From e2196d6f8082ed3f3361c7243cc2535f71c3eb6a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 29 Jul 2020 15:01:56 +0800 Subject: [PATCH 001/109] [td-988] --- src/tsdb/src/tsdbRead.c | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 0e3a657fde..9d66e5c569 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -316,17 +316,20 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh assert(pCheckInfo->iter == NULL && pCheckInfo->iiter == NULL); - // TODO: add uid check - if (pHandle->mem && pCheckInfo->tableId.tid < pHandle->mem->maxTables && - pHandle->mem->tData[pCheckInfo->tableId.tid] != NULL) { - pCheckInfo->iter = tSkipListCreateIterFromVal(pHandle->mem->tData[pCheckInfo->tableId.tid]->pData, - (const char*)&pCheckInfo->lastKey, TSDB_DATA_TYPE_TIMESTAMP, order); + if (pHandle->mem && pCheckInfo->tableId.tid < pHandle->mem->maxTables) { + STableData* ptd = pHandle->mem->tData[pCheckInfo->tableId.tid]; + if (ptd != NULL && ptd->uid == pCheckInfo->tableId.uid) { // check uid + pCheckInfo->iter = + tSkipListCreateIterFromVal(ptd->pData, (const char*)&pCheckInfo->lastKey, TSDB_DATA_TYPE_TIMESTAMP, order); + } } - if (pHandle->imem && pCheckInfo->tableId.tid < pHandle->imem->maxTables && - pHandle->imem->tData[pCheckInfo->tableId.tid] != NULL) { - pCheckInfo->iiter = tSkipListCreateIterFromVal(pHandle->imem->tData[pCheckInfo->tableId.tid]->pData, - (const char*)&pCheckInfo->lastKey, TSDB_DATA_TYPE_TIMESTAMP, order); + if (pHandle->imem && pCheckInfo->tableId.tid < pHandle->imem->maxTables) { + STableData* ptd = pHandle->imem->tData[pCheckInfo->tableId.tid]; + if (ptd != NULL && ptd->uid == pCheckInfo->tableId.uid) { // check uid + pCheckInfo->iiter = + tSkipListCreateIterFromVal(ptd->pData, (const char*)&pCheckInfo->lastKey, TSDB_DATA_TYPE_TIMESTAMP, order); + } } // both iterators are NULL, no data in buffer right now From f86022f6c11dfdaecfac7ff7608e5c1294f0acd8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 29 Jul 2020 15:02:33 +0800 Subject: [PATCH 002/109] [td-960] --- src/client/src/tscSQLParser.c | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 9488b37fe3..48f932203d 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -1191,7 +1191,15 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel tExprTreeDestroy(&pNode, NULL); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "invalid arithmetic expression in select clause"); } - + + size_t numOfNode = taosArrayGetSize(colList); + for(int32_t k = 0; k < numOfNode; ++k) { + SColIndex* pIndex = taosArrayGet(colList, k); + if (pIndex->flag == 1) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "tag columns can not be used in arithmetic expression"); + } + } + SBufferWriter bw = tbufInitWriter(NULL, false); TRY(0) { @@ -6115,12 +6123,14 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSQLExpr* pS if (ret != TSDB_CODE_SUCCESS) { return ret; } - + + STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta; + int32_t numOfColumns = tscGetNumOfColumns(pTableMeta); + *pExpr = calloc(1, sizeof(tExprNode)); (*pExpr)->nodeType = TSQL_NODE_COL; (*pExpr)->pSchema = calloc(1, sizeof(SSchema)); - - STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta; + SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, index.columnIndex); *(*pExpr)->pSchema = *pSchema; @@ -6129,7 +6139,8 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSQLExpr* pS tstrncpy(colIndex.name, pSchema->name, sizeof(colIndex.name)); colIndex.colId = pSchema->colId; colIndex.colIndex = index.columnIndex; - + colIndex.flag = (index.columnIndex >= numOfColumns)? 1:0; + taosArrayPush(pCols, &colIndex); } From 6fd03c905d20ff2fc05b6d1876cf42397570bb24 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 29 Jul 2020 17:24:43 +0800 Subject: [PATCH 003/109] [td-225] fix bug in aggregation function in arithmetic expression. --- src/client/src/tscLocalMerge.c | 6 +- src/client/src/tscSQLParser.c | 239 +++++++++++++++++++-------------- src/client/src/tscServer.c | 17 +-- src/client/src/tscUtil.c | 74 ++++++---- 4 files changed, 192 insertions(+), 144 deletions(-) diff --git a/src/client/src/tscLocalMerge.c b/src/client/src/tscLocalMerge.c index bf76b8cbe8..eaf78235a4 100644 --- a/src/client/src/tscLocalMerge.c +++ b/src/client/src/tscLocalMerge.c @@ -68,7 +68,7 @@ static void tscInitSqlContext(SSqlCmd *pCmd, SLocalReducer *pReducer, tOrderDesc SSqlExpr * pExpr = tscSqlExprGet(pQueryInfo, i); pCtx->aOutputBuf = - pReducer->pResultBuf->data + tscFieldInfoGetOffset(pQueryInfo, i) * pReducer->resColModel->capacity; + pReducer->pResultBuf->data + pExpr->offset * pReducer->resColModel->capacity; pCtx->order = pQueryInfo->order.order; pCtx->functionId = pExpr->functionId; @@ -321,6 +321,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd pReducer->finalRowSize = tscGetResRowLength(pQueryInfo->exprList); pReducer->resColModel = finalmodel; pReducer->resColModel->capacity = pReducer->nResultBufSize; + assert(pReducer->finalRowSize > 0); if (pReducer->finalRowSize > 0) { pReducer->resColModel->capacity /= pReducer->finalRowSize; @@ -328,10 +329,9 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd assert(pReducer->finalRowSize <= pReducer->rowSize); pReducer->pFinalRes = calloc(1, pReducer->rowSize * pReducer->resColModel->capacity); -// pReducer->pBufForInterpo = calloc(1, pReducer->nResultBufSize); if (pReducer->pTempBuffer == NULL || pReducer->discardData == NULL || pReducer->pResultBuf == NULL || - /*pReducer->pBufForInterpo == NULL || */pReducer->pFinalRes == NULL || pReducer->prevRowOfInput == NULL) { + pReducer->pFinalRes == NULL || pReducer->prevRowOfInput == NULL) { tfree(pReducer->pTempBuffer); tfree(pReducer->discardData); tfree(pReducer->pResultBuf); diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 48f932203d..07afd82150 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -87,7 +87,7 @@ static int32_t parseOrderbyClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQueryS static int32_t tsRewriteFieldNameIfNecessary(SSqlCmd* pCmd, SQueryInfo* pQueryInfo); static int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo); static int32_t validateSqlFunctionInStreamSql(SSqlCmd* pCmd, SQueryInfo* pQueryInfo); -static int32_t buildArithmeticExprString(tSQLExpr* pExpr, char** exprString); +static int32_t arithmeticExprToString(tSQLExpr* pExpr, char** exprString); static int32_t validateFunctionsInIntervalOrGroupbyQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo); static int32_t validateArithmeticSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryInfo* pQueryInfo, SColumnList* pList, int32_t* type); static int32_t validateEp(char* ep); @@ -1107,13 +1107,128 @@ static void extractColumnNameFromString(tSQLExprItem* pItem) { } } +static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t exprIndex, tSQLExprItem* pItem) { + const char* msg1 = "invalid column name, or illegal column type"; + const char* msg2 = "invalid arithmetic expression in select clause"; + const char* msg3 = "tag columns can not be used in arithmetic expression"; + const char* msg4 = "columns from different table mixed up in arithmetic expression"; + + // arithmetic function in select clause + SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, clauseIndex); + + SColumnList columnList = {0}; + int32_t arithmeticType = NON_ARITHMEIC_EXPR; + + if (validateArithmeticSQLExpr(pCmd, pItem->pNode, pQueryInfo, &columnList, &arithmeticType) != TSDB_CODE_SUCCESS) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + } + + int32_t tableIndex = columnList.ids[0].tableIndex; + + // todo potential data overflow + char arithmeticExprStr[1024*12]; + char* p = arithmeticExprStr; + + if (arithmeticType == NORMAL_ARITHMETIC) { + pQueryInfo->type |= TSDB_QUERY_TYPE_PROJECTION_QUERY; + + // all columns in arithmetic expression must belong to the same table + for (int32_t f = 1; f < columnList.num; ++f) { + if (columnList.ids[f].tableIndex != tableIndex) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); + } + } + + if (arithmeticExprToString(pItem->pNode, &p) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_SQL; + } + + // expr string is set as the parameter of function + SColumnIndex index = {.tableIndex = tableIndex}; + + SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_ARITHM, &index, TSDB_DATA_TYPE_DOUBLE, sizeof(double), + sizeof(double), false); + + char* name = (pItem->aliasName != NULL)? pItem->aliasName:arithmeticExprStr; + tstrncpy(pExpr->aliasName, name, sizeof(pExpr->aliasName)); + + tExprNode* pNode = NULL; + SArray* colList = taosArrayInit(10, sizeof(SColIndex)); + + int32_t ret = exprTreeFromSqlExpr(pCmd, &pNode, pItem->pNode, pQueryInfo->exprList, pQueryInfo, colList); + if (ret != TSDB_CODE_SUCCESS) { + tExprTreeDestroy(&pNode, NULL); + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); + } + + size_t numOfNode = taosArrayGetSize(colList); + for(int32_t k = 0; k < numOfNode; ++k) { + SColIndex* pIndex = taosArrayGet(colList, k); + if (pIndex->flag == 1) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + } + } + + SBufferWriter bw = tbufInitWriter(NULL, false); + + TRY(0) { + exprTreeToBinary(&bw, pNode); + } CATCH(code) { + tbufCloseWriter(&bw); + UNUSED(code); + // TODO: other error handling + } END_TRY + + size_t len = tbufTell(&bw); + char* c = tbufGetData(&bw, true); + + // set the serialized binary string as the parameter of arithmetic expression + addExprParams(pExpr, c, TSDB_DATA_TYPE_BINARY, len, index.tableIndex); + + insertResultField(pQueryInfo, exprIndex, &columnList, sizeof(double), TSDB_DATA_TYPE_DOUBLE, pExpr->aliasName, pExpr); + + taosArrayDestroy(colList); + tExprTreeDestroy(&pNode, NULL); + } else { + if (arithmeticExprToString(pItem->pNode, &p) != TSDB_CODE_SUCCESS) { + return TSDB_CODE_TSC_INVALID_SQL; + } + + columnList.num = 0; + columnList.ids[0] = (SColumnIndex) {0, 0}; + + char* name = (pItem->aliasName != NULL)? pItem->aliasName:arithmeticExprStr; + insertResultField(pQueryInfo, exprIndex, &columnList, sizeof(double), TSDB_DATA_TYPE_DOUBLE, name, NULL); + + int32_t slot = tscNumOfFields(pQueryInfo) - 1; + SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, slot); + + if (pInfo->pSqlExpr == NULL) { + SExprInfo* pArithExprInfo = calloc(1, sizeof(SExprInfo)); + + // arithmetic expression always return result in the format of double float + pArithExprInfo->bytes = sizeof(double); + pArithExprInfo->interBytes = sizeof(double); + pArithExprInfo->type = TSDB_DATA_TYPE_DOUBLE; + + int32_t ret = exprTreeFromSqlExpr(pCmd, &pArithExprInfo->pExpr, pItem->pNode, pQueryInfo->exprList, pQueryInfo, NULL); + if (ret != TSDB_CODE_SUCCESS) { + tExprTreeDestroy(&pArithExprInfo->pExpr, NULL); + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "invalid expression in select clause"); + } + + pInfo->pArithExprInfo = pArithExprInfo; + } + } + + return TSDB_CODE_SUCCESS; +} + int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSelection, bool isSTable) { assert(pSelection != NULL && pCmd != NULL); - const char* msg1 = "invalid column name, or illegal column type"; const char* msg2 = "functions can not be mixed up"; const char* msg3 = "not support query expression"; - const char* msg4 = "columns from different table mixed up in arithmetic expression"; const char* msg5 = "invalid function name"; SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, clauseIndex); @@ -1148,104 +1263,11 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel } } else if (pItem->pNode->nSQLOptr >= TK_PLUS && pItem->pNode->nSQLOptr <= TK_REM) { - // arithmetic function in select clause - SColumnList columnList = {0}; - int32_t arithmeticType = NON_ARITHMEIC_EXPR; - - if (validateArithmeticSQLExpr(pCmd, pItem->pNode, pQueryInfo, &columnList, &arithmeticType) != TSDB_CODE_SUCCESS) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); + int32_t code = handleArithmeticExpr(pCmd, clauseIndex, i, pItem); + if (code != TSDB_CODE_SUCCESS) { + return code; } - - int32_t tableIndex = columnList.ids[0].tableIndex; - char arithmeticExprStr[1024] = {0}; - char* p = arithmeticExprStr; - - if (arithmeticType == NORMAL_ARITHMETIC) { - pQueryInfo->type |= TSDB_QUERY_TYPE_PROJECTION_QUERY; - - // all columns in arithmetic expression must belong to the same table - for (int32_t f = 1; f < columnList.num; ++f) { - if (columnList.ids[f].tableIndex != tableIndex) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); - } - } - - if (buildArithmeticExprString(pItem->pNode, &p) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; - } - - // expr string is set as the parameter of function - SColumnIndex index = {.tableIndex = tableIndex}; - SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_ARITHM, &index, TSDB_DATA_TYPE_DOUBLE, - sizeof(double), sizeof(double), false); - - /* todo alias name should use the original sql string */ - char* name = (pItem->aliasName != NULL)? pItem->aliasName:arithmeticExprStr; - tstrncpy(pExpr->aliasName, name, sizeof(pExpr->aliasName)); - - tExprNode* pNode = NULL; - SArray* colList = taosArrayInit(10, sizeof(SColIndex)); - - int32_t ret = exprTreeFromSqlExpr(pCmd, &pNode, pItem->pNode, pQueryInfo->exprList, pQueryInfo, colList); - if (ret != TSDB_CODE_SUCCESS) { - tExprTreeDestroy(&pNode, NULL); - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "invalid arithmetic expression in select clause"); - } - size_t numOfNode = taosArrayGetSize(colList); - for(int32_t k = 0; k < numOfNode; ++k) { - SColIndex* pIndex = taosArrayGet(colList, k); - if (pIndex->flag == 1) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "tag columns can not be used in arithmetic expression"); - } - } - - SBufferWriter bw = tbufInitWriter(NULL, false); - - TRY(0) { - exprTreeToBinary(&bw, pNode); - } CATCH(code) { - tbufCloseWriter(&bw); - UNUSED(code); - // TODO: other error handling - } END_TRY - - size_t len = tbufTell(&bw); - char* c = tbufGetData(&bw, true); - - // set the serialized binary string as the parameter of arithmetic expression - addExprParams(pExpr, c, TSDB_DATA_TYPE_BINARY, len, index.tableIndex); - - insertResultField(pQueryInfo, i, &columnList, sizeof(double), TSDB_DATA_TYPE_DOUBLE, pExpr->aliasName, pExpr); - - taosArrayDestroy(colList); - tExprTreeDestroy(&pNode, NULL); - } else { - columnList.num = 0; - columnList.ids[0] = (SColumnIndex) {0, 0}; - - insertResultField(pQueryInfo, i, &columnList, sizeof(double), TSDB_DATA_TYPE_DOUBLE, "dummy_column", NULL); - - int32_t slot = tscNumOfFields(pQueryInfo) - 1; - SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, slot); - - if (pInfo->pSqlExpr == NULL) { - SExprInfo* pArithExprInfo = calloc(1, sizeof(SExprInfo)); - - // arithmetic expression always return result in the format of double float - pArithExprInfo->bytes = sizeof(double); - pArithExprInfo->interBytes = sizeof(double); - pArithExprInfo->type = TSDB_DATA_TYPE_DOUBLE; - - int32_t ret = exprTreeFromSqlExpr(pCmd, &pArithExprInfo->pExpr, pItem->pNode, pQueryInfo->exprList, pQueryInfo, NULL); - if (ret != TSDB_CODE_SUCCESS) { - tExprTreeDestroy(&pArithExprInfo->pExpr, NULL); - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "invalid expression in select clause"); - } - - pInfo->pArithExprInfo = pArithExprInfo; - } - } } else { /* * not support such expression @@ -3090,14 +3112,14 @@ static int32_t getJoinCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr* } // todo error handle / such as and /or mixed with +/-/*/ -int32_t buildArithmeticExprString(tSQLExpr* pExpr, char** exprString) { +int32_t doArithmeticExprToString(tSQLExpr* pExpr, char** exprString) { tSQLExpr* pLeft = pExpr->pLeft; tSQLExpr* pRight = pExpr->pRight; *(*exprString)++ = '('; if (pLeft->nSQLOptr >= TK_PLUS && pLeft->nSQLOptr <= TK_REM) { - buildArithmeticExprString(pLeft, exprString); + doArithmeticExprToString(pLeft, exprString); } else { int32_t ret = tSQLExprNodeToString(pLeft, exprString); if (ret != TSDB_CODE_SUCCESS) { @@ -3108,7 +3130,7 @@ int32_t buildArithmeticExprString(tSQLExpr* pExpr, char** exprString) { optrToString(pExpr, exprString); if (pRight->nSQLOptr >= TK_PLUS && pRight->nSQLOptr <= TK_REM) { - buildArithmeticExprString(pRight, exprString); + doArithmeticExprToString(pRight, exprString); } else { int32_t ret = tSQLExprNodeToString(pRight, exprString); if (ret != TSDB_CODE_SUCCESS) { @@ -3121,6 +3143,19 @@ int32_t buildArithmeticExprString(tSQLExpr* pExpr, char** exprString) { return TSDB_CODE_SUCCESS; } +static int32_t arithmeticExprToString(tSQLExpr* pExpr, char** str) { + char* start = *str; + + int32_t code = doArithmeticExprToString(pExpr, str); + if (code == TSDB_CODE_SUCCESS) { // remove out the parenthesis + int32_t len = strlen(start); + memmove(start, start + 1, len - 2); + start[len - 2] = 0; + } + + return code; +} + static int32_t validateSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryInfo* pQueryInfo, SColumnList* pList, int32_t* type) { if (pExpr->nSQLOptr == TK_ID) { if (*type == NON_ARITHMEIC_EXPR) { diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 521280af87..690a5f790a 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1461,7 +1461,7 @@ int tscProcessRetrieveLocalMergeRsp(SSqlObj *pSql) { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); if (pRes->code == TSDB_CODE_SUCCESS && pRes->numOfRows > 0) { - tscSetResultPointer(pQueryInfo, pRes); + tscCreateResPointerInfo(pRes, pQueryInfo); } pRes->row = 0; @@ -2115,21 +2115,6 @@ int tscProcessRetrieveRspFromNode(SSqlObj *pSql) { return 0; } -int tscProcessRetrieveRspFromLocal(SSqlObj *pSql) { - SSqlRes * pRes = &pSql->res; - SSqlCmd * pCmd = &pSql->cmd; - SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); - - SRetrieveTableRsp *pRetrieve = (SRetrieveTableRsp *)pRes->pRsp; - - pRes->numOfRows = htonl(pRetrieve->numOfRows); - pRes->data = pRetrieve->data; - - tscSetResultPointer(pQueryInfo, pRes); - pRes->row = 0; - return 0; -} - void tscTableMetaCallBack(void *param, TAOS_RES *res, int code); static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 73d6f0e592..bbb3af473b 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -249,23 +249,25 @@ void tscClearInterpInfo(SQueryInfo* pQueryInfo) { } int32_t tscCreateResPointerInfo(SSqlRes* pRes, SQueryInfo* pQueryInfo) { - if (pRes->tsrow == NULL) { - int32_t numOfOutput = pQueryInfo->fieldsInfo.numOfOutput; - pRes->numOfCols = numOfOutput; - - pRes->tsrow = calloc(numOfOutput, POINTER_BYTES); - pRes->length = calloc(numOfOutput, sizeof(int32_t)); // todo refactor - pRes->buffer = calloc(numOfOutput, POINTER_BYTES); - - // not enough memory - if (pRes->tsrow == NULL || (pRes->buffer == NULL && pRes->numOfCols > 0)) { - tfree(pRes->tsrow); - tfree(pRes->buffer); - tfree(pRes->length); - - pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; - return pRes->code; - } + if (pRes->tsrow != NULL) { + return TSDB_CODE_SUCCESS; + } + + int32_t numOfOutput = pQueryInfo->fieldsInfo.numOfOutput; + pRes->numOfCols = numOfOutput; + + pRes->tsrow = calloc(numOfOutput, POINTER_BYTES); + pRes->length = calloc(numOfOutput, sizeof(int32_t)); // todo refactor + pRes->buffer = calloc(numOfOutput, POINTER_BYTES); + + // not enough memory + if (pRes->tsrow == NULL || (pRes->buffer == NULL && pRes->numOfCols > 0)) { + tfree(pRes->tsrow); + tfree(pRes->buffer); + tfree(pRes->length); + + pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; + return pRes->code; } return TSDB_CODE_SUCCESS; @@ -858,12 +860,13 @@ void tscFieldInfoCopy(SFieldInfo* dst, const SFieldInfo* src) { } TAOS_FIELD* tscFieldInfoGetField(SFieldInfo* pFieldInfo, int32_t index) { + assert(index < pFieldInfo->numOfOutput); return TARRAY_GET_ELEM(pFieldInfo->pFields, index); } int16_t tscFieldInfoGetOffset(SQueryInfo* pQueryInfo, int32_t index) { SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, index); - assert(pInfo != NULL); + assert(pInfo != NULL && pInfo->pSqlExpr != NULL); return pInfo->pSqlExpr->offset; } @@ -1773,11 +1776,36 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); if (pExpr->uid == uid) { - TAOS_FIELD* p = tscFieldInfoGetField(pFieldInfo, i); - SFieldSupInfo* pInfo = tscFieldInfoGetSupp(pFieldInfo, i); - - SFieldSupInfo* pInfo1 = tscFieldInfoAppend(&pNewQueryInfo->fieldsInfo, p); - *pInfo1 = *pInfo; + if (i < pFieldInfo->numOfOutput) { + SFieldSupInfo* pInfo = tscFieldInfoGetSupp(pFieldInfo, i); + if (pInfo->pSqlExpr != NULL) { + TAOS_FIELD* p = tscFieldInfoGetField(pFieldInfo, i); + assert(strcmp(p->name, pExpr->aliasName) == 0 && pInfo->pSqlExpr == pExpr); + + SFieldSupInfo* pInfo1 = tscFieldInfoAppend(&pNewQueryInfo->fieldsInfo, p); + *pInfo1 = *pInfo; + } else { + // current sql function is not direct output result, so create a dummy output field + assert(pInfo->pArithExprInfo != NULL); + + TAOS_FIELD f = {.type = pExpr->resType, .bytes = pExpr->resBytes}; + tstrncpy(f.name, pExpr->aliasName, sizeof(f.name)); + + SFieldSupInfo* pInfo1 = tscFieldInfoAppend(&pNewQueryInfo->fieldsInfo, &f); + + pInfo1->pSqlExpr = pExpr; + pInfo1->visible = false; + } + } else { + // current sql function is not direct output result, so create a dummy output field + TAOS_FIELD f = {.type = pExpr->resType, .bytes = pExpr->resBytes}; + tstrncpy(f.name, pExpr->aliasName, sizeof(f.name)); + + SFieldSupInfo* pInfo1 = tscFieldInfoAppend(&pNewQueryInfo->fieldsInfo, &f); + + pInfo1->pSqlExpr = pExpr; + pInfo1->visible = false; + } } } From e8f9f503fcaadf103a4e10c2f39c265db1a56f68 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 29 Jul 2020 18:04:20 +0800 Subject: [PATCH 004/109] [td-225] fix bugs in join query. --- src/client/inc/tsclient.h | 1 - src/client/src/tscSubquery.c | 8 +++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 7a44870938..dbbb6be128 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -419,7 +419,6 @@ char *tscGetErrorMsgPayload(SSqlCmd *pCmd); int32_t tscInvalidSQLErrMsg(char *msg, const char *additionalInfo, const char *sql); int32_t tscToSQLCmd(SSqlObj *pSql, struct SSqlInfo *pInfo); -//void tscGetResultColumnChr(SSqlRes *pRes, SFieldInfo* pFieldInfo, int32_t column); static FORCE_INLINE void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pFieldInfo, int32_t columnIndex) { SFieldSupInfo* pInfo = (SFieldSupInfo*) TARRAY_GET_ELEM(pFieldInfo->pSupportInfo, columnIndex); diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 15e02799aa..c396c81310 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -1054,6 +1054,10 @@ void tscSetupOutputColumnIndex(SSqlObj* pSql) { } } } + + // restore the offset value for super table query in case of final result. + tscRestoreSQLFuncForSTableQuery(pQueryInfo); + tscFieldInfoUpdateOffset(pQueryInfo); } void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) { @@ -2046,9 +2050,7 @@ void tscBuildResFromSubqueries(SSqlObj *pSql) { } while (1) { - if (pRes->row < pRes->numOfRows) { - assert(0); - } + assert (pRes->row >= pRes->numOfRows); doBuildResFromSubqueries(pSql); sem_post(&pSql->rspSem); From 6e45e83242abed03a643b85e45c3a1471c52ab3c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 29 Jul 2020 18:08:47 +0800 Subject: [PATCH 005/109] [td-225] fix bugs in join query. --- src/client/src/tscUtil.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index bbb3af473b..bd1fd9905a 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1780,7 +1780,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void SFieldSupInfo* pInfo = tscFieldInfoGetSupp(pFieldInfo, i); if (pInfo->pSqlExpr != NULL) { TAOS_FIELD* p = tscFieldInfoGetField(pFieldInfo, i); - assert(strcmp(p->name, pExpr->aliasName) == 0 && pInfo->pSqlExpr == pExpr); + assert(strcmp(p->name, pExpr->aliasName) == 0); SFieldSupInfo* pInfo1 = tscFieldInfoAppend(&pNewQueryInfo->fieldsInfo, p); *pInfo1 = *pInfo; From 4e2ed952ec89c106d9b565bf8cc1eea3518da629 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 29 Jul 2020 18:51:27 +0800 Subject: [PATCH 006/109] [td-225] fix bugs in join query. --- src/client/inc/tsclient.h | 2 +- src/client/src/tscSubquery.c | 10 ++- src/client/src/tscUtil.c | 140 ++++++++++++++++++----------------- 3 files changed, 79 insertions(+), 73 deletions(-) diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index dbbb6be128..e3e1d44514 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -106,7 +106,7 @@ typedef struct SColumnIndex { typedef struct SFieldSupInfo { bool visible; SExprInfo *pArithExprInfo; - SSqlExpr * pSqlExpr; + SSqlExpr *pSqlExpr; } SFieldSupInfo; typedef struct SFieldInfo { diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index c396c81310..c7e7d1323b 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -1026,9 +1026,11 @@ void tscSetupOutputColumnIndex(SSqlObj* pSql) { } SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - pRes->pColumnIndex = calloc(1, sizeof(SColumnIndex) * pQueryInfo->fieldsInfo.numOfOutput); - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { + int32_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); + pRes->pColumnIndex = calloc(1, sizeof(SColumnIndex) * numOfExprs); + + for (int32_t i = 0; i < numOfExprs; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); int32_t tableIndexOfSub = -1; @@ -1045,8 +1047,8 @@ void tscSetupOutputColumnIndex(SSqlObj* pSql) { SSqlCmd* pSubCmd = &pSql->pSubs[tableIndexOfSub]->cmd; SQueryInfo* pSubQueryInfo = tscGetQueryInfoDetail(pSubCmd, 0); - size_t numOfExprs = taosArrayGetSize(pSubQueryInfo->exprList); - for (int32_t k = 0; k < numOfExprs; ++k) { + size_t numOfSubExpr = taosArrayGetSize(pSubQueryInfo->exprList); + for (int32_t k = 0; k < numOfSubExpr; ++k) { SSqlExpr* pSubExpr = tscSqlExprGet(pSubQueryInfo, k); if (pExpr->functionId == pSubExpr->functionId && pExpr->colInfo.colId == pSubExpr->colInfo.colId) { pRes->pColumnIndex[i] = (SColumnIndex){.tableIndex = tableIndexOfSub, .columnIndex = k}; diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index bd1fd9905a..32a82a080f 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1673,6 +1673,77 @@ SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cm return pNew; } +// current sql function is not direct output result, so create a dummy output field +static void doSetNewFieldInfo(SQueryInfo* pNewQueryInfo, SSqlExpr* pExpr) { + TAOS_FIELD f = {.type = pExpr->resType, .bytes = pExpr->resBytes}; + tstrncpy(f.name, pExpr->aliasName, sizeof(f.name)); + + SFieldSupInfo* pInfo1 = tscFieldInfoAppend(&pNewQueryInfo->fieldsInfo, &f); + + pInfo1->pSqlExpr = pExpr; + pInfo1->visible = false; +} + +static void doSetSqlExprAndResultFieldInfo(SQueryInfo* pQueryInfo, SQueryInfo* pNewQueryInfo, int64_t uid) { + int32_t numOfOutput = tscSqlExprNumOfExprs(pNewQueryInfo); + if (numOfOutput == 0) { + return; + } + + size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); + SFieldInfo* pFieldInfo = &pQueryInfo->fieldsInfo; + + // set the field info in pNewQueryInfo object + for (int32_t i = 0; i < numOfExprs; ++i) { + SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); + + if (pExpr->uid == uid) { + if (i < pFieldInfo->numOfOutput) { + SFieldSupInfo* pInfo = tscFieldInfoGetSupp(pFieldInfo, i); + + if (pInfo->pSqlExpr != NULL) { + TAOS_FIELD* p = tscFieldInfoGetField(pFieldInfo, i); + assert(strcmp(p->name, pExpr->aliasName) == 0); + + SFieldSupInfo* pInfo1 = tscFieldInfoAppend(&pNewQueryInfo->fieldsInfo, p); + *pInfo1 = *pInfo; + } else { + assert(pInfo->pArithExprInfo != NULL); + doSetNewFieldInfo(pNewQueryInfo, pExpr); + } + } else { // it is a arithmetic column, does not have actual field for sqlExpr, so build it + doSetNewFieldInfo(pNewQueryInfo, pExpr); + } + } + } + + // make sure the the sqlExpr for each fields is correct + numOfExprs = tscSqlExprNumOfExprs(pNewQueryInfo); + + // update the pSqlExpr pointer in SFieldSupInfo according the field name + // make sure the pSqlExpr point to the correct SqlExpr in pNewQueryInfo, not SqlExpr in pQueryInfo + for (int32_t f = 0; f < pNewQueryInfo->fieldsInfo.numOfOutput; ++f) { + TAOS_FIELD* field = tscFieldInfoGetField(&pNewQueryInfo->fieldsInfo, f); + + bool matched = false; + for (int32_t k1 = 0; k1 < numOfExprs; ++k1) { + SSqlExpr* pExpr1 = tscSqlExprGet(pNewQueryInfo, k1); + + if (strcmp(field->name, pExpr1->aliasName) == 0) { // establish link according to the result field name + SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pNewQueryInfo->fieldsInfo, f); + pInfo->pSqlExpr = pExpr1; + + matched = true; + break; + } + } + + assert(matched); + } + + tscFieldInfoUpdateOffset(pNewQueryInfo); +} + SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void* param, int32_t cmd, SSqlObj* pPrevSql) { SSqlCmd* pCmd = &pSql->cmd; SSqlObj* pNew = (SSqlObj*)calloc(1, sizeof(SSqlObj)); @@ -1766,74 +1837,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void uint64_t uid = pTableMetaInfo->pTableMeta->id.uid; tscSqlExprCopy(pNewQueryInfo->exprList, pQueryInfo->exprList, uid, true); - int32_t numOfOutput = tscSqlExprNumOfExprs(pNewQueryInfo); - - if (numOfOutput > 0) { // todo refactor to extract method - size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); - SFieldInfo* pFieldInfo = &pQueryInfo->fieldsInfo; - - for (int32_t i = 0; i < numOfExprs; ++i) { - SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); - - if (pExpr->uid == uid) { - if (i < pFieldInfo->numOfOutput) { - SFieldSupInfo* pInfo = tscFieldInfoGetSupp(pFieldInfo, i); - if (pInfo->pSqlExpr != NULL) { - TAOS_FIELD* p = tscFieldInfoGetField(pFieldInfo, i); - assert(strcmp(p->name, pExpr->aliasName) == 0); - - SFieldSupInfo* pInfo1 = tscFieldInfoAppend(&pNewQueryInfo->fieldsInfo, p); - *pInfo1 = *pInfo; - } else { - // current sql function is not direct output result, so create a dummy output field - assert(pInfo->pArithExprInfo != NULL); - - TAOS_FIELD f = {.type = pExpr->resType, .bytes = pExpr->resBytes}; - tstrncpy(f.name, pExpr->aliasName, sizeof(f.name)); - - SFieldSupInfo* pInfo1 = tscFieldInfoAppend(&pNewQueryInfo->fieldsInfo, &f); - - pInfo1->pSqlExpr = pExpr; - pInfo1->visible = false; - } - } else { - // current sql function is not direct output result, so create a dummy output field - TAOS_FIELD f = {.type = pExpr->resType, .bytes = pExpr->resBytes}; - tstrncpy(f.name, pExpr->aliasName, sizeof(f.name)); - - SFieldSupInfo* pInfo1 = tscFieldInfoAppend(&pNewQueryInfo->fieldsInfo, &f); - - pInfo1->pSqlExpr = pExpr; - pInfo1->visible = false; - } - } - } - - // make sure the the sqlExpr for each fields is correct - // todo handle the agg arithmetic expression - numOfExprs = tscSqlExprNumOfExprs(pNewQueryInfo); - - for(int32_t f = 0; f < pNewQueryInfo->fieldsInfo.numOfOutput; ++f) { - TAOS_FIELD* field = tscFieldInfoGetField(&pNewQueryInfo->fieldsInfo, f); - bool matched = false; - - for(int32_t k1 = 0; k1 < numOfExprs; ++k1) { - SSqlExpr* pExpr1 = tscSqlExprGet(pNewQueryInfo, k1); - - if (strcmp(field->name, pExpr1->aliasName) == 0) { // establish link according to the result field name - SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pNewQueryInfo->fieldsInfo, f); - pInfo->pSqlExpr = pExpr1; - - matched = true; - break; - } - } - - assert(matched); - } - - tscFieldInfoUpdateOffset(pNewQueryInfo); - } + doSetSqlExprAndResultFieldInfo(pQueryInfo, pNewQueryInfo, uid); pNew->fp = fp; pNew->fetchFp = fp; From d58310d9d73b404a9138b934cb9e2f9b9e8507f5 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 30 Jul 2020 15:46:50 +0800 Subject: [PATCH 007/109] [td-225] fix bugs in join/union query. --- src/client/src/tscAsync.c | 12 +- src/client/src/tscLocal.c | 4 +- src/client/src/tscServer.c | 58 ++++------ src/client/src/tscSubquery.c | 2 +- src/client/src/tscUtil.c | 4 + src/query/src/qExecutor.c | 2 +- tests/script/general/parser/join.sim | 24 ++++ tests/script/general/parser/union.sim | 154 +++++++++++++------------- 8 files changed, 138 insertions(+), 122 deletions(-) diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index 85cff4ba17..59d906e8fd 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -220,14 +220,13 @@ void taos_fetch_rows_a(TAOS_RES *taosa, void (*fp)(void *, TAOS_RES *, int), voi if (pCmd->command == TSDB_SQL_TABLE_JOIN_RETRIEVE) { tscFetchDatablockFromSubquery(pSql); } else if (pRes->completed) { - if(pCmd->command == TSDB_SQL_FETCH) { + if(pCmd->command == TSDB_SQL_FETCH || (pCmd->command >= TSDB_SQL_SERV_STATUS && pCmd->command <= TSDB_SQL_CURRENT_USER)) { if (hasMoreVnodesToTry(pSql)) { // sequentially retrieve data from remain vnodes. tscTryQueryNextVnode(pSql, tscAsyncQueryRowsForNextVnode); - return; } else { /* - * all available virtual node has been checked already, now we need to check - * for the next subclause queries + * all available virtual nodes in current clause has been checked already, now try the + * next one in the following union subclause */ if (pCmd->clauseIndex < pCmd->numOfClause - 1) { tscTryQueryNextClause(pSql, tscAsyncQueryRowsForNextVnode); @@ -235,11 +234,12 @@ void taos_fetch_rows_a(TAOS_RES *taosa, void (*fp)(void *, TAOS_RES *, int), voi } /* - * 1. has reach the limitation - * 2. no remain virtual nodes to be retrieved anymore + * 1. has reach the limitation + * 2. no remain virtual nodes to be retrieved anymore */ (*pSql->fetchFp)(param, pSql, 0); } + return; } else if (pCmd->command == TSDB_SQL_RETRIEVE || pCmd->command == TSDB_SQL_RETRIEVE_LOCALMERGE) { // in case of show command, return no data diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c index 7f336daa91..6822851d84 100644 --- a/src/client/src/tscLocal.c +++ b/src/client/src/tscLocal.c @@ -293,7 +293,7 @@ static void tscProcessCurrentDB(SSqlObj *pSql) { char db[TSDB_DB_NAME_LEN] = {0}; extractDBName(pSql->pTscObj->db, db); - SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); + SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, pSql->cmd.clauseIndex); SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, 0); pExpr->resType = TSDB_DATA_TYPE_BINARY; @@ -314,7 +314,7 @@ static void tscProcessCurrentDB(SSqlObj *pSql) { static void tscProcessServerVer(SSqlObj *pSql) { const char* v = pSql->pTscObj->sversion; - SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); + SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, pSql->cmd.clauseIndex); SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, 0); pExpr->resType = TSDB_DATA_TYPE_BINARY; diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 690a5f790a..6b9fc0551e 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -46,19 +46,27 @@ void tscSaveSubscriptionProgress(void* sub); static int32_t minMsgSize() { return tsRpcHeadSize + 100; } static void tscSetDnodeEpSet(SSqlObj* pSql, SCMVgroupInfo* pVgroupInfo) { + assert(pSql != NULL && pVgroupInfo != NULL && pVgroupInfo->numOfEps > 0); + SRpcEpSet* pEpSet = &pSql->epSet; - pEpSet->inUse = 0; - if (pVgroupInfo == NULL) { - pEpSet->numOfEps = 0; - return; - } + pEpSet->inUse = 0; + + // apply the FQDN string length check here + bool hasFqdn = false; pEpSet->numOfEps = pVgroupInfo->numOfEps; for(int32_t i = 0; i < pVgroupInfo->numOfEps; ++i) { strcpy(pEpSet->fqdn[i], pVgroupInfo->epAddr[i].fqdn); pEpSet->port[i] = pVgroupInfo->epAddr[i].port; + + if (!hasFqdn) { + hasFqdn = (strlen(pEpSet->fqdn[i]) > 0); + } } + + assert(hasFqdn); } + static void tscDumpMgmtEpSet(SRpcEpSet *epSet) { taosCorBeginRead(&tscMgmtEpSet.version); *epSet = tscMgmtEpSet.epSet; @@ -128,21 +136,6 @@ void tscPrintMgmtEp() { } } -/* - * For each management node, try twice at least in case of poor network situation. - * If the client start to connect to a non-management node from the client, and the first retry may fail due to - * the poor network quality. And then, the second retry get the response with redirection command. - * The retry will not be executed since only *two* retry is allowed in case of single management node in the cluster. - * Therefore, we need to multiply the retry times by factor of 2 to fix this problem. - */ -UNUSED_FUNC -static int32_t tscGetMgmtConnMaxRetryTimes() { - int32_t factor = 2; - SRpcEpSet dump; - tscDumpMgmtEpSet(&dump); - return dump.numOfEps * factor; -} - void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { STscObj *pObj = (STscObj *)param; if (pObj == NULL) return; @@ -425,21 +418,18 @@ int doProcessSql(SSqlObj *pSql) { } int tscProcessSql(SSqlObj *pSql) { - char * name = NULL; + char *name = NULL; SSqlCmd *pCmd = &pSql->cmd; - SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); + SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); STableMetaInfo *pTableMetaInfo = NULL; uint32_t type = 0; if (pQueryInfo != NULL) { pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - if (pTableMetaInfo != NULL) { - name = pTableMetaInfo->name; - } - + name = (pTableMetaInfo != NULL)? pTableMetaInfo->name:NULL; type = pQueryInfo->type; - + // while numOfTables equals to 0, it must be Heartbeat assert((pQueryInfo->numOfTables == 0 && pQueryInfo->command == TSDB_SQL_HB) || pQueryInfo->numOfTables > 0); } @@ -451,7 +441,6 @@ int tscProcessSql(SSqlObj *pSql) { return pSql->res.code; } } else if (pCmd->command < TSDB_SQL_LOCAL) { - //pSql->epSet = tscMgmtEpSet; } else { // local handler return (*tscProcessMsgRsp[pCmd->command])(pSql); @@ -598,11 +587,11 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char } else { pVgroupInfo = &pTableMeta->vgroupInfo; } - tscSetDnodeEpSet(pSql, pVgroupInfo); - if (pVgroupInfo != NULL) { - pQueryMsg->head.vgId = htonl(pVgroupInfo->vgId); - } + assert(pVgroupInfo != NULL); + + tscSetDnodeEpSet(pSql, pVgroupInfo); + pQueryMsg->head.vgId = htonl(pVgroupInfo->vgId); STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg; pTableIdInfo->tid = htonl(pTableMeta->id.tid); @@ -1885,11 +1874,10 @@ int tscProcessSTableVgroupRsp(SSqlObj *pSql) { for (int32_t k = 0; k < pVgroups->numOfEps; ++k) { pVgroups->epAddr[k].port = htons(pVgroups->epAddr[k].port); - } - - pMsg += size; } + + pMsg += size; } return pSql->res.code; diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index c7e7d1323b..7258ac528e 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -1085,7 +1085,7 @@ void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) { if (taos_errno(pSql) != TSDB_CODE_SUCCESS) { assert(taos_errno(pSql) == code); - tscError("%p abort query, code:%d, global code:%d", pSql, code, pParentSql->res.code); + tscError("%p abort query, code:%s, global code:%s", pSql, tstrerror(code), tstrerror(pParentSql->res.code)); pParentSql->res.code = code; quitAllSubquery(pParentSql, pSupporter); diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 32a82a080f..5ee3db36d1 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -2034,6 +2034,10 @@ bool hasMoreVnodesToTry(SSqlObj* pSql) { } int32_t numOfVgroups = pTableMetaInfo->vgroupList->numOfVgroups; + if (pTableMetaInfo->pVgroupTables != NULL) { + numOfVgroups = taosArrayGetSize(pTableMetaInfo->pVgroupTables); + } + return tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && (!tscHasReachLimitation(pQueryInfo, pRes)) && (pTableMetaInfo->vgroupIndex < numOfVgroups - 1); } diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 6efc8a827e..fb13972689 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -2216,7 +2216,7 @@ static void ensureOutputBufferSimple(SQueryRuntimeEnv* pRuntimeEnv, int32_t capa static void ensureOutputBuffer(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pBlockInfo) { // in case of prj/diff query, ensure the output buffer is sufficient to accommodate the results of current block SQuery* pQuery = pRuntimeEnv->pQuery; - if (!QUERY_IS_INTERVAL_QUERY(pQuery) && !pRuntimeEnv->groupbyNormalCol && !isFixedOutputQuery(pRuntimeEnv)) { + if (!QUERY_IS_INTERVAL_QUERY(pQuery) && !pRuntimeEnv->groupbyNormalCol && !isFixedOutputQuery(pRuntimeEnv) && !isTSCompQuery(pQuery)) { SResultRec *pRec = &pQuery->rec; if (pQuery->rec.capacity - pQuery->rec.rows < pBlockInfo->rows) { diff --git a/tests/script/general/parser/join.sim b/tests/script/general/parser/join.sim index 07f2cd3f77..882f561ae1 100644 --- a/tests/script/general/parser/join.sim +++ b/tests/script/general/parser/join.sim @@ -257,6 +257,21 @@ if $data01 != $val then return -1 endi +sql select count(join_tb1.*) + count(join_tb0.*) from join_tb1 , join_tb0 where join_tb1.ts = join_tb0.ts and join_tb1.ts >= 100000 and join_tb0.c7 = false;; +if $rows != 1 then + return -1 +endi + +if $data00 != 20.000000000 then + print expect 20.000000000 actual $data00 + return -1 +endi + +sql select count(join_tb1.*)/10 from join_tb1 , join_tb0 where join_tb1.ts = join_tb0.ts and join_tb1.ts >= 100000 and join_tb0.c7 = false;; +if $data00 != 1.000000000 then + return -1 +endi + print 3 #agg + where condition sql select count(join_tb1.c3), count(join_tb0.ts) from $tb1 , $tb2 where $ts1 = $ts2 and join_tb1.ts <= 100002 and join_tb0.c7 = true; @@ -381,6 +396,15 @@ if $data00 != $val then return -1 endi +sql select sum(join_mt0.c1)+sum(join_mt0.c1) from join_mt0, join_mt1 where join_mt0.ts = join_mt1.ts and join_mt0.t1=join_mt1.t1 and join_mt0.c2=99 and join_mt1.ts=100999; +if $rows != 1 then + return -1 +endi + +if $data00 != 396.000000000 then + return -1 +endi + # first/last sql select count(join_mt0.c1), sum(join_mt1.c2), first(join_mt0.c5), last(join_mt1.c7) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts and join_mt0.t1=1 interval(10a) order by join_mt0.ts asc; diff --git a/tests/script/general/parser/union.sim b/tests/script/general/parser/union.sim index 358bcb8a40..b9dc8e8e1f 100644 --- a/tests/script/general/parser/union.sim +++ b/tests/script/general/parser/union.sim @@ -1,10 +1,10 @@ -#system sh/stop_dnodes.sh -# -#system sh/deploy.sh -n dnode1 -i 1 -#system sh/cfg.sh -n dnode1 -c walLevel -v 0 -#system sh/cfg.sh -n dnode1 -c debugFlag -v 135 -#system sh/cfg.sh -n dnode1 -c rpcDebugFlag -v 135 -#system sh/exec.sh -n dnode1 -s start +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c debugFlag -v 135 +system sh/cfg.sh -n dnode1 -c rpcDebugFlag -v 135 +system sh/exec.sh -n dnode1 -s start sleep 1000 sql connect @@ -24,77 +24,77 @@ $mt = $mtPrefix . $i $j = 1 $mt1 = $mtPrefix . $j -# -#sql drop database if exits $db -x step1 -#step1: -#sql create database if not exists $db maxtables 4 + +sql drop database if exits $db -x step1 +step1: +sql create database if not exists $db maxtables 4 sql use $db -#sql create table $mt (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int) -# -#$i = 0 -#$t = 1578203484000 -# -#while $i < $tbNum -# $tb = $tbPrefix . $i -# sql create table $tb using $mt tags( $i ) -# -# $x = 0 -# while $x < $rowNum -# $ms = $x * 1000 -# $ms = $ms * 60 -# -# $c = $x / 100 -# $c = $c * 100 -# $c = $x - $c -# $binary = 'binary . $c -# $binary = $binary . ' -# $nchar = 'nchar . $c -# $nchar = $nchar . ' -# -# $t1 = $t + $ms -# sql insert into $tb values ($t1 , $c , $c , $c , $c , $c , $c , $c , $binary , $nchar ) -# $x = $x + 1 -# endw -# -# $i = $i + 1 -#endw -# -#sql create table $mt1 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int) -# -#$j = 0 -#$t = 1578203484000 -#$rowNum = 1000 -#$tbNum = 5 -#$i = 0 -# -#while $i < $tbNum -# $tb1 = $tbPrefix1 . $j -# sql create table $tb1 using $mt1 tags( $i ) -# -# $x = 0 -# while $x < $rowNum -# $ms = $x * 1000 -# $ms = $ms * 60 -# -# $c = $x / 100 -# $c = $c * 100 -# $c = $x - $c -# $binary = 'binary . $c -# $binary = $binary . ' -# $nchar = 'nchar . $c -# $nchar = $nchar . ' -# -# $t1 = $t + $ms -# sql insert into $tb1 values ($t1 , $c , $c , $c , $c , $c , $c , $c , $binary , $nchar ) -# $x = $x + 1 -# endw -# -# $i = $i + 1 -# $j = $j + 1 -#endw -# -#print sleep 1sec. -#sleep 1000 +sql create table $mt (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int) + +$i = 0 +$t = 1578203484000 + +while $i < $tbNum + $tb = $tbPrefix . $i + sql create table $tb using $mt tags( $i ) + + $x = 0 + while $x < $rowNum + $ms = $x * 1000 + $ms = $ms * 60 + + $c = $x / 100 + $c = $c * 100 + $c = $x - $c + $binary = 'binary . $c + $binary = $binary . ' + $nchar = 'nchar . $c + $nchar = $nchar . ' + + $t1 = $t + $ms + sql insert into $tb values ($t1 , $c , $c , $c , $c , $c , $c , $c , $binary , $nchar ) + $x = $x + 1 + endw + + $i = $i + 1 +endw + +sql create table $mt1 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int) + +$j = 0 +$t = 1578203484000 +$rowNum = 1000 +$tbNum = 5 +$i = 0 + +while $i < $tbNum + $tb1 = $tbPrefix1 . $j + sql create table $tb1 using $mt1 tags( $i ) + + $x = 0 + while $x < $rowNum + $ms = $x * 1000 + $ms = $ms * 60 + + $c = $x / 100 + $c = $c * 100 + $c = $x - $c + $binary = 'binary . $c + $binary = $binary . ' + $nchar = 'nchar . $c + $nchar = $nchar . ' + + $t1 = $t + $ms + sql insert into $tb1 values ($t1 , $c , $c , $c , $c , $c , $c , $c , $binary , $nchar ) + $x = $x + 1 + endw + + $i = $i + 1 + $j = $j + 1 +endw + +print sleep 1sec. +sleep 1000 $i = 1 $tb = $tbPrefix . $i From 1dd3d2ba05e065c2cb72535ca6f6a1d68b1dd21f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 30 Jul 2020 15:57:32 +0800 Subject: [PATCH 008/109] [td-225] update the script --- tests/script/general/parser/sliding.sim | 4 ++-- tests/script/general/parser/union.sim | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/script/general/parser/sliding.sim b/tests/script/general/parser/sliding.sim index 177c95651f..f85211beb8 100644 --- a/tests/script/general/parser/sliding.sim +++ b/tests/script/general/parser/sliding.sim @@ -1,7 +1,7 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 system sh/cfg.sh -n dnode1 -c debugFlag -v 135 system sh/cfg.sh -n dnode1 -c rpcDebugFlag -v 135 system sh/exec.sh -n dnode1 -s start @@ -28,7 +28,7 @@ $mt = $mtPrefix . $i sql drop database if exits $db -x step1 step1: -sql create database if not exists $db tables 4 keep 36500 +sql create database if not exists $db maxtables 4 keep 36500 sql use $db sql create table $mt (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int, t2 binary(12)) diff --git a/tests/script/general/parser/union.sim b/tests/script/general/parser/union.sim index b9dc8e8e1f..fbd1c211b9 100644 --- a/tests/script/general/parser/union.sim +++ b/tests/script/general/parser/union.sim @@ -412,4 +412,8 @@ if $data10 != @union_db0@ then return -1 endi +sql_error show tables union all show tables +sql_error show stables union all show stables +sql_error show databases union all show databases + system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file From a09378a4db084835bbaf2d221b2c977e58ffe544 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 30 Jul 2020 17:02:42 +0800 Subject: [PATCH 009/109] [td-225] refactors --- src/query/src/qExecutor.c | 10 +++++----- src/vnode/src/vnodeRead.c | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index fb13972689..66762a1ca5 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -6670,9 +6670,9 @@ void qQueryMgmtNotifyClosed(void* pQMgmt) { SQueryMgmt* pQueryMgmt = pQMgmt; qDebug("vgId:%d, set querymgmt closed, wait for all queries cancelled", pQueryMgmt->vgId); - pthread_mutex_lock(&pQueryMgmt->lock); +// pthread_mutex_lock(&pQueryMgmt->lock); pQueryMgmt->closed = true; - pthread_mutex_unlock(&pQueryMgmt->lock); +// pthread_mutex_unlock(&pQueryMgmt->lock); taosCacheRefresh(pQueryMgmt->qinfoPool, queryMgmtKillQueryFn); } @@ -6710,16 +6710,16 @@ void** qRegisterQInfo(void* pMgmt, uint64_t qInfo) { return NULL; } - pthread_mutex_lock(&pQueryMgmt->lock); +// pthread_mutex_lock(&pQueryMgmt->lock); if (pQueryMgmt->closed) { - pthread_mutex_unlock(&pQueryMgmt->lock); +// pthread_mutex_unlock(&pQueryMgmt->lock); qError("QInfo:%p failed to add qhandle into cache, since qMgmt is colsing", (void *)qInfo); return NULL; } else { uint64_t handleVal = (uint64_t) qInfo; void** handle = taosCachePut(pQueryMgmt->qinfoPool, &handleVal, sizeof(int64_t), &qInfo, POINTER_BYTES, DEFAULT_QHANDLE_LIFE_SPAN); - pthread_mutex_unlock(&pQueryMgmt->lock); +// pthread_mutex_unlock(&pQueryMgmt->lock); return handle; } diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index 066770e1bb..3c642b5098 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -14,7 +14,7 @@ */ #define _DEFAULT_SOURCE -#include +//#include #include "os.h" #include "tglobal.h" From fa560fd3a754c78bb136ee07bf262b1f5683c1b6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 30 Jul 2020 17:14:17 +0800 Subject: [PATCH 010/109] [td-225] refactor codes --- cmake/platform.inc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmake/platform.inc b/cmake/platform.inc index 7a371df70b..dac0509b12 100755 --- a/cmake/platform.inc +++ b/cmake/platform.inc @@ -61,6 +61,8 @@ execute_process(COMMAND chmod 777 ${TD_COMMUNITY_DIR}/packaging/tools/get_os.sh) execute_process(COMMAND ${TD_COMMUNITY_DIR}/packaging/tools/get_os.sh "" OUTPUT_VARIABLE TD_OS_INFO) MESSAGE(STATUS "The current os is " ${TD_OS_INFO}) +ADD_DEFINITIONS(-D_TD_LINUX_64) + IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux") IF (${CMAKE_SIZEOF_VOID_P} MATCHES 8) SET(TD_LINUX_64 TRUE) From 3b80eb725593acf3301bd264ddfd39ab26d91676 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 30 Jul 2020 17:18:15 +0800 Subject: [PATCH 011/109] [td-225] refactor codes. --- src/util/src/tcache.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index dc9128d4a9..8470eedd0e 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -276,7 +276,7 @@ void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const v return NULL; } - __cache_wr_lock(pCacheObj); + __cache_rd_lock(pCacheObj); SCacheDataNode **pt = (SCacheDataNode **)taosHashGet(pCacheObj->pHashTable, key, keyLen); SCacheDataNode * pOld = (pt != NULL) ? (*pt) : NULL; From ae244c9e849c71b02c99307b84d7590fedcdad3d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 30 Jul 2020 17:22:13 +0800 Subject: [PATCH 012/109] [td-225] refactor codes. --- src/util/src/tcache.c | 2 +- src/vnode/src/vnodeRead.c | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index 8470eedd0e..dc9128d4a9 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -276,7 +276,7 @@ void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const v return NULL; } - __cache_rd_lock(pCacheObj); + __cache_wr_lock(pCacheObj); SCacheDataNode **pt = (SCacheDataNode **)taosHashGet(pCacheObj->pHashTable, key, keyLen); SCacheDataNode * pOld = (pt != NULL) ? (*pt) : NULL; diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index 3c642b5098..6c05091cec 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -207,6 +207,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { vDebug("vgId:%d, QInfo:%p, start to build result rsp after query paused, %p", pVnode->vgId, *handle, pReadMsg->rpcMsg.handle); code = vnodeDumpQueryResult(&pReadMsg->rspRet, pVnode, *handle, &freehandle); + freehandle = false; // todo test the error code case if (code == TSDB_CODE_SUCCESS) { code = TSDB_CODE_QRY_HAS_RSP; @@ -266,6 +267,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { } code = vnodeDumpQueryResult(pRet, pVnode, *handle, &freeHandle); + freeHandle = false; } qReleaseQInfo(pVnode->qMgmt, (void**) &handle, freeHandle); From c2ce0fc4f88aeb31d8114d61aed15ec52adbfa0f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 30 Jul 2020 23:47:06 +0800 Subject: [PATCH 013/109] [td-225] refactor codes --- src/util/inc/hash.h | 12 ++-- src/util/src/hash.c | 127 ++++++++++++------------------------- src/util/src/tcache.c | 130 +++++++++++++++++++------------------- src/vnode/src/vnodeRead.c | 2 - 4 files changed, 113 insertions(+), 158 deletions(-) diff --git a/src/util/inc/hash.h b/src/util/inc/hash.h index 137777f3cb..900aa112e4 100644 --- a/src/util/inc/hash.h +++ b/src/util/inc/hash.h @@ -29,16 +29,12 @@ extern "C" { typedef void (*_hash_free_fn_t)(void *param); typedef struct SHashNode { - char *key; -// union { - struct SHashNode * prev; -// struct SHashEntry *prev1; -// }; -// + char *key; + struct SHashNode *prev; struct SHashNode *next; uint32_t hashVal; // the hash value of key, if hashVal == HASH_VALUE_IN_TRASH, this node is moved to trash uint32_t keyLen; // length of the key - char data[]; + char *data; } SHashNode; typedef struct SHashObj { @@ -109,6 +105,8 @@ void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen); */ void taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen); +void taosHashRemoveNode(); + /** * clean up hash table * @param handle diff --git a/src/util/src/hash.c b/src/util/src/hash.c index 5dff6286f0..37e8c37cb6 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -19,6 +19,8 @@ #include "tulog.h" #include "tutil.h" +#define HASH_NEED_RESIZE(_h) ((_h)->size >= (_h)->capacity * HASH_DEFAULT_LOAD_FACTOR) + static FORCE_INLINE void __wr_lock(void *lock) { if (lock == NULL) { return; @@ -95,29 +97,19 @@ static FORCE_INLINE int32_t taosHashCapacity(int32_t length) { * @param hashVal hash value by hash function * @return */ -FORCE_INLINE SHashNode *doGetNodeFromHashTable(SHashObj *pHashObj, const void *key, uint32_t keyLen, uint32_t *hashVal) { - uint32_t hash = (*pHashObj->hashFp)(key, keyLen); - - int32_t slot = HASH_INDEX(hash, pHashObj->capacity); +FORCE_INLINE SHashNode *doGetNodeFromHashTable(SHashObj *pHashObj, const void *key, uint32_t keyLen, uint32_t hashVal) { + int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); + SHashNode *pNode = pHashObj->hashList[slot]; - while (pNode) { if ((pNode->keyLen == keyLen) && (memcmp(pNode->key, key, keyLen) == 0)) { + assert(pNode->hashVal == hashVal); break; } pNode = pNode->next; } - if (pNode) { - assert(HASH_INDEX(pNode->hashVal, pHashObj->capacity) == slot); - } - - // return the calculated hash value, to avoid calculating it again in other functions - if (hashVal != NULL) { - *hashVal = hash; - } - return pNode; } @@ -148,7 +140,13 @@ static SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *p * @param dsize size of actual data * @return hash node */ -static SHashNode *doUpdateHashNode(SHashNode *pNode, const void *key, size_t keyLen, const void *pData, size_t dsize); +static FORCE_INLINE SHashNode *doUpdateHashNode(SHashNode *pNode, SHashNode *pNewNode) { + assert(pNode->keyLen == pNewNode->keyLen); + SWAP(pNode->key, pNewNode->key, void*); + SWAP(pNode->data, pNewNode->data, void*); + + return pNewNode; +} /** * insert the hash node at the front of the linked list @@ -217,58 +215,43 @@ size_t taosHashGetSize(const SHashObj *pHashObj) { } int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size) { - __wr_lock(pHashObj->lock); + uint32_t hashVal = (*pHashObj->hashFp)(key, keyLen); + SHashNode *pNewNode = doCreateHashNode(key, keyLen, data, size, hashVal); + if (pNewNode == NULL) { + return -1; + } - uint32_t hashVal = 0; - SHashNode *pNode = doGetNodeFromHashTable(pHashObj, key, keyLen, &hashVal); + __wr_lock(pHashObj->lock); + SHashNode *pNode = doGetNodeFromHashTable(pHashObj, key, keyLen, hashVal); if (pNode == NULL) { // no data in hash table with the specified key, add it into hash table - taosHashTableResize(pHashObj); - SHashNode *pNewNode = doCreateHashNode(key, keyLen, data, size, hashVal); - if (pNewNode == NULL) { - __unlock(pHashObj->lock); - - return -1; + if (HASH_NEED_RESIZE(pHashObj)) { + taosHashTableResize(pHashObj); } doAddToHashTable(pHashObj, pNewNode); + __unlock(pHashObj->lock); } else { - SHashNode *pNewNode = doUpdateHashNode(pNode, key, keyLen, data, size); - if (pNewNode == NULL) { - __unlock(pHashObj->lock); - return -1; - } + doUpdateHashNode(pNode, pNewNode); + __unlock(pHashObj->lock); - if (pNewNode->prev) { - pNewNode->prev->next = pNewNode; - } else { - int32_t slot = HASH_INDEX(pNewNode->hashVal, pHashObj->capacity); - - assert(pHashObj->hashList[slot] == pNode); - pHashObj->hashList[slot] = pNewNode; - } - - if (pNewNode->next) { - (pNewNode->next)->prev = pNewNode; - } + tfree(pNewNode->data) + tfree(pNewNode); } - __unlock(pHashObj->lock); return 0; } void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen) { + uint32_t hashVal = (*pHashObj->hashFp)(key, keyLen); + __rd_lock(pHashObj->lock); - - uint32_t hashVal = 0; - SHashNode *pNode = doGetNodeFromHashTable(pHashObj, key, keyLen, &hashVal); - + SHashNode *pNode = doGetNodeFromHashTable(pHashObj, key, keyLen, hashVal); __unlock(pHashObj->lock); - if (pNode != NULL) { + if (pNode) { assert(pNode->hashVal == hashVal); - return pNode->data; } else { return NULL; @@ -276,10 +259,10 @@ void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen) { } void taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen) { - __wr_lock(pHashObj->lock); + uint32_t hashVal = (*pHashObj->hashFp)(key, keyLen); - uint32_t val = 0; - SHashNode *pNode = doGetNodeFromHashTable(pHashObj, key, keyLen, &val); + __wr_lock(pHashObj->lock); + SHashNode *pNode = doGetNodeFromHashTable(pHashObj, key, keyLen, hashVal); if (pNode == NULL) { __unlock(pHashObj->lock); return; @@ -287,7 +270,7 @@ void taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen) { SHashNode *pNext = pNode->next; if (pNode->prev == NULL) { - int32_t slot = HASH_INDEX(val, pHashObj->capacity); + int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); assert(pHashObj->hashList[slot] == pNode); pHashObj->hashList[slot] = pNext; @@ -299,13 +282,13 @@ void taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen) { pNext->prev = pNode->prev; } - pHashObj->size--; + pHashObj->size -= 1; + __unlock(pHashObj->lock); pNode->next = NULL; pNode->prev = NULL; tfree(pNode); - __unlock(pHashObj->lock); } void taosHashCleanup(SHashObj *pHashObj) { @@ -341,14 +324,6 @@ void taosHashCleanup(SHashObj *pHashObj) { free(pHashObj); } -void taosHashSetFreecb(SHashObj *pHashObj, _hash_free_fn_t freeFp) { - if (pHashObj == NULL || freeFp == NULL) { - return; - } - - pHashObj->freeFp = freeFp; -} - SHashMutableIterator *taosHashCreateIter(SHashObj *pHashObj) { SHashMutableIterator *pIter = calloc(1, sizeof(SHashMutableIterator)); if (pIter == NULL) { @@ -528,38 +503,20 @@ void taosHashTableResize(SHashObj *pHashObj) { } SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, size_t dsize, uint32_t hashVal) { - size_t totalSize = dsize + sizeof(SHashNode) + keyLen; - - SHashNode *pNewNode = calloc(1, totalSize); + SHashNode *pNewNode = calloc(1, sizeof(SHashNode)); if (pNewNode == NULL) { uError("failed to allocate memory, reason:%s", strerror(errno)); return NULL; } - - memcpy(pNewNode->data, pData, dsize); - - pNewNode->key = pNewNode->data + dsize; - memcpy(pNewNode->key, key, keyLen); - pNewNode->keyLen = keyLen; - - pNewNode->hashVal = hashVal; - return pNewNode; -} -SHashNode *doUpdateHashNode(SHashNode *pNode, const void *key, size_t keyLen, const void *pData, size_t dsize) { - size_t size = dsize + sizeof(SHashNode) + keyLen; - - SHashNode *pNewNode = (SHashNode *)realloc(pNode, size); - if (pNewNode == NULL) { - return NULL; - } - + pNewNode->data = malloc(dsize + keyLen); memcpy(pNewNode->data, pData, dsize); pNewNode->key = pNewNode->data + dsize; - assert(memcmp(pNewNode->key, key, keyLen) == 0 && keyLen == pNewNode->keyLen); - memcpy(pNewNode->key, key, keyLen); + + pNewNode->keyLen = keyLen; + pNewNode->hashVal = hashVal; return pNewNode; } diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index dc9128d4a9..b89cd836e1 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -147,49 +147,18 @@ static FORCE_INLINE void taosCacheMoveToTrash(SCacheObj *pCacheObj, SCacheDataNo * @param dataSize * @return */ -static SCacheDataNode *taosUpdateCacheImpl(SCacheObj *pCacheObj, SCacheDataNode *pNode, const char *key, int32_t keyLen, - const void *pData, uint32_t dataSize, uint64_t duration) { - SCacheDataNode *pNewNode = NULL; - +static UNUSED_FUNC SCacheDataNode *taosUpdateCacheImpl(SCacheObj *pCacheObj, SCacheDataNode* pNode, SCacheDataNode* pNewNode, + const char *key, int32_t keyLen) { + // only a node is not referenced by any other object, in-place update it - if (T_REF_VAL_GET(pNode) == 0) { - size_t newSize = sizeof(SCacheDataNode) + dataSize + keyLen + 1; - - pNewNode = (SCacheDataNode *)realloc(pNode, newSize); - if (pNewNode == NULL) { - return NULL; - } - - memset(pNewNode, 0, newSize); - pNewNode->signature = (uint64_t)pNewNode; - memcpy(pNewNode->data, pData, dataSize); - - pNewNode->key = (char *)pNewNode + sizeof(SCacheDataNode) + dataSize; - pNewNode->keySize = keyLen; - memcpy(pNewNode->key, key, keyLen); - - // update the timestamp information for updated key/value - pNewNode->addedTime = taosGetTimestampMs(); - pNewNode->lifespan = duration; - - T_REF_INC(pNewNode); - - // the address of this node may be changed, so the prev and next element should update the corresponding pointer - taosHashPut(pCacheObj->pHashTable, key, keyLen, &pNewNode, sizeof(void *)); - } else { + if (T_REF_VAL_GET(pNode) > 0) { taosCacheMoveToTrash(pCacheObj, pNode); - - pNewNode = taosCreateCacheNode(key, keyLen, pData, dataSize, duration); - if (pNewNode == NULL) { - return NULL; - } - - T_REF_INC(pNewNode); - - // addedTime new element to hashtable - taosHashPut(pCacheObj->pHashTable, key, keyLen, &pNewNode, sizeof(void *)); } - + + T_REF_INC(pNewNode); + + // addedTime new element to hashtable + taosHashPut(pCacheObj->pHashTable, key, keyLen, &pNewNode, sizeof(void *)); return pNewNode; } @@ -238,7 +207,7 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool ext return NULL; } - pCacheObj->pHashTable = taosHashInit(128, taosGetDefaultHashFunction(keyType), false); + pCacheObj->pHashTable = taosHashInit(128, taosGetDefaultHashFunction(keyType), true); pCacheObj->name = strdup(cacheName); if (pCacheObj->pHashTable == NULL) { free(pCacheObj); @@ -270,36 +239,59 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool ext } void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const void *pData, size_t dataSize, int duration) { - SCacheDataNode *pNode; +// SCacheDataNode *pNode = NULL; if (pCacheObj == NULL || pCacheObj->pHashTable == NULL) { return NULL; } + SCacheDataNode *pNode1 = taosCreateCacheNode(key, keyLen, pData, dataSize, duration); + if (pNode1 == NULL) { + uError("cache:%s, key:%p, failed to added into cache, out of memory", pCacheObj->name, key); + return NULL; + } + __cache_wr_lock(pCacheObj); SCacheDataNode **pt = (SCacheDataNode **)taosHashGet(pCacheObj->pHashTable, key, keyLen); SCacheDataNode * pOld = (pt != NULL) ? (*pt) : NULL; if (pOld == NULL) { // do addedTime to cache - pNode = taosAddToCacheImpl(pCacheObj, key, keyLen, pData, dataSize, duration * 1000L); - if (NULL != pNode) { - pCacheObj->totalSize += pNode->size; + T_REF_INC(pNode1); + taosHashPut(pCacheObj->pHashTable, key, keyLen, &pNode1, sizeof(void *)); + __cache_unlock(pCacheObj); - uDebug("cache:%s, key:%p, %p added into cache, added:%" PRIu64 ", expire:%" PRIu64 ", totalNum:%d totalSize:%" PRId64 - "bytes size:%" PRId64 "bytes", - pCacheObj->name, key, pNode->data, pNode->addedTime, pNode->expireTime, - (int32_t)taosHashGetSize(pCacheObj->pHashTable), pCacheObj->totalSize, (int64_t)dataSize); - } else { - uError("cache:%s, key:%p, failed to added into cache, out of memory", pCacheObj->name, key); - } + atomic_add_fetch_64(&pCacheObj->totalSize, pNode1->size); + uDebug("cache:%s, key:%p, %p added into cache, added:%" PRIu64 ", expire:%" PRIu64 ", totalNum:%d totalSize:%" PRId64 + "bytes size:%" PRId64 "bytes", + pCacheObj->name, key, pNode1->data, pNode1->addedTime, pNode1->expireTime, + (int32_t)taosHashGetSize(pCacheObj->pHashTable), pCacheObj->totalSize, (int64_t)dataSize); } else { // old data exists, update the node - pNode = taosUpdateCacheImpl(pCacheObj, pOld, key, keyLen, pData, dataSize, duration * 1000L); - uDebug("cache:%s, key:%p, %p exist in cache, updated old:%p", pCacheObj->name, key, pNode->data, pOld); + + bool addToTrashcan = false; + if (T_REF_VAL_GET(pOld) > 0) { + // todo removed by node, instead of by key + taosHashRemove(pCacheObj->pHashTable, pOld->key, pOld->keySize); + } else { + addToTrashcan = true; + } + + T_REF_INC(pNode1); + + // addedTime new element to hashtable + taosHashPut(pCacheObj->pHashTable, key, keyLen, &pNode1, sizeof(void *)); + __cache_unlock(pCacheObj); + + // todo add trashcan lock + if (addToTrashcan) { + taosAddToTrash(pCacheObj, pOld); + } else { + tfree(pOld); + } + + uDebug("cache:%s, key:%p, %p exist in cache, updated old:%p", pCacheObj->name, key, pNode1->data, pOld); } - __cache_unlock(pCacheObj); - - return (pNode != NULL) ? pNode->data : NULL; + return pNode1->data; } void *taosCacheAcquireByKey(SCacheObj *pCacheObj, const void *key, size_t keyLen) { @@ -310,7 +302,6 @@ void *taosCacheAcquireByKey(SCacheObj *pCacheObj, const void *key, size_t keyLen void *pData = NULL; __cache_rd_lock(pCacheObj); - SCacheDataNode **ptNode = (SCacheDataNode **)taosHashGet(pCacheObj->pHashTable, key, keyLen); int32_t ref = 0; @@ -439,21 +430,33 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { * that tries to do the same thing. */ if (pNode->inTrashCan) { + __cache_unlock(pCacheObj); + if (ref == 0) { assert(pNode->pTNodeHeader->pData == pNode); taosRemoveFromTrashCan(pCacheObj, pNode->pTNodeHeader); } } else { + taosHashRemove(pCacheObj->pHashTable, pNode->key, pNode->keySize); + __cache_unlock(pCacheObj); + if (ref > 0) { assert(pNode->pTNodeHeader == NULL); - taosCacheMoveToTrash(pCacheObj, pNode); + + // todo trashcan lock + taosAddToTrash(pCacheObj, pNode); } else { - taosCacheReleaseNode(pCacheObj, pNode); +// taosCacheReleaseNode(pCacheObj, pNode); + atomic_fetch_sub_ptr(&pCacheObj->totalSize, pNode->size); + uDebug("cache:%s, key:%p, %p is destroyed from cache, totalNum:%d totalSize:%" PRId64 "bytes size:%dbytes", + pCacheObj->name, pNode->key, pNode->data, (int32_t)taosHashGetSize(pCacheObj->pHashTable), pCacheObj->totalSize, + pNode->size); + + if (pCacheObj->freeFp) pCacheObj->freeFp(pNode->data); + free(pNode); } } - __cache_unlock(pCacheObj); - } else { // NOTE: once refcount is decrease, pNode may be freed by other thread immediately. int32_t ref = T_REF_DEC(pNode); @@ -497,8 +500,7 @@ void taosCacheCleanup(SCacheObj *pCacheObj) { doCleanupDataCache(pCacheObj); } -SCacheDataNode *taosCreateCacheNode(const char *key, size_t keyLen, const char *pData, size_t size, - uint64_t duration) { +SCacheDataNode *taosCreateCacheNode(const char *key, size_t keyLen, const char *pData, size_t size, uint64_t duration) { size_t totalSize = size + sizeof(SCacheDataNode) + keyLen; SCacheDataNode *pNewNode = calloc(1, totalSize); diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index 6c05091cec..3c642b5098 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -207,7 +207,6 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { vDebug("vgId:%d, QInfo:%p, start to build result rsp after query paused, %p", pVnode->vgId, *handle, pReadMsg->rpcMsg.handle); code = vnodeDumpQueryResult(&pReadMsg->rspRet, pVnode, *handle, &freehandle); - freehandle = false; // todo test the error code case if (code == TSDB_CODE_SUCCESS) { code = TSDB_CODE_QRY_HAS_RSP; @@ -267,7 +266,6 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { } code = vnodeDumpQueryResult(pRet, pVnode, *handle, &freeHandle); - freeHandle = false; } qReleaseQInfo(pVnode->qMgmt, (void**) &handle, freeHandle); From 4a508cdf3e6f9f200cbe97d6eb08c2b50a6171cf Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 31 Jul 2020 00:15:59 +0800 Subject: [PATCH 014/109] [td-225] refactor codes. --- src/util/src/tcache.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index b89cd836e1..715a142c90 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -207,7 +207,7 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool ext return NULL; } - pCacheObj->pHashTable = taosHashInit(128, taosGetDefaultHashFunction(keyType), true); + pCacheObj->pHashTable = taosHashInit(128, taosGetDefaultHashFunction(keyType), false); pCacheObj->name = strdup(cacheName); if (pCacheObj->pHashTable == NULL) { free(pCacheObj); @@ -239,8 +239,6 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool ext } void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const void *pData, size_t dataSize, int duration) { -// SCacheDataNode *pNode = NULL; - if (pCacheObj == NULL || pCacheObj->pHashTable == NULL) { return NULL; } @@ -266,12 +264,11 @@ void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const v pCacheObj->name, key, pNode1->data, pNode1->addedTime, pNode1->expireTime, (int32_t)taosHashGetSize(pCacheObj->pHashTable), pCacheObj->totalSize, (int64_t)dataSize); } else { // old data exists, update the node - bool addToTrashcan = false; if (T_REF_VAL_GET(pOld) > 0) { + // todo removed by node, instead of by key taosHashRemove(pCacheObj->pHashTable, pOld->key, pOld->keySize); - } else { addToTrashcan = true; } @@ -285,7 +282,7 @@ void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const v if (addToTrashcan) { taosAddToTrash(pCacheObj, pOld); } else { - tfree(pOld); + free(pOld); } uDebug("cache:%s, key:%p, %p exist in cache, updated old:%p", pCacheObj->name, key, pNode1->data, pOld); From e47e6300e4b25f961b016a634a366af9b137a08b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 31 Jul 2020 00:49:24 +0800 Subject: [PATCH 015/109] [td-225] refactor codes. --- src/util/inc/hash.h | 2 +- src/util/src/hash.c | 5 +++-- src/util/src/tcache.c | 20 +++++++++++++------- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/src/util/inc/hash.h b/src/util/inc/hash.h index 900aa112e4..d3edc89585 100644 --- a/src/util/inc/hash.h +++ b/src/util/inc/hash.h @@ -103,7 +103,7 @@ void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen); * @param key * @param keyLen */ -void taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen); +int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen); void taosHashRemoveNode(); diff --git a/src/util/src/hash.c b/src/util/src/hash.c index 37e8c37cb6..f59f25d153 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -258,14 +258,14 @@ void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen) { } } -void taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen) { +int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen) { uint32_t hashVal = (*pHashObj->hashFp)(key, keyLen); __wr_lock(pHashObj->lock); SHashNode *pNode = doGetNodeFromHashTable(pHashObj, key, keyLen, hashVal); if (pNode == NULL) { __unlock(pHashObj->lock); - return; + return -1; } SHashNode *pNext = pNode->next; @@ -289,6 +289,7 @@ void taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen) { pNode->prev = NULL; tfree(pNode); + return 0; } void taosHashCleanup(SHashObj *pHashObj) { diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index 715a142c90..a5317cdda1 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -207,7 +207,7 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool ext return NULL; } - pCacheObj->pHashTable = taosHashInit(128, taosGetDefaultHashFunction(keyType), false); + pCacheObj->pHashTable = taosHashInit(128, taosGetDefaultHashFunction(keyType), true); pCacheObj->name = strdup(cacheName); if (pCacheObj->pHashTable == NULL) { free(pCacheObj); @@ -268,7 +268,9 @@ void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const v if (T_REF_VAL_GET(pOld) > 0) { // todo removed by node, instead of by key - taosHashRemove(pCacheObj->pHashTable, pOld->key, pOld->keySize); + int32_t succ = taosHashRemove(pCacheObj->pHashTable, pOld->key, pOld->keySize); + assert(succ == 0); + addToTrashcan = true; } @@ -413,7 +415,7 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { } if (_remove) { - __cache_wr_lock(pCacheObj); +// __cache_wr_lock(pCacheObj); // NOTE: once refcount is decrease, pNode may be freed by other thread immediately. int32_t ref = T_REF_DEC(pNode); @@ -427,22 +429,26 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { * that tries to do the same thing. */ if (pNode->inTrashCan) { - __cache_unlock(pCacheObj); +// __cache_unlock(pCacheObj); if (ref == 0) { assert(pNode->pTNodeHeader->pData == pNode); taosRemoveFromTrashCan(pCacheObj, pNode->pTNodeHeader); } } else { - taosHashRemove(pCacheObj->pHashTable, pNode->key, pNode->keySize); - __cache_unlock(pCacheObj); + int32_t success = taosHashRemove(pCacheObj->pHashTable, pNode->key, pNode->keySize); if (ref > 0) { assert(pNode->pTNodeHeader == NULL); // todo trashcan lock - taosAddToTrash(pCacheObj, pNode); + if (success) { + taosAddToTrash(pCacheObj, pNode); + } +// __cache_unlock(pCacheObj); } else { +// __cache_unlock(pCacheObj); + // taosCacheReleaseNode(pCacheObj, pNode); atomic_fetch_sub_ptr(&pCacheObj->totalSize, pNode->size); uDebug("cache:%s, key:%p, %p is destroyed from cache, totalNum:%d totalSize:%" PRId64 "bytes size:%dbytes", From b52bae71ae2e7f8054ba77a9b6af37328972f729 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 31 Jul 2020 00:54:31 +0800 Subject: [PATCH 016/109] [td-225] refactor codes. --- src/util/src/tcache.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index a5317cdda1..77f3ea2db0 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -300,7 +300,7 @@ void *taosCacheAcquireByKey(SCacheObj *pCacheObj, const void *key, size_t keyLen void *pData = NULL; - __cache_rd_lock(pCacheObj); +// __cache_rd_lock(pCacheObj); SCacheDataNode **ptNode = (SCacheDataNode **)taosHashGet(pCacheObj->pHashTable, key, keyLen); int32_t ref = 0; @@ -309,7 +309,7 @@ void *taosCacheAcquireByKey(SCacheObj *pCacheObj, const void *key, size_t keyLen pData = (*ptNode)->data; } - __cache_unlock(pCacheObj); +// __cache_unlock(pCacheObj); if (pData != NULL) { atomic_add_fetch_32(&pCacheObj->statistics.hitCount, 1); From ce4a2802cd5332b12c1c9034d3436a4a1a7aba90 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 31 Jul 2020 11:32:54 +0800 Subject: [PATCH 017/109] [td-225] update the hash func --- src/client/src/tscParseInsert.c | 2 +- src/client/src/tscUtil.c | 2 +- src/mnode/src/mnodeSdb.c | 2 +- src/mnode/src/mnodeTable.c | 2 +- src/query/src/qExecutor.c | 2 +- src/query/src/qResultbuf.c | 4 +- src/query/src/qTokenizer.c | 2 +- src/query/src/qUtil.c | 4 +- src/rpc/src/rpcMain.c | 2 +- src/sync/src/syncMain.c | 2 +- src/tsdb/src/tsdbMeta.c | 2 +- src/util/inc/hash.h | 6 +- src/util/src/hash.c | 50 ++++++++++++++-- src/util/src/tcache.c | 103 +++++++++++++++----------------- src/util/src/tkvstore.c | 2 +- src/vnode/src/vnodeMain.c | 2 +- 16 files changed, 114 insertions(+), 75 deletions(-) diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index ae2370cd56..d113a43ecf 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -1039,7 +1039,7 @@ int tsParseInsertSql(SSqlObj *pSql) { } if (NULL == pCmd->pTableList) { - pCmd->pTableList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false); + pCmd->pTableList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); pCmd->pDataBlocks = taosArrayInit(4, POINTER_BYTES); if (NULL == pCmd->pTableList || NULL == pSql->cmd.pDataBlocks) { code = TSDB_CODE_TSC_OUT_OF_MEMORY; diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 5ee3db36d1..9c2454e76c 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -646,7 +646,7 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SArray* pTableDataBlockList) { STableDataBlocks* pOneTableBlock = taosArrayGetP(pTableDataBlockList, 0); int32_t expandSize = getRowExpandSize(pOneTableBlock->pTableMeta); - void* pVnodeDataBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false); + void* pVnodeDataBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); SArray* pVnodeDataBlockList = taosArrayInit(8, POINTER_BYTES); size_t total = taosArrayGetSize(pTableDataBlockList); diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 26efcfeac0..f09541d306 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -814,7 +814,7 @@ void *sdbOpenTable(SSdbTableDesc *pDesc) { if (pTable->keyType == SDB_KEY_STRING || pTable->keyType == SDB_KEY_VAR_STRING) { hashFp = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); } - pTable->iHandle = taosHashInit(pTable->hashSessions, hashFp, true); + pTable->iHandle = taosHashInit(pTable->hashSessions, hashFp, true, true); tsSdbObj.numOfTables++; tsSdbObj.tableList[pTable->tableId] = pTable; diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 053a1522a7..85afd70b83 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -378,7 +378,7 @@ static void mnodeAddTableIntoStable(SSuperTableObj *pStable, SChildTableObj *pCt atomic_add_fetch_32(&pStable->numOfTables, 1); if (pStable->vgHash == NULL) { - pStable->vgHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false); + pStable->vgHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false); } if (pStable->vgHash != NULL) { diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 66762a1ca5..8afc44f939 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -5841,7 +5841,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList, pQInfo->tableqinfoGroupInfo.pGroupList = taosArrayInit(numOfGroups, POINTER_BYTES); pQInfo->tableqinfoGroupInfo.numOfTables = pTableGroupInfo->numOfTables; pQInfo->tableqinfoGroupInfo.map = taosHashInit(pTableGroupInfo->numOfTables, - taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false); + taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false); } int tableIndex = 0; diff --git a/src/query/src/qResultbuf.c b/src/query/src/qResultbuf.c index 93ccad8e27..d2a124aa30 100644 --- a/src/query/src/qResultbuf.c +++ b/src/query/src/qResultbuf.c @@ -34,9 +34,9 @@ int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t ro pResBuf->lruList = tdListNew(POINTER_BYTES); // init id hash table - pResBuf->groupSet = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false); + pResBuf->groupSet = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false); pResBuf->assistBuf = malloc(pResBuf->pageSize + 2); // EXTRA BYTES - pResBuf->all = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false); + pResBuf->all = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false); char path[PATH_MAX] = {0}; getTmpfilePath("qbuf", path); diff --git a/src/query/src/qTokenizer.c b/src/query/src/qTokenizer.c index 0ea0ff7bf3..a22c4d5f65 100644 --- a/src/query/src/qTokenizer.c +++ b/src/query/src/qTokenizer.c @@ -256,7 +256,7 @@ static void* KeywordHashTable = NULL; static void doInitKeywordsTable() { int numOfEntries = tListLen(keywordTable); - KeywordHashTable = taosHashInit(numOfEntries, MurmurHash3_32, false); + KeywordHashTable = taosHashInit(numOfEntries, MurmurHash3_32, true, false); for (int32_t i = 0; i < numOfEntries; i++) { keywordTable[i].len = strlen(keywordTable[i].name); void* ptr = &keywordTable[i]; diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index c9143b3a53..ff3c6b20bb 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -40,7 +40,7 @@ int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, SQueryRuntimeEnv *pRun pWindowResInfo->type = type; _hash_fn_t fn = taosGetDefaultHashFunction(type); - pWindowResInfo->hashList = taosHashInit(threshold, fn, false); + pWindowResInfo->hashList = taosHashInit(threshold, fn, true, false); if (pWindowResInfo->hashList == NULL) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } @@ -107,7 +107,7 @@ void resetTimeWindowInfo(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowR pWindowResInfo->size = 0; _hash_fn_t fn = taosGetDefaultHashFunction(pWindowResInfo->type); - pWindowResInfo->hashList = taosHashInit(pWindowResInfo->capacity, fn, false); + pWindowResInfo->hashList = taosHashInit(pWindowResInfo->capacity, fn, true, false); pWindowResInfo->startTime = TSKEY_INITIAL_VAL; pWindowResInfo->prevSKey = TSKEY_INITIAL_VAL; diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index 7425558535..4acf95d4f4 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -261,7 +261,7 @@ void *rpcOpen(const SRpcInit *pInit) { } if (pRpc->connType == TAOS_CONN_SERVER) { - pRpc->hash = taosHashInit(pRpc->sessions, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true); + pRpc->hash = taosHashInit(pRpc->sessions, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, true); if (pRpc->hash == NULL) { tError("%s failed to init string hash", pRpc->label); rpcClose(pRpc); diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index 93c4a9402f..afd765f2c2 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -96,7 +96,7 @@ static void syncModuleInitFunc() { return; } - vgIdHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true); + vgIdHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, true); if (vgIdHash == NULL) { taosTmrCleanUp(syncTmrCtrl); taosCloseTcpThreadPool(tsTcpPool); diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 09bbbd8f4d..5fc2f3e253 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -436,7 +436,7 @@ STsdbMeta *tsdbNewMeta(STsdbCfg *pCfg) { goto _err; } - pMeta->uidMap = taosHashInit(TSDB_INIT_NTABLES * 1.1, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false); + pMeta->uidMap = taosHashInit(TSDB_INIT_NTABLES * 1.1, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); if (pMeta->uidMap == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _err; diff --git a/src/util/inc/hash.h b/src/util/inc/hash.h index d3edc89585..3c99a36764 100644 --- a/src/util/inc/hash.h +++ b/src/util/inc/hash.h @@ -44,6 +44,7 @@ typedef struct SHashObj { _hash_fn_t hashFp; // hash function _hash_free_fn_t freeFp; // hash node free callback function + bool enableUpdate; // enable update #if defined(LINUX) pthread_rwlock_t *lock; #else @@ -67,7 +68,7 @@ typedef struct SHashMutableIterator { * @param threadsafe thread safe or not * @return */ -SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool threadsafe); +SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, bool threadsafe); /** * return the size of hash table @@ -105,7 +106,8 @@ void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen); */ int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen); -void taosHashRemoveNode(); + +void* taosHashRemoveNode(SHashObj *pHashObj, const void *key, size_t keyLen); /** * clean up hash table diff --git a/src/util/src/hash.c b/src/util/src/hash.c index f59f25d153..e560a9c744 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -163,7 +163,7 @@ static void doAddToHashTable(SHashObj *pHashObj, SHashNode *pNode); */ static SHashNode *getNextHashNode(SHashMutableIterator *pIter); -SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool threadsafe) { +SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, bool threadsafe) { if (capacity == 0 || fn == NULL) { return NULL; } @@ -179,6 +179,7 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool threadsafe) { assert((pHashObj->capacity & (pHashObj->capacity - 1)) == 0); pHashObj->hashFp = fn; + pHashObj->enableUpdate = update; pHashObj->hashList = (SHashNode **)calloc(pHashObj->capacity, POINTER_BYTES); if (pHashObj->hashList == NULL) { @@ -232,15 +233,21 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da doAddToHashTable(pHashObj, pNewNode); __unlock(pHashObj->lock); + + return 0; } else { - doUpdateHashNode(pNode, pNewNode); + // not support the update operation, return error + if (pHashObj->enableUpdate) { + doUpdateHashNode(pNode, pNewNode); + } + __unlock(pHashObj->lock); tfree(pNewNode->data) tfree(pNewNode); - } - return 0; + return pHashObj->enableUpdate? 0:-1; + } } void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen) { @@ -288,10 +295,45 @@ int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen) { pNode->next = NULL; pNode->prev = NULL; + tfree(pNode->data); tfree(pNode); + return 0; } +void* taosHashRemoveNode(SHashObj *pHashObj, const void *key, size_t keyLen) { + uint32_t hashVal = (*pHashObj->hashFp)(key, keyLen); + + __wr_lock(pHashObj->lock); + SHashNode *pNode = doGetNodeFromHashTable(pHashObj, key, keyLen, hashVal); + if (pNode == NULL) { + __unlock(pHashObj->lock); + return NULL; + } + + SHashNode *pNext = pNode->next; + if (pNode->prev == NULL) { + int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); + assert(pHashObj->hashList[slot] == pNode); + + pHashObj->hashList[slot] = pNext; + } else { + pNode->prev->next = pNext; + } + + if (pNext != NULL) { + pNext->prev = pNode->prev; + } + + pHashObj->size -= 1; + __unlock(pHashObj->lock); + + pNode->next = NULL; + pNode->prev = NULL; + + return pNode; +} + void taosHashCleanup(SHashObj *pHashObj) { if (pHashObj == NULL) return; diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index 77f3ea2db0..2432e97552 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -207,7 +207,7 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool ext return NULL; } - pCacheObj->pHashTable = taosHashInit(128, taosGetDefaultHashFunction(keyType), true); + pCacheObj->pHashTable = taosHashInit(128, taosGetDefaultHashFunction(keyType), false, true); pCacheObj->name = strdup(cacheName); if (pCacheObj->pHashTable == NULL) { free(pCacheObj); @@ -249,45 +249,49 @@ void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const v return NULL; } - __cache_wr_lock(pCacheObj); - SCacheDataNode **pt = (SCacheDataNode **)taosHashGet(pCacheObj->pHashTable, key, keyLen); - SCacheDataNode * pOld = (pt != NULL) ? (*pt) : NULL; - - if (pOld == NULL) { // do addedTime to cache - T_REF_INC(pNode1); - taosHashPut(pCacheObj->pHashTable, key, keyLen, &pNode1, sizeof(void *)); - __cache_unlock(pCacheObj); +// __cache_wr_lock(pCacheObj); + T_REF_INC(pNode1); + int32_t succ = taosHashPut(pCacheObj->pHashTable, key, keyLen, &pNode1, sizeof(void *)); + if (succ == 0) { atomic_add_fetch_64(&pCacheObj->totalSize, pNode1->size); - uDebug("cache:%s, key:%p, %p added into cache, added:%" PRIu64 ", expire:%" PRIu64 ", totalNum:%d totalSize:%" PRId64 - "bytes size:%" PRId64 "bytes", + uDebug("cache:%s, key:%p, %p added into cache, added:%" PRIu64 ", expire:%" PRIu64 + ", totalNum:%d totalSize:%" PRId64 "bytes size:%" PRId64 "bytes", pCacheObj->name, key, pNode1->data, pNode1->addedTime, pNode1->expireTime, (int32_t)taosHashGetSize(pCacheObj->pHashTable), pCacheObj->totalSize, (int64_t)dataSize); - } else { // old data exists, update the node - bool addToTrashcan = false; - if (T_REF_VAL_GET(pOld) > 0) { - + } else { // duplicated key exists + while (1) { // todo removed by node, instead of by key - int32_t succ = taosHashRemove(pCacheObj->pHashTable, pOld->key, pOld->keySize); - assert(succ == 0); + SHashNode *p = taosHashRemoveNode(pCacheObj->pHashTable, key, keyLen); - addToTrashcan = true; + // add to trashcan + if (p != NULL) { + SCacheDataNode* pCachedNode = *(SCacheDataNode**)p->data; + if (T_REF_VAL_GET(pCachedNode) == 0) { + tfree(pCachedNode); + } else { + taosAddToTrash(pCacheObj, pCachedNode); + uDebug("cache:%s, key:%p, %p exist in cache, updated old:%p", pCacheObj->name, key, pNode1->data, pCachedNode); + } + } + + assert(T_REF_VAL_GET(pNode1) == 1); + + int32_t ret = taosHashPut(pCacheObj->pHashTable, key, keyLen, &pNode1, sizeof(void *)); + if (ret == 0) { + atomic_add_fetch_64(&pCacheObj->totalSize, pNode1->size); + + uDebug("cache:%s, key:%p, %p added into cache, added:%" PRIu64 ", expire:%" PRIu64 + ", totalNum:%d totalSize:%" PRId64 "bytes size:%" PRId64 "bytes", + pCacheObj->name, key, pNode1->data, pNode1->addedTime, pNode1->expireTime, + (int32_t)taosHashGetSize(pCacheObj->pHashTable), pCacheObj->totalSize, (int64_t)dataSize); + + return pNode1; + + } else { + // failed, try again + } } - - T_REF_INC(pNode1); - - // addedTime new element to hashtable - taosHashPut(pCacheObj->pHashTable, key, keyLen, &pNode1, sizeof(void *)); - __cache_unlock(pCacheObj); - - // todo add trashcan lock - if (addToTrashcan) { - taosAddToTrash(pCacheObj, pOld); - } else { - free(pOld); - } - - uDebug("cache:%s, key:%p, %p exist in cache, updated old:%p", pCacheObj->name, key, pNode1->data, pOld); } return pNode1->data; @@ -415,8 +419,6 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { } if (_remove) { -// __cache_wr_lock(pCacheObj); - // NOTE: once refcount is decrease, pNode may be freed by other thread immediately. int32_t ref = T_REF_DEC(pNode); uDebug("cache:%s, key:%p, %p is released, refcnt:%d", pCacheObj->name, pNode->key, pNode->data, ref); @@ -429,34 +431,27 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { * that tries to do the same thing. */ if (pNode->inTrashCan) { -// __cache_unlock(pCacheObj); - if (ref == 0) { assert(pNode->pTNodeHeader->pData == pNode); taosRemoveFromTrashCan(pCacheObj, pNode->pTNodeHeader); } } else { int32_t success = taosHashRemove(pCacheObj->pHashTable, pNode->key, pNode->keySize); + if (success) { + if (ref > 0) { + assert(pNode->pTNodeHeader == NULL); - if (ref > 0) { - assert(pNode->pTNodeHeader == NULL); - - // todo trashcan lock - if (success) { + // todo trashcan lock taosAddToTrash(pCacheObj, pNode); + } else { // ref == 0 + atomic_fetch_sub_ptr(&pCacheObj->totalSize, pNode->size); + uDebug("cache:%s, key:%p, %p is destroyed from cache, totalNum:%d totalSize:%" PRId64 "bytes size:%dbytes", + pCacheObj->name, pNode->key, pNode->data, (int32_t)taosHashGetSize(pCacheObj->pHashTable), + pCacheObj->totalSize, pNode->size); + + if (pCacheObj->freeFp) pCacheObj->freeFp(pNode->data); + free(pNode); } -// __cache_unlock(pCacheObj); - } else { -// __cache_unlock(pCacheObj); - -// taosCacheReleaseNode(pCacheObj, pNode); - atomic_fetch_sub_ptr(&pCacheObj->totalSize, pNode->size); - uDebug("cache:%s, key:%p, %p is destroyed from cache, totalNum:%d totalSize:%" PRId64 "bytes size:%dbytes", - pCacheObj->name, pNode->key, pNode->data, (int32_t)taosHashGetSize(pCacheObj->pHashTable), pCacheObj->totalSize, - pNode->size); - - if (pCacheObj->freeFp) pCacheObj->freeFp(pNode->data); - free(pNode); } } diff --git a/src/util/src/tkvstore.c b/src/util/src/tkvstore.c index d7bf9d7857..602ea8c96d 100644 --- a/src/util/src/tkvstore.c +++ b/src/util/src/tkvstore.c @@ -419,7 +419,7 @@ static SKVStore *tdNewKVStore(char *fname, iterFunc iFunc, afterFunc aFunc, void pStore->iFunc = iFunc; pStore->aFunc = aFunc; pStore->appH = appH; - pStore->map = taosHashInit(4096, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false); + pStore->map = taosHashInit(4096, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); if (pStore->map == NULL) { terrno = TSDB_CODE_COM_OUT_OF_MEMORY; goto _err; diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 77d4503d9d..e8919e5fce 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -50,7 +50,7 @@ int32_t vnodeInitResources() { vnodeInitWriteFp(); vnodeInitReadFp(); - tsDnodeVnodesHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true); + tsDnodeVnodesHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, true); if (tsDnodeVnodesHash == NULL) { vError("failed to init vnode list"); return TSDB_CODE_VND_OUT_OF_MEMORY; From 97616d2ad1c664f95f549bbc61d51575dbe8dd96 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 31 Jul 2020 11:50:13 +0800 Subject: [PATCH 018/109] [td-225] update the hash func --- src/util/src/tcache.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index 2432e97552..4e2c41d1bc 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -436,8 +436,8 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { taosRemoveFromTrashCan(pCacheObj, pNode->pTNodeHeader); } } else { - int32_t success = taosHashRemove(pCacheObj->pHashTable, pNode->key, pNode->keySize); - if (success) { + int32_t ret = taosHashRemove(pCacheObj->pHashTable, pNode->key, pNode->keySize); + if (ret == 0) { if (ref > 0) { assert(pNode->pTNodeHeader == NULL); From 285d732e7fa994d3fa09790c1134451a727a07f2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 31 Jul 2020 16:16:32 +0800 Subject: [PATCH 019/109] [td-225] update the hash func --- src/util/inc/hash.h | 49 +++-- src/util/src/hash.c | 418 ++++++++++++++++++++++++------------ src/util/src/tcache.c | 32 +-- src/util/tests/hashTest.cpp | 6 +- 4 files changed, 330 insertions(+), 175 deletions(-) diff --git a/src/util/inc/hash.h b/src/util/inc/hash.h index 3c99a36764..c5168a92ee 100644 --- a/src/util/inc/hash.h +++ b/src/util/inc/hash.h @@ -20,7 +20,9 @@ extern "C" { #endif +#include "tarray.h" #include "hashfunc.h" +#include "tlockfree.h" #define HASH_MAX_CAPACITY (1024 * 1024 * 16) #define HASH_DEFAULT_LOAD_FACTOR (0.75) @@ -37,23 +39,41 @@ typedef struct SHashNode { char *data; } SHashNode; -typedef struct SHashObj { - SHashNode **hashList; - size_t capacity; // number of slots - size_t size; // number of elements in hash table - _hash_fn_t hashFp; // hash function - _hash_free_fn_t freeFp; // hash node free callback function +typedef enum SHashLockTypeE { + HASH_NO_LOCK = 0, + HASH_GLOBAL_LOCK = 1, + HASH_ENTRY_LOCK = 2, +} SHashLockTypeE; - bool enableUpdate; // enable update +typedef struct SHashLock { #if defined(LINUX) pthread_rwlock_t *lock; #else - pthread_mutex_t *lock; + pthread_mutex_t *lock; #endif +} SHashLock; + +typedef struct SHashEntry { + int32_t num; // number of elements in current entry + SRWLatch latch; // entry latch + SHashNode head; // dummy head +} SHashEntry; + +typedef struct SHashObj { + SHashEntry **hashList; + size_t capacity; // number of slots + size_t size; // number of elements in hash table + _hash_fn_t hashFp; // hash function + _hash_free_fn_t freeFp; // hash node free callback function + + SHashLock lock; + SHashLockTypeE lockType; // lock type + bool enableUpdate; // enable update + SArray *pMemBlock; // memory block allocated for SHashEntry } SHashObj; typedef struct SHashMutableIterator { - SHashObj * pHashObj; + SHashObj *pHashObj; int32_t entryIndex; SHashNode *pCur; SHashNode *pNext; // current node can be deleted for mutable iterator, so keep the next one before return current @@ -68,7 +88,7 @@ typedef struct SHashMutableIterator { * @param threadsafe thread safe or not * @return */ -SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, bool threadsafe); +SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTypeE type); /** * return the size of hash table @@ -107,7 +127,7 @@ void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen); int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen); -void* taosHashRemoveNode(SHashObj *pHashObj, const void *key, size_t keyLen); +int32_t taosHashRemoveNode(SHashObj *pHashObj, const void *key, size_t keyLen, void* data, size_t dsize); /** * clean up hash table @@ -115,13 +135,6 @@ void* taosHashRemoveNode(SHashObj *pHashObj, const void *key, size_t keyLen); */ void taosHashCleanup(SHashObj *pHashObj); -/** - * Set the free callback function - * This function if set will be invoked right before freeing each hash node - * @param pHashObj - */ -void taosHashSetFreecb(SHashObj *pHashObj, _hash_free_fn_t freeFp); - /** * * @param pHashObj diff --git a/src/util/src/hash.c b/src/util/src/hash.c index e560a9c744..bff8d1e50a 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -100,7 +100,19 @@ static FORCE_INLINE int32_t taosHashCapacity(int32_t length) { FORCE_INLINE SHashNode *doGetNodeFromHashTable(SHashObj *pHashObj, const void *key, uint32_t keyLen, uint32_t hashVal) { int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); - SHashNode *pNode = pHashObj->hashList[slot]; + SHashEntry *pe = pHashObj->hashList[slot]; + + // no data, return directly + int32_t num = atomic_load_32(&pe->num); + if (num == 0) { + return NULL; + } + + if (pHashObj->lockType == HASH_ENTRY_LOCK) { + taosRLockLatch(&pe->latch); + } + + SHashNode* pNode = pe->head.next; while (pNode) { if ((pNode->keyLen == keyLen) && (memcmp(pNode->key, key, keyLen) == 0)) { assert(pNode->hashVal == hashVal); @@ -109,7 +121,11 @@ FORCE_INLINE SHashNode *doGetNodeFromHashTable(SHashObj *pHashObj, const void *k pNode = pNode->next; } - + + if (pHashObj->lockType == HASH_ENTRY_LOCK) { + taosRUnLockLatch(&pe->latch); + } + return pNode; } @@ -154,7 +170,7 @@ static FORCE_INLINE SHashNode *doUpdateHashNode(SHashNode *pNode, SHashNode *pNe * @param pHashObj * @param pNode */ -static void doAddToHashTable(SHashObj *pHashObj, SHashNode *pNode); +static void pushfrontNode(SHashEntry* pEntry, SHashNode *pNode); /** * Get the next element in hash table for iterator @@ -163,7 +179,7 @@ static void doAddToHashTable(SHashObj *pHashObj, SHashNode *pNode); */ static SHashNode *getNextHashNode(SHashMutableIterator *pIter); -SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, bool threadsafe) { +SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTypeE type) { if (capacity == 0 || fn == NULL) { return NULL; } @@ -179,24 +195,35 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, bool threads assert((pHashObj->capacity & (pHashObj->capacity - 1)) == 0); pHashObj->hashFp = fn; + pHashObj->lockType = type; pHashObj->enableUpdate = update; - pHashObj->hashList = (SHashNode **)calloc(pHashObj->capacity, POINTER_BYTES); + pHashObj->hashList = (SHashEntry **)calloc(pHashObj->capacity, sizeof(void*)); if (pHashObj->hashList == NULL) { free(pHashObj); uError("failed to allocate memory, reason:%s", strerror(errno)); return NULL; + } else { + + pHashObj->pMemBlock = taosArrayInit(8, sizeof(void*)); + + void* p = calloc(pHashObj->capacity, sizeof(SHashEntry)); + for(int32_t i = 0; i < pHashObj->capacity; ++i) { + pHashObj->hashList[i] = p + i * sizeof(SHashEntry); + } + + taosArrayPush(pHashObj->pMemBlock, &p); } - if (threadsafe) { + if (pHashObj->lockType != HASH_NO_LOCK) { #if defined(LINUX) - pHashObj->lock = calloc(1, sizeof(pthread_rwlock_t)); + pHashObj->lock.lock = calloc(1, sizeof(pthread_rwlock_t)); #else - pHashObj->lock = calloc(1, sizeof(pthread_mutex_t)); + pHashObj->lock.lock = calloc(1, sizeof(pthread_mutex_t)); #endif } - if (__lock_init(pHashObj->lock) != 0) { + if (__lock_init(pHashObj->lock.lock) != 0) { free(pHashObj->hashList); free(pHashObj); @@ -208,11 +235,7 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, bool threads } size_t taosHashGetSize(const SHashObj *pHashObj) { - if (pHashObj == NULL) { - return 0; - } - - return pHashObj->size; + return (pHashObj == NULL)? 0:pHashObj->size; } int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size) { @@ -222,17 +245,43 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da return -1; } - __wr_lock(pHashObj->lock); - SHashNode *pNode = doGetNodeFromHashTable(pHashObj, key, keyLen, hashVal); + // need the resize process, write lock applied + if (HASH_NEED_RESIZE(pHashObj)) { + __wr_lock(pHashObj->lock.lock); + taosHashTableResize(pHashObj); + __unlock(pHashObj->lock.lock); + } - if (pNode == NULL) { // no data in hash table with the specified key, add it into hash table + __rd_lock(pHashObj->lock.lock); - if (HASH_NEED_RESIZE(pHashObj)) { - taosHashTableResize(pHashObj); + int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); + SHashEntry *pe = pHashObj->hashList[slot]; + + if (pHashObj->lockType == HASH_ENTRY_LOCK) { + taosWLockLatch(&pe->latch); + } + + SHashNode* pNode = pe->head.next; + while (pNode) { + if ((pNode->keyLen == keyLen) && (memcmp(pNode->key, key, keyLen) == 0)) { + assert(pNode->hashVal == hashVal); + break; } - doAddToHashTable(pHashObj, pNewNode); - __unlock(pHashObj->lock); + pNode = pNode->next; + } + + if (pNode == NULL) { + // no data in hash table with the specified key, add it into hash table + pushfrontNode(pe, pNewNode); + + if (pHashObj->lockType == HASH_ENTRY_LOCK) { + taosWUnLockLatch(&pe->latch); + } + + // enable resize + __unlock(pHashObj->lock.lock); + atomic_add_fetch_64(&pHashObj->size, 1); return 0; } else { @@ -241,7 +290,12 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da doUpdateHashNode(pNode, pNewNode); } - __unlock(pHashObj->lock); + if (pHashObj->lockType == HASH_ENTRY_LOCK) { + taosWUnLockLatch(&pe->latch); + } + + // enable resize + __unlock(pHashObj->lock.lock); tfree(pNewNode->data) tfree(pNewNode); @@ -251,11 +305,18 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da } void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen) { + if (pHashObj->size <= 0 || keyLen == 0 || key == NULL) { + return NULL; + } + uint32_t hashVal = (*pHashObj->hashFp)(key, keyLen); - __rd_lock(pHashObj->lock); + // only add the read lock to disable the resize process + __rd_lock(pHashObj->lock.lock); + SHashNode *pNode = doGetNodeFromHashTable(pHashObj, key, keyLen, hashVal); - __unlock(pHashObj->lock); + + __unlock(pHashObj->lock.lock); if (pNode) { assert(pNode->hashVal == hashVal); @@ -266,91 +327,167 @@ void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen) { } int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen) { - uint32_t hashVal = (*pHashObj->hashFp)(key, keyLen); - - __wr_lock(pHashObj->lock); - SHashNode *pNode = doGetNodeFromHashTable(pHashObj, key, keyLen, hashVal); - if (pNode == NULL) { - __unlock(pHashObj->lock); + if (pHashObj->size <= 0) { return -1; } - SHashNode *pNext = pNode->next; - if (pNode->prev == NULL) { - int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); - assert(pHashObj->hashList[slot] == pNode); - - pHashObj->hashList[slot] = pNext; - } else { - pNode->prev->next = pNext; - } - - if (pNext != NULL) { - pNext->prev = pNode->prev; - } - - pHashObj->size -= 1; - __unlock(pHashObj->lock); - - pNode->next = NULL; - pNode->prev = NULL; - - tfree(pNode->data); - tfree(pNode); - - return 0; -} - -void* taosHashRemoveNode(SHashObj *pHashObj, const void *key, size_t keyLen) { uint32_t hashVal = (*pHashObj->hashFp)(key, keyLen); - __wr_lock(pHashObj->lock); - SHashNode *pNode = doGetNodeFromHashTable(pHashObj, key, keyLen, hashVal); - if (pNode == NULL) { - __unlock(pHashObj->lock); - return NULL; + // disable the resize process + __rd_lock(pHashObj->lock.lock); + + int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); + SHashEntry *pe = pHashObj->hashList[slot]; + + // no data, return directly + if (pe->num == 0) { + __unlock(pHashObj->lock.lock); + return -1; } - SHashNode *pNext = pNode->next; - if (pNode->prev == NULL) { - int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); - assert(pHashObj->hashList[slot] == pNode); + if (pHashObj->lockType == HASH_ENTRY_LOCK) { + taosWLockLatch(&pe->latch); + } - pHashObj->hashList[slot] = pNext; - } else { + SHashNode* pNode = pe->head.next; + while (pNode) { + if ((pNode->keyLen == keyLen) && (memcmp(pNode->key, key, keyLen) == 0)) { + assert(pNode->hashVal == hashVal); + break; + } + + pNode = pNode->next; + } + + if (pNode != NULL) { + assert(pNode->prev != NULL); + + SHashNode *pNext = pNode->next; pNode->prev->next = pNext; + + if (pNext != NULL) { + pNext->prev = pNode->prev; + } + + pe->num -= 1; } - if (pNext != NULL) { - pNext->prev = pNode->prev; + if (pHashObj->lockType == HASH_ENTRY_LOCK) { + taosWUnLockLatch(&pe->latch); } - pHashObj->size -= 1; - __unlock(pHashObj->lock); + __unlock(pHashObj->lock.lock); - pNode->next = NULL; - pNode->prev = NULL; + if (pNode != NULL) { + atomic_sub_fetch_64(&pHashObj->size, 1); - return pNode; + pNode->next = NULL; + pNode->prev = NULL; + + tfree(pNode->data); + tfree(pNode); + + return 0; + } else { + return -1; + } +} + +int32_t taosHashRemoveNode(SHashObj *pHashObj, const void *key, size_t keyLen, void* data, size_t dsize) { + if (pHashObj->size <= 0) { + return -1; + } + + uint32_t hashVal = (*pHashObj->hashFp)(key, keyLen); + + // disable the resize process + __rd_lock(pHashObj->lock.lock); + + int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); + SHashEntry *pe = pHashObj->hashList[slot]; + + // no data, return directly + if (pe->num == 0) { + __unlock(pHashObj->lock.lock); + return -1; + } + + if (pHashObj->lockType == HASH_ENTRY_LOCK) { + taosWLockLatch(&pe->latch); + } + + SHashNode* pNode = pe->head.next; + while (pNode) { + if ((pNode->keyLen == keyLen) && (memcmp(pNode->key, key, keyLen) == 0)) { + assert(pNode->hashVal == hashVal); + break; + } + + pNode = pNode->next; + } + + if (pNode != NULL) { + assert(pNode->prev != NULL); + + SHashNode *pNext = pNode->next; + pNode->prev->next = pNext; + + if (pNext != NULL) { + pNext->prev = pNode->prev; + } + } + + if (pHashObj->lockType == HASH_ENTRY_LOCK) { + pe->num -= 1; + taosWUnLockLatch(&pe->latch); + } + + __unlock(pHashObj->lock.lock); + + atomic_sub_fetch_64(&pHashObj->size, 1); + + if (data != NULL) { + memcpy(data, pNode->data, dsize); + } + + if (pNode != NULL) { + pNode->next = NULL; + pNode->prev = NULL; + + tfree(pNode->data); + tfree(pNode); + + return 0; + } else { + return -1; + } } void taosHashCleanup(SHashObj *pHashObj) { - if (pHashObj == NULL) return; + if (pHashObj == NULL) { + return; + } SHashNode *pNode, *pNext; - __wr_lock(pHashObj->lock); + __wr_lock(pHashObj->lock.lock); if (pHashObj->hashList) { for (int32_t i = 0; i < pHashObj->capacity; ++i) { - pNode = pHashObj->hashList[i]; + SHashEntry* pEntry = pHashObj->hashList[i]; + if (pEntry->num == 0) { + assert(pEntry->head.next == 0); + continue; + } + pNode = pEntry->head.next; while (pNode) { pNext = pNode->next; if (pHashObj->freeFp) { pHashObj->freeFp(pNode->data); } + free(pNode->data); free(pNode); pNode = pNext; } @@ -359,10 +496,20 @@ void taosHashCleanup(SHashObj *pHashObj) { free(pHashObj->hashList); } - __unlock(pHashObj->lock); - __lock_destroy(pHashObj->lock); + __unlock(pHashObj->lock.lock); + __lock_destroy(pHashObj->lock.lock); + + tfree(pHashObj->lock.lock); + + // destroy mem block + size_t memBlock = taosArrayGetSize(pHashObj->pMemBlock); + for(int32_t i = 0; i < memBlock; ++i) { + void* p = taosArrayGetP(pHashObj->pMemBlock, i); + tfree(p); + } + + taosArrayDestroy(pHashObj->pMemBlock); - tfree(pHashObj->lock); memset(pHashObj, 0, sizeof(SHashObj)); free(pHashObj); } @@ -392,13 +539,13 @@ bool taosHashIterNext(SHashMutableIterator *pIter) { assert(pIter->pCur == NULL && pIter->pNext == NULL); while (1) { - SHashNode *pEntry = pIter->pHashObj->hashList[pIter->entryIndex]; - if (pEntry == NULL) { + SHashEntry *pEntry = pIter->pHashObj->hashList[pIter->entryIndex]; + if (pEntry->num == 0) { pIter->entryIndex++; continue; } - pIter->pCur = pEntry; + pIter->pCur = pEntry->head.next; if (pIter->pCur->next) { pIter->pNext = pIter->pCur->next; @@ -451,19 +598,9 @@ int32_t taosHashGetMaxOverflowLinkLength(const SHashObj *pHashObj) { int32_t num = 0; for (int32_t i = 0; i < pHashObj->size; ++i) { - SHashNode *pEntry = pHashObj->hashList[i]; - if (pEntry == NULL) { - continue; - } - - int32_t j = 0; - while(pEntry != NULL) { - pEntry = pEntry->next; - j++; - } - - if (num < j) { - num = j; + SHashEntry *pEntry = pHashObj->hashList[i]; + if (num < pEntry->num) { + num = pEntry->num; } } @@ -471,7 +608,7 @@ int32_t taosHashGetMaxOverflowLinkLength(const SHashObj *pHashObj) { } void taosHashTableResize(SHashObj *pHashObj) { - if (pHashObj->size < pHashObj->capacity * HASH_DEFAULT_LOAD_FACTOR) { + if (!HASH_NEED_RESIZE(pHashObj)) { return; } @@ -486,37 +623,43 @@ void taosHashTableResize(SHashObj *pHashObj) { return; } - int32_t pointerSize = POINTER_BYTES; - void *pNewEntry = realloc(pHashObj->hashList, pointerSize * newSize); - if (pNewEntry == NULL) {// todo handle error + void *pNewEntryList = realloc(pHashObj->hashList, sizeof(SHashEntry) * newSize); + if (pNewEntryList == NULL) {// todo handle error // uDebug("cache resize failed due to out of memory, capacity remain:%d", pHashObj->capacity); return; } - pHashObj->hashList = pNewEntry; - memset(&pHashObj->hashList[pHashObj->capacity], 0, POINTER_BYTES * (newSize - pHashObj->capacity)); - + pHashObj->hashList = pNewEntryList; + + size_t inc = newSize - pHashObj->capacity; + void* p = calloc(inc, sizeof(SHashEntry)); + + for(int32_t i = 0; i < inc; ++i) { + pHashObj->hashList[i + pHashObj->capacity] = p + i * sizeof(SHashEntry); + } + + taosArrayPush(pHashObj->pMemBlock, &p); + pHashObj->capacity = newSize; - for (int32_t i = 0; i < pHashObj->capacity; ++i) { - pNode = pHashObj->hashList[i]; - if (pNode != NULL) { - assert(pNode->prev == NULL); + SHashEntry* pe = pHashObj->hashList[i]; + if (pe->num == 0) { + assert(pe->head.next == NULL); + continue; } - + + pNode = pe->head.next; while (pNode) { int32_t j = HASH_INDEX(pNode->hashVal, pHashObj->capacity); if (j == i) { // this key locates in the same slot, no need to relocate it pNode = pNode->next; + assert(pNode == NULL || pNode->next != pNode); } else { pNext = pNode->next; - - if (pNode->prev == NULL) { // first node of the overflow linked list - pHashObj->hashList[i] = pNext; - } else { - pNode->prev->next = pNext; - } - + assert(pNode != pNext && (pNext == NULL || pNext->prev == pNode) && pNode->prev->next == pNode); + + assert(pNode->prev != NULL); + pNode->prev->next = pNext; if (pNext != NULL) { pNext->prev = pNode->prev; } @@ -524,17 +667,12 @@ void taosHashTableResize(SHashObj *pHashObj) { // clear pointer pNode->next = NULL; pNode->prev = NULL; + pe->num -= 1; // added into new slot - SHashNode *pNew = pHashObj->hashList[j]; - if (pNew != NULL) { - assert(pNew->prev == NULL); - pNew->prev = pNode; - } - - pNode->next = pNew; - pHashObj->hashList[j] = pNode; - + SHashEntry *pNewEntry = pHashObj->hashList[j]; + pushfrontNode(pNewEntry, pNode); + // continue pNode = pNext; } @@ -563,21 +701,19 @@ SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, s return pNewNode; } -void doAddToHashTable(SHashObj *pHashObj, SHashNode *pNode) { - assert(pNode != NULL); +void pushfrontNode(SHashEntry* pEntry, SHashNode *pNode) { + assert(pNode != NULL && pEntry != NULL); - int32_t index = HASH_INDEX(pNode->hashVal, pHashObj->capacity); - - SHashNode* pEntry = pHashObj->hashList[index]; - if (pEntry != NULL) { - pEntry->prev = pNode; - - pNode->next = pEntry; - pNode->prev = NULL; + SHashNode* pNext = pEntry->head.next; + if (pNext != NULL) { + pNext->prev = pNode; } - - pHashObj->hashList[index] = pNode; - pHashObj->size++; + + pNode->next = pNext; + pNode->prev = &pEntry->head; + pEntry->head.next = pNode; + + pEntry->num += 1; } SHashNode *getNextHashNode(SHashMutableIterator *pIter) { @@ -585,13 +721,13 @@ SHashNode *getNextHashNode(SHashMutableIterator *pIter) { pIter->entryIndex++; while (pIter->entryIndex < pIter->pHashObj->capacity) { - SHashNode *pNode = pIter->pHashObj->hashList[pIter->entryIndex]; - if (pNode == NULL) { + SHashEntry*pEntry = pIter->pHashObj->hashList[pIter->entryIndex]; + if (pEntry->num == 0) { pIter->entryIndex++; continue; } - return pNode; + return pEntry->head.next; } return NULL; diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index 4e2c41d1bc..65b2e99337 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -207,7 +207,7 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool ext return NULL; } - pCacheObj->pHashTable = taosHashInit(128, taosGetDefaultHashFunction(keyType), false, true); + pCacheObj->pHashTable = taosHashInit(128, taosGetDefaultHashFunction(keyType), false, HASH_ENTRY_LOCK); pCacheObj->name = strdup(cacheName); if (pCacheObj->pHashTable == NULL) { free(pCacheObj); @@ -249,7 +249,6 @@ void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const v return NULL; } -// __cache_wr_lock(pCacheObj); T_REF_INC(pNode1); int32_t succ = taosHashPut(pCacheObj->pHashTable, key, keyLen, &pNode1, sizeof(void *)); @@ -261,23 +260,27 @@ void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const v (int32_t)taosHashGetSize(pCacheObj->pHashTable), pCacheObj->totalSize, (int64_t)dataSize); } else { // duplicated key exists while (1) { - // todo removed by node, instead of by key - SHashNode *p = taosHashRemoveNode(pCacheObj->pHashTable, key, keyLen); + SCacheDataNode* p = NULL; + int32_t ret = taosHashRemoveNode(pCacheObj->pHashTable, key, keyLen, (void*) &p, sizeof(void*)); // add to trashcan - if (p != NULL) { - SCacheDataNode* pCachedNode = *(SCacheDataNode**)p->data; - if (T_REF_VAL_GET(pCachedNode) == 0) { - tfree(pCachedNode); + if (ret == 0) { + if (T_REF_VAL_GET(p) == 0) { + + if (pCacheObj->freeFp) { + pCacheObj->freeFp(p->data); + } + + tfree(p); } else { - taosAddToTrash(pCacheObj, pCachedNode); - uDebug("cache:%s, key:%p, %p exist in cache, updated old:%p", pCacheObj->name, key, pNode1->data, pCachedNode); + taosAddToTrash(pCacheObj, p); + uDebug("cache:%s, key:%p, %p exist in cache, updated old:%p", pCacheObj->name, key, pNode1->data, p); } } assert(T_REF_VAL_GET(pNode1) == 1); - int32_t ret = taosHashPut(pCacheObj->pHashTable, key, keyLen, &pNode1, sizeof(void *)); + ret = taosHashPut(pCacheObj->pHashTable, key, keyLen, &pNode1, sizeof(void *)); if (ret == 0) { atomic_add_fetch_64(&pCacheObj->totalSize, pNode1->size); @@ -430,14 +433,16 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { * NOTE: previous ref is 0, and current ref is still 0, remove it. If previous is not 0, there is another thread * that tries to do the same thing. */ - if (pNode->inTrashCan) { + if (inTrashCan) { if (ref == 0) { assert(pNode->pTNodeHeader->pData == pNode); + + // todo add lock here taosRemoveFromTrashCan(pCacheObj, pNode->pTNodeHeader); } } else { int32_t ret = taosHashRemove(pCacheObj->pHashTable, pNode->key, pNode->keySize); - if (ret == 0) { + if (ret == 0) { // successfully remove from hash table if (ref > 0) { assert(pNode->pTNodeHeader == NULL); @@ -459,6 +464,7 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { // NOTE: once refcount is decrease, pNode may be freed by other thread immediately. int32_t ref = T_REF_DEC(pNode); + // todo so, invalid read here! uDebug("cache:%s, key:%p, %p released, refcnt:%d, data in trancan:%d", pCacheObj->name, pNode->key, pNode->data, ref, inTrashCan); } diff --git a/src/util/tests/hashTest.cpp b/src/util/tests/hashTest.cpp index 93a1989741..16b300a831 100644 --- a/src/util/tests/hashTest.cpp +++ b/src/util/tests/hashTest.cpp @@ -10,7 +10,7 @@ namespace { // the simple test code for basic operations void simpleTest() { - auto* hashTable = (SHashObj*) taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false); + SHashObj* hashTable = (SHashObj*) taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_ENTRY_LOCK); ASSERT_EQ(taosHashGetSize(hashTable), 0); // put 400 elements in the hash table @@ -47,7 +47,7 @@ void simpleTest() { } void stringKeyTest() { - auto* hashTable = (SHashObj*) taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false); + auto* hashTable = (SHashObj*) taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); ASSERT_EQ(taosHashGetSize(hashTable), 0); char key[128] = {0}; @@ -97,7 +97,7 @@ void functionTest() { * a single threads situation */ void noLockPerformanceTest() { - auto* hashTable = (SHashObj*) taosHashInit(4096, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false); + auto* hashTable = (SHashObj*) taosHashInit(4096, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); ASSERT_EQ(taosHashGetSize(hashTable), 0); char key[128] = {0}; From 99e15e0d6d1644721442ef3c187232c29b595605 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 31 Jul 2020 16:31:19 +0800 Subject: [PATCH 020/109] [td-225] update the hash func --- src/query/src/qExecutor.c | 2 +- src/util/src/tcache.c | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 8afc44f939..9ae407e078 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -6640,7 +6640,7 @@ void freeqinfoFn(void *qhandle) { } void* qOpenQueryMgmt(int32_t vgId) { - const int32_t REFRESH_HANDLE_INTERVAL = 30; // every 30 seconds, refresh handle pool + const int32_t REFRESH_HANDLE_INTERVAL = 60; // every 30 seconds, refresh handle pool char cacheName[128] = {0}; sprintf(cacheName, "qhandle_%d", vgId); diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index 65b2e99337..3efa31dc84 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -108,7 +108,7 @@ static void taosTrashCanEmpty(SCacheObj *pCacheObj, bool force); /** * release node * @param pCacheObj cache object - * @param pNode data node + * @param pNode data node */ static FORCE_INLINE void taosCacheReleaseNode(SCacheObj *pCacheObj, SCacheDataNode *pNode) { if (pNode->signature != (uint64_t)pNode) { @@ -207,7 +207,7 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool ext return NULL; } - pCacheObj->pHashTable = taosHashInit(128, taosGetDefaultHashFunction(keyType), false, HASH_ENTRY_LOCK); + pCacheObj->pHashTable = taosHashInit(4096, taosGetDefaultHashFunction(keyType), false, HASH_ENTRY_LOCK); pCacheObj->name = strdup(cacheName); if (pCacheObj->pHashTable == NULL) { free(pCacheObj); @@ -648,12 +648,12 @@ void doCleanupDataCache(SCacheObj *pCacheObj) { static void doCacheRefresh(SCacheObj* pCacheObj, int64_t time, __cache_free_fn_t fp) { SHashMutableIterator *pIter = taosHashCreateIter(pCacheObj->pHashTable); - __cache_wr_lock(pCacheObj); +// __cache_wr_lock(pCacheObj); while (taosHashIterNext(pIter)) { SCacheDataNode *pNode = *(SCacheDataNode **)taosHashIterGet(pIter); if (pNode->expireTime < time && T_REF_VAL_GET(pNode) <= 0) { - taosCacheReleaseNode(pCacheObj, pNode); +// taosCacheReleaseNode(pCacheObj, pNode); continue; } @@ -662,7 +662,7 @@ static void doCacheRefresh(SCacheObj* pCacheObj, int64_t time, __cache_free_fn_t } } - __cache_unlock(pCacheObj); +// __cache_unlock(pCacheObj); taosHashDestroyIter(pIter); } From 6f3f7f7ff993c1c54d6589b1d81f58e1eea5442b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 31 Jul 2020 17:27:12 +0800 Subject: [PATCH 021/109] [td-225] update the hash func --- src/util/inc/hash.h | 22 ++-- src/util/src/hash.c | 233 +++++++++++------------------------------- src/util/src/tcache.c | 2 +- 3 files changed, 72 insertions(+), 185 deletions(-) diff --git a/src/util/inc/hash.h b/src/util/inc/hash.h index c5168a92ee..e22716241d 100644 --- a/src/util/inc/hash.h +++ b/src/util/inc/hash.h @@ -41,17 +41,17 @@ typedef struct SHashNode { typedef enum SHashLockTypeE { HASH_NO_LOCK = 0, - HASH_GLOBAL_LOCK = 1, - HASH_ENTRY_LOCK = 2, +// HASH_GLOBAL_LOCK = 1, + HASH_ENTRY_LOCK = 1, } SHashLockTypeE; -typedef struct SHashLock { -#if defined(LINUX) - pthread_rwlock_t *lock; -#else - pthread_mutex_t *lock; -#endif -} SHashLock; +//typedef struct SHashLock { +//#if defined(LINUX) +// pthread_rwlock_t *lock; +//#else +// pthread_mutex_t *lock; +//#endif +//} SHashLock; typedef struct SHashEntry { int32_t num; // number of elements in current entry @@ -66,8 +66,8 @@ typedef struct SHashObj { _hash_fn_t hashFp; // hash function _hash_free_fn_t freeFp; // hash node free callback function - SHashLock lock; - SHashLockTypeE lockType; // lock type + SRWLatch lock; // read-write spin lock + SHashLockTypeE type; // lock type bool enableUpdate; // enable update SArray *pMemBlock; // memory block allocated for SHashEntry } SHashObj; diff --git a/src/util/src/hash.c b/src/util/src/hash.c index bff8d1e50a..5db9d26d04 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -21,64 +21,35 @@ #define HASH_NEED_RESIZE(_h) ((_h)->size >= (_h)->capacity * HASH_DEFAULT_LOAD_FACTOR) -static FORCE_INLINE void __wr_lock(void *lock) { - if (lock == NULL) { +static FORCE_INLINE void __wr_lock(void *lock, int32_t type) { + if (type == HASH_NO_LOCK) { + return; + } + taosWLockLatch(lock); +} + +static FORCE_INLINE void __rd_lock(void *lock, int32_t type) { + if (type == HASH_NO_LOCK) { return; } -#if defined(LINUX) - pthread_rwlock_wrlock(lock); -#else - pthread_mutex_lock(lock); -#endif + taosRLockLatch(lock); } -static FORCE_INLINE void __rd_lock(void *lock) { - if (lock == NULL) { +static FORCE_INLINE void __rd_unlock(void *lock, int32_t type) { + if (type == HASH_NO_LOCK) { return; } -#if defined(LINUX) - pthread_rwlock_rdlock(lock); -#else - pthread_mutex_lock(lock); -#endif + taosRUnLockLatch(lock); } -static FORCE_INLINE void __unlock(void *lock) { - if (lock == NULL) { +static FORCE_INLINE void __wr_unlock(void *lock, int32_t type) { + if (type == HASH_NO_LOCK) { return; } -#if defined(LINUX) - pthread_rwlock_unlock(lock); -#else - pthread_mutex_unlock(lock); -#endif -} - -static FORCE_INLINE int32_t __lock_init(void *lock) { - if (lock == NULL) { - return 0; - } - -#if defined(LINUX) - return pthread_rwlock_init(lock, NULL); -#else - return pthread_mutex_init(lock, NULL); -#endif -} - -static FORCE_INLINE void __lock_destroy(void *lock) { - if (lock == NULL) { - return; - } - -#if defined(LINUX) - pthread_rwlock_destroy(lock); -#else - pthread_mutex_destroy(lock); -#endif + taosWUnLockLatch(lock); } static FORCE_INLINE int32_t taosHashCapacity(int32_t length) { @@ -97,32 +68,38 @@ static FORCE_INLINE int32_t taosHashCapacity(int32_t length) { * @param hashVal hash value by hash function * @return */ -FORCE_INLINE SHashNode *doGetNodeFromHashTable(SHashObj *pHashObj, const void *key, uint32_t keyLen, uint32_t hashVal) { - int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); - - SHashEntry *pe = pHashObj->hashList[slot]; - - // no data, return directly - int32_t num = atomic_load_32(&pe->num); - if (num == 0) { - return NULL; - } - - if (pHashObj->lockType == HASH_ENTRY_LOCK) { - taosRLockLatch(&pe->latch); - } +static FORCE_INLINE SHashNode* doSearchEntryList(SHashEntry* pe, const void* key, size_t keyLen, uint32_t hashVal) { SHashNode* pNode = pe->head.next; while (pNode) { if ((pNode->keyLen == keyLen) && (memcmp(pNode->key, key, keyLen) == 0)) { assert(pNode->hashVal == hashVal); break; } - + pNode = pNode->next; } - if (pHashObj->lockType == HASH_ENTRY_LOCK) { + return pNode; +} + +static FORCE_INLINE SHashNode *doGetNodeFromHashTable(SHashObj *pHashObj, const void *key, uint32_t keyLen, uint32_t hashVal) { + int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); + + SHashEntry *pe = pHashObj->hashList[slot]; + + // no data, return directly + if (atomic_load_32(&pe->num) == 0) { + return NULL; + } + + if (pHashObj->type == HASH_ENTRY_LOCK) { + taosRLockLatch(&pe->latch); + } + + SHashNode* pNode = doSearchEntryList(pe, key, keyLen, hashVal); + + if (pHashObj->type == HASH_ENTRY_LOCK) { taosRUnLockLatch(&pe->latch); } @@ -195,7 +172,7 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTyp assert((pHashObj->capacity & (pHashObj->capacity - 1)) == 0); pHashObj->hashFp = fn; - pHashObj->lockType = type; + pHashObj->type = type; pHashObj->enableUpdate = update; pHashObj->hashList = (SHashEntry **)calloc(pHashObj->capacity, sizeof(void*)); @@ -215,22 +192,6 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTyp taosArrayPush(pHashObj->pMemBlock, &p); } - if (pHashObj->lockType != HASH_NO_LOCK) { -#if defined(LINUX) - pHashObj->lock.lock = calloc(1, sizeof(pthread_rwlock_t)); -#else - pHashObj->lock.lock = calloc(1, sizeof(pthread_mutex_t)); -#endif - } - - if (__lock_init(pHashObj->lock.lock) != 0) { - free(pHashObj->hashList); - free(pHashObj); - - uError("failed to init lock, reason:%s", strerror(errno)); - return NULL; - } - return pHashObj; } @@ -247,17 +208,17 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da // need the resize process, write lock applied if (HASH_NEED_RESIZE(pHashObj)) { - __wr_lock(pHashObj->lock.lock); + __wr_lock(&pHashObj->lock, pHashObj->type); taosHashTableResize(pHashObj); - __unlock(pHashObj->lock.lock); + __wr_unlock(&pHashObj->lock, pHashObj->type); } - __rd_lock(pHashObj->lock.lock); + __rd_lock(&pHashObj->lock, pHashObj->type); int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); SHashEntry *pe = pHashObj->hashList[slot]; - if (pHashObj->lockType == HASH_ENTRY_LOCK) { + if (pHashObj->type == HASH_ENTRY_LOCK) { taosWLockLatch(&pe->latch); } @@ -275,12 +236,12 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da // no data in hash table with the specified key, add it into hash table pushfrontNode(pe, pNewNode); - if (pHashObj->lockType == HASH_ENTRY_LOCK) { + if (pHashObj->type == HASH_ENTRY_LOCK) { taosWUnLockLatch(&pe->latch); } // enable resize - __unlock(pHashObj->lock.lock); + __rd_unlock(&pHashObj->lock, pHashObj->type); atomic_add_fetch_64(&pHashObj->size, 1); return 0; @@ -290,12 +251,12 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da doUpdateHashNode(pNode, pNewNode); } - if (pHashObj->lockType == HASH_ENTRY_LOCK) { + if (pHashObj->type == HASH_ENTRY_LOCK) { taosWUnLockLatch(&pe->latch); } // enable resize - __unlock(pHashObj->lock.lock); + __rd_unlock(&pHashObj->lock, pHashObj->type); tfree(pNewNode->data) tfree(pNewNode); @@ -312,11 +273,11 @@ void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen) { uint32_t hashVal = (*pHashObj->hashFp)(key, keyLen); // only add the read lock to disable the resize process - __rd_lock(pHashObj->lock.lock); + __rd_lock(&pHashObj->lock, pHashObj->type); SHashNode *pNode = doGetNodeFromHashTable(pHashObj, key, keyLen, hashVal); - __unlock(pHashObj->lock.lock); + __rd_unlock(&pHashObj->lock, pHashObj->type); if (pNode) { assert(pNode->hashVal == hashVal); @@ -327,70 +288,7 @@ void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen) { } int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen) { - if (pHashObj->size <= 0) { - return -1; - } - - uint32_t hashVal = (*pHashObj->hashFp)(key, keyLen); - - // disable the resize process - __rd_lock(pHashObj->lock.lock); - - int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); - SHashEntry *pe = pHashObj->hashList[slot]; - - // no data, return directly - if (pe->num == 0) { - __unlock(pHashObj->lock.lock); - return -1; - } - - if (pHashObj->lockType == HASH_ENTRY_LOCK) { - taosWLockLatch(&pe->latch); - } - - SHashNode* pNode = pe->head.next; - while (pNode) { - if ((pNode->keyLen == keyLen) && (memcmp(pNode->key, key, keyLen) == 0)) { - assert(pNode->hashVal == hashVal); - break; - } - - pNode = pNode->next; - } - - if (pNode != NULL) { - assert(pNode->prev != NULL); - - SHashNode *pNext = pNode->next; - pNode->prev->next = pNext; - - if (pNext != NULL) { - pNext->prev = pNode->prev; - } - - pe->num -= 1; - } - - if (pHashObj->lockType == HASH_ENTRY_LOCK) { - taosWUnLockLatch(&pe->latch); - } - - __unlock(pHashObj->lock.lock); - - if (pNode != NULL) { - atomic_sub_fetch_64(&pHashObj->size, 1); - - pNode->next = NULL; - pNode->prev = NULL; - - tfree(pNode->data); - tfree(pNode); - - return 0; - } else { - return -1; - } + return taosHashRemoveNode(pHashObj, key, keyLen, NULL, 0); } int32_t taosHashRemoveNode(SHashObj *pHashObj, const void *key, size_t keyLen, void* data, size_t dsize) { @@ -401,31 +299,22 @@ int32_t taosHashRemoveNode(SHashObj *pHashObj, const void *key, size_t keyLen, v uint32_t hashVal = (*pHashObj->hashFp)(key, keyLen); // disable the resize process - __rd_lock(pHashObj->lock.lock); + __rd_lock(&pHashObj->lock, pHashObj->type); int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); SHashEntry *pe = pHashObj->hashList[slot]; // no data, return directly if (pe->num == 0) { - __unlock(pHashObj->lock.lock); + __rd_unlock(&pHashObj->lock, pHashObj->type); return -1; } - if (pHashObj->lockType == HASH_ENTRY_LOCK) { + if (pHashObj->type == HASH_ENTRY_LOCK) { taosWLockLatch(&pe->latch); } - SHashNode* pNode = pe->head.next; - while (pNode) { - if ((pNode->keyLen == keyLen) && (memcmp(pNode->key, key, keyLen) == 0)) { - assert(pNode->hashVal == hashVal); - break; - } - - pNode = pNode->next; - } - + SHashNode* pNode = doSearchEntryList(pe, key, keyLen, hashVal); if (pNode != NULL) { assert(pNode->prev != NULL); @@ -435,14 +324,15 @@ int32_t taosHashRemoveNode(SHashObj *pHashObj, const void *key, size_t keyLen, v if (pNext != NULL) { pNext->prev = pNode->prev; } + + pe->num -= 1; } - if (pHashObj->lockType == HASH_ENTRY_LOCK) { - pe->num -= 1; + if (pHashObj->type == HASH_ENTRY_LOCK) { taosWUnLockLatch(&pe->latch); } - __unlock(pHashObj->lock.lock); + __rd_unlock(&pHashObj->lock, pHashObj->type); atomic_sub_fetch_64(&pHashObj->size, 1); @@ -470,7 +360,7 @@ void taosHashCleanup(SHashObj *pHashObj) { SHashNode *pNode, *pNext; - __wr_lock(pHashObj->lock.lock); + __wr_lock(&pHashObj->lock, pHashObj->type); if (pHashObj->hashList) { for (int32_t i = 0; i < pHashObj->capacity; ++i) { @@ -496,10 +386,7 @@ void taosHashCleanup(SHashObj *pHashObj) { free(pHashObj->hashList); } - __unlock(pHashObj->lock.lock); - __lock_destroy(pHashObj->lock.lock); - - tfree(pHashObj->lock.lock); + __wr_unlock(&pHashObj->lock, pHashObj->type); // destroy mem block size_t memBlock = taosArrayGetSize(pHashObj->pMemBlock); diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index 3efa31dc84..9f6f0f77ec 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -289,7 +289,7 @@ void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const v pCacheObj->name, key, pNode1->data, pNode1->addedTime, pNode1->expireTime, (int32_t)taosHashGetSize(pCacheObj->pHashTable), pCacheObj->totalSize, (int64_t)dataSize); - return pNode1; + return pNode1->data; } else { // failed, try again From abd8b930fad1d433269c45be1600a177448ea555 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 31 Jul 2020 17:54:21 +0800 Subject: [PATCH 022/109] [td-225] --- src/util/src/hash.c | 50 +++++++++++++++++++-------------------- src/util/src/tcache.c | 2 +- src/vnode/src/vnodeRead.c | 4 ++-- 3 files changed, 28 insertions(+), 28 deletions(-) diff --git a/src/util/src/hash.c b/src/util/src/hash.c index 5db9d26d04..171c8ec100 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -21,6 +21,12 @@ #define HASH_NEED_RESIZE(_h) ((_h)->size >= (_h)->capacity * HASH_DEFAULT_LOAD_FACTOR) +#define FREE_HASH_NODE(_n) \ + do { \ + tfree((_n)->data); \ + tfree(_n); \ + } while (0) + static FORCE_INLINE void __wr_lock(void *lock, int32_t type) { if (type == HASH_NO_LOCK) { return; @@ -258,9 +264,7 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da // enable resize __rd_unlock(&pHashObj->lock, pHashObj->type); - tfree(pNewNode->data) - tfree(pNewNode); - + FREE_HASH_NODE(pNewNode); return pHashObj->enableUpdate? 0:-1; } } @@ -273,7 +277,7 @@ void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen) { uint32_t hashVal = (*pHashObj->hashFp)(key, keyLen); // only add the read lock to disable the resize process - __rd_lock(&pHashObj->lock, pHashObj->type); + __rd_lock(&pHashObj->lock, pHashObj->type); SHashNode *pNode = doGetNodeFromHashTable(pHashObj, key, keyLen, hashVal); @@ -291,6 +295,18 @@ int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen) { return taosHashRemoveNode(pHashObj, key, keyLen, NULL, 0); } +static FORCE_INLINE void popNodeFromEntryList(SHashEntry* pe, SHashNode* pNode) { + SHashNode* pNext = pNode->next; + + assert(pNode->prev != NULL); + pNode->prev->next = pNext; + if (pNext != NULL) { + pNext->prev = pNode->prev; + } + + pe->num -= 1; +} + int32_t taosHashRemoveNode(SHashObj *pHashObj, const void *key, size_t keyLen, void* data, size_t dsize) { if (pHashObj->size <= 0) { return -1; @@ -316,16 +332,7 @@ int32_t taosHashRemoveNode(SHashObj *pHashObj, const void *key, size_t keyLen, v SHashNode* pNode = doSearchEntryList(pe, key, keyLen, hashVal); if (pNode != NULL) { - assert(pNode->prev != NULL); - - SHashNode *pNext = pNode->next; - pNode->prev->next = pNext; - - if (pNext != NULL) { - pNext->prev = pNode->prev; - } - - pe->num -= 1; + popNodeFromEntryList(pe, pNode); } if (pHashObj->type == HASH_ENTRY_LOCK) { @@ -344,8 +351,7 @@ int32_t taosHashRemoveNode(SHashObj *pHashObj, const void *key, size_t keyLen, v pNode->next = NULL; pNode->prev = NULL; - tfree(pNode->data); - tfree(pNode); + FREE_HASH_NODE(pNode); return 0; } else { @@ -540,22 +546,16 @@ void taosHashTableResize(SHashObj *pHashObj) { int32_t j = HASH_INDEX(pNode->hashVal, pHashObj->capacity); if (j == i) { // this key locates in the same slot, no need to relocate it pNode = pNode->next; - assert(pNode == NULL || pNode->next != pNode); } else { pNext = pNode->next; assert(pNode != pNext && (pNext == NULL || pNext->prev == pNode) && pNode->prev->next == pNode); - assert(pNode->prev != NULL); - pNode->prev->next = pNext; - if (pNext != NULL) { - pNext->prev = pNode->prev; - } - + popNodeFromEntryList(pe, pNode); + // clear pointer pNode->next = NULL; pNode->prev = NULL; - pe->num -= 1; - + // added into new slot SHashEntry *pNewEntry = pHashObj->hashList[j]; pushfrontNode(pNewEntry, pNode); diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index 9f6f0f77ec..f8711828d1 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -653,7 +653,7 @@ static void doCacheRefresh(SCacheObj* pCacheObj, int64_t time, __cache_free_fn_t SCacheDataNode *pNode = *(SCacheDataNode **)taosHashIterGet(pIter); if (pNode->expireTime < time && T_REF_VAL_GET(pNode) <= 0) { -// taosCacheReleaseNode(pCacheObj, pNode); + taosCacheReleaseNode(pCacheObj, pNode); continue; } diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index 3c642b5098..9b16cce66c 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -213,7 +213,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { } } - qReleaseQInfo(pVnode->qMgmt, (void**) &handle, freehandle); + qReleaseQInfo(pVnode->qMgmt, (void**) &handle, false); } } @@ -268,7 +268,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { code = vnodeDumpQueryResult(pRet, pVnode, *handle, &freeHandle); } - qReleaseQInfo(pVnode->qMgmt, (void**) &handle, freeHandle); + qReleaseQInfo(pVnode->qMgmt, (void**) &handle, false); return code; } From 07726783e1ff732558b0b513b52356f8cc5a27b6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 31 Jul 2020 17:58:07 +0800 Subject: [PATCH 023/109] [td-225] --- src/query/src/qExecutor.c | 1 - src/vnode/src/vnodeRead.c | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 9ae407e078..899c49139a 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -6742,7 +6742,6 @@ void** qAcquireQInfo(void* pMgmt, uint64_t key) { void** qReleaseQInfo(void* pMgmt, void* pQInfo, bool freeHandle) { SQueryMgmt *pQueryMgmt = pMgmt; - if (pQueryMgmt->qinfoPool == NULL) { return NULL; } diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index 9b16cce66c..3c642b5098 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -213,7 +213,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { } } - qReleaseQInfo(pVnode->qMgmt, (void**) &handle, false); + qReleaseQInfo(pVnode->qMgmt, (void**) &handle, freehandle); } } @@ -268,7 +268,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { code = vnodeDumpQueryResult(pRet, pVnode, *handle, &freeHandle); } - qReleaseQInfo(pVnode->qMgmt, (void**) &handle, false); + qReleaseQInfo(pVnode->qMgmt, (void**) &handle, freeHandle); return code; } From adcbb40fbef1fce1288532113c8d06ad759597df Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 1 Aug 2020 13:41:24 +0800 Subject: [PATCH 024/109] [td-225] --- src/util/inc/hash.h | 10 +- src/util/src/hash.c | 222 +++++++++++++++++++++++++----------- src/util/src/tcache.c | 259 ++++++++++++++++++++++-------------------- 3 files changed, 299 insertions(+), 192 deletions(-) diff --git a/src/util/inc/hash.h b/src/util/inc/hash.h index e22716241d..688bf317d6 100644 --- a/src/util/inc/hash.h +++ b/src/util/inc/hash.h @@ -76,8 +76,9 @@ typedef struct SHashMutableIterator { SHashObj *pHashObj; int32_t entryIndex; SHashNode *pCur; - SHashNode *pNext; // current node can be deleted for mutable iterator, so keep the next one before return current - int32_t num; // already check number of elements in hash table + SHashNode *pNext; // current node can be deleted for mutable iterator, so keep the next one before return current + size_t numOfChecked; // already check number of elements in hash table + size_t numOfEntries; // number of entries while the iterator is created } SHashMutableIterator; /** @@ -118,6 +119,8 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da */ void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen); +void *taosHashGetCB(SHashObj *pHashObj, const void *key, size_t keyLen, void(*fp)(void*)); + /** * remove item with the specified key * @param pHashObj @@ -126,8 +129,9 @@ void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen); */ int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen); +int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLen, void* data, size_t dsize); -int32_t taosHashRemoveNode(SHashObj *pHashObj, const void *key, size_t keyLen, void* data, size_t dsize); +int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), void *param); /** * clean up hash table diff --git a/src/util/src/hash.c b/src/util/src/hash.c index 171c8ec100..be4baf85d2 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -19,7 +19,7 @@ #include "tulog.h" #include "tutil.h" -#define HASH_NEED_RESIZE(_h) ((_h)->size >= (_h)->capacity * HASH_DEFAULT_LOAD_FACTOR) +#define HASH_NEED_RESIZE(_h) ((_h)->size >= (_h)->capacity * HASH_DEFAULT_LOAD_FACTOR) #define FREE_HASH_NODE(_n) \ do { \ @@ -75,8 +75,8 @@ static FORCE_INLINE int32_t taosHashCapacity(int32_t length) { * @return */ -static FORCE_INLINE SHashNode* doSearchEntryList(SHashEntry* pe, const void* key, size_t keyLen, uint32_t hashVal) { - SHashNode* pNode = pe->head.next; +static FORCE_INLINE SHashNode *doSearchInEntryList(SHashEntry *pe, const void *key, size_t keyLen, uint32_t hashVal) { + SHashNode *pNode = pe->head.next; while (pNode) { if ((pNode->keyLen == keyLen) && (memcmp(pNode->key, key, keyLen) == 0)) { assert(pNode->hashVal == hashVal); @@ -89,7 +89,8 @@ static FORCE_INLINE SHashNode* doSearchEntryList(SHashEntry* pe, const void* key return pNode; } -static FORCE_INLINE SHashNode *doGetNodeFromHashTable(SHashObj *pHashObj, const void *key, uint32_t keyLen, uint32_t hashVal) { +static FORCE_INLINE SHashNode *doGetNodeFromHashTable(SHashObj *pHashObj, const void *key, uint32_t keyLen, + uint32_t hashVal) { int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); SHashEntry *pe = pHashObj->hashList[slot]; @@ -103,7 +104,7 @@ static FORCE_INLINE SHashNode *doGetNodeFromHashTable(SHashObj *pHashObj, const taosRLockLatch(&pe->latch); } - SHashNode* pNode = doSearchEntryList(pe, key, keyLen, hashVal); + SHashNode *pNode = doSearchInEntryList(pe, key, keyLen, hashVal); if (pHashObj->type == HASH_ENTRY_LOCK) { taosRUnLockLatch(&pe->latch); @@ -141,8 +142,8 @@ static SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *p */ static FORCE_INLINE SHashNode *doUpdateHashNode(SHashNode *pNode, SHashNode *pNewNode) { assert(pNode->keyLen == pNewNode->keyLen); - SWAP(pNode->key, pNewNode->key, void*); - SWAP(pNode->data, pNewNode->data, void*); + SWAP(pNode->key, pNewNode->key, void *); + SWAP(pNode->data, pNewNode->data, void *); return pNewNode; } @@ -153,7 +154,7 @@ static FORCE_INLINE SHashNode *doUpdateHashNode(SHashNode *pNode, SHashNode *pNe * @param pHashObj * @param pNode */ -static void pushfrontNode(SHashEntry* pEntry, SHashNode *pNode); +static void pushfrontNode(SHashEntry *pEntry, SHashNode *pNode); /** * Get the next element in hash table for iterator @@ -181,17 +182,16 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTyp pHashObj->type = type; pHashObj->enableUpdate = update; - pHashObj->hashList = (SHashEntry **)calloc(pHashObj->capacity, sizeof(void*)); + pHashObj->hashList = (SHashEntry **)calloc(pHashObj->capacity, sizeof(void *)); if (pHashObj->hashList == NULL) { free(pHashObj); uError("failed to allocate memory, reason:%s", strerror(errno)); return NULL; } else { + pHashObj->pMemBlock = taosArrayInit(8, sizeof(void *)); - pHashObj->pMemBlock = taosArrayInit(8, sizeof(void*)); - - void* p = calloc(pHashObj->capacity, sizeof(SHashEntry)); - for(int32_t i = 0; i < pHashObj->capacity; ++i) { + void *p = calloc(pHashObj->capacity, sizeof(SHashEntry)); + for (int32_t i = 0; i < pHashObj->capacity; ++i) { pHashObj->hashList[i] = p + i * sizeof(SHashEntry); } @@ -201,9 +201,7 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTyp return pHashObj; } -size_t taosHashGetSize(const SHashObj *pHashObj) { - return (pHashObj == NULL)? 0:pHashObj->size; -} +size_t taosHashGetSize(const SHashObj *pHashObj) { return (pHashObj == NULL) ? 0 : pHashObj->size; } int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size) { uint32_t hashVal = (*pHashObj->hashFp)(key, keyLen); @@ -221,14 +219,14 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da __rd_lock(&pHashObj->lock, pHashObj->type); - int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); + int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); SHashEntry *pe = pHashObj->hashList[slot]; if (pHashObj->type == HASH_ENTRY_LOCK) { taosWLockLatch(&pe->latch); } - SHashNode* pNode = pe->head.next; + SHashNode *pNode = pe->head.next; while (pNode) { if ((pNode->keyLen == keyLen) && (memcmp(pNode->key, key, keyLen) == 0)) { assert(pNode->hashVal == hashVal); @@ -265,11 +263,15 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da __rd_unlock(&pHashObj->lock, pHashObj->type); FREE_HASH_NODE(pNewNode); - return pHashObj->enableUpdate? 0:-1; + return pHashObj->enableUpdate ? 0 : -1; } } void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen) { + return taosHashGetCB(pHashObj, key, keyLen, NULL); +} + +void *taosHashGetCB(SHashObj *pHashObj, const void *key, size_t keyLen, void (*fp)(void *)) { if (pHashObj->size <= 0 || keyLen == 0 || key == NULL) { return NULL; } @@ -279,24 +281,42 @@ void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen) { // only add the read lock to disable the resize process __rd_lock(&pHashObj->lock, pHashObj->type); - SHashNode *pNode = doGetNodeFromHashTable(pHashObj, key, keyLen, hashVal); + int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); + SHashEntry *pe = pHashObj->hashList[slot]; - __rd_unlock(&pHashObj->lock, pHashObj->type); - - if (pNode) { - assert(pNode->hashVal == hashVal); - return pNode->data; - } else { + // no data, return directly + if (atomic_load_32(&pe->num) == 0) { + __rd_unlock(&pHashObj->lock, pHashObj->type); return NULL; } + + char *data = NULL; + + // lock entry + if (pHashObj->type == HASH_ENTRY_LOCK) { + taosRLockLatch(&pe->latch); + } + + SHashNode *pNode = doSearchInEntryList(pe, key, keyLen, hashVal); + if (fp != NULL) { + fp(pNode->data); + } + + data = pNode->data; + if (pHashObj->type == HASH_ENTRY_LOCK) { + taosRUnLockLatch(&pe->latch); + } + + __rd_unlock(&pHashObj->lock, pHashObj->type); + return data; } int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen) { - return taosHashRemoveNode(pHashObj, key, keyLen, NULL, 0); + return taosHashRemoveWithData(pHashObj, key, keyLen, NULL, 0); } -static FORCE_INLINE void popNodeFromEntryList(SHashEntry* pe, SHashNode* pNode) { - SHashNode* pNext = pNode->next; +static FORCE_INLINE void doPopFromEntryList(SHashEntry *pe, SHashNode *pNode) { + SHashNode *pNext = pNode->next; assert(pNode->prev != NULL); pNode->prev->next = pNext; @@ -307,17 +327,17 @@ static FORCE_INLINE void popNodeFromEntryList(SHashEntry* pe, SHashNode* pNode) pe->num -= 1; } -int32_t taosHashRemoveNode(SHashObj *pHashObj, const void *key, size_t keyLen, void* data, size_t dsize) { - if (pHashObj->size <= 0) { +int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t dsize) { + if (pHashObj == NULL || pHashObj->size <= 0) { return -1; } uint32_t hashVal = (*pHashObj->hashFp)(key, keyLen); // disable the resize process - __rd_lock(&pHashObj->lock, pHashObj->type); + __rd_lock(&pHashObj->lock, pHashObj->type); - int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); + int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); SHashEntry *pe = pHashObj->hashList[slot]; // no data, return directly @@ -330,9 +350,9 @@ int32_t taosHashRemoveNode(SHashObj *pHashObj, const void *key, size_t keyLen, v taosWLockLatch(&pe->latch); } - SHashNode* pNode = doSearchEntryList(pe, key, keyLen, hashVal); + SHashNode *pNode = doSearchInEntryList(pe, key, keyLen, hashVal); if (pNode != NULL) { - popNodeFromEntryList(pe, pNode); + doPopFromEntryList(pe, pNode); } if (pHashObj->type == HASH_ENTRY_LOCK) { @@ -341,13 +361,13 @@ int32_t taosHashRemoveNode(SHashObj *pHashObj, const void *key, size_t keyLen, v __rd_unlock(&pHashObj->lock, pHashObj->type); - atomic_sub_fetch_64(&pHashObj->size, 1); - if (data != NULL) { memcpy(data, pNode->data, dsize); } if (pNode != NULL) { + atomic_sub_fetch_64(&pHashObj->size, 1); + pNode->next = NULL; pNode->prev = NULL; @@ -359,6 +379,49 @@ int32_t taosHashRemoveNode(SHashObj *pHashObj, const void *key, size_t keyLen, v } } +int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), void *param) { + if (pHashObj == NULL || pHashObj->size == 0) { + return 0; + } + + // disable the resize process + __rd_lock(&pHashObj->lock, pHashObj->type); + + int32_t numOfEntries = pHashObj->capacity; + for (int32_t i = 0; i < numOfEntries; ++i) { + SHashEntry *pEntry = pHashObj->hashList[i]; + if (pEntry->num <= 0) { + continue; + } + + if (pHashObj->type == HASH_ENTRY_LOCK) { + taosWLockLatch(&pEntry->latch); + } + + SHashNode *pNode = pEntry->head.next; + assert(pNode != NULL); + + SHashNode *pNext = NULL; + while (pNode != NULL) { + pNext = pNode->next; + + // not qualified, remove it + if (fp && (!fp(param, pNode->data))) { + doPopFromEntryList(pEntry, pNode); + } + + pNode = pNext; + } + + if (pHashObj->type == HASH_ENTRY_LOCK) { + taosWUnLockLatch(&pEntry->latch); + } + } + + __rd_unlock(&pHashObj->lock, pHashObj->type); + return 0; +} + void taosHashCleanup(SHashObj *pHashObj) { if (pHashObj == NULL) { return; @@ -370,7 +433,7 @@ void taosHashCleanup(SHashObj *pHashObj) { if (pHashObj->hashList) { for (int32_t i = 0; i < pHashObj->capacity; ++i) { - SHashEntry* pEntry = pHashObj->hashList[i]; + SHashEntry *pEntry = pHashObj->hashList[i]; if (pEntry->num == 0) { assert(pEntry->head.next == 0); continue; @@ -396,8 +459,8 @@ void taosHashCleanup(SHashObj *pHashObj) { // destroy mem block size_t memBlock = taosArrayGetSize(pHashObj->pMemBlock); - for(int32_t i = 0; i < memBlock; ++i) { - void* p = taosArrayGetP(pHashObj->pMemBlock, i); + for (int32_t i = 0; i < memBlock; ++i) { + void *p = taosArrayGetP(pHashObj->pMemBlock, i); tfree(p); } @@ -414,6 +477,9 @@ SHashMutableIterator *taosHashCreateIter(SHashObj *pHashObj) { } pIter->pHashObj = pHashObj; + + // keep it in local variable, in case the resize operation expand the size + pIter->numOfEntries = pHashObj->capacity; return pIter; } @@ -428,7 +494,7 @@ bool taosHashIterNext(SHashMutableIterator *pIter) { } // check the first one - if (pIter->num == 0) { + if (pIter->numOfChecked == 0) { assert(pIter->pCur == NULL && pIter->pNext == NULL); while (1) { @@ -438,18 +504,30 @@ bool taosHashIterNext(SHashMutableIterator *pIter) { continue; } + if (pIter->pHashObj->type == HASH_ENTRY_LOCK) { + taosRLockLatch(&pEntry->latch); + } + pIter->pCur = pEntry->head.next; if (pIter->pCur->next) { pIter->pNext = pIter->pCur->next; + + if (pIter->pHashObj->type == HASH_ENTRY_LOCK) { + taosRUnLockLatch(&pEntry->latch); + } } else { + if (pIter->pHashObj->type == HASH_ENTRY_LOCK) { + taosRUnLockLatch(&pEntry->latch); + } + pIter->pNext = getNextHashNode(pIter); } break; } - pIter->num++; + pIter->numOfChecked++; return true; } else { assert(pIter->pCur != NULL); @@ -459,7 +537,7 @@ bool taosHashIterNext(SHashMutableIterator *pIter) { return false; } - pIter->num++; + pIter->numOfChecked++; if (pIter->pCur->next) { pIter->pNext = pIter->pCur->next; @@ -504,30 +582,30 @@ void taosHashTableResize(SHashObj *pHashObj) { if (!HASH_NEED_RESIZE(pHashObj)) { return; } - + // double the original capacity SHashNode *pNode = NULL; SHashNode *pNext = NULL; - + int32_t newSize = pHashObj->capacity << 1u; if (newSize > HASH_MAX_CAPACITY) { -// uDebug("current capacity:%d, maximum capacity:%d, no resize applied due to limitation is reached", -// pHashObj->capacity, HASH_MAX_CAPACITY); + // uDebug("current capacity:%d, maximum capacity:%d, no resize applied due to limitation is reached", + // pHashObj->capacity, HASH_MAX_CAPACITY); return; } void *pNewEntryList = realloc(pHashObj->hashList, sizeof(SHashEntry) * newSize); - if (pNewEntryList == NULL) {// todo handle error -// uDebug("cache resize failed due to out of memory, capacity remain:%d", pHashObj->capacity); + if (pNewEntryList == NULL) { // todo handle error + // uDebug("cache resize failed due to out of memory, capacity remain:%d", pHashObj->capacity); return; } - + pHashObj->hashList = pNewEntryList; size_t inc = newSize - pHashObj->capacity; - void* p = calloc(inc, sizeof(SHashEntry)); + void * p = calloc(inc, sizeof(SHashEntry)); - for(int32_t i = 0; i < inc; ++i) { + for (int32_t i = 0; i < inc; ++i) { pHashObj->hashList[i + pHashObj->capacity] = p + i * sizeof(SHashEntry); } @@ -535,7 +613,7 @@ void taosHashTableResize(SHashObj *pHashObj) { pHashObj->capacity = newSize; for (int32_t i = 0; i < pHashObj->capacity; ++i) { - SHashEntry* pe = pHashObj->hashList[i]; + SHashEntry *pe = pHashObj->hashList[i]; if (pe->num == 0) { assert(pe->head.next == NULL); continue; @@ -550,7 +628,7 @@ void taosHashTableResize(SHashObj *pHashObj) { pNext = pNode->next; assert(pNode != pNext && (pNext == NULL || pNext->prev == pNode) && pNode->prev->next == pNode); - popNodeFromEntryList(pe, pNode); + doPopFromEntryList(pe, pNode); // clear pointer pNode->next = NULL; @@ -566,8 +644,8 @@ void taosHashTableResize(SHashObj *pHashObj) { } } -// uDebug("hash table resize completed, new capacity:%d, load factor:%f, elapsed time:%fms", pHashObj->capacity, -// ((double)pHashObj->size) / pHashObj->capacity, (et - st) / 1000.0); + // uDebug("hash table resize completed, new capacity:%d, load factor:%f, elapsed time:%fms", pHashObj->capacity, + // ((double)pHashObj->size) / pHashObj->capacity, (et - st) / 1000.0); } SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, size_t dsize, uint32_t hashVal) { @@ -579,7 +657,7 @@ SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, s pNewNode->data = malloc(dsize + keyLen); memcpy(pNewNode->data, pData, dsize); - + pNewNode->key = pNewNode->data + dsize; memcpy(pNewNode->key, key, keyLen); @@ -588,10 +666,10 @@ SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, s return pNewNode; } -void pushfrontNode(SHashEntry* pEntry, SHashNode *pNode) { +void pushfrontNode(SHashEntry *pEntry, SHashNode *pNode) { assert(pNode != NULL && pEntry != NULL); - - SHashNode* pNext = pEntry->head.next; + + SHashNode *pNext = pEntry->head.next; if (pNext != NULL) { pNext->prev = pNode; } @@ -605,17 +683,29 @@ void pushfrontNode(SHashEntry* pEntry, SHashNode *pNode) { SHashNode *getNextHashNode(SHashMutableIterator *pIter) { assert(pIter != NULL); - + pIter->entryIndex++; - while (pIter->entryIndex < pIter->pHashObj->capacity) { - SHashEntry*pEntry = pIter->pHashObj->hashList[pIter->entryIndex]; + SHashNode *p = NULL; + + while (pIter->entryIndex < pIter->numOfEntries) { + SHashEntry *pEntry = pIter->pHashObj->hashList[pIter->entryIndex]; if (pEntry->num == 0) { pIter->entryIndex++; continue; } - - return pEntry->head.next; + + if (pIter->pHashObj->type == HASH_ENTRY_LOCK) { + taosRLockLatch(&pEntry->latch); + } + + p = pEntry->head.next; + + if (pIter->pHashObj->type == HASH_ENTRY_LOCK) { + taosRUnLockLatch(&pEntry->latch); + } + + return p; } - + return NULL; } diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index f8711828d1..8523a6f8b6 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -63,13 +63,6 @@ static FORCE_INLINE void __cache_lock_destroy(SCacheObj *pCacheObj) { #endif } -#if 0 -static FORCE_INLINE void taosFreeNode(void *data) { - SCacheDataNode *pNode = *(SCacheDataNode **)data; - free(pNode); -} -#endif - /** * @param key key of object for hash, usually a null-terminated string * @param keyLen length of key @@ -89,13 +82,6 @@ static SCacheDataNode *taosCreateCacheNode(const char *key, size_t keyLen, const */ static void taosAddToTrash(SCacheObj *pCacheObj, SCacheDataNode *pNode); -/** - * remove node in trash can - * @param pCacheObj - * @param pElem - */ -static void taosRemoveFromTrashCan(SCacheObj *pCacheObj, STrashElem *pElem); - /** * remove nodes in trash with refCount == 0 in cache * @param pNode @@ -113,17 +99,19 @@ static void taosTrashCanEmpty(SCacheObj *pCacheObj, bool force); static FORCE_INLINE void taosCacheReleaseNode(SCacheObj *pCacheObj, SCacheDataNode *pNode) { if (pNode->signature != (uint64_t)pNode) { uError("key:%s, %p data is invalid, or has been released", pNode->key, pNode); + assert(0); return; } - taosHashRemove(pCacheObj->pHashTable, pNode->key, pNode->keySize); - pCacheObj->totalSize -= pNode->size; + int32_t size = taosHashGetSize(pCacheObj->pHashTable); uDebug("cache:%s, key:%p, %p is destroyed from cache, totalNum:%d totalSize:%" PRId64 "bytes size:%dbytes", - pCacheObj->name, pNode->key, pNode->data, (int32_t)taosHashGetSize(pCacheObj->pHashTable), pCacheObj->totalSize, - pNode->size); + pCacheObj->name, pNode->key, pNode->data, size, pCacheObj->totalSize, pNode->size); + + if (pCacheObj->freeFp) { + pCacheObj->freeFp(pNode->data); + } - if (pCacheObj->freeFp) pCacheObj->freeFp(pNode->data); free(pNode); } @@ -137,6 +125,32 @@ static FORCE_INLINE void taosCacheMoveToTrash(SCacheObj *pCacheObj, SCacheDataNo taosAddToTrash(pCacheObj, pNode); } +static FORCE_INLINE void doRemoveElemInTrashcan(SCacheObj* pCacheObj, STrashElem *pElem) { + if (pElem->pData->signature != (uint64_t) pElem->pData) { + uError("key:sig:0x%" PRIx64 " %p data has been released, ignore", pElem->pData->signature, pElem->pData); + return; + } + + pCacheObj->numOfElemsInTrash--; + if (pElem->prev) { + pElem->prev->next = pElem->next; + } else { // pnode is the header, update header + pCacheObj->pTrash = pElem->next; + } + + if (pElem->next) { + pElem->next->prev = pElem->prev; + } +} + +static FORCE_INLINE void doDestroyTrashcanElem(SCacheObj* pCacheObj, STrashElem *pElem) { + if (pCacheObj->freeFp) { + pCacheObj->freeFp(pElem->pData->data); + } + + free(pElem->pData); + free(pElem); +} /** * update data in cache * @param pCacheObj @@ -261,12 +275,11 @@ void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const v } else { // duplicated key exists while (1) { SCacheDataNode* p = NULL; - int32_t ret = taosHashRemoveNode(pCacheObj->pHashTable, key, keyLen, (void*) &p, sizeof(void*)); + int32_t ret = taosHashRemoveWithData(pCacheObj->pHashTable, key, keyLen, (void*) &p, sizeof(void*)); // add to trashcan if (ret == 0) { if (T_REF_VAL_GET(p) == 0) { - if (pCacheObj->freeFp) { pCacheObj->freeFp(p->data); } @@ -300,27 +313,25 @@ void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const v return pNode1->data; } +static void incRefFn(void* ptNode) { + assert(ptNode != NULL); + + SCacheDataNode** p = (SCacheDataNode**) ptNode; + int32_t ret = T_REF_INC(*p); + assert(ret > 0); +} + void *taosCacheAcquireByKey(SCacheObj *pCacheObj, const void *key, size_t keyLen) { if (pCacheObj == NULL || taosHashGetSize(pCacheObj->pHashTable) == 0) { return NULL; } - void *pData = NULL; - -// __cache_rd_lock(pCacheObj); - SCacheDataNode **ptNode = (SCacheDataNode **)taosHashGet(pCacheObj->pHashTable, key, keyLen); - - int32_t ref = 0; - if (ptNode != NULL) { - ref = T_REF_INC(*ptNode); - pData = (*ptNode)->data; - } - -// __cache_unlock(pCacheObj); + SCacheDataNode **ptNode = (SCacheDataNode **)taosHashGetCB(pCacheObj->pHashTable, key, keyLen, incRefFn); + void* pData = (ptNode != NULL)? (*ptNode)->data:NULL; if (pData != NULL) { atomic_add_fetch_32(&pCacheObj->statistics.hitCount, 1); - uDebug("cache:%s, key:%p, %p is retrieved from cache, refcnt:%d", pCacheObj->name, key, pData, ref); + uDebug("cache:%s, key:%p, %p is retrieved from cache, refcnt:%d", pCacheObj->name, key, pData, T_REF_VAL_GET(*ptNode)); } else { atomic_add_fetch_32(&pCacheObj->statistics.missCount, 1); uDebug("cache:%s, key:%p, not in cache, retrieved failed", pCacheObj->name, key); @@ -423,8 +434,11 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { if (_remove) { // NOTE: once refcount is decrease, pNode may be freed by other thread immediately. + char* key = pNode->key; + char* d = pNode->data; + int32_t ref = T_REF_DEC(pNode); - uDebug("cache:%s, key:%p, %p is released, refcnt:%d", pCacheObj->name, pNode->key, pNode->data, ref); + uDebug("cache:%s, key:%p, %p is released, refcnt:%d", pCacheObj->name, key, d, ref); /* * If it is not referenced by other users, remove it immediately. Otherwise move this node to trashcan wait for all users @@ -437,24 +451,35 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { if (ref == 0) { assert(pNode->pTNodeHeader->pData == pNode); - // todo add lock here - taosRemoveFromTrashCan(pCacheObj, pNode->pTNodeHeader); + __cache_wr_lock(pCacheObj); + doRemoveElemInTrashcan(pCacheObj, pNode->pTNodeHeader); + __cache_unlock(pCacheObj); + + doDestroyTrashcanElem(pCacheObj, pNode->pTNodeHeader); } } else { int32_t ret = taosHashRemove(pCacheObj->pHashTable, pNode->key, pNode->keySize); - if (ret == 0) { // successfully remove from hash table + + // successfully remove from hash table, if failed, this node must have been move to trash already, do nothing. + // note that the remove operation can be executed only once. + if (ret == 0) { if (ref > 0) { assert(pNode->pTNodeHeader == NULL); - // todo trashcan lock + __cache_wr_lock(pCacheObj); taosAddToTrash(pCacheObj, pNode); + __cache_unlock(pCacheObj); } else { // ref == 0 - atomic_fetch_sub_ptr(&pCacheObj->totalSize, pNode->size); + atomic_sub_fetch_64(&pCacheObj->totalSize, pNode->size); + uDebug("cache:%s, key:%p, %p is destroyed from cache, totalNum:%d totalSize:%" PRId64 "bytes size:%dbytes", pCacheObj->name, pNode->key, pNode->data, (int32_t)taosHashGetSize(pCacheObj->pHashTable), pCacheObj->totalSize, pNode->size); - if (pCacheObj->freeFp) pCacheObj->freeFp(pNode->data); + if (pCacheObj->freeFp) { + pCacheObj->freeFp(pNode->data); + } + free(pNode); } } @@ -462,33 +487,40 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { } else { // NOTE: once refcount is decrease, pNode may be freed by other thread immediately. - int32_t ref = T_REF_DEC(pNode); + char* key = pNode->key; + char* p = pNode->data; - // todo so, invalid read here! - uDebug("cache:%s, key:%p, %p released, refcnt:%d, data in trancan:%d", pCacheObj->name, pNode->key, pNode->data, - ref, inTrashCan); + int32_t ref = T_REF_DEC(pNode); + uDebug("cache:%s, key:%p, %p released, refcnt:%d, data in trancan:%d", pCacheObj->name, key, p, ref, inTrashCan); } } -void taosCacheEmpty(SCacheObj *pCacheObj) { - SHashMutableIterator *pIter = taosHashCreateIter(pCacheObj->pHashTable); - - __cache_wr_lock(pCacheObj); - while (taosHashIterNext(pIter)) { - if (pCacheObj->deleting == 1) { - break; - } - - SCacheDataNode *pNode = *(SCacheDataNode **) taosHashIterGet(pIter); - if (T_REF_VAL_GET(pNode) == 0) { - taosCacheReleaseNode(pCacheObj, pNode); - } else { - taosCacheMoveToTrash(pCacheObj, pNode); - } +typedef struct SHashTravSupp { + SCacheObj* pCacheObj; + int64_t time; + __cache_free_fn_t fp; +} SHashTravSupp; + +static bool travHashTableEmptyFn(void* param, void* data) { + SHashTravSupp* ps = (SHashTravSupp*) param; + SCacheObj* pCacheObj= ps->pCacheObj; + + SCacheDataNode *pNode = *(SCacheDataNode **) data; + + if (T_REF_VAL_GET(pNode) == 0) { + taosCacheReleaseNode(pCacheObj, pNode); + } else { // do add to trashcan + taosAddToTrash(pCacheObj, pNode); } - __cache_unlock(pCacheObj); - - taosHashDestroyIter(pIter); + + // this node should be remove from hash table + return false; +} + +void taosCacheEmpty(SCacheObj *pCacheObj) { + SHashTravSupp sup = {.pCacheObj = pCacheObj, .fp = NULL, .time = taosGetTimestampMs()}; + + taosHashCondTraverse(pCacheObj->pHashTable, travHashTableEmptyFn, &sup); taosTrashCanEmpty(pCacheObj, false); } @@ -553,33 +585,6 @@ void taosAddToTrash(SCacheObj *pCacheObj, SCacheDataNode *pNode) { uDebug("key:%p, %p move to trash, numOfElem in trash:%d", pNode->key, pNode->data, pCacheObj->numOfElemsInTrash); } -void taosRemoveFromTrashCan(SCacheObj *pCacheObj, STrashElem *pElem) { - if (pElem->pData->signature != (uint64_t)pElem->pData) { - uError("key:sig:0x%" PRIx64 " %p data has been released, ignore", pElem->pData->signature, pElem->pData); - return; - } - - pCacheObj->numOfElemsInTrash--; - if (pElem->prev) { - pElem->prev->next = pElem->next; - } else { /* pnode is the header, update header */ - pCacheObj->pTrash = pElem->next; - } - - if (pElem->next) { - pElem->next->prev = pElem->prev; - } - - pElem->pData->signature = 0; - if (pCacheObj->freeFp) { - pCacheObj->freeFp(pElem->pData->data); - } - - free(pElem->pData); - free(pElem); -} - -// TODO add another lock when scanning trashcan void taosTrashCanEmpty(SCacheObj *pCacheObj, bool force) { __cache_wr_lock(pCacheObj); @@ -587,8 +592,8 @@ void taosTrashCanEmpty(SCacheObj *pCacheObj, bool force) { if (pCacheObj->pTrash != NULL) { uError("key:inconsistency data in cache, numOfElem in trash:%d", pCacheObj->numOfElemsInTrash); } - pCacheObj->pTrash = NULL; + pCacheObj->pTrash = NULL; __cache_unlock(pCacheObj); return; } @@ -604,10 +609,12 @@ void taosTrashCanEmpty(SCacheObj *pCacheObj, bool force) { if (force || (T_REF_VAL_GET(pElem->pData) == 0)) { uDebug("key:%p, %p removed from trash. numOfElem in trash:%d", pElem->pData->key, pElem->pData->data, pCacheObj->numOfElemsInTrash - 1); - STrashElem *p = pElem; + STrashElem *p = pElem; pElem = pElem->next; - taosRemoveFromTrashCan(pCacheObj, p); + + doRemoveElemInTrashcan(pCacheObj, p); + doDestroyTrashcanElem(pCacheObj, p); } else { pElem = pElem->next; } @@ -617,26 +624,27 @@ void taosTrashCanEmpty(SCacheObj *pCacheObj, bool force) { } void doCleanupDataCache(SCacheObj *pCacheObj) { - __cache_wr_lock(pCacheObj); - SHashMutableIterator *pIter = taosHashCreateIter(pCacheObj->pHashTable); - while (taosHashIterNext(pIter)) { - SCacheDataNode *pNode = *(SCacheDataNode **)taosHashIterGet(pIter); +// SHashMutableIterator *pIter = taosHashCreateIter(pCacheObj->pHashTable); +// while (taosHashIterNext(pIter)) { +// SCacheDataNode *pNode = *(SCacheDataNode **)taosHashIterGet(pIter); +// +// int32_t c = T_REF_VAL_GET(pNode); +// if (c <= 0) { +// taosCacheReleaseNode(pCacheObj, pNode); +// } else { +// uDebug("cache:%s key:%p, %p will not remove from cache, refcnt:%d", pCacheObj->name, pNode->key, +// pNode->data, T_REF_VAL_GET(pNode)); +// } +// } +// +// taosHashDestroyIter(pIter); - int32_t c = T_REF_VAL_GET(pNode); - if (c <= 0) { - taosCacheReleaseNode(pCacheObj, pNode); - } else { - uDebug("cache:%s key:%p, %p will not remove from cache, refcnt:%d", pCacheObj->name, pNode->key, - pNode->data, T_REF_VAL_GET(pNode)); - } - } - taosHashDestroyIter(pIter); + SHashTravSupp sup = {.pCacheObj = pCacheObj, .fp = NULL, .time = taosGetTimestampMs()}; + taosHashCondTraverse(pCacheObj->pHashTable, travHashTableEmptyFn, &sup); // todo memory leak if there are object with refcount greater than 0 in hash table? taosHashCleanup(pCacheObj->pHashTable); - __cache_unlock(pCacheObj); - taosTrashCanEmpty(pCacheObj, true); __cache_lock_destroy(pCacheObj); @@ -645,26 +653,31 @@ void doCleanupDataCache(SCacheObj *pCacheObj) { free(pCacheObj); } -static void doCacheRefresh(SCacheObj* pCacheObj, int64_t time, __cache_free_fn_t fp) { - SHashMutableIterator *pIter = taosHashCreateIter(pCacheObj->pHashTable); +bool travHashTableFn(void* param, void* data) { + SHashTravSupp* ps = (SHashTravSupp*) param; + SCacheObj* pCacheObj= ps->pCacheObj; -// __cache_wr_lock(pCacheObj); - while (taosHashIterNext(pIter)) { - SCacheDataNode *pNode = *(SCacheDataNode **)taosHashIterGet(pIter); + SCacheDataNode* pNode = *(SCacheDataNode **) data; + if (pNode->expireTime < ps->time && T_REF_VAL_GET(pNode) <= 0) { + taosCacheReleaseNode(pCacheObj, pNode); - if (pNode->expireTime < time && T_REF_VAL_GET(pNode) <= 0) { - taosCacheReleaseNode(pCacheObj, pNode); - continue; - } - - if (fp) { - fp(pNode->data); - } + // this node should be remove from hash table + return false; } -// __cache_unlock(pCacheObj); + if (ps->fp) { + (ps->fp)(pNode->data); + } - taosHashDestroyIter(pIter); + // do not remove element in hash table + return true; +} + +static void doCacheRefresh(SCacheObj* pCacheObj, int64_t time, __cache_free_fn_t fp) { + assert(pCacheObj != NULL); + + SHashTravSupp sup = {.pCacheObj = pCacheObj, .fp = fp, .time = time}; + taosHashCondTraverse(pCacheObj->pHashTable, travHashTableFn, &sup); } void* taosCacheTimedRefresh(void *handle) { From 2b413844d32fa4f51b6e4a2716a8f539d56f3c54 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 1 Aug 2020 13:50:42 +0800 Subject: [PATCH 025/109] [td-225] --- src/util/src/hash.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/util/src/hash.c b/src/util/src/hash.c index be4baf85d2..698bcbb15f 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -298,11 +298,13 @@ void *taosHashGetCB(SHashObj *pHashObj, const void *key, size_t keyLen, void (*f } SHashNode *pNode = doSearchInEntryList(pe, key, keyLen, hashVal); - if (fp != NULL) { - fp(pNode->data); + if (pNode != NULL) { + if (fp != NULL) { + fp(pNode->data); + } + data = pNode->data; } - data = pNode->data; if (pHashObj->type == HASH_ENTRY_LOCK) { taosRUnLockLatch(&pe->latch); } From 02bdbce35af7a75fef1b63cc3534cb6c61d69245 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 1 Aug 2020 16:15:07 +0800 Subject: [PATCH 026/109] [td-225] merge develop --- src/mnode/src/mnodeProfile.c | 2 +- src/util/inc/hash.h | 9 ------ src/util/inc/tcache.h | 2 +- src/util/src/hash.c | 4 +-- src/util/src/tcache.c | 54 ++++++++++++++++++------------------ 5 files changed, 31 insertions(+), 40 deletions(-) diff --git a/src/mnode/src/mnodeProfile.c b/src/mnode/src/mnodeProfile.c index 353dd59671..e8f37f1422 100644 --- a/src/mnode/src/mnodeProfile.c +++ b/src/mnode/src/mnodeProfile.c @@ -112,7 +112,7 @@ void mnodeReleaseConn(SConnObj *pConn) { SConnObj *mnodeAccquireConn(int32_t connId, char *user, uint32_t ip, uint16_t port) { uint64_t expireTime = CONN_KEEP_TIME * 1000 + (uint64_t)taosGetTimestampMs(); - SConnObj *pConn = taosCacheUpdateExpireTimeByName(tsMnodeConnCache, &connId, sizeof(int32_t), expireTime); + SConnObj *pConn = taosCacheAcquireByKey(tsMnodeConnCache, &connId, sizeof(int32_t)); if (pConn == NULL) { mDebug("connId:%d, is already destroyed, user:%s ip:%s:%u", connId, user, taosIpStr(ip), port); return NULL; diff --git a/src/util/inc/hash.h b/src/util/inc/hash.h index 688bf317d6..71493788ac 100644 --- a/src/util/inc/hash.h +++ b/src/util/inc/hash.h @@ -41,18 +41,9 @@ typedef struct SHashNode { typedef enum SHashLockTypeE { HASH_NO_LOCK = 0, -// HASH_GLOBAL_LOCK = 1, HASH_ENTRY_LOCK = 1, } SHashLockTypeE; -//typedef struct SHashLock { -//#if defined(LINUX) -// pthread_rwlock_t *lock; -//#else -// pthread_mutex_t *lock; -//#endif -//} SHashLock; - typedef struct SHashEntry { int32_t num; // number of elements in current entry SRWLatch latch; // entry latch diff --git a/src/util/inc/tcache.h b/src/util/inc/tcache.h index 1e2aeae394..11121fcf3b 100644 --- a/src/util/inc/tcache.h +++ b/src/util/inc/tcache.h @@ -121,7 +121,7 @@ void *taosCacheAcquireByKey(SCacheObj *pCacheObj, const void *key, size_t keyLen * @param expireTime new expire time of data * @return */ -void* taosCacheUpdateExpireTimeByName(SCacheObj *pCacheObj, void *key, size_t keyLen, uint64_t expireTime); +//void* taosCacheUpdateExpireTimeByName(SCacheObj *pCacheObj, void *key, size_t keyLen, uint64_t expireTime); /** * Add one reference count for the exist data, and assign this data for a new owner. diff --git a/src/util/src/hash.c b/src/util/src/hash.c index ed4f445795..83e2630e41 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -447,8 +447,7 @@ void taosHashCleanup(SHashObj *pHashObj) { pHashObj->freeFp(pNode->data); } - free(pNode->data); - free(pNode); + FREE_HASH_NODE(pNode); pNode = pNext; } } @@ -651,6 +650,7 @@ void taosHashTableResize(SHashObj *pHashObj) { SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, size_t dsize, uint32_t hashVal) { SHashNode *pNewNode = calloc(1, sizeof(SHashNode)); + if (pNewNode == NULL) { uError("failed to allocate memory, reason:%s", strerror(errno)); return NULL; diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index c360d09a4a..e7f4b29744 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -340,33 +340,33 @@ void *taosCacheAcquireByKey(SCacheObj *pCacheObj, const void *key, size_t keyLen return pData; } -void* taosCacheUpdateExpireTimeByName(SCacheObj *pCacheObj, void *key, size_t keyLen, uint64_t expireTime) { - if (pCacheObj == NULL || taosHashGetSize(pCacheObj->pHashTable) == 0) { - return NULL; - } - - __cache_rd_lock(pCacheObj); - - SCacheDataNode **ptNode = (SCacheDataNode **)taosHashGet(pCacheObj->pHashTable, key, keyLen); - if (ptNode != NULL) { - T_REF_INC(*ptNode); - (*ptNode)->expireTime = expireTime; // taosGetTimestampMs() + (*ptNode)->lifespan; - } - - __cache_unlock(pCacheObj); - - if (ptNode != NULL) { - atomic_add_fetch_32(&pCacheObj->statistics.hitCount, 1); - uDebug("cache:%s, key:%p, %p expireTime is updated in cache, refcnt:%d", pCacheObj->name, key, - (*ptNode)->data, T_REF_VAL_GET(*ptNode)); - } else { - atomic_add_fetch_32(&pCacheObj->statistics.missCount, 1); - uDebug("cache:%s, key:%p, not in cache, retrieved failed", pCacheObj->name, key); - } - - atomic_add_fetch_32(&pCacheObj->statistics.totalAccess, 1); - return (ptNode != NULL) ? (*ptNode)->data : NULL; -} +//void* taosCacheUpdateExpireTimeByName(SCacheObj *pCacheObj, void *key, size_t keyLen, uint64_t expireTime) { +// if (pCacheObj == NULL || taosHashGetSize(pCacheObj->pHashTable) == 0) { +// return NULL; +// } +// +// __cache_rd_lock(pCacheObj); +// +// SCacheDataNode **ptNode = (SCacheDataNode **)taosHashGet(pCacheObj->pHashTable, key, keyLen); +// if (ptNode != NULL) { +// T_REF_INC(*ptNode); +// (*ptNode)->expireTime = expireTime; // taosGetTimestampMs() + (*ptNode)->lifespan; +// } +// +// __cache_unlock(pCacheObj); +// +// if (ptNode != NULL) { +// atomic_add_fetch_32(&pCacheObj->statistics.hitCount, 1); +// uDebug("cache:%s, key:%p, %p expireTime is updated in cache, refcnt:%d", pCacheObj->name, key, +// (*ptNode)->data, T_REF_VAL_GET(*ptNode)); +// } else { +// atomic_add_fetch_32(&pCacheObj->statistics.missCount, 1); +// uDebug("cache:%s, key:%p, not in cache, retrieved failed", pCacheObj->name, key); +// } +// +// atomic_add_fetch_32(&pCacheObj->statistics.totalAccess, 1); +// return (ptNode != NULL) ? (*ptNode)->data : NULL; +//} void *taosCacheAcquireByData(SCacheObj *pCacheObj, void *data) { if (pCacheObj == NULL || data == NULL) return NULL; From 9be35563a2284d8867f446ce7c26b453df8ee834 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 3 Aug 2020 11:06:08 +0800 Subject: [PATCH 027/109] [td-225] fix bugs found by sim. --- src/inc/tsdb.h | 27 ++--- src/mnode/src/mnodeProfile.c | 1 - src/query/inc/qExecutor.h | 2 - src/query/src/qExecutor.c | 88 ++++++++++++----- src/tsdb/src/tsdbRead.c | 131 ++++++++++++++++--------- src/util/src/tcache.c | 34 +++---- tests/script/general/parser/topbot.sim | 12 ++- 7 files changed, 186 insertions(+), 109 deletions(-) diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index bab17322f0..4776d1cda7 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -167,9 +167,14 @@ typedef struct SDataBlockInfo { } SDataBlockInfo; typedef struct { - size_t numOfTables; + void *pTable; + TSKEY lastKey; +} STableKeyInfo; + +typedef struct { + size_t numOfTables; SArray *pGroupList; - SHashObj *map; // speedup acquire the tableQueryInfo from STableId + SHashObj *map; // speedup acquire the tableQueryInfo by table uid } STableGroupInfo; /** @@ -177,24 +182,24 @@ typedef struct { * * @param tsdb tsdb handle * @param pCond query condition, including time window, result set order, and basic required columns for each block - * @param tableqinfoGroupInfo tableId list in the form of set, seperated into different groups according to group by condition + * @param tableInfoGroup table object list in the form of set, grouped into different sets according to the + * group by condition * @param qinfo query info handle from query processor * @return */ -TsdbQueryHandleT *tsdbQueryTables(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableqinfoGroupInfo, void *qinfo); +TsdbQueryHandleT *tsdbQueryTables(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfoGroup, void *qinfo); /** * Get the last row of the given query time window for all the tables in STableGroupInfo object. * Note that only one data block with only row will be returned while invoking retrieve data block function for * all tables in this group. * - * @param tsdb tsdb handle - * @param pCond query condition, including time window, result set order, and basic required columns for each - * block - * @param tableqinfoGroupInfo tableId list. + * @param tsdb tsdb handle + * @param pCond query condition, including time window, result set order, and basic required columns for each block + * @param tableInfo table list. * @return */ -TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableqinfoGroupInfo, void *qinfo); +TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfo, void *qinfo); /** * get the queried table object list @@ -260,7 +265,7 @@ SArray *tsdbRetrieveDataBlock(TsdbQueryHandleT *pQueryHandle, SArray *pColumnIdL * @param stableid. super table sid * @param pTagCond. tag query condition */ -int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T *tsdb, uint64_t uid, const char *pTagCond, size_t len, +int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T *tsdb, uint64_t uid, TSKEY key, const char *pTagCond, size_t len, int16_t tagNameRelType, const char *tbnameCond, STableGroupInfo *pGroupList, SColIndex *pColIndex, int32_t numOfCols); @@ -278,7 +283,7 @@ void tsdbDestroyTableGroup(STableGroupInfo *pGroupList); * @param pGroupInfo the generated result * @return */ -int32_t tsdbGetOneTableGroup(TSDB_REPO_T *tsdb, uint64_t uid, STableGroupInfo *pGroupInfo); +int32_t tsdbGetOneTableGroup(TSDB_REPO_T *tsdb, uint64_t uid, TSKEY startKey, STableGroupInfo *pGroupInfo); /** * diff --git a/src/mnode/src/mnodeProfile.c b/src/mnode/src/mnodeProfile.c index e8f37f1422..7079b1a26a 100644 --- a/src/mnode/src/mnodeProfile.c +++ b/src/mnode/src/mnodeProfile.c @@ -73,7 +73,6 @@ int32_t mnodeInitProfile() { void mnodeCleanupProfile() { if (tsMnodeConnCache != NULL) { - mInfo("conn cache is cleanup"); taosCacheCleanup(tsMnodeConnCache); tsMnodeConnCache = NULL; } diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index bd2e0a4470..328078fb60 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -186,8 +186,6 @@ typedef struct SQInfo { void* signature; int32_t pointsInterpo; int32_t code; // error code to returned to client -// sem_t dataReady; - void* tsdb; int32_t vgId; STableGroupInfo tableGroupInfo; // table id list < only includes the STable list> diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 6bc8bf31ea..613c3ae14c 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -1813,10 +1813,14 @@ static void doExchangeTimeWindow(SQInfo* pQInfo) { for(int32_t i = 0; i < t; ++i) { SArray* p1 = GET_TABLEGROUP(pQInfo, i); + SArray* tableKeyGroup = taosArrayGetP(pQInfo->tableGroupInfo.pGroupList, i); size_t len = taosArrayGetSize(p1); for(int32_t j = 0; j < len; ++j) { STableQueryInfo* pTableQueryInfo = (STableQueryInfo*) taosArrayGetP(p1, j); SWAP(pTableQueryInfo->win.skey, pTableQueryInfo->win.ekey, TSKEY); + + STableKeyInfo* pInfo = taosArrayGet(tableKeyGroup, j); + pInfo->lastKey = pTableQueryInfo->win.skey; } } } @@ -2925,7 +2929,7 @@ static void updateTableQueryInfoForReverseScan(SQuery *pQuery, STableQueryInfo * return; } - // order has change already! + // order has changed already int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); // TODO validate the assertion @@ -2934,9 +2938,13 @@ static void updateTableQueryInfoForReverseScan(SQuery *pQuery, STableQueryInfo * // } else { // assert(pTableQueryInfo->win.ekey <= pTableQueryInfo->lastKey + step); // } - - pTableQueryInfo->win.ekey = pTableQueryInfo->lastKey + step; - + + if (pTableQueryInfo->lastKey == pTableQueryInfo->win.skey) { + // do nothing, no results + } else { + pTableQueryInfo->win.ekey = pTableQueryInfo->lastKey + step; + } + SWAP(pTableQueryInfo->win.skey, pTableQueryInfo->win.ekey, TSKEY); pTableQueryInfo->lastKey = pTableQueryInfo->win.skey; @@ -2998,16 +3006,26 @@ void disableFuncInReverseScan(SQInfo *pQInfo) { } } } - +} + +static void setupQueryRangeForReverseScan(SQInfo* pQInfo) { + SQuery* pQuery = pQInfo->runtimeEnv.pQuery; int32_t numOfGroups = GET_NUM_OF_TABLEGROUP(pQInfo); - + for(int32_t i = 0; i < numOfGroups; ++i) { SArray *group = GET_TABLEGROUP(pQInfo, i); - + SArray *tableKeyGroup = taosArrayGetP(pQInfo->tableGroupInfo.pGroupList, i); + size_t t = taosArrayGetSize(group); for (int32_t j = 0; j < t; ++j) { STableQueryInfo *pCheckInfo = taosArrayGetP(group, j); updateTableQueryInfoForReverseScan(pQuery, pCheckInfo); + + // update the last key in tableKeyInfo list + STableKeyInfo *pTableKeyInfo = taosArrayGet(tableKeyGroup, j); + pTableKeyInfo->lastKey = pCheckInfo->lastKey; + + assert(pCheckInfo->pTable == pTableKeyInfo->pTable); } } } @@ -3252,20 +3270,20 @@ static void setEnvBeforeReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatusI .numOfCols = pQuery->numOfCols, }; + setQueryStatus(pQuery, QUERY_NOT_COMPLETED); + switchCtxOrder(pRuntimeEnv); + disableFuncInReverseScan(pQInfo); + setupQueryRangeForReverseScan(pQInfo); + // clean unused handle if (pRuntimeEnv->pSecQueryHandle != NULL) { tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle); } - // add ref for table pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo); if (pRuntimeEnv->pSecQueryHandle == NULL) { longjmp(pRuntimeEnv->env, terrno); } - - setQueryStatus(pQuery, QUERY_NOT_COMPLETED); - switchCtxOrder(pRuntimeEnv); - disableFuncInReverseScan(pQInfo); } static void clearEnvAfterReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatusInfo *pStatus) { @@ -3290,6 +3308,13 @@ static void clearEnvAfterReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatus pQuery->window = pTableQueryInfo->win; } +static void restoreTimeWindow(STableGroupInfo* pTableGroupInfo, STsdbQueryCond* pCond) { + assert(pTableGroupInfo->numOfTables == 1); + SArray* pTableKeyGroup = taosArrayGetP(pTableGroupInfo->pGroupList, 0); + STableKeyInfo* pKeyInfo = taosArrayGet(pTableKeyGroup, 0); + pKeyInfo->lastKey = pCond->twindow.skey; +} + void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { SQInfo *pQInfo = (SQInfo *) GET_QINFO_ADDR(pRuntimeEnv); SQuery *pQuery = pRuntimeEnv->pQuery; @@ -3337,6 +3362,7 @@ void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle); } + restoreTimeWindow(&pQInfo->tableGroupInfo, &cond); pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo); if (pRuntimeEnv->pSecQueryHandle == NULL) { longjmp(pRuntimeEnv->env, terrno); @@ -4409,9 +4435,11 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) { // todo refactor SArray *g1 = taosArrayInit(1, POINTER_BYTES); - SArray *tx = taosArrayInit(1, POINTER_BYTES); + SArray *tx = taosArrayInit(1, sizeof(STableKeyInfo)); + + STableKeyInfo info = {.pTable = pCheckInfo->pTable, .lastKey = pCheckInfo->lastKey}; + taosArrayPush(tx, &info); - taosArrayPush(tx, &pCheckInfo->pTable); taosArrayPush(g1, &tx); STableGroupInfo gp = {.numOfTables = 1, .pGroupList = g1}; @@ -4561,7 +4589,9 @@ static void sequentialTableProcess(SQInfo *pQInfo) { pRuntimeEnv->pQueryHandle = NULL; } + // no need to update the lastkey for each table pRuntimeEnv->pQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &gp, pQInfo); + taosArrayDestroy(g1); taosArrayDestroy(tx); if (pRuntimeEnv->pQueryHandle == NULL) { @@ -4687,8 +4717,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { taosArrayPush(pQInfo->arrTableIdInfo, &tidInfo); // if the buffer is full or group by each table, we need to jump out of the loop - if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL) /*|| - isGroupbyEachTable(pQuery->pGroupbyExpr, pSupporter->pSidSet)*/) { + if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)) { break; } @@ -4753,21 +4782,22 @@ static void doSaveContext(SQInfo *pQInfo) { .colList = pQuery->colList, .numOfCols = pQuery->numOfCols, }; - + // clean unused handle if (pRuntimeEnv->pSecQueryHandle != NULL) { tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle); } + setQueryStatus(pQuery, QUERY_NOT_COMPLETED); + switchCtxOrder(pRuntimeEnv); + disableFuncInReverseScan(pQInfo); + setupQueryRangeForReverseScan(pQInfo); + pRuntimeEnv->prevGroupId = INT32_MIN; pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo); if (pRuntimeEnv->pSecQueryHandle == NULL) { longjmp(pRuntimeEnv->env, terrno); } - - setQueryStatus(pQuery, QUERY_NOT_COMPLETED); - switchCtxOrder(pRuntimeEnv); - disableFuncInReverseScan(pQInfo); } static void doRestoreContext(SQInfo *pQInfo) { @@ -5861,8 +5891,8 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList, } for(int32_t j = 0; j < s; ++j) { - void* pTable = taosArrayGetP(pa, j); - STableId* id = TSDB_TABLEID(pTable); + STableKeyInfo* info = taosArrayGet(pa, j); + STableId* id = TSDB_TABLEID(info->pTable); STableIdInfo* pTableId = taosArraySearch(pTableIdList, id, compareTableIdInfo); if (pTableId != NULL ) { @@ -5872,10 +5902,11 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList, } void* buf = pQInfo->pBuf + index * sizeof(STableQueryInfo); - STableQueryInfo* item = createTableQueryInfo(&pQInfo->runtimeEnv, pTable, window, buf); + STableQueryInfo* item = createTableQueryInfo(&pQInfo->runtimeEnv, info->pTable, window, buf); if (item == NULL) { goto _cleanup; } + item->groupIndex = i; taosArrayPush(p1, &item); taosHashPut(pQInfo->tableqinfoGroupInfo.map, &id->tid, sizeof(id->tid), &item, POINTER_BYTES); @@ -5904,6 +5935,7 @@ _cleanup_query: taosArrayDestroy(pGroupbyExpr->columnInfo); free(pGroupbyExpr); } + taosTFree(pTagCols); for (int32_t i = 0; i < numOfOutput; ++i) { SExprInfo* pExprInfo = &pExprs[i]; @@ -5911,6 +5943,7 @@ _cleanup_query: tExprTreeDestroy(&pExprInfo->pExpr, NULL); } } + taosTFree(pExprs); _cleanup: @@ -6198,7 +6231,7 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qi STableIdInfo *id = taosArrayGet(pTableIdList, 0); qDebug("qmsg:%p query normal table, uid:%"PRId64", tid:%d", pQueryMsg, id->uid, id->tid); - if ((code = tsdbGetOneTableGroup(tsdb, id->uid, &tableGroupInfo)) != TSDB_CODE_SUCCESS) { + if ((code = tsdbGetOneTableGroup(tsdb, id->uid, pQueryMsg->window.skey, &tableGroupInfo)) != TSDB_CODE_SUCCESS) { goto _over; } } else if (TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_MULTITABLE_QUERY|TSDB_QUERY_TYPE_STABLE_QUERY)) { @@ -6215,8 +6248,9 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qi } qDebug("qmsg:%p query stable, uid:%"PRId64", tid:%d", pQueryMsg, id->uid, id->tid); - code = tsdbQuerySTableByTagCond(tsdb, id->uid, tagCond, pQueryMsg->tagCondLen, pQueryMsg->tagNameRelType, tbnameCond, &tableGroupInfo, pGroupColIndex, - numOfGroupByCols); + code = tsdbQuerySTableByTagCond(tsdb, id->uid, pQueryMsg->window.skey, tagCond, pQueryMsg->tagCondLen, + pQueryMsg->tagNameRelType, tbnameCond, &tableGroupInfo, pGroupColIndex, numOfGroupByCols); + if (code != TSDB_CODE_SUCCESS) { qError("qmsg:%p failed to query stable, reason: %s", pQueryMsg, tstrerror(code)); goto _over; diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 2537e0e822..b086451dd1 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -172,6 +172,7 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab if (pQueryHandle == NULL) { goto out_of_memory; } + pQueryHandle->order = pCond->order; pQueryHandle->window = pCond->twindow; pQueryHandle->pTsdb = tsdb; @@ -190,9 +191,6 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab tsdbTakeMemSnapshot(pQueryHandle->pTsdb, &pQueryHandle->mem, &pQueryHandle->imem); - size_t sizeOfGroup = taosArrayGetSize(groupList->pGroupList); - assert(sizeOfGroup >= 1 && pCond != NULL && pCond->numOfCols > 0); - // allocate buffer in order to load data blocks from file int32_t numOfCols = pCond->numOfCols; @@ -200,6 +198,7 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab if (pQueryHandle->statis == NULL) { goto out_of_memory; } + pQueryHandle->pColumns = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); // todo: use list instead of array? if (pQueryHandle->pColumns == NULL) { goto out_of_memory; @@ -221,9 +220,13 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab if (pQueryHandle->pTableCheckInfo == NULL) { goto out_of_memory; } + STsdbMeta* pMeta = tsdbGetMeta(tsdb); assert(pMeta != NULL); + size_t sizeOfGroup = taosArrayGetSize(groupList->pGroupList); + assert(sizeOfGroup >= 1 && pCond != NULL && pCond->numOfCols > 0); + for (int32_t i = 0; i < sizeOfGroup; ++i) { SArray* group = *(SArray**) taosArrayGet(groupList->pGroupList, i); @@ -231,17 +234,23 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab assert(gsize > 0); for (int32_t j = 0; j < gsize; ++j) { - STable* pTable = (STable*) taosArrayGetP(group, j); + STableKeyInfo* pKeyInfo = (STableKeyInfo*) taosArrayGet(group, j); STableCheckInfo info = { - .lastKey = pQueryHandle->window.skey, - .tableId = pTable->tableId, - .pTableObj = pTable, + .lastKey = pKeyInfo->lastKey, + .tableId = ((STable*)(pKeyInfo->pTable))->tableId, + .pTableObj = pKeyInfo->pTable, }; assert(info.pTableObj != NULL && (info.pTableObj->type == TSDB_NORMAL_TABLE || info.pTableObj->type == TSDB_CHILD_TABLE || info.pTableObj->type == TSDB_STREAM_TABLE)); + if (ASCENDING_TRAVERSE(pQueryHandle->order)) { + assert(info.lastKey >= pQueryHandle->window.skey); + } else { + assert(info.lastKey <= pQueryHandle->window.skey); + } + taosArrayPush(pQueryHandle->pTableCheckInfo, &info); } } @@ -315,19 +324,22 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh assert(pCheckInfo->iter == NULL && pCheckInfo->iiter == NULL); + STableData* pMem = NULL; + STableData* pIMem = NULL; + if (pHandle->mem && pCheckInfo->tableId.tid < pHandle->mem->maxTables) { - STableData* ptd = pHandle->mem->tData[pCheckInfo->tableId.tid]; - if (ptd != NULL && ptd->uid == pCheckInfo->tableId.uid) { // check uid + pMem = pHandle->mem->tData[pCheckInfo->tableId.tid]; + if (pMem != NULL && pMem->uid == pCheckInfo->tableId.uid) { // check uid pCheckInfo->iter = - tSkipListCreateIterFromVal(ptd->pData, (const char*)&pCheckInfo->lastKey, TSDB_DATA_TYPE_TIMESTAMP, order); + tSkipListCreateIterFromVal(pMem->pData, (const char*)&pCheckInfo->lastKey, TSDB_DATA_TYPE_TIMESTAMP, order); } } if (pHandle->imem && pCheckInfo->tableId.tid < pHandle->imem->maxTables) { - STableData* ptd = pHandle->imem->tData[pCheckInfo->tableId.tid]; - if (ptd != NULL && ptd->uid == pCheckInfo->tableId.uid) { // check uid + pIMem = pHandle->imem->tData[pCheckInfo->tableId.tid]; + if (pIMem != NULL && pIMem->uid == pCheckInfo->tableId.uid) { // check uid pCheckInfo->iiter = - tSkipListCreateIterFromVal(ptd->pData, (const char*)&pCheckInfo->lastKey, TSDB_DATA_TYPE_TIMESTAMP, order); + tSkipListCreateIterFromVal(pIMem->pData, (const char*)&pCheckInfo->lastKey, TSDB_DATA_TYPE_TIMESTAMP, order); } } @@ -348,8 +360,17 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh SDataRow row = SL_GET_NODE_DATA(node); TSKEY key = dataRowKey(row); // first timestamp in buffer - tsdbDebug("%p uid:%" PRId64", tid:%d check data in mem from skey:%" PRId64 ", order:%d, %p", pHandle, - pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, key, order, pHandle->qinfo); + tsdbDebug("%p uid:%" PRId64 ", tid:%d check data in mem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 + "-%" PRId64 ", lastKey:%" PRId64 ", %p", + pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, key, order, pMem->keyFirst, pMem->keyLast, + pCheckInfo->lastKey, pHandle->qinfo); + + if (ASCENDING_TRAVERSE(order)) { + assert(pCheckInfo->lastKey <= key); + } else { + assert(pCheckInfo->lastKey >= key); + } + } else { tsdbDebug("%p uid:%"PRId64", tid:%d no data in mem, %p", pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, pHandle->qinfo); @@ -361,8 +382,16 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh SDataRow row = SL_GET_NODE_DATA(node); TSKEY key = dataRowKey(row); // first timestamp in buffer - tsdbDebug("%p uid:%" PRId64", tid:%d check data in imem from skey:%" PRId64 ", order:%d, %p", pHandle, - pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, key, order, pHandle->qinfo); + tsdbDebug("%p uid:%" PRId64 ", tid:%d check data in mem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 + "-%" PRId64 ", lastKey:%" PRId64 ", %p", + pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, key, order, pIMem->keyFirst, pIMem->keyLast, + pCheckInfo->lastKey, pHandle->qinfo); + + if (ASCENDING_TRAVERSE(order)) { + assert(pCheckInfo->lastKey <= key); + } else { + assert(pCheckInfo->lastKey >= key); + } } else { tsdbDebug("%p uid:%"PRId64", tid:%d no data in imem, %p", pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, pHandle->qinfo); @@ -2033,7 +2062,9 @@ static int32_t getAllTableList(STable* pSuperTable, SArray* list) { SSkipListNode* pNode = tSkipListIterGet(iter); STable** pTable = (STable**) SL_GET_NODE_DATA((SSkipListNode*) pNode); - taosArrayPush(list, pTable); + + STableKeyInfo info = {.pTable = *pTable, .lastKey = TSKEY_INITIAL_VAL}; + taosArrayPush(list, &info); } tSkipListDestroyIter(iter); @@ -2089,8 +2120,8 @@ typedef struct STableGroupSupporter { int32_t tableGroupComparFn(const void *p1, const void *p2, const void *param) { STableGroupSupporter* pTableGroupSupp = (STableGroupSupporter*) param; - STable* pTable1 = *(STable**) p1; - STable* pTable2 = *(STable**) p2; + STable* pTable1 = ((STableKeyInfo*) p1)->pTable; + STable* pTable2 = ((STableKeyInfo*) p2)->pTable; for (int32_t i = 0; i < pTableGroupSupp->numOfCols; ++i) { SColIndex* pColIndex = &pTableGroupSupp->pCols[i]; @@ -2140,12 +2171,14 @@ int32_t tableGroupComparFn(const void *p1, const void *p2, const void *param) { return 0; } -void createTableGroupImpl(SArray* pGroups, SArray* pTableList, size_t numOfTables, STableGroupSupporter* pSupp, +void createTableGroupImpl(SArray* pGroups, SArray* pTableList, size_t numOfTables, TSKEY skey, STableGroupSupporter* pSupp, __ext_compar_fn_t compareFn) { STable* pTable = taosArrayGetP(pTableList, 0); - SArray* g = taosArrayInit(16, POINTER_BYTES); - taosArrayPush(g, &pTable); + SArray* g = taosArrayInit(16, sizeof(STableKeyInfo)); + + STableKeyInfo info = {.pTable = pTable, .lastKey = skey}; + taosArrayPush(g, &info); tsdbRefTable(pTable); for (int32_t i = 1; i < numOfTables; ++i) { @@ -2159,18 +2192,21 @@ void createTableGroupImpl(SArray* pGroups, SArray* pTableList, size_t numOfTable assert((*p)->type == TSDB_CHILD_TABLE); if (ret == 0) { - taosArrayPush(g, p); + STableKeyInfo info1 = {.pTable = *p, .lastKey = skey}; + taosArrayPush(g, &info1); } else { taosArrayPush(pGroups, &g); // current group is ended, start a new group - g = taosArrayInit(16, POINTER_BYTES); - taosArrayPush(g, p); + g = taosArrayInit(16, sizeof(STableKeyInfo)); + + STableKeyInfo info1 = {.pTable = *p, .lastKey = skey}; + taosArrayPush(g, &info1); } } taosArrayPush(pGroups, &g); } -SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pCols, int32_t numOfOrderCols) { +SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pCols, int32_t numOfOrderCols, TSKEY skey) { assert(pTableList != NULL); SArray* pTableGroup = taosArrayInit(1, POINTER_BYTES); @@ -2181,13 +2217,16 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC } if (numOfOrderCols == 0 || size == 1) { // no group by tags clause or only one table - SArray* sa = taosArrayInit(size, POINTER_BYTES); - for(int32_t i = 0; i < size; ++i) { - STable** pTable = taosArrayGet(pTableList, i); - assert((*pTable)->type == TSDB_CHILD_TABLE); + SArray* sa = taosArrayInit(size, sizeof(STableKeyInfo)); - tsdbRefTable(*pTable); - taosArrayPush(sa, pTable); + for(int32_t i = 0; i < size; ++i) { + STableKeyInfo *pKeyInfo = taosArrayGet(pTableList, i); + assert(((STable*)pKeyInfo->pTable)->type == TSDB_CHILD_TABLE); + + tsdbRefTable(pKeyInfo->pTable); + + STableKeyInfo info = {.pTable = pKeyInfo->pTable, .lastKey = skey}; + taosArrayPush(sa, &info); } taosArrayPush(pTableGroup, &sa); @@ -2198,8 +2237,8 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC pSupp->pTagSchema = pTagSchema; pSupp->pCols = pCols; - taosqsort(pTableList->pData, size, POINTER_BYTES, pSupp, tableGroupComparFn); - createTableGroupImpl(pTableGroup, pTableList, size, pSupp, tableGroupComparFn); + taosqsort(pTableList->pData, size, sizeof(STableKeyInfo), pSupp, tableGroupComparFn); + createTableGroupImpl(pTableGroup, pTableList, size, skey, pSupp, tableGroupComparFn); taosTFree(pSupp); } @@ -2272,7 +2311,7 @@ static int32_t doQueryTableList(STable* pSTable, SArray* pRes, tExprNode* pExpr) return TSDB_CODE_SUCCESS; } -int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T* tsdb, uint64_t uid, const char* pTagCond, size_t len, +int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T* tsdb, uint64_t uid, TSKEY skey, const char* pTagCond, size_t len, int16_t tagNameRelType, const char* tbnameCond, STableGroupInfo* pGroupInfo, SColIndex* pColIndex, int32_t numOfCols) { if (tsdbRLockRepoMeta(tsdb) < 0) goto _error; @@ -2296,7 +2335,7 @@ int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T* tsdb, uint64_t uid, const char* pT } //NOTE: not add ref count for super table - SArray* res = taosArrayInit(8, POINTER_BYTES); + SArray* res = taosArrayInit(8, sizeof(STableKeyInfo)); STSchema* pTagSchema = tsdbGetTableTagSchema(pTable); // no tags and tbname condition, all child tables of this stable are involved @@ -2308,7 +2347,7 @@ int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T* tsdb, uint64_t uid, const char* pT } pGroupInfo->numOfTables = taosArrayGetSize(res); - pGroupInfo->pGroupList = createTableGroup(res, pTagSchema, pColIndex, numOfCols); + pGroupInfo->pGroupList = createTableGroup(res, pTagSchema, pColIndex, numOfCols, skey); tsdbDebug("%p no table name/tag condition, all tables belong to one group, numOfTables:%zu", tsdb, pGroupInfo->numOfTables); taosArrayDestroy(res); @@ -2351,7 +2390,7 @@ int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T* tsdb, uint64_t uid, const char* pT doQueryTableList(pTable, res, expr); pGroupInfo->numOfTables = taosArrayGetSize(res); - pGroupInfo->pGroupList = createTableGroup(res, pTagSchema, pColIndex, numOfCols); + pGroupInfo->pGroupList = createTableGroup(res, pTagSchema, pColIndex, numOfCols, skey); tsdbDebug("%p stable tid:%d, uid:%"PRIu64" query, numOfTables:%zu, belong to %zu groups", tsdb, pTable->tableId.tid, pTable->tableId.uid, pGroupInfo->numOfTables, taosArrayGetSize(pGroupInfo->pGroupList)); @@ -2365,7 +2404,7 @@ int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T* tsdb, uint64_t uid, const char* pT return terrno; } -int32_t tsdbGetOneTableGroup(TSDB_REPO_T* tsdb, uint64_t uid, STableGroupInfo* pGroupInfo) { +int32_t tsdbGetOneTableGroup(TSDB_REPO_T* tsdb, uint64_t uid, TSKEY startKey, STableGroupInfo* pGroupInfo) { if (tsdbRLockRepoMeta(tsdb) < 0) goto _error; STable* pTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), uid); @@ -2382,9 +2421,11 @@ int32_t tsdbGetOneTableGroup(TSDB_REPO_T* tsdb, uint64_t uid, STableGroupInfo* p pGroupInfo->numOfTables = 1; pGroupInfo->pGroupList = taosArrayInit(1, POINTER_BYTES); - SArray* group = taosArrayInit(1, POINTER_BYTES); + SArray* group = taosArrayInit(1, sizeof(STableKeyInfo)); + + STableKeyInfo info = {.pTable = pTable, .lastKey = startKey}; + taosArrayPush(group, &info); - taosArrayPush(group, &pTable); taosArrayPush(pGroupInfo->pGroupList, &group); return TSDB_CODE_SUCCESS; @@ -2401,7 +2442,7 @@ int32_t tsdbGetTableGroupFromIdList(TSDB_REPO_T* tsdb, SArray* pTableIdList, STa assert(pTableIdList != NULL); size_t size = taosArrayGetSize(pTableIdList); pGroupInfo->pGroupList = taosArrayInit(1, POINTER_BYTES); - SArray* group = taosArrayInit(1, POINTER_BYTES); + SArray* group = taosArrayInit(1, sizeof(STableKeyInfo)); int32_t i = 0; for(; i < size; ++i) { @@ -2419,7 +2460,9 @@ int32_t tsdbGetTableGroupFromIdList(TSDB_REPO_T* tsdb, SArray* pTableIdList, STa } tsdbRefTable(pTable); - taosArrayPush(group, &pTable); + + STableKeyInfo info = {.pTable = pTable, .lastKey = id->key}; + taosArrayPush(group, &info); } if (tsdbUnlockRepoMeta(tsdb) < 0) { diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index e7f4b29744..b5f8515f78 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -104,8 +104,10 @@ static FORCE_INLINE void taosCacheReleaseNode(SCacheObj *pCacheObj, SCacheDataNo pCacheObj->totalSize -= pNode->size; int32_t size = taosHashGetSize(pCacheObj->pHashTable); - uDebug("cache:%s, key:%p, %p is destroyed from cache, totalNum:%d totalSize:%" PRId64 "bytes size:%dbytes", - pCacheObj->name, pNode->key, pNode->data, size, pCacheObj->totalSize, pNode->size); + assert(size > 0); + + uDebug("cache:%s, key:%p, %p is destroyed from cache, size:%dbytes, num:%d size:%" PRId64 "bytes", + pCacheObj->name, pNode->key, pNode->data, pNode->size, size - 1, pCacheObj->totalSize); if (pCacheObj->freeFp) { pCacheObj->freeFp(pNode->data); @@ -428,7 +430,7 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { if (pCacheObj->extendLifespan && (!inTrashCan) && (!_remove)) { atomic_store_64(&pNode->expireTime, pNode->lifespan + taosGetTimestampMs()); - uDebug("cache:%s data:%p extend life time to %"PRId64 " before release", pCacheObj->name, pNode->data, pNode->expireTime); + uDebug("cache:%s data:%p extend expire time: %"PRId64, pCacheObj->name, pNode->data, pNode->expireTime); } if (_remove) { @@ -471,9 +473,9 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { } else { // ref == 0 atomic_sub_fetch_64(&pCacheObj->totalSize, pNode->size); - uDebug("cache:%s, key:%p, %p is destroyed from cache, totalNum:%d totalSize:%" PRId64 "bytes size:%dbytes", - pCacheObj->name, pNode->key, pNode->data, (int32_t)taosHashGetSize(pCacheObj->pHashTable), - pCacheObj->totalSize, pNode->size); + int32_t size = taosHashGetSize(pCacheObj->pHashTable); + uDebug("cache:%s, key:%p, %p is destroyed from cache, size:%dbytes, num:%d size:%" PRId64 "bytes", + pCacheObj->name, pNode->key, pNode->data, pNode->size, size, pCacheObj->totalSize); if (pCacheObj->freeFp) { pCacheObj->freeFp(pNode->data); @@ -581,7 +583,8 @@ void taosAddToTrash(SCacheObj *pCacheObj, SCacheDataNode *pNode) { pNode->pTNodeHeader = pElem; pCacheObj->numOfElemsInTrash++; - uDebug("key:%p, %p move to trash, numOfElem in trash:%d", pNode->key, pNode->data, pCacheObj->numOfElemsInTrash); + uDebug("%s key:%p, %p move to trash, numOfElem in trash:%d", pCacheObj->name, pNode->key, pNode->data, + pCacheObj->numOfElemsInTrash); } void taosTrashCanEmpty(SCacheObj *pCacheObj, bool force) { @@ -623,28 +626,13 @@ void taosTrashCanEmpty(SCacheObj *pCacheObj, bool force) { } void doCleanupDataCache(SCacheObj *pCacheObj) { - -// SHashMutableIterator *pIter = taosHashCreateIter(pCacheObj->pHashTable); -// while (taosHashIterNext(pIter)) { -// SCacheDataNode *pNode = *(SCacheDataNode **)taosHashIterGet(pIter); -// -// int32_t c = T_REF_VAL_GET(pNode); -// if (c <= 0) { -// taosCacheReleaseNode(pCacheObj, pNode); -// } else { -// uDebug("cache:%s key:%p, %p will not remove from cache, refcnt:%d", pCacheObj->name, pNode->key, -// pNode->data, T_REF_VAL_GET(pNode)); -// } -// } -// -// taosHashDestroyIter(pIter); - SHashTravSupp sup = {.pCacheObj = pCacheObj, .fp = NULL, .time = taosGetTimestampMs()}; taosHashCondTraverse(pCacheObj->pHashTable, travHashTableEmptyFn, &sup); // todo memory leak if there are object with refcount greater than 0 in hash table? taosHashCleanup(pCacheObj->pHashTable); taosTrashCanEmpty(pCacheObj, true); + __cache_lock_destroy(pCacheObj); taosTFree(pCacheObj->name); diff --git a/tests/script/general/parser/topbot.sim b/tests/script/general/parser/topbot.sim index a0c46dbc65..57858ae04e 100644 --- a/tests/script/general/parser/topbot.sim +++ b/tests/script/general/parser/topbot.sim @@ -66,9 +66,19 @@ if $row != 100 then return -1 endi -sql select last(c2) from tb_tb9 +sql select last(*) from tb_tb9 if $row != 1 then return -1 endi +sql select last(c2) from tb_tb9 +if $row != 0 then + return -1 +endi + +sql select first(c2), last(c2) from tb_tb9 +if $row != 0 then + return -1 +endi + system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file From 8bdf3a05e0b463922b8db2a611c4eec9c0354714 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 3 Aug 2020 14:36:39 +0800 Subject: [PATCH 028/109] [td-225] fix memory leaks --- src/mnode/src/mnodeProfile.c | 2 +- src/util/src/hash.c | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/mnode/src/mnodeProfile.c b/src/mnode/src/mnodeProfile.c index 7079b1a26a..5e11ce98de 100644 --- a/src/mnode/src/mnodeProfile.c +++ b/src/mnode/src/mnodeProfile.c @@ -99,7 +99,7 @@ SConnObj *mnodeCreateConn(char *user, uint32_t ip, uint16_t port) { tstrncpy(connObj.user, user, sizeof(connObj.user)); SConnObj *pConn = taosCachePut(tsMnodeConnCache, &connId, sizeof(int32_t), &connObj, sizeof(connObj), CONN_KEEP_TIME); - + mDebug("connId:%d, is created, user:%s ip:%s:%u", connId, user, taosIpStr(ip), port); return pConn; } diff --git a/src/util/src/hash.c b/src/util/src/hash.c index 83e2630e41..71a51c4cea 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -409,6 +409,7 @@ int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), voi // not qualified, remove it if (fp && (!fp(param, pNode->data))) { doPopFromEntryList(pEntry, pNode); + FREE_HASH_NODE(pNode); } pNode = pNext; From aadc31f73b3bf4e3e804dfa2aa840e95ae43d88a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 3 Aug 2020 14:46:29 +0800 Subject: [PATCH 029/109] [td-225] fix bugs in bottom query. --- src/client/src/tscFunctionImpl.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index c4f768f6ac..6dff3ddc8e 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -1942,11 +1942,12 @@ static void do_top_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, static void do_bottom_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, int64_t ts, uint16_t type, SExtTagsInfo *pTagInfo, char *pTags, int16_t stage) { - tValuePair **pList = pInfo->res; - tVariant val = {0}; tVariantCreateFromBinary(&val, pData, tDataTypeDesc[type].nSize, type); - + + tValuePair **pList = pInfo->res; + assert(pList != NULL); + if (pInfo->num < maxLen) { if (pInfo->num == 0) { valuePairAssign(pList[pInfo->num], type, (const char*) &val.i64Key, ts, pTags, pTagInfo, stage); @@ -2379,8 +2380,9 @@ static void bottom_func_second_merge(SQLFunctionCtx *pCtx) { // the intermediate result is binary, we only use the output data type for (int32_t i = 0; i < pInput->num; ++i) { + int16_t type = (pCtx->outputType == TSDB_DATA_TYPE_FLOAT)? TSDB_DATA_TYPE_DOUBLE:pCtx->outputType; do_bottom_function_add(pOutput, pCtx->param[0].i64Key, &pInput->res[i]->v.i64Key, pInput->res[i]->timestamp, - pCtx->outputType, &pCtx->tagInfo, pInput->res[i]->pTags, pCtx->currentStage); + type, &pCtx->tagInfo, pInput->res[i]->pTags, pCtx->currentStage); } SET_VAL(pCtx, pInput->num, pOutput->num); From a79f608dd4d98615dc9d31cc8bd3b1effbc6dfd3 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 3 Aug 2020 14:46:56 +0800 Subject: [PATCH 030/109] [td-225] update the sim script. --- tests/script/general/parser/topbot.sim | 37 ++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/tests/script/general/parser/topbot.sim b/tests/script/general/parser/topbot.sim index 57858ae04e..fdda79451d 100644 --- a/tests/script/general/parser/topbot.sim +++ b/tests/script/general/parser/topbot.sim @@ -66,6 +66,11 @@ if $row != 100 then return -1 endi +sql select bottom(c1, 100) from tb_stb0 +if $row != 100 then + return -1 +endi + sql select last(*) from tb_tb9 if $row != 1 then return -1 @@ -81,4 +86,36 @@ if $row != 0 then return -1 endi +sql create table test(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, col7 bool, col8 binary(20), col9 nchar(20)) tags(loc nchar(20)); +sql create table test1 using test tags('beijing'); +sql insert into test1 values(1537146000000, 1, 1, 1, 1, 0.100000, 0.100000, 0, 'taosdata1', '涛æ€æ•°æ®1'); +sql insert into test1 values(1537146000001, 2, 2, 2, 2, 1.100000, 1.100000, 1, 'taosdata2', '涛æ€æ•°æ®2'); +sql insert into test1 values(1537146000002, 3, 3, 3, 3, 2.100000, 2.100000, 0, 'taosdata3', '涛æ€æ•°æ®3'); +sql insert into test1 values(1537146000003, 4, 4, 4, 4, 3.100000, 3.100000, 1, 'taosdata4', '涛æ€æ•°æ®4'); +sql insert into test1 values(1537146000004, 5, 5, 5, 5, 4.100000, 4.100000, 0, 'taosdata5', '涛æ€æ•°æ®5'); +sql insert into test1 values(1537146000005, 6, 6, 6, 6, 5.100000, 5.100000, 1, 'taosdata6', '涛æ€æ•°æ®6'); +sql insert into test1 values(1537146000006, 7, 7, 7, 7, 6.100000, 6.100000, 0, 'taosdata7', '涛æ€æ•°æ®7'); +sql insert into test1 values(1537146000007, 8, 8, 8, 8, 7.100000, 7.100000, 1, 'taosdata8', '涛æ€æ•°æ®8'); +sql insert into test1 values(1537146000008, 9, 9, 9, 9, 8.100000, 8.100000, 0, 'taosdata9', '涛æ€æ•°æ®9'); +sql insert into test1 values(1537146000009, 10, 10, 10, 10, 9.100000, 9.100000, 1, 'taosdata10', '涛æ€æ•°æ®10'); +sql select bottom(col5, 10) from test +if $rows != 10 then + return -1 +endi + +if $data01 != 0.10000 then + print expect 0.10000 actual: $data01 + return -1 +endi + +if $data11 != 1.10000 then + print expect 1.10000 actual: $data11 + return -1 +endi + +if $data21 != 2.10000 then + print expect 2.10000 actual: $data21 + return -1 +endi + system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file From 34c8f42a8072c75a72f144ef52298b34a2847d27 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 3 Aug 2020 17:52:32 +0800 Subject: [PATCH 031/109] minor changes --- src/mnode/src/mnodeProfile.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/mnode/src/mnodeProfile.c b/src/mnode/src/mnodeProfile.c index 5e11ce98de..85457d7a26 100644 --- a/src/mnode/src/mnodeProfile.c +++ b/src/mnode/src/mnodeProfile.c @@ -79,6 +79,7 @@ void mnodeCleanupProfile() { } SConnObj *mnodeCreateConn(char *user, uint32_t ip, uint16_t port) { +#if 0 int32_t connSize = taosHashGetSize(tsMnodeConnCache->pHashTable); if (connSize > tsMaxShellConns) { mError("failed to create conn for user:%s ip:%s:%u, conns:%d larger than maxShellConns:%d, ", user, taosIpStr(ip), @@ -86,6 +87,7 @@ SConnObj *mnodeCreateConn(char *user, uint32_t ip, uint16_t port) { terrno = TSDB_CODE_MND_TOO_MANY_SHELL_CONNS; return NULL; } +#endif int32_t connId = atomic_add_fetch_32(&tsConnIndex, 1); if (connId == 0) atomic_add_fetch_32(&tsConnIndex, 1); From 9a91f0e4c3eb52bf0c456ca957fafc0c47896cb2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 4 Aug 2020 18:13:28 +0800 Subject: [PATCH 032/109] [td-225] add some logs. --- src/dnode/src/dnodeVRead.c | 5 +- src/inc/taoserror.h | 1 + src/query/inc/qExecutor.h | 4 +- src/query/src/qExecutor.c | 30 +++-- src/query/src/qResultbuf.c | 13 ++- src/util/inc/hash.h | 4 +- src/util/src/hash.c | 233 +++++++++++++++++++++---------------- src/vnode/src/vnodeRead.c | 3 +- 8 files changed, 170 insertions(+), 123 deletions(-) diff --git a/src/dnode/src/dnodeVRead.c b/src/dnode/src/dnodeVRead.c index cb53bb5e60..c99cf87b21 100644 --- a/src/dnode/src/dnodeVRead.c +++ b/src/dnode/src/dnodeVRead.c @@ -202,8 +202,9 @@ static void *dnodeProcessReadQueue(void *param) { break; } - dDebug("%p, msg:%s will be processed in vread queue, qtype:%d", pReadMsg->rpcMsg.ahandle, - taosMsg[pReadMsg->rpcMsg.msgType], type); + dDebug("%p, msg:%s will be processed in vread queue, qtype:%d, msg:%p", pReadMsg->rpcMsg.ahandle, + taosMsg[pReadMsg->rpcMsg.msgType], type, pReadMsg); + int32_t code = vnodeProcessRead(pVnode, pReadMsg); if (type == TAOS_QTYPE_RPC && code != TSDB_CODE_QRY_NOT_READY) { diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index ece08ae173..9af4cee28a 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -219,6 +219,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QRY_DUP_JOIN_KEY, 0, 0x0705, "Duplicated TAOS_DEFINE_ERROR(TSDB_CODE_QRY_EXCEED_TAGS_LIMIT, 0, 0x0706, "Tag conditon too many") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_READY, 0, 0x0707, "Query not ready") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_HAS_RSP, 0, 0x0708, "Query should response") +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_IN_EXEC, 0, 0x0709, "Multiple retrieval of this query") // grant TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_EXPIRED, 0, 0x0800, "License expired") diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index 328078fb60..44d5d26f71 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -184,8 +184,8 @@ enum { typedef struct SQInfo { void* signature; - int32_t pointsInterpo; - int32_t code; // error code to returned to client + int32_t code; // error code to returned to client + pthread_t owner; // if it is in execution void* tsdb; int32_t vgId; STableGroupInfo tableGroupInfo; // table id list < only includes the STable list> diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 613c3ae14c..ac90ca9595 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -2784,6 +2784,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { setWindowResultInfo(pResultInfo, pQuery, pRuntimeEnv->stableQuery, buf); resetMergeResultBuf(pQuery, pRuntimeEnv->pCtx, pResultInfo); + // todo add windowRes iterator int64_t lastTimestamp = -1; int64_t startt = taosGetTimestampMs(); @@ -2791,7 +2792,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { int32_t pos = pTree->pNode[0].index; SWindowResInfo *pWindowResInfo = &pTableList[pos]->windowResInfo; - SWindowResult * pWindowRes = getWindowResult(pWindowResInfo, cs.position[pos]); + SWindowResult *pWindowRes = getWindowResult(pWindowResInfo, cs.position[pos]); tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pos.pageId); char *b = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes, page); @@ -2828,6 +2829,9 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { lastTimestamp = ts; + // move to the next element of current entry + int32_t currentPageId = pWindowRes->pos.pageId; + cs.position[pos] += 1; if (cs.position[pos] >= pWindowResInfo->size) { cs.position[pos] = -1; @@ -2836,6 +2840,12 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { if (--numOfTables == 0) { break; } + } else { + // current page is not needed anymore + SWindowResult *pNextWindowRes = getWindowResult(pWindowResInfo, cs.position[pos]); + if (pNextWindowRes->pos.pageId != currentPageId) { + releaseResBufPage(pRuntimeEnv->pResultBuf, page); + } } } @@ -5081,8 +5091,6 @@ static void tableIntervalProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { copyFromWindowResToSData(pQInfo, &pRuntimeEnv->windowResInfo); clearFirstNTimeWindow(pRuntimeEnv, pQInfo->groupIndex); } - - pQInfo->pointsInterpo += numOfFilled; } static void tableQueryImpl(SQInfo *pQInfo) { @@ -6330,16 +6338,24 @@ static bool doBuildResCheck(SQInfo* pQInfo) { pthread_mutex_unlock(&pQInfo->lock); + // clear qhandle owner +// assert(pQInfo->owner == pthread_self()); +// pQInfo->owner = 0; + return buildRes; } bool qTableQuery(qinfo_t qinfo) { SQInfo *pQInfo = (SQInfo *)qinfo; + assert(pQInfo && pQInfo->signature == pQInfo); +// int64_t threadId = pthread_self(); - if (pQInfo == NULL || pQInfo->signature != pQInfo) { - qDebug("QInfo:%p has been freed, no need to execute", pQInfo); - return false; - } +// int64_t curOwner = 0; +// if ((curOwner = atomic_val_compare_exchange_64(&pQInfo->owner, 0, threadId)) != 0) { +// qError("QInfo:%p qhandle is now executed by thread:%p", pQInfo, (void*) curOwner); +// pQInfo->code = TSDB_CODE_QRY_IN_EXEC; +// return false; +// } if (IS_QUERY_KILLED(pQInfo)) { qDebug("QInfo:%p it is already killed, abort", pQInfo); diff --git a/src/query/src/qResultbuf.c b/src/query/src/qResultbuf.c index a577070970..8235bd7b1f 100644 --- a/src/query/src/qResultbuf.c +++ b/src/query/src/qResultbuf.c @@ -186,7 +186,7 @@ static char* loadPageFromDisk(SDiskbasedResultBuf* pResultBuf, SPageInfo* pg) { return GET_DATA_PAYLOAD(pg); } -#define NO_AVAILABLE_PAGES(_b) ((_b)->numOfPages >= (_b)->inMemPages) +#define NO_IN_MEM_AVAILABLE_PAGES(_b) (listNEles((_b)->lruList) >= (_b)->inMemPages) static SIDList addNewGroup(SDiskbasedResultBuf* pResultBuf, int32_t groupId) { assert(taosHashGet(pResultBuf->groupSet, (const char*) &groupId, sizeof(int32_t)) == NULL); @@ -281,7 +281,7 @@ tFilePage* getNewDataBuf(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32 pResultBuf->statis.getPages += 1; char* availablePage = NULL; - if (NO_AVAILABLE_PAGES(pResultBuf)) { + if (NO_IN_MEM_AVAILABLE_PAGES(pResultBuf)) { availablePage = evicOneDataPage(pResultBuf); } @@ -340,7 +340,7 @@ tFilePage* getResBufPage(SDiskbasedResultBuf* pResultBuf, int32_t id) { assert((*pi)->pData == NULL && (*pi)->pn == NULL && (*pi)->info.length >= 0 && (*pi)->info.offset >= 0); char* availablePage = NULL; - if (NO_AVAILABLE_PAGES(pResultBuf)) { + if (NO_IN_MEM_AVAILABLE_PAGES(pResultBuf)) { availablePage = evicOneDataPage(pResultBuf); } @@ -396,12 +396,13 @@ void destroyResultBuf(SDiskbasedResultBuf* pResultBuf) { } if (pResultBuf->file != NULL) { - qDebug("QInfo:%p disk-based output buffer closed, total:%" PRId64 " bytes, file size:%"PRId64" bytes", - pResultBuf->handle, pResultBuf->totalBufSize, pResultBuf->fileSize); + qDebug("QInfo:%p res output buffer closed, total:%" PRId64 " bytes, inmem size:%dbytes, file size:%"PRId64" bytes", + pResultBuf->handle, pResultBuf->totalBufSize, listNEles(pResultBuf->lruList) * pResultBuf->pageSize, + pResultBuf->fileSize); fclose(pResultBuf->file); } else { - qDebug("QInfo:%p disk-based output buffer closed, total:%" PRId64 " bytes, no file created", pResultBuf->handle, + qDebug("QInfo:%p res output buffer closed, total:%" PRId64 " bytes, no file created", pResultBuf->handle, pResultBuf->totalBufSize); } diff --git a/src/util/inc/hash.h b/src/util/inc/hash.h index 71493788ac..f289a4e8c3 100644 --- a/src/util/inc/hash.h +++ b/src/util/inc/hash.h @@ -32,7 +32,7 @@ typedef void (*_hash_free_fn_t)(void *param); typedef struct SHashNode { char *key; - struct SHashNode *prev; +// struct SHashNode *prev; struct SHashNode *next; uint32_t hashVal; // the hash value of key, if hashVal == HASH_VALUE_IN_TRASH, this node is moved to trash uint32_t keyLen; // length of the key @@ -47,7 +47,7 @@ typedef enum SHashLockTypeE { typedef struct SHashEntry { int32_t num; // number of elements in current entry SRWLatch latch; // entry latch - SHashNode head; // dummy head + SHashNode *next; } SHashEntry; typedef struct SHashObj { diff --git a/src/util/src/hash.c b/src/util/src/hash.c index 71a51c4cea..8c74db0082 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -20,12 +20,21 @@ #define HASH_NEED_RESIZE(_h) ((_h)->size >= (_h)->capacity * HASH_DEFAULT_LOAD_FACTOR) -#define FREE_HASH_NODE(_n) \ - do { \ - taosTFree((_n)->data); \ - taosTFree(_n); \ +#define DO_FREE_HASH_NODE(_n) \ + do { \ + taosTFree((_n)->data); \ + taosTFree(_n); \ } while (0) +#define FREE_HASH_NODE(_h, _n) \ + do { \ + if ((_h)->freeFp) { \ + (_h)->freeFp((_n)->data); \ + } \ + \ + DO_FREE_HASH_NODE(_n); \ + } while (0); + static FORCE_INLINE void __wr_lock(void *lock, int32_t type) { if (type == HASH_NO_LOCK) { return; @@ -65,17 +74,8 @@ static FORCE_INLINE int32_t taosHashCapacity(int32_t length) { return i; } -/** - * Get SHashNode from hashlist, nodes from trash are not included. - * @param pHashObj Cache objection - * @param key key for hash - * @param keyLen key length - * @param hashVal hash value by hash function - * @return - */ - static FORCE_INLINE SHashNode *doSearchInEntryList(SHashEntry *pe, const void *key, size_t keyLen, uint32_t hashVal) { - SHashNode *pNode = pe->head.next; + SHashNode *pNode = pe->next; while (pNode) { if ((pNode->keyLen == keyLen) && (memcmp(pNode->key, key, keyLen) == 0)) { assert(pNode->hashVal == hashVal); @@ -88,28 +88,21 @@ static FORCE_INLINE SHashNode *doSearchInEntryList(SHashEntry *pe, const void *k return pNode; } -static FORCE_INLINE SHashNode *doGetNodeFromHashTable(SHashObj *pHashObj, const void *key, uint32_t keyLen, - uint32_t hashVal) { - int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); +static FORCE_INLINE SHashNode *doSerchPrevInEntryList(SHashEntry *pe, const void *key, size_t keyLen, uint32_t hashVal) { + SHashNode *prev= NULL; + SHashNode *pNode = pe->next; - SHashEntry *pe = pHashObj->hashList[slot]; + while (pNode) { + if ((pNode->keyLen == keyLen) && (memcmp(pNode->key, key, keyLen) == 0)) { + assert(pNode->hashVal == hashVal); + break; + } - // no data, return directly - if (atomic_load_32(&pe->num) == 0) { - return NULL; + prev = pNode; + pNode = pNode->next; } - if (pHashObj->type == HASH_ENTRY_LOCK) { - taosRLockLatch(&pe->latch); - } - - SHashNode *pNode = doSearchInEntryList(pe, key, keyLen, hashVal); - - if (pHashObj->type == HASH_ENTRY_LOCK) { - taosRUnLockLatch(&pe->latch); - } - - return pNode; + return prev; } /** @@ -153,7 +146,7 @@ static FORCE_INLINE SHashNode *doUpdateHashNode(SHashNode *pNode, SHashNode *pNe * @param pHashObj * @param pNode */ -static void pushfrontNode(SHashEntry *pEntry, SHashNode *pNode); +static void pushfrontNodeInEntryList(SHashEntry *pEntry, SHashNode *pNode); /** * Get the next element in hash table for iterator @@ -225,7 +218,13 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da taosWLockLatch(&pe->latch); } - SHashNode *pNode = pe->head.next; + SHashNode *pNode = pe->next; + if (pe->num > 0) { + assert(pNode != NULL); + } else { + assert(pNode == NULL); + } + while (pNode) { if ((pNode->keyLen == keyLen) && (memcmp(pNode->key, key, keyLen) == 0)) { assert(pNode->hashVal == hashVal); @@ -237,7 +236,7 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da if (pNode == NULL) { // no data in hash table with the specified key, add it into hash table - pushfrontNode(pe, pNewNode); + pushfrontNodeInEntryList(pe, pNewNode); if (pHashObj->type == HASH_ENTRY_LOCK) { taosWUnLockLatch(&pe->latch); @@ -261,7 +260,7 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da // enable resize __rd_unlock(&pHashObj->lock, pHashObj->type); - FREE_HASH_NODE(pNewNode); + DO_FREE_HASH_NODE(pNewNode); return pHashObj->enableUpdate ? 0 : -1; } } @@ -301,6 +300,7 @@ void *taosHashGetCB(SHashObj *pHashObj, const void *key, size_t keyLen, void (*f if (fp != NULL) { fp(pNode->data); } + data = pNode->data; } @@ -316,13 +316,12 @@ int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen) { return taosHashRemoveWithData(pHashObj, key, keyLen, NULL, 0); } -static FORCE_INLINE void doPopFromEntryList(SHashEntry *pe, SHashNode *pNode) { +static FORCE_INLINE void doPopNextFromEntryList(SHashEntry *pe, SHashNode *pNode) { SHashNode *pNext = pNode->next; - - assert(pNode->prev != NULL); - pNode->prev->next = pNext; if (pNext != NULL) { - pNext->prev = pNode->prev; + pNode->next = pNext->next; + } else { + pNode->next = NULL; } pe->num -= 1; @@ -351,9 +350,27 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe taosWLockLatch(&pe->latch); } - SHashNode *pNode = doSearchInEntryList(pe, key, keyLen, hashVal); - if (pNode != NULL) { - doPopFromEntryList(pe, pNode); + SHashNode *pNode = pe->next; + SHashNode *pRes = NULL; + // remove it + if ((pNode->keyLen == keyLen) && (memcmp(pNode->key, key, keyLen) == 0)) { + pe->next = pNode->next; + pRes = pNode; + } else { + while (pNode->next != NULL) { + if (((pNode->next)->keyLen == keyLen) && (memcmp((pNode->next)->key, key, keyLen) == 0)) { + assert((pNode->next)->hashVal == hashVal); + break; + } + + pNode = pNode->next; + } + + + if (pNode->next != NULL) { + pRes = pNode->next; + pNode->next = pNode->next->next; + } } if (pHashObj->type == HASH_ENTRY_LOCK) { @@ -362,18 +379,14 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe __rd_unlock(&pHashObj->lock, pHashObj->type); - if (data != NULL) { - memcpy(data, pNode->data, dsize); + if (data != NULL && pRes != NULL) { + memcpy(data, pRes->data, dsize); } - if (pNode != NULL) { + if (pRes != NULL) { + pe->num -= 1; atomic_sub_fetch_64(&pHashObj->size, 1); - - pNode->next = NULL; - pNode->prev = NULL; - - FREE_HASH_NODE(pNode); - + FREE_HASH_NODE(pHashObj, pRes); return 0; } else { return -1; @@ -391,7 +404,7 @@ int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), voi int32_t numOfEntries = pHashObj->capacity; for (int32_t i = 0; i < numOfEntries; ++i) { SHashEntry *pEntry = pHashObj->hashList[i]; - if (pEntry->num <= 0) { + if (pEntry->num == 0) { continue; } @@ -399,20 +412,35 @@ int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), voi taosWLockLatch(&pEntry->latch); } - SHashNode *pNode = pEntry->head.next; - assert(pNode != NULL); - - SHashNode *pNext = NULL; - while (pNode != NULL) { - pNext = pNode->next; - - // not qualified, remove it + // todo remove first node + SHashNode *pNode = NULL; + while((pNode = pEntry->next) != NULL) { if (fp && (!fp(param, pNode->data))) { - doPopFromEntryList(pEntry, pNode); - FREE_HASH_NODE(pNode); - } + pEntry->num -= 1; + pEntry->next = pNode->next; - pNode = pNext; + FREE_HASH_NODE(pHashObj, pNode); + } else { + break; + } + } + + // handle the following node + if (pNode != NULL) { + assert(pNode == pEntry->next); + SHashNode *pNext = NULL; + + while ((pNext = pNode->next) != NULL) { + // not qualified, remove it + if (fp && (!fp(param, pNext->data))) { + pNode->next = pNext->next; + pEntry->num -= 1; + + FREE_HASH_NODE(pHashObj, pNext); + } else { + pNode = pNext; + } + } } if (pHashObj->type == HASH_ENTRY_LOCK) { @@ -437,18 +465,15 @@ void taosHashCleanup(SHashObj *pHashObj) { for (int32_t i = 0; i < pHashObj->capacity; ++i) { SHashEntry *pEntry = pHashObj->hashList[i]; if (pEntry->num == 0) { - assert(pEntry->head.next == 0); + assert(pEntry->next == 0); continue; } - pNode = pEntry->head.next; + pNode = pEntry->next; while (pNode) { pNext = pNode->next; - if (pHashObj->freeFp) { - pHashObj->freeFp(pNode->data); - } + FREE_HASH_NODE(pHashObj, pNode); - FREE_HASH_NODE(pNode); pNode = pNext; } } @@ -501,6 +526,8 @@ bool taosHashIterNext(SHashMutableIterator *pIter) { while (1) { SHashEntry *pEntry = pIter->pHashObj->hashList[pIter->entryIndex]; if (pEntry->num == 0) { + assert(pEntry->next == NULL); + pIter->entryIndex++; continue; } @@ -509,7 +536,7 @@ bool taosHashIterNext(SHashMutableIterator *pIter) { taosRLockLatch(&pEntry->latch); } - pIter->pCur = pEntry->head.next; + pIter->pCur = pEntry->next; if (pIter->pCur->next) { pIter->pNext = pIter->pCur->next; @@ -595,7 +622,7 @@ void taosHashTableResize(SHashObj *pHashObj) { return; } - void *pNewEntryList = realloc(pHashObj->hashList, sizeof(SHashEntry) * newSize); + void *pNewEntryList = realloc(pHashObj->hashList, sizeof(void*) * newSize); if (pNewEntryList == NULL) { // todo handle error // uDebug("cache resize failed due to out of memory, capacity remain:%d", pHashObj->capacity); return; @@ -616,33 +643,39 @@ void taosHashTableResize(SHashObj *pHashObj) { for (int32_t i = 0; i < pHashObj->capacity; ++i) { SHashEntry *pe = pHashObj->hashList[i]; if (pe->num == 0) { - assert(pe->head.next == NULL); + assert(pe->next == NULL); continue; } - pNode = pe->head.next; - while (pNode) { + while ((pNode = pe->next) != NULL) { int32_t j = HASH_INDEX(pNode->hashVal, pHashObj->capacity); - if (j == i) { // this key locates in the same slot, no need to relocate it - pNode = pNode->next; - } else { - pNext = pNode->next; - assert(pNode != pNext && (pNext == NULL || pNext->prev == pNode) && pNode->prev->next == pNode); + if (j != i) { + pe->num -= 1; + pe->next = pNode->next; - doPopFromEntryList(pe, pNode); - - // clear pointer - pNode->next = NULL; - pNode->prev = NULL; - - // added into new slot SHashEntry *pNewEntry = pHashObj->hashList[j]; - pushfrontNode(pNewEntry, pNode); - - // continue - pNode = pNext; + pushfrontNodeInEntryList(pNewEntry, pNode); + } else { + break; } } + + if (pNode != NULL) { + while ((pNext = pNode->next) != NULL) { + int32_t j = HASH_INDEX(pNext->hashVal, pHashObj->capacity); + if (j != i) { + pNode->next = pNext->next; + pNext->next = NULL; + + // added into new slot + SHashEntry *pNewEntry = pHashObj->hashList[j]; + pushfrontNodeInEntryList(pNewEntry, pNext); + } else { + pNode = pNext; + } + } + } + } // uDebug("hash table resize completed, new capacity:%d, load factor:%f, elapsed time:%fms", pHashObj->capacity, @@ -668,17 +701,11 @@ SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, s return pNewNode; } -void pushfrontNode(SHashEntry *pEntry, SHashNode *pNode) { +void pushfrontNodeInEntryList(SHashEntry *pEntry, SHashNode *pNode) { assert(pNode != NULL && pEntry != NULL); - SHashNode *pNext = pEntry->head.next; - if (pNext != NULL) { - pNext->prev = pNode; - } - - pNode->next = pNext; - pNode->prev = &pEntry->head; - pEntry->head.next = pNode; + pNode->next = pEntry->next; + pEntry->next = pNode; pEntry->num += 1; } @@ -700,7 +727,7 @@ SHashNode *getNextHashNode(SHashMutableIterator *pIter) { taosRLockLatch(&pEntry->latch); } - p = pEntry->head.next; + p = pEntry->next; if (pIter->pHashObj->type == HASH_ENTRY_LOCK) { taosRUnLockLatch(&pEntry->latch); diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index 3c642b5098..3cf5c1382f 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -74,6 +74,8 @@ static void vnodePutItemIntoReadQueue(SVnodeObj *pVnode, void *qhandle) { pRead->rpcMsg.handle = NULL; atomic_add_fetch_32(&pVnode->refCount, 1); + + vDebug("QInfo:%p add to query task queue for exec, msg:%p", qhandle, pRead); taosWriteQitem(pVnode->rqueue, TAOS_QTYPE_QUERY, pRead); } @@ -83,7 +85,6 @@ static int32_t vnodeDumpQueryResult(SRspRet *pRet, void* pVnode, void* handle, b int32_t code = TSDB_CODE_SUCCESS; if ((code = qDumpRetrieveResult(handle, (SRetrieveTableRsp **)&pRet->rsp, &pRet->len, &continueExec)) == TSDB_CODE_SUCCESS) { if (continueExec) { - vDebug("QInfo:%p add to query task queue for exec", handle); vnodePutItemIntoReadQueue(pVnode, handle); pRet->qhandle = handle; *freeHandle = false; From 6f6896293b6336d7809ce0b2350c36f88bbfe4ec Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 4 Aug 2020 18:57:48 +0800 Subject: [PATCH 033/109] [td-225] add some logs. --- src/util/src/hash.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/util/src/hash.c b/src/util/src/hash.c index 8c74db0082..8e4aa48127 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -352,6 +352,7 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe SHashNode *pNode = pe->next; SHashNode *pRes = NULL; + // remove it if ((pNode->keyLen == keyLen) && (memcmp(pNode->key, key, keyLen) == 0)) { pe->next = pNode->next; @@ -622,6 +623,7 @@ void taosHashTableResize(SHashObj *pHashObj) { return; } + int64_t st = taosGetTimestampUs(); void *pNewEntryList = realloc(pHashObj->hashList, sizeof(void*) * newSize); if (pNewEntryList == NULL) { // todo handle error // uDebug("cache resize failed due to out of memory, capacity remain:%d", pHashObj->capacity); @@ -678,8 +680,10 @@ void taosHashTableResize(SHashObj *pHashObj) { } - // uDebug("hash table resize completed, new capacity:%d, load factor:%f, elapsed time:%fms", pHashObj->capacity, - // ((double)pHashObj->size) / pHashObj->capacity, (et - st) / 1000.0); + int64_t et = taosGetTimestampUs(); + + uDebug("hash table resize completed, new capacity:%"PRId64", load factor:%f, elapsed time:%fms", pHashObj->capacity, + ((double)pHashObj->size) / pHashObj->capacity, (et - st) / 1000.0); } SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, size_t dsize, uint32_t hashVal) { From bd2b9b2b3563b0c40836ad2514dfd1baceeede7a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 4 Aug 2020 19:12:04 +0800 Subject: [PATCH 034/109] [td-225] add some logs. --- src/util/src/hash.c | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/src/util/src/hash.c b/src/util/src/hash.c index 8e4aa48127..19f22766d6 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -284,6 +284,7 @@ void *taosHashGetCB(SHashObj *pHashObj, const void *key, size_t keyLen, void (*f // no data, return directly if (atomic_load_32(&pe->num) == 0) { + __rd_unlock(&pHashObj->lock, pHashObj->type); return NULL; } @@ -295,6 +296,12 @@ void *taosHashGetCB(SHashObj *pHashObj, const void *key, size_t keyLen, void (*f taosRLockLatch(&pe->latch); } + if (pe->num > 0) { + assert(pe->next != NULL); + } else { + assert(pe->next == NULL); + } + SHashNode *pNode = doSearchInEntryList(pe, key, keyLen, hashVal); if (pNode != NULL) { if (fp != NULL) { @@ -342,6 +349,8 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe // no data, return directly if (pe->num == 0) { + assert(pe->next == NULL); + __rd_unlock(&pHashObj->lock, pHashObj->type); return -1; } @@ -374,6 +383,12 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe } } + if (pe->num == 0) { + assert(pe->next == NULL); + } else { + assert(pe->next != NULL); + } + if (pHashObj->type == HASH_ENTRY_LOCK) { taosWUnLockLatch(&pe->latch); } @@ -471,6 +486,8 @@ void taosHashCleanup(SHashObj *pHashObj) { } pNode = pEntry->next; + assert(pNode != NULL); + while (pNode) { pNext = pNode->next; FREE_HASH_NODE(pHashObj, pNode); @@ -655,6 +672,12 @@ void taosHashTableResize(SHashObj *pHashObj) { pe->num -= 1; pe->next = pNode->next; + if (pe->num == 0) { + assert(pe->next == NULL); + } else { + assert(pe->next != NULL); + } + SHashEntry *pNewEntry = pHashObj->hashList[j]; pushfrontNodeInEntryList(pNewEntry, pNode); } else { @@ -671,11 +694,25 @@ void taosHashTableResize(SHashObj *pHashObj) { // added into new slot SHashEntry *pNewEntry = pHashObj->hashList[j]; + + if (pNewEntry->num == 0) { + assert(pNewEntry->next == NULL); + } else { + assert(pNewEntry->next != NULL); + } + pushfrontNodeInEntryList(pNewEntry, pNext); } else { pNode = pNext; } } + + if (pe->num == 0) { + assert(pe->next == NULL); + } else { + assert(pe->next != NULL); + } + } } From d6c19bbb5e609ca2293c3f070add384bba07889a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 4 Aug 2020 19:23:24 +0800 Subject: [PATCH 035/109] [td-225] add some logs. --- src/util/src/hash.c | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/src/util/src/hash.c b/src/util/src/hash.c index 19f22766d6..4c98e59ef0 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -347,6 +347,12 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); SHashEntry *pe = pHashObj->hashList[slot]; + if (pe->num == 0) { + assert(pe->next == NULL); + } else { + assert(pe->next != NULL); + } + // no data, return directly if (pe->num == 0) { assert(pe->next == NULL); @@ -383,12 +389,6 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe } } - if (pe->num == 0) { - assert(pe->next == NULL); - } else { - assert(pe->next != NULL); - } - if (pHashObj->type == HASH_ENTRY_LOCK) { taosWUnLockLatch(&pe->latch); } @@ -403,8 +403,22 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe pe->num -= 1; atomic_sub_fetch_64(&pHashObj->size, 1); FREE_HASH_NODE(pHashObj, pRes); + + if (pe->num == 0) { + assert(pe->next == NULL); + } else { + assert(pe->next != NULL); + } + return 0; } else { + + if (pe->num == 0) { + assert(pe->next == NULL); + } else { + assert(pe->next != NULL); + } + return -1; } } From 6eca95b6ce48c77fbd4376ec04bed75534521f19 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 4 Aug 2020 19:36:01 +0800 Subject: [PATCH 036/109] [td-225] add some logs. --- src/util/src/hash.c | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/util/src/hash.c b/src/util/src/hash.c index 4c98e59ef0..e7e46cd328 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -238,6 +238,12 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da // no data in hash table with the specified key, add it into hash table pushfrontNodeInEntryList(pe, pNewNode); + if (pe->num == 0) { + assert(pe->next == NULL); + } else { + assert(pe->next != NULL); + } + if (pHashObj->type == HASH_ENTRY_LOCK) { taosWUnLockLatch(&pe->latch); } @@ -449,6 +455,12 @@ int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), voi pEntry->num -= 1; pEntry->next = pNode->next; + if (pEntry->num == 0) { + assert(pEntry->next == NULL); + } else { + assert(pEntry->next != NULL); + } + FREE_HASH_NODE(pHashObj, pNode); } else { break; @@ -466,6 +478,12 @@ int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), voi pNode->next = pNext->next; pEntry->num -= 1; + if (pEntry->num == 0) { + assert(pEntry->next == NULL); + } else { + assert(pEntry->next != NULL); + } + FREE_HASH_NODE(pHashObj, pNext); } else { pNode = pNext; @@ -675,6 +693,13 @@ void taosHashTableResize(SHashObj *pHashObj) { pHashObj->capacity = newSize; for (int32_t i = 0; i < pHashObj->capacity; ++i) { SHashEntry *pe = pHashObj->hashList[i]; + + if (pe->num == 0) { + assert(pe->next == NULL); + } else { + assert(pe->next != NULL); + } + if (pe->num == 0) { assert(pe->next == NULL); continue; From 28f2f102aff40f131ab454f2c4dca2e30c0b90b1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 4 Aug 2020 19:57:16 +0800 Subject: [PATCH 037/109] [td-225] add some logs. --- src/util/src/hash.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/util/src/hash.c b/src/util/src/hash.c index e7e46cd328..298d81251c 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -728,6 +728,8 @@ void taosHashTableResize(SHashObj *pHashObj) { while ((pNext = pNode->next) != NULL) { int32_t j = HASH_INDEX(pNext->hashVal, pHashObj->capacity); if (j != i) { + pe->num -= 1; + pNode->next = pNext->next; pNext->next = NULL; From d08e9972b6f05f7b6e36d9d138ef5d4d6664f580 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 5 Aug 2020 00:48:19 +0800 Subject: [PATCH 038/109] [td-225] prevent qinfo from being released while it is in queue. --- src/query/src/qExecutor.c | 3 ++- src/vnode/src/vnodeRead.c | 26 +++++++++++++++++--------- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index ac90ca9595..b5fc67e4f1 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -6408,6 +6408,7 @@ int32_t qRetrieveQueryResultInfo(qinfo_t qinfo, bool* buildRes, void* pRspContex return TSDB_CODE_QRY_INVALID_QHANDLE; } + *buildRes = false; SQuery *pQuery = pQInfo->runtimeEnv.pQuery; if (IS_QUERY_KILLED(pQInfo)) { qDebug("QInfo:%p query is killed, code:%d", pQInfo, pQInfo->code); @@ -6751,7 +6752,7 @@ void** qRegisterQInfo(void* pMgmt, uint64_t qInfo) { return NULL; } - const int32_t DEFAULT_QHANDLE_LIFE_SPAN = tsShellActivityTimer * 2; + const int32_t DEFAULT_QHANDLE_LIFE_SPAN = tsShellActivityTimer * 2 * 1000; SQueryMgmt *pQueryMgmt = pMgmt; if (pQueryMgmt->qinfoPool == NULL) { diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index 3cf5c1382f..896b27e03f 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -159,7 +159,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { // current connect is broken if (code == TSDB_CODE_SUCCESS) { handle = qRegisterQInfo(pVnode->qMgmt, (uint64_t) pQInfo); - if (handle == NULL) { // failed to register qhandle + if (handle == NULL) { // failed to register qhandle, todo add error test case vError("vgId:%d QInfo:%p register qhandle failed, return to app, code:%s", pVnode->vgId, (void *)pQInfo, tstrerror(pRsp->code)); pRsp->code = TSDB_CODE_QRY_INVALID_QHANDLE; @@ -180,12 +180,9 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { } if (handle != NULL) { - vDebug("vgId:%d, QInfo:%p, dnode query msg disposed, register qhandle and return to app", vgId, *handle); - + vDebug("vgId:%d, QInfo:%p, dnode query msg disposed, create qhandle and returns to app", vgId, *handle); vnodePutItemIntoReadQueue(pVnode, *handle); - qReleaseQInfo(pVnode->qMgmt, (void**) &handle, false); } - } else { assert(pCont != NULL); @@ -194,18 +191,19 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { vWarn("QInfo:%p invalid qhandle in continuing exec query, conn:%p", (void*) pCont, pReadMsg->rpcMsg.handle); code = TSDB_CODE_QRY_INVALID_QHANDLE; } else { - vDebug("vgId:%d, QInfo:%p, dnode continue exec query", pVnode->vgId, (void*) pCont); + vDebug("vgId:%d, QInfo:%p, dnode continues to exec query", pVnode->vgId, (void*) pCont); bool freehandle = false; bool buildRes = qTableQuery(*handle); // do execute query - // build query rsp + // build query rsp, the retrieve request has reached here already if (buildRes) { // update the connection info according to the retrieve connection pReadMsg->rpcMsg.handle = qGetResultRetrieveMsg(*handle); assert(pReadMsg->rpcMsg.handle != NULL); - vDebug("vgId:%d, QInfo:%p, start to build result rsp after query paused, %p", pVnode->vgId, *handle, pReadMsg->rpcMsg.handle); + vDebug("vgId:%d, QInfo:%p, start to build retrieval rsp after query paused, %p", pVnode->vgId, *handle, + pReadMsg->rpcMsg.handle); code = vnodeDumpQueryResult(&pReadMsg->rspRet, pVnode, *handle, &freehandle); // todo test the error code case @@ -214,7 +212,17 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { } } - qReleaseQInfo(pVnode->qMgmt, (void**) &handle, freehandle); + // If retrieval request has not arrived, release the qhandle and decrease the reference count to allow + // the queryMgmt to free it when expired + qReleaseQInfo(pVnode->qMgmt, (void**) &handle, false); + + // NOTE: + // if the qhandle is put into query vread queue and wait to be executed by worker in read queue, + // the reference count of qhandle can not be decreased. Otherwise, qhandle may be released before or in the + // procedure of query execution + if (freehandle) { + qReleaseQInfo(pVnode->qMgmt, (void **)&handle, freehandle); + } } } From 7bb42b5f57c076a67b494b91a94aaf531d568d71 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 5 Aug 2020 11:42:36 +0800 Subject: [PATCH 039/109] [td-225] fix mem leak --- src/vnode/src/vnodeRead.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index 896b27e03f..5c7bc869ba 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -85,12 +85,12 @@ static int32_t vnodeDumpQueryResult(SRspRet *pRet, void* pVnode, void* handle, b int32_t code = TSDB_CODE_SUCCESS; if ((code = qDumpRetrieveResult(handle, (SRetrieveTableRsp **)&pRet->rsp, &pRet->len, &continueExec)) == TSDB_CODE_SUCCESS) { if (continueExec) { + *freeHandle = false; vnodePutItemIntoReadQueue(pVnode, handle); pRet->qhandle = handle; - *freeHandle = false; } else { - vDebug("QInfo:%p exec completed", handle); *freeHandle = true; + vDebug("QInfo:%p exec completed, free handle:%d", handle, *freeHandle); } } else { pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp)); @@ -214,6 +214,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { // If retrieval request has not arrived, release the qhandle and decrease the reference count to allow // the queryMgmt to free it when expired + void** dup = handle; qReleaseQInfo(pVnode->qMgmt, (void**) &handle, false); // NOTE: @@ -221,7 +222,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { // the reference count of qhandle can not be decreased. Otherwise, qhandle may be released before or in the // procedure of query execution if (freehandle) { - qReleaseQInfo(pVnode->qMgmt, (void **)&handle, freehandle); + qReleaseQInfo(pVnode->qMgmt, (void **)&dup, freehandle); } } } @@ -268,16 +269,23 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { //TODO handle malloc failure pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp)); memset(pRet->rsp, 0, sizeof(SRetrieveTableRsp)); + qReleaseQInfo(pVnode->qMgmt, (void**) &handle, freeHandle); } else { // result is not ready, return immediately if (!buildRes) { qReleaseQInfo(pVnode->qMgmt, (void**) &handle, false); return TSDB_CODE_QRY_NOT_READY; } + void** dup = handle; code = vnodeDumpQueryResult(pRet, pVnode, *handle, &freeHandle); + qReleaseQInfo(pVnode->qMgmt, (void**) &handle, false); + + // not added into task queue, free it immediate + if (freeHandle) { + qReleaseQInfo(pVnode->qMgmt, (void**) &dup, freeHandle); + } } - qReleaseQInfo(pVnode->qMgmt, (void**) &handle, freeHandle); return code; } From 4222cfcda663152409b1fb04594d4104a7f6df3c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 5 Aug 2020 15:48:35 +0800 Subject: [PATCH 040/109] [td-225] fix bug in hash --- src/util/src/hash.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/util/src/hash.c b/src/util/src/hash.c index 298d81251c..06326e037e 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -376,8 +376,9 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe // remove it if ((pNode->keyLen == keyLen) && (memcmp(pNode->key, key, keyLen) == 0)) { - pe->next = pNode->next; + pe->num -= 1; pRes = pNode; + pe->next = pNode->next; } else { while (pNode->next != NULL) { if (((pNode->next)->keyLen == keyLen) && (memcmp((pNode->next)->key, key, keyLen) == 0)) { @@ -390,6 +391,7 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe if (pNode->next != NULL) { + pe->num -= 1; pRes = pNode->next; pNode->next = pNode->next->next; } @@ -406,7 +408,6 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe } if (pRes != NULL) { - pe->num -= 1; atomic_sub_fetch_64(&pHashObj->size, 1); FREE_HASH_NODE(pHashObj, pRes); From 0ce566550d061a3dcdde5da6e862b2034ace4997 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 5 Aug 2020 16:24:42 +0800 Subject: [PATCH 041/109] [td-225] fix bug in hash and queryhandle management. --- src/inc/query.h | 3 ++ src/query/src/qExecutor.c | 58 ++++++++++++------------------------- src/util/src/hash.c | 22 +++++++++----- src/util/src/tcache.c | 9 ++++-- src/vnode/src/vnodeRead.c | 61 ++++++++++++++++++--------------------- 5 files changed, 72 insertions(+), 81 deletions(-) diff --git a/src/inc/query.h b/src/inc/query.h index ec1e458b62..0c18f85dc3 100644 --- a/src/inc/query.h +++ b/src/inc/query.h @@ -76,6 +76,9 @@ void* qGetResultRetrieveMsg(qinfo_t qinfo); */ int32_t qKillQuery(qinfo_t qinfo); +int32_t qQueryCompleted(qinfo_t qinfo); + + /** * destroy query info structure * @param qHandle diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index b5fc67e4f1..0ffb1a4cde 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -6432,34 +6432,6 @@ int32_t qRetrieveQueryResultInfo(qinfo_t qinfo, bool* buildRes, void* pRspContex return code; } -bool qHasMoreResultsToRetrieve(qinfo_t qinfo) { - SQInfo *pQInfo = (SQInfo *)qinfo; - - if (!isValidQInfo(pQInfo) || pQInfo->code != TSDB_CODE_SUCCESS) { - qDebug("QInfo:%p invalid qhandle or error occurs, abort query, code:%x", pQInfo, pQInfo->code); - return false; - } - - SQuery *pQuery = pQInfo->runtimeEnv.pQuery; - - bool ret = false; - if (Q_STATUS_EQUAL(pQuery->status, QUERY_OVER)) { - ret = false; - } else if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)) { - ret = true; - } else if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { - ret = true; - } else { - assert(0); - } - - if (ret) { - qDebug("QInfo:%p has more results waits for client retrieve", pQInfo); - } - - return ret; -} - int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *contLen, bool* continueExec) { SQInfo *pQInfo = (SQInfo *)qinfo; @@ -6487,11 +6459,11 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co int32_t code = pQInfo->code; if (code == TSDB_CODE_SUCCESS) { - (*pRsp)->offset = htobe64(pQuery->limit.offset); + (*pRsp)->offset = htobe64(pQuery->limit.offset); (*pRsp)->useconds = htobe64(pRuntimeEnv->summary.elapsedTime); } else { - (*pRsp)->useconds = 0; - (*pRsp)->offset = 0; + (*pRsp)->offset = 0; + (*pRsp)->useconds = htobe64(pRuntimeEnv->summary.elapsedTime); } (*pRsp)->precision = htons(pQuery->precision); @@ -6503,22 +6475,30 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co } pQInfo->rspContext = NULL; - pQInfo->dataReady = QUERY_RESULT_NOT_READY; + pQInfo->dataReady = QUERY_RESULT_NOT_READY; if (IS_QUERY_KILLED(pQInfo) || Q_STATUS_EQUAL(pQuery->status, QUERY_OVER)) { - (*pRsp)->completed = 1; // notify no more result to client - } - - if (qHasMoreResultsToRetrieve(pQInfo)) { - *continueExec = true; - } else { // failed to dump result, free qhandle immediately *continueExec = false; - qKillQuery(pQInfo); + (*pRsp)->completed = 1; // notify no more result to client + } else { + *continueExec = true; + qDebug("QInfo:%p has more results waits for client retrieve", pQInfo); } return code; } +int32_t qQueryCompleted(qinfo_t qinfo) { + SQInfo *pQInfo = (SQInfo *)qinfo; + + if (pQInfo == NULL || !isValidQInfo(pQInfo)) { + return TSDB_CODE_QRY_INVALID_QHANDLE; + } + + SQuery* pQuery = pQInfo->runtimeEnv.pQuery; + return IS_QUERY_KILLED(pQInfo) || Q_STATUS_EQUAL(pQuery->status, QUERY_OVER); +} + int32_t qKillQuery(qinfo_t qinfo) { SQInfo *pQInfo = (SQInfo *)qinfo; diff --git a/src/util/src/hash.c b/src/util/src/hash.c index 06326e037e..96b4e9cd28 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -353,16 +353,9 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); SHashEntry *pe = pHashObj->hashList[slot]; - if (pe->num == 0) { - assert(pe->next == NULL); - } else { - assert(pe->next != NULL); - } - // no data, return directly if (pe->num == 0) { assert(pe->next == NULL); - __rd_unlock(&pHashObj->lock, pHashObj->type); return -1; } @@ -371,6 +364,21 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe taosWLockLatch(&pe->latch); } + if (pe->num == 0) { + assert(pe->next == NULL); + } else { + assert(pe->next != NULL); + } + + // double check after locked + if (pe->num == 0) { + assert(pe->next == NULL); + taosWUnLockLatch(&pe->latch); + + __rd_unlock(&pHashObj->lock, pHashObj->type); + return -1; + } + SHashNode *pNode = pe->next; SHashNode *pRes = NULL; diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index b5f8515f78..8a4145f2f8 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -438,8 +438,8 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { char* key = pNode->key; char* d = pNode->data; - int32_t ref = T_REF_DEC(pNode); - uDebug("cache:%s, key:%p, %p is released, refcnt:%d", pCacheObj->name, key, d, ref); + int32_t ref = T_REF_VAL_GET(pNode); + uDebug("cache:%s, key:%p, %p is released, refcnt:%d", pCacheObj->name, key, d, ref - 1); /* * If it is not referenced by other users, remove it immediately. Otherwise move this node to trashcan wait for all users @@ -449,6 +449,8 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { * that tries to do the same thing. */ if (inTrashCan) { + ref = T_REF_DEC(pNode); + if (ref == 0) { assert(pNode->pTNodeHeader->pData == pNode); @@ -459,7 +461,10 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { doDestroyTrashcanElem(pCacheObj, pNode->pTNodeHeader); } } else { + // NOTE: remove it from hash in the first place, otherwise, the pNode may have been released by other thread + // when reaches here. int32_t ret = taosHashRemove(pCacheObj->pHashTable, pNode->key, pNode->keySize); + ref = T_REF_DEC(pNode); // successfully remove from hash table, if failed, this node must have been move to trash already, do nothing. // note that the remove operation can be executed only once. diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index 5c7bc869ba..0d30be7662 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -66,7 +66,7 @@ int32_t vnodeProcessRead(void *param, SReadMsg *pReadMsg) { return (*vnodeProcessReadMsgFp[msgType])(pVnode, pReadMsg); } -static void vnodePutItemIntoReadQueue(SVnodeObj *pVnode, void *qhandle) { +static void vnodePutItemIntoReadQueue(SVnodeObj *pVnode, void **qhandle) { SReadMsg *pRead = (SReadMsg *)taosAllocateQitem(sizeof(SReadMsg)); pRead->rpcMsg.msgType = TSDB_MSG_TYPE_QUERY; pRead->pCont = qhandle; @@ -75,22 +75,22 @@ static void vnodePutItemIntoReadQueue(SVnodeObj *pVnode, void *qhandle) { atomic_add_fetch_32(&pVnode->refCount, 1); - vDebug("QInfo:%p add to query task queue for exec, msg:%p", qhandle, pRead); + vDebug("QInfo:%p add to vread queue for exec query, msg:%p", *qhandle, pRead); taosWriteQitem(pVnode->rqueue, TAOS_QTYPE_QUERY, pRead); } -static int32_t vnodeDumpQueryResult(SRspRet *pRet, void* pVnode, void* handle, bool* freeHandle) { +static int32_t vnodeDumpQueryResult(SRspRet *pRet, void* pVnode, void** handle, bool* freeHandle) { bool continueExec = false; int32_t code = TSDB_CODE_SUCCESS; - if ((code = qDumpRetrieveResult(handle, (SRetrieveTableRsp **)&pRet->rsp, &pRet->len, &continueExec)) == TSDB_CODE_SUCCESS) { + if ((code = qDumpRetrieveResult(*handle, (SRetrieveTableRsp **)&pRet->rsp, &pRet->len, &continueExec)) == TSDB_CODE_SUCCESS) { if (continueExec) { *freeHandle = false; vnodePutItemIntoReadQueue(pVnode, handle); - pRet->qhandle = handle; + pRet->qhandle = *handle; } else { *freeHandle = true; - vDebug("QInfo:%p exec completed, free handle:%d", handle, *freeHandle); + vDebug("QInfo:%p exec completed, free handle:%d", *handle, *freeHandle); } } else { pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp)); @@ -181,50 +181,45 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { if (handle != NULL) { vDebug("vgId:%d, QInfo:%p, dnode query msg disposed, create qhandle and returns to app", vgId, *handle); - vnodePutItemIntoReadQueue(pVnode, *handle); + vnodePutItemIntoReadQueue(pVnode, handle); } } else { assert(pCont != NULL); + void** qhandle = (void**) pCont; +// *handle = /*(void*) */pCont; - handle = qAcquireQInfo(pVnode->qMgmt, (uint64_t) pCont); - if (handle == NULL) { - vWarn("QInfo:%p invalid qhandle in continuing exec query, conn:%p", (void*) pCont, pReadMsg->rpcMsg.handle); - code = TSDB_CODE_QRY_INVALID_QHANDLE; - } else { - vDebug("vgId:%d, QInfo:%p, dnode continues to exec query", pVnode->vgId, (void*) pCont); +// handle = qAcquireQInfo(pVnode->qMgmt, (uint64_t) pCont); +// if (handle == NULL) { +// vWarn("QInfo:%p invalid qhandle in continuing exec query, conn:%p", (void*) pCont, pReadMsg->rpcMsg.handle); +// code = TSDB_CODE_QRY_INVALID_QHANDLE; +// } else { + vDebug("vgId:%d, QInfo:%p, dnode continues to exec query", pVnode->vgId, *qhandle); bool freehandle = false; - bool buildRes = qTableQuery(*handle); // do execute query + bool buildRes = qTableQuery(*qhandle); // do execute query // build query rsp, the retrieve request has reached here already if (buildRes) { // update the connection info according to the retrieve connection - pReadMsg->rpcMsg.handle = qGetResultRetrieveMsg(*handle); + pReadMsg->rpcMsg.handle = qGetResultRetrieveMsg(*qhandle); assert(pReadMsg->rpcMsg.handle != NULL); - vDebug("vgId:%d, QInfo:%p, start to build retrieval rsp after query paused, %p", pVnode->vgId, *handle, + vDebug("vgId:%d, QInfo:%p, start to build retrieval rsp after query paused, %p", pVnode->vgId, *qhandle, pReadMsg->rpcMsg.handle); - code = vnodeDumpQueryResult(&pReadMsg->rspRet, pVnode, *handle, &freehandle); + code = vnodeDumpQueryResult(&pReadMsg->rspRet, pVnode, qhandle, &freehandle); // todo test the error code case if (code == TSDB_CODE_SUCCESS) { code = TSDB_CODE_QRY_HAS_RSP; } + } else { + freehandle = qQueryCompleted(*qhandle); } - // If retrieval request has not arrived, release the qhandle and decrease the reference count to allow - // the queryMgmt to free it when expired - void** dup = handle; - qReleaseQInfo(pVnode->qMgmt, (void**) &handle, false); - - // NOTE: - // if the qhandle is put into query vread queue and wait to be executed by worker in read queue, - // the reference count of qhandle can not be decreased. Otherwise, qhandle may be released before or in the - // procedure of query execution - if (freehandle) { - qReleaseQInfo(pVnode->qMgmt, (void **)&dup, freehandle); + // NOTE: if the qhandle is not put into vread queue or query is completed, free the qhandle. + if (freehandle || (!buildRes)) { + qReleaseQInfo(pVnode->qMgmt, (void **)&qhandle, freehandle); } - } } return code; @@ -269,7 +264,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { //TODO handle malloc failure pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp)); memset(pRet->rsp, 0, sizeof(SRetrieveTableRsp)); - qReleaseQInfo(pVnode->qMgmt, (void**) &handle, freeHandle); + qReleaseQInfo(pVnode->qMgmt, (void**) &handle, true); } else { // result is not ready, return immediately if (!buildRes) { qReleaseQInfo(pVnode->qMgmt, (void**) &handle, false); @@ -277,12 +272,12 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { } void** dup = handle; - code = vnodeDumpQueryResult(pRet, pVnode, *handle, &freeHandle); + code = vnodeDumpQueryResult(pRet, pVnode, handle, &freeHandle); qReleaseQInfo(pVnode->qMgmt, (void**) &handle, false); - // not added into task queue, free it immediate + // not added into task queue, the query must be completed already, free qhandle immediate if (freeHandle) { - qReleaseQInfo(pVnode->qMgmt, (void**) &dup, freeHandle); + qReleaseQInfo(pVnode->qMgmt, (void**) &dup, true); } } From 175dfb56a02e3691441a30124bf334fd6b8b8932 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 5 Aug 2020 17:03:59 +0800 Subject: [PATCH 042/109] [td-225] fix bug in hash --- src/util/src/tcache.c | 2 ++ src/vnode/src/vnodeRead.c | 52 +++++++++++++++++---------------------- 2 files changed, 25 insertions(+), 29 deletions(-) diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index 8a4145f2f8..51991974ff 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -318,6 +318,8 @@ static void incRefFn(void* ptNode) { assert(ptNode != NULL); SCacheDataNode** p = (SCacheDataNode**) ptNode; + + assert(T_REF_VAL_GET(*p) >= 0); int32_t ret = T_REF_INC(*p); assert(ret > 0); } diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index 0d30be7662..e391fc5eb8 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -186,40 +186,35 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { } else { assert(pCont != NULL); void** qhandle = (void**) pCont; -// *handle = /*(void*) */pCont; -// handle = qAcquireQInfo(pVnode->qMgmt, (uint64_t) pCont); -// if (handle == NULL) { -// vWarn("QInfo:%p invalid qhandle in continuing exec query, conn:%p", (void*) pCont, pReadMsg->rpcMsg.handle); -// code = TSDB_CODE_QRY_INVALID_QHANDLE; -// } else { - vDebug("vgId:%d, QInfo:%p, dnode continues to exec query", pVnode->vgId, *qhandle); + vDebug("vgId:%d, QInfo:%p, dnode continues to exec query", pVnode->vgId, *qhandle); - bool freehandle = false; - bool buildRes = qTableQuery(*qhandle); // do execute query + bool freehandle = false; + bool buildRes = qTableQuery(*qhandle); // do execute query - // build query rsp, the retrieve request has reached here already - if (buildRes) { - // update the connection info according to the retrieve connection - pReadMsg->rpcMsg.handle = qGetResultRetrieveMsg(*qhandle); - assert(pReadMsg->rpcMsg.handle != NULL); + // build query rsp, the retrieve request has reached here already + if (buildRes) { + // update the connection info according to the retrieve connection + pReadMsg->rpcMsg.handle = qGetResultRetrieveMsg(*qhandle); + assert(pReadMsg->rpcMsg.handle != NULL); - vDebug("vgId:%d, QInfo:%p, start to build retrieval rsp after query paused, %p", pVnode->vgId, *qhandle, - pReadMsg->rpcMsg.handle); - code = vnodeDumpQueryResult(&pReadMsg->rspRet, pVnode, qhandle, &freehandle); + vDebug("vgId:%d, QInfo:%p, start to build retrieval rsp after query paused, %p", pVnode->vgId, *qhandle, + pReadMsg->rpcMsg.handle); + code = vnodeDumpQueryResult(&pReadMsg->rspRet, pVnode, qhandle, &freehandle); - // todo test the error code case - if (code == TSDB_CODE_SUCCESS) { - code = TSDB_CODE_QRY_HAS_RSP; - } - } else { - freehandle = qQueryCompleted(*qhandle); + // todo test the error code case + if (code == TSDB_CODE_SUCCESS) { + code = TSDB_CODE_QRY_HAS_RSP; } + } else { + freehandle = qQueryCompleted(*qhandle); + } - // NOTE: if the qhandle is not put into vread queue or query is completed, free the qhandle. - if (freehandle || (!buildRes)) { - qReleaseQInfo(pVnode->qMgmt, (void **)&qhandle, freehandle); - } + // NOTE: if the qhandle is not put into vread queue or query is completed, free the qhandle. + // if not build result, free it not by forced. + if (freehandle || (!buildRes)) { + qReleaseQInfo(pVnode->qMgmt, (void **)&qhandle, freehandle); + } } return code; @@ -273,9 +268,8 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { void** dup = handle; code = vnodeDumpQueryResult(pRet, pVnode, handle, &freeHandle); - qReleaseQInfo(pVnode->qMgmt, (void**) &handle, false); - // not added into task queue, the query must be completed already, free qhandle immediate + // not added into task queue, the query must be completed already, free qhandle immediately if (freeHandle) { qReleaseQInfo(pVnode->qMgmt, (void**) &dup, true); } From 321f7b276bebaef6c485361ccee469ba8737d7ec Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 5 Aug 2020 23:07:34 +0800 Subject: [PATCH 043/109] [td-225] fix bug in hash --- src/util/src/hash.c | 13 +----- src/util/src/tcache.c | 86 +-------------------------------------- src/vnode/src/vnodeRead.c | 12 +++--- 3 files changed, 8 insertions(+), 103 deletions(-) diff --git a/src/util/src/hash.c b/src/util/src/hash.c index 96b4e9cd28..a70256666f 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -329,17 +329,6 @@ int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen) { return taosHashRemoveWithData(pHashObj, key, keyLen, NULL, 0); } -static FORCE_INLINE void doPopNextFromEntryList(SHashEntry *pe, SHashNode *pNode) { - SHashNode *pNext = pNode->next; - if (pNext != NULL) { - pNode->next = pNext->next; - } else { - pNode->next = NULL; - } - - pe->num -= 1; -} - int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t dsize) { if (pHashObj == NULL || pHashObj->size <= 0) { return -1; @@ -457,7 +446,7 @@ int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), voi taosWLockLatch(&pEntry->latch); } - // todo remove first node + // todo remove the first node SHashNode *pNode = NULL; while((pNode = pEntry->next) != NULL) { if (fp && (!fp(param, pNode->data))) { diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index 51991974ff..b6df852876 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -116,16 +116,6 @@ static FORCE_INLINE void taosCacheReleaseNode(SCacheObj *pCacheObj, SCacheDataNo free(pNode); } -/** - * move the old node into trash - * @param pCacheObj - * @param pNode - */ -static FORCE_INLINE void taosCacheMoveToTrash(SCacheObj *pCacheObj, SCacheDataNode *pNode) { - taosHashRemove(pCacheObj->pHashTable, pNode->key, pNode->keySize); - taosAddToTrash(pCacheObj, pNode); -} - static FORCE_INLINE void doRemoveElemInTrashcan(SCacheObj* pCacheObj, STrashElem *pElem) { if (pElem->pData->signature != (uint64_t) pElem->pData) { uError("key:sig:0x%" PRIx64 " %p data has been released, ignore", pElem->pData->signature, pElem->pData); @@ -152,52 +142,6 @@ static FORCE_INLINE void doDestroyTrashcanElem(SCacheObj* pCacheObj, STrashElem free(pElem->pData); free(pElem); } -/** - * update data in cache - * @param pCacheObj - * @param pNode - * @param key - * @param keyLen - * @param pData - * @param dataSize - * @return - */ -static UNUSED_FUNC SCacheDataNode *taosUpdateCacheImpl(SCacheObj *pCacheObj, SCacheDataNode* pNode, SCacheDataNode* pNewNode, - const char *key, int32_t keyLen) { - - // only a node is not referenced by any other object, in-place update it - if (T_REF_VAL_GET(pNode) > 0) { - taosCacheMoveToTrash(pCacheObj, pNode); - } - - T_REF_INC(pNewNode); - - // addedTime new element to hashtable - taosHashPut(pCacheObj->pHashTable, key, keyLen, &pNewNode, sizeof(void *)); - return pNewNode; -} - -/** - * addedTime data into hash table - * @param key - * @param pData - * @param size - * @param pCacheObj - * @param keyLen - * @param pNode - * @return - */ -static FORCE_INLINE SCacheDataNode *taosAddToCacheImpl(SCacheObj *pCacheObj, const char *key, size_t keyLen, const void *pData, - size_t dataSize, uint64_t duration) { - SCacheDataNode *pNode = taosCreateCacheNode(key, keyLen, pData, dataSize, duration); - if (pNode == NULL) { - return NULL; - } - - T_REF_INC(pNode); - taosHashPut(pCacheObj->pHashTable, key, keyLen, &pNode, sizeof(void *)); - return pNode; -} /** * do cleanup the taos cache @@ -318,8 +262,8 @@ static void incRefFn(void* ptNode) { assert(ptNode != NULL); SCacheDataNode** p = (SCacheDataNode**) ptNode; - assert(T_REF_VAL_GET(*p) >= 0); + int32_t ret = T_REF_INC(*p); assert(ret > 0); } @@ -344,34 +288,6 @@ void *taosCacheAcquireByKey(SCacheObj *pCacheObj, const void *key, size_t keyLen return pData; } -//void* taosCacheUpdateExpireTimeByName(SCacheObj *pCacheObj, void *key, size_t keyLen, uint64_t expireTime) { -// if (pCacheObj == NULL || taosHashGetSize(pCacheObj->pHashTable) == 0) { -// return NULL; -// } -// -// __cache_rd_lock(pCacheObj); -// -// SCacheDataNode **ptNode = (SCacheDataNode **)taosHashGet(pCacheObj->pHashTable, key, keyLen); -// if (ptNode != NULL) { -// T_REF_INC(*ptNode); -// (*ptNode)->expireTime = expireTime; // taosGetTimestampMs() + (*ptNode)->lifespan; -// } -// -// __cache_unlock(pCacheObj); -// -// if (ptNode != NULL) { -// atomic_add_fetch_32(&pCacheObj->statistics.hitCount, 1); -// uDebug("cache:%s, key:%p, %p expireTime is updated in cache, refcnt:%d", pCacheObj->name, key, -// (*ptNode)->data, T_REF_VAL_GET(*ptNode)); -// } else { -// atomic_add_fetch_32(&pCacheObj->statistics.missCount, 1); -// uDebug("cache:%s, key:%p, not in cache, retrieved failed", pCacheObj->name, key); -// } -// -// atomic_add_fetch_32(&pCacheObj->statistics.totalAccess, 1); -// return (ptNode != NULL) ? (*ptNode)->data : NULL; -//} - void *taosCacheAcquireByData(SCacheObj *pCacheObj, void *data) { if (pCacheObj == NULL || data == NULL) return NULL; diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index e391fc5eb8..26c1062479 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -259,20 +259,20 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { //TODO handle malloc failure pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp)); memset(pRet->rsp, 0, sizeof(SRetrieveTableRsp)); - qReleaseQInfo(pVnode->qMgmt, (void**) &handle, true); + freeHandle = true; } else { // result is not ready, return immediately if (!buildRes) { qReleaseQInfo(pVnode->qMgmt, (void**) &handle, false); return TSDB_CODE_QRY_NOT_READY; } - void** dup = handle; code = vnodeDumpQueryResult(pRet, pVnode, handle, &freeHandle); + } - // not added into task queue, the query must be completed already, free qhandle immediately - if (freeHandle) { - qReleaseQInfo(pVnode->qMgmt, (void**) &dup, true); - } + // if qhandle is not added into task queue, the query must be completed already or paused with error , + // free qhandle immediately + if (freeHandle) { + qReleaseQInfo(pVnode->qMgmt, (void**) &handle, true); } return code; From 846704de87b6dab093180ce946f91407a137cd05 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 6 Aug 2020 10:36:41 +0800 Subject: [PATCH 044/109] [td-473] --- src/client/inc/tsclient.h | 9 +++--- src/client/src/tscSQLParser.c | 56 ++++++++++++++++++++--------------- 2 files changed, 37 insertions(+), 28 deletions(-) diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index e3e1d44514..4fd9549e75 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -80,8 +80,9 @@ typedef struct STableMetaInfo { * 2. keep the vgroup index for multi-vnode insertion */ int32_t vgroupIndex; - char name[TSDB_TABLE_ID_LEN]; // (super) table name - SArray* tagColList; // SArray, involved tag columns + char name[TSDB_TABLE_FNAME_LEN]; // (super) table name + char aliasName[TSDB_TABLE_NAME_LEN]; // alias name of table specified in query sql + SArray* tagColList; // SArray, involved tag columns } STableMetaInfo; /* the structure for sql function in select clause */ @@ -128,7 +129,7 @@ typedef struct SCond { } SCond; typedef struct SJoinNode { - char tableId[TSDB_TABLE_ID_LEN]; + char tableId[TSDB_TABLE_FNAME_LEN]; uint64_t uid; int16_t tagColId; } SJoinNode; @@ -162,7 +163,7 @@ typedef struct SParamInfo { } SParamInfo; typedef struct STableDataBlocks { - char tableId[TSDB_TABLE_ID_LEN]; + char tableId[TSDB_TABLE_FNAME_LEN]; int8_t tsSource; // where does the UNIX timestamp come from, server or client bool ordered; // if current rows are ordered or not int64_t vgId; // virtual group id diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 4b70e54265..e5d8269ff9 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -1087,11 +1087,11 @@ int32_t setObjFullName(char* fullName, const char* account, SSQLToken* pDB, SSQL *xlen = totalLen; } - if (totalLen < TSDB_TABLE_ID_LEN) { + if (totalLen < TSDB_TABLE_FNAME_LEN) { fullName[totalLen] = 0; } - return (totalLen < TSDB_TABLE_ID_LEN) ? TSDB_CODE_SUCCESS : TSDB_CODE_TSC_INVALID_SQL; + return (totalLen < TSDB_TABLE_FNAME_LEN) ? TSDB_CODE_SUCCESS : TSDB_CODE_TSC_INVALID_SQL; } static void extractColumnNameFromString(tSQLExprItem* pItem) { @@ -2136,13 +2136,10 @@ int32_t getTableIndexImpl(SSQLToken* pTableToken, SQueryInfo* pQueryInfo, SColum } pIndex->tableIndex = COLUMN_INDEX_INITIAL_VAL; - char tableName[TSDB_TABLE_ID_LEN] = {0}; - for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); - extractTableName(pTableMetaInfo->name, tableName); - - if (strncasecmp(tableName, pTableToken->z, pTableToken->n) == 0 && strlen(tableName) == pTableToken->n) { + char* name = pTableMetaInfo->aliasName; + if (strncasecmp(name, pTableToken->z, pTableToken->n) == 0 && strlen(name) == pTableToken->n) { pIndex->tableIndex = i; break; } @@ -3658,7 +3655,7 @@ static int32_t setTableCondForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SStringBuilder sb1; memset(&sb1, 0, sizeof(sb1)); taosStringBuilderAppendStringLen(&sb1, QUERY_COND_REL_PREFIX_IN, QUERY_COND_REL_PREFIX_IN_LEN); - char db[TSDB_TABLE_ID_LEN] = {0}; + char db[TSDB_TABLE_FNAME_LEN] = {0}; // remove the duplicated input table names int32_t num = 0; @@ -3683,7 +3680,7 @@ static int32_t setTableCondForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, taosStringBuilderAppendStringLen(&sb1, TBNAME_LIST_SEP, 1); } - char idBuf[TSDB_TABLE_ID_LEN] = {0}; + char idBuf[TSDB_TABLE_FNAME_LEN] = {0}; int32_t xlen = strlen(segments[i]); SSQLToken t = {.z = segments[i], .n = xlen, .type = TK_STRING}; @@ -5915,15 +5912,16 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { assert(pQuerySql != NULL && (pQuerySql->from == NULL || pQuerySql->from->nExpr > 0)); - const char* msg0 = "invalid table name"; - const char* msg1 = "table name too long"; - const char* msg2 = "point interpolation query needs timestamp"; - const char* msg5 = "fill only available for interval query"; - const char* msg6 = "start(end) time of query range required or time range too large"; - const char* msg7 = "illegal number of tables in from clause"; - const char* msg8 = "too many columns in selection clause"; - const char* msg9 = "TWA query requires both the start and end time"; - const char* msg10= "too many tables in from clause"; + const char* msg0 = "invalid table name"; + const char* msg1 = "table name too long"; + const char* msg2 = "point interpolation query needs timestamp"; + const char* msg5 = "fill only available for interval query"; + const char* msg6 = "start(end) time of query range required or time range too large"; + const char* msg7 = "illegal number of tables in from clause"; + const char* msg8 = "too many columns in selection clause"; + const char* msg9 = "TWA query requires both the start and end time"; + const char* msg10 = "too many tables in from clause"; + const char* msg11 = "invalid table alias name"; int32_t code = TSDB_CODE_SUCCESS; @@ -5966,7 +5964,7 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { } // set all query tables, which are maybe more than one. - for (int32_t i = 0; i < pQuerySql->from->nExpr; ++i) { + for (int32_t i = 0; i < pQuerySql->from->nExpr; ) { tVariant* pTableItem = &pQuerySql->from->a[i].pVar; if (pTableItem->nType != TSDB_DATA_TYPE_BINARY) { @@ -5980,24 +5978,34 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg0); } - if (pQueryInfo->numOfTables <= i) { // more than one table + if (pQueryInfo->numOfTables <= i/2) { // more than one table tscAddEmptyMetaInfo(pQueryInfo); } - STableMetaInfo* pMeterInfo1 = tscGetMetaInfo(pQueryInfo, i); + STableMetaInfo* pTableMetaInfo1 = tscGetMetaInfo(pQueryInfo, i); SSQLToken t = {.type = TSDB_DATA_TYPE_BINARY, .n = pTableItem->nLen, .z = pTableItem->pz}; - if (tscSetTableFullName(pMeterInfo1, &t, pSql) != TSDB_CODE_SUCCESS) { + if (tscSetTableFullName(pTableMetaInfo1, &t, pSql) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } - code = tscGetTableMeta(pSql, pMeterInfo1); + tVariant* pTableItem1 = &pQuerySql->from->a[i + 1].pVar; + SSQLToken aliasName = {.z = pTableItem1->pz, .n = pTableItem1->nLen, .type = TK_STRING}; + if (tscValidateName(&aliasName) != TSDB_CODE_SUCCESS) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg11); + } + + tstrncpy(pTableMetaInfo1->aliasName, pTableItem1->pz, sizeof(pTableMetaInfo1->aliasName)); + + code = tscGetTableMeta(pSql, pTableMetaInfo1); if (code != TSDB_CODE_SUCCESS) { return code; } + + i += 2; } - assert(pQueryInfo->numOfTables == pQuerySql->from->nExpr); + assert(pQueryInfo->numOfTables == pQuerySql->from->nExpr / 2); bool isSTable = false; if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { From cd163fc0d4aef5cee099f4c0306e6d68cb3a5259 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 6 Aug 2020 10:37:20 +0800 Subject: [PATCH 045/109] [td-473] --- src/query/inc/sql.y | 38 +++++++++++++++++++++++++++++++++++--- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/src/query/inc/sql.y b/src/query/inc/sql.y index 15cfeee6b2..2b2a967edf 100644 --- a/src/query/inc/sql.y +++ b/src/query/inc/sql.y @@ -26,7 +26,12 @@ #include #include #include +#include "qSqlparser.h" +#include "tcmdtype.h" +#include "tstoken.h" +#include "ttokendef.h" #include "tutil.h" +#include "tvariant.h" } %syntax_error { @@ -254,7 +259,7 @@ alter_db_optr(Y) ::= alter_db_optr(Z) keep(X). { Y = Z; Y.keep = X; } alter_db_optr(Y) ::= alter_db_optr(Z) blocks(X). { Y = Z; Y.numOfBlocks = strtol(X.z, NULL, 10); } alter_db_optr(Y) ::= alter_db_optr(Z) comp(X). { Y = Z; Y.compressionLevel = strtol(X.z, NULL, 10); } alter_db_optr(Y) ::= alter_db_optr(Z) wal(X). { Y = Z; Y.walLevel = strtol(X.z, NULL, 10); } -alter_db_optr(Y) ::= alter_db_optr(Z) fsync(X). { Y = Z; Y.fsyncPeriod = strtod(X.z, NULL, 10); } +alter_db_optr(Y) ::= alter_db_optr(Z) fsync(X). { Y = Z; Y.fsyncPeriod = strtol(X.z, NULL, 10); } %type typename {TAOS_FIELD} typename(A) ::= ids(X). { @@ -422,8 +427,35 @@ as(X) ::= . { X.n = 0; } from(A) ::= FROM tablelist(X). {A = X;} %type tablelist {tVariantList*} -tablelist(A) ::= ids(X) cpxName(Y). { toTSDBType(X.type); X.n += Y.n; A = tVariantListAppendToken(NULL, &X, -1);} -tablelist(A) ::= tablelist(Y) COMMA ids(X) cpxName(Z). { toTSDBType(X.type); X.n += Z.n; A = tVariantListAppendToken(Y, &X, -1); } +tablelist(A) ::= ids(X) cpxName(Y). { + toTSDBType(X.type); + X.n += Y.n; + A = tVariantListAppendToken(NULL, &X, -1); + A = tVariantListAppendToken(A, &X, -1); // table alias name +} + +tablelist(A) ::= ids(X) cpxName(Y) ids(Z). { + toTSDBType(X.type); + toTSDBType(Z.type); + X.n += Y.n; + A = tVariantListAppendToken(NULL, &X, -1); + A = tVariantListAppendToken(A, &Z, -1); +} + +tablelist(A) ::= tablelist(Y) COMMA ids(X) cpxName(Z). { + toTSDBType(X.type); + X.n += Z.n; + A = tVariantListAppendToken(Y, &X, -1); + A = tVariantListAppendToken(A, &X, -1); +} + +tablelist(A) ::= tablelist(Y) COMMA ids(X) cpxName(Z) ids(F). { + toTSDBType(X.type); + toTSDBType(F.type); + X.n += Z.n; + A = tVariantListAppendToken(Y, &X, -1); + A = tVariantListAppendToken(A, &F, -1); +} // The value of interval should be the form of "number+[a,s,m,h,d,n,y]" or "now" %type tmvar {SSQLToken} From 49f476c3a6a631cb1d1dd5fd836c09ae7b34afbb Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 6 Aug 2020 10:38:46 +0800 Subject: [PATCH 046/109] [td-473] --- src/query/src/sql.c | 783 +++++++++++++++++++++++--------------------- 1 file changed, 409 insertions(+), 374 deletions(-) diff --git a/src/query/src/sql.c b/src/query/src/sql.c index fe26f61725..75ef2f3218 100644 --- a/src/query/src/sql.c +++ b/src/query/src/sql.c @@ -126,17 +126,17 @@ typedef union { #define ParseARG_FETCH SSqlInfo* pInfo = yypParser->pInfo #define ParseARG_STORE yypParser->pInfo = pInfo #define YYFALLBACK 1 -#define YYNSTATE 243 -#define YYNRULE 226 +#define YYNSTATE 245 +#define YYNRULE 228 #define YYNTOKEN 207 -#define YY_MAX_SHIFT 242 -#define YY_MIN_SHIFTREDUCE 405 -#define YY_MAX_SHIFTREDUCE 630 -#define YY_ERROR_ACTION 631 -#define YY_ACCEPT_ACTION 632 -#define YY_NO_ACTION 633 -#define YY_MIN_REDUCE 634 -#define YY_MAX_REDUCE 859 +#define YY_MAX_SHIFT 244 +#define YY_MIN_SHIFTREDUCE 407 +#define YY_MAX_SHIFTREDUCE 634 +#define YY_ERROR_ACTION 635 +#define YY_ACCEPT_ACTION 636 +#define YY_NO_ACTION 637 +#define YY_MIN_REDUCE 638 +#define YY_MAX_REDUCE 865 /************* End control #defines *******************************************/ /* Define the yytestcase() macro to be a no-op if is not already defined @@ -202,64 +202,64 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (552) +#define YY_ACTTAB_COUNT (554) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 103, 446, 135, 673, 632, 242, 126, 515, 135, 447, - /* 10 */ 135, 158, 847, 41, 43, 11, 35, 36, 846, 157, - /* 20 */ 847, 29, 134, 446, 197, 39, 37, 40, 38, 155, - /* 30 */ 103, 447, 139, 34, 33, 217, 216, 32, 31, 30, - /* 40 */ 41, 43, 767, 35, 36, 32, 31, 30, 29, 756, - /* 50 */ 446, 197, 39, 37, 40, 38, 182, 802, 447, 192, - /* 60 */ 34, 33, 21, 21, 32, 31, 30, 406, 407, 408, - /* 70 */ 409, 410, 411, 412, 413, 414, 415, 416, 417, 241, - /* 80 */ 41, 43, 228, 35, 36, 194, 843, 58, 29, 21, - /* 90 */ 842, 197, 39, 37, 40, 38, 166, 167, 753, 753, - /* 100 */ 34, 33, 168, 56, 32, 31, 30, 778, 841, 16, - /* 110 */ 235, 208, 234, 233, 207, 206, 205, 232, 204, 231, - /* 120 */ 230, 229, 203, 215, 151, 753, 732, 586, 719, 720, - /* 130 */ 721, 722, 723, 724, 725, 726, 727, 728, 729, 730, - /* 140 */ 731, 43, 8, 35, 36, 61, 113, 21, 29, 153, - /* 150 */ 240, 197, 39, 37, 40, 38, 239, 238, 95, 775, - /* 160 */ 34, 33, 165, 99, 32, 31, 30, 169, 35, 36, - /* 170 */ 214, 213, 592, 29, 595, 103, 197, 39, 37, 40, - /* 180 */ 38, 220, 756, 753, 236, 34, 33, 175, 12, 32, - /* 190 */ 31, 30, 162, 599, 179, 178, 590, 767, 593, 103, - /* 200 */ 596, 161, 162, 599, 756, 17, 590, 148, 593, 152, - /* 210 */ 596, 154, 26, 88, 87, 142, 185, 567, 568, 16, - /* 220 */ 235, 147, 234, 233, 159, 160, 219, 232, 196, 231, - /* 230 */ 230, 229, 801, 76, 159, 160, 162, 599, 547, 228, - /* 240 */ 590, 3, 593, 17, 596, 74, 78, 83, 86, 77, - /* 250 */ 26, 39, 37, 40, 38, 80, 59, 754, 21, 34, - /* 260 */ 33, 544, 60, 32, 31, 30, 18, 140, 159, 160, - /* 270 */ 181, 737, 539, 736, 27, 734, 735, 150, 682, 184, - /* 280 */ 738, 126, 740, 741, 739, 674, 531, 141, 126, 528, - /* 290 */ 42, 529, 558, 530, 752, 591, 46, 594, 34, 33, - /* 300 */ 42, 598, 32, 31, 30, 116, 117, 68, 64, 67, - /* 310 */ 588, 598, 143, 50, 73, 72, 597, 170, 171, 130, - /* 320 */ 128, 91, 90, 89, 98, 47, 597, 144, 559, 616, - /* 330 */ 51, 26, 14, 13, 42, 145, 600, 521, 520, 201, - /* 340 */ 13, 46, 22, 22, 48, 598, 589, 10, 9, 535, - /* 350 */ 533, 536, 534, 85, 84, 146, 137, 133, 856, 138, - /* 360 */ 597, 136, 755, 812, 811, 163, 808, 807, 164, 777, - /* 370 */ 747, 218, 794, 100, 793, 769, 114, 115, 26, 684, - /* 380 */ 112, 202, 131, 183, 24, 211, 681, 212, 855, 532, - /* 390 */ 70, 854, 93, 852, 118, 702, 554, 25, 23, 132, - /* 400 */ 671, 79, 52, 186, 669, 81, 82, 667, 666, 172, - /* 410 */ 127, 664, 190, 663, 662, 661, 660, 652, 129, 658, - /* 420 */ 49, 656, 654, 766, 781, 782, 795, 104, 195, 44, - /* 430 */ 193, 191, 189, 187, 210, 105, 75, 28, 221, 199, - /* 440 */ 222, 223, 53, 225, 224, 149, 226, 62, 65, 703, - /* 450 */ 227, 237, 630, 173, 174, 629, 177, 665, 628, 119, - /* 460 */ 176, 92, 121, 125, 120, 751, 122, 123, 659, 124, - /* 470 */ 108, 106, 107, 109, 110, 111, 94, 1, 2, 621, - /* 480 */ 180, 184, 541, 55, 57, 555, 101, 156, 188, 198, - /* 490 */ 19, 63, 5, 560, 102, 4, 6, 601, 20, 15, - /* 500 */ 7, 488, 484, 200, 482, 481, 480, 477, 450, 209, - /* 510 */ 66, 45, 22, 69, 71, 517, 516, 514, 471, 54, - /* 520 */ 469, 461, 467, 463, 465, 459, 457, 487, 486, 485, - /* 530 */ 483, 479, 478, 476, 46, 448, 421, 419, 634, 633, - /* 540 */ 633, 633, 633, 633, 633, 633, 633, 633, 633, 633, - /* 550 */ 96, 97, + /* 0 */ 105, 448, 137, 677, 636, 244, 128, 517, 137, 449, + /* 10 */ 137, 160, 853, 41, 43, 11, 35, 36, 852, 159, + /* 20 */ 853, 29, 136, 448, 199, 39, 37, 40, 38, 157, + /* 30 */ 105, 449, 141, 34, 33, 219, 218, 32, 31, 30, + /* 40 */ 41, 43, 771, 35, 36, 32, 31, 30, 29, 760, + /* 50 */ 448, 199, 39, 37, 40, 38, 184, 808, 449, 194, + /* 60 */ 34, 33, 21, 21, 32, 31, 30, 408, 409, 410, + /* 70 */ 411, 412, 413, 414, 415, 416, 417, 418, 419, 243, + /* 80 */ 41, 43, 230, 35, 36, 196, 849, 60, 29, 21, + /* 90 */ 848, 199, 39, 37, 40, 38, 168, 169, 757, 757, + /* 100 */ 34, 33, 170, 56, 32, 31, 30, 782, 847, 16, + /* 110 */ 237, 210, 236, 235, 209, 208, 207, 234, 206, 233, + /* 120 */ 232, 231, 205, 217, 153, 757, 736, 590, 723, 724, + /* 130 */ 725, 726, 727, 728, 729, 730, 731, 732, 733, 734, + /* 140 */ 735, 43, 8, 35, 36, 63, 115, 21, 29, 155, + /* 150 */ 242, 199, 39, 37, 40, 38, 241, 240, 97, 779, + /* 160 */ 34, 33, 167, 101, 32, 31, 30, 171, 35, 36, + /* 170 */ 216, 215, 596, 29, 599, 105, 199, 39, 37, 40, + /* 180 */ 38, 222, 760, 757, 238, 34, 33, 177, 12, 32, + /* 190 */ 31, 30, 164, 603, 181, 180, 594, 771, 597, 105, + /* 200 */ 600, 163, 164, 603, 760, 17, 594, 150, 597, 154, + /* 210 */ 600, 156, 26, 90, 89, 144, 187, 571, 572, 16, + /* 220 */ 237, 149, 236, 235, 161, 162, 221, 234, 198, 233, + /* 230 */ 232, 231, 807, 78, 161, 162, 164, 603, 549, 230, + /* 240 */ 594, 3, 597, 17, 600, 76, 80, 85, 88, 79, + /* 250 */ 26, 39, 37, 40, 38, 82, 61, 758, 21, 34, + /* 260 */ 33, 546, 62, 32, 31, 30, 18, 142, 161, 162, + /* 270 */ 183, 741, 541, 740, 27, 738, 739, 152, 686, 186, + /* 280 */ 742, 128, 744, 745, 743, 678, 533, 143, 128, 530, + /* 290 */ 42, 531, 562, 532, 756, 595, 46, 598, 34, 33, + /* 300 */ 42, 602, 32, 31, 30, 118, 119, 70, 66, 69, + /* 310 */ 592, 602, 145, 50, 75, 74, 601, 172, 173, 132, + /* 320 */ 130, 93, 92, 91, 100, 47, 601, 146, 563, 620, + /* 330 */ 51, 26, 14, 13, 42, 147, 604, 523, 522, 203, + /* 340 */ 13, 46, 22, 22, 48, 602, 593, 10, 9, 537, + /* 350 */ 535, 538, 536, 87, 86, 148, 139, 135, 862, 140, + /* 360 */ 601, 138, 759, 818, 817, 165, 814, 813, 166, 781, + /* 370 */ 751, 220, 800, 786, 788, 773, 102, 799, 26, 116, + /* 380 */ 114, 117, 688, 185, 204, 133, 24, 213, 685, 534, + /* 390 */ 214, 861, 95, 72, 860, 858, 558, 120, 706, 25, + /* 400 */ 23, 134, 52, 188, 675, 81, 673, 83, 84, 671, + /* 410 */ 670, 174, 192, 129, 668, 667, 666, 665, 664, 656, + /* 420 */ 49, 131, 662, 660, 658, 770, 57, 58, 801, 44, + /* 430 */ 197, 195, 193, 191, 189, 28, 106, 212, 77, 223, + /* 440 */ 224, 225, 226, 201, 53, 227, 228, 229, 239, 64, + /* 450 */ 67, 634, 151, 175, 176, 633, 178, 179, 632, 669, + /* 460 */ 186, 625, 94, 96, 123, 127, 2, 122, 707, 755, + /* 470 */ 121, 124, 125, 111, 107, 108, 126, 109, 110, 112, + /* 480 */ 663, 113, 182, 1, 543, 55, 59, 559, 103, 158, + /* 490 */ 564, 5, 190, 104, 6, 65, 490, 605, 4, 19, + /* 500 */ 20, 15, 200, 7, 202, 486, 484, 483, 482, 479, + /* 510 */ 452, 211, 68, 45, 22, 71, 73, 519, 518, 516, + /* 520 */ 54, 473, 471, 463, 469, 465, 467, 461, 459, 489, + /* 530 */ 488, 487, 485, 481, 480, 478, 46, 450, 423, 421, + /* 540 */ 638, 637, 637, 637, 637, 637, 637, 637, 637, 637, + /* 550 */ 637, 637, 98, 99, }; static const YYCODETYPE yy_lookahead[] = { /* 0 */ 211, 1, 262, 215, 208, 209, 218, 5, 262, 9, @@ -299,25 +299,25 @@ static const YYCODETYPE yy_lookahead[] = { /* 340 */ 104, 104, 104, 104, 123, 110, 37, 129, 130, 5, /* 350 */ 5, 7, 7, 72, 73, 262, 262, 262, 248, 262, /* 360 */ 125, 262, 248, 243, 243, 243, 243, 243, 243, 211, - /* 370 */ 244, 243, 269, 211, 269, 246, 211, 211, 106, 211, + /* 370 */ 244, 243, 269, 211, 211, 246, 211, 269, 106, 211, /* 380 */ 250, 211, 211, 246, 211, 211, 211, 211, 211, 105, /* 390 */ 211, 211, 59, 211, 211, 211, 110, 211, 211, 211, /* 400 */ 211, 211, 120, 265, 211, 211, 211, 211, 211, 211, /* 410 */ 211, 211, 265, 211, 211, 211, 211, 211, 211, 211, - /* 420 */ 122, 211, 211, 259, 212, 212, 212, 258, 114, 119, - /* 430 */ 118, 113, 112, 111, 75, 257, 84, 124, 83, 212, - /* 440 */ 49, 80, 212, 53, 82, 212, 81, 216, 216, 226, - /* 450 */ 79, 75, 5, 136, 5, 5, 5, 212, 5, 225, - /* 460 */ 136, 213, 220, 219, 224, 246, 223, 221, 212, 222, - /* 470 */ 254, 256, 255, 253, 252, 251, 213, 217, 214, 87, - /* 480 */ 127, 107, 100, 108, 104, 100, 99, 1, 99, 101, - /* 490 */ 104, 72, 115, 100, 99, 99, 115, 100, 104, 99, - /* 500 */ 99, 9, 5, 101, 5, 5, 5, 5, 76, 15, - /* 510 */ 72, 16, 104, 130, 130, 5, 5, 100, 5, 99, - /* 520 */ 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - /* 530 */ 5, 5, 5, 5, 104, 76, 59, 58, 0, 273, - /* 540 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - /* 550 */ 21, 21, 273, 273, 273, 273, 273, 273, 273, 273, + /* 420 */ 122, 211, 211, 211, 211, 259, 212, 212, 212, 119, + /* 430 */ 114, 118, 113, 112, 111, 124, 258, 75, 84, 83, + /* 440 */ 49, 80, 82, 212, 212, 53, 81, 79, 75, 216, + /* 450 */ 216, 5, 212, 136, 5, 5, 136, 5, 5, 212, + /* 460 */ 107, 87, 213, 213, 220, 219, 214, 224, 226, 246, + /* 470 */ 225, 223, 221, 253, 257, 256, 222, 255, 254, 252, + /* 480 */ 212, 251, 127, 217, 100, 108, 104, 100, 99, 1, + /* 490 */ 100, 115, 99, 99, 115, 72, 9, 100, 99, 104, + /* 500 */ 104, 99, 101, 99, 101, 5, 5, 5, 5, 5, + /* 510 */ 76, 15, 72, 16, 104, 130, 130, 5, 5, 100, + /* 520 */ 99, 5, 5, 5, 5, 5, 5, 5, 5, 5, + /* 530 */ 5, 5, 5, 5, 5, 5, 104, 76, 59, 58, + /* 540 */ 0, 273, 273, 273, 273, 273, 273, 273, 273, 273, + /* 550 */ 273, 273, 21, 21, 273, 273, 273, 273, 273, 273, /* 560 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, /* 570 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, /* 580 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, @@ -337,83 +337,84 @@ static const YYCODETYPE yy_lookahead[] = { /* 720 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, /* 730 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, /* 740 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, - /* 750 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, + /* 750 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, + /* 760 */ 273, }; -#define YY_SHIFT_COUNT (242) +#define YY_SHIFT_COUNT (244) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (538) +#define YY_SHIFT_MAX (540) static const unsigned short int yy_shift_ofst[] = { /* 0 */ 144, 24, 134, 191, 235, 49, 49, 49, 49, 49, /* 10 */ 49, 0, 22, 235, 284, 284, 284, 106, 49, 49, - /* 20 */ 49, 49, 49, 161, 4, 4, 552, 201, 235, 235, + /* 20 */ 49, 49, 49, 161, 4, 4, 554, 201, 235, 235, /* 30 */ 235, 235, 235, 235, 235, 235, 235, 235, 235, 235, /* 40 */ 235, 235, 235, 235, 235, 284, 284, 2, 2, 2, - /* 50 */ 2, 2, 2, 43, 2, 225, 49, 49, 101, 101, - /* 60 */ 157, 49, 49, 49, 49, 49, 49, 49, 49, 49, + /* 50 */ 2, 2, 2, 43, 2, 225, 49, 49, 49, 49, + /* 60 */ 101, 101, 157, 49, 49, 49, 49, 49, 49, 49, /* 70 */ 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, /* 80 */ 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, - /* 90 */ 49, 49, 49, 49, 49, 49, 49, 49, 272, 333, - /* 100 */ 333, 286, 286, 333, 282, 298, 310, 314, 312, 318, - /* 110 */ 320, 322, 313, 272, 333, 333, 359, 359, 333, 352, - /* 120 */ 355, 391, 361, 362, 390, 365, 371, 333, 376, 333, - /* 130 */ 376, 552, 552, 27, 67, 67, 67, 127, 152, 226, - /* 140 */ 226, 226, 181, 265, 265, 265, 265, 241, 255, 39, - /* 150 */ 60, 8, 8, 96, 172, 192, 228, 229, 236, 167, - /* 160 */ 290, 309, 142, 221, 209, 237, 238, 239, 185, 218, - /* 170 */ 344, 345, 281, 447, 317, 449, 450, 324, 451, 453, - /* 180 */ 392, 353, 374, 382, 375, 380, 385, 387, 486, 389, - /* 190 */ 393, 395, 386, 377, 394, 381, 397, 396, 400, 388, - /* 200 */ 401, 402, 419, 492, 497, 499, 500, 501, 502, 432, - /* 210 */ 494, 438, 495, 383, 384, 408, 510, 511, 417, 420, - /* 220 */ 408, 513, 515, 516, 517, 518, 519, 520, 521, 522, - /* 230 */ 523, 524, 525, 526, 527, 528, 430, 459, 529, 530, - /* 240 */ 477, 479, 538, + /* 90 */ 49, 49, 49, 49, 49, 49, 49, 49, 49, 49, + /* 100 */ 272, 333, 333, 286, 286, 333, 282, 298, 310, 316, + /* 110 */ 313, 319, 321, 323, 311, 272, 333, 333, 362, 362, + /* 120 */ 333, 354, 356, 391, 361, 360, 392, 365, 368, 333, + /* 130 */ 373, 333, 373, 554, 554, 27, 67, 67, 67, 127, + /* 140 */ 152, 226, 226, 226, 181, 265, 265, 265, 265, 241, + /* 150 */ 255, 39, 60, 8, 8, 96, 172, 192, 228, 229, + /* 160 */ 236, 167, 290, 309, 142, 221, 209, 237, 238, 239, + /* 170 */ 185, 218, 344, 345, 281, 446, 317, 449, 450, 320, + /* 180 */ 452, 453, 374, 355, 353, 384, 377, 382, 387, 389, + /* 190 */ 488, 393, 390, 394, 395, 376, 396, 379, 397, 399, + /* 200 */ 402, 401, 404, 403, 423, 487, 500, 501, 502, 503, + /* 210 */ 504, 434, 496, 440, 497, 385, 386, 410, 512, 513, + /* 220 */ 419, 421, 410, 516, 517, 518, 519, 520, 521, 522, + /* 230 */ 523, 524, 525, 526, 527, 528, 529, 530, 432, 461, + /* 240 */ 531, 532, 479, 481, 540, }; -#define YY_REDUCE_COUNT (132) +#define YY_REDUCE_COUNT (134) #define YY_REDUCE_MIN (-260) -#define YY_REDUCE_MAX (264) +#define YY_REDUCE_MAX (268) static const short yy_reduce_ofst[] = { /* 0 */ -204, -101, 44, -260, -252, -211, -181, -149, -148, -122, /* 10 */ -64, -104, -61, -254, -199, -66, -44, -49, -48, -36, /* 20 */ -12, 15, 47, -212, 63, 70, 13, -247, -240, -230, /* 30 */ -176, -172, -154, -138, -53, 5, 25, 50, 65, 73, /* 40 */ 93, 94, 95, 97, 99, 110, 114, 120, 121, 122, - /* 50 */ 123, 124, 125, 126, 128, 129, 158, 162, 103, 105, - /* 60 */ 130, 165, 166, 168, 170, 171, 173, 174, 175, 176, + /* 50 */ 123, 124, 125, 126, 128, 129, 158, 162, 163, 165, + /* 60 */ 103, 108, 130, 168, 170, 171, 173, 174, 175, 176, /* 70 */ 177, 179, 180, 182, 183, 184, 186, 187, 188, 189, /* 80 */ 190, 193, 194, 195, 196, 197, 198, 199, 200, 202, - /* 90 */ 203, 204, 205, 206, 207, 208, 210, 211, 137, 212, - /* 100 */ 213, 138, 147, 214, 164, 169, 178, 215, 217, 216, - /* 110 */ 220, 222, 224, 219, 227, 230, 231, 232, 233, 223, - /* 120 */ 234, 240, 242, 243, 246, 247, 244, 245, 248, 256, - /* 130 */ 263, 260, 264, + /* 90 */ 203, 204, 205, 206, 207, 208, 210, 211, 212, 213, + /* 100 */ 137, 214, 215, 138, 147, 216, 166, 178, 217, 219, + /* 110 */ 222, 224, 220, 227, 230, 223, 231, 232, 233, 234, + /* 120 */ 240, 242, 245, 243, 244, 248, 251, 254, 246, 247, + /* 130 */ 249, 268, 250, 266, 252, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 631, 683, 672, 849, 849, 631, 631, 631, 631, 631, - /* 10 */ 631, 779, 649, 849, 631, 631, 631, 631, 631, 631, - /* 20 */ 631, 631, 631, 685, 685, 685, 774, 631, 631, 631, - /* 30 */ 631, 631, 631, 631, 631, 631, 631, 631, 631, 631, - /* 40 */ 631, 631, 631, 631, 631, 631, 631, 631, 631, 631, - /* 50 */ 631, 631, 631, 631, 631, 631, 631, 631, 798, 798, - /* 60 */ 772, 631, 631, 631, 631, 631, 631, 631, 631, 631, - /* 70 */ 631, 631, 631, 631, 631, 631, 631, 631, 631, 670, - /* 80 */ 631, 668, 631, 631, 631, 631, 631, 631, 631, 631, - /* 90 */ 631, 631, 631, 631, 631, 657, 631, 631, 631, 651, - /* 100 */ 651, 631, 631, 651, 805, 809, 803, 791, 799, 790, - /* 110 */ 786, 785, 813, 631, 651, 651, 680, 680, 651, 701, - /* 120 */ 699, 697, 689, 695, 691, 693, 687, 651, 678, 651, - /* 130 */ 678, 718, 733, 631, 814, 848, 804, 832, 831, 844, - /* 140 */ 838, 837, 631, 836, 835, 834, 833, 631, 631, 631, - /* 150 */ 631, 840, 839, 631, 631, 631, 631, 631, 631, 631, - /* 160 */ 631, 631, 816, 810, 806, 631, 631, 631, 631, 631, - /* 170 */ 631, 631, 631, 631, 631, 631, 631, 631, 631, 631, - /* 180 */ 631, 631, 771, 631, 631, 780, 631, 631, 631, 631, - /* 190 */ 631, 631, 800, 631, 792, 631, 631, 631, 631, 631, - /* 200 */ 631, 748, 631, 631, 631, 631, 631, 631, 631, 631, - /* 210 */ 631, 631, 631, 631, 631, 853, 631, 631, 631, 742, - /* 220 */ 851, 631, 631, 631, 631, 631, 631, 631, 631, 631, - /* 230 */ 631, 631, 631, 631, 631, 631, 704, 631, 655, 653, - /* 240 */ 631, 647, 631, + /* 0 */ 635, 687, 676, 855, 855, 635, 635, 635, 635, 635, + /* 10 */ 635, 783, 653, 855, 635, 635, 635, 635, 635, 635, + /* 20 */ 635, 635, 635, 689, 689, 689, 778, 635, 635, 635, + /* 30 */ 635, 635, 635, 635, 635, 635, 635, 635, 635, 635, + /* 40 */ 635, 635, 635, 635, 635, 635, 635, 635, 635, 635, + /* 50 */ 635, 635, 635, 635, 635, 635, 635, 785, 787, 635, + /* 60 */ 804, 804, 776, 635, 635, 635, 635, 635, 635, 635, + /* 70 */ 635, 635, 635, 635, 635, 635, 635, 635, 635, 635, + /* 80 */ 635, 674, 635, 672, 635, 635, 635, 635, 635, 635, + /* 90 */ 635, 635, 635, 635, 635, 635, 635, 661, 635, 635, + /* 100 */ 635, 655, 655, 635, 635, 655, 811, 815, 809, 797, + /* 110 */ 805, 796, 792, 791, 819, 635, 655, 655, 684, 684, + /* 120 */ 655, 705, 703, 701, 693, 699, 695, 697, 691, 655, + /* 130 */ 682, 655, 682, 722, 737, 635, 820, 854, 810, 838, + /* 140 */ 837, 850, 844, 843, 635, 842, 841, 840, 839, 635, + /* 150 */ 635, 635, 635, 846, 845, 635, 635, 635, 635, 635, + /* 160 */ 635, 635, 635, 635, 822, 816, 812, 635, 635, 635, + /* 170 */ 635, 635, 635, 635, 635, 635, 635, 635, 635, 635, + /* 180 */ 635, 635, 635, 635, 775, 635, 635, 784, 635, 635, + /* 190 */ 635, 635, 635, 635, 806, 635, 798, 635, 635, 635, + /* 200 */ 635, 635, 635, 752, 635, 635, 635, 635, 635, 635, + /* 210 */ 635, 635, 635, 635, 635, 635, 635, 859, 635, 635, + /* 220 */ 635, 746, 857, 635, 635, 635, 635, 635, 635, 635, + /* 230 */ 635, 635, 635, 635, 635, 635, 635, 635, 708, 635, + /* 240 */ 659, 657, 635, 651, 635, }; /********** End of lemon-generated parsing tables *****************************/ @@ -1154,84 +1155,86 @@ static const char *const yyRuleName[] = { /* 145 */ "as ::=", /* 146 */ "from ::= FROM tablelist", /* 147 */ "tablelist ::= ids cpxName", - /* 148 */ "tablelist ::= tablelist COMMA ids cpxName", - /* 149 */ "tmvar ::= VARIABLE", - /* 150 */ "interval_opt ::= INTERVAL LP tmvar RP", - /* 151 */ "interval_opt ::=", - /* 152 */ "fill_opt ::=", - /* 153 */ "fill_opt ::= FILL LP ID COMMA tagitemlist RP", - /* 154 */ "fill_opt ::= FILL LP ID RP", - /* 155 */ "sliding_opt ::= SLIDING LP tmvar RP", - /* 156 */ "sliding_opt ::=", - /* 157 */ "orderby_opt ::=", - /* 158 */ "orderby_opt ::= ORDER BY sortlist", - /* 159 */ "sortlist ::= sortlist COMMA item sortorder", - /* 160 */ "sortlist ::= item sortorder", - /* 161 */ "item ::= ids cpxName", - /* 162 */ "sortorder ::= ASC", - /* 163 */ "sortorder ::= DESC", - /* 164 */ "sortorder ::=", - /* 165 */ "groupby_opt ::=", - /* 166 */ "groupby_opt ::= GROUP BY grouplist", - /* 167 */ "grouplist ::= grouplist COMMA item", - /* 168 */ "grouplist ::= item", - /* 169 */ "having_opt ::=", - /* 170 */ "having_opt ::= HAVING expr", - /* 171 */ "limit_opt ::=", - /* 172 */ "limit_opt ::= LIMIT signed", - /* 173 */ "limit_opt ::= LIMIT signed OFFSET signed", - /* 174 */ "limit_opt ::= LIMIT signed COMMA signed", - /* 175 */ "slimit_opt ::=", - /* 176 */ "slimit_opt ::= SLIMIT signed", - /* 177 */ "slimit_opt ::= SLIMIT signed SOFFSET signed", - /* 178 */ "slimit_opt ::= SLIMIT signed COMMA signed", - /* 179 */ "where_opt ::=", - /* 180 */ "where_opt ::= WHERE expr", - /* 181 */ "expr ::= LP expr RP", - /* 182 */ "expr ::= ID", - /* 183 */ "expr ::= ID DOT ID", - /* 184 */ "expr ::= ID DOT STAR", - /* 185 */ "expr ::= INTEGER", - /* 186 */ "expr ::= MINUS INTEGER", - /* 187 */ "expr ::= PLUS INTEGER", - /* 188 */ "expr ::= FLOAT", - /* 189 */ "expr ::= MINUS FLOAT", - /* 190 */ "expr ::= PLUS FLOAT", - /* 191 */ "expr ::= STRING", - /* 192 */ "expr ::= NOW", - /* 193 */ "expr ::= VARIABLE", - /* 194 */ "expr ::= BOOL", - /* 195 */ "expr ::= ID LP exprlist RP", - /* 196 */ "expr ::= ID LP STAR RP", - /* 197 */ "expr ::= expr AND expr", - /* 198 */ "expr ::= expr OR expr", - /* 199 */ "expr ::= expr LT expr", - /* 200 */ "expr ::= expr GT expr", - /* 201 */ "expr ::= expr LE expr", - /* 202 */ "expr ::= expr GE expr", - /* 203 */ "expr ::= expr NE expr", - /* 204 */ "expr ::= expr EQ expr", - /* 205 */ "expr ::= expr PLUS expr", - /* 206 */ "expr ::= expr MINUS expr", - /* 207 */ "expr ::= expr STAR expr", - /* 208 */ "expr ::= expr SLASH expr", - /* 209 */ "expr ::= expr REM expr", - /* 210 */ "expr ::= expr LIKE expr", - /* 211 */ "expr ::= expr IN LP exprlist RP", - /* 212 */ "exprlist ::= exprlist COMMA expritem", - /* 213 */ "exprlist ::= expritem", - /* 214 */ "expritem ::= expr", - /* 215 */ "expritem ::=", - /* 216 */ "cmd ::= RESET QUERY CACHE", - /* 217 */ "cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist", - /* 218 */ "cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids", - /* 219 */ "cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist", - /* 220 */ "cmd ::= ALTER TABLE ids cpxName DROP TAG ids", - /* 221 */ "cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids", - /* 222 */ "cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem", - /* 223 */ "cmd ::= KILL CONNECTION INTEGER", - /* 224 */ "cmd ::= KILL STREAM INTEGER COLON INTEGER", - /* 225 */ "cmd ::= KILL QUERY INTEGER COLON INTEGER", + /* 148 */ "tablelist ::= ids cpxName ids", + /* 149 */ "tablelist ::= tablelist COMMA ids cpxName", + /* 150 */ "tablelist ::= tablelist COMMA ids cpxName ids", + /* 151 */ "tmvar ::= VARIABLE", + /* 152 */ "interval_opt ::= INTERVAL LP tmvar RP", + /* 153 */ "interval_opt ::=", + /* 154 */ "fill_opt ::=", + /* 155 */ "fill_opt ::= FILL LP ID COMMA tagitemlist RP", + /* 156 */ "fill_opt ::= FILL LP ID RP", + /* 157 */ "sliding_opt ::= SLIDING LP tmvar RP", + /* 158 */ "sliding_opt ::=", + /* 159 */ "orderby_opt ::=", + /* 160 */ "orderby_opt ::= ORDER BY sortlist", + /* 161 */ "sortlist ::= sortlist COMMA item sortorder", + /* 162 */ "sortlist ::= item sortorder", + /* 163 */ "item ::= ids cpxName", + /* 164 */ "sortorder ::= ASC", + /* 165 */ "sortorder ::= DESC", + /* 166 */ "sortorder ::=", + /* 167 */ "groupby_opt ::=", + /* 168 */ "groupby_opt ::= GROUP BY grouplist", + /* 169 */ "grouplist ::= grouplist COMMA item", + /* 170 */ "grouplist ::= item", + /* 171 */ "having_opt ::=", + /* 172 */ "having_opt ::= HAVING expr", + /* 173 */ "limit_opt ::=", + /* 174 */ "limit_opt ::= LIMIT signed", + /* 175 */ "limit_opt ::= LIMIT signed OFFSET signed", + /* 176 */ "limit_opt ::= LIMIT signed COMMA signed", + /* 177 */ "slimit_opt ::=", + /* 178 */ "slimit_opt ::= SLIMIT signed", + /* 179 */ "slimit_opt ::= SLIMIT signed SOFFSET signed", + /* 180 */ "slimit_opt ::= SLIMIT signed COMMA signed", + /* 181 */ "where_opt ::=", + /* 182 */ "where_opt ::= WHERE expr", + /* 183 */ "expr ::= LP expr RP", + /* 184 */ "expr ::= ID", + /* 185 */ "expr ::= ID DOT ID", + /* 186 */ "expr ::= ID DOT STAR", + /* 187 */ "expr ::= INTEGER", + /* 188 */ "expr ::= MINUS INTEGER", + /* 189 */ "expr ::= PLUS INTEGER", + /* 190 */ "expr ::= FLOAT", + /* 191 */ "expr ::= MINUS FLOAT", + /* 192 */ "expr ::= PLUS FLOAT", + /* 193 */ "expr ::= STRING", + /* 194 */ "expr ::= NOW", + /* 195 */ "expr ::= VARIABLE", + /* 196 */ "expr ::= BOOL", + /* 197 */ "expr ::= ID LP exprlist RP", + /* 198 */ "expr ::= ID LP STAR RP", + /* 199 */ "expr ::= expr AND expr", + /* 200 */ "expr ::= expr OR expr", + /* 201 */ "expr ::= expr LT expr", + /* 202 */ "expr ::= expr GT expr", + /* 203 */ "expr ::= expr LE expr", + /* 204 */ "expr ::= expr GE expr", + /* 205 */ "expr ::= expr NE expr", + /* 206 */ "expr ::= expr EQ expr", + /* 207 */ "expr ::= expr PLUS expr", + /* 208 */ "expr ::= expr MINUS expr", + /* 209 */ "expr ::= expr STAR expr", + /* 210 */ "expr ::= expr SLASH expr", + /* 211 */ "expr ::= expr REM expr", + /* 212 */ "expr ::= expr LIKE expr", + /* 213 */ "expr ::= expr IN LP exprlist RP", + /* 214 */ "exprlist ::= exprlist COMMA expritem", + /* 215 */ "exprlist ::= expritem", + /* 216 */ "expritem ::= expr", + /* 217 */ "expritem ::=", + /* 218 */ "cmd ::= RESET QUERY CACHE", + /* 219 */ "cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist", + /* 220 */ "cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids", + /* 221 */ "cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist", + /* 222 */ "cmd ::= ALTER TABLE ids cpxName DROP TAG ids", + /* 223 */ "cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids", + /* 224 */ "cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem", + /* 225 */ "cmd ::= KILL CONNECTION INTEGER", + /* 226 */ "cmd ::= KILL STREAM INTEGER COLON INTEGER", + /* 227 */ "cmd ::= KILL QUERY INTEGER COLON INTEGER", }; #endif /* NDEBUG */ @@ -1837,84 +1840,86 @@ static const struct { { 263, 0 }, /* (145) as ::= */ { 250, -2 }, /* (146) from ::= FROM tablelist */ { 264, -2 }, /* (147) tablelist ::= ids cpxName */ - { 264, -4 }, /* (148) tablelist ::= tablelist COMMA ids cpxName */ - { 265, -1 }, /* (149) tmvar ::= VARIABLE */ - { 252, -4 }, /* (150) interval_opt ::= INTERVAL LP tmvar RP */ - { 252, 0 }, /* (151) interval_opt ::= */ - { 253, 0 }, /* (152) fill_opt ::= */ - { 253, -6 }, /* (153) fill_opt ::= FILL LP ID COMMA tagitemlist RP */ - { 253, -4 }, /* (154) fill_opt ::= FILL LP ID RP */ - { 254, -4 }, /* (155) sliding_opt ::= SLIDING LP tmvar RP */ - { 254, 0 }, /* (156) sliding_opt ::= */ - { 256, 0 }, /* (157) orderby_opt ::= */ - { 256, -3 }, /* (158) orderby_opt ::= ORDER BY sortlist */ - { 266, -4 }, /* (159) sortlist ::= sortlist COMMA item sortorder */ - { 266, -2 }, /* (160) sortlist ::= item sortorder */ - { 268, -2 }, /* (161) item ::= ids cpxName */ - { 269, -1 }, /* (162) sortorder ::= ASC */ - { 269, -1 }, /* (163) sortorder ::= DESC */ - { 269, 0 }, /* (164) sortorder ::= */ - { 255, 0 }, /* (165) groupby_opt ::= */ - { 255, -3 }, /* (166) groupby_opt ::= GROUP BY grouplist */ - { 270, -3 }, /* (167) grouplist ::= grouplist COMMA item */ - { 270, -1 }, /* (168) grouplist ::= item */ - { 257, 0 }, /* (169) having_opt ::= */ - { 257, -2 }, /* (170) having_opt ::= HAVING expr */ - { 259, 0 }, /* (171) limit_opt ::= */ - { 259, -2 }, /* (172) limit_opt ::= LIMIT signed */ - { 259, -4 }, /* (173) limit_opt ::= LIMIT signed OFFSET signed */ - { 259, -4 }, /* (174) limit_opt ::= LIMIT signed COMMA signed */ - { 258, 0 }, /* (175) slimit_opt ::= */ - { 258, -2 }, /* (176) slimit_opt ::= SLIMIT signed */ - { 258, -4 }, /* (177) slimit_opt ::= SLIMIT signed SOFFSET signed */ - { 258, -4 }, /* (178) slimit_opt ::= SLIMIT signed COMMA signed */ - { 251, 0 }, /* (179) where_opt ::= */ - { 251, -2 }, /* (180) where_opt ::= WHERE expr */ - { 262, -3 }, /* (181) expr ::= LP expr RP */ - { 262, -1 }, /* (182) expr ::= ID */ - { 262, -3 }, /* (183) expr ::= ID DOT ID */ - { 262, -3 }, /* (184) expr ::= ID DOT STAR */ - { 262, -1 }, /* (185) expr ::= INTEGER */ - { 262, -2 }, /* (186) expr ::= MINUS INTEGER */ - { 262, -2 }, /* (187) expr ::= PLUS INTEGER */ - { 262, -1 }, /* (188) expr ::= FLOAT */ - { 262, -2 }, /* (189) expr ::= MINUS FLOAT */ - { 262, -2 }, /* (190) expr ::= PLUS FLOAT */ - { 262, -1 }, /* (191) expr ::= STRING */ - { 262, -1 }, /* (192) expr ::= NOW */ - { 262, -1 }, /* (193) expr ::= VARIABLE */ - { 262, -1 }, /* (194) expr ::= BOOL */ - { 262, -4 }, /* (195) expr ::= ID LP exprlist RP */ - { 262, -4 }, /* (196) expr ::= ID LP STAR RP */ - { 262, -3 }, /* (197) expr ::= expr AND expr */ - { 262, -3 }, /* (198) expr ::= expr OR expr */ - { 262, -3 }, /* (199) expr ::= expr LT expr */ - { 262, -3 }, /* (200) expr ::= expr GT expr */ - { 262, -3 }, /* (201) expr ::= expr LE expr */ - { 262, -3 }, /* (202) expr ::= expr GE expr */ - { 262, -3 }, /* (203) expr ::= expr NE expr */ - { 262, -3 }, /* (204) expr ::= expr EQ expr */ - { 262, -3 }, /* (205) expr ::= expr PLUS expr */ - { 262, -3 }, /* (206) expr ::= expr MINUS expr */ - { 262, -3 }, /* (207) expr ::= expr STAR expr */ - { 262, -3 }, /* (208) expr ::= expr SLASH expr */ - { 262, -3 }, /* (209) expr ::= expr REM expr */ - { 262, -3 }, /* (210) expr ::= expr LIKE expr */ - { 262, -5 }, /* (211) expr ::= expr IN LP exprlist RP */ - { 271, -3 }, /* (212) exprlist ::= exprlist COMMA expritem */ - { 271, -1 }, /* (213) exprlist ::= expritem */ - { 272, -1 }, /* (214) expritem ::= expr */ - { 272, 0 }, /* (215) expritem ::= */ - { 209, -3 }, /* (216) cmd ::= RESET QUERY CACHE */ - { 209, -7 }, /* (217) cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ - { 209, -7 }, /* (218) cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ - { 209, -7 }, /* (219) cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ - { 209, -7 }, /* (220) cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ - { 209, -8 }, /* (221) cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ - { 209, -9 }, /* (222) cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ - { 209, -3 }, /* (223) cmd ::= KILL CONNECTION INTEGER */ - { 209, -5 }, /* (224) cmd ::= KILL STREAM INTEGER COLON INTEGER */ - { 209, -5 }, /* (225) cmd ::= KILL QUERY INTEGER COLON INTEGER */ + { 264, -3 }, /* (148) tablelist ::= ids cpxName ids */ + { 264, -4 }, /* (149) tablelist ::= tablelist COMMA ids cpxName */ + { 264, -5 }, /* (150) tablelist ::= tablelist COMMA ids cpxName ids */ + { 265, -1 }, /* (151) tmvar ::= VARIABLE */ + { 252, -4 }, /* (152) interval_opt ::= INTERVAL LP tmvar RP */ + { 252, 0 }, /* (153) interval_opt ::= */ + { 253, 0 }, /* (154) fill_opt ::= */ + { 253, -6 }, /* (155) fill_opt ::= FILL LP ID COMMA tagitemlist RP */ + { 253, -4 }, /* (156) fill_opt ::= FILL LP ID RP */ + { 254, -4 }, /* (157) sliding_opt ::= SLIDING LP tmvar RP */ + { 254, 0 }, /* (158) sliding_opt ::= */ + { 256, 0 }, /* (159) orderby_opt ::= */ + { 256, -3 }, /* (160) orderby_opt ::= ORDER BY sortlist */ + { 266, -4 }, /* (161) sortlist ::= sortlist COMMA item sortorder */ + { 266, -2 }, /* (162) sortlist ::= item sortorder */ + { 268, -2 }, /* (163) item ::= ids cpxName */ + { 269, -1 }, /* (164) sortorder ::= ASC */ + { 269, -1 }, /* (165) sortorder ::= DESC */ + { 269, 0 }, /* (166) sortorder ::= */ + { 255, 0 }, /* (167) groupby_opt ::= */ + { 255, -3 }, /* (168) groupby_opt ::= GROUP BY grouplist */ + { 270, -3 }, /* (169) grouplist ::= grouplist COMMA item */ + { 270, -1 }, /* (170) grouplist ::= item */ + { 257, 0 }, /* (171) having_opt ::= */ + { 257, -2 }, /* (172) having_opt ::= HAVING expr */ + { 259, 0 }, /* (173) limit_opt ::= */ + { 259, -2 }, /* (174) limit_opt ::= LIMIT signed */ + { 259, -4 }, /* (175) limit_opt ::= LIMIT signed OFFSET signed */ + { 259, -4 }, /* (176) limit_opt ::= LIMIT signed COMMA signed */ + { 258, 0 }, /* (177) slimit_opt ::= */ + { 258, -2 }, /* (178) slimit_opt ::= SLIMIT signed */ + { 258, -4 }, /* (179) slimit_opt ::= SLIMIT signed SOFFSET signed */ + { 258, -4 }, /* (180) slimit_opt ::= SLIMIT signed COMMA signed */ + { 251, 0 }, /* (181) where_opt ::= */ + { 251, -2 }, /* (182) where_opt ::= WHERE expr */ + { 262, -3 }, /* (183) expr ::= LP expr RP */ + { 262, -1 }, /* (184) expr ::= ID */ + { 262, -3 }, /* (185) expr ::= ID DOT ID */ + { 262, -3 }, /* (186) expr ::= ID DOT STAR */ + { 262, -1 }, /* (187) expr ::= INTEGER */ + { 262, -2 }, /* (188) expr ::= MINUS INTEGER */ + { 262, -2 }, /* (189) expr ::= PLUS INTEGER */ + { 262, -1 }, /* (190) expr ::= FLOAT */ + { 262, -2 }, /* (191) expr ::= MINUS FLOAT */ + { 262, -2 }, /* (192) expr ::= PLUS FLOAT */ + { 262, -1 }, /* (193) expr ::= STRING */ + { 262, -1 }, /* (194) expr ::= NOW */ + { 262, -1 }, /* (195) expr ::= VARIABLE */ + { 262, -1 }, /* (196) expr ::= BOOL */ + { 262, -4 }, /* (197) expr ::= ID LP exprlist RP */ + { 262, -4 }, /* (198) expr ::= ID LP STAR RP */ + { 262, -3 }, /* (199) expr ::= expr AND expr */ + { 262, -3 }, /* (200) expr ::= expr OR expr */ + { 262, -3 }, /* (201) expr ::= expr LT expr */ + { 262, -3 }, /* (202) expr ::= expr GT expr */ + { 262, -3 }, /* (203) expr ::= expr LE expr */ + { 262, -3 }, /* (204) expr ::= expr GE expr */ + { 262, -3 }, /* (205) expr ::= expr NE expr */ + { 262, -3 }, /* (206) expr ::= expr EQ expr */ + { 262, -3 }, /* (207) expr ::= expr PLUS expr */ + { 262, -3 }, /* (208) expr ::= expr MINUS expr */ + { 262, -3 }, /* (209) expr ::= expr STAR expr */ + { 262, -3 }, /* (210) expr ::= expr SLASH expr */ + { 262, -3 }, /* (211) expr ::= expr REM expr */ + { 262, -3 }, /* (212) expr ::= expr LIKE expr */ + { 262, -5 }, /* (213) expr ::= expr IN LP exprlist RP */ + { 271, -3 }, /* (214) exprlist ::= exprlist COMMA expritem */ + { 271, -1 }, /* (215) exprlist ::= expritem */ + { 272, -1 }, /* (216) expritem ::= expr */ + { 272, 0 }, /* (217) expritem ::= */ + { 209, -3 }, /* (218) cmd ::= RESET QUERY CACHE */ + { 209, -7 }, /* (219) cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ + { 209, -7 }, /* (220) cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ + { 209, -7 }, /* (221) cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ + { 209, -7 }, /* (222) cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ + { 209, -8 }, /* (223) cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ + { 209, -9 }, /* (224) cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ + { 209, -3 }, /* (225) cmd ::= KILL CONNECTION INTEGER */ + { 209, -5 }, /* (226) cmd ::= KILL STREAM INTEGER COLON INTEGER */ + { 209, -5 }, /* (227) cmd ::= KILL QUERY INTEGER COLON INTEGER */ }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -2465,29 +2470,59 @@ static void yy_reduce( {yymsp[-1].minor.yy498 = yymsp[0].minor.yy498;} break; case 147: /* tablelist ::= ids cpxName */ -{ toTSDBType(yymsp[-1].minor.yy0.type); yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yylhsminor.yy498 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1);} +{ + toTSDBType(yymsp[-1].minor.yy0.type); + yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; + yylhsminor.yy498 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1); + yylhsminor.yy498 = tVariantListAppendToken(yylhsminor.yy498, &yymsp[-1].minor.yy0, -1); // table alias name +} yymsp[-1].minor.yy498 = yylhsminor.yy498; break; - case 148: /* tablelist ::= tablelist COMMA ids cpxName */ -{ toTSDBType(yymsp[-1].minor.yy0.type); yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yylhsminor.yy498 = tVariantListAppendToken(yymsp[-3].minor.yy498, &yymsp[-1].minor.yy0, -1); } + case 148: /* tablelist ::= ids cpxName ids */ +{ + toTSDBType(yymsp[-2].minor.yy0.type); + toTSDBType(yymsp[0].minor.yy0.type); + yymsp[-2].minor.yy0.n += yymsp[-1].minor.yy0.n; + yylhsminor.yy498 = tVariantListAppendToken(NULL, &yymsp[-2].minor.yy0, -1); + yylhsminor.yy498 = tVariantListAppendToken(yylhsminor.yy498, &yymsp[0].minor.yy0, -1); +} + yymsp[-2].minor.yy498 = yylhsminor.yy498; + break; + case 149: /* tablelist ::= tablelist COMMA ids cpxName */ +{ + toTSDBType(yymsp[-1].minor.yy0.type); + yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; + yylhsminor.yy498 = tVariantListAppendToken(yymsp[-3].minor.yy498, &yymsp[-1].minor.yy0, -1); + yylhsminor.yy498 = tVariantListAppendToken(yylhsminor.yy498, &yymsp[-1].minor.yy0, -1); +} yymsp[-3].minor.yy498 = yylhsminor.yy498; break; - case 149: /* tmvar ::= VARIABLE */ + case 150: /* tablelist ::= tablelist COMMA ids cpxName ids */ +{ + toTSDBType(yymsp[-2].minor.yy0.type); + toTSDBType(yymsp[0].minor.yy0.type); + yymsp[-2].minor.yy0.n += yymsp[-1].minor.yy0.n; + yylhsminor.yy498 = tVariantListAppendToken(yymsp[-4].minor.yy498, &yymsp[-2].minor.yy0, -1); + yylhsminor.yy498 = tVariantListAppendToken(yylhsminor.yy498, &yymsp[0].minor.yy0, -1); +} + yymsp[-4].minor.yy498 = yylhsminor.yy498; + break; + case 151: /* tmvar ::= VARIABLE */ {yylhsminor.yy0 = yymsp[0].minor.yy0;} yymsp[0].minor.yy0 = yylhsminor.yy0; break; - case 150: /* interval_opt ::= INTERVAL LP tmvar RP */ - case 155: /* sliding_opt ::= SLIDING LP tmvar RP */ yytestcase(yyruleno==155); + case 152: /* interval_opt ::= INTERVAL LP tmvar RP */ + case 157: /* sliding_opt ::= SLIDING LP tmvar RP */ yytestcase(yyruleno==157); {yymsp[-3].minor.yy0 = yymsp[-1].minor.yy0; } break; - case 151: /* interval_opt ::= */ - case 156: /* sliding_opt ::= */ yytestcase(yyruleno==156); + case 153: /* interval_opt ::= */ + case 158: /* sliding_opt ::= */ yytestcase(yyruleno==158); {yymsp[1].minor.yy0.n = 0; yymsp[1].minor.yy0.z = NULL; yymsp[1].minor.yy0.type = 0; } break; - case 152: /* fill_opt ::= */ + case 154: /* fill_opt ::= */ {yymsp[1].minor.yy498 = 0; } break; - case 153: /* fill_opt ::= FILL LP ID COMMA tagitemlist RP */ + case 155: /* fill_opt ::= FILL LP ID COMMA tagitemlist RP */ { tVariant A = {0}; toTSDBType(yymsp[-3].minor.yy0.type); @@ -2497,33 +2532,33 @@ static void yy_reduce( yymsp[-5].minor.yy498 = yymsp[-1].minor.yy498; } break; - case 154: /* fill_opt ::= FILL LP ID RP */ + case 156: /* fill_opt ::= FILL LP ID RP */ { toTSDBType(yymsp[-1].minor.yy0.type); yymsp[-3].minor.yy498 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1); } break; - case 157: /* orderby_opt ::= */ - case 165: /* groupby_opt ::= */ yytestcase(yyruleno==165); + case 159: /* orderby_opt ::= */ + case 167: /* groupby_opt ::= */ yytestcase(yyruleno==167); {yymsp[1].minor.yy498 = 0;} break; - case 158: /* orderby_opt ::= ORDER BY sortlist */ - case 166: /* groupby_opt ::= GROUP BY grouplist */ yytestcase(yyruleno==166); + case 160: /* orderby_opt ::= ORDER BY sortlist */ + case 168: /* groupby_opt ::= GROUP BY grouplist */ yytestcase(yyruleno==168); {yymsp[-2].minor.yy498 = yymsp[0].minor.yy498;} break; - case 159: /* sortlist ::= sortlist COMMA item sortorder */ + case 161: /* sortlist ::= sortlist COMMA item sortorder */ { yylhsminor.yy498 = tVariantListAppend(yymsp[-3].minor.yy498, &yymsp[-1].minor.yy134, yymsp[0].minor.yy46); } yymsp[-3].minor.yy498 = yylhsminor.yy498; break; - case 160: /* sortlist ::= item sortorder */ + case 162: /* sortlist ::= item sortorder */ { yylhsminor.yy498 = tVariantListAppend(NULL, &yymsp[-1].minor.yy134, yymsp[0].minor.yy46); } yymsp[-1].minor.yy498 = yylhsminor.yy498; break; - case 161: /* item ::= ids cpxName */ + case 163: /* item ::= ids cpxName */ { toTSDBType(yymsp[-1].minor.yy0.type); yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; @@ -2532,196 +2567,196 @@ static void yy_reduce( } yymsp[-1].minor.yy134 = yylhsminor.yy134; break; - case 162: /* sortorder ::= ASC */ + case 164: /* sortorder ::= ASC */ {yymsp[0].minor.yy46 = TSDB_ORDER_ASC; } break; - case 163: /* sortorder ::= DESC */ + case 165: /* sortorder ::= DESC */ {yymsp[0].minor.yy46 = TSDB_ORDER_DESC;} break; - case 164: /* sortorder ::= */ + case 166: /* sortorder ::= */ {yymsp[1].minor.yy46 = TSDB_ORDER_ASC;} break; - case 167: /* grouplist ::= grouplist COMMA item */ + case 169: /* grouplist ::= grouplist COMMA item */ { yylhsminor.yy498 = tVariantListAppend(yymsp[-2].minor.yy498, &yymsp[0].minor.yy134, -1); } yymsp[-2].minor.yy498 = yylhsminor.yy498; break; - case 168: /* grouplist ::= item */ + case 170: /* grouplist ::= item */ { yylhsminor.yy498 = tVariantListAppend(NULL, &yymsp[0].minor.yy134, -1); } yymsp[0].minor.yy498 = yylhsminor.yy498; break; - case 169: /* having_opt ::= */ - case 179: /* where_opt ::= */ yytestcase(yyruleno==179); - case 215: /* expritem ::= */ yytestcase(yyruleno==215); + case 171: /* having_opt ::= */ + case 181: /* where_opt ::= */ yytestcase(yyruleno==181); + case 217: /* expritem ::= */ yytestcase(yyruleno==217); {yymsp[1].minor.yy64 = 0;} break; - case 170: /* having_opt ::= HAVING expr */ - case 180: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==180); + case 172: /* having_opt ::= HAVING expr */ + case 182: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==182); {yymsp[-1].minor.yy64 = yymsp[0].minor.yy64;} break; - case 171: /* limit_opt ::= */ - case 175: /* slimit_opt ::= */ yytestcase(yyruleno==175); + case 173: /* limit_opt ::= */ + case 177: /* slimit_opt ::= */ yytestcase(yyruleno==177); {yymsp[1].minor.yy216.limit = -1; yymsp[1].minor.yy216.offset = 0;} break; - case 172: /* limit_opt ::= LIMIT signed */ - case 176: /* slimit_opt ::= SLIMIT signed */ yytestcase(yyruleno==176); + case 174: /* limit_opt ::= LIMIT signed */ + case 178: /* slimit_opt ::= SLIMIT signed */ yytestcase(yyruleno==178); {yymsp[-1].minor.yy216.limit = yymsp[0].minor.yy207; yymsp[-1].minor.yy216.offset = 0;} break; - case 173: /* limit_opt ::= LIMIT signed OFFSET signed */ - case 177: /* slimit_opt ::= SLIMIT signed SOFFSET signed */ yytestcase(yyruleno==177); + case 175: /* limit_opt ::= LIMIT signed OFFSET signed */ + case 179: /* slimit_opt ::= SLIMIT signed SOFFSET signed */ yytestcase(yyruleno==179); {yymsp[-3].minor.yy216.limit = yymsp[-2].minor.yy207; yymsp[-3].minor.yy216.offset = yymsp[0].minor.yy207;} break; - case 174: /* limit_opt ::= LIMIT signed COMMA signed */ - case 178: /* slimit_opt ::= SLIMIT signed COMMA signed */ yytestcase(yyruleno==178); + case 176: /* limit_opt ::= LIMIT signed COMMA signed */ + case 180: /* slimit_opt ::= SLIMIT signed COMMA signed */ yytestcase(yyruleno==180); {yymsp[-3].minor.yy216.limit = yymsp[0].minor.yy207; yymsp[-3].minor.yy216.offset = yymsp[-2].minor.yy207;} break; - case 181: /* expr ::= LP expr RP */ + case 183: /* expr ::= LP expr RP */ {yymsp[-2].minor.yy64 = yymsp[-1].minor.yy64; } break; - case 182: /* expr ::= ID */ + case 184: /* expr ::= ID */ {yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_ID);} yymsp[0].minor.yy64 = yylhsminor.yy64; break; - case 183: /* expr ::= ID DOT ID */ + case 185: /* expr ::= ID DOT ID */ {yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[-2].minor.yy0, TK_ID);} yymsp[-2].minor.yy64 = yylhsminor.yy64; break; - case 184: /* expr ::= ID DOT STAR */ + case 186: /* expr ::= ID DOT STAR */ {yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[-2].minor.yy0, TK_ALL);} yymsp[-2].minor.yy64 = yylhsminor.yy64; break; - case 185: /* expr ::= INTEGER */ + case 187: /* expr ::= INTEGER */ {yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_INTEGER);} yymsp[0].minor.yy64 = yylhsminor.yy64; break; - case 186: /* expr ::= MINUS INTEGER */ - case 187: /* expr ::= PLUS INTEGER */ yytestcase(yyruleno==187); + case 188: /* expr ::= MINUS INTEGER */ + case 189: /* expr ::= PLUS INTEGER */ yytestcase(yyruleno==189); {yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_INTEGER; yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[-1].minor.yy0, TK_INTEGER);} yymsp[-1].minor.yy64 = yylhsminor.yy64; break; - case 188: /* expr ::= FLOAT */ + case 190: /* expr ::= FLOAT */ {yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_FLOAT);} yymsp[0].minor.yy64 = yylhsminor.yy64; break; - case 189: /* expr ::= MINUS FLOAT */ - case 190: /* expr ::= PLUS FLOAT */ yytestcase(yyruleno==190); + case 191: /* expr ::= MINUS FLOAT */ + case 192: /* expr ::= PLUS FLOAT */ yytestcase(yyruleno==192); {yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_FLOAT; yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[-1].minor.yy0, TK_FLOAT);} yymsp[-1].minor.yy64 = yylhsminor.yy64; break; - case 191: /* expr ::= STRING */ + case 193: /* expr ::= STRING */ {yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_STRING);} yymsp[0].minor.yy64 = yylhsminor.yy64; break; - case 192: /* expr ::= NOW */ + case 194: /* expr ::= NOW */ {yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_NOW); } yymsp[0].minor.yy64 = yylhsminor.yy64; break; - case 193: /* expr ::= VARIABLE */ + case 195: /* expr ::= VARIABLE */ {yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_VARIABLE);} yymsp[0].minor.yy64 = yylhsminor.yy64; break; - case 194: /* expr ::= BOOL */ + case 196: /* expr ::= BOOL */ {yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_BOOL);} yymsp[0].minor.yy64 = yylhsminor.yy64; break; - case 195: /* expr ::= ID LP exprlist RP */ + case 197: /* expr ::= ID LP exprlist RP */ { yylhsminor.yy64 = tSQLExprCreateFunction(yymsp[-1].minor.yy290, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } yymsp[-3].minor.yy64 = yylhsminor.yy64; break; - case 196: /* expr ::= ID LP STAR RP */ + case 198: /* expr ::= ID LP STAR RP */ { yylhsminor.yy64 = tSQLExprCreateFunction(NULL, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } yymsp[-3].minor.yy64 = yylhsminor.yy64; break; - case 197: /* expr ::= expr AND expr */ + case 199: /* expr ::= expr AND expr */ {yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_AND);} yymsp[-2].minor.yy64 = yylhsminor.yy64; break; - case 198: /* expr ::= expr OR expr */ + case 200: /* expr ::= expr OR expr */ {yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_OR); } yymsp[-2].minor.yy64 = yylhsminor.yy64; break; - case 199: /* expr ::= expr LT expr */ + case 201: /* expr ::= expr LT expr */ {yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_LT);} yymsp[-2].minor.yy64 = yylhsminor.yy64; break; - case 200: /* expr ::= expr GT expr */ + case 202: /* expr ::= expr GT expr */ {yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_GT);} yymsp[-2].minor.yy64 = yylhsminor.yy64; break; - case 201: /* expr ::= expr LE expr */ + case 203: /* expr ::= expr LE expr */ {yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_LE);} yymsp[-2].minor.yy64 = yylhsminor.yy64; break; - case 202: /* expr ::= expr GE expr */ + case 204: /* expr ::= expr GE expr */ {yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_GE);} yymsp[-2].minor.yy64 = yylhsminor.yy64; break; - case 203: /* expr ::= expr NE expr */ + case 205: /* expr ::= expr NE expr */ {yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_NE);} yymsp[-2].minor.yy64 = yylhsminor.yy64; break; - case 204: /* expr ::= expr EQ expr */ + case 206: /* expr ::= expr EQ expr */ {yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_EQ);} yymsp[-2].minor.yy64 = yylhsminor.yy64; break; - case 205: /* expr ::= expr PLUS expr */ + case 207: /* expr ::= expr PLUS expr */ {yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_PLUS); } yymsp[-2].minor.yy64 = yylhsminor.yy64; break; - case 206: /* expr ::= expr MINUS expr */ + case 208: /* expr ::= expr MINUS expr */ {yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_MINUS); } yymsp[-2].minor.yy64 = yylhsminor.yy64; break; - case 207: /* expr ::= expr STAR expr */ + case 209: /* expr ::= expr STAR expr */ {yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_STAR); } yymsp[-2].minor.yy64 = yylhsminor.yy64; break; - case 208: /* expr ::= expr SLASH expr */ + case 210: /* expr ::= expr SLASH expr */ {yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_DIVIDE);} yymsp[-2].minor.yy64 = yylhsminor.yy64; break; - case 209: /* expr ::= expr REM expr */ + case 211: /* expr ::= expr REM expr */ {yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_REM); } yymsp[-2].minor.yy64 = yylhsminor.yy64; break; - case 210: /* expr ::= expr LIKE expr */ + case 212: /* expr ::= expr LIKE expr */ {yylhsminor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_LIKE); } yymsp[-2].minor.yy64 = yylhsminor.yy64; break; - case 211: /* expr ::= expr IN LP exprlist RP */ + case 213: /* expr ::= expr IN LP exprlist RP */ {yylhsminor.yy64 = tSQLExprCreate(yymsp[-4].minor.yy64, (tSQLExpr*)yymsp[-1].minor.yy290, TK_IN); } yymsp[-4].minor.yy64 = yylhsminor.yy64; break; - case 212: /* exprlist ::= exprlist COMMA expritem */ + case 214: /* exprlist ::= exprlist COMMA expritem */ {yylhsminor.yy290 = tSQLExprListAppend(yymsp[-2].minor.yy290,yymsp[0].minor.yy64,0);} yymsp[-2].minor.yy290 = yylhsminor.yy290; break; - case 213: /* exprlist ::= expritem */ + case 215: /* exprlist ::= expritem */ {yylhsminor.yy290 = tSQLExprListAppend(0,yymsp[0].minor.yy64,0);} yymsp[0].minor.yy290 = yylhsminor.yy290; break; - case 214: /* expritem ::= expr */ + case 216: /* expritem ::= expr */ {yylhsminor.yy64 = yymsp[0].minor.yy64;} yymsp[0].minor.yy64 = yylhsminor.yy64; break; - case 216: /* cmd ::= RESET QUERY CACHE */ + case 218: /* cmd ::= RESET QUERY CACHE */ { setDCLSQLElems(pInfo, TSDB_SQL_RESET_CACHE, 0);} break; - case 217: /* cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ + case 219: /* cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy523, NULL, TSDB_ALTER_TABLE_ADD_COLUMN); setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 218: /* cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ + case 220: /* cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; @@ -2732,14 +2767,14 @@ static void yy_reduce( setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 219: /* cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ + case 221: /* cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy523, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN); setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 220: /* cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ + case 222: /* cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; @@ -2750,7 +2785,7 @@ static void yy_reduce( setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 221: /* cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ + case 223: /* cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ { yymsp[-5].minor.yy0.n += yymsp[-4].minor.yy0.n; @@ -2764,7 +2799,7 @@ static void yy_reduce( setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 222: /* cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ + case 224: /* cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ { yymsp[-6].minor.yy0.n += yymsp[-5].minor.yy0.n; @@ -2776,13 +2811,13 @@ static void yy_reduce( setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 223: /* cmd ::= KILL CONNECTION INTEGER */ + case 225: /* cmd ::= KILL CONNECTION INTEGER */ {setKillSQL(pInfo, TSDB_SQL_KILL_CONNECTION, &yymsp[0].minor.yy0);} break; - case 224: /* cmd ::= KILL STREAM INTEGER COLON INTEGER */ + case 226: /* cmd ::= KILL STREAM INTEGER COLON INTEGER */ {yymsp[-2].minor.yy0.n += (yymsp[-1].minor.yy0.n + yymsp[0].minor.yy0.n); setKillSQL(pInfo, TSDB_SQL_KILL_STREAM, &yymsp[-2].minor.yy0);} break; - case 225: /* cmd ::= KILL QUERY INTEGER COLON INTEGER */ + case 227: /* cmd ::= KILL QUERY INTEGER COLON INTEGER */ {yymsp[-2].minor.yy0.n += (yymsp[-1].minor.yy0.n + yymsp[0].minor.yy0.n); setKillSQL(pInfo, TSDB_SQL_KILL_QUERY, &yymsp[-2].minor.yy0);} break; default: From 80d44b2ff0b12b2bdb65823335588796d98ef594 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 6 Aug 2020 10:39:25 +0800 Subject: [PATCH 047/109] [td-225] refactor codes --- src/client/src/tscParseInsert.c | 4 ++-- src/client/src/tscServer.c | 6 +++--- src/client/src/tscSql.c | 2 +- src/inc/taosdef.h | 2 +- src/inc/taosmsg.h | 26 +++++++++++++------------- src/mnode/src/mnodeDb.c | 2 +- src/mnode/src/mnodeShow.c | 2 +- src/mnode/src/mnodeTable.c | 22 +++++++++++----------- src/plugins/monitor/src/monitorMain.c | 2 +- 9 files changed, 34 insertions(+), 34 deletions(-) diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index c23f40b4ed..af875a28c5 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -989,7 +989,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { } int validateTableName(char *tblName, int len, SSQLToken* psTblToken) { - tstrncpy(psTblToken->z, tblName, TSDB_TABLE_ID_LEN); + tstrncpy(psTblToken->z, tblName, TSDB_TABLE_FNAME_LEN); psTblToken->n = len; psTblToken->type = TK_ID; @@ -1077,7 +1077,7 @@ int tsParseInsertSql(SSqlObj *pSql) { } pCmd->curSql = sToken.z; - char buf[TSDB_TABLE_ID_LEN]; + char buf[TSDB_TABLE_FNAME_LEN]; SSQLToken sTblToken; sTblToken.z = buf; // Check if the table name available or not diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index b1c69dfdc0..a9dee643b2 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1551,7 +1551,7 @@ int tscBuildMultiMeterMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { // fill head info SMgmtHead *pMgmt = (SMgmtHead *)(pCmd->payload + tsRpcHeadSize); - memset(pMgmt->db, 0, TSDB_TABLE_ID_LEN); // server don't need the db + memset(pMgmt->db, 0, TSDB_TABLE_FNAME_LEN); // server don't need the db SCMMultiTableInfoMsg *pInfoMsg = (SCMMultiTableInfoMsg *)(pCmd->payload + tsRpcHeadSize + sizeof(SMgmtHead)); pInfoMsg->numOfTables = htonl((int32_t)pCmd->count); @@ -1592,7 +1592,7 @@ int tscBuildMultiMeterMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { //// tagLen += strlen(pQueryInfo->tagCond.tbnameCond.cond) * TSDB_NCHAR_SIZE; //// } //// -//// int32_t joinCondLen = (TSDB_TABLE_ID_LEN + sizeof(int16_t)) * 2; +//// int32_t joinCondLen = (TSDB_TABLE_FNAME_LEN + sizeof(int16_t)) * 2; //// int32_t elemSize = sizeof(SSuperTableMetaElemMsg) * pQueryInfo->numOfTables; //// //// int32_t colSize = pQueryInfo->groupbyExpr.numOfGroupCols*sizeof(SColIndex); @@ -1954,7 +1954,7 @@ int tscProcessShowRsp(SSqlObj *pSql) { } int tscProcessConnectRsp(SSqlObj *pSql) { - char temp[TSDB_TABLE_ID_LEN * 2]; + char temp[TSDB_TABLE_FNAME_LEN * 2]; STscObj *pObj = pSql->pTscObj; SSqlRes *pRes = &pSql->res; diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 10720874e2..af8e913563 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -812,7 +812,7 @@ static int tscParseTblNameList(SSqlObj *pSql, const char *tblNameList, int32_t t } char *nextStr; - char tblName[TSDB_TABLE_ID_LEN]; + char tblName[TSDB_TABLE_FNAME_LEN]; int payloadLen = 0; char *pMsg = pCmd->payload; while (1) { diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index c3a808b765..ea445ee543 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -232,7 +232,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TSDB_NODE_NAME_LEN 64 #define TSDB_TABLE_NAME_LEN 193 // it is a null-terminated string #define TSDB_DB_NAME_LEN 33 -#define TSDB_TABLE_ID_LEN (TSDB_ACCT_LEN + TSDB_DB_NAME_LEN + TSDB_TABLE_NAME_LEN) +#define TSDB_TABLE_FNAME_LEN (TSDB_ACCT_LEN + TSDB_DB_NAME_LEN + TSDB_TABLE_NAME_LEN) #define TSDB_COL_NAME_LEN 65 #define TSDB_MAX_SAVED_SQL_LEN TSDB_MAX_COLUMNS * 64 #define TSDB_MAX_SQL_LEN TSDB_PAYLOAD_SIZE diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 57e0b46f06..0b47971353 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -246,13 +246,13 @@ typedef struct { uint64_t uid; uint64_t superTableUid; uint64_t createdTime; - char tableId[TSDB_TABLE_ID_LEN]; - char superTableId[TSDB_TABLE_ID_LEN]; + char tableId[TSDB_TABLE_FNAME_LEN]; + char superTableId[TSDB_TABLE_FNAME_LEN]; char data[]; } SMDCreateTableMsg; typedef struct { - char tableId[TSDB_TABLE_ID_LEN]; + char tableId[TSDB_TABLE_FNAME_LEN]; char db[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN]; int8_t igExists; int8_t getMeta; @@ -265,12 +265,12 @@ typedef struct { } SCMCreateTableMsg; typedef struct { - char tableId[TSDB_TABLE_ID_LEN]; + char tableId[TSDB_TABLE_FNAME_LEN]; int8_t igNotExists; } SCMDropTableMsg; typedef struct { - char tableId[TSDB_TABLE_ID_LEN]; + char tableId[TSDB_TABLE_FNAME_LEN]; char db[TSDB_ACCT_LEN + TSDB_DB_NAME_LEN]; int16_t type; /* operation type */ int16_t numOfCols; /* number of schema */ @@ -297,7 +297,7 @@ typedef struct { typedef struct { char clientVersion[TSDB_VERSION_LEN]; char msgVersion[TSDB_VERSION_LEN]; - char db[TSDB_TABLE_ID_LEN]; + char db[TSDB_TABLE_FNAME_LEN]; } SCMConnectMsg; typedef struct { @@ -347,14 +347,14 @@ typedef struct { int32_t vgId; int32_t sid; uint64_t uid; - char tableId[TSDB_TABLE_ID_LEN]; + char tableId[TSDB_TABLE_FNAME_LEN]; } SMDDropTableMsg; typedef struct { int32_t contLen; int32_t vgId; uint64_t uid; - char tableId[TSDB_TABLE_ID_LEN]; + char tableId[TSDB_TABLE_FNAME_LEN]; } SMDDropSTableMsg; typedef struct { @@ -527,7 +527,7 @@ typedef struct { } SCMCreateDbMsg, SCMAlterDbMsg; typedef struct { - char db[TSDB_TABLE_ID_LEN]; + char db[TSDB_TABLE_FNAME_LEN]; uint8_t ignoreNotExists; } SCMDropDbMsg, SCMUseDbMsg; @@ -637,7 +637,7 @@ typedef struct { } SMDCreateVnodeMsg, SMDAlterVnodeMsg; typedef struct { - char tableId[TSDB_TABLE_ID_LEN]; + char tableId[TSDB_TABLE_FNAME_LEN]; int16_t createFlag; char tags[]; } SCMTableInfoMsg; @@ -664,7 +664,7 @@ typedef struct { typedef struct STableMetaMsg { int32_t contLen; - char tableId[TSDB_TABLE_ID_LEN]; // table id + char tableId[TSDB_TABLE_FNAME_LEN]; // table id uint8_t numOfTags; uint8_t precision; uint8_t tableType; @@ -685,7 +685,7 @@ typedef struct SMultiTableMeta { typedef struct { int32_t dataLen; - char name[TSDB_TABLE_ID_LEN]; + char name[TSDB_TABLE_FNAME_LEN]; char data[TSDB_MAX_TAGS_LEN + TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * TSDB_MAX_TAGS]; } STagData; @@ -771,7 +771,7 @@ typedef struct { uint64_t uid; uint64_t stime; // stream starting time int32_t status; - char tableId[TSDB_TABLE_ID_LEN]; + char tableId[TSDB_TABLE_FNAME_LEN]; } SMDAlterStreamMsg; typedef struct { diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c index 48acc6787c..11013a8ef9 100644 --- a/src/mnode/src/mnodeDb.c +++ b/src/mnode/src/mnodeDb.c @@ -189,7 +189,7 @@ void mnodeDecDbRef(SDbObj *pDb) { } SDbObj *mnodeGetDbByTableId(char *tableId) { - char db[TSDB_TABLE_ID_LEN], *pos; + char db[TSDB_TABLE_FNAME_LEN], *pos; // tableId format should be : acct.db.table pos = strstr(tableId, TS_PATH_DELIMITER); diff --git a/src/mnode/src/mnodeShow.c b/src/mnode/src/mnodeShow.c index 995bfbe840..9983c111f6 100644 --- a/src/mnode/src/mnodeShow.c +++ b/src/mnode/src/mnodeShow.c @@ -302,7 +302,7 @@ static int32_t mnodeProcessConnectMsg(SMnodeMsg *pMsg) { SAcctObj *pAcct = pUser->pAcct; if (pConnectMsg->db[0]) { - char dbName[TSDB_TABLE_ID_LEN * 3] = {0}; + char dbName[TSDB_TABLE_FNAME_LEN * 3] = {0}; sprintf(dbName, "%x%s%s", pAcct->acctId, TS_PATH_DELIMITER, pConnectMsg->db); SDbObj *pDb = mnodeGetDb(dbName); if (pDb == NULL) { diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index e599e1df19..73dc7dbd54 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -215,7 +215,7 @@ static int32_t mnodeChildTableActionEncode(SSdbOper *pOper) { assert(pTable != NULL && pOper->rowData != NULL); int32_t len = strlen(pTable->info.tableId); - if (len >= TSDB_TABLE_ID_LEN) return TSDB_CODE_MND_INVALID_TABLE_ID; + if (len >= TSDB_TABLE_FNAME_LEN) return TSDB_CODE_MND_INVALID_TABLE_ID; memcpy(pOper->rowData, pTable->info.tableId, len); memset(pOper->rowData + len, 0, 1); @@ -246,7 +246,7 @@ static int32_t mnodeChildTableActionDecode(SSdbOper *pOper) { if (pTable == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY; int32_t len = strlen(pOper->rowData); - if (len >= TSDB_TABLE_ID_LEN) { + if (len >= TSDB_TABLE_FNAME_LEN) { free(pTable); return TSDB_CODE_MND_INVALID_TABLE_ID; } @@ -348,7 +348,7 @@ static int32_t mnodeInitChildTables() { .tableId = SDB_TABLE_CTABLE, .tableName = "ctables", .hashSessions = TSDB_DEFAULT_CTABLES_HASH_SIZE, - .maxRowSize = sizeof(SChildTableObj) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16) + TSDB_TABLE_ID_LEN + TSDB_CQ_SQL_SIZE, + .maxRowSize = sizeof(SChildTableObj) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16) + TSDB_TABLE_FNAME_LEN + TSDB_CQ_SQL_SIZE, .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, .keyType = SDB_KEY_VAR_STRING, .insertFp = mnodeChildTableActionInsert, @@ -471,7 +471,7 @@ static int32_t mnodeSuperTableActionEncode(SSdbOper *pOper) { assert(pOper->pObj != NULL && pOper->rowData != NULL); int32_t len = strlen(pStable->info.tableId); - if (len >= TSDB_TABLE_ID_LEN) len = TSDB_CODE_MND_INVALID_TABLE_ID; + if (len >= TSDB_TABLE_FNAME_LEN) len = TSDB_CODE_MND_INVALID_TABLE_ID; memcpy(pOper->rowData, pStable->info.tableId, len); memset(pOper->rowData + len, 0, 1); @@ -495,7 +495,7 @@ static int32_t mnodeSuperTableActionDecode(SSdbOper *pOper) { if (pStable == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY; int32_t len = strlen(pOper->rowData); - if (len >= TSDB_TABLE_ID_LEN){ + if (len >= TSDB_TABLE_FNAME_LEN){ free(pStable); return TSDB_CODE_MND_INVALID_TABLE_ID; } @@ -531,7 +531,7 @@ static int32_t mnodeInitSuperTables() { .tableId = SDB_TABLE_STABLE, .tableName = "stables", .hashSessions = TSDB_DEFAULT_STABLES_HASH_SIZE, - .maxRowSize = sizeof(SSuperTableObj) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16) + TSDB_TABLE_ID_LEN, + .maxRowSize = sizeof(SSuperTableObj) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16) + TSDB_TABLE_FNAME_LEN, .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, .keyType = SDB_KEY_VAR_STRING, .insertFp = mnodeSuperTableActionInsert, @@ -1456,7 +1456,7 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) { // reserve space int32_t contLen = sizeof(SCMSTableVgroupRspMsg) + 32 * sizeof(SCMVgroupInfo) + sizeof(SVgroupsInfo); for (int32_t i = 0; i < numOfTable; ++i) { - char *stableName = (char*)pInfo + sizeof(SCMSTableVgroupMsg) + (TSDB_TABLE_ID_LEN) * i; + char *stableName = (char*)pInfo + sizeof(SCMSTableVgroupMsg) + (TSDB_TABLE_FNAME_LEN) * i; SSuperTableObj *pTable = mnodeGetSuperTable(stableName); if (pTable != NULL && pTable->vgHash != NULL) { contLen += (taosHashGetSize(pTable->vgHash) * sizeof(SCMVgroupInfo) + sizeof(SVgroupsInfo)); @@ -1473,7 +1473,7 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) { char *msg = (char *)pRsp + sizeof(SCMSTableVgroupRspMsg); for (int32_t i = 0; i < numOfTable; ++i) { - char * stableName = (char *)pInfo + sizeof(SCMSTableVgroupMsg) + (TSDB_TABLE_ID_LEN)*i; + char * stableName = (char *)pInfo + sizeof(SCMSTableVgroupMsg) + (TSDB_TABLE_FNAME_LEN)*i; SSuperTableObj *pTable = mnodeGetSuperTable(stableName); if (pTable == NULL) { mError("app:%p:%p, stable:%s, not exist while get stable vgroup info", pMsg->rpcMsg.ahandle, pMsg, stableName); @@ -1820,7 +1820,7 @@ static int32_t mnodeSendDropChildTableMsg(SMnodeMsg *pMsg, bool needReturn) { return TSDB_CODE_MND_OUT_OF_MEMORY; } - tstrncpy(pDrop->tableId, pTable->info.tableId, TSDB_TABLE_ID_LEN); + tstrncpy(pDrop->tableId, pTable->info.tableId, TSDB_TABLE_FNAME_LEN); pDrop->vgId = htonl(pTable->vgId); pDrop->contLen = htonl(sizeof(SMDDropTableMsg)); pDrop->sid = htonl(pTable->sid); @@ -2071,7 +2071,7 @@ static int32_t mnodeDoGetChildTableMeta(SMnodeMsg *pMsg, STableMetaMsg *pMeta) { pMeta->sid = htonl(pTable->sid); pMeta->precision = pDb->cfg.precision; pMeta->tableType = pTable->info.type; - tstrncpy(pMeta->tableId, pTable->info.tableId, TSDB_TABLE_ID_LEN); + tstrncpy(pMeta->tableId, pTable->info.tableId, TSDB_TABLE_FNAME_LEN); if (pTable->info.type == TSDB_CHILD_TABLE) { pMeta->sversion = htons(pTable->superTable->sversion); @@ -2440,7 +2440,7 @@ static int32_t mnodeProcessMultiTableMetaMsg(SMnodeMsg *pMsg) { pMultiMeta->numOfTables = 0; for (int32_t t = 0; t < pInfo->numOfTables; ++t) { - char * tableId = (char *)(pInfo->tableIds + t * TSDB_TABLE_ID_LEN); + char * tableId = (char *)(pInfo->tableIds + t * TSDB_TABLE_FNAME_LEN); SChildTableObj *pTable = mnodeGetChildTable(tableId); if (pTable == NULL) continue; diff --git a/src/plugins/monitor/src/monitorMain.c b/src/plugins/monitor/src/monitorMain.c index b31fc368af..4dc79aef68 100644 --- a/src/plugins/monitor/src/monitorMain.c +++ b/src/plugins/monitor/src/monitorMain.c @@ -189,7 +189,7 @@ static void dnodeBuildMonitorSql(char *sql, int32_t cmd) { snprintf(sql, SQL_LENGTH, "create table if not exists %s.slowquery(ts timestamp, username " "binary(%d), created_time timestamp, time bigint, sql binary(%d))", - tsMonitorDbName, TSDB_TABLE_ID_LEN - 1, TSDB_SLOW_QUERY_SQL_LEN); + tsMonitorDbName, TSDB_TABLE_FNAME_LEN - 1, TSDB_SLOW_QUERY_SQL_LEN); } else if (cmd == MONITOR_CMD_CREATE_TB_LOG) { snprintf(sql, SQL_LENGTH, "create table if not exists %s.log(ts timestamp, level tinyint, " From 8b3ed9e50f6c5e6b94bd51fe7a3ec689cf496df6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 6 Aug 2020 11:49:18 +0800 Subject: [PATCH 048/109] [td-225] update the page size --- src/query/inc/qExtbuffer.h | 2 +- src/query/src/qExtbuffer.c | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/query/inc/qExtbuffer.h b/src/query/inc/qExtbuffer.h index 36fc0c9820..0bdcf5c45e 100644 --- a/src/query/inc/qExtbuffer.h +++ b/src/query/inc/qExtbuffer.h @@ -29,7 +29,7 @@ extern "C" { #define MAX_TMPFILE_PATH_LENGTH PATH_MAX #define INITIAL_ALLOCATION_BUFFER_SIZE 64 -#define DEFAULT_PAGE_SIZE (4096L) // 16k larger than the SHistoInfo +#define DEFAULT_PAGE_SIZE (1024L) // 16k larger than the SHistoInfo typedef enum EXT_BUFFER_FLUSH_MODEL { /* diff --git a/src/query/src/qExtbuffer.c b/src/query/src/qExtbuffer.c index 21b5361acb..f11a16810c 100644 --- a/src/query/src/qExtbuffer.c +++ b/src/query/src/qExtbuffer.c @@ -12,15 +12,14 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#include "qExtbuffer.h" #include "os.h" +#include "qExtbuffer.h" #include "queryLog.h" #include "taos.h" #include "taosdef.h" #include "taosmsg.h" #include "tsqlfunction.h" #include "tulog.h" -#include "tutil.h" #define COLMODEL_GET_VAL(data, schema, allrow, rowId, colId) \ (data + (schema)->pFields[colId].offset * (allrow) + (rowId) * (schema)->pFields[colId].field.bytes) From 9191700f1048124789bd4ca1049f291aeb7d6a02 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 6 Aug 2020 13:24:13 +0800 Subject: [PATCH 049/109] [td-805] opt perf. --- src/query/src/qExecutor.c | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 0ffb1a4cde..e9d1ffa639 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -125,6 +125,9 @@ static void finalizeQueryResult(SQueryRuntimeEnv *pRuntimeEnv); (tw)->ekey = (tw)->skey + ((_q)->intervalTime - 1); \ } while (0) +#define SET_STABLE_QUERY_OVER(_q) ((_q)->tableIndex = (_q)->tableqinfoGroupInfo.numOfTables) +#define IS_STASBLE_QUERY_OVER(_q) ((_q)->tableIndex >= (_q)->tableqinfoGroupInfo.numOfTables) + // todo move to utility static int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *group); @@ -2656,6 +2659,10 @@ int32_t mergeIntoGroupResult(SQInfo *pQInfo) { qDebug("QInfo:%p no result in group %d, continue", pQInfo, pQInfo->groupIndex - 1); } + if (pQInfo->groupIndex == numOfGroups) { + SET_STABLE_QUERY_OVER(pQInfo); + } + qDebug("QInfo:%p merge res data into group, index:%d, total group:%d, elapsed time:%" PRId64 "ms", pQInfo, pQInfo->groupIndex - 1, numOfGroups, taosGetTimestampMs() - st); @@ -2674,7 +2681,7 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { // check if all results has been sent to client int32_t numOfGroup = GET_NUM_OF_TABLEGROUP(pQInfo); if (pQInfo->numOfGroupResultPages == 0 && pQInfo->groupIndex == numOfGroup) { - pQInfo->tableIndex = pQInfo->tableqinfoGroupInfo.numOfTables; // set query completed + SET_STABLE_QUERY_OVER(pQInfo); return; } } @@ -3821,7 +3828,7 @@ static void stableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBloc } } -bool queryHasRemainResults(SQueryRuntimeEnv* pRuntimeEnv) { +bool queryHasRemainResForTableQuery(SQueryRuntimeEnv* pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; SFillInfo *pFillInfo = pRuntimeEnv->pFillInfo; @@ -3830,8 +3837,7 @@ bool queryHasRemainResults(SQueryRuntimeEnv* pRuntimeEnv) { } if (pQuery->fillType != TSDB_FILL_NONE && !isPointInterpoQuery(pQuery)) { - // There are results not returned to client yet, so filling operation applied to the remain result is required - // in the first place. + // There are results not returned to client yet, so filling applied to the remain result is required firstly. int32_t remain = taosNumOfRemainRows(pFillInfo); if (remain > 0) { return true; @@ -3885,14 +3891,14 @@ static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data data += sizeof(STableIdInfo); } - // all data returned, set query over + // Check if query is completed or not for stable query or normal table query respectively. if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { if (pQInfo->runtimeEnv.stableQuery) { - if (pQInfo->tableIndex >= pQInfo->tableqinfoGroupInfo.numOfTables) { + if (IS_STASBLE_QUERY_OVER(pQInfo)) { setQueryStatus(pQuery, QUERY_OVER); } } else { - if (!queryHasRemainResults(&pQInfo->runtimeEnv)) { + if (!queryHasRemainResForTableQuery(&pQInfo->runtimeEnv)) { setQueryStatus(pQuery, QUERY_OVER); } } @@ -3938,7 +3944,7 @@ int32_t doFillGapsInResults(SQueryRuntimeEnv* pRuntimeEnv, tFilePage **pDst, int ret = 0; } - if (!queryHasRemainResults(pRuntimeEnv)) { + if (!queryHasRemainResForTableQuery(pRuntimeEnv)) { return ret; } } @@ -4702,7 +4708,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { // the limitation of output result is reached, set the query completed if (limitResults(pRuntimeEnv)) { - pQInfo->tableIndex = pQInfo->tableqinfoGroupInfo.numOfTables; + SET_STABLE_QUERY_OVER(pQInfo); break; } @@ -5097,7 +5103,7 @@ static void tableQueryImpl(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; - if (queryHasRemainResults(pRuntimeEnv)) { + if (queryHasRemainResForTableQuery(pRuntimeEnv)) { if (pQuery->fillType != TSDB_FILL_NONE) { /* @@ -6598,7 +6604,7 @@ static void buildTagQueryResult(SQInfo* pQInfo) { *(int64_t*) pQuery->sdata[0]->data = num; count = 1; - pQInfo->tableIndex = num; //set query completed + SET_STABLE_QUERY_OVER(pQInfo); qDebug("QInfo:%p create count(tbname) query, res:%d rows:1", pQInfo, count); } else { // return only the tags|table name etc. count = 0; From f14c76d2f6f17516ff493d1c3b380a3ccb222ec7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 6 Aug 2020 15:33:39 +0800 Subject: [PATCH 050/109] [td-805] decrease the memory consumption during interval query. --- src/query/inc/qExecutor.h | 3 ++- src/query/src/qExecutor.c | 24 +++++++++++++++++++----- src/vnode/src/vnodeRead.c | 4 ++++ 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index 44d5d26f71..c7026b45c6 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -52,10 +52,10 @@ typedef struct SWindowStatus { typedef struct SWindowResult { uint16_t numOfRows; // number of rows of current time window + SWindowStatus status; // this result status: closed or opened SPosInfo pos; // Position of current result in disk-based output buffer SResultInfo* resultInfo; // For each result column, there is a resultInfo STimeWindow window; // The time window that current result covers. - SWindowStatus status; // this result status: closed or opened } SWindowResult; /** @@ -122,6 +122,7 @@ typedef struct SQueryCostInfo { uint32_t discardBlocks; uint64_t elapsedTime; uint64_t computTime; + uint64_t internalSupSize; } SQueryCostInfo; typedef struct SQuery { diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index e9d1ffa639..6f4afcd6bd 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -398,8 +398,18 @@ static SWindowResult *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWin // more than the capacity, reallocate the resources if (pWindowResInfo->size >= pWindowResInfo->capacity) { - int64_t newCap = pWindowResInfo->capacity * 1.5; + int64_t newCap = 0; + if (pWindowResInfo->capacity > 10000) { + newCap = pWindowResInfo->capacity * 1.25; + } else { + newCap = pWindowResInfo->capacity * 1.5; + } + + printf("%ld\n", newCap); + char *t = realloc(pWindowResInfo->pResult, newCap * sizeof(SWindowResult)); + pRuntimeEnv->summary.internalSupSize += (newCap - pWindowResInfo->capacity) * sizeof(SWindowResult); + if (t == NULL) { longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -2659,7 +2669,7 @@ int32_t mergeIntoGroupResult(SQInfo *pQInfo) { qDebug("QInfo:%p no result in group %d, continue", pQInfo, pQInfo->groupIndex - 1); } - if (pQInfo->groupIndex == numOfGroups) { + if (pQInfo->groupIndex == numOfGroups && pQInfo->offset == pQInfo->numOfGroupResultPages) { SET_STABLE_QUERY_OVER(pQInfo); } @@ -2705,7 +2715,6 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { memcpy(pDest + offset * bytes, pData->data + pRuntimeEnv->offset[i] * pData->num, bytes * pData->num); } -// rows += pData->num; offset += pData->num; } @@ -2796,6 +2805,11 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { int64_t startt = taosGetTimestampMs(); while (1) { + if (IS_QUERY_KILLED(pQInfo)) { + qDebug("QInfo:%p it is already killed, abort", pQInfo); + longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); + } + int32_t pos = pTree->pNode[0].index; SWindowResInfo *pWindowResInfo = &pTableList[pos]->windowResInfo; @@ -3958,6 +3972,8 @@ static void queryCostStatis(SQInfo *pQInfo) { " load data block:%d, total rows:%"PRId64 ", check rows:%"PRId64, pQInfo, pSummary->elapsedTime, pSummary->totalBlocks, pSummary->loadBlockStatis, pSummary->loadBlocks, pSummary->totalRows, pSummary->totalCheckedRows); + + qDebug("QInfo:%p :cost summary: internal size:%"PRId64, pQInfo, pSummary->internalSupSize); } static void updateOffsetVal(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBlockInfo) { @@ -6039,8 +6055,6 @@ static void freeQInfo(SQInfo *pQInfo) { } SQuery *pQuery = pQInfo->runtimeEnv.pQuery; - setQueryKilled(pQInfo); - qDebug("QInfo:%p start to free QInfo", pQInfo); for (int32_t col = 0; col < pQuery->numOfOutput; ++col) { taosTFree(pQuery->sdata[col]); diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index 26c1062479..a02413822e 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -251,6 +251,10 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { return code; } + // todo add more error check here + // register the qhandle to connect to quit query immediate if connection is broken + vnodeNotifyCurrentQhandle(pReadMsg->rpcMsg.handle, *handle, pVnode->vgId); + bool freeHandle = true; bool buildRes = false; From 4545cfaa2976811282772b761f109a853d8297d4 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 7 Aug 2020 10:58:55 +0800 Subject: [PATCH 051/109] [td-805] add log for internal structure. --- src/query/src/qExecutor.c | 2 ++ src/query/src/qUtil.c | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 6f4afcd6bd..68239c5c63 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -419,6 +419,8 @@ static SWindowResult *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWin int32_t inc = newCap - pWindowResInfo->capacity; memset(&pWindowResInfo->pResult[pWindowResInfo->capacity], 0, sizeof(SWindowResult) * inc); + pRuntimeEnv->summary.internalSupSize += (pQuery->numOfOutput * sizeof(SResultInfo) + pRuntimeEnv->interBufSize) * inc; + for (int32_t i = pWindowResInfo->capacity; i < newCap; ++i) { createQueryResultInfo(pQuery, &pWindowResInfo->pResult[i], pRuntimeEnv->stableQuery, pRuntimeEnv->interBufSize); } diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index 007a25f772..a01eb33ec7 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -46,12 +46,17 @@ int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, SQueryRuntimeEnv *pRun pWindowResInfo->size = 0; pWindowResInfo->prevSKey = TSKEY_INITIAL_VAL; + pRuntimeEnv->summary.internalSupSize += sizeof(SWindowResult) * threshold; + // use the pointer arraylist pWindowResInfo->pResult = calloc(threshold, sizeof(SWindowResult)); if (pWindowResInfo->pResult == NULL) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } + pRuntimeEnv->summary.internalSupSize += sizeof(SWindowResult) * threshold; + pRuntimeEnv->summary.internalSupSize += (pRuntimeEnv->pQuery->numOfOutput * sizeof(SResultInfo) + pRuntimeEnv->interBufSize) * pWindowResInfo->capacity; + for (int32_t i = 0; i < pWindowResInfo->capacity; ++i) { int32_t code = createQueryResultInfo(pRuntimeEnv->pQuery, &pWindowResInfo->pResult[i], pRuntimeEnv->stableQuery, pRuntimeEnv->interBufSize); if (code != TSDB_CODE_SUCCESS) { From 39dc3c422c95101817e21f3d94818d1055ffc796 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 7 Aug 2020 12:08:42 +0800 Subject: [PATCH 052/109] [td-1065] --- src/query/src/qResultbuf.c | 20 ++++++++++++-------- src/query/tests/resultBufferTest.cpp | 1 - 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/query/src/qResultbuf.c b/src/query/src/qResultbuf.c index 8235bd7b1f..777210bd11 100644 --- a/src/query/src/qResultbuf.c +++ b/src/query/src/qResultbuf.c @@ -7,6 +7,7 @@ #include "taoserror.h" #define GET_DATA_PAYLOAD(_p) ((_p)->pData + POINTER_BYTES) +#define NO_IN_MEM_AVAILABLE_PAGES(_b) (listNEles((_b)->lruList) >= (_b)->inMemPages) int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t rowSize, int32_t pagesize, int32_t inMemBufSize, const void* handle) { @@ -25,7 +26,7 @@ int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t ro pResBuf->comp = true; pResBuf->file = NULL; pResBuf->handle = handle; - pResBuf->fileSize = 0; + pResBuf->fileSize = 0; // at least more than 2 pages must be in memory assert(inMemBufSize >= pagesize * 2); @@ -186,8 +187,6 @@ static char* loadPageFromDisk(SDiskbasedResultBuf* pResultBuf, SPageInfo* pg) { return GET_DATA_PAYLOAD(pg); } -#define NO_IN_MEM_AVAILABLE_PAGES(_b) (listNEles((_b)->lruList) >= (_b)->inMemPages) - static SIDList addNewGroup(SDiskbasedResultBuf* pResultBuf, int32_t groupId) { assert(taosHashGet(pResultBuf->groupSet, (const char*) &groupId, sizeof(int32_t)) == NULL); @@ -211,11 +210,12 @@ static SPageInfo* registerPage(SDiskbasedResultBuf* pResultBuf, int32_t groupId, pResultBuf->numOfPages += 1; SPageInfo* ppi = malloc(sizeof(SPageInfo));//{ .info = PAGE_INFO_INITIALIZER, .pageId = pageId, .pn = NULL}; - ppi->info = PAGE_INFO_INITIALIZER; - ppi->pageId = pageId; - ppi->pData = NULL; - ppi->pn = NULL; - ppi->used = true; + + ppi->pageId = pageId; + ppi->pData = NULL; + ppi->info = PAGE_INFO_INITIALIZER; + ppi->used = true; + ppi->pn = NULL; return *(SPageInfo**) taosArrayPush(list, &ppi); } @@ -246,6 +246,8 @@ static char* evicOneDataPage(SDiskbasedResultBuf* pResultBuf) { // all pages are referenced by user, try to allocate new space if (pn == NULL) { int32_t prev = pResultBuf->inMemPages; + + // increase by 50% of previous mem pages pResultBuf->inMemPages = pResultBuf->inMemPages * 1.5; qWarn("%p in memory buf page not sufficient, expand from %d to %d, page size:%d", pResultBuf, prev, @@ -353,6 +355,8 @@ tFilePage* getResBufPage(SDiskbasedResultBuf* pResultBuf, int32_t id) { ((void**)((*pi)->pData))[0] = (*pi); lruListPushFront(pResultBuf->lruList, *pi); + (*pi)->used = true; + loadPageFromDisk(pResultBuf, *pi); return GET_DATA_PAYLOAD(*pi); } diff --git a/src/query/tests/resultBufferTest.cpp b/src/query/tests/resultBufferTest.cpp index e9611a3232..7b946d8589 100644 --- a/src/query/tests/resultBufferTest.cpp +++ b/src/query/tests/resultBufferTest.cpp @@ -130,7 +130,6 @@ void recyclePageTest() { ASSERT_TRUE(t4 == pBufPage4); ASSERT_TRUE(pageId == 4); releaseResBufPage(pResultBuf, t4); - releaseResBufPage(pResultBuf, t4); tFilePage* pBufPage5 = getNewDataBuf(pResultBuf, groupId, &pageId); tFilePage* t5 = getResBufPage(pResultBuf, pageId); From 68ca5f522557380ff1f0e02252c8e5d6443d15ee Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 7 Aug 2020 14:11:47 +0800 Subject: [PATCH 053/109] [td-225]fix memory leaks. --- src/query/src/qTsbuf.c | 1 + src/query/tests/CMakeLists.txt | 2 +- src/query/tests/tsBufTest.cpp | 10 ++++++++++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/query/src/qTsbuf.c b/src/query/src/qTsbuf.c index fe39fe4e4a..95581e1869 100644 --- a/src/query/src/qTsbuf.c +++ b/src/query/src/qTsbuf.c @@ -72,6 +72,7 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { // invalid file if (header.magic != TS_COMP_FILE_MAGIC) { + tsBufDestroy(pTSBuf); return NULL; } diff --git a/src/query/tests/CMakeLists.txt b/src/query/tests/CMakeLists.txt index 1856223391..bd7dcd4b89 100644 --- a/src/query/tests/CMakeLists.txt +++ b/src/query/tests/CMakeLists.txt @@ -10,6 +10,6 @@ IF (HEADER_GTEST_INCLUDE_DIR AND LIB_GTEST_STATIC_DIR) INCLUDE_DIRECTORIES(${HEADER_GTEST_INCLUDE_DIR}) AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) - ADD_EXECUTABLE(queryTest ${SOURCE_LIST}) + ADD_EXECUTABLE(queryTest ./unitTest.cpp ./tsBufTest.cpp) TARGET_LINK_LIBRARIES(queryTest taos query gtest pthread gcov) ENDIF() diff --git a/src/query/tests/tsBufTest.cpp b/src/query/tests/tsBufTest.cpp index 28b1d9cefe..f8738eec9c 100644 --- a/src/query/tests/tsBufTest.cpp +++ b/src/query/tests/tsBufTest.cpp @@ -47,6 +47,8 @@ void simpleTest() { EXPECT_EQ(pTSBuf->block.numOfElem, num); tsBufDestroy(pTSBuf); + + free(list); } // one large list of ts, the ts list need to be split into several small blocks @@ -71,6 +73,7 @@ void largeTSTest() { EXPECT_EQ(pTSBuf->block.numOfElem, num); tsBufDestroy(pTSBuf); + free(list); } void multiTagsTest() { @@ -208,6 +211,8 @@ void loadDataTest() { int64_t e = taosGetTimestampUs(); printf("end:%" PRIu64 ", elapsed:%" PRIu64 ", total obj:%d\n", e, e - s, x); + tsBufDestroy(pTSBuf); + tsBufDestroy(pNewBuf); } void randomIncTsTest() {} @@ -338,6 +343,8 @@ void TSTraverse() { } } } + + tsBufDestroy(pTSBuf); } void performanceTest() {} @@ -352,9 +359,12 @@ void invalidFileTest() { STSBuf* pNewBuf = tsBufCreateFromFile("/tmp/test", true); EXPECT_TRUE(pNewBuf == NULL); + tsBufDestroy(pNewBuf); pNewBuf = tsBufCreateFromFile("/tmp/911", true); EXPECT_TRUE(pNewBuf == NULL); + + tsBufDestroy(pNewBuf); } void mergeDiffVnodeBufferTest() { From 0ee12789ddb23eed26b7b15d622ffc81ed6956a9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 7 Aug 2020 14:12:30 +0800 Subject: [PATCH 054/109] [td-225]update test cmake --- src/query/tests/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/tests/CMakeLists.txt b/src/query/tests/CMakeLists.txt index bd7dcd4b89..1856223391 100644 --- a/src/query/tests/CMakeLists.txt +++ b/src/query/tests/CMakeLists.txt @@ -10,6 +10,6 @@ IF (HEADER_GTEST_INCLUDE_DIR AND LIB_GTEST_STATIC_DIR) INCLUDE_DIRECTORIES(${HEADER_GTEST_INCLUDE_DIR}) AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) - ADD_EXECUTABLE(queryTest ./unitTest.cpp ./tsBufTest.cpp) + ADD_EXECUTABLE(queryTest ${SOURCE_LIST}) TARGET_LINK_LIBRARIES(queryTest taos query gtest pthread gcov) ENDIF() From b2c166213b4a6abf21934583cfdd95e4528f40d4 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 7 Aug 2020 14:37:14 +0800 Subject: [PATCH 055/109] [td-225] add check after killing query. --- src/query/src/qExecutor.c | 25 ++++++++++++++++--------- src/vnode/src/vnodeRead.c | 19 ++++++++++++------- 2 files changed, 28 insertions(+), 16 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 68239c5c63..c7e0a53502 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -6361,8 +6361,8 @@ static bool doBuildResCheck(SQInfo* pQInfo) { pthread_mutex_unlock(&pQInfo->lock); // clear qhandle owner -// assert(pQInfo->owner == pthread_self()); -// pQInfo->owner = 0; + assert(pQInfo->owner == pthread_self()); + pQInfo->owner = 0; return buildRes; } @@ -6370,14 +6370,14 @@ static bool doBuildResCheck(SQInfo* pQInfo) { bool qTableQuery(qinfo_t qinfo) { SQInfo *pQInfo = (SQInfo *)qinfo; assert(pQInfo && pQInfo->signature == pQInfo); -// int64_t threadId = pthread_self(); + int64_t threadId = pthread_self(); -// int64_t curOwner = 0; -// if ((curOwner = atomic_val_compare_exchange_64(&pQInfo->owner, 0, threadId)) != 0) { -// qError("QInfo:%p qhandle is now executed by thread:%p", pQInfo, (void*) curOwner); -// pQInfo->code = TSDB_CODE_QRY_IN_EXEC; -// return false; -// } + int64_t curOwner = 0; + if ((curOwner = atomic_val_compare_exchange_64(&pQInfo->owner, 0, threadId)) != 0) { + qError("QInfo:%p qhandle is now executed by thread:%p", pQInfo, (void*) curOwner); + pQInfo->code = TSDB_CODE_QRY_IN_EXEC; + return false; + } if (IS_QUERY_KILLED(pQInfo)) { qDebug("QInfo:%p it is already killed, abort", pQInfo); @@ -6529,6 +6529,13 @@ int32_t qKillQuery(qinfo_t qinfo) { } setQueryKilled(pQInfo); + + // Wait for the query executing thread being stopped/ + // Once the query is stopped, the owner of qHandle will be cleared immediately. + while(pQInfo->owner != 0) { + taosMsleep(100); + } + return TSDB_CODE_SUCCESS; } diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index a02413822e..8ca76ef22d 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -225,8 +225,8 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { SRspRet *pRet = &pReadMsg->rspRet; SRetrieveTableMsg *pRetrieve = pCont; - pRetrieve->qhandle = htobe64(pRetrieve->qhandle); pRetrieve->free = htons(pRetrieve->free); + pRetrieve->qhandle = htobe64(pRetrieve->qhandle); vDebug("vgId:%d, QInfo:%p, retrieve msg is disposed, free:%d, conn:%p", pVnode->vgId, (void*) pRetrieve->qhandle, pRetrieve->free, pReadMsg->rpcMsg.handle); @@ -236,24 +236,29 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { void** handle = qAcquireQInfo(pVnode->qMgmt, pRetrieve->qhandle); if (handle == NULL || (*handle) != (void*) pRetrieve->qhandle) { code = TSDB_CODE_QRY_INVALID_QHANDLE; - vDebug("vgId:%d, invalid qhandle in fetch result, QInfo:%p", pVnode->vgId, (void*) pRetrieve->qhandle); + vDebug("vgId:%d, invalid qhandle in retrieving result, QInfo:%p", pVnode->vgId, (void*) pRetrieve->qhandle); vnodeBuildNoResultQueryRsp(pRet); return code; } if (pRetrieve->free == 1) { - vDebug("vgId:%d, QInfo:%p, retrieve msg received to kill query and free qhandle", pVnode->vgId, *handle); + vWarn("vgId:%d, QInfo:%p, retrieve msg received to kill query and free qhandle", pVnode->vgId, *handle); qKillQuery(*handle); qReleaseQInfo(pVnode->qMgmt, (void**) &handle, true); vnodeBuildNoResultQueryRsp(pRet); + code = TSDB_CODE_TSC_QUERY_CANCELLED; return code; } - // todo add more error check here // register the qhandle to connect to quit query immediate if connection is broken - vnodeNotifyCurrentQhandle(pReadMsg->rpcMsg.handle, *handle, pVnode->vgId); + if (vnodeNotifyCurrentQhandle(pReadMsg->rpcMsg.handle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) { + vError("vgId:%d, QInfo:%p, retrieve discarded since link is broken, %p", pVnode->vgId, *handle, pReadMsg->rpcMsg.handle); + code = TSDB_CODE_RPC_NETWORK_UNAVAIL; + qReleaseQInfo(pVnode->qMgmt, (void**) &handle, true); + return code; + } bool freeHandle = true; bool buildRes = false; @@ -273,8 +278,8 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) { code = vnodeDumpQueryResult(pRet, pVnode, handle, &freeHandle); } - // if qhandle is not added into task queue, the query must be completed already or paused with error , - // free qhandle immediately + // If qhandle is not added into vread queue, the query should be completed already or paused with error. + // Here free qhandle immediately if (freeHandle) { qReleaseQInfo(pVnode->qMgmt, (void**) &handle, true); } From 24aa1a357d1d1aab6093e49388a6add44e0456aa Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 7 Aug 2020 14:47:32 +0800 Subject: [PATCH 056/109] [td-979] --- src/client/src/tscSQLParser.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index e5d8269ff9..5519dd69c2 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -4385,7 +4385,8 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { const char* msg16 = "only support one column"; const char* msg17 = "invalid column name"; const char* msg18 = "primary timestamp column cannot be dropped"; - + const char* msg19 = "invalid new tag name"; + SSqlCmd* pCmd = &pSql->cmd; SAlterTableSQL* pAlterSQL = pInfo->pAlterInfo; SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); @@ -4486,12 +4487,12 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { SSQLToken srcToken = {.z = pSrcItem->pVar.pz, .n = pSrcItem->pVar.nLen, .type = TK_STRING}; if (getColumnIndexByName(pCmd, &srcToken, pQueryInfo, &srcIndex) != TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg17); } SSQLToken destToken = {.z = pDstItem->pVar.pz, .n = pDstItem->pVar.nLen, .type = TK_STRING}; if (getColumnIndexByName(pCmd, &destToken, pQueryInfo, &destIndex) == TSDB_CODE_SUCCESS) { - return TSDB_CODE_TSC_INVALID_SQL; + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg19); } char name[TSDB_COL_NAME_LEN] = {0}; From 364e3f52b30a79afb3e54b8eb4f89415dfd679d7 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 7 Aug 2020 06:48:28 +0000 Subject: [PATCH 057/109] minor changes --- src/plugins/http/src/httpContext.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/plugins/http/src/httpContext.c b/src/plugins/http/src/httpContext.c index e367911695..f7694ded97 100644 --- a/src/plugins/http/src/httpContext.c +++ b/src/plugins/http/src/httpContext.c @@ -167,8 +167,8 @@ bool httpInitContext(HttpContext *pContext) { memset(pParser, 0, sizeof(HttpParser)); pParser->pCur = pParser->pLast = pParser->buffer; - httpDebug("context:%p, fd:%d, ip:%s, thread:%s, accessTimes:%d, parsed:%d", - pContext, pContext->fd, pContext->ipstr, pContext->pThread->label, pContext->accessTimes, pContext->parsed); + httpDebug("context:%p, fd:%d, ip:%s, accessTimes:%d, parsed:%d", pContext, pContext->fd, pContext->ipstr, + pContext->accessTimes, pContext->parsed); return true; } From fce5d817bc7f7b3bebee002422fca0d28ed23db2 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 7 Aug 2020 15:18:54 +0800 Subject: [PATCH 058/109] TD-1057 change file format from ansi to unicode --- deps/MsvcLibX/include/debugm.h | 4 ++-- deps/MsvcLibX/include/direct.h | 4 ++-- deps/MsvcLibX/include/dirent.h | 4 ++-- deps/MsvcLibX/include/error.h | 4 ++-- deps/MsvcLibX/include/fadvise.h | 4 ++-- deps/MsvcLibX/include/fcntl.h | 4 ++-- deps/MsvcLibX/include/fnmatch.h | 4 ++-- deps/MsvcLibX/include/getopt.h | 2 +- deps/MsvcLibX/include/iconv.h | 4 ++-- deps/MsvcLibX/include/inttypes.h | 4 ++-- deps/MsvcLibX/include/libgen.h | 4 ++-- deps/MsvcLibX/include/limits.h | 4 ++-- deps/MsvcLibX/include/msvclibx.h | 4 ++-- deps/MsvcLibX/include/netdb.h | 4 ++-- deps/MsvcLibX/include/process.h | 4 ++-- deps/MsvcLibX/include/regex.h | 2 +- deps/MsvcLibX/include/reparsept.h | 4 ++-- deps/MsvcLibX/include/stdbool.h | 4 ++-- deps/MsvcLibX/include/stdint.h | 2 +- deps/MsvcLibX/include/stdio--.h | 4 ++-- deps/MsvcLibX/include/stdio.h | 4 ++-- deps/MsvcLibX/include/stdlib.h | 4 ++-- deps/MsvcLibX/include/sys/param.h | 4 ++-- deps/MsvcLibX/include/sys/stat.h | 4 ++-- deps/MsvcLibX/include/sys/time.h | 4 ++-- deps/MsvcLibX/include/sys/types.h | 4 ++-- deps/MsvcLibX/include/sys/utsname.h | 4 ++-- deps/MsvcLibX/include/system.h | 2 +- deps/MsvcLibX/include/time.h | 4 ++-- deps/MsvcLibX/include/unistd.h | 4 ++-- deps/MsvcLibX/include/utime.h | 4 ++-- deps/MsvcLibX/include/windows.h | 4 ++-- deps/MsvcLibX/include/xfreopen.h | 4 ++-- deps/MsvcLibX/src/Files.mak | 4 ++-- deps/MsvcLibX/src/GetFileAttributes.c | 4 ++-- deps/MsvcLibX/src/GetFileAttributesEx.c | 4 ++-- deps/MsvcLibX/src/GetFullPathName.c | 4 ++-- deps/MsvcLibX/src/GetLongPathName.c | 4 ++-- deps/MsvcLibX/src/NMakefile | 4 ++-- deps/MsvcLibX/src/access.c | 4 ++-- deps/MsvcLibX/src/basename.c | 4 ++-- deps/MsvcLibX/src/chdir.c | 4 ++-- deps/MsvcLibX/src/clock_gettime.c | 4 ++-- deps/MsvcLibX/src/debugv.c | 4 ++-- deps/MsvcLibX/src/dirent.c | 4 ++-- deps/MsvcLibX/src/dirname.c | 4 ++-- deps/MsvcLibX/src/err2errno.c | 4 ++-- deps/MsvcLibX/src/filetime.c | 4 ++-- deps/MsvcLibX/src/fnmatch.c | 4 ++-- deps/MsvcLibX/src/fstat.c | 4 ++-- deps/MsvcLibX/src/fstat64.c | 4 ++-- deps/MsvcLibX/src/fstat64i32.c | 4 ++-- deps/MsvcLibX/src/fullpath.c | 4 ++-- deps/MsvcLibX/src/getcwd.c | 4 ++-- deps/MsvcLibX/src/getopt.c | 2 +- deps/MsvcLibX/src/getppid.c | 4 ++-- deps/MsvcLibX/src/gettimeofday.c | 4 ++-- deps/MsvcLibX/src/iconv.c | 4 ++-- deps/MsvcLibX/src/lstat.c | 4 ++-- deps/MsvcLibX/src/lstat32.c | 4 ++-- deps/MsvcLibX/src/lstat32i64.c | 4 ++-- deps/MsvcLibX/src/lstat64.c | 4 ++-- deps/MsvcLibX/src/lstat64i32.c | 4 ++-- deps/MsvcLibX/src/main.c | 4 ++-- deps/MsvcLibX/src/mb2wpath.c | 4 ++-- deps/MsvcLibX/src/mkdir.c | 4 ++-- deps/MsvcLibX/src/mkdtemp.c | 4 ++-- deps/MsvcLibX/src/mkstemp.c | 4 ++-- deps/MsvcLibX/src/open.c | 4 ++-- deps/MsvcLibX/src/readlink.c | 4 ++-- deps/MsvcLibX/src/realpath.c | 4 ++-- deps/MsvcLibX/src/spawn.c | 4 ++-- deps/MsvcLibX/src/strerror.c | 4 ++-- deps/MsvcLibX/src/strndup.c | 4 ++-- deps/MsvcLibX/src/strptime.c | 2 +- deps/MsvcLibX/src/symlink.c | 4 ++-- deps/MsvcLibX/src/uname.c | 4 ++-- deps/MsvcLibX/src/utime.c | 4 ++-- deps/MsvcLibX/src/utimes.c | 4 ++-- deps/MsvcLibX/src/xfreopen.c | 2 +- 80 files changed, 153 insertions(+), 153 deletions(-) diff --git a/deps/MsvcLibX/include/debugm.h b/deps/MsvcLibX/include/debugm.h index b20a7035c6..5b3c38f45b 100644 --- a/deps/MsvcLibX/include/debugm.h +++ b/deps/MsvcLibX/include/debugm.h @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename: debugm.h * * * @@ -44,7 +44,7 @@ * 2016-10-04 JFL Added macros DEBUG_OFF(), DEBUG_MORE(), DEBUG_LESS(). * * Allow using DEBUG_ON()/MORE()/LESS()/OFF() in release mode. * * -* © Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/include/direct.h b/deps/MsvcLibX/include/direct.h index 7cedc1cf7d..0ffbe1e5f4 100644 --- a/deps/MsvcLibX/include/direct.h +++ b/deps/MsvcLibX/include/direct.h @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename: direct.h * * * @@ -10,7 +10,7 @@ * 2014-03-24 JFL Created this file, with content moved from unistd.h. * * 2015-11-15 JFL Visual Studio 2015 moved this file to the Windows Kit UCRT. * * -* © Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/include/dirent.h b/deps/MsvcLibX/include/dirent.h index ac702e01dd..db146a5998 100644 --- a/deps/MsvcLibX/include/dirent.h +++ b/deps/MsvcLibX/include/dirent.h @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename: dirent.h * * * @@ -24,7 +24,7 @@ * 2015-12-07 JFL Added the conditional definition of symlink constants, so * * that our code builds even in XP and older Windows SDKs. * * * -* © Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/include/error.h b/deps/MsvcLibX/include/error.h index e8edad7c67..9a0a790a9a 100644 --- a/deps/MsvcLibX/include/error.h +++ b/deps/MsvcLibX/include/error.h @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename: error.h * * * @@ -11,7 +11,7 @@ * History: * * 2012-10-21 JFL Created this file. * * * -* © Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/include/fadvise.h b/deps/MsvcLibX/include/fadvise.h index 37d4c310f0..89600ebd05 100644 --- a/deps/MsvcLibX/include/fadvise.h +++ b/deps/MsvcLibX/include/fadvise.h @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename: fadvise.h * * * @@ -10,7 +10,7 @@ * History: * * 2012-10-17 JFL Created this file. * * * -* © Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/include/fcntl.h b/deps/MsvcLibX/include/fcntl.h index 735287c0d4..0c4a97df8f 100644 --- a/deps/MsvcLibX/include/fcntl.h +++ b/deps/MsvcLibX/include/fcntl.h @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename: fcntl.h * * * @@ -9,7 +9,7 @@ * History: * * 2017-02-16 JFL Created this file. * * * -* © Copyright 2017 Hewlett Packard Enterprise Development LP * +* Copyright 2017 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/include/fnmatch.h b/deps/MsvcLibX/include/fnmatch.h index b41a2b5305..4b7d15b2e8 100644 --- a/deps/MsvcLibX/include/fnmatch.h +++ b/deps/MsvcLibX/include/fnmatch.h @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename: fnmatch.h * * * @@ -10,7 +10,7 @@ * History: * * 2012-01-17 JFL Created this file. * * * -* © Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/include/getopt.h b/deps/MsvcLibX/include/getopt.h index ad4b2f2baa..fa0d75b3ff 100644 --- a/deps/MsvcLibX/include/getopt.h +++ b/deps/MsvcLibX/include/getopt.h @@ -1,4 +1,4 @@ -/*- +/*- * Copyright (c) 2000 The NetBSD Foundation, Inc. * All rights reserved. * diff --git a/deps/MsvcLibX/include/iconv.h b/deps/MsvcLibX/include/iconv.h index 396296fb53..8ad5e2f67f 100644 --- a/deps/MsvcLibX/include/iconv.h +++ b/deps/MsvcLibX/include/iconv.h @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename iconv.h * * * @@ -10,7 +10,7 @@ * History: * * 2014-02-27 JFL Created this file. * * * -* © Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/include/inttypes.h b/deps/MsvcLibX/include/inttypes.h index 51740fade3..addc380ed0 100644 --- a/deps/MsvcLibX/include/inttypes.h +++ b/deps/MsvcLibX/include/inttypes.h @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename inttypes.h * * * @@ -10,7 +10,7 @@ * 2014-02-07 JFL Added definitions for PRIdMAX and PRIiMAX. * * 2016-01-07 JFL Restructured and improved support for MS-DOS. * * * -* © Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/include/libgen.h b/deps/MsvcLibX/include/libgen.h index f7ce9feff4..c3051a2fb2 100644 --- a/deps/MsvcLibX/include/libgen.h +++ b/deps/MsvcLibX/include/libgen.h @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename: libgen.h * * * @@ -9,7 +9,7 @@ * History: * * 2016-09-08 JFL Created this file. * * * -* © Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/include/limits.h b/deps/MsvcLibX/include/limits.h index d4a9e09d5e..b26d935ac8 100644 --- a/deps/MsvcLibX/include/limits.h +++ b/deps/MsvcLibX/include/limits.h @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename: limits.h * * * @@ -9,7 +9,7 @@ * History: * * 2014-06-30 JFL Created this file. * * * -* ?Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/include/msvclibx.h b/deps/MsvcLibX/include/msvclibx.h index 6395737f4b..31f0ddec33 100644 --- a/deps/MsvcLibX/include/msvclibx.h +++ b/deps/MsvcLibX/include/msvclibx.h @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename MsvcLibX.h * * * @@ -15,7 +15,7 @@ * 2016-09-28 JFL Can also be included by MS' Resource Compiler. * * 2017-02-05 JFL Changed the UTF-8 programs initialization method. * * * -* ?Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/include/netdb.h b/deps/MsvcLibX/include/netdb.h index 417150afc8..cc874580e1 100644 --- a/deps/MsvcLibX/include/netdb.h +++ b/deps/MsvcLibX/include/netdb.h @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename: netdb.h * * * @@ -9,7 +9,7 @@ * History: * * 2012-01-24 JFL Created this file. * * * -* © Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/include/process.h b/deps/MsvcLibX/include/process.h index 105b71669b..60eee46b2f 100644 --- a/deps/MsvcLibX/include/process.h +++ b/deps/MsvcLibX/include/process.h @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename: process.h * * * @@ -10,7 +10,7 @@ * 2014-03-27 JFL Created this file. * * 2015-11-15 JFL Visual Studio 2015 moved this file to the Windows Kit UCRT. * * -* © Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/include/regex.h b/deps/MsvcLibX/include/regex.h index 82158c7b2e..66dd78fbc3 100644 --- a/deps/MsvcLibX/include/regex.h +++ b/deps/MsvcLibX/include/regex.h @@ -1,4 +1,4 @@ -#ifndef _HSREGEX_H_ +#ifndef _HSREGEX_H_ #define _HSREGEX_H_ #ifndef _HSREGEX_H #define _HSREGEX_H /* never again */ diff --git a/deps/MsvcLibX/include/reparsept.h b/deps/MsvcLibX/include/reparsept.h index 1f2f7bea0c..04ba02f9a2 100644 --- a/deps/MsvcLibX/include/reparsept.h +++ b/deps/MsvcLibX/include/reparsept.h @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename: reparsept.h * * * @@ -9,7 +9,7 @@ * History: * * 2014-02-28 JFL Created this file. * * * -* © Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/include/stdbool.h b/deps/MsvcLibX/include/stdbool.h index 4697fd5255..85b42c6585 100644 --- a/deps/MsvcLibX/include/stdbool.h +++ b/deps/MsvcLibX/include/stdbool.h @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename: stdbool.h * * * @@ -9,7 +9,7 @@ * History: * * 2012-10-17 JFL Created this file. * * * -* © Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/include/stdint.h b/deps/MsvcLibX/include/stdint.h index db321df86a..55b75fe46f 100644 --- a/deps/MsvcLibX/include/stdint.h +++ b/deps/MsvcLibX/include/stdint.h @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename stdint.h * * * diff --git a/deps/MsvcLibX/include/stdio--.h b/deps/MsvcLibX/include/stdio--.h index 66b7e6e326..54cc883248 100644 --- a/deps/MsvcLibX/include/stdio--.h +++ b/deps/MsvcLibX/include/stdio--.h @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename: stdio--.h * * * @@ -10,7 +10,7 @@ * History: * * 2012-10-17 JFL Created this file. * * * -* © Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/include/stdio.h b/deps/MsvcLibX/include/stdio.h index e819d13189..a609149982 100644 --- a/deps/MsvcLibX/include/stdio.h +++ b/deps/MsvcLibX/include/stdio.h @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename: stdio.h * * * @@ -11,7 +11,7 @@ * 2015-11-15 JFL Visual Studio 2015 moved this file to the Windows Kit UCRT. * 2015-12-09 JFL Alias fputs to fputsU, and vfprintf to vfprintfU. * * * -* © Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/include/stdlib.h b/deps/MsvcLibX/include/stdlib.h index 110641d427..1db1e044ea 100644 --- a/deps/MsvcLibX/include/stdlib.h +++ b/deps/MsvcLibX/include/stdlib.h @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename: stdlib.h * * * @@ -9,7 +9,7 @@ * History: * * 2016-09-13 JFL Created this file. * * * -* © Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/include/sys/param.h b/deps/MsvcLibX/include/sys/param.h index ebb0430fc9..096af578cd 100644 --- a/deps/MsvcLibX/include/sys/param.h +++ b/deps/MsvcLibX/include/sys/param.h @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename sys/param.h * * * @@ -9,7 +9,7 @@ * History * * 2014-06-10 JFL Created this file. * * * -* © Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/include/sys/stat.h b/deps/MsvcLibX/include/sys/stat.h index 2cbe715148..a3b039aa01 100644 --- a/deps/MsvcLibX/include/sys/stat.h +++ b/deps/MsvcLibX/include/sys/stat.h @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename: stat.h * * * @@ -19,7 +19,7 @@ * 2015-11-15 JFL Visual Studio 2015 moved this file to the Windows Kit UCRT. * 2016-09-15 JFL Fixed a warning in Visual Studio 2015. * * * -* ?Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/include/sys/time.h b/deps/MsvcLibX/include/sys/time.h index 2dd8f4a5e4..d538f6aa76 100644 --- a/deps/MsvcLibX/include/sys/time.h +++ b/deps/MsvcLibX/include/sys/time.h @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename sys/time.h * * * @@ -21,7 +21,7 @@ * Added macros TIMEVAL_TO_TIMESPEC & TIMESPEC_TO_TIMEVAL. * * 2016-07-06 JFL Avoid error if winsocks2.h has been previously included. * * * -* ?Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/include/sys/types.h b/deps/MsvcLibX/include/sys/types.h index 5b0f413322..4057e031e1 100644 --- a/deps/MsvcLibX/include/sys/types.h +++ b/deps/MsvcLibX/include/sys/types.h @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename: sys/types.h * * * @@ -11,7 +11,7 @@ * 2014-06-06 JFL Moved mode_t & off*_t definitions here, from sys\stat.h. * * 2015-11-15 JFL Visual Studio 2015 moved this file to the Windows Kit UCRT. * * -* © Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/include/sys/utsname.h b/deps/MsvcLibX/include/sys/utsname.h index 4b2a85527f..41d97a3719 100644 --- a/deps/MsvcLibX/include/sys/utsname.h +++ b/deps/MsvcLibX/include/sys/utsname.h @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename: sys/utsname.h * * * @@ -9,7 +9,7 @@ * History: * * 2014-05-30 JFL Created this file. * * * -* © Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/include/system.h b/deps/MsvcLibX/include/system.h index 56f102c33d..59c875b114 100644 --- a/deps/MsvcLibX/include/system.h +++ b/deps/MsvcLibX/include/system.h @@ -1,2 +1,2 @@ -/* CoreUtils global system configuration definitions */ +/* CoreUtils global system configuration definitions */ diff --git a/deps/MsvcLibX/include/time.h b/deps/MsvcLibX/include/time.h index f95ec6ddfa..afe0769b52 100644 --- a/deps/MsvcLibX/include/time.h +++ b/deps/MsvcLibX/include/time.h @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename: time.h * * * @@ -10,7 +10,7 @@ * 2014-06-04 JFL Created this file. * * 2015-11-15 JFL Visual Studio 2015 moved this file to the Windows Kit UCRT. * * -* ?Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/include/unistd.h b/deps/MsvcLibX/include/unistd.h index 748de62d24..0058c50173 100644 --- a/deps/MsvcLibX/include/unistd.h +++ b/deps/MsvcLibX/include/unistd.h @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename: unistd.h * * * @@ -18,7 +18,7 @@ * 2014-06-30 JFL Moved PATH_MAX definition to limits.h. * * 2016-08-25 JFL Implemented ResolveLinksA(). * * * -* ?Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/include/utime.h b/deps/MsvcLibX/include/utime.h index e6771fa7a8..ec84687167 100644 --- a/deps/MsvcLibX/include/utime.h +++ b/deps/MsvcLibX/include/utime.h @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename: utime.h * * * @@ -11,7 +11,7 @@ * History: * * 2014-12-13 JFL Created this file. * * * -* © Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/include/windows.h b/deps/MsvcLibX/include/windows.h index 03404d7653..564e1ea2fc 100644 --- a/deps/MsvcLibX/include/windows.h +++ b/deps/MsvcLibX/include/windows.h @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename: windows.h * * * @@ -9,7 +9,7 @@ * History: * * 2016-09-12 JFL Created this file. * * * -* © Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/include/xfreopen.h b/deps/MsvcLibX/include/xfreopen.h index 9ec6e493e9..89fb542564 100644 --- a/deps/MsvcLibX/include/xfreopen.h +++ b/deps/MsvcLibX/include/xfreopen.h @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename: error.h * * * @@ -9,7 +9,7 @@ * History: * * 2014-02-10 JFL Created this file. * * * -* © Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/src/Files.mak b/deps/MsvcLibX/src/Files.mak index 0b9ad8ee8c..b4868a2b5b 100644 --- a/deps/MsvcLibX/src/Files.mak +++ b/deps/MsvcLibX/src/Files.mak @@ -1,4 +1,4 @@ -############################################################################### +############################################################################### # # # File name Files.mak # # # @@ -31,7 +31,7 @@ # 2016-10-11 JFL moved debugm.h to SysToolsLib global C include dir. # # 2017-02-16 JFL Added open.obj. # # # -# © Copyright 2016 Hewlett Packard Enterprise Development LP # +# Copyright 2016 Hewlett Packard Enterprise Development LP # # Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 # ############################################################################### diff --git a/deps/MsvcLibX/src/GetFileAttributes.c b/deps/MsvcLibX/src/GetFileAttributes.c index 8bf3a53e86..4a74ff2757 100644 --- a/deps/MsvcLibX/src/GetFileAttributes.c +++ b/deps/MsvcLibX/src/GetFileAttributes.c @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename GetFileAttributes.c * * * @@ -9,7 +9,7 @@ * History: * * 2016-09-12 JFL Created this file. * * * -* ?Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/src/GetFileAttributesEx.c b/deps/MsvcLibX/src/GetFileAttributesEx.c index c24664e8fa..9f2b799f4a 100644 --- a/deps/MsvcLibX/src/GetFileAttributesEx.c +++ b/deps/MsvcLibX/src/GetFileAttributesEx.c @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename GetFileAttributesEx.c * * * @@ -9,7 +9,7 @@ * History: * * 2016-09-12 JFL Created this file. * * * -* © Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/src/GetFullPathName.c b/deps/MsvcLibX/src/GetFullPathName.c index 2f87f36fdf..560285269b 100644 --- a/deps/MsvcLibX/src/GetFullPathName.c +++ b/deps/MsvcLibX/src/GetFullPathName.c @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename GetFullPathName.c * * * @@ -9,7 +9,7 @@ * History: * * 2016-09-12 JFL Created this file, from the routine in truename.c. * * * -* © Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/src/GetLongPathName.c b/deps/MsvcLibX/src/GetLongPathName.c index 58a10b616e..ed4ad8162b 100644 --- a/deps/MsvcLibX/src/GetLongPathName.c +++ b/deps/MsvcLibX/src/GetLongPathName.c @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename GetLongPathName.c * * * @@ -9,7 +9,7 @@ * History: * * 2016-09-12 JFL Created this file, from the routine in truename.c. * * * -* © Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/src/NMakefile b/deps/MsvcLibX/src/NMakefile index 398576c915..8be23a7c28 100644 --- a/deps/MsvcLibX/src/NMakefile +++ b/deps/MsvcLibX/src/NMakefile @@ -1,4 +1,4 @@ -############################################################################### +############################################################################### # # # File name NMakefile # # # @@ -30,7 +30,7 @@ # 2017-02-16 JFL Default goals now depend on the existence of their # # corresponding make file. # # # -# © Copyright 2016 Hewlett Packard Enterprise Development LP # +# Copyright 2016 Hewlett Packard Enterprise Development LP # # Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 # ############################################################################### diff --git a/deps/MsvcLibX/src/access.c b/deps/MsvcLibX/src/access.c index e221c892cb..3f8f1ec117 100644 --- a/deps/MsvcLibX/src/access.c +++ b/deps/MsvcLibX/src/access.c @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename access.c * * * @@ -10,7 +10,7 @@ * 2014-03-24 JFL Created this module. * * 2014-07-02 JFL Added support for pathnames >= 260 characters. * * * -* © Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/src/basename.c b/deps/MsvcLibX/src/basename.c index f89aa22e1f..bf2af0efbe 100644 --- a/deps/MsvcLibX/src/basename.c +++ b/deps/MsvcLibX/src/basename.c @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename basename.c * * * @@ -12,7 +12,7 @@ * History * * 2016-09-08 JFL Created this file. * * * -* © Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/src/chdir.c b/deps/MsvcLibX/src/chdir.c index 48ab62fe06..e438cf937e 100644 --- a/deps/MsvcLibX/src/chdir.c +++ b/deps/MsvcLibX/src/chdir.c @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename chdir.c * * * @@ -10,7 +10,7 @@ * 2014-02-28 JFL Created this module. * * 2014-07-02 JFL Added support for pathnames >= 260 characters. * * * -* © Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/src/clock_gettime.c b/deps/MsvcLibX/src/clock_gettime.c index 02202e5d2e..8a5bb8f3fb 100644 --- a/deps/MsvcLibX/src/clock_gettime.c +++ b/deps/MsvcLibX/src/clock_gettime.c @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename: clock_gettime.c * * * @@ -9,7 +9,7 @@ * History: * * 2014-06-04 JFL Created this file. * * * -* ?Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/src/debugv.c b/deps/MsvcLibX/src/debugv.c index a0a757032c..fb2d6fd101 100644 --- a/deps/MsvcLibX/src/debugv.c +++ b/deps/MsvcLibX/src/debugv.c @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename: debugv.c * * * @@ -9,7 +9,7 @@ * History: * * 2013-03-27 JFL jf.larvoire@hp.com created this file. * * * -* © Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/src/dirent.c b/deps/MsvcLibX/src/dirent.c index 5fbf3be6ca..fa664a7353 100644 --- a/deps/MsvcLibX/src/dirent.c +++ b/deps/MsvcLibX/src/dirent.c @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename: dirent.c * * * @@ -30,7 +30,7 @@ * 2015-12-14 JFL Bug fix: WIN32 readdirW always read the root on "D:". * * Bug fix: DOS opendir failed on root dirs, like "D:\". * * * -* © Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/src/dirname.c b/deps/MsvcLibX/src/dirname.c index 97fffcfe8c..df4e4d1ebc 100644 --- a/deps/MsvcLibX/src/dirname.c +++ b/deps/MsvcLibX/src/dirname.c @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename dirname.c * * * @@ -12,7 +12,7 @@ * History * * 2016-09-08 JFL Created this file. * * * -* © Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/src/err2errno.c b/deps/MsvcLibX/src/err2errno.c index ec4260df0b..87f9c6c688 100644 --- a/deps/MsvcLibX/src/err2errno.c +++ b/deps/MsvcLibX/src/err2errno.c @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename err2errno.c * * * @@ -12,7 +12,7 @@ * Removed a few useless special cases, and added EZERO case.* * Make sure the global errno is _not_ changed by this funct.* * * -* © Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/src/filetime.c b/deps/MsvcLibX/src/filetime.c index 7531ba3daa..e22bc2f2a9 100644 --- a/deps/MsvcLibX/src/filetime.c +++ b/deps/MsvcLibX/src/filetime.c @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename filetime.c * * * @@ -12,7 +12,7 @@ * 2014-07-03 JFL Filetime2String: Output time with µs precision if possib. * * 2016-09-13 JFL Fixed a warning. * * * -* © Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/src/fnmatch.c b/deps/MsvcLibX/src/fnmatch.c index 20c2256008..021cfc5900 100644 --- a/deps/MsvcLibX/src/fnmatch.c +++ b/deps/MsvcLibX/src/fnmatch.c @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename: fnmatch.c * * * @@ -16,7 +16,7 @@ * 2014-02-28 JFL Added support for UTF-8 pathnames. * * 2014-03-05 JFL In debug mode, hide recursive calls. * * * -* © Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/src/fstat.c b/deps/MsvcLibX/src/fstat.c index ba6a635bb0..ba767a7bae 100644 --- a/deps/MsvcLibX/src/fstat.c +++ b/deps/MsvcLibX/src/fstat.c @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename fstat.c * * * @@ -9,7 +9,7 @@ * History: * * 2014-06-24 JFL Created this module. * * * -* ?Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/src/fstat64.c b/deps/MsvcLibX/src/fstat64.c index de6a2c6dcd..5a54962c88 100644 --- a/deps/MsvcLibX/src/fstat64.c +++ b/deps/MsvcLibX/src/fstat64.c @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename fstat64.c * * * @@ -9,7 +9,7 @@ * History: * * 2014-06-24 JFL Created this module. * * * -* © Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/src/fstat64i32.c b/deps/MsvcLibX/src/fstat64i32.c index 6275260720..cb1dfd445d 100644 --- a/deps/MsvcLibX/src/fstat64i32.c +++ b/deps/MsvcLibX/src/fstat64i32.c @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename fstat64i32.c * * * @@ -9,7 +9,7 @@ * History: * * 2014-06-24 JFL Created this module. * * * -* © Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/src/fullpath.c b/deps/MsvcLibX/src/fullpath.c index 8d94d159db..0d60b53002 100644 --- a/deps/MsvcLibX/src/fullpath.c +++ b/deps/MsvcLibX/src/fullpath.c @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename fullpath.c * * * @@ -9,7 +9,7 @@ * History * * 2016-09-13 JFL Created this module with routine from truename.c. * * * -* © Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/src/getcwd.c b/deps/MsvcLibX/src/getcwd.c index b04127c211..766a40f003 100644 --- a/deps/MsvcLibX/src/getcwd.c +++ b/deps/MsvcLibX/src/getcwd.c @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename getcwd.c * * * @@ -10,7 +10,7 @@ * 2014-02-28 JFL Created this module. * * 2014-07-02 JFL Added support for pathnames >= 260 characters. * * * -* ?Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/src/getopt.c b/deps/MsvcLibX/src/getopt.c index 81976204cc..67baf949a1 100644 --- a/deps/MsvcLibX/src/getopt.c +++ b/deps/MsvcLibX/src/getopt.c @@ -1,4 +1,4 @@ -/*- +/*- * Copyright (c) 2000 The NetBSD Foundation, Inc. * All rights reserved. * diff --git a/deps/MsvcLibX/src/getppid.c b/deps/MsvcLibX/src/getppid.c index 73f66e5144..17bd075916 100644 --- a/deps/MsvcLibX/src/getppid.c +++ b/deps/MsvcLibX/src/getppid.c @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename getppid.c * * * @@ -17,7 +17,7 @@ * History: * * 2013-03-27 JFL Created this module. * * * -* © Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/src/gettimeofday.c b/deps/MsvcLibX/src/gettimeofday.c index 9cc548366b..f9150b9b7c 100644 --- a/deps/MsvcLibX/src/gettimeofday.c +++ b/deps/MsvcLibX/src/gettimeofday.c @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename gettimeofday.c * * * @@ -9,7 +9,7 @@ * History * * 2014-06-04 JFL Created this file. * * * -* ?Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/src/iconv.c b/deps/MsvcLibX/src/iconv.c index b60ed61029..c76d655e25 100644 --- a/deps/MsvcLibX/src/iconv.c +++ b/deps/MsvcLibX/src/iconv.c @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename iconv.c * * * @@ -12,7 +12,7 @@ * 2015-12-09 JFL Added routines fputsU and vfprintfU. * * 2016-09-13 JFL Fixed warnings in fputsU. Do not change the input buffer. * * * -* © Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/src/lstat.c b/deps/MsvcLibX/src/lstat.c index bd688c13b9..9a42b0f587 100644 --- a/deps/MsvcLibX/src/lstat.c +++ b/deps/MsvcLibX/src/lstat.c @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename lstat.c * * * @@ -17,7 +17,7 @@ * 2014-03-24 JFL Renamed "statx.h" as the standard . * * 2014-06-30 JFL Added support for 32K Unicode paths. * * * -* ?Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/src/lstat32.c b/deps/MsvcLibX/src/lstat32.c index 5486f45b8e..67023ea2bd 100644 --- a/deps/MsvcLibX/src/lstat32.c +++ b/deps/MsvcLibX/src/lstat32.c @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename lstat32.c * * * @@ -9,7 +9,7 @@ * History: * * 2014-02-14 JFL Created this module. * * * -* © Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/src/lstat32i64.c b/deps/MsvcLibX/src/lstat32i64.c index 8eafed1d3e..483ff471a2 100644 --- a/deps/MsvcLibX/src/lstat32i64.c +++ b/deps/MsvcLibX/src/lstat32i64.c @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename lstat32i64.c * * * @@ -9,7 +9,7 @@ * History: * * 2014-02-14 JFL Created this module. * * * -* © Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/src/lstat64.c b/deps/MsvcLibX/src/lstat64.c index b84b3271e3..56cd0e91a9 100644 --- a/deps/MsvcLibX/src/lstat64.c +++ b/deps/MsvcLibX/src/lstat64.c @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename lstat.c * * * @@ -9,7 +9,7 @@ * History: * * 2014-02-14 JFL Created this module. * * * -* © Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/src/lstat64i32.c b/deps/MsvcLibX/src/lstat64i32.c index f5325d1832..6c86979191 100644 --- a/deps/MsvcLibX/src/lstat64i32.c +++ b/deps/MsvcLibX/src/lstat64i32.c @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename lstat64i32.c * * * @@ -9,7 +9,7 @@ * History: * * 2014-02-14 JFL Created this module. * * * -* © Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/src/main.c b/deps/MsvcLibX/src/main.c index 7c87c7fb7e..c90c111f10 100644 --- a/deps/MsvcLibX/src/main.c +++ b/deps/MsvcLibX/src/main.c @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename main.c * * * @@ -12,7 +12,7 @@ * 2017-02-05 JFL Redesigned to override libc's _setargv(). This avoids * * having to encapsulate the main() routine with one here. * * * -* © Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/src/mb2wpath.c b/deps/MsvcLibX/src/mb2wpath.c index a41ff75d94..db6897a3d8 100644 --- a/deps/MsvcLibX/src/mb2wpath.c +++ b/deps/MsvcLibX/src/mb2wpath.c @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename mb2wpath.c * * * @@ -10,7 +10,7 @@ * History: * * 2014-07-01 JFL Created this module. * * * -* © Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/src/mkdir.c b/deps/MsvcLibX/src/mkdir.c index e83f4c1773..9d23c14f64 100644 --- a/deps/MsvcLibX/src/mkdir.c +++ b/deps/MsvcLibX/src/mkdir.c @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename mkdir.c * * * @@ -11,7 +11,7 @@ * 2014-03-24 JFL Renamed "statx.h" as the standard . * * 2014-07-02 JFL Added support for pathnames >= 260 characters. * * * -* © Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/src/mkdtemp.c b/deps/MsvcLibX/src/mkdtemp.c index 602cdd4b55..99bd5197ce 100644 --- a/deps/MsvcLibX/src/mkdtemp.c +++ b/deps/MsvcLibX/src/mkdtemp.c @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename mkdtemp.c * * * @@ -9,7 +9,7 @@ * History: * * 2014-02-13 JFL Created this module. * * * -* © Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/src/mkstemp.c b/deps/MsvcLibX/src/mkstemp.c index ab576b6542..573f601bf4 100644 --- a/deps/MsvcLibX/src/mkstemp.c +++ b/deps/MsvcLibX/src/mkstemp.c @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename mkstemp.c * * * @@ -9,7 +9,7 @@ * History: * * 2014-03-03 JFL Created this module. * * * -* © Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/src/open.c b/deps/MsvcLibX/src/open.c index 8876ff43eb..4e33390e2a 100644 --- a/deps/MsvcLibX/src/open.c +++ b/deps/MsvcLibX/src/open.c @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename open.c * * * @@ -9,7 +9,7 @@ * History: * * 2017-02-16 JFL Created this module. * * * -* © Copyright 2017 Hewlett Packard Enterprise Development LP * +* Copyright 2017 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/src/readlink.c b/deps/MsvcLibX/src/readlink.c index d7c728c9fc..5e691ff86f 100644 --- a/deps/MsvcLibX/src/readlink.c +++ b/deps/MsvcLibX/src/readlink.c @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename readlink.c * * * @@ -19,7 +19,7 @@ * 2014-07-03 JFL Added support for pathnames >= 260 characters. * * 2016-09-09 JFL Fixed a crash in debug mode, due to stack overflows. * * * -* ?Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/src/realpath.c b/deps/MsvcLibX/src/realpath.c index 8d60c9f554..0d99ff631a 100644 --- a/deps/MsvcLibX/src/realpath.c +++ b/deps/MsvcLibX/src/realpath.c @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename realpath.c * * * @@ -23,7 +23,7 @@ * Convert short WIN32 paths to long paths. * * 2016-09-13 JFL Resize output buffers, to avoid wasting lots of memory. * * * -* © Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/src/spawn.c b/deps/MsvcLibX/src/spawn.c index ab70172b3e..08b81449c1 100644 --- a/deps/MsvcLibX/src/spawn.c +++ b/deps/MsvcLibX/src/spawn.c @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename spawn.c * * * @@ -10,7 +10,7 @@ * 2014-03-27 JFL Created this module. * * 2014-07-03 JFL Added support for pathnames >= 260 characters. * * * -* © Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/src/strerror.c b/deps/MsvcLibX/src/strerror.c index 2f8f324829..a10fc45753 100644 --- a/deps/MsvcLibX/src/strerror.c +++ b/deps/MsvcLibX/src/strerror.c @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename strerror.c * * * @@ -9,7 +9,7 @@ * History: * * 2014-03-06 JFL Created this module. * * * -* © Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/src/strndup.c b/deps/MsvcLibX/src/strndup.c index 5f68b8e5fc..2c75cb9ed5 100644 --- a/deps/MsvcLibX/src/strndup.c +++ b/deps/MsvcLibX/src/strndup.c @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename strndup.c * * * @@ -9,7 +9,7 @@ * History: * * 2014-02-13 JFL Created this module. * * * -* © Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/src/strptime.c b/deps/MsvcLibX/src/strptime.c index 142a2408dd..716b573a34 100644 --- a/deps/MsvcLibX/src/strptime.c +++ b/deps/MsvcLibX/src/strptime.c @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename strptime.c * * * diff --git a/deps/MsvcLibX/src/symlink.c b/deps/MsvcLibX/src/symlink.c index e791cea35a..0784fc9f41 100644 --- a/deps/MsvcLibX/src/symlink.c +++ b/deps/MsvcLibX/src/symlink.c @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename symlink.c * * * @@ -19,7 +19,7 @@ * do not support symlinks. * * 2016-08-25 JFL Fixed two warnings. * * * -* ?Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/src/uname.c b/deps/MsvcLibX/src/uname.c index 10baef0fe6..81a4c3c49a 100644 --- a/deps/MsvcLibX/src/uname.c +++ b/deps/MsvcLibX/src/uname.c @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename: uname.c * * * @@ -17,7 +17,7 @@ * History: * * 2014-05-30 JFL Created this file. * * * -* © Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/src/utime.c b/deps/MsvcLibX/src/utime.c index c365ae9f60..0a1a323e0b 100644 --- a/deps/MsvcLibX/src/utime.c +++ b/deps/MsvcLibX/src/utime.c @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename utime.c * * * @@ -13,7 +13,7 @@ * 2014-07-02 JFL Added support for pathnames >= 260 characters. * * 2016-08-25 JFL Added missing routine utimeA(). * * * -* © Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/src/utimes.c b/deps/MsvcLibX/src/utimes.c index 482883b87b..e7c760ce5b 100644 --- a/deps/MsvcLibX/src/utimes.c +++ b/deps/MsvcLibX/src/utimes.c @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename utimes.c * * * @@ -14,7 +14,7 @@ * 2014-06-04 JFL Added handling of UTIME_NOW and UTIME_OMIT. * * 2014-07-02 JFL Added support for pathnames >= 260 characters. * * * -* ?Copyright 2016 Hewlett Packard Enterprise Development LP * +* Copyright 2016 Hewlett Packard Enterprise Development LP * * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ diff --git a/deps/MsvcLibX/src/xfreopen.c b/deps/MsvcLibX/src/xfreopen.c index 724fb8e13b..772e81f34f 100644 --- a/deps/MsvcLibX/src/xfreopen.c +++ b/deps/MsvcLibX/src/xfreopen.c @@ -1,4 +1,4 @@ -/*****************************************************************************\ +/*****************************************************************************\ * * * Filename xfreopen.c * * * From 8e53280b3c240a0d564708b35607b592e9dd907e Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 7 Aug 2020 16:58:55 +0800 Subject: [PATCH 059/109] TD-1057 --- deps/MsvcLibX/src/utimes.c | 2 +- src/os/src/windows/CMakeLists.txt | 2 +- src/query/CMakeLists.txt | 2 +- src/tsdb/CMakeLists.txt | 20 ++++++++++++++++++++ src/tsdb/src/tsdbFile.c | 2 ++ 5 files changed, 25 insertions(+), 3 deletions(-) diff --git a/deps/MsvcLibX/src/utimes.c b/deps/MsvcLibX/src/utimes.c index e7c760ce5b..7638b932b4 100644 --- a/deps/MsvcLibX/src/utimes.c +++ b/deps/MsvcLibX/src/utimes.c @@ -64,7 +64,7 @@ void Filetime2Timeval(const FILETIME *pFT, struct timeval *ptv) { ULARGE_INTEGER ull; ull.LowPart = pFT->dwLowDateTime; ull.HighPart = pFT->dwHighDateTime; - ptv->tv_sec = ull.QuadPart / 10000000ULL - 11644473600ULL; + ptv->tv_sec = (long)(ull.QuadPart / 10000000ULL - 11644473600ULL); ptv->tv_usec = (int32_t)((ull.QuadPart % 10000000ULL)/10); } diff --git a/src/os/src/windows/CMakeLists.txt b/src/os/src/windows/CMakeLists.txt index 131533c5c5..588d3b7f68 100644 --- a/src/os/src/windows/CMakeLists.txt +++ b/src/os/src/windows/CMakeLists.txt @@ -4,4 +4,4 @@ PROJECT(TDengine) AUX_SOURCE_DIRECTORY(. SRC) ADD_LIBRARY(os ${SRC}) -TARGET_LINK_LIBRARIES(os winmm IPHLPAPI ws2_32) +TARGET_LINK_LIBRARIES(os winmm IPHLPAPI ws2_32 MsvcLibXw64) diff --git a/src/query/CMakeLists.txt b/src/query/CMakeLists.txt index b9e7ff59e5..e2bee4285f 100644 --- a/src/query/CMakeLists.txt +++ b/src/query/CMakeLists.txt @@ -13,5 +13,5 @@ IF (TD_LINUX) TARGET_LINK_LIBRARIES(query tsdb tutil m rt) ADD_SUBDIRECTORY(tests) ELSEIF (TD_WINDOWS) - TARGET_LINK_LIBRARIES(query tutil) + TARGET_LINK_LIBRARIES(query tsdb tutil) ENDIF () diff --git a/src/tsdb/CMakeLists.txt b/src/tsdb/CMakeLists.txt index cef1d0bba7..7831e492ca 100644 --- a/src/tsdb/CMakeLists.txt +++ b/src/tsdb/CMakeLists.txt @@ -10,6 +10,26 @@ IF (TD_LINUX) # Someone has no gtest directory, so comment it # ADD_SUBDIRECTORY(tests) ELSEIF (TD_WINDOWS) +SET(CMAKE_MODULE_PATH ${TD_COMMUNITY_DIR}/deps/MsvcLibX/cmake) + include(FindWindowsSDK) + get_mywindowssdk_include_dirs(${WINDOWSSDK_PREFERRED_DIR} WINSDK_INCLUDE_DIR) + get_ucrt_include_dirs(${WINDOWSSDK_PREFERRED_DIR} UCRT_INCLUDE_DIR) + + Add_Definitions("-DWSDKINCLUDE=${WINSDK_INCLUDE_DIR}") + + include(FindMSVC) + Add_Definitions("-DMSVCINCLUDE=${VC_INCLUDE_DIR}") + + IF ((NOT DEFINED UCRT_INCLUDE_DIR) OR (UCRT_INCLUDE_DIR STREQUAL "NOTFOUND")) + Message(STATUS "UCRT_INCLUDE_DIR notu found, set it to vc incude dir") + SET(UCRT_INCLUDE_DIR "${VC_INCLUDE_DIR}") + Add_Definitions("-DUCRTINCLUDE=${VC_INCLUDE_DIR}") + ELSE () + Add_Definitions("-DUCRTINCLUDE=${UCRT_INCLUDE_DIR}") + ENDIF () + + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/MsvcLibX/include) + ADD_LIBRARY(tsdb ${SRC}) TARGET_LINK_LIBRARIES(tsdb common tutil) ENDIF () diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index 8cec7c08e1..4cb5f76414 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -18,6 +18,8 @@ #include "tchecksum.h" #include "tsdbMain.h" #include "tutil.h" +#include "dirent.h" + #define TAOS_RANDOM_FILE_FAIL_TEST #ifdef TSDB_IDX From 17e1c19b81f9f73b2d0c678b74a469743e475fcb Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 7 Aug 2020 09:30:52 +0000 Subject: [PATCH 060/109] remove Idx file --- src/tsdb/inc/tsdbMain.h | 15 ------------ src/tsdb/src/tsdbFile.c | 4 ---- src/tsdb/src/tsdbMemTable.c | 5 ---- src/tsdb/src/tsdbRWHelper.c | 48 ------------------------------------- 4 files changed, 72 deletions(-) diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 6119f7086c..16a2e17f1e 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -151,18 +151,10 @@ typedef struct { // ------------------ tsdbFile.c extern const char* tsdbFileSuffix[]; typedef enum { -#ifdef TSDB_IDX - TSDB_FILE_TYPE_IDX = 0, - TSDB_FILE_TYPE_HEAD, -#else TSDB_FILE_TYPE_HEAD = 0, -#endif TSDB_FILE_TYPE_DATA, TSDB_FILE_TYPE_LAST, TSDB_FILE_TYPE_MAX, -#ifdef TSDB_IDX - TSDB_FILE_TYPE_NIDX, -#endif TSDB_FILE_TYPE_NHEAD, TSDB_FILE_TYPE_NLAST } TSDB_FILE_TYPE; @@ -281,9 +273,6 @@ typedef struct { TSKEY minKey; TSKEY maxKey; SFileGroup fGroup; -#ifdef TSDB_IDX - SFile nIdxF; -#endif SFile nHeadF; SFile nLastF; } SHelperFile; @@ -497,10 +486,6 @@ void tsdbGetFidKeyRange(int daysPerFile, int8_t precision, int fileId, TS #define helperState(h) (h)->state #define TSDB_NLAST_FILE_OPENED(h) ((h)->files.nLastF.fd > 0) #define helperFileId(h) ((h)->files.fGroup.fileId) -#ifdef TSDB_IDX -#define helperIdxF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_IDX])) -#define helperNewIdxF(h) (&((h)->files.nIdxF)) -#endif #define helperHeadF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_HEAD])) #define helperDataF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_DATA])) #define helperLastF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_LAST])) diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index 8cec7c08e1..a5435ad872 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -20,11 +20,7 @@ #include "tutil.h" #define TAOS_RANDOM_FILE_FAIL_TEST -#ifdef TSDB_IDX -const char *tsdbFileSuffix[] = {".idx", ".head", ".data", ".last", "", ".i", ".h", ".l"}; -#else const char *tsdbFileSuffix[] = {".head", ".data", ".last", "", ".h", ".l"}; -#endif static int tsdbInitFile(SFile *pFile, STsdbRepo *pRepo, int fid, int type); static void tsdbDestroyFile(SFile *pFile); diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 990db76b7e..0303c47146 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -683,11 +683,6 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe pthread_rwlock_wrlock(&(pFileH->fhlock)); -#ifdef TSDB_IDX - rename(helperNewIdxF(pHelper)->fname, helperIdxF(pHelper)->fname); - pGroup->files[TSDB_FILE_TYPE_IDX].info = helperNewIdxF(pHelper)->info; -#endif - rename(helperNewHeadF(pHelper)->fname, helperHeadF(pHelper)->fname); pGroup->files[TSDB_FILE_TYPE_HEAD].info = helperNewHeadF(pHelper)->info; diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index fb79747d30..4b5acd8fe4 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -110,31 +110,16 @@ int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup) { // Set the files pHelper->files.fGroup = *pGroup; if (helperType(pHelper) == TSDB_WRITE_HELPER) { -#ifdef TSDB_IDX - tsdbGetDataFileName(pHelper->pRepo, pGroup->fileId, TSDB_FILE_TYPE_NIDX, helperNewIdxF(pHelper)->fname); -#endif tsdbGetDataFileName(pHelper->pRepo, pGroup->fileId, TSDB_FILE_TYPE_NHEAD, helperNewHeadF(pHelper)->fname); tsdbGetDataFileName(pHelper->pRepo, pGroup->fileId, TSDB_FILE_TYPE_NLAST, helperNewLastF(pHelper)->fname); } // Open the files -#ifdef TSDB_IDX - if (tsdbOpenFile(helperIdxF(pHelper), O_RDONLY) < 0) return -1; -#endif if (tsdbOpenFile(helperHeadF(pHelper), O_RDONLY) < 0) return -1; if (helperType(pHelper) == TSDB_WRITE_HELPER) { if (tsdbOpenFile(helperDataF(pHelper), O_RDWR) < 0) return -1; if (tsdbOpenFile(helperLastF(pHelper), O_RDWR) < 0) return -1; -#ifdef TSDB_IDX - // Create and open .i file - pFile = helperNewIdxF(pHelper); - if (tsdbOpenFile(pFile, O_WRONLY | O_CREAT) < 0) return -1; - pFile->info.size = TSDB_FILE_HEAD_SIZE; - pFile->info.magic = TSDB_FILE_INIT_MAGIC; - if (tsdbUpdateFileHeader(pFile) < 0) return -1; -#endif - // Create and open .h pFile = helperNewHeadF(pHelper); if (tsdbOpenFile(pFile, O_WRONLY | O_CREAT) < 0) return -1; @@ -164,11 +149,6 @@ int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup) { int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError) { SFile *pFile = NULL; -#ifdef TSDB_IDX - pFile = helperIdxF(pHelper); - tsdbCloseFile(pFile); -#endif - pFile = helperHeadF(pHelper); tsdbCloseFile(pFile); @@ -199,18 +179,6 @@ int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError) { } if (helperType(pHelper) == TSDB_WRITE_HELPER) { -#ifdef TSDB_IDX - pFile = helperNewIdxF(pHelper); - if (pFile->fd > 0) { - if (!hasError) { - tsdbUpdateFileHeader(pFile); - fsync(pFile->fd); - } - tsdbCloseFile(pFile); - if (hasError) (void)remove(pFile->fname); - } -#endif - pFile = helperNewHeadF(pHelper); if (pFile->fd > 0) { if (!hasError) { @@ -412,10 +380,6 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) { return -1; } -#ifdef TSDB_IDX - pFile = helperNewIdxF(pHelper); -#endif - if (taosTSizeof(pHelper->pWIdx) < pFile->info.len + sizeof(SCompIdx) + 12) { pHelper->pWIdx = taosTRealloc(pHelper->pWIdx, taosTSizeof(pHelper->pWIdx) == 0 ? 1024 : taosTSizeof(pHelper->pWIdx) * 2); if (pHelper->pWIdx == NULL) { @@ -435,11 +399,7 @@ int tsdbWriteCompIdx(SRWHelper *pHelper) { ASSERT(helperType(pHelper) == TSDB_WRITE_HELPER); off_t offset = 0; -#ifdef TSDB_IDX - SFile *pFile = helperNewIdxF(pHelper); -#else SFile *pFile = helperNewHeadF(pHelper); -#endif pFile->info.len += sizeof(TSCKSUM); if (taosTSizeof(pHelper->pWIdx) < pFile->info.len) { @@ -474,11 +434,7 @@ int tsdbWriteCompIdx(SRWHelper *pHelper) { int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) { ASSERT(pHelper->state == TSDB_HELPER_FILE_SET_AND_OPEN); -#ifdef TSDB_IDX - SFile *pFile = helperIdxF(pHelper); -#else SFile *pFile = helperHeadF(pHelper); -#endif int fd = pFile->fd; if (!helperHasState(pHelper, TSDB_HELPER_IDX_LOAD)) { @@ -1052,10 +1008,6 @@ static void tsdbResetHelperFileImpl(SRWHelper *pHelper) { helperLastF(pHelper)->fd = -1; helperNewHeadF(pHelper)->fd = -1; helperNewLastF(pHelper)->fd = -1; -#ifdef TSDB_IDX - helperIdxF(pHelper)->fd = -1; - helperNewIdxF(pHelper)->fd = -1; -#endif } static int tsdbInitHelperFile(SRWHelper *pHelper) { From 562e62e9330a959aa9e9134ea69e195c2476fa4c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 7 Aug 2020 17:44:32 +0800 Subject: [PATCH 061/109] [td-225] add check after killing query. --- src/client/inc/tscUtil.h | 3 +++ src/client/inc/tsclient.h | 1 + src/client/src/tscSql.c | 54 +++++++++++++-------------------------- src/client/src/tscUtil.c | 18 +++++++++++++ 4 files changed, 40 insertions(+), 36 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 590f205e1d..72ca96891a 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -270,6 +270,9 @@ void tscAsyncQuerySingleRowForNextVnode(void *param, TAOS_RES *tres, int numOfRo void tscTryQueryNextClause(SSqlObj* pSql, __async_cb_func_t fp); int tscSetMgmtEpSetFromCfg(const char *first, const char *second); +bool tscSetSqlOwner(SSqlObj* pSql); +void tscClearSqlOwner(SSqlObj* pSql); + void* malloc_throw(size_t size); void* calloc_throw(size_t nmemb, size_t size); char* strdup_throw(const char* str); diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 3ad4cd9455..49f7cec889 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -302,6 +302,7 @@ typedef struct STscObj { typedef struct SSqlObj { void *signature; + pthread_t owner; // owner of sql object, by which it is executed STscObj *pTscObj; void *pRpcCtx; void (*fp)(); diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index af8e913563..f01f1aa384 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -233,35 +233,6 @@ static void waitForRetrieveRsp(void *param, TAOS_RES *tres, int numOfRows) { sem_post(&pSql->rspSem); } -TAOS_RES* taos_query(TAOS *taos, const char *sqlstr) { - STscObj *pObj = (STscObj *)taos; - if (pObj == NULL || pObj->signature != pObj) { - terrno = TSDB_CODE_TSC_DISCONNECTED; - return NULL; - } - - int32_t sqlLen = strlen(sqlstr); - if (sqlLen > tsMaxSQLStringLen) { - tscError("sql string exceeds max length:%d", tsMaxSQLStringLen); - terrno = TSDB_CODE_TSC_INVALID_SQL; - return NULL; - } - - taosNotePrintTsc(sqlstr); - - SSqlObj* pSql = calloc(1, sizeof(SSqlObj)); - if (pSql == NULL) { - tscError("failed to malloc sqlObj"); - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - return NULL; - } - - doAsyncQuery(pObj, pSql, waitForQueryRsp, taos, sqlstr, sqlLen); - - // wait for the callback function to post the semaphore - tsem_wait(&pSql->rspSem); - return pSql; -} TAOS_RES* taos_query_c(TAOS *taos, const char *sqlstr, uint32_t sqlLen) { STscObj *pObj = (STscObj *)taos; if (pObj == NULL || pObj->signature != pObj) { @@ -274,7 +245,9 @@ TAOS_RES* taos_query_c(TAOS *taos, const char *sqlstr, uint32_t sqlLen) { terrno = TSDB_CODE_TSC_INVALID_SQL; return NULL; } - + + taosNotePrintTsc(sqlstr); + SSqlObj* pSql = calloc(1, sizeof(SSqlObj)); if (pSql == NULL) { tscError("failed to malloc sqlObj"); @@ -287,6 +260,11 @@ TAOS_RES* taos_query_c(TAOS *taos, const char *sqlstr, uint32_t sqlLen) { tsem_wait(&pSql->rspSem); return pSql; } + +TAOS_RES* taos_query(TAOS *taos, const char *sqlstr) { + return taos_query_c(taos, sqlstr, strlen(sqlstr)); +} + int taos_result_precision(TAOS_RES *res) { SSqlObj *pSql = (SSqlObj *)res; if (pSql == NULL || pSql->signature != pSql) return 0; @@ -422,7 +400,10 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { pCmd->command == TSDB_SQL_INSERT) { return NULL; } - + + // set the sql object owner + tscSetSqlOwner(pSql); + // current data set are exhausted, fetch more data from node if (pRes->row >= pRes->numOfRows && (pRes->completed != true || hasMoreVnodesToTry(pSql) || hasMoreClauseToTry(pSql)) && (pCmd->command == TSDB_SQL_RETRIEVE || @@ -441,7 +422,10 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { sem_wait(&pSql->rspSem); } - return doSetResultRowData(pSql, true); + void* data = doSetResultRowData(pSql, true); + + tscClearSqlOwner(pSql); + return data; } int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) { @@ -509,7 +493,7 @@ int taos_select_db(TAOS *taos, const char *db) { } // send free message to vnode to free qhandle and corresponding resources in vnode -static bool tscFreeQhandleInVnode(SSqlObj* pSql) { +static bool tscKillQueryInVnode(SSqlObj* pSql) { SSqlCmd* pCmd = &pSql->cmd; SSqlRes* pRes = &pSql->res; @@ -557,16 +541,14 @@ void taos_free_result(TAOS_RES *res) { } pQueryInfo->type = TSDB_QUERY_TYPE_FREE_RESOURCE; - if (!tscFreeQhandleInVnode(pSql)) { + if (!tscKillQueryInVnode(pSql)) { tscFreeSqlObj(pSql); tscDebug("%p sqlObj is freed by app", pSql); } } -// todo should not be used in async query int taos_errno(TAOS_RES *tres) { SSqlObj *pSql = (SSqlObj *) tres; - if (pSql == NULL || pSql->signature != pSql) { return terrno; } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 5409d90939..fdc019e97b 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -2223,3 +2223,21 @@ int tscSetMgmtEpSetFromCfg(const char *first, const char *second) { return 0; } + +bool tscSetSqlOwner(SSqlObj* pSql) { + SSqlRes* pRes = &pSql->res; + + // set the sql object owner + uint64_t threadId = taosGetPthreadId(); + if (atomic_val_compare_exchange_64(&pSql->owner, 0, threadId) != 0) { + pRes->code = TSDB_CODE_QRY_IN_EXEC; + return false; + } + + return true; +} + +void tscClearSqlOwner(SSqlObj* pSql) { + assert(pSql->owner != 0); + atomic_store_64(&pSql->owner, 0); +} \ No newline at end of file From ef2f966da5817fd04e3993e594c51b38ab5b2603 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 7 Aug 2020 18:59:52 +0800 Subject: [PATCH 062/109] TD-1057 --- cmake/define.inc | 1 + deps/MsvcLibX/CMakeLists.txt | 25 +- deps/MsvcLibX/README.md | 820 ------------------ deps/MsvcLibX/config.DESKTOP-U79TD6T.bat | 146 ---- deps/MsvcLibX/configure.bat | 35 - .../include/{debugm.h => msvcDebugm.h} | 0 .../include/{direct.h => msvcDirect.h} | 2 +- .../include/{dirent.h => msvcDirent.h} | 0 .../MsvcLibX/include/{error.h => msvcError.h} | 0 .../include/{fadvise.h => msvcFadvise.h} | 0 .../MsvcLibX/include/{fcntl.h => msvcFcntl.h} | 2 +- .../include/{fnmatch.h => msvcFnmatch.h} | 0 .../include/{getopt.h => msvcGetopt.h} | 0 .../MsvcLibX/include/{iconv.h => msvcIconv.h} | 0 .../include/{inttypes.h => msvcInttypes.h} | 0 .../include/{libgen.h => msvcLibgen.h} | 0 .../include/{limits.h => msvcLimits.h} | 2 +- .../MsvcLibX/include/{netdb.h => msvcNetdb.h} | 0 .../include/{process.h => msvcProcess.h} | 2 +- .../MsvcLibX/include/{regex.h => msvcRegex.h} | 0 .../include/{reparsept.h => msvcReparsept.h} | 0 .../include/{stdbool.h => msvcStdbool.h} | 0 .../include/{stdint.h => msvcStdint.h} | 0 .../MsvcLibX/include/{stdio.h => msvcStdio.h} | 2 +- .../include/{stdlib.h => msvcStdlib.h} | 2 +- .../include/{system.h => msvcSystem.h} | 0 deps/MsvcLibX/include/{time.h => msvcTime.h} | 6 +- .../include/{unistd.h => msvcUnistd.h} | 4 +- .../MsvcLibX/include/{utime.h => msvcUtime.h} | 0 .../include/{windows.h => msvcWindows.h} | 2 +- .../include/{xfreopen.h => msvcXfreopen.h} | 0 deps/MsvcLibX/include/stdio--.h | 17 - .../include/sys/{param.h => msvcParam.h} | 0 .../include/sys/{stat.h => msvcStat.h} | 12 +- .../include/sys/{time.h => msvcTime.h} | 0 .../include/sys/{utsname.h => msvcTsname.h} | 0 .../include/sys/{types.h => msvcTypes.h} | 3 +- deps/MsvcLibX/make.bat | 35 - deps/MsvcLibX/src/Files.mak | 261 ------ deps/MsvcLibX/src/GetFileAttributes.c | 2 +- deps/MsvcLibX/src/GetFileAttributesEx.c | 2 +- deps/MsvcLibX/src/GetFullPathName.c | 4 +- deps/MsvcLibX/src/GetLongPathName.c | 4 +- deps/MsvcLibX/src/NMakefile | 239 ----- deps/MsvcLibX/src/access.c | 5 +- deps/MsvcLibX/src/basename.c | 6 +- deps/MsvcLibX/src/chdir.c | 10 +- deps/MsvcLibX/src/clock_gettime.c | 4 +- deps/MsvcLibX/src/config.DESKTOP-U79TD6T.bat | 151 ---- deps/MsvcLibX/src/configure.MsvcLibX.bat | 36 - deps/MsvcLibX/src/configure.bat | 35 - deps/MsvcLibX/src/debugv.c | 2 +- deps/MsvcLibX/src/dirent.c | 9 +- deps/MsvcLibX/src/dirname.c | 4 +- deps/MsvcLibX/src/err2errno.c | 3 +- deps/MsvcLibX/src/exe | 99 --- deps/MsvcLibX/src/exe.bat | 82 -- deps/MsvcLibX/src/filetime.c | 10 +- deps/MsvcLibX/src/fnmatch.c | 4 +- deps/MsvcLibX/src/fstat.c | 6 +- deps/MsvcLibX/src/fullpath.c | 4 +- deps/MsvcLibX/src/getcwd.c | 5 +- deps/MsvcLibX/src/getopt.c | 2 +- deps/MsvcLibX/src/getppid.c | 2 +- deps/MsvcLibX/src/gettimeofday.c | 4 +- deps/MsvcLibX/src/iconv.c | 5 +- deps/MsvcLibX/src/lstat.c | 11 +- deps/MsvcLibX/src/main.c | 4 +- deps/MsvcLibX/src/make.bat | 35 - deps/MsvcLibX/src/mb2wpath.c | 2 +- deps/MsvcLibX/src/mkdir.c | 3 +- deps/MsvcLibX/src/mkdtemp.c | 2 +- deps/MsvcLibX/src/mkstemp.c | 2 +- deps/MsvcLibX/src/open.c | 5 +- deps/MsvcLibX/src/readlink.c | 7 +- deps/MsvcLibX/src/realpath.c | 6 +- deps/MsvcLibX/src/spawn.c | 3 +- deps/MsvcLibX/src/src2objs.bat | 125 --- deps/MsvcLibX/src/symlink.c | 7 +- deps/MsvcLibX/src/uname.c | 2 +- deps/MsvcLibX/src/utime.c | 10 +- deps/MsvcLibX/src/utimes.c | 10 +- deps/MsvcLibX/src/xfreopen.c | 2 +- deps/pthread/pthread.h | 2 + deps/pthread/sched.h | 25 +- deps/pthread/semaphore.h | 2 + src/os/inc/osFile.h | 12 - src/os/inc/osWindows.h | 21 +- src/os/src/windows/w64Time.c | 12 - src/tsdb/src/tsdbFile.c | 3 +- 90 files changed, 140 insertions(+), 2291 deletions(-) delete mode 100644 deps/MsvcLibX/README.md delete mode 100644 deps/MsvcLibX/config.DESKTOP-U79TD6T.bat delete mode 100644 deps/MsvcLibX/configure.bat rename deps/MsvcLibX/include/{debugm.h => msvcDebugm.h} (100%) rename deps/MsvcLibX/include/{direct.h => msvcDirect.h} (91%) rename deps/MsvcLibX/include/{dirent.h => msvcDirent.h} (100%) rename deps/MsvcLibX/include/{error.h => msvcError.h} (100%) rename deps/MsvcLibX/include/{fadvise.h => msvcFadvise.h} (100%) rename deps/MsvcLibX/include/{fcntl.h => msvcFcntl.h} (92%) rename deps/MsvcLibX/include/{fnmatch.h => msvcFnmatch.h} (100%) rename deps/MsvcLibX/include/{getopt.h => msvcGetopt.h} (100%) rename deps/MsvcLibX/include/{iconv.h => msvcIconv.h} (100%) rename deps/MsvcLibX/include/{inttypes.h => msvcInttypes.h} (100%) rename deps/MsvcLibX/include/{libgen.h => msvcLibgen.h} (100%) rename deps/MsvcLibX/include/{limits.h => msvcLimits.h} (94%) rename deps/MsvcLibX/include/{netdb.h => msvcNetdb.h} (100%) rename deps/MsvcLibX/include/{process.h => msvcProcess.h} (92%) rename deps/MsvcLibX/include/{regex.h => msvcRegex.h} (100%) rename deps/MsvcLibX/include/{reparsept.h => msvcReparsept.h} (100%) rename deps/MsvcLibX/include/{stdbool.h => msvcStdbool.h} (100%) rename deps/MsvcLibX/include/{stdint.h => msvcStdint.h} (100%) rename deps/MsvcLibX/include/{stdio.h => msvcStdio.h} (95%) rename deps/MsvcLibX/include/{stdlib.h => msvcStdlib.h} (93%) rename deps/MsvcLibX/include/{system.h => msvcSystem.h} (100%) rename deps/MsvcLibX/include/{time.h => msvcTime.h} (87%) rename deps/MsvcLibX/include/{unistd.h => msvcUnistd.h} (96%) rename deps/MsvcLibX/include/{utime.h => msvcUtime.h} (100%) rename deps/MsvcLibX/include/{windows.h => msvcWindows.h} (93%) rename deps/MsvcLibX/include/{xfreopen.h => msvcXfreopen.h} (100%) delete mode 100644 deps/MsvcLibX/include/stdio--.h rename deps/MsvcLibX/include/sys/{param.h => msvcParam.h} (100%) rename deps/MsvcLibX/include/sys/{stat.h => msvcStat.h} (96%) rename deps/MsvcLibX/include/sys/{time.h => msvcTime.h} (100%) rename deps/MsvcLibX/include/sys/{utsname.h => msvcTsname.h} (100%) rename deps/MsvcLibX/include/sys/{types.h => msvcTypes.h} (94%) delete mode 100644 deps/MsvcLibX/make.bat delete mode 100644 deps/MsvcLibX/src/Files.mak delete mode 100644 deps/MsvcLibX/src/NMakefile delete mode 100644 deps/MsvcLibX/src/config.DESKTOP-U79TD6T.bat delete mode 100644 deps/MsvcLibX/src/configure.MsvcLibX.bat delete mode 100644 deps/MsvcLibX/src/configure.bat delete mode 100644 deps/MsvcLibX/src/exe delete mode 100644 deps/MsvcLibX/src/exe.bat delete mode 100644 deps/MsvcLibX/src/make.bat delete mode 100644 deps/MsvcLibX/src/src2objs.bat diff --git a/cmake/define.inc b/cmake/define.inc index 5c005b0cb5..09c21e2912 100755 --- a/cmake/define.inc +++ b/cmake/define.inc @@ -119,6 +119,7 @@ IF (TD_WINDOWS) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/iconv) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/regex) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/wepoll/inc) + INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/MsvcLibX/include) ENDIF () IF (TD_WINDOWS_64) diff --git a/deps/MsvcLibX/CMakeLists.txt b/deps/MsvcLibX/CMakeLists.txt index 7f24d34dd5..fc77a3b447 100644 --- a/deps/MsvcLibX/CMakeLists.txt +++ b/deps/MsvcLibX/CMakeLists.txt @@ -1,30 +1,7 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) -IF (TD_WINDOWS_64) - SET(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake) - - include(FindWindowsSDK) - get_mywindowssdk_include_dirs(${WINDOWSSDK_PREFERRED_DIR} WINSDK_INCLUDE_DIR) - get_ucrt_include_dirs(${WINDOWSSDK_PREFERRED_DIR} UCRT_INCLUDE_DIR) - - Add_Definitions("-DWSDKINCLUDE=${WINSDK_INCLUDE_DIR}") - - include(FindMSVC) - Add_Definitions("-DMSVCINCLUDE=${VC_INCLUDE_DIR}") - - IF ((NOT DEFINED UCRT_INCLUDE_DIR) OR (UCRT_INCLUDE_DIR STREQUAL "NOTFOUND")) - Message(STATUS "UCRT_INCLUDE_DIR notu found, set it to vc incude dir") - SET(UCRT_INCLUDE_DIR "${VC_INCLUDE_DIR}") - Add_Definitions("-DUCRTINCLUDE=${VC_INCLUDE_DIR}") - ELSE () - Add_Definitions("-DUCRTINCLUDE=${UCRT_INCLUDE_DIR}") - ENDIF () - - Message(STATUS "WINSDK_INCLUDE_DIR: ${WINSDK_INCLUDE_DIR}") - Message(STATUS "UCRT_INCLUDE_DIR: ${UCRT_INCLUDE_DIR}") - Message(STATUS "VC_INCLUDE_DIR: ${VC_INCLUDE_DIR}") - +IF (TD_WINDOWS) INCLUDE_DIRECTORIES(include) AUX_SOURCE_DIRECTORY(src SRC) ADD_LIBRARY(MsvcLibXw64 ${SRC}) diff --git a/deps/MsvcLibX/README.md b/deps/MsvcLibX/README.md deleted file mode 100644 index 8f689412cb..0000000000 --- a/deps/MsvcLibX/README.md +++ /dev/null @@ -1,820 +0,0 @@ -MsvcLibX - A set of MSVC Library eXtensions -=========================================== - - -Introduction ------------- - -The MsvcLibX library is built upon 30 years of work. -It exists because the Microsoft Visual C/C++ library is only a subset of the -standard C library, and I wanted to port to DOS and Windows programs using some -of the missing parts. -Initially Microsoft C compilers for MS-DOS tried to be as compliant as possible -with the standard C library. Then Microsoft rewrote their library for Windows. -But when the WIN32 API became prevalent, Microsoft apparently lost interest, -and stopped porting to MS-DOS and Windows the new C include files and library -routines that got standardized over the years. -To the point that's it's now impossible to rebuild any recent Posix/Unix/Linux -program using the MSVC library. - -MsvcLibX adds a number of standard include files and library routines that I've -needed over the years. For example the directory access routines defined in -dirent.h, or the symbolic link support in unistd.h. -It also includes a number of useful routines and macros, either needed -internally to implement and debug the above, or made necessary by irresolvable -incompatibilities between Unix and Windows. - -Other major features of the MsvcLibX library are: - -* A powerful make system, based on Microsoft nmake files, that allow building - multiple versions of the same program (Ex: DOS/WIN32/WIN64) with one simple - command. And in most cases without having to write any dedicated make file. -* Support for UTF-8 sources, allowing to build programs working in any code page. -* Support for NTFS symlinks and junctions. -* Support for DOS, and Windows 95 targets, for targeting old computers. -* Support for bound DOS+Windows programs, for truly universal executables that - work in *all* versions of DOS and Windows. - -An obvious alternative exists for building Windows programs: MinGW. -But I started all this before MinGW even existed. And, surprisingly, even MinGW -still has holes in 2014, where MsvcLibX in significantly more advanced. -Another alternative is CygWin. This one is more complete, but also much more -heavyweight. Using programs built with CygWin requires installing CygWin DLLs. -Copying the program .exe from one system to the next is not sufficient. In that -sense, MinGW or MsvcLibX are much better. - -Contrary to MinGW and CygWin, MsvcLibX does not attempt to be complete, and is -unlikely to ever be. But any contribution of improvements is welcome. -Likewise, any help at contributing unique MsvcLibX features to MinGW is welcome. - -Jean-François Larvoire -2016-09-29 - - -Building the MsvcLibX library ------------------------------ - -On a recent Windows PC, with Microsoft Visual C++ compilers installed: - -- Select a base work directory. I'll call it %BASEDIR% in the examples below: - - set "HOME=%HOMEDRIVE%%HOMEPATH%" - set "BASEDIR=%HOME%\Documents\SRC" - md "%BASEDIR%" - cd "%BASEDIR%" - -- Extract the MsvcLibX archive into a work subdirectory. Ex: %BASEDIR%\MsvcLibX\ - This will put files in several subdirectories: include, src - -- Open a cmd window, and run: - - cd "%BASEDIR%\MsvcLibX\src" - configure - make - -The configure.bat script will locate your MSVC tools, and generate a config -file. It needs to be run once initially, then again if new MSVC tool versions -are installed. -The make.bat script should take care of everything, and rebuild normal and debug -versions of the MsvcLibX.lib library for all operating systems your tools support. -In the end, it defines system environment variable MSVCLIBX, necessary for -building programs using the MsvcLibX.lib library. - -Requirements: - -- Microsoft Visual C++ compiler and linker for Windows 32 and 64 bits targets. -- Microsoft Windows SDK. (May be installed along with Visual C++) - -As of 2015-12-10, I've tested the make.bat script and make files with Visual C++ -2005, 2008, 2012, and 2015, and Windows SDK 5.2, 8.1, and 10. -Support for older versions is still built-in, but I've not tested it for long. -Support for newer versions will require minor tweaks in configure.bat. -Note that configure.bat does not depend on MSVC's vcvars.bat. It will attempt -to locate and use the latest usable version of MSVC it finds in Program Files. - -Optional: - -- Microsoft Visual C++ 1.52 compiler and linker for MS-DOS targets. - If present in C:\MSVC, make.bat will also generate libraries for DOS. - Note that generating DOS versions is still more useful than it looks: - Experience has shown that the severe memory constraints under DOS are a very - good revelator of memory leaks, and other bad pointer issues. Build your - programs for DOS too if you can, and test them in a VM with Windows XP. - This takes a very small extra build time. And it'll be very much worth the time - if the Windows version appears to work, but the DOS version does not. - Visual C++ 1.52 is available on the Visual Studio 2005 CD. -- The Visual C++ 8 compiler and tools from Microsoft Visual Studio 2005. - This is the last version that allowed building programs for Windows 95/NT4. - -The MsvcLibX.lib libraries are generated in subdirectories of src, then copied -into the %BASEDIR%\MsvcLibX\lib directory, and renamed as shown in this table: - -Subdirectory | Description | Renamed as ----------------- | -------------------------------------- | ---------------- -DOS\BIN\T\ | DOS normal version, tiny memory model | MsvcLibXdt.lib -DOS\DEBUG\BIN\T\ | DOS debug version, tiny memory model | MsvcLibXdtd.lib -DOS\BIN\S\ | DOS normal version, small memory model | MsvcLibXds.lib -DOS\DEBUG\BIN\S\ | DOS debug version, small memory model | MsvcLibXdsd.lib -DOS\BIN\L\ | DOS normal version, large memory model | MsvcLibXdl.lib -DOS\DEBUG\BIN\L\ | DOS debug version, large memory model | MsvcLibXdld.lib -WIN95\ | WIN32 normal version for Windows 95 | MsvcLibXw32.lib -WIN95\DEBUG\ | WIN32 debug version for Windows 95 | MsvcLibXw32d.lib -WIN32\ | WIN32 (X86) normal version | MsvcLibXw32.lib -WIN32\DEBUG\ | WIN32 (X86) debug version | MsvcLibXw32d.lib -WIN64\ | WIN64 (AMD64) normal version | MsvcLibXw64.lib -WIN64\DEBUG\ | WIN64 (AMD64) debug version | MsvcLibXw64d.lib - - -Building programs using the MsvcLibX library --------------------------------------------- - -Create a work directory, distinct from the MsvcLibX directories. Ex: - - set "HOME=%HOMEDRIVE%%HOMEPATH%" - set "BASEDIR=%HOME%\Documents\SRC" - md "%BASEDIR%\MyTools" - cd "%BASEDIR%\MyTools" - :# Define a variable giving the location of the MsvcLibX base directory - :# (Automatically defined if you built the library already on that same system.) - set "MSVCLIBX=%BASEDIR%\MsvcLibX" - :# Get batch files and make files from MsvcLibX sources - copy "%MSVCLIBX%\src\*.bat" - copy "%MSVCLIBX%\src\*.mak" - :# Create the configuration file (To be done just once) - configure - :# Compile and link your C or C++ program. - :# Ex, for the dirc.c sample, to create all dirc.exe versions, type: - make dirc.exe - :# If there is any error, the dirc.log file will pop up. - :# If there's no error, it's possible to check for warnings by reading dirc.log: - notepad dirc.log - :# All generated object files, listings, executables, etc, are in - :# target-OS-specific subdirectories, like for the MsvcLibX builds above. - :# They're automatically linked with the corresponding (renamed) MsvcLibX*.lib. - -make.bat will generate WIN32 (X86) and WIN64 (AMD64) versions by default, -and put them respectively in the WIN32\ and WIN64\ subdirectories. -It will also generate a DOS version in DOS\ if MSVC 1.52 is installed. -It will also generate a WIN95 version in WIN95\ if MSVC 8 (aka. 2005) is installed. -Run `make -?` get a help screen for make.bat. - -Note that the configure.bat and make.bat scripts are actually independent of the -rest of the MsvcLibX library. They can be used to easily build any console -programs for DOS and Windows, without loading the Visual Studio GUI. - -An exe.bat script is a front-end to make.bat, saving a few characters to type: -`exe dirc` <==> `make dirc.exe` - - -Building Linux versions of the same programs --------------------------------------------- - -Install virtual machines with Linux, and give them access to the host's file -system, for example in the /c directory. -Then execute the following commands, adapting the paths as needed: - - # Go to the work directory - BASEDIR=/c/Users/YOURNAME/Documents/SRC - mkdir $BASEDIR/MyTools - cd $BASEDIR/MyTools - # Get a bash script to build Linux versions using similar directory outputs - cp $BASEDIR/MsvcLibX/src/exe . - # Make sure the Linux C compiler finds MsvcLibX debug macros, but not other MsvcLibX include files. - # Important: Do not point C_INCLUDE_PATH at MsvcLibX/include, as this directory - # contains duplicates for standard include files (Ex: stdio.h), that will fail - # to compile in Linux. - mkdir ~/include - cp $BASEDIR/MsvcLibX/include/debugm.h ~/include - export C_INCLUDE_PATH=~/include - # Make sure that variable is defined in all future sessions - echo "export C_INCLUDE_PATH=~/include" >>~/.bashrc - # Compile your sample dirc.c program (which uses MsvcLibX debug macros) - ./exe dirc - # The output will be in subdirectories such as Linux.i686/ or Linux.x86_64/. - # (The exact name depends on `echo "$(uname -s).$(uname -m)"` output) - - -Adding new files to the library -------------------------------- - -All new files should have headers and comments similar to the existing ones. - -To add a new source file (let's call it newfile.c) into the library: - -- Put newfile.c in the %MSVCLIBX%\src directory. -- Add a +newfile.obj entry to the OBJECTS list in %MSVCLIBX%\src\Files.mak -- Also add a rule with newfile.c include files dependencies in src\Files.mak. -- Rebuild the library, and make sure there are no errors nor warnings. - -To add a new include file into the library: - -- All new include file names should be standard C library include files names! -- Put it in the include or include\sys subdirectory. - -To add an include file overriding an homonym one in the MSVC library: - -- The trick is to make it include MSVC's one, then define its own extensions. - As MSVC compilers do not support the #include_next directive, I've implemented - a mechanism for including MSVC include files using their full pathname. - As an example, see include/direct.h or include/sys/stat.h. - - -The make file system --------------------- - -Microsoft Visual Studio tools contain a build tool called nmake, very similar -to, but slightly incompatible with, Unix make. -MsvcLibX's make.bat is designed to give a similar feel to Unix make. -It uses internally a number of make files: - -Make file | Description ------------ | ---------------------------------------------------------------- -NMakefile | Default make file to use, if none is specified. -DOS.mak | Generic rules for building MS-DOS programs into the DOS subdir. -WIN32.mak | Generic rules for building 32-bits Windows programs into WIN32. -WIN64.mak | Generic rules for building 64-bits Windows programs into WIN64. -WIN95.mak | Generic rules for building Windows 95 programs into WIN95. -All.mak | Generic rules for building one or more of the above. -Files.mak | Application-specific source file dependancy rules. Intended to be includable from both Windows' nmakefile and Linux' makefile. - -DOS.mak, WIN32.mak, WIN64.mak, and All.mak are pretty stable, and should not -be changed. The only likely case where a change is needed would be to add extra -libraries to link with _all_ programs. -To add a library for one particular program, it'd be better to add a specific -rule in its own make file, as described below. -In all cases, see the detailed notes in each make file header. - -Specific make files: - -To build a target called program.exe, the 5 make files with generic rules -(DOS.mak, WIN32.mak, WIN64.mak, WIN95.mak, and All.mak) first look for a -specific make file called program.mak. They include it if present. This allows -defining application-specific rules. -These rules should use macros defined in the 4 make files to specify the -build environment and parameters. Here's a list of the most useful ones: - -Macro | Description ---------- | ---------------------------------------------------------------- -T | Target OS. One of DOS, WIN32, WIN64, WIN95. -S | Path of the source files -O | Path where to put the object files -B | Path where to put the binary executable files -L | Path where to put the listings -CFLAGS | Flags for Microsoft C compiler -INCLUDE | List of C include file paths, serarated by ';' -LFLAGS | Flags for Microsoft linker -PROGRAM | The program base name, infered from the target name. -SOURCES | The list of sources for the program. Default: PROGRAM.c or .cpp -OBJECTS | The list of objects to link. Default: PROGRAM.obj - -Scripts: - -Script | Description ---------------------- | -------------------------------------------------------- -make.bat | The main build tool. Invokes nmake. -config.%HOSTNAME%.bat | Defines paths to all tools used by make. Do not edit. -configure.bat | Analyses your system, and generates a config.%HOSTNAME%.bat file. -configure.*.bat | Define user or task-specific extensions to configure.bat. -exe.bat | Front end to make.bat, generating multiple goals. -exe | Linux shell script, invoking cc with multiple goals. -src2objs.bat | Internal script used by make files to convert SOURCES to OBJECTS - -configure.bat with search for configure.*.bat scripts in %windir%, then in %HOME%, -then in the current directory. -Put configuration scripts with your global preferences in %windir% for example. -Within each directory, the files are called in the alphabetic order, allowing -to manage predictable dependancies. - -Example 1: The 2clip program has no MS-DOS version. To prevent the make system -from attempting to build a DOS version (Only necessary if you DO have MSVC 1.52 -installed), create a 2clip.mak file with this content: - - !IF "$(T)"=="DOS" - complain: - @echo>con There's no DOS version of this program. - - dirs $(O)\2clip.obj $(B)\2clip.exe: complain - @rem Do nothing - !ENDIF - -Example 2: Porting to Windows a resize.c program manipulating jpeg images, -and using the libjpeg library. Create a resize.mak file with lines like these: - - INCLUDE=$(INCLUDE);C:\JFL\SRC\Libs\libjpeg;C:\JFL\SRC\Libs\libjpeg\jpeg-8d - LFLAGS=$(LFLAGS) C:\JFL\SRC\Libs\libjpeg\$(B)\libjpeg.lib - -Example 3: Some Windows programs need to include additional resources, defined -in a .rc file. Ex: The update program uses a manifest to control its rights. -Create an update.mak file with directives like this: - - !IF "$(T)"=="WIN32" || "$(T)"=="WIN64" - SOURCES=update.c update.rc - LFLAGS=$(LFLAGS) /MANIFEST - !ENDIF - - -The debug system ----------------- - -The MsvcLibX library makes it easy to build two versions of each program: - -- A release version, small lean and fast. -- A debug version, with additional code to help developers debug the program. - -It follows Microsoft's convention of defining the macro _DEBUG when compiling -a C/C++ source for a debug build. -But it goes much further, by providing in debugm.h a set of macros to assist -debugging. - -The general principle is that by default, the debug version operates exactly -like the release version. (Except possibly for performance) -Then, if the "debug mode" is enabled, it outputs debug messages on stdout. -A major property of the MsvcLibX debugging output is that is is intended by -function call depth. This makes it considerably easier to read the debug output. - -The MsvcLibX library itself is built in debug and release versions. -The make.bat system will link the release version of your program with the -release version of the MsvcLibX library, and likewise for the debug versions. -To use it, include debugm.h in your main module, and add to your main() routine -a command-line option (-d or --debug maybe?) that calls DEBUG_ON(). -There's also an "extra debug mode", displaying even more details than the -debug mode. It is enabled by calling DEBUG_ON() twice. ==> Invoke with -d -d. - -Debug macros: - -Macro | Description ---------------------------- | ------------------------------------------------ -DEBUG_ON() | Enable the debug mode. -DEBUG_CODE(...) | The code within parentheses is only compiled in the debug version -DEBUG_PRINTF((format, ...)) | Generates a printf instruction in the debug version only, that prints only if debug mode is enabled, with the output indented by call depth. -DEBUG_ENTER((format, ...)) | Like DEBUG_PRINTF, but for use at the beginning of a function. Increases the indent level. -DEBUG_LEAVE((format, ...)) | Like DEBUG_PRINTF, but for use before returning from a function. Decreases the indent level. - -Note that every use of DEBUG_ENTER must be matched by one DEBUG_LEAVE. So if a -function has several return instructions, every return must be preceded by a -DEBUG_LEAVE. - -DEBUG_LEAVE alternatives: -To further simplify the source, a pair DEBUG_LEAVE()/return can be replaced by -one of the following macros: - -Macro | Simplified description ------------------------------ | ------------------------------------------------ -RETURN_INT(i) | DEBUG_LEAVE(("return %d\n", i)); return i; -RETURN_INT_COMMENT(i, (args)) | Idem, plus prints a comment behind the return -RETURN_BOOL(b) | DEBUG_LEAVE(("return %s\n", b?"TRUE":"FALSE")); return b; -RETURN_BOOL_COMMENT(b, (...)) | Idem, plus prints a comment behind the return -RETURN_CHAR(c) | DEBUG_LEAVE(("return %c\n", c)); return c; -RETURN_STRING(s) | DEBUG_LEAVE(("return %s\n", s)); return s; - -For all the above, the release version just does return retValue; - -Example for a recursive function factorial: - - int fact(int n) { - DEBUG_ENTER((__FUNCTION__ "(%d);\n", n)); - if (n) n *= fact(n-1); else n = 1; - RETURN_INT(n); - } - -The debug version, in debug mode, invoked with argument 4, prints: - - fact(4); - fact(3); - fact(2); - fact(1); - fact(0); - return 1; - return 1; - return 2; - return 6; - return 24; - - -Support for UTF-8 sources -------------------------- - -The MsvcLibX library supports writing C programs using 8-bit characters, -with strings encoded as UTF-8, and that will work for any cmd.exe code page. -This makes the sources much more simple and readable that using full-fledged -Unicode, with 16-bits wchar_t or WCHAR and L"strings" or _T("strings"). - -Note: The cmd.exe code page can be read and changed with the CHCP command. -The most common code pages are: - -CP | Description ------ | ---------------------------------------------------------------------------- -437 | MS-DOS OEM code page, still used by cmd.exe in US and west-European systems. -1252 | Windows "ANSI" code page, used by most GUI programs, like notepad.exe. -65001 | UTF-8 code page. Allows display any Unicode character. - -Important: Changing the code page will only work correctly if cmd.exe is using -a TrueType font. The default "Raster" font supports code page 437 only. - -To enable that UTF-8 support: - -1. Set the C or C++ source encoding to UTF-8 with BOM. (BOM = Byte-Order Mark) -Having a BOM is important, as without it some Windows editors will incorrectly -detect the encoding, and then sometimes corrupt the source. -2. Define one of the following constants in the .c source, _before_ including -any .h include files: - - #define _BSD_SOURCE 1 /* Defined by many standard BSD-Unix programs */ - #define _GNU_SOURCE 1 /* Defined by many standard GNU-Unix/Linux programs */ - #define _UTF8_SOURCE 1 /* MsvcLibX-specific */ - -Note that most modern Linux compilers do expect C sources encoded as UTF-8, -and will silently ignore the UTF-8 BOM if present. - -Internally, MsvcLibX extends Microsoft's convention of having two ANSI and Wide -versions of each routine, respectively with an 'A' and a 'W' suffix. Ex: -FindFirstFile() being an alias to either FindFirstFileA() or FindFirstFileW(). -MsvcLibX uses two additional suffixes: 'U' for the UTF-8 version, and 'M' for -the common MultiByte subroutine used by both the 'A' and 'U' versions. Ex: - -Function | Description ----------- | --------------------------------------------------------------- -readlinkW | Posix routine readlink - Wide char version -readlinkM | MultiByte char sub-routine, used by the next two routines. -readlinkA | Posix routine readlink - ANSI version -readlinkU | Posix routine readlink - UTF-8 version -readlink | Posix routine readlink - Alias to either readlinkA or readlinkU - -Note that the M version has one additional argument: The code page to use for -converting strings to and from Unicode. In that sense, it's not Posix-compliant. - -Gotcha: As of 2014-03-25, most file I/O and enumeration routines have been -restructured this way, but a few have not yet been: -scandir() and lstat() only support UTF-8 file names, not ANSI names. - -Gotcha: As of 2014-03-25, there's a potential issue with the C main() routine: -Supporting UTF-8 file names is not just supporting UTF-8 strings in library -functions. It's also necessary to process the command line, so that command line -arguments are passed in to the main() routine as UTF-8 strings. -Currently this is implemented as a macro that redefines the main token, so -that it generates a main() routine just calling a _mainU0() routine from -MsvcLibX.lib, followed by another local _mainU() routine with the body intended -for your main routine. Ex: - - int main(int argc, char *argv[]) { /* your main body */ } - -Becomes: - - int main(int argc, char *argv[]) {return _mainU0()} - int _mainU(int argc, char *argv[]) { /* your main body */ } - -The _mainU0() routine from MsvcLibX.lib reprocesses the Win32 command line as -UTF-8 argv[] arguments, then calls _mainU(argc, argv[]). -This works well and transparently, except in one case: -If one of your sources or include files contain a prototype for the main() -routine, then MsvcLibX' main macro will break that prototype, and cause -compilation and/or link errors. -If this happens, simply remove the main() prototype, which is useless anyway. - - -Support for NTFS symlinks and junctions ---------------------------------------- - -Posix defines only a single kind of symbolic links, usable for any kind of -targets, whether they're files, directories, or further symbolic links. -The standard C library defines functions for managing symbolic links: - -Function | Description ----------- | ---------------------------------------------------------------- -readlink() | Read a link target -symlink() | Create a link, or change its target -lstat() | Read the metadata (timestamp, access rights) for the link itself -lchmod() | Change the link access rights -lchown() | Change the link owner -realpath() | Generate an absolute path, with all links resolved - -Windows defines three types of links: - -Type | Description ---------- | --------------------------------------------------------------- -SYMLINK | Symbolic link to a file -SYMLINKD | Symbolic link to a directory -JUNCTION | Mount point, often used as a symbolic link to another directory - -All three types can be created in Vista and later by the mklink command. -JUNCTIONS can also be created in 2000 and later by Microsoft's SysInternal's -junction command. -One important difference is that JUNCTIONs can be created with normal user -rights, whereas SYMLINKs and SYMLINKDs require administrator rights in an -elevated command window. -A second important difference is that on networks, SYMLINKs and SYMLINKDs are -interpreted on the client side, whereas JUNCTIONs are interpreted on the server -side (despite having their target readable from the client side). - -The MsvcLibcX library tries to hide that complexity, and implements the standard -functions as if there were only one kind of link. -It also provides non-standard functions symlinkd() and junction() to -specifically create SYMLINKDs and JUNCTIONs respectively. - -Notes about readlink(): -For SYMLINKs and SYMLINKDs, the case is straightforward. -For JUNCTIONs, there are two problems: - -- The target is stored as an absolute path, even when the JUNCTION was created - with a relative path. readlink() tries to convert the absolute path to a - relative path, so that file copying utilities can clone trees with internal - JUNCTIONs. -- When the JUNCTION is on a network drive, the target refers to the absolute - path on the server. This may not be accessible from the client through the - existing mounted shares. Even when this is accessible, it's not always easy - to map the server path to a valid client path. - readlink() uses heuristics which always work for drives shared at the drive - level. Ex: a C: drive shared as C$, a D: drive shared as D$, etc. - The heuristic also very likely works for drives shared at the root or first - directory level. Ex: C:\ shared as CROOT, or C:\Public shared as PUBLIC. - As of 2014-03-25, it'll fail in any other case. Ex: - C:\Users\YOURNAME shared as YOURHOME, - or a C:\Public share mounted through its subdirectory C:\Public\Temp - -Notes about symlink(): -symlink() will attempt to create a SYMLINK or a SYMLINKD, depending on the type -of the target. This works well when the target exists. But when it does not -(which is legal), it will create a SYMLINK by default, or a SYMLINKD if the -target ends with a '/' or a '\'. Posix allows, but does not require, providing -a trailing '/', so the link type may possibly be incorrect. -junction() will have the same issues as readlink() above. - -Notes about junction(): -The problem for junction() is to convert a client-side target pathname provided -by the program running on the client, to a server-side pathname, necessary for -junctions to work. -For full-drive shares (i.e. any share named like D$), this is easy. -For non-full-drive shares (i.e. anything else), it will assume this is a -first level shared directory on the C: drive. Ex: A link on share PUBLIC -will be targeted at C:\PUBLIC. -Problem: The junction will not work if the share actually resides anywhere else -on the server. But, surprisingly, there's an easy workaround: -Simply create manually on the server itself, a junction called C:\PUBLIC, -pointing at the actual directory shared as PUBLIC. -This way the junctions created from the client side will work correctly, both -on the client and on the server side, after being resolved on the server side -through that second junction. - - -Support for Windows 95/98 -------------------------- - -The configure.bat script searches for old versions of the Visual C++ compiler, -which can build WIN32 programs that can run in Windows 95/98/ME/NT4. -The most recent of these is Visual C++ 8, from Visual Studio 2005. -The make.bat script can then use rules in win95.mak to builds WIN32 programs -that will run in any version of Windows from Windown 95 to Windows 10. - -Note however that Windows 95/98/ME only have a very limited support for Unicode -built in. The rest of this section mentions 95, but applies to 98 & ME as well. -The MsvcLibX library uses a number of Unicode functions not available in default -installations of Windows 95. This includes all file management functions. -Thus most of our WIN95 executables will not work by default in Windows 95. -To allow them to work, it is necessary to download from Microsoft a "Microsoft -Layer for Unicode on Windows 95/98/ME Systems" (MSLU for short), and install it -on the Windows 95 system. See the following links for details: -https://en.wikipedia.org/wiki/Microsoft_Layer_for_Unicode -https://msdn.microsoft.com/en-us/goglobal/bb688166.aspx -MSLU installation procedure: - -- Download the MSLU redistributable setup (unicows.exe) from: - http://go.microsoft.com/fwlink/?LinkId=14851 -- Extract unicows.dll from the unicows.exe archive. -- Copy that unicows.dll to the Windows 95 system, into %windir%\System. - -Testing WIN95 executables in a Windows 95 VM -VMWare Player does not have Windows 95 drivers for the HGFS file system. -This prevents accessing the host's files directly as network files, as is -usually done for Windows XP and later versions of Windows. -It is not possible to use network shares either, as Windows 95 only supports -the SMB 1 protocol, which is actively blocked by Windows Vista and later hosts. -It is possible, but inconvenient, to transit through a web server, and download -the files in the Windows 95 VM using Internet Explorer 4. -The best solution probably is to transit through a floppy or CD image, and -mount that image in the VM Player. Many tools, including our own HpMkIso.exe -can create CD images. A very convenient shareware called WinImage allows to -create floppy images. -Another solution is to install a Web server on the host PC, and use Internet -Explorer 4 in Windows 95 to download the executable files into the VM. - - -Support for DOS, and bound DOS+Windows programs ------------------------------------------------ - -- If Visual C++ 1.52 is installed, configure.bat will setup the make system -for building the MS-DOS version of your programs. (Output in the DOS directory) -- If Visual C++ 8 is installed, configure.bat will setup the make system -for building 32-bits versions of your programs compatible with Windows 95 -and all later versions, including 64-bits ones. (Output in the WIN95 directory) - -Both are available as part of the Visual Studio 2005 CD, still available for -download for MSDN subscribers from the MSDN web site. - -Win32 programs have an "MS-DOS stub", that runs when the Windows program is -executed under MS-DOS. -The default stub used by the Win32 linker if a tiny DOS program that displays -an error message such as: "This program can only run in Windows" - -When it builds the WIN95 or WIN32 version of a program, and when it has built -the DOS version already, the MsvcLibX make system uses that DOS version as the -stub for the Windows version. -This allows building executables that work in *all* versions of DOS and Windows! - -- When run in MS-DOS, it's the DOS stub of the exe that runs. -- When run in Windows (even 64-bits versions), it's the Win32 part that runs. - -Note that the make system will build such bound executables for any WIN32 -build made with more recent compilers. But these recent compilers generate -executables that cannot run in old versions of Windows. For example, the -Visual C++ 14 compiler can only target Windows Vista and later systems. -Having an executable that can run in DOS and Windows 7, but not in Windows XP, -is not very useful. Make sure to install the Visual C++ 8 compiler in parallel -with Visual C++ 1.52, and the latest compiler (Visual C++ 14 at the time of -this writing), to generate truly universal WIN95 builds, that run in -DOS/95/98/ME/2000/XP/Vista/7/8/10. - - -History -------- - -**1986** - -I (Jean-François Larvoire) started writing command-line tools for MS-DOS. -Some were enumerating files, using inline assembly language to make MS-DOS -interrupt 21h system calls: update, dirsize, redo, backnum, which, dirc... -To make it simple I factored these out in subroutines srch1st and srchnext, -that I manually copied from one program to the next. -Things got a bit tricky to support recursion, which was not straightforward -in MS-DOS. - - -**1992** - -We got an OS/2 server, and I started porting the above programs to OS/2. -MS-DOS was still important, so I used conditional compilation to support -both operating systems with the same source. -I wrote a second version of srch1st and srchnext for OS/2, and had to -include a new routine srchdone due to OS/2 API constraints. -dirc was the first program I ported, then I slowly duplicated the code into -the other programs. Synchronizing bug fixes became more difficult. -A nice trick was the OS/2 supported dual-mode EXEs, with both the DOS and -OS/2 version bound in the same .exe file. - - -**1995** - -We ditched the OS/2 server, and got a new one running Windows NT. -Again, I created a third conditionally compiled version of srch1st/srchnext/ -srchdone for WIN32, for use in dirc. -Manually back porting the updates and bug fixes to all programs became -really painful, and took several years. -Like OS/2, Windows supported dual-mode EXEs, and I updated my make files -to include both the DOS and Windows version in the same file. - - -**2005** - -I started working on Linux projects. Despite the bonanza of command-line -tools available, I couldn't find one equivalent to dirc. -So, yet another time, I created a fourth conditionally compiled version of -srch1st/srchnext/srchdone for Linux, and obtained a Linux version of dirc. -I also ported it and a couple of other programs to Tru64, which was the -first 64-bits OS I worked with. This exposed a few unsuspected bugs. -The thing became so complex than porting the changes and updates to the -other programs was a nightmare. I did it for a few of them, but never -got the time to port them all. - - -**2010** - -Building WIN64 versions was relatively easier due to the Tru64 precedent, -but added yet another set of conditional compilations. -The porting nightmare worsened. - - -**2012** - -I tried porting some unrelated Linux programs to Windows, and hit a wall: -Many include files and standard C library routines were missing in MSVC. -I then tried using MinGW, but it too was missing many required features! -I considered contributing updates to MinGW, but that project was too -complex and ill-documented at that time, and I eventually gave up. -Instead, I started writing my own "libc-ext" library of include files and -routines to supplement MSVC. -Ex: stdint.h, inttypes.h, fnmatch.h/fnmatch.c, ... -Also it became obvious that multiplying conditional compilations and -duplicating directory access code everywhere was wrong. Instead I had to -write standard dirent.h/dirent.c routines for DOS/OS2/Windows, and -rewrite all my programs around these standard routines. -I did it for DOS and Windows versions, and left stubs of the OS/2 versions, -for the unlikely case where somebody wants to revive them. - -**2013** - -The restructuration was over for dirc, dirsize, which, backnum, update. -The library got its final name, "MsvcLibX", to better match it purpose: -An extension of the Microsoft Visual C library, not of the standard libc. -As the library grew, debugging it became more difficult, and I decided to -use my debugm.h within the library: This file contains a set of debugging -macros I had developed for other programs. -I started thinking about sorely needed improvements: - -- There was no support for Windows symlinks and junctions. -- The Windows versions of the programs output ANSI file names into a - cmd.exe window using the OEM character set. This caused all files with - French names to be shown as garbage. -- Worse still, I had a few files and directories with non-ANSI names - (In Russian and Chinese), that caused the programs to fail! - -The Linux version of the program did not have these issues, as all recent -versions of Linux use the UTF-8 encoding, and this works transparently -even for old programs using 8-bits characters like mine. - - -**2014** - -I got a bit more time, and started working on a redesign of MsvcLibX's -dirent.c/dirent.h, and added lstat.c, symlink.c, readlink.c, etc, to -support symlinks, junctions, and UTF-8 file names. -This proved to be much more work than I initially thought, but the result -was worth the effort: dirc, update got easily adapted, with remarkably -few changes to their source. Adding recursion to update was easy. -I'm now able to use update to backup all files on my system, including -symbolic links, and junctions used as poor man's links. And the programs -now display names correctly in any code page with the right font. -A seventh program, truename, joined the collection using dirent.h routines. -Later in the year, I made significant other changes: - -- Split make.bat into a generic make.bat/configure.bat pair of scripts. - configure.bat needs to be run once, plus everytime the environment - changes. (Such as if a new MSVC version is installed.) - Project-specific definitions are moved to new files, such as our - configure.MsvcLibX.bat. -- Added a general mechanism for defining extensions to existing MSVC - include files. This allowed defining our own homonym include files, - containing just the missing definitions. Which in turn allowed to move - many such definitions, that were initially stored in unistd.h for lack - of an alternative, to their standard location. -- Updated all file functions to support WIN32 pathnames > 260 characters. - (The basic WIN32 functions that I used until then had that limitation; - I'm now using extended functions supporting path lengths up to 64K.) - -All tools like dirc, dirsize, which, backnum, update, truename, redo -benefit from that last change. - -**2015** - -A major improvement was the addition of support for new operating systems -and processor targets. This required redesigning several things: -The OS-specific definition macros have been renamed to more generic names. -The DosWin.mak file has been replaced by a more generic All.mak, -supporting the old dos.mak, win32.mak, and win64.mak, plus the new... - -- bios.mak Allows building 16-bits programs that can run in BIOS option - ROMs. These programs are based on the BiosLib library, - documented separately. -- win95.mak Allows building win32 programs that run in Windows 95. -- ia64.mak Allows building 64-bits programs that run in IA64 versions - of Windows. Not tested. -- arm.mak Allows building win32 programs for Windows RT. - Tests failed so far, due to a missing "ARM Desktop SDK". - -All.mak also skips building versions for which no tools are available; -This prevents problems in the normal case when people only have the latest -version of MSVC. In this case, it just builds the WIN32 and WIN64 versions. -The support for Windows 95 required fixing numerous issues with old -Microsoft tools and Windows SDKs. -Conversely, the support for Visual Studio 15 also required fixing lots of -issues, as the MSVC library has been completely redesigned, and split into -two parts: - -- A compiler-specific part, part of the compiler tree as before. -- A generic part, called Universal C runtime (UCRT) that's now part of - the windows SDK. - -And of course there were also issues with MsvcLibX recent improvements, -like Unicode support and symlink support, which had never been tested, -and of course did not work, in Windows 95 and XP. - - -**2016** - -Many significant changes and improvements this year: - -- Changed the UTF-8 C source support to require a UTF-8 BOM. - This is to prevent problems with Windows tools that tend to corrupt files - without a UTF-8 BOM, but with other UTF-8 characters. - This required changing all C/C++ compilation rules to first remove the - UTF-8 BOM, as MS compilers do not react correctly when they find one. -- Added a windows.h extension. - Just like Standard C library APIs, windows 8-bit APIs can be overriden to - refer to custom routines that support UTF-8 strings. - The MsvcLibX library was beginning to use a significant number of these - custom routines internally. They're now accessible directly to outside - UTF-8 programs targeting only Windows. -- Finished implementing support for the OUTDIR variable. (Started in 2015.) - All make files now optionally create output files in an %OUTDIR% directory. - This is intended for testing builds in VMs, with shared sources on the - host, but the output locally in the VM, avoiding to overwrite the main - version on the host. IF %OUTDIR% is not defined, the default output - still goes below the source directory as before. diff --git a/deps/MsvcLibX/config.DESKTOP-U79TD6T.bat b/deps/MsvcLibX/config.DESKTOP-U79TD6T.bat deleted file mode 100644 index b1b68e1540..0000000000 --- a/deps/MsvcLibX/config.DESKTOP-U79TD6T.bat +++ /dev/null @@ -1,146 +0,0 @@ -:# config.DESKTOP-U79TD6T.bat generated by configure.bat on 2020/05/19 Öܶþ 7:49:05.47 -:# -:# If changes are needeed, do not edit this file, but instead create a new script -:# called configure.YOURCHOICE.bat. This new script will be invoked automatically -:# by configure.bat while creating this file. Then your script can write extra -:# definitions, or change some of the variables before configure.bat writes them. -:# -:# Invoke configure.bat manually if anything changes in the tools config, such as -:# installing a Visual Studio update, or updating a configure.XXX.bat script. - -set "HAS_STINCLUDE=1" &:# Found the System Tools global C includes -set "STINCLUDE=C:\Users\admin\Desktop\temp\MyGitHub\C\include" &:# System Tools global C includes -set "HAS_SDK_FLAGS=/DHAS_STINCLUDE=1" &:# SDK detection flags for the C compiler - -SET "PF32=C:\Program Files (x86)" &:# 32-bits Program Files -SET "PF64=C:\Program Files" &:# 64-bits Program Files -SET "ARCH=AMD64" &:# PROCESSOR_ARCHITECTURE - -SET "MASM=" &:# Microsoft 16-bits Assembler base path -SET "MSVC=" &:# Microsoft 16-bits Visual C++ base path -SET "MAPSYM=" &:# 16-bits debugging symbols generator - -SET "VSTUDIOLONG=C:\Program Files (x86)\Microsoft Visual Studio 14.0" &:# Microsoft Visual Studio (Long path) -SET "VSTUDIO=C:\PROGRA~2\MICROS~2.0" &:# Microsoft Visual Studio (Short path) -SET "VSCOMMONLONG=C:\Program Files (x86)\Microsoft Visual Studio 14.0\Common7" &:# Microsoft Visual Studio Common Files (Long path) -SET "VSCOMMON=C:\PROGRA~2\MICROS~2.0\Common7" &:# Microsoft Visual Studio Common Files (Short path) -SET "VSIDELONG=C:\Program Files (x86)\Microsoft Visual Studio 14.0\Common7\IDE" &:# Microsoft Visual Studio IDE Files (Long path) -SET "VSIDE=C:\PROGRA~2\MICROS~2.0\Common7\IDE" &:# Microsoft Visual Studio IDE Files (Short path) -SET "VSTOOLSLONG=C:\Program Files (x86)\Microsoft Visual Studio 14.0\Common7\Tools" &:# Microsoft Visual Studio Tools (Long paths) -SET "VSTOOLS=C:\PROGRA~2\MICROS~2.0\Common7\Tools" &:# Microsoft Visual Studio Tools (Short paths) -SET "MSVC32LONG=C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC" &:# Microsoft Visual C++ 32/64 bits (Long path) -SET "MSVC32=C:\PROGRA~2\MICROS~2.0\VC" &:# Microsoft Visual C++ 32/64 bits (Short path) - -SET "WIN_CP=936" &:# Windows Code Page -SET "WIN_CS=gb2312" &:# Windows Character Set -SET "DOS_CP=936" &:# DOS Code Page -SET "DOS_CS=gb2312" &:# DOS Character Set - -SET "AS=" &:# Assembler -SET "CC=" &:# C compiler -SET "INCLUDE=C:\Users\admin\Desktop\temp\MyGitHub\C\include" &:# Include paths. Define USER_INCLUDE if needed. -SET "LK=" &:# Linker -SET "LIB=" &:# Libraries paths. Define USER_LIBS if needed. -SET "LB=" &:# Library manager -SET "RC=" &:# Resource compiler -SET "MT=" &:# Manifest tool - -SET "DOS_CC=" &:# Microsoft Visual C++ 16-bits compiler -SET "DOS_AS=" &:# Microsoft 16-bits assembler -SET "DOS_LK=" &:# Microsoft 16-bits linker -SET "DOS_LB=" &:# Microsoft 16-bits librarian -SET "DOS_RC=" &:# Microsoft 16-bits resource compiler -SET "DOS_MT=" &:# Microsoft 16-bits manifest tool -SET "DOS_PATH=;C:\Windows\System32;C:\Windows" &:# All tools paths for 16-bits compilation -SET "DOS_VCINC=" &:# Visual C++ 16-bits compiler include dir for MsvcLibX include_next -SET "DOS_CRTINC=" &:# Visual C++ 16-bits CRT library include dir for MsvcLibX include_next -SET "DOS_INCPATH=" &:# Include paths for 16-bits compilation -SET "DOS_LIBPATH=" &:# Libraries paths for 16-bits linking -SET "DOS_WINSDK=" &:# Microsoft Windows 16-bits SDK -SET "DOS_WINSDKINC=" &:# Microsoft Windows 16-bits SDK Include directory - -SET "WIN95_CC=" &:# Microsoft Visual C++ 32-bits compiler -SET "WIN95_AS=" &:# Microsoft 32-bits assembler -SET "WIN95_LK=" &:# Microsoft 32-bits linker -SET "WIN95_LB=" &:# Microsoft 32-bits librarian -SET "WIN95_RC=" &:# Microsoft 32-bits resource compiler -SET "WIN95_MT=" &:# Microsoft 32-bits manifest tool -SET "WIN95_PATH=;C:\Windows\System32;C:\Windows" &:# All tools paths for 32-bits compilation -SET "WIN95_VCINC=" &:# Visual C++ 32-bits compiler include dir for MsvcLibX include_next -SET "WIN95_CRTINC=" &:# Visual C++ 32-bits CRT library include dir for MsvcLibX include_next -SET "WIN95_INCPATH=" &:# Include paths for 32-bits compilation -SET "WIN95_LIBPATH=" &:# Libraries paths for 32-bits linking -SET "WIN95_WINSDK=" &:# Microsoft Windows 32-bits SDK -SET "WIN95_WINSDKINC=" &:# Microsoft Windows 32-bits SDK Include directory - -SET "WIN32_CC="C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\CL.EXE"" &:# Microsoft Visual C++ 32-bits compiler -SET "WIN32_AS="C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\ML.EXE"" &:# Microsoft 32-bits assembler -SET "WIN32_LK="C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\LINK.EXE"" &:# Microsoft 32-bits linker -SET "WIN32_LB="C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\LIB.EXE"" &:# Microsoft 32-bits librarian -SET "WIN32_RC=" &:# Microsoft 32-bits resource compiler -SET "WIN32_MT=" &:# Microsoft 32-bits manifest tool -SET "WIN32_PATH=C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin;C:\Program Files (x86)\Microsoft Visual Studio 14.0\Common7\IDE;C:\Program Files (x86)\Microsoft Visual Studio 14.0\Common7\Tools;C:\Windows\System32;C:\Windows" &:# All tools paths for 32-bits compilation -SET "WIN32_VCINC=C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\include" &:# Visual C++ 32-bits compiler include dir for MsvcLibX include_next -SET "WIN32_CRTINC=C:\Program Files (x86)\Windows Kits\10\Include\10.0.18362.0\ucrt" &:# Visual C++ 32-bits CRT library include dir for MsvcLibX include_next -SET "WIN32_INCPATH=C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\include;:\Program Files (x86)\Windows Kits\10\Include\10.0.17763.0\ucrt;C:\Program Files (x86)\Windows Kits\10\Include\10.0.17763.0\shared;C:\Program Files (x86)\Windows Kits\10\Include\10.0.17763.0\um;C:\Program Files (x86)\Windows Kits\10\Include\10.0.17763.0\winrt;C:\Program Files (x86)\Windows Kits\10\Include\10.0.18362.0\ucrt;C:\Program Files (x86)\Windows Kits\10\Include\10.0.18362.0\shared;C:\Program Files (x86)\Windows Kits\10\Include\10.0.18362.0\um;C:\Program Files (x86)\Windows Kits\10\Include\10.0.18362.0\winrt" &:# Include paths for 32-bits compilation -SET "WIN32_LIBPATH=C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\lib;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.18362.0\um\x86;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.10240.0\ucrt\x86;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.17763.0\ucrt\x86;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.18362.0\ucrt\x86;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.10240.0\ucrt\x86;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.17763.0\ucrt\x86;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.18362.0\ucrt\x86;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.10240.0\ucrt\x86;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.17763.0\ucrt\x86;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.18362.0\ucrt\x86;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.10240.0\ucrt\x86;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.17763.0\ucrt\x86;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.18362.0\ucrt\x86;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.10240.0\ucrt\x86;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.17763.0\ucrt\x86;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.18362.0\ucrt\x86;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.10240.0\ucrt\x86;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.17763.0\ucrt\x86;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.18362.0\ucrt\x86;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.10240.0\ucrt\x86;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.17763.0\ucrt\x86;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.18362.0\ucrt\x86;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.10240.0\ucrt\x86;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.17763.0\ucrt\x86;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.18362.0\ucrt\x86" &:# Libraries paths for 32-bits linking -SET "WIN32_WINSDK=C:\Program Files (x86)\Windows Kits\10" &:# Microsoft Windows 32-bits SDK -SET "WIN32_WINSDKINC=C:\Program Files (x86)\Windows Kits\10\Include\10.0.18362.0" &:# Microsoft Windows 32-bits SDK Include directory - -SET "IA64_CC=" &:# Microsoft Visual C++ IA64 compiler -SET "IA64_AS=" &:# Microsoft IA64 assembler -SET "IA64_LK=" &:# Microsoft IA64 linker -SET "IA64_LB=" &:# Microsoft IA64 librarian -SET "IA64_RC=" &:# Microsoft IA64 resource compiler -SET "IA64_MT=" &:# Microsoft IA64 manifest tool -SET "IA64_PATH=;C:\Windows\System32;C:\Windows" &:# All tools paths for IA64 compilation -SET "IA64_VCINC=" &:# Visual C++ IA64 compiler include dir for MsvcLibX include_next -SET "IA64_CRTINC=" &:# Visual C++ IA64 CRT library include dir for MsvcLibX include_next -SET "IA64_INCPATH=" &:# Include paths for IA64 compilation -SET "IA64_LIBPATH=" &:# Libraries paths for IA64 linking -SET "IA64_WINSDK=" &:# Microsoft Windows IA64 SDK -SET "IA64_WINSDKINC=" &:# Microsoft Windows IA64 SDK Include directory - -SET "WIN64_CC="C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\amd64\CL.EXE"" &:# Microsoft Visual C++ 64-bits compiler -SET "WIN64_AS="C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\amd64\ML64.EXE"" &:# Microsoft 64-bits assembler -SET "WIN64_LK="C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\amd64\LINK.EXE"" &:# Microsoft 64-bits linker -SET "WIN64_LB="C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\amd64\LIB.EXE"" &:# Microsoft 64-bits librarian -SET "WIN64_RC=" &:# Microsoft 64-bits resource compiler -SET "WIN64_MT=" &:# Microsoft 64-bits manifest tool -SET "WIN64_PATH=C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\amd64;C:\Program Files (x86)\Microsoft Visual Studio 14.0\Common7\IDE;C:\Program Files (x86)\Microsoft Visual Studio 14.0\Common7\Tools;C:\Windows\System32;C:\Windows" &:# All tools paths for 64-bits compilation -SET "WIN64_VCINC=C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\include" &:# Visual C++ 64-bits compiler include dir for MsvcLibX include_next -SET "WIN64_CRTINC=C:\Program Files (x86)\Windows Kits\10\Include\10.0.18362.0\ucrt" &:# Visual C++ 64-bits CRT library include dir for MsvcLibX include_next -SET "WIN64_INCPATH=C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\include;:\Program Files (x86)\Windows Kits\10\Include\10.0.17763.0\ucrt;C:\Program Files (x86)\Windows Kits\10\Include\10.0.17763.0\shared;C:\Program Files (x86)\Windows Kits\10\Include\10.0.17763.0\um;C:\Program Files (x86)\Windows Kits\10\Include\10.0.17763.0\winrt;C:\Program Files (x86)\Windows Kits\10\Include\10.0.18362.0\ucrt;C:\Program Files (x86)\Windows Kits\10\Include\10.0.18362.0\shared;C:\Program Files (x86)\Windows Kits\10\Include\10.0.18362.0\um;C:\Program Files (x86)\Windows Kits\10\Include\10.0.18362.0\winrt" &:# Include paths for 64-bits compilation -SET "WIN64_LIBPATH=C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\lib\amd64;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.18362.0\um\x64;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.10240.0\ucrt\x64;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.17763.0\ucrt\x64;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.18362.0\ucrt\x64;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.10240.0\ucrt\x64;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.17763.0\ucrt\x64;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.18362.0\ucrt\x64;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.10240.0\ucrt\x64;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.17763.0\ucrt\x64;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.18362.0\ucrt\x64;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.10240.0\ucrt\x64;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.17763.0\ucrt\x64;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.18362.0\ucrt\x64;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.10240.0\ucrt\x64;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.17763.0\ucrt\x64;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.18362.0\ucrt\x64;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.10240.0\ucrt\x64;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.17763.0\ucrt\x64;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.18362.0\ucrt\x64;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.10240.0\ucrt\x64;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.17763.0\ucrt\x64;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.18362.0\ucrt\x64;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.10240.0\ucrt\x64;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.17763.0\ucrt\x64;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.18362.0\ucrt\x64" &:# Libraries paths for 64-bits linking -SET "WIN64_WINSDK=C:\Program Files (x86)\Windows Kits\10" &:# Microsoft Windows 64-bits SDK -SET "WIN64_WINSDKINC=C:\Program Files (x86)\Windows Kits\10\Include\10.0.18362.0" &:# Microsoft Windows 64-bits SDK Include directory - -SET "ARM_CC="C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\amd64_arm\CL.EXE"" &:# Microsoft Visual C++ ARM compiler -SET "ARM_AS="C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\amd64_arm\ARMASM.EXE"" &:# Microsoft ARM assembler -SET "ARM_LK="C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\amd64_arm\LINK.EXE"" &:# Microsoft ARM linker -SET "ARM_LB="C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\amd64_arm\LIB.EXE"" &:# Microsoft ARM librarian -SET "ARM_RC=" &:# Microsoft ARM resource compiler -SET "ARM_MT=" &:# Microsoft ARM manifest tool -SET "ARM_PATH=C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\amd64_arm;C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\amd64;C:\Program Files (x86)\Microsoft Visual Studio 14.0\Common7\IDE;C:\Program Files (x86)\Microsoft Visual Studio 14.0\Common7\Tools;C:\Windows\System32;C:\Windows" &:# All tools paths for ARM compilation -SET "ARM_VCINC=C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\include" &:# Visual C++ ARM compiler include dir for MsvcLibX include_next -SET "ARM_CRTINC=C:\Program Files (x86)\Windows Kits\10\Include\10.0.18362.0\ucrt" &:# Visual C++ ARM CRT library include dir for MsvcLibX include_next -SET "ARM_INCPATH=C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\include;:\Program Files (x86)\Windows Kits\10\Include\10.0.17763.0\ucrt;C:\Program Files (x86)\Windows Kits\10\Include\10.0.17763.0\shared;C:\Program Files (x86)\Windows Kits\10\Include\10.0.17763.0\um;C:\Program Files (x86)\Windows Kits\10\Include\10.0.17763.0\winrt;C:\Program Files (x86)\Windows Kits\10\Include\10.0.18362.0\ucrt;C:\Program Files (x86)\Windows Kits\10\Include\10.0.18362.0\shared;C:\Program Files (x86)\Windows Kits\10\Include\10.0.18362.0\um;C:\Program Files (x86)\Windows Kits\10\Include\10.0.18362.0\winrt" &:# Include paths for ARM compilation -SET "ARM_LIBPATH=C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\lib\arm;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.18362.0\um\arm;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.10240.0\ucrt\arm;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.17763.0\ucrt\arm;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.18362.0\ucrt\arm;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.10240.0\ucrt\arm;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.17763.0\ucrt\arm;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.18362.0\ucrt\arm;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.10240.0\ucrt\arm;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.17763.0\ucrt\arm;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.18362.0\ucrt\arm;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.10240.0\ucrt\arm;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.17763.0\ucrt\arm;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.18362.0\ucrt\arm;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.10240.0\ucrt\arm;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.17763.0\ucrt\arm;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.18362.0\ucrt\arm;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.10240.0\ucrt\arm;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.17763.0\ucrt\arm;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.18362.0\ucrt\arm;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.10240.0\ucrt\arm;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.17763.0\ucrt\arm;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.18362.0\ucrt\arm;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.10240.0\ucrt\arm;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.17763.0\ucrt\arm;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.18362.0\ucrt\arm" &:# Libraries paths for ARM linking -SET "ARM_WINSDK=C:\Program Files (x86)\Windows Kits\10" &:# Microsoft Windows ARM SDK -SET "ARM_WINSDKINC=C:\Program Files (x86)\Windows Kits\10\Include\10.0.18362.0" &:# Microsoft Windows ARM SDK Include directory - -SET "ARM64_CC=" &:# Microsoft Visual C++ ARM64 compiler -SET "ARM64_AS=" &:# Microsoft ARM64 assembler -SET "ARM64_LK=" &:# Microsoft ARM64 linker -SET "ARM64_LB=" &:# Microsoft ARM64 librarian -SET "ARM64_RC=" &:# Microsoft ARM64 resource compiler -SET "ARM64_MT=" &:# Microsoft ARM64 manifest tool -SET "ARM64_PATH=;C:\Windows\System32;C:\Windows" &:# All tools paths for ARM64 compilation -SET "ARM64_VCINC=" &:# Visual C++ ARM64 compiler include dir for MsvcLibX include_next -SET "ARM64_CRTINC=" &:# Visual C++ ARM64 CRT library include dir for MsvcLibX include_next -SET "ARM64_INCPATH=" &:# Include paths for ARM64 compilation -SET "ARM64_LIBPATH=" &:# Libraries paths for ARM64 linking -SET "ARM64_WINSDK=" &:# Microsoft Windows ARM64 SDK -SET "ARM64_WINSDKINC=" &:# Microsoft Windows ARM64 SDK Include directory - -exit /b 0 &:# Configuration done successfully diff --git a/deps/MsvcLibX/configure.bat b/deps/MsvcLibX/configure.bat deleted file mode 100644 index 7037eb92af..0000000000 --- a/deps/MsvcLibX/configure.bat +++ /dev/null @@ -1,35 +0,0 @@ -@echo off -:#***************************************************************************** -:# * -:# Filename: configure.bat * -:# * -:# Description: Detect system-specific settings and create config.*.bat * -:# * -:# Notes: Proxy script for %STINCLUDE%\configure.bat. * -:# * -:# Make any change needed in %STINCLUDE%\configure.bat. * -:# * -:# History: * -:# 2016-10-10 JFL jf.larvoire@hpe.com created this file. * -:# 2016-12-15 JFL Search for the real make.bat in [.|..|../..]\include. * -:# * -:# © Copyright 2016 Hewlett Packard Enterprise Development LP * -:# Licensed under the Apache 2.0 license www.apache.org/licenses/LICENSE-2.0 * -:#***************************************************************************** - -:# Get the full pathname of the STINCLUDE library directory -if defined STINCLUDE if not exist "%STINCLUDE%\make.bat" set "STINCLUDE=" &:# Allow overriding with another alias name, but ignore invalid overrides -for %%p in (. .. ..\..) do if not defined STINCLUDE if exist %%p\include\make.bat ( :# Default: Search it the current directory, and 2 levels above. - for /f "delims=" %%d in ('"pushd %%p\include & cd & popd"') do SET "STINCLUDE=%%d" -) -if not defined STINCLUDE ( :# Try getting the copy in the master environment - for /f "tokens=3" %%v in ('reg query "HKCU\Environment" /v STINCLUDE 2^>NUL') do set "STINCLUDE=%%v" -) - -if not exist %STINCLUDE%\make.bat ( - >&2 echo %0 Error: Cannot find SysToolsLib's global C include directory. Please define variable STINCLUDE. - exit /b 1 -) - -if [%1]==[-d] echo "%STINCLUDE%\configure.bat" %* -"%STINCLUDE%\configure.bat" %* diff --git a/deps/MsvcLibX/include/debugm.h b/deps/MsvcLibX/include/msvcDebugm.h similarity index 100% rename from deps/MsvcLibX/include/debugm.h rename to deps/MsvcLibX/include/msvcDebugm.h diff --git a/deps/MsvcLibX/include/direct.h b/deps/MsvcLibX/include/msvcDirect.h similarity index 91% rename from deps/MsvcLibX/include/direct.h rename to deps/MsvcLibX/include/msvcDirect.h index 0ffbe1e5f4..31017ce50c 100644 --- a/deps/MsvcLibX/include/direct.h +++ b/deps/MsvcLibX/include/msvcDirect.h @@ -19,7 +19,7 @@ #include "msvclibx.h" -#include UCRT_INCLUDE_FILE(direct.h) /* Include MSVC's own file */ +#include /* Include MSVC's own file */ #undef mkdir /* This MSVC macro is incompatible with mkdir() function in unistd.h */ diff --git a/deps/MsvcLibX/include/dirent.h b/deps/MsvcLibX/include/msvcDirent.h similarity index 100% rename from deps/MsvcLibX/include/dirent.h rename to deps/MsvcLibX/include/msvcDirent.h diff --git a/deps/MsvcLibX/include/error.h b/deps/MsvcLibX/include/msvcError.h similarity index 100% rename from deps/MsvcLibX/include/error.h rename to deps/MsvcLibX/include/msvcError.h diff --git a/deps/MsvcLibX/include/fadvise.h b/deps/MsvcLibX/include/msvcFadvise.h similarity index 100% rename from deps/MsvcLibX/include/fadvise.h rename to deps/MsvcLibX/include/msvcFadvise.h diff --git a/deps/MsvcLibX/include/fcntl.h b/deps/MsvcLibX/include/msvcFcntl.h similarity index 92% rename from deps/MsvcLibX/include/fcntl.h rename to deps/MsvcLibX/include/msvcFcntl.h index 0c4a97df8f..fc981b5fc9 100644 --- a/deps/MsvcLibX/include/fcntl.h +++ b/deps/MsvcLibX/include/msvcFcntl.h @@ -18,7 +18,7 @@ #include "msvclibx.h" -#include UCRT_INCLUDE_FILE(fcntl.h) /* Include MSVC's own file */ +#include /* Include MSVC's own file */ /* Microsoft defines _open() in io.h */ #include diff --git a/deps/MsvcLibX/include/fnmatch.h b/deps/MsvcLibX/include/msvcFnmatch.h similarity index 100% rename from deps/MsvcLibX/include/fnmatch.h rename to deps/MsvcLibX/include/msvcFnmatch.h diff --git a/deps/MsvcLibX/include/getopt.h b/deps/MsvcLibX/include/msvcGetopt.h similarity index 100% rename from deps/MsvcLibX/include/getopt.h rename to deps/MsvcLibX/include/msvcGetopt.h diff --git a/deps/MsvcLibX/include/iconv.h b/deps/MsvcLibX/include/msvcIconv.h similarity index 100% rename from deps/MsvcLibX/include/iconv.h rename to deps/MsvcLibX/include/msvcIconv.h diff --git a/deps/MsvcLibX/include/inttypes.h b/deps/MsvcLibX/include/msvcInttypes.h similarity index 100% rename from deps/MsvcLibX/include/inttypes.h rename to deps/MsvcLibX/include/msvcInttypes.h diff --git a/deps/MsvcLibX/include/libgen.h b/deps/MsvcLibX/include/msvcLibgen.h similarity index 100% rename from deps/MsvcLibX/include/libgen.h rename to deps/MsvcLibX/include/msvcLibgen.h diff --git a/deps/MsvcLibX/include/limits.h b/deps/MsvcLibX/include/msvcLimits.h similarity index 94% rename from deps/MsvcLibX/include/limits.h rename to deps/MsvcLibX/include/msvcLimits.h index b26d935ac8..496fe9f24c 100644 --- a/deps/MsvcLibX/include/limits.h +++ b/deps/MsvcLibX/include/msvcLimits.h @@ -18,7 +18,7 @@ #include "msvclibx.h" -#include MSVC_INCLUDE_FILE(limits.h) /* Include MSVC's own file */ +#include /* Include MSVC's own file */ /************************ MS-DOS-specific definitions ************************/ diff --git a/deps/MsvcLibX/include/netdb.h b/deps/MsvcLibX/include/msvcNetdb.h similarity index 100% rename from deps/MsvcLibX/include/netdb.h rename to deps/MsvcLibX/include/msvcNetdb.h diff --git a/deps/MsvcLibX/include/process.h b/deps/MsvcLibX/include/msvcProcess.h similarity index 92% rename from deps/MsvcLibX/include/process.h rename to deps/MsvcLibX/include/msvcProcess.h index 60eee46b2f..66cf94de14 100644 --- a/deps/MsvcLibX/include/process.h +++ b/deps/MsvcLibX/include/msvcProcess.h @@ -19,7 +19,7 @@ #include "msvclibx.h" -#include UCRT_INCLUDE_FILE(process.h) /* Include MSVC's own file */ +#include /* Include MSVC's own file */ #if defined(_WIN32) extern intptr_t _spawnvpU(int iMode, const char *pszCommand, char *const *argv); diff --git a/deps/MsvcLibX/include/regex.h b/deps/MsvcLibX/include/msvcRegex.h similarity index 100% rename from deps/MsvcLibX/include/regex.h rename to deps/MsvcLibX/include/msvcRegex.h diff --git a/deps/MsvcLibX/include/reparsept.h b/deps/MsvcLibX/include/msvcReparsept.h similarity index 100% rename from deps/MsvcLibX/include/reparsept.h rename to deps/MsvcLibX/include/msvcReparsept.h diff --git a/deps/MsvcLibX/include/stdbool.h b/deps/MsvcLibX/include/msvcStdbool.h similarity index 100% rename from deps/MsvcLibX/include/stdbool.h rename to deps/MsvcLibX/include/msvcStdbool.h diff --git a/deps/MsvcLibX/include/stdint.h b/deps/MsvcLibX/include/msvcStdint.h similarity index 100% rename from deps/MsvcLibX/include/stdint.h rename to deps/MsvcLibX/include/msvcStdint.h diff --git a/deps/MsvcLibX/include/stdio.h b/deps/MsvcLibX/include/msvcStdio.h similarity index 95% rename from deps/MsvcLibX/include/stdio.h rename to deps/MsvcLibX/include/msvcStdio.h index a609149982..65af04340d 100644 --- a/deps/MsvcLibX/include/stdio.h +++ b/deps/MsvcLibX/include/msvcStdio.h @@ -20,7 +20,7 @@ #include "msvclibx.h" -#include UCRT_INCLUDE_FILE(stdio.h) /* Include MSVC's own file */ +#include /* Include MSVC's own file */ #include diff --git a/deps/MsvcLibX/include/stdlib.h b/deps/MsvcLibX/include/msvcStdlib.h similarity index 93% rename from deps/MsvcLibX/include/stdlib.h rename to deps/MsvcLibX/include/msvcStdlib.h index 1db1e044ea..b9bcc83f2d 100644 --- a/deps/MsvcLibX/include/stdlib.h +++ b/deps/MsvcLibX/include/msvcStdlib.h @@ -18,7 +18,7 @@ #include "msvclibx.h" -#include UCRT_INCLUDE_FILE(stdlib.h) /* Include MSVC's own file */ +#include /* Include MSVC's own file */ #ifdef __cplusplus extern "C" { diff --git a/deps/MsvcLibX/include/system.h b/deps/MsvcLibX/include/msvcSystem.h similarity index 100% rename from deps/MsvcLibX/include/system.h rename to deps/MsvcLibX/include/msvcSystem.h diff --git a/deps/MsvcLibX/include/time.h b/deps/MsvcLibX/include/msvcTime.h similarity index 87% rename from deps/MsvcLibX/include/time.h rename to deps/MsvcLibX/include/msvcTime.h index afe0769b52..1897da5857 100644 --- a/deps/MsvcLibX/include/time.h +++ b/deps/MsvcLibX/include/msvcTime.h @@ -19,8 +19,8 @@ #include "msvclibx.h" -#include WINSDK_INCLUDE_FILE(winsock2.h) -#include UCRT_INCLUDE_FILE(time.h) /* Include MSVC's own file */ +#include +#include /* Include MSVC's own file */ #ifdef _MSDOS @@ -33,7 +33,7 @@ #ifdef _WIN32 -#include "sys\time.h" /* for struct timespec */ +#include "sys\msvcTime.h" /* for struct timespec */ typedef int clockid_t; /* Supported values for clockid_t */ diff --git a/deps/MsvcLibX/include/unistd.h b/deps/MsvcLibX/include/msvcUnistd.h similarity index 96% rename from deps/MsvcLibX/include/unistd.h rename to deps/MsvcLibX/include/msvcUnistd.h index 0058c50173..9ad60625e0 100644 --- a/deps/MsvcLibX/include/unistd.h +++ b/deps/MsvcLibX/include/msvcUnistd.h @@ -30,8 +30,8 @@ #endif #include "msvclibx.h" /* Generate a library search record to load MsvcLibX.lib. */ -#include "sys/types.h" /* Define pid_t and getppid(). */ -#include "dirent.h" /* Define pid_t and getppid(). */ +#include "sys/msvcTypes.h" /* Define pid_t and getppid(). */ +#include "msvcDirent.h" /* Define pid_t and getppid(). */ #include /* For functions like _chdir() and _getcwd() */ #include /* For _getpid() */ diff --git a/deps/MsvcLibX/include/utime.h b/deps/MsvcLibX/include/msvcUtime.h similarity index 100% rename from deps/MsvcLibX/include/utime.h rename to deps/MsvcLibX/include/msvcUtime.h diff --git a/deps/MsvcLibX/include/windows.h b/deps/MsvcLibX/include/msvcWindows.h similarity index 93% rename from deps/MsvcLibX/include/windows.h rename to deps/MsvcLibX/include/msvcWindows.h index 564e1ea2fc..1c2d72554b 100644 --- a/deps/MsvcLibX/include/windows.h +++ b/deps/MsvcLibX/include/msvcWindows.h @@ -20,7 +20,7 @@ #ifdef _WIN32 -#include WINSDK_INCLUDE_FILE(windows.h) /* Include Windows SDK's own windows.h */ +#include /* Include Windows SDK's own windows.h */ /****************** Define UTF-8 versions of WIN32 routines ******************/ diff --git a/deps/MsvcLibX/include/xfreopen.h b/deps/MsvcLibX/include/msvcXfreopen.h similarity index 100% rename from deps/MsvcLibX/include/xfreopen.h rename to deps/MsvcLibX/include/msvcXfreopen.h diff --git a/deps/MsvcLibX/include/stdio--.h b/deps/MsvcLibX/include/stdio--.h deleted file mode 100644 index 54cc883248..0000000000 --- a/deps/MsvcLibX/include/stdio--.h +++ /dev/null @@ -1,17 +0,0 @@ -/*****************************************************************************\ -* * -* Filename: stdio--.h * -* * -* Description: front-end to stdio.h that redefines some unsafe functions * -* * -* Notes: This header is part of the GNU CoreUtils library. * -* msvclibx: Pass through to the standard stdio.h. * -* * -* History: * -* 2012-10-17 JFL Created this file. * -* * -* Copyright 2016 Hewlett Packard Enterprise Development LP * -* Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * -\*****************************************************************************/ - -#include diff --git a/deps/MsvcLibX/include/sys/param.h b/deps/MsvcLibX/include/sys/msvcParam.h similarity index 100% rename from deps/MsvcLibX/include/sys/param.h rename to deps/MsvcLibX/include/sys/msvcParam.h diff --git a/deps/MsvcLibX/include/sys/stat.h b/deps/MsvcLibX/include/sys/msvcStat.h similarity index 96% rename from deps/MsvcLibX/include/sys/stat.h rename to deps/MsvcLibX/include/sys/msvcStat.h index a3b039aa01..b2b82d0be8 100644 --- a/deps/MsvcLibX/include/sys/stat.h +++ b/deps/MsvcLibX/include/sys/msvcStat.h @@ -28,11 +28,11 @@ #include "msvclibx.h" -#include -#include UCRT_INCLUDE_FILE(sys\stat.h) /* Include MSVC's own file */ -#include /* For dirent2stat() arguments definitions */ -#include /* for time_t definition */ -#include /* for timespec definition */ +#include "sys/msvcTypes.h" +#include /* Include MSVC's own file */ +#include "msvcDirent.h" /* For dirent2stat() arguments definitions */ +#include "msvcTime.h" /* for time_t definition */ +#include "sys/msvcTime.h" /* for timespec definition */ /* Include MsvcLibX's override, to avoid conflict with the standard mkdir defined here, should be manually included later on in the C source */ #include @@ -154,7 +154,7 @@ extern char *Filetime2String(uint16_t date, uint16_t time, char *pBuf, size_t nB if we were to use MsvcLibX extended stat structures and routines */ #define _LIBX_stat _CONCAT(_MSVC_stat,_ns) #define _LIBX_stat64 _CONCAT(_MSVC_stat64,_ns) -#include "debugm.h" +#include "msvcDebugm.h" #pragma message("Defining type struct " VALUEIZE(_LIBX_stat)) struct _LIBX_stat { /* MSVC standard stat structure fields */ diff --git a/deps/MsvcLibX/include/sys/time.h b/deps/MsvcLibX/include/sys/msvcTime.h similarity index 100% rename from deps/MsvcLibX/include/sys/time.h rename to deps/MsvcLibX/include/sys/msvcTime.h diff --git a/deps/MsvcLibX/include/sys/utsname.h b/deps/MsvcLibX/include/sys/msvcTsname.h similarity index 100% rename from deps/MsvcLibX/include/sys/utsname.h rename to deps/MsvcLibX/include/sys/msvcTsname.h diff --git a/deps/MsvcLibX/include/sys/types.h b/deps/MsvcLibX/include/sys/msvcTypes.h similarity index 94% rename from deps/MsvcLibX/include/sys/types.h rename to deps/MsvcLibX/include/sys/msvcTypes.h index 4057e031e1..bde2ac9a8a 100644 --- a/deps/MsvcLibX/include/sys/types.h +++ b/deps/MsvcLibX/include/sys/msvcTypes.h @@ -20,7 +20,7 @@ #include "msvclibx.h" -#include UCRT_INCLUDE_FILE(sys\types.h) /* Include MSVC's own file */ +#include /* Include MSVC's own file */ /************************ MS-DOS-specific definitions ************************/ @@ -90,6 +90,7 @@ typedef int uid_t; /* MsvcLibX uses mode_t in sys/stat.h */ typedef int mode_t; +#define HAVE_MODE_T /* File link counts type (not used by MsvcLibX so far) */ typedef int nlink_t; /* Is short in some Unix versions */ diff --git a/deps/MsvcLibX/make.bat b/deps/MsvcLibX/make.bat deleted file mode 100644 index 7a27256122..0000000000 --- a/deps/MsvcLibX/make.bat +++ /dev/null @@ -1,35 +0,0 @@ -@echo off -:#***************************************************************************** -:# * -:# Filename: make.bat * -:# * -:# Description: Build DOS and Windows targets * -:# * -:# Notes: Proxy script for %STINCLUDE%\make.bat. * -:# * -:# If any change is needed, put it in %STINCLUDE%\make.bat. * -:# * -:# History: * -:# 2016-10-10 JFL jf.larvoire@hpe.com created this file. * -:# 2016-12-15 JFL Search for the real make.bat in [.|..|../..]\include. * -:# * -:# © Copyright 2016 Hewlett Packard Enterprise Development LP * -:# Licensed under the Apache 2.0 license www.apache.org/licenses/LICENSE-2.0 * -:#***************************************************************************** - -:# Get the full pathname of the STINCLUDE library directory -if defined STINCLUDE if not exist "%STINCLUDE%\make.bat" set "STINCLUDE=" &:# Allow overriding with another alias name, but ignore invalid overrides -for %%p in (. .. ..\..) do if not defined STINCLUDE if exist %%p\include\make.bat ( :# Default: Search it the current directory, and 2 levels above. - for /f "delims=" %%d in ('"pushd %%p\include & cd & popd"') do SET "STINCLUDE=%%d" -) -if not defined STINCLUDE ( :# Try getting the copy in the master environment - for /f "tokens=3" %%v in ('reg query "HKCU\Environment" /v STINCLUDE 2^>NUL') do set "STINCLUDE=%%v" -) - -if not exist %STINCLUDE%\make.bat ( - >&2 echo %0 Error: Cannot find SysToolsLib's global C include directory. Please define variable STINCLUDE. - exit /b 1 -) - -if [%1]==[-d] echo "%STINCLUDE%\make.bat" %* -"%STINCLUDE%\make.bat" %* diff --git a/deps/MsvcLibX/src/Files.mak b/deps/MsvcLibX/src/Files.mak deleted file mode 100644 index b4868a2b5b..0000000000 --- a/deps/MsvcLibX/src/Files.mak +++ /dev/null @@ -1,261 +0,0 @@ -############################################################################### -# # -# File name Files.mak # -# # -# Description MsvcLibX Specific file dependancies # -# # -# Notes # -# # -# History # -# 2012-10-21 JFL Initial version # -# 2013-03-27 JFL Added debugv.obj and getppid.obj. # -# 2014-02-03 JFL Added readlink.obj. # -# 2014-02-05 JFL Added symlink.obj. # -# 2014-02-06 JFL Added lstat*.obj. # -# 2014-02-10 JFL Added realpath.obj. # -# 2014-02-17 JFL Added err2errno.obj. # -# 2014-02-26 JFL Added filetime.obj. # -# 2014-02-27 JFL Added iconv.obj. # -# 2014-02-28 JFL Added chdir.obj and getcwd.obj. # -# 2014-03-04 JFL Added fopen.obj. # -# 2014-03-06 JFL Added strerror.obj. # -# 2014-03-24 JFL Added access.obj. # -# 2014-03-27 JFL Added spawn.obj. # -# 2014-05-30 JFL Moved here the OBJECTS macro definition from NMakeFile. # -# Added uname.obj and utimes.obj. # -# 2014-06-04 JFL Added clock_gettime.obj and gettimeofday.obj. # -# 2014-06-24 JFL Added fstat64.obj and fstat64i32.obj. # -# 2014-07-01 JFL Added mb2wpath.obj. # -# 2016-09-08 JFL Added basename.obj and dirname.obj. # -# 2016-09-12 JFL Added WIN32_OBJECTS, and several WIN32 UTF-8 routines. # -# 2016-10-11 JFL moved debugm.h to SysToolsLib global C include dir. # -# 2017-02-16 JFL Added open.obj. # -# # -# Copyright 2016 Hewlett Packard Enterprise Development LP # -# Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 # -############################################################################### - -# List of object files to build and include in the MsvcLibX library -# IMPORTANT NOTE: Every time you add an object file in the list here, also -# store its specific source file dependancies below. -OBJECTS = \ - +access.obj \ - +basename.obj \ - +chdir.obj \ - +clock_gettime.obj \ - +debugv.obj \ - +dirent.obj \ - +dirname.obj \ - +err2errno.obj \ - +filetime.obj \ - +fnmatch.obj \ - +fopen.obj \ - +fstat64i32.obj \ - +fstat64.obj \ - +getcwd.obj \ - +getopt.obj \ - +getppid.obj \ - +gettimeofday.obj \ - +iconv.obj \ - +lstat64i32.obj \ - +lstat64.obj \ - +main.obj \ - +mb2wpath.obj \ - +mkdir.obj \ - +mkdtemp.obj \ - +mkstemp.obj \ - +open.obj \ - +readlink.obj \ - +realpath.obj \ - +spawn.obj \ - +strerror.obj \ - +strndup.obj \ - +strptime.obj \ - +symlink.obj \ - +uname.obj \ - +utime.obj \ - +utimes.obj \ - +xfreopen.obj \ -# +lstat32.obj \ -# +lstat32i64.obj \ - -# WIN32 UTF-8 extension routines, used for implementing UTF-8 support for WIN32 libc. -WIN32_OBJECTS = \ - +GetFileAttributes.obj \ - +GetFileAttributesEx.obj \ - +GetFullPathName.obj \ - +GetLongPathName.obj \ - +fullpath.obj \ - -# GnuLib routines that I mistakenly defined here -REMOVED_OBJECTS = \ - +error.obj \ - +initmain.obj \ - +xnmalloc.obj \ - -############################################################################### -# Include files dependancies # -############################################################################### - -I=..\include -CI=$(STINCLUDE) - -$(I)\chdir.h: $(I)\unistd.h $(I)\iconv.h $(CI)\debugm.h - -$(I)\config.h: $(I)\msvclibx.h $(I)\stdbool.h $(I)\unistd.h - -$(I)\direct.h: $(I)\msvclibx.h - -$(I)\dirent.h: $(I)\inttypes.h $(I)\sys\stat.h - -$(I)\error.h: $(I)\msvclibx.h - -# $(I)\fadvise.h: - -$(I)\fcntl.h: $(I)\msvclibx.h - -$(I)\fnmatch.h: $(I)\msvclibx.h - -$(I)\getcwd.h: $(I)\unistd.h $(CI)\debugm.h - -# $(I)\getopt.h: - -$(I)\grp.h: $(I)\msvclibx.h - -# $(I)\inttypes.h: - -# $(I)\msvclibx.h: - -# $(I)\netdb.h: - -$(I)\process.h: $(I)\msvclibx.h - -$(I)\pwd.h: $(I)\msvclibx.h - -# $(I)\regex.h: - -$(I)\sys\stat.h: $(I)\msvclibx.h $(I)\sys\types.h - -# $(I)\stdbool.h: - -# $(I)\stdint.h: - -$(I)\stdio.h: $(I)\msvclibx.h - -# $(I)\stdio--.h: - -$(I)\stdlib.h: $(I)\msvclibx.h - -# $(I)\system.h: - -$(I)\unistd.h: $(I)\msvclibx.h $(I)\dirent.h - -# $(I)\utime.h: - -$(I)\windowsU.h: $(I)\msvclibx.h - -$(I)\xfreopen.h: $(I)\msvclibx.h - -$(I)\sys\types.h: $(I)\msvclibx.h - - -############################################################################### -# Source files dependancies # -############################################################################### - -access.c: $(I)\MsvcLibX.h $(CI)\debugm.h - -basename.c: $(I)\libgen.h - -chdir.c: $(CI)\debugm.h $(I)\iconv.h $(I)\unistd.h - -clock_gettime.c: $(I)\MsvcLibX.h $(I)\time.h $(I)\sys\stat.h - -debugv.c: $(CI)\debugm.h - -dirent.c: $(CI)\debugm.h $(I)\dirent.h $(I)\sys\stat.h $(I)\unistd.h - -dirname.c: $(I)\libgen.h - -err2errno.c: $(I)\MsvcLibX.h $(CI)\debugm.h - -error.c: $(I)\config.h $(I)\error.h - -filetime.c: $(I)\sys\stat.h - -fnmatch.c: $(CI)\debugm.h $(I)\fnmatch.h - -fopen.c: $(I)\MsvcLibX.h - -fstat64.c: fstat.c $(CI)\debugm.h $(I)\dirent.h $(I)\MsvcLibX.h $(I)\sys\stat.h $(I)\stdint.h - -fstat64i32.c: fstat.c $(CI)\debugm.h $(I)\dirent.h $(I)\MsvcLibX.h $(I)\sys\stat.h $(I)\stdint.h - -fullpath.c: $(I)\stdlib.h $(I)\limits.h - -getcwd.c: $(CI)\debugm.h $(I)\unistd.h - -GetFileAttributesU.c: $(I)\windowsU.h $(I)\limits.h - -GetFileAttributesExU.c: $(I)\windowsU.h $(I)\limits.h - -GetFullPathNameU.c: $(I)\windowsU.h $(I)\limits.h - -GetLongPathNameU.c: $(I)\windowsU.h $(I)\limits.h - -getopt.c: $(I)\getopt.h - -# getppid.c: - -gettimeofday.c: $(I)\MsvcLibX.h $(I)\time.h $(I)\sys\time.h - -grp.c: $(I)\grp.h - -iconv.c: $(I)\iconv.h - -initmain.c: $(I)\config.h - -lstat32.c: lstat.c $(CI)\debugm.h $(I)\dirent.h $(I)\MsvcLibX.h $(I)\sys\stat.h $(I)\stdint.h $(I)\unistd.h - -lstat32i64.c: lstat.c $(CI)\debugm.h $(I)\dirent.h $(I)\MsvcLibX.h $(I)\sys\stat.h $(I)\stdint.h $(I)\unistd.h - -lstat64.c: lstat.c $(CI)\debugm.h $(I)\dirent.h $(I)\MsvcLibX.h $(I)\sys\stat.h $(I)\stdint.h $(I)\unistd.h - -lstat64i32.c: lstat.c $(CI)\debugm.h $(I)\dirent.h $(I)\MsvcLibX.h $(I)\sys\stat.h $(I)\stdint.h $(I)\unistd.h - -main.c: $(I)\MsvcLibX.h - -mb2wpath.c: $(I)\MsvcLibX.h $(CI)\debugm.h - -mkdir.c: $(I)\MsvcLibX.h $(I)\sys\stat.h - -mkdtemp.c: $(I)\unistd.h - -mkstemp.c: $(I)\unistd.h - -open.c: $(I)\MsvcLibX.h $(I)\fcntl.h $(CI)\debugm.h - -pwd.c: $(I)\pwd.h - -readlink.c: $(CI)\debugm.h $(I)\unistd.h $(I)\reparsept.h - -realpath.c: $(CI)\debugm.h $(I)\unistd.h - -spawm.c: $(CI)\debugm.h $(I)\MsvcLibX.h $(I)\process.h - -strerror.c: $(I)\MsvcLibX.h - -# strndup.c: - -# strptime.c: - -symlink.c: $(CI)\debugm.h $(I)\reparsept.h $(I)\unistd.h - -uname.c: $(I)\MsvcLibX.h $(I)\sys\utsname.h - -utime.c: $(CI)\debugm.h $(I)\unistd.h $(I)\utime.h $(I)\sys\time.h - -xfreopen.c: $(I)\xfreopen.h - -xnmalloc.c: $(I)\config.h - diff --git a/deps/MsvcLibX/src/GetFileAttributes.c b/deps/MsvcLibX/src/GetFileAttributes.c index 4a74ff2757..386114ed70 100644 --- a/deps/MsvcLibX/src/GetFileAttributes.c +++ b/deps/MsvcLibX/src/GetFileAttributes.c @@ -16,7 +16,7 @@ #ifdef _WIN32 /* Automatically defined when targeting a Win32 application */ #include /* Also includes MsvcLibX' WIN32 UTF-8 extensions */ -#include +#include "msvcLimits.h" /*---------------------------------------------------------------------------*\ * * diff --git a/deps/MsvcLibX/src/GetFileAttributesEx.c b/deps/MsvcLibX/src/GetFileAttributesEx.c index 9f2b799f4a..caf3c27342 100644 --- a/deps/MsvcLibX/src/GetFileAttributesEx.c +++ b/deps/MsvcLibX/src/GetFileAttributesEx.c @@ -16,7 +16,7 @@ #ifdef _WIN32 /* Automatically defined when targeting a Win32 application */ #include /* Also includes MsvcLibX' WIN32 UTF-8 extensions */ -#include +#include "msvcLimits.h" /*---------------------------------------------------------------------------*\ * * diff --git a/deps/MsvcLibX/src/GetFullPathName.c b/deps/MsvcLibX/src/GetFullPathName.c index 560285269b..4e9afeec3e 100644 --- a/deps/MsvcLibX/src/GetFullPathName.c +++ b/deps/MsvcLibX/src/GetFullPathName.c @@ -16,8 +16,8 @@ #ifdef _WIN32 /* Automatically defined when targeting a Win32 application */ #include /* Also includes MsvcLibX' WIN32 UTF-8 extensions */ -#include -#include "debugm.h" /* MsvcLibX debugging macros */ +#include "msvcLimits.h" +#include "msvcDebugm.h" /* MsvcLibX debugging macros */ /*---------------------------------------------------------------------------*\ * * diff --git a/deps/MsvcLibX/src/GetLongPathName.c b/deps/MsvcLibX/src/GetLongPathName.c index ed4ad8162b..b0bbc245d9 100644 --- a/deps/MsvcLibX/src/GetLongPathName.c +++ b/deps/MsvcLibX/src/GetLongPathName.c @@ -16,8 +16,8 @@ #ifdef _WIN32 /* Automatically defined when targeting a Win32 application */ #include /* Also includes MsvcLibX' WIN32 UTF-8 extensions */ -#include -#include "debugm.h" /* MsvcLibX debugging macros */ +#include "msvcLimits.h" +#include "msvcDebugm.h" /* MsvcLibX debugging macros */ /*---------------------------------------------------------------------------*\ * * diff --git a/deps/MsvcLibX/src/NMakefile b/deps/MsvcLibX/src/NMakefile deleted file mode 100644 index 8be23a7c28..0000000000 --- a/deps/MsvcLibX/src/NMakefile +++ /dev/null @@ -1,239 +0,0 @@ -############################################################################### -# # -# File name NMakefile # -# # -# Description An NMake file for making all MsvcLibX library versions # -# # -# Notes make.bat looks for a default nmake file called nmakefile. # -# # -# History # -# 2012-10-21 JFL Initial version # -# 2014-03-05 JFL Generate the DOS version only if DOS tools are present. # -# 2014-04-22 JFL Fixed the clean target to work even without 16-bits tools.# -# 2014-05-30 JFL Moved the OBJECTS macro definition to Files.mak. # -# 2014-12-03 JFL Fixed the zip target creation. # -# 2015-11-06 JFL Added support for a Win95 version. # -# 2016-01-11 JFL Added a rule to create the common ..\lib directory. # -# 2016-07-05 JFL Fixed a typo preventing the build of DOS S Debug version. # -# 2016-09-26 JFL Added macro LIBDIR, depending on OUTDIR. # -# Copy the include files to $(OUTDIR). # -# 2016-10-04 JFL Make sure nothing is displayed when doing a make clean. # -# 2016-10-06 JFL Added targets mostlyclean & distclean. # -# 2016-10-11 JFL Adapted for use with make files in the global include dir.# -# 2016-10-13 JFL Added target cleanenv. # -# 2016-11-03 JFL Added target config. # -# Updated the cleanenv: script to display the reg command # -# it uses to delete the global environment variable. # -# 2016-11-07 JFL Place LIBDIR in the parent directory even if OUTDIR defnd.# -# Do not copy include files to OUTDIR anymore. # -# 2016-11-16 JFL Removed the rule for copying INCDIR, not used anymore. # -# 2017-02-16 JFL Default goals now depend on the existence of their # -# corresponding make file. # -# # -# Copyright 2016 Hewlett Packard Enterprise Development LP # -# Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 # -############################################################################### - -!IF DEFINED(MESSAGES) -!MESSAGE Started MsvcLibX/src/NMakefile in $(MAKEDIR) # Display this make file name -!ENDIF - -!IF DEFINED(OUTDIR) -OD=$(OUTDIR)^\ -!IF "$(OUTDIR:\=)"=="$(OUTDIR)" -LIBDIR=..\$(OUTDIR)\lib # If OUTDIR is relative, put it in MsvcLibX top directory -!ELSE -LIBDIR=$(OUTDIR)\lib # If OUTDIR is absolute, use it as is -!ENDIF -!ELSE -OD= -LIBDIR=..\lib -!ENDIF - -MSG=>con echo # Command for writing a progress message on the console -HEADLINE=$(MSG).&$(MSG) # Output a blank line, then a message - -# Default goal: Generate all versions -all: headline \ -!IF DEFINED(DOS_CC) && EXIST("$(STINCLUDE)\DOS.mak") - $(OD)DOS\MsvcLibX.lib \ -!ENDIF -!IF DEFINED(WIN95_CC) && EXIST("$(STINCLUDE)\WIN95.mak") - $(OD)WIN95\MsvcLibX.lib \ -!ENDIF -!IF DEFINED(WIN32_CC) && EXIST("$(STINCLUDE)\WIN32.mak") - $(OD)WIN32\MsvcLibX.lib \ -!ENDIF -!IF DEFINED(WIN64_CC) && EXIST("$(STINCLUDE)\WIN64.mak") - $(OD)WIN64\MsvcLibX.lib \ -!ENDIF -!IF DEFINED(DOS_CC) && EXIST("$(STINCLUDE)\DOS.mak") - $(OD)DOS\DEBUG\MsvcLibX.lib \ -!ENDIF -!IF DEFINED(WIN95_CC) && EXIST("$(STINCLUDE)\WIN95.mak") - $(OD)WIN95\DEBUG\MsvcLibX.lib \ -!ENDIF -!IF DEFINED(WIN32_CC) && EXIST("$(STINCLUDE)\WIN32.mak") - $(OD)WIN32\DEBUG\MsvcLibX.lib \ -!ENDIF -!IF DEFINED(WIN64_CC) && EXIST("$(STINCLUDE)\WIN64.mak") - $(OD)WIN64\DEBUG\MsvcLibX.lib \ -!ENDIF - -headline: - $(HEADLINE) Building all MsvcLibX library versions - -# Define the OBJECTS macro = the list of object files to include in the library -!INCLUDE "Files.mak" - -# Create the common libs directory -$(LIBDIR): - $(HEADLINE) Creating directory $(LIBDIR:..=MsvcLibX) - md $(LIBDIR) - -# Rules for building specific versions of the MsvcLibX library -$(OD)DOS\MsvcLibX.lib: $(OD)DOS\BIN\T\MsvcLibX.lib $(OD)DOS\BIN\S\MsvcLibX.lib $(OD)DOS\BIN\L\MsvcLibX.lib - @echo Done building all DOS release libraries - -$(OD)DOS\DEBUG\MsvcLibX.lib: $(OD)DOS\DEBUG\BIN\T\MsvcLibX.lib $(OD)DOS\DEBUG\BIN\S\MsvcLibX.lib $(OD)DOS\DEBUG\BIN\L\MsvcLibX.lib - @echo Done building all DOS debug libraries - -$(OD)DOS\BIN\T\MsvcLibX.lib: $(LIBDIR) NUL - $(HEADLINE) Building MsvcLibX library DOS tiny release version - set OBJECTS=$(OBJECTS) - $(MAKE) /$(MAKEFLAGS) /f $(STINCLUDE)\DOS.mak "DEBUG=0" "PROGRAM=MsvcLibX" "OBJECTS=%OBJECTS:+=+$(OD)DOS\OBJ\T\%" "MEM=T" dirs $@ - if exist $@ copy $@ $(LIBDIR)\MsvcLibXdt.lib - -$(OD)DOS\BIN\S\MsvcLibX.lib: $(LIBDIR) NUL - $(HEADLINE) Building MsvcLibX library DOS small release version - set OBJECTS=$(OBJECTS) - $(MAKE) /$(MAKEFLAGS) /f $(STINCLUDE)\DOS.mak "DEBUG=0" "PROGRAM=MsvcLibX" "OBJECTS=%OBJECTS:+=+$(OD)DOS\OBJ\S\%" "MEM=S" dirs $@ - if exist $@ copy $@ $(LIBDIR)\MsvcLibXds.lib - -$(OD)DOS\BIN\L\MsvcLibX.lib: $(LIBDIR) NUL - $(HEADLINE) Building MsvcLibX library DOS large release version - set OBJECTS=$(OBJECTS) - $(MAKE) /$(MAKEFLAGS) /f $(STINCLUDE)\DOS.mak "DEBUG=0" "PROGRAM=MsvcLibX" "OBJECTS=%OBJECTS:+=+$(OD)DOS\OBJ\L\%" "MEM=L" dirs $@ - if exist $@ copy $@ $(LIBDIR)\MsvcLibXdl.lib - -$(OD)WIN95\MsvcLibX.lib: $(LIBDIR) NUL - $(HEADLINE) Building MsvcLibX library WIN95 release version - set OBJECTS=$(OBJECTS) $(WIN32_OBJECTS) - $(MAKE) /$(MAKEFLAGS) /f $(STINCLUDE)\WIN95.mak "DEBUG=0" "PROGRAM=MsvcLibX" "OBJECTS=%OBJECTS:+=+$(OD)WIN95\OBJ\%" dirs $@ - if exist $@ copy $@ $(LIBDIR)\MsvcLibXw95.lib - -$(OD)WIN32\MsvcLibX.lib: $(LIBDIR) NUL - $(HEADLINE) Building MsvcLibX library WIN32 release version - set OBJECTS=$(OBJECTS) $(WIN32_OBJECTS) - set OD=$(OD) - $(MAKE) /$(MAKEFLAGS) /f $(STINCLUDE)\WIN32.mak "DEBUG=0" "PROGRAM=MsvcLibX" "OBJECTS=%OBJECTS:+=+$(OD)WIN32\OBJ\%" dirs $@ - if exist $@ copy $@ $(LIBDIR)\MsvcLibXw32.lib - -$(OD)WIN64\MsvcLibX.lib: $(LIBDIR) NUL - $(HEADLINE) Building MsvcLibX library WIN64 release version - set OBJECTS=$(OBJECTS) $(WIN32_OBJECTS) - $(MAKE) /$(MAKEFLAGS) /f $(STINCLUDE)\WIN64.mak "DEBUG=0" "PROGRAM=MsvcLibX" "OBJECTS=%OBJECTS:+=+$(OD)WIN64\OBJ\%" dirs $@ - if exist $@ copy $@ $(LIBDIR)\MsvcLibXw64.lib - -$(OD)DOS\DEBUG\BIN\T\MsvcLibX.lib: $(LIBDIR) NUL - $(HEADLINE) Building MsvcLibX library DOS tiny debug version - set OBJECTS=$(OBJECTS) - $(MAKE) /$(MAKEFLAGS) /f $(STINCLUDE)\DOS.mak "DEBUG=1" "PROGRAM=MsvcLibX" "OBJECTS=%OBJECTS:+=+$(OD)DOS\DEBUG\OBJ\T\%" "MEM=T" dirs $@ - if exist $@ copy $@ $(LIBDIR)\MsvcLibXdtd.lib - -$(OD)DOS\DEBUG\BIN\S\MsvcLibX.lib: $(LIBDIR) NUL - $(HEADLINE) Building MsvcLibX library DOS small debug version - set OBJECTS=$(OBJECTS) - $(MAKE) /$(MAKEFLAGS) /f $(STINCLUDE)\DOS.mak "DEBUG=1" "PROGRAM=MsvcLibX" "OBJECTS=%OBJECTS:+=+$(OD)DOS\DEBUG\OBJ\S\%" "MEM=S" dirs $@ - if exist $@ copy $@ $(LIBDIR)\MsvcLibXdsd.lib - -$(OD)DOS\DEBUG\BIN\L\MsvcLibX.lib: $(LIBDIR) NUL - $(HEADLINE) Building MsvcLibX library DOS large debug version - set OBJECTS=$(OBJECTS) - $(MAKE) /$(MAKEFLAGS) /f $(STINCLUDE)\DOS.mak "DEBUG=1" "PROGRAM=MsvcLibX" "OBJECTS=%OBJECTS:+=+$(OD)DOS\DEBUG\OBJ\L\%" "MEM=L" dirs $@ - if exist $@ copy $@ $(LIBDIR)\MsvcLibXdld.lib - -$(OD)WIN95\DEBUG\MsvcLibX.lib: $(LIBDIR) NUL - $(HEADLINE) Building MsvcLibX library WIN95 debug version - set OBJECTS=$(OBJECTS) $(WIN32_OBJECTS) - $(MAKE) /$(MAKEFLAGS) /f $(STINCLUDE)\WIN95.mak "DEBUG=1" "PROGRAM=MsvcLibX" "OBJECTS=%OBJECTS:+=+$(OD)WIN95\DEBUG\OBJ\%" dirs $@ - if exist $@ copy $@ $(LIBDIR)\MsvcLibXw95d.lib - -$(OD)WIN32\DEBUG\MsvcLibX.lib: $(LIBDIR) NUL - $(HEADLINE) Building MsvcLibX library WIN32 debug version - set OBJECTS=$(OBJECTS) $(WIN32_OBJECTS) - $(MAKE) /$(MAKEFLAGS) /f $(STINCLUDE)\WIN32.mak "DEBUG=1" "PROGRAM=MsvcLibX" "OBJECTS=%OBJECTS:+=+$(OD)WIN32\DEBUG\OBJ\%" dirs $@ - if exist $@ copy $@ $(LIBDIR)\MsvcLibXw32d.lib - -$(OD)WIN64\DEBUG\MsvcLibX.lib: $(LIBDIR) NUL - $(HEADLINE) Building MsvcLibX library WIN64 debug version - set OBJECTS=$(OBJECTS) $(WIN32_OBJECTS) - $(MAKE) /$(MAKEFLAGS) /f $(STINCLUDE)\WIN64.mak "DEBUG=1" "PROGRAM=MsvcLibX" "OBJECTS=%OBJECTS:+=+$(OD)WIN64\DEBUG\OBJ\%" dirs $@ - if exist $@ copy $@ $(LIBDIR)\MsvcLibXw64d.lib - -# Erase all global environment variables created by this build -VARS=MSVCLIBX # Global environment variables generated here -cleanenv: - call <<"$(TMP)\cleanenv-$(PID).bat" &:# Delete variables created here - @echo off - setlocal EnableExtensions EnableDelayedExpansion - set "KEY=HKCU\Environment" - if not "$(VARS)"=="" for %%v in ($(VARS)) do @( - >>"$(TMP)\cleanenv-$(PID).lst" (echo %%v) &:# Pass the name back to make.bat, for deleting it in the current shell environment - set "VALUE=" - for /f "tokens=1,3" %%a in ('reg query "%KEY%" ^| findstr /i /b /r /c:" *%%v "') do set VALUE="%%b" - if defined VALUE ( :# The global environment variable exists. Delete it, using its actual name with the correct case. - set CMD=reg delete "%KEY%" /v %%v /f - echo !CMD! - !CMD! >NUL - ) - ) -<< - -# Dummy target, to delete all files built by these make files -clean mostlyclean distclean: - rem # Delete temporary files - -for /f "delims=" %f in ('dir /b /s *~ *.bak #*# 2^>NUL') do @del "%f" - rem # Delete files built by this nmakefile - -for /f "delims=" %f in ('dir /b /s ..\*.zip *.log 2^>NUL') do @del "%f" - rem # Delete files built by the DOS.mak, WIN32.mak, WIN64.mak make files - rem # But make DOS.mak will fail if 16-bit tools are missing. Workaround: - -if exist $(STINCLUDE)\DOS.mak $(MAKE) /$(MAKEFLAGS) /f $(STINCLUDE)\DOS.mak clean - if exist $(OD)DOS rd /s /q $(OD)DOS - rem # No such problem with the others, but just in case, do the same. - -if exist $(STINCLUDE)\WIN95.mak $(MAKE) /$(MAKEFLAGS) /f $(STINCLUDE)\WIN95.mak clean - if exist $(OD)WIN95 rd /s /q $(OD)WIN95 - -if exist $(STINCLUDE)\WIN32.mak $(MAKE) /$(MAKEFLAGS) /f $(STINCLUDE)\WIN32.mak clean - if exist $(OD)WIN32 rd /s /q $(OD)WIN32 - -if exist $(STINCLUDE)\WIN64.mak $(MAKE) /$(MAKEFLAGS) /f $(STINCLUDE)\WIN64.mak clean - if exist $(OD)WIN64 rd /s /q $(OD)WIN64 -!IF DEFINED(OUTDIR) - -rd /S /Q $(OUTDIR) >NUL 2>&1 -!ENDIF - -if "$@"=="distclean" del /Q config.*.bat >NUL 2>&1 - -# Dummy target, to build a source archive -dist zip: - $(MSG) Building ..\MsvcLibX.zip - cd .. - if exist MsvcLibX.zip del MsvcLibX.zip - set PATH=$(PATH);C:\Program Files\7-zip - 7z.exe a MsvcLibX.zip *.txt src\*.bat src\*.mak src\*makefile src\exe src\*.c -r include\*.h lib\ - rem # Delete files that got dragged in by wild cards, but that we don't want in the source zip. - 7z.exe d MsvcLibX.zip src\config.*.bat lib\*.lib - cd src - -# Run the configure.bat script in every subdirectory -config: - rem Nothing to do in $(MAKEDIR) as there is no further child level - -# Dummy target, to display a help screen -help: - type << -Targets: - all Rebuild all library versions (Default) - clean Delete all files built here, and all backup files - distclean Do a clean, then delete config.*.bat files - zip Build a source archive in ..\MsvcLibX.zip. Requires 7-zip. -< #include #include +#include "msvcStdio.h" #include "msvclibx.h" -#include "debugm.h" +#include "msvcDebugm.h" +#include "msvcLimits.h" #ifdef _WIN32 diff --git a/deps/MsvcLibX/src/basename.c b/deps/MsvcLibX/src/basename.c index bf2af0efbe..c442a40626 100644 --- a/deps/MsvcLibX/src/basename.c +++ b/deps/MsvcLibX/src/basename.c @@ -20,11 +20,11 @@ #define _CRT_SECURE_NO_WARNINGS 1 /* Avoid Visual C++ 2005 security warnings */ -#include #include #include -#include -#include +#include "msvcStdlib.h" +#include "msvcLibgen.h" +#include "msvcLimits.h" #define TRUE 1 #define FALSE 0 diff --git a/deps/MsvcLibX/src/chdir.c b/deps/MsvcLibX/src/chdir.c index e438cf937e..8d562f885f 100644 --- a/deps/MsvcLibX/src/chdir.c +++ b/deps/MsvcLibX/src/chdir.c @@ -18,12 +18,14 @@ /* Microsoft C libraries include files */ #include -#include #include /* MsvcLibX library extensions */ -#include -#include -#include "debugm.h" +#include "msvcStdio.h" +#include "msvcUnistd.h" +#include "msvcIconv.h" +#include "msvcDebugm.h" +#include "msvcLimits.h" +#include "sys/msvcTypes.h" #if defined(_MSDOS) diff --git a/deps/MsvcLibX/src/clock_gettime.c b/deps/MsvcLibX/src/clock_gettime.c index 8a5bb8f3fb..fee7532e2e 100644 --- a/deps/MsvcLibX/src/clock_gettime.c +++ b/deps/MsvcLibX/src/clock_gettime.c @@ -31,8 +31,8 @@ #define WIN32_LEAN_AND_MEAN /* Avoid lots of unnecessary inclusions */ #include - -#include /* For MsvcLibX's Filetime2Timespec */ +#include "msvcTime.h" +#include "sys/msvcStat.h" /* For MsvcLibX's Filetime2Timespec */ int clock_gettime(clockid_t clock_id, struct timespec *pTS) { FILETIME ft; diff --git a/deps/MsvcLibX/src/config.DESKTOP-U79TD6T.bat b/deps/MsvcLibX/src/config.DESKTOP-U79TD6T.bat deleted file mode 100644 index 6d57a1358a..0000000000 --- a/deps/MsvcLibX/src/config.DESKTOP-U79TD6T.bat +++ /dev/null @@ -1,151 +0,0 @@ -:# config.DESKTOP-U79TD6T.bat generated by configure.bat on 2020/05/19 Öܶþ 7:50:15.02 -:# -:# If changes are needeed, do not edit this file, but instead create a new script -:# called configure.YOURCHOICE.bat. This new script will be invoked automatically -:# by configure.bat while creating this file. Then your script can write extra -:# definitions, or change some of the variables before configure.bat writes them. -:# -:# Invoke configure.bat manually if anything changes in the tools config, such as -:# installing a Visual Studio update, or updating a configure.XXX.bat script. - -set "HAS_STINCLUDE=1" &:# Found the System Tools global C includes -set "STINCLUDE=C:\Users\admin\Desktop\temp\MyGitHub\C\include" &:# System Tools global C includes -set "HAS_MSVCLIBX=1" &:# Found the MSVC Library eXtensions library -set "MSVCLIBX=C:\Users\admin\Desktop\temp\MyGitHub\C\MsvcLibX" &:# MSVC Library eXtensions library -set "HAS_SDK_FLAGS=/DHAS_STINCLUDE=1 /DHAS_MSVCLIBX=1" &:# SDK detection flags for the C compiler - -SET "PF32=C:\Program Files (x86)" &:# 32-bits Program Files -SET "PF64=C:\Program Files" &:# 64-bits Program Files -SET "ARCH=AMD64" &:# PROCESSOR_ARCHITECTURE - -SET "MASM=" &:# Microsoft 16-bits Assembler base path -SET "MSVC=" &:# Microsoft 16-bits Visual C++ base path -SET "MAPSYM=" &:# 16-bits debugging symbols generator - -SET "VSTUDIOLONG=C:\Program Files (x86)\Microsoft Visual Studio 14.0" &:# Microsoft Visual Studio (Long path) -SET "VSTUDIO=C:\PROGRA~2\MICROS~2.0" &:# Microsoft Visual Studio (Short path) -SET "VSCOMMONLONG=C:\Program Files (x86)\Microsoft Visual Studio 14.0\Common7" &:# Microsoft Visual Studio Common Files (Long path) -SET "VSCOMMON=C:\PROGRA~2\MICROS~2.0\Common7" &:# Microsoft Visual Studio Common Files (Short path) -SET "VSIDELONG=C:\Program Files (x86)\Microsoft Visual Studio 14.0\Common7\IDE" &:# Microsoft Visual Studio IDE Files (Long path) -SET "VSIDE=C:\PROGRA~2\MICROS~2.0\Common7\IDE" &:# Microsoft Visual Studio IDE Files (Short path) -SET "VSTOOLSLONG=C:\Program Files (x86)\Microsoft Visual Studio 14.0\Common7\Tools" &:# Microsoft Visual Studio Tools (Long paths) -SET "VSTOOLS=C:\PROGRA~2\MICROS~2.0\Common7\Tools" &:# Microsoft Visual Studio Tools (Short paths) -SET "MSVC32LONG=C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC" &:# Microsoft Visual C++ 32/64 bits (Long path) -SET "MSVC32=C:\PROGRA~2\MICROS~2.0\VC" &:# Microsoft Visual C++ 32/64 bits (Short path) - -SET "WIN_CP=936" &:# Windows Code Page -SET "WIN_CS=gb2312" &:# Windows Character Set -SET "DOS_CP=936" &:# DOS Code Page -SET "DOS_CS=gb2312" &:# DOS Character Set - -SET "AS=" &:# Assembler -SET "CC=" &:# C compiler -SET "INCLUDE=C:\Users\admin\Desktop\temp\MyGitHub\C\include" &:# Include paths. Define USER_INCLUDE if needed. -SET "LK=" &:# Linker -SET "LIB=" &:# Libraries paths. Define USER_LIBS if needed. -SET "LB=" &:# Library manager -SET "RC=" &:# Resource compiler -SET "MT=" &:# Manifest tool - -SET "DOS_CC=" &:# Microsoft Visual C++ 16-bits compiler -SET "DOS_AS=" &:# Microsoft 16-bits assembler -SET "DOS_LK=" &:# Microsoft 16-bits linker -SET "DOS_LB=" &:# Microsoft 16-bits librarian -SET "DOS_RC=" &:# Microsoft 16-bits resource compiler -SET "DOS_MT=" &:# Microsoft 16-bits manifest tool -SET "DOS_PATH=;C:\Windows\System32;C:\Windows" &:# All tools paths for 16-bits compilation -SET "DOS_VCINC=" &:# Visual C++ 16-bits compiler include dir for MsvcLibX include_next -SET "DOS_CRTINC=" &:# Visual C++ 16-bits CRT library include dir for MsvcLibX include_next -SET "DOS_INCPATH=" &:# Include paths for 16-bits compilation -SET "DOS_LIBPATH=" &:# Libraries paths for 16-bits linking -SET "DOS_WINSDK=" &:# Microsoft Windows 16-bits SDK -SET "DOS_WINSDKINC=" &:# Microsoft Windows 16-bits SDK Include directory - -SET "WIN95_CC=" &:# Microsoft Visual C++ 32-bits compiler -SET "WIN95_AS=" &:# Microsoft 32-bits assembler -SET "WIN95_LK=" &:# Microsoft 32-bits linker -SET "WIN95_LB=" &:# Microsoft 32-bits librarian -SET "WIN95_RC=" &:# Microsoft 32-bits resource compiler -SET "WIN95_MT=" &:# Microsoft 32-bits manifest tool -SET "WIN95_PATH=;C:\Windows\System32;C:\Windows" &:# All tools paths for 32-bits compilation -SET "WIN95_VCINC=" &:# Visual C++ 32-bits compiler include dir for MsvcLibX include_next -SET "WIN95_CRTINC=" &:# Visual C++ 32-bits CRT library include dir for MsvcLibX include_next -SET "WIN95_INCPATH=" &:# Include paths for 32-bits compilation -SET "WIN95_LIBPATH=" &:# Libraries paths for 32-bits linking -SET "WIN95_WINSDK=" &:# Microsoft Windows 32-bits SDK -SET "WIN95_WINSDKINC=" &:# Microsoft Windows 32-bits SDK Include directory - -SET "WIN32_CC="C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\CL.EXE"" &:# Microsoft Visual C++ 32-bits compiler -SET "WIN32_AS="C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\ML.EXE"" &:# Microsoft 32-bits assembler -SET "WIN32_LK="C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\LINK.EXE"" &:# Microsoft 32-bits linker -SET "WIN32_LB="C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\LIB.EXE"" &:# Microsoft 32-bits librarian -SET "WIN32_RC=" &:# Microsoft 32-bits resource compiler -SET "WIN32_MT=" &:# Microsoft 32-bits manifest tool -SET "WIN32_PATH=C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin;C:\Program Files (x86)\Microsoft Visual Studio 14.0\Common7\IDE;C:\Program Files (x86)\Microsoft Visual Studio 14.0\Common7\Tools;C:\Windows\System32;C:\Windows" &:# All tools paths for 32-bits compilation -SET "WIN32_VCINC=C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\include" &:# Visual C++ 32-bits compiler include dir for MsvcLibX include_next -SET "WIN32_CRTINC=C:\Program Files (x86)\Windows Kits\10\Include\10.0.18362.0\ucrt" &:# Visual C++ 32-bits CRT library include dir for MsvcLibX include_next -SET "WIN32_INCPATH=C:\Users\admin\Desktop\temp\MyGitHub\C\MsvcLibX\include;C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\include;:\Program Files (x86)\Windows Kits\10\Include\10.0.17763.0\ucrt;C:\Program Files (x86)\Windows Kits\10\Include\10.0.17763.0\shared;C:\Program Files (x86)\Windows Kits\10\Include\10.0.17763.0\um;C:\Program Files (x86)\Windows Kits\10\Include\10.0.17763.0\winrt;C:\Program Files (x86)\Windows Kits\10\Include\10.0.18362.0\ucrt;C:\Program Files (x86)\Windows Kits\10\Include\10.0.18362.0\shared;C:\Program Files (x86)\Windows Kits\10\Include\10.0.18362.0\um;C:\Program Files (x86)\Windows Kits\10\Include\10.0.18362.0\winrt" &:# Include paths for 32-bits compilation -SET "WIN32_LIBPATH=C:\Users\admin\Desktop\temp\MyGitHub\C\MsvcLibX\lib;C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\lib;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.18362.0\um\x86;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.10240.0\ucrt\x86;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.17763.0\ucrt\x86;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.18362.0\ucrt\x86;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.10240.0\ucrt\x86;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.17763.0\ucrt\x86;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.18362.0\ucrt\x86;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.10240.0\ucrt\x86;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.17763.0\ucrt\x86;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.18362.0\ucrt\x86;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.10240.0\ucrt\x86;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.17763.0\ucrt\x86;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.18362.0\ucrt\x86;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.10240.0\ucrt\x86;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.17763.0\ucrt\x86;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.18362.0\ucrt\x86;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.10240.0\ucrt\x86;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.17763.0\ucrt\x86;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.18362.0\ucrt\x86;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.10240.0\ucrt\x86;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.17763.0\ucrt\x86;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.18362.0\ucrt\x86;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.10240.0\ucrt\x86;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.17763.0\ucrt\x86;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.18362.0\ucrt\x86" &:# Libraries paths for 32-bits linking -SET "WIN32_WINSDK=C:\Program Files (x86)\Windows Kits\10" &:# Microsoft Windows 32-bits SDK -SET "WIN32_WINSDKINC=C:\Program Files (x86)\Windows Kits\10\Include\10.0.18362.0" &:# Microsoft Windows 32-bits SDK Include directory - -SET "IA64_CC=" &:# Microsoft Visual C++ IA64 compiler -SET "IA64_AS=" &:# Microsoft IA64 assembler -SET "IA64_LK=" &:# Microsoft IA64 linker -SET "IA64_LB=" &:# Microsoft IA64 librarian -SET "IA64_RC=" &:# Microsoft IA64 resource compiler -SET "IA64_MT=" &:# Microsoft IA64 manifest tool -SET "IA64_PATH=;C:\Windows\System32;C:\Windows" &:# All tools paths for IA64 compilation -SET "IA64_VCINC=" &:# Visual C++ IA64 compiler include dir for MsvcLibX include_next -SET "IA64_CRTINC=" &:# Visual C++ IA64 CRT library include dir for MsvcLibX include_next -SET "IA64_INCPATH=" &:# Include paths for IA64 compilation -SET "IA64_LIBPATH=" &:# Libraries paths for IA64 linking -SET "IA64_WINSDK=" &:# Microsoft Windows IA64 SDK -SET "IA64_WINSDKINC=" &:# Microsoft Windows IA64 SDK Include directory - -SET "WIN64_CC="C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\amd64\CL.EXE"" &:# Microsoft Visual C++ 64-bits compiler -SET "WIN64_AS="C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\amd64\ML64.EXE"" &:# Microsoft 64-bits assembler -SET "WIN64_LK="C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\amd64\LINK.EXE"" &:# Microsoft 64-bits linker -SET "WIN64_LB="C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\amd64\LIB.EXE"" &:# Microsoft 64-bits librarian -SET "WIN64_RC=" &:# Microsoft 64-bits resource compiler -SET "WIN64_MT=" &:# Microsoft 64-bits manifest tool -SET "WIN64_PATH=C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\amd64;C:\Program Files (x86)\Microsoft Visual Studio 14.0\Common7\IDE;C:\Program Files (x86)\Microsoft Visual Studio 14.0\Common7\Tools;C:\Windows\System32;C:\Windows" &:# All tools paths for 64-bits compilation -SET "WIN64_VCINC=C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\include" &:# Visual C++ 64-bits compiler include dir for MsvcLibX include_next -SET "WIN64_CRTINC=C:\Program Files (x86)\Windows Kits\10\Include\10.0.18362.0\ucrt" &:# Visual C++ 64-bits CRT library include dir for MsvcLibX include_next -SET "WIN64_INCPATH=C:\Users\admin\Desktop\temp\MyGitHub\C\MsvcLibX\include;C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\include;:\Program Files (x86)\Windows Kits\10\Include\10.0.17763.0\ucrt;C:\Program Files (x86)\Windows Kits\10\Include\10.0.17763.0\shared;C:\Program Files (x86)\Windows Kits\10\Include\10.0.17763.0\um;C:\Program Files (x86)\Windows Kits\10\Include\10.0.17763.0\winrt;C:\Program Files (x86)\Windows Kits\10\Include\10.0.18362.0\ucrt;C:\Program Files (x86)\Windows Kits\10\Include\10.0.18362.0\shared;C:\Program Files (x86)\Windows Kits\10\Include\10.0.18362.0\um;C:\Program Files (x86)\Windows Kits\10\Include\10.0.18362.0\winrt" &:# Include paths for 64-bits compilation -SET "WIN64_LIBPATH=C:\Users\admin\Desktop\temp\MyGitHub\C\MsvcLibX\lib;C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\lib\amd64;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.18362.0\um\x64;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.10240.0\ucrt\x64;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.17763.0\ucrt\x64;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.18362.0\ucrt\x64;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.10240.0\ucrt\x64;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.17763.0\ucrt\x64;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.18362.0\ucrt\x64;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.10240.0\ucrt\x64;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.17763.0\ucrt\x64;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.18362.0\ucrt\x64;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.10240.0\ucrt\x64;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.17763.0\ucrt\x64;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.18362.0\ucrt\x64;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.10240.0\ucrt\x64;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.17763.0\ucrt\x64;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.18362.0\ucrt\x64;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.10240.0\ucrt\x64;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.17763.0\ucrt\x64;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.18362.0\ucrt\x64;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.10240.0\ucrt\x64;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.17763.0\ucrt\x64;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.18362.0\ucrt\x64;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.10240.0\ucrt\x64;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.17763.0\ucrt\x64;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.18362.0\ucrt\x64" &:# Libraries paths for 64-bits linking -SET "WIN64_WINSDK=C:\Program Files (x86)\Windows Kits\10" &:# Microsoft Windows 64-bits SDK -SET "WIN64_WINSDKINC=C:\Program Files (x86)\Windows Kits\10\Include\10.0.18362.0" &:# Microsoft Windows 64-bits SDK Include directory - -SET "ARM_CC="C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\amd64_arm\CL.EXE"" &:# Microsoft Visual C++ ARM compiler -SET "ARM_AS="C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\amd64_arm\ARMASM.EXE"" &:# Microsoft ARM assembler -SET "ARM_LK="C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\amd64_arm\LINK.EXE"" &:# Microsoft ARM linker -SET "ARM_LB="C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\amd64_arm\LIB.EXE"" &:# Microsoft ARM librarian -SET "ARM_RC=" &:# Microsoft ARM resource compiler -SET "ARM_MT=" &:# Microsoft ARM manifest tool -SET "ARM_PATH=C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\amd64_arm;C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\amd64;C:\Program Files (x86)\Microsoft Visual Studio 14.0\Common7\IDE;C:\Program Files (x86)\Microsoft Visual Studio 14.0\Common7\Tools;C:\Windows\System32;C:\Windows" &:# All tools paths for ARM compilation -SET "ARM_VCINC=C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\include" &:# Visual C++ ARM compiler include dir for MsvcLibX include_next -SET "ARM_CRTINC=C:\Program Files (x86)\Windows Kits\10\Include\10.0.18362.0\ucrt" &:# Visual C++ ARM CRT library include dir for MsvcLibX include_next -SET "ARM_INCPATH=C:\Users\admin\Desktop\temp\MyGitHub\C\MsvcLibX\include;C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\include;:\Program Files (x86)\Windows Kits\10\Include\10.0.17763.0\ucrt;C:\Program Files (x86)\Windows Kits\10\Include\10.0.17763.0\shared;C:\Program Files (x86)\Windows Kits\10\Include\10.0.17763.0\um;C:\Program Files (x86)\Windows Kits\10\Include\10.0.17763.0\winrt;C:\Program Files (x86)\Windows Kits\10\Include\10.0.18362.0\ucrt;C:\Program Files (x86)\Windows Kits\10\Include\10.0.18362.0\shared;C:\Program Files (x86)\Windows Kits\10\Include\10.0.18362.0\um;C:\Program Files (x86)\Windows Kits\10\Include\10.0.18362.0\winrt" &:# Include paths for ARM compilation -SET "ARM_LIBPATH=C:\Users\admin\Desktop\temp\MyGitHub\C\MsvcLibX\lib;C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\lib\arm;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.18362.0\um\arm;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.10240.0\ucrt\arm;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.17763.0\ucrt\arm;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.18362.0\ucrt\arm;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.10240.0\ucrt\arm;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.17763.0\ucrt\arm;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.18362.0\ucrt\arm;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.10240.0\ucrt\arm;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.17763.0\ucrt\arm;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.18362.0\ucrt\arm;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.10240.0\ucrt\arm;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.17763.0\ucrt\arm;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.18362.0\ucrt\arm;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.10240.0\ucrt\arm;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.17763.0\ucrt\arm;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.18362.0\ucrt\arm;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.10240.0\ucrt\arm;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.17763.0\ucrt\arm;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.18362.0\ucrt\arm;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.10240.0\ucrt\arm;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.17763.0\ucrt\arm;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.18362.0\ucrt\arm;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.10240.0\ucrt\arm;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.17763.0\ucrt\arm;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.18362.0\ucrt\arm" &:# Libraries paths for ARM linking -SET "ARM_WINSDK=C:\Program Files (x86)\Windows Kits\10" &:# Microsoft Windows ARM SDK -SET "ARM_WINSDKINC=C:\Program Files (x86)\Windows Kits\10\Include\10.0.18362.0" &:# Microsoft Windows ARM SDK Include directory - -SET "ARM64_CC=" &:# Microsoft Visual C++ ARM64 compiler -SET "ARM64_AS=" &:# Microsoft ARM64 assembler -SET "ARM64_LK=" &:# Microsoft ARM64 linker -SET "ARM64_LB=" &:# Microsoft ARM64 librarian -SET "ARM64_RC=" &:# Microsoft ARM64 resource compiler -SET "ARM64_MT=" &:# Microsoft ARM64 manifest tool -SET "ARM64_PATH=;C:\Windows\System32;C:\Windows" &:# All tools paths for ARM64 compilation -SET "ARM64_VCINC=" &:# Visual C++ ARM64 compiler include dir for MsvcLibX include_next -SET "ARM64_CRTINC=" &:# Visual C++ ARM64 CRT library include dir for MsvcLibX include_next -SET "ARM64_INCPATH=" &:# Include paths for ARM64 compilation -SET "ARM64_LIBPATH=" &:# Libraries paths for ARM64 linking -SET "ARM64_WINSDK=" &:# Microsoft Windows ARM64 SDK -SET "ARM64_WINSDKINC=" &:# Microsoft Windows ARM64 SDK Include directory - -:# List of commands to run when make.bat exits -SET "POST_MAKE_ACTIONS=set "MSVCLIBX=C:\Users\admin\Desktop\temp\MyGitHub\C\MsvcLibX"" - -exit /b 0 &:# Configuration done successfully diff --git a/deps/MsvcLibX/src/configure.MsvcLibX.bat b/deps/MsvcLibX/src/configure.MsvcLibX.bat deleted file mode 100644 index 767225480a..0000000000 --- a/deps/MsvcLibX/src/configure.MsvcLibX.bat +++ /dev/null @@ -1,36 +0,0 @@ -@echo off -:#***************************************************************************** -:# * -:# Filename: configure.MsvcLibX.bat * -:# * -:# Description: Special make actions for rebuilding the MsvcLibX library * -:# * -:# Notes: * -:# * -:# History: * -:# 2015-11-06 JFL Created this script. * -:# 2016-09-27 JFL Correct the final MSVCLIBX if there's a different OUTDIR. * -:# 2016-11-03 JFL Removed the side effect creating %OUTDIR%. * -:# 2016-11-07 JFL Removed the dependency on OUTDIR. * -:# Immediately set the system environment. * -:# 2016-11-16 JFL Allow using a predefined alias for this lib base path. * -:# 2016-12-16 JFL Only use setx if requested by user, with PERSISTENT_VARS. * -:# * -:# © Copyright 2016 Hewlett Packard Enterprise Development LP * -:# Licensed under the Apache 2.0 license www.apache.org/licenses/LICENSE-2.0 * -:#***************************************************************************** - -:# Get the full pathname of the MsvcLibX library base directory -if defined MSVCLIBX if not exist "%MSVCLIBX%\include\msvclibx.h" set "MSVCLIBX=" &:# Allow overriding with another alias name, but ignore invalid overrides -if not defined MSVCLIBX for /f "delims=" %%d in ('"pushd .. & cd & popd"') do SET "MSVCLIBX=%%d" &:# Default: Use the current directory - -:# Declare the SDKs and libraries we need -%BEGIN_SDK_DEFS% -%USE_SDK% MSVCLIBX &:# Triggers the emission of a %CONFIG% record for MSVCLIBX -%END_SDK_DEFS% - -:# Set the local environment variable just before make exits, so that future commands in this CMD window have it. -%ADD_POST_MAKE_ACTION% set "MSVCLIBX=%MSVCLIBX%" - -:# Set the system environment variable, so that other CMD windows opened later on inherit it -if defined PERSISTENT_VARS setx MSVCLIBX "%MSVCLIBX%" >NUL diff --git a/deps/MsvcLibX/src/configure.bat b/deps/MsvcLibX/src/configure.bat deleted file mode 100644 index 7037eb92af..0000000000 --- a/deps/MsvcLibX/src/configure.bat +++ /dev/null @@ -1,35 +0,0 @@ -@echo off -:#***************************************************************************** -:# * -:# Filename: configure.bat * -:# * -:# Description: Detect system-specific settings and create config.*.bat * -:# * -:# Notes: Proxy script for %STINCLUDE%\configure.bat. * -:# * -:# Make any change needed in %STINCLUDE%\configure.bat. * -:# * -:# History: * -:# 2016-10-10 JFL jf.larvoire@hpe.com created this file. * -:# 2016-12-15 JFL Search for the real make.bat in [.|..|../..]\include. * -:# * -:# © Copyright 2016 Hewlett Packard Enterprise Development LP * -:# Licensed under the Apache 2.0 license www.apache.org/licenses/LICENSE-2.0 * -:#***************************************************************************** - -:# Get the full pathname of the STINCLUDE library directory -if defined STINCLUDE if not exist "%STINCLUDE%\make.bat" set "STINCLUDE=" &:# Allow overriding with another alias name, but ignore invalid overrides -for %%p in (. .. ..\..) do if not defined STINCLUDE if exist %%p\include\make.bat ( :# Default: Search it the current directory, and 2 levels above. - for /f "delims=" %%d in ('"pushd %%p\include & cd & popd"') do SET "STINCLUDE=%%d" -) -if not defined STINCLUDE ( :# Try getting the copy in the master environment - for /f "tokens=3" %%v in ('reg query "HKCU\Environment" /v STINCLUDE 2^>NUL') do set "STINCLUDE=%%v" -) - -if not exist %STINCLUDE%\make.bat ( - >&2 echo %0 Error: Cannot find SysToolsLib's global C include directory. Please define variable STINCLUDE. - exit /b 1 -) - -if [%1]==[-d] echo "%STINCLUDE%\configure.bat" %* -"%STINCLUDE%\configure.bat" %* diff --git a/deps/MsvcLibX/src/debugv.c b/deps/MsvcLibX/src/debugv.c index fb2d6fd101..5d410c1309 100644 --- a/deps/MsvcLibX/src/debugv.c +++ b/deps/MsvcLibX/src/debugv.c @@ -13,7 +13,7 @@ * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ -#include "debugm.h" +#include "msvcDebugm.h" #if defined(_DEBUG) diff --git a/deps/MsvcLibX/src/dirent.c b/deps/MsvcLibX/src/dirent.c index fa664a7353..1ab2e9c002 100644 --- a/deps/MsvcLibX/src/dirent.c +++ b/deps/MsvcLibX/src/dirent.c @@ -38,7 +38,7 @@ #define _UTF8_SOURCE /* Generate the UTF-8 version of WIN32 printf & scandir */ -#include "dirent.h" /* Include our associated .h, in the same dir as this .c. Do not use <>. */ +#include "msvcDirent.h" /* Include our associated .h, in the same dir as this .c. Do not use <>. */ #ifndef _DIRENT_FOR_DOS_WINDOWS #error "This requires MsvcLibX own version of dirent.h for DOS/Windows" #endif @@ -49,9 +49,10 @@ #include #include /* MsvcLibX library extensions */ -#include /* For readlink() */ -#include /* For Filetime2String() */ -#include "debugm.h" /* Use our house debugging framework */ +#include "msvcUnistd.h" /* For readlink() */ +#include "sys/msvcStat.h" /* For Filetime2String() */ +#include "msvcDebugm.h" /* Use our house debugging framework */ +#include "msvcLimits.h" /* Use our house debugging framework */ /*****************************************************************************\ * * diff --git a/deps/MsvcLibX/src/dirname.c b/deps/MsvcLibX/src/dirname.c index df4e4d1ebc..1bdc9d9ba3 100644 --- a/deps/MsvcLibX/src/dirname.c +++ b/deps/MsvcLibX/src/dirname.c @@ -23,8 +23,8 @@ #include #include #include -#include -#include +#include "msvcLibgen.h" +#include "msvcLimits.h" #define TRUE 1 #define FALSE 0 diff --git a/deps/MsvcLibX/src/err2errno.c b/deps/MsvcLibX/src/err2errno.c index 87f9c6c688..306160f664 100644 --- a/deps/MsvcLibX/src/err2errno.c +++ b/deps/MsvcLibX/src/err2errno.c @@ -20,7 +20,7 @@ #include #include /* MsvcLibX library extensions */ -#include "debugm.h" +#include "msvcDebugm.h" #ifdef _WIN32 @@ -65,7 +65,6 @@ int _get_errno_from_oserr(unsigned long dwErr) { return errno; } #else -#pragma message("Using the default " MSVCLIBX_STRINGIZE(_get_errno_from_oserr) "()") /* Equivalent function in MSVC library. Does not know about symlink errors. */ extern int __cdecl _get_errno_from_oserr(unsigned long oserrno); #endif diff --git a/deps/MsvcLibX/src/exe b/deps/MsvcLibX/src/exe deleted file mode 100644 index 0afeaf5d6b..0000000000 --- a/deps/MsvcLibX/src/exe +++ /dev/null @@ -1,99 +0,0 @@ -#!/bin/bash -#*****************************************************************************# -# # -# Filename: exe # -# # -# Description: Build simple C/C++ programs for Unix/Linux # -# # -# Notes: Usage: ./exe PROGRAM # -# # -# Stores the executables in $OS.$PROC/[Debug/] for # -# consistency with the Windows build tools. # -# This allows sharing sources in a host system, and # -# using VMs for building the various Windows and Linux # -# versions in a set of OS-specific subdirectories. # -# # -# History: # -# 2013-12-16 JFL Added support for MinGW64. # -# 2015-12-12 JFL Help now displays the output directory name. # -# 2016-01-07 JFL Added compilation option -Wall. # -# # -# © Copyright 2016 Hewlett Packard Enterprise Development LP # -# Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 # -#*****************************************************************************# - -FileNoCase() # Case-independant search for a file. -{ - find . -type f | grep -i -E "./$1$" | sed s=./== -} - -# Identify the OS -OS=`uname -s` -PROC=`uname -p` -if [[ "$OS" == "OSF1" && "`uname -m`" == "alpha" ]] ; then - OS=Tru64 -fi -if [[ "$OS" == "WindowsNT" ]] ; then - OS=WIN32 -fi -OUTDIR=$OS.$PROC -if [[ "${OS:0:7}" == "MINGW32" ]] ; then # Ex: "MINGW32_NT-6.1" - OUTDIR=MINGW32 # MigGW shell if NOT case sensitive - # 2013-12-16 Actually, the 64-bits tool chain also reports MINGW32_NT-6.1 - # So distinguish the two by whether /mingw is mounted on C:\MinGW or C:\MinGW64 - if mount | grep /mingw | grep 64 > /dev/null ; then - OUTDIR=MINGW64 # MigGW shell if NOT case sensitive - fi -fi -if [[ "${OS:0:7}" == "MINGW64" ]] ; then # Ex: ? - OUTDIR=MINGW64 -fi -if [[ "${OS:0:6}" == "CYGWIN" ]] ; then # Ex: "CYGWIN_NT-6.1-WOW64" - OUTDIR=cygwin # Cygwin shell if case sensitive, so use lower case -fi - -# Command line analysis. -case "$1" in - "" | "-h" | "-?" | --help) - echo "Build simple C/C++ programs, storing the executables in $OUTDIR/" - echo "Usage: ./exe PROGRAM" - exit 0 - ;; -esac - -# Identify the source file and program to build. -PROGRAM=$1 -shift -SOURCES=`FileNoCase ${PROGRAM}.c` -CFLAGS="-std=c99 -Wall" # Force compilation in C, even if there are // comments. -if [[ "${SOURCES}" == "" ]] ; then - SOURCES=`FileNoCase ${PROGRAM}.cpp` - CFLAGS="-std=gnu++98 -lstdc++" # Force compilation in C++, even if plain C. - # -lstdc++ prevents error "undefined reference to '__gxx_personality_v0'" -fi -if [[ "${SOURCES}" == "" ]] ; then - echo "Failed to find ${PROGRAM} source." - exit 1 -fi - -# Make sure our include directories are accessible -if [[ -d "/u/JFL/SRC/Include" ]] ; then - if [[ ":$C_INCLUDE_PATH:" != *:/u/JFL/SRC/Include:* ]] ; then - if [[ "$C_INCLUDE_PATH" == "" ]] ; then - export C_INCLUDE_PATH="/u/JFL/SRC/Include" - else - export C_INCLUDE_PATH="$C_INCLUDE_PATH:/u/JFL/SRC/Include" - fi - fi -fi -echo "# C_INCLUDE_PATH=\"$C_INCLUDE_PATH\"" - -# Build it. -# gmake CC=gcc CFLAGS="$CFLAGS" SOURCES="$SOURCES" PROGRAM="$PROGRAM" OS="$OS" $* -mkdir -p $OUTDIR -echo "gcc $CFLAGS -U_DEBUG $SOURCES -o $OUTDIR/$PROGRAM" -gcc $CFLAGS -U_DEBUG $SOURCES -o $OUTDIR/$PROGRAM -mkdir -p $OUTDIR/debug -echo "gcc $CFLAGS -D_DEBUG $SOURCES -o $OUTDIR/debug/$PROGRAM" -gcc $CFLAGS -D_DEBUG $SOURCES -o $OUTDIR/debug/$PROGRAM - diff --git a/deps/MsvcLibX/src/exe.bat b/deps/MsvcLibX/src/exe.bat deleted file mode 100644 index f93d6ba399..0000000000 --- a/deps/MsvcLibX/src/exe.bat +++ /dev/null @@ -1,82 +0,0 @@ -@echo off -:#***************************************************************************** -:# * -:# Filename: exe.bat * -:# * -:# Description: Front end to make.bat, to simply build multiple targets * -:# * -:# Arguments: Use option -? to display a help screen * -:# * -:# Notes: Builds the 16-bits MS-DOS version if Visual C++ 1.52 is * -:# installed in its default location in C:\MSVC. * -:# * -:# History: * -:# 2003-03-31 JFL Adapted from previous projects * -:# 2014-03-21 JFL Builds the 16-bits MS-DOS version if Visual C++ 1.52 is * -:# installed in its default location in C:\MSVC. * -:# 2014-03-27 JFL Changed option -f to use nmake option /A. * -:# Added option -r for completeness. * -:# 2015-11-13 JFL Adapted to the new multitarget make system. * -:# * -:# © Copyright 2016 Hewlett Packard Enterprise Development LP * -:# Licensed under the Apache 2.0 license www.apache.org/licenses/LICENSE-2.0 * -:#***************************************************************************** - -setlocal enableextensions enabledelayedexpansion -goto main - -:main -set "FORCE=0" -set "ACTION=default" -set "EXEC=" -set "MAKEOPTS=" - -goto get_arg -:next_arg -shift -:get_arg -if .%1.==.-?. goto help -if .%1.==./?. goto help -if .%1.==.-a. set "ACTION=all" & goto next_arg -if .%1.==.-d. set "ACTION=debug" & goto next_arg -if .%1.==.-f. set "FORCE=1" & goto next_arg -if .%1.==.-r. set "ACTION=release" & goto next_arg -if .%1.==.-X. set "EXEC=echo" & goto next_arg -set MAKEOPTS=%2 %3 %4 %5 %6 %7 %8 %9 -if "%FORCE%"=="1" set "MAKEOPTS=%MAKEOPTS% /A" -goto %ACTION% - -:help -echo. -echo..exe program builder from a C or C++ source -echo. -echo.Usage: exe [options] program [nmake_options] -echo. -echo.Options: -echo. -echo. -? Display this help page -echo. -a Builds all release ^& debug versions (default) -echo. -d Builds all debug versions only -echo. -f Force building all program targets, irrespective of file dates -echo. -r Builds all release versions only -echo. -X Display the make command generated and exit -echo. -echo.Notes: -echo.* exe myprog ^<==^> make myprog.exe debug\myprog.exe -echo.* This builds all possible OS targets -echo.* To force rebuilding all targets, irrespective of their date, -echo. use nmake option /A. So: exe -f myprog ^<==^> exe myprog /A -goto :eof - -:release -%EXEC% make %MAKEOPTS% %1.exe -goto :eof - -:debug -%EXEC% make %MAKEOPTS% debug\%1.exe -goto :eof - -:default -:all -%EXEC% make %MAKEOPTS% %1.exe debug\%1.exe -goto :eof diff --git a/deps/MsvcLibX/src/filetime.c b/deps/MsvcLibX/src/filetime.c index e22bc2f2a9..084b5eef67 100644 --- a/deps/MsvcLibX/src/filetime.c +++ b/deps/MsvcLibX/src/filetime.c @@ -9,7 +9,7 @@ * History: * * 2014-02-26 JFL Created this module. * * 2014-03-24 JFL Renamed "statx.h" as the standard . * -* 2014-07-03 JFL Filetime2String: Output time with µs precision if possib. * +* 2014-07-03 JFL Filetime2String: Output time with �s precision if possib. * * 2016-09-13 JFL Fixed a warning. * * * * Copyright 2016 Hewlett Packard Enterprise Development LP * @@ -18,8 +18,8 @@ #define _CRT_SECURE_NO_WARNINGS 1 /* Avoid Visual C++ security warnings */ -#include /* Define time_t */ -#include +#include "msvcTime.h" /* Define time_t */ +#include "sys/msvcStat.h" #ifdef _MSDOS @@ -230,7 +230,7 @@ struct tm *LocalFileTime(const time_t *pt) { } /* Generate a string with the local file time, in the ISO 8601 date/time format */ -/* 2014-07-03 Output time with µs precision if possible */ +/* 2014-07-03 Output time with �s precision if possible */ char *Filetime2String(const FILETIME *pFT, char *pBuf, size_t nBufSize) { FILETIME lft; SYSTEMTIME sTime; @@ -246,7 +246,7 @@ char *Filetime2String(const FILETIME *pFT, char *pBuf, size_t nBufSize) { uli.LowPart = lft.dwLowDateTime; uli.HighPart = lft.dwHighDateTime; iFraction = (int)(uli.QuadPart % 10000000); /* FILETIME has 100ns resolution */ - iFraction /= 10; /* Convert 100ns resolution to 1µs resolution */ + iFraction /= 10; /* Convert 100ns resolution to 1�s resolution */ wsprintf(pBuf+19, ".%06d", iFraction); } else if (nBufSize >= 24) { wsprintf(pBuf+19, ".%03d", sTime.wMilliseconds); diff --git a/deps/MsvcLibX/src/fnmatch.c b/deps/MsvcLibX/src/fnmatch.c index 021cfc5900..d6f640f4c0 100644 --- a/deps/MsvcLibX/src/fnmatch.c +++ b/deps/MsvcLibX/src/fnmatch.c @@ -26,8 +26,8 @@ #include #include -#include "fnmatch.h" /* Include our associated .h, in the same dir as this .c. Do not use <>. */ -#include "debugm.h" +#include "msvcFnmatch.h" /* Include our associated .h, in the same dir as this .c. Do not use <>. */ +#include "msvcDebugm.h" #ifndef TRUE #define TRUE 1 diff --git a/deps/MsvcLibX/src/fstat.c b/deps/MsvcLibX/src/fstat.c index ba767a7bae..124eda8119 100644 --- a/deps/MsvcLibX/src/fstat.c +++ b/deps/MsvcLibX/src/fstat.c @@ -21,10 +21,10 @@ #include #include #include -/* MsvcLibX library extensions */ -#include -#include "debugm.h" #include +/* MsvcLibX library extensions */ +#include "sys/msvcStat.h" +#include "msvcDebugm.h" #if defined(_MSDOS) diff --git a/deps/MsvcLibX/src/fullpath.c b/deps/MsvcLibX/src/fullpath.c index 0d60b53002..1cc2bf4271 100644 --- a/deps/MsvcLibX/src/fullpath.c +++ b/deps/MsvcLibX/src/fullpath.c @@ -18,12 +18,12 @@ #define _UTF8_SOURCE /* Generate the UTF-8 version of routines */ #include -#include #include +#include "msvcLimits.h" #ifdef _WIN32 /* Automatically defined when targeting a Win32 application */ -#include /* Also includes MsvcLibX' WIN32 UTF-8 extensions */ +#include "msvcWindows.h" /* Also includes MsvcLibX' WIN32 UTF-8 extensions */ /*---------------------------------------------------------------------------*\ * * diff --git a/deps/MsvcLibX/src/getcwd.c b/deps/MsvcLibX/src/getcwd.c index 766a40f003..314c59504a 100644 --- a/deps/MsvcLibX/src/getcwd.c +++ b/deps/MsvcLibX/src/getcwd.c @@ -20,8 +20,9 @@ #include #include /* MsvcLibX library extensions */ -#include -#include "debugm.h" +#include "msvcUnistd.h" +#include "msvcDebugm.h" +#include "msvcLimits.h" #if defined(_MSDOS) diff --git a/deps/MsvcLibX/src/getopt.c b/deps/MsvcLibX/src/getopt.c index 67baf949a1..ba0b2ccb33 100644 --- a/deps/MsvcLibX/src/getopt.c +++ b/deps/MsvcLibX/src/getopt.c @@ -40,7 +40,7 @@ #include #include #include -#include "getopt.h" +#include "msvcGetopt.h" #ifndef _DIAGASSERT #ifdef NDEBUG diff --git a/deps/MsvcLibX/src/getppid.c b/deps/MsvcLibX/src/getppid.c index 17bd075916..062beff3b0 100644 --- a/deps/MsvcLibX/src/getppid.c +++ b/deps/MsvcLibX/src/getppid.c @@ -21,7 +21,7 @@ * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ -#include +#include "msvcUnistd.h" #ifdef _WIN32 diff --git a/deps/MsvcLibX/src/gettimeofday.c b/deps/MsvcLibX/src/gettimeofday.c index f9150b9b7c..b485edeb9e 100644 --- a/deps/MsvcLibX/src/gettimeofday.c +++ b/deps/MsvcLibX/src/gettimeofday.c @@ -15,8 +15,8 @@ #include "msvclibx.h" -#include -#include +#include "msvcTime.h" +#include "sys/msvcTime.h" #ifdef _MSDOS /* MS-DOS only has a 1-second resolution on system time. diff --git a/deps/MsvcLibX/src/iconv.c b/deps/MsvcLibX/src/iconv.c index c76d655e25..40b6e6462d 100644 --- a/deps/MsvcLibX/src/iconv.c +++ b/deps/MsvcLibX/src/iconv.c @@ -23,8 +23,9 @@ #include #include /* MsvcLibX library extensions */ -#include "iconv.h" -#include "debugm.h" +#include "msvcIconv.h" +#include "msvcDebugm.h" +#include "msvcLimits.h" #if defined(_MSDOS) diff --git a/deps/MsvcLibX/src/lstat.c b/deps/MsvcLibX/src/lstat.c index 9a42b0f587..e107fd8706 100644 --- a/deps/MsvcLibX/src/lstat.c +++ b/deps/MsvcLibX/src/lstat.c @@ -29,13 +29,14 @@ #include #include #include +#include /* MsvcLibX library extensions */ #include "msvclibx.h" -#include -#include -#include /* For ResolveLinks() definition */ -#include "debugm.h" -#include +#include +#include "msvcDirent.h" +#include "msvcUnistd.h" /* For ResolveLinks() definition */ +#include "msvcDebugm.h" +#include "msvcLimits.h" #if defined(_MSDOS) /* Make sure it's only defined it in one of the lstatxxx versions */ diff --git a/deps/MsvcLibX/src/main.c b/deps/MsvcLibX/src/main.c index c90c111f10..f366b081ad 100644 --- a/deps/MsvcLibX/src/main.c +++ b/deps/MsvcLibX/src/main.c @@ -19,13 +19,13 @@ #define _UTF8_SOURCE #define _CRT_SECURE_NO_WARNINGS /* Avoid depreciation warnings */ -#include #include "msvclibx.h" +#include "msvcStdio.h" #ifdef _WIN32 #include -#include /* For MsvcLibX' codePage global variable */ +#include "msvcIconv.h" /* For MsvcLibX' codePage global variable */ /*---------------------------------------------------------------------------*\ * * diff --git a/deps/MsvcLibX/src/make.bat b/deps/MsvcLibX/src/make.bat deleted file mode 100644 index 7a27256122..0000000000 --- a/deps/MsvcLibX/src/make.bat +++ /dev/null @@ -1,35 +0,0 @@ -@echo off -:#***************************************************************************** -:# * -:# Filename: make.bat * -:# * -:# Description: Build DOS and Windows targets * -:# * -:# Notes: Proxy script for %STINCLUDE%\make.bat. * -:# * -:# If any change is needed, put it in %STINCLUDE%\make.bat. * -:# * -:# History: * -:# 2016-10-10 JFL jf.larvoire@hpe.com created this file. * -:# 2016-12-15 JFL Search for the real make.bat in [.|..|../..]\include. * -:# * -:# © Copyright 2016 Hewlett Packard Enterprise Development LP * -:# Licensed under the Apache 2.0 license www.apache.org/licenses/LICENSE-2.0 * -:#***************************************************************************** - -:# Get the full pathname of the STINCLUDE library directory -if defined STINCLUDE if not exist "%STINCLUDE%\make.bat" set "STINCLUDE=" &:# Allow overriding with another alias name, but ignore invalid overrides -for %%p in (. .. ..\..) do if not defined STINCLUDE if exist %%p\include\make.bat ( :# Default: Search it the current directory, and 2 levels above. - for /f "delims=" %%d in ('"pushd %%p\include & cd & popd"') do SET "STINCLUDE=%%d" -) -if not defined STINCLUDE ( :# Try getting the copy in the master environment - for /f "tokens=3" %%v in ('reg query "HKCU\Environment" /v STINCLUDE 2^>NUL') do set "STINCLUDE=%%v" -) - -if not exist %STINCLUDE%\make.bat ( - >&2 echo %0 Error: Cannot find SysToolsLib's global C include directory. Please define variable STINCLUDE. - exit /b 1 -) - -if [%1]==[-d] echo "%STINCLUDE%\make.bat" %* -"%STINCLUDE%\make.bat" %* diff --git a/deps/MsvcLibX/src/mb2wpath.c b/deps/MsvcLibX/src/mb2wpath.c index db6897a3d8..097d3e7367 100644 --- a/deps/MsvcLibX/src/mb2wpath.c +++ b/deps/MsvcLibX/src/mb2wpath.c @@ -17,7 +17,7 @@ #define _UTF8_SOURCE /* Generate the UTF-8 version of printf routines */ #include "msvclibx.h" -#include "debugm.h" +#include "msvcDebugm.h" #if defined(_WIN32) diff --git a/deps/MsvcLibX/src/mkdir.c b/deps/MsvcLibX/src/mkdir.c index 9d23c14f64..24e98a52ba 100644 --- a/deps/MsvcLibX/src/mkdir.c +++ b/deps/MsvcLibX/src/mkdir.c @@ -22,8 +22,9 @@ #include /* For _mkdir() */ #include /* MsvcLibX library extensions */ -#include +#include #include "msvclibx.h" +#include "msvcLimits.h" #ifdef _WIN32 diff --git a/deps/MsvcLibX/src/mkdtemp.c b/deps/MsvcLibX/src/mkdtemp.c index 99bd5197ce..86bfbf25cf 100644 --- a/deps/MsvcLibX/src/mkdtemp.c +++ b/deps/MsvcLibX/src/mkdtemp.c @@ -13,11 +13,11 @@ * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ -#include #include #include #include #include +#include "msvcUnistd.h" char *mkdtemp(char *pszName) { char *pszXXX, *pc; diff --git a/deps/MsvcLibX/src/mkstemp.c b/deps/MsvcLibX/src/mkstemp.c index 573f601bf4..f587cff48d 100644 --- a/deps/MsvcLibX/src/mkstemp.c +++ b/deps/MsvcLibX/src/mkstemp.c @@ -15,7 +15,7 @@ #define _CRT_SECURE_NO_WARNINGS 1 /* Avoid Visual C++ security warnings */ -#include +#include "msvcUnistd.h" #include #include #include diff --git a/deps/MsvcLibX/src/open.c b/deps/MsvcLibX/src/open.c index 4e33390e2a..dd70409410 100644 --- a/deps/MsvcLibX/src/open.c +++ b/deps/MsvcLibX/src/open.c @@ -19,8 +19,9 @@ #include #include "msvclibx.h" -#include "fcntl.h" -#include "debugm.h" +#include "msvcFcntl.h" +#include "msvcDebugm.h" +#include "msvcLimits.h" #ifdef _WIN32 diff --git a/deps/MsvcLibX/src/readlink.c b/deps/MsvcLibX/src/readlink.c index 5e691ff86f..4abc91e8e2 100644 --- a/deps/MsvcLibX/src/readlink.c +++ b/deps/MsvcLibX/src/readlink.c @@ -27,16 +27,17 @@ #define _UTF8_SOURCE /* Generate the UTF-8 version of routines */ -#include +#include "msvcUnistd.h" #pragma comment(lib, "Mpr.lib") #include -#include "debugm.h" +#include "msvcDebugm.h" +#include "msvcLimits.h" #ifdef _WIN32 #include -#include "reparsept.h" +#include "msvcReparsept.h" /* Get the Reparse Point Tag for a mount point - Wide char version */ /* See http://msdn.microsoft.com/en-us/library/windows/desktop/aa365511(v=vs.85).aspx */ diff --git a/deps/MsvcLibX/src/realpath.c b/deps/MsvcLibX/src/realpath.c index 0d99ff631a..5fbcf773a2 100644 --- a/deps/MsvcLibX/src/realpath.c +++ b/deps/MsvcLibX/src/realpath.c @@ -31,13 +31,15 @@ #define _UTF8_SOURCE /* Generate the UTF-8 version of routines */ -#include #include #include #include #include /* For _getdcwd() */ #include /* For toupper() */ -#include "debugm.h" +#include "msvcDebugm.h" +#include "msvcLimits.h" +#include "msvcUnistd.h" +#include "msvcWindows.h" #define TRUE 1 #define FALSE 0 diff --git a/deps/MsvcLibX/src/spawn.c b/deps/MsvcLibX/src/spawn.c index 08b81449c1..1dede9b712 100644 --- a/deps/MsvcLibX/src/spawn.c +++ b/deps/MsvcLibX/src/spawn.c @@ -19,8 +19,9 @@ /* Microsoft C libraries include files */ #include /* MsvcLibX library extensions */ -#include "debugm.h" +#include "msvcDebugm.h" #include "msvclibx.h" +#include "msvcLimits.h" #ifdef _WIN32 diff --git a/deps/MsvcLibX/src/src2objs.bat b/deps/MsvcLibX/src/src2objs.bat deleted file mode 100644 index b840e308c4..0000000000 --- a/deps/MsvcLibX/src/src2objs.bat +++ /dev/null @@ -1,125 +0,0 @@ -@echo off -:****************************************************************************** -:* * -:* Filename: src2objs.bat * -:* * -:* Description: Generate object files names from source files names * -:* * -:* Notes: * -:* * -:* History: * -:* 2010-04-07 JFL Created this batch. * -:* * -:# © Copyright 2016 Hewlett Packard Enterprise Development LP * -:# Licensed under the Apache 2.0 license www.apache.org/licenses/LICENSE-2.0 * -:****************************************************************************** - -setlocal 2>NUL -set ARG0=%0 - -:# Mechanism for calling subroutines. Done by {call %0 _call_ label [arguments]}. -if .%1.==._call_. shift /2 & goto %2 -set CALL=call %0 _call_ -set RETURN=goto end -:# Silent return. Used for routines silently called via call :label. -set _RETURN=goto _end -set PUTVARS=call :putvars - -:# Set global defaults -setlocal -set OUTFILE=obj\objects.mak -set OUTPATH=obj -set NO_EXEC=0 -goto get_args - -:help -echo Generate object files names from source files names -echo. -echo Usage: %ARG0% [options] source ... -echo. -echo Options: -echo -?^|-h This help -echo -o {pathname} Output file pathname. Default: %OUTFILE% -echo -X Display object files names, but do not create the output file -goto end - -:get_args -if .%1.==.. goto help -if .%1.==.-?. goto help -if .%1.==./?. goto help -if %1==-o shift & goto set_out -if %1==-X shift & goto no_exec -goto go - -:set_out -set OUTFILE=%1 -:# Split the path. Uses a fake drive @ to prevent prepending the current path on an existing drive. -for %%F in (@:"%OUTFILE%") do ( - set "OUTPATH=%%~dpF" -) -:# Remove the head "@:\ and tail \ to the path -set "OUTPATH=%OUTPATH:~4,-1%" -shift -goto get_args - -:no_exec -set NO_EXEC=1 -goto get_args - -:# Check prerequisites -:check -verify other 2>nul -setlocal enableextensions -if errorlevel 1 ( - echo>&2 Error: Unable to enable command extensions. - exit /b 1 -) -set VAR=before -if "%VAR%" == "before" ( - set VAR=after - if not "!VAR!" == "after" ( - echo>&2 Error: Delayed environment variable expansion must be enabled. - echo>&2 Please restart your cmd.exe shell with the /V option, - echo>&2 or set HKLM\Software\Microsoft\Command Processor\DelayedExpansion=1 - exit /b 1 - ) -) -%_RETURN% - -:go -%CALL% check -if errorlevel 1 exit /b 1 -set OBJECTS= -:next -if .%1.==.. goto done -for %%s in (%1) do ( - set EXT=%%~xs - set OBJ= - if .!EXT!.==..c. set OBJ=obj - if .!EXT!.==..C. set OBJ=obj - if .!EXT!.==..cpp. set OBJ=obj - if .!EXT!.==..CPP. set OBJ=obj - if .!EXT!.==..asm. set OBJ=obj - if .!EXT!.==..ASM. set OBJ=obj - if .!EXT!.==..rc. set OBJ=res - if .!EXT!.==..RC. set OBJ=res - if .!OBJ!.==.. ( - echo>&2 Error: Unsupported source type: !EXT! - echo>&2 Please add a conversion rule in %ARG0% - ) else ( - if .!OBJECTS!.==.. ( - set OBJECTS=%OUTPATH%\%%~ns.!OBJ! - ) else ( - set OBJECTS=!OBJECTS! %OUTPATH%\%%~ns.!OBJ! - ) - ) -) -shift -goto next -:done -echo OBJECTS=%OBJECTS% -if %NO_EXEC%==0 echo>%OUTFILE% OBJECTS=%OBJECTS% - -:end -:_end - diff --git a/deps/MsvcLibX/src/symlink.c b/deps/MsvcLibX/src/symlink.c index 0784fc9f41..ce2bbe58a2 100644 --- a/deps/MsvcLibX/src/symlink.c +++ b/deps/MsvcLibX/src/symlink.c @@ -27,17 +27,18 @@ #define _UTF8_SOURCE /* Generate the UTF-8 version of routines */ -#include +#include "msvcUnistd.h" #pragma comment(lib, "Mpr.lib") #include -#include "debugm.h" +#include "msvcDebugm.h" +#include "msvcLimits.h" #ifdef _WIN32 #include -#include "reparsept.h" +#include "msvcReparsept.h" /*---------------------------------------------------------------------------*\ * * diff --git a/deps/MsvcLibX/src/uname.c b/deps/MsvcLibX/src/uname.c index 81a4c3c49a..afa2ef0d5e 100644 --- a/deps/MsvcLibX/src/uname.c +++ b/deps/MsvcLibX/src/uname.c @@ -21,7 +21,7 @@ * Licensed under the Apache 2.0 license - www.apache.org/licenses/LICENSE-2.0 * \*****************************************************************************/ -#include "sys/utsname.h" +#include "sys/msvcTsname.h" #include /* For itoa() */ static char major[4] = {0}; diff --git a/deps/MsvcLibX/src/utime.c b/deps/MsvcLibX/src/utime.c index 0a1a323e0b..cbddfecb01 100644 --- a/deps/MsvcLibX/src/utime.c +++ b/deps/MsvcLibX/src/utime.c @@ -22,10 +22,10 @@ #define _CRT_SECURE_NO_WARNINGS 1 /* Avoid Visual C++ security warnings */ #include -#include /* Must be included before any direct or indirect inclusion */ -#include - -#include "debugm.h" +#include "sys/msvcTime.h" /* Must be included before any direct or indirect inclusion */ +#include "msvcUtime.h" +#include "msvcDebugm.h" +#include "msvcLimits.h" #if defined(_DEBUG) #include @@ -35,7 +35,7 @@ #include #include /* For MSVC's _get_osfhandle() */ -#include /* For MsvcLibX's ResolveLinks() */ +#include "msvcUnistd.h" /* For MsvcLibX's ResolveLinks() */ /* Convert a Windows FILETIME to a Unix time_t. A FILETIME is the number of 100-nanosecond intervals since January 1, 1601. diff --git a/deps/MsvcLibX/src/utimes.c b/deps/MsvcLibX/src/utimes.c index 7638b932b4..9326bb5f74 100644 --- a/deps/MsvcLibX/src/utimes.c +++ b/deps/MsvcLibX/src/utimes.c @@ -23,10 +23,10 @@ #define _CRT_SECURE_NO_WARNINGS 1 /* Avoid Visual C++ security warnings */ #include -#include /* Must be included before any direct or indirect inclusion */ -#include - -#include "debugm.h" +#include "sys/msvcTime.h" /* Must be included before any direct or indirect inclusion */ +#include "sys/msvcStat.h" +#include "msvcDebugm.h" +#include "msvcLimits.h" #if defined(_DEBUG) #include @@ -36,7 +36,7 @@ #include #include /* For MSVC's _get_osfhandle() */ -#include /* For MsvcLibX's ResolveLinks() */ +#include "msvcUnistd.h" /* For MsvcLibX's ResolveLinks() */ DEBUG_CODE( int Timeval2String(char *buf, size_t bufsize, const struct timeval *tvp) { diff --git a/deps/MsvcLibX/src/xfreopen.c b/deps/MsvcLibX/src/xfreopen.c index 772e81f34f..80311bbac6 100644 --- a/deps/MsvcLibX/src/xfreopen.c +++ b/deps/MsvcLibX/src/xfreopen.c @@ -18,7 +18,7 @@ #include #include #include -#include "xfreopen.h" +#include "msvcXfreopen.h" FILE *xfreopen(const char *filename, const char *mode, FILE *stream) { int iMode = 0; diff --git a/deps/pthread/pthread.h b/deps/pthread/pthread.h index e08bf22abc..87673398d0 100644 --- a/deps/pthread/pthread.h +++ b/deps/pthread/pthread.h @@ -228,6 +228,8 @@ typedef unsigned long ULONG_PTR; #include "config.h" #endif /* HAVE_PTW32_CONFIG_H */ +#include "msvcTime.h" + #if !defined(NEED_FTIME) #include #else /* NEED_FTIME */ diff --git a/deps/pthread/sched.h b/deps/pthread/sched.h index 8c1096d887..d9bcc2d27f 100644 --- a/deps/pthread/sched.h +++ b/deps/pthread/sched.h @@ -100,18 +100,19 @@ #endif #endif /* PTW32_SCHED_LEVEL >= PTW32_SCHED_LEVEL_MAX */ -#if (defined(__MINGW64__) || defined(__MINGW32__)) || defined(_UWIN) -# if PTW32_SCHED_LEVEL >= PTW32_SCHED_LEVEL_MAX -/* For pid_t */ -# include -/* Required by Unix 98 */ -# include -# else - typedef int pid_t; -# endif -#else - typedef int pid_t; -#endif +// #if (defined(__MINGW64__) || defined(__MINGW32__)) || defined(_UWIN) +// # if PTW32_SCHED_LEVEL >= PTW32_SCHED_LEVEL_MAX +// /* For pid_t */ +// # include +// /* Required by Unix 98 */ +// # include +// # else +// typedef int pid_t; +// # endif +// #else +// typedef int pid_t; +// #endif +#include /* Thread scheduling policies */ diff --git a/deps/pthread/semaphore.h b/deps/pthread/semaphore.h index ecc29fc33d..9714365b61 100644 --- a/deps/pthread/semaphore.h +++ b/deps/pthread/semaphore.h @@ -99,6 +99,8 @@ #endif #endif /* PTW32_SEMAPHORE_LEVEL >= PTW32_SEMAPHORE_LEVEL_MAX */ +#include "sys/msvcTypes.h" + #define _POSIX_SEMAPHORES #if defined(__cplusplus) diff --git a/src/os/inc/osFile.h b/src/os/inc/osFile.h index e99c64c2e0..dc19c8177c 100644 --- a/src/os/inc/osFile.h +++ b/src/os/inc/osFile.h @@ -56,18 +56,6 @@ int32_t taosFileRename(char *fullPath, char *suffix, char delimiter, char **dstP // TAOS_OS_FUNC_FILE_GETTMPFILEPATH void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath); -#ifdef TAOS_OS_FUNC_FILE_ISDIR - #define S_ISDIR(m) (((m) & 0170000) == (0040000)) -#endif - -#ifdef TAOS_OS_FUNC_FILE_ISREG - #define S_ISREG(m) !(S_ISDIR(m)) -#endif - -#ifdef TAOS_OS_FUNC_FILE_ISLNK - #define S_ISLNK(m) 0 -#endif - #ifndef TAOS_OS_FUNC_FILE_FTRUNCATE #define taosFtruncate ftruncate #endif diff --git a/src/os/inc/osWindows.h b/src/os/inc/osWindows.h index bdfb4acb5b..778474db0f 100644 --- a/src/os/inc/osWindows.h +++ b/src/os/inc/osWindows.h @@ -18,9 +18,7 @@ #include #include -#include #include -#include #include #include #include @@ -35,15 +33,17 @@ #include #include #include -#include -#include -#include -#include #include "winsock2.h" #include #include #include -#include +#include +#include +#include "msvcProcess.h" +#include "msvcDirect.h" +#include "msvcFcntl.h" +#include "sys/msvcStat.h" +#include "sys/msvcTypes.h" #ifdef __cplusplus extern "C" { @@ -126,7 +126,6 @@ char *stpncpy (char *dest, const char *src, size_t n); typedef int (*__compar_fn_t)(const void *, const void *); #define ssize_t int #define bzero(ptr, size) memset((ptr), 0, (size)) -#define mkdir(pathname, mode) _mkdir(pathname) #define strcasecmp _stricmp #define strncasecmp _strnicmp #define wcsncasecmp _wcsnicmp @@ -138,7 +137,6 @@ typedef int (*__compar_fn_t)(const void *, const void *); #define twrite write #define getpid _getpid -int gettimeofday(struct timeval *tv, struct timezone *tz); struct tm *localtime_r(const time_t *timep, struct tm *result); char * strptime(const char *buf, const char *fmt, struct tm *tm); char * strsep(char **stringp, const char *delim); @@ -147,11 +145,6 @@ int flock(int fd, int option); int fsync(int filedes); char * strndup(const char *s, size_t n); -// for function open in stat.h -#define S_IRWXU _S_IREAD -#define S_IRWXG _S_IWRITE -#define S_IRWXO _S_IWRITE - // for access function in io.h #define F_OK 00 //Existence only #define W_OK 02 //Write - only diff --git a/src/os/src/windows/w64Time.c b/src/os/src/windows/w64Time.c index 1484b13843..ce7eada6a0 100644 --- a/src/os/src/windows/w64Time.c +++ b/src/os/src/windows/w64Time.c @@ -16,18 +16,6 @@ #include #include -int gettimeofday(struct timeval *tv, struct timezone *tz) { - time_t t; - t = time(NULL); - SYSTEMTIME st; - GetLocalTime(&st); - - tv->tv_sec = (long)t; - tv->tv_usec = st.wMilliseconds * 1000; - - return 0; -} - struct tm *localtime_r(const time_t *timep, struct tm *result) { localtime_s(result, timep); return result; diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index 4cb5f76414..b73db35bff 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -18,7 +18,6 @@ #include "tchecksum.h" #include "tsdbMain.h" #include "tutil.h" -#include "dirent.h" #define TAOS_RANDOM_FILE_FAIL_TEST @@ -203,7 +202,7 @@ void tsdbSeekFileGroupIter(SFileGroupIter *pIter, int fid) { pIter->index = -1; pIter->fileId = -1; } else { - pIter->index = POINTER_DISTANCE(ptr, pFileH->pFGroup) / sizeof(SFileGroup); + pIter->index = (int)(POINTER_DISTANCE(ptr, pFileH->pFGroup) / sizeof(SFileGroup)); pIter->fileId = ((SFileGroup *)ptr)->fileId; } } From 1db9831c5a554aa96ad22fd040858cdbe5a6ddb7 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 7 Aug 2020 19:04:48 +0800 Subject: [PATCH 063/109] TD-1057 --- src/tsdb/CMakeLists.txt | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/src/tsdb/CMakeLists.txt b/src/tsdb/CMakeLists.txt index 7831e492ca..cef1d0bba7 100644 --- a/src/tsdb/CMakeLists.txt +++ b/src/tsdb/CMakeLists.txt @@ -10,26 +10,6 @@ IF (TD_LINUX) # Someone has no gtest directory, so comment it # ADD_SUBDIRECTORY(tests) ELSEIF (TD_WINDOWS) -SET(CMAKE_MODULE_PATH ${TD_COMMUNITY_DIR}/deps/MsvcLibX/cmake) - include(FindWindowsSDK) - get_mywindowssdk_include_dirs(${WINDOWSSDK_PREFERRED_DIR} WINSDK_INCLUDE_DIR) - get_ucrt_include_dirs(${WINDOWSSDK_PREFERRED_DIR} UCRT_INCLUDE_DIR) - - Add_Definitions("-DWSDKINCLUDE=${WINSDK_INCLUDE_DIR}") - - include(FindMSVC) - Add_Definitions("-DMSVCINCLUDE=${VC_INCLUDE_DIR}") - - IF ((NOT DEFINED UCRT_INCLUDE_DIR) OR (UCRT_INCLUDE_DIR STREQUAL "NOTFOUND")) - Message(STATUS "UCRT_INCLUDE_DIR notu found, set it to vc incude dir") - SET(UCRT_INCLUDE_DIR "${VC_INCLUDE_DIR}") - Add_Definitions("-DUCRTINCLUDE=${VC_INCLUDE_DIR}") - ELSE () - Add_Definitions("-DUCRTINCLUDE=${UCRT_INCLUDE_DIR}") - ENDIF () - - INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/MsvcLibX/include) - ADD_LIBRARY(tsdb ${SRC}) TARGET_LINK_LIBRARIES(tsdb common tutil) ENDIF () From 87a73142a14160262fe4c7b89f7a56c9ab1cd8f9 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 7 Aug 2020 19:28:05 +0800 Subject: [PATCH 064/109] TD-1057 --- deps/MsvcLibX/include/msvclibx.h | 2 +- src/os/inc/osWindows.h | 1 + src/tsdb/src/tsdbFile.c | 6 +-- src/tsdb/src/tsdbMain.c | 14 +++---- src/tsdb/src/tsdbMemTable.c | 8 ++-- src/tsdb/src/tsdbMeta.c | 12 +++--- src/tsdb/src/tsdbRWHelper.c | 54 ++++++++++++------------ src/tsdb/src/tsdbRead.c | 70 ++++++++++++++++---------------- 8 files changed, 85 insertions(+), 82 deletions(-) diff --git a/deps/MsvcLibX/include/msvclibx.h b/deps/MsvcLibX/include/msvclibx.h index 31f0ddec33..4f5fc2733b 100644 --- a/deps/MsvcLibX/include/msvclibx.h +++ b/deps/MsvcLibX/include/msvclibx.h @@ -58,7 +58,7 @@ /* Generate the OS-and-debug-mode-specific library name */ #define _MSVCLIBX_LIB "MsvcLibX" _MSVCLIBX_LIB_OS_SUFFIX _MSVCLIBX_LIB_DBG_SUFFIX ".lib" -#pragma message("Adding pragma comment(lib, \"" _MSVCLIBX_LIB "\")") +//#pragma message("Adding pragma comment(lib, \"" _MSVCLIBX_LIB "\")") #pragma comment(lib, _MSVCLIBX_LIB) /* Library-specific routine used internally by many standard routines */ diff --git a/src/os/inc/osWindows.h b/src/os/inc/osWindows.h index 778474db0f..825ef4769d 100644 --- a/src/os/inc/osWindows.h +++ b/src/os/inc/osWindows.h @@ -144,6 +144,7 @@ char * getpass(const char *prefix); int flock(int fd, int option); int fsync(int filedes); char * strndup(const char *s, size_t n); +char * dirname(char *pszPathname); // for access function in io.h #define F_OK 00 //Existence only diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index b73db35bff..ee3bad9835 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -305,8 +305,8 @@ void tsdbFitRetention(STsdbRepo *pRepo) { STsdbFileH *pFileH = pRepo->tsdbFileH; SFileGroup *pGroup = pFileH->pFGroup; - int mfid = TSDB_KEY_FILEID(taosGetTimestamp(pCfg->precision), pCfg->daysPerFile, pCfg->precision) - - TSDB_MAX_FILE(pCfg->keep, pCfg->daysPerFile); + int mfid = (int)(TSDB_KEY_FILEID(taosGetTimestamp(pCfg->precision), pCfg->daysPerFile, pCfg->precision) - + TSDB_MAX_FILE(pCfg->keep, pCfg->daysPerFile)); pthread_rwlock_wrlock(&(pFileH->fhlock)); @@ -371,7 +371,7 @@ void tsdbRemoveFileGroup(STsdbRepo *pRepo, SFileGroup *pFGroup) { SFileGroup fileGroup = *pFGroup; - int nFilesLeft = pFileH->nFGroups - (POINTER_DISTANCE(pFGroup, pFileH->pFGroup) / sizeof(SFileGroup) + 1); + int nFilesLeft = pFileH->nFGroups - (int)(POINTER_DISTANCE(pFGroup, pFileH->pFGroup) / sizeof(SFileGroup) + 1); if (nFilesLeft > 0) { memmove((void *)pFGroup, POINTER_SHIFT(pFGroup, sizeof(SFileGroup)), sizeof(SFileGroup) * nFilesLeft); } diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 46801d0788..3acad05504 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -211,7 +211,7 @@ uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_ char *sdup = strdup(pRepo->rootDir); char *prefix = dirname(sdup); - int prefixLen = strlen(prefix); + int prefixLen = (int)strlen(prefix); taosTFree(sdup); if (name[0] == 0) { // get the file from index or after, but not larger than eindex @@ -232,7 +232,7 @@ uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_ fname = strdup(pFGroup->files[(*index) % TSDB_FILE_TYPE_MAX].fname); magic = pFGroup->files[(*index) % TSDB_FILE_TYPE_MAX].info.magic; } else { - if ((pFGroup->fileId + 1) * TSDB_FILE_TYPE_MAX - 1 < eindex) { + if ((pFGroup->fileId + 1) * TSDB_FILE_TYPE_MAX - 1 < (int)eindex) { fname = strdup(pFGroup->files[0].fname); *index = pFGroup->fileId * TSDB_FILE_TYPE_MAX; magic = pFGroup->files[0].info.magic; @@ -327,7 +327,7 @@ void tsdbReportStat(void *repo, int64_t *totalPoints, int64_t *totalStorage, int // ----------------- INTERNAL FUNCTIONS ----------------- char *tsdbGetMetaFileName(char *rootDir) { - int tlen = strlen(rootDir) + strlen(TSDB_META_FILE_NAME) + 2; + int tlen = (int)(strlen(rootDir) + strlen(TSDB_META_FILE_NAME) + 2); char *fname = calloc(1, tlen); if (fname == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; @@ -366,7 +366,7 @@ int tsdbUnlockRepo(STsdbRepo *pRepo) { } char *tsdbGetDataDirName(char *rootDir) { - int tlen = strlen(rootDir) + strlen(TSDB_DATA_DIR_NAME) + 2; + int tlen = (int)(strlen(rootDir) + strlen(TSDB_DATA_DIR_NAME) + 2); char *fname = calloc(1, tlen); if (fname == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; @@ -627,7 +627,7 @@ _err: } static char *tsdbGetCfgFname(char *rootDir) { - int tlen = strlen(rootDir) + strlen(TSDB_CFG_FILE_NAME) + 2; + int tlen = (int)(strlen(rootDir) + strlen(TSDB_CFG_FILE_NAME) + 2); char *fname = calloc(1, tlen); if (fname == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; @@ -853,8 +853,8 @@ static int tsdbAlterKeep(STsdbRepo *pRepo, int32_t keep) { return -1; } - int mfid = TSDB_KEY_FILEID(taosGetTimestamp(pCfg->precision), pCfg->daysPerFile, pCfg->precision) - - TSDB_MAX_FILE(keep, pCfg->daysPerFile); + int mfid = (int)(TSDB_KEY_FILEID(taosGetTimestamp(pCfg->precision), pCfg->daysPerFile, pCfg->precision) - + TSDB_MAX_FILE(keep, pCfg->daysPerFile)); int i = 0; for (; i < pFileH->nFGroups; i++) { diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 990db76b7e..02c3ee4d85 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -307,7 +307,7 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey keyNext = tsdbNextIterKey(pIter); if (keyNext < 0 || keyNext > maxKey) return numOfRows; void *ptr = taosbsearch((void *)(&keyNext), (void *)filterKeys, nFilterKeys, sizeof(TSKEY), compTSKEY, TD_GE); - filterIter = (ptr == NULL) ? nFilterKeys : (POINTER_DISTANCE(ptr, filterKeys) / sizeof(TSKEY)); + filterIter = (ptr == NULL) ? nFilterKeys : (int)((POINTER_DISTANCE(ptr, filterKeys) / sizeof(TSKEY))); } do { @@ -364,7 +364,7 @@ static void tsdbFreeBytes(STsdbRepo *pRepo, void *ptr, int bytes) { tsdbTrace("vgId:%d free %d bytes to TSDB buffer pool, nBlocks %d offset %d remain %d", REPO_ID(pRepo), bytes, listNEles(pRepo->mem->bufBlockList), pBufBlock->offset, pBufBlock->remain); } else { - SListNode *pNode = (SListNode *)POINTER_SHIFT(ptr, -sizeof(SListNode)); + SListNode *pNode = (SListNode *)POINTER_SHIFT(ptr, -(int)(sizeof(SListNode))); ASSERT(listTail(pRepo->mem->extraBuffList) == pNode); tdListPopNode(pRepo->mem->extraBuffList, pNode); free(pNode); @@ -495,8 +495,8 @@ static void *tsdbCommitData(void *arg) { goto _exit; } - int sfid = TSDB_KEY_FILEID(pMem->keyFirst, pCfg->daysPerFile, pCfg->precision); - int efid = TSDB_KEY_FILEID(pMem->keyLast, pCfg->daysPerFile, pCfg->precision); + int sfid = (int)(TSDB_KEY_FILEID(pMem->keyFirst, pCfg->daysPerFile, pCfg->precision)); + int efid = (int)(TSDB_KEY_FILEID(pMem->keyLast, pCfg->daysPerFile, pCfg->precision)); // Loop to commit to each file for (int fid = sfid; fid <= efid; fid++) { diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index c0d58f6332..eb1f1fc340 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -443,7 +443,7 @@ STsdbMeta *tsdbNewMeta(STsdbCfg *pCfg) { goto _err; } - pMeta->uidMap = taosHashInit(TSDB_INIT_NTABLES * 1.1, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false); + pMeta->uidMap = taosHashInit((size_t)(TSDB_INIT_NTABLES * 1.1), taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false); if (pMeta->uidMap == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _err; @@ -672,7 +672,7 @@ static STable *tsdbNewTable(STableCfg *pCfg, bool isSuper) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _err; } - STR_WITH_SIZE_TO_VARSTR(pTable->name, pCfg->sname, tsize); + STR_WITH_SIZE_TO_VARSTR(pTable->name, pCfg->sname, (VarDataLenT)tsize); TABLE_UID(pTable) = pCfg->superUid; TABLE_TID(pTable) = -1; TABLE_SUID(pTable) = -1; @@ -690,7 +690,7 @@ static STable *tsdbNewTable(STableCfg *pCfg, bool isSuper) { } pTable->tagVal = NULL; STColumn *pCol = schemaColAt(pTable->tagSchema, DEFAULT_TAG_INDEX_COLUMN); - pTable->pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, colType(pCol), colBytes(pCol), 1, 0, 1, getTagIndexKey); + pTable->pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, colType(pCol), (uint8_t)(colBytes(pCol)), 1, 0, 1, getTagIndexKey); if (pTable->pIndex == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _err; @@ -703,7 +703,7 @@ static STable *tsdbNewTable(STableCfg *pCfg, bool isSuper) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _err; } - STR_WITH_SIZE_TO_VARSTR(pTable->name, pCfg->name, tsize); + STR_WITH_SIZE_TO_VARSTR(pTable->name, pCfg->name, (VarDataLenT)tsize); TABLE_UID(pTable) = pCfg->tableId.uid; TABLE_TID(pTable) = pCfg->tableId.tid; @@ -1165,7 +1165,7 @@ static void *tsdbDecodeTable(void *buf, STable **pRTable) { buf = tdDecodeSchema(buf, &(pTable->tagSchema)); STColumn *pCol = schemaColAt(pTable->tagSchema, DEFAULT_TAG_INDEX_COLUMN); pTable->pIndex = - tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, colType(pCol), colBytes(pCol), 1, 0, 1, getTagIndexKey); + tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, colType(pCol), (uint8_t)(colBytes(pCol)), 1, 0, 1, getTagIndexKey); if (pTable->pIndex == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; tsdbFreeTable(pTable); @@ -1191,7 +1191,7 @@ static int tsdbGetTableEncodeSize(int8_t act, STable *pTable) { tlen = sizeof(SListNode) + sizeof(SActObj) + sizeof(SActCont) + tsdbEncodeTable(NULL, pTable) + sizeof(TSCKSUM); } else { if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) { - tlen = (sizeof(SListNode) + sizeof(SActObj)) * (tSkipListGetSize(pTable->pIndex) + 1); + tlen = (int)((sizeof(SListNode) + sizeof(SActObj)) * (tSkipListGetSize(pTable->pIndex) + 1)); } else { tlen = sizeof(SListNode) + sizeof(SActObj); } diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index fb79747d30..d63bf8ab78 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -302,7 +302,7 @@ int tsdbCommitTableData(SRWHelper *pHelper, SCommitIter *pCommitIter, SDataCols if (tsdbLoadCompInfo(pHelper, NULL) < 0) return -1; while (true) { - ASSERT(blkIdx <= pIdx->numOfBlocks); + ASSERT(blkIdx <= (int)pIdx->numOfBlocks); TSKEY keyFirst = tsdbNextIterKey(pCommitIter->pIter); if (keyFirst < 0 || keyFirst > maxKey) break; // iter over @@ -405,7 +405,7 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) { pIdx->tid = pHelper->tableInfo.tid; ASSERT(pIdx->offset >= TSDB_FILE_HEAD_SIZE); - if (taosTWrite(pFile->fd, (void *)(pHelper->pCompInfo), pIdx->len) < pIdx->len) { + if (taosTWrite(pFile->fd, (void *)(pHelper->pCompInfo), pIdx->len) < (int)pIdx->len) { tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(pHelper->pRepo), pIdx->len, pFile->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); @@ -462,7 +462,7 @@ int tsdbWriteCompIdx(SRWHelper *pHelper) { pFile->info.offset = offset; - if (taosTWrite(pFile->fd, (void *)pHelper->pWIdx, pFile->info.len) < pFile->info.len) { + if (taosTWrite(pFile->fd, (void *)pHelper->pWIdx, pFile->info.len) < (int)pFile->info.len) { tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(pHelper->pRepo), pFile->info.len, pFile->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); @@ -495,7 +495,7 @@ int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) { return -1; } - if (taosTRead(fd, (void *)(pHelper->pBuffer), pFile->info.len) < pFile->info.len) { + if (taosTRead(fd, (void *)(pHelper->pBuffer), pFile->info.len) < (int)pFile->info.len) { tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pHelper->pRepo), pFile->info.len, pFile->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); @@ -512,7 +512,7 @@ int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) { // Decode it pHelper->idxH.numOfIdx = 0; void *ptr = pHelper->pBuffer; - while (POINTER_DISTANCE(ptr, pHelper->pBuffer) < (pFile->info.len - sizeof(TSCKSUM))) { + while (POINTER_DISTANCE(ptr, pHelper->pBuffer) < (int)(pFile->info.len - sizeof(TSCKSUM))) { size_t tlen = taosTSizeof(pHelper->idxH.pIdxArray); pHelper->idxH.numOfIdx++; @@ -535,7 +535,7 @@ int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) { ASSERT(pHelper->idxH.numOfIdx == 1 || pHelper->idxH.pIdxArray[pHelper->idxH.numOfIdx - 1].tid > pHelper->idxH.pIdxArray[pHelper->idxH.numOfIdx - 2].tid); - ASSERT(POINTER_DISTANCE(ptr, pHelper->pBuffer) <= pFile->info.len - sizeof(TSCKSUM)); + ASSERT(POINTER_DISTANCE(ptr, pHelper->pBuffer) <= (int)(pFile->info.len - sizeof(TSCKSUM))); } } } @@ -566,7 +566,7 @@ int tsdbLoadCompInfo(SRWHelper *pHelper, void *target) { } pHelper->pCompInfo = taosTRealloc((void *)pHelper->pCompInfo, pIdx->len); - if (taosTRead(fd, (void *)(pHelper->pCompInfo), pIdx->len) < pIdx->len) { + if (taosTRead(fd, (void *)(pHelper->pCompInfo), pIdx->len) < (int)pIdx->len) { tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pHelper->pRepo), pIdx->len, helperHeadF(pHelper)->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); @@ -594,7 +594,7 @@ int tsdbLoadCompData(SRWHelper *pHelper, SCompBlock *pCompBlock, void *target) { ASSERT(pCompBlock->numOfSubBlocks <= 1); SFile *pFile = (pCompBlock->last) ? helperLastF(pHelper) : helperDataF(pHelper); - if (lseek(pFile->fd, pCompBlock->offset, SEEK_SET) < 0) { + if (lseek(pFile->fd, (off_t)pCompBlock->offset, SEEK_SET) < 0) { tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -614,7 +614,7 @@ int tsdbLoadCompData(SRWHelper *pHelper, SCompBlock *pCompBlock, void *target) { return -1; } - if (!taosCheckChecksumWhole((uint8_t *)pHelper->pCompData, tsize)) { + if (!taosCheckChecksumWhole((uint8_t *)pHelper->pCompData, (uint32_t)tsize)) { tsdbError("vgId:%d file %s is broken, offset %" PRId64 " size %zu", REPO_ID(pHelper->pRepo), pFile->fname, (int64_t)pCompBlock->offset, tsize); terrno = TSDB_CODE_TDB_FILE_CORRUPTED; @@ -787,8 +787,8 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa } flen = (*(tDataTypeDesc[pDataCol->type].compFunc))((char *)pDataCol->pData, tlen, rowsToWrite, tptr, - taosTSizeof(pHelper->pBuffer) - lsize, pCfg->compression, - pHelper->compBuffer, taosTSizeof(pHelper->compBuffer)); + (int32_t)taosTSizeof(pHelper->pBuffer) - lsize, pCfg->compression, + pHelper->compBuffer, (int32_t)taosTSizeof(pHelper->compBuffer)); } else { flen = tlen; memcpy(tptr, pDataCol->pData, flen); @@ -879,7 +879,7 @@ static int tsdbAdjustInfoSizeIfNeeded(SRWHelper *pHelper, size_t esize) { static int tsdbInsertSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx) { SCompIdx *pIdx = &(pHelper->curCompIdx); - ASSERT(blkIdx >= 0 && blkIdx <= pIdx->numOfBlocks); + ASSERT(blkIdx >= 0 && blkIdx <= (int)pIdx->numOfBlocks); ASSERT(pCompBlock->numOfSubBlocks == 1); // Adjust memory if no more room @@ -887,7 +887,7 @@ static int tsdbInsertSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int if (tsdbAdjustInfoSizeIfNeeded(pHelper, pIdx->len + sizeof(SCompInfo)) < 0) goto _err; // Change the offset - for (int i = 0; i < pIdx->numOfBlocks; i++) { + for (uint32_t i = 0; i < pIdx->numOfBlocks; i++) { SCompBlock *pTCompBlock = &pHelper->pCompInfo->blocks[i]; if (pTCompBlock->numOfSubBlocks > 1) pTCompBlock->offset += sizeof(SCompBlock); } @@ -906,7 +906,7 @@ static int tsdbInsertSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int pIdx->len += sizeof(SCompBlock); ASSERT(pIdx->len <= taosTSizeof(pHelper->pCompInfo)); pIdx->maxKey = blockAtIdx(pHelper, pIdx->numOfBlocks - 1)->keyLast; - pIdx->hasLast = blockAtIdx(pHelper, pIdx->numOfBlocks - 1)->last; + pIdx->hasLast = (uint32_t)blockAtIdx(pHelper, pIdx->numOfBlocks - 1)->last; if (pIdx->numOfBlocks > 1) { ASSERT(pHelper->pCompInfo->blocks[0].keyLast < pHelper->pCompInfo->blocks[1].keyFirst); @@ -925,7 +925,7 @@ static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkId ASSERT(pCompBlock->numOfSubBlocks == 0); SCompIdx *pIdx = &(pHelper->curCompIdx); - ASSERT(blkIdx >= 0 && blkIdx < pIdx->numOfBlocks); + ASSERT(blkIdx >= 0 && blkIdx < (int)pIdx->numOfBlocks); SCompBlock *pSCompBlock = pHelper->pCompInfo->blocks + blkIdx; ASSERT(pSCompBlock->numOfSubBlocks >= 1 && pSCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS); @@ -943,7 +943,7 @@ static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkId memmove((void *)((char *)(pHelper->pCompInfo) + pSCompBlock->offset + pSCompBlock->len + sizeof(SCompBlock)), (void *)((char *)(pHelper->pCompInfo) + pSCompBlock->offset + pSCompBlock->len), tsize); - for (int i = blkIdx + 1; i < pIdx->numOfBlocks; i++) { + for (uint32_t i = blkIdx + 1; i < pIdx->numOfBlocks; i++) { SCompBlock *pTCompBlock = &pHelper->pCompInfo->blocks[i]; if (pTCompBlock->numOfSubBlocks > 1) pTCompBlock->offset += sizeof(SCompBlock); } @@ -960,7 +960,7 @@ static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkId pIdx->len += sizeof(SCompBlock); } else { // Need to create two sub-blocks void *ptr = NULL; - for (int i = blkIdx + 1; i < pIdx->numOfBlocks; i++) { + for (uint32_t i = blkIdx + 1; i < pIdx->numOfBlocks; i++) { SCompBlock *pTCompBlock = pHelper->pCompInfo->blocks + i; if (pTCompBlock->numOfSubBlocks > 1) { ptr = POINTER_SHIFT(pHelper->pCompInfo, pTCompBlock->offset); @@ -973,7 +973,7 @@ static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkId size_t tsize = pIdx->len - ((char *)ptr - (char *)(pHelper->pCompInfo)); if (tsize > 0) { memmove(POINTER_SHIFT(ptr, sizeof(SCompBlock) * 2), ptr, tsize); - for (int i = blkIdx + 1; i < pIdx->numOfBlocks; i++) { + for (uint32_t i = blkIdx + 1; i < pIdx->numOfBlocks; i++) { SCompBlock *pTCompBlock = pHelper->pCompInfo->blocks + i; if (pTCompBlock->numOfSubBlocks > 1) pTCompBlock->offset += (sizeof(SCompBlock) * 2); } @@ -995,7 +995,7 @@ static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkId } pIdx->maxKey = pHelper->pCompInfo->blocks[pIdx->numOfBlocks - 1].keyLast; - pIdx->hasLast = pHelper->pCompInfo->blocks[pIdx->numOfBlocks - 1].last; + pIdx->hasLast = (uint32_t)pHelper->pCompInfo->blocks[pIdx->numOfBlocks - 1].last; tsdbDebug("vgId:%d tid:%d a subblock is added at index %d", REPO_ID(pHelper->pRepo), pHelper->tableInfo.tid, blkIdx); @@ -1010,7 +1010,7 @@ static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int SCompIdx *pIdx = &(pHelper->curCompIdx); - ASSERT(blkIdx >= 0 && blkIdx < pIdx->numOfBlocks); + ASSERT(blkIdx >= 0 && blkIdx < (int)pIdx->numOfBlocks); SCompBlock *pSCompBlock = pHelper->pCompInfo->blocks + blkIdx; @@ -1024,7 +1024,7 @@ static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int POINTER_SHIFT(pHelper->pCompInfo, pSCompBlock->offset + pSCompBlock->len), tsize); } - for (int i = blkIdx + 1; i < pIdx->numOfBlocks; i++) { + for (uint32_t i = blkIdx + 1; i < pIdx->numOfBlocks; i++) { SCompBlock *pTCompBlock = &pHelper->pCompInfo->blocks[i]; if (pTCompBlock->numOfSubBlocks > 1) pTCompBlock->offset -= (sizeof(SCompBlock) * pSCompBlock->numOfSubBlocks); } @@ -1035,7 +1035,7 @@ static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int *pSCompBlock = *pCompBlock; pIdx->maxKey = blockAtIdx(pHelper, pIdx->numOfBlocks - 1)->keyLast; - pIdx->hasLast = blockAtIdx(pHelper, pIdx->numOfBlocks - 1)->last; + pIdx->hasLast = (uint32_t)blockAtIdx(pHelper, pIdx->numOfBlocks - 1)->last; tsdbDebug("vgId:%d tid:%d a super block is updated at index %d", REPO_ID(pHelper->pRepo), pHelper->tableInfo.tid, blkIdx); @@ -1197,7 +1197,7 @@ static int tsdbLoadColData(SRWHelper *pHelper, SFile *pFile, SCompBlock *pCompBl } int64_t offset = pCompBlock->offset + TSDB_GET_COMPCOL_LEN(pCompBlock->numOfCols) + pCompCol->offset; - if (lseek(pFile->fd, offset, SEEK_SET) < 0) { + if (lseek(pFile->fd, (off_t)offset, SEEK_SET) < 0) { tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; @@ -1212,7 +1212,7 @@ static int tsdbLoadColData(SRWHelper *pHelper, SFile *pFile, SCompBlock *pCompBl if (tsdbCheckAndDecodeColumnData(pDataCol, pHelper->pBuffer, pCompCol->len, pCompBlock->algorithm, pCompBlock->numOfRows, pHelper->pRepo->config.maxRowsPerFileBlock, - pHelper->compBuffer, taosTSizeof(pHelper->compBuffer)) < 0) { + pHelper->compBuffer, (int32_t)taosTSizeof(pHelper->compBuffer)) < 0) { tsdbError("vgId:%d file %s is broken at column %d offset %" PRId64, REPO_ID(pHelper->pRepo), pFile->fname, pCompCol->colId, offset); return -1; @@ -1312,7 +1312,7 @@ static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDa SCompData *pCompData = (SCompData *)pHelper->pBuffer; int fd = pFile->fd; - if (lseek(fd, pCompBlock->offset, SEEK_SET) < 0) { + if (lseek(fd, (off_t)pCompBlock->offset, SEEK_SET) < 0) { tsdbError("vgId:%d tid:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pHelper->tableInfo.tid, pFile->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); @@ -1375,7 +1375,7 @@ static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDa } if (tsdbCheckAndDecodeColumnData(pDataCol, (char *)pCompData + tsize + toffset, tlen, pCompBlock->algorithm, pCompBlock->numOfRows, pDataCols->maxPoints, pHelper->compBuffer, - taosTSizeof(pHelper->compBuffer)) < 0) { + (int32_t)taosTSizeof(pHelper->compBuffer)) < 0) { tsdbError("vgId:%d file %s is broken at column %d block offset %" PRId64 " column offset %d", REPO_ID(pHelper->pRepo), pFile->fname, tcolId, (int64_t)pCompBlock->offset, toffset); goto _err; @@ -1499,7 +1499,7 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, SCompBlock *pCompBlock = taosbsearch((void *)(&keyFirst), (void *)blockAtIdx(pHelper, *blkIdx), pIdx->numOfBlocks - *blkIdx, sizeof(SCompBlock), compareKeyBlock, TD_GE); ASSERT(pCompBlock != NULL); - int tblkIdx = TSDB_GET_COMPBLOCK_IDX(pHelper, pCompBlock); + int tblkIdx = (int32_t)(TSDB_GET_COMPBLOCK_IDX(pHelper, pCompBlock)); if (pCompBlock->last) { ASSERT(pCompBlock->numOfRows < pCfg->minRowsPerFileBlock && tblkIdx == pIdx->numOfBlocks - 1); diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index a34216d2fb..97fc467ba4 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -235,9 +235,10 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab STableCheckInfo info = { .lastKey = pQueryHandle->window.skey, - .tableId = pTable->tableId, + //.tableId = pTable->tableId, .pTableObj = pTable, }; + info.tableId = pTable->tableId; assert(info.pTableObj != NULL && (info.pTableObj->type == TSDB_NORMAL_TABLE || info.pTableObj->type == TSDB_CHILD_TABLE || info.pTableObj->type == TSDB_STREAM_TABLE)); @@ -523,7 +524,7 @@ static int32_t getFileIdFromKey(TSKEY key, int32_t daysPerFile, int32_t precisio fid = INT32_MAX; } - return fid; + return (int32_t)fid; } static int32_t binarySearchForBlock(SCompBlock* pBlock, int32_t numOfBlocks, TSKEY skey, int32_t order) { @@ -571,7 +572,7 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo continue; // no data blocks in the file belongs to pCheckInfo->pTable } - if (pCheckInfo->compSize < compIndex->len) { + if (pCheckInfo->compSize < (int32_t)compIndex->len) { assert(compIndex->len > 0); char* t = realloc(pCheckInfo->pCompInfo, compIndex->len); @@ -604,7 +605,7 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo } // todo speedup the procedure of located end block - while (end < compIndex->numOfBlocks && (pCompInfo->blocks[end].keyFirst <= e)) { + while (end < (int32_t)compIndex->numOfBlocks && (pCompInfo->blocks[end].keyFirst <= e)) { end += 1; } @@ -644,7 +645,7 @@ static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlo tdInitDataCols(pQueryHandle->rhelper.pDataCols[1], pSchema); int16_t* colIds = pQueryHandle->defaultLoadColumn->pData; - int32_t ret = tsdbLoadBlockDataCols(&(pQueryHandle->rhelper), pBlock, pCheckInfo->pCompInfo, colIds, QH_GET_NUM_OF_COLS(pQueryHandle)); + int32_t ret = tsdbLoadBlockDataCols(&(pQueryHandle->rhelper), pBlock, pCheckInfo->pCompInfo, colIds, (int)(QH_GET_NUM_OF_COLS(pQueryHandle))); if (ret == TSDB_CODE_SUCCESS) { SDataBlockLoadInfo* pBlockLoadInfo = &pQueryHandle->dataBlockLoadInfo; @@ -846,7 +847,7 @@ static int32_t copyDataFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t cap return numOfRows; } - int32_t requiredNumOfCols = taosArrayGetSize(pQueryHandle->pColumns); + int32_t requiredNumOfCols = (int32_t)taosArrayGetSize(pQueryHandle->pColumns); //data in buffer has greater timestamp, copy data in file block int32_t i = 0, j = 0; @@ -862,15 +863,15 @@ static int32_t copyDataFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t cap int32_t bytes = pColInfo->info.bytes; if (ASCENDING_TRAVERSE(pQueryHandle->order)) { - pData = pColInfo->pData + numOfRows * pColInfo->info.bytes; + pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; } else { - pData = pColInfo->pData + (capacity - numOfRows - num) * pColInfo->info.bytes; + pData = (char*)pColInfo->pData + (capacity - numOfRows - num) * pColInfo->info.bytes; } if (pColInfo->info.colId == src->colId) { if (pColInfo->info.type != TSDB_DATA_TYPE_BINARY && pColInfo->info.type != TSDB_DATA_TYPE_NCHAR) { - memmove(pData, src->pData + bytes * start, bytes * num); + memmove(pData, (char*)src->pData + bytes * start, bytes * num); } else { // handle the var-string char* dst = pData; @@ -902,9 +903,9 @@ static int32_t copyDataFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t cap while (i < requiredNumOfCols) { // the remain columns are all null data SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); if (ASCENDING_TRAVERSE(pQueryHandle->order)) { - pData = pColInfo->pData + numOfRows * pColInfo->info.bytes; + pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; } else { - pData = pColInfo->pData + (capacity - numOfRows - num) * pColInfo->info.bytes; + pData = (char*)pColInfo->pData + (capacity - numOfRows - num) * pColInfo->info.bytes; } if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { @@ -944,13 +945,13 @@ static void copyOneRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, } if (ASCENDING_TRAVERSE(pQueryHandle->order)) { - pData = pColInfo->pData + numOfRows * pColInfo->info.bytes; + pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; } else { - pData = pColInfo->pData + (capacity - numOfRows - 1) * pColInfo->info.bytes; + pData = (char*)pColInfo->pData + (capacity - numOfRows - 1) * pColInfo->info.bytes; } if (pSchema->columns[j].colId == pColInfo->info.colId) { - void* value = tdGetRowDataOfCol(row, pColInfo->info.type, TD_DATA_ROW_HEAD_SIZE + pSchema->columns[j].offset); + void* value = tdGetRowDataOfCol(row, (int8_t)pColInfo->info.type, TD_DATA_ROW_HEAD_SIZE + pSchema->columns[j].offset); if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { memcpy(pData, value, varDataTLen(value)); } else { @@ -972,9 +973,9 @@ static void copyOneRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, while (i < numOfCols) { // the remain columns are all null data SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); if (ASCENDING_TRAVERSE(pQueryHandle->order)) { - pData = pColInfo->pData + numOfRows * pColInfo->info.bytes; + pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; } else { - pData = pColInfo->pData + (capacity - numOfRows - 1) * pColInfo->info.bytes; + pData = (char*)pColInfo->pData + (capacity - numOfRows - 1) * pColInfo->info.bytes; } if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { @@ -997,7 +998,7 @@ static void moveDataToFront(STsdbQueryHandle* pQueryHandle, int32_t numOfRows, i int32_t emptySize = pQueryHandle->outputCapacity - numOfRows; for(int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); - memmove(pColInfo->pData, pColInfo->pData + emptySize * pColInfo->info.bytes, numOfRows * pColInfo->info.bytes); + memmove((char*)pColInfo->pData, (char*)pColInfo->pData + emptySize * pColInfo->info.bytes, numOfRows * pColInfo->info.bytes); } } } @@ -1075,7 +1076,7 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* int32_t order = (pQueryHandle->order == TSDB_ORDER_ASC)? TSDB_ORDER_DESC:TSDB_ORDER_ASC; int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1:-1; - int32_t numOfCols = QH_GET_NUM_OF_COLS(pQueryHandle); + int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pQueryHandle)); STable* pTable = pCheckInfo->pTableObj; int32_t endPos = cur->pos; @@ -1326,7 +1327,7 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO size_t size = sizeof(STableBlockInfo) * numOfBlocks; if (pQueryHandle->allocSize < size) { - pQueryHandle->allocSize = size; + pQueryHandle->allocSize = (int32_t)size; char* tmp = realloc(pQueryHandle->pDataBlockInfo, pQueryHandle->allocSize); if (tmp == NULL) { return TSDB_CODE_TDB_OUT_OF_MEMORY; @@ -1338,7 +1339,7 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO memset(pQueryHandle->pDataBlockInfo, 0, size); *numOfAllocBlocks = numOfBlocks; - int32_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); + int32_t numOfTables = (int32_t)taosArrayGetSize(pQueryHandle->pTableCheckInfo); SBlockOrderSupporter sup = {0}; sup.numOfTables = numOfTables; @@ -1445,7 +1446,7 @@ static int32_t getDataBlocksInFilesImpl(STsdbQueryHandle* pQueryHandle, bool* ex int32_t code = TSDB_CODE_SUCCESS; int32_t numOfBlocks = 0; - int32_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); + int32_t numOfTables = (int32_t)taosArrayGetSize(pQueryHandle->pTableCheckInfo); STsdbCfg* pCfg = &pQueryHandle->pTsdb->config; STimeWindow win = TSWINDOW_INITIALIZER; @@ -1615,10 +1616,10 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { if (pQueryHandle->cur.win.ekey == pQueryHandle->window.skey) { // data already retrieve, discard other data rows and return - int32_t numOfCols = QH_GET_NUM_OF_COLS(pQueryHandle); + int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pQueryHandle)); for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i); - memcpy(pCol->pData, pCol->pData + pCol->info.bytes * (pQueryHandle->cur.rows-1), pCol->info.bytes); + memcpy((char*)pCol->pData, (char*)pCol->pData + pCol->info.bytes * (pQueryHandle->cur.rows - 1), pCol->info.bytes); } pQueryHandle->cur.win = (STimeWindow){pQueryHandle->window.skey, pQueryHandle->window.skey}; @@ -1646,7 +1647,7 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { tsdbTakeMemSnapshot(pSecQueryHandle->pTsdb, &pSecQueryHandle->mem, &pSecQueryHandle->imem); // allocate buffer in order to load data blocks from file - int32_t numOfCols = QH_GET_NUM_OF_COLS(pQueryHandle); + int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pQueryHandle)); pSecQueryHandle->statis = calloc(numOfCols, sizeof(SDataStatis)); pSecQueryHandle->pColumns = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); @@ -1669,9 +1670,10 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { STableCheckInfo* pCheckInfo = (STableCheckInfo*) taosArrayGet(pQueryHandle->pTableCheckInfo, j); STableCheckInfo info = { .lastKey = pSecQueryHandle->window.skey, - .tableId = pCheckInfo->tableId, + //.tableId = pCheckInfo->tableId, .pTableObj = pCheckInfo->pTableObj, }; + info.tableId = pCheckInfo->tableId; taosArrayPush(pSecQueryHandle->pTableCheckInfo, &info); } @@ -1688,12 +1690,12 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i); - memcpy(pCol->pData, pCol->pData + pCol->info.bytes * (pQueryHandle->cur.rows-1), pCol->info.bytes); + memcpy((char*)pCol->pData, (char*)pCol->pData + pCol->info.bytes * (pQueryHandle->cur.rows - 1), pCol->info.bytes); SColumnInfoData* pCol1 = taosArrayGet(pSecQueryHandle->pColumns, i); assert(pCol->info.colId == pCol1->info.colId); - memcpy(pCol->pData + pCol->info.bytes, pCol1->pData, pCol1->info.bytes); + memcpy((char*)pCol->pData + pCol->info.bytes, pCol1->pData, pCol1->info.bytes); } SColumnInfoData* pTSCol = taosArrayGet(pQueryHandle->pColumns, 0); @@ -1839,7 +1841,7 @@ static void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle) { static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, STimeWindow* win, STsdbQueryHandle* pQueryHandle) { int numOfRows = 0; - int32_t numOfCols = taosArrayGetSize(pQueryHandle->pColumns); + int32_t numOfCols = (int32_t)taosArrayGetSize(pQueryHandle->pColumns); win->skey = TSKEY_INITIAL_VAL; int64_t st = taosGetTimestampUs(); @@ -1881,7 +1883,7 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int for(int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); - memmove(pColInfo->pData, pColInfo->pData + emptySize * pColInfo->info.bytes, numOfRows * pColInfo->info.bytes); + memmove((char*)pColInfo->pData, (char*)pColInfo->pData + emptySize * pColInfo->info.bytes, numOfRows * pColInfo->info.bytes); } } @@ -1910,7 +1912,7 @@ void tsdbRetrieveDataBlockInfo(TsdbQueryHandleT* pQueryHandle, SDataBlockInfo* p pDataBlockInfo->tid = pTable->tableId.tid; pDataBlockInfo->rows = cur->rows; pDataBlockInfo->window = cur->win; - pDataBlockInfo->numOfCols = QH_GET_NUM_OF_COLS(pHandle); + pDataBlockInfo->numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pHandle)); } /* @@ -1945,7 +1947,7 @@ int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT* pQueryHandle, SDataSta pHandle->statis[i].colId = colIds[i]; } - tsdbGetDataStatis(&pHandle->rhelper, pHandle->statis, numOfCols); + tsdbGetDataStatis(&pHandle->rhelper, pHandle->statis, (int)numOfCols); // always load the first primary timestamp column data SDataStatis* pPrimaryColStatis = &pHandle->statis[0]; @@ -2010,11 +2012,11 @@ SArray* tsdbRetrieveDataBlock(TsdbQueryHandleT* pQueryHandle, SArray* pIdList) { // if the buffer is not full in case of descending order query, move the data in the front of the buffer if (!ASCENDING_TRAVERSE(pHandle->order) && numOfRows < pHandle->outputCapacity) { int32_t emptySize = pHandle->outputCapacity - numOfRows; - int32_t reqNumOfCols = taosArrayGetSize(pHandle->pColumns); + int32_t reqNumOfCols = (int32_t)taosArrayGetSize(pHandle->pColumns); for(int32_t i = 0; i < reqNumOfCols; ++i) { SColumnInfoData* pColInfo = taosArrayGet(pHandle->pColumns, i); - memmove(pColInfo->pData, pColInfo->pData + emptySize * pColInfo->info.bytes, numOfRows * pColInfo->info.bytes); + memmove((char*)pColInfo->pData, (char*)pColInfo->pData + emptySize * pColInfo->info.bytes, numOfRows * pColInfo->info.bytes); } } @@ -2332,7 +2334,7 @@ int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T* tsdb, uint64_t uid, const char* pT THROW( TSDB_CODE_TDB_OUT_OF_MEMORY ); } expr->nodeType = TSQL_NODE_EXPR; - expr->_node.optr = tagNameRelType; + expr->_node.optr = (uint8_t)tagNameRelType; expr->_node.pLeft = tagExpr; expr->_node.pRight = tbnameExpr; } From aea35d6b23248e7cfc950e6bf65297452e348c55 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 8 Aug 2020 10:35:57 +0800 Subject: [PATCH 065/109] [td-225] fix bugs. --- src/client/inc/tsclient.h | 1 + src/client/src/tscLocalMerge.c | 73 +++++++++++++--------------------- src/client/src/tscSQLParser.c | 6 +-- 3 files changed, 32 insertions(+), 48 deletions(-) diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 49f7cec889..6d02bc7fbd 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -256,6 +256,7 @@ typedef struct SResRec { typedef struct { int64_t numOfRows; // num of results in current retrieved + int64_t numOfRowsGroup; // num of results of current group int64_t numOfTotal; // num of total results int64_t numOfClauseTotal; // num of total result in current subclause char * pRsp; diff --git a/src/client/src/tscLocalMerge.c b/src/client/src/tscLocalMerge.c index 95f45ad105..ff5fbb2699 100644 --- a/src/client/src/tscLocalMerge.c +++ b/src/client/src/tscLocalMerge.c @@ -856,24 +856,6 @@ void savePrevRecordAndSetupInterpoInfo(SLocalReducer *pLocalReducer, SQueryInfo tColModelAppend(pModel, pLocalReducer->discardData, pLocalReducer->prevRowOfInput, 0, 1, 1); } -// todo merge with following function -// static void reversedCopyResultToDstBuf(SQueryInfo* pQueryInfo, SSqlRes *pRes, tFilePage *pFinalDataPage) { -// -// for (int32_t i = 0; i < pQueryInfo->exprList.numOfExprs; ++i) { -// TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); -// -// int32_t offset = tscFieldInfoGetOffset(pQueryInfo, i); -// char * src = pFinalDataPage->data + (pRes->numOfRows - 1) * pField->bytes + pRes->numOfRows * offset; -// char * dst = pRes->data + pRes->numOfRows * offset; -// -// for (int32_t j = 0; j < pRes->numOfRows; ++j) { -// memcpy(dst, src, (size_t)pField->bytes); -// dst += pField->bytes; -// src -= pField->bytes; -// } -// } -//} - static void reversedCopyFromInterpolationToDstBuf(SQueryInfo *pQueryInfo, SSqlRes *pRes, tFilePage **pResPages, SLocalReducer *pLocalReducer) { assert(0); @@ -907,20 +889,10 @@ static void doFillResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneO tFilePage * pFinalDataPage = pLocalReducer->pResultBuf; SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); -// if (pRes->pLocalReducer != pLocalReducer) { -// /* -// * Release the SSqlObj is called, and it is int destroying function invoked by other thread. -// * However, the other thread will WAIT until current process fully completes. -// * Since the flag of release struct is set by doLocalReduce function -// */ -// assert(pRes->pLocalReducer == NULL); -// } - // no interval query, no fill operation if (pQueryInfo->intervalTime == 0 || pQueryInfo->fillType == TSDB_FILL_NONE) { pRes->data = pLocalReducer->pFinalRes; pRes->numOfRows = pFinalDataPage->num; - pRes->numOfClauseTotal += pRes->numOfRows; if (pQueryInfo->limit.offset > 0) { if (pQueryInfo->limit.offset < pRes->numOfRows) { @@ -931,22 +903,22 @@ static void doFillResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneO tColModelCompact(pLocalReducer->resColModel, pFinalDataPage, prevSize); pRes->numOfRows -= pQueryInfo->limit.offset; - pRes->numOfClauseTotal -= pQueryInfo->limit.offset; pQueryInfo->limit.offset = 0; } else { pQueryInfo->limit.offset -= pRes->numOfRows; pRes->numOfRows = 0; - pRes->numOfClauseTotal = 0; } } - if (pQueryInfo->limit.limit >= 0 && pRes->numOfClauseTotal > pQueryInfo->limit.limit) { + pRes->numOfRowsGroup += pRes->numOfRows; + + if (pQueryInfo->limit.limit >= 0 && pRes->numOfRowsGroup > pQueryInfo->limit.limit) { /* impose the limitation of output rows on the final result */ int32_t prevSize = pFinalDataPage->num; - int32_t overflow = pRes->numOfClauseTotal - pQueryInfo->limit.limit; + int32_t overflow = pRes->numOfRowsGroup - pQueryInfo->limit.limit; assert(overflow < pRes->numOfRows); - pRes->numOfClauseTotal = pQueryInfo->limit.limit; + pRes->numOfRowsGroup = pQueryInfo->limit.limit; pRes->numOfRows -= overflow; pFinalDataPage->num -= overflow; @@ -957,6 +929,8 @@ static void doFillResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneO } memcpy(pRes->data, pFinalDataPage->data, pRes->numOfRows * pLocalReducer->finalRowSize); + + pRes->numOfClauseTotal += pRes->numOfRows; pFinalDataPage->num = 0; return; } @@ -969,7 +943,7 @@ static void doFillResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneO TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); pResPages[i] = calloc(1, sizeof(tFilePage) + pField->bytes * pLocalReducer->resColModel->capacity); } - + while (1) { int64_t newRows = taosGenerateDataBlock(pFillInfo, pResPages, pLocalReducer->resColModel->capacity); @@ -986,7 +960,6 @@ static void doFillResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneO pRes->data = pLocalReducer->pFinalRes; pRes->numOfRows = newRows; - pRes->numOfClauseTotal += newRows; pQueryInfo->limit.offset = 0; break; @@ -1010,15 +983,13 @@ static void doFillResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneO } if (pRes->numOfRows > 0) { - if (pQueryInfo->limit.limit >= 0 && pRes->numOfClauseTotal > pQueryInfo->limit.limit) { - int32_t overflow = pRes->numOfClauseTotal - pQueryInfo->limit.limit; + if (pQueryInfo->limit.limit >= 0 && pRes->numOfRows > pQueryInfo->limit.limit) { + int32_t overflow = pRes->numOfRows - pQueryInfo->limit.limit; pRes->numOfRows -= overflow; - - assert(pRes->numOfRows >= 0); - - pRes->numOfClauseTotal = pQueryInfo->limit.limit; pFinalDataPage->num -= overflow; + assert(pRes->numOfRows >= 0 && pFinalDataPage->num > 0); + /* set remain data to be discarded, and reset the interpolation information */ savePrevRecordAndSetupInterpoInfo(pLocalReducer, pQueryInfo, pFillInfo); } @@ -1032,6 +1003,9 @@ static void doFillResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneO } else { // todo bug?? reversedCopyFromInterpolationToDstBuf(pQueryInfo, pRes, pResPages, pLocalReducer); } + + pRes->numOfRowsGroup += pRes->numOfRows; + pRes->numOfClauseTotal += pRes->numOfRows; } pFinalDataPage->num = 0; @@ -1227,7 +1201,10 @@ static bool saveGroupResultInfo(SSqlObj *pSql) { SSqlRes *pRes = &pSql->res; SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - pRes->numOfGroups += 1; + + if (pRes->numOfRowsGroup > 0) { + pRes->numOfGroups += 1; + } // the output group is limited by the slimit clause if (reachGroupResultLimit(pQueryInfo, pRes)) { @@ -1266,7 +1243,12 @@ bool doGenerateFinalResults(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool no pRes->numOfRows = 0; pQueryInfo->slimit.offset -= 1; pLocalReducer->discard = !noMoreCurrentGroupRes; - + + if (pLocalReducer->discard) { + SColumnModel *pInternModel = pLocalReducer->pDesc->pColumnModel; + tColModelAppend(pInternModel, pLocalReducer->discardData, pLocalReducer->pTempBuffer->data, 0, 1, 1); + } + return false; } @@ -1299,7 +1281,7 @@ void resetOutputBuf(SQueryInfo *pQueryInfo, SLocalReducer *pLocalReducer) { // static void resetEnvForNewResultset(SSqlRes *pRes, SSqlCmd *pCmd, SLocalReducer *pLocalReducer) { // In handling data in other groups, we need to reset the interpolation information for a new group data pRes->numOfRows = 0; - pRes->numOfClauseTotal = 0; + pRes->numOfRowsGroup = 0; SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); @@ -1363,7 +1345,8 @@ static bool doHandleLastRemainData(SSqlObj *pSql) { if ((isAllSourcesCompleted(pLocalReducer) && !pLocalReducer->hasPrevRow) || pLocalReducer->pLocalDataSrc[0] == NULL || prevGroupCompleted) { // if fillType == TSDB_FILL_NONE, return directly - if (pQueryInfo->fillType != TSDB_FILL_NONE) { + if (pQueryInfo->fillType != TSDB_FILL_NONE && + ((pRes->numOfRowsGroup < pQueryInfo->limit.limit && pQueryInfo->limit.limit > 0) || (pQueryInfo->limit.limit < 0))) { int64_t etime = (pQueryInfo->window.skey < pQueryInfo->window.ekey) ? pQueryInfo->window.ekey : pQueryInfo->window.skey; assert(pFillInfo->numOfRows == 0); diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 5519dd69c2..40be0efab6 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -5954,13 +5954,13 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { return doLocalQueryProcess(pCmd, pQueryInfo, pQuerySql); } - if (pQuerySql->from->nExpr > TSDB_MAX_JOIN_TABLE_NUM) { + if (pQuerySql->from->nExpr > TSDB_MAX_JOIN_TABLE_NUM * 2) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg7); } pQueryInfo->command = TSDB_SQL_SELECT; - if (pQuerySql->from->nExpr > 2) { + if (pQuerySql->from->nExpr > 4) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg10); } @@ -5983,7 +5983,7 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { tscAddEmptyMetaInfo(pQueryInfo); } - STableMetaInfo* pTableMetaInfo1 = tscGetMetaInfo(pQueryInfo, i); + STableMetaInfo* pTableMetaInfo1 = tscGetMetaInfo(pQueryInfo, i/2); SSQLToken t = {.type = TSDB_DATA_TYPE_BINARY, .n = pTableItem->nLen, .z = pTableItem->pz}; if (tscSetTableFullName(pTableMetaInfo1, &t, pSql) != TSDB_CODE_SUCCESS) { From 3c2786021660ca9e1210411cad967ecec41f7b6f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 8 Aug 2020 10:55:27 +0800 Subject: [PATCH 066/109] [td-225] --- src/query/src/qExecutor.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index c7e0a53502..c64f9d5c48 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -405,8 +405,6 @@ static SWindowResult *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWin newCap = pWindowResInfo->capacity * 1.5; } - printf("%ld\n", newCap); - char *t = realloc(pWindowResInfo->pResult, newCap * sizeof(SWindowResult)); pRuntimeEnv->summary.internalSupSize += (newCap - pWindowResInfo->capacity) * sizeof(SWindowResult); From a18e287ca55916367aa5f0f9cc02c7d33a8c89c8 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 8 Aug 2020 11:06:32 +0800 Subject: [PATCH 067/109] TD-1057 --- deps/iconv/relocatable.c | 2 +- src/client/src/tscUtil.c | 1 + src/kit/shell/src/shellDarwin.c | 2 +- src/kit/shell/src/shellEngine.c | 9 ++++----- src/kit/shell/src/shellWindows.c | 10 +++++++--- src/kit/taosdemo/CMakeLists.txt | 4 +++- src/kit/taosdemo/taosdemo.c | 2 +- src/os/inc/osWindows.h | 1 + src/os/src/windows/w64Time.c | 12 ++++++++++++ src/util/inc/tscompression.h | 16 ++++++++++++++++ src/util/src/tcompression.c | 1 + 11 files changed, 48 insertions(+), 12 deletions(-) diff --git a/deps/iconv/relocatable.c b/deps/iconv/relocatable.c index 79906a399c..52cf4c1372 100644 --- a/deps/iconv/relocatable.c +++ b/deps/iconv/relocatable.c @@ -360,7 +360,7 @@ find_shared_library_fullname () ungetc (c, fp); shared_library_fullname = NULL; size = 0; - len = getline (&shared_library_fullname, &size, fp); + len = taosGetline(&shared_library_fullname, &size, fp); if (len >= 0) { /* Success: filled shared_library_fullname. */ diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index c74cc82f85..fc60e24d8e 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1931,6 +1931,7 @@ int16_t tscGetJoinTagColIdByUid(STagCond* pTagCond, uint64_t uid) { return pTagCond->joinInfo.right.tagColId; } else { assert(0); + return -1; } } diff --git a/src/kit/shell/src/shellDarwin.c b/src/kit/shell/src/shellDarwin.c index 3cb324abe9..ce41827462 100644 --- a/src/kit/shell/src/shellDarwin.c +++ b/src/kit/shell/src/shellDarwin.c @@ -62,7 +62,7 @@ void printHelp() { exit(EXIT_SUCCESS); } -void shellParseArgument(int argc, char *argv[], struct arguments *arguments) { +void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) { wordexp_t full_path; for (int i = 1; i < argc; i++) { // for host diff --git a/src/kit/shell/src/shellEngine.c b/src/kit/shell/src/shellEngine.c index 5d877ba6f0..750335a037 100644 --- a/src/kit/shell/src/shellEngine.c +++ b/src/kit/shell/src/shellEngine.c @@ -577,7 +577,7 @@ static int verticalPrintResult(TAOS_RES* tres) { int maxColNameLen = 0; for (int col = 0; col < num_fields; col++) { - int len = strlen(fields[col].name); + int len = (int)strlen(fields[col].name); if (len > maxColNameLen) { maxColNameLen = len; } @@ -604,9 +604,8 @@ static int verticalPrintResult(TAOS_RES* tres) { return numOfRows; } - static int calcColWidth(TAOS_FIELD* field, int precision) { - int width = strlen(field->name); + int width = (int)strlen(field->name); switch (field->type) { case TSDB_DATA_TYPE_BOOL: @@ -741,7 +740,7 @@ void read_history() { return; } - while ((read_size = getline(&line, &line_size, f)) != -1) { + while ((read_size = taosGetline(&line, &line_size, f)) != -1) { line[read_size - 1] = '\0'; history.hist[history.hend] = strdup(line); @@ -822,7 +821,7 @@ void source_file(TAOS *con, char *fptr) { return; } - while ((read_len = getline(&line, &line_len, f)) != -1) { + while ((read_len = taosGetline(&line, &line_len, f)) != -1) { if (read_len >= tsMaxSQLStringLen) continue; line[--read_len] = '\0'; diff --git a/src/kit/shell/src/shellWindows.c b/src/kit/shell/src/shellWindows.c index 8a7996d682..7297f23931 100644 --- a/src/kit/shell/src/shellWindows.c +++ b/src/kit/shell/src/shellWindows.c @@ -9,12 +9,16 @@ * * ****************************************************************/ -#include "shell.h" #include #include #include +#include "os.h" +#include "shell.h" +#include "taos.h" #include "shellCommand.h" +extern char configDir[]; + void printHelp() { char indent[10] = " "; printf("taos shell is used to test the TDEngine database\n"); @@ -43,7 +47,7 @@ void printHelp() { exit(EXIT_SUCCESS); } -void shellParseArgument(int argc, char *argv[], struct arguments *arguments) { +void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) { for (int i = 1; i < argc; i++) { // for host if (strcmp(argv[i], "-h") == 0) { @@ -81,7 +85,7 @@ void shellParseArgument(int argc, char *argv[], struct arguments *arguments) { fprintf(stderr, "config file path: %s overflow max len %d\n", argv[i], TSDB_FILENAME_LEN - 1); exit(EXIT_FAILURE); } - strcpy(configDir, argv[i]); + strcpy(configDir, argv[++i]); } else { fprintf(stderr, "Option -c requires an argument\n"); exit(EXIT_FAILURE); diff --git a/src/kit/taosdemo/CMakeLists.txt b/src/kit/taosdemo/CMakeLists.txt index a13bb767fc..55ad64db13 100644 --- a/src/kit/taosdemo/CMakeLists.txt +++ b/src/kit/taosdemo/CMakeLists.txt @@ -13,5 +13,7 @@ IF (TD_LINUX) # ELSE () # TARGET_LINK_LIBRARIES(taosdemo taos_static) # ENDIF () - +ELSEIF (TD_WINDOWS) + AUX_SOURCE_DIRECTORY(. SRC) + ADD_EXECUTABLE(taosdemo ${SRC}) ENDIF () diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 21e5b963c8..56a07b2ab7 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -1,4 +1,4 @@ -/* +/* * Copyright (c) 2019 TAOS Data, Inc. * * This program is free software: you can use, redistribute, and/or modify diff --git a/src/os/inc/osWindows.h b/src/os/inc/osWindows.h index 825ef4769d..9a430c406d 100644 --- a/src/os/inc/osWindows.h +++ b/src/os/inc/osWindows.h @@ -145,6 +145,7 @@ int flock(int fd, int option); int fsync(int filedes); char * strndup(const char *s, size_t n); char * dirname(char *pszPathname); +int gettimeofday(struct timeval *ptv, void *pTimeZone); // for access function in io.h #define F_OK 00 //Existence only diff --git a/src/os/src/windows/w64Time.c b/src/os/src/windows/w64Time.c index ce7eada6a0..1484b13843 100644 --- a/src/os/src/windows/w64Time.c +++ b/src/os/src/windows/w64Time.c @@ -16,6 +16,18 @@ #include #include +int gettimeofday(struct timeval *tv, struct timezone *tz) { + time_t t; + t = time(NULL); + SYSTEMTIME st; + GetLocalTime(&st); + + tv->tv_sec = (long)t; + tv->tv_usec = st.wMilliseconds * 1000; + + return 0; +} + struct tm *localtime_r(const time_t *timep, struct tm *result) { localtime_s(result, timep); return result; diff --git a/src/util/inc/tscompression.h b/src/util/inc/tscompression.h index 9398ff8243..bd1ccf3ca5 100644 --- a/src/util/inc/tscompression.h +++ b/src/util/inc/tscompression.h @@ -56,6 +56,7 @@ static FORCE_INLINE int tsCompressTinyint(const char *const input, int inputSize return tsCompressStringImp(buffer, len, output, outputSize); } else { assert(0); + return -1; } } @@ -68,6 +69,7 @@ static FORCE_INLINE int tsDecompressTinyint(const char *const input, int compres return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_TINYINT); } else { assert(0); + return -1; } } @@ -80,6 +82,7 @@ static FORCE_INLINE int tsCompressSmallint(const char *const input, int inputSiz return tsCompressStringImp(buffer, len, output, outputSize); } else { assert(0); + return -1; } } @@ -92,6 +95,7 @@ static FORCE_INLINE int tsDecompressSmallint(const char *const input, int compre return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_SMALLINT); } else { assert(0); + return -1; } } @@ -104,6 +108,7 @@ static FORCE_INLINE int tsCompressInt(const char *const input, int inputSize, co return tsCompressStringImp(buffer, len, output, outputSize); } else { assert(0); + return -1; } } @@ -116,6 +121,7 @@ static FORCE_INLINE int tsDecompressInt(const char *const input, int compressedS return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_INT); } else { assert(0); + return -1; } } @@ -128,6 +134,7 @@ static FORCE_INLINE int tsCompressBigint(const char *const input, int inputSize, return tsCompressStringImp(buffer, len, output, outputSize); } else { assert(0); + return -1; } } @@ -140,6 +147,7 @@ static FORCE_INLINE int tsDecompressBigint(const char *const input, int compress return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_BIGINT); } else { assert(0); + return -1; } } @@ -152,6 +160,7 @@ static FORCE_INLINE int tsCompressBool(const char *const input, int inputSize, c return tsCompressStringImp(buffer, len, output, outputSize); } else { assert(0); + return -1; } } @@ -164,6 +173,7 @@ static FORCE_INLINE int tsDecompressBool(const char *const input, int compressed return tsDecompressBoolImp(buffer, nelements, output); } else { assert(0); + return -1; } } @@ -186,6 +196,7 @@ static FORCE_INLINE int tsCompressFloat(const char *const input, int inputSize, return tsCompressStringImp(buffer, len, output, outputSize); } else { assert(0); + return -1; } } @@ -198,6 +209,7 @@ static FORCE_INLINE int tsDecompressFloat(const char *const input, int compresse return tsDecompressFloatImp(buffer, nelements, output); } else { assert(0); + return -1; } } @@ -210,6 +222,7 @@ static FORCE_INLINE int tsCompressDouble(const char *const input, int inputSize, return tsCompressStringImp(buffer, len, output, outputSize); } else { assert(0); + return -1; } } @@ -222,6 +235,7 @@ static FORCE_INLINE int tsDecompressDouble(const char *const input, int compress return tsDecompressDoubleImp(buffer, nelements, output); } else { assert(0); + return -1; } } @@ -234,6 +248,7 @@ static FORCE_INLINE int tsCompressTimestamp(const char *const input, int inputSi return tsCompressStringImp(buffer, len, output, outputSize); } else { assert(0); + return -1; } } @@ -246,6 +261,7 @@ static FORCE_INLINE int tsDecompressTimestamp(const char *const input, int compr return tsDecompressTimestampImp(buffer, nelements, output); } else { assert(0); + return -1; } } diff --git a/src/util/src/tcompression.c b/src/util/src/tcompression.c index c20ba59ac9..8c5828d32d 100644 --- a/src/util/src/tcompression.c +++ b/src/util/src/tcompression.c @@ -591,6 +591,7 @@ int tsDecompressTimestampImp(const char *const input, const int nelements, char } else { assert(0); + return -1; } } /* --------------------------------------------Double Compression From 7e3cd84df026a7ebc6c4b492a14dd351f359a7fe Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 8 Aug 2020 13:04:29 +0800 Subject: [PATCH 068/109] TD-1057 --- src/kit/taosdemo/CMakeLists.txt | 1 + src/kit/taosdemo/taosdemo.c | 560 +++++++++++++++++++++----------- src/os/inc/osWindows.h | 1 + tests/tsim/CMakeLists.txt | 4 - tests/tsim/src/simExe.c | 14 +- tests/tsim/src/simParse.c | 68 ++-- 6 files changed, 407 insertions(+), 241 deletions(-) diff --git a/src/kit/taosdemo/CMakeLists.txt b/src/kit/taosdemo/CMakeLists.txt index 55ad64db13..1698c85915 100644 --- a/src/kit/taosdemo/CMakeLists.txt +++ b/src/kit/taosdemo/CMakeLists.txt @@ -16,4 +16,5 @@ IF (TD_LINUX) ELSEIF (TD_WINDOWS) AUX_SOURCE_DIRECTORY(. SRC) ADD_EXECUTABLE(taosdemo ${SRC}) + TARGET_LINK_LIBRARIES(taosdemo taos_static) ENDIF () diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 56a07b2ab7..a34964cb2e 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -15,24 +15,31 @@ #define _GNU_SOURCE -#include -#include -#include - -#ifndef _ALPINE -#include -#endif -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#ifdef LINUX + #include "os.h" + #include + #include + #include + #ifndef _ALPINE + #include + #endif + #include + #include + #include + #include + #include + #include + #include + #include + #include + #include + #include +#else + #include + #include + #include + #include "os.h" +#endif #include "taos.h" #include "tutil.h" @@ -48,32 +55,6 @@ extern char configDir[]; #define STRING_LEN 60000 #define MAX_PREPARED_RAND 1000000 -/* The options we understand. */ -static struct argp_option options[] = { - {0, 'h', "host", 0, "The host to connect to TDEngine. Default is localhost.", 0}, - {0, 'p', "port", 0, "The TCP/IP port number to use for the connection. Default is 0.", 1}, - {0, 'u', "user", 0, "The TDEngine user name to use when connecting to the server. Default is 'root'.", 2}, - {0, 'P', "password", 0, "The password to use when connecting to the server. Default is 'taosdata'.", 3}, - {0, 'd', "database", 0, "Destination database. Default is 'test'.", 3}, - {0, 'm', "table_prefix", 0, "Table prefix name. Default is 't'.", 3}, - {0, 's', "sql file", 0, "The select sql file.", 3}, - {0, 'M', 0, 0, "Use metric flag.", 13}, - {0, 'o', "outputfile", 0, "Direct output to the named file. Default is './output.txt'.", 14}, - {0, 'q', "query_mode", 0, "Query mode--0: SYNC, 1: ASYNC. Default is SYNC.", 6}, - {0, 'b', "type_of_cols", 0, "The data_type of columns: 'INT', 'TINYINT', 'SMALLINT', 'BIGINT', 'FLOAT', 'DOUBLE', 'BINARY'. Default is 'INT'.", 7}, - {0, 'w', "length_of_binary", 0, "The length of data_type 'BINARY'. Only applicable when type of cols is 'BINARY'. Default is 8", 8}, - {0, 'l', "num_of_cols_per_record", 0, "The number of columns per record. Default is 3.", 8}, - {0, 'T', "num_of_threads", 0, "The number of threads. Default is 10.", 9}, - {0, 'r', "num_of_records_per_req", 0, "The number of records per request. Default is 1000.", 10}, - {0, 't', "num_of_tables", 0, "The number of tables. Default is 10000.", 11}, - {0, 'n', "num_of_records_per_table", 0, "The number of records per table. Default is 100000.", 12}, - {0, 'c', "config_directory", 0, "Configuration directory. Default is '/etc/taos/'.", 14}, - {0, 'x', 0, 0, "Insert only flag.", 13}, - {0, 'O', "order", 0, "Insert mode--0: In order, 1: Out of order. Default is in order.", 14}, - {0, 'R', "rate", 0, "Out of order data's rate--if order=1 Default 10, min: 0, max: 50.", 14}, - {0, 'D', "delete table", 0, "Delete data methods——0: don't delete, 1: delete by table, 2: delete by stable, 3: delete by database", 14}, - {0}}; - /* Used by main to communicate with parse_opt. */ typedef struct DemoArguments { char *host; @@ -101,143 +82,337 @@ typedef struct DemoArguments { char **arg_list; } SDemoArguments; -/* Parse a single option. */ -static error_t parse_opt(int key, char *arg, struct argp_state *state) { - /* Get the input argument from argp_parse, which we - know is a pointer to our arguments structure. */ - SDemoArguments *arguments = state->input; - wordexp_t full_path; - char **sptr; - switch (key) { - case 'h': - arguments->host = arg; - break; - case 'p': - arguments->port = atoi(arg); - break; - case 'u': - arguments->user = arg; - break; - case 'P': - arguments->password = arg; - break; - case 'o': - arguments->output_file = arg; - break; - case 's': - arguments->sqlFile = arg; - break; - case 'q': - arguments->mode = atoi(arg); - break; - case 'T': - arguments->num_of_threads = atoi(arg); - break; - case 'r': - arguments->num_of_RPR = atoi(arg); - break; - case 't': - arguments->num_of_tables = atoi(arg); - break; - case 'n': - arguments->num_of_DPT = atoi(arg); - break; - case 'd': - arguments->database = arg; - break; - case 'l': - arguments->num_of_CPR = atoi(arg); - break; - case 'b': - sptr = arguments->datatype; - if (strstr(arg, ",") == NULL) { - if (strcasecmp(arg, "INT") != 0 && strcasecmp(arg, "FLOAT") != 0 && - strcasecmp(arg, "TINYINT") != 0 && strcasecmp(arg, "BOOL") != 0 && - strcasecmp(arg, "SMALLINT") != 0 && - strcasecmp(arg, "BIGINT") != 0 && strcasecmp(arg, "DOUBLE") != 0 && - strcasecmp(arg, "BINARY") && strcasecmp(arg, "NCHAR")) { - argp_error(state, "Invalid data_type!"); - } - sptr[0] = arg; - } else { - int index = 0; - char *dupstr = strdup(arg); - char *running = dupstr; - char *token = strsep(&running, ","); - while (token != NULL) { - if (strcasecmp(token, "INT") != 0 && - strcasecmp(token, "FLOAT") != 0 && - strcasecmp(token, "TINYINT") != 0 && - strcasecmp(token, "BOOL") != 0 && - strcasecmp(token, "SMALLINT") != 0 && - strcasecmp(token, "BIGINT") != 0 && - strcasecmp(token, "DOUBLE") != 0 && strcasecmp(token, "BINARY") && strcasecmp(token, "NCHAR")) { +#ifdef LINUX + /* The options we understand. */ + static struct argp_option options[] = { + {0, 'h', "host", 0, "The host to connect to TDEngine. Default is localhost.", 0}, + {0, 'p', "port", 0, "The TCP/IP port number to use for the connection. Default is 0.", 1}, + {0, 'u', "user", 0, "The TDEngine user name to use when connecting to the server. Default is 'root'.", 2}, + {0, 'P', "password", 0, "The password to use when connecting to the server. Default is 'taosdata'.", 3}, + {0, 'd', "database", 0, "Destination database. Default is 'test'.", 3}, + {0, 'm', "table_prefix", 0, "Table prefix name. Default is 't'.", 3}, + {0, 's', "sql file", 0, "The select sql file.", 3}, + {0, 'M', 0, 0, "Use metric flag.", 13}, + {0, 'o', "outputfile", 0, "Direct output to the named file. Default is './output.txt'.", 14}, + {0, 'q', "query_mode", 0, "Query mode--0: SYNC, 1: ASYNC. Default is SYNC.", 6}, + {0, 'b', "type_of_cols", 0, "The data_type of columns: 'INT', 'TINYINT', 'SMALLINT', 'BIGINT', 'FLOAT', 'DOUBLE', 'BINARY'. Default is 'INT'.", 7}, + {0, 'w', "length_of_binary", 0, "The length of data_type 'BINARY'. Only applicable when type of cols is 'BINARY'. Default is 8", 8}, + {0, 'l', "num_of_cols_per_record", 0, "The number of columns per record. Default is 3.", 8}, + {0, 'T', "num_of_threads", 0, "The number of threads. Default is 10.", 9}, + {0, 'r', "num_of_records_per_req", 0, "The number of records per request. Default is 1000.", 10}, + {0, 't', "num_of_tables", 0, "The number of tables. Default is 10000.", 11}, + {0, 'n', "num_of_records_per_table", 0, "The number of records per table. Default is 100000.", 12}, + {0, 'c', "config_directory", 0, "Configuration directory. Default is '/etc/taos/'.", 14}, + {0, 'x', 0, 0, "Insert only flag.", 13}, + {0, 'O', "order", 0, "Insert mode--0: In order, 1: Out of order. Default is in order.", 14}, + {0, 'R', "rate", 0, "Out of order data's rate--if order=1 Default 10, min: 0, max: 50.", 14}, + {0, 'D', "delete table", 0, "Delete data methods——0: don't delete, 1: delete by table, 2: delete by stable, 3: delete by database", 14}, + {0}}; + + /* Parse a single option. */ + static error_t parse_opt(int key, char *arg, struct argp_state *state) { + /* Get the input argument from argp_parse, which we + know is a pointer to our arguments structure. */ + SDemoArguments *arguments = state->input; + wordexp_t full_path; + char **sptr; + switch (key) { + case 'h': + arguments->host = arg; + break; + case 'p': + arguments->port = atoi(arg); + break; + case 'u': + arguments->user = arg; + break; + case 'P': + arguments->password = arg; + break; + case 'o': + arguments->output_file = arg; + break; + case 's': + arguments->sqlFile = arg; + break; + case 'q': + arguments->mode = atoi(arg); + break; + case 'T': + arguments->num_of_threads = atoi(arg); + break; + case 'r': + arguments->num_of_RPR = atoi(arg); + break; + case 't': + arguments->num_of_tables = atoi(arg); + break; + case 'n': + arguments->num_of_DPT = atoi(arg); + break; + case 'd': + arguments->database = arg; + break; + case 'l': + arguments->num_of_CPR = atoi(arg); + break; + case 'b': + sptr = arguments->datatype; + if (strstr(arg, ",") == NULL) { + if (strcasecmp(arg, "INT") != 0 && strcasecmp(arg, "FLOAT") != 0 && + strcasecmp(arg, "TINYINT") != 0 && strcasecmp(arg, "BOOL") != 0 && + strcasecmp(arg, "SMALLINT") != 0 && + strcasecmp(arg, "BIGINT") != 0 && strcasecmp(arg, "DOUBLE") != 0 && + strcasecmp(arg, "BINARY") && strcasecmp(arg, "NCHAR")) { argp_error(state, "Invalid data_type!"); } - sptr[index++] = token; - token = strsep(&running, ","); - if (index >= MAX_NUM_DATATYPE) break; + sptr[0] = arg; + } else { + int index = 0; + char *dupstr = strdup(arg); + char *running = dupstr; + char *token = strsep(&running, ","); + while (token != NULL) { + if (strcasecmp(token, "INT") != 0 && + strcasecmp(token, "FLOAT") != 0 && + strcasecmp(token, "TINYINT") != 0 && + strcasecmp(token, "BOOL") != 0 && + strcasecmp(token, "SMALLINT") != 0 && + strcasecmp(token, "BIGINT") != 0 && + strcasecmp(token, "DOUBLE") != 0 && strcasecmp(token, "BINARY") && strcasecmp(token, "NCHAR")) { + argp_error(state, "Invalid data_type!"); + } + sptr[index++] = token; + token = strsep(&running, ","); + if (index >= MAX_NUM_DATATYPE) break; + } } - } - break; - case 'w': - arguments->len_of_binary = atoi(arg); - break; - case 'm': - arguments->tb_prefix = arg; - break; - case 'M': - arguments->use_metric = false; - break; - case 'x': - arguments->insert_only = false; - break; - case 'c': - if (wordexp(arg, &full_path, 0) != 0) { - fprintf(stderr, "Invalid path %s\n", arg); - return -1; - } - taos_options(TSDB_OPTION_CONFIGDIR, full_path.we_wordv[0]); - wordfree(&full_path); - break; - case 'O': - arguments->order = atoi(arg); - if (arguments->order > 1 || arguments->order < 0) - { - arguments->order = 0; - } else if (arguments->order == 1) - { - arguments->rate = 10; - } - break; - case 'R': - arguments->rate = atoi(arg); - if (arguments->order == 1 && (arguments->rate > 50 || arguments->rate <= 0)) - { - arguments->rate = 10; - } - break; - case 'D': - arguments->method_of_delete = atoi(arg); - if (arguments->method_of_delete < 0 || arguments->method_of_delete > 3) - { - arguments->method_of_delete = 0; - } - break; - case OPT_ABORT: - arguments->abort = 1; - break; - case ARGP_KEY_ARG: - /*arguments->arg_list = &state->argv[state->next-1]; - state->next = state->argc;*/ - argp_usage(state); - break; + break; + case 'w': + arguments->len_of_binary = atoi(arg); + break; + case 'm': + arguments->tb_prefix = arg; + break; + case 'M': + arguments->use_metric = false; + break; + case 'x': + arguments->insert_only = false; + break; + case 'c': + if (wordexp(arg, &full_path, 0) != 0) { + fprintf(stderr, "Invalid path %s\n", arg); + return -1; + } + taos_options(TSDB_OPTION_CONFIGDIR, full_path.we_wordv[0]); + wordfree(&full_path); + break; + case 'O': + arguments->order = atoi(arg); + if (arguments->order > 1 || arguments->order < 0) + { + arguments->order = 0; + } else if (arguments->order == 1) + { + arguments->rate = 10; + } + break; + case 'R': + arguments->rate = atoi(arg); + if (arguments->order == 1 && (arguments->rate > 50 || arguments->rate <= 0)) + { + arguments->rate = 10; + } + break; + case 'D': + arguments->method_of_delete = atoi(arg); + if (arguments->method_of_delete < 0 || arguments->method_of_delete > 3) + { + arguments->method_of_delete = 0; + } + break; + case OPT_ABORT: + arguments->abort = 1; + break; + case ARGP_KEY_ARG: + /*arguments->arg_list = &state->argv[state->next-1]; + state->next = state->argc;*/ + argp_usage(state); + break; - default: - return ARGP_ERR_UNKNOWN; + default: + return ARGP_ERR_UNKNOWN; + } + return 0; } - return 0; -} + + static struct argp argp = {options, parse_opt, 0, 0}; + + void parse_args(int argc, char *argv[], SDemoArguments *arguments) { + argp_parse(&argp, argc, argv, 0, 0, &arguments); + if (arguments.abort) { + #ifndef _ALPINE + error(10, 0, "ABORTED"); + #else + abort(); + #endif + } + } + +#else + void printHelp() { + char indent[10] = " "; + printf("%s%s\n", indent, "-h"); + printf("%s%s%s\n", indent, indent, "host, The host to connect to TDEngine. Default is localhost."); + printf("%s%s\n", indent, "-p"); + printf("%s%s%s\n", indent, indent, "port, The TCP/IP port number to use for the connection. Default is 0."); + printf("%s%s\n", indent, "-u"); + printf("%s%s%s\n", indent, indent, "user, The TDEngine user name to use when connecting to the server. Default is 'root'."); + printf("%s%s\n", indent, "-p"); + printf("%s%s%s\n", indent, indent, "password, The password to use when connecting to the server. Default is 'taosdata'."); + printf("%s%s\n", indent, "-d"); + printf("%s%s%s\n", indent, indent, "database, Destination database. Default is 'test'."); + printf("%s%s\n", indent, "-m"); + printf("%s%s%s\n", indent, indent, "table_prefix, Table prefix name. Default is 't'."); + printf("%s%s\n", indent, "-s"); + printf("%s%s%s\n", indent, indent, "sql file, The select sql file."); + printf("%s%s\n", indent, "-M"); + printf("%s%s%s\n", indent, indent, "meteric, Use metric flag."); + printf("%s%s\n", indent, "-o"); + printf("%s%s%s\n", indent, indent, "outputfile, Direct output to the named file. Default is './output.txt'."); + printf("%s%s\n", indent, "-q"); + printf("%s%s%s\n", indent, indent, "query_mode, Query mode--0: SYNC, 1: ASYNC. Default is SYNC."); + printf("%s%s\n", indent, "-b"); + printf("%s%s%s\n", indent, indent, "type_of_cols, data_type of columns: 'INT', 'TINYINT', 'SMALLINT', 'BIGINT', 'FLOAT', 'DOUBLE', 'BINARY'. Default is 'INT'."); + printf("%s%s\n", indent, "-w"); + printf("%s%s%s\n", indent, indent, "length_of_binary, The length of data_type 'BINARY'. Only applicable when type of cols is 'BINARY'. Default is 8"); + printf("%s%s\n", indent, "-l"); + printf("%s%s%s\n", indent, indent, "num_of_cols_per_record, The number of columns per record. Default is 3."); + printf("%s%s\n", indent, "-T"); + printf("%s%s%s\n", indent, indent, "num_of_threads, The number of threads. Default is 10."); + printf("%s%s\n", indent, "-r"); + printf("%s%s%s\n", indent, indent, "num_of_records_per_req, The number of records per request. Default is 1000."); + printf("%s%s\n", indent, "-t"); + printf("%s%s%s\n", indent, indent, "num_of_tables, The number of tables. Default is 10000."); + printf("%s%s\n", indent, "-n"); + printf("%s%s%s\n", indent, indent, "num_of_records_per_table, The number of records per table. Default is 100000."); + printf("%s%s\n", indent, "-c"); + printf("%s%s%s\n", indent, indent, "config_directory, Configuration directory. Default is '/etc/taos/'."); + printf("%s%s\n", indent, "-x"); + printf("%s%s%s\n", indent, indent, "flag, Insert only flag."); + printf("%s%s\n", indent, "-O"); + printf("%s%s%s\n", indent, indent, "order, Insert mode--0: In order, 1: Out of order. Default is in order."); + printf("%s%s\n", indent, "-R"); + printf("%s%s%s\n", indent, indent, "rate, Out of order data's rate--if order=1 Default 10, min: 0, max: 50."); + printf("%s%s\n", indent, "-D"); + printf("%s%s%s\n", indent, indent, "Delete data methods——0: don't delete, 1: delete by table, 2: delete by stable, 3: delete by database."); + } + + void parse_args(int argc, char *argv[], SDemoArguments *arguments) { + char **sptr; + for (int i = 1; i < argc; i++) { + if (strcmp(argv[i], "-h") == 0) { + arguments->host = argv[++i]; + } else if (strcmp(argv[i], "-p") == 0) { + arguments->port = atoi(argv[++i]); + } else if (strcmp(argv[i], "-u") == 0) { + arguments->user = argv[++i]; + } else if (strcmp(argv[i], "-P") == 0) { + arguments->password = argv[++i]; + } else if (strcmp(argv[i], "-o") == 0) { + arguments->output_file = argv[++i]; + } else if (strcmp(argv[i], "-s") == 0) { + arguments->sqlFile = argv[++i]; + } else if (strcmp(argv[i], "-q") == 0) { + arguments->mode = atoi(argv[++i]); + } else if (strcmp(argv[i], "-T") == 0) { + arguments->num_of_threads = atoi(argv[++i]); + } else if (strcmp(argv[i], "-r") == 0) { + arguments->num_of_RPR = atoi(argv[++i]); + } else if (strcmp(argv[i], "-t") == 0) { + arguments->num_of_tables = atoi(argv[++i]); + } else if (strcmp(argv[i], "-n") == 0) { + arguments->num_of_DPT = atoi(argv[++i]); + } else if (strcmp(argv[i], "-d") == 0) { + arguments->database = argv[++i]; + } else if (strcmp(argv[i], "-l") == 0) { + arguments->num_of_CPR = atoi(argv[++i]); + } else if (strcmp(argv[i], "-b") == 0) { + sptr = arguments->datatype; + ++i; + if (strstr(argv[i], ",") == NULL) { + if (strcasecmp(argv[i], "INT") != 0 && strcasecmp(argv[i], "FLOAT") != 0 && + strcasecmp(argv[i], "TINYINT") != 0 && strcasecmp(argv[i], "BOOL") != 0 && + strcasecmp(argv[i], "SMALLINT") != 0 && + strcasecmp(argv[i], "BIGINT") != 0 && strcasecmp(argv[i], "DOUBLE") != 0 && + strcasecmp(argv[i], "BINARY") && strcasecmp(argv[i], "NCHAR")) { + fprintf(stderr, "Invalid data_type!\n"); + printHelp(); + exit(EXIT_FAILURE); + } + sptr[0] = argv[i]; + } else { + int index = 0; + char *dupstr = strdup(argv[i]); + char *running = dupstr; + char *token = strsep(&running, ","); + while (token != NULL) { + if (strcasecmp(token, "INT") != 0 && + strcasecmp(token, "FLOAT") != 0 && + strcasecmp(token, "TINYINT") != 0 && + strcasecmp(token, "BOOL") != 0 && + strcasecmp(token, "SMALLINT") != 0 && + strcasecmp(token, "BIGINT") != 0 && + strcasecmp(token, "DOUBLE") != 0 && strcasecmp(token, "BINARY") && strcasecmp(token, "NCHAR")) { + fprintf(stderr, "Invalid data_type!\n"); + printHelp(); + exit(EXIT_FAILURE); + } + sptr[index++] = token; + token = strsep(&running, ","); + if (index >= MAX_NUM_DATATYPE) break; + } + } + } else if (strcmp(argv[i], "-w") == 0) { + arguments->len_of_binary = atoi(argv[++i]); + } else if (strcmp(argv[i], "-m") == 0) { + arguments->tb_prefix = argv[++i]; + } else if (strcmp(argv[i], "-M") == 0) { + arguments->use_metric = false; + } else if (strcmp(argv[i], "-x") == 0) { + arguments->insert_only = false; + } else if (strcmp(argv[i], "-c") == 0) { + strcpy(configDir, argv[++i]); + } else if (strcmp(argv[i], "-O") == 0) { + arguments->order = atoi(argv[++i]); + if (arguments->order > 1 || arguments->order < 0) { + arguments->order = 0; + } else if (arguments->order == 1) { + arguments->rate = 10; + } + } else if (strcmp(argv[i], "-R") == 0) { + arguments->rate = atoi(argv[++i]); + if (arguments->order == 1 && (arguments->rate > 50 || arguments->rate <= 0)) { + arguments->rate = 10; + } + } else if (strcmp(argv[i], "-D") == 0) { + arguments->method_of_delete = atoi(argv[++i]); + if (arguments->method_of_delete < 0 || arguments->method_of_delete > 3) { + arguments->method_of_delete = 0; + } + } else if (strcmp(argv[i], "--help") == 0) { + printHelp(); + exit(EXIT_FAILURE); + } else { + fprintf(stderr, "wrong options\n"); + printHelp(); + exit(EXIT_FAILURE); + } + } + } + +#endif /* ******************************* Structure * definition******************************* */ @@ -293,9 +468,6 @@ typedef struct { * variables******************************* */ char *aggreFunc[] = {"*", "count(*)", "avg(f1)", "sum(f1)", "max(f1)", "min(f1)", "first(f1)", "last(f1)"}; -/* ******************************* Global - * functions******************************* */ -static struct argp argp = {options, parse_opt, 0, 0}; void queryDB(TAOS *taos, char *command); @@ -368,15 +540,7 @@ int main(int argc, char *argv[]) { arguments.insert_only = true; // end change - argp_parse(&argp, argc, argv, 0, 0, &arguments); - - if (arguments.abort) { - #ifndef _ALPINE - error(10, 0, "ABORTED"); - #else - abort(); - #endif - } + parse_args(argc, argv, &arguments); enum MODE query_mode = arguments.mode; char *ip_addr = arguments.host; @@ -438,7 +602,7 @@ int main(int argc, char *argv[]) { printf("# Use metric: %s\n", use_metric ? "true" : "false"); printf("# Datatype of Columns: %s\n", dataString); printf("# Binary Length(If applicable): %d\n", - (strcasestr(dataString, "BINARY") != NULL || strcasestr(dataString, "NCHAR") != NULL ) ? len_of_binary : -1); + (strncasecmp(dataString, "BINARY", 6) == 0 || strncasecmp(dataString, "NCHAR", 5) == 0) ? len_of_binary : -1); printf("# Number of Columns per record: %d\n", ncols_per_record); printf("# Number of Threads: %d\n", threads); printf("# Number of Tables: %d\n", ntables); @@ -466,7 +630,7 @@ int main(int argc, char *argv[]) { fprintf(fp, "# Use metric: %s\n", use_metric ? "true" : "false"); fprintf(fp, "# Datatype of Columns: %s\n", dataString); fprintf(fp, "# Binary Length(If applicable): %d\n", - (strcasestr(dataString, "BINARY") != NULL || strcasestr(dataString, "NCHAR") != NULL ) ? len_of_binary : -1); + (strncasecmp(dataString, "BINARY", 6) == 0 || strncasecmp(dataString, "NCHAR", 5) == 0) ? len_of_binary : -1); fprintf(fp, "# Number of Columns per record: %d\n", ncols_per_record); fprintf(fp, "# Number of Threads: %d\n", threads); fprintf(fp, "# Number of Tables: %d\n", ntables); @@ -753,7 +917,7 @@ void querySqlFile(TAOS* taos, char* sqlFile) double t = getCurrentTime(); - while ((read_len = getline(&line, &line_len, fp)) != -1) { + while ((read_len = taosGetline(&line, &line_len, fp)) != -1) { if (read_len >= MAX_SQL_SIZE) continue; line[--read_len] = '\0'; @@ -1035,7 +1199,7 @@ void *syncWrite(void *sarg) { char **data_type = winfo->datatype; int len_of_binary = winfo->len_of_binary; int ncols_per_record = winfo->ncols_per_record; - srand(time(NULL)); + srand((uint32_t)time(NULL)); int64_t time_counter = winfo->start_time; for (int i = 0; i < winfo->nrecords_per_table;) { for (int tID = winfo->start_table_id; tID <= winfo->end_table_id; tID++) { @@ -1049,7 +1213,7 @@ void *syncWrite(void *sarg) { int rand_num = rand() % 100; int len = -1; if (winfo->data_of_order ==1 && rand_num < winfo->data_of_rate) { - long d = tmp_time - rand() % 1000000 + rand_num; + int64_t d = tmp_time - rand() % 1000000 + rand_num; len = generateData(data, data_type, ncols_per_record, d, len_of_binary); } else { len = generateData(data, data_type, ncols_per_record, tmp_time += 1000, len_of_binary); @@ -1144,7 +1308,7 @@ void callBack(void *param, TAOS_RES *res, int code) { int rand_num = rand() % 100; if (tb_info->data_of_order ==1 && rand_num < tb_info->data_of_rate) { - long d = tmp_time - rand() % 1000000 + rand_num; + int64_t d = tmp_time - rand() % 1000000 + rand_num; generateData(data, datatype, ncols_per_record, d, len_of_binary); } else { @@ -1269,13 +1433,15 @@ int32_t generateData(char *res, char **data_type, int num_of_cols, int64_t times bool b = rand() & 1; pstr += sprintf(pstr, ", %s", b ? "true" : "false"); } else if (strcasecmp(data_type[i % c], "binary") == 0) { - char s[len_of_binary]; + char *s = malloc(len_of_binary); rand_string(s, len_of_binary); pstr += sprintf(pstr, ", \"%s\"", s); + free(s); }else if (strcasecmp(data_type[i % c], "nchar") == 0) { - char s[len_of_binary]; + char *s = malloc(len_of_binary); rand_string(s, len_of_binary); pstr += sprintf(pstr, ", \"%s\"", s); + free(s); } if (pstr - res > MAX_DATA_SIZE) { @@ -1286,7 +1452,7 @@ int32_t generateData(char *res, char **data_type, int num_of_cols, int64_t times pstr += sprintf(pstr, ")"); - return pstr - res; + return (int32_t)(pstr - res); } static const char charset[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJK1234567890"; diff --git a/src/os/inc/osWindows.h b/src/os/inc/osWindows.h index 9a430c406d..dcf087beeb 100644 --- a/src/os/inc/osWindows.h +++ b/src/os/inc/osWindows.h @@ -42,6 +42,7 @@ #include "msvcProcess.h" #include "msvcDirect.h" #include "msvcFcntl.h" +#include "msvcStdio.h" #include "sys/msvcStat.h" #include "sys/msvcTypes.h" diff --git a/tests/tsim/CMakeLists.txt b/tests/tsim/CMakeLists.txt index 64adce5945..50b42941af 100644 --- a/tests/tsim/CMakeLists.txt +++ b/tests/tsim/CMakeLists.txt @@ -4,10 +4,6 @@ INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/cJson/inc) INCLUDE_DIRECTORIES(inc) -IF (TD_WINDOWS) - INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/pthread) -ENDIF () - AUX_SOURCE_DIRECTORY(src SRC) ADD_EXECUTABLE(tsim ${SRC}) TARGET_LINK_LIBRARIES(tsim taos_static trpc tutil pthread cJson) diff --git a/tests/tsim/src/simExe.c b/tests/tsim/src/simExe.c index 1078b32981..8bc9a76545 100644 --- a/tests/tsim/src/simExe.c +++ b/tests/tsim/src/simExe.c @@ -313,7 +313,9 @@ bool simExecuteSystemCmd(SScript *script, char *option) { simError("script:%s, failed to execute %s , code %d, errno:%d %s, repeatTimes:%d", script->fileName, buf, code, errno, strerror(errno), repeatTimes); taosMsleep(1000); +#ifdef LINUX signal(SIGCHLD, SIG_DFL); +#endif if (repeatTimes++ >= 10) { exit(0); } @@ -418,14 +420,14 @@ void simVisuallizeOption(SScript *script, char *src, char *dst) { var = strchr(src, '$'); if (var == NULL) break; if (var && ((var - src - 1) > 0) && *(var - 1) == '\\') { - srcLen = var - src - 1; + srcLen = (int)(var - src - 1); memcpy(dst + dstLen, src, srcLen); dstLen += srcLen; src = var; break; } - srcLen = var - src; + srcLen = (int)(var - src); memcpy(dst + dstLen, src, srcLen); dstLen += srcLen; @@ -433,7 +435,7 @@ void simVisuallizeOption(SScript *script, char *src, char *dst) { value = simGetVariable(script, token, tokenLen); strcpy(dst + dstLen, value); - dstLen += strlen(value); + dstLen += (int)strlen(value); } strcpy(dst + dstLen, src); @@ -455,9 +457,9 @@ void simCloseNativeConnect(SScript *script) { void simCloseTaosdConnect(SScript *script) { if (simAsyncQuery) { - return simCloseRestFulConnect(script); + simCloseRestFulConnect(script); } else { - return simCloseNativeConnect(script); + simCloseNativeConnect(script); } } // {"status":"succ","code":0,"desc":"/KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04"} @@ -575,7 +577,7 @@ int simExecuteRestFulCommand(SScript *script, char *command) { while (!feof(fp)) { int availSize = mallocSize - alreadyReadSize; - int len = fread(content + alreadyReadSize, 1, availSize, fp); + int len = (int)fread(content + alreadyReadSize, 1, availSize, fp); if (len >= availSize) { alreadyReadSize += len; mallocSize *= 2; diff --git a/tests/tsim/src/simParse.c b/tests/tsim/src/simParse.c index 2528fde9f5..f201f149a1 100644 --- a/tests/tsim/src/simParse.c +++ b/tests/tsim/src/simParse.c @@ -105,7 +105,7 @@ void simAddCmdIntoHash(SCommand *pCmd) { int hash; SCommand *node; - hash = simHashCmd(pCmd->name, strlen(pCmd->name)); + hash = simHashCmd(pCmd->name, (int)strlen(pCmd->name)); node = cmdHashList[hash]; pCmd->next = node; cmdHashList[hash] = pCmd; @@ -199,7 +199,7 @@ SScript *simParseScript(char *fileName) { if (fgets(buffer, sizeof(buffer), fd) == NULL) continue; lineNum++; - int cmdlen = strlen(buffer); + int cmdlen = (int)strlen(buffer); if (buffer[cmdlen - 1] == '\r' || buffer[cmdlen - 1] == '\n') buffer[cmdlen - 1] = 0; rest = buffer; @@ -294,10 +294,10 @@ int simCheckExpression(char *exp) { rest = paGetToken(rest, &op, &opLen); - if (opLen == 0) return rest - exp; + if (opLen == 0) return (int)(rest - exp); /* if it is key word "then" */ - if (strncmp(op, "then", 4) == 0) return op - exp; + if (strncmp(op, "then", 4) == 0) return (int)(op - exp); rest = paGetToken(rest, &op2, &op2Len); if (op2Len == 0) { @@ -312,7 +312,7 @@ int simCheckExpression(char *exp) { if (op[0] == '+' || op[0] == '-' || op[0] == '*' || op[0] == '/' || op[0] == '.') { - return rest - exp; + return (int)(rest - exp); } return -1; @@ -655,7 +655,7 @@ bool simParsePrintCmd(char *rest, SCommand *pCmd, int lineNum) { cmdLine[numOfLines].cmdno = SIM_CMD_PRINT; cmdLine[numOfLines].lineNum = lineNum; cmdLine[numOfLines].optionOffset = optionOffset; - expLen = strlen(rest); + expLen = (int)strlen(rest); memcpy(optionBuffer + optionOffset, rest, expLen); optionOffset += expLen + 1; *(optionBuffer + optionOffset - 1) = 0; @@ -690,7 +690,7 @@ bool simParseSqlCmd(char *rest, SCommand *pCmd, int lineNum) { cmdLine[numOfLines].cmdno = SIM_CMD_SQL; cmdLine[numOfLines].lineNum = lineNum; cmdLine[numOfLines].optionOffset = optionOffset; - expLen = strlen(rest); + expLen = (int)strlen(rest); memcpy(optionBuffer + optionOffset, rest, expLen); optionOffset += expLen + 1; *(optionBuffer + optionOffset - 1) = 0; @@ -706,7 +706,7 @@ bool simParseSqlErrorCmd(char *rest, SCommand *pCmd, int lineNum) { cmdLine[numOfLines].cmdno = SIM_CMD_SQL_ERROR; cmdLine[numOfLines].lineNum = lineNum; cmdLine[numOfLines].optionOffset = optionOffset; - expLen = strlen(rest); + expLen = (int)strlen(rest); memcpy(optionBuffer + optionOffset, rest, expLen); optionOffset += expLen + 1; *(optionBuffer + optionOffset - 1) = 0; @@ -728,7 +728,7 @@ bool simParseSystemCmd(char *rest, SCommand *pCmd, int lineNum) { cmdLine[numOfLines].cmdno = SIM_CMD_SYSTEM; cmdLine[numOfLines].lineNum = lineNum; cmdLine[numOfLines].optionOffset = optionOffset; - expLen = strlen(rest); + expLen = (int)strlen(rest); memcpy(optionBuffer + optionOffset, rest, expLen); optionOffset += expLen + 1; *(optionBuffer + optionOffset - 1) = 0; @@ -840,14 +840,14 @@ void simInitsimCmdList() { cmdno = SIM_CMD_EXP; simCmdList[cmdno].cmdno = cmdno; strcpy(simCmdList[cmdno].name, "exp"); - simCmdList[cmdno].nlen = strlen(simCmdList[cmdno].name); + simCmdList[cmdno].nlen = (int)strlen(simCmdList[cmdno].name); simCmdList[cmdno].parseCmd = NULL; simCmdList[cmdno].executeCmd = simExecuteExpCmd; cmdno = SIM_CMD_IF; simCmdList[cmdno].cmdno = cmdno; strcpy(simCmdList[cmdno].name, "if"); - simCmdList[cmdno].nlen = strlen(simCmdList[cmdno].name); + simCmdList[cmdno].nlen = (int)strlen(simCmdList[cmdno].name); simCmdList[cmdno].parseCmd = simParseIfCmd; simCmdList[cmdno].executeCmd = NULL; simAddCmdIntoHash(&(simCmdList[cmdno])); @@ -855,7 +855,7 @@ void simInitsimCmdList() { cmdno = SIM_CMD_ELIF; simCmdList[cmdno].cmdno = cmdno; strcpy(simCmdList[cmdno].name, "elif"); - simCmdList[cmdno].nlen = strlen(simCmdList[cmdno].name); + simCmdList[cmdno].nlen = (int)strlen(simCmdList[cmdno].name); simCmdList[cmdno].parseCmd = simParseElifCmd; simCmdList[cmdno].executeCmd = NULL; simAddCmdIntoHash(&(simCmdList[cmdno])); @@ -863,7 +863,7 @@ void simInitsimCmdList() { cmdno = SIM_CMD_ELSE; simCmdList[cmdno].cmdno = cmdno; strcpy(simCmdList[cmdno].name, "else"); - simCmdList[cmdno].nlen = strlen(simCmdList[cmdno].name); + simCmdList[cmdno].nlen = (int)strlen(simCmdList[cmdno].name); simCmdList[cmdno].parseCmd = simParseElseCmd; simCmdList[cmdno].executeCmd = NULL; simAddCmdIntoHash(&(simCmdList[cmdno])); @@ -871,7 +871,7 @@ void simInitsimCmdList() { cmdno = SIM_CMD_ENDI; simCmdList[cmdno].cmdno = cmdno; strcpy(simCmdList[cmdno].name, "endi"); - simCmdList[cmdno].nlen = strlen(simCmdList[cmdno].name); + simCmdList[cmdno].nlen = (int)strlen(simCmdList[cmdno].name); simCmdList[cmdno].parseCmd = simParseEndiCmd; simCmdList[cmdno].executeCmd = NULL; simAddCmdIntoHash(&(simCmdList[cmdno])); @@ -879,7 +879,7 @@ void simInitsimCmdList() { cmdno = SIM_CMD_WHILE; simCmdList[cmdno].cmdno = cmdno; strcpy(simCmdList[cmdno].name, "while"); - simCmdList[cmdno].nlen = strlen(simCmdList[cmdno].name); + simCmdList[cmdno].nlen = (int)strlen(simCmdList[cmdno].name); simCmdList[cmdno].parseCmd = simParseWhileCmd; simCmdList[cmdno].executeCmd = NULL; simAddCmdIntoHash(&(simCmdList[cmdno])); @@ -887,7 +887,7 @@ void simInitsimCmdList() { cmdno = SIM_CMD_ENDW; simCmdList[cmdno].cmdno = cmdno; strcpy(simCmdList[cmdno].name, "endw"); - simCmdList[cmdno].nlen = strlen(simCmdList[cmdno].name); + simCmdList[cmdno].nlen = (int)strlen(simCmdList[cmdno].name); simCmdList[cmdno].parseCmd = simParseEndwCmd; simCmdList[cmdno].executeCmd = NULL; simAddCmdIntoHash(&(simCmdList[cmdno])); @@ -895,7 +895,7 @@ void simInitsimCmdList() { cmdno = SIM_CMD_SWITCH; simCmdList[cmdno].cmdno = cmdno; strcpy(simCmdList[cmdno].name, "switch"); - simCmdList[cmdno].nlen = strlen(simCmdList[cmdno].name); + simCmdList[cmdno].nlen = (int)strlen(simCmdList[cmdno].name); simCmdList[cmdno].parseCmd = simParseSwitchCmd; simCmdList[cmdno].executeCmd = NULL; simAddCmdIntoHash(&(simCmdList[cmdno])); @@ -903,7 +903,7 @@ void simInitsimCmdList() { cmdno = SIM_CMD_CASE; simCmdList[cmdno].cmdno = cmdno; strcpy(simCmdList[cmdno].name, "case"); - simCmdList[cmdno].nlen = strlen(simCmdList[cmdno].name); + simCmdList[cmdno].nlen = (int)strlen(simCmdList[cmdno].name); simCmdList[cmdno].parseCmd = simParseCaseCmd; simCmdList[cmdno].executeCmd = NULL; simAddCmdIntoHash(&(simCmdList[cmdno])); @@ -911,7 +911,7 @@ void simInitsimCmdList() { cmdno = SIM_CMD_DEFAULT; simCmdList[cmdno].cmdno = cmdno; strcpy(simCmdList[cmdno].name, "default"); - simCmdList[cmdno].nlen = strlen(simCmdList[cmdno].name); + simCmdList[cmdno].nlen = (int)strlen(simCmdList[cmdno].name); simCmdList[cmdno].parseCmd = simParseDefaultCmd; simCmdList[cmdno].executeCmd = NULL; simAddCmdIntoHash(&(simCmdList[cmdno])); @@ -919,7 +919,7 @@ void simInitsimCmdList() { cmdno = SIM_CMD_BREAK; simCmdList[cmdno].cmdno = cmdno; strcpy(simCmdList[cmdno].name, "break"); - simCmdList[cmdno].nlen = strlen(simCmdList[cmdno].name); + simCmdList[cmdno].nlen = (int)strlen(simCmdList[cmdno].name); simCmdList[cmdno].parseCmd = simParseBreakCmd; simCmdList[cmdno].executeCmd = NULL; simAddCmdIntoHash(&(simCmdList[cmdno])); @@ -927,7 +927,7 @@ void simInitsimCmdList() { cmdno = SIM_CMD_CONTINUE; simCmdList[cmdno].cmdno = cmdno; strcpy(simCmdList[cmdno].name, "continue"); - simCmdList[cmdno].nlen = strlen(simCmdList[cmdno].name); + simCmdList[cmdno].nlen = (int)strlen(simCmdList[cmdno].name); simCmdList[cmdno].parseCmd = simParseContinueCmd; simCmdList[cmdno].executeCmd = NULL; simAddCmdIntoHash(&(simCmdList[cmdno])); @@ -935,7 +935,7 @@ void simInitsimCmdList() { cmdno = SIM_CMD_ENDS; simCmdList[cmdno].cmdno = cmdno; strcpy(simCmdList[cmdno].name, "ends"); - simCmdList[cmdno].nlen = strlen(simCmdList[cmdno].name); + simCmdList[cmdno].nlen = (int)strlen(simCmdList[cmdno].name); simCmdList[cmdno].parseCmd = simParseEndsCmd; simCmdList[cmdno].executeCmd = NULL; simAddCmdIntoHash(&(simCmdList[cmdno])); @@ -943,7 +943,7 @@ void simInitsimCmdList() { cmdno = SIM_CMD_SLEEP; simCmdList[cmdno].cmdno = cmdno; strcpy(simCmdList[cmdno].name, "sleep"); - simCmdList[cmdno].nlen = strlen(simCmdList[cmdno].name); + simCmdList[cmdno].nlen = (int)strlen(simCmdList[cmdno].name); simCmdList[cmdno].parseCmd = simParseSleepCmd; simCmdList[cmdno].executeCmd = simExecuteSleepCmd; simAddCmdIntoHash(&(simCmdList[cmdno])); @@ -951,7 +951,7 @@ void simInitsimCmdList() { cmdno = SIM_CMD_GOTO; simCmdList[cmdno].cmdno = cmdno; strcpy(simCmdList[cmdno].name, "goto"); - simCmdList[cmdno].nlen = strlen(simCmdList[cmdno].name); + simCmdList[cmdno].nlen = (int)strlen(simCmdList[cmdno].name); simCmdList[cmdno].parseCmd = simParseGotoCmd; simCmdList[cmdno].executeCmd = simExecuteGotoCmd; simAddCmdIntoHash(&(simCmdList[cmdno])); @@ -959,7 +959,7 @@ void simInitsimCmdList() { cmdno = SIM_CMD_RUN; simCmdList[cmdno].cmdno = cmdno; strcpy(simCmdList[cmdno].name, "run"); - simCmdList[cmdno].nlen = strlen(simCmdList[cmdno].name); + simCmdList[cmdno].nlen = (int)strlen(simCmdList[cmdno].name); simCmdList[cmdno].parseCmd = simParseRunCmd; simCmdList[cmdno].executeCmd = simExecuteRunCmd; simAddCmdIntoHash(&(simCmdList[cmdno])); @@ -967,7 +967,7 @@ void simInitsimCmdList() { cmdno = SIM_CMD_RUN_BACK; simCmdList[cmdno].cmdno = cmdno; strcpy(simCmdList[cmdno].name, "run_back"); - simCmdList[cmdno].nlen = strlen(simCmdList[cmdno].name); + simCmdList[cmdno].nlen = (int)strlen(simCmdList[cmdno].name); simCmdList[cmdno].parseCmd = simParseRunBackCmd; simCmdList[cmdno].executeCmd = simExecuteRunBackCmd; simAddCmdIntoHash(&(simCmdList[cmdno])); @@ -975,7 +975,7 @@ void simInitsimCmdList() { cmdno = SIM_CMD_SYSTEM; simCmdList[cmdno].cmdno = cmdno; strcpy(simCmdList[cmdno].name, "system"); - simCmdList[cmdno].nlen = strlen(simCmdList[cmdno].name); + simCmdList[cmdno].nlen = (int)strlen(simCmdList[cmdno].name); simCmdList[cmdno].parseCmd = simParseSystemCmd; simCmdList[cmdno].executeCmd = simExecuteSystemCmd; simAddCmdIntoHash(&(simCmdList[cmdno])); @@ -983,7 +983,7 @@ void simInitsimCmdList() { cmdno = SIM_CMD_SYSTEM_CONTENT; simCmdList[cmdno].cmdno = cmdno; strcpy(simCmdList[cmdno].name, "system_content"); - simCmdList[cmdno].nlen = strlen(simCmdList[cmdno].name); + simCmdList[cmdno].nlen = (int)strlen(simCmdList[cmdno].name); simCmdList[cmdno].parseCmd = simParseSystemContentCmd; simCmdList[cmdno].executeCmd = simExecuteSystemContentCmd; simAddCmdIntoHash(&(simCmdList[cmdno])); @@ -991,7 +991,7 @@ void simInitsimCmdList() { cmdno = SIM_CMD_PRINT; simCmdList[cmdno].cmdno = cmdno; strcpy(simCmdList[cmdno].name, "print"); - simCmdList[cmdno].nlen = strlen(simCmdList[cmdno].name); + simCmdList[cmdno].nlen = (int)strlen(simCmdList[cmdno].name); simCmdList[cmdno].parseCmd = simParsePrintCmd; simCmdList[cmdno].executeCmd = simExecutePrintCmd; simAddCmdIntoHash(&(simCmdList[cmdno])); @@ -999,7 +999,7 @@ void simInitsimCmdList() { cmdno = SIM_CMD_SQL; simCmdList[cmdno].cmdno = cmdno; strcpy(simCmdList[cmdno].name, "sql"); - simCmdList[cmdno].nlen = strlen(simCmdList[cmdno].name); + simCmdList[cmdno].nlen = (int)strlen(simCmdList[cmdno].name); simCmdList[cmdno].parseCmd = simParseSqlCmd; simCmdList[cmdno].executeCmd = simExecuteSqlCmd; simAddCmdIntoHash(&(simCmdList[cmdno])); @@ -1007,7 +1007,7 @@ void simInitsimCmdList() { cmdno = SIM_CMD_SQL_ERROR; simCmdList[cmdno].cmdno = cmdno; strcpy(simCmdList[cmdno].name, "sql_error"); - simCmdList[cmdno].nlen = strlen(simCmdList[cmdno].name); + simCmdList[cmdno].nlen = (int)strlen(simCmdList[cmdno].name); simCmdList[cmdno].parseCmd = simParseSqlErrorCmd; simCmdList[cmdno].executeCmd = simExecuteSqlErrorCmd; simAddCmdIntoHash(&(simCmdList[cmdno])); @@ -1015,7 +1015,7 @@ void simInitsimCmdList() { cmdno = SIM_CMD_SQL_SLOW; simCmdList[cmdno].cmdno = cmdno; strcpy(simCmdList[cmdno].name, "sql_slow"); - simCmdList[cmdno].nlen = strlen(simCmdList[cmdno].name); + simCmdList[cmdno].nlen = (int)strlen(simCmdList[cmdno].name); simCmdList[cmdno].parseCmd = simParseSqlSlowCmd; simCmdList[cmdno].executeCmd = simExecuteSqlSlowCmd; simAddCmdIntoHash(&(simCmdList[cmdno])); @@ -1024,14 +1024,14 @@ void simInitsimCmdList() { cmdno = SIM_CMD_TEST; simCmdList[cmdno].cmdno = cmdno; strcpy(simCmdList[cmdno].name, "test"); - simCmdList[cmdno].nlen = strlen(simCmdList[cmdno].name); + simCmdList[cmdno].nlen = (int)strlen(simCmdList[cmdno].name); simCmdList[cmdno].parseCmd = NULL; simCmdList[cmdno].executeCmd = simExecuteTestCmd; cmdno = SIM_CMD_RETURN; simCmdList[cmdno].cmdno = cmdno; strcpy(simCmdList[cmdno].name, "return"); - simCmdList[cmdno].nlen = strlen(simCmdList[cmdno].name); + simCmdList[cmdno].nlen = (int)strlen(simCmdList[cmdno].name); simCmdList[cmdno].parseCmd = simParseReturnCmd; simCmdList[cmdno].executeCmd = simExecuteReturnCmd; simAddCmdIntoHash(&(simCmdList[cmdno])); From 1d7626df1c2f83513d991ad33296da5c6e5194ef Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Fri, 7 Aug 2020 17:58:52 +0800 Subject: [PATCH 069/109] fix several bugs related to subscribe including: * server side crash * client side crash * server side memory leak --- src/client/inc/tscUtil.h | 6 ++++++ src/client/src/tscServer.c | 2 +- src/client/src/tscSub.c | 3 +++ src/query/src/qExecutor.c | 5 ++++- src/tsdb/src/tsdbRead.c | 7 ++----- 5 files changed, 16 insertions(+), 7 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 590f205e1d..fbec2e2167 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -53,12 +53,18 @@ typedef struct SParsedDataColInfo { bool hasVal[TSDB_MAX_COLUMNS]; } SParsedDataColInfo; +#pragma pack(push,1) +// this struct is transfered as binary, padding two bytes to avoid +// an 'uid' whose low bytes is 0xff being recoginized as NULL, +// and set 'pack' to 1 to avoid break existing code. typedef struct STidTags { + int16_t padding; int64_t uid; int32_t tid; int32_t vgId; char tag[]; } STidTags; +#pragma pack(pop) typedef struct SJoinSupporter { SSubqueryState* pState; diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 4d4254fa46..2977ed01ec 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -912,7 +912,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pSql->cmd.msgType = TSDB_MSG_TYPE_QUERY; pQueryMsg->head.contLen = htonl(msgLen); - assert(msgLen + minMsgSize() <= size); + assert(msgLen + minMsgSize() <= pCmd->allocSize); return TSDB_CODE_SUCCESS; } diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c index 81d73236a4..bef754d69f 100644 --- a/src/client/src/tscSub.c +++ b/src/client/src/tscSub.c @@ -410,6 +410,9 @@ TAOS_RES *taos_consume(TAOS_SUB *tsub) { } } + size_t size = taosArrayGetSize(pSub->progress) * sizeof(STableIdInfo); + size += sizeof(SQueryTableMsg) + 4096; + tscAllocPayload(&pSql->cmd, size); for (int retry = 0; retry < 3; retry++) { tscRemoveFromSqlList(pSql); diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 22c2bb5c9a..be3d476be5 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -6549,12 +6549,15 @@ static void buildTagQueryResult(SQInfo* pQInfo) { int32_t i = pQInfo->tableIndex++; STableQueryInfo *item = taosArrayGetP(pa, i); - char *output = pQuery->sdata[0]->data + i * rsize; + char *output = pQuery->sdata[0]->data + count * rsize; varDataSetLen(output, rsize - VARSTR_HEADER_SIZE); output = varDataVal(output); STableId* id = TSDB_TABLEID(item->pTable); + *(int16_t *)output = 0; + output += sizeof(int16_t); + *(int64_t *)output = id->uid; // memory align problem, todo serialize output += sizeof(id->uid); diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index a34216d2fb..7fd2e62070 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -2442,11 +2442,8 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) { STableCheckInfo* pTableCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); destroyTableMemIterator(pTableCheckInfo); - if (pTableCheckInfo->pDataCols != NULL) { - taosTFree(pTableCheckInfo->pDataCols->buf); - } - - taosTFree(pTableCheckInfo->pDataCols); + tdFreeDataCols(pTableCheckInfo->pDataCols); + pTableCheckInfo->pDataCols = NULL; taosTFree(pTableCheckInfo->pCompInfo); } taosArrayDestroy(pQueryHandle->pTableCheckInfo); From 941705c5d6be4c76d54e40aa879b95db4677e34b Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Sat, 8 Aug 2020 10:29:50 +0800 Subject: [PATCH 070/109] td-1089: fix memory leak and invalid read --- src/client/inc/tscUtil.h | 1 + src/client/src/tscSub.c | 16 +++++++++++++++- src/client/src/tscUtil.c | 12 +++++++++++- 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index fbec2e2167..b15286fe80 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -230,6 +230,7 @@ int32_t tscAddSubqueryInfo(SSqlCmd *pCmd); void tscInitQueryInfo(SQueryInfo* pQueryInfo); void tscClearSubqueryInfo(SSqlCmd* pCmd); +void tscFreeVgroupTableInfo(SArray* pVgroupTables); int tscGetSTableVgroupInfo(SSqlObj* pSql, int32_t clauseIndex); int tscGetTableMeta(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo); diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c index bef754d69f..f740e3cce9 100644 --- a/src/client/src/tscSub.c +++ b/src/client/src/tscSub.c @@ -230,6 +230,19 @@ static SArray* getTableList( SSqlObj* pSql ) { return result; } +static int32_t compareTidTag(const void* p1, const void* p2) { + const STidTags* t1 = (const STidTags*)p1; + const STidTags* t2 = (const STidTags*)p2; + + if (t1->vgId != t2->vgId) { + return (t1->vgId > t2->vgId) ? 1 : -1; + } + if (t1->tid != t2->tid) { + return (t1->tid > t2->tid) ? 1 : -1; + } + return 0; +} + static int tscUpdateSubscription(STscObj* pObj, SSub* pSub) { SSqlObj* pSql = pSub->pSql; @@ -270,7 +283,8 @@ static int tscUpdateSubscription(STscObj* pObj, SSub* pSub) { pSub->progress = progress; if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - taosArraySort( tables, tscCompareTidTags ); + taosArraySort( tables, compareTidTag ); + tscFreeVgroupTableInfo(pTableMetaInfo->pVgroupTables); tscBuildVgroupTableInfo(pSql, pTableMetaInfo, tables); } taosArrayDestroy(tables); diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index c74cc82f85..11add5d47e 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1556,12 +1556,22 @@ void tscClearSubqueryInfo(SSqlCmd* pCmd) { } } +void tscFreeVgroupTableInfo(SArray* pVgroupTables) { + if (pVgroupTables != NULL) { + for (size_t i = 0; i < taosArrayGetSize(pVgroupTables); i++) { + SVgroupTableInfo* pInfo = taosArrayGet(pVgroupTables, i); + taosArrayDestroy(pInfo->itemList); + } + taosArrayDestroy(pVgroupTables); + } +} + void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool removeFromCache) { tscDebug("%p deref the table meta in cache, numOfTables:%d", address, pQueryInfo->numOfTables); for(int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); - + tscFreeVgroupTableInfo(pTableMetaInfo->pVgroupTables); tscClearTableMetaInfo(pTableMetaInfo, removeFromCache); free(pTableMetaInfo); } From 2b36e2a2c32558acc7f5ad42a4e227e34cc6daca Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Sat, 8 Aug 2020 11:33:23 +0800 Subject: [PATCH 071/109] add and update testcases --- src/client/src/tscFunctionImpl.c | 2 +- tests/examples/c/subscribe.c | 103 +++++++++++++++------------ tests/pytest/subscribe/stability.py | 93 ++++++++++++++++++++++++ tests/pytest/subscribe/supertable.py | 26 ++++--- 4 files changed, 167 insertions(+), 57 deletions(-) create mode 100644 tests/pytest/subscribe/stability.py diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index d1f5b510a4..9c0951cdd6 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -169,7 +169,7 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI // (uid, tid) + VGID + TAGSIZE + VARSTR_HEADER_SIZE if (functionId == TSDB_FUNC_TID_TAG) { // todo use struct *type = TSDB_DATA_TYPE_BINARY; - *bytes = (int16_t)(dataBytes + sizeof(int64_t) + sizeof(int32_t) + sizeof(int32_t) + VARSTR_HEADER_SIZE); + *bytes = (int16_t)(dataBytes + sizeof(int16_t) + sizeof(int64_t) + sizeof(int32_t) + sizeof(int32_t) + VARSTR_HEADER_SIZE); *interBytes = *bytes; return TSDB_CODE_SUCCESS; } diff --git a/tests/examples/c/subscribe.c b/tests/examples/c/subscribe.c index f9acf2bb10..db5ad34ee7 100644 --- a/tests/examples/c/subscribe.c +++ b/tests/examples/c/subscribe.c @@ -7,28 +7,31 @@ #include // include TDengine header file #include +int nTotalRows; + void print_result(TAOS_RES* res, int blockFetch) { TAOS_ROW row = NULL; int num_fields = taos_num_fields(res); TAOS_FIELD* fields = taos_fetch_fields(res); int nRows = 0; + char buf[4096]; + if (blockFetch) { nRows = taos_fetch_block(res, &row); for (int i = 0; i < nRows; i++) { - char temp[256]; - taos_print_row(temp, row + i, fields, num_fields); - puts(temp); + taos_print_row(buf, row + i, fields, num_fields); + puts(buf); } } else { while ((row = taos_fetch_row(res))) { - char temp[256]; - taos_print_row(temp, row, fields, num_fields); - puts(temp); + taos_print_row(buf, row, fields, num_fields); + puts(buf); nRows++; } } + nTotalRows += nRows; printf("%d rows consumed.\n", nRows); } @@ -52,47 +55,52 @@ void check_row_count(int line, TAOS_RES* res, int expected) { } +void do_query(TAOS* taos, const char* sql) { + TAOS_RES* res = taos_query(taos, "drop database if exists test;"); + taos_free_result(res); +} + + void run_test(TAOS* taos) { - taos_query(taos, "drop database if exists test;"); + do_query(taos, "drop database if exists test;"); usleep(100000); - //taos_query(taos, "create database test tables 5;"); - taos_query(taos, "create database test;"); + do_query(taos, "create database test;"); usleep(100000); - taos_query(taos, "use test;"); + do_query(taos, "use test;"); usleep(100000); - taos_query(taos, "create table meters(ts timestamp, a int) tags(area int);"); + do_query(taos, "create table meters(ts timestamp, a int) tags(area int);"); - taos_query(taos, "create table t0 using meters tags(0);"); - taos_query(taos, "create table t1 using meters tags(1);"); - taos_query(taos, "create table t2 using meters tags(2);"); - taos_query(taos, "create table t3 using meters tags(3);"); - taos_query(taos, "create table t4 using meters tags(4);"); - taos_query(taos, "create table t5 using meters tags(5);"); - taos_query(taos, "create table t6 using meters tags(6);"); - taos_query(taos, "create table t7 using meters tags(7);"); - taos_query(taos, "create table t8 using meters tags(8);"); - taos_query(taos, "create table t9 using meters tags(9);"); + do_query(taos, "create table t0 using meters tags(0);"); + do_query(taos, "create table t1 using meters tags(1);"); + do_query(taos, "create table t2 using meters tags(2);"); + do_query(taos, "create table t3 using meters tags(3);"); + do_query(taos, "create table t4 using meters tags(4);"); + do_query(taos, "create table t5 using meters tags(5);"); + do_query(taos, "create table t6 using meters tags(6);"); + do_query(taos, "create table t7 using meters tags(7);"); + do_query(taos, "create table t8 using meters tags(8);"); + do_query(taos, "create table t9 using meters tags(9);"); - taos_query(taos, "insert into t0 values('2020-01-01 00:00:00.000', 0);"); - taos_query(taos, "insert into t0 values('2020-01-01 00:01:00.000', 0);"); - taos_query(taos, "insert into t0 values('2020-01-01 00:02:00.000', 0);"); - taos_query(taos, "insert into t1 values('2020-01-01 00:00:00.000', 0);"); - taos_query(taos, "insert into t1 values('2020-01-01 00:01:00.000', 0);"); - taos_query(taos, "insert into t1 values('2020-01-01 00:02:00.000', 0);"); - taos_query(taos, "insert into t1 values('2020-01-01 00:03:00.000', 0);"); - taos_query(taos, "insert into t2 values('2020-01-01 00:00:00.000', 0);"); - taos_query(taos, "insert into t2 values('2020-01-01 00:01:00.000', 0);"); - taos_query(taos, "insert into t2 values('2020-01-01 00:01:01.000', 0);"); - taos_query(taos, "insert into t2 values('2020-01-01 00:01:02.000', 0);"); - taos_query(taos, "insert into t3 values('2020-01-01 00:01:02.000', 0);"); - taos_query(taos, "insert into t4 values('2020-01-01 00:01:02.000', 0);"); - taos_query(taos, "insert into t5 values('2020-01-01 00:01:02.000', 0);"); - taos_query(taos, "insert into t6 values('2020-01-01 00:01:02.000', 0);"); - taos_query(taos, "insert into t7 values('2020-01-01 00:01:02.000', 0);"); - taos_query(taos, "insert into t8 values('2020-01-01 00:01:02.000', 0);"); - taos_query(taos, "insert into t9 values('2020-01-01 00:01:02.000', 0);"); + do_query(taos, "insert into t0 values('2020-01-01 00:00:00.000', 0);"); + do_query(taos, "insert into t0 values('2020-01-01 00:01:00.000', 0);"); + do_query(taos, "insert into t0 values('2020-01-01 00:02:00.000', 0);"); + do_query(taos, "insert into t1 values('2020-01-01 00:00:00.000', 0);"); + do_query(taos, "insert into t1 values('2020-01-01 00:01:00.000', 0);"); + do_query(taos, "insert into t1 values('2020-01-01 00:02:00.000', 0);"); + do_query(taos, "insert into t1 values('2020-01-01 00:03:00.000', 0);"); + do_query(taos, "insert into t2 values('2020-01-01 00:00:00.000', 0);"); + do_query(taos, "insert into t2 values('2020-01-01 00:01:00.000', 0);"); + do_query(taos, "insert into t2 values('2020-01-01 00:01:01.000', 0);"); + do_query(taos, "insert into t2 values('2020-01-01 00:01:02.000', 0);"); + do_query(taos, "insert into t3 values('2020-01-01 00:01:02.000', 0);"); + do_query(taos, "insert into t4 values('2020-01-01 00:01:02.000', 0);"); + do_query(taos, "insert into t5 values('2020-01-01 00:01:02.000', 0);"); + do_query(taos, "insert into t6 values('2020-01-01 00:01:02.000', 0);"); + do_query(taos, "insert into t7 values('2020-01-01 00:01:02.000', 0);"); + do_query(taos, "insert into t8 values('2020-01-01 00:01:02.000', 0);"); + do_query(taos, "insert into t9 values('2020-01-01 00:01:02.000', 0);"); // super tables subscription usleep(1000000); @@ -104,23 +112,23 @@ void run_test(TAOS* taos) { res = taos_consume(tsub); check_row_count(__LINE__, res, 0); - taos_query(taos, "insert into t0 values('2020-01-01 00:02:00.001', 0);"); - taos_query(taos, "insert into t8 values('2020-01-01 00:01:03.000', 0);"); + do_query(taos, "insert into t0 values('2020-01-01 00:02:00.001', 0);"); + do_query(taos, "insert into t8 values('2020-01-01 00:01:03.000', 0);"); res = taos_consume(tsub); check_row_count(__LINE__, res, 2); - taos_query(taos, "insert into t2 values('2020-01-01 00:01:02.001', 0);"); - taos_query(taos, "insert into t1 values('2020-01-01 00:03:00.001', 0);"); + do_query(taos, "insert into t2 values('2020-01-01 00:01:02.001', 0);"); + do_query(taos, "insert into t1 values('2020-01-01 00:03:00.001', 0);"); res = taos_consume(tsub); check_row_count(__LINE__, res, 2); - taos_query(taos, "insert into t1 values('2020-01-01 00:03:00.002', 0);"); + do_query(taos, "insert into t1 values('2020-01-01 00:03:00.002', 0);"); res = taos_consume(tsub); check_row_count(__LINE__, res, 1); // keep progress information and restart subscription taos_unsubscribe(tsub, 1); - taos_query(taos, "insert into t0 values('2020-01-01 00:04:00.000', 0);"); + do_query(taos, "insert into t0 values('2020-01-01 00:04:00.000', 0);"); tsub = taos_subscribe(taos, 1, "test", "select * from meters;", NULL, NULL, 0); res = taos_consume(tsub); check_row_count(__LINE__, res, 24); @@ -147,7 +155,7 @@ void run_test(TAOS* taos) { res = taos_consume(tsub); check_row_count(__LINE__, res, 0); - taos_query(taos, "insert into t0 values('2020-01-01 00:04:00.001', 0);"); + do_query(taos, "insert into t0 values('2020-01-01 00:04:00.001', 0);"); res = taos_consume(tsub); check_row_count(__LINE__, res, 1); @@ -223,7 +231,7 @@ int main(int argc, char *argv[]) { exit(0); } - taos_query(taos, "use test;"); + taos_select_db(taos, "test"); TAOS_SUB* tsub = NULL; if (async) { // create an asynchronized subscription, the callback function will be called every 1s @@ -251,6 +259,7 @@ int main(int argc, char *argv[]) { } } + printf("total rows consumed: %d\n", nTotalRows); taos_unsubscribe(tsub, keep); taos_close(taos); diff --git a/tests/pytest/subscribe/stability.py b/tests/pytest/subscribe/stability.py new file mode 100644 index 0000000000..ddd8b3282a --- /dev/null +++ b/tests/pytest/subscribe/stability.py @@ -0,0 +1,93 @@ +################################################################### + # Copyright (c) 2020 by TAOS Technologies, Inc. + # All rights reserved. + # + # This file is proprietary and confidential to TAOS Technologies. + # No part of this file may be reproduced, stored, transmitted, + # disclosed or used in any form or by any means other than as + # expressly provided by the written permission from Jianhui Tao + # +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +import time +import random +import string +from util.log import * +from util.cases import * +from util.sql import * +from util.sub import * + +class TDTestCase: + maxTables = 10000 + maxCols = 50 + rowsPerSecond = 1000 + + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdLog.notice("NOTE: this case does not stop automatically, Ctrl+C to stop") + tdSql.init(conn.cursor(), logSql) + self.conn = conn + + + def generateString(self, length): + chars = string.ascii_uppercase + string.ascii_lowercase + v = "" + for i in range(length): + v += random.choice(chars) + return v + + + def insert(self): + id = random.randint(0, self.maxTables - 1) + cola = self.generateString(40) + sql = "insert into car%d values(now, '%s', %f, %d" % (id, cola, random.random()*100, random.randint(0, 2)) + for i in range(self.maxCols): + sql += ", %d" % random.randint(0, self.maxTables) + sql += ")" + tdSql.execute(sql) + + + def prepare(self): + tdLog.info("prepare database: test") + tdSql.execute('reset query cache') + tdSql.execute('drop database if exists test') + tdSql.execute('create database test') + tdSql.execute('use test') + + def run(self): + self.prepare() + + sql = "create table cars (ts timestamp, a binary(50), b float, c bool" + for i in range(self.maxCols): + sql += ", c%d int" % i + sql += ") tags(id int, category binary(30), brand binary(30));" + tdSql.execute(sql) + + for i in range(self.maxTables): + tdSql.execute("create table car%d using cars tags(%d, 'category%d', 'brand%d')" % (i, i, i % 30, i // 30)) + + time.sleep(0.1) + + total = 0 + while True: + start = time.time() + for i in range(self.rowsPerSecond): + self.insert() + total = total + 1 + d = time.time() - start + tdLog.info("%d rows inserted in %f seconds, total %d" % (self.rowsPerSecond, d, total)) + if d < 1: + time.sleep(1 - d) + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/subscribe/supertable.py b/tests/pytest/subscribe/supertable.py index c6cc2969aa..ee3aa225bd 100644 --- a/tests/pytest/subscribe/supertable.py +++ b/tests/pytest/subscribe/supertable.py @@ -31,16 +31,19 @@ class TDTestCase: now = int(time.time() * 1000) tdSql.prepare() - tdLog.info("create a super table and 10 sub-tables, then insert 5 rows into each sub-table.") + numTables = 2000 + rowsPerTable = 5 + totalRows = numTables * rowsPerTable + tdLog.info("create a super table and %d sub-tables, then insert %d rows into each sub-table." % (numTables, rowsPerTable)) tdSql.execute("create table meters(ts timestamp, a int, b int) tags(area int, loc binary(20));") - for i in range(0, 10): - for j in range(0, 5): + for i in range(0, numTables): + for j in range(0, rowsPerTable): tdSql.execute("insert into t%d using meters tags(%d, 'area%d') values (%d, %d, %d);" % (i, i, i, now + j, j, j)) tdLog.info("consumption 01.") tdSub.init(self.conn.subscribe(True, topic, sqlstr, 0)) tdSub.consume() - tdSub.checkRows(50) + tdSub.checkRows(totalRows) tdLog.info("consumption 02: no new rows inserted") tdSub.consume() @@ -61,17 +64,17 @@ class TDTestCase: tdSub.close(False) tdSub.init(self.conn.subscribe(False, topic, sqlstr, 0)) tdSub.consume() - tdSub.checkRows(51) + tdSub.checkRows(totalRows + 1) tdLog.info("consumption 06: keep progress and restart the subscription") tdSub.close(True) tdSub.init(self.conn.subscribe(True, topic, sqlstr, 0)) tdSub.consume() - tdSub.checkRows(51) + tdSub.checkRows(totalRows + 1) tdLog.info("consumption 07: insert one row to two table then remove one table") tdSql.execute("insert into t0 values (%d, 11, 11);" % (now + 11)) - tdSql.execute("insert into t1 values (%d, 11, 11);" % (now + 11)) + tdSql.execute("insert into t%d values (%d, 11, 11);" % ((numTables-1), (now + 11))) tdSql.execute("drop table t0") tdSub.consume() tdSub.checkRows(1) @@ -80,7 +83,7 @@ class TDTestCase: tdSub.close(False) tdSub.init(self.conn.subscribe(True, topic, sqlstr + " where ts > %d" % now, 0)) tdSub.consume() - tdSub.checkRows(37) + tdSub.checkRows((numTables-1) * (rowsPerTable-1) + 1) tdLog.info("consumption 09: insert large timestamp to t2 then insert smaller timestamp to t1") tdSql.execute("insert into t2 values (%d, 100, 100);" % (now + 100)) @@ -101,10 +104,15 @@ class TDTestCase: tdLog.info("consumption 11: two vnodes") tdSql.execute("insert into t2 values (%d, 102, 100);" % (now + 104)) - tdSql.execute("insert into t9 values (%d, 102, 100);" % (now + 104)) + tdSql.execute("insert into t1299 values (%d, 102, 100);" % (now + 104)) tdSub.consume() tdSub.checkRows(2) + tdLog.info("consumption 12: create a new table") + tdSql.execute("insert into t%d using meters tags(%d, 'area%d') values (%d, 102, 100);" % (numTables, numTables, numTables, now + 105)) + tdSub.consume() + tdSub.checkRows(1) + def stop(self): tdSub.close(False) tdSql.close() From ed896f976b706bde52a5875b08183871465aa656 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 8 Aug 2020 13:14:12 +0800 Subject: [PATCH 072/109] TD-1057 --- cmake/install.inc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/install.inc b/cmake/install.inc index 8d3acde45d..98a60ace7b 100755 --- a/cmake/install.inc +++ b/cmake/install.inc @@ -27,7 +27,7 @@ ELSEIF (TD_WINDOWS_64) #INSTALL(TARGETS taos RUNTIME DESTINATION driver) #INSTALL(TARGETS shell RUNTIME DESTINATION .) IF (TD_MVN_INSTALLED) - INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-1.0.2-dist.jar DESTINATION connector/jdbc) + INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.0-dist.jar DESTINATION connector/jdbc) ENDIF () ELSE () INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/libtaos.dll DESTINATION driver) From a945922c67f60af866cea10bb82dd298279f2c7a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 8 Aug 2020 13:44:26 +0800 Subject: [PATCH 073/109] [td-225] fix invalid read. --- src/query/src/qAst.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/query/src/qAst.c b/src/query/src/qAst.c index 44051c7e3b..05df8d95ba 100644 --- a/src/query/src/qAst.c +++ b/src/query/src/qAst.c @@ -689,7 +689,8 @@ static void tQueryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, } if (addToResult) { - taosArrayPush(res, pData); + STableKeyInfo info = {.pTable = *(void**)pData, .lastKey = -1}; + taosArrayPush(res, &info); } } From 4706b400d089065663d837ca30dc5eeaaaedf93b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 8 Aug 2020 14:18:58 +0800 Subject: [PATCH 074/109] [td-225] fix invalid read. --- src/query/inc/qAst.h | 2 +- src/query/src/qAst.c | 26 ++++++++++++++++---------- src/tsdb/src/tsdbRead.c | 2 +- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/src/query/inc/qAst.h b/src/query/inc/qAst.h index 01b4c16ac1..ec568a6cdb 100644 --- a/src/query/inc/qAst.h +++ b/src/query/inc/qAst.h @@ -48,7 +48,7 @@ typedef struct tQueryInfo { SSchema sch; // schema of tags char* q; __compar_fn_t compare; // filter function - void* param; // STSchema + bool indexed; // indexed columns } tQueryInfo; typedef struct SExprTraverseSupp { diff --git a/src/query/src/qAst.c b/src/query/src/qAst.c index 05df8d95ba..6d9e5856e2 100644 --- a/src/query/src/qAst.c +++ b/src/query/src/qAst.c @@ -427,8 +427,9 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr if (ret != 0) { break; } - - taosArrayPush(result, SL_GET_NODE_DATA(pNode)); + + STableKeyInfo info = {.pTable = *(void**)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; + taosArrayPush(result, &info); } } else if (optr == TSDB_RELATION_GREATER || optr == TSDB_RELATION_GREATER_EQUAL) { // greater equal bool comp = true; @@ -445,7 +446,8 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr if (ret == 0 && optr == TSDB_RELATION_GREATER) { continue; } else { - taosArrayPush(result, SL_GET_NODE_DATA(pNode)); + STableKeyInfo info = {.pTable = *(void**)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; + taosArrayPush(result, &info); comp = false; } } @@ -458,8 +460,9 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr if (comp) { continue; } - - taosArrayPush(result, SL_GET_NODE_DATA(pNode)); + + STableKeyInfo info = {.pTable = *(void**)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; + taosArrayPush(result, &info); } tSkipListDestroyIter(iter); @@ -472,8 +475,9 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr if (comp) { continue; } - - taosArrayPush(result, SL_GET_NODE_DATA(pNode)); + + STableKeyInfo info = {.pTable = *(void**)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; + taosArrayPush(result, &info); } } else { @@ -496,12 +500,14 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr if (ret == 0 && optr == TSDB_RELATION_LESS) { continue; } else { - taosArrayPush(result, SL_GET_NODE_DATA(pNode)); + STableKeyInfo info = {.pTable = *(void**)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; + taosArrayPush(result, &info); comp = false; // no need to compare anymore } } } } + free(cond.start); free(cond.end); tSkipListDestroyIter(iter); @@ -689,7 +695,7 @@ static void tQueryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, } if (addToResult) { - STableKeyInfo info = {.pTable = *(void**)pData, .lastKey = -1}; + STableKeyInfo info = {.pTable = *(void**)pData, .lastKey = TSKEY_INITIAL_VAL}; taosArrayPush(res, &info); } } @@ -717,7 +723,7 @@ void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, S } tQueryInfo *pQueryInfo = pExpr->_node.info; - if (pQueryInfo->sch.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX && pQueryInfo->optr != TSDB_RELATION_LIKE) { + if (pQueryInfo->indexed && pQueryInfo->optr != TSDB_RELATION_LIKE) { tQueryIndexColumn(pSkipList, pQueryInfo, result); } else { tQueryIndexlessColumn(pSkipList, pQueryInfo, result, param->nodeFilterFn); diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index b086451dd1..e06bced896 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -2102,7 +2102,7 @@ void filterPrepare(void* expr, void* param) { pInfo->sch = *pSchema; pInfo->optr = pExpr->_node.optr; pInfo->compare = getComparFunc(pSchema->type, pInfo->optr); - pInfo->param = pTSSchema; + pInfo->indexed = pTSSchema->columns->colId == pInfo->sch.colId; if (pInfo->optr == TSDB_RELATION_IN) { pInfo->q = (char*) pCond->arr; From 0bc29a0130cf8f42fc9cb6d42aeb560e8d10c43b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 8 Aug 2020 14:51:46 +0800 Subject: [PATCH 075/109] TD-1057 --- src/kit/taosdemo/taosdemo.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index a34964cb2e..9d46ac5055 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -251,7 +251,7 @@ typedef struct DemoArguments { void parse_args(int argc, char *argv[], SDemoArguments *arguments) { argp_parse(&argp, argc, argv, 0, 0, &arguments); - if (arguments.abort) { + if (arguments->abort) { #ifndef _ALPINE error(10, 0, "ABORTED"); #else From 8e30461cff375e9aa8013cbfc5a795854677a8ad Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 8 Aug 2020 16:33:38 +0800 Subject: [PATCH 076/109] [td-225] fix memory leak --- src/tsdb/src/tsdbRead.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index e06bced896..17cee360f3 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -2488,11 +2488,7 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) { STableCheckInfo* pTableCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); destroyTableMemIterator(pTableCheckInfo); - if (pTableCheckInfo->pDataCols != NULL) { - taosTFree(pTableCheckInfo->pDataCols->buf); - } - - taosTFree(pTableCheckInfo->pDataCols); + tdFreeDataCols(pTableCheckInfo->pDataCols); taosTFree(pTableCheckInfo->pCompInfo); } taosArrayDestroy(pQueryHandle->pTableCheckInfo); From ef8135a609f5b8ff15b095acb3a589f5d66b8ee4 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 8 Aug 2020 16:38:44 +0800 Subject: [PATCH 077/109] TD-1057 --- src/common/src/tglobal.c | 1 + src/os/inc/osSocket.h | 6 +++++- src/os/inc/osWindows.h | 3 +++ src/os/src/windows/w64Env.c | 3 +++ src/plugins/http/src/httpServer.c | 1 + src/rpc/src/rpcTcp.c | 3 ++- src/sync/src/taosTcpPool.c | 3 ++- 7 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index c20061d11b..386ea2f9a7 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -295,6 +295,7 @@ bool taosCfgDynamicOptions(char *msg) { } static void doInitGlobalConfig(void) { + osInit(); SGlobalCfg cfg = {0}; // ip address diff --git a/src/os/inc/osSocket.h b/src/os/inc/osSocket.h index b9be39307c..58f95c3c2d 100644 --- a/src/os/inc/osSocket.h +++ b/src/os/inc/osSocket.h @@ -33,7 +33,11 @@ extern "C" { } \ } typedef int SOCKET; -#endif +#endif + +#ifndef TAOS_OS_DEF_EPOLL + #define TAOS_EPOLL_WAIT_TIME -1 +#endif #define taosClose(x) taosCloseSocket(x) diff --git a/src/os/inc/osWindows.h b/src/os/inc/osWindows.h index dcf087beeb..caab61536e 100644 --- a/src/os/inc/osWindows.h +++ b/src/os/inc/osWindows.h @@ -94,6 +94,9 @@ extern "C" { typedef SOCKET eventfd_t; #define eventfd(a, b) -1 +#define TAOS_OS_DEF_EPOLL + #define TAOS_EPOLL_WAIT_TIME 100 + #define TAOS_OS_FUNC_STRING_WCHAR int twcslen(const wchar_t *wcs); #define TAOS_OS_FUNC_STRING_GETLINE diff --git a/src/os/src/windows/w64Env.c b/src/os/src/windows/w64Env.c index 840a8d9f7a..333871e96c 100644 --- a/src/os/src/windows/w64Env.c +++ b/src/os/src/windows/w64Env.c @@ -18,6 +18,8 @@ #include "tglobal.h" #include "tulog.h" +extern void taosWinSocketInit(); + void osInit() { strcpy(configDir, "C:/TDengine/cfg"); strcpy(tsVnodeDir, "C:/TDengine/data"); @@ -27,4 +29,5 @@ void osInit() { strcpy(tsLogDir, "C:/TDengine/log"); strcpy(tsScriptDir, "C:/TDengine/script"); strcpy(tsOsName, "Windows"); + taosWinSocketInit(); } \ No newline at end of file diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c index 5c910a3311..8ee92be31c 100644 --- a/src/plugins/http/src/httpServer.c +++ b/src/plugins/http/src/httpServer.c @@ -38,6 +38,7 @@ static void httpStopThread(HttpThread* pThread) { eventfd_t fd = eventfd(1, 0); if (fd == -1) { httpError("%s, failed to create eventfd, will call pthread_cancel instead, which may result in data corruption: %s", pThread->label, strerror(errno)); + pThread->stop = true; pthread_cancel(pThread->thread); } else if (epoll_ctl(pThread->pollFd, EPOLL_CTL_ADD, fd, &event) < 0) { httpError("%s, failed to call epoll_ctl, will call pthread_cancel instead, which may result in data corruption: %s", pThread->label, strerror(errno)); diff --git a/src/rpc/src/rpcTcp.c b/src/rpc/src/rpcTcp.c index ac617edfc3..9da11831e5 100644 --- a/src/rpc/src/rpcTcp.c +++ b/src/rpc/src/rpcTcp.c @@ -174,6 +174,7 @@ static void taosStopTcpThread(SThreadObj* pThreadObj) { if (fd == -1) { // failed to create eventfd, call pthread_cancel instead, which may result in data corruption: tError("%s, failed to create eventfd(%s)", pThreadObj->label, strerror(errno)); + pThreadObj->stop = true; pthread_cancel(pThreadObj->thread); } else if (epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_ADD, fd, &event) < 0) { // failed to call epoll_ctl, call pthread_cancel instead, which may result in data corruption: @@ -464,7 +465,7 @@ static void *taosProcessTcpData(void *param) { SRecvInfo recvInfo; while (1) { - int fdNum = epoll_wait(pThreadObj->pollFd, events, maxEvents, -1); + int fdNum = epoll_wait(pThreadObj->pollFd, events, maxEvents, TAOS_EPOLL_WAIT_TIME); if (pThreadObj->stop) { tDebug("%s TCP thread get stop event, exiting...", pThreadObj->label); break; diff --git a/src/sync/src/taosTcpPool.c b/src/sync/src/taosTcpPool.c index 88c2c53be4..b523728bf9 100644 --- a/src/sync/src/taosTcpPool.c +++ b/src/sync/src/taosTcpPool.c @@ -184,7 +184,7 @@ static void *taosProcessTcpData(void *param) { while (1) { if (pThread->stop) break; - int fdNum = epoll_wait(pThread->pollFd, events, maxEvents, -1); + int fdNum = epoll_wait(pThread->pollFd, events, maxEvents, TAOS_EPOLL_WAIT_TIME); if (pThread->stop) { uDebug("%p TCP epoll thread is exiting...", pThread); break; @@ -313,6 +313,7 @@ static void taosStopPoolThread(SThreadObj* pThread) { // failed to create eventfd, call pthread_cancel instead, which may result in data corruption uError("failed to create eventfd(%s)", strerror(errno)); pthread_cancel(pThread->thread); + pThread->stop = true; } else if (epoll_ctl(pThread->pollFd, EPOLL_CTL_ADD, fd, &event) < 0) { // failed to call epoll_ctl, call pthread_cancel instead, which may result in data corruption uError("failed to call epoll_ctl(%s)", strerror(errno)); From 222ebe3602bf0ac9fd7766a9d38b5a268b336a1b Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 8 Aug 2020 17:14:57 +0800 Subject: [PATCH 078/109] add file size statistics --- src/tsdb/inc/tsdbMain.h | 14 +++++++++++--- src/tsdb/src/tsdbFile.c | 2 +- src/tsdb/src/tsdbMemTable.c | 4 ++-- src/tsdb/src/tsdbRWHelper.c | 36 ++++++++++++++++++++++++++++-------- 4 files changed, 42 insertions(+), 14 deletions(-) diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 16a2e17f1e..f42c00f171 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -154,11 +154,19 @@ typedef enum { TSDB_FILE_TYPE_HEAD = 0, TSDB_FILE_TYPE_DATA, TSDB_FILE_TYPE_LAST, - TSDB_FILE_TYPE_MAX, + TSDB_FILE_TYPE_STAT, TSDB_FILE_TYPE_NHEAD, - TSDB_FILE_TYPE_NLAST + TSDB_FILE_TYPE_NDATA, + TSDB_FILE_TYPE_NLAST, + TSDB_FILE_TYPE_NSTAT } TSDB_FILE_TYPE; +#ifndef TDINTERNAL +#define TSDB_FILE_TYPE_MAX (TSDB_FILE_TYPE_LAST+1) +#else +#define TSDB_FILE_TYPE_MAX (TSDB_FILE_TYPE_STAT+1) +#endif + typedef struct { uint32_t magic; uint32_t len; @@ -497,7 +505,7 @@ int tsdbInitWriteHelper(SRWHelper* pHelper, STsdbRepo* pRepo); void tsdbDestroyHelper(SRWHelper* pHelper); void tsdbResetHelper(SRWHelper* pHelper); int tsdbSetAndOpenHelperFile(SRWHelper* pHelper, SFileGroup* pGroup); -int tsdbCloseHelperFile(SRWHelper* pHelper, bool hasError); +int tsdbCloseHelperFile(SRWHelper* pHelper, bool hasError, SFileGroup* pGroup); int tsdbSetHelperTable(SRWHelper* pHelper, STable* pTable, STsdbRepo* pRepo); int tsdbCommitTableData(SRWHelper* pHelper, SCommitIter* pCommitIter, SDataCols* pDataCols, TSKEY maxKey); int tsdbMoveLastBlockIfNeccessary(SRWHelper* pHelper); diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index a5435ad872..d30377ad9f 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -20,7 +20,7 @@ #include "tutil.h" #define TAOS_RANDOM_FILE_FAIL_TEST -const char *tsdbFileSuffix[] = {".head", ".data", ".last", "", ".h", ".l"}; +const char *tsdbFileSuffix[] = {".head", ".data", ".last", ".stat", ".h", ".d", ".l", ".s"}; static int tsdbInitFile(SFile *pFile, STsdbRepo *pRepo, int fid, int type); static void tsdbDestroyFile(SFile *pFile); diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 0303c47146..04b8b64e06 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -679,7 +679,7 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe } taosTFree(dataDir); - tsdbCloseHelperFile(pHelper, 0); + tsdbCloseHelperFile(pHelper, 0, pGroup); pthread_rwlock_wrlock(&(pFileH->fhlock)); @@ -701,7 +701,7 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe _err: taosTFree(dataDir); - tsdbCloseHelperFile(pHelper, 1); + tsdbCloseHelperFile(pHelper, 1, NULL); return -1; } diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index 4b5acd8fe4..a00ed8998a 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -91,7 +91,7 @@ void tsdbResetHelper(SRWHelper *pHelper) { tsdbResetHelperTableImpl(pHelper); // Reset the file part - tsdbCloseHelperFile(pHelper, false); + tsdbCloseHelperFile(pHelper, false, NULL); tsdbResetHelperFileImpl(pHelper); pHelper->state = TSDB_HELPER_CLEAR_STATE; @@ -120,6 +120,14 @@ int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup) { if (tsdbOpenFile(helperDataF(pHelper), O_RDWR) < 0) return -1; if (tsdbOpenFile(helperLastF(pHelper), O_RDWR) < 0) return -1; + // NOTE: For data file compatibility + if (helperDataF(pHelper)->info.size == TSDB_FILE_HEAD_SIZE) { + helperDataF(pHelper)->info.size = (uint64_t)lseek(helperDataF(pHelper)->fd, 0, SEEK_END); + } + if (helperLastF(pHelper)->info.size == TSDB_FILE_HEAD_SIZE) { + helperLastF(pHelper)->info.size = (uint64_t)lseek(helperLastF(pHelper)->fd, 0, SEEK_END); + } + // Create and open .h pFile = helperNewHeadF(pHelper); if (tsdbOpenFile(pFile, O_WRONLY | O_CREAT) < 0) return -1; @@ -146,7 +154,7 @@ int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup) { return 0; } -int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError) { +int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError, SFileGroup *pGroup) { SFile *pFile = NULL; pFile = helperHeadF(pHelper); @@ -157,10 +165,11 @@ int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError) { if (helperType(pHelper) == TSDB_WRITE_HELPER) { if (!hasError) { tsdbUpdateFileHeader(pFile); - fsync(pFile->fd); } else { - // TODO: shrink back to origin + ASSERT(pGroup != NULL); + taosFtruncate(pFile->fd, pGroup->files[TSDB_FILE_TYPE_DATA].info.size); } + fsync(pFile->fd); } tsdbCloseFile(pFile); } @@ -170,10 +179,11 @@ int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError) { if (helperType(pHelper) == TSDB_WRITE_HELPER && !TSDB_NLAST_FILE_OPENED(pHelper)) { if (!hasError) { tsdbUpdateFileHeader(pFile); - fsync(pFile->fd); } else { - // TODO: shrink back to origin + ASSERT(pGroup != NULL); + taosFtruncate(pFile->fd, pGroup->files[TSDB_FILE_TYPE_LAST].info.size); } + fsync(pFile->fd); } tsdbCloseFile(pFile); } @@ -390,6 +400,9 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) { void *pBuf = POINTER_SHIFT(pHelper->pWIdx, pFile->info.len); pFile->info.len += tsdbEncodeSCompIdx(&pBuf, &(pHelper->curCompIdx)); + + pFile->info.size += pIdx->len; + ASSERT(pFile->info.size == lseek(pFile->fd, 0, SEEK_CUR)); } return 0; @@ -420,7 +433,7 @@ int tsdbWriteCompIdx(SRWHelper *pHelper) { return -1; } - pFile->info.offset = offset; + ASSERT(offset == pFile->info.size); if (taosTWrite(pFile->fd, (void *)pHelper->pWIdx, pFile->info.len) < pFile->info.len) { tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(pHelper->pRepo), pFile->info.len, @@ -429,6 +442,10 @@ int tsdbWriteCompIdx(SRWHelper *pHelper) { return -1; } + pFile->info.offset = offset; + pFile->info.size += pFile->info.len; + ASSERT(pFile->info.size == lseek(pFile->fd, 0, SEEK_CUR)); + return 0; } @@ -803,6 +820,9 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa (int)(pCompBlock->numOfRows), pCompBlock->len, pCompBlock->numOfCols, pCompBlock->keyFirst, pCompBlock->keyLast); + pFile->info.size += pCompBlock->len; + ASSERT(pFile->info.size == lseek(pFile->fd, 0, SEEK_CUR)); + return 0; _err: @@ -1016,7 +1036,7 @@ static int tsdbInitHelperFile(SRWHelper *pHelper) { } static void tsdbDestroyHelperFile(SRWHelper *pHelper) { - tsdbCloseHelperFile(pHelper, false); + tsdbCloseHelperFile(pHelper, false, NULL); tsdbResetHelperFileImpl(pHelper); taosTZfree(pHelper->idxH.pIdxArray); taosTZfree(pHelper->pWIdx); From c1b3156563520e9a4bdc5c13a230fb0d7f06c28b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 8 Aug 2020 17:48:08 +0800 Subject: [PATCH 079/109] [td-225] fix memory leak --- src/client/src/tscSystem.c | 3 ++- src/mnode/src/mnodeDb.c | 2 +- src/mnode/src/mnodeTable.c | 2 +- src/plugins/monitor/src/monitorMain.c | 24 +++++++++++++++++------- src/query/src/qTokenizer.c | 6 +++++- src/util/inc/tstoken.h | 4 ++-- 6 files changed, 28 insertions(+), 13 deletions(-) diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c index a252beec33..2cc237dd83 100644 --- a/src/client/src/tscSystem.c +++ b/src/client/src/tscSystem.c @@ -164,7 +164,8 @@ void taos_cleanup() { taosCleanUpScheduler(tscQhandle); tscQhandle = NULL; } - + + taosCleanupKeywordsTable(); taosCloseLog(); taosTmrCleanUp(tscTmr); diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c index 803e44cea2..f6249986fd 100644 --- a/src/mnode/src/mnodeDb.c +++ b/src/mnode/src/mnodeDb.c @@ -1046,7 +1046,7 @@ static int32_t mnodeProcessDropDbMsg(SMnodeMsg *pMsg) { if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDb(pDrop->db); if (pMsg->pDb == NULL) { if (pDrop->ignoreNotExists) { - mDebug("db:%s, db is not exist, think drop success", pDrop->db); + mDebug("db:%s, db is not exist, treat as success", pDrop->db); return TSDB_CODE_SUCCESS; } else { mError("db:%s, failed to drop, invalid db", pDrop->db); diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 0912a6f3ff..dbc1bffa5c 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -751,7 +751,7 @@ static int32_t mnodeProcessDropTableMsg(SMnodeMsg *pMsg) { if (pMsg->pTable == NULL) pMsg->pTable = mnodeGetTable(pDrop->tableId); if (pMsg->pTable == NULL) { if (pDrop->igNotExists) { - mDebug("app:%p:%p, table:%s, table is not exist, think drop success", pMsg->rpcMsg.ahandle, pMsg, pDrop->tableId); + mDebug("app:%p:%p, table:%s, table is not exist, treat as success", pMsg->rpcMsg.ahandle, pMsg, pDrop->tableId); return TSDB_CODE_SUCCESS; } else { mError("app:%p:%p, table:%s, failed to drop table, table not exist", pMsg->rpcMsg.ahandle, pMsg, pDrop->tableId); diff --git a/src/plugins/monitor/src/monitorMain.c b/src/plugins/monitor/src/monitorMain.c index 4dc79aef68..644835abc0 100644 --- a/src/plugins/monitor/src/monitorMain.c +++ b/src/plugins/monitor/src/monitorMain.c @@ -131,7 +131,10 @@ static void monitorInitConn(void *para, void *unused) { } static void monitorInitConnCb(void *param, TAOS_RES *result, int32_t code) { - if (code < 0) { + // free it firstly in any cases. + taos_free_result(result); + + if (code != TSDB_CODE_SUCCESS) { monitorError("monitor:%p, connect to database failed, reason:%s", tsMonitorConn.conn, tstrerror(code)); taos_close(tsMonitorConn.conn); tsMonitorConn.conn = NULL; @@ -214,7 +217,7 @@ static void monitorInitDatabase() { } static void monitorInitDatabaseCb(void *param, TAOS_RES *result, int32_t code) { - if (-code == TSDB_CODE_MND_TABLE_ALREADY_EXIST || -code == TSDB_CODE_MND_DB_ALREADY_EXIST || code >= 0) { + if (code == TSDB_CODE_MND_TABLE_ALREADY_EXIST || code == TSDB_CODE_MND_DB_ALREADY_EXIST || code >= 0) { monitorDebug("monitor:%p, sql success, reason:%s, %s", tsMonitorConn.conn, tstrerror(code), tsMonitorConn.sql); if (tsMonitorConn.cmdIndex == MONITOR_CMD_CREATE_TB_LOG) { monitorInfo("dnode:%s is started", tsLocalEp); @@ -226,6 +229,8 @@ static void monitorInitDatabaseCb(void *param, TAOS_RES *result, int32_t code) { tsMonitorConn.state = MONITOR_STATE_UN_INIT; monitorStartSystemRetry(); } + + taos_free_result(result); } void monitorStopSystem() { @@ -238,6 +243,8 @@ void monitorStopSystem() { if (tsMonitorConn.timer != NULL) { taosTmrStopA(&(tsMonitorConn.timer)); } + + taos_close(tsMonitorConn.conn); } void monitorCleanUpSystem() { @@ -250,13 +257,16 @@ static void monitorStartTimer() { } static void dnodeMontiorLogCallback(void *param, TAOS_RES *result, int32_t code) { - if (code < 0) { - monitorError("monitor:%p, save %s failed, reason:%s", tsMonitorConn.conn, (char *)param, tstrerror(code)); - } else if (code == 0) { - monitorError("monitor:%p, save %s failed, affect rows:%d", tsMonitorConn.conn, (char *)param, code); + int32_t c = taos_errno(result); + + if (c != TSDB_CODE_SUCCESS) { + monitorError("monitor:%p, save %s failed, reason:%s", tsMonitorConn.conn, (char *)param, tstrerror(c)); } else { - monitorDebug("monitor:%p, save %s info success, reason:%s", tsMonitorConn.conn, (char *)param, tstrerror(code)); + int32_t rows = taos_affected_rows(result); + monitorDebug("monitor:%p, save %s succ, rows:%d", tsMonitorConn.conn, (char *)param, rows); } + + taos_free_result(result); } // unit is MB diff --git a/src/query/src/qTokenizer.c b/src/query/src/qTokenizer.c index 7418bc6895..1fa565ca90 100644 --- a/src/query/src/qTokenizer.c +++ b/src/query/src/qTokenizer.c @@ -25,7 +25,7 @@ // All the keywords of the SQL language are stored in a hash table typedef struct SKeyword { const char* name; // The keyword name - uint16_t type; // type + uint16_t type; // type uint8_t len; // length } SKeyword; @@ -659,3 +659,7 @@ SSQLToken tStrGetToken(char* str, int32_t* i, bool isPrevOptr, uint32_t numOfIgn } bool isKeyWord(const char* z, int32_t len) { return (tSQLKeywordCode((char*)z, len) != TK_ID); } + +void taosCleanupKeywordsTable() { + taosHashCleanup(KeywordHashTable); +} diff --git a/src/util/inc/tstoken.h b/src/util/inc/tstoken.h index 258e62cf8a..c1c6f2de7a 100644 --- a/src/util/inc/tstoken.h +++ b/src/util/inc/tstoken.h @@ -24,8 +24,6 @@ extern "C" { #include "tutil.h" #include "ttokendef.h" - - #define TSQL_TBNAME "TBNAME" #define TSQL_TBNAME_L "tbname" @@ -182,6 +180,8 @@ static FORCE_INLINE int32_t isValidNumber(const SSQLToken* pToken) { return (i < pToken->n)? TK_ILLEGAL:type; } +void taosCleanupKeywordsTable(); + #ifdef __cplusplus } #endif From ab6151cd9016ab7f0710bc49130369a43ae4551d Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 8 Aug 2020 17:52:48 +0800 Subject: [PATCH 080/109] TD-1057 --- src/common/src/tglobal.c | 6 +++--- src/os/src/alpine/alpineEnv.c | 5 ++++- src/os/src/darwin/darwinEnv.c | 5 ++++- src/os/src/linux64/linuxEnv.c | 5 ++++- src/os/src/windows/w64Env.c | 5 ++++- 5 files changed, 19 insertions(+), 7 deletions(-) diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 386ea2f9a7..0d7df38b83 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -147,12 +147,12 @@ int32_t tsMonitorInterval = 30; // seconds // internal int32_t tscEmbedded = 0; -char configDir[TSDB_FILENAME_LEN] = "/etc/taos"; +char configDir[TSDB_FILENAME_LEN] = {0}; char tsVnodeDir[TSDB_FILENAME_LEN] = {0}; char tsDnodeDir[TSDB_FILENAME_LEN] = {0}; char tsMnodeDir[TSDB_FILENAME_LEN] = {0}; -char tsDataDir[TSDB_FILENAME_LEN] = "/var/lib/taos"; -char tsScriptDir[TSDB_FILENAME_LEN] = "/etc/taos"; +char tsDataDir[TSDB_FILENAME_LEN] = {0}; +char tsScriptDir[TSDB_FILENAME_LEN] = {0}; char tsVnodeBakDir[TSDB_FILENAME_LEN] = {0}; /* diff --git a/src/os/src/alpine/alpineEnv.c b/src/os/src/alpine/alpineEnv.c index 4f84412075..811d98ad7f 100644 --- a/src/os/src/alpine/alpineEnv.c +++ b/src/os/src/alpine/alpineEnv.c @@ -18,7 +18,10 @@ #include "tglobal.h" void osInit() { - strcpy(configDir, "/etc/taos"); + if (configDir[0] == 0) { + strcpy(configDir, "/etc/taos"); + } + strcpy(tsVnodeDir, ""); strcpy(tsDnodeDir, ""); strcpy(tsMnodeDir, ""); diff --git a/src/os/src/darwin/darwinEnv.c b/src/os/src/darwin/darwinEnv.c index 7e1031a5af..6adebabec0 100644 --- a/src/os/src/darwin/darwinEnv.c +++ b/src/os/src/darwin/darwinEnv.c @@ -18,7 +18,10 @@ #include "tglobal.h" void osInit() { - strcpy(configDir, "~/TDengine/cfg"); + if (configDir[0] == 0) { + strcpy(configDir, "~/TDengine/cfg"); + } + strcpy(tsVnodeDir, ""); strcpy(tsDnodeDir, ""); strcpy(tsMnodeDir, ""); diff --git a/src/os/src/linux64/linuxEnv.c b/src/os/src/linux64/linuxEnv.c index 4f84412075..811d98ad7f 100644 --- a/src/os/src/linux64/linuxEnv.c +++ b/src/os/src/linux64/linuxEnv.c @@ -18,7 +18,10 @@ #include "tglobal.h" void osInit() { - strcpy(configDir, "/etc/taos"); + if (configDir[0] == 0) { + strcpy(configDir, "/etc/taos"); + } + strcpy(tsVnodeDir, ""); strcpy(tsDnodeDir, ""); strcpy(tsMnodeDir, ""); diff --git a/src/os/src/windows/w64Env.c b/src/os/src/windows/w64Env.c index 333871e96c..c6046ecae7 100644 --- a/src/os/src/windows/w64Env.c +++ b/src/os/src/windows/w64Env.c @@ -21,7 +21,10 @@ extern void taosWinSocketInit(); void osInit() { - strcpy(configDir, "C:/TDengine/cfg"); + if (configDir[0] == 0) { + strcpy(configDir, "~/TDengine/cfg"); + } + strcpy(tsVnodeDir, "C:/TDengine/data"); strcpy(tsDnodeDir, ""); strcpy(tsMnodeDir, ""); From 16ac4b66c92cd3381a6348ee0295cc0bc772c968 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 8 Aug 2020 18:07:51 +0800 Subject: [PATCH 081/109] [td-225] fix memory leak --- src/client/src/tscSystem.c | 3 ++- src/plugins/monitor/src/monitorMain.c | 24 +++++++++++++++++------- src/query/src/qTokenizer.c | 4 ++++ src/util/inc/tstoken.h | 2 ++ 4 files changed, 25 insertions(+), 8 deletions(-) diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c index 878fdd500f..c742db42ab 100644 --- a/src/client/src/tscSystem.c +++ b/src/client/src/tscSystem.c @@ -164,7 +164,8 @@ void taos_cleanup() { taosCleanUpScheduler(tscQhandle); tscQhandle = NULL; } - + + taosCleanupKeywordsTable(); taosCloseLog(); taosTmrCleanUp(tscTmr); diff --git a/src/plugins/monitor/src/monitorMain.c b/src/plugins/monitor/src/monitorMain.c index 8bf9277a43..6c7ecbb951 100644 --- a/src/plugins/monitor/src/monitorMain.c +++ b/src/plugins/monitor/src/monitorMain.c @@ -131,7 +131,10 @@ static void monitorInitConn(void *para, void *unused) { } static void monitorInitConnCb(void *param, TAOS_RES *result, int32_t code) { - if (code < 0) { + // free it firstly in any cases. + taos_free_result(result); + + if (code != TSDB_CODE_SUCCESS) { monitorError("monitor:%p, connect to database failed, reason:%s", tsMonitorConn.conn, tstrerror(code)); taos_close(tsMonitorConn.conn); tsMonitorConn.conn = NULL; @@ -214,7 +217,7 @@ static void monitorInitDatabase() { } static void monitorInitDatabaseCb(void *param, TAOS_RES *result, int32_t code) { - if (-code == TSDB_CODE_MND_TABLE_ALREADY_EXIST || -code == TSDB_CODE_MND_DB_ALREADY_EXIST || code >= 0) { + if (code == TSDB_CODE_MND_TABLE_ALREADY_EXIST || code == TSDB_CODE_MND_DB_ALREADY_EXIST || code >= 0) { monitorDebug("monitor:%p, sql success, reason:%s, %s", tsMonitorConn.conn, tstrerror(code), tsMonitorConn.sql); if (tsMonitorConn.cmdIndex == MONITOR_CMD_CREATE_TB_LOG) { monitorInfo("dnode:%s is started", tsLocalEp); @@ -226,6 +229,8 @@ static void monitorInitDatabaseCb(void *param, TAOS_RES *result, int32_t code) { tsMonitorConn.state = MONITOR_STATE_UN_INIT; monitorStartSystemRetry(); } + + taos_free_result(result); } void monitorStopSystem() { @@ -238,6 +243,8 @@ void monitorStopSystem() { if (tsMonitorConn.timer != NULL) { taosTmrStopA(&(tsMonitorConn.timer)); } + + taos_close(tsMonitorConn.conn); } void monitorCleanUpSystem() { @@ -250,13 +257,16 @@ static void monitorStartTimer() { } static void dnodeMontiorLogCallback(void *param, TAOS_RES *result, int32_t code) { - if (code < 0) { - monitorError("monitor:%p, save %s failed, reason:%s", tsMonitorConn.conn, (char *)param, tstrerror(code)); - } else if (code == 0) { - monitorError("monitor:%p, save %s failed, affect rows:%d", tsMonitorConn.conn, (char *)param, code); + int32_t c = taos_errno(result); + + if (c != TSDB_CODE_SUCCESS) { + monitorError("monitor:%p, save %s failed, reason:%s", tsMonitorConn.conn, (char *)param, tstrerror(c)); } else { - monitorDebug("monitor:%p, save %s info success, reason:%s", tsMonitorConn.conn, (char *)param, tstrerror(code)); + int32_t rows = taos_affected_rows(result); + monitorDebug("monitor:%p, save %s succ, rows:%d", tsMonitorConn.conn, (char *)param, rows); } + + taos_free_result(result); } // unit is MB diff --git a/src/query/src/qTokenizer.c b/src/query/src/qTokenizer.c index d0191ec732..d730b3cab0 100644 --- a/src/query/src/qTokenizer.c +++ b/src/query/src/qTokenizer.c @@ -659,3 +659,7 @@ SSQLToken tStrGetToken(char* str, int32_t* i, bool isPrevOptr, uint32_t numOfIgn } bool isKeyWord(const char* z, int32_t len) { return (tSQLKeywordCode((char*)z, len) != TK_ID); } + +void taosCleanupKeywordsTable() { + taosHashCleanup(KeywordHashTable); +} \ No newline at end of file diff --git a/src/util/inc/tstoken.h b/src/util/inc/tstoken.h index 258e62cf8a..705c52af47 100644 --- a/src/util/inc/tstoken.h +++ b/src/util/inc/tstoken.h @@ -182,6 +182,8 @@ static FORCE_INLINE int32_t isValidNumber(const SSQLToken* pToken) { return (i < pToken->n)? TK_ILLEGAL:type; } +void taosCleanupKeywordsTable(); + #ifdef __cplusplus } #endif From b1473342e4b1c1dcd52e146b35f2edf88dda21d8 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 8 Aug 2020 21:34:56 +0800 Subject: [PATCH 082/109] TD-1057 --- deps/iconv/relocatable.c | 2 +- src/client/src/tscServer.c | 2 +- src/client/src/tscSub.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/deps/iconv/relocatable.c b/deps/iconv/relocatable.c index 52cf4c1372..5676eb703f 100644 --- a/deps/iconv/relocatable.c +++ b/deps/iconv/relocatable.c @@ -360,7 +360,7 @@ find_shared_library_fullname () ungetc (c, fp); shared_library_fullname = NULL; size = 0; - len = taosGetline(&shared_library_fullname, &size, fp); + len = getline(&shared_library_fullname, &size, fp); if (len >= 0) { /* Success: filled shared_library_fullname. */ diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 2977ed01ec..3390710472 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -912,7 +912,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pSql->cmd.msgType = TSDB_MSG_TYPE_QUERY; pQueryMsg->head.contLen = htonl(msgLen); - assert(msgLen + minMsgSize() <= pCmd->allocSize); + assert(msgLen + minMsgSize() <= (int32_t)pCmd->allocSize); return TSDB_CODE_SUCCESS; } diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c index f740e3cce9..e9f2c1dc1d 100644 --- a/src/client/src/tscSub.c +++ b/src/client/src/tscSub.c @@ -426,7 +426,7 @@ TAOS_RES *taos_consume(TAOS_SUB *tsub) { size_t size = taosArrayGetSize(pSub->progress) * sizeof(STableIdInfo); size += sizeof(SQueryTableMsg) + 4096; - tscAllocPayload(&pSql->cmd, size); + tscAllocPayload(&pSql->cmd, (int)size); for (int retry = 0; retry < 3; retry++) { tscRemoveFromSqlList(pSql); From 976b4e863a0895208e1b7247792326a178f532c7 Mon Sep 17 00:00:00 2001 From: slguan Date: Sat, 8 Aug 2020 23:06:02 +0800 Subject: [PATCH 083/109] scripts --- tests/script/windows/account/authority.sim | 317 ------------------ tests/script/windows/account/user_create.sim | 80 ----- tests/script/windows/account/user_len.sim | 81 ----- tests/script/windows/alter/metrics.sim | 16 +- tests/script/windows/alter/table.sim | 16 +- tests/script/windows/compute/diff.sim | 2 +- tests/script/windows/compute/first.sim | 2 +- tests/script/windows/compute/interval.sim | 2 +- tests/script/windows/compute/last.sim | 2 +- tests/script/windows/compute/leastsquare.sim | 18 +- tests/script/windows/compute/max.sim | 2 +- tests/script/windows/compute/min.sim | 2 +- tests/script/windows/compute/percentile.sim | 2 +- tests/script/windows/compute/stddev.sim | 2 +- tests/script/windows/compute/sum.sim | 2 +- tests/script/windows/compute/top.sim | 2 +- tests/script/windows/db/basic.sim | 21 +- tests/script/windows/db/len.sim | 6 +- tests/script/windows/field/binary.sim | 15 +- tests/script/windows/import/basic.sim | 109 +++--- tests/script/windows/insert/basic.sim | 12 +- .../windows/insert/query_multi_file.sim | 2 +- tests/script/windows/table/binary.sim | 3 +- tests/script/windows/table/bool.sim | 12 +- tests/script/windows/table/column_value.sim | 13 +- tests/script/windows/table/float.sim | 10 +- tests/script/windows/table/table.sim | 4 +- tests/script/windows/table/table_len.sim | 2 +- tests/script/windows/tag/add.sim | 117 +++---- tests/script/windows/tag/change.sim | 36 +- tests/script/windows/tag/create.sim | 11 +- tests/script/windows/tag/delete.sim | 84 ++--- tests/script/windows/tag/filter.sim | 11 +- tests/script/windows/tag/set.sim | 52 +-- tests/script/windows/vector/multi.sim | 1 - 35 files changed, 276 insertions(+), 793 deletions(-) delete mode 100644 tests/script/windows/account/authority.sim delete mode 100644 tests/script/windows/account/user_create.sim delete mode 100644 tests/script/windows/account/user_len.sim diff --git a/tests/script/windows/account/authority.sim b/tests/script/windows/account/authority.sim deleted file mode 100644 index a352faf23b..0000000000 --- a/tests/script/windows/account/authority.sim +++ /dev/null @@ -1,317 +0,0 @@ -sql connect -sleep 3000 - -print ============= step1 - -sql create user read pass 'taosdata' -sql create user write pass 'taosdata' -sql create user manage pass 'taosdata' - -sql create user a PASS 'ade' privilege -x step11 - return -1 -step11: - -sql create user a PASS 'ade' privilege a -x step12 - return -1 -step12: - -sql create user a PASS 'ade' privilege read -x step13 - return -1 -step13: - -sql show users -if $rows != 6 then - return -1 -endi - -sql alter user read privilege read -sql alter user write privilege write -sql alter user manage privilege super - -print ============= step2 -sql close -sql connect write -sleep 2000 - -sql create database d1 -sql create database d2 -sql create table d1.t1 (ts timestamp, i int) -sql create table d2.t2 (ts timestamp, i int) -sql insert into d1.t1 values(now, 1) -sql insert into d2.t2 values(now, 1) -sql insert into d2.t2 values(now+1s, 2) - -sql show users -if $rows != 6 then - return -1 -endi -sql show databases -if $rows != 2 then - return -1 -endi -sql select * from d1.t1 -if $rows != 1 then - return -1 -endi -sql select * from d2.t2 -if $rows != 2 then - return -1 -endi - -sql create account t1 pass 'taosdata' -x step21 - return -1 -step21: - -sql create user t1 pass 'taosdata' -x step22 - return -1 -step22: - -sql alter user read pass 'taosdata' -x step23 - return -1 -step23: - -sql create dnode $hostname2 -x step24 - return -1 -step24: - -sql drop dnode $hostname2 -x step25 - return -1 -step25: - -sql create mnode 192.168.0.2 -x step26 - return -1 -step26: - -sql drop mnode 192.168.0.2 -x step27 - return -1 -step27: - -sql drop account root -x step28 - return -1 -step28: - -sql alter user write pass 'taosdata' - -print ============= step3 -sql close -sql connect read -sleep 2000 - -sql create database d3 -x step31 - return -1 -step31: - -sql create table d1.t3 (ts timestamp, i int) -x step32 - return -1 -step32: - -#sql insert into d1.t1 values(now, 2) -x step33 -# return -1 -#step33: - -sql show accounts -if $rows != 1 then - return -1 -endi -sql show users -if $rows != 6 then - return -1 -endi -sql show databases -if $rows != 2 then - return -1 -endi -sql select * from d1.t1 -if $rows != 1 then - return -1 -endi - -sql select * from d2.t2 -if $rows != 2 then - return -1 -endi - -sql sql create account t1 pass 'taosdata' -x step34 - return -1 -step34: - -sql sql create user t1 pass 'taosdata' -x step35 - return -1 -step35: - -print ============= step4 -sql close -sql connect manage -sleep 2000 - -sql create database d3 -sql create database d4 -sql create table d3.t3 (ts timestamp, i int) -sql create table d4.t4 (ts timestamp, i int) - -sql show accounts -if $rows != 1 then - return -1 -endi -sql show users -if $rows != 6 then - return -1 -endi -sql show databases -if $rows != 4 then - return -1 -endi -sql select * from d1.t1 -if $rows != 1 then - return -1 -endi -sql select * from d2.t2 -if $rows != 2 then - return -1 -endi - -sql create account other pass 'taosdata' -x step41 - return -1 -step41: - -sql close -sql connect -sleep 2000 -sql create account other pass 'taosdata' - -print ============= step5 -sql close -sql connect other -sleep 2000 -sql create user read pass 'taosdata' -x step51 - return -1 -step51: -sql create other write pass 'taosdata' -x step52 - return -1 -step52: - -sql create user oread pass 'taosdata' -sql create user owrite pass 'taosdata' -sql create user omanage pass 'taosdata' - -sql show users -print show users $rows -if $rows != 5 then - return -1 -endi - -sql alter user oread privilege read -sql alter user owrite privilege write -sql alter user oroot privilege super -x step53 - return -1 -step53: -sql alter user read privilege read -x step54 - return -1 -step54: - -print ============= step6 -sql close -sql connect owrite -sleep 2000 - -sql create database d1 -sql create database d3 -sql create table d1.t1 (ts timestamp, i int) -sql create table d3.t3 (ts timestamp, i int) -sql insert into d1.t1 values(now, 11) -sql insert into d3.t3 values(now, 11) -sql insert into d3.t3 values(now+1s, 12) - -sql show databases -if $rows != 2 then - return -1 -endi -sql select * from d1.t1 -if $rows != 1 then - return -1 -endi -sql select * from d2.t2 -x step6 - return -1 -step6: -sql select * from d3.t3 -if $rows != 2 then - return -1 -endi - -sql sql create user t1 pass 'taosdata' -x step62 - return -1 -step62: - -print ============= step7 -sql close -sql connect oread -sleep 2000 - -sql create database d7 -x step71 - return -1 -step71: - -sql show databases -if $rows != 2 then - return -1 -endi -sql select * from d1.t1 -if $rows != 1 then - return -1 -endi -sql select * from d2.t2 -x step72 - return -1 -step72: -sql select * from d3.t3 -if $rows != 2 then - return -1 -endi - -sql sql create user t1 pass 'taosdata' -x step74 - return -1 -step74: - -print ============= step8 -sql close -sql connect omanage -sleep 2000 - -sql create database d4 -sql create table d4.t4 (ts timestamp, i int) - -sql show databases -if $rows != 3 then - return -1 -endi -sql select * from d1.t1 -if $rows != 1 then - return -1 -endi -sql select * from d2.t2 -x step82 - return -1 -step82: -sql select * from d3.t3 -if $rows != 2 then - return -1 -endi - -print ============= step9 -sql close -sql connect -sleep 2000 -sql show databases -if $rows != 4 then - return -1 -endi - -sql drop user read -sql drop user manage -sql drop user write - -sql close -sql connect -sleep 2000 -sql drop database d1 -sql drop database d2 -sql drop database d3 -sql drop database d4 diff --git a/tests/script/windows/account/user_create.sim b/tests/script/windows/account/user_create.sim deleted file mode 100644 index 6dc9ab05b3..0000000000 --- a/tests/script/windows/account/user_create.sim +++ /dev/null @@ -1,80 +0,0 @@ -sql connect -sleep 3000 - -print =============== step1 -sql show users -if $rows != 3 then - return -1 -endi - -sql create user read PASS 'pass123' -sql create user read PASS 'pass123' -x step1 - return -1 -step1: - -sql show users -if $rows != 4 then - return -1 -endi - -sql alter user read PASS 'taosdata' - -print =============== step2 -sql close -sql connect read -sleep 2000 - -sql alter user read PASS 'taosdata' - -print =============== step3 -sql drop user read -x step31 - return -1 -step31: -sql drop user _root -x step32 - return -1 -step32: -sql drop user monitor -x step33 - return -1 -step33: - -print =============== step4 -sql close -sql connect -sleep 2000 - -sql alter user read privilege read -sql show users -if $data1_read != read then - return -1 -endi - -sql alter user read privilege super -sql show users -if $data1_read != super then - return -1 -endi - -sql alter user read privilege write -sql show users -if $data1_read != write then - return -1 -endi - -sql alter user read privilege 1 -x step43 - return -1 -step43: - -sql drop user _root -x step41 - return -1 -step41: - -sql drop user monitor -x step42 - return -1 -step42: - -sql drop user read - - - - - diff --git a/tests/script/windows/account/user_len.sim b/tests/script/windows/account/user_len.sim deleted file mode 100644 index b4a344fe8d..0000000000 --- a/tests/script/windows/account/user_len.sim +++ /dev/null @@ -1,81 +0,0 @@ -sql connect -sleep 3000 - -$i = 0 -$dbPrefix = lm_us_db -$tbPrefix = lm_us_tb -$db = $dbPrefix . $i -$tb = $tbPrefix . $i - -print =============== step1 -sql drop user ac -x step0 - return -1 -step0: - -sql create user PASS '123' -x step1 - return -1 -step1: - -sql show users -if $rows != 3 then - return -1 -endi - -print =============== step2 -sql drop user a -x step2 -step2: -sql create user a PASS '123' -sql show users -if $rows != 4 then - return -1 -endi - -sql drop user a -sql show users -if $rows != 3 then - return -1 -endi - -print =============== step3 -sql drop user abc01234567890123456789 -x step3 -step3: - -sql create user abc01234567890123456789 PASS '123' -sql show users -if $rows != 4 then - return -1 -endi - -sql drop user abc01234567890123456789 -sql show users -if $rows != 3 then - return -1 -endi - -print =============== step4 -sql create user abcd0123456789012345678901234567890111 PASS '123' -x step4 - return -1 -step4: -sql show users -if $rows != 3 then - return -1 -endi - -print =============== step5 -sql drop user 123 -x step5 -step5: -sql create user 123 PASS '123' -x step61 - return -1 -step61: - -sql create user a123 PASS '123' -sql show users -if $rows != 4 then - return -1 -endi - -sql drop user a123 -sql show users -if $rows != 3 then - return -1 -endi diff --git a/tests/script/windows/alter/metrics.sim b/tests/script/windows/alter/metrics.sim index 323e150d41..3717d8c1ed 100644 --- a/tests/script/windows/alter/metrics.sim +++ b/tests/script/windows/alter/metrics.sim @@ -35,7 +35,6 @@ endi print ======== step2 sql alter table mt add column b smallint -sleep 2500 sql describe tb if $data00 != ts then return -1 @@ -64,7 +63,6 @@ endi print ======== step3 sql alter table mt add column c tinyint -sleep 2500 sql describe tb if $data00 != ts then return -1 @@ -99,7 +97,6 @@ endi print ======== step4 sql alter table mt add column d int -sleep 2500 sql describe tb if $data00 != ts then return -1 @@ -140,7 +137,6 @@ endi print ======== step5 sql alter table mt add column e bigint -sleep 2500 sql describe tb if $data00 != ts then return -1 @@ -187,7 +183,6 @@ endi print ======== step6 sql alter table mt add column f float -sleep 2500 sql describe tb if $data00 != ts then return -1 @@ -240,7 +235,6 @@ endi print ======== step7 sql alter table mt add column g double -sleep 2500 sql describe tb if $data00 != ts then return -1 @@ -299,7 +293,6 @@ endi print ======== step8 sql alter table mt add column h binary(10) -sleep 2500 sql describe tb if $data00 != ts then return -1 @@ -461,7 +454,6 @@ print ======== step11 print ======== step12 sql alter table mt drop column b -sleep 2500 sql describe tb if $data00 != ts then return -1 @@ -523,7 +515,6 @@ endi print ======== step13 sql alter table mt drop column c -sleep 2500 sql describe tb if $data00 != ts then return -1 @@ -579,7 +570,6 @@ endi print ======== step14 sql alter table mt drop column d -sleep 2500 sql describe tb if $data00 != ts then return -1 @@ -629,7 +619,6 @@ endi print ======== step15 sql alter table mt drop column e -sleep 2500 sql describe tb if $data00 != ts then return -1 @@ -673,7 +662,6 @@ endi print ======== step16 sql alter table mt drop column f -sleep 2500 sql describe tb if $data00 != ts then return -1 @@ -711,7 +699,6 @@ endi print ======== step17 sql alter table mt drop column g -sleep 2500 sql describe tb if $data00 != ts then return -1 @@ -743,7 +730,6 @@ endi print ============= step18 sql alter table mt drop column h -sleep 2500 sql describe tb if $data00 != ts then return -1 @@ -763,7 +749,7 @@ endi if $data21 != INT then return -1 endi -if $data30 != NULL then +if $data30 != null then return -1 endi diff --git a/tests/script/windows/alter/table.sim b/tests/script/windows/alter/table.sim index e6b5d69551..3b811a065e 100644 --- a/tests/script/windows/alter/table.sim +++ b/tests/script/windows/alter/table.sim @@ -28,7 +28,6 @@ endi print ======== step2 sql alter table tb add column b smallint -sleep 2500 sql describe tb if $data00 != ts then return -1 @@ -51,7 +50,6 @@ endi print ======== step3 sql alter table tb add column c tinyint -sleep 2500 sql describe tb if $data00 != ts then return -1 @@ -80,7 +78,6 @@ endi print ======== step4 sql alter table tb add column d int -sleep 2500 sql describe tb if $data00 != ts then return -1 @@ -115,7 +112,6 @@ endi print ======== step5 sql alter table tb add column e bigint -sleep 2500 sql describe tb if $data00 != ts then return -1 @@ -156,7 +152,6 @@ endi print ======== step6 sql alter table tb add column f float -sleep 2500 sql describe tb if $data00 != ts then return -1 @@ -203,7 +198,6 @@ endi print ======== step7 sql alter table tb add column g double -sleep 2500 sql describe tb if $data00 != ts then return -1 @@ -256,7 +250,6 @@ endi print ======== step8 sql alter table tb add column h binary(10) -sleep 2500 sql describe tb if $data00 != ts then return -1 @@ -406,7 +399,6 @@ step115: print ======== step12 sql alter table tb drop column b -sleep 2500 sql describe tb if $data00 != ts then return -1 @@ -462,7 +454,6 @@ endi print ======== step13 sql alter table tb drop column c -sleep 2500 sql describe tb if $data00 != ts then return -1 @@ -512,7 +503,6 @@ endi print ======== step14 sql alter table tb drop column d -sleep 2500 sql describe tb if $data00 != ts then return -1 @@ -556,7 +546,6 @@ endi print ======== step15 sql alter table tb drop column e -sleep 2500 sql describe tb if $data00 != ts then return -1 @@ -594,7 +583,6 @@ endi print ======== step16 sql alter table tb drop column f -sleep 2500 sql describe tb if $data00 != ts then return -1 @@ -626,7 +614,6 @@ endi print ======== step17 sql alter table tb drop column g -sleep 2500 sql describe tb if $data00 != ts then return -1 @@ -652,7 +639,6 @@ endi print ============= step18 sql alter table tb drop column h -sleep 2500 sql describe tb if $data00 != ts then return -1 @@ -666,7 +652,7 @@ endi if $data11 != INT then return -1 endi -if $data20 != NULL then +if $data20 != null then return -1 endi diff --git a/tests/script/windows/compute/diff.sim b/tests/script/windows/compute/diff.sim index e9d76f8c78..6c2829872a 100644 --- a/tests/script/windows/compute/diff.sim +++ b/tests/script/windows/compute/diff.sim @@ -1,5 +1,5 @@ -sql connect sleep 3000 +sql connect $dbPrefix = m_di_db $tbPrefix = m_di_tb diff --git a/tests/script/windows/compute/first.sim b/tests/script/windows/compute/first.sim index 7b6a831004..9a0c02fe4b 100644 --- a/tests/script/windows/compute/first.sim +++ b/tests/script/windows/compute/first.sim @@ -1,5 +1,5 @@ -sql connect sleep 3000 +sql connect $dbPrefix = m_fi_db $tbPrefix = m_fi_tb diff --git a/tests/script/windows/compute/interval.sim b/tests/script/windows/compute/interval.sim index d40c6ad0c9..365c6d9d31 100644 --- a/tests/script/windows/compute/interval.sim +++ b/tests/script/windows/compute/interval.sim @@ -1,5 +1,5 @@ -sql connect sleep 3000 +sql connect $dbPrefix = m_in_db $tbPrefix = m_in_tb diff --git a/tests/script/windows/compute/last.sim b/tests/script/windows/compute/last.sim index 4b686766ae..aa9699778f 100644 --- a/tests/script/windows/compute/last.sim +++ b/tests/script/windows/compute/last.sim @@ -1,5 +1,5 @@ -sql connect sleep 3000 +sql connect $dbPrefix = m_la_db $tbPrefix = m_la_tb diff --git a/tests/script/windows/compute/leastsquare.sim b/tests/script/windows/compute/leastsquare.sim index 81f37c3228..bb7404edd0 100644 --- a/tests/script/windows/compute/leastsquare.sim +++ b/tests/script/windows/compute/leastsquare.sim @@ -1,5 +1,5 @@ -sql connect sleep 3000 +sql connect $dbPrefix = m_le_db $tbPrefix = m_le_tb @@ -15,7 +15,7 @@ $mt = $mtPrefix . $i sql drop database $db -x step1 step1: -sql create database $db +sql create database $db keep 36500 sql use $db sql create table $mt (ts timestamp, tbcol int) TAGS(tgcol int) @@ -43,41 +43,41 @@ $tb = $tbPrefix . $i sql select leastsquares(tbcol, 1, 1) from $tb print ===> $data00 -if $data00 != @(1.000000, 1.000000)@ then +if $data00 != @{slop:1.000000, intercept:1.000000}@ then return -1 endi print =============== step3 sql select leastsquares(tbcol, 1, 1) from $tb where ts < now + 4m print ===> $data00 -if $data00 != @(1.000000, 1.000000)@ then +if $data00 != @{slop:1.000000, intercept:1.000000}@ then return -1 endi print =============== step4 sql select leastsquares(tbcol, 1, 1) as b from $tb print ===> $data00 -if $data00 != @(1.000000, 1.000000)@ then +if $data00 != @{slop:1.000000, intercept:1.000000}@ then return -1 endi print =============== step5 sql select leastsquares(tbcol, 1, 1) as b from $tb interval(1m) print ===> $data01 -if $data01 != @(1.000000, 1.000000)@ then +if $data01 != @{slop:1.000000, intercept:1.000000}@ then return -1 endi sql select leastsquares(tbcol, 1, 1) as b from $tb interval(1d) print ===> $data01 -if $data01 != @(1.000000, 1.000000)@ then +if $data01 != @{slop:1.000000, intercept:1.000000}@ then return -1 endi print =============== step6 sql select leastsquares(tbcol, 1, 1) as b from $tb where ts < now + 4m interval(1m) print ===> $data01 -if $data01 != @(1.000000, 1.000000)@ then +if $data01 != @{slop:1.000000, intercept:1.000000}@ then return -1 endi print ===> $rows @@ -90,4 +90,4 @@ sql drop database $db sql show databases if $rows != 0 then return -1 -endi \ No newline at end of file +endi diff --git a/tests/script/windows/compute/max.sim b/tests/script/windows/compute/max.sim index 60147358b3..a19d122ecd 100644 --- a/tests/script/windows/compute/max.sim +++ b/tests/script/windows/compute/max.sim @@ -1,5 +1,5 @@ -sql connect sleep 3000 +sql connect $dbPrefix = m_ma_db $tbPrefix = m_ma_tb diff --git a/tests/script/windows/compute/min.sim b/tests/script/windows/compute/min.sim index 928404babc..216f2061d7 100644 --- a/tests/script/windows/compute/min.sim +++ b/tests/script/windows/compute/min.sim @@ -1,5 +1,5 @@ -sql connect sleep 3000 +sql connect $dbPrefix = m_mi_db $tbPrefix = m_mi_tb diff --git a/tests/script/windows/compute/percentile.sim b/tests/script/windows/compute/percentile.sim index 6068722b54..20b2740d6e 100644 --- a/tests/script/windows/compute/percentile.sim +++ b/tests/script/windows/compute/percentile.sim @@ -1,5 +1,5 @@ -sql connect sleep 3000 +sql connect $dbPrefix = m_pe_db $tbPrefix = m_pe_tb diff --git a/tests/script/windows/compute/stddev.sim b/tests/script/windows/compute/stddev.sim index b5ba704bef..c02b3e4ab3 100644 --- a/tests/script/windows/compute/stddev.sim +++ b/tests/script/windows/compute/stddev.sim @@ -1,5 +1,5 @@ -sql connect sleep 3000 +sql connect $dbPrefix = m_st_db $tbPrefix = m_st_tb diff --git a/tests/script/windows/compute/sum.sim b/tests/script/windows/compute/sum.sim index 39eb771f34..04af1d457a 100644 --- a/tests/script/windows/compute/sum.sim +++ b/tests/script/windows/compute/sum.sim @@ -1,5 +1,5 @@ -sql connect sleep 3000 +sql connect $dbPrefix = m_su_db $tbPrefix = m_su_tb diff --git a/tests/script/windows/compute/top.sim b/tests/script/windows/compute/top.sim index b64c24b03f..b3c698c064 100644 --- a/tests/script/windows/compute/top.sim +++ b/tests/script/windows/compute/top.sim @@ -1,5 +1,5 @@ -sql connect sleep 3000 +sql connect $dbPrefix = m_to_db $tbPrefix = m_to_tb diff --git a/tests/script/windows/db/basic.sim b/tests/script/windows/db/basic.sim index 4222cfe3b2..f1e18d15a5 100644 --- a/tests/script/windows/db/basic.sim +++ b/tests/script/windows/db/basic.sim @@ -1,5 +1,6 @@ -sql connect sleep 3000 +sql connect + print ============================ dnode1 start $i = 0 @@ -9,7 +10,7 @@ $db = $dbPrefix . $i $tb = $tbPrefix . $i print =============== step1 -sql create database $db replica 1 days 20 keep 2000 +sql create database $db replica 1 days 20 keep 2000 cache 16 sql show databases print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 if $data00 != $db then @@ -24,10 +25,10 @@ endi if $data04 != 1 then return -1 endi -if $data05 != 20 then +if $data06 != 20 then return -1 endi -if $data07 != 1000 then +if $data08 != 16 then return -1 endi @@ -46,7 +47,7 @@ if $rows != 0 then endi print =============== step4 -sql drop database $db +sql_error drop database $db print =============== step5 sql create database $db replica 1 days 15 keep 1500 @@ -64,14 +65,10 @@ endi if $data04 != 1 then return -1 endi -if $data05 != 15 then +if $data06 != 15 then return -1 endi -if $data07 != 1000 then - return -1 -endi - print =============== step6 sql use $db sql create table $tb (ts timestamp, speed int) @@ -95,9 +92,7 @@ $db = $dbPrefix . $i $tb = $tbPrefix . $i sql create database $db sql use $db -sql create table $tb (ts timestamp, speed int) -x step6 - return -1 -step6: +sql create table $tb (ts timestamp, speed int) print =============== step7 $i = 0 diff --git a/tests/script/windows/db/len.sim b/tests/script/windows/db/len.sim index 32f6476177..f922e7e05a 100644 --- a/tests/script/windows/db/len.sim +++ b/tests/script/windows/db/len.sim @@ -1,8 +1,8 @@ -sql connect sleep 3000 +sql connect print =============== step1 -sql drop database dd +sql_error drop database dd sql create database -x step1 return -1 @@ -40,7 +40,7 @@ if $rows != 0 then endi print =============== step4 -sql create database a012345678901201234567890120123456789012 -x step4 +sql create database a012345678901201234567890120123456789012a012345678901201234567890120123456789012 -x step4 return -1 step4: sql show databases diff --git a/tests/script/windows/field/binary.sim b/tests/script/windows/field/binary.sim index cd75b63814..8b86c4dbea 100644 --- a/tests/script/windows/field/binary.sim +++ b/tests/script/windows/field/binary.sim @@ -55,17 +55,10 @@ if $rows != 75 then return -1 endi -sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where tbcol = '1' group by tgcol -x step13 - return -1 -step13: - -sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where ts < now + 4m and tbcol = '1' group by tgcol -x step14 - return -1 -step14: - -sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where tbcol = '1' interval(1d) group by tgcol -x step15 - return -1 -step15: +print select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where tbcol = '1' +sql_error select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where tbcol = '1' group by tgcol +sql_error select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where ts < now + 4m and tbcol = '1' group by tgcol +sql_error select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt where tbcol = '1' interval(1d) group by tgcol #can't filter binary fields diff --git a/tests/script/windows/import/basic.sim b/tests/script/windows/import/basic.sim index 0621721a6b..c20378ee88 100644 --- a/tests/script/windows/import/basic.sim +++ b/tests/script/windows/import/basic.sim @@ -6,111 +6,112 @@ sql use ibadb sql create table tb(ts timestamp, i int) print ================= step1 -sql import into tb values(10000, 10000) + +sql import into tb values(1564641710000, 10000) sql select * from tb; if $rows != 1 then return -1 endi print ================= step2 -sql insert into tb values(8000, 8000) -sql select * from tb; -if $rows != 1 then - return -1 -endi - -print ================= step3 -sql insert into tb values(20000, 20000) +sql insert into tb values(1564641708000, 8000) sql select * from tb; if $rows != 2 then return -1 endi +print ================= step3 +sql insert into tb values(1564641720000, 20000) +sql select * from tb; +if $rows != 3 then + return -1 +endi + print ================= step4 -sql import into tb values(8000, 8000) -sql import into tb values(15000, 15000) -sql import into tb values(30000, 30000) +sql import into tb values(1564641708000, 8000) +sql import into tb values(1564641715000, 15000) +sql import into tb values(1564641730000, 30000) sql select * from tb; if $rows != 5 then return -1 endi print ================= step5 -sql insert into tb values(8000, 8000) -sql insert into tb values(14000, 14000) -sql insert into tb values(25000, 25000) -sql insert into tb values(40000, 40000) +sql insert into tb values(1564641708000, 8000) +sql insert into tb values(1564641714000, 14000) +sql insert into tb values(1564641725000, 25000) +sql insert into tb values(1564641740000, 40000) sql select * from tb; -if $rows != 6 then +if $rows != 8 then return -1 endi print ================= step6 -sql import into tb values(7000, 7000) -sql import into tb values(12000, 12000) -sql import into tb values(23000, 23000) -sql import into tb values(34000, 34000) -sql import into tb values(50000, 50000) +sql import into tb values(1564641707000, 7000) +sql import into tb values(1564641712000, 12000) +sql import into tb values(1564641723000, 23000) +sql import into tb values(1564641734000, 34000) +sql import into tb values(1564641750000, 50000) sql select * from tb; -if $rows != 11 then +if $rows != 13 then return -1 endi print ================= step7 -sql import into tb values(7001, 7001) -sql import into tb values(12001, 12001) -sql import into tb values(23001, 23001) -sql import into tb values(34001, 34001) -sql import into tb values(50001, 50001) -sql select * from tb; -if $rows != 16 then - return -1 -endi - -print ================= step8 -sql insert into tb values(8002, 8002) -sql insert into tb values(14002, 14002) -sql insert into tb values(25002, 25002) -sql insert into tb values(200000, 60000) -sql select * from tb; -if $rows != 17 then - return -1 -endi - -print ================= step9 only insert last one -sql import into tb values(5000, 5000)(18000, 18000)(700000, 700000) +sql import into tb values(1564641707001, 7001) +sql import into tb values(1564641712001, 12001) +sql import into tb values(1564641723001, 23001) +sql import into tb values(1564641734001, 34001) +sql import into tb values(1564641750001, 50001) sql select * from tb; if $rows != 18 then return -1 endi -print ================= step10 -sql import into tb values(5000, 5000)(18000, 18000)(700000, 70000) +print ================= step8 +sql insert into tb values(1564641708002, 8002) +sql insert into tb values(1564641714002, 14002) +sql insert into tb values(1564641725002, 25002) +sql insert into tb values(1564641900000, 200000) sql select * from tb; -if $rows != 19 then +if $rows != 22 then + return -1 +endi + +print ================= step9 only insert last one +sql import into tb values(1564641705000, 5000)(1564641718000, 18000)(1564642400000, 700000) +sql select * from tb; +if $rows != 25 then + return -1 +endi + +print ================= step10 +sql import into tb values(1564641705000, 5000)(1564641718000, 18000)(1564642400000, 70000) +sql select * from tb; +if $rows != 25 then return -1 endi print ================= step11 -sql import into tb values(700000, 700000) +sql import into tb values(1564642400000, 700000) sql select * from tb; -if $rows != 19 then +if $rows != 25 then return -1 endi print ================= step12 -sql import into tb values(9527, 9527)(9527, 9528) +sql import into tb values(1564641709527, 9527)(1564641709527, 9528) sql select * from tb; print rows=> $rows -if $rows != 21 then +if $rows != 26 then return -1 endi print ================= step13 -sql import into tb values(9898, 9898)(9897, 9897) +sql import into tb values(1564641709898, 9898)(1564641709897, 9897) sql select * from tb; print rows=> $rows -if $rows != 23 then +if $rows != 28 then return -1 endi diff --git a/tests/script/windows/insert/basic.sim b/tests/script/windows/insert/basic.sim index 42fdf87098..be0980a2d4 100644 --- a/tests/script/windows/insert/basic.sim +++ b/tests/script/windows/insert/basic.sim @@ -22,20 +22,20 @@ while $x < 10 endw print =============== step 2 -sql insert into $tb values (now - 5m , 10) -x error_insert -sql insert into $tb values (now - 6m , 10) -x error_insert -sql insert into $tb values (now - 7m , 10) -x error_insert -sql insert into $tb values (now - 8m , 10) -x error_insert -error_insert: +sql insert into $tb values (now - 5m , 10) +sql insert into $tb values (now - 6m , 10) +sql insert into $tb values (now - 7m , 10) +sql insert into $tb values (now - 8m , 10) sql select * from $tb print $rows points data are retrieved -if $rows != 10 then +if $rows != 14 then return -1 endi sql drop database $db +sleep 1000 sql show databases if $rows != 0 then return -1 diff --git a/tests/script/windows/insert/query_multi_file.sim b/tests/script/windows/insert/query_multi_file.sim index 3170b43582..84c091fb21 100644 --- a/tests/script/windows/insert/query_multi_file.sim +++ b/tests/script/windows/insert/query_multi_file.sim @@ -27,7 +27,7 @@ $N = 20000 $x = 0 while $x < $N - $ms = $x . m + $ms = $x . s #print insert into $tb values (now + $ms , $x ) sql insert into $tb values (now + $ms , $x ) -x error_insert $x = $x + 1 diff --git a/tests/script/windows/table/binary.sim b/tests/script/windows/table/binary.sim index 1f3df987bb..69354ed5c8 100644 --- a/tests/script/windows/table/binary.sim +++ b/tests/script/windows/table/binary.sim @@ -38,7 +38,8 @@ if $data00 != 23456 then endi print =============== step4 -sql insert into $tb values (now+3a, '345678') +sql_error insert into $tb values (now+3a, '345678') +sql insert into $tb values (now+3a, '34567') sql select speed from $tb order by ts desc if $rows != 3 then return -1 diff --git a/tests/script/windows/table/bool.sim b/tests/script/windows/table/bool.sim index afaaf46f51..9e434d801a 100644 --- a/tests/script/windows/table/bool.sim +++ b/tests/script/windows/table/bool.sim @@ -19,7 +19,7 @@ if $rows != 1 then return -1 endi -if $data01 != true then +if $data01 != 1 then return -1 endi @@ -30,7 +30,7 @@ if $rows != 2 then return -1 endi -if $data01 != true then +if $data01 != 1 then return -1 endi @@ -41,7 +41,7 @@ if $rows != 3 then return -1 endi -if $data01 != true then +if $data01 != 1 then return -1 endi @@ -52,7 +52,7 @@ if $rows != 4 then return -1 endi -if $data01 != false then +if $data01 != 0 then return -1 endi @@ -63,7 +63,7 @@ if $rows != 5 then return -1 endi -if $data01 != true then +if $data01 != 1 then return -1 endi @@ -74,7 +74,7 @@ if $rows != 6 then return -1 endi -if $data01 != false then +if $data01 != 0 then return -1 endi diff --git a/tests/script/windows/table/column_value.sim b/tests/script/windows/table/column_value.sim index d3b120d08e..9dbaf7ceab 100644 --- a/tests/script/windows/table/column_value.sim +++ b/tests/script/windows/table/column_value.sim @@ -29,13 +29,12 @@ if $rows != 0 then endi print =============== step2 -sql create table $tb (ts timestamp, speed bigint, v1 binary(1500), v2 binary(1500), v3 binary(1500), v4 binary(500), v5 binary(500)) -x step2 - return -1 -step2: +sql create table $tb (ts timestamp, speed bigint, v1 binary(1500), v2 binary(1500), v3 binary(1500), v4 binary(500), v5 binary(500)) sql show tables -if $rows != 0 then +if $rows != 1 then return -1 endi +sql drop table $tb print =============== step3 sql create table $tb (ts timestamp, speed float, v1 binary(100), v2 binary(100), v3 binary(100), v4 binary(100), v5 binary(100)) @@ -56,11 +55,9 @@ if $rows != 0 then endi print =============== step4 -sql create table $tb (ts timestamp, speed double, v1 binary(1500), v2 binary(1500), v3 binary(1500), v4 binary(500), v5 binary(500)) -x step4 - return -1 -step4: +sql create table $tb (ts timestamp, speed double, v1 binary(1500), v2 binary(1500), v3 binary(1500), v4 binary(500), v5 binary(500)) sql show tables -if $rows != 0 then +if $rows != 1 then return -1 endi diff --git a/tests/script/windows/table/float.sim b/tests/script/windows/table/float.sim index 4188f62166..57b626f865 100644 --- a/tests/script/windows/table/float.sim +++ b/tests/script/windows/table/float.sim @@ -31,7 +31,7 @@ sql select * from $tb order by ts desc if $rows != 1 then return -1 endi -if $data01 != 2.8500 then +if $data01 != 2.85000 then return -1 endi @@ -41,7 +41,7 @@ sql select * from $tb order by ts desc if $rows != 2 then return -1 endi -if $data01 != 3.4000 then +if $data01 != 3.40000 then return -1 endi @@ -54,7 +54,7 @@ sql select * from $tb order by ts desc if $rows != 3 then return -1 endi -if $data01 != 0.0000 then +if $data01 != 0.00000 then return -1 endi @@ -67,7 +67,7 @@ sql select * from $tb order by ts desc if $rows != 4 then return -1 endi -if $data01 != 2.0000 then +if $data01 != 2.00000 then return -1 endi @@ -80,7 +80,7 @@ sql select * from $tb order by ts desc if $rows != 5 then return -1 endi -if $data01 != 2.0000 then +if $data01 != 2.00000 then return -1 endi diff --git a/tests/script/windows/table/table.sim b/tests/script/windows/table/table.sim index fe93fec6cd..55be8af851 100644 --- a/tests/script/windows/table/table.sim +++ b/tests/script/windows/table/table.sim @@ -58,7 +58,7 @@ sql create table $tb (ts timestamp, val float, val2 float) sql insert into $tb values(now, 5, 5) sql select * from $tb print ==> $data01 $data02 -if $data01 != 5.0000 then +if $data01 != 5.00000 then return -1 endi @@ -162,7 +162,7 @@ endi sql insert into $tb values(now, 5, 5) sql select * from $tb print ==> $data01 $data02 -if $data01 != 5.0000 then +if $data01 != 5.00000 then return -1 endi diff --git a/tests/script/windows/table/table_len.sim b/tests/script/windows/table/table_len.sim index b792044b40..cdd1f31731 100644 --- a/tests/script/windows/table/table_len.sim +++ b/tests/script/windows/table/table_len.sim @@ -51,7 +51,7 @@ if $rows != 0 then endi print =============== step4 -sql create table ab01234567890123456789a0123456789a0123456789ab01234567890123456789a0123456789a0123456789 (ts timestamp, speed int) -x step4 +sql create table ab01234567890123456789a0123456789a0123456789ab01234567890123456789a0123456789a0123456789ab01234567890123456789a0123456789a0123456789ab01234567890123456789a0123456789a0123456789ab01234567890123456789a0123456789a0123456789ab01234567890123456789a0123456789a0123456789 (ts timestamp, speed int) -x step4 return -1 step4: sql show tables diff --git a/tests/script/windows/tag/add.sim b/tests/script/windows/tag/add.sim index adcff4f62d..2c72d01955 100644 --- a/tests/script/windows/tag/add.sim +++ b/tests/script/windows/tag/add.sim @@ -30,7 +30,7 @@ endi if $data01 != 1 then return -1 endi -if $data02 != true then +if $data02 != 1 then return -1 endi if $data03 != 2 then @@ -40,7 +40,7 @@ endi sql alter table $mt drop tag tgcol2 sql alter table $mt add tag tgcol4 int sql reset query cache -sql alter table $tb set tgcol4 =4 +sql alter table $tb set tag tgcol4 =4 sql reset query cache sql select * from $mt where tgcol4 = 4 @@ -51,7 +51,7 @@ endi if $data01 != 1 then return -1 endi -if $data02 != true then +if $data02 != 1 then return -1 endi if $data03 != 4 then @@ -86,7 +86,7 @@ endi sql alter table $mt drop tag tgcol2 sql alter table $mt add tag tgcol4 tinyint sql reset query cache -sql alter table $tb set tgcol4=4 +sql alter table $tb set tag tgcol4=4 sql reset query cache sql select * from $mt where tgcol4 = 4 @@ -125,28 +125,29 @@ endi if $data02 != 1 then return -1 endi -if $data03 != 2.0000 then +if $data03 != 2.00000 then return -1 endi sql describe $tb +print sql describe $tb if $data21 != BIGINT then return -1 endi if $data31 != FLOAT then return -1 endi -if $data23 != 1 then +if $data23 != TAG then return -1 endi -if $data33 != 2.000000 then +if $data33 != TAG then return -1 endi sql alter table $mt drop tag tgcol2 sql alter table $mt add tag tgcol4 float sql reset query cache -sql alter table $tb set tgcol4=4 +sql alter table $tb set tag tgcol4=4 sql reset query cache sql select * from $mt where tgcol4 = 4 @@ -160,7 +161,7 @@ endi if $data02 != 1 then return -1 endi -if $data03 != 4.0000 then +if $data03 != 4.00000 then return -1 endi @@ -192,7 +193,7 @@ endi sql alter table $mt drop tag tgcol2 sql alter table $mt add tag tgcol4 smallint sql reset query cache -sql alter table $tb set tgcol4=4 +sql alter table $tb set tag tgcol4=4 sql reset query cache sql select * from $mt where tgcol4 = 4 @@ -228,7 +229,7 @@ endi if $data01 != 1 then return -1 endi -if $data02 != true then +if $data02 != 1 then return -1 endi if $data03 != 2 then @@ -245,9 +246,9 @@ sql alter table $mt add tag tgcol5 binary(10) sql alter table $mt add tag tgcol6 binary(10) sql reset query cache -sql alter table $tb set tgcol4=false -sql alter table $tb set tgcol5=5 -sql alter table $tb set tgcol6=6 +sql alter table $tb set tag tgcol4=false +sql alter table $tb set tag tgcol5=5 +sql alter table $tb set tag tgcol6=6 sql reset query cache sql select * from $mt where tgcol5 = '5' @@ -258,7 +259,7 @@ endi if $data01 != 1 then return -1 endi -if $data02 != false then +if $data02 != 0 then return -1 endi if $data03 != 5 then @@ -276,7 +277,7 @@ endi if $data01 != 1 then return -1 endi -if $data02 != false then +if $data02 != 0 then return -1 endi if $data03 != 5 then @@ -325,9 +326,9 @@ sql alter table $mt add tag tgcol5 bigint sql alter table $mt add tag tgcol6 tinyint sql reset query cache -sql alter table $tb set tgcol4=4 -sql alter table $tb set tgcol5=5 -sql alter table $tb set tgcol6=6 +sql alter table $tb set tag tgcol4=4 +sql alter table $tb set tag tgcol5=5 +sql alter table $tb set tag tgcol6=6 sql reset query cache sql select * from $mt where tgcol6 = 6 @@ -372,7 +373,7 @@ endi if $data02 != 1 then return -1 endi -if $data03 != 2.0000 then +if $data03 != 2.00000 then return -1 endi if $data04 != 3 then @@ -385,9 +386,9 @@ sql alter table $mt drop tag tgcol3 sql alter table $mt add tag tgcol5 binary(17) sql alter table $mt add tag tgcol6 bool sql reset query cache -sql alter table $tb set tgcol4=4 -sql alter table $tb set tgcol5=5 -sql alter table $tb set tgcol6=true +sql alter table $tb set tag tgcol4=4 +sql alter table $tb set tag tgcol5=5 +sql alter table $tb set tag tgcol6=1 sql reset query cache sql select * from $mt where tgcol5 = '5' @@ -405,7 +406,7 @@ endi if $data03 != 5 then return -1 endi -if $data04 != true then +if $data04 != 1 then return -1 endi @@ -447,12 +448,12 @@ sql alter table $mt add tag tgcol5 bool sql alter table $mt add tag tgcol6 float sql reset query cache -sql alter table $tb set tgcol4=4 -sql alter table $tb set tgcol5=true -sql alter table $tb set tgcol6=6 +sql alter table $tb set tag tgcol4=4 +sql alter table $tb set tag tgcol5=1 +sql alter table $tb set tag tgcol6=6 sql reset query cache -sql select * from $mt where tgcol5 = true +sql select * from $mt where tgcol5 = 1 print $data01 $data02 $data03 if $rows != 1 then return -1 @@ -463,10 +464,10 @@ endi if $data02 != 4.000000000 then return -1 endi -if $data03 != true then +if $data03 != 1 then return -1 endi -if $data04 != 6.0000 then +if $data04 != 6.00000 then return -1 endi @@ -516,8 +517,8 @@ sql alter table $mt add tag tgcol4 binary(10) sql alter table $mt add tag tgcol5 bool sql reset query cache -sql alter table $tb set tgcol4=4 -sql alter table $tb set tgcol5=false +sql alter table $tb set tag tgcol4=4 +sql alter table $tb set tag tgcol5=false sql reset query cache sql select * from $mt where tgcol4 = '4' @@ -534,10 +535,10 @@ endi if $data03 != 4 then return -1 endi -if $data04 != false then +if $data04 != 0 then return -1 endi -if $data05 != NULL then +if $data05 != null then return -1 endi @@ -562,7 +563,7 @@ endi if $data01 != 1 then return -1 endi -if $data02 != true then +if $data02 != 1 then return -1 endi if $data03 != 2 then @@ -571,7 +572,7 @@ endi if $data04 != 3 then return -1 endi -if $data05 != 4.0000 then +if $data05 != 4.00000 then return -1 endi if $data06 != 5 then @@ -594,11 +595,11 @@ sql alter table $mt add tag tgcol7 bigint sql alter table $mt add tag tgcol8 smallint sql reset query cache -sql alter table $tb set tgcol4=4 -sql alter table $tb set tgcol5=5 -sql alter table $tb set tgcol6=6 -sql alter table $tb set tgcol7=7 -sql alter table $tb set tgcol8=8 +sql alter table $tb set tag tgcol4=4 +sql alter table $tb set tag tgcol5=5 +sql alter table $tb set tag tgcol6=6 +sql alter table $tb set tag tgcol7=7 +sql alter table $tb set tag tgcol8=8 sql reset query cache sql select * from $mt where tgcol5 =5 @@ -609,7 +610,7 @@ endi if $data01 != 1 then return -1 endi -if $data02 != true then +if $data02 != 1 then return -1 endi if $data03 != 4 then @@ -652,13 +653,13 @@ endi if $data01 != 1 then return -1 endi -if $data02 != true then +if $data02 != 1 then return -1 endi if $data03 != 2 then return -1 endi -if $data04 != 3.0000 then +if $data04 != 3.00000 then return -1 endi if $data05 != 4.000000000 then @@ -682,12 +683,12 @@ sql alter table $mt add tag tgcol4 binary(10) sql alter table $mt add tag tgcol5 bigint sql reset query cache -sql alter table $tb set tgcol1=false -sql alter table $tb set tgcol2=5 -sql alter table $tb set tgcol3=4 -sql alter table $tb set tgcol4=3 -sql alter table $tb set tgcol5=2 -sql alter table $tb set tgcol6=1 +sql alter table $tb set tag tgcol1=false +sql alter table $tb set tag tgcol2=5 +sql alter table $tb set tag tgcol3=4 +sql alter table $tb set tag tgcol4=3 +sql alter table $tb set tag tgcol5=2 +sql alter table $tb set tag tgcol6=1 sql reset query cache sql select * from $mt where tgcol4 = '3' @@ -698,7 +699,7 @@ endi if $data01 != 1 then return -1 endi -if $data02 != false then +if $data02 != 0 then return -1 endi if $data03 != 1 then @@ -779,12 +780,12 @@ sql alter table $mt add tag tgcol4 int sql alter table $mt add tag tgcol6 bigint sql reset query cache -sql alter table $tb set tgcol1=7 -sql alter table $tb set tgcol2=8 -sql alter table $tb set tgcol3=9 -sql alter table $tb set tgcol4=10 -sql alter table $tb set tgcol5=11 -sql alter table $tb set tgcol6=12 +sql alter table $tb set tag tgcol1=7 +sql alter table $tb set tag tgcol2=8 +sql alter table $tb set tag tgcol3=9 +sql alter table $tb set tag tgcol4=10 +sql alter table $tb set tag tgcol5=11 +sql alter table $tb set tag tgcol6=12 sql reset query cache sql select * from $mt where tgcol2 = '8' @@ -826,6 +827,8 @@ sql alter table $mt add tag tgcol3 binary(10) sql alter table $mt add tag tgcol4 int sql alter table $mt add tag tgcol5 bigint sql alter table $mt add tag tgcol6 bigint + +return sql alter table $mt add tag tgcol7 bigint -x step141 return -1 step141: diff --git a/tests/script/windows/tag/change.sim b/tests/script/windows/tag/change.sim index 95a767da49..2901842190 100644 --- a/tests/script/windows/tag/change.sim +++ b/tests/script/windows/tag/change.sim @@ -30,7 +30,7 @@ endi if $data01 != 1 then return -1 endi -if $data02 != true then +if $data02 != 1 then return -1 endi if $data03 != 2 then @@ -43,9 +43,9 @@ step21: sql alter table $mt change tag tgcol1 tgcol2 -x step22 return -1 step22: -sql alter table $mt change tag tgcol1 xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx -x step20 - return -1 -step20: +#sql alter table $mt change tag tgcol1 xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx -x step20 +# return -1 +#step20: sql alter table $mt change tag tgcol1 tgcol3 sql alter table $mt change tag tgcol2 tgcol4 @@ -94,7 +94,7 @@ endi if $data02 != 1 then return -1 endi -if $data03 != 2.0000 then +if $data03 != 2.00000 then return -1 endi @@ -191,7 +191,7 @@ endi if $data01 != 1 then return -1 endi -if $data02 != true then +if $data02 != 1 then return -1 endi if $data03 != 2 then @@ -206,7 +206,7 @@ endi if $data01 != 1 then return -1 endi -if $data02 != true then +if $data02 != 1 then return -1 endi if $data03 != 2 then @@ -278,22 +278,22 @@ endi if $data02 != 1 then return -1 endi -if $data03 != 2.0000 then +if $data03 != 2.00000 then return -1 endi sql select * from $mt where tgcol4 = 2 print $data01 $data02 $data03 -if $rows != 1 then +if $rows != 1 then return -1 endi -if $data01 != 1 then +if $data01 != 1 then return -1 endi -if $data02 != 1 then +if $data02 != 1 then return -1 endi -if $data03 != 2.0000 then +if $data03 != 2.00000 then return -1 endi @@ -386,7 +386,7 @@ endi if $data06 != 6 then return -1 endi -if $data07 != NULL then +if $data07 != null then return -1 endi @@ -413,7 +413,7 @@ endi if $data06 != 6 then return -1 endi -if $data07 != NULL then +if $data07 != null then return -1 endi @@ -440,7 +440,7 @@ endi if $data06 != 6 then return -1 endi -if $data07 != NULL then +if $data07 != null then return -1 endi @@ -467,7 +467,7 @@ endi if $data06 != 6 then return -1 endi -if $data07 != NULL then +if $data07 != null then return -1 endi @@ -494,7 +494,7 @@ endi if $data06 != 6 then return -1 endi -if $data07 != NULL then +if $data07 != null then return -1 endi @@ -503,4 +503,4 @@ sql drop database $db sql show databases if $rows != 0 then return -1 -endi \ No newline at end of file +endi diff --git a/tests/script/windows/tag/create.sim b/tests/script/windows/tag/create.sim index e7f7163160..5beba21727 100644 --- a/tests/script/windows/tag/create.sim +++ b/tests/script/windows/tag/create.sim @@ -126,7 +126,8 @@ if $data01 != 1 then return -1 endi sql select * from $mt where tgcol = 0 -if $rows != 0 then +if $rows != 0 then + print expect 0, actual: $rows return -1 endi @@ -176,7 +177,8 @@ sql create table $mt (ts timestamp, tbcol int) TAGS(tgcol bool, tgcol2 bool) sql create table $tb using $mt tags( 1, 2 ) sql insert into $tb values(now, 1) sql select * from $mt where tgcol2 = 2 -if $rows != 1 then +if $rows != 1 then + print expect 1, actual: $rows return -1 endi if $data01 != 1 then @@ -569,7 +571,7 @@ $i = 30 $mt = $mtPrefix . $i $tb = $tbPrefix . $i sql create table $mt (ts timestamp, tbcol int) TAGS(tgcol binary(250), tgcol2 binary(250), tgcol3 binary(30)) -x step30 - return -1 +# return -1 step30: print =============== step31 @@ -577,7 +579,8 @@ $i = 31 $mt = $mtPrefix . $i $tb = $tbPrefix . $i sql create table $mt (ts timestamp, tbcol int) TAGS(tgcol binary(5)) -sql create table $tb using $mt tags('1234567') +sql_error create table $tb using $mt tags('1234567') +sql create table $tb using $mt tags('12345') sql insert into $tb values(now, 1) sql select * from $mt print sql select * from $mt diff --git a/tests/script/windows/tag/delete.sim b/tests/script/windows/tag/delete.sim index 75702fe9ee..e2395c8f97 100644 --- a/tests/script/windows/tag/delete.sim +++ b/tests/script/windows/tag/delete.sim @@ -30,7 +30,7 @@ endi if $data01 != 1 then return -1 endi -if $data02 != true then +if $data02 != 1 then return -1 endi if $data03 != 2 then @@ -79,7 +79,7 @@ endi if $data02 != 1 then return -1 endi -if $data03 != 2.0000 then +if $data03 != 2.00000 then return -1 endi @@ -90,7 +90,7 @@ endi if $data31 != FLOAT then return -1 endi -if $data23 != 1 then +if $data23 != TAG then return -1 endi @@ -139,7 +139,7 @@ endi if $data01 != 1 then return -1 endi -if $data02 != true then +if $data02 != 1 then return -1 endi if $data03 != 2 then @@ -195,13 +195,13 @@ endi if $data42 != 10 then return -1 endi -if $data23 != 1 then +if $data23 != TAG then return -1 endi -if $data33 != 2 then +if $data33 != TAG then return -1 endi -if $data43 != 3 then +if $data43 != TAG then return -1 endi @@ -225,7 +225,7 @@ endi if $data02 != 1 then return -1 endi -if $data03 != 2.0000 then +if $data03 != 2.00000 then return -1 endi if $data04 != 3 then @@ -307,7 +307,7 @@ endi if $data01 != 1 then return -1 endi -if $data02 != true then +if $data02 != 1 then return -1 endi if $data03 != 2 then @@ -316,7 +316,7 @@ endi if $data04 != 3 then return -1 endi -if $data05 != 4.0000 then +if $data05 != 4.00000 then return -1 endi if $data06 != 5 then @@ -341,13 +341,13 @@ endi if $data01 != 1 then return -1 endi -if $data02 != true then +if $data02 != 1 then return -1 endi if $data03 != 2 then return -1 endi -if $data04 != 3.0000 then +if $data04 != 3.00000 then return -1 endi if $data05 != 4.000000000 then @@ -417,10 +417,10 @@ endi if $data01 != 1 then return -1 endi -if $data02 != true then +if $data02 != 1 then return -1 endi -if $data03 != NULL then +if $data03 != null then return -1 endi @@ -444,7 +444,7 @@ endi if $data02 != 1 then return -1 endi -if $data03 != NULL then +if $data03 != null then return -1 endi @@ -468,7 +468,7 @@ endi if $data02 != 1 then return -1 endi -if $data03 != NULL then +if $data03 != null then return -1 endi @@ -492,7 +492,7 @@ endi if $data02 != 1.000000000 then return -1 endi -if $data03 != NULL then +if $data03 != null then return -1 endi @@ -513,13 +513,13 @@ endi if $data01 != 1 then return -1 endi -if $data02 != true then +if $data02 != 1 then return -1 endi -if $data03 != NULL then +if $data03 != null then return -1 endi -if $data04 != NULL then +if $data04 != null then return -1 endi @@ -546,10 +546,10 @@ endi if $data02 != 1 then return -1 endi -if $data03 != NULL then +if $data03 != null then return -1 endi -if $data04 != NULL then +if $data04 != null then return -1 endi @@ -576,10 +576,10 @@ endi if $data02 != 1 then return -1 endi -if $data03 != NULL then +if $data03 != null then return -1 endi -if $data04 != NULL then +if $data04 != null then return -1 endi @@ -606,10 +606,10 @@ endi if $data02 != 1.000000000 then return -1 endi -if $data03 != NULL then +if $data03 != null then return -1 endi -if $data04 != NULL then +if $data04 != null then return -1 endi @@ -636,13 +636,13 @@ endi if $data02 != 1 then return -1 endi -if $data03 != NULL then +if $data03 != null then return -1 endi -if $data04 != NULL then +if $data04 != null then return -1 endi -if $data05 != NULL then +if $data05 != null then return -1 endi @@ -669,19 +669,19 @@ endi if $data01 != 1 then return -1 endi -if $data02 != true then +if $data02 != 1 then return -1 endi -if $data03 != 4.0000 then +if $data03 != 4.00000 then return -1 endi -if $data04 != NULL then +if $data04 != null then return -1 endi -if $data05 != NULL then +if $data05 != null then return -1 endi -if $data06 != NULL then +if $data06 != null then return -1 endi @@ -708,22 +708,22 @@ endi if $data01 != 1 then return -1 endi -if $data02 != true then +if $data02 != 1 then return -1 endi if $data03 != 4.000000000 then return -1 endi -if $data04 != NULL then +if $data04 != null then return -1 endi -if $data05 != NULL then +if $data05 != null then return -1 endi -if $data06 != NULL then +if $data06 != null then return -1 endi -if $data07 != NULL then +if $data07 != null then return -1 endi @@ -763,13 +763,13 @@ endi if $data04 != 5.000000000 then return -1 endi -if $data05 != NULL then +if $data05 != null then return -1 endi -if $data06 != NULL then +if $data06 != null then return -1 endi -if $data07 != NULL then +if $data07 != null then return -1 endi diff --git a/tests/script/windows/tag/filter.sim b/tests/script/windows/tag/filter.sim index 72a8e39dc8..b70e56fdb6 100644 --- a/tests/script/windows/tag/filter.sim +++ b/tests/script/windows/tag/filter.sim @@ -85,9 +85,7 @@ sql select count(tgcol), avg(tgcol), sum(tgcol), min(tgcol), max(tgcol), first(t step7: print =============== step8 -sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt group by tbcol -x step8 - return -1 -step8: +sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt group by tbcol print =============== step9 sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from $mt group by noexist -x step9 @@ -102,9 +100,7 @@ if $data00 != 100 then endi print =============== step11 -sql select count(tbcol) as c from $mt group by tbcol -x step11 - return -1 -step11: +sql select count(tbcol) as c from $mt group by tbcol print =============== step12 sql select count(tbcol) as c from $mt group by noexist -x step12 @@ -121,7 +117,8 @@ endi print =============== step14 sql select count(tbcol) as c from $mt where ts > 1000 group by tgcol print $data00 $data01 $data02 $data03 $data04 $data05 $data06 -if $data00 != 100 then +if $data00 != 100 then + print expect 100, actual $data00 return -1 endi diff --git a/tests/script/windows/tag/set.sim b/tests/script/windows/tag/set.sim index b264164604..580f91cb49 100644 --- a/tests/script/windows/tag/set.sim +++ b/tests/script/windows/tag/set.sim @@ -30,7 +30,7 @@ endi if $data01 != 1 then return -1 endi -if $data02 != true then +if $data02 != 1 then return -1 endi if $data03 != 2 then @@ -40,8 +40,8 @@ endi sql alter table $tb set tag tagcx 1 -x step21 return -1 step21: -sql alter table $tb set tgcol1=false -sql alter table $tb set tgcol2=4 +sql alter table $tb set tag tgcol1=false +sql alter table $tb set tag tgcol2=4 sql reset query cache @@ -53,7 +53,7 @@ endi if $data01 != 1 then return -1 endi -if $data02 != false then +if $data02 != 0 then return -1 endi if $data03 != 4 then @@ -68,7 +68,7 @@ endi if $data01 != 1 then return -1 endi -if $data02 != false then +if $data02 != 0 then return -1 endi if $data03 != 4 then @@ -83,10 +83,10 @@ endi if $data31 != INT then return -1 endi -if $data23 != false then +if $data23 != TAG then return -1 endi -if $data33 != 4 then +if $data33 != TAG then return -1 endi @@ -111,8 +111,8 @@ if $data03 != 2 then return -1 endi -sql alter table $tb set tgcol1=3 -sql alter table $tb set tgcol2=4 +sql alter table $tb set tag tgcol1=3 +sql alter table $tb set tag tgcol2=4 sql reset query cache @@ -169,12 +169,12 @@ endi if $data02 != 1 then return -1 endi -if $data03 != 2.0000 then +if $data03 != 2.00000 then return -1 endi -sql alter table $tb set tgcol1=3 -sql alter table $tb set tgcol2=4 +sql alter table $tb set tag tgcol1=3 +sql alter table $tb set tag tgcol2=4 sql reset query cache @@ -189,7 +189,7 @@ endi if $data02 != 3 then return -1 endi -if $data03 != 4.0000 then +if $data03 != 4.00000 then return -1 endi @@ -204,7 +204,7 @@ endi if $data02 != 3 then return -1 endi -if $data03 != 4.0000 then +if $data03 != 4.00000 then return -1 endi @@ -230,8 +230,8 @@ if $data03 != 2 then return -1 endi -sql alter table $tb set tgcol1=3 -sql alter table $tb set tgcol2='4' +sql alter table $tb set tag tgcol1=3 +sql alter table $tb set tag tgcol2='4' sql reset query cache @@ -299,11 +299,11 @@ if $data07 != 6 then endi sql alter table $mt drop tag tgcol3 -sql alter table $tb set tgcol1='7' -sql alter table $tb set tgcol2=8 -sql alter table $tb set tgcol4='9' -sql alter table $tb set tgcol5=10 -sql alter table $tb set tgcol6='11' +sql alter table $tb set tag tgcol1='7' +sql alter table $tb set tag tgcol2=8 +sql alter table $tb set tag tgcol4='9' +sql alter table $tb set tag tgcol5=10 +sql alter table $tb set tag tgcol6='11' sql reset query cache @@ -330,7 +330,7 @@ endi if $data06 != 11 then return -1 endi -if $data07 != NULL then +if $data07 != null then return -1 endi @@ -357,7 +357,7 @@ endi if $data06 != 11 then return -1 endi -if $data07 != NULL then +if $data07 != null then return -1 endi @@ -384,7 +384,7 @@ endi if $data06 != 11 then return -1 endi -if $data07 != NULL then +if $data07 != null then return -1 endi @@ -411,7 +411,7 @@ endi if $data06 != 11 then return -1 endi -if $data07 != NULL then +if $data07 != null then return -1 endi @@ -438,7 +438,7 @@ endi if $data06 != 11 then return -1 endi -if $data07 != NULL then +if $data07 != null then return -1 endi diff --git a/tests/script/windows/vector/multi.sim b/tests/script/windows/vector/multi.sim index 969ba5aa19..adcc94db3b 100644 --- a/tests/script/windows/vector/multi.sim +++ b/tests/script/windows/vector/multi.sim @@ -200,7 +200,6 @@ sql select a + f from $tb where g = 2 and ts > now + 4m order by ts asc -x step return -1 step74: - print =============== clear sql drop database $db sql show databases From 592b9d1cd123d0eb1653cad494c36958d6542f25 Mon Sep 17 00:00:00 2001 From: slguan Date: Sat, 8 Aug 2020 23:15:23 +0800 Subject: [PATCH 084/109] scripts --- tests/script/windows/testSuite.sim | 146 ++++++++++++++--------------- 1 file changed, 71 insertions(+), 75 deletions(-) diff --git a/tests/script/windows/testSuite.sim b/tests/script/windows/testSuite.sim index 9aa7147ef6..e372217b62 100644 --- a/tests/script/windows/testSuite.sim +++ b/tests/script/windows/testSuite.sim @@ -1,97 +1,93 @@ -run windows/account/user_create.sim -run windows/account/user_len.sim - -run windows/db/len.sim - -run windows/table/table.sim -run windows/table/table_len.sim -run windows/table/column_num.sim -run windows/table/column_name.sim -run windows/table/column_value.sim -run windows/table/binary.sim -run windows/table/bool.sim -run windows/table/double.sim -run windows/table/float.sim -run windows/table/db.table.sim - -run windows/import/basic.sim - run windows/alter/table.sim run windows/alter/metrics.sim -run windows/compute/count.sim run windows/compute/avg.sim -run windows/compute/sum.sim -run windows/compute/min.sim -run windows/compute/max.sim -run windows/compute/first.sim -run windows/compute/last.sim -run windows/compute/stddev.sim -#run windows/compute/leastsquare.sim -run windows/compute/top.sim run windows/compute/bottom.sim -run windows/compute/percentile.sim +run windows/compute/count.sim run windows/compute/diff.sim +run windows/compute/first.sim run windows/compute/interval.sim - -run windows/field/single.sim -run windows/field/bool.sim -run windows/field/smallint.sim -run windows/field/tinyint.sim -run windows/field/int.sim -run windows/field/bigint.sim -run windows/field/float.sim -run windows/field/double.sim -run windows/field/binary.sim +run windows/compute/last.sim +run windows/compute/leastsquare.sim +run windows/compute/max.sim +run windows/compute/min.sim +run windows/compute/percentile.sim +run windows/compute/stddev.sim +run windows/compute/sum.sim +run windows/compute/top.sim + +run windows/db/basic.sim +run windows/db/len.sim + run windows/field/2.sim run windows/field/3.sim run windows/field/4.sim run windows/field/5.sim run windows/field/6.sim +run windows/field/bigint.sim +run windows/field/binary.sim +run windows/field/bool.sim +run windows/field/double.sim +run windows/field/float.sim +run windows/field/int.sim +run windows/field/single.sim +run windows/field/smallint.sim +run windows/field/tinyint.sim + +run windows/import/basic.sim + +run windows/insert/basic.sim +run windows/insert/query_block1_file.sim +run windows/insert/query_block1_memory.sim +run windows/insert/query_block2_file.sim +run windows/insert/query_block2_memory.sim +run windows/insert/query_file_memory.sim +run windows/insert/query_multi_file.sim + +run windows/table/binary.sim +run windows/table/bool.sim +run windows/table/column_num.sim +run windows/table/column_name.sim +run windows/table/column_value.sim +run windows/table/db.table.sim +run windows/table/double.sim +run windows/table/float.sim +run windows/table/table_len.sim +run windows/table/table.sim -run windows/tag/filter.sim -run windows/tag/column.sim -run windows/tag/bool.sim -run windows/tag/smallint.sim -run windows/tag/tinyint.sim -run windows/tag/int.sim -run windows/tag/bigint.sim -run windows/tag/float.sim -run windows/tag/double.sim -run windows/tag/binary.sim -run windows/tag/bool_int.sim -run windows/tag/bool_binary.sim -run windows/tag/int_float.sim -run windows/tag/int_binary.sim -run windows/tag/binary_binary.sim run windows/tag/3.sim run windows/tag/4.sim run windows/tag/5.sim run windows/tag/6.sim +run windows/tag/add.sim +run windows/tag/bigint.sim +run windows/tag/binary_binary.sim +run windows/tag/binary.sim +run windows/tag/bool_binary.sim +run windows/tag/bool_int.sim +run windows/tag/bool.sim +run windows/tag/change.sim +run windows/tag/column.sim run windows/tag/create.sim -#run windows/tag/delete.sim +run windows/tag/delete.sim +run windows/tag/double.sim +run windows/tag/filter.sim +run windows/tag/float.sim +run windows/tag/int_binary.sim +run windows/tag/int_float.sim +run windows/tag/int.sim +run windows/tag/set.sim +run windows/tag/smallint.sim +run windows/tag/tinyint.sim -#run windows/tag/set.sim -#run windows/tag/add.sim - -run windows/vector/single.sim -run windows/vector/multi.sim -run windows/vector/table_query.sim -run windows/vector/table_time.sim -run windows/vector/table_field.sim -run windows/vector/table_mix.sim +run windows/vector/metrics_field.sim +run windows/vector/metrics_mix.sim run windows/vector/metrics_query.sim run windows/vector/metrics_tag.sim run windows/vector/metrics_time.sim -run windows/vector/metrics_field.sim -run windows/vector/metrics_mix.sim - -run windows/insert/basic.sim -run windows/insert/query_block1_memory.sim -run windows/insert/query_block2_memory.sim -run windows/insert/query_block1_file.sim -#run windows/insert/query_block2_file.sim -#run windows/insert/query_file_memory.sim -#run windows/insert/query_multi_file.sim - -run windows/tag/change.sim +run windows/vector/multi.sim +run windows/vector/single.sim +run windows/vector/table_field.sim +run windows/vector/table_mix.sim +run windows/vector/table_query.sim +run windows/vector/table_time.sim From 2118830b90e5f63e3da691d7a9759b381f9c13b3 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sun, 9 Aug 2020 12:37:10 +0800 Subject: [PATCH 085/109] fix hard code ip in query code. --- .../src/main/java/com/opentsdb/test/OpentsdbTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/comparisonTest/opentsdb/opentsdbtest/src/main/java/com/opentsdb/test/OpentsdbTest.java b/tests/comparisonTest/opentsdb/opentsdbtest/src/main/java/com/opentsdb/test/OpentsdbTest.java index 4d02922766..0e82c480ec 100644 --- a/tests/comparisonTest/opentsdb/opentsdbtest/src/main/java/com/opentsdb/test/OpentsdbTest.java +++ b/tests/comparisonTest/opentsdb/opentsdbtest/src/main/java/com/opentsdb/test/OpentsdbTest.java @@ -198,7 +198,7 @@ public class OpentsdbTest{ case "q2": //count startTime = System.currentTimeMillis(); - get_url = "http://192.168.1.114:4242/api/query?"; + get_url = "http://127.0.0.1:4242/api/query?"; httpPost = new HttpPost(get_url); qjson = " {\n" + " \"start\": 1563249700,\n" + @@ -351,7 +351,7 @@ public class OpentsdbTest{ break; case "q3": startTime = System.currentTimeMillis(); - get_url = "http://192.168.1.114:4242/api/query?"; + get_url = "http://127.0.0.1:4242/api/query?"; httpPost = new HttpPost(get_url); qjson = " {\n" + " \"start\": 1563249700,\n" + @@ -411,7 +411,7 @@ public class OpentsdbTest{ break; case "q4": startTime = System.currentTimeMillis(); - get_url = "http://192.168.1.114:4242/api/query?"; + get_url = "http://127.0.0.1:4242/api/query?"; httpPost = new HttpPost(get_url); qjson = " {\n" + " \"start\": 1563249700,\n" + From fb4a40260b45b02af3c35243001d9bc1c3095210 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sun, 9 Aug 2020 04:52:47 +0000 Subject: [PATCH 086/109] if redirect does not include Ep List, treat it as NOT_READY --- src/rpc/src/rpcMain.c | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index 37bf9c3f34..70e5713ee6 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -322,6 +322,8 @@ void *rpcMallocCont(int contLen) { if (start == NULL) { tError("failed to malloc msg, size:%d", size); return NULL; + } else { + tDebug("malloc mem: %p", start); } return start + sizeof(SRpcReqContext) + sizeof(SRpcHead); @@ -331,7 +333,7 @@ void rpcFreeCont(void *cont) { if ( cont ) { char *temp = ((char *)cont) - sizeof(SRpcHead) - sizeof(SRpcReqContext); free(temp); - // tTrace("free mem: %p", temp); + tDebug("free mem: %p", temp); } } @@ -551,7 +553,7 @@ static void rpcFreeMsg(void *msg) { if ( msg ) { char *temp = (char *)msg - sizeof(SRpcReqContext); free(temp); - // tTrace("free mem: %p", temp); + tDebug("free mem: %p", temp); } } @@ -1098,10 +1100,15 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead) { } if (pHead->code == TSDB_CODE_RPC_REDIRECT) { - pContext->redirect++; - if (pContext->redirect > TSDB_MAX_REPLICA) { - pHead->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; - tWarn("%s, too many redirects, quit", pConn->info); + if (rpcMsg.contLen < sizeof(SRpcEpSet)) { + // if EpSet is not included in the msg, treat it as NOT_READY + pHead->code = TSDB_CODE_RPC_NOT_READY; + } else { + pContext->redirect++; + if (pContext->redirect > TSDB_MAX_REPLICA) { + pHead->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; + tWarn("%s, too many redirects, quit", pConn->info); + } } } From 90173d94c814182c6b5f7470e001d68f5c2b458a Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 9 Aug 2020 05:33:08 +0000 Subject: [PATCH 087/109] TD-1102 --- src/vnode/src/vnodeMain.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 6abd255dc4..9695f90c30 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -251,10 +251,17 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { appH.cqCreateFunc = cqCreate; appH.cqDropFunc = cqDrop; sprintf(temp, "%s/tsdb", rootDir); + + terrno = 0; pVnode->tsdb = tsdbOpenRepo(temp, &appH); if (pVnode->tsdb == NULL) { vnodeCleanUp(pVnode); return terrno; + } else if (terrno != 0 && pVnode->syncCfg.replica <= 1) { + vError("vgId:%d, failed to open tsdb, replica:%d reason:%s", pVnode->vgId, pVnode->syncCfg.replica, + tstrerror(terrno)); + vnodeCleanUp(pVnode); + return terrno; } sprintf(temp, "%s/wal", rootDir); From cced7da57e0c441a58e22d0f5c4ac1857d2b64c9 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sun, 9 Aug 2020 16:10:06 +0800 Subject: [PATCH 088/109] add opentsdb query loop test script. --- tests/perftest-scripts/opentsdbTestQ1Loop.sh | 99 +++++++ tests/perftest-scripts/opentsdbTestQ2Loop.sh | 276 +++++++++++++++++++ tests/perftest-scripts/opentsdbTestQ3Loop.sh | 96 +++++++ tests/perftest-scripts/opentsdbTestQ4Loop.sh | 103 +++++++ 4 files changed, 574 insertions(+) create mode 100755 tests/perftest-scripts/opentsdbTestQ1Loop.sh create mode 100755 tests/perftest-scripts/opentsdbTestQ2Loop.sh create mode 100755 tests/perftest-scripts/opentsdbTestQ3Loop.sh create mode 100755 tests/perftest-scripts/opentsdbTestQ4Loop.sh diff --git a/tests/perftest-scripts/opentsdbTestQ1Loop.sh b/tests/perftest-scripts/opentsdbTestQ1Loop.sh new file mode 100755 index 0000000000..2f9ca0dfdb --- /dev/null +++ b/tests/perftest-scripts/opentsdbTestQ1Loop.sh @@ -0,0 +1,99 @@ +#!/bin/bash + +NUM_LOOP=5 + +function printTo { + if $verbose ; then + echo $1 + fi +} + +TSDBTESTQ1OUT=opentsdbTestQ1.out + +function runTest { + totalG0=0 + totalG10=0 + totalG20=0 + totalG30=0 + totalG40=0 + totalG50=0 + totalG60=0 + totalG70=0 + totalG80=0 + totalG90=0 + for i in `seq 1 $NUM_LOOP`; do + printTo "loop i:$i, java -jar \ + $TSDBTEST_DIR/opentsdbtest/target/opentsdbtest-1.0-SNAPSHOT-jar-with-dependencies.jar \ + -sql q1" + java -jar \ + $TSDBTEST_DIR/opentsdbtest/target/opentsdbtest-1.0-SNAPSHOT-jar-with-dependencies.jar \ + -sql q1 2>&1 \ + | tee $TSDBTESTQ1OUT + G0=`grep "devgroup = 0" $TSDBTESTQ1OUT| awk '{print $2}'` + totalG0=`echo "scale=4; $totalG0 + $G0" | bc` + G10=`grep "devgroup = 10" $TSDBTESTQ1OUT| awk '{print $2}'` + totalG10=`echo "scale=4; $totalG10 + $G10" | bc` + G20=`grep "devgroup = 20" $TSDBTESTQ1OUT| awk '{print $2}'` + totalG20=`echo "scale=4; $totalG20 + $G20" | bc` + G30=`grep "devgroup = 30" $TSDBTESTQ1OUT| awk '{print $2}'` + totalG30=`echo "scale=4; $totalG30 + $G30" | bc` + G40=`grep "devgroup = 40" $TSDBTESTQ1OUT| awk '{print $2}'` + totalG40=`echo "scale=4; $totalG40 + $G40" | bc` + G50=`grep "devgroup = 50" $TSDBTESTQ1OUT| awk '{print $2}'` + totalG50=`echo "scale=4; $totalG50 + $G50" | bc` + G60=`grep "devgroup = 60" $TSDBTESTQ1OUT| awk '{print $2}'` + totalG60=`echo "scale=4; $totalG60 + $G60" | bc` + G70=`grep "devgroup = 70" $TSDBTESTQ1OUT| awk '{print $2}'` + totalG70=`echo "scale=4; $totalG70 + $G70" | bc` + G80=`grep "devgroup = 80" $TSDBTESTQ1OUT| awk '{print $2}'` + totalG80=`echo "scale=4; $totalG80 + $G80" | bc` + G90=`grep "devgroup = 90" $TSDBTESTQ1OUT| awk '{print $2}'` + totalG90=`echo "scale=4; $totalG90 + $G90" | bc` + done + avgG0=`echo "scale=4; x = $totalG0 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG10=`echo "scale=4; x = $totalG10 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG20=`echo "scale=4; x = $totalG20 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG30=`echo "scale=4; x = $totalG30 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG40=`echo "scale=4; x = $totalG40 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG50=`echo "scale=4; x = $totalG50 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG60=`echo "scale=4; x = $totalG60 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG70=`echo "scale=4; x = $totalG70 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG80=`echo "scale=4; x = $totalG80 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG90=`echo "scale=4; x = $totalG90 / $NUM_LOOP; if(x<1) print 0; x" | bc` + echo "Latency, G-0, G-10, G-20, G-30, G-40, G-50, G-60, G-70, G-80, G-90" + echo "OpenTSDB, $avgG0, $avgG10, $avgG20, $avgG30, $avgG40, $avgG50, $avgG60, $avgG70, $avgG80, $avgG90" +} + +################ Main ################ + +master=false +develop=true +verbose=false + +clients=1 + +while : ; do + case $1 in + -v) + verbose=true + shift ;; + + -c) + clients=$2 + shift 2;; + + -n) + NUM_LOOP=$2 + shift 2;; + + *) + break ;; + esac +done + +WORK_DIR=/mnt/root/TDengine +TSDBTEST_DIR=$WORK_DIR/tests/comparisonTest/opentsdb + +runTest + +printTo "Test done!" diff --git a/tests/perftest-scripts/opentsdbTestQ2Loop.sh b/tests/perftest-scripts/opentsdbTestQ2Loop.sh new file mode 100755 index 0000000000..db2d0435a3 --- /dev/null +++ b/tests/perftest-scripts/opentsdbTestQ2Loop.sh @@ -0,0 +1,276 @@ +#!/bin/bash + +NUM_LOOP=5 + +function printTo { + if $verbose ; then + echo $1 + fi +} + +TSDBTESTQ2OUT=opentsdbTestQ2.out + +function runTest { + totalCount10=0 + totalCount20=0 + totalCount30=0 + totalCount40=0 + totalCount50=0 + totalCount60=0 + totalCount70=0 + totalCount80=0 + totalCount90=0 + totalCount100=0 + + totalAvg10=0 + totalAvg20=0 + totalAvg30=0 + totalAvg40=0 + totalAvg50=0 + totalAvg60=0 + totalAvg70=0 + totalAvg80=0 + totalAvg90=0 + totalAvg100=0 + + totalSum10=0 + totalSum20=0 + totalSum30=0 + totalSum40=0 + totalSum50=0 + totalSum60=0 + totalSum70=0 + totalSum80=0 + totalSum90=0 + totalSum100=0 + + totalMax10=0 + totalMax20=0 + totalMax30=0 + totalMax40=0 + totalMax50=0 + totalMax60=0 + totalMax70=0 + totalMax80=0 + totalMax90=0 + totalMax100=0 + + totalMin10=0 + totalMin20=0 + totalMin30=0 + totalMin40=0 + totalMin50=0 + totalMin60=0 + totalMin70=0 + totalMin80=0 + totalMin90=0 + totalMin100=0 + + for i in `seq 1 $NUM_LOOP`; do + printTo "loop i:$i, java -jar \ + $TSDBTEST_DIR/opentsdbtest/target/opentsdbtest-1.0-SNAPSHOT-jar-with-dependencies.jar \ + -sql q2" + java -jar \ + $TSDBTEST_DIR/opentsdbtest/target/opentsdbtest-1.0-SNAPSHOT-jar-with-dependencies.jar \ + -sql q2 2>&1 \ + | tee $TSDBTESTQ2OUT + + Count10=`cat $TSDBTESTQ2OUT | grep count | grep "devgroup < 10" | awk '{print $2}'` + totalCount10=`echo "scale=4; $totalCount10 + $Count10" | bc` + Count20=`cat $TSDBTESTQ2OUT | grep count | grep "devgroup < 20" | awk '{print $2}'` + totalCount20=`echo "scale=4; $totalCount20 + $Count20" | bc` + Count30=`cat $TSDBTESTQ2OUT | grep count | grep "devgroup < 30" | awk '{print $2}'` + totalCount30=`echo "scale=4; $totalCount30 + $Count30" | bc` + Count40=`cat $TSDBTESTQ2OUT | grep count | grep "devgroup < 40" | awk '{print $2}'` + totalCount40=`echo "scale=4; $totalCount40 + $Count40" | bc` + Count50=`cat $TSDBTESTQ2OUT | grep count | grep "devgroup < 50" | awk '{print $2}'` + totalCount50=`echo "scale=4; $totalCount50 + $Count50" | bc` + Count60=`cat $TSDBTESTQ2OUT | grep count | grep "devgroup < 60" | awk '{print $2}'` + totalCount60=`echo "scale=4; $totalCount60 + $Count60" | bc` + Count70=`cat $TSDBTESTQ2OUT | grep count | grep "devgroup < 70" | awk '{print $2}'` + totalCount70=`echo "scale=4; $totalCount70 + $Count70" | bc` + Count80=`cat $TSDBTESTQ2OUT | grep count | grep "devgroup < 80" | awk '{print $2}'` + totalCount80=`echo "scale=4; $totalCount80 + $Count80" | bc` + Count90=`cat $TSDBTESTQ2OUT | grep count | grep "devgroup < 90" | awk '{print $2}'` + totalCount90=`echo "scale=4; $totalCount90 + $Count90" | bc` + Count100=`cat $TSDBTESTQ2OUT | grep count | grep "devgroup < 100" | awk '{print $2}'` + totalCount100=`echo "scale=4; $totalCount100 + $Count100" | bc` + + Avg10=`cat $TSDBTESTQ2OUT | grep avg | grep "devgroup < 10" | awk '{print $2}'` + totalAvg10=`echo "scale=4; $totalAvg10 + $Avg10" | bc` + Avg20=`cat $TSDBTESTQ2OUT | grep avg | grep "devgroup < 20" | awk '{print $2}'` + totalAvg20=`echo "scale=4; $totalAvg20 + $Avg20" | bc` + Avg30=`cat $TSDBTESTQ2OUT | grep avg | grep "devgroup < 30" | awk '{print $2}'` + totalAvg30=`echo "scale=4; $totalAvg30 + $Avg30" | bc` + Avg40=`cat $TSDBTESTQ2OUT | grep avg | grep "devgroup < 40" | awk '{print $2}'` + totalAvg40=`echo "scale=4; $totalAvg40 + $Avg40" | bc` + Avg50=`cat $TSDBTESTQ2OUT | grep avg | grep "devgroup < 50" | awk '{print $2}'` + totalAvg50=`echo "scale=4; $totalAvg50 + $Avg50" | bc` + Avg60=`cat $TSDBTESTQ2OUT | grep avg | grep "devgroup < 60" | awk '{print $2}'` + totalAvg60=`echo "scale=4; $totalAvg60 + $Avg60" | bc` + Avg70=`cat $TSDBTESTQ2OUT | grep avg | grep "devgroup < 70" | awk '{print $2}'` + totalAvg70=`echo "scale=4; $totalAvg70 + $Avg70" | bc` + Avg80=`cat $TSDBTESTQ2OUT | grep avg | grep "devgroup < 80" | awk '{print $2}'` + totalAvg80=`echo "scale=4; $totalAvg80 + $Avg80" | bc` + Avg90=`cat $TSDBTESTQ2OUT | grep avg | grep "devgroup < 90" | awk '{print $2}'` + totalAvg90=`echo "scale=4; $totalAvg90 + $Avg90" | bc` + Avg100=`cat $TSDBTESTQ2OUT | grep avg | grep "devgroup < 100" | awk '{print $2}'` + totalAvg100=`echo "scale=4; $totalAvg100 + $Avg100" | bc` + + Sum10=`cat $TSDBTESTQ2OUT | grep sum | grep "devgroup < 10" | awk '{print $2}'` + totalSum10=`echo "scale=4; $totalSum10 + $Sum10" | bc` + Sum20=`cat $TSDBTESTQ2OUT | grep sum | grep "devgroup < 20" | awk '{print $2}'` + totalSum20=`echo "scale=4; $totalSum20 + $Sum20" | bc` + Sum30=`cat $TSDBTESTQ2OUT | grep sum | grep "devgroup < 30" | awk '{print $2}'` + totalSum30=`echo "scale=4; $totalSum30 + $Sum30" | bc` + Sum40=`cat $TSDBTESTQ2OUT | grep sum | grep "devgroup < 40" | awk '{print $2}'` + totalSum40=`echo "scale=4; $totalSum40 + $Sum40" | bc` + Sum50=`cat $TSDBTESTQ2OUT | grep sum | grep "devgroup < 50" | awk '{print $2}'` + totalSum50=`echo "scale=4; $totalSum50 + $Sum50" | bc` + Sum60=`cat $TSDBTESTQ2OUT | grep sum | grep "devgroup < 60" | awk '{print $2}'` + totalSum60=`echo "scale=4; $totalSum60 + $Sum60" | bc` + Sum70=`cat $TSDBTESTQ2OUT | grep sum | grep "devgroup < 70" | awk '{print $2}'` + totalSum70=`echo "scale=4; $totalSum70 + $Sum70" | bc` + Sum80=`cat $TSDBTESTQ2OUT | grep sum | grep "devgroup < 80" | awk '{print $2}'` + totalSum80=`echo "scale=4; $totalSum80 + $Sum80" | bc` + Sum90=`cat $TSDBTESTQ2OUT | grep sum | grep "devgroup < 90" | awk '{print $2}'` + totalSum90=`echo "scale=4; $totalSum90 + $Sum90" | bc` + Sum100=`cat $TSDBTESTQ2OUT | grep sum | grep "devgroup < 100" | awk '{print $2}'` + totalSum100=`echo "scale=4; $totalSum100 + $Sum100" | bc` + + Max10=`cat $TSDBTESTQ2OUT | grep max | grep "devgroup < 10" | awk '{print $2}'` + totalMax10=`echo "scale=4; $totalMax10 + $Max10" | bc` + Max20=`cat $TSDBTESTQ2OUT | grep max | grep "devgroup < 20" | awk '{print $2}'` + totalMax20=`echo "scale=4; $totalMax20 + $Max20" | bc` + Max30=`cat $TSDBTESTQ2OUT | grep max | grep "devgroup < 30" | awk '{print $2}'` + totalMax30=`echo "scale=4; $totalMax30 + $Max30" | bc` + Max40=`cat $TSDBTESTQ2OUT | grep max | grep "devgroup < 40" | awk '{print $2}'` + totalMax40=`echo "scale=4; $totalMax40 + $Max40" | bc` + Max50=`cat $TSDBTESTQ2OUT | grep max | grep "devgroup < 50" | awk '{print $2}'` + totalMax50=`echo "scale=4; $totalMax50 + $Max50" | bc` + Max60=`cat $TSDBTESTQ2OUT | grep max | grep "devgroup < 60" | awk '{print $2}'` + totalMax60=`echo "scale=4; $totalMax60 + $Max60" | bc` + Max70=`cat $TSDBTESTQ2OUT | grep max | grep "devgroup < 70" | awk '{print $2}'` + totalMax70=`echo "scale=4; $totalMax70 + $Max70" | bc` + Max80=`cat $TSDBTESTQ2OUT | grep max | grep "devgroup < 80" | awk '{print $2}'` + totalMax80=`echo "scale=4; $totalMax80 + $Max80" | bc` + Max90=`cat $TSDBTESTQ2OUT | grep max | grep "devgroup < 90" | awk '{print $2}'` + totalMax90=`echo "scale=4; $totalMax90 + $Max90" | bc` + Max100=`cat $TSDBTESTQ2OUT | grep max | grep "devgroup < 100" | awk '{print $2}'` + totalMax100=`echo "scale=4; $totalMax100 + $Max100" | bc` + + Min10=`cat $TSDBTESTQ2OUT | grep min | grep "devgroup < 10" | awk '{print $2}'` + totalMin10=`echo "scale=4; $totalMin10 + $Min10" | bc` + Min20=`cat $TSDBTESTQ2OUT | grep min | grep "devgroup < 20" | awk '{print $2}'` + totalMin20=`echo "scale=4; $totalMin20 + $Min20" | bc` + Min30=`cat $TSDBTESTQ2OUT | grep min | grep "devgroup < 30" | awk '{print $2}'` + totalMin30=`echo "scale=4; $totalMin30 + $Min30" | bc` + Min40=`cat $TSDBTESTQ2OUT | grep min | grep "devgroup < 40" | awk '{print $2}'` + totalMin40=`echo "scale=4; $totalMin40 + $Min40" | bc` + Min50=`cat $TSDBTESTQ2OUT | grep min | grep "devgroup < 50" | awk '{print $2}'` + totalMin50=`echo "scale=4; $totalMin50 + $Min50" | bc` + Min60=`cat $TSDBTESTQ2OUT | grep min | grep "devgroup < 60" | awk '{print $2}'` + totalMin60=`echo "scale=4; $totalMin60 + $Min60" | bc` + Min70=`cat $TSDBTESTQ2OUT | grep min | grep "devgroup < 70" | awk '{print $2}'` + totalMin70=`echo "scale=4; $totalMin70 + $Min70" | bc` + Min80=`cat $TSDBTESTQ2OUT | grep min | grep "devgroup < 80" | awk '{print $2}'` + totalMin80=`echo "scale=4; $totalMin80 + $Min80" | bc` + Min90=`cat $TSDBTESTQ2OUT | grep min | grep "devgroup < 90" | awk '{print $2}'` + totalMin90=`echo "scale=4; $totalMin90 + $Min90" | bc` + Min100=`cat $TSDBTESTQ2OUT | grep min | grep "devgroup < 100" | awk '{print $2}'` + totalMin100=`echo "scale=4; $totalMin100 + $Min100" | bc` + + done + avgCount10=`echo "scale=4; x = $totalCount10 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgCount20=`echo "scale=4; x = $totalCount20 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgCount30=`echo "scale=4; x = $totalCount30 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgCount40=`echo "scale=4; x = $totalCount40 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgCount50=`echo "scale=4; x = $totalCount50 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgCount60=`echo "scale=4; x = $totalCount60 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgCount70=`echo "scale=4; x = $totalCount70 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgCount80=`echo "scale=4; x = $totalCount80 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgCount90=`echo "scale=4; x = $totalCount90 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgCount100=`echo "scale=4; x = $totalCount100 / $NUM_LOOP; if(x<1) print 0; x" | bc` + + avgAvg10=`echo "scale=4; x = $totalAvg10 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgAvg20=`echo "scale=4; x = $totalAvg20 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgAvg30=`echo "scale=4; x = $totalAvg30 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgAvg40=`echo "scale=4; x = $totalAvg40 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgAvg50=`echo "scale=4; x = $totalAvg50 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgAvg60=`echo "scale=4; x = $totalAvg60 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgAvg70=`echo "scale=4; x = $totalAvg70 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgAvg80=`echo "scale=4; x = $totalAvg80 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgAvg90=`echo "scale=4; x = $totalAvg90 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgAvg100=`echo "scale=4; x = $totalAvg100 / $NUM_LOOP; if(x<1) print 0; x" | bc` + + avgSum10=`echo "scale=4; x = $totalSum10 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgSum20=`echo "scale=4; x = $totalSum20 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgSum30=`echo "scale=4; x = $totalSum30 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgSum40=`echo "scale=4; x = $totalSum40 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgSum50=`echo "scale=4; x = $totalSum50 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgSum60=`echo "scale=4; x = $totalSum60 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgSum70=`echo "scale=4; x = $totalSum70 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgSum80=`echo "scale=4; x = $totalSum80 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgSum90=`echo "scale=4; x = $totalSum90 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgSum100=`echo "scale=4; x = $totalSum100 / $NUM_LOOP; if(x<1) print 0; x" | bc` + + avgMax10=`echo "scale=4; x = $totalMax10 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMax20=`echo "scale=4; x = $totalMax20 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMax30=`echo "scale=4; x = $totalMax30 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMax40=`echo "scale=4; x = $totalMax40 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMax50=`echo "scale=4; x = $totalMax50 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMax60=`echo "scale=4; x = $totalMax60 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMax70=`echo "scale=4; x = $totalMax70 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMax80=`echo "scale=4; x = $totalMax80 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMax90=`echo "scale=4; x = $totalMax90 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMax100=`echo "scale=4; x = $totalMax100 / $NUM_LOOP; if(x<1) print 0; x" | bc` + + avgMin10=`echo "scale=4; x = $totalMin10 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMin20=`echo "scale=4; x = $totalMin20 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMin30=`echo "scale=4; x = $totalMin30 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMin40=`echo "scale=4; x = $totalMin40 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMin50=`echo "scale=4; x = $totalMin50 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMin60=`echo "scale=4; x = $totalMin60 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMin70=`echo "scale=4; x = $totalMin70 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMin80=`echo "scale=4; x = $totalMin80 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMin90=`echo "scale=4; x = $totalMin90 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMin100=`echo "scale=4; x = $totalMin100 / $NUM_LOOP; if(x<1) print 0; x" | bc` + + echo "Latency, 10%, 20%, 30%, 40%, 50%, 60%, 70%, 80%, 90%, 100%" + echo "Count, $avgCount10, $avgCount20, $avgCount30, $avgCount40, $avgCount50, $avgCount60, $avgCount70, $avgCount80, $avgCount90, $avgCount100" + echo "Avg, $avgAvg10, $avgAvg20, $avgAvg30, $avgAvg40, $avgAvg50, $avgAvg60, $avgAvg70, $avgAvg80, $avgAvg90, $avgAvg100" + echo "Sum, $avgSum10, $avgSum20, $avgSum30, $avgSum40, $avgSum50, $avgSum60, $avgSum70, $avgSum80, $avgSum90, $avgSum100" + echo "Max, $avgMax10, $avgMax20, $avgMax30, $avgMax40, $avgMax50, $avgMax60, $avgMax70, $avgMax80, $avgMax90, $avgMax100" + echo "Min, $avgMin10, $avgMin20, $avgMin30, $avgMin40, $avgMin50, $avgMin60, $avgMin70, $avgMin80, $avgMin90, $avgMin100" +} + +################ Main ################ + +verbose=false + +for arg in "$@" +do + case $arg in + -v) + verbose=true + shift ;; + + -c) + clients=$2 + shift 2;; + + -n) + NUM_LOOP=$2 + shift 2;; + + *) + ;; + esac +done + +WORK_DIR=/mnt/root/TDengine +TSDBTEST_DIR=$WORK_DIR/tests/comparisonTest/opentsdb + +runTest + +printTo "Test done!" diff --git a/tests/perftest-scripts/opentsdbTestQ3Loop.sh b/tests/perftest-scripts/opentsdbTestQ3Loop.sh new file mode 100755 index 0000000000..6f69cf85a0 --- /dev/null +++ b/tests/perftest-scripts/opentsdbTestQ3Loop.sh @@ -0,0 +1,96 @@ +#!/bin/bash + +NUM_LOOP=5 + +function printTo { + if $verbose ; then + echo $1 + fi +} + +TSDBTESTQ3OUT=opentsdbTestQ3.out + +function runTest { + totalG10=0 + totalG20=0 + totalG30=0 + totalG40=0 + totalG50=0 + totalG60=0 + totalG70=0 + totalG80=0 + totalG90=0 + totalG100=0 + for i in `seq 1 $NUM_LOOP`; do + printTo "loop i:$i, java -jar \ + $TSDBTEST_DIR/opentsdbtest/target/opentsdbtest-1.0-SNAPSHOT-jar-with-dependencies.jar \ + -sql q3" + java -jar \ + $TSDBTEST_DIR/opentsdbtest/target/opentsdbtest-1.0-SNAPSHOT-jar-with-dependencies.jar \ + -sql q3 2>&1 \ + | tee $TSDBTESTQ3OUT + G10=`grep -w "devgroup < 10" $TSDBTESTQ3OUT| awk '{print $2}'` + totalG10=`echo "scale=4; $totalG10 + $G10" | bc` + G20=`grep -w "devgroup < 20" $TSDBTESTQ3OUT| awk '{print $2}'` + totalG20=`echo "scale=4; $totalG20 + $G20" | bc` + G30=`grep -w "devgroup < 30" $TSDBTESTQ3OUT| awk '{print $2}'` + totalG30=`echo "scale=4; $totalG30 + $G30" | bc` + G40=`grep -w "devgroup < 40" $TSDBTESTQ3OUT| awk '{print $2}'` + totalG40=`echo "scale=4; $totalG40 + $G40" | bc` + G50=`grep -w "devgroup < 50" $TSDBTESTQ3OUT| awk '{print $2}'` + totalG50=`echo "scale=4; $totalG50 + $G50" | bc` + G60=`grep -w "devgroup < 60" $TSDBTESTQ3OUT| awk '{print $2}'` + totalG60=`echo "scale=4; $totalG60 + $G60" | bc` + G70=`grep -w "devgroup < 70" $TSDBTESTQ3OUT| awk '{print $2}'` + totalG70=`echo "scale=4; $totalG70 + $G70" | bc` + G80=`grep -w "devgroup < 80" $TSDBTESTQ3OUT| awk '{print $2}'` + totalG80=`echo "scale=4; $totalG80 + $G80" | bc` + G90=`grep -w "devgroup < 90" $TSDBTESTQ3OUT| awk '{print $2}'` + totalG90=`echo "scale=4; $totalG90 + $G90" | bc` + G100=`grep -w "devgroup < 100" $TSDBTESTQ3OUT| awk '{print $2}'` + totalG100=`echo "scale=4; $totalG100 + $G100" | bc` + done + avgG10=`echo "scale=4; x = $totalG10 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG20=`echo "scale=4; x = $totalG20 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG30=`echo "scale=4; x = $totalG30 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG40=`echo "scale=4; x = $totalG40 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG50=`echo "scale=4; x = $totalG50 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG60=`echo "scale=4; x = $totalG60 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG70=`echo "scale=4; x = $totalG70 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG80=`echo "scale=4; x = $totalG80 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG90=`echo "scale=4; x = $totalG90 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG100=`echo "scale=4; x = $totalG100 / $NUM_LOOP; if(x<1) print 0; x" | bc` + echo "Latency, 10%, 20%, 30%, 40%, 50%, 60%, 70%, 80%, 90%, 100%" + echo "OpenTSDB, $avgG10, $avgG20, $avgG30, $avgG40, $avgG50, $avgG60, $avgG70, $avgG80, $avgG90, $avgG100" +} + +################ Main ################ + +verbose=false + +for arg in "$@" +do + case $arg in + -v) + verbose=true + shift ;; + + -c) + clients=$2 + shift 2;; + + -n) + NUM_LOOP=$2 + shift 2;; + + *) + ;; + esac +done + +WORK_DIR=/mnt/root/TDengine +TSDBTEST_DIR=$WORK_DIR/tests/comparisonTest/opentsdb + +runTest + +printTo "Test done!" diff --git a/tests/perftest-scripts/opentsdbTestQ4Loop.sh b/tests/perftest-scripts/opentsdbTestQ4Loop.sh new file mode 100755 index 0000000000..4341f2165c --- /dev/null +++ b/tests/perftest-scripts/opentsdbTestQ4Loop.sh @@ -0,0 +1,103 @@ +#!/bin/bash + +DATA_DIR=/mnt/root/testdata +NUM_LOOP=5 + +function printTo { + if $verbose ; then + echo $1 + fi +} + +TSDBTESTQ4OUT=opentsdbTestQ4.out + +function runTest { + totalG10=0 + totalG20=0 + totalG30=0 + totalG40=0 + totalG50=0 + totalG60=0 + totalG70=0 + totalG80=0 + totalG90=0 + totalG100=0 + for i in `seq 1 $NUM_LOOP`; do + printTo "loop i:$i, java -jar \ + $TSDBTEST_DIR/opentsdbtest/target/opentsdbtest-1.0-SNAPSHOT-jar-with-dependencies.jar \ + -sql q4" + + java -jar \ + $TSDBTEST_DIR/opentsdbtest/target/opentsdbtest-1.0-SNAPSHOT-jar-with-dependencies.jar \ + -sql q4 2>&1 | + tee $TSDBTESTQ4OUT + G10=`grep -w "devgroup < 10" $TSDBTESTQ4OUT| awk '{print $2}'` + totalG10=`echo "scale=4; $totalG10 + $G10" | bc` + G20=`grep -w "devgroup < 20" $TSDBTESTQ4OUT| awk '{print $2}'` + totalG20=`echo "scale=4; $totalG20 + $G20" | bc` + G30=`grep -w "devgroup < 30" $TSDBTESTQ4OUT| awk '{print $2}'` + totalG30=`echo "scale=4; $totalG30 + $G30" | bc` + G40=`grep -w "devgroup < 40" $TSDBTESTQ4OUT| awk '{print $2}'` + totalG40=`echo "scale=4; $totalG40 + $G40" | bc` + G50=`grep -w "devgroup < 50" $TSDBTESTQ4OUT| awk '{print $2}'` + totalG50=`echo "scale=4; $totalG50 + $G50" | bc` + G60=`grep -w "devgroup < 60" $TSDBTESTQ4OUT| awk '{print $2}'` + totalG60=`echo "scale=4; $totalG60 + $G60" | bc` + G70=`grep -w "devgroup < 70" $TSDBTESTQ4OUT| awk '{print $2}'` + totalG70=`echo "scale=4; $totalG70 + $G70" | bc` + G80=`grep -w "devgroup < 80" $TSDBTESTQ4OUT| awk '{print $2}'` + totalG80=`echo "scale=4; $totalG80 + $G80" | bc` + G90=`grep -w "devgroup < 90" $TSDBTESTQ4OUT| awk '{print $2}'` + totalG90=`echo "scale=4; $totalG90 + $G90" | bc` + G100=`grep -w "devgroup < 100" $TSDBTESTQ4OUT| awk '{print $2}'` + totalG100=`echo "scale=4; $totalG100 + $G100" | bc` + done + avgG10=`echo "scale=4; x = $totalG10 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG20=`echo "scale=4; x = $totalG20 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG30=`echo "scale=4; x = $totalG30 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG40=`echo "scale=4; x = $totalG40 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG50=`echo "scale=4; x = $totalG50 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG60=`echo "scale=4; x = $totalG60 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG70=`echo "scale=4; x = $totalG70 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG80=`echo "scale=4; x = $totalG80 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG90=`echo "scale=4; x = $totalG90 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG100=`echo "scale=4; x = $totalG100 / $NUM_LOOP; if(x<1) print 0; x" | bc` + echo "Latency, 10%, 20%, 30%, 40%, 50%, 60%, 70%, 80%, 90%, 100%" + echo "OpenTSDB, $avgG10, $avgG20, $avgG30, $avgG40, $avgG50, $avgG60, $avgG70, $avgG80, $avgG90, $avgG100" +} + +################ Main ################ + +verbose=false +regeneratedata=false + +for arg in "$@" +do + case $arg in + -v) + verbose=true + shift ;; + + -c) + clients=$2 + shift 2;; + + -r) + regeneratedata=true + ;; + + -n) + NUM_LOOP=$2 + shift 2;; + + *) + ;; + esac +done + +WORK_DIR=/mnt/root/TDengine +TSDBTEST_DIR=$WORK_DIR/tests/comparisonTest/opentsdb + +runTest + +printTo "Test done!" From 4ae1387995840946dc789dc0f5a92659110e0f26 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sun, 9 Aug 2020 16:10:06 +0800 Subject: [PATCH 089/109] add opentsdb query loop test script. --- tests/perftest-scripts/opentsdbTestQ1Loop.sh | 99 +++++++ tests/perftest-scripts/opentsdbTestQ2Loop.sh | 276 +++++++++++++++++++ tests/perftest-scripts/opentsdbTestQ3Loop.sh | 96 +++++++ tests/perftest-scripts/opentsdbTestQ4Loop.sh | 103 +++++++ 4 files changed, 574 insertions(+) create mode 100755 tests/perftest-scripts/opentsdbTestQ1Loop.sh create mode 100755 tests/perftest-scripts/opentsdbTestQ2Loop.sh create mode 100755 tests/perftest-scripts/opentsdbTestQ3Loop.sh create mode 100755 tests/perftest-scripts/opentsdbTestQ4Loop.sh diff --git a/tests/perftest-scripts/opentsdbTestQ1Loop.sh b/tests/perftest-scripts/opentsdbTestQ1Loop.sh new file mode 100755 index 0000000000..2f9ca0dfdb --- /dev/null +++ b/tests/perftest-scripts/opentsdbTestQ1Loop.sh @@ -0,0 +1,99 @@ +#!/bin/bash + +NUM_LOOP=5 + +function printTo { + if $verbose ; then + echo $1 + fi +} + +TSDBTESTQ1OUT=opentsdbTestQ1.out + +function runTest { + totalG0=0 + totalG10=0 + totalG20=0 + totalG30=0 + totalG40=0 + totalG50=0 + totalG60=0 + totalG70=0 + totalG80=0 + totalG90=0 + for i in `seq 1 $NUM_LOOP`; do + printTo "loop i:$i, java -jar \ + $TSDBTEST_DIR/opentsdbtest/target/opentsdbtest-1.0-SNAPSHOT-jar-with-dependencies.jar \ + -sql q1" + java -jar \ + $TSDBTEST_DIR/opentsdbtest/target/opentsdbtest-1.0-SNAPSHOT-jar-with-dependencies.jar \ + -sql q1 2>&1 \ + | tee $TSDBTESTQ1OUT + G0=`grep "devgroup = 0" $TSDBTESTQ1OUT| awk '{print $2}'` + totalG0=`echo "scale=4; $totalG0 + $G0" | bc` + G10=`grep "devgroup = 10" $TSDBTESTQ1OUT| awk '{print $2}'` + totalG10=`echo "scale=4; $totalG10 + $G10" | bc` + G20=`grep "devgroup = 20" $TSDBTESTQ1OUT| awk '{print $2}'` + totalG20=`echo "scale=4; $totalG20 + $G20" | bc` + G30=`grep "devgroup = 30" $TSDBTESTQ1OUT| awk '{print $2}'` + totalG30=`echo "scale=4; $totalG30 + $G30" | bc` + G40=`grep "devgroup = 40" $TSDBTESTQ1OUT| awk '{print $2}'` + totalG40=`echo "scale=4; $totalG40 + $G40" | bc` + G50=`grep "devgroup = 50" $TSDBTESTQ1OUT| awk '{print $2}'` + totalG50=`echo "scale=4; $totalG50 + $G50" | bc` + G60=`grep "devgroup = 60" $TSDBTESTQ1OUT| awk '{print $2}'` + totalG60=`echo "scale=4; $totalG60 + $G60" | bc` + G70=`grep "devgroup = 70" $TSDBTESTQ1OUT| awk '{print $2}'` + totalG70=`echo "scale=4; $totalG70 + $G70" | bc` + G80=`grep "devgroup = 80" $TSDBTESTQ1OUT| awk '{print $2}'` + totalG80=`echo "scale=4; $totalG80 + $G80" | bc` + G90=`grep "devgroup = 90" $TSDBTESTQ1OUT| awk '{print $2}'` + totalG90=`echo "scale=4; $totalG90 + $G90" | bc` + done + avgG0=`echo "scale=4; x = $totalG0 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG10=`echo "scale=4; x = $totalG10 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG20=`echo "scale=4; x = $totalG20 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG30=`echo "scale=4; x = $totalG30 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG40=`echo "scale=4; x = $totalG40 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG50=`echo "scale=4; x = $totalG50 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG60=`echo "scale=4; x = $totalG60 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG70=`echo "scale=4; x = $totalG70 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG80=`echo "scale=4; x = $totalG80 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG90=`echo "scale=4; x = $totalG90 / $NUM_LOOP; if(x<1) print 0; x" | bc` + echo "Latency, G-0, G-10, G-20, G-30, G-40, G-50, G-60, G-70, G-80, G-90" + echo "OpenTSDB, $avgG0, $avgG10, $avgG20, $avgG30, $avgG40, $avgG50, $avgG60, $avgG70, $avgG80, $avgG90" +} + +################ Main ################ + +master=false +develop=true +verbose=false + +clients=1 + +while : ; do + case $1 in + -v) + verbose=true + shift ;; + + -c) + clients=$2 + shift 2;; + + -n) + NUM_LOOP=$2 + shift 2;; + + *) + break ;; + esac +done + +WORK_DIR=/mnt/root/TDengine +TSDBTEST_DIR=$WORK_DIR/tests/comparisonTest/opentsdb + +runTest + +printTo "Test done!" diff --git a/tests/perftest-scripts/opentsdbTestQ2Loop.sh b/tests/perftest-scripts/opentsdbTestQ2Loop.sh new file mode 100755 index 0000000000..db2d0435a3 --- /dev/null +++ b/tests/perftest-scripts/opentsdbTestQ2Loop.sh @@ -0,0 +1,276 @@ +#!/bin/bash + +NUM_LOOP=5 + +function printTo { + if $verbose ; then + echo $1 + fi +} + +TSDBTESTQ2OUT=opentsdbTestQ2.out + +function runTest { + totalCount10=0 + totalCount20=0 + totalCount30=0 + totalCount40=0 + totalCount50=0 + totalCount60=0 + totalCount70=0 + totalCount80=0 + totalCount90=0 + totalCount100=0 + + totalAvg10=0 + totalAvg20=0 + totalAvg30=0 + totalAvg40=0 + totalAvg50=0 + totalAvg60=0 + totalAvg70=0 + totalAvg80=0 + totalAvg90=0 + totalAvg100=0 + + totalSum10=0 + totalSum20=0 + totalSum30=0 + totalSum40=0 + totalSum50=0 + totalSum60=0 + totalSum70=0 + totalSum80=0 + totalSum90=0 + totalSum100=0 + + totalMax10=0 + totalMax20=0 + totalMax30=0 + totalMax40=0 + totalMax50=0 + totalMax60=0 + totalMax70=0 + totalMax80=0 + totalMax90=0 + totalMax100=0 + + totalMin10=0 + totalMin20=0 + totalMin30=0 + totalMin40=0 + totalMin50=0 + totalMin60=0 + totalMin70=0 + totalMin80=0 + totalMin90=0 + totalMin100=0 + + for i in `seq 1 $NUM_LOOP`; do + printTo "loop i:$i, java -jar \ + $TSDBTEST_DIR/opentsdbtest/target/opentsdbtest-1.0-SNAPSHOT-jar-with-dependencies.jar \ + -sql q2" + java -jar \ + $TSDBTEST_DIR/opentsdbtest/target/opentsdbtest-1.0-SNAPSHOT-jar-with-dependencies.jar \ + -sql q2 2>&1 \ + | tee $TSDBTESTQ2OUT + + Count10=`cat $TSDBTESTQ2OUT | grep count | grep "devgroup < 10" | awk '{print $2}'` + totalCount10=`echo "scale=4; $totalCount10 + $Count10" | bc` + Count20=`cat $TSDBTESTQ2OUT | grep count | grep "devgroup < 20" | awk '{print $2}'` + totalCount20=`echo "scale=4; $totalCount20 + $Count20" | bc` + Count30=`cat $TSDBTESTQ2OUT | grep count | grep "devgroup < 30" | awk '{print $2}'` + totalCount30=`echo "scale=4; $totalCount30 + $Count30" | bc` + Count40=`cat $TSDBTESTQ2OUT | grep count | grep "devgroup < 40" | awk '{print $2}'` + totalCount40=`echo "scale=4; $totalCount40 + $Count40" | bc` + Count50=`cat $TSDBTESTQ2OUT | grep count | grep "devgroup < 50" | awk '{print $2}'` + totalCount50=`echo "scale=4; $totalCount50 + $Count50" | bc` + Count60=`cat $TSDBTESTQ2OUT | grep count | grep "devgroup < 60" | awk '{print $2}'` + totalCount60=`echo "scale=4; $totalCount60 + $Count60" | bc` + Count70=`cat $TSDBTESTQ2OUT | grep count | grep "devgroup < 70" | awk '{print $2}'` + totalCount70=`echo "scale=4; $totalCount70 + $Count70" | bc` + Count80=`cat $TSDBTESTQ2OUT | grep count | grep "devgroup < 80" | awk '{print $2}'` + totalCount80=`echo "scale=4; $totalCount80 + $Count80" | bc` + Count90=`cat $TSDBTESTQ2OUT | grep count | grep "devgroup < 90" | awk '{print $2}'` + totalCount90=`echo "scale=4; $totalCount90 + $Count90" | bc` + Count100=`cat $TSDBTESTQ2OUT | grep count | grep "devgroup < 100" | awk '{print $2}'` + totalCount100=`echo "scale=4; $totalCount100 + $Count100" | bc` + + Avg10=`cat $TSDBTESTQ2OUT | grep avg | grep "devgroup < 10" | awk '{print $2}'` + totalAvg10=`echo "scale=4; $totalAvg10 + $Avg10" | bc` + Avg20=`cat $TSDBTESTQ2OUT | grep avg | grep "devgroup < 20" | awk '{print $2}'` + totalAvg20=`echo "scale=4; $totalAvg20 + $Avg20" | bc` + Avg30=`cat $TSDBTESTQ2OUT | grep avg | grep "devgroup < 30" | awk '{print $2}'` + totalAvg30=`echo "scale=4; $totalAvg30 + $Avg30" | bc` + Avg40=`cat $TSDBTESTQ2OUT | grep avg | grep "devgroup < 40" | awk '{print $2}'` + totalAvg40=`echo "scale=4; $totalAvg40 + $Avg40" | bc` + Avg50=`cat $TSDBTESTQ2OUT | grep avg | grep "devgroup < 50" | awk '{print $2}'` + totalAvg50=`echo "scale=4; $totalAvg50 + $Avg50" | bc` + Avg60=`cat $TSDBTESTQ2OUT | grep avg | grep "devgroup < 60" | awk '{print $2}'` + totalAvg60=`echo "scale=4; $totalAvg60 + $Avg60" | bc` + Avg70=`cat $TSDBTESTQ2OUT | grep avg | grep "devgroup < 70" | awk '{print $2}'` + totalAvg70=`echo "scale=4; $totalAvg70 + $Avg70" | bc` + Avg80=`cat $TSDBTESTQ2OUT | grep avg | grep "devgroup < 80" | awk '{print $2}'` + totalAvg80=`echo "scale=4; $totalAvg80 + $Avg80" | bc` + Avg90=`cat $TSDBTESTQ2OUT | grep avg | grep "devgroup < 90" | awk '{print $2}'` + totalAvg90=`echo "scale=4; $totalAvg90 + $Avg90" | bc` + Avg100=`cat $TSDBTESTQ2OUT | grep avg | grep "devgroup < 100" | awk '{print $2}'` + totalAvg100=`echo "scale=4; $totalAvg100 + $Avg100" | bc` + + Sum10=`cat $TSDBTESTQ2OUT | grep sum | grep "devgroup < 10" | awk '{print $2}'` + totalSum10=`echo "scale=4; $totalSum10 + $Sum10" | bc` + Sum20=`cat $TSDBTESTQ2OUT | grep sum | grep "devgroup < 20" | awk '{print $2}'` + totalSum20=`echo "scale=4; $totalSum20 + $Sum20" | bc` + Sum30=`cat $TSDBTESTQ2OUT | grep sum | grep "devgroup < 30" | awk '{print $2}'` + totalSum30=`echo "scale=4; $totalSum30 + $Sum30" | bc` + Sum40=`cat $TSDBTESTQ2OUT | grep sum | grep "devgroup < 40" | awk '{print $2}'` + totalSum40=`echo "scale=4; $totalSum40 + $Sum40" | bc` + Sum50=`cat $TSDBTESTQ2OUT | grep sum | grep "devgroup < 50" | awk '{print $2}'` + totalSum50=`echo "scale=4; $totalSum50 + $Sum50" | bc` + Sum60=`cat $TSDBTESTQ2OUT | grep sum | grep "devgroup < 60" | awk '{print $2}'` + totalSum60=`echo "scale=4; $totalSum60 + $Sum60" | bc` + Sum70=`cat $TSDBTESTQ2OUT | grep sum | grep "devgroup < 70" | awk '{print $2}'` + totalSum70=`echo "scale=4; $totalSum70 + $Sum70" | bc` + Sum80=`cat $TSDBTESTQ2OUT | grep sum | grep "devgroup < 80" | awk '{print $2}'` + totalSum80=`echo "scale=4; $totalSum80 + $Sum80" | bc` + Sum90=`cat $TSDBTESTQ2OUT | grep sum | grep "devgroup < 90" | awk '{print $2}'` + totalSum90=`echo "scale=4; $totalSum90 + $Sum90" | bc` + Sum100=`cat $TSDBTESTQ2OUT | grep sum | grep "devgroup < 100" | awk '{print $2}'` + totalSum100=`echo "scale=4; $totalSum100 + $Sum100" | bc` + + Max10=`cat $TSDBTESTQ2OUT | grep max | grep "devgroup < 10" | awk '{print $2}'` + totalMax10=`echo "scale=4; $totalMax10 + $Max10" | bc` + Max20=`cat $TSDBTESTQ2OUT | grep max | grep "devgroup < 20" | awk '{print $2}'` + totalMax20=`echo "scale=4; $totalMax20 + $Max20" | bc` + Max30=`cat $TSDBTESTQ2OUT | grep max | grep "devgroup < 30" | awk '{print $2}'` + totalMax30=`echo "scale=4; $totalMax30 + $Max30" | bc` + Max40=`cat $TSDBTESTQ2OUT | grep max | grep "devgroup < 40" | awk '{print $2}'` + totalMax40=`echo "scale=4; $totalMax40 + $Max40" | bc` + Max50=`cat $TSDBTESTQ2OUT | grep max | grep "devgroup < 50" | awk '{print $2}'` + totalMax50=`echo "scale=4; $totalMax50 + $Max50" | bc` + Max60=`cat $TSDBTESTQ2OUT | grep max | grep "devgroup < 60" | awk '{print $2}'` + totalMax60=`echo "scale=4; $totalMax60 + $Max60" | bc` + Max70=`cat $TSDBTESTQ2OUT | grep max | grep "devgroup < 70" | awk '{print $2}'` + totalMax70=`echo "scale=4; $totalMax70 + $Max70" | bc` + Max80=`cat $TSDBTESTQ2OUT | grep max | grep "devgroup < 80" | awk '{print $2}'` + totalMax80=`echo "scale=4; $totalMax80 + $Max80" | bc` + Max90=`cat $TSDBTESTQ2OUT | grep max | grep "devgroup < 90" | awk '{print $2}'` + totalMax90=`echo "scale=4; $totalMax90 + $Max90" | bc` + Max100=`cat $TSDBTESTQ2OUT | grep max | grep "devgroup < 100" | awk '{print $2}'` + totalMax100=`echo "scale=4; $totalMax100 + $Max100" | bc` + + Min10=`cat $TSDBTESTQ2OUT | grep min | grep "devgroup < 10" | awk '{print $2}'` + totalMin10=`echo "scale=4; $totalMin10 + $Min10" | bc` + Min20=`cat $TSDBTESTQ2OUT | grep min | grep "devgroup < 20" | awk '{print $2}'` + totalMin20=`echo "scale=4; $totalMin20 + $Min20" | bc` + Min30=`cat $TSDBTESTQ2OUT | grep min | grep "devgroup < 30" | awk '{print $2}'` + totalMin30=`echo "scale=4; $totalMin30 + $Min30" | bc` + Min40=`cat $TSDBTESTQ2OUT | grep min | grep "devgroup < 40" | awk '{print $2}'` + totalMin40=`echo "scale=4; $totalMin40 + $Min40" | bc` + Min50=`cat $TSDBTESTQ2OUT | grep min | grep "devgroup < 50" | awk '{print $2}'` + totalMin50=`echo "scale=4; $totalMin50 + $Min50" | bc` + Min60=`cat $TSDBTESTQ2OUT | grep min | grep "devgroup < 60" | awk '{print $2}'` + totalMin60=`echo "scale=4; $totalMin60 + $Min60" | bc` + Min70=`cat $TSDBTESTQ2OUT | grep min | grep "devgroup < 70" | awk '{print $2}'` + totalMin70=`echo "scale=4; $totalMin70 + $Min70" | bc` + Min80=`cat $TSDBTESTQ2OUT | grep min | grep "devgroup < 80" | awk '{print $2}'` + totalMin80=`echo "scale=4; $totalMin80 + $Min80" | bc` + Min90=`cat $TSDBTESTQ2OUT | grep min | grep "devgroup < 90" | awk '{print $2}'` + totalMin90=`echo "scale=4; $totalMin90 + $Min90" | bc` + Min100=`cat $TSDBTESTQ2OUT | grep min | grep "devgroup < 100" | awk '{print $2}'` + totalMin100=`echo "scale=4; $totalMin100 + $Min100" | bc` + + done + avgCount10=`echo "scale=4; x = $totalCount10 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgCount20=`echo "scale=4; x = $totalCount20 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgCount30=`echo "scale=4; x = $totalCount30 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgCount40=`echo "scale=4; x = $totalCount40 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgCount50=`echo "scale=4; x = $totalCount50 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgCount60=`echo "scale=4; x = $totalCount60 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgCount70=`echo "scale=4; x = $totalCount70 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgCount80=`echo "scale=4; x = $totalCount80 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgCount90=`echo "scale=4; x = $totalCount90 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgCount100=`echo "scale=4; x = $totalCount100 / $NUM_LOOP; if(x<1) print 0; x" | bc` + + avgAvg10=`echo "scale=4; x = $totalAvg10 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgAvg20=`echo "scale=4; x = $totalAvg20 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgAvg30=`echo "scale=4; x = $totalAvg30 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgAvg40=`echo "scale=4; x = $totalAvg40 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgAvg50=`echo "scale=4; x = $totalAvg50 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgAvg60=`echo "scale=4; x = $totalAvg60 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgAvg70=`echo "scale=4; x = $totalAvg70 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgAvg80=`echo "scale=4; x = $totalAvg80 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgAvg90=`echo "scale=4; x = $totalAvg90 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgAvg100=`echo "scale=4; x = $totalAvg100 / $NUM_LOOP; if(x<1) print 0; x" | bc` + + avgSum10=`echo "scale=4; x = $totalSum10 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgSum20=`echo "scale=4; x = $totalSum20 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgSum30=`echo "scale=4; x = $totalSum30 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgSum40=`echo "scale=4; x = $totalSum40 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgSum50=`echo "scale=4; x = $totalSum50 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgSum60=`echo "scale=4; x = $totalSum60 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgSum70=`echo "scale=4; x = $totalSum70 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgSum80=`echo "scale=4; x = $totalSum80 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgSum90=`echo "scale=4; x = $totalSum90 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgSum100=`echo "scale=4; x = $totalSum100 / $NUM_LOOP; if(x<1) print 0; x" | bc` + + avgMax10=`echo "scale=4; x = $totalMax10 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMax20=`echo "scale=4; x = $totalMax20 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMax30=`echo "scale=4; x = $totalMax30 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMax40=`echo "scale=4; x = $totalMax40 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMax50=`echo "scale=4; x = $totalMax50 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMax60=`echo "scale=4; x = $totalMax60 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMax70=`echo "scale=4; x = $totalMax70 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMax80=`echo "scale=4; x = $totalMax80 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMax90=`echo "scale=4; x = $totalMax90 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMax100=`echo "scale=4; x = $totalMax100 / $NUM_LOOP; if(x<1) print 0; x" | bc` + + avgMin10=`echo "scale=4; x = $totalMin10 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMin20=`echo "scale=4; x = $totalMin20 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMin30=`echo "scale=4; x = $totalMin30 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMin40=`echo "scale=4; x = $totalMin40 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMin50=`echo "scale=4; x = $totalMin50 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMin60=`echo "scale=4; x = $totalMin60 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMin70=`echo "scale=4; x = $totalMin70 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMin80=`echo "scale=4; x = $totalMin80 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMin90=`echo "scale=4; x = $totalMin90 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMin100=`echo "scale=4; x = $totalMin100 / $NUM_LOOP; if(x<1) print 0; x" | bc` + + echo "Latency, 10%, 20%, 30%, 40%, 50%, 60%, 70%, 80%, 90%, 100%" + echo "Count, $avgCount10, $avgCount20, $avgCount30, $avgCount40, $avgCount50, $avgCount60, $avgCount70, $avgCount80, $avgCount90, $avgCount100" + echo "Avg, $avgAvg10, $avgAvg20, $avgAvg30, $avgAvg40, $avgAvg50, $avgAvg60, $avgAvg70, $avgAvg80, $avgAvg90, $avgAvg100" + echo "Sum, $avgSum10, $avgSum20, $avgSum30, $avgSum40, $avgSum50, $avgSum60, $avgSum70, $avgSum80, $avgSum90, $avgSum100" + echo "Max, $avgMax10, $avgMax20, $avgMax30, $avgMax40, $avgMax50, $avgMax60, $avgMax70, $avgMax80, $avgMax90, $avgMax100" + echo "Min, $avgMin10, $avgMin20, $avgMin30, $avgMin40, $avgMin50, $avgMin60, $avgMin70, $avgMin80, $avgMin90, $avgMin100" +} + +################ Main ################ + +verbose=false + +for arg in "$@" +do + case $arg in + -v) + verbose=true + shift ;; + + -c) + clients=$2 + shift 2;; + + -n) + NUM_LOOP=$2 + shift 2;; + + *) + ;; + esac +done + +WORK_DIR=/mnt/root/TDengine +TSDBTEST_DIR=$WORK_DIR/tests/comparisonTest/opentsdb + +runTest + +printTo "Test done!" diff --git a/tests/perftest-scripts/opentsdbTestQ3Loop.sh b/tests/perftest-scripts/opentsdbTestQ3Loop.sh new file mode 100755 index 0000000000..6f69cf85a0 --- /dev/null +++ b/tests/perftest-scripts/opentsdbTestQ3Loop.sh @@ -0,0 +1,96 @@ +#!/bin/bash + +NUM_LOOP=5 + +function printTo { + if $verbose ; then + echo $1 + fi +} + +TSDBTESTQ3OUT=opentsdbTestQ3.out + +function runTest { + totalG10=0 + totalG20=0 + totalG30=0 + totalG40=0 + totalG50=0 + totalG60=0 + totalG70=0 + totalG80=0 + totalG90=0 + totalG100=0 + for i in `seq 1 $NUM_LOOP`; do + printTo "loop i:$i, java -jar \ + $TSDBTEST_DIR/opentsdbtest/target/opentsdbtest-1.0-SNAPSHOT-jar-with-dependencies.jar \ + -sql q3" + java -jar \ + $TSDBTEST_DIR/opentsdbtest/target/opentsdbtest-1.0-SNAPSHOT-jar-with-dependencies.jar \ + -sql q3 2>&1 \ + | tee $TSDBTESTQ3OUT + G10=`grep -w "devgroup < 10" $TSDBTESTQ3OUT| awk '{print $2}'` + totalG10=`echo "scale=4; $totalG10 + $G10" | bc` + G20=`grep -w "devgroup < 20" $TSDBTESTQ3OUT| awk '{print $2}'` + totalG20=`echo "scale=4; $totalG20 + $G20" | bc` + G30=`grep -w "devgroup < 30" $TSDBTESTQ3OUT| awk '{print $2}'` + totalG30=`echo "scale=4; $totalG30 + $G30" | bc` + G40=`grep -w "devgroup < 40" $TSDBTESTQ3OUT| awk '{print $2}'` + totalG40=`echo "scale=4; $totalG40 + $G40" | bc` + G50=`grep -w "devgroup < 50" $TSDBTESTQ3OUT| awk '{print $2}'` + totalG50=`echo "scale=4; $totalG50 + $G50" | bc` + G60=`grep -w "devgroup < 60" $TSDBTESTQ3OUT| awk '{print $2}'` + totalG60=`echo "scale=4; $totalG60 + $G60" | bc` + G70=`grep -w "devgroup < 70" $TSDBTESTQ3OUT| awk '{print $2}'` + totalG70=`echo "scale=4; $totalG70 + $G70" | bc` + G80=`grep -w "devgroup < 80" $TSDBTESTQ3OUT| awk '{print $2}'` + totalG80=`echo "scale=4; $totalG80 + $G80" | bc` + G90=`grep -w "devgroup < 90" $TSDBTESTQ3OUT| awk '{print $2}'` + totalG90=`echo "scale=4; $totalG90 + $G90" | bc` + G100=`grep -w "devgroup < 100" $TSDBTESTQ3OUT| awk '{print $2}'` + totalG100=`echo "scale=4; $totalG100 + $G100" | bc` + done + avgG10=`echo "scale=4; x = $totalG10 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG20=`echo "scale=4; x = $totalG20 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG30=`echo "scale=4; x = $totalG30 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG40=`echo "scale=4; x = $totalG40 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG50=`echo "scale=4; x = $totalG50 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG60=`echo "scale=4; x = $totalG60 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG70=`echo "scale=4; x = $totalG70 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG80=`echo "scale=4; x = $totalG80 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG90=`echo "scale=4; x = $totalG90 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG100=`echo "scale=4; x = $totalG100 / $NUM_LOOP; if(x<1) print 0; x" | bc` + echo "Latency, 10%, 20%, 30%, 40%, 50%, 60%, 70%, 80%, 90%, 100%" + echo "OpenTSDB, $avgG10, $avgG20, $avgG30, $avgG40, $avgG50, $avgG60, $avgG70, $avgG80, $avgG90, $avgG100" +} + +################ Main ################ + +verbose=false + +for arg in "$@" +do + case $arg in + -v) + verbose=true + shift ;; + + -c) + clients=$2 + shift 2;; + + -n) + NUM_LOOP=$2 + shift 2;; + + *) + ;; + esac +done + +WORK_DIR=/mnt/root/TDengine +TSDBTEST_DIR=$WORK_DIR/tests/comparisonTest/opentsdb + +runTest + +printTo "Test done!" diff --git a/tests/perftest-scripts/opentsdbTestQ4Loop.sh b/tests/perftest-scripts/opentsdbTestQ4Loop.sh new file mode 100755 index 0000000000..4341f2165c --- /dev/null +++ b/tests/perftest-scripts/opentsdbTestQ4Loop.sh @@ -0,0 +1,103 @@ +#!/bin/bash + +DATA_DIR=/mnt/root/testdata +NUM_LOOP=5 + +function printTo { + if $verbose ; then + echo $1 + fi +} + +TSDBTESTQ4OUT=opentsdbTestQ4.out + +function runTest { + totalG10=0 + totalG20=0 + totalG30=0 + totalG40=0 + totalG50=0 + totalG60=0 + totalG70=0 + totalG80=0 + totalG90=0 + totalG100=0 + for i in `seq 1 $NUM_LOOP`; do + printTo "loop i:$i, java -jar \ + $TSDBTEST_DIR/opentsdbtest/target/opentsdbtest-1.0-SNAPSHOT-jar-with-dependencies.jar \ + -sql q4" + + java -jar \ + $TSDBTEST_DIR/opentsdbtest/target/opentsdbtest-1.0-SNAPSHOT-jar-with-dependencies.jar \ + -sql q4 2>&1 | + tee $TSDBTESTQ4OUT + G10=`grep -w "devgroup < 10" $TSDBTESTQ4OUT| awk '{print $2}'` + totalG10=`echo "scale=4; $totalG10 + $G10" | bc` + G20=`grep -w "devgroup < 20" $TSDBTESTQ4OUT| awk '{print $2}'` + totalG20=`echo "scale=4; $totalG20 + $G20" | bc` + G30=`grep -w "devgroup < 30" $TSDBTESTQ4OUT| awk '{print $2}'` + totalG30=`echo "scale=4; $totalG30 + $G30" | bc` + G40=`grep -w "devgroup < 40" $TSDBTESTQ4OUT| awk '{print $2}'` + totalG40=`echo "scale=4; $totalG40 + $G40" | bc` + G50=`grep -w "devgroup < 50" $TSDBTESTQ4OUT| awk '{print $2}'` + totalG50=`echo "scale=4; $totalG50 + $G50" | bc` + G60=`grep -w "devgroup < 60" $TSDBTESTQ4OUT| awk '{print $2}'` + totalG60=`echo "scale=4; $totalG60 + $G60" | bc` + G70=`grep -w "devgroup < 70" $TSDBTESTQ4OUT| awk '{print $2}'` + totalG70=`echo "scale=4; $totalG70 + $G70" | bc` + G80=`grep -w "devgroup < 80" $TSDBTESTQ4OUT| awk '{print $2}'` + totalG80=`echo "scale=4; $totalG80 + $G80" | bc` + G90=`grep -w "devgroup < 90" $TSDBTESTQ4OUT| awk '{print $2}'` + totalG90=`echo "scale=4; $totalG90 + $G90" | bc` + G100=`grep -w "devgroup < 100" $TSDBTESTQ4OUT| awk '{print $2}'` + totalG100=`echo "scale=4; $totalG100 + $G100" | bc` + done + avgG10=`echo "scale=4; x = $totalG10 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG20=`echo "scale=4; x = $totalG20 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG30=`echo "scale=4; x = $totalG30 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG40=`echo "scale=4; x = $totalG40 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG50=`echo "scale=4; x = $totalG50 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG60=`echo "scale=4; x = $totalG60 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG70=`echo "scale=4; x = $totalG70 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG80=`echo "scale=4; x = $totalG80 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG90=`echo "scale=4; x = $totalG90 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG100=`echo "scale=4; x = $totalG100 / $NUM_LOOP; if(x<1) print 0; x" | bc` + echo "Latency, 10%, 20%, 30%, 40%, 50%, 60%, 70%, 80%, 90%, 100%" + echo "OpenTSDB, $avgG10, $avgG20, $avgG30, $avgG40, $avgG50, $avgG60, $avgG70, $avgG80, $avgG90, $avgG100" +} + +################ Main ################ + +verbose=false +regeneratedata=false + +for arg in "$@" +do + case $arg in + -v) + verbose=true + shift ;; + + -c) + clients=$2 + shift 2;; + + -r) + regeneratedata=true + ;; + + -n) + NUM_LOOP=$2 + shift 2;; + + *) + ;; + esac +done + +WORK_DIR=/mnt/root/TDengine +TSDBTEST_DIR=$WORK_DIR/tests/comparisonTest/opentsdb + +runTest + +printTo "Test done!" From 824158587034416837cb60aa1b79315b86ea85dd Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sun, 9 Aug 2020 09:45:07 +0000 Subject: [PATCH 090/109] fix influxdbTest timeout error. [TD-1051] --- tests/comparisonTest/influxdb/main.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/comparisonTest/influxdb/main.go b/tests/comparisonTest/influxdb/main.go index a6550013e3..2fb16fad89 100644 --- a/tests/comparisonTest/influxdb/main.go +++ b/tests/comparisonTest/influxdb/main.go @@ -110,6 +110,7 @@ func writeDataImp(wInfo *WriteInfo, wg *sync.WaitGroup, arguments *ProArgs) { Addr: arguments.host, Username: arguments.username, Password: arguments.password, + Timeout: 300 * time.Second, }) if err != nil { @@ -220,6 +221,7 @@ func readData(arguments *ProArgs) { Addr: arguments.host, Username: arguments.username, Password: arguments.password, + Timeout: 300 * time.Second, }) if err != nil { From 6cfbaefea7656764fc1650b752373d781539dbb3 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sun, 9 Aug 2020 09:48:24 +0000 Subject: [PATCH 091/109] add influxdb write loop test script. --- .../perftest-scripts/influxdbTestWriteLoop.sh | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100755 tests/perftest-scripts/influxdbTestWriteLoop.sh diff --git a/tests/perftest-scripts/influxdbTestWriteLoop.sh b/tests/perftest-scripts/influxdbTestWriteLoop.sh new file mode 100755 index 0000000000..a9b10ac45d --- /dev/null +++ b/tests/perftest-scripts/influxdbTestWriteLoop.sh @@ -0,0 +1,84 @@ +#!/bin/bash + +DATA_DIR=/mnt/root/testdata +NUM_LOOP=1 +NUM_OF_FILES=100 + +rowsPerRequest=(1 100 500 1000 2000) + +function printTo { + if $verbose ; then + echo $1 + fi +} + +function runTest { + printf "R/R, " + for c in `seq 1 $clients`; do + if [ "$c" == "1" ]; then + printf "$c client, " + else + printf "$c clients, " + fi + done + printf "\n" + + for r in ${rowsPerRequest[@]}; do + printf "$r, " + for c in `seq 1 $clients`; do + totalRPR=0 + OUTPUT_FILE=influxdbTestWrite-RPR$r-clients$c.out + for i in `seq 1 $NUM_LOOP`; do + printTo "loop i:$i, $INF_TEST_DIR/influxdbTest \ + -dataDir $DATA_DIR \ + -numOfFiles $NUM_OF_FILES \ + -writeClients $c \ + -rowsPerRequest $r" + $INF_TEST_DIR/influxdbTest \ + -dataDir $DATA_DIR \ + -numOfFiles $NUM_OF_FILES \ + -writeClients $c \ + -rowsPerRequest $r 2>&1 \ + | tee $OUTPUT_FILE + RPR=`cat $OUTPUT_FILE | grep speed | awk '{print $(NF-1)}'` + totalRPR=`echo "scale=4; $totalRPR + $RPR" | bc` + printTo "rows:$r, clients:$c, i:$i RPR:$RPR" + done + avgRPR=`echo "scale=4; $totalRPR / $NUM_LOOP" | bc` + printf "$avgRPR, " + done + printf "\n" + done +} + +################ Main ################ + +verbose=false +clients=1 + +while : ; do + case $1 in + -v) + verbose=true + shift ;; + + -n) + NUM_LOOP=$2 + shift 2;; + + -c) + clients=$2 + shift 2;; + + *) + break ;; + esac +done + +WORK_DIR=/mnt/root/TDengine + +INF_TEST_DIR=$WORK_DIR/tests/comparisonTest/influxdb + +runTest + +echo "Test done!" From ceab359b914840df830f72832af9456c550ba69c Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 9 Aug 2020 20:59:23 +0800 Subject: [PATCH 092/109] TD-1057 --- src/os/src/windows/w64Env.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/os/src/windows/w64Env.c b/src/os/src/windows/w64Env.c index c6046ecae7..57d34d4003 100644 --- a/src/os/src/windows/w64Env.c +++ b/src/os/src/windows/w64Env.c @@ -22,7 +22,7 @@ extern void taosWinSocketInit(); void osInit() { if (configDir[0] == 0) { - strcpy(configDir, "~/TDengine/cfg"); + strcpy(configDir, "C:/TDengine/cfg"); } strcpy(tsVnodeDir, "C:/TDengine/data"); From a9cabb54fa67d8081780f216f33c969a0c933b05 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 9 Aug 2020 22:59:42 +0800 Subject: [PATCH 093/109] TD-1057 --- src/client/src/tscSql.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 3e5280401f..6b3653ff63 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -256,6 +256,7 @@ TAOS_RES* taos_query(TAOS *taos, const char *sqlstr) { return NULL; } + tsem_init(&pSql->rspSem, 0, 0); doAsyncQuery(pObj, pSql, waitForQueryRsp, taos, sqlstr, sqlLen); // wait for the callback function to post the semaphore From 41d97eeea7257ce966d5e9c0a69e19b52f1afcd2 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 9 Aug 2020 15:21:03 +0000 Subject: [PATCH 094/109] TD-1057 --- src/client/src/tscSql.c | 14 +++++++------- src/client/src/tscSub.c | 8 ++++---- src/client/src/tscSubquery.c | 4 ++-- src/client/src/tscUtil.c | 2 +- src/dnode/src/dnodeSystem.c | 8 ++++---- src/dnode/src/dnodeTelemetry.c | 8 ++++---- src/kit/taosdemo/taosdemo.c | 30 +++++++++++++++--------------- src/query/inc/qExecutor.h | 2 +- src/rpc/test/rclient.c | 10 +++++----- src/rpc/test/rsclient.c | 6 +++--- src/sync/src/tarbitrator.c | 8 ++++---- src/sync/test/syncClient.c | 10 +++++----- 12 files changed, 55 insertions(+), 55 deletions(-) diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 6b3653ff63..7bb9be5d5c 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -141,7 +141,7 @@ static void syncConnCallback(void *param, TAOS_RES *tres, int code) { SSqlObj *pSql = (SSqlObj *) tres; assert(pSql != NULL); - sem_post(&pSql->rspSem); + tsem_post(&pSql->rspSem); } TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port) { @@ -156,7 +156,7 @@ TAOS *taos_connect(const char *ip, const char *user, const char *pass, const cha pSql->param = pSql; tscProcessSql(pSql); - sem_wait(&pSql->rspSem); + tsem_wait(&pSql->rspSem); if (pSql->res.code != TSDB_CODE_SUCCESS) { terrno = pSql->res.code; @@ -225,12 +225,12 @@ void waitForQueryRsp(void *param, TAOS_RES *tres, int code) { assert(tres != NULL); SSqlObj *pSql = (SSqlObj *) tres; - sem_post(&pSql->rspSem); + tsem_post(&pSql->rspSem); } static void waitForRetrieveRsp(void *param, TAOS_RES *tres, int numOfRows) { SSqlObj* pSql = (SSqlObj*) tres; - sem_post(&pSql->rspSem); + tsem_post(&pSql->rspSem); } TAOS_RES* taos_query(TAOS *taos, const char *sqlstr) { @@ -439,7 +439,7 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { pCmd->command == TSDB_SQL_CLI_VERSION || pCmd->command == TSDB_SQL_CURRENT_USER )) { taos_fetch_rows_a(res, waitForRetrieveRsp, pSql->pTscObj); - sem_wait(&pSql->rspSem); + tsem_wait(&pSql->rspSem); } return doSetResultRowData(pSql, true); @@ -729,7 +729,7 @@ static void asyncCallback(void *param, TAOS_RES *tres, int code) { assert(param != NULL); SSqlObj *pSql = ((SSqlObj *)param); pSql->res.code = code; - sem_post(&pSql->rspSem); + tsem_post(&pSql->rspSem); } int taos_validate_sql(TAOS *taos, const char *sql) { @@ -780,7 +780,7 @@ int taos_validate_sql(TAOS *taos, const char *sql) { pSql->param = pSql; int code = tsParseSql(pSql, true); if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - sem_wait(&pSql->rspSem); + tsem_wait(&pSql->rspSem); code = pSql->res.code; } if (code != TSDB_CODE_SUCCESS) { diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c index e9f2c1dc1d..608551c7f3 100644 --- a/src/client/src/tscSub.c +++ b/src/client/src/tscSub.c @@ -33,7 +33,7 @@ typedef struct SSubscriptionProgress { typedef struct SSub { void * signature; char topic[32]; - sem_t sem; + tsem_t sem; int64_t lastSyncTime; int64_t lastConsumeTime; TAOS * taos; @@ -85,7 +85,7 @@ static void asyncCallback(void *param, TAOS_RES *tres, int code) { assert(param != NULL); SSub *pSub = ((SSub *)param); pSub->pSql->res.code = code; - sem_post(&pSub->sem); + tsem_post(&pSub->sem); } @@ -154,7 +154,7 @@ static SSub* tscCreateSubscription(STscObj* pObj, const char* topic, const char* code = tsParseSql(pSql, false); if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - sem_wait(&pSub->sem); + tsem_wait(&pSub->sem); code = pSql->res.code; } if (code != TSDB_CODE_SUCCESS) { @@ -451,7 +451,7 @@ TAOS_RES *taos_consume(TAOS_SUB *tsub) { pSql->fetchFp = asyncCallback; pSql->param = pSub; tscDoQuery(pSql); - sem_wait(&pSub->sem); + tsem_wait(&pSub->sem); if (pRes->code != TSDB_CODE_SUCCESS) { continue; diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 8a596d8893..4e188d4fb6 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -2057,7 +2057,7 @@ void tscBuildResFromSubqueries(SSqlObj *pSql) { } doBuildResFromSubqueries(pSql); - sem_post(&pSql->rspSem); + tsem_post(&pSql->rspSem); return; @@ -2083,7 +2083,7 @@ void tscBuildResFromSubqueries(SSqlObj *pSql) { // free(pState); // // pRes->completed = true; // set query completed -// sem_post(&pSql->rspSem); +// tsem_post(&pSql->rspSem); // return; // } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 1b6d18be0c..582411fc0c 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -387,7 +387,7 @@ void tscFreeSqlObj(SSqlObj* pSql) { pCmd->allocSize = 0; taosTFree(pSql->sqlstr); - sem_destroy(&pSql->rspSem); + tsem_destroy(&pSql->rspSem); free(pSql); } diff --git a/src/dnode/src/dnodeSystem.c b/src/dnode/src/dnodeSystem.c index 2519684878..543e1c9639 100644 --- a/src/dnode/src/dnodeSystem.c +++ b/src/dnode/src/dnodeSystem.c @@ -22,7 +22,7 @@ #include "dnodeMain.h" static void signal_handler(int32_t signum, siginfo_t *sigInfo, void *context); -static sem_t exitSem; +static tsem_t exitSem; int32_t main(int32_t argc, char *argv[]) { // Set global configuration file @@ -88,7 +88,7 @@ int32_t main(int32_t argc, char *argv[]) { #endif } - if (sem_init(&exitSem, 0, 0) != 0) { + if (tsem_init(&exitSem, 0, 0) != 0) { printf("failed to create exit semphore\n"); exit(EXIT_FAILURE); } @@ -117,7 +117,7 @@ int32_t main(int32_t argc, char *argv[]) { syslog(LOG_INFO, "Started TDengine service successfully."); - for (int res = sem_wait(&exitSem); res != 0; res = sem_wait(&exitSem)) { + for (int res = tsem_wait(&exitSem); res != 0; res = tsem_wait(&exitSem)) { if (res != EINTR) { syslog(LOG_ERR, "failed to wait exit semphore: %d", res); break; @@ -157,5 +157,5 @@ static void signal_handler(int32_t signum, siginfo_t *sigInfo, void *context) { sigaction(SIGUSR2, &act, NULL); // inform main thread to exit - sem_post(&exitSem); + tsem_post(&exitSem); } diff --git a/src/dnode/src/dnodeTelemetry.c b/src/dnode/src/dnodeTelemetry.c index 892fd1d903..8ed4a9518b 100644 --- a/src/dnode/src/dnodeTelemetry.c +++ b/src/dnode/src/dnodeTelemetry.c @@ -36,7 +36,7 @@ #include "dnodeInt.h" #include "dnodeTelemetry.h" -static sem_t tsExitSem; +static tsem_t tsExitSem; static pthread_t tsTelemetryThread; #define TELEMETRY_SERVER "telemetry.taosdata.com" @@ -266,7 +266,7 @@ int32_t dnodeInitTelemetry() { return 0; } - if (sem_init(&tsExitSem, 0, 0) == -1) { + if (tsem_init(&tsExitSem, 0, 0) == -1) { // just log the error, it is ok for telemetry to fail dTrace("failed to create semaphore for telemetry, reason:%s", strerror(errno)); return 0; @@ -291,8 +291,8 @@ void dnodeCleanupTelemetry() { } if (tsTelemetryThread) { - sem_post(&tsExitSem); + tsem_post(&tsExitSem); pthread_join(tsTelemetryThread, NULL); - sem_destroy(&tsExitSem); + tsem_destroy(&tsExitSem); } } diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 9d46ac5055..859e22a178 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -440,9 +440,9 @@ typedef struct { char* cols; bool use_metric; - sem_t mutex_sem; + tsem_t mutex_sem; int notFinished; - sem_t lock_sem; + tsem_t lock_sem; } info; typedef struct { @@ -459,9 +459,9 @@ typedef struct { int data_of_order; int data_of_rate; - sem_t *mutex_sem; - int *notFinished; - sem_t *lock_sem; + tsem_t *mutex_sem; + int *notFinished; + tsem_t *lock_sem; } sTable; /* ******************************* Global @@ -729,9 +729,9 @@ int main(int argc, char *argv[]) { t_info->end_table_id = i < b ? last + a : last + a - 1; last = t_info->end_table_id + 1; - sem_init(&(t_info->mutex_sem), 0, 1); + tsem_init(&(t_info->mutex_sem), 0, 1); t_info->notFinished = t_info->end_table_id - t_info->start_table_id + 1; - sem_init(&(t_info->lock_sem), 0, 0); + tsem_init(&(t_info->lock_sem), 0, 0); if (query_mode == SYNC) { pthread_create(pids + i, NULL, syncWrite, t_info); @@ -762,8 +762,8 @@ int main(int argc, char *argv[]) { for (int i = 0; i < threads; i++) { info *t_info = infos + i; taos_close(t_info->taos); - sem_destroy(&(t_info->mutex_sem)); - sem_destroy(&(t_info->lock_sem)); + tsem_destroy(&(t_info->mutex_sem)); + tsem_destroy(&(t_info->lock_sem)); } free(pids); @@ -1021,8 +1021,8 @@ void multiThreadCreateTable(char* cols, bool use_metric, int threads, int ntable for (int i = 0; i < threads; i++) { info *t_info = infos + i; - sem_destroy(&(t_info->mutex_sem)); - sem_destroy(&(t_info->lock_sem)); + tsem_destroy(&(t_info->mutex_sem)); + tsem_destroy(&(t_info->lock_sem)); } free(pids); @@ -1272,7 +1272,7 @@ void *asyncWrite(void *sarg) { taos_query_a(winfo->taos, "show databases", callBack, tb_info); } - sem_wait(&(winfo->lock_sem)); + tsem_wait(&(winfo->lock_sem)); free(tb_infos); return NULL; @@ -1292,10 +1292,10 @@ void callBack(void *param, TAOS_RES *res, int code) { // If finished; if (tb_info->counter >= tb_info->target) { - sem_wait(tb_info->mutex_sem); + tsem_wait(tb_info->mutex_sem); (*(tb_info->notFinished))--; - if (*(tb_info->notFinished) == 0) sem_post(tb_info->lock_sem); - sem_post(tb_info->mutex_sem); + if (*(tb_info->notFinished) == 0) tsem_post(tb_info->lock_sem); + tsem_post(tb_info->mutex_sem); return; } diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index bd2e0a4470..9757036783 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -186,7 +186,7 @@ typedef struct SQInfo { void* signature; int32_t pointsInterpo; int32_t code; // error code to returned to client -// sem_t dataReady; +//tsem_t dataReady; void* tsdb; int32_t vgId; diff --git a/src/rpc/test/rclient.c b/src/rpc/test/rclient.c index 6ec2d82445..7a963e9ce4 100644 --- a/src/rpc/test/rclient.c +++ b/src/rpc/test/rclient.c @@ -26,8 +26,8 @@ typedef struct { int num; int numOfReqs; int msgSize; - sem_t rspSem; - sem_t *pOverSem; + tsem_t rspSem; + tsem_t *pOverSem; pthread_t thread; void *pRpc; } SInfo; @@ -39,7 +39,7 @@ static void processResponse(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { if (pEpSet) pInfo->epSet = *pEpSet; rpcFreeCont(pMsg->pCont); - sem_post(&pInfo->rspSem); + tsem_post(&pInfo->rspSem); } static int tcount = 0; @@ -60,7 +60,7 @@ static void *sendRequest(void *param) { rpcSendRequest(pInfo->pRpc, &pInfo->epSet, &rpcMsg); if ( pInfo->num % 20000 == 0 ) tInfo("thread:%d, %d requests have been sent", pInfo->index, pInfo->num); - sem_wait(&pInfo->rspSem); + tsem_wait(&pInfo->rspSem); } tDebug("thread:%d, it is over", pInfo->index); @@ -171,7 +171,7 @@ int main(int argc, char *argv[]) { pInfo->epSet = epSet; pInfo->numOfReqs = numOfReqs; pInfo->msgSize = msgSize; - sem_init(&pInfo->rspSem, 0, 0); + tsem_init(&pInfo->rspSem, 0, 0); pInfo->pRpc = pRpc; pthread_create(&pInfo->thread, &thattr, sendRequest, pInfo); pInfo++; diff --git a/src/rpc/test/rsclient.c b/src/rpc/test/rsclient.c index 6e6961784b..a152d8e4a5 100644 --- a/src/rpc/test/rsclient.c +++ b/src/rpc/test/rsclient.c @@ -27,8 +27,8 @@ typedef struct { int num; int numOfReqs; int msgSize; - sem_t rspSem; - sem_t *pOverSem; + tsem_t rspSem; + tsem_t *pOverSem; pthread_t thread; void *pRpc; } SInfo; @@ -171,7 +171,7 @@ int main(int argc, char *argv[]) { pInfo->epSet = epSet; pInfo->numOfReqs = numOfReqs; pInfo->msgSize = msgSize; - sem_init(&pInfo->rspSem, 0, 0); + tsem_init(&pInfo->rspSem, 0, 0); pInfo->pRpc = pRpc; pthread_create(&pInfo->thread, &thattr, sendRequest, pInfo); pInfo++; diff --git a/src/sync/src/tarbitrator.c b/src/sync/src/tarbitrator.c index 3c6db88a9c..3538391a94 100644 --- a/src/sync/src/tarbitrator.c +++ b/src/sync/src/tarbitrator.c @@ -31,7 +31,7 @@ static void arbSignalHandler(int32_t signum, siginfo_t *sigInfo, void *context); static void arbProcessIncommingConnection(int connFd, uint32_t sourceIp); static void arbProcessBrokenLink(void *param); static int arbProcessPeerMsg(void *param, void *buffer); -static sem_t tsArbSem; +static tsem_t tsArbSem; static ttpool_h tsArbTcpPool; typedef struct { @@ -61,7 +61,7 @@ int main(int argc, char *argv[]) { } } - if (sem_init(&tsArbSem, 0, 0) != 0) { + if (tsem_init(&tsArbSem, 0, 0) != 0) { printf("failed to create exit semphore\n"); exit(EXIT_FAILURE); } @@ -98,7 +98,7 @@ int main(int argc, char *argv[]) { sInfo("TAOS arbitrator: %s:%d is running", tsNodeFqdn, tsServerPort); - for (int res = sem_wait(&tsArbSem); res != 0; res = sem_wait(&tsArbSem)) { + for (int res = tsem_wait(&tsArbSem); res != 0; res = tsem_wait(&tsArbSem)) { if (res != EINTR) break; } @@ -185,6 +185,6 @@ static void arbSignalHandler(int32_t signum, siginfo_t *sigInfo, void *context) sInfo("shut down signal is %d, sender PID:%d", signum, sigInfo->si_pid); // inform main thread to exit - sem_post(&tsArbSem); + tsem_post(&tsArbSem); } diff --git a/src/sync/test/syncClient.c b/src/sync/test/syncClient.c index cd873b758b..16053d1088 100644 --- a/src/sync/test/syncClient.c +++ b/src/sync/test/syncClient.c @@ -25,8 +25,8 @@ typedef struct { int num; int numOfReqs; int msgSize; - sem_t rspSem; - sem_t *pOverSem; + tsem_t rspSem; + tsem_t *pOverSem; pthread_t thread; void *pRpc; } SInfo; @@ -38,7 +38,7 @@ void processResponse(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { if (pEpSet) pInfo->epSet = *pEpSet; rpcFreeCont(pMsg->pCont); - sem_post(&pInfo->rspSem); + tsem_post(&pInfo->rspSem); } int tcount = 0; @@ -59,7 +59,7 @@ void *sendRequest(void *param) { rpcSendRequest(pInfo->pRpc, &pInfo->epSet, &rpcMsg); if ( pInfo->num % 20000 == 0 ) uInfo("thread:%d, %d requests have been sent", pInfo->index, pInfo->num); - sem_wait(&pInfo->rspSem); + tsem_wait(&pInfo->rspSem); } uDebug("thread:%d, it is over", pInfo->index); @@ -169,7 +169,7 @@ int main(int argc, char *argv[]) { pInfo->epSet = epSet; pInfo->numOfReqs = numOfReqs; pInfo->msgSize = msgSize; - sem_init(&pInfo->rspSem, 0, 0); + tsem_init(&pInfo->rspSem, 0, 0); pInfo->pRpc = pRpc; pthread_create(&pInfo->thread, &thattr, sendRequest, pInfo); pInfo++; From 9ccdaec95d89d887e0ea35b04696ffa48a436bb7 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Sun, 9 Aug 2020 23:25:43 +0800 Subject: [PATCH 095/109] fix typo in taos sql md file --- documentation20/webdocs/markdowndocs/TAOS SQL-ch.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation20/webdocs/markdowndocs/TAOS SQL-ch.md b/documentation20/webdocs/markdowndocs/TAOS SQL-ch.md index 3cf811ab66..6c6a3afb60 100644 --- a/documentation20/webdocs/markdowndocs/TAOS SQL-ch.md +++ b/documentation20/webdocs/markdowndocs/TAOS SQL-ch.md @@ -157,7 +157,7 @@ TDengine缺çœçš„时间戳是毫秒精度，但通过修改é…ç½®å‚æ•°enableMic ```mysql DROP TABLE [IF EXISTS] stb_name; ``` - 删除STable会自动删除通过STable创建的字表。 + 删除STable会自动删除通过STable创建的å­è¡¨ã€‚ - **显示当å‰æ•°æ®åº“下的所有超级表信æ¯** @@ -206,7 +206,7 @@ TDengine缺çœçš„时间戳是毫秒精度，但通过修改é…ç½®å‚æ•°enableMic ``` 修改超级表的标签å，从超级表修改æŸä¸ªæ ‡ç­¾ååŽï¼Œè¯¥è¶…级表下的所有å­è¡¨ä¹Ÿä¼šè‡ªåŠ¨æ›´æ–°è¯¥æ ‡ç­¾å。 -- **修改字表标签值** +- **修改å­è¡¨æ ‡ç­¾å€¼** ```mysql ALTER TABLE tb_name SET TAG tag_name=new_tag_value; From 77cfb02ea0dcf9380ed0cd239e9d1ebd96d600d4 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 10 Aug 2020 09:59:15 +0800 Subject: [PATCH 096/109] add file size statistics --- src/tsdb/src/tsdbFile.c | 4 ++++ src/tsdb/src/tsdbRWHelper.c | 8 -------- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index b913070e8a..b8173d41b3 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -409,6 +409,10 @@ static int tsdbInitFile(SFile *pFile, STsdbRepo *pRepo, int fid, int type) { pBuf = taosDecodeFixedU32(pBuf, &version); pBuf = tsdbDecodeSFileInfo(pBuf, &(pFile->info)); + if (pFile->info.size == TSDB_FILE_HEAD_SIZE) { + pFile->info.size = lseek(pFile->fd, 0, SEEK_END); + } + if (version != TSDB_FILE_VERSION) { tsdbError("vgId:%d file %s version %u is not the same as program version %u which may cause problem", REPO_ID(pRepo), pFile->fname, version, TSDB_FILE_VERSION); diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index d30a060c2b..5d0e9f16b2 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -120,14 +120,6 @@ int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup) { if (tsdbOpenFile(helperDataF(pHelper), O_RDWR) < 0) return -1; if (tsdbOpenFile(helperLastF(pHelper), O_RDWR) < 0) return -1; - // NOTE: For data file compatibility - if (helperDataF(pHelper)->info.size == TSDB_FILE_HEAD_SIZE) { - helperDataF(pHelper)->info.size = (uint64_t)lseek(helperDataF(pHelper)->fd, 0, SEEK_END); - } - if (helperLastF(pHelper)->info.size == TSDB_FILE_HEAD_SIZE) { - helperLastF(pHelper)->info.size = (uint64_t)lseek(helperLastF(pHelper)->fd, 0, SEEK_END); - } - // Create and open .h pFile = helperNewHeadF(pHelper); if (tsdbOpenFile(pFile, O_WRONLY | O_CREAT) < 0) return -1; From 1f66342c4b12cd121ca637939576534a8fd6f753 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 10 Aug 2020 11:12:16 +0800 Subject: [PATCH 097/109] remove part of assert --- src/tsdb/src/tsdbRWHelper.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index 5d0e9f16b2..1e082e39be 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -394,7 +394,7 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) { pFile->info.len += tsdbEncodeSCompIdx(&pBuf, &(pHelper->curCompIdx)); pFile->info.size += pIdx->len; - ASSERT(pFile->info.size == lseek(pFile->fd, 0, SEEK_CUR)); + // ASSERT(pFile->info.size == lseek(pFile->fd, 0, SEEK_CUR)); } return 0; @@ -436,7 +436,7 @@ int tsdbWriteCompIdx(SRWHelper *pHelper) { pFile->info.offset = offset; pFile->info.size += pFile->info.len; - ASSERT(pFile->info.size == lseek(pFile->fd, 0, SEEK_CUR)); + // ASSERT(pFile->info.size == lseek(pFile->fd, 0, SEEK_CUR)); return 0; } @@ -813,7 +813,7 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa pCompBlock->keyLast); pFile->info.size += pCompBlock->len; - ASSERT(pFile->info.size == lseek(pFile->fd, 0, SEEK_CUR)); + // ASSERT(pFile->info.size == lseek(pFile->fd, 0, SEEK_CUR)); return 0; From 2cee0797b428bf7e7192ec58066bed70f7105ae5 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 10 Aug 2020 15:56:41 +0800 Subject: [PATCH 098/109] [td-1101]add some logs --- src/tsdb/src/tsdbRead.c | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index ccc631fb58..2e9520c360 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -172,6 +172,7 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab if (pQueryHandle == NULL) { goto out_of_memory; } + pQueryHandle->order = pCond->order; pQueryHandle->window = pCond->twindow; pQueryHandle->pTsdb = tsdb; @@ -183,6 +184,7 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab pQueryHandle->qinfo = qinfo; pQueryHandle->outputCapacity = ((STsdbRepo*)tsdb)->config.maxRowsPerFileBlock; pQueryHandle->allocSize = 0; + pQueryHandle->locateStart = false; if (tsdbInitReadHelper(&pQueryHandle->rhelper, (STsdbRepo*) tsdb) != 0) { goto out_of_memory; @@ -193,6 +195,12 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab size_t sizeOfGroup = taosArrayGetSize(groupList->pGroupList); assert(sizeOfGroup >= 1 && pCond != NULL && pCond->numOfCols > 0); + if (ASCENDING_TRAVERSE(pCond->order)) { + assert(pQueryHandle->window.skey >= pQueryHandle->window.ekey); + } else { + assert(pQueryHandle->window.skey <= pQueryHandle->window.ekey); + } + // allocate buffer in order to load data blocks from file int32_t numOfCols = pCond->numOfCols; @@ -243,6 +251,8 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab info.pTableObj->type == TSDB_CHILD_TABLE || info.pTableObj->type == TSDB_STREAM_TABLE)); taosArrayPush(pQueryHandle->pTableCheckInfo, &info); + tsdbDebug("%p check table uid:%"PRId64", tid:%d from lastKey:%"PRId64" %p", pQueryHandle, info.tableId.uid, + info.tableId.tid, info.lastKey, qinfo); } } @@ -645,7 +655,7 @@ static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlo int16_t* colIds = pQueryHandle->defaultLoadColumn->pData; int32_t ret = tsdbLoadBlockDataCols(&(pQueryHandle->rhelper), pBlock, pCheckInfo->pCompInfo, colIds, QH_GET_NUM_OF_COLS(pQueryHandle)); - if (ret == TSDB_CODE_SUCCESS) { + if (ret == TSDB_CODE_SUCCESS) { SDataBlockLoadInfo* pBlockLoadInfo = &pQueryHandle->dataBlockLoadInfo; pBlockLoadInfo->fileGroup = pQueryHandle->pFileGroup; @@ -1071,6 +1081,14 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* TSKEY* tsArray = pCols->cols[0].pData; + if (ASCENDING_TRAVERSE(pQueryHandle->order)) { + TSKEY s = tsArray[cur->pos]; + assert(s >= pQueryHandle->window.skey && s <= pQueryHandle->window.ekey); + } else { + TSKEY s = tsArray[cur->pos]; + assert(s <= pQueryHandle->window.skey && s >= pQueryHandle->window.ekey); + } + // for search the endPos, so the order needs to reverse int32_t order = (pQueryHandle->order == TSDB_ORDER_ASC)? TSDB_ORDER_DESC:TSDB_ORDER_ASC; @@ -1550,7 +1568,7 @@ static int32_t getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle, bool* exists STableCheckInfo* pCheckInfo = pBlockInfo->pTableCheckInfo; // current block is done, try next - if (!cur->mixBlock || cur->blockCompleted) { + if ((!cur->mixBlock) || cur->blockCompleted) { if ((cur->slot == pQueryHandle->numOfBlocks - 1 && ASCENDING_TRAVERSE(pQueryHandle->order)) || (cur->slot == 0 && !ASCENDING_TRAVERSE(pQueryHandle->order))) { // all data blocks in current file has been checked already, try next file if exists @@ -1569,6 +1587,7 @@ static int32_t getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle, bool* exists return TSDB_CODE_SUCCESS; } } else { + tsdbDebug("%p continue in current data block, index:%d, %p", pQueryHandle, cur->slot, pQueryHandle->qinfo); handleDataMergeIfNeeded(pQueryHandle, pBlockInfo->compBlock, pCheckInfo); *exists = pQueryHandle->realNumOfRows > 0; From 8aad94d7eb8d192801aa45abbe86d15b83e9aa6a Mon Sep 17 00:00:00 2001 From: eurake Date: Mon, 10 Aug 2020 17:31:12 +0800 Subject: [PATCH 099/109] Update cluster-ch.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 删除多余英文字符 --- documentation20/webdocs/markdowndocs/cluster-ch.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation20/webdocs/markdowndocs/cluster-ch.md b/documentation20/webdocs/markdowndocs/cluster-ch.md index 2df6d2cb0e..afe0272387 100644 --- a/documentation20/webdocs/markdowndocs/cluster-ch.md +++ b/documentation20/webdocs/markdowndocs/cluster-ch.md @@ -107,7 +107,7 @@ CREATE DATABASE demo replica 3; ``` 一个DB里的数æ®ä¼šè¢«åˆ‡ç‰‡åˆ†åˆ°å¤šä¸ªvnode group，vnode group里的vnode数目就是DB的副本数，åŒä¸€ä¸ªvnode group里å„vnodeçš„æ•°æ®æ˜¯å®Œå…¨ä¸€è‡´çš„。为ä¿è¯é«˜å¯ç”¨æ€§ï¼Œvnode group里的vnode一定è¦åˆ†å¸ƒåœ¨ä¸åŒçš„dnode里（实际部署时，需è¦åœ¨ä¸åŒçš„物ç†æœºä¸Šï¼‰ï¼Œåªè¦ä¸€ä¸ªvgroup里超过åŠæ•°çš„vnode处于工作状æ€ï¼Œè¿™ä¸ªvgroup就能正常的对外æœåŠ¡ã€‚ -一个dnode里å¯èƒ½æœ‰å¤šä¸ªDBçš„æ•°æ®ï¼Œå› æ­¤ä¸€ä¸ªdnode离线时，å¯èƒ½ä¼šå½±å“到多个DB。如果一个vnode group里的一åŠæˆ–一åŠä»¥ä¸Šçš„vnodeä¸å·¥ä½œï¼Œé‚£ä¹ˆè¯¥vnode group就无法对外æœåŠ¡ï¼Œæ— æ³•æ’入或读å–æ•°æ®ï¼Œè¿™æ ·ä¼šå½±å“到它所属的DB的一部分表的d读写æ“作。 +一个dnode里å¯èƒ½æœ‰å¤šä¸ªDBçš„æ•°æ®ï¼Œå› æ­¤ä¸€ä¸ªdnode离线时，å¯èƒ½ä¼šå½±å“到多个DB。如果一个vnode group里的一åŠæˆ–一åŠä»¥ä¸Šçš„vnodeä¸å·¥ä½œï¼Œé‚£ä¹ˆè¯¥vnode group就无法对外æœåŠ¡ï¼Œæ— æ³•æ’入或读å–æ•°æ®ï¼Œè¿™æ ·ä¼šå½±å“到它所属的DB的一部分表的读写æ“作。 因为vnode的引入，无法简å•çš„给出结论：“集群中过åŠdnode工作，集群就应该工作â€ã€‚但是对于简å•çš„情形，很好下结论。比如副本数为3，åªæœ‰ä¸‰ä¸ªdnode，那如果仅有一个节点ä¸å·¥ä½œï¼Œæ•´ä¸ªé›†ç¾¤è¿˜æ˜¯å¯ä»¥æ­£å¸¸å·¥ä½œçš„，但如果有两个节点ä¸å·¥ä½œï¼Œé‚£æ•´ä¸ªé›†ç¾¤å°±æ— æ³•æ­£å¸¸å·¥ä½œäº†ã€‚ From e00d0728872db857a7b4eb6ce5c92ae3d1e503c3 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 10 Aug 2020 17:38:09 +0800 Subject: [PATCH 100/109] [td-1101] add time range check, fix bugs for projection query. --- src/query/src/qExecutor.c | 5 +++++ src/tsdb/src/tsdbRead.c | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 1277e7bfbb..a08a5476e3 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -1342,6 +1342,11 @@ static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBl if ((pQuery->limit.limit >= 0) && (pQuery->limit.limit + pQuery->limit.offset) <= numOfRes) { setQueryStatus(pQuery, QUERY_COMPLETED); } + + if (((pTableQInfo->lastKey > pTableQInfo->win.ekey) && QUERY_IS_ASC_QUERY(pQuery)) || + ((pTableQInfo->lastKey < pTableQInfo->win.ekey) && (!QUERY_IS_ASC_QUERY(pQuery)))) { + setQueryStatus(pQuery, QUERY_COMPLETED); + } } } diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 2e9520c360..d40084be23 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -196,9 +196,9 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab assert(sizeOfGroup >= 1 && pCond != NULL && pCond->numOfCols > 0); if (ASCENDING_TRAVERSE(pCond->order)) { - assert(pQueryHandle->window.skey >= pQueryHandle->window.ekey); - } else { assert(pQueryHandle->window.skey <= pQueryHandle->window.ekey); + } else { + assert(pQueryHandle->window.skey >= pQueryHandle->window.ekey); } // allocate buffer in order to load data blocks from file From dcfab65c6a4498d64ac3568a6ff370683a67ddcd Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 10 Aug 2020 18:10:06 +0800 Subject: [PATCH 101/109] [td-255] fix errors. --- src/tsdb/src/tsdbRead.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 0aae3b2ad9..07b50301d3 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -232,10 +232,7 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab } STsdbMeta* pMeta = tsdbGetMeta(tsdb); - assert(pMeta != NULL); - - size_t sizeOfGroup = taosArrayGetSize(groupList->pGroupList); - assert(sizeOfGroup >= 1 && pCond != NULL && pCond->numOfCols > 0); + assert(pMeta != NULL && sizeOfGroup >= 1 && pCond != NULL && pCond->numOfCols > 0); for (int32_t i = 0; i < sizeOfGroup; ++i) { SArray* group = *(SArray**) taosArrayGet(groupList->pGroupList, i); @@ -251,11 +248,13 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab .tableId = ((STable*)(pKeyInfo->pTable))->tableId, .pTableObj = pKeyInfo->pTable, }; - info.tableId = pTable->tableId; assert(info.pTableObj != NULL && (info.pTableObj->type == TSDB_NORMAL_TABLE || info.pTableObj->type == TSDB_CHILD_TABLE || info.pTableObj->type == TSDB_STREAM_TABLE)); + info.tableId.tid = info.pTableObj->tableId.tid; + info.tableId.uid = info.pTableObj->tableId.uid; + if (ASCENDING_TRAVERSE(pQueryHandle->order)) { assert(info.lastKey >= pQueryHandle->window.skey); } else { From 4bd41a0fbf2d1fb400169a7df1ba9fccdd39f9dd Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 10 Aug 2020 18:17:14 +0800 Subject: [PATCH 102/109] TD-1057 --- src/client/src/tscSub.c | 2 +- src/client/src/tscSubquery.c | 12 +++--- src/client/src/tscUtil.c | 2 +- src/os/inc/os.h | 1 + src/os/inc/osCommon.h | 31 +++++++++++++++ src/os/inc/osWindows.h | 3 ++ src/os/src/detail/osMemory.c | 8 ++-- src/plugins/http/src/httpContext.c | 2 +- src/plugins/http/src/httpSession.c | 2 +- src/query/src/qExecutor.c | 12 +++--- src/tsdb/src/tsdbRWHelper.c | 4 +- src/tsdb/src/tsdbRead.c | 8 ++-- src/util/src/tkvstore.c | 2 +- tests/script/test.bat | 60 ++++++++++++++++++++++++++++++ 14 files changed, 122 insertions(+), 27 deletions(-) create mode 100644 src/os/inc/osCommon.h create mode 100644 tests/script/test.bat diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c index 608551c7f3..5d8e601882 100644 --- a/src/client/src/tscSub.c +++ b/src/client/src/tscSub.c @@ -339,7 +339,7 @@ static int tscLoadSubscriptionProgress(SSub* pSub) { fclose(fp); taosArraySort(progress, tscCompareSubscriptionProgress); - tscDebug("subscription progress loaded, %zu tables: %s", taosArrayGetSize(progress), pSub->topic); + tscDebug("subscription progress loaded, %%" PRIzu " tables: %s", taosArrayGetSize(progress), pSub->topic); return 1; } diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 4e188d4fb6..0718338237 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -362,7 +362,7 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) { } size_t numOfCols = taosArrayGetSize(pNewQueryInfo->colList); - tscDebug("%p subquery:%p tableIndex:%d, vgroupIndex:%d, type:%d, exprInfo:%zu, colList:%zu, fieldsInfo:%d, name:%s", + tscDebug("%p subquery:%p tableIndex:%d, vgroupIndex:%d, type:%d, exprInfo:%" PRIzu ", colList:%" PRIzu ", fieldsInfo:%d, name:%s", pSql, pNew, 0, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type, taosArrayGetSize(pNewQueryInfo->exprList), numOfCols, pNewQueryInfo->fieldsInfo.numOfOutput, pTableMetaInfo->name); } @@ -522,7 +522,7 @@ static void issueTSCompQuery(SSqlObj* pSql, SJoinSupporter* pSupporter, SSqlObj* tscDebug( "%p subquery:%p tableIndex:%d, vgroupIndex:%d, numOfVgroups:%d, type:%d, ts_comp query to retrieve timestamps, " - "numOfExpr:%zu, colList:%zu, numOfOutputFields:%d, name:%s", + "numOfExpr:%" PRIzu ", colList:%" PRIzu ", numOfOutputFields:%d, name:%s", pParent, pSql, 0, pTableMetaInfo->vgroupIndex, pTableMetaInfo->vgroupList->numOfVgroups, pQueryInfo->type, tscSqlExprNumOfExprs(pQueryInfo), numOfCols, pQueryInfo->fieldsInfo.numOfOutput, pTableMetaInfo->name); @@ -1225,7 +1225,7 @@ int32_t tscLaunchJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter tscDebug( "%p subquery:%p tableIndex:%d, vgroupIndex:%d, type:%d, transfer to tid_tag query to retrieve (tableId, tags), " - "exprInfo:%zu, colList:%zu, fieldsInfo:%d, tagIndex:%d, name:%s", + "exprInfo:%" PRIzu ", colList:%" PRIzu ", fieldsInfo:%d, tagIndex:%d, name:%s", pSql, pNew, tableIndex, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type, tscSqlExprNumOfExprs(pNewQueryInfo), numOfCols, pNewQueryInfo->fieldsInfo.numOfOutput, index.columnIndex, pNewQueryInfo->pTableMetaInfo[0]->name); } else { @@ -1260,7 +1260,7 @@ int32_t tscLaunchJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter tscDebug( "%p subquery:%p tableIndex:%d, vgroupIndex:%d, type:%u, transfer to ts_comp query to retrieve timestamps, " - "exprInfo:%zu, colList:%zu, fieldsInfo:%d, name:%s", + "exprInfo:%" PRIzu ", colList:%" PRIzu ", fieldsInfo:%d, name:%s", pSql, pNew, tableIndex, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type, tscSqlExprNumOfExprs(pNewQueryInfo), numOfCols, pNewQueryInfo->fieldsInfo.numOfOutput, pNewQueryInfo->pTableMetaInfo[0]->name); } @@ -1915,7 +1915,7 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) { pSql->pSubs = calloc(size, POINTER_BYTES); pSql->numOfSubs = (uint16_t)size; - tscDebug("%p submit data to %zu vnode(s)", pSql, size); + tscDebug("%p submit data to %" PRIzu " vnode(s)", pSql, size); SSubqueryState *pState = calloc(1, sizeof(SSubqueryState)); pState->numOfTotal = pSql->numOfSubs; @@ -1949,7 +1949,7 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) { tscDebug("%p sub:%p create subObj success. orderOfSub:%d", pSql, pNew, numOfSub); numOfSub++; } else { - tscDebug("%p prepare submit data block failed in async insertion, vnodeIdx:%d, total:%zu, code:%s", pSql, numOfSub, + tscDebug("%p prepare submit data block failed in async insertion, vnodeIdx:%d, total:%" PRIzu ", code:%s", pSql, numOfSub, size, tstrerror(pRes->code)); goto _error; } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 582411fc0c..3cd91c5ad1 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1864,7 +1864,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void size_t size = taosArrayGetSize(pNewQueryInfo->colList); tscDebug( - "%p new subquery:%p, tableIndex:%d, vgroupIndex:%d, type:%d, exprInfo:%zu, colList:%zu," + "%p new subquery:%p, tableIndex:%d, vgroupIndex:%d, type:%d, exprInfo:%" PRIzu ", colList:%" PRIzu "," "fieldInfo:%d, name:%s, qrang:%" PRId64 " - %" PRId64 " order:%d, limit:%" PRId64, pSql, pNew, tableIndex, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type, tscSqlExprNumOfExprs(pNewQueryInfo), size, pNewQueryInfo->fieldsInfo.numOfOutput, pFinalInfo->name, pNewQueryInfo->window.skey, diff --git a/src/os/inc/os.h b/src/os/inc/os.h index 700b29ce98..11c423a500 100644 --- a/src/os/inc/os.h +++ b/src/os/inc/os.h @@ -41,6 +41,7 @@ extern "C" { #endif #include "osAtomic.h" +#include "osCommon.h" #include "osDef.h" #include "osDir.h" #include "osFile.h" diff --git a/src/os/inc/osCommon.h b/src/os/inc/osCommon.h new file mode 100644 index 0000000000..70d2b2c0c2 --- /dev/null +++ b/src/os/inc/osCommon.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_OS_COMMON_H +#define TDENGINE_OS_COMMON_H + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef TAOS_OS_DEF_ZU + #define PRIzu "zu" +#endif + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/os/inc/osWindows.h b/src/os/inc/osWindows.h index caab61536e..6665dcd920 100644 --- a/src/os/inc/osWindows.h +++ b/src/os/inc/osWindows.h @@ -97,6 +97,9 @@ typedef SOCKET eventfd_t; #define TAOS_OS_DEF_EPOLL #define TAOS_EPOLL_WAIT_TIME 100 +#define TAOS_OS_DEF_ZU + #define PRIzu "ld" + #define TAOS_OS_FUNC_STRING_WCHAR int twcslen(const wchar_t *wcs); #define TAOS_OS_FUNC_STRING_GETLINE diff --git a/src/os/src/detail/osMemory.c b/src/os/src/detail/osMemory.c index 3bbe806369..dfd320be89 100644 --- a/src/os/src/detail/osMemory.c +++ b/src/os/src/detail/osMemory.c @@ -42,7 +42,7 @@ static bool random_alloc_fail(size_t size, const char* file, uint32_t line) { } if (fpAllocLog != NULL) { - fprintf(fpAllocLog, "%s:%d: memory allocation of %zu bytes will fail.\n", file, line, size); + fprintf(fpAllocLog, "%s:%d: memory allocation of %" PRIzu " bytes will fail.\n", file, line, size); } return true; @@ -159,7 +159,7 @@ static void* malloc_detect_leak(size_t size, const char* file, uint32_t line) { } if (size > UINT32_MAX && fpAllocLog != NULL) { - fprintf(fpAllocLog, "%s:%d: size too large: %zu.\n", file, line, size); + fprintf(fpAllocLog, "%s:%d: size too large: %" PRIzu ".\n", file, line, size); } blk->file = file; @@ -207,7 +207,7 @@ static void* realloc_detect_leak(void* ptr, size_t size, const char* file, uint3 } if (size > UINT32_MAX && fpAllocLog != NULL) { - fprintf(fpAllocLog, "%s:%d: size too large: %zu.\n", file, line, size); + fprintf(fpAllocLog, "%s:%d: size too large: %" PRIzu ".\n", file, line, size); } blk = (SMemBlock*)p; @@ -295,7 +295,7 @@ static void dump_memory_leak() { atomic_store_ptr(&lock, 0); - fprintf(fpAllocLog, "\nnumber of blocks: %zu, total bytes: %zu\n", numOfBlk, totalSize); + fprintf(fpAllocLog, "\nnumber of blocks: %" PRIzu ", total bytes: %" PRIzu "\n", numOfBlk, totalSize); fflush(fpAllocLog); } diff --git a/src/plugins/http/src/httpContext.c b/src/plugins/http/src/httpContext.c index e367911695..7d6cca511e 100644 --- a/src/plugins/http/src/httpContext.c +++ b/src/plugins/http/src/httpContext.c @@ -70,7 +70,7 @@ bool httpInitContexts() { void httpCleanupContexts() { if (tsHttpServer.contextCache != NULL) { SCacheObj *cache = tsHttpServer.contextCache; - httpInfo("context cache is cleanuping, size:%zu", taosHashGetSize(cache->pHashTable)); + httpInfo("context cache is cleanuping, size:%" PRIzu "", taosHashGetSize(cache->pHashTable)); taosCacheCleanup(tsHttpServer.contextCache); tsHttpServer.contextCache = NULL; } diff --git a/src/plugins/http/src/httpSession.c b/src/plugins/http/src/httpSession.c index fce85df45e..ad57f0fc29 100644 --- a/src/plugins/http/src/httpSession.c +++ b/src/plugins/http/src/httpSession.c @@ -107,7 +107,7 @@ static void httpDestroySession(void *data) { void httpCleanUpSessions() { if (tsHttpServer.sessionCache != NULL) { SCacheObj *cache = tsHttpServer.sessionCache; - httpInfo("session cache is cleanuping, size:%zu", taosHashGetSize(cache->pHashTable)); + httpInfo("session cache is cleanuping, size:%" PRIzu "", taosHashGetSize(cache->pHashTable)); taosCacheCleanup(tsHttpServer.sessionCache); tsHttpServer.sessionCache = NULL; } diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index be3d476be5..71bf0eaa5a 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -4478,7 +4478,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { while (pQInfo->groupIndex < numOfGroups) { SArray* group = taosArrayGetP(pQInfo->tableGroupInfo.pGroupList, pQInfo->groupIndex); - qDebug("QInfo:%p last_row query on group:%d, total group:%zu, current group:%p", pQInfo, pQInfo->groupIndex, + qDebug("QInfo:%p last_row query on group:%d, total group:%" PRIzu ", current group:%p", pQInfo, pQInfo->groupIndex, numOfGroups, group); STsdbQueryCond cond = { @@ -4552,7 +4552,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { while (pQInfo->groupIndex < numOfGroups) { SArray* group = taosArrayGetP(pQInfo->tableGroupInfo.pGroupList, pQInfo->groupIndex); - qDebug("QInfo:%p group by normal columns group:%d, total group:%zu", pQInfo, pQInfo->groupIndex, numOfGroups); + qDebug("QInfo:%p group by normal columns group:%d, total group:%" PRIzu "", pQInfo, pQInfo->groupIndex, numOfGroups); STsdbQueryCond cond = { .colList = pQuery->colList, @@ -4743,7 +4743,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { } qDebug( - "QInfo %p numOfTables:%"PRIu64", index:%d, numOfGroups:%zu, %"PRId64" points returned, total:%"PRId64", offset:%" PRId64, + "QInfo %p numOfTables:%"PRIu64", index:%d, numOfGroups:%" PRIzu ", %"PRId64" points returned, total:%"PRId64", offset:%" PRId64, pQInfo, pQInfo->tableqinfoGroupInfo.numOfTables, pQInfo->tableIndex, numOfGroups, pQuery->rec.rows, pQuery->rec.total, pQuery->limit.offset); } @@ -6241,11 +6241,11 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qi goto _over; } - qDebug("qmsg:%p query on %zu tables in one group from client", pQueryMsg, tableGroupInfo.numOfTables); + qDebug("qmsg:%p query on %" PRIzu " tables in one group from client", pQueryMsg, tableGroupInfo.numOfTables); } int64_t el = taosGetTimestampUs() - st; - qDebug("qmsg:%p tag filter completed, numOfTables:%zu, elapsed time:%"PRId64"us", pQueryMsg, tableGroupInfo.numOfTables, el); + qDebug("qmsg:%p tag filter completed, numOfTables:%" PRIzu ", elapsed time:%"PRId64"us", pQueryMsg, tableGroupInfo.numOfTables, el); } else { assert(0); } @@ -6356,7 +6356,7 @@ bool qTableQuery(qinfo_t qinfo) { if (IS_QUERY_KILLED(pQInfo)) { qDebug("QInfo:%p query is killed", pQInfo); } else if (pQuery->rec.rows == 0) { - qDebug("QInfo:%p over, %zu tables queried, %"PRId64" rows are returned", pQInfo, pQInfo->tableqinfoGroupInfo.numOfTables, pQuery->rec.total); + qDebug("QInfo:%p over, %" PRIzu " tables queried, %"PRId64" rows are returned", pQInfo, pQInfo->tableqinfoGroupInfo.numOfTables, pQuery->rec.total); } else { qDebug("QInfo:%p query paused, %" PRId64 " rows returned, numOfTotal:%" PRId64 " rows", pQInfo, pQuery->rec.rows, pQuery->rec.total + pQuery->rec.rows); diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index d63bf8ab78..41da1b2dc6 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -608,14 +608,14 @@ int tsdbLoadCompData(SRWHelper *pHelper, SCompBlock *pCompBlock, void *target) { } if (taosTRead(pFile->fd, (void *)pHelper->pCompData, tsize) < tsize) { - tsdbError("vgId:%d failed to read %zu bytes from file %s since %s", REPO_ID(pHelper->pRepo), tsize, pFile->fname, + tsdbError("vgId:%d failed to read %" PRIzu " bytes from file %s since %s", REPO_ID(pHelper->pRepo), tsize, pFile->fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); return -1; } if (!taosCheckChecksumWhole((uint8_t *)pHelper->pCompData, (uint32_t)tsize)) { - tsdbError("vgId:%d file %s is broken, offset %" PRId64 " size %zu", REPO_ID(pHelper->pRepo), pFile->fname, + tsdbError("vgId:%d file %s is broken, offset %" PRId64 " size %" PRIzu "", REPO_ID(pHelper->pRepo), pFile->fname, (int64_t)pCompBlock->offset, tsize); terrno = TSDB_CODE_TDB_FILE_CORRUPTED; return -1; diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 17b0239e3b..051dcf7adf 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -250,7 +250,7 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab taosArraySort(pQueryHandle->pTableCheckInfo, tsdbCheckInfoCompar); pQueryHandle->defaultLoadColumn = getDefaultLoadColumns(pQueryHandle, true); - tsdbDebug("%p total numOfTable:%zu in query, %p", pQueryHandle, taosArrayGetSize(pQueryHandle->pTableCheckInfo), pQueryHandle->qinfo); + tsdbDebug("%p total numOfTable:%" PRIzu " in query, %p", pQueryHandle, taosArrayGetSize(pQueryHandle->pTableCheckInfo), pQueryHandle->qinfo); tsdbInitDataBlockLoadInfo(&pQueryHandle->dataBlockLoadInfo); tsdbInitCompBlockLoadInfo(&pQueryHandle->compBlockLoadInfo); @@ -2190,7 +2190,7 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC } taosArrayPush(pTableGroup, &sa); - tsdbDebug("all %zu tables belong to one group", size); + tsdbDebug("all %" PRIzu " tables belong to one group", size); } else { STableGroupSupporter *pSupp = (STableGroupSupporter *) calloc(1, sizeof(STableGroupSupporter)); pSupp->numOfCols = numOfOrderCols; @@ -2309,7 +2309,7 @@ int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T* tsdb, uint64_t uid, const char* pT pGroupInfo->numOfTables = taosArrayGetSize(res); pGroupInfo->pGroupList = createTableGroup(res, pTagSchema, pColIndex, numOfCols); - tsdbDebug("%p no table name/tag condition, all tables belong to one group, numOfTables:%zu", tsdb, pGroupInfo->numOfTables); + tsdbDebug("%p no table name/tag condition, all tables belong to one group, numOfTables:%" PRIzu "", tsdb, pGroupInfo->numOfTables); taosArrayDestroy(res); if (tsdbUnlockRepoMeta(tsdb) < 0) goto _error; @@ -2352,7 +2352,7 @@ int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T* tsdb, uint64_t uid, const char* pT pGroupInfo->numOfTables = taosArrayGetSize(res); pGroupInfo->pGroupList = createTableGroup(res, pTagSchema, pColIndex, numOfCols); - tsdbDebug("%p stable tid:%d, uid:%"PRIu64" query, numOfTables:%zu, belong to %zu groups", tsdb, pTable->tableId.tid, + tsdbDebug("%p stable tid:%d, uid:%"PRIu64" query, numOfTables:%" PRIzu ", belong to %" PRIzu " groups", tsdb, pTable->tableId.tid, pTable->tableId.uid, pGroupInfo->numOfTables, taosArrayGetSize(pGroupInfo->pGroupList)); taosArrayDestroy(res); diff --git a/src/util/src/tkvstore.c b/src/util/src/tkvstore.c index 9657d82773..c10e882d57 100644 --- a/src/util/src/tkvstore.c +++ b/src/util/src/tkvstore.c @@ -509,7 +509,7 @@ static int tdRestoreKVStore(SKVStore *pStore) { ssize_t tsize = taosTRead(pStore->fd, tbuf, sizeof(SKVRecord)); if (tsize == 0) break; if (tsize < sizeof(SKVRecord)) { - uError("failed to read %zu bytes from file %s at offset %" PRId64 "since %s", sizeof(SKVRecord), pStore->fname, + uError("failed to read %" PRIzu " bytes from file %s at offset %" PRId64 "since %s", sizeof(SKVRecord), pStore->fname, pStore->info.size, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); goto _err; diff --git a/tests/script/test.bat b/tests/script/test.bat new file mode 100644 index 0000000000..1574b5013e --- /dev/null +++ b/tests/script/test.bat @@ -0,0 +1,60 @@ +@echo off + +echo TDengine in windows +echo Start TDengine Testing Case ... + +set "SCRIPT_DIR=%~dp0" +echo SCRIPT_DIR: %SCRIPT_DIR% + +set "BUILD_DIR=%~dp0..\..\debug\build\bin" +set "TSIM=%~dp0..\..\debug\build\bin\tsim" +echo BUILD_DIR: %BUILD_DIR% + +set "SIM_DIR=%~dp0..\..\sim" +echo SIM_DIR: %SIM_DIR% + +set "TSIM_DIR=%~dp0..\..\sim\tsim" +echo TSIM_DIR: %TSIM_DIR% + +set "CFG_DIR=%~dp0..\..\sim\tsim\cfg" +echo CFG_DIR: %CFG_DIR% + +set "LOG_DIR=%~dp0..\..\sim\tsim\log" +echo LOG_DIR: %LOG_DIR% + +set "TAOS_CFG=%~dp0..\..\sim\tsim\cfg\taos.cfg" +echo TAOS_CFG: %TAOS_CFG% + +if not exist %SIM_DIR% mkdir %SIM_DIR% +if not exist %TSIM_DIR% mkdir %TSIM_DIR% +if exist %CFG_DIR% rmdir /s/q %CFG_DIR% +if exist %LOG_DIR% rmdir /s/q %LOG_DIR% +if not exist %CFG_DIR% mkdir %CFG_DIR% +if not exist %LOG_DIR% mkdir %LOG_DIR% + +echo firstEp %FIRSTEP% > %TAOS_CFG% +echo serverPort 6030 >> %TAOS_CFG% +echo wal 2 >> %TAOS_CFG% +echo asyncLog 0 >> %TAOS_CFG% +echo locale en_US.UTF-8 >> %TAOS_CFG% +echo logDir %LOG_DIR% >> %TAOS_CFG% +echo scriptDir %SCRIPT_DIR% >> %TAOS_CFG% +echo numOfLogLines 100000000 >> %TAOS_CFG% +echo tmrDebugFlag 131 >> %TAOS_CFG% +echo rpcDebugFlag 143 >> %TAOS_CFG% +echo cDebugFlag 143 >> %TAOS_CFG% +echo qdebugFlag 143 >> %TAOS_CFG% +echo udebugFlag 143 >> %TAOS_CFG% + +set "FILE_NAME=windows\testSuite.sim" +set "FIRSTEP=localhost" +if "%1" == "-f" set "FILE_NAME=%2" +if "%1" == "-h" set "FIRSTEP=%2" +if "%3" == "-f" set "FILE_NAME=%4" +if "%3" == "-h" set "FIRSTEP=%4" + +echo FILE_NAME: %FILE_NAME% +echo FIRSTEP: %FIRSTEP% +echo ExcuteCmd: %tsim% -c %CFG_DIR% -f %FILE_NAME% + +%tsim% -c %CFG_DIR% -f %FILE_NAME% \ No newline at end of file From 98cec99dc16a2c46e28bbaeb4783682f80cf7482 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 10 Aug 2020 18:30:16 +0800 Subject: [PATCH 103/109] [td-255] fix counting value error bug. --- src/util/src/hash.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/util/src/hash.c b/src/util/src/hash.c index 9e262916ce..93514f87cf 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -451,6 +451,8 @@ int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), voi while((pNode = pEntry->next) != NULL) { if (fp && (!fp(param, pNode->data))) { pEntry->num -= 1; + atomic_sub_fetch_64(&pHashObj->size, 1); + pEntry->next = pNode->next; if (pEntry->num == 0) { @@ -475,6 +477,7 @@ int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), voi if (fp && (!fp(param, pNext->data))) { pNode->next = pNext->next; pEntry->num -= 1; + atomic_sub_fetch_64(&pHashObj->size, 1); if (pEntry->num == 0) { assert(pEntry->next == NULL); From f6712d984ac70b1743151cc15d940c3976488b51 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 10 Aug 2020 18:43:16 +0800 Subject: [PATCH 104/109] [td-255]remove unused functions. --- src/util/src/hash.c | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/src/util/src/hash.c b/src/util/src/hash.c index 93514f87cf..8df3e3d4d3 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -88,23 +88,6 @@ static FORCE_INLINE SHashNode *doSearchInEntryList(SHashEntry *pe, const void *k return pNode; } -static FORCE_INLINE SHashNode *doSerchPrevInEntryList(SHashEntry *pe, const void *key, size_t keyLen, uint32_t hashVal) { - SHashNode *prev= NULL; - SHashNode *pNode = pe->next; - - while (pNode) { - if ((pNode->keyLen == keyLen) && (memcmp(pNode->key, key, keyLen) == 0)) { - assert(pNode->hashVal == hashVal); - break; - } - - prev = pNode; - pNode = pNode->next; - } - - return prev; -} - /** * Resize the hash list if the threshold is reached * From 5aae4b90786096e4438bd66b23d3f37a0943d1e6 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 10 Aug 2020 18:46:20 +0800 Subject: [PATCH 105/109] TD-1057 first compile version of windows 64 client --- src/client/src/tscSub.c | 2 +- tests/script/windows/testSuite.sim | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c index 5d8e601882..0e1be926f4 100644 --- a/src/client/src/tscSub.c +++ b/src/client/src/tscSub.c @@ -339,7 +339,7 @@ static int tscLoadSubscriptionProgress(SSub* pSub) { fclose(fp); taosArraySort(progress, tscCompareSubscriptionProgress); - tscDebug("subscription progress loaded, %%" PRIzu " tables: %s", taosArrayGetSize(progress), pSub->topic); + tscDebug("subscription progress loaded, %" PRIzu " tables: %s", taosArrayGetSize(progress), pSub->topic); return 1; } diff --git a/tests/script/windows/testSuite.sim b/tests/script/windows/testSuite.sim index e372217b62..fc574ed9c4 100644 --- a/tests/script/windows/testSuite.sim +++ b/tests/script/windows/testSuite.sim @@ -1,14 +1,14 @@ run windows/alter/table.sim run windows/alter/metrics.sim -run windows/compute/avg.sim +#run windows/compute/avg.sim run windows/compute/bottom.sim run windows/compute/count.sim run windows/compute/diff.sim run windows/compute/first.sim run windows/compute/interval.sim run windows/compute/last.sim -run windows/compute/leastsquare.sim +##run windows/compute/leastsquare.sim run windows/compute/max.sim run windows/compute/min.sim run windows/compute/percentile.sim @@ -37,12 +37,12 @@ run windows/field/tinyint.sim run windows/import/basic.sim run windows/insert/basic.sim -run windows/insert/query_block1_file.sim -run windows/insert/query_block1_memory.sim -run windows/insert/query_block2_file.sim -run windows/insert/query_block2_memory.sim -run windows/insert/query_file_memory.sim -run windows/insert/query_multi_file.sim +#run windows/insert/query_block1_file.sim +#run windows/insert/query_block1_memory.sim +#run windows/insert/query_block2_file.sim +#run windows/insert/query_block2_memory.sim +#run windows/insert/query_file_memory.sim +#run windows/insert/query_multi_file.sim run windows/table/binary.sim run windows/table/bool.sim From bc73587498eecfdac33c80c7f8ca0fe2df6a456c Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Mon, 10 Aug 2020 18:47:44 +0800 Subject: [PATCH 106/109] udpate taos-jdbcdriver version for jdbc demos --- tests/examples/JDBC/JDBCDemo/pom.xml | 2 +- tests/examples/JDBC/SpringJdbcTemplate/pom.xml | 2 +- tests/examples/JDBC/springbootdemo/pom.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/examples/JDBC/JDBCDemo/pom.xml b/tests/examples/JDBC/JDBCDemo/pom.xml index d015c62772..50313a0a0c 100644 --- a/tests/examples/JDBC/JDBCDemo/pom.xml +++ b/tests/examples/JDBC/JDBCDemo/pom.xml @@ -63,7 +63,7 @@ com.taosdata.jdbc taos-jdbcdriver - 1.0.1 + 2.0.2 diff --git a/tests/examples/JDBC/SpringJdbcTemplate/pom.xml b/tests/examples/JDBC/SpringJdbcTemplate/pom.xml index 45abc5354a..b796d52d28 100644 --- a/tests/examples/JDBC/SpringJdbcTemplate/pom.xml +++ b/tests/examples/JDBC/SpringJdbcTemplate/pom.xml @@ -41,7 +41,7 @@ com.taosdata.jdbc taos-jdbcdriver - 1.0.3 + 2.0.2 diff --git a/tests/examples/JDBC/springbootdemo/pom.xml b/tests/examples/JDBC/springbootdemo/pom.xml index 74522979c0..5f31d36d6e 100644 --- a/tests/examples/JDBC/springbootdemo/pom.xml +++ b/tests/examples/JDBC/springbootdemo/pom.xml @@ -63,7 +63,7 @@ com.taosdata.jdbc taos-jdbcdriver - 1.0.3 + 2.0.2 From 7e4cf6511a4cf01efa8d5635992e5fbb37c9d8ae Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 10 Aug 2020 19:05:00 +0800 Subject: [PATCH 107/109] [td-255]remove unused functions. --- src/util/src/tcache.c | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index e1ba80c9d0..54a4b289b7 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -30,14 +30,6 @@ static FORCE_INLINE void __cache_wr_lock(SCacheObj *pCacheObj) { #endif } -static FORCE_INLINE void __cache_rd_lock(SCacheObj *pCacheObj) { -#if defined(LINUX) - pthread_rwlock_rdlock(&pCacheObj->lock); -#else - pthread_mutex_lock(&pCacheObj->lock); -#endif -} - static FORCE_INLINE void __cache_unlock(SCacheObj *pCacheObj) { #if defined(LINUX) pthread_rwlock_unlock(&pCacheObj->lock); From e57965f64ace0f9c08f8e6ad70dd99cee249b7e6 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 10 Aug 2020 13:48:59 +0000 Subject: [PATCH 108/109] TD-1057 --- tests/script/windows/alter/metrics.sim | 7 +++++++ tests/script/windows/alter/table.sim | 7 +++++++ tests/script/windows/compute/avg.sim | 7 +++++++ tests/script/windows/compute/bottom.sim | 7 +++++++ tests/script/windows/compute/count.sim | 7 +++++++ tests/script/windows/compute/diff.sim | 7 +++++++ tests/script/windows/compute/first.sim | 7 +++++++ tests/script/windows/compute/interval.sim | 7 +++++++ tests/script/windows/compute/last.sim | 7 +++++++ tests/script/windows/compute/leastsquare.sim | 7 +++++++ tests/script/windows/compute/max.sim | 7 +++++++ tests/script/windows/compute/min.sim | 7 +++++++ tests/script/windows/compute/percentile.sim | 7 +++++++ tests/script/windows/compute/stddev.sim | 7 +++++++ tests/script/windows/compute/sum.sim | 7 +++++++ tests/script/windows/compute/top.sim | 7 +++++++ tests/script/windows/db/basic.sim | 7 +++++++ tests/script/windows/db/len.sim | 7 +++++++ tests/script/windows/field/2.sim | 7 +++++++ tests/script/windows/field/3.sim | 7 +++++++ tests/script/windows/field/4.sim | 7 +++++++ tests/script/windows/field/5.sim | 7 +++++++ tests/script/windows/field/6.sim | 7 +++++++ tests/script/windows/field/bigint.sim | 7 +++++++ tests/script/windows/field/binary.sim | 7 +++++++ tests/script/windows/field/bool.sim | 7 +++++++ tests/script/windows/field/double.sim | 7 +++++++ tests/script/windows/field/float.sim | 7 +++++++ tests/script/windows/field/int.sim | 7 +++++++ tests/script/windows/field/single.sim | 7 +++++++ tests/script/windows/field/smallint.sim | 7 +++++++ tests/script/windows/field/tinyint.sim | 7 +++++++ tests/script/windows/import/basic.sim | 7 +++++++ tests/script/windows/insert/basic.sim | 7 +++++++ .../script/windows/insert/query_block1_file.sim | 9 ++++++++- .../windows/insert/query_block1_memory.sim | 7 +++++++ .../script/windows/insert/query_block2_file.sim | 7 +++++++ .../windows/insert/query_block2_memory.sim | 7 +++++++ .../script/windows/insert/query_file_memory.sim | 7 +++++++ tests/script/windows/insert/query_multi_file.sim | 7 +++++++ tests/script/windows/table/binary.sim | 7 +++++++ tests/script/windows/table/bool.sim | 7 +++++++ tests/script/windows/table/column_name.sim | 8 ++++++++ tests/script/windows/table/column_num.sim | 8 ++++++++ tests/script/windows/table/column_value.sim | 8 ++++++++ tests/script/windows/table/db.table.sim | 8 ++++++++ tests/script/windows/table/double.sim | 7 +++++++ tests/script/windows/table/float.sim | 7 +++++++ tests/script/windows/table/table.sim | 7 +++++++ tests/script/windows/table/table_len.sim | 7 +++++++ tests/script/windows/tag/3.sim | 7 +++++++ tests/script/windows/tag/4.sim | 7 +++++++ tests/script/windows/tag/5.sim | 7 +++++++ tests/script/windows/tag/6.sim | 7 +++++++ tests/script/windows/tag/add.sim | 7 +++++++ tests/script/windows/tag/bigint.sim | 7 +++++++ tests/script/windows/tag/binary.sim | 7 +++++++ tests/script/windows/tag/binary_binary.sim | 7 +++++++ tests/script/windows/tag/bool.sim | 7 +++++++ tests/script/windows/tag/bool_binary.sim | 7 +++++++ tests/script/windows/tag/bool_int.sim | 7 +++++++ tests/script/windows/tag/change.sim | 7 +++++++ tests/script/windows/tag/column.sim | 7 +++++++ tests/script/windows/tag/create.sim | 7 +++++++ tests/script/windows/tag/delete.sim | 7 +++++++ tests/script/windows/tag/double.sim | 7 +++++++ tests/script/windows/tag/filter.sim | 7 +++++++ tests/script/windows/tag/float.sim | 7 +++++++ tests/script/windows/tag/int.sim | 7 +++++++ tests/script/windows/tag/int_binary.sim | 7 +++++++ tests/script/windows/tag/int_float.sim | 7 +++++++ tests/script/windows/tag/set.sim | 7 +++++++ tests/script/windows/tag/smallint.sim | 7 +++++++ tests/script/windows/tag/tinyint.sim | 7 +++++++ tests/script/windows/testSuite.sim | 16 ++++++++-------- tests/script/windows/vector/metrics_field.sim | 7 +++++++ tests/script/windows/vector/metrics_mix.sim | 7 +++++++ tests/script/windows/vector/metrics_query.sim | 7 +++++++ tests/script/windows/vector/metrics_tag.sim | 7 +++++++ tests/script/windows/vector/metrics_time.sim | 7 +++++++ tests/script/windows/vector/multi.sim | 7 +++++++ tests/script/windows/vector/single.sim | 7 +++++++ tests/script/windows/vector/table_field.sim | 7 +++++++ tests/script/windows/vector/table_mix.sim | 7 +++++++ tests/script/windows/vector/table_query.sim | 7 +++++++ tests/script/windows/vector/table_time.sim | 7 +++++++ 86 files changed, 608 insertions(+), 9 deletions(-) diff --git a/tests/script/windows/alter/metrics.sim b/tests/script/windows/alter/metrics.sim index 3717d8c1ed..7d5dc77949 100644 --- a/tests/script/windows/alter/metrics.sim +++ b/tests/script/windows/alter/metrics.sim @@ -1,6 +1,13 @@ sleep 3000 sql connect +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + print ======== step1 sql create database d2 sql use d2 diff --git a/tests/script/windows/alter/table.sim b/tests/script/windows/alter/table.sim index 3b811a065e..03182e613d 100644 --- a/tests/script/windows/alter/table.sim +++ b/tests/script/windows/alter/table.sim @@ -1,6 +1,13 @@ sleep 3000 sql connect +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + print ======== step1 sql create database d1 sql use d1 diff --git a/tests/script/windows/compute/avg.sim b/tests/script/windows/compute/avg.sim index 1374ca5a25..b655abf163 100644 --- a/tests/script/windows/compute/avg.sim +++ b/tests/script/windows/compute/avg.sim @@ -1,6 +1,13 @@ sql connect sleep 3000 +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = m_av_db $tbPrefix = m_av_tb $mtPrefix = m_av_mt diff --git a/tests/script/windows/compute/bottom.sim b/tests/script/windows/compute/bottom.sim index e908c774e4..dc104a8ebd 100644 --- a/tests/script/windows/compute/bottom.sim +++ b/tests/script/windows/compute/bottom.sim @@ -1,6 +1,13 @@ sql connect sleep 3000 +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = m_bo_db $tbPrefix = m_bo_tb $mtPrefix = m_bo_mt diff --git a/tests/script/windows/compute/count.sim b/tests/script/windows/compute/count.sim index 54544f0354..9c9d8821b0 100644 --- a/tests/script/windows/compute/count.sim +++ b/tests/script/windows/compute/count.sim @@ -1,6 +1,13 @@ sql connect sleep 3000 +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = m_co_db $tbPrefix = m_co_tb $mtPrefix = m_co_mt diff --git a/tests/script/windows/compute/diff.sim b/tests/script/windows/compute/diff.sim index 6c2829872a..667fcdbcff 100644 --- a/tests/script/windows/compute/diff.sim +++ b/tests/script/windows/compute/diff.sim @@ -1,6 +1,13 @@ sleep 3000 sql connect +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = m_di_db $tbPrefix = m_di_tb $mtPrefix = m_di_mt diff --git a/tests/script/windows/compute/first.sim b/tests/script/windows/compute/first.sim index 9a0c02fe4b..d6e1b1deea 100644 --- a/tests/script/windows/compute/first.sim +++ b/tests/script/windows/compute/first.sim @@ -1,6 +1,13 @@ sleep 3000 sql connect +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = m_fi_db $tbPrefix = m_fi_tb $mtPrefix = m_fi_mt diff --git a/tests/script/windows/compute/interval.sim b/tests/script/windows/compute/interval.sim index 365c6d9d31..4bf548ccf2 100644 --- a/tests/script/windows/compute/interval.sim +++ b/tests/script/windows/compute/interval.sim @@ -1,6 +1,13 @@ sleep 3000 sql connect +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = m_in_db $tbPrefix = m_in_tb $mtPrefix = m_in_mt diff --git a/tests/script/windows/compute/last.sim b/tests/script/windows/compute/last.sim index aa9699778f..63d4d3ecbd 100644 --- a/tests/script/windows/compute/last.sim +++ b/tests/script/windows/compute/last.sim @@ -1,6 +1,13 @@ sleep 3000 sql connect +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = m_la_db $tbPrefix = m_la_tb $mtPrefix = m_la_mt diff --git a/tests/script/windows/compute/leastsquare.sim b/tests/script/windows/compute/leastsquare.sim index bb7404edd0..4cd3ad1fb9 100644 --- a/tests/script/windows/compute/leastsquare.sim +++ b/tests/script/windows/compute/leastsquare.sim @@ -1,6 +1,13 @@ sleep 3000 sql connect +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = m_le_db $tbPrefix = m_le_tb $mtPrefix = m_le_mt diff --git a/tests/script/windows/compute/max.sim b/tests/script/windows/compute/max.sim index a19d122ecd..e480736550 100644 --- a/tests/script/windows/compute/max.sim +++ b/tests/script/windows/compute/max.sim @@ -1,6 +1,13 @@ sleep 3000 sql connect +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = m_ma_db $tbPrefix = m_ma_tb $mtPrefix = m_ma_mt diff --git a/tests/script/windows/compute/min.sim b/tests/script/windows/compute/min.sim index 216f2061d7..1ff637cecd 100644 --- a/tests/script/windows/compute/min.sim +++ b/tests/script/windows/compute/min.sim @@ -1,6 +1,13 @@ sleep 3000 sql connect +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = m_mi_db $tbPrefix = m_mi_tb $mtPrefix = m_mi_mt diff --git a/tests/script/windows/compute/percentile.sim b/tests/script/windows/compute/percentile.sim index 20b2740d6e..5e327055a8 100644 --- a/tests/script/windows/compute/percentile.sim +++ b/tests/script/windows/compute/percentile.sim @@ -1,6 +1,13 @@ sleep 3000 sql connect +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = m_pe_db $tbPrefix = m_pe_tb $mtPrefix = m_pe_mt diff --git a/tests/script/windows/compute/stddev.sim b/tests/script/windows/compute/stddev.sim index c02b3e4ab3..2aa481248a 100644 --- a/tests/script/windows/compute/stddev.sim +++ b/tests/script/windows/compute/stddev.sim @@ -1,6 +1,13 @@ sleep 3000 sql connect +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = m_st_db $tbPrefix = m_st_tb $mtPrefix = m_st_mt diff --git a/tests/script/windows/compute/sum.sim b/tests/script/windows/compute/sum.sim index 04af1d457a..30e98a5b25 100644 --- a/tests/script/windows/compute/sum.sim +++ b/tests/script/windows/compute/sum.sim @@ -1,6 +1,13 @@ sleep 3000 sql connect +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = m_su_db $tbPrefix = m_su_tb $mtPrefix = m_su_mt diff --git a/tests/script/windows/compute/top.sim b/tests/script/windows/compute/top.sim index b3c698c064..9590997ef7 100644 --- a/tests/script/windows/compute/top.sim +++ b/tests/script/windows/compute/top.sim @@ -1,6 +1,13 @@ sleep 3000 sql connect +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = m_to_db $tbPrefix = m_to_tb $mtPrefix = m_to_mt diff --git a/tests/script/windows/db/basic.sim b/tests/script/windows/db/basic.sim index f1e18d15a5..fffde94d66 100644 --- a/tests/script/windows/db/basic.sim +++ b/tests/script/windows/db/basic.sim @@ -1,6 +1,13 @@ sleep 3000 sql connect +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + print ============================ dnode1 start $i = 0 diff --git a/tests/script/windows/db/len.sim b/tests/script/windows/db/len.sim index f922e7e05a..5afa2496dd 100644 --- a/tests/script/windows/db/len.sim +++ b/tests/script/windows/db/len.sim @@ -1,6 +1,13 @@ sleep 3000 sql connect +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + print =============== step1 sql_error drop database dd diff --git a/tests/script/windows/field/2.sim b/tests/script/windows/field/2.sim index 3d4492083e..8ac6fa1a1b 100644 --- a/tests/script/windows/field/2.sim +++ b/tests/script/windows/field/2.sim @@ -2,6 +2,13 @@ sql connect sleep 3000 print ======================== dnode1 start +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = fi_bt_db $tbPrefix = fi_bt_tb $mtPrefix = fi_bt_mt diff --git a/tests/script/windows/field/3.sim b/tests/script/windows/field/3.sim index fb7d60d12a..331e930b31 100644 --- a/tests/script/windows/field/3.sim +++ b/tests/script/windows/field/3.sim @@ -2,6 +2,13 @@ sql connect sleep 3000 print ======================== dnode1 start +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = fi_3_db $tbPrefix = fi_3_tb $mtPrefix = fi_3_mt diff --git a/tests/script/windows/field/4.sim b/tests/script/windows/field/4.sim index f7ffa9807c..c6224c46ee 100644 --- a/tests/script/windows/field/4.sim +++ b/tests/script/windows/field/4.sim @@ -2,6 +2,13 @@ sql connect sleep 3000 print ======================== dnode1 start +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = fi_4_db $tbPrefix = fi_4_tb $mtPrefix = fi_4_mt diff --git a/tests/script/windows/field/5.sim b/tests/script/windows/field/5.sim index e408871693..d1f40059d0 100644 --- a/tests/script/windows/field/5.sim +++ b/tests/script/windows/field/5.sim @@ -2,6 +2,13 @@ sql connect sleep 3000 print ======================== dnode1 start +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = fi_5_db $tbPrefix = fi_5_tb $mtPrefix = fi_5_mt diff --git a/tests/script/windows/field/6.sim b/tests/script/windows/field/6.sim index d1551d63b5..98071f87df 100644 --- a/tests/script/windows/field/6.sim +++ b/tests/script/windows/field/6.sim @@ -2,6 +2,13 @@ sql connect sleep 3000 print ======================== dnode1 start +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = fi_6_db $tbPrefix = fi_6_tb $mtPrefix = fi_6_mt diff --git a/tests/script/windows/field/bigint.sim b/tests/script/windows/field/bigint.sim index 9ccaeb3723..bef571f445 100644 --- a/tests/script/windows/field/bigint.sim +++ b/tests/script/windows/field/bigint.sim @@ -2,6 +2,13 @@ sql connect sleep 3000 print ======================== dnode1 start +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = fi_bi_db $tbPrefix = fi_bi_tb $mtPrefix = fi_bi_mt diff --git a/tests/script/windows/field/binary.sim b/tests/script/windows/field/binary.sim index 8b86c4dbea..72a356e684 100644 --- a/tests/script/windows/field/binary.sim +++ b/tests/script/windows/field/binary.sim @@ -2,6 +2,13 @@ sql connect sleep 3000 print ======================== dnode1 start +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = fi_by_db $tbPrefix = fi_by_tb $mtPrefix = fi_by_mt diff --git a/tests/script/windows/field/bool.sim b/tests/script/windows/field/bool.sim index 5f2c61475c..abc970264d 100644 --- a/tests/script/windows/field/bool.sim +++ b/tests/script/windows/field/bool.sim @@ -2,6 +2,13 @@ sql connect sleep 3000 print ======================== dnode1 start +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = fi_bo_db $tbPrefix = fi_bo_tb $mtPrefix = fi_bo_mt diff --git a/tests/script/windows/field/double.sim b/tests/script/windows/field/double.sim index ea7e075208..e805e0373b 100644 --- a/tests/script/windows/field/double.sim +++ b/tests/script/windows/field/double.sim @@ -2,6 +2,13 @@ sql connect sleep 3000 print ======================== dnode1 start +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = fi_do_db $tbPrefix = fi_do_tb $mtPrefix = fi_do_mt diff --git a/tests/script/windows/field/float.sim b/tests/script/windows/field/float.sim index 5be59bae3b..4178ab4e1e 100644 --- a/tests/script/windows/field/float.sim +++ b/tests/script/windows/field/float.sim @@ -2,6 +2,13 @@ sql connect sleep 3000 print ======================== dnode1 start +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = fi_fl_db $tbPrefix = fi_fl_tb $mtPrefix = fi_fl_mt diff --git a/tests/script/windows/field/int.sim b/tests/script/windows/field/int.sim index d7d26b7341..05dc19094d 100644 --- a/tests/script/windows/field/int.sim +++ b/tests/script/windows/field/int.sim @@ -2,6 +2,13 @@ sql connect sleep 3000 print ======================== dnode1 start +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = fi_in_db $tbPrefix = fi_in_tb $mtPrefix = fi_in_mt diff --git a/tests/script/windows/field/single.sim b/tests/script/windows/field/single.sim index 0199133ecd..6422b7f697 100644 --- a/tests/script/windows/field/single.sim +++ b/tests/script/windows/field/single.sim @@ -2,6 +2,13 @@ sql connect sleep 3000 print ======================== dnode1 start +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = fi_si_db $tbPrefix = fi_si_tb $mtPrefix = fi_si_mt diff --git a/tests/script/windows/field/smallint.sim b/tests/script/windows/field/smallint.sim index 8bee463292..8bf41f45a5 100644 --- a/tests/script/windows/field/smallint.sim +++ b/tests/script/windows/field/smallint.sim @@ -2,6 +2,13 @@ sql connect sleep 3000 print ======================== dnode1 start +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = fi_sm_db $tbPrefix = fi_sm_tb $mtPrefix = fi_sm_mt diff --git a/tests/script/windows/field/tinyint.sim b/tests/script/windows/field/tinyint.sim index 65bffca095..16c19ba38d 100644 --- a/tests/script/windows/field/tinyint.sim +++ b/tests/script/windows/field/tinyint.sim @@ -2,6 +2,13 @@ sql connect sleep 3000 print ======================== dnode1 start +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = fi_ti_db $tbPrefix = fi_ti_tb $mtPrefix = fi_ti_mt diff --git a/tests/script/windows/import/basic.sim b/tests/script/windows/import/basic.sim index c20378ee88..491b4f8b34 100644 --- a/tests/script/windows/import/basic.sim +++ b/tests/script/windows/import/basic.sim @@ -1,6 +1,13 @@ sql connect sleep 3000 +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + sql create database ibadb sql use ibadb sql create table tb(ts timestamp, i int) diff --git a/tests/script/windows/insert/basic.sim b/tests/script/windows/insert/basic.sim index be0980a2d4..54cbd3f4d9 100644 --- a/tests/script/windows/insert/basic.sim +++ b/tests/script/windows/insert/basic.sim @@ -1,6 +1,13 @@ sql connect sleep 3000 +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $i = 0 $dbPrefix = tb_in_db $tbPrefix = tb_in_tb diff --git a/tests/script/windows/insert/query_block1_file.sim b/tests/script/windows/insert/query_block1_file.sim index 3eb1d402e8..388ed061e5 100644 --- a/tests/script/windows/insert/query_block1_file.sim +++ b/tests/script/windows/insert/query_block1_file.sim @@ -1,6 +1,13 @@ sql connect sleep 3000 +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $i = 0 $dbPrefix = tb_1f_db $tbPrefix = tb_1f_tb @@ -24,7 +31,7 @@ $y = $N / 2 while $x > $y $ms = $x . m $xt = - . $x - sql insert into $tb values (now - $ms , - $x ) + sql insert into $tb values (now - $ms , $x ) $x = $x - 1 endw diff --git a/tests/script/windows/insert/query_block1_memory.sim b/tests/script/windows/insert/query_block1_memory.sim index 60d31e52d6..9e4fc68d09 100644 --- a/tests/script/windows/insert/query_block1_memory.sim +++ b/tests/script/windows/insert/query_block1_memory.sim @@ -1,6 +1,13 @@ sql connect sleep 3000 +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $i = 0 $dbPrefix = tb_1m_db $tbPrefix = tb_1m_tb diff --git a/tests/script/windows/insert/query_block2_file.sim b/tests/script/windows/insert/query_block2_file.sim index e9f562a538..9fd4434476 100644 --- a/tests/script/windows/insert/query_block2_file.sim +++ b/tests/script/windows/insert/query_block2_file.sim @@ -1,6 +1,13 @@ sql connect sleep 3000 +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $i = 0 $dbPrefix = tb_2f_db $tbPrefix = tb_2f_tb diff --git a/tests/script/windows/insert/query_block2_memory.sim b/tests/script/windows/insert/query_block2_memory.sim index fd173f1356..ede7f3efc6 100644 --- a/tests/script/windows/insert/query_block2_memory.sim +++ b/tests/script/windows/insert/query_block2_memory.sim @@ -1,6 +1,13 @@ sql connect sleep 3000 +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $i = 0 $dbPrefix = tb_2m_db $tbPrefix = tb_2m_tb diff --git a/tests/script/windows/insert/query_file_memory.sim b/tests/script/windows/insert/query_file_memory.sim index e9b0c69ea5..083beb4ac5 100644 --- a/tests/script/windows/insert/query_file_memory.sim +++ b/tests/script/windows/insert/query_file_memory.sim @@ -1,6 +1,13 @@ sql connect sleep 3000 +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $i = 0 $dbPrefix = tb_fm_db $tbPrefix = tb_fm_tb diff --git a/tests/script/windows/insert/query_multi_file.sim b/tests/script/windows/insert/query_multi_file.sim index 84c091fb21..465970f942 100644 --- a/tests/script/windows/insert/query_multi_file.sim +++ b/tests/script/windows/insert/query_multi_file.sim @@ -1,6 +1,13 @@ sql connect sleep 3000 +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $i = 0 $dbPrefix = tb_mf_db $tbPrefix = tb_mf_tb diff --git a/tests/script/windows/table/binary.sim b/tests/script/windows/table/binary.sim index 69354ed5c8..64a081c72f 100644 --- a/tests/script/windows/table/binary.sim +++ b/tests/script/windows/table/binary.sim @@ -1,6 +1,13 @@ sql connect sleep 3000 +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $i = 0 $dbPrefix = lm_bn_db $tbPrefix = lm_bn_tb diff --git a/tests/script/windows/table/bool.sim b/tests/script/windows/table/bool.sim index 9e434d801a..9486c42221 100644 --- a/tests/script/windows/table/bool.sim +++ b/tests/script/windows/table/bool.sim @@ -1,6 +1,13 @@ sql connect sleep 3000 +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $i = 0 $dbPrefix = lm_bo_db $tbPrefix = lm_bo_tb diff --git a/tests/script/windows/table/column_name.sim b/tests/script/windows/table/column_name.sim index bbac293fed..fffb1334e5 100644 --- a/tests/script/windows/table/column_name.sim +++ b/tests/script/windows/table/column_name.sim @@ -1,6 +1,14 @@ sql connect sleep 3000 +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + + $i = 0 $dbPrefix = lm_cm_db $tbPrefix = lm_cm_tb diff --git a/tests/script/windows/table/column_num.sim b/tests/script/windows/table/column_num.sim index f7ead41437..d182696ce0 100644 --- a/tests/script/windows/table/column_num.sim +++ b/tests/script/windows/table/column_num.sim @@ -1,6 +1,14 @@ sql connect sleep 3000 +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + + $i = 0 $dbPrefix = lm_cn_db $tbPrefix = lm_cn_tb diff --git a/tests/script/windows/table/column_value.sim b/tests/script/windows/table/column_value.sim index 9dbaf7ceab..c59e7af8ba 100644 --- a/tests/script/windows/table/column_value.sim +++ b/tests/script/windows/table/column_value.sim @@ -1,6 +1,14 @@ sql connect sleep 3000 +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + + $i = 0 $dbPrefix = lm_cv_db $tbPrefix = lm_cv_tb diff --git a/tests/script/windows/table/db.table.sim b/tests/script/windows/table/db.table.sim index 8d244e011f..97a9e6fbe9 100644 --- a/tests/script/windows/table/db.table.sim +++ b/tests/script/windows/table/db.table.sim @@ -1,6 +1,14 @@ sql connect sleep 3000 +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + + $i = 0 $dbPrefix = lm_dt_db $tbPrefix = lm_dt_tb diff --git a/tests/script/windows/table/double.sim b/tests/script/windows/table/double.sim index 1402982c98..93bf3bb149 100644 --- a/tests/script/windows/table/double.sim +++ b/tests/script/windows/table/double.sim @@ -1,6 +1,13 @@ sql connect sleep 3000 +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $i = 0 $dbPrefix = lm_do_db $tbPrefix = lm_do_tb diff --git a/tests/script/windows/table/float.sim b/tests/script/windows/table/float.sim index 57b626f865..684f78a386 100644 --- a/tests/script/windows/table/float.sim +++ b/tests/script/windows/table/float.sim @@ -1,6 +1,13 @@ sql connect sleep 3000 +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $i = 0 $dbPrefix = lm_fl_db $tbPrefix = lm_fl_tb diff --git a/tests/script/windows/table/table.sim b/tests/script/windows/table/table.sim index 55be8af851..985620152a 100644 --- a/tests/script/windows/table/table.sim +++ b/tests/script/windows/table/table.sim @@ -2,6 +2,13 @@ sql connect sleep 3000 print ============================ dnode1 start +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $i = 0 $dbPrefix = ob_tb_db $tbPrefix = ob_tb_tb diff --git a/tests/script/windows/table/table_len.sim b/tests/script/windows/table/table_len.sim index cdd1f31731..367f1c6895 100644 --- a/tests/script/windows/table/table_len.sim +++ b/tests/script/windows/table/table_len.sim @@ -1,6 +1,13 @@ sql connect sleep 3000 +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $i = 0 $dbPrefix = lm_tb_db $tbPrefix = lm_tb_tb diff --git a/tests/script/windows/tag/3.sim b/tests/script/windows/tag/3.sim index 9ffa11b03f..63a8766727 100644 --- a/tests/script/windows/tag/3.sim +++ b/tests/script/windows/tag/3.sim @@ -2,6 +2,13 @@ sql connect sleep 3000 print ======================== dnode1 start +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = ta_3_db $tbPrefix = ta_3_tb $mtPrefix = ta_3_mt diff --git a/tests/script/windows/tag/4.sim b/tests/script/windows/tag/4.sim index beabe1fd8f..7e9af7ece7 100644 --- a/tests/script/windows/tag/4.sim +++ b/tests/script/windows/tag/4.sim @@ -2,6 +2,13 @@ sql connect sleep 3000 print ======================== dnode1 start +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = ta_4_db $tbPrefix = ta_4_tb $mtPrefix = ta_4_mt diff --git a/tests/script/windows/tag/5.sim b/tests/script/windows/tag/5.sim index 161d98756c..5dc128a0e0 100644 --- a/tests/script/windows/tag/5.sim +++ b/tests/script/windows/tag/5.sim @@ -2,6 +2,13 @@ sql connect sleep 3000 print ======================== dnode1 start +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = ta_5_db $tbPrefix = ta_5_tb $mtPrefix = ta_5_mt diff --git a/tests/script/windows/tag/6.sim b/tests/script/windows/tag/6.sim index b8666305bd..12e9c597f0 100644 --- a/tests/script/windows/tag/6.sim +++ b/tests/script/windows/tag/6.sim @@ -2,6 +2,13 @@ sql connect sleep 3000 print ======================== dnode1 start +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = ta_6_db $tbPrefix = ta_6_tb $mtPrefix = ta_6_mt diff --git a/tests/script/windows/tag/add.sim b/tests/script/windows/tag/add.sim index 2c72d01955..0a1416b68c 100644 --- a/tests/script/windows/tag/add.sim +++ b/tests/script/windows/tag/add.sim @@ -2,6 +2,13 @@ sql connect sleep 3000 print ======================== dnode1 start +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = ta_ad_db $tbPrefix = ta_ad_tb $mtPrefix = ta_ad_mt diff --git a/tests/script/windows/tag/bigint.sim b/tests/script/windows/tag/bigint.sim index 4406c7386d..d988ad1fdc 100644 --- a/tests/script/windows/tag/bigint.sim +++ b/tests/script/windows/tag/bigint.sim @@ -2,6 +2,13 @@ sql connect sleep 3000 print ======================== dnode1 start +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = ta_bi_db $tbPrefix = ta_bi_tb $mtPrefix = ta_bi_mt diff --git a/tests/script/windows/tag/binary.sim b/tests/script/windows/tag/binary.sim index deeae81117..9dc18cfa94 100644 --- a/tests/script/windows/tag/binary.sim +++ b/tests/script/windows/tag/binary.sim @@ -2,6 +2,13 @@ sql connect sleep 3000 print ======================== dnode1 start +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = ta_by_db $tbPrefix = ta_by_tb $mtPrefix = ta_by_mt diff --git a/tests/script/windows/tag/binary_binary.sim b/tests/script/windows/tag/binary_binary.sim index c1f93bc656..ba688aa80e 100644 --- a/tests/script/windows/tag/binary_binary.sim +++ b/tests/script/windows/tag/binary_binary.sim @@ -2,6 +2,13 @@ sql connect sleep 3000 print ======================== dnode1 start +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = ta_bib_db $tbPrefix = ta_bib_tb $mtPrefix = ta_bib_mt diff --git a/tests/script/windows/tag/bool.sim b/tests/script/windows/tag/bool.sim index 81ea20064b..a7e5d909c5 100644 --- a/tests/script/windows/tag/bool.sim +++ b/tests/script/windows/tag/bool.sim @@ -2,6 +2,13 @@ sql connect sleep 3000 print ======================== dnode1 start +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = ta_bo_db $tbPrefix = ta_bo_tb $mtPrefix = ta_bo_mt diff --git a/tests/script/windows/tag/bool_binary.sim b/tests/script/windows/tag/bool_binary.sim index c3daf2c242..639f6c5f2f 100644 --- a/tests/script/windows/tag/bool_binary.sim +++ b/tests/script/windows/tag/bool_binary.sim @@ -2,6 +2,13 @@ sql connect sleep 3000 print ======================== dnode1 start +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = ta_bob_db $tbPrefix = ta_bob_tb $mtPrefix = ta_bob_mt diff --git a/tests/script/windows/tag/bool_int.sim b/tests/script/windows/tag/bool_int.sim index 79e4b67bfa..900cc9e8a1 100644 --- a/tests/script/windows/tag/bool_int.sim +++ b/tests/script/windows/tag/bool_int.sim @@ -2,6 +2,13 @@ sql connect sleep 3000 print ======================== dnode1 start +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = ta_boi_db $tbPrefix = ta_boi_tb $mtPrefix = ta_boi_mt diff --git a/tests/script/windows/tag/change.sim b/tests/script/windows/tag/change.sim index 2901842190..75a976bbb1 100644 --- a/tests/script/windows/tag/change.sim +++ b/tests/script/windows/tag/change.sim @@ -2,6 +2,13 @@ sql connect sleep 3000 print ======================== dnode1 start +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = ta_ch_db $tbPrefix = ta_ch_tb $mtPrefix = ta_ch_mt diff --git a/tests/script/windows/tag/column.sim b/tests/script/windows/tag/column.sim index 131f3e06ea..9f5bfce07e 100644 --- a/tests/script/windows/tag/column.sim +++ b/tests/script/windows/tag/column.sim @@ -2,6 +2,13 @@ sql connect sleep 3000 print ======================== dnode1 start +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = ta_co_db $tbPrefix = ta_co_tb $mtPrefix = ta_co_mt diff --git a/tests/script/windows/tag/create.sim b/tests/script/windows/tag/create.sim index 5beba21727..6a76c93d83 100644 --- a/tests/script/windows/tag/create.sim +++ b/tests/script/windows/tag/create.sim @@ -2,6 +2,13 @@ sql connect sleep 3000 print ======================== dnode1 start +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = ta_cr_db $tbPrefix = ta_cr_tb $mtPrefix = ta_cr_mt diff --git a/tests/script/windows/tag/delete.sim b/tests/script/windows/tag/delete.sim index e2395c8f97..9e8ea9aba0 100644 --- a/tests/script/windows/tag/delete.sim +++ b/tests/script/windows/tag/delete.sim @@ -2,6 +2,13 @@ sql connect sleep 3000 print ======================== dnode1 start +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = ta_de_db $tbPrefix = ta_de_tb $mtPrefix = ta_de_mt diff --git a/tests/script/windows/tag/double.sim b/tests/script/windows/tag/double.sim index c08351a41b..5445b1dbea 100644 --- a/tests/script/windows/tag/double.sim +++ b/tests/script/windows/tag/double.sim @@ -2,6 +2,13 @@ sql connect sleep 3000 print ======================== dnode1 start +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = ta_do_db $tbPrefix = ta_do_tb $mtPrefix = ta_do_mt diff --git a/tests/script/windows/tag/filter.sim b/tests/script/windows/tag/filter.sim index b70e56fdb6..f704f32cd2 100644 --- a/tests/script/windows/tag/filter.sim +++ b/tests/script/windows/tag/filter.sim @@ -2,6 +2,13 @@ sql connect sleep 3000 print ======================== dnode1 start +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = ta_fi_db $tbPrefix = ta_fi_tb $mtPrefix = ta_fi_mt diff --git a/tests/script/windows/tag/float.sim b/tests/script/windows/tag/float.sim index 79eabb2cb4..64424c1e20 100644 --- a/tests/script/windows/tag/float.sim +++ b/tests/script/windows/tag/float.sim @@ -2,6 +2,13 @@ sql connect sleep 3000 print ======================== dnode1 start +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = ta_fl_db $tbPrefix = ta_fl_tb $mtPrefix = ta_fl_mt diff --git a/tests/script/windows/tag/int.sim b/tests/script/windows/tag/int.sim index d3921218fd..7d7b5271d1 100644 --- a/tests/script/windows/tag/int.sim +++ b/tests/script/windows/tag/int.sim @@ -2,6 +2,13 @@ sql connect sleep 3000 print ======================== dnode1 start +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = ta_in_db $tbPrefix = ta_in_tb $mtPrefix = ta_in_mt diff --git a/tests/script/windows/tag/int_binary.sim b/tests/script/windows/tag/int_binary.sim index 96f4f18966..1dd4771605 100644 --- a/tests/script/windows/tag/int_binary.sim +++ b/tests/script/windows/tag/int_binary.sim @@ -2,6 +2,13 @@ sql connect sleep 3000 print ======================== dnode1 start +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = ta_inb_db $tbPrefix = ta_inb_tb $mtPrefix = ta_inb_mt diff --git a/tests/script/windows/tag/int_float.sim b/tests/script/windows/tag/int_float.sim index 768e86b96d..cdb9032d8c 100644 --- a/tests/script/windows/tag/int_float.sim +++ b/tests/script/windows/tag/int_float.sim @@ -2,6 +2,13 @@ sql connect sleep 3000 print ======================== dnode1 start +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = ta_inf_db $tbPrefix = ta_inf_tb $mtPrefix = ta_inf_mt diff --git a/tests/script/windows/tag/set.sim b/tests/script/windows/tag/set.sim index 580f91cb49..16103c6ce8 100644 --- a/tests/script/windows/tag/set.sim +++ b/tests/script/windows/tag/set.sim @@ -2,6 +2,13 @@ sql connect sleep 3000 print ======================== dnode1 start +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = ta_se_db $tbPrefix = ta_se_tb $mtPrefix = ta_se_mt diff --git a/tests/script/windows/tag/smallint.sim b/tests/script/windows/tag/smallint.sim index 1b7ff0860b..dbab4f2d43 100644 --- a/tests/script/windows/tag/smallint.sim +++ b/tests/script/windows/tag/smallint.sim @@ -2,6 +2,13 @@ sql connect sleep 3000 print ======================== dnode1 start +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = ta_sm_db $tbPrefix = ta_sm_tb $mtPrefix = ta_sm_mt diff --git a/tests/script/windows/tag/tinyint.sim b/tests/script/windows/tag/tinyint.sim index 8fe957664c..7a0237c0d9 100644 --- a/tests/script/windows/tag/tinyint.sim +++ b/tests/script/windows/tag/tinyint.sim @@ -2,6 +2,13 @@ sql connect sleep 3000 print ======================== dnode1 start +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = ta_ti_db $tbPrefix = ta_ti_tb $mtPrefix = ta_ti_mt diff --git a/tests/script/windows/testSuite.sim b/tests/script/windows/testSuite.sim index fc574ed9c4..e372217b62 100644 --- a/tests/script/windows/testSuite.sim +++ b/tests/script/windows/testSuite.sim @@ -1,14 +1,14 @@ run windows/alter/table.sim run windows/alter/metrics.sim -#run windows/compute/avg.sim +run windows/compute/avg.sim run windows/compute/bottom.sim run windows/compute/count.sim run windows/compute/diff.sim run windows/compute/first.sim run windows/compute/interval.sim run windows/compute/last.sim -##run windows/compute/leastsquare.sim +run windows/compute/leastsquare.sim run windows/compute/max.sim run windows/compute/min.sim run windows/compute/percentile.sim @@ -37,12 +37,12 @@ run windows/field/tinyint.sim run windows/import/basic.sim run windows/insert/basic.sim -#run windows/insert/query_block1_file.sim -#run windows/insert/query_block1_memory.sim -#run windows/insert/query_block2_file.sim -#run windows/insert/query_block2_memory.sim -#run windows/insert/query_file_memory.sim -#run windows/insert/query_multi_file.sim +run windows/insert/query_block1_file.sim +run windows/insert/query_block1_memory.sim +run windows/insert/query_block2_file.sim +run windows/insert/query_block2_memory.sim +run windows/insert/query_file_memory.sim +run windows/insert/query_multi_file.sim run windows/table/binary.sim run windows/table/bool.sim diff --git a/tests/script/windows/vector/metrics_field.sim b/tests/script/windows/vector/metrics_field.sim index 8e9a87239e..e7c926e141 100644 --- a/tests/script/windows/vector/metrics_field.sim +++ b/tests/script/windows/vector/metrics_field.sim @@ -1,6 +1,13 @@ sql connect sleep 3000 +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = m_mf_db $tbPrefix = m_mf_tb $mtPrefix = m_mf_mt diff --git a/tests/script/windows/vector/metrics_mix.sim b/tests/script/windows/vector/metrics_mix.sim index 0f960deb4d..3d94a96385 100644 --- a/tests/script/windows/vector/metrics_mix.sim +++ b/tests/script/windows/vector/metrics_mix.sim @@ -1,6 +1,13 @@ sql connect sleep 3000 +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = m_mx_db $tbPrefix = m_mx_tb $mtPrefix = m_mx_mt diff --git a/tests/script/windows/vector/metrics_query.sim b/tests/script/windows/vector/metrics_query.sim index a0df4c9b04..c292c6b306 100644 --- a/tests/script/windows/vector/metrics_query.sim +++ b/tests/script/windows/vector/metrics_query.sim @@ -1,6 +1,13 @@ sql connect sleep 3000 +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = m_mq_db $tbPrefix = m_mq_tb $mtPrefix = m_mq_mt diff --git a/tests/script/windows/vector/metrics_tag.sim b/tests/script/windows/vector/metrics_tag.sim index 22fd19bc89..f51a449d71 100644 --- a/tests/script/windows/vector/metrics_tag.sim +++ b/tests/script/windows/vector/metrics_tag.sim @@ -1,6 +1,13 @@ sql connect sleep 3000 +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = m_mtg_db $tbPrefix = m_mtg_tb $mtPrefix = m_mtg_mt diff --git a/tests/script/windows/vector/metrics_time.sim b/tests/script/windows/vector/metrics_time.sim index da102b64a3..716f49d1e5 100644 --- a/tests/script/windows/vector/metrics_time.sim +++ b/tests/script/windows/vector/metrics_time.sim @@ -1,6 +1,13 @@ sql connect sleep 3000 +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = m_mt_db $tbPrefix = m_mt_tb $mtPrefix = m_mt_mt diff --git a/tests/script/windows/vector/multi.sim b/tests/script/windows/vector/multi.sim index adcc94db3b..415384d243 100644 --- a/tests/script/windows/vector/multi.sim +++ b/tests/script/windows/vector/multi.sim @@ -1,6 +1,13 @@ sql connect sleep 3000 +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = m_mu_db $tbPrefix = m_mu_tb $mtPrefix = m_mu_mt diff --git a/tests/script/windows/vector/single.sim b/tests/script/windows/vector/single.sim index 61135fc6b5..f3f3862e54 100644 --- a/tests/script/windows/vector/single.sim +++ b/tests/script/windows/vector/single.sim @@ -1,6 +1,13 @@ sql connect sleep 3000 +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = m_si_db $tbPrefix = m_si_tb $mtPrefix = m_si_mt diff --git a/tests/script/windows/vector/table_field.sim b/tests/script/windows/vector/table_field.sim index ec50bc7a2a..0c5df695fb 100644 --- a/tests/script/windows/vector/table_field.sim +++ b/tests/script/windows/vector/table_field.sim @@ -1,6 +1,13 @@ sql connect sleep 3000 +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = m_tf_db $tbPrefix = m_tf_tb $mtPrefix = m_tf_mt diff --git a/tests/script/windows/vector/table_mix.sim b/tests/script/windows/vector/table_mix.sim index 653171b302..3d660b5611 100644 --- a/tests/script/windows/vector/table_mix.sim +++ b/tests/script/windows/vector/table_mix.sim @@ -1,6 +1,13 @@ sql connect sleep 3000 +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = m_tm_db $tbPrefix = m_tm_tb $mtPrefix = m_tm_mt diff --git a/tests/script/windows/vector/table_query.sim b/tests/script/windows/vector/table_query.sim index cdbd96f4d0..27fcd0f654 100644 --- a/tests/script/windows/vector/table_query.sim +++ b/tests/script/windows/vector/table_query.sim @@ -1,6 +1,13 @@ sql connect sleep 3000 +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = m_tq_db $tbPrefix = m_tq_tb $mtPrefix = m_tq_mt diff --git a/tests/script/windows/vector/table_time.sim b/tests/script/windows/vector/table_time.sim index 48bcf86dca..8a3804c619 100644 --- a/tests/script/windows/vector/table_time.sim +++ b/tests/script/windows/vector/table_time.sim @@ -1,6 +1,13 @@ sql connect sleep 3000 +sql show databases +sql drop database $data00 -x e1 +e1: +sql show databases +sql drop database $data00 -x e2 +e2: + $dbPrefix = m_tt_db $tbPrefix = m_tt_tb $mtPrefix = m_tt_mt From d79cad4b641fbf66fdebd876edd14830e8c4f990 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 10 Aug 2020 14:34:50 +0000 Subject: [PATCH 109/109] TD-1057 --- tests/script/{test.bat => wtest.bat} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/script/{test.bat => wtest.bat} (100%) diff --git a/tests/script/test.bat b/tests/script/wtest.bat similarity index 100% rename from tests/script/test.bat rename to tests/script/wtest.bat