From e2b5d27c2a7ec71109c2f90b66d58afc613f3df2 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 11 Apr 2023 10:22:47 +0800 Subject: [PATCH 01/12] enh: stmt ignore get fields errors --- source/client/inc/clientStmt.h | 9 +++++++++ source/client/src/clientStmt.c | 30 ++++++++++++++++++------------ 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/source/client/inc/clientStmt.h b/source/client/inc/clientStmt.h index 0c9696f1c8..cbef80b6da 100644 --- a/source/client/inc/clientStmt.h +++ b/source/client/inc/clientStmt.h @@ -144,6 +144,15 @@ extern char *gStmtStatusStr[]; goto _return; \ } \ } while (0) +#define STMT_ERRI_JRET(c) \ + do { \ + code = c; \ + if (code != TSDB_CODE_SUCCESS) { \ + terrno = code; \ + goto _return; \ + } \ + } while (0) + #define STMT_ELOG(param, ...) qError("stmt:%p " param, pStmt, __VA_ARGS__) #define STMT_DLOG(param, ...) qDebug("stmt:%p " param, pStmt, __VA_ARGS__) diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index 71a41c68e7..ac60a069eb 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -975,15 +975,16 @@ int stmtIsInsert(TAOS_STMT* stmt, int* insert) { } int stmtGetTagFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields) { + int32_t code = 0; STscStmt* pStmt = (STscStmt*)stmt; STMT_DLOG_E("start to get tag fields"); if (STMT_TYPE_QUERY == pStmt->sql.type) { - STMT_RET(TSDB_CODE_TSC_STMT_API_ERROR); + STMT_ERRI_JRET(TSDB_CODE_TSC_STMT_API_ERROR); } - STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_FETCH_FIELDS)); + STMT_ERRI_JRET(stmtSwitchStatus(pStmt, STMT_FETCH_FIELDS)); if (pStmt->bInfo.needParse && pStmt->sql.runTimes && pStmt->sql.type > 0 && STMT_TYPE_MULTI_INSERT != pStmt->sql.type) { @@ -995,27 +996,30 @@ int stmtGetTagFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields) { pStmt->exec.pRequest = NULL; } - STMT_ERR_RET(stmtCreateRequest(pStmt)); + STMT_ERRI_JRET(stmtCreateRequest(pStmt)); if (pStmt->bInfo.needParse) { - STMT_ERR_RET(stmtParseSql(pStmt)); + STMT_ERRI_JRET(stmtParseSql(pStmt)); } - STMT_ERR_RET(stmtFetchTagFields(stmt, nums, fields)); + STMT_ERRI_JRET(stmtFetchTagFields(stmt, nums, fields)); - return TSDB_CODE_SUCCESS; +_return: + + return code; } int stmtGetColFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields) { + int32_t code = 0; STscStmt* pStmt = (STscStmt*)stmt; STMT_DLOG_E("start to get col fields"); if (STMT_TYPE_QUERY == pStmt->sql.type) { - STMT_RET(TSDB_CODE_TSC_STMT_API_ERROR); + STMT_ERRI_JRET(TSDB_CODE_TSC_STMT_API_ERROR); } - STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_FETCH_FIELDS)); + STMT_ERRI_JRET(stmtSwitchStatus(pStmt, STMT_FETCH_FIELDS)); if (pStmt->bInfo.needParse && pStmt->sql.runTimes && pStmt->sql.type > 0 && STMT_TYPE_MULTI_INSERT != pStmt->sql.type) { @@ -1027,15 +1031,17 @@ int stmtGetColFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields) { pStmt->exec.pRequest = NULL; } - STMT_ERR_RET(stmtCreateRequest(pStmt)); + STMT_ERRI_JRET(stmtCreateRequest(pStmt)); if (pStmt->bInfo.needParse) { - STMT_ERR_RET(stmtParseSql(pStmt)); + STMT_ERRI_JRET(stmtParseSql(pStmt)); } - STMT_ERR_RET(stmtFetchColFields(stmt, nums, fields)); + STMT_ERRI_JRET(stmtFetchColFields(stmt, nums, fields)); - return TSDB_CODE_SUCCESS; +_return: + + return code; } int stmtGetParamNum(TAOS_STMT* stmt, int* nums) { From 8060e77be35e4b9e281e057a9d32a94aecff6078 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 11 Apr 2023 10:41:35 +0800 Subject: [PATCH 02/12] fix(udf): make strncat working for release building --- source/libs/function/src/udfd.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index c368788243..e444a974f9 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -571,13 +571,13 @@ int32_t udfdLoadUdf(char *udfName, SUdf *udf) { char initFuncName[TSDB_FUNC_NAME_LEN + 5] = {0}; char *initSuffix = "_init"; strcpy(initFuncName, udfName); - strncat(initFuncName, initSuffix, strlen(initSuffix)); + strncat(initFuncName, initSuffix, TSDB_FUNC_NAME_LEN + 5 - strlen(initFuncName) - 1); uv_dlsym(&udf->lib, initFuncName, (void **)(&udf->initFunc)); char destroyFuncName[TSDB_FUNC_NAME_LEN + 5] = {0}; char *destroySuffix = "_destroy"; strcpy(destroyFuncName, udfName); - strncat(destroyFuncName, destroySuffix, strlen(destroySuffix)); + strncat(destroyFuncName, destroySuffix, TSDB_FUNC_NAME_LEN + 5 - strlen(destroyFuncName) - 1); uv_dlsym(&udf->lib, destroyFuncName, (void **)(&udf->destroyFunc)); if (udf->funcType == TSDB_FUNC_TYPE_SCALAR) { @@ -591,17 +591,17 @@ int32_t udfdLoadUdf(char *udfName, SUdf *udf) { char startFuncName[TSDB_FUNC_NAME_LEN + 6] = {0}; char *startSuffix = "_start"; strncpy(startFuncName, processFuncName, sizeof(startFuncName)); - strncat(startFuncName, startSuffix, strlen(startSuffix)); + strncat(startFuncName, startSuffix, TSDB_FUNC_NAME_LEN + 6 - strlen(startSuffix) - 1); uv_dlsym(&udf->lib, startFuncName, (void **)(&udf->aggStartFunc)); char finishFuncName[TSDB_FUNC_NAME_LEN + 7] = {0}; char *finishSuffix = "_finish"; strncpy(finishFuncName, processFuncName, sizeof(finishFuncName)); - strncat(finishFuncName, finishSuffix, strlen(finishSuffix)); + strncat(finishFuncName, finishSuffix, TSDB_FUNC_NAME_LEN + 7 - strlen(finishFuncName) - 1); uv_dlsym(&udf->lib, finishFuncName, (void **)(&udf->aggFinishFunc)); char mergeFuncName[TSDB_FUNC_NAME_LEN + 6] = {0}; char *mergeSuffix = "_merge"; strncpy(mergeFuncName, processFuncName, sizeof(mergeFuncName)); - strncat(mergeFuncName, mergeSuffix, strlen(mergeSuffix)); + strncat(mergeFuncName, mergeSuffix, TSDB_FUNC_NAME_LEN + 6 - strlen(mergeFuncName) - 1); uv_dlsym(&udf->lib, mergeFuncName, (void **)(&udf->aggMergeFunc)); } return 0; From d99a849956536825b71e0530564d3f075b12815e Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 11 Apr 2023 10:47:15 +0800 Subject: [PATCH 03/12] fix: ignore stmt get fields error --- source/client/src/clientStmt.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index ac60a069eb..6e529f1a0b 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -977,6 +977,7 @@ int stmtIsInsert(TAOS_STMT* stmt, int* insert) { int stmtGetTagFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields) { int32_t code = 0; STscStmt* pStmt = (STscStmt*)stmt; + int32_t preCode = pStmt->errCode; STMT_DLOG_E("start to get tag fields"); @@ -1006,12 +1007,15 @@ int stmtGetTagFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields) { _return: + pStmt->errCode = preCode; + return code; } int stmtGetColFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields) { int32_t code = 0; STscStmt* pStmt = (STscStmt*)stmt; + int32_t preCode = pStmt->errCode; STMT_DLOG_E("start to get col fields"); @@ -1041,6 +1045,8 @@ int stmtGetColFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields) { _return: + pStmt->errCode = preCode; + return code; } From c40c695ea74b9af6427431481c705fd3631544a4 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 11 Apr 2023 13:49:18 +0800 Subject: [PATCH 04/12] fix:optimize log & change the length of tag if tag is null in schemaless --- source/client/src/clientSml.c | 2 +- source/dnode/mgmt/mgmt_vnode/src/vmWorker.c | 4 ++-- source/dnode/vnode/src/tq/tqRead.c | 3 +-- source/libs/executor/src/executor.c | 7 ++++++- source/libs/executor/src/executorimpl.c | 2 ++ source/libs/wal/src/walRead.c | 10 +--------- 6 files changed, 13 insertions(+), 15 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 4c28ee76c3..2e040052c3 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -699,7 +699,7 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, pReq.numOfTags = 1; SField field = {0}; field.type = TSDB_DATA_TYPE_NCHAR; - field.bytes = 1; + field.bytes = TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE; strcpy(field.name, tsSmlTagName); taosArrayPush(pReq.pTags, &field); } diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index e4e0d608de..756ca008b0 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -114,11 +114,11 @@ static void vmProcessFetchQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO int32_t code = vnodeProcessFetchMsg(pVnode->pImpl, pMsg, pInfo); if (code != 0) { if (terrno != 0) code = terrno; - dGError("vgId:%d, msg:%p failed to fetch since %s", pVnode->vgId, pMsg, terrstr(code)); + dGError("vnodeProcessFetchMsg vgId:%d, msg:%p failed to fetch since %s", pVnode->vgId, pMsg, terrstr()); vmSendRsp(pMsg, code); } - dGTrace("vgId:%d, msg:%p is freed, code:0x%x", pVnode->vgId, pMsg, code); + dGTrace("vnodeProcessFetchMsg vgId:%d, msg:%p is freed, code:0x%x", pVnode->vgId, pMsg, code); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 54e4e393ec..72b478e6bf 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -296,10 +296,9 @@ void tqCloseReader(STqReader* pReader) { int32_t tqSeekVer(STqReader* pReader, int64_t ver, const char* id) { if (walReadSeekVer(pReader->pWalReader, ver) < 0) { - tqDebug("tmq poll: wal reader failed to seek to ver:%"PRId64" code:%s, %s", ver, tstrerror(terrno), id); return -1; } - tqDebug("tmq poll: wal reader seek to ver:%"PRId64" %s", ver, id); + tqDebug("tmq poll: wal reader seek to ver success ver:%"PRId64" %s", ver, id); return 0; } diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 1670eb3c59..cd7a76f7ad 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -1062,6 +1062,7 @@ int32_t qStreamSetScanMemData(qTaskInfo_t tinfo, SPackedData submit) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; if((pTaskInfo->execModel != OPTR_EXEC_MODEL_QUEUE) || (pTaskInfo->streamInfo.submit.msgStr != NULL)){ qError("qStreamSetScanMemData err:%d,%p", pTaskInfo->execModel, pTaskInfo->streamInfo.submit.msgStr); + terrno = TSDB_CODE_PAR_INTERNAL_ERROR; return -1; } qDebug("set the submit block for future scan"); @@ -1102,7 +1103,6 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT // let's seek to the next version in wal file if (tqSeekVer(pInfo->tqReader, pOffset->version + 1, id) < 0) { - qError("tqSeekVer failed ver:%"PRId64", %s", pOffset->version + 1, id); return -1; } } else if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) { @@ -1125,6 +1125,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT } else { taosRUnLockLatch(&pTaskInfo->lock); qError("no table in table list, %s", id); + terrno = TSDB_CODE_PAR_INTERNAL_ERROR; return -1; } } @@ -1143,6 +1144,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT } else { qError("vgId:%d uid:%" PRIu64 " not found in table list, total:%d, index:%d %s", pTaskInfo->id.vgId, uid, numOfTables, pScanInfo->currentTable, id); + terrno = TSDB_CODE_PAR_INTERNAL_ERROR; return -1; } @@ -1175,6 +1177,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT pScanBaseInfo->cond.twindows.skey = oldSkey; } else { qError("invalid pOffset->type:%d, %s", pOffset->type, id); + terrno = TSDB_CODE_PAR_INTERNAL_ERROR; return -1; } @@ -1189,6 +1192,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT if (setForSnapShot(sContext, pOffset->uid) != 0) { qError("setDataForSnapShot error. uid:%" PRId64" , %s", pOffset->uid, id); + terrno = TSDB_CODE_PAR_INTERNAL_ERROR; return -1; } @@ -1224,6 +1228,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT SSnapContext* sContext = pInfo->sContext; if (setForSnapShot(sContext, pOffset->uid) != 0) { qError("setForSnapShot error. uid:%" PRIu64 " ,version:%" PRId64, pOffset->uid, pOffset->version); + terrno = TSDB_CODE_PAR_INTERNAL_ERROR; return -1; } qDebug("tmqsnap qStreamPrepareScan snapshot meta uid:%" PRId64 " ts %" PRId64 " %s", pOffset->uid, pOffset->ts, id); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 11753c181c..4e44e0e223 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -999,6 +999,7 @@ int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scan SOperatorInfo* extractOperatorInTree(SOperatorInfo* pOperator, int32_t type, const char* id) { if (pOperator == NULL) { qError("invalid operator, failed to find tableScanOperator %s", id); + terrno = TSDB_CODE_PAR_INTERNAL_ERROR; return NULL; } @@ -1007,6 +1008,7 @@ SOperatorInfo* extractOperatorInTree(SOperatorInfo* pOperator, int32_t type, con } else { if (pOperator->pDownstream == NULL || pOperator->pDownstream[0] == NULL) { qError("invalid operator, failed to find tableScanOperator %s", id); + terrno = TSDB_CODE_PAR_INTERNAL_ERROR; return NULL; } diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index ad6127ead2..db4e3a4759 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -207,17 +207,12 @@ int32_t walReadSeekVer(SWalReader *pReader, int64_t ver) { return 0; } -// pReader->curInvalid = 1; -// pReader->curVersion = ver; - if (ver > pWal->vers.lastVer || ver < pWal->vers.firstVer) { - wDebug("vgId:%d, invalid index:%" PRId64 ", first index:%" PRId64 ", last index:%" PRId64, pReader->pWal->cfg.vgId, + wInfo("vgId:%d, invalid index:%" PRId64 ", first index:%" PRId64 ", last index:%" PRId64, pReader->pWal->cfg.vgId, ver, pWal->vers.firstVer, pWal->vers.lastVer); terrno = TSDB_CODE_WAL_LOG_NOT_EXIST; return -1; } -// if (ver < pWal->vers.snapshotVer) { -// } if (walReadSeekVerImpl(pReader, ver) < 0) { return -1; @@ -236,8 +231,6 @@ static int32_t walFetchHeadNew(SWalReader *pRead, int64_t fetchVer) { if (pRead->curVersion != fetchVer) { if (walReadSeekVer(pRead, fetchVer) < 0) { -// pRead->curVersion = fetchVer; -// pRead->curInvalid = 1; return -1; } seeked = true; @@ -256,7 +249,6 @@ static int32_t walFetchHeadNew(SWalReader *pRead, int64_t fetchVer) { } else { terrno = TSDB_CODE_WAL_FILE_CORRUPTED; } -// pRead->curInvalid = 1; return -1; } } From 42a5a87e964b4d54b0cbfaa7b1bc14f7de4f7ea1 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Tue, 11 Apr 2023 15:08:42 +0800 Subject: [PATCH 05/12] fix: table level privilege --- source/dnode/mnode/impl/src/mndUser.c | 31 ++++++++++++++++++++++++--- source/libs/catalog/src/ctgUtil.c | 3 ++- source/libs/parser/src/parInsertSql.c | 2 +- 3 files changed, 31 insertions(+), 5 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 34914d80f0..3a1c4ce58f 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -156,7 +156,7 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) { size_t valueLen = 0; valueLen = strlen(stb); size += sizeof(int32_t); - size += keyLen; + size += valueLen; stb = taosHashIterate(pUser->writeTbs, stb); } @@ -369,7 +369,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { int32_t valuelen = 0; SDB_GET_INT32(pRaw, dataPos, &valuelen, _OVER); char *value = taosMemoryCalloc(valuelen, sizeof(char)); - memset(value, 0, keyLen); + memset(value, 0, valuelen); SDB_GET_BINARY(pRaw, dataPos, value, valuelen, _OVER) taosHashPut(pUser->writeTbs, key, keyLen, value, valuelen); @@ -458,6 +458,31 @@ SHashObj *mndDupTableHash(SHashObj *pOld) { return pNew; } +SHashObj *mndDupUseDbHash(SHashObj *pOld) { + SHashObj *pNew = + taosHashInit(taosHashGetSize(pOld), taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + if (pNew == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + int32_t *db = taosHashIterate(pOld, NULL); + while (db != NULL) { + size_t keyLen = 0; + char *key = taosHashGetKey(db, &keyLen); + + if (taosHashPut(pNew, key, keyLen, db, sizeof(*db)) != 0) { + taosHashCancelIterate(pOld, db); + taosHashCleanup(pNew); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + db = taosHashIterate(pOld, db); + } + + return pNew; +} + static int32_t mndUserDupObj(SUserObj *pUser, SUserObj *pNew) { memcpy(pNew, pUser, sizeof(SUserObj)); pNew->authVersion++; @@ -469,7 +494,7 @@ static int32_t mndUserDupObj(SUserObj *pUser, SUserObj *pNew) { pNew->readTbs = mndDupTableHash(pUser->readTbs); pNew->writeTbs = mndDupTableHash(pUser->writeTbs); pNew->topics = mndDupTopicHash(pUser->topics); - pNew->useDbs = mndDupDbHash(pUser->useDbs); + pNew->useDbs = mndDupUseDbHash(pUser->useDbs); taosRUnLockLatch(&pUser->lock); if (pNew->readDbs == NULL || pNew->writeDbs == NULL || pNew->topics == NULL) { diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index d02f94c939..f7c7522a88 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -1457,7 +1457,8 @@ int32_t ctgChkSetAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) { } case AUTH_TYPE_READ_OR_WRITE: { if ((pInfo->readDbs && taosHashGet(pInfo->readDbs, dbFName, strlen(dbFName))) || - (pInfo->writeDbs && taosHashGet(pInfo->writeDbs, dbFName, strlen(dbFName)))) { + (pInfo->writeDbs && taosHashGet(pInfo->writeDbs, dbFName, strlen(dbFName))) || + (pInfo->useDbs && taosHashGet(pInfo->useDbs, dbFName, strlen(dbFName)))) { pRes->pass = true; return TSDB_CODE_SUCCESS; } diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index d734479b98..83528288fc 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -2022,7 +2022,7 @@ static int32_t buildInsertUserAuthReq(const char* pUser, SName* pName, SArray** SUserAuthInfo userAuth = {.type = AUTH_TYPE_WRITE}; snprintf(userAuth.user, sizeof(userAuth.user), "%s", pUser); - // tNameGetFullDbName(pName, userAuth.dbFName); + memcpy(&userAuth.tbName, pName, sizeof(SName)); taosArrayPush(*pUserAuth, &userAuth); return TSDB_CODE_SUCCESS; From 171647bdbd5730b73097e45ba4ed5a0876f22884 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 11 Apr 2023 15:46:36 +0800 Subject: [PATCH 06/12] fix(shell): update the double display. --- tools/shell/src/shellEngine.c | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index f9602aaa4d..4798151d38 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -315,7 +315,6 @@ void shellDumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, i quotationStr[0] = '\"'; quotationStr[1] = 0; - int n; char buf[TSDB_MAX_BYTES_PER_ROW]; switch (field->type) { case TSDB_DATA_TYPE_BOOL: @@ -346,15 +345,11 @@ void shellDumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, i taosFprintfFile(pFile, "%" PRIu64, *((uint64_t *)val)); break; case TSDB_DATA_TYPE_FLOAT: - taosFprintfFile(pFile, "%.5f", GET_FLOAT_VAL(val)); + taosFprintfFile(pFile, "%e", GET_FLOAT_VAL(val)); break; case TSDB_DATA_TYPE_DOUBLE: - n = snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.9f", length, GET_DOUBLE_VAL(val)); - if (n > TMAX(25, length)) { - taosFprintfFile(pFile, "%*.15e", length, GET_DOUBLE_VAL(val)); - } else { - taosFprintfFile(pFile, "%s", buf); - } + snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.15e", 23, GET_DOUBLE_VAL(val)); + taosFprintfFile(pFile, "%s", buf); break; case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: @@ -510,7 +505,6 @@ void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t return; } - int n; char buf[TSDB_MAX_BYTES_PER_ROW]; switch (field->type) { case TSDB_DATA_TYPE_BOOL: @@ -541,15 +535,11 @@ void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t printf("%*" PRIu64, width, *((uint64_t *)val)); break; case TSDB_DATA_TYPE_FLOAT: - printf("%*ef", width, GET_FLOAT_VAL(val)); + printf("%*e", width, GET_FLOAT_VAL(val)); break; case TSDB_DATA_TYPE_DOUBLE: - n = snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.9f", width, GET_DOUBLE_VAL(val)); - if (n > TMAX(25, width)) { - printf("%*.15e", width, GET_DOUBLE_VAL(val)); - } else { - printf("%s", buf); - } + snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%.15e", GET_DOUBLE_VAL(val)); + printf("%*s", width, buf); break; case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: From 20013b3f77cb4af8da5c7703d0ac3409a2a6b09b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 11 Apr 2023 17:26:05 +0800 Subject: [PATCH 07/12] fix(shell): update the display of double value. --- tools/shell/src/shellEngine.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 4798151d38..11758c76dc 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -538,7 +538,7 @@ void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t printf("%*e", width, GET_FLOAT_VAL(val)); break; case TSDB_DATA_TYPE_DOUBLE: - snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%.15e", GET_DOUBLE_VAL(val)); + snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%.9e", GET_DOUBLE_VAL(val)); printf("%*s", width, buf); break; case TSDB_DATA_TYPE_BINARY: From bf45ff56cafb2d21cfdf7cbbb91af9e15fa48023 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 11 Apr 2023 18:27:05 +0800 Subject: [PATCH 08/12] fix:modify fileContent for data compare --- tests/system-test/7-tmq/tmq3mnodeSwitch.py | 30 +------------------- tests/system-test/7-tmq/tmqCheckData.py | 33 ++-------------------- tests/system-test/7-tmq/tmqCommon.py | 22 +++++++++------ 3 files changed, 17 insertions(+), 68 deletions(-) diff --git a/tests/system-test/7-tmq/tmq3mnodeSwitch.py b/tests/system-test/7-tmq/tmq3mnodeSwitch.py index 7c95f7a3db..54ccc88226 100644 --- a/tests/system-test/7-tmq/tmq3mnodeSwitch.py +++ b/tests/system-test/7-tmq/tmq3mnodeSwitch.py @@ -138,34 +138,6 @@ class TDTestCase: else: tdLog.exit("three mnodes is not ready in 10s ") - def checkFileContent(self, consumerId, queryString): - buildPath = tdCom.getBuildPath() - cfgPath = tdCom.getClientCfgPath() - dstFile = '%s/../log/dstrows_%d.txt'%(cfgPath, consumerId) - cmdStr = '%s/build/bin/taos -c %s -s "%s >> %s"'%(buildPath, cfgPath, queryString, dstFile) - tdLog.info(cmdStr) - os.system(cmdStr) - - consumeRowsFile = '%s/../log/consumerid_%d.txt'%(cfgPath, consumerId) - tdLog.info("rows file: %s, %s"%(consumeRowsFile, dstFile)) - - consumeFile = open(consumeRowsFile, mode='r') - queryFile = open(dstFile, mode='r') - - # skip first line for it is schema - queryFile.readline() - - while True: - dst = queryFile.readline() - src = consumeFile.readline() - - if dst: - if dst != src: - tdLog.exit("consumerId %d consume rows is not match the rows by direct query"%consumerId) - else: - break - return - def tmqCase1(self): tdLog.printNoPrefix("======== test case 1: ") paraDict = {'dbName': 'db1', @@ -256,7 +228,7 @@ class TDTestCase: tdLog.exit("0 tmq consume rows error!") if expectRowsList[0] == resultList[0]: - self.checkFileContent(consumerId, queryString) + tqCom.checkFileContent(consumerId, queryString) time.sleep(10) for i in range(len(topicNameList)): diff --git a/tests/system-test/7-tmq/tmqCheckData.py b/tests/system-test/7-tmq/tmqCheckData.py index 04d0744ab5..1b45646c79 100644 --- a/tests/system-test/7-tmq/tmqCheckData.py +++ b/tests/system-test/7-tmq/tmqCheckData.py @@ -5,6 +5,7 @@ import time import socket import os import threading +import math from util.log import * from util.sql import * @@ -21,34 +22,6 @@ class TDTestCase: tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file - def checkFileContent(self, consumerId, queryString): - buildPath = tdCom.getBuildPath() - cfgPath = tdCom.getClientCfgPath() - dstFile = '%s/../log/dstrows_%d.txt'%(cfgPath, consumerId) - cmdStr = '%s/build/bin/taos -c %s -s "%s >> %s"'%(buildPath, cfgPath, queryString, dstFile) - tdLog.info(cmdStr) - os.system(cmdStr) - - consumeRowsFile = '%s/../log/consumerid_%d.txt'%(cfgPath, consumerId) - tdLog.info("rows file: %s, %s"%(consumeRowsFile, dstFile)) - - consumeFile = open(consumeRowsFile, mode='r') - queryFile = open(dstFile, mode='r') - - # skip first line for it is schema - queryFile.readline() - - while True: - dst = queryFile.readline() - src = consumeFile.readline() - - if dst: - if dst != src: - tdLog.exit("consumerId %d consume rows is not match the rows by direct query"%consumerId) - else: - break - return - def tmqCase1(self): tdLog.printNoPrefix("======== test case 1: ") paraDict = {'dbName': 'db1', @@ -109,7 +82,7 @@ class TDTestCase: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0])) tdLog.exit("0 tmq consume rows error!") - self.checkFileContent(consumerId, queryString) + tmqCom.checkFileContent(consumerId, queryString) # reinit consume info, and start tmq_sim, then check consume result tmqCom.initConsumerTable() @@ -135,7 +108,7 @@ class TDTestCase: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[1], resultList[0])) tdLog.exit("1 tmq consume rows error!") - self.checkFileContent(consumerId, queryString) + tmqCom.checkFileContent(consumerId, queryString) # reinit consume info, and start tmq_sim, then check consume result tmqCom.initConsumerTable() diff --git a/tests/system-test/7-tmq/tmqCommon.py b/tests/system-test/7-tmq/tmqCommon.py index 895da95e5d..d17fea7b5a 100644 --- a/tests/system-test/7-tmq/tmqCommon.py +++ b/tests/system-test/7-tmq/tmqCommon.py @@ -10,7 +10,7 @@ ################################################################### # -*- coding: utf-8 -*- - +import math from asyncore import loop from collections import defaultdict import subprocess @@ -462,18 +462,22 @@ class TMQCom: for i in range(0,skipRowsOfCons): consumeFile.readline() - lines = 0 while True: dst = queryFile.readline() src = consumeFile.readline() - lines += 1 - if dst: - if dst != src: - tdLog.info("src row: %s"%src) - tdLog.info("dst row: %s"%dst) - tdLog.exit("consumerId %d consume rows[%d] is not match the rows by direct query"%(consumerId, lines)) - else: + dstSplit = dst.split(',') + srcSplit = src.split(',') + + if len(dstSplit) != len(srcSplit): + tdLog.exit("consumerId %d consume rows len is not match the rows by direct query"%consumerId) + if not dst or not src: break + for i in range(len(dstSplit)): + if srcSplit[i] != dstSplit[i]: + srcFloat = float(srcSplit[i]) + dstFloat = float(dstSplit[i]) + if not math.isclose(srcFloat, dstFloat, abs_tol=1e-9): + tdLog.exit("consumerId %d consume rows is not match the rows by direct query"%consumerId) return def getResultFileByTaosShell(self, consumerId, queryString): From 54a414a93418f20e90b521dd5be0bdc43b3f8fa6 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 11 Apr 2023 18:35:05 +0800 Subject: [PATCH 09/12] fix:rm useless fileContent function --- tests/system-test/2-query/sml.py | 4 +-- .../7-tmq/dropDbR3ConflictTransaction.py | 28 --------------- tests/system-test/7-tmq/tmq3mnodeSwitch.py | 2 +- tests/system-test/7-tmq/tmqCheckData.py | 2 +- tests/system-test/7-tmq/tmqCheckData1.py | 34 ++---------------- tests/system-test/7-tmq/tmqConsumerGroup.py | 28 --------------- .../7-tmq/tmqUdf-multCtb-snapshot0.py | 36 +++---------------- .../7-tmq/tmqUdf-multCtb-snapshot1.py | 36 +++---------------- tests/system-test/7-tmq/tmqUdf.py | 36 +++---------------- tests/system-test/99-TDcase/TD-16821.py | 34 ++---------------- 10 files changed, 22 insertions(+), 218 deletions(-) diff --git a/tests/system-test/2-query/sml.py b/tests/system-test/2-query/sml.py index ec6309c71a..f96ed8a3ff 100644 --- a/tests/system-test/2-query/sml.py +++ b/tests/system-test/2-query/sml.py @@ -24,7 +24,7 @@ class TDTestCase: tdSql.init(conn.cursor(), True) #tdSql.init(conn.cursor(), logSql) # output sql.txt file - def checkFileContent(self, dbname="sml_db"): + def checkContent(self, dbname="sml_db"): simClientCfg="%s/taos.cfg"%tdDnodes.getSimCfgPath() buildPath = tdCom.getBuildPath() cmdStr = '%s/build/bin/sml_test %s'%(buildPath, simClientCfg) @@ -102,7 +102,7 @@ class TDTestCase: def run(self): tdSql.prepare() - self.checkFileContent() + self.checkContent() def stop(self): tdSql.close() diff --git a/tests/system-test/7-tmq/dropDbR3ConflictTransaction.py b/tests/system-test/7-tmq/dropDbR3ConflictTransaction.py index 4371a909c2..e25fb412af 100644 --- a/tests/system-test/7-tmq/dropDbR3ConflictTransaction.py +++ b/tests/system-test/7-tmq/dropDbR3ConflictTransaction.py @@ -32,34 +32,6 @@ class TDTestCase: tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file - def checkFileContent(self, consumerId, queryString): - buildPath = tdCom.getBuildPath() - cfgPath = tdCom.getClientCfgPath() - dstFile = '%s/../log/dstrows_%d.txt'%(cfgPath, consumerId) - cmdStr = '%s/build/bin/taos -c %s -s "%s >> %s"'%(buildPath, cfgPath, queryString, dstFile) - tdLog.info(cmdStr) - os.system(cmdStr) - - consumeRowsFile = '%s/../log/consumerid_%d.txt'%(cfgPath, consumerId) - tdLog.info("rows file: %s, %s"%(consumeRowsFile, dstFile)) - - consumeFile = open(consumeRowsFile, mode='r') - queryFile = open(dstFile, mode='r') - - # skip first line for it is schema - queryFile.readline() - - while True: - dst = queryFile.readline() - src = consumeFile.readline() - - if dst: - if dst != src: - tdLog.exit("consumerId %d consume rows is not match the rows by direct query"%consumerId) - else: - break - return - def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") paraDict = {'dbName': 'dbt', diff --git a/tests/system-test/7-tmq/tmq3mnodeSwitch.py b/tests/system-test/7-tmq/tmq3mnodeSwitch.py index 54ccc88226..8c5dc5e693 100644 --- a/tests/system-test/7-tmq/tmq3mnodeSwitch.py +++ b/tests/system-test/7-tmq/tmq3mnodeSwitch.py @@ -228,7 +228,7 @@ class TDTestCase: tdLog.exit("0 tmq consume rows error!") if expectRowsList[0] == resultList[0]: - tqCom.checkFileContent(consumerId, queryString) + tmqCom.checkFileContent(consumerId, queryString) time.sleep(10) for i in range(len(topicNameList)): diff --git a/tests/system-test/7-tmq/tmqCheckData.py b/tests/system-test/7-tmq/tmqCheckData.py index 1b45646c79..4d5edf87f1 100644 --- a/tests/system-test/7-tmq/tmqCheckData.py +++ b/tests/system-test/7-tmq/tmqCheckData.py @@ -134,7 +134,7 @@ class TDTestCase: # tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[2], resultList[0])) # tdLog.exit("2 tmq consume rows error!") - # self.checkFileContent(consumerId, queryString) + # tmqCom.checkFileContent(consumerId, queryString) time.sleep(10) for i in range(len(topicNameList)): diff --git a/tests/system-test/7-tmq/tmqCheckData1.py b/tests/system-test/7-tmq/tmqCheckData1.py index b9dac62833..1209c2812c 100644 --- a/tests/system-test/7-tmq/tmqCheckData1.py +++ b/tests/system-test/7-tmq/tmqCheckData1.py @@ -21,34 +21,6 @@ class TDTestCase: tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file - def checkFileContent(self, consumerId, queryString): - buildPath = tdCom.getBuildPath() - cfgPath = tdCom.getClientCfgPath() - dstFile = '%s/../log/dstrows_%d.txt'%(cfgPath, consumerId) - cmdStr = '%s/build/bin/taos -c %s -s "%s >> %s"'%(buildPath, cfgPath, queryString, dstFile) - tdLog.info(cmdStr) - os.system(cmdStr) - - consumeRowsFile = '%s/../log/consumerid_%d.txt'%(cfgPath, consumerId) - tdLog.info("rows file: %s, %s"%(consumeRowsFile, dstFile)) - - consumeFile = open(consumeRowsFile, mode='r') - queryFile = open(dstFile, mode='r') - - # skip first line for it is schema - queryFile.readline() - - while True: - dst = queryFile.readline() - src = consumeFile.readline() - - if dst: - if dst != src: - tdLog.exit("consumerId %d consume rows is not match the rows by direct query"%consumerId) - else: - break - return - def tmqCase1(self): tdLog.printNoPrefix("======== test case 1: ") paraDict = {'dbName': 'db1', @@ -109,7 +81,7 @@ class TDTestCase: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0])) tdLog.exit("0 tmq consume rows error!") - self.checkFileContent(consumerId, queryString) + tmqCom.checkFileContent(consumerId, queryString) # reinit consume info, and start tmq_sim, then check consume result tmqCom.initConsumerTable() @@ -134,7 +106,7 @@ class TDTestCase: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[1], resultList[0])) tdLog.exit("1 tmq consume rows error!") - self.checkFileContent(consumerId, queryString) + tmqCom.checkFileContent(consumerId, queryString) # reinit consume info, and start tmq_sim, then check consume result tmqCom.initConsumerTable() @@ -159,7 +131,7 @@ class TDTestCase: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[2], resultList[0])) tdLog.exit("2 tmq consume rows error!") - self.checkFileContent(consumerId, queryString) + tmqCom.checkFileContent(consumerId, queryString) time.sleep(10) for i in range(len(topicNameList)): diff --git a/tests/system-test/7-tmq/tmqConsumerGroup.py b/tests/system-test/7-tmq/tmqConsumerGroup.py index 02093a2d88..ed70866c5d 100644 --- a/tests/system-test/7-tmq/tmqConsumerGroup.py +++ b/tests/system-test/7-tmq/tmqConsumerGroup.py @@ -21,34 +21,6 @@ class TDTestCase: tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file - def checkFileContent(self, consumerId, queryString): - buildPath = tdCom.getBuildPath() - cfgPath = tdCom.getClientCfgPath() - dstFile = '%s/../log/dstrows_%d.txt'%(cfgPath, consumerId) - cmdStr = '%s/build/bin/taos -c %s -s "%s >> %s"'%(buildPath, cfgPath, queryString, dstFile) - tdLog.info(cmdStr) - os.system(cmdStr) - - consumeRowsFile = '%s/../log/consumerid_%d.txt'%(cfgPath, consumerId) - tdLog.info("rows file: %s, %s"%(consumeRowsFile, dstFile)) - - consumeFile = open(consumeRowsFile, mode='r') - queryFile = open(dstFile, mode='r') - - # skip first line for it is schema - queryFile.readline() - - while True: - dst = queryFile.readline() - src = consumeFile.readline() - - if dst: - if dst != src: - tdLog.exit("consumerId %d consume rows is not match the rows by direct query"%consumerId) - else: - break - return - def tmqCase1(self): tdLog.printNoPrefix("======== test case 1: ") paraDict = {'dbName': 'db1', diff --git a/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot0.py b/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot0.py index 297429b495..6ce798ab87 100644 --- a/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot0.py +++ b/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot0.py @@ -60,34 +60,6 @@ class TDTestCase: tdLog.exit("create udf functions fail") return - def checkFileContent(self, consumerId, queryString): - buildPath = tdCom.getBuildPath() - cfgPath = tdCom.getClientCfgPath() - dstFile = '%s/../log/dstrows_%d.txt'%(cfgPath, consumerId) - cmdStr = '%s/build/bin/taos -c %s -s "%s >> %s"'%(buildPath, cfgPath, queryString, dstFile) - tdLog.info(cmdStr) - os.system(cmdStr) - - consumeRowsFile = '%s/../log/consumerid_%d.txt'%(cfgPath, consumerId) - tdLog.info("rows file: %s, %s"%(consumeRowsFile, dstFile)) - - consumeFile = open(consumeRowsFile, mode='r') - queryFile = open(dstFile, mode='r') - - # skip first line for it is schema - queryFile.readline() - - while True: - dst = queryFile.readline() - src = consumeFile.readline() - - if dst: - if dst != src: - tdLog.exit("consumerId %d consume rows is not match the rows by direct query"%consumerId) - else: - break - return - def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") paraDict = {'dbName': 'dbt', @@ -199,7 +171,7 @@ class TDTestCase: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0])) tdLog.exit("0 tmq consume rows error!") - # self.checkFileContent(consumerId, queryString) + # tmqCom.checkFileContent(consumerId, queryString) # tdLog.printNoPrefix("consumerId %d check data ok!"%(consumerId)) # reinit consume info, and start tmq_sim, then check consume result @@ -226,7 +198,7 @@ class TDTestCase: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[1], resultList[0])) tdLog.exit("1 tmq consume rows error!") - # self.checkFileContent(consumerId, queryString) + # tmqCom.checkFileContent(consumerId, queryString) # tdLog.printNoPrefix("consumerId %d check data ok!"%(consumerId)) time.sleep(10) @@ -309,7 +281,7 @@ class TDTestCase: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0])) tdLog.exit("2 tmq consume rows error!") - # self.checkFileContent(consumerId, queryString) + # tmqCom.checkFileContent(consumerId, queryString) # tdLog.printNoPrefix("consumerId %d check data ok!"%(consumerId)) # reinit consume info, and start tmq_sim, then check consume result @@ -336,7 +308,7 @@ class TDTestCase: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[1], resultList[0])) tdLog.exit("3 tmq consume rows error!") - # self.checkFileContent(consumerId, queryString) + # tmqCom.checkFileContent(consumerId, queryString) # tdLog.printNoPrefix("consumerId %d check data ok!"%(consumerId)) time.sleep(10) diff --git a/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot1.py b/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot1.py index 9c139b50de..143c8762b8 100644 --- a/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot1.py +++ b/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot1.py @@ -60,34 +60,6 @@ class TDTestCase: tdLog.exit("create udf functions fail") return - def checkFileContent(self, consumerId, queryString): - buildPath = tdCom.getBuildPath() - cfgPath = tdCom.getClientCfgPath() - dstFile = '%s/../log/dstrows_%d.txt'%(cfgPath, consumerId) - cmdStr = '%s/build/bin/taos -c %s -s "%s >> %s"'%(buildPath, cfgPath, queryString, dstFile) - tdLog.info(cmdStr) - os.system(cmdStr) - - consumeRowsFile = '%s/../log/consumerid_%d.txt'%(cfgPath, consumerId) - tdLog.info("rows file: %s, %s"%(consumeRowsFile, dstFile)) - - consumeFile = open(consumeRowsFile, mode='r') - queryFile = open(dstFile, mode='r') - - # skip first line for it is schema - queryFile.readline() - - while True: - dst = queryFile.readline() - src = consumeFile.readline() - - if dst: - if dst != src: - tdLog.exit("consumerId %d consume rows is not match the rows by direct query"%consumerId) - else: - break - return - def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") paraDict = {'dbName': 'dbt', @@ -199,7 +171,7 @@ class TDTestCase: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0])) tdLog.exit("0 tmq consume rows error!") - # self.checkFileContent(consumerId, queryString) + # tmqCom.checkFileContent(consumerId, queryString) # tdLog.printNoPrefix("consumerId %d check data ok!"%(consumerId)) # reinit consume info, and start tmq_sim, then check consume result @@ -226,7 +198,7 @@ class TDTestCase: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[1], resultList[0])) tdLog.exit("1 tmq consume rows error!") - # self.checkFileContent(consumerId, queryString) + # tmqCom.checkFileContent(consumerId, queryString) # tdLog.printNoPrefix("consumerId %d check data ok!"%(consumerId)) time.sleep(10) @@ -309,7 +281,7 @@ class TDTestCase: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0])) tdLog.exit("2 tmq consume rows error!") - # self.checkFileContent(consumerId, queryString) + # tmqCom.checkFileContent(consumerId, queryString) # tdLog.printNoPrefix("consumerId %d check data ok!"%(consumerId)) # reinit consume info, and start tmq_sim, then check consume result @@ -336,7 +308,7 @@ class TDTestCase: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[1], resultList[0])) tdLog.exit("3 tmq consume rows error!") - # self.checkFileContent(consumerId, queryString) + # tmqCom.checkFileContent(consumerId, queryString) # tdLog.printNoPrefix("consumerId %d check data ok!"%(consumerId)) time.sleep(10) diff --git a/tests/system-test/7-tmq/tmqUdf.py b/tests/system-test/7-tmq/tmqUdf.py index 8593fd4f1e..04a3a2301d 100644 --- a/tests/system-test/7-tmq/tmqUdf.py +++ b/tests/system-test/7-tmq/tmqUdf.py @@ -60,34 +60,6 @@ class TDTestCase: tdLog.exit("create udf functions fail") return - def checkFileContent(self, consumerId, queryString): - buildPath = tdCom.getBuildPath() - cfgPath = tdCom.getClientCfgPath() - dstFile = '%s/../log/dstrows_%d.txt'%(cfgPath, consumerId) - cmdStr = '%s/build/bin/taos -c %s -s "%s >> %s"'%(buildPath, cfgPath, queryString, dstFile) - tdLog.info(cmdStr) - os.system(cmdStr) - - consumeRowsFile = '%s/../log/consumerid_%d.txt'%(cfgPath, consumerId) - tdLog.info("rows file: %s, %s"%(consumeRowsFile, dstFile)) - - consumeFile = open(consumeRowsFile, mode='r') - queryFile = open(dstFile, mode='r') - - # skip first line for it is schema - queryFile.readline() - - while True: - dst = queryFile.readline() - src = consumeFile.readline() - - if dst: - if dst != src: - tdLog.exit("consumerId %d consume rows is not match the rows by direct query"%consumerId) - else: - break - return - def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") paraDict = {'dbName': 'dbt', @@ -199,7 +171,7 @@ class TDTestCase: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0])) tdLog.exit("0 tmq consume rows error!") - self.checkFileContent(consumerId, queryString) + tmqCom.checkFileContent(consumerId, queryString) tdLog.printNoPrefix("consumerId %d check data ok!"%(consumerId)) @@ -227,7 +199,7 @@ class TDTestCase: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[1], resultList[0])) tdLog.exit("1 tmq consume rows error!") - self.checkFileContent(consumerId, queryString) + tmqCom.checkFileContent(consumerId, queryString) tdLog.printNoPrefix("consumerId %d check data ok!"%(consumerId)) time.sleep(10) @@ -310,7 +282,7 @@ class TDTestCase: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0])) tdLog.exit("2 tmq consume rows error!") - self.checkFileContent(consumerId, queryString) + tmqCom.checkFileContent(consumerId, queryString) tdLog.printNoPrefix("consumerId %d check data ok!"%(consumerId)) # reinit consume info, and start tmq_sim, then check consume result @@ -337,7 +309,7 @@ class TDTestCase: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[1], resultList[0])) tdLog.exit("3 tmq consume rows error!") - self.checkFileContent(consumerId, queryString) + tmqCom.checkFileContent(consumerId, queryString) tdLog.printNoPrefix("consumerId %d check data ok!"%(consumerId)) time.sleep(10) diff --git a/tests/system-test/99-TDcase/TD-16821.py b/tests/system-test/99-TDcase/TD-16821.py index f57fae752f..26b41e6afc 100644 --- a/tests/system-test/99-TDcase/TD-16821.py +++ b/tests/system-test/99-TDcase/TD-16821.py @@ -21,34 +21,6 @@ class TDTestCase: tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file - def checkFileContent(self, consumerId, queryString): - buildPath = tdCom.getBuildPath() - cfgPath = tdCom.getClientCfgPath() - dstFile = '%s/../log/dstrows_%d.txt'%(cfgPath, consumerId) - cmdStr = '%s/build/bin/taos -c %s -s "%s >> %s"'%(buildPath, cfgPath, queryString, dstFile) - tdLog.info(cmdStr) - os.system(cmdStr) - - consumeRowsFile = '%s/../log/consumerid_%d.txt'%(cfgPath, consumerId) - tdLog.info("rows file: %s, %s"%(consumeRowsFile, dstFile)) - - consumeFile = open(consumeRowsFile, mode='r') - queryFile = open(dstFile, mode='r') - - # skip first line for it is schema - queryFile.readline() - - while True: - dst = queryFile.readline() - src = consumeFile.readline() - - if dst: - if dst != src: - tdLog.exit("consumerId %d consume rows is not match the rows by direct query"%consumerId) - else: - break - return - def tmqCase1(self): tdLog.printNoPrefix("======== test case 1: ") paraDict = {'dbName': 'db1', @@ -113,7 +85,7 @@ class TDTestCase: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0])) tdLog.exit("0 tmq consume rows error!") - self.checkFileContent(consumerId, queryString) + tmqCom.checkFileContent(consumerId, queryString) # reinit consume info, and start tmq_sim, then check consume result tmqCom.initConsumerTable() @@ -139,7 +111,7 @@ class TDTestCase: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[1], resultList[0])) tdLog.exit("1 tmq consume rows error!") - self.checkFileContent(consumerId, queryString) + tmqCom.checkFileContent(consumerId, queryString) # reinit consume info, and start tmq_sim, then check consume result tmqCom.initConsumerTable() @@ -165,7 +137,7 @@ class TDTestCase: # tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[2], resultList[0])) # tdLog.exit("2 tmq consume rows error!") - # self.checkFileContent(consumerId, queryString) + # tmqCom.checkFileContent(consumerId, queryString) time.sleep(10) for i in range(len(topicNameList)): From 985a2382425fb17495846b5ad5911ab71782886c Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 11 Apr 2023 19:38:10 +0800 Subject: [PATCH 10/12] fix:modify checkFileContent if one is empty --- tests/system-test/7-tmq/tmqCommon.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/system-test/7-tmq/tmqCommon.py b/tests/system-test/7-tmq/tmqCommon.py index d17fea7b5a..b47601620a 100644 --- a/tests/system-test/7-tmq/tmqCommon.py +++ b/tests/system-test/7-tmq/tmqCommon.py @@ -468,10 +468,12 @@ class TMQCom: dstSplit = dst.split(',') srcSplit = src.split(',') - if len(dstSplit) != len(srcSplit): - tdLog.exit("consumerId %d consume rows len is not match the rows by direct query"%consumerId) if not dst or not src: break + if len(dstSplit) != len(srcSplit): + tdLog.exit("consumerId %d consume rows len is not match the rows by direct query,len(dstSplit):%d != len(srcSplit):%d, dst:%s, src:%s" + %(consumerId, len(dstSplit), len(srcSplit), dst, src)) + for i in range(len(dstSplit)): if srcSplit[i] != dstSplit[i]: srcFloat = float(srcSplit[i]) From 71283d6c4198d8305309743c090758638bcb819d Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 12 Apr 2023 09:35:47 +0800 Subject: [PATCH 11/12] fix: script if share not exist (#20875) --- packaging/tools/install.sh | 10 ++++++---- packaging/tools/makepkg.sh | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index 849ab5a92a..a1a95d9f30 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -573,16 +573,18 @@ function install_config() { } function install_share_etc() { + [ ! -d ${script_dir}/share/etc ] && return for c in `ls ${script_dir}/share/etc/`; do if [ -e /etc/$c ]; then out=/etc/$c.new.`date +%F` - ${csudo}cp -f ${script_dir}/share/etc/$c $out + ${csudo}cp -f ${script_dir}/share/etc/$c $out ||: else - ${csudo}cp -f ${script_dir}/share/etc/$c /etc/$c + ${csudo}cp -f ${script_dir}/share/etc/$c /etc/$c ||: fi done - ${csudo} cp ${script_dir}/share/srv/* ${service_config_dir} + [ ! -d ${script_dir}/share/srv ] && return + ${csudo} cp ${script_dir}/share/srv/* ${service_config_dir} ||: } function install_log() { @@ -612,7 +614,7 @@ function install_examples() { function install_web() { if [ -d "${script_dir}/share" ]; then - ${csudo}cp -rf ${script_dir}/share/* ${install_main_dir}/share + ${csudo}cp -rf ${script_dir}/share/* ${install_main_dir}/share > /dev/null 2>&1 ||: fi } diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index 12e215c62b..0dce526db6 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -150,7 +150,7 @@ fi mkdir -p ${install_dir}/bin && cp ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* || : mkdir -p ${install_dir}/init.d && cp ${init_file_deb} ${install_dir}/init.d/${serverName}.deb mkdir -p ${install_dir}/init.d && cp ${init_file_rpm} ${install_dir}/init.d/${serverName}.rpm -mkdir -p ${install_dir}/share && cp -rf ${build_dir}/share/{etc,srv} ${install_dir}/share +mkdir -p ${install_dir}/share && cp -rf ${build_dir}/share/{etc,srv} ${install_dir}/share ||: if [ $adapterName != "taosadapter" ]; then mv ${install_dir}/cfg/${clientName2}adapter.toml ${install_dir}/cfg/$adapterName.toml From 5d9f3c4133f5ccd0d2734db8166ce32404dc994b Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Wed, 12 Apr 2023 10:44:04 +0800 Subject: [PATCH 12/12] merge main --- source/libs/catalog/src/ctgUtil.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index 6beadf09da..b2b2b5a87e 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -1400,10 +1400,10 @@ int32_t ctgChkSetAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) { pRes->pass = false; pRes->pCond = NULL; - // if (!pInfo->enable) { - // pRes->pass = false; - // return TSDB_CODE_SUCCESS; - // } + if (!pInfo->enable) { + pRes->pass = false; + return TSDB_CODE_SUCCESS; + } if (pInfo->superAuth) { pRes->pass = true;