diff --git a/contrib/test/azure/CMakeLists.txt b/contrib/test/azure/CMakeLists.txt index 35c87312d0..68571dce46 100644 --- a/contrib/test/azure/CMakeLists.txt +++ b/contrib/test/azure/CMakeLists.txt @@ -14,13 +14,13 @@ find_library(CRYPTO_LIBRARY crypto $ENV{HOME}/.cos-local.2/lib64 $ENV{HOME}/.cos #find_library(CoreFoundation_Library CoreFoundation) #find_library(SystemConfiguration_Library SystemConfiguration) - target_link_libraries( - azure-test - PRIVATE _azure_sdk - PRIVATE ${CURL_LIBRARY} - PRIVATE ${XML2_LIBRARY} - PRIVATE ${SSL_LIBRARY} - PRIVATE ${CRYPTO_LIBRARY} - PRIVATE dl - PRIVATE pthread +target_link_libraries( + azure-test + PRIVATE _azure_sdk + PRIVATE ${CURL_LIBRARY} + PRIVATE ${XML2_LIBRARY} + PRIVATE ${SSL_LIBRARY} + PRIVATE ${CRYPTO_LIBRARY} + PRIVATE dl + PRIVATE pthread ) diff --git a/contrib/test/azure/main.cpp b/contrib/test/azure/main.cpp index 09eadecd69..badfef623b 100644 --- a/contrib/test/azure/main.cpp +++ b/contrib/test/azure/main.cpp @@ -22,6 +22,9 @@ int main(int argc, char* argv[]) { try { auto containerClient = BlobContainerClient::CreateFromConnectionString(std::getenv("STORAGE_CONNECTION_STRING"), "td-test"); + + // Create the container if it does not exist + // std::cout << "Creating container: " << containerName << std::endl; // containerClient.CreateIfNotExists(); /**************** Container SDK client ************************/ @@ -30,6 +33,21 @@ int main(int argc, char* argv[]) { // auto response = containerClient.ListBlobs(); // auto blobListPage = response.Value; // auto blobListPage = response.Blobs; + //(void)_azUploadFrom(blobClient, file, offset, size); + /* + auto blockBlobClient = BlockBlobClient(endpointUrl, sharedKeyCredential); + + // Create some data to upload into the blob. + std::vector data = {1, 2, 3, 4}; + Azure::Core::IO::MemoryBodyStream stream(data); + + Azure::Response response = blockBlobClient.Upload(stream); + + Models::UploadBlockBlobResult model = response.Value; + std::cout << "Last modified date of uploaded blob: " << model.LastModified.ToString() + << std::endl; + */ + for (auto page = containerClient.ListBlobs(/*options*/); page.HasPage(); page.MoveToNextPage()) { for (auto& blob : page.Blobs) { std::cout << blob.Name << std::endl; diff --git a/source/libs/azure/src/az.cpp b/source/libs/azure/src/az.cpp index 3da4b6e808..a2411c883d 100644 --- a/source/libs/azure/src/az.cpp +++ b/source/libs/azure/src/az.cpp @@ -222,10 +222,6 @@ int32_t azPutObjectFromFileOffset(const char *file, const char *object_name, int std::string containerName = tsS3BucketName; auto containerClient = blobServiceClient.GetBlobContainerClient(containerName); - // Create the container if it does not exist - // std::cout << "Creating container: " << containerName << std::endl; - // containerClient.CreateIfNotExists(); - std::string blobName = "blob.txt"; uint8_t blobContent[] = "Hello Azure!"; // Create the block blob client @@ -237,20 +233,6 @@ int32_t azPutObjectFromFileOffset(const char *file, const char *object_name, int // std::cout << "Uploading blob: " << blobName << std::endl; // blobClient.UploadFrom(blobContent, sizeof(blobContent)); blobClient.UploadFrom(file, offset, size); - //(void)_azUploadFrom(blobClient, file, offset, size); - /* - auto blockBlobClient = BlockBlobClient(endpointUrl, sharedKeyCredential); - - // Create some data to upload into the blob. - std::vector data = {1, 2, 3, 4}; - Azure::Core::IO::MemoryBodyStream stream(data); - - Azure::Response response = blockBlobClient.Upload(stream); - - Models::UploadBlockBlobResult model = response.Value; - std::cout << "Last modified date of uploaded blob: " << model.LastModified.ToString() - << std::endl; - */ } catch (const Azure::Core::RequestFailedException &e) { /* std::cout << "Status Code: " << static_cast(e.StatusCode) << ", Reason Phrase: " << e.ReasonPhrase diff --git a/source/libs/azure/test/azTest.cpp b/source/libs/azure/test/azTest.cpp index 9e963508f8..2780cddacc 100644 --- a/source/libs/azure/test/azTest.cpp +++ b/source/libs/azure/test/azTest.cpp @@ -2,456 +2,78 @@ #include #include #include -/* -#include "walInt.h" -const char* ranStr = "tvapq02tcp"; -const int ranStrLen = strlen(ranStr); -SWalSyncInfo syncMeta = {0}; +#include "az.h" -class WalCleanEnv : public ::testing::Test { - protected: - static void SetUpTestCase() { - int code = walInit(NULL); - ASSERT(code == 0); - } +extern int8_t tsS3Enabled; - static void TearDownTestCase() { walCleanUp(); } +int32_t azInitEnv() { + int32_t code = 0; - void SetUp() override { - taosRemoveDir(pathName); - SWalCfg* pCfg = (SWalCfg*)taosMemoryMalloc(sizeof(SWalCfg)); - memset(pCfg, 0, sizeof(SWalCfg)); - pCfg->rollPeriod = -1; - pCfg->segSize = -1; - pCfg->retentionPeriod = 0; - pCfg->retentionSize = 0; - pCfg->level = TAOS_WAL_FSYNC; - pWal = walOpen(pathName, pCfg); - taosMemoryFree(pCfg); - ASSERT(pWal != NULL); - } + extern int8_t tsS3EpNum; - void TearDown() override { - walClose(pWal); - pWal = NULL; - } + extern char tsS3Hostname[][TSDB_FQDN_LEN]; + extern char tsS3AccessKeyId[][TSDB_FQDN_LEN]; + extern char tsS3AccessKeySecret[][TSDB_FQDN_LEN]; + extern char tsS3BucketName[TSDB_FQDN_LEN]; - SWal* pWal = NULL; - const char* pathName = TD_TMP_DIR_PATH "wal_test"; -}; + /* TCS parameter format + tsS3Hostname[0] = "endpoint/.blob.core.windows.net"; + tsS3AccessKeyId[0] = ""; + tsS3AccessKeySecret[0] = ""; + tsS3BucketName = ""; + */ -class WalCleanDeleteEnv : public ::testing::Test { - protected: - static void SetUpTestCase() { - int code = walInit(NULL); - ASSERT(code == 0); - } + const char *hostname = "endpoint/.blob.core.windows.net"; + const char *accessKeyId = ""; + const char *accessKeySecret = ""; + const char *bucketName = ""; - static void TearDownTestCase() { walCleanUp(); } + tstrncpy(&tsS3Hostname[0][0], hostname, TSDB_FQDN_LEN); + tstrncpy(&tsS3AccessKeyId[0][0], accessKeyId, TSDB_FQDN_LEN); + tstrncpy(&tsS3AccessKeySecret[0][0], accessKeySecret, TSDB_FQDN_LEN); + tstrncpy(tsS3BucketName, bucketName, TSDB_FQDN_LEN); - void SetUp() override { - taosRemoveDir(pathName); - SWalCfg* pCfg = (SWalCfg*)taosMemoryMalloc(sizeof(SWalCfg)); - memset(pCfg, 0, sizeof(SWalCfg)); - pCfg->retentionPeriod = 0; - pCfg->retentionSize = 0; - pCfg->level = TAOS_WAL_FSYNC; - pWal = walOpen(pathName, pCfg); - taosMemoryFree(pCfg); - ASSERT(pWal != NULL); - } + tstrncpy(tsTempDir, "/tmp/", PATH_MAX); - void TearDown() override { - walClose(pWal); - pWal = NULL; - } + tsS3Enabled = true; - SWal* pWal = NULL; - const char* pathName = TD_TMP_DIR_PATH "wal_test"; -}; - -class WalKeepEnv : public ::testing::Test { - protected: - static void SetUpTestCase() { - int code = walInit(NULL); - ASSERT(code == 0); - } - - static void TearDownTestCase() { walCleanUp(); } - - void walResetEnv() { - TearDown(); - taosRemoveDir(pathName); - SetUp(); - } - - void SetUp() override { - SWalCfg* pCfg = (SWalCfg*)taosMemoryMalloc(sizeof(SWalCfg)); - memset(pCfg, 0, sizeof(SWalCfg)); - pCfg->rollPeriod = -1; - pCfg->segSize = -1; - pCfg->retentionPeriod = 0; - pCfg->retentionSize = 0; - pCfg->level = TAOS_WAL_FSYNC; - pWal = walOpen(pathName, pCfg); - taosMemoryFree(pCfg); - ASSERT(pWal != NULL); - } - - void TearDown() override { - walClose(pWal); - pWal = NULL; - } - - SWal* pWal = NULL; - const char* pathName = TD_TMP_DIR_PATH "wal_test"; -}; - -class WalRetentionEnv : public ::testing::Test { - protected: - static void SetUpTestCase() { - int code = walInit(NULL); - ASSERT(code == 0); - } - - static void TearDownTestCase() { walCleanUp(); } - - void walResetEnv() { - TearDown(); - taosRemoveDir(pathName); - SetUp(); - } - - void SetUp() override { - SWalCfg cfg; - cfg.rollPeriod = -1; - cfg.segSize = -1; - cfg.retentionPeriod = -1; - cfg.retentionSize = 0; - cfg.rollPeriod = 0; - cfg.vgId = 0; - cfg.level = TAOS_WAL_FSYNC; - pWal = walOpen(pathName, &cfg); - ASSERT(pWal != NULL); - } - - void TearDown() override { - walClose(pWal); - pWal = NULL; - } - - SWal* pWal = NULL; - const char* pathName = TD_TMP_DIR_PATH "wal_test"; -}; - -TEST_F(WalCleanEnv, createNew) { - walRollFileInfo(pWal); - ASSERT(pWal->fileInfoSet != NULL); - ASSERT_EQ(pWal->fileInfoSet->size, 1); - SWalFileInfo* pInfo = (SWalFileInfo*)taosArrayGetLast(pWal->fileInfoSet); - ASSERT_EQ(pInfo->firstVer, 0); - ASSERT_EQ(pInfo->lastVer, -1); - ASSERT_EQ(pInfo->closeTs, -1); - ASSERT_EQ(pInfo->fileSize, 0); + return code; } -TEST_F(WalCleanEnv, serialize) { - int code = walRollFileInfo(pWal); - ASSERT(code == 0); - ASSERT(pWal->fileInfoSet != NULL); +// TEST(AzTest, DISABLED_InterfaceTest) { +TEST(AzTest, InterfaceTest) { + int code = 0; - code = walRollFileInfo(pWal); - ASSERT(code == 0); - code = walRollFileInfo(pWal); - ASSERT(code == 0); - code = walRollFileInfo(pWal); - ASSERT(code == 0); - code = walRollFileInfo(pWal); - ASSERT(code == 0); - code = walRollFileInfo(pWal); - ASSERT(code == 0); - char* ss = NULL; - code = walMetaSerialize(pWal, &ss); - ASSERT(code == 0); - printf("%s\n", ss); - taosMemoryFree(ss); - code = walSaveMeta(pWal); - ASSERT(code == 0); + code = azInitEnv(); + GTEST_ASSERT_EQ(code, 0); + GTEST_ASSERT_EQ(tsS3Enabled, 1); + + code = azBegin(); + GTEST_ASSERT_EQ(code, 0); + + code = azCheckCfg(); + GTEST_ASSERT_EQ(code, 0); + /* + code = azPutObjectFromFileOffset(file, object_name, offset, size); + GTEST_ASSERT_EQ(code, 0); + code = azGetObjectBlock(object_name, offset, size, check, ppBlock); + GTEST_ASSERT_EQ(code, 0); + + azDeleteObjectsByPrefix(prefix); + // list object to check + + code = azPutObjectFromFile2(file, object, withcp); + GTEST_ASSERT_EQ(code, 0); + code = azGetObjectsByPrefix(prefix, path); + GTEST_ASSERT_EQ(code, 0); + code = azDeleteObjects(object_name, nobject); + GTEST_ASSERT_EQ(code, 0); + code = azGetObjectToFile(object_name, fileName); + GTEST_ASSERT_EQ(code, 0); + + // GTEST_ASSERT_NE(pEnv, nullptr); + */ + + azEnd(); } - -TEST_F(WalCleanEnv, removeOldMeta) { - int code = walRollFileInfo(pWal); - ASSERT(code == 0); - ASSERT(pWal->fileInfoSet != NULL); - code = walSaveMeta(pWal); - ASSERT(code == 0); - code = walRollFileInfo(pWal); - ASSERT(code == 0); - code = walSaveMeta(pWal); - ASSERT(code == 0); -} - -TEST_F(WalKeepEnv, readOldMeta) { - walResetEnv(); - int code; - - syncMeta.isWeek = -1; - syncMeta.seqNum = UINT64_MAX; - syncMeta.term = UINT64_MAX; - - for (int i = 0; i < 10; i++) { - code = walAppendLog(pWal, i, i + 1, syncMeta, (void*)ranStr, ranStrLen); - ASSERT_EQ(code, 0); - ASSERT_EQ(pWal->vers.lastVer, i); - code = walAppendLog(pWal, i + 2, i, syncMeta, (void*)ranStr, ranStrLen); - ASSERT_EQ(code, TSDB_CODE_WAL_INVALID_VER); - ASSERT_EQ(pWal->vers.lastVer, i); - } - char* oldss = NULL; - code = walMetaSerialize(pWal, &oldss); - ASSERT(code == 0); - - TearDown(); - SetUp(); - - ASSERT_EQ(pWal->vers.firstVer, 0); - ASSERT_EQ(pWal->vers.lastVer, 9); - - char* newss = NULL; - code = walMetaSerialize(pWal, &newss); - ASSERT(code == 0); - - int len = strlen(oldss); - ASSERT_EQ(len, strlen(newss)); - for (int i = 0; i < len; i++) { - EXPECT_EQ(oldss[i], newss[i]); - } - taosMemoryFree(oldss); - taosMemoryFree(newss); -} - -TEST_F(WalCleanEnv, write) { - int code; - for (int i = 0; i < 10; i++) { - code = walAppendLog(pWal, i, i + 1, syncMeta, (void*)ranStr, ranStrLen); - ASSERT_EQ(code, 0); - ASSERT_EQ(pWal->vers.lastVer, i); - code = walAppendLog(pWal, i + 2, i, syncMeta, (void*)ranStr, ranStrLen); - ASSERT_EQ(code, TSDB_CODE_WAL_INVALID_VER); - ASSERT_EQ(pWal->vers.lastVer, i); - } - code = walSaveMeta(pWal); - ASSERT_EQ(code, 0); -} - -TEST_F(WalCleanEnv, rollback) { - int code; - for (int i = 0; i < 10; i++) { - code = walAppendLog(pWal, i, i + 1, syncMeta, (void*)ranStr, ranStrLen); - ASSERT_EQ(code, 0); - ASSERT_EQ(pWal->vers.lastVer, i); - } - code = walRollback(pWal, 12); - ASSERT_NE(code, 0); - ASSERT_EQ(pWal->vers.lastVer, 9); - code = walRollback(pWal, 9); - ASSERT_EQ(code, 0); - ASSERT_EQ(pWal->vers.lastVer, 8); - code = walRollback(pWal, 5); - ASSERT_EQ(code, 0); - ASSERT_EQ(pWal->vers.lastVer, 4); - code = walRollback(pWal, 3); - ASSERT_EQ(code, 0); - ASSERT_EQ(pWal->vers.lastVer, 2); - code = walSaveMeta(pWal); - ASSERT_EQ(code, 0); -} - -TEST_F(WalCleanEnv, rollbackMultiFile) { - int code; - for (int i = 0; i < 10; i++) { - code = walAppendLog(pWal, i, i + 1, syncMeta, (void*)ranStr, ranStrLen); - ASSERT_EQ(code, 0); - ASSERT_EQ(pWal->vers.lastVer, i); - if (i == 5) { - walBeginSnapshot(pWal, i, 0); - walEndSnapshot(pWal); - } - } - code = walRollback(pWal, 12); - ASSERT_NE(code, 0); - ASSERT_EQ(pWal->vers.lastVer, 9); - code = walRollback(pWal, 9); - ASSERT_EQ(code, 0); - ASSERT_EQ(pWal->vers.lastVer, 8); - code = walRollback(pWal, 6); - ASSERT_EQ(code, 0); - ASSERT_EQ(pWal->vers.lastVer, 5); - code = walRollback(pWal, 5); - ASSERT_NE(code, 0); - - ASSERT_EQ(pWal->vers.lastVer, 5); - - code = walAppendLog(pWal, 6, 6, syncMeta, (void*)ranStr, ranStrLen); - ASSERT_EQ(code, 0); - ASSERT_EQ(pWal->vers.lastVer, 6); - - code = walSaveMeta(pWal); - ASSERT_EQ(code, 0); -} - -TEST_F(WalCleanDeleteEnv, roll) { - int code; - int i; - for (i = 0; i < 100; i++) { - code = walAppendLog(pWal, i, 0, syncMeta, (void*)ranStr, ranStrLen); - ASSERT_EQ(code, 0); - ASSERT_EQ(pWal->vers.lastVer, i); - code = walCommit(pWal, i); - ASSERT_EQ(pWal->vers.commitVer, i); - } - - walBeginSnapshot(pWal, i - 1, 0); - ASSERT_EQ(pWal->vers.verInSnapshotting, i - 1); - walEndSnapshot(pWal); - ASSERT_EQ(pWal->vers.snapshotVer, i - 1); - ASSERT_EQ(pWal->vers.verInSnapshotting, -1); - - code = walAppendLog(pWal, 5, 0, syncMeta, (void*)ranStr, ranStrLen); - ASSERT_NE(code, 0); - - for (; i < 200; i++) { - code = walAppendLog(pWal, i, 0, syncMeta, (void*)ranStr, ranStrLen); - ASSERT_EQ(code, 0); - code = walCommit(pWal, i); - ASSERT_EQ(pWal->vers.commitVer, i); - } - - code = walBeginSnapshot(pWal, i - 1, 0); - ASSERT_EQ(code, 0); - code = walEndSnapshot(pWal); - ASSERT_EQ(code, 0); -} - -TEST_F(WalKeepEnv, readHandleRead) { - walResetEnv(); - int code; - SWalReader* pRead = walOpenReader(pWal, NULL, 0); - ASSERT(pRead != NULL); - - int i; - for (i = 0; i < 100; i++) { - char newStr[100]; - sprintf(newStr, "%s-%d", ranStr, i); - int len = strlen(newStr); - code = walAppendLog(pWal, i, 0, syncMeta, newStr, len); - ASSERT_EQ(code, 0); - } - for (int i = 0; i < 1000; i++) { - int ver = taosRand() % 100; - code = walReadVer(pRead, ver); - ASSERT_EQ(code, 0); - - // printf("rrbody: \n"); - // for(int i = 0; i < pRead->pHead->head.len; i++) { - // printf("%d ", pRead->pHead->head.body[i]); - //} - // printf("\n"); - - ASSERT_EQ(pRead->pHead->head.version, ver); - ASSERT_EQ(pRead->curVersion, ver + 1); - char newStr[100]; - sprintf(newStr, "%s-%d", ranStr, ver); - int len = strlen(newStr); - ASSERT_EQ(pRead->pHead->head.bodyLen, len); - for (int j = 0; j < len; j++) { - EXPECT_EQ(newStr[j], pRead->pHead->head.body[j]); - } - } - walCloseReader(pRead); -} - -TEST_F(WalRetentionEnv, repairMeta1) { - walResetEnv(); - int code; - - int i; - for (i = 0; i < 100; i++) { - char newStr[100]; - sprintf(newStr, "%s-%d", ranStr, i); - int len = strlen(newStr); - code = walAppendLog(pWal, i, 0, syncMeta, newStr, len); - ASSERT_EQ(code, 0); - } - - TearDown(); - - // getchar(); - char buf[100]; - sprintf(buf, "%s/meta-ver%d", pathName, 0); - taosRemoveFile(buf); - sprintf(buf, "%s/meta-ver%d", pathName, 1); - taosRemoveFile(buf); - SetUp(); - // getchar(); - - ASSERT_EQ(pWal->vers.lastVer, 99); - - SWalReader* pRead = walOpenReader(pWal, NULL, 0); - ASSERT(pRead != NULL); - - for (int i = 0; i < 1000; i++) { - int ver = taosRand() % 100; - code = walReadVer(pRead, ver); - ASSERT_EQ(code, 0); - - // printf("rrbody: \n"); - // for(int i = 0; i < pRead->pHead->head.len; i++) { - // printf("%d ", pRead->pHead->head.body[i]); - //} - // printf("\n"); - - ASSERT_EQ(pRead->pHead->head.version, ver); - ASSERT_EQ(pRead->curVersion, ver + 1); - char newStr[100]; - sprintf(newStr, "%s-%d", ranStr, ver); - int len = strlen(newStr); - ASSERT_EQ(pRead->pHead->head.bodyLen, len); - for (int j = 0; j < len; j++) { - EXPECT_EQ(newStr[j], pRead->pHead->head.body[j]); - } - } - - for (i = 100; i < 200; i++) { - char newStr[100]; - sprintf(newStr, "%s-%d", ranStr, i); - int len = strlen(newStr); - code = walAppendLog(pWal, i, 0, syncMeta, newStr, len); - ASSERT_EQ(code, 0); - } - - for (int i = 0; i < 1000; i++) { - int ver = taosRand() % 200; - code = walReadVer(pRead, ver); - ASSERT_EQ(code, 0); - - // printf("rrbody: \n"); - // for(int i = 0; i < pRead->pHead->head.len; i++) { - // printf("%d ", pRead->pHead->head.body[i]); - //} - // printf("\n"); - - ASSERT_EQ(pRead->pHead->head.version, ver); - ASSERT_EQ(pRead->curVersion, ver + 1); - char newStr[100]; - sprintf(newStr, "%s-%d", ranStr, ver); - int len = strlen(newStr); - ASSERT_EQ(pRead->pHead->head.bodyLen, len); - for (int j = 0; j < len; j++) { - EXPECT_EQ(newStr[j], pRead->pHead->head.body[j]); - } - } - walCloseReader(pRead); -} -*/