az/test: test cases for az apis

contrib/test/azure/CMakeLists.txt

@@ -14,13 +14,13 @@ find_library(CRYPTO_LIBRARY crypto $ENV{HOME}/.cos-local.2/lib64 $ENV{HOME}/.cos
 #find_library(CoreFoundation_Library CoreFoundation)
 #find_library(SystemConfiguration_Library SystemConfiguration)
 
- target_link_libraries(
+target_link_libraries(
-   azure-test
+  azure-test
- PRIVATE _azure_sdk
+  PRIVATE _azure_sdk
- PRIVATE ${CURL_LIBRARY}
+  PRIVATE ${CURL_LIBRARY}
- PRIVATE ${XML2_LIBRARY}
+  PRIVATE ${XML2_LIBRARY}
- PRIVATE ${SSL_LIBRARY}
+  PRIVATE ${SSL_LIBRARY}
- PRIVATE ${CRYPTO_LIBRARY}
+  PRIVATE ${CRYPTO_LIBRARY}
- PRIVATE dl
+  PRIVATE dl
- PRIVATE pthread
+  PRIVATE pthread
 )

contrib/test/azure/main.cpp

@@ -22,6 +22,9 @@ int main(int argc, char* argv[]) {
   try {
     auto containerClient =
         BlobContainerClient::CreateFromConnectionString(std::getenv("STORAGE_CONNECTION_STRING"), "td-test");
+
+    // Create the container if it does not exist
+    // std::cout << "Creating container: " << containerName << std::endl;
     // containerClient.CreateIfNotExists();
 
     /**************** Container SDK client ************************/
@@ -30,6 +33,21 @@ int main(int argc, char* argv[]) {
     // auto response = containerClient.ListBlobs();
     // auto blobListPage = response.Value;
     // auto blobListPage = response.Blobs;
+    //(void)_azUploadFrom(blobClient, file, offset, size);
+    /*
+        auto blockBlobClient = BlockBlobClient(endpointUrl, sharedKeyCredential);
+
+        // Create some data to upload into the blob.
+        std::vector<uint8_t> data = {1, 2, 3, 4};
+        Azure::Core::IO::MemoryBodyStream stream(data);
+
+        Azure::Response<Models::UploadBlockBlobResult> response = blockBlobClient.Upload(stream);
+
+        Models::UploadBlockBlobResult model = response.Value;
+        std::cout << "Last modified date of uploaded blob: " << model.LastModified.ToString()
+                  << std::endl;
+    */
+
     for (auto page = containerClient.ListBlobs(/*options*/); page.HasPage(); page.MoveToNextPage()) {
       for (auto& blob : page.Blobs) {
         std::cout << blob.Name << std::endl;

source/libs/azure/src/az.cpp

@@ -222,10 +222,6 @@ int32_t azPutObjectFromFileOffset(const char *file, const char *object_name, int
     std::string containerName = tsS3BucketName;
     auto        containerClient = blobServiceClient.GetBlobContainerClient(containerName);
 
-    // Create the container if it does not exist
-    // std::cout << "Creating container: " << containerName << std::endl;
-    // containerClient.CreateIfNotExists();
-
     std::string blobName = "blob.txt";
     uint8_t     blobContent[] = "Hello Azure!";
     // Create the block blob client
@@ -237,20 +233,6 @@ int32_t azPutObjectFromFileOffset(const char *file, const char *object_name, int
     // std::cout << "Uploading blob: " << blobName << std::endl;
     // blobClient.UploadFrom(blobContent, sizeof(blobContent));
     blobClient.UploadFrom(file, offset, size);
-    //(void)_azUploadFrom(blobClient, file, offset, size);
-    /*
-        auto blockBlobClient = BlockBlobClient(endpointUrl, sharedKeyCredential);
-
-        // Create some data to upload into the blob.
-        std::vector<uint8_t> data = {1, 2, 3, 4};
-        Azure::Core::IO::MemoryBodyStream stream(data);
-
-        Azure::Response<Models::UploadBlockBlobResult> response = blockBlobClient.Upload(stream);
-
-        Models::UploadBlockBlobResult model = response.Value;
-        std::cout << "Last modified date of uploaded blob: " << model.LastModified.ToString()
-                  << std::endl;
-    */
   } catch (const Azure::Core::RequestFailedException &e) {
     /*
     std::cout << "Status Code: " << static_cast<int>(e.StatusCode) << ", Reason Phrase: " << e.ReasonPhrase

source/libs/azure/test/azTest.cpp

@@ -2,456 +2,78 @@
 #include <cstring>
 #include <iostream>
 #include <queue>
-/*
-#include "walInt.h"
 
-const char*  ranStr = "tvapq02tcp";
+#include "az.h"
-const int    ranStrLen = strlen(ranStr);
-SWalSyncInfo syncMeta = {0};
 
-class WalCleanEnv : public ::testing::Test {
+extern int8_t tsS3Enabled;
- protected:
-  static void SetUpTestCase() {
-    int code = walInit(NULL);
-    ASSERT(code == 0);
-  }
 
-  static void TearDownTestCase() { walCleanUp(); }
+int32_t azInitEnv() {
+  int32_t code = 0;
 
-  void SetUp() override {
+  extern int8_t tsS3EpNum;
-    taosRemoveDir(pathName);
-    SWalCfg* pCfg = (SWalCfg*)taosMemoryMalloc(sizeof(SWalCfg));
-    memset(pCfg, 0, sizeof(SWalCfg));
-    pCfg->rollPeriod = -1;
-    pCfg->segSize = -1;
-    pCfg->retentionPeriod = 0;
-    pCfg->retentionSize = 0;
-    pCfg->level = TAOS_WAL_FSYNC;
-    pWal = walOpen(pathName, pCfg);
-    taosMemoryFree(pCfg);
-    ASSERT(pWal != NULL);
-  }
 
-  void TearDown() override {
+  extern char tsS3Hostname[][TSDB_FQDN_LEN];
-    walClose(pWal);
+  extern char tsS3AccessKeyId[][TSDB_FQDN_LEN];
-    pWal = NULL;
+  extern char tsS3AccessKeySecret[][TSDB_FQDN_LEN];
-  }
+  extern char tsS3BucketName[TSDB_FQDN_LEN];
 
-  SWal*       pWal = NULL;
+  /* TCS parameter format
-  const char* pathName = TD_TMP_DIR_PATH "wal_test";
+  tsS3Hostname[0] = "endpoint/<account-name>.blob.core.windows.net";
-};
+  tsS3AccessKeyId[0] = "<access-key-id/account-name>";
+  tsS3AccessKeySecret[0] = "<access-key-secret/account-key>";
+  tsS3BucketName = "<bucket/container-name>";
+  */
 
-class WalCleanDeleteEnv : public ::testing::Test {
+  const char *hostname = "endpoint/<account-name>.blob.core.windows.net";
- protected:
+  const char *accessKeyId = "<access-key-id/account-name>";
-  static void SetUpTestCase() {
+  const char *accessKeySecret = "<access-key-secret/account-key>";
-    int code = walInit(NULL);
+  const char *bucketName = "<bucket/container-name>";
-    ASSERT(code == 0);
-  }
 
-  static void TearDownTestCase() { walCleanUp(); }
+  tstrncpy(&tsS3Hostname[0][0], hostname, TSDB_FQDN_LEN);
+  tstrncpy(&tsS3AccessKeyId[0][0], accessKeyId, TSDB_FQDN_LEN);
+  tstrncpy(&tsS3AccessKeySecret[0][0], accessKeySecret, TSDB_FQDN_LEN);
+  tstrncpy(tsS3BucketName, bucketName, TSDB_FQDN_LEN);
 
-  void SetUp() override {
+  tstrncpy(tsTempDir, "/tmp/", PATH_MAX);
-    taosRemoveDir(pathName);
-    SWalCfg* pCfg = (SWalCfg*)taosMemoryMalloc(sizeof(SWalCfg));
-    memset(pCfg, 0, sizeof(SWalCfg));
-    pCfg->retentionPeriod = 0;
-    pCfg->retentionSize = 0;
-    pCfg->level = TAOS_WAL_FSYNC;
-    pWal = walOpen(pathName, pCfg);
-    taosMemoryFree(pCfg);
-    ASSERT(pWal != NULL);
-  }
 
-  void TearDown() override {
+  tsS3Enabled = true;
-    walClose(pWal);
-    pWal = NULL;
-  }
 
-  SWal*       pWal = NULL;
+  return code;
-  const char* pathName = TD_TMP_DIR_PATH "wal_test";
-};
-
-class WalKeepEnv : public ::testing::Test {
- protected:
-  static void SetUpTestCase() {
-    int code = walInit(NULL);
-    ASSERT(code == 0);
-  }
-
-  static void TearDownTestCase() { walCleanUp(); }
-
-  void walResetEnv() {
-    TearDown();
-    taosRemoveDir(pathName);
-    SetUp();
-  }
-
-  void SetUp() override {
-    SWalCfg* pCfg = (SWalCfg*)taosMemoryMalloc(sizeof(SWalCfg));
-    memset(pCfg, 0, sizeof(SWalCfg));
-    pCfg->rollPeriod = -1;
-    pCfg->segSize = -1;
-    pCfg->retentionPeriod = 0;
-    pCfg->retentionSize = 0;
-    pCfg->level = TAOS_WAL_FSYNC;
-    pWal = walOpen(pathName, pCfg);
-    taosMemoryFree(pCfg);
-    ASSERT(pWal != NULL);
-  }
-
-  void TearDown() override {
-    walClose(pWal);
-    pWal = NULL;
-  }
-
-  SWal*       pWal = NULL;
-  const char* pathName = TD_TMP_DIR_PATH "wal_test";
-};
-
-class WalRetentionEnv : public ::testing::Test {
- protected:
-  static void SetUpTestCase() {
-    int code = walInit(NULL);
-    ASSERT(code == 0);
-  }
-
-  static void TearDownTestCase() { walCleanUp(); }
-
-  void walResetEnv() {
-    TearDown();
-    taosRemoveDir(pathName);
-    SetUp();
-  }
-
-  void SetUp() override {
-    SWalCfg cfg;
-    cfg.rollPeriod = -1;
-    cfg.segSize = -1;
-    cfg.retentionPeriod = -1;
-    cfg.retentionSize = 0;
-    cfg.rollPeriod = 0;
-    cfg.vgId = 0;
-    cfg.level = TAOS_WAL_FSYNC;
-    pWal = walOpen(pathName, &cfg);
-    ASSERT(pWal != NULL);
-  }
-
-  void TearDown() override {
-    walClose(pWal);
-    pWal = NULL;
-  }
-
-  SWal*       pWal = NULL;
-  const char* pathName = TD_TMP_DIR_PATH "wal_test";
-};
-
-TEST_F(WalCleanEnv, createNew) {
-  walRollFileInfo(pWal);
-  ASSERT(pWal->fileInfoSet != NULL);
-  ASSERT_EQ(pWal->fileInfoSet->size, 1);
-  SWalFileInfo* pInfo = (SWalFileInfo*)taosArrayGetLast(pWal->fileInfoSet);
-  ASSERT_EQ(pInfo->firstVer, 0);
-  ASSERT_EQ(pInfo->lastVer, -1);
-  ASSERT_EQ(pInfo->closeTs, -1);
-  ASSERT_EQ(pInfo->fileSize, 0);
 }
 
-TEST_F(WalCleanEnv, serialize) {
+// TEST(AzTest, DISABLED_InterfaceTest) {
-  int code = walRollFileInfo(pWal);
+TEST(AzTest, InterfaceTest) {
-  ASSERT(code == 0);
+  int code = 0;
-  ASSERT(pWal->fileInfoSet != NULL);
 
-  code = walRollFileInfo(pWal);
+  code = azInitEnv();
-  ASSERT(code == 0);
+  GTEST_ASSERT_EQ(code, 0);
-  code = walRollFileInfo(pWal);
+  GTEST_ASSERT_EQ(tsS3Enabled, 1);
-  ASSERT(code == 0);
+
-  code = walRollFileInfo(pWal);
+  code = azBegin();
-  ASSERT(code == 0);
+  GTEST_ASSERT_EQ(code, 0);
-  code = walRollFileInfo(pWal);
+
-  ASSERT(code == 0);
+  code = azCheckCfg();
-  code = walRollFileInfo(pWal);
+  GTEST_ASSERT_EQ(code, 0);
-  ASSERT(code == 0);
+  /*
-  char* ss = NULL;
+  code = azPutObjectFromFileOffset(file, object_name, offset, size);
-  code = walMetaSerialize(pWal, &ss);
+  GTEST_ASSERT_EQ(code, 0);
-  ASSERT(code == 0);
+  code = azGetObjectBlock(object_name, offset, size, check, ppBlock);
-  printf("%s\n", ss);
+  GTEST_ASSERT_EQ(code, 0);
-  taosMemoryFree(ss);
+
-  code = walSaveMeta(pWal);
+  azDeleteObjectsByPrefix(prefix);
-  ASSERT(code == 0);
+  // list object to check
+
+  code = azPutObjectFromFile2(file, object, withcp);
+  GTEST_ASSERT_EQ(code, 0);
+  code = azGetObjectsByPrefix(prefix, path);
+  GTEST_ASSERT_EQ(code, 0);
+  code = azDeleteObjects(object_name, nobject);
+  GTEST_ASSERT_EQ(code, 0);
+  code = azGetObjectToFile(object_name, fileName);
+  GTEST_ASSERT_EQ(code, 0);
+
+  // GTEST_ASSERT_NE(pEnv, nullptr);
+  */
+
+  azEnd();
 }
-
-TEST_F(WalCleanEnv, removeOldMeta) {
-  int code = walRollFileInfo(pWal);
-  ASSERT(code == 0);
-  ASSERT(pWal->fileInfoSet != NULL);
-  code = walSaveMeta(pWal);
-  ASSERT(code == 0);
-  code = walRollFileInfo(pWal);
-  ASSERT(code == 0);
-  code = walSaveMeta(pWal);
-  ASSERT(code == 0);
-}
-
-TEST_F(WalKeepEnv, readOldMeta) {
-  walResetEnv();
-  int code;
-
-  syncMeta.isWeek = -1;
-  syncMeta.seqNum = UINT64_MAX;
-  syncMeta.term = UINT64_MAX;
-
-  for (int i = 0; i < 10; i++) {
-    code = walAppendLog(pWal, i, i + 1, syncMeta, (void*)ranStr, ranStrLen);
-    ASSERT_EQ(code, 0);
-    ASSERT_EQ(pWal->vers.lastVer, i);
-    code = walAppendLog(pWal, i + 2, i, syncMeta, (void*)ranStr, ranStrLen);
-    ASSERT_EQ(code, TSDB_CODE_WAL_INVALID_VER);
-    ASSERT_EQ(pWal->vers.lastVer, i);
-  }
-  char* oldss = NULL;
-  code = walMetaSerialize(pWal, &oldss);
-  ASSERT(code == 0);
-
-  TearDown();
-  SetUp();
-
-  ASSERT_EQ(pWal->vers.firstVer, 0);
-  ASSERT_EQ(pWal->vers.lastVer, 9);
-
-  char* newss = NULL;
-  code = walMetaSerialize(pWal, &newss);
-  ASSERT(code == 0);
-
-  int len = strlen(oldss);
-  ASSERT_EQ(len, strlen(newss));
-  for (int i = 0; i < len; i++) {
-    EXPECT_EQ(oldss[i], newss[i]);
-  }
-  taosMemoryFree(oldss);
-  taosMemoryFree(newss);
-}
-
-TEST_F(WalCleanEnv, write) {
-  int code;
-  for (int i = 0; i < 10; i++) {
-    code = walAppendLog(pWal, i, i + 1, syncMeta, (void*)ranStr, ranStrLen);
-    ASSERT_EQ(code, 0);
-    ASSERT_EQ(pWal->vers.lastVer, i);
-    code = walAppendLog(pWal, i + 2, i, syncMeta, (void*)ranStr, ranStrLen);
-    ASSERT_EQ(code, TSDB_CODE_WAL_INVALID_VER);
-    ASSERT_EQ(pWal->vers.lastVer, i);
-  }
-  code = walSaveMeta(pWal);
-  ASSERT_EQ(code, 0);
-}
-
-TEST_F(WalCleanEnv, rollback) {
-  int code;
-  for (int i = 0; i < 10; i++) {
-    code = walAppendLog(pWal, i, i + 1, syncMeta, (void*)ranStr, ranStrLen);
-    ASSERT_EQ(code, 0);
-    ASSERT_EQ(pWal->vers.lastVer, i);
-  }
-  code = walRollback(pWal, 12);
-  ASSERT_NE(code, 0);
-  ASSERT_EQ(pWal->vers.lastVer, 9);
-  code = walRollback(pWal, 9);
-  ASSERT_EQ(code, 0);
-  ASSERT_EQ(pWal->vers.lastVer, 8);
-  code = walRollback(pWal, 5);
-  ASSERT_EQ(code, 0);
-  ASSERT_EQ(pWal->vers.lastVer, 4);
-  code = walRollback(pWal, 3);
-  ASSERT_EQ(code, 0);
-  ASSERT_EQ(pWal->vers.lastVer, 2);
-  code = walSaveMeta(pWal);
-  ASSERT_EQ(code, 0);
-}
-
-TEST_F(WalCleanEnv, rollbackMultiFile) {
-  int code;
-  for (int i = 0; i < 10; i++) {
-    code = walAppendLog(pWal, i, i + 1, syncMeta, (void*)ranStr, ranStrLen);
-    ASSERT_EQ(code, 0);
-    ASSERT_EQ(pWal->vers.lastVer, i);
-    if (i == 5) {
-      walBeginSnapshot(pWal, i, 0);
-      walEndSnapshot(pWal);
-    }
-  }
-  code = walRollback(pWal, 12);
-  ASSERT_NE(code, 0);
-  ASSERT_EQ(pWal->vers.lastVer, 9);
-  code = walRollback(pWal, 9);
-  ASSERT_EQ(code, 0);
-  ASSERT_EQ(pWal->vers.lastVer, 8);
-  code = walRollback(pWal, 6);
-  ASSERT_EQ(code, 0);
-  ASSERT_EQ(pWal->vers.lastVer, 5);
-  code = walRollback(pWal, 5);
-  ASSERT_NE(code, 0);
-
-  ASSERT_EQ(pWal->vers.lastVer, 5);
-
-  code = walAppendLog(pWal, 6, 6, syncMeta, (void*)ranStr, ranStrLen);
-  ASSERT_EQ(code, 0);
-  ASSERT_EQ(pWal->vers.lastVer, 6);
-
-  code = walSaveMeta(pWal);
-  ASSERT_EQ(code, 0);
-}
-
-TEST_F(WalCleanDeleteEnv, roll) {
-  int code;
-  int i;
-  for (i = 0; i < 100; i++) {
-    code = walAppendLog(pWal, i, 0, syncMeta, (void*)ranStr, ranStrLen);
-    ASSERT_EQ(code, 0);
-    ASSERT_EQ(pWal->vers.lastVer, i);
-    code = walCommit(pWal, i);
-    ASSERT_EQ(pWal->vers.commitVer, i);
-  }
-
-  walBeginSnapshot(pWal, i - 1, 0);
-  ASSERT_EQ(pWal->vers.verInSnapshotting, i - 1);
-  walEndSnapshot(pWal);
-  ASSERT_EQ(pWal->vers.snapshotVer, i - 1);
-  ASSERT_EQ(pWal->vers.verInSnapshotting, -1);
-
-  code = walAppendLog(pWal, 5, 0, syncMeta, (void*)ranStr, ranStrLen);
-  ASSERT_NE(code, 0);
-
-  for (; i < 200; i++) {
-    code = walAppendLog(pWal, i, 0, syncMeta, (void*)ranStr, ranStrLen);
-    ASSERT_EQ(code, 0);
-    code = walCommit(pWal, i);
-    ASSERT_EQ(pWal->vers.commitVer, i);
-  }
-
-  code = walBeginSnapshot(pWal, i - 1, 0);
-  ASSERT_EQ(code, 0);
-  code = walEndSnapshot(pWal);
-  ASSERT_EQ(code, 0);
-}
-
-TEST_F(WalKeepEnv, readHandleRead) {
-  walResetEnv();
-  int         code;
-  SWalReader* pRead = walOpenReader(pWal, NULL, 0);
-  ASSERT(pRead != NULL);
-
-  int i;
-  for (i = 0; i < 100; i++) {
-    char newStr[100];
-    sprintf(newStr, "%s-%d", ranStr, i);
-    int len = strlen(newStr);
-    code = walAppendLog(pWal, i, 0, syncMeta, newStr, len);
-    ASSERT_EQ(code, 0);
-  }
-  for (int i = 0; i < 1000; i++) {
-    int ver = taosRand() % 100;
-    code = walReadVer(pRead, ver);
-    ASSERT_EQ(code, 0);
-
-    // printf("rrbody: \n");
-    // for(int i = 0; i < pRead->pHead->head.len; i++) {
-    // printf("%d ", pRead->pHead->head.body[i]);
-    //}
-    // printf("\n");
-
-    ASSERT_EQ(pRead->pHead->head.version, ver);
-    ASSERT_EQ(pRead->curVersion, ver + 1);
-    char newStr[100];
-    sprintf(newStr, "%s-%d", ranStr, ver);
-    int len = strlen(newStr);
-    ASSERT_EQ(pRead->pHead->head.bodyLen, len);
-    for (int j = 0; j < len; j++) {
-      EXPECT_EQ(newStr[j], pRead->pHead->head.body[j]);
-    }
-  }
-  walCloseReader(pRead);
-}
-
-TEST_F(WalRetentionEnv, repairMeta1) {
-  walResetEnv();
-  int code;
-
-  int i;
-  for (i = 0; i < 100; i++) {
-    char newStr[100];
-    sprintf(newStr, "%s-%d", ranStr, i);
-    int len = strlen(newStr);
-    code = walAppendLog(pWal, i, 0, syncMeta, newStr, len);
-    ASSERT_EQ(code, 0);
-  }
-
-  TearDown();
-
-  // getchar();
-  char buf[100];
-  sprintf(buf, "%s/meta-ver%d", pathName, 0);
-  taosRemoveFile(buf);
-  sprintf(buf, "%s/meta-ver%d", pathName, 1);
-  taosRemoveFile(buf);
-  SetUp();
-  // getchar();
-
-  ASSERT_EQ(pWal->vers.lastVer, 99);
-
-  SWalReader* pRead = walOpenReader(pWal, NULL, 0);
-  ASSERT(pRead != NULL);
-
-  for (int i = 0; i < 1000; i++) {
-    int ver = taosRand() % 100;
-    code = walReadVer(pRead, ver);
-    ASSERT_EQ(code, 0);
-
-    // printf("rrbody: \n");
-    // for(int i = 0; i < pRead->pHead->head.len; i++) {
-    // printf("%d ", pRead->pHead->head.body[i]);
-    //}
-    // printf("\n");
-
-    ASSERT_EQ(pRead->pHead->head.version, ver);
-    ASSERT_EQ(pRead->curVersion, ver + 1);
-    char newStr[100];
-    sprintf(newStr, "%s-%d", ranStr, ver);
-    int len = strlen(newStr);
-    ASSERT_EQ(pRead->pHead->head.bodyLen, len);
-    for (int j = 0; j < len; j++) {
-      EXPECT_EQ(newStr[j], pRead->pHead->head.body[j]);
-    }
-  }
-
-  for (i = 100; i < 200; i++) {
-    char newStr[100];
-    sprintf(newStr, "%s-%d", ranStr, i);
-    int len = strlen(newStr);
-    code = walAppendLog(pWal, i, 0, syncMeta, newStr, len);
-    ASSERT_EQ(code, 0);
-  }
-
-  for (int i = 0; i < 1000; i++) {
-    int ver = taosRand() % 200;
-    code = walReadVer(pRead, ver);
-    ASSERT_EQ(code, 0);
-
-    // printf("rrbody: \n");
-    // for(int i = 0; i < pRead->pHead->head.len; i++) {
-    // printf("%d ", pRead->pHead->head.body[i]);
-    //}
-    // printf("\n");
-
-    ASSERT_EQ(pRead->pHead->head.version, ver);
-    ASSERT_EQ(pRead->curVersion, ver + 1);
-    char newStr[100];
-    sprintf(newStr, "%s-%d", ranStr, ver);
-    int len = strlen(newStr);
-    ASSERT_EQ(pRead->pHead->head.bodyLen, len);
-    for (int j = 0; j < len; j++) {
-      EXPECT_EQ(newStr[j], pRead->pHead->head.body[j]);
-    }
-  }
-  walCloseReader(pRead);
-}
-*/