Merge branch '3.0' into enh/refactorBackend

source/common/src/cos.c

@@ -70,7 +70,7 @@ static void s3PrintError(const char *func, S3Status status, char error_details[]
   if (status < S3StatusErrorAccessDenied) {
     uError("%s: %s", __func__, S3_get_status_name(status));
   } else {
-    uError("%s: %s, %s", __func__, S3_get_status_name(status), error_details);
+    uError("%s: %s, %s, %d", __func__, S3_get_status_name(status), error_details, status);
   }
 }
 
@@ -466,6 +466,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) {
   data.gb = 0;
   data.noStatus = noStatus;
 
+  uError("ERROR: %s stat file %s: ", __func__, file);
   if (taosStatFile(file, &contentLength, NULL, NULL) < 0) {
     uError("ERROR: %s Failed to stat file %s: ", __func__, file);
     code = TAOS_SYSTEM_ERROR(errno);

source/libs/stream/inc/streamBackendRocksdb.h

@@ -254,5 +254,5 @@ int32_t  bkdMgtGetDelta(SBkdMgt* bm, char* taskId, int64_t chkpId, SArray* list,
 int32_t  bkdMgtDumpTo(SBkdMgt* bm, char* taskId, char* dname);
 void     bkdMgtDestroy(SBkdMgt* bm);
 
-int32_t taskDbGenChkpUploadPath(void* arg, void* bkdMgt, int64_t chkpId, int8_t type, char** pathkj);
+int32_t taskDbGenChkpUploadData(void* arg, void* bkdMgt, int64_t chkpId, int8_t type, char** pathkj);
 #endif

source/libs/stream/src/streamBackendRocksdb.c

@@ -1736,7 +1736,7 @@ int32_t taskDbGenChkpUploadData__s3(STaskDbWrapper* pDb, void* bkdChkpMgt, int64
 
   return 0;
 }
-int32_t taskDbGenChkpUploadPath(void* arg, void* mgt, int64_t chkpId, int8_t type, char** path) {
+int32_t taskDbGenChkpUploadData(void* arg, void* mgt, int64_t chkpId, int8_t type, char** path) {
   STaskDbWrapper* pDb = arg;
   UPLOAD_TYPE     utype = type;
 
@@ -3363,13 +3363,27 @@ _err:
   return code >= 0 ? 0 : -1;
 }
 
+int32_t isBkdDataMeta(char* name) {
+  const char* pCurrent = "CURRENT";
+  int32_t     currLen = strlen(pCurrent);
+
+  const char* pManifest = "MANIFEST-";
+  int32_t     maniLen = strlen(pManifest);
+
+  if (strlen(name) >= maniLen && strncmp(name, pManifest, maniLen) == 0) {
+    return 1;
+  } else if (strlen(name) == currLen && strcmp(name, pCurrent) == 0) {
+    return 1;
+  }
+  return 0;
+}
 int32_t compareHashTableImpl(SHashObj* p1, SHashObj* p2, SArray* diff) {
   int32_t code = 0;
   size_t  len = 0;
   void*   pIter = taosHashIterate(p2, NULL);
   while (pIter) {
     char* name = taosHashGetKey(pIter, &len);
-    if (!taosHashGet(p1, name, len)) {
+    if (!isBkdDataMeta(name) && !taosHashGet(p1, name, len)) {
       char* p = taosStrdup(name);
       taosArrayPush(diff, &p);
     }
@@ -3431,13 +3445,22 @@ int32_t dbChkpGetDelta(SDbChkp* p, int64_t chkpId, SArray* list) {
       continue;
     }
   }
+
+  void* pIter = taosHashIterate(p->pSstTbl[1 - p->idx], NULL);
+  while (pIter) {
+    char *name = taosHashGetKey(pIter, NULL);
+    stError("curr file list:  %s", name);
+    pIter = taosHashIterate(p->pSstTbl[1 - p->idx], pIter);
+  }
+
   if (p->init == 0) {
     void* pIter = taosHashIterate(p->pSstTbl[1 - p->idx], NULL);
     while (pIter) {
       size_t len;
       char*  name = taosHashGetKey(pIter, &len);
-      if (name != NULL && len != 0) {
+      if (name != NULL && !isBkdDataMeta(name)) {
-        taosArrayPush(p->pAdd, &name);
+        char* fname = taosStrdup(name);
+        taosArrayPush(p->pAdd, &fname);
       }
       pIter = taosHashIterate(p->pSstTbl[1 - p->idx], pIter);
     }
@@ -3538,12 +3561,12 @@ int32_t dbChkpDumpTo(SDbChkp* p, char* dname) {
     goto _ERROR;
   }
 
-  code = taosMkDir(dstDir);
+  // code = taosMkDir(dstDir);
-  if (code != 0) {
+  // if (code != 0) {
-    terrno = TAOS_SYSTEM_ERROR(errno);
+  //   terrno = TAOS_SYSTEM_ERROR(errno);
-    stError("failed to mkdir srcDir %s, reason: %s", dstDir, terrstr());
+  //   stError("failed to mkdir srcDir %s, reason: %s", dstDir, terrstr());
-    goto _ERROR;
+  //   goto _ERROR;
-  }
+  // }
 
   // clear current file
   memset(dstBuf, 0, len);
@@ -3563,7 +3586,9 @@ int32_t dbChkpDumpTo(SDbChkp* p, char* dname) {
     sprintf(srcBuf, "%s%s%s", srcDir, TD_DIRSEP, filename);
     sprintf(dstBuf, "%s%s%s", dstDir, TD_DIRSEP, filename);
 
-    taosCopyFile(srcBuf, dstBuf);
+    if (taosCopyFile(srcBuf, dstBuf) < 0) {
+      stError("failed to copy file from %s to %s", srcBuf, dstBuf);
+    }
   }
   // del file in $name
   for (int i = 0; i < taosArrayGetSize(p->pDel); i++) {
@@ -3580,14 +3605,18 @@ int32_t dbChkpDumpTo(SDbChkp* p, char* dname) {
   memset(dstBuf, 0, len);
   sprintf(srcBuf, "%s%s%s", srcDir, TD_DIRSEP, p->pCurrent);
   sprintf(dstBuf, "%s%s%s", dstDir, TD_DIRSEP, p->pCurrent);
-  taosCopyFile(srcBuf, dstBuf);
+  if (taosCopyFile(srcBuf, dstBuf) < 0) {
+    stError("failed to copy file from %s to %s", srcBuf, dstBuf);
+  }
 
   // copy manifest file to dst dir
   memset(srcBuf, 0, len);
   memset(dstBuf, 0, len);
   sprintf(srcBuf, "%s%s%s", srcDir, TD_DIRSEP, p->pManifest);
   sprintf(dstBuf, "%s%s%s", dstDir, TD_DIRSEP, p->pManifest);
-  taosCopyFile(srcBuf, dstBuf);
+  if (taosCopyFile(srcBuf, dstBuf) < 0) {
+    stError("failed to copy file from %s to %s", srcBuf, dstBuf);
+  }
 
   // clear delta data buf
   taosArrayClearP(p->pAdd, taosMemoryFree);

source/libs/stream/src/streamCheckpoint.c

@@ -341,12 +341,12 @@ int32_t doUploadChkp(void* param) {
   char*            path = NULL;
   int32_t          code = 0;
 
-  if ((code = taskDbGenChkpUploadPath(arg->pTask->pBackend, arg->pTask->pMeta->bkdChkptMgt, arg->chkpId,
+  if ((code = taskDbGenChkpUploadData(arg->pTask->pBackend, arg->pTask->pMeta->bkdChkptMgt, arg->chkpId,
                                       (int8_t)(arg->type), &path)) != 0) {
     stError("s-task:%s failed to gen upload checkpoint:%" PRId64 "", arg->pTask->id.idStr, arg->chkpId);
   }
   if (code == 0 && uploadCheckpoint(arg->taskId, path) != 0) {
-    stError("s-task:%s faile to upload checkpoint:%" PRId64 "", arg->pTask->id.idStr, arg->chkpId);
+    stError("s-task:%s failed to upload checkpoint:%" PRId64, arg->pTask->id.idStr, arg->chkpId);
   }
 
   taosMemoryFree(path);