Support backup and restore (#4206)

#### What type of PR is this?

/kind feature
/area core

#### What this PR does / why we need it:

See https://github.com/JohnNiang/halo/blob/9921deb0768cb22fb4ba738e20fdc099f1678926/docs/backup-and-restore.md for more.

<img width="1906" alt="image" src="https://github.com/halo-dev/halo/assets/21301288/41531186-d305-44fd-8bdc-30df9b71af43">
<img width="1909" alt="image" src="https://github.com/halo-dev/halo/assets/21301288/3d7af1b9-37ad-4a40-9b81-f15ed0f1f6e8">


#### Which issue(s) this PR fixes:

Fixes #4059
Fixes #3274

#### Special notes for your reviewer:

#### Does this PR introduce a user-facing change?

```release-note
支持备份和恢复功能。
```

.editorconfig

@@ -508,3 +508,6 @@ ij_html_text_wrap = normal
 indent_size = 2
 ij_yaml_keep_indents_on_empty_lines = false
 ij_yaml_keep_line_breaks = true
+
+[*.md]
+indent_size = 2

api/src/main/java/run/halo/app/extension/ExtensionUtil.java

@@ -1,5 +1,6 @@
 package run.halo.app.extension;
 
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.Set;
 
@@ -11,21 +12,26 @@ public static boolean isDeleted(ExtensionOperator extension) {
             && extension.getMetadata().getDeletionTimestamp() != null;
     }
 
-    public static void addFinalizers(MetadataOperator metadata, Set<String> finalizers) {
-        var existingFinalizers = metadata.getFinalizers();
-        if (existingFinalizers == null) {
-            existingFinalizers = new HashSet<>();
+    public static boolean addFinalizers(MetadataOperator metadata, Set<String> finalizers) {
+        var modifiableFinalizers = new HashSet<>(
+            metadata.getFinalizers() == null ? Collections.emptySet() : metadata.getFinalizers());
+        var added = modifiableFinalizers.addAll(finalizers);
+        if (added) {
+            metadata.setFinalizers(modifiableFinalizers);
         }
-        existingFinalizers.addAll(finalizers);
-        metadata.setFinalizers(existingFinalizers);
+        return added;
     }
 
-    public static void removeFinalizers(MetadataOperator metadata, Set<String> finalizers) {
-        var existingFinalizers = metadata.getFinalizers();
-        if (existingFinalizers != null) {
-            existingFinalizers.removeAll(finalizers);
+    public static boolean removeFinalizers(MetadataOperator metadata, Set<String> finalizers) {
+        if (metadata.getFinalizers() == null) {
+            return false;
         }
-        metadata.setFinalizers(existingFinalizers);
+        var existingFinalizers = new HashSet<>(metadata.getFinalizers());
+        var removed = existingFinalizers.removeAll(finalizers);
+        if (removed) {
+            metadata.setFinalizers(existingFinalizers);
+        }
+        return removed;
     }
 
 }

api/src/test/java/run/halo/app/extension/ExtensionUtilTest.java

@@ -34,25 +34,25 @@ void testIsNotDeleted() {
     void addFinalizers() {
         var metadata = new Metadata();
         assertNull(metadata.getFinalizers());
-        ExtensionUtil.addFinalizers(metadata, Set.of("fake"));
+        assertTrue(ExtensionUtil.addFinalizers(metadata, Set.of("fake")));
 
         assertEquals(Set.of("fake"), metadata.getFinalizers());
 
-        ExtensionUtil.addFinalizers(metadata, Set.of("fake"));
+        assertFalse(ExtensionUtil.addFinalizers(metadata, Set.of("fake")));
         assertEquals(Set.of("fake"), metadata.getFinalizers());
 
-        ExtensionUtil.addFinalizers(metadata, Set.of("another-fake"));
+        assertTrue(ExtensionUtil.addFinalizers(metadata, Set.of("another-fake")));
         assertEquals(Set.of("fake", "another-fake"), metadata.getFinalizers());
     }
 
     @Test
     void removeFinalizers() {
         var metadata = new Metadata();
-        ExtensionUtil.removeFinalizers(metadata, Set.of("fake"));
+        assertFalse(ExtensionUtil.removeFinalizers(metadata, Set.of("fake")));
         assertNull(metadata.getFinalizers());
 
         metadata.setFinalizers(new HashSet<>(Set.of("fake")));
-        ExtensionUtil.removeFinalizers(metadata, Set.of("fake"));
+        assertTrue(ExtensionUtil.removeFinalizers(metadata, Set.of("fake")));
         assertEquals(Set.of(), metadata.getFinalizers());
     }
 

application/src/main/java/run/halo/app/infra/SchemeInitializer.java

@@ -32,6 +32,7 @@
 import run.halo.app.extension.ConfigMap;
 import run.halo.app.extension.SchemeManager;
 import run.halo.app.extension.Secret;
+import run.halo.app.migration.Backup;
 import run.halo.app.plugin.extensionpoint.ExtensionDefinition;
 import run.halo.app.plugin.extensionpoint.ExtensionPointDefinition;
 import run.halo.app.search.extension.SearchEngine;
@@ -89,6 +90,9 @@ public void onApplicationEvent(@NonNull ApplicationStartedEvent event) {
         schemeManager.register(AuthProvider.class);
         schemeManager.register(UserConnection.class);
 
+        // migration.halo.run
+        schemeManager.register(Backup.class);
+
         eventPublisher.publishEvent(new SchemeInitializedEvent(this));
     }
 }

application/src/main/java/run/halo/app/infra/utils/FileUtils.java

@@ -1,5 +1,6 @@
 package run.halo.app.infra.utils;
 
+import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
 import static org.springframework.util.FileSystemUtils.deleteRecursively;
 
 import java.io.Closeable;
@@ -12,7 +13,9 @@
 import java.nio.file.Paths;
 import java.nio.file.SimpleFileVisitor;
 import java.nio.file.attribute.BasicFileAttributes;
+import java.util.Set;
 import java.util.function.Consumer;
+import java.util.function.Predicate;
 import java.util.jar.JarEntry;
 import java.util.jar.JarOutputStream;
 import java.util.stream.Stream;
@@ -21,6 +24,7 @@
 import java.util.zip.ZipOutputStream;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.lang.NonNull;
+import org.springframework.util.AntPathMatcher;
 import org.springframework.util.Assert;
 import run.halo.app.infra.exception.AccessDeniedException;
 
@@ -246,4 +250,31 @@ public static void copy(Path source, Path dest, CopyOption... options) {
             throw new RuntimeException(e);
         }
     }
+
+    public static void copyRecursively(Path src, Path target, Set<String> excludes)
+        throws IOException {
+        var pathMatcher = new AntPathMatcher();
+        Predicate<Path> shouldExclude = path -> excludes.stream()
+            .anyMatch(pattern -> pathMatcher.match(pattern, path.toString()));
+        Files.walkFileTree(src, new SimpleFileVisitor<>() {
+            @Override
+            public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
+                throws IOException {
+                if (!shouldExclude.test(src.relativize(file))) {
+                    Files.copy(file, target.resolve(src.relativize(file)), REPLACE_EXISTING);
+                }
+                return super.visitFile(file, attrs);
+            }
+
+            @Override
+            public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs)
+                throws IOException {
+                if (shouldExclude.test(src.relativize(dir))) {
+                    return FileVisitResult.SKIP_SUBTREE;
+                }
+                Files.createDirectories(target.resolve(src.relativize(dir)));
+                return super.preVisitDirectory(dir, attrs);
+            }
+        });
+    }
 }

application/src/main/java/run/halo/app/migration/Backup.java

@@ -0,0 +1,65 @@
+package run.halo.app.migration;
+
+import io.swagger.v3.oas.annotations.media.Schema;
+import java.time.Instant;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.ToString;
+import run.halo.app.extension.AbstractExtension;
+import run.halo.app.extension.GVK;
+
+@Data
+@ToString(callSuper = true)
+@EqualsAndHashCode(callSuper = true)
+@GVK(group = "migration.halo.run", version = "v1alpha1", kind = "Backup",
+    plural = "backups", singular = "backup")
+public class Backup extends AbstractExtension {
+
+    private Spec spec = new Spec();
+
+    private Status status = new Status();
+
+    @Data
+    @Schema(name = "BackupSpec")
+    public static class Spec {
+
+        @Schema(description = "Backup file format. Currently, only zip format is supported.")
+        private String format;
+
+        private Instant expiresAt;
+
+    }
+
+    @Data
+    @Schema(name = "BackupStatus")
+    public static class Status {
+
+        private Phase phase = Phase.PENDING;
+
+        private Instant startTimestamp;
+
+        private Instant completionTimestamp;
+
+        private String failureReason;
+
+        private String failureMessage;
+
+        /**
+         * Size of backup file. Data unit: byte
+         */
+        private Long size;
+
+        /**
+         * Name of backup file.
+         */
+        private String filename;
+    }
+
+    public enum Phase {
+        PENDING,
+        RUNNING,
+        SUCCEEDED,
+        FAILED,
+    }
+
+}

application/src/main/java/run/halo/app/migration/BackupReconciler.java

@@ -0,0 +1,133 @@
+package run.halo.app.migration;
+
+import static run.halo.app.extension.ExtensionUtil.addFinalizers;
+import static run.halo.app.extension.ExtensionUtil.isDeleted;
+import static run.halo.app.extension.ExtensionUtil.removeFinalizers;
+import static run.halo.app.extension.controller.Reconciler.Result.doNotRetry;
+import static run.halo.app.migration.Constant.HOUSE_KEEPER_FINALIZER;
+
+import java.time.Clock;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.Set;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Component;
+import reactor.core.Exceptions;
+import run.halo.app.extension.ExtensionClient;
+import run.halo.app.extension.controller.Controller;
+import run.halo.app.extension.controller.ControllerBuilder;
+import run.halo.app.extension.controller.Reconciler;
+import run.halo.app.extension.controller.Reconciler.Request;
+import run.halo.app.migration.Backup.Phase;
+
+@Slf4j
+@Component
+public class BackupReconciler implements Reconciler<Request> {
+
+    private final ExtensionClient client;
+
+    private final MigrationService migrationService;
+
+    private Clock clock;
+
+    public BackupReconciler(ExtensionClient client, MigrationService migrationService) {
+        this.client = client;
+        this.migrationService = migrationService;
+        clock = Clock.systemDefaultZone();
+    }
+
+    /**
+     * Set clock. The method is only for unit test.
+     *
+     * @param clock is new clock
+     */
+    void setClock(Clock clock) {
+        this.clock = clock;
+    }
+
+    @Override
+    public Result reconcile(Request request) {
+        return client.fetch(Backup.class, request.name())
+            .map(backup -> {
+                var metadata = backup.getMetadata();
+                var status = backup.getStatus();
+                var spec = backup.getSpec();
+                if (isDeleted(backup)) {
+                    if (removeFinalizers(metadata, Set.of(HOUSE_KEEPER_FINALIZER))) {
+                        migrationService.cleanup(backup).block();
+                        client.update(backup);
+                    }
+                    return doNotRetry();
+                }
+                if (addFinalizers(metadata, Set.of(HOUSE_KEEPER_FINALIZER))) {
+                    client.update(backup);
+                }
+
+                if (Phase.PENDING.equals(status.getPhase())) {
+                    // Do backup
+                    try {
+                        status.setPhase(Phase.RUNNING);
+                        status.setStartTimestamp(Instant.now(clock));
+                        updateStatus(request.name(), status);
+                        // Long period execution when backing up
+                        migrationService.backup(backup).block();
+                        status.setPhase(Phase.SUCCEEDED);
+                        status.setCompletionTimestamp(Instant.now(clock));
+                        updateStatus(request.name(), status);
+                    } catch (Throwable t) {
+                        var unwrapped = Exceptions.unwrap(t);
+                        log.error("Failed to backup", unwrapped);
+                        // Only happen when shutting down
+                        status.setPhase(Phase.FAILED);
+                        if (unwrapped instanceof InterruptedException) {
+                            status.setFailureReason("Interrupted");
+                            status.setFailureMessage("The backup process was interrupted.");
+                        } else {
+                            status.setFailureReason("SystemError");
+                            status.setFailureMessage(
+                                "Something went wrong! Error message: " + unwrapped.getMessage());
+                        }
+                        updateStatus(request.name(), status);
+                    }
+                }
+                // Only happen when failing to update status when interrupted
+                if (Phase.RUNNING.equals(status.getPhase())) {
+                    status.setPhase(Phase.FAILED);
+                    status.setFailureReason("UnexpectedExit");
+                    status.setFailureMessage("The backup process may exit abnormally.");
+                    updateStatus(request.name(), status);
+                }
+                // Check the expires at and requeue if necessary
+                if (isTerminal(status.getPhase())) {
+                    var expiresAt = spec.getExpiresAt();
+                    if (expiresAt != null) {
+                        var now = Instant.now(clock);
+                        if (now.isBefore(expiresAt)) {
+                            return new Result(true, Duration.between(now, expiresAt));
+                        }
+                        client.delete(backup);
+                    }
+                }
+                return doNotRetry();
+            }).orElseGet(Result::doNotRetry);
+    }
+
+    private void updateStatus(String name, Backup.Status status) {
+        client.fetch(Backup.class, name)
+            .ifPresent(backup -> {
+                backup.setStatus(status);
+                client.update(backup);
+            });
+    }
+
+    private static boolean isTerminal(Phase phase) {
+        return Phase.FAILED.equals(phase) || Phase.SUCCEEDED.equals(phase);
+    }
+
+    @Override
+    public Controller setupWith(ControllerBuilder builder) {
+        return builder
+            .extension(new Backup())
+            .build();
+    }
+}

application/src/main/java/run/halo/app/migration/Constant.java

@@ -0,0 +1,12 @@
+package run.halo.app.migration;
+
+public enum Constant {
+    ;
+
+    public static final String GROUP = "migration.halo.run";
+
+    public static final String VERSION = "v1alpha1";
+
+    public static final String HOUSE_KEEPER_FINALIZER = "housekeeper";
+
+}