From 2c11dc7e568fcc081ae8037d840f2c56a2ccac42 Mon Sep 17 00:00:00 2001 From: Four <8122332+HungryFour@users.noreply.github.com> Date: Mon, 20 Jul 2026 11:43:13 +0800 Subject: [PATCH 01/28] feat: add SQLite-backed VexFS filesystem --- agent_files/cli/test/vexfs_cli_smoke.sh | 66 + agent_files/cli/vexfs_fskit_state.h | 9 + agent_files/cli/vexfs_fskit_state.mm | 70 + agent_files/cli/vexfs_main.cpp | 995 ++++++++ .../macos/VexFS.xcodeproj/project.pbxproj | 562 +++++ agent_files/macos/VexFSApp/ContentView.swift | 32 + .../macos/VexFSApp/VexFSApp.entitlements | 8 + agent_files/macos/VexFSApp/VexFSApp.swift | 11 + agent_files/macos/VexFSAppEx/Info.plist | 46 + .../macos/VexFSAppEx/VexFS-Bridging-Header.h | 6 + .../macos/VexFSAppEx/VexFSAppEx.entitlements | 10 + agent_files/macos/VexFSAppEx/VexFSAppEx.swift | 12 + .../macos/VexFSAppEx/VexFSBackend.swift | 279 +++ .../macos/VexFSAppEx/VexFSFileSystem.swift | 119 + agent_files/macos/VexFSAppEx/VexFSItem.swift | 28 + .../VexFSAppEx/VexFSVolume+Operations.swift | 258 ++ .../macos/VexFSAppEx/VexFSVolume.swift | 185 ++ agent_files/macos/generate_xcode_project.rb | 121 + agent_files/macos/package/Install.command | 6 + agent_files/macos/package/README.md | 89 + agent_files/macos/package/install.sh | 31 + agent_files/macos/package/uninstall.sh | 13 + agent_files/macos/package_preview.sh | 122 + .../common/include/vexfs_mount_contract.h | 151 ++ .../src/vexfs_mount_contract_sqlite.cpp | 812 +++++++ .../test/vexfs_mount_contract_smoke.cpp | 228 ++ build_sqlite.sh | 25 +- tests/eval/vexfs/README.md | 64 + tests/eval/vexfs/run.py | 1902 +++++++++++++++ .../sqlite/agent_files/vexfs_contract.yaml | 111 + vexdb_sqlite/CMakeLists.txt | 40 +- .../include/agent_files/vexfs_sqlite.h | 25 + vexdb_sqlite/src/agent_files/vexfs_sqlite.cpp | 2139 +++++++++++++++++ vexdb_sqlite/src/vexdb_sqlite_init.cpp | 4 + vexdb_sqlite/test/vexfs_static_smoke.c | 270 +++ 35 files changed, 8845 insertions(+), 4 deletions(-) create mode 100755 agent_files/cli/test/vexfs_cli_smoke.sh create mode 100644 agent_files/cli/vexfs_fskit_state.h create mode 100644 agent_files/cli/vexfs_fskit_state.mm create mode 100644 agent_files/cli/vexfs_main.cpp create mode 100644 agent_files/macos/VexFS.xcodeproj/project.pbxproj create mode 100644 agent_files/macos/VexFSApp/ContentView.swift create mode 100644 agent_files/macos/VexFSApp/VexFSApp.entitlements create mode 100644 agent_files/macos/VexFSApp/VexFSApp.swift create mode 100644 agent_files/macos/VexFSAppEx/Info.plist create mode 100644 agent_files/macos/VexFSAppEx/VexFS-Bridging-Header.h create mode 100644 agent_files/macos/VexFSAppEx/VexFSAppEx.entitlements create mode 100644 agent_files/macos/VexFSAppEx/VexFSAppEx.swift create mode 100644 agent_files/macos/VexFSAppEx/VexFSBackend.swift create mode 100644 agent_files/macos/VexFSAppEx/VexFSFileSystem.swift create mode 100644 agent_files/macos/VexFSAppEx/VexFSItem.swift create mode 100644 agent_files/macos/VexFSAppEx/VexFSVolume+Operations.swift create mode 100644 agent_files/macos/VexFSAppEx/VexFSVolume.swift create mode 100644 agent_files/macos/generate_xcode_project.rb create mode 100755 agent_files/macos/package/Install.command create mode 100644 agent_files/macos/package/README.md create mode 100755 agent_files/macos/package/install.sh create mode 100755 agent_files/macos/package/uninstall.sh create mode 100755 agent_files/macos/package_preview.sh create mode 100644 agent_files/mount/common/include/vexfs_mount_contract.h create mode 100644 agent_files/mount/common/src/vexfs_mount_contract_sqlite.cpp create mode 100644 agent_files/mount/common/test/vexfs_mount_contract_smoke.cpp create mode 100644 tests/eval/vexfs/README.md create mode 100644 tests/eval/vexfs/run.py create mode 100644 tests/spec/sqlite/agent_files/vexfs_contract.yaml create mode 100644 vexdb_sqlite/include/agent_files/vexfs_sqlite.h create mode 100644 vexdb_sqlite/src/agent_files/vexfs_sqlite.cpp create mode 100644 vexdb_sqlite/test/vexfs_static_smoke.c diff --git a/agent_files/cli/test/vexfs_cli_smoke.sh b/agent_files/cli/test/vexfs_cli_smoke.sh new file mode 100755 index 0000000000..2ed08e5acf --- /dev/null +++ b/agent_files/cli/test/vexfs_cli_smoke.sh @@ -0,0 +1,66 @@ +#!/bin/bash +set -euo pipefail + +VEXFS="${1:?usage: vexfs_cli_smoke.sh /path/to/vexfs}" +TMP_DIR="$(mktemp -d -t vexfs-cli-smoke)" +DB="$TMP_DIR/vexfs.sqlite3" +cleanup() { + rm -f "$DB" "$DB-wal" "$DB-shm" + rmdir "$TMP_DIR" 2>/dev/null || true +} +trap cleanup EXIT + +set +e +MISSING_DOCTOR="$("$VEXFS" --db "$DB" --workspace smoke --json doctor)" +MISSING_STATUS=$? +set -e +[ "$MISSING_STATUS" -eq 1 ] +[ ! -e "$DB" ] +printf '%s' "$MISSING_DOCTOR" | /usr/bin/python3 -c \ + 'import json,sys; value=json.load(sys.stdin); assert "error" in value["database"]' + +"$VEXFS" --db "$DB" --workspace smoke setup >/dev/null +"$VEXFS" --db "$DB" --workspace smoke mkdir /agent +printf 'hello from cli' | "$VEXFS" --db "$DB" --workspace smoke write /agent/task.txt >/dev/null +printf 'alpha' | "$VEXFS" --db "$DB" --workspace smoke write /agent/version.txt >/dev/null +printf 'beta' | "$VEXFS" --db "$DB" --workspace smoke write /agent/version.txt >/dev/null + +CONTENT="$("$VEXFS" --db "$DB" --workspace smoke cat /agent/task.txt)" +[ "$CONTENT" = "hello from cli" ] + +LIST="$("$VEXFS" --db "$DB" --workspace smoke --json ls /agent)" +printf '%s' "$LIST" | /usr/bin/python3 -c \ + 'import json,sys; rows=json.load(sys.stdin); assert rows[0]["name"] == "task.txt"' + +HISTORY="$("$VEXFS" --db "$DB" --workspace smoke --json history /agent/version.txt)" +printf '%s' "$HISTORY" | /usr/bin/python3 -c \ + 'import json,sys; page=json.load(sys.stdin); rows=page["entries"]; assert [row["version"] for row in rows] == [2,1]; assert rows[0]["current"] is True; assert page["next_before"] is None' +[ "$("$VEXFS" --db "$DB" --workspace smoke show /agent/version.txt --version 1)" = "alpha" ] +set +e +DIFF="$("$VEXFS" --db "$DB" --workspace smoke diff /agent/version.txt --from 1 --to 2)" +DIFF_STATUS=$? +set -e +[ "$DIFF_STATUS" -eq 1 ] +printf '%s' "$DIFF" | grep -q -- '-alpha' +set +e +"$VEXFS" --db "$DB" --workspace smoke diff /agent/version.txt --from 99 --to 2 >/dev/null 2>&1 +MISSING_VERSION_STATUS=$? +set -e +[ "$MISSING_VERSION_STATUS" -eq 3 ] +"$VEXFS" --db "$DB" --workspace smoke restore /agent/version.txt --version 1 --dry-run >/dev/null +[ "$("$VEXFS" --db "$DB" --workspace smoke stat /agent/version.txt | /usr/bin/python3 -c 'import json,sys; print(json.load(sys.stdin)["version"])')" = "2" ] +[ "$("$VEXFS" --db "$DB" --workspace smoke restore /agent/version.txt --version 1)" = "3" ] +[ "$("$VEXFS" --db "$DB" --workspace smoke cat /agent/version.txt)" = "alpha" ] + +set +e +DOCTOR="$("$VEXFS" --db "$DB" --workspace smoke --json doctor)" +DOCTOR_STATUS=$? +set -e +[ "$DOCTOR_STATUS" -eq 0 ] || [ "$DOCTOR_STATUS" -eq 1 ] +printf '%s' "$DOCTOR" | /usr/bin/python3 -c \ + 'import json,sys; value=json.load(sys.stdin); assert value["database"]["contract_version"] == "0.3.0"' + +MOUNTS="$("$VEXFS" --json mount status)" +printf '%s' "$MOUNTS" | /usr/bin/python3 -c 'import json,sys; assert isinstance(json.load(sys.stdin), list)' + +echo "VEXFS CLI SMOKE: PASS" diff --git a/agent_files/cli/vexfs_fskit_state.h b/agent_files/cli/vexfs_fskit_state.h new file mode 100644 index 0000000000..7c82746b6d --- /dev/null +++ b/agent_files/cli/vexfs_fskit_state.h @@ -0,0 +1,9 @@ +#pragma once + +#include + +// Returns 2 for enabled, 1 for disabled, 0 for missing, and -1 when FSKit +// cannot answer. The URL is optional and only populated for an installed module. +extern "C" int vexfs_fskit_extension_state(const char *bundle_identifier, + char *url, + std::size_t url_size); diff --git a/agent_files/cli/vexfs_fskit_state.mm b/agent_files/cli/vexfs_fskit_state.mm new file mode 100644 index 0000000000..6f836d993c --- /dev/null +++ b/agent_files/cli/vexfs_fskit_state.mm @@ -0,0 +1,70 @@ +#include "vexfs_fskit_state.h" + +#import +#import + +#include +#include + +#include +#include + +extern "C" int vexfs_fskit_extension_state(const char *bundle_identifier, + char *url, + std::size_t url_size) { + if (bundle_identifier == nullptr) return -1; + if (url != nullptr && url_size > 0) url[0] = '\0'; + + @autoreleasepool { + static void *fskit_framework = dlopen( + "/System/Library/Frameworks/FSKit.framework/FSKit", RTLD_LAZY | RTLD_LOCAL); + if (fskit_framework == nullptr) return -1; + Class client_class = NSClassFromString(@"FSClient"); + SEL shared_selector = NSSelectorFromString(@"sharedInstance"); + SEL fetch_selector = + NSSelectorFromString(@"fetchInstalledExtensionsWithCompletionHandler:"); + if (client_class == Nil || ![client_class respondsToSelector:shared_selector]) return -1; + + using SendNoArguments = id (*)(id, SEL); + id client = reinterpret_cast(objc_msgSend)(client_class, + shared_selector); + if (client == nil || ![client respondsToSelector:fetch_selector]) return -1; + + dispatch_semaphore_t finished = dispatch_semaphore_create(0); + __block NSArray *extensions = nil; + __block NSError *fetch_error = nil; + void (^completion)(NSArray *, NSError *) = ^(NSArray *values, NSError *error) { + extensions = values; + fetch_error = error; + dispatch_semaphore_signal(finished); + }; + + using SendCompletion = void (*)(id, SEL, id); + reinterpret_cast(objc_msgSend)(client, fetch_selector, completion); + const long wait_result = dispatch_semaphore_wait( + finished, dispatch_time(DISPATCH_TIME_NOW, 5 * NSEC_PER_SEC)); + if (wait_result != 0) return -2; + if (fetch_error != nil) { + if (std::getenv("VEXFS_DEBUG") != nullptr) { + std::fprintf(stderr, "vexfs: FSClient error=%s\n", + fetch_error.localizedDescription.UTF8String); + } + return -3; + } + if (extensions == nil) return -4; + + NSString *wanted = [NSString stringWithUTF8String:bundle_identifier]; + for (id extension in extensions) { + NSString *identifier = [extension valueForKey:@"bundleIdentifier"]; + if (![identifier isEqualToString:wanted]) continue; + + NSURL *module_url = [extension valueForKey:@"url"]; + if (url != nullptr && url_size > 0 && module_url != nil) { + std::snprintf(url, url_size, "%s", module_url.path.UTF8String); + } + NSNumber *enabled = [extension valueForKey:@"enabled"]; + return enabled.boolValue ? 2 : 1; + } + return 0; + } +} diff --git a/agent_files/cli/vexfs_main.cpp b/agent_files/cli/vexfs_main.cpp new file mode 100644 index 0000000000..db5e1cabe1 --- /dev/null +++ b/agent_files/cli/vexfs_main.cpp @@ -0,0 +1,995 @@ +#include "vexfs_mount_contract.h" +#include "vexfs_fskit_state.h" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +struct Options { + std::string database; + std::string workspace = "default"; + std::string mount_point; + bool json = false; + std::vector arguments; +}; + +std::string HomeDirectory() { + const char *home = std::getenv("HOME"); + return home == nullptr ? std::string(".") : std::string(home); +} + +std::string DefaultDatabasePath() { + if (const char *configured = std::getenv("VEXFS_DATABASE")) return configured; + const std::filesystem::path directory = std::filesystem::path(HomeDirectory()) / + "Library/Application Support/VexFS"; + return (directory / "vexfs.sqlite3").string(); +} + +void Usage(std::ostream &output) { + output << + "Usage: vexfs [--db PATH] [--workspace NAME] COMMAND [ARGS]\n" + "\n" + "Commands:\n" + " setup [--mount PATH] Initialize and optionally mount a workspace\n" + " init Alias for setup without mount\n" + " mkdir PATH... Create directories (including parents)\n" + " write PATH [LOCAL_FILE] Write a local file or stdin\n" + " cat PATH Print a file\n" + " ls [PATH] [--json] List a directory\n" + " stat PATH Print file metadata as JSON\n" + " history PATH [--limit N] [--before N] [--json]\n" + " List one page of file versions\n" + " show PATH --version N Print one historical version\n" + " diff PATH --from N [--to N] Compare two versions (default: current)\n" + " restore PATH --version N [--dry-run]\n" + " Restore as a new version\n" + " mv SOURCE DESTINATION Move a file or directory\n" + " rm [-r] PATH Remove a file or directory\n" + " descriptor OUTPUT Write an FSKit .vexfs descriptor\n" + " mount MOUNT_POINT Mount through the enabled FSKit extension\n" + " mount status [MOUNT_POINT] Show active VexFS mounts\n" + " unmount MOUNT_POINT Unmount a VexFS mount\n" + " doctor [--json] Check macOS, extension and SQLite state\n"; +} + +Options ParseOptions(int argc, char **argv) { + Options options; + options.database = DefaultDatabasePath(); + for (int index = 1; index < argc; ++index) { + std::string argument = argv[index]; + if (argument == "--db" && index + 1 < argc) { + options.database = argv[++index]; + } else if (argument == "--workspace" && index + 1 < argc) { + options.workspace = argv[++index]; + } else if (argument == "--mount" && index + 1 < argc) { + options.mount_point = argv[++index]; + } else if (argument == "--json") { + options.json = true; + } else if (argument == "--help" || argument == "-h") { + Usage(std::cout); + std::exit(0); + } else { + options.arguments.push_back(std::move(argument)); + } + } + return options; +} + +std::string ErrorMessage(const vexfs_mount_error &error) { + return std::string(error.message); +} + +const char *ErrorCode(vexfs_mount_status status) { + switch (status) { + case VEXFS_MOUNT_INVALID_ARGUMENT: return "VEXFS_INVALID_ARGUMENT"; + case VEXFS_MOUNT_NOT_FOUND: return "VEXFS_NOT_FOUND"; + case VEXFS_MOUNT_CONFLICT: return "VEXFS_CONFLICT"; + case VEXFS_MOUNT_READ_ONLY: return "VEXFS_READ_ONLY"; + case VEXFS_MOUNT_BUSY: return "VEXFS_BUSY"; + case VEXFS_MOUNT_PERMISSION_DENIED: return "VEXFS_PERMISSION_DENIED"; + case VEXFS_MOUNT_NO_SPACE: return "VEXFS_NO_SPACE"; + case VEXFS_MOUNT_CORRUPTION: return "VEXFS_CORRUPTION"; + case VEXFS_MOUNT_UNSUPPORTED: return "VEXFS_UNSUPPORTED"; + case VEXFS_MOUNT_DATABASE_ERROR: return "VEXFS_DATABASE_ERROR"; + case VEXFS_MOUNT_INTERNAL_ERROR: return "VEXFS_INTERNAL_ERROR"; + default: return "VEXFS_ERROR"; + } +} + +int ExitCode(vexfs_mount_status status) { + switch (status) { + case VEXFS_MOUNT_INVALID_ARGUMENT: return 2; + case VEXFS_MOUNT_NOT_FOUND: return 3; + case VEXFS_MOUNT_PERMISSION_DENIED: + case VEXFS_MOUNT_READ_ONLY: return 4; + case VEXFS_MOUNT_CONFLICT: return 5; + case VEXFS_MOUNT_NO_SPACE: return 6; + case VEXFS_MOUNT_BUSY: + case VEXFS_MOUNT_DATABASE_ERROR: return 7; + case VEXFS_MOUNT_CORRUPTION: return 8; + case VEXFS_MOUNT_UNSUPPORTED: return 9; + default: return 1; + } +} + +class CliError : public std::runtime_error { + public: + CliError(vexfs_mount_status status, const std::string &message) + : std::runtime_error(message), status(status), exit_code(ExitCode(status)), + code(ErrorCode(status)) {} + + vexfs_mount_status status; + int exit_code; + std::string code; +}; + +class Session { + public: + explicit Session(const Options &options, bool initialize = true) { + const std::filesystem::path path(options.database); + if (initialize && path.has_parent_path()) { + const bool existed = std::filesystem::exists(path.parent_path()); + std::filesystem::create_directories(path.parent_path()); + if (!existed && chmod(path.parent_path().c_str(), 0700) != 0) + throw std::runtime_error(std::strerror(errno)); + } + vexfs_mount_config config{}; + config.abi_version = VEXFS_MOUNT_ABI_VERSION; + config.database_path = options.database.c_str(); + config.workspace = options.workspace.c_str(); + config.busy_timeout_ms = 5000; + config.flags = initialize ? 0 : VEXFS_MOUNT_OPEN_NO_CREATE; + vexfs_mount_error error{}; + const auto status = vexfs_mount_session_open(&config, &session_, &error); + if (status != VEXFS_MOUNT_OK) throw CliError(status, ErrorMessage(error)); + } + ~Session() { vexfs_mount_session_close(session_); } + vexfs_mount_session *get() const { return session_; } + + private: + vexfs_mount_session *session_ = nullptr; +}; + +std::vector ReadInput(const std::vector &arguments, size_t index) { + if (arguments.size() > index) { + std::ifstream input(arguments[index], std::ios::binary); + if (!input) throw std::runtime_error("cannot open local file: " + arguments[index]); + return {std::istreambuf_iterator(input), std::istreambuf_iterator()}; + } + return {std::istreambuf_iterator(std::cin), std::istreambuf_iterator()}; +} + +void Check(vexfs_mount_status status, const vexfs_mount_error &error) { + if (status != VEXFS_MOUNT_OK) throw CliError(status, ErrorMessage(error)); +} + +std::string BytesToString(vexfs_mount_bytes *bytes) { + std::string result; + if (bytes->data != nullptr) { + result.assign(static_cast(bytes->data), static_cast(bytes->size)); + vexfs_mount_free(bytes->data); + } + bytes->data = nullptr; + bytes->size = 0; + return result; +} + +std::string JsonUnescape(const std::string &json, size_t *position) { + std::string value; + while (*position < json.size()) { + const char c = json[(*position)++]; + if (c == '"') break; + if (c != '\\' || *position >= json.size()) { + value.push_back(c); + continue; + } + const char escaped = json[(*position)++]; + switch (escaped) { + case 'n': value.push_back('\n'); break; + case 'r': value.push_back('\r'); break; + case 't': value.push_back('\t'); break; + case 'b': value.push_back('\b'); break; + case 'f': value.push_back('\f'); break; + case '"': value.push_back('"'); break; + case '\\': value.push_back('\\'); break; + default: value.push_back('?'); break; + } + } + return value; +} + +std::string JsonEscape(const std::string &value); + +void PrintNames(const std::string &json) { + const std::string marker = "\"name\":\""; + size_t position = 0; + while ((position = json.find(marker, position)) != std::string::npos) { + position += marker.size(); + std::cout << JsonUnescape(json, &position) << '\n'; + } +} + +struct ParsedCommand { + std::vector positional; + std::vector> values; + std::vector flags; + + std::string Value(const std::string &name, bool required = true) const { + for (const auto &entry : values) { + if (entry.first == name) return entry.second; + } + if (required) throw std::runtime_error("missing required option: " + name); + return {}; + } + + bool Flag(const std::string &name) const { + for (const std::string &flag : flags) if (flag == name) return true; + return false; + } +}; + +bool Contains(const std::vector &values, const std::string &value) { + for (const std::string &candidate : values) if (candidate == value) return true; + return false; +} + +ParsedCommand ParseCommand(const std::vector &arguments, + const std::vector &value_options, + const std::vector &flag_options) { + ParsedCommand parsed; + for (size_t index = 1; index < arguments.size(); ++index) { + const std::string &argument = arguments[index]; + if (Contains(value_options, argument)) { + if (index + 1 >= arguments.size()) + throw std::runtime_error("option needs a value: " + argument); + for (const auto &existing : parsed.values) { + if (existing.first == argument) + throw std::runtime_error("option may only be used once: " + argument); + } + parsed.values.emplace_back(argument, arguments[++index]); + } else if (Contains(flag_options, argument)) { + if (parsed.Flag(argument)) + throw std::runtime_error("option may only be used once: " + argument); + parsed.flags.push_back(argument); + } else if (!argument.empty() && argument.front() == '-') { + throw std::runtime_error("unknown option: " + argument); + } else { + parsed.positional.push_back(argument); + } + } + return parsed; +} + +int64_t PositiveInteger(const std::string &value, const std::string &name) { + if (value.empty()) + throw CliError(VEXFS_MOUNT_INVALID_ARGUMENT, name + " must be a positive integer"); + char *end = nullptr; + errno = 0; + const long long result = std::strtoll(value.c_str(), &end, 10); + if (errno != 0 || end == value.c_str() || *end != '\0' || result <= 0) { + throw CliError(VEXFS_MOUNT_INVALID_ARGUMENT, name + " must be a positive integer"); + } + return static_cast(result); +} + +int64_t NonnegativeInteger(const std::string &value, const std::string &name) { + if (value.empty()) + throw CliError(VEXFS_MOUNT_INVALID_ARGUMENT, name + " must be a non-negative integer"); + char *end = nullptr; + errno = 0; + const long long result = std::strtoll(value.c_str(), &end, 10); + if (errno != 0 || end == value.c_str() || *end != '\0' || result < 0) { + throw CliError(VEXFS_MOUNT_INVALID_ARGUMENT, name + " must be a non-negative integer"); + } + return static_cast(result); +} + +int64_t JsonInteger(const std::string &json, const std::string &name) { + const std::string marker = "\"" + name + "\":"; + size_t position = json.find(marker); + if (position == std::string::npos) + throw std::runtime_error("invalid JSON result: missing " + name); + position += marker.size(); + char *end = nullptr; + const long long value = std::strtoll(json.c_str() + position, &end, 10); + if (end == json.c_str() + position) + throw std::runtime_error("invalid JSON result: bad " + name); + return static_cast(value); +} + +std::string JsonString(const std::string &json, const std::string &name) { + const std::string marker = "\"" + name + "\":\""; + size_t position = json.find(marker); + if (position == std::string::npos) + throw std::runtime_error("invalid JSON result: missing " + name); + position += marker.size(); + return JsonUnescape(json, &position); +} + +bool JsonBoolean(const std::string &json, const std::string &name) { + const std::string marker = "\"" + name + "\":"; + const size_t position = json.find(marker); + if (position == std::string::npos) + throw std::runtime_error("invalid JSON result: missing " + name); + return json.compare(position + marker.size(), 4, "true") == 0; +} + +void PrintHistory(const std::string &json) { + std::cout << "VERSION\tCOMMIT\tSIZE\tCREATED_AT\tCURRENT\tMESSAGE\n"; + const std::string entries_marker = "\"entries\":["; + size_t position = json.find(entries_marker); + if (position == std::string::npos) throw std::runtime_error("invalid history JSON"); + position += entries_marker.size(); + while ((position = json.find('{', position)) != std::string::npos) { + const size_t entries_end = json.find(']', position); + if (entries_end != std::string::npos && entries_end < position) break; + const size_t end = json.find('}', position); + if (end == std::string::npos) throw std::runtime_error("invalid history JSON"); + const std::string row = json.substr(position, end - position + 1); + std::cout << JsonInteger(row, "version") << '\t' + << JsonInteger(row, "commit") << '\t' + << JsonInteger(row, "size") << '\t' + << JsonInteger(row, "created_at") << '\t' + << (JsonBoolean(row, "current") ? "*" : "") << '\t' + << JsonString(row, "message") << '\n'; + position = end + 1; + } + const std::string next_marker = "\"next_before\":"; + const size_t next = json.find(next_marker); + if (next != std::string::npos && json.compare(next + next_marker.size(), 4, "null") != 0) { + std::cout << "NEXT_BEFORE\t" << JsonInteger(json, "next_before") << '\n'; + } +} + +std::vector SplitLines(const std::string &value) { + std::vector lines; + size_t start = 0; + while (start < value.size()) { + const size_t newline = value.find('\n', start); + if (newline == std::string::npos) { + lines.emplace_back(value.data() + start, value.size() - start); + break; + } + lines.emplace_back(value.data() + start, newline - start + 1); + start = newline + 1; + } + return lines; +} + +void PrintDiffLine(char prefix, std::string_view line) { + std::cout << prefix << line; + if (line.empty() || line.back() != '\n') { + std::cout << "\n\\ No newline at end of file\n"; + } +} + +bool HasNul(const std::string &value) { + return value.find('\0') != std::string::npos; +} + +bool PrintDiff(const std::string &path, int64_t from_version, const std::string &from, + int64_t to_version, const std::string &to, bool json) { + const bool changed = from != to; + const bool binary = HasNul(from) || HasNul(to); + if (json) { + std::cout << "{\"path\":\"" << JsonEscape(path) << "\",\"from\":" + << from_version << ",\"to\":" << to_version + << ",\"changed\":" << (changed ? "true" : "false") + << ",\"binary\":" << (binary ? "true" : "false") + << ",\"from_size\":" << from.size() + << ",\"to_size\":" << to.size() << "}\n"; + return changed; + } + if (!changed) return false; + std::cout << "--- " << path << '@' << from_version << '\n' + << "+++ " << path << '@' << to_version << '\n'; + if (binary) { + std::cout << "Binary files differ\n"; + return true; + } + constexpr size_t kDetailedDiffLimit = 4u * 1024u * 1024u; + if (from.size() > kDetailedDiffLimit || to.size() > kDetailedDiffLimit) { + std::cout << "Text files differ; detailed diff omitted for files larger than 4 MiB\n"; + return true; + } + const auto old_lines = SplitLines(from); + const auto new_lines = SplitLines(to); + size_t prefix = 0; + while (prefix < old_lines.size() && prefix < new_lines.size() && + old_lines[prefix] == new_lines[prefix]) ++prefix; + size_t suffix = 0; + while (suffix < old_lines.size() - prefix && suffix < new_lines.size() - prefix && + old_lines[old_lines.size() - suffix - 1] == new_lines[new_lines.size() - suffix - 1]) { + ++suffix; + } + const size_t context_start = prefix > 3 ? prefix - 3 : 0; + const size_t old_change_end = old_lines.size() - suffix; + const size_t new_change_end = new_lines.size() - suffix; + const size_t old_end = std::min(old_lines.size(), old_change_end + 3); + const size_t new_end = std::min(new_lines.size(), new_change_end + 3); + std::cout << "@@ -" << context_start + 1 << ',' << old_end - context_start + << " +" << context_start + 1 << ',' << new_end - context_start << " @@\n"; + for (size_t index = context_start; index < prefix; ++index) + PrintDiffLine(' ', old_lines[index]); + for (size_t index = prefix; index < old_change_end; ++index) + PrintDiffLine('-', old_lines[index]); + for (size_t index = prefix; index < new_change_end; ++index) + PrintDiffLine('+', new_lines[index]); + for (size_t index = new_change_end; index < new_end; ++index) + PrintDiffLine(' ', new_lines[index]); + return true; +} + +std::string JsonEscape(const std::string &value) { + std::string output; + output.reserve(value.size() + 8); + for (const unsigned char c : value) { + switch (c) { + case '"': output += "\\\""; break; + case '\\': output += "\\\\"; break; + case '\b': output += "\\b"; break; + case '\f': output += "\\f"; break; + case '\n': output += "\\n"; break; + case '\r': output += "\\r"; break; + case '\t': output += "\\t"; break; + default: + if (c < 0x20) { + char escaped[7] = {}; + std::snprintf(escaped, sizeof(escaped), "\\u%04x", c); + output += escaped; + } else { + output.push_back(static_cast(c)); + } + } + } + return output; +} + +void WriteDescriptor(const Options &options, const std::filesystem::path &path) { + if (path.has_parent_path()) std::filesystem::create_directories(path.parent_path()); + std::ofstream output(path, std::ios::binary | std::ios::trunc); + if (!output) throw std::runtime_error("cannot write descriptor: " + path.string()); + output << "{\n \"version\": 1,\n \"database_path\": \"" + << JsonEscape(std::filesystem::absolute(options.database).string()) + << "\",\n \"workspace\": \"" << JsonEscape(options.workspace) << "\"\n}\n"; +} + +std::filesystem::path WriteMountResourceDescriptor(const Options &options) { + const std::filesystem::path database = + std::filesystem::absolute(options.database).lexically_normal(); + const std::filesystem::path directory = database.parent_path(); + if (directory.empty() || database.filename().empty()) { + throw std::runtime_error("database path must include a file name"); + } + std::filesystem::create_directories(directory); + const std::filesystem::path descriptor = directory / ".vexfs-volume.json"; + std::ofstream output(descriptor, std::ios::binary | std::ios::trunc); + if (!output) throw std::runtime_error("cannot write mount resource: " + descriptor.string()); + output << "{\n \"version\": 2,\n \"database_file\": \"" + << JsonEscape(database.filename().string()) + << "\",\n \"workspace\": \"" << JsonEscape(options.workspace) << "\"\n}\n"; + return directory; +} + +int RunProcess(const char *program, const std::vector &arguments) { + const pid_t child = fork(); + if (child < 0) throw std::runtime_error(std::strerror(errno)); + if (child == 0) { + std::vector values; + values.push_back(const_cast(program)); + for (const std::string &argument : arguments) + values.push_back(const_cast(argument.c_str())); + values.push_back(nullptr); + execv(program, values.data()); + _exit(127); + } + int status = 0; + if (waitpid(child, &status, 0) < 0) throw std::runtime_error(std::strerror(errno)); + return WIFEXITED(status) ? WEXITSTATUS(status) : 1; +} + +struct ProcessOutput { + int exit_code = 1; + std::string output; +}; + +ProcessOutput CaptureProcess(const char *program, const std::vector &arguments) { + int descriptors[2] = {-1, -1}; + if (pipe(descriptors) != 0) throw std::runtime_error(std::strerror(errno)); + const pid_t child = fork(); + if (child < 0) { + close(descriptors[0]); + close(descriptors[1]); + throw std::runtime_error(std::strerror(errno)); + } + if (child == 0) { + close(descriptors[0]); + dup2(descriptors[1], STDOUT_FILENO); + dup2(descriptors[1], STDERR_FILENO); + close(descriptors[1]); + std::vector values; + values.push_back(const_cast(program)); + for (const std::string &argument : arguments) + values.push_back(const_cast(argument.c_str())); + values.push_back(nullptr); + execv(program, values.data()); + _exit(127); + } + close(descriptors[1]); + ProcessOutput result; + char buffer[4096]; + for (;;) { + const ssize_t count = read(descriptors[0], buffer, sizeof(buffer)); + if (count > 0) result.output.append(buffer, static_cast(count)); + else if (count == 0) break; + else if (errno != EINTR) break; + } + close(descriptors[0]); + int status = 0; + if (waitpid(child, &status, 0) < 0) throw std::runtime_error(std::strerror(errno)); + result.exit_code = WIFEXITED(status) ? WEXITSTATUS(status) : 1; + return result; +} + +enum class ExtensionState { kMissing, kEnabled, kDisabled, kRegistered }; + +ExtensionState VexFSExtensionState() { + char module_url[4096]{}; + const int fskit_state = vexfs_fskit_extension_state( + "io.vexdb.vexfs.extension", module_url, sizeof(module_url)); + if (std::getenv("VEXFS_DEBUG") != nullptr) { + std::cerr << "vexfs: FSKit state=" << fskit_state + << " module=" << module_url << '\n'; + } + if (fskit_state == 2) return ExtensionState::kEnabled; + if (fskit_state == 1) return ExtensionState::kDisabled; + + // FSKit filters out modules whose signature or provisioning it cannot accept. + // pluginkit can still distinguish that state from a package that is not installed. + // Its '+' marker is registration/election, not the FSKit per-user enable switch, + // so it must never claim that mounting is ready. + const ProcessOutput result = CaptureProcess("/usr/bin/pluginkit", + {"-m", "-A", "-D", "-i", "io.vexdb.vexfs.extension"}); + if (result.exit_code != 0 || result.output.empty()) return ExtensionState::kMissing; + for (size_t position = 0; position < result.output.size();) { + const size_t end = result.output.find('\n', position); + const std::string line = result.output.substr(position, + end == std::string::npos ? std::string::npos : end - position); + const size_t marker = line.find_first_not_of(" \t"); + if (marker != std::string::npos) { + if (line[marker] == '+' || line[marker] == '!') return ExtensionState::kRegistered; + if (line[marker] == '-') return ExtensionState::kDisabled; + } + if (end == std::string::npos) break; + position = end + 1; + } + return ExtensionState::kRegistered; +} + +const char *ExtensionStateName(ExtensionState state) { + switch (state) { + case ExtensionState::kEnabled: return "enabled"; + case ExtensionState::kDisabled: return "disabled"; + case ExtensionState::kRegistered: return "registered"; + case ExtensionState::kMissing: return "missing"; + } + return "missing"; +} + +struct MountedVolume { + std::string source; + std::string target; + std::string type; +}; + +std::string NormalizedPath(const std::string &path) { + std::error_code error; + const auto canonical = std::filesystem::weakly_canonical(path, error); + if (!error) return canonical.string(); + return std::filesystem::absolute(path).lexically_normal().string(); +} + +std::vector VexFSMounts() { + struct statfs *entries = nullptr; + const int count = getmntinfo(&entries, MNT_NOWAIT); + std::vector mounts; + for (int index = 0; index < count; ++index) { + if (std::strcmp(entries[index].f_fstypename, "vexfs") != 0) continue; + mounts.push_back({entries[index].f_mntfromname, entries[index].f_mntonname, + entries[index].f_fstypename}); + } + return mounts; +} + +bool ProductVersionSupported(const std::string &version) { + char *end = nullptr; + const long major = std::strtol(version.c_str(), &end, 10); + return end != version.c_str() && major >= 26; +} + +std::string ProductVersion() { + char buffer[128] = {}; + size_t size = sizeof(buffer); + if (sysctlbyname("kern.osproductversion", buffer, &size, nullptr, 0) != 0) return "unknown"; + return buffer; +} + +std::string DatabaseDiagnostics(Session &session) { + vexfs_mount_bytes json{}; + vexfs_mount_error error{}; + Check(vexfs_mount_diagnostics(session.get(), &json, &error), error); + return BytesToString(&json); +} + +std::string ReadHistoricalVersion(Session &session, const std::string &path, int64_t version) { + vexfs_mount_bytes content{}; + vexfs_mount_error error{}; + Check(vexfs_mount_read_version(session.get(), path.c_str(), version, &content, &error), error); + return BytesToString(&content); +} + +int64_t CurrentFileVersion(Session &session, const std::string &path) { + vexfs_mount_bytes json{}; + vexfs_mount_error error{}; + Check(vexfs_mount_stat(session.get(), path.c_str(), &json, &error), error); + return JsonInteger(BytesToString(&json), "version"); +} + +struct VersionComparison { + std::string json; + bool changed = false; + bool binary = false; + int64_t from_size = 0; + int64_t to_size = 0; +}; + +VersionComparison CompareHistoricalVersions(Session &session, const std::string &path, + int64_t from_version, int64_t to_version) { + vexfs_mount_bytes json{}; + vexfs_mount_error error{}; + Check(vexfs_mount_compare_versions(session.get(), path.c_str(), from_version, to_version, + &json, &error), error); + VersionComparison result; + result.json = BytesToString(&json); + result.changed = JsonBoolean(result.json, "changed"); + result.binary = JsonBoolean(result.json, "binary"); + result.from_size = JsonInteger(result.json, "from_size"); + result.to_size = JsonInteger(result.json, "to_size"); + return result; +} + +bool RunVersionDiff(Session &session, const std::string &path, int64_t from_version, + int64_t to_version, bool json) { + const VersionComparison comparison = + CompareHistoricalVersions(session, path, from_version, to_version); + if (json) { + std::cout << comparison.json << '\n'; + return comparison.changed; + } + if (!comparison.changed) return false; + constexpr int64_t kDetailedDiffLimit = 4LL * 1024LL * 1024LL; + if (comparison.binary || comparison.from_size > kDetailedDiffLimit || + comparison.to_size > kDetailedDiffLimit) { + std::cout << "--- " << path << '@' << from_version << '\n' + << "+++ " << path << '@' << to_version << '\n'; + if (comparison.binary) std::cout << "Binary files differ\n"; + else std::cout << "Text files differ; detailed diff omitted for files larger than 4 MiB\n"; + return true; + } + const std::string from = ReadHistoricalVersion(session, path, from_version); + const std::string to = ReadHistoricalVersion(session, path, to_version); + return PrintDiff(path, from_version, from, to_version, to, false); +} + +int PrintMountStatus(const Options &options, const std::string &requested_path) { + const std::string normalized = requested_path.empty() ? "" : NormalizedPath(requested_path); + std::vector mounts; + for (const auto &mount : VexFSMounts()) { + if (normalized.empty() || NormalizedPath(mount.target) == normalized) mounts.push_back(mount); + } + if (options.json) { + std::cout << '['; + for (size_t index = 0; index < mounts.size(); ++index) { + if (index != 0) std::cout << ','; + std::cout << "{\"source\":\"" << JsonEscape(mounts[index].source) + << "\",\"target\":\"" << JsonEscape(mounts[index].target) + << "\",\"type\":\"" << JsonEscape(mounts[index].type) << "\"}"; + } + std::cout << "]\n"; + } else if (mounts.empty()) { + std::cout << "no VexFS mounts\n"; + } else { + for (const auto &mount : mounts) + std::cout << mount.target << " <- " << mount.source << " (" << mount.type << ")\n"; + } + return requested_path.empty() || !mounts.empty() ? 0 : 1; +} + +void PrepareMountPoint(const std::string &path) { + const std::filesystem::path mount_point(path); + if (std::filesystem::exists(mount_point) && !std::filesystem::is_directory(mount_point)) { + throw std::runtime_error("mount point is not a directory: " + path); + } + std::filesystem::create_directories(mount_point); + if (chmod(mount_point.c_str(), 0700) != 0) throw std::runtime_error(std::strerror(errno)); + if (std::filesystem::directory_iterator(mount_point) != std::filesystem::directory_iterator()) { + throw std::runtime_error("mount point must be empty: " + path); + } +} + +std::string MountedFileSystemAt(const std::string &path) { + const std::string normalized = NormalizedPath(path); + struct statfs *entries = nullptr; + const int count = getmntinfo(&entries, MNT_NOWAIT); + for (int index = 0; index < count; ++index) { + if (NormalizedPath(entries[index].f_mntonname) == normalized) + return entries[index].f_fstypename; + } + return {}; +} + +int MountWorkspace(const Options &options, const std::string &mount_point) { + const std::string mounted_type = MountedFileSystemAt(mount_point); + if (mounted_type == "vexfs") { + std::cout << mount_point << " is already mounted\n"; + return 0; + } + if (!mounted_type.empty()) { + throw std::runtime_error("mount point is already used by " + mounted_type); + } + const ExtensionState extension = VexFSExtensionState(); + if (extension != ExtensionState::kEnabled) { + throw std::runtime_error(std::string("VexFS extension is ") + + ExtensionStateName(extension) + + "; install VexFS.app and enable its file system extension in System Settings"); + } + PrepareMountPoint(mount_point); + const std::filesystem::path resource_directory = WriteMountResourceDescriptor(options); + int result = 1; + for (int attempt = 0; attempt < 3; ++attempt) { + result = RunProcess("/sbin/mount", + {"-F", "-t", "vexfs", resource_directory.string(), mount_point}); + if (MountedFileSystemAt(mount_point) == "vexfs") { + result = 0; + break; + } + if (result == 0) break; + if (attempt != 2) usleep(500 * 1000); + } + if (result == 0 && MountedFileSystemAt(mount_point) != "vexfs") { + throw std::runtime_error("mount command returned success but VexFS is not mounted"); + } + return result; +} + +int RunDoctor(const Options &options) { + const std::string version = ProductVersion(); + const bool supported_macos = ProductVersionSupported(version); + void *framework = dlopen("/System/Library/Frameworks/FSKit.framework/FSKit", RTLD_LAZY); + const bool fskit = framework != nullptr; + if (framework != nullptr) dlclose(framework); + const ExtensionState extension = VexFSExtensionState(); + const auto mounts = VexFSMounts(); + std::string database_details; + std::string database_error; + try { + Session session(options, false); + database_details = DatabaseDiagnostics(session); + } catch (const std::exception &error) { + database_error = error.what(); + } + const bool database_readable = !database_details.empty(); + const bool database_ok = database_readable && JsonBoolean(database_details, "compatible"); + if (options.json) { + std::cout << "{\"macos\":\"" << JsonEscape(version) + << "\",\"macos_supported\":" << (supported_macos ? "true" : "false") + << ",\"fskit\":" << (fskit ? "true" : "false") + << ",\"extension\":\"" << ExtensionStateName(extension) << "\"" + << ",\"database\":"; + if (database_readable) std::cout << database_details; + else std::cout << "{\"path\":\"" << JsonEscape(options.database) + << "\",\"error\":\"" << JsonEscape(database_error) << "\"}"; + std::cout << ",\"mount_count\":" << mounts.size() << "}\n"; + } else { + std::cout << "macOS: " << version << (supported_macos ? " (supported)" : " (requires 26+)") << '\n' + << "FSKit: " << (fskit ? "available" : "missing") << '\n' + << "extension: " << ExtensionStateName(extension) << '\n' + << "database: " << options.database << '\n' + << "workspace: " << options.workspace << '\n' + << "SQLite contract: " << (database_ok ? "ok" : + (database_readable ? "upgrade required or incompatible" : database_error)) << '\n' + << "active mounts: " << mounts.size() << '\n'; + if (database_readable) std::cout << "database details: " << database_details << '\n'; + } + return supported_macos && fskit && extension == ExtensionState::kEnabled && database_ok ? 0 : 1; +} + +int Run(const Options &options) { + if (options.arguments.empty()) { + Usage(std::cerr); + return 2; + } + const std::string &command = options.arguments[0]; + + if (command == "descriptor") { + if (options.arguments.size() != 2) throw std::runtime_error("descriptor needs OUTPUT"); + WriteDescriptor(options, options.arguments[1]); + std::cout << options.arguments[1] << '\n'; + return 0; + } + if (command == "mount") { + if (options.arguments.size() >= 2 && options.arguments[1] == "status") { + if (options.arguments.size() > 3) throw std::runtime_error("mount status accepts one path"); + return PrintMountStatus(options, + options.arguments.size() == 3 ? options.arguments[2] : std::string()); + } + if (options.arguments.size() != 2) throw std::runtime_error("mount needs MOUNT_POINT"); + return MountWorkspace(options, options.arguments[1]); + } + if (command == "unmount") { + if (options.arguments.size() != 2) throw std::runtime_error("unmount needs MOUNT_POINT"); + return RunProcess("/sbin/umount", {options.arguments[1]}); + } + if (command == "doctor") { + if (options.arguments.size() != 1) throw std::runtime_error("doctor accepts only global options"); + return RunDoctor(options); + } + + Session session(options); + vexfs_mount_error error{}; + if (command == "init" || command == "setup") { + if (options.arguments.size() != 1) throw std::runtime_error(command + " accepts only global options"); + std::cout << options.database << " [" << options.workspace << "]\n"; + if (!options.mount_point.empty()) return MountWorkspace(options, options.mount_point); + } else if (command == "mkdir") { + if (options.arguments.size() < 2) throw std::runtime_error("mkdir needs PATH"); + for (size_t i = 1; i < options.arguments.size(); ++i) + Check(vexfs_mount_mkdir(session.get(), options.arguments[i].c_str(), &error), error); + } else if (command == "write") { + if (options.arguments.size() < 2 || options.arguments.size() > 3) + throw std::runtime_error("write needs PATH and optional LOCAL_FILE"); + const auto content = ReadInput(options.arguments, 2); + int64_t version = 0; + Check(vexfs_mount_write_file(session.get(), options.arguments[1].c_str(), + content.data(), content.size(), &version, &error), error); + std::cout << version << '\n'; + } else if (command == "cat") { + if (options.arguments.size() != 2) throw std::runtime_error("cat needs PATH"); + vexfs_mount_bytes content{}; + Check(vexfs_mount_read_file(session.get(), options.arguments[1].c_str(), &content, &error), error); + if (content.data != nullptr) std::cout.write(static_cast(content.data), content.size); + vexfs_mount_free(content.data); + } else if (command == "ls") { + const std::string path = options.arguments.size() >= 2 ? options.arguments[1] : "/"; + vexfs_mount_bytes json{}; + Check(vexfs_mount_list(session.get(), path.c_str(), &json, &error), error); + const std::string value = BytesToString(&json); + if (options.json) std::cout << value << '\n'; else PrintNames(value); + } else if (command == "stat") { + if (options.arguments.size() != 2) throw std::runtime_error("stat needs PATH"); + vexfs_mount_bytes json{}; + Check(vexfs_mount_stat(session.get(), options.arguments[1].c_str(), &json, &error), error); + std::cout << BytesToString(&json) << '\n'; + } else if (command == "history") { + const ParsedCommand parsed = ParseCommand(options.arguments, {"--limit", "--before"}, {}); + if (parsed.positional.size() != 1) throw std::runtime_error("history needs PATH"); + const std::string limit_value = parsed.Value("--limit", false); + const std::string before_value = parsed.Value("--before", false); + const int64_t limit = limit_value.empty() ? 100 : PositiveInteger(limit_value, "limit"); + const int64_t before = before_value.empty() ? 0 : NonnegativeInteger(before_value, "before"); + if (limit > 1000) + throw CliError(VEXFS_MOUNT_INVALID_ARGUMENT, "limit must be at most 1000"); + vexfs_mount_bytes json{}; + Check(vexfs_mount_history_page(session.get(), parsed.positional[0].c_str(), + static_cast(limit), before, &json, &error), error); + const std::string value = BytesToString(&json); + if (options.json) std::cout << value << '\n'; else PrintHistory(value); + } else if (command == "show") { + const ParsedCommand parsed = ParseCommand(options.arguments, {"--version"}, {}); + if (parsed.positional.size() != 1) + throw std::runtime_error("show needs PATH and --version N"); + const int64_t version = PositiveInteger(parsed.Value("--version"), "version"); + const std::string content = ReadHistoricalVersion(session, parsed.positional[0], version); + if (!content.empty()) std::cout.write(content.data(), static_cast(content.size())); + } else if (command == "diff") { + const ParsedCommand parsed = ParseCommand(options.arguments, {"--from", "--to"}, {}); + if (parsed.positional.size() != 1) + throw std::runtime_error("diff needs PATH, --from N and optional --to N"); + const std::string &path = parsed.positional[0]; + const int64_t from_version = PositiveInteger(parsed.Value("--from"), "from version"); + const std::string to_value = parsed.Value("--to", false); + const int64_t to_version = to_value.empty() ? CurrentFileVersion(session, path) : + PositiveInteger(to_value, "to version"); + return RunVersionDiff(session, path, from_version, to_version, options.json) ? 1 : 0; + } else if (command == "restore") { + const ParsedCommand parsed = ParseCommand(options.arguments, {"--version"}, {"--dry-run"}); + if (parsed.positional.size() != 1) + throw std::runtime_error("restore needs PATH and --version N"); + const std::string &path = parsed.positional[0]; + const int64_t target_version = PositiveInteger(parsed.Value("--version"), "version"); + const int64_t current_version = CurrentFileVersion(session, path); + if (parsed.Flag("--dry-run")) { + RunVersionDiff(session, path, current_version, target_version, options.json); + } else { + int64_t new_version = 0; + Check(vexfs_mount_restore_version(session.get(), path.c_str(), target_version, + current_version, &new_version, &error), error); + if (options.json) { + std::cout << "{\"path\":\"" << JsonEscape(path) << "\",\"restored_from\":" + << target_version << ",\"previous_version\":" << current_version + << ",\"version\":" << new_version << "}\n"; + } else { + std::cout << new_version << '\n'; + } + } + } else if (command == "mv") { + if (options.arguments.size() != 3) throw std::runtime_error("mv needs SOURCE DESTINATION"); + Check(vexfs_mount_move(session.get(), options.arguments[1].c_str(), + options.arguments[2].c_str(), &error), error); + } else if (command == "rm") { + bool recursive = false; + size_t path_index = 1; + if (options.arguments.size() > 1 && (options.arguments[1] == "-r" || options.arguments[1] == "-R")) { + recursive = true; + path_index = 2; + } + if (options.arguments.size() != path_index + 1) throw std::runtime_error("rm needs PATH"); + Check(vexfs_mount_remove(session.get(), options.arguments[path_index].c_str(), + recursive ? 1 : 0, &error), error); + } else { + throw std::runtime_error("unknown command: " + command); + } + return 0; +} + +bool WantsJson(int argc, char **argv) { + for (int index = 1; index < argc; ++index) { + if (std::strcmp(argv[index], "--json") == 0) return true; + } + return false; +} + +void PrintError(const std::string &code, const std::string &message, int exit_code, + bool json) { + if (json) { + std::cerr << "{\"error\":{\"code\":\"" << JsonEscape(code) + << "\",\"message\":\"" << JsonEscape(message) + << "\",\"exit_code\":" << exit_code << "}}\n"; + } else { + std::cerr << "vexfs: " << message << '\n'; + } +} + +} // namespace + +int main(int argc, char **argv) { + try { + return Run(ParseOptions(argc, argv)); + } catch (const CliError &error) { + PrintError(error.code, error.what(), error.exit_code, WantsJson(argc, argv)); + return error.exit_code; + } catch (const std::exception &error) { + PrintError("VEXFS_ERROR", error.what(), 1, WantsJson(argc, argv)); + return 1; + } +} diff --git a/agent_files/macos/VexFS.xcodeproj/project.pbxproj b/agent_files/macos/VexFS.xcodeproj/project.pbxproj new file mode 100644 index 0000000000..21adaca682 --- /dev/null +++ b/agent_files/macos/VexFS.xcodeproj/project.pbxproj @@ -0,0 +1,562 @@ +// !$*UTF8*$! +{ + archiveVersion = 1; + classes = { + }; + objectVersion = 46; + objects = { + +/* Begin PBXBuildFile section */ + 0BC68A3B319449C6CEE91C8D /* VexFSAppEx.appex in Embed ExtensionKit Extensions */ = {isa = PBXBuildFile; fileRef = 6CCDB74FA186F6FCD7A300FB /* VexFSAppEx.appex */; settings = {ATTRIBUTES = (RemoveHeadersOnCopy, ); }; }; + 23DDC1E282C33F2F633E7872 /* VexFSBackend.swift in Sources */ = {isa = PBXBuildFile; fileRef = F36BCE283AA3A5947E3A75BC /* VexFSBackend.swift */; }; + 3B11C6F78EA62D945AF6E165 /* FSKit.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 24ABADDA7C830A7635A906E5 /* FSKit.framework */; }; + A6C51020C5D7076193EA157A /* VexFSVolume+Operations.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0157B8EFB8B783DBB2A4B15D /* VexFSVolume+Operations.swift */; }; + A89A6A7839DECBD962C9FA02 /* Cocoa.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = D48085DD8AFB52024CD9512B /* Cocoa.framework */; }; + B383BA27662F9E6DD90C2C81 /* VexFSFileSystem.swift in Sources */ = {isa = PBXBuildFile; fileRef = 612EC9965B79CDECF771CFE7 /* VexFSFileSystem.swift */; }; + B6B84C145BFAEA0C42E15916 /* ContentView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 339D10E1B23713660842B8ED /* ContentView.swift */; }; + BD1E4AE3BF6880BE89877841 /* Cocoa.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = D48085DD8AFB52024CD9512B /* Cocoa.framework */; }; + BF65C046A873125AC5174F58 /* VexFSVolume.swift in Sources */ = {isa = PBXBuildFile; fileRef = 12A53F9C5F028FC3DD4BE3E3 /* VexFSVolume.swift */; }; + C9D129F35E11C5025B10D5F6 /* VexFSItem.swift in Sources */ = {isa = PBXBuildFile; fileRef = 79162BBEE0BBD0A628DBB6BD /* VexFSItem.swift */; }; + CCA1446D1EA1AA3AD47E37D0 /* VexFSAppEx.swift in Sources */ = {isa = PBXBuildFile; fileRef = 94204BA19A4C73FA79C190C9 /* VexFSAppEx.swift */; }; + D180822993DA1907CE7B6A10 /* VexFSApp.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0D63C4F4452011ECF470D69E /* VexFSApp.swift */; }; +/* End PBXBuildFile section */ + +/* Begin PBXContainerItemProxy section */ + 83549E04744B8CA7BF0D59B9 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = AC4A42524ED5CBEE19926577 /* Project object */; + proxyType = 1; + remoteGlobalIDString = 1A5D042A492F7D6E68921B31; + remoteInfo = VexFSAppEx; + }; +/* End PBXContainerItemProxy section */ + +/* Begin PBXCopyFilesBuildPhase section */ + C9410A46AA6747711A591D0A /* Embed ExtensionKit Extensions */ = { + isa = PBXCopyFilesBuildPhase; + buildActionMask = 2147483647; + dstPath = "$(EXTENSIONS_FOLDER_PATH)"; + dstSubfolderSpec = 16; + files = ( + 0BC68A3B319449C6CEE91C8D /* VexFSAppEx.appex in Embed ExtensionKit Extensions */, + ); + name = "Embed ExtensionKit Extensions"; + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXCopyFilesBuildPhase section */ + +/* Begin PBXFileReference section */ + 0157B8EFB8B783DBB2A4B15D /* VexFSVolume+Operations.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = "VexFSVolume+Operations.swift"; sourceTree = ""; }; + 0D63C4F4452011ECF470D69E /* VexFSApp.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = VexFSApp.swift; sourceTree = ""; }; + 12A53F9C5F028FC3DD4BE3E3 /* VexFSVolume.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = VexFSVolume.swift; sourceTree = ""; }; + 24ABADDA7C830A7635A906E5 /* FSKit.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = FSKit.framework; path = System/Library/Frameworks/FSKit.framework; sourceTree = SDKROOT; }; + 339D10E1B23713660842B8ED /* ContentView.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = ContentView.swift; sourceTree = ""; }; + 4A51E7C4A79392F1B486CBDC /* VexFSApp.entitlements */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.plist.entitlements; path = VexFSApp.entitlements; sourceTree = ""; }; + 5B9E7D35E13C5EDA8EB0EC02 /* VexFS-Bridging-Header.h */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.c.h; path = "VexFS-Bridging-Header.h"; sourceTree = ""; }; + 612EC9965B79CDECF771CFE7 /* VexFSFileSystem.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = VexFSFileSystem.swift; sourceTree = ""; }; + 6CCDB74FA186F6FCD7A300FB /* VexFSAppEx.appex */ = {isa = PBXFileReference; explicitFileType = "wrapper.extensionkit-extension"; includeInIndex = 0; path = VexFSAppEx.appex; sourceTree = BUILT_PRODUCTS_DIR; }; + 79162BBEE0BBD0A628DBB6BD /* VexFSItem.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = VexFSItem.swift; sourceTree = ""; }; + 8B79A91CD5F1353E7C4DBC40 /* VexFSApp.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = VexFSApp.app; sourceTree = BUILT_PRODUCTS_DIR; }; + 94204BA19A4C73FA79C190C9 /* VexFSAppEx.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = VexFSAppEx.swift; sourceTree = ""; }; + C9358B6392411F7B776722E1 /* VexFSAppEx.entitlements */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.plist.entitlements; path = VexFSAppEx.entitlements; sourceTree = ""; }; + D48085DD8AFB52024CD9512B /* Cocoa.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = Cocoa.framework; path = Platforms/MacOSX.platform/Developer/SDKs/MacOSX15.0.sdk/System/Library/Frameworks/Cocoa.framework; sourceTree = DEVELOPER_DIR; }; + E1821EECDA039CD17708C8F8 /* Info.plist */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; + F36BCE283AA3A5947E3A75BC /* VexFSBackend.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = VexFSBackend.swift; sourceTree = ""; }; +/* End PBXFileReference section */ + +/* Begin PBXFrameworksBuildPhase section */ + 537E5BFB297AFC43B476E8BA /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + A89A6A7839DECBD962C9FA02 /* Cocoa.framework in Frameworks */, + 3B11C6F78EA62D945AF6E165 /* FSKit.framework in Frameworks */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; + F9E2F58F5023DE4798CEA89E /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + BD1E4AE3BF6880BE89877841 /* Cocoa.framework in Frameworks */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXFrameworksBuildPhase section */ + +/* Begin PBXGroup section */ + 1398F869C8324455467B1337 /* VexFSAppEx */ = { + isa = PBXGroup; + children = ( + 94204BA19A4C73FA79C190C9 /* VexFSAppEx.swift */, + F36BCE283AA3A5947E3A75BC /* VexFSBackend.swift */, + 612EC9965B79CDECF771CFE7 /* VexFSFileSystem.swift */, + 79162BBEE0BBD0A628DBB6BD /* VexFSItem.swift */, + 12A53F9C5F028FC3DD4BE3E3 /* VexFSVolume.swift */, + 0157B8EFB8B783DBB2A4B15D /* VexFSVolume+Operations.swift */, + 5B9E7D35E13C5EDA8EB0EC02 /* VexFS-Bridging-Header.h */, + C9358B6392411F7B776722E1 /* VexFSAppEx.entitlements */, + E1821EECDA039CD17708C8F8 /* Info.plist */, + ); + name = VexFSAppEx; + path = VexFSAppEx; + sourceTree = ""; + }; + 43A182FDE5C8742E9D66F9DE /* OS X */ = { + isa = PBXGroup; + children = ( + D48085DD8AFB52024CD9512B /* Cocoa.framework */, + ); + name = "OS X"; + sourceTree = ""; + }; + 5C3B6513C200006C7D3ED342 /* Products */ = { + isa = PBXGroup; + children = ( + 8B79A91CD5F1353E7C4DBC40 /* VexFSApp.app */, + 6CCDB74FA186F6FCD7A300FB /* VexFSAppEx.appex */, + ); + name = Products; + sourceTree = ""; + }; + 667505ECC9BC5B1D8210D9F4 /* Frameworks */ = { + isa = PBXGroup; + children = ( + 43A182FDE5C8742E9D66F9DE /* OS X */, + 24ABADDA7C830A7635A906E5 /* FSKit.framework */, + ); + name = Frameworks; + sourceTree = ""; + }; + 975A9C6292155E57E5284D04 = { + isa = PBXGroup; + children = ( + 5C3B6513C200006C7D3ED342 /* Products */, + 667505ECC9BC5B1D8210D9F4 /* Frameworks */, + D6D01A997E4D4C079763432B /* VexFSApp */, + 1398F869C8324455467B1337 /* VexFSAppEx */, + ); + sourceTree = ""; + }; + D6D01A997E4D4C079763432B /* VexFSApp */ = { + isa = PBXGroup; + children = ( + 0D63C4F4452011ECF470D69E /* VexFSApp.swift */, + 339D10E1B23713660842B8ED /* ContentView.swift */, + 4A51E7C4A79392F1B486CBDC /* VexFSApp.entitlements */, + ); + name = VexFSApp; + path = VexFSApp; + sourceTree = ""; + }; +/* End PBXGroup section */ + +/* Begin PBXNativeTarget section */ + 1A5D042A492F7D6E68921B31 /* VexFSAppEx */ = { + isa = PBXNativeTarget; + buildConfigurationList = 81F32C156A3B0AF8EA9C65C0 /* Build configuration list for PBXNativeTarget "VexFSAppEx" */; + buildPhases = ( + C4F2425D1C43106F2CDA1C6C /* Build VexFS SQLite Core */, + FE25A55875F5753214F89004 /* Sources */, + 537E5BFB297AFC43B476E8BA /* Frameworks */, + 3F8ACBA139F4993B190B48B6 /* Resources */, + ); + buildRules = ( + ); + dependencies = ( + ); + name = VexFSAppEx; + productName = VexFSAppEx; + productReference = 6CCDB74FA186F6FCD7A300FB /* VexFSAppEx.appex */; + productType = "com.apple.product-type.extensionkit-extension"; + }; + 6FF435D5C1BC036A4F7B4BA6 /* VexFSApp */ = { + isa = PBXNativeTarget; + buildConfigurationList = C235E4ED94D989172CFFDA17 /* Build configuration list for PBXNativeTarget "VexFSApp" */; + buildPhases = ( + 046F460A8B6C8F5B35D1F22D /* Sources */, + F9E2F58F5023DE4798CEA89E /* Frameworks */, + 99F228151C9C8F18C28D2EBB /* Resources */, + C9410A46AA6747711A591D0A /* Embed ExtensionKit Extensions */, + ); + buildRules = ( + ); + dependencies = ( + 479D729D1D8E7D50EEF80968 /* PBXTargetDependency */, + ); + name = VexFSApp; + productName = VexFSApp; + productReference = 8B79A91CD5F1353E7C4DBC40 /* VexFSApp.app */; + productType = "com.apple.product-type.application"; + }; +/* End PBXNativeTarget section */ + +/* Begin PBXProject section */ + AC4A42524ED5CBEE19926577 /* Project object */ = { + isa = PBXProject; + attributes = { + LastSwiftUpdateCheck = 2630; + LastUpgradeCheck = 2630; + }; + buildConfigurationList = 3DCF428E65687F1FDC0726AD /* Build configuration list for PBXProject "VexFS" */; + compatibilityVersion = "Xcode 3.2"; + developmentRegion = en; + hasScannedForEncodings = 0; + knownRegions = ( + en, + Base, + ); + mainGroup = 975A9C6292155E57E5284D04; + minimizedProjectReferenceProxies = 0; + preferredProjectObjectVersion = 77; + productRefGroup = 5C3B6513C200006C7D3ED342 /* Products */; + projectDirPath = ""; + projectRoot = ""; + targets = ( + 6FF435D5C1BC036A4F7B4BA6 /* VexFSApp */, + 1A5D042A492F7D6E68921B31 /* VexFSAppEx */, + ); + }; +/* End PBXProject section */ + +/* Begin PBXResourcesBuildPhase section */ + 3F8ACBA139F4993B190B48B6 /* Resources */ = { + isa = PBXResourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; + 99F228151C9C8F18C28D2EBB /* Resources */ = { + isa = PBXResourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXResourcesBuildPhase section */ + +/* Begin PBXShellScriptBuildPhase section */ + C4F2425D1C43106F2CDA1C6C /* Build VexFS SQLite Core */ = { + isa = PBXShellScriptBuildPhase; + alwaysOutOfDate = 1; + buildActionMask = 2147483647; + files = ( + ); + inputFileListPaths = ( + ); + inputPaths = ( + ); + name = "Build VexFS SQLite Core"; + outputFileListPaths = ( + ); + outputPaths = ( + ); + runOnlyForDeploymentPostprocessing = 0; + shellPath = /bin/sh; + shellScript = "set -e\nCMAKE_BIN=\"$(command -v cmake || true)\"\n[ -x \"$CMAKE_BIN\" ] || CMAKE_BIN=/opt/homebrew/bin/cmake\n[ -x \"$CMAKE_BIN\" ] || CMAKE_BIN=/usr/local/bin/cmake\n[ -x \"$CMAKE_BIN\" ] || { echo \"cmake is required\" >&2; exit 1; }\nBUILD_DIR=\"$SRCROOT/../../vexdb_sqlite/build-fskit\"\n\"$CMAKE_BIN\" -S \"$SRCROOT/../../vexdb_sqlite\" -B \"$BUILD_DIR\" \\\n -DCMAKE_BUILD_TYPE=\"$CONFIGURATION\" -DVEXDB_SQLITE_BUILD_TESTS=OFF\n\"$CMAKE_BIN\" --build \"$BUILD_DIR\" --target vexfs_mount_contract -j \"$(sysctl -n hw.ncpu)\"\n"; + }; +/* End PBXShellScriptBuildPhase section */ + +/* Begin PBXSourcesBuildPhase section */ + 046F460A8B6C8F5B35D1F22D /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + D180822993DA1907CE7B6A10 /* VexFSApp.swift in Sources */, + B6B84C145BFAEA0C42E15916 /* ContentView.swift in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; + FE25A55875F5753214F89004 /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + CCA1446D1EA1AA3AD47E37D0 /* VexFSAppEx.swift in Sources */, + 23DDC1E282C33F2F633E7872 /* VexFSBackend.swift in Sources */, + B383BA27662F9E6DD90C2C81 /* VexFSFileSystem.swift in Sources */, + C9D129F35E11C5025B10D5F6 /* VexFSItem.swift in Sources */, + BF65C046A873125AC5174F58 /* VexFSVolume.swift in Sources */, + A6C51020C5D7076193EA157A /* VexFSVolume+Operations.swift in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXSourcesBuildPhase section */ + +/* Begin PBXTargetDependency section */ + 479D729D1D8E7D50EEF80968 /* PBXTargetDependency */ = { + isa = PBXTargetDependency; + name = VexFSAppEx; + target = 1A5D042A492F7D6E68921B31 /* VexFSAppEx */; + targetProxy = 83549E04744B8CA7BF0D59B9 /* PBXContainerItemProxy */; + }; +/* End PBXTargetDependency section */ + +/* Begin XCBuildConfiguration section */ + 09491CF975FB59BD73BDAA62 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + CLANG_ANALYZER_NONNULL = YES; + CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++14"; + CLANG_CXX_LIBRARY = "libc++"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_ENABLE_OBJC_WEAK = YES; + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_COMMA = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_DOCUMENTATION_COMMENTS = YES; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; + CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES; + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; + CLANG_WARN_STRICT_PROTOTYPES = YES; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; + CLANG_WARN_UNREACHABLE_CODE = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + COPY_PHASE_STRIP = NO; + DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; + ENABLE_NS_ASSERTIONS = NO; + ENABLE_STRICT_OBJC_MSGSEND = YES; + GCC_C_LANGUAGE_STANDARD = gnu11; + GCC_NO_COMMON_BLOCKS = YES; + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + MACOSX_DEPLOYMENT_TARGET = 26.0; + MTL_ENABLE_DEBUG_INFO = NO; + MTL_FAST_MATH = YES; + PRODUCT_NAME = "$(TARGET_NAME)"; + SDKROOT = macosx; + SWIFT_COMPILATION_MODE = wholemodule; + SWIFT_OPTIMIZATION_LEVEL = "-O"; + SWIFT_VERSION = 5.0; + }; + name = Release; + }; + 31A0A5719E39E7BB401D99FE /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; + CODE_SIGN_ENTITLEMENTS = VexFSApp/VexFSApp.entitlements; + CODE_SIGN_STYLE = Automatic; + COMBINE_HIDPI_IMAGES = YES; + CURRENT_PROJECT_VERSION = 1; + DEVELOPMENT_TEAM = ""; + ENABLE_APP_SANDBOX = YES; + GENERATE_INFOPLIST_FILE = YES; + INFOPLIST_KEY_CFBundleDisplayName = VexFS; + LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/../Frameworks"; + MACOSX_DEPLOYMENT_TARGET = 26.0; + MARKETING_VERSION = 0.1.0; + PRODUCT_BUNDLE_IDENTIFIER = io.vexdb.vexfs; + PRODUCT_NAME = VexFS; + SDKROOT = macosx; + }; + name = Debug; + }; + A6AECEA8884F34F863EE1397 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; + CODE_SIGN_ENTITLEMENTS = VexFSApp/VexFSApp.entitlements; + CODE_SIGN_STYLE = Automatic; + COMBINE_HIDPI_IMAGES = YES; + CURRENT_PROJECT_VERSION = 1; + DEVELOPMENT_TEAM = ""; + ENABLE_APP_SANDBOX = YES; + GENERATE_INFOPLIST_FILE = YES; + INFOPLIST_KEY_CFBundleDisplayName = VexFS; + LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/../Frameworks"; + MACOSX_DEPLOYMENT_TARGET = 26.0; + MARKETING_VERSION = 0.1.0; + PRODUCT_BUNDLE_IDENTIFIER = io.vexdb.vexfs; + PRODUCT_NAME = VexFS; + SDKROOT = macosx; + }; + name = Release; + }; + B5F77B8B9328312919A0E4EA /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + APPLICATION_EXTENSION_API_ONLY = YES; + CODE_SIGN_ENTITLEMENTS = VexFSAppEx/VexFSAppEx.entitlements; + CODE_SIGN_STYLE = Automatic; + CURRENT_PROJECT_VERSION = 1; + DEVELOPMENT_TEAM = ""; + ENABLE_APP_SANDBOX = YES; + ENABLE_USER_SCRIPT_SANDBOXING = NO; + GENERATE_INFOPLIST_FILE = YES; + HEADER_SEARCH_PATHS = ( + "$(inherited)", + "\"$(SRCROOT)/../mount/common/include\"", + ); + INFOPLIST_FILE = VexFSAppEx/Info.plist; + INFOPLIST_KEY_CFBundleDisplayName = "VexFS file system"; + LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/../Frameworks @executable_path/../../../../Frameworks"; + MACOSX_DEPLOYMENT_TARGET = 26.0; + MARKETING_VERSION = 0.1.0; + OTHER_LDFLAGS = ( + "$(inherited)", + "\"$(SRCROOT)/../../vexdb_sqlite/build-fskit/libvexfs_mount_contract.a\"", + "\"$(SRCROOT)/../../vexdb_sqlite/build-fskit/libvexdb_lite_static.a\"", + "-lsqlite3", + "-lc++", + ); + PRODUCT_BUNDLE_IDENTIFIER = io.vexdb.vexfs.extension; + PRODUCT_NAME = VexFSAppEx; + SDKROOT = macosx; + SKIP_INSTALL = YES; + SWIFT_OBJC_BRIDGING_HEADER = "VexFSAppEx/VexFS-Bridging-Header.h"; + }; + name = Release; + }; + B6EE28519513696CE1930079 /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + CLANG_ANALYZER_NONNULL = YES; + CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++14"; + CLANG_CXX_LIBRARY = "libc++"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_ENABLE_OBJC_WEAK = YES; + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_COMMA = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_DOCUMENTATION_COMMENTS = YES; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; + CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES; + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; + CLANG_WARN_STRICT_PROTOTYPES = YES; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; + CLANG_WARN_UNREACHABLE_CODE = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + COPY_PHASE_STRIP = NO; + DEBUG_INFORMATION_FORMAT = dwarf; + ENABLE_STRICT_OBJC_MSGSEND = YES; + ENABLE_TESTABILITY = YES; + GCC_C_LANGUAGE_STANDARD = gnu11; + GCC_DYNAMIC_NO_PIC = NO; + GCC_NO_COMMON_BLOCKS = YES; + GCC_OPTIMIZATION_LEVEL = 0; + GCC_PREPROCESSOR_DEFINITIONS = ( + "DEBUG=1", + "$(inherited)", + ); + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + MACOSX_DEPLOYMENT_TARGET = 26.0; + MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; + MTL_FAST_MATH = YES; + ONLY_ACTIVE_ARCH = YES; + PRODUCT_NAME = "$(TARGET_NAME)"; + SDKROOT = macosx; + SWIFT_ACTIVE_COMPILATION_CONDITIONS = DEBUG; + SWIFT_OPTIMIZATION_LEVEL = "-Onone"; + SWIFT_VERSION = 5.0; + }; + name = Debug; + }; + E8620A8E6E1735705B0CE3C8 /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + APPLICATION_EXTENSION_API_ONLY = YES; + CODE_SIGN_ENTITLEMENTS = VexFSAppEx/VexFSAppEx.entitlements; + CODE_SIGN_STYLE = Automatic; + CURRENT_PROJECT_VERSION = 1; + DEVELOPMENT_TEAM = ""; + ENABLE_APP_SANDBOX = YES; + ENABLE_USER_SCRIPT_SANDBOXING = NO; + GENERATE_INFOPLIST_FILE = YES; + HEADER_SEARCH_PATHS = ( + "$(inherited)", + "\"$(SRCROOT)/../mount/common/include\"", + ); + INFOPLIST_FILE = VexFSAppEx/Info.plist; + INFOPLIST_KEY_CFBundleDisplayName = "VexFS file system"; + LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/../Frameworks @executable_path/../../../../Frameworks"; + MACOSX_DEPLOYMENT_TARGET = 26.0; + MARKETING_VERSION = 0.1.0; + OTHER_LDFLAGS = ( + "$(inherited)", + "\"$(SRCROOT)/../../vexdb_sqlite/build-fskit/libvexfs_mount_contract.a\"", + "\"$(SRCROOT)/../../vexdb_sqlite/build-fskit/libvexdb_lite_static.a\"", + "-lsqlite3", + "-lc++", + ); + PRODUCT_BUNDLE_IDENTIFIER = io.vexdb.vexfs.extension; + PRODUCT_NAME = VexFSAppEx; + SDKROOT = macosx; + SKIP_INSTALL = YES; + SWIFT_OBJC_BRIDGING_HEADER = "VexFSAppEx/VexFS-Bridging-Header.h"; + }; + name = Debug; + }; +/* End XCBuildConfiguration section */ + +/* Begin XCConfigurationList section */ + 3DCF428E65687F1FDC0726AD /* Build configuration list for PBXProject "VexFS" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + B6EE28519513696CE1930079 /* Debug */, + 09491CF975FB59BD73BDAA62 /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; + 81F32C156A3B0AF8EA9C65C0 /* Build configuration list for PBXNativeTarget "VexFSAppEx" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + B5F77B8B9328312919A0E4EA /* Release */, + E8620A8E6E1735705B0CE3C8 /* Debug */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; + C235E4ED94D989172CFFDA17 /* Build configuration list for PBXNativeTarget "VexFSApp" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + A6AECEA8884F34F863EE1397 /* Release */, + 31A0A5719E39E7BB401D99FE /* Debug */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; +/* End XCConfigurationList section */ + }; + rootObject = AC4A42524ED5CBEE19926577 /* Project object */; +} diff --git a/agent_files/macos/VexFSApp/ContentView.swift b/agent_files/macos/VexFSApp/ContentView.swift new file mode 100644 index 0000000000..2b80016c04 --- /dev/null +++ b/agent_files/macos/VexFSApp/ContentView.swift @@ -0,0 +1,32 @@ +import SwiftUI + +struct ContentView: View { + private var databasePath: String { + FileManager.default.homeDirectoryForCurrentUser + .appendingPathComponent("Library/Application Support/VexFS/vexfs.sqlite3").path + } + + var body: some View { + VStack(alignment: .leading, spacing: 18) { + Label("VexFS", systemImage: "externaldrive.connected.to.line.below") + .font(.system(size: 32, weight: .semibold)) + Text("由 SQLite 管理、可直接通过 Bash 使用的文件系统。") + .font(.title3) + GroupBox("启用步骤") { + VStack(alignment: .leading, spacing: 10) { + Text("1. 在 系统设置 → 通用 → 登录项与扩展 → 文件系统扩展 中启用 VexFS。") + Text("2. 运行 vexfs setup 初始化数据库，再用 vexfs doctor 检查状态。") + Text("3. 运行 vexfs mount ~/VexFS 挂载，然后直接使用 ls、cat、grep、cp、mv 等命令。") + } + .frame(maxWidth: .infinity, alignment: .leading) + .padding(4) + } + Text("默认数据库：\(databasePath)") + .font(.system(.footnote, design: .monospaced)) + .textSelection(.enabled) + .foregroundStyle(.secondary) + Spacer() + } + .padding(28) + } +} diff --git a/agent_files/macos/VexFSApp/VexFSApp.entitlements b/agent_files/macos/VexFSApp/VexFSApp.entitlements new file mode 100644 index 0000000000..13cb114cf8 --- /dev/null +++ b/agent_files/macos/VexFSApp/VexFSApp.entitlements @@ -0,0 +1,8 @@ + + + + + com.apple.security.app-sandbox + + + diff --git a/agent_files/macos/VexFSApp/VexFSApp.swift b/agent_files/macos/VexFSApp/VexFSApp.swift new file mode 100644 index 0000000000..7f867f54b5 --- /dev/null +++ b/agent_files/macos/VexFSApp/VexFSApp.swift @@ -0,0 +1,11 @@ +import SwiftUI + +@main +struct VexFSApp: App { + var body: some Scene { + WindowGroup { + ContentView() + .frame(minWidth: 620, minHeight: 380) + } + } +} diff --git a/agent_files/macos/VexFSAppEx/Info.plist b/agent_files/macos/VexFSAppEx/Info.plist new file mode 100644 index 0000000000..df80a37d5c --- /dev/null +++ b/agent_files/macos/VexFSAppEx/Info.plist @@ -0,0 +1,46 @@ + + + + + EXAppExtensionAttributes + + EXExtensionPointIdentifier + com.apple.fskit.fsmodule + FSActivateOptionSyntax + + shortOptions + u:g:m:o: + + FSCheckOptionSyntax + + shortOptions + nqy + + FSFormatOptionSyntax + + shortOptions + v + + FSMediaTypes + + FSPersonalities + + VexFS + + FSfileObjectsAreCaseSensitive + + FSName + VexFS + + + FSRequiresSecurityScopedPathURLResources + + FSShortName + vexfs + FSSupportsBlockResources + + FSSupportsPathURLs + + + + diff --git a/agent_files/macos/VexFSAppEx/VexFS-Bridging-Header.h b/agent_files/macos/VexFSAppEx/VexFS-Bridging-Header.h new file mode 100644 index 0000000000..1e1a311632 --- /dev/null +++ b/agent_files/macos/VexFSAppEx/VexFS-Bridging-Header.h @@ -0,0 +1,6 @@ +#ifndef VEXFS_BRIDGING_HEADER_H +#define VEXFS_BRIDGING_HEADER_H + +#include "vexfs_mount_contract.h" + +#endif diff --git a/agent_files/macos/VexFSAppEx/VexFSAppEx.entitlements b/agent_files/macos/VexFSAppEx/VexFSAppEx.entitlements new file mode 100644 index 0000000000..40a67e20d7 --- /dev/null +++ b/agent_files/macos/VexFSAppEx/VexFSAppEx.entitlements @@ -0,0 +1,10 @@ + + + + + com.apple.security.app-sandbox + + com.apple.developer.fskit.fsmodule + + + diff --git a/agent_files/macos/VexFSAppEx/VexFSAppEx.swift b/agent_files/macos/VexFSAppEx/VexFSAppEx.swift new file mode 100644 index 0000000000..14c6c86836 --- /dev/null +++ b/agent_files/macos/VexFSAppEx/VexFSAppEx.swift @@ -0,0 +1,12 @@ +import Foundation +import FSKit + +@main +@available(macOS 26.0, *) +struct VexFSAppEx: UnaryFileSystemExtension { + typealias FileSystem = FSUnaryFileSystem & FSUnaryFileSystemOperations + + var fileSystem: FSUnaryFileSystem & FSUnaryFileSystemOperations { + VexFSFileSystem() + } +} diff --git a/agent_files/macos/VexFSAppEx/VexFSBackend.swift b/agent_files/macos/VexFSAppEx/VexFSBackend.swift new file mode 100644 index 0000000000..6b007583d6 --- /dev/null +++ b/agent_files/macos/VexFSAppEx/VexFSBackend.swift @@ -0,0 +1,279 @@ +import Foundation + +struct VexFSDescriptor: Codable { + let version: Int + let database_file: String + let workspace: String +} + +struct VexFSStatRecord: Codable { + let path: String + let inode: UInt64 + let kind: String + let mode: UInt32 + let size: UInt64 + let version: UInt64 + let created_at: Int64 + let updated_at: Int64 +} + +struct VexFSDirectoryEntry: Codable { + let name: String + let inode: UInt64 + let kind: String + let size: UInt64 + let version: UInt64 + let mode: UInt32 + let created_at: Int64 + let updated_at: Int64 +} + +final class VexFSBackend { + private var session: OpaquePointer? + + init(databasePath: String, workspace: String) throws { + var config = vexfs_mount_config() + config.abi_version = UInt32(VEXFS_MOUNT_ABI_VERSION) + config.busy_timeout_ms = 5_000 + config.flags = UInt32(VEXFS_MOUNT_EXCLUSIVE_GATEWAY) + var opened: OpaquePointer? + var error = vexfs_mount_error() + let status = databasePath.withCString { databasePointer in + workspace.withCString { workspacePointer in + config.database_path = databasePointer + config.workspace = workspacePointer + return vexfs_mount_session_open(&config, &opened, &error) + } + } + try Self.check(status, error: &error) + session = opened + } + + deinit { + vexfs_mount_session_close(session) + } + + func stat(path: String) throws -> VexFSStatRecord { + var output = vexfs_mount_bytes() + var error = vexfs_mount_error() + let status = path.withCString { vexfs_mount_stat(session, $0, &output, &error) } + try Self.check(status, error: &error) + return try JSONDecoder().decode(VexFSStatRecord.self, from: Self.take(&output)) + } + + func path(inode: UInt64) throws -> String { + var output = vexfs_mount_bytes() + var error = vexfs_mount_error() + let status = vexfs_mount_path_for_inode(session, Int64(bitPattern: inode), &output, &error) + try Self.check(status, error: &error) + guard let value = String(data: Self.take(&output), encoding: .utf8) else { + throw POSIXError(.EIO) + } + return value + } + + func list(path: String) throws -> [VexFSDirectoryEntry] { + var output = vexfs_mount_bytes() + var error = vexfs_mount_error() + let status = path.withCString { vexfs_mount_list(session, $0, &output, &error) } + try Self.check(status, error: &error) + return try JSONDecoder().decode([VexFSDirectoryEntry].self, from: Self.take(&output)) + } + + func mkdir(path: String) throws { + var error = vexfs_mount_error() + let status = path.withCString { vexfs_mount_mkdir(session, $0, &error) } + try Self.check(status, error: &error) + } + + @discardableResult + func writeFile(path: String, data: Data) throws -> Int64 { + var version: Int64 = 0 + var error = vexfs_mount_error() + let status = path.withCString { pathPointer in + data.withUnsafeBytes { bytes in + vexfs_mount_write_file(session, pathPointer, bytes.baseAddress, + UInt64(bytes.count), &version, &error) + } + } + try Self.check(status, error: &error) + return version + } + + func readFile(path: String) throws -> Data { + var output = vexfs_mount_bytes() + var error = vexfs_mount_error() + let status = path.withCString { vexfs_mount_read_file(session, $0, &output, &error) } + try Self.check(status, error: &error) + return Self.take(&output) + } + + func move(source: String, destination: String) throws { + var error = vexfs_mount_error() + let status = source.withCString { sourcePointer in + destination.withCString { destinationPointer in + vexfs_mount_move(session, sourcePointer, destinationPointer, &error) + } + } + try Self.check(status, error: &error) + } + + func rename(source: String, destination: String, replace: Bool) throws { + var error = vexfs_mount_error() + let status = source.withCString { sourcePointer in + destination.withCString { destinationPointer in + vexfs_mount_rename(session, sourcePointer, destinationPointer, + replace ? 1 : 0, &error) + } + } + try Self.check(status, error: &error) + } + + func remove(path: String, recursive: Bool = false) throws { + var error = vexfs_mount_error() + let status = path.withCString { + vexfs_mount_remove(session, $0, recursive ? 1 : 0, &error) + } + try Self.check(status, error: &error) + } + + func openHandle(path: String, flags: String) throws -> String { + var output = vexfs_mount_bytes() + var error = vexfs_mount_error() + let request = UUID().uuidString + let status = path.withCString { pathPointer in + flags.withCString { flagsPointer in + request.withCString { requestPointer in + vexfs_mount_handle_open(session, pathPointer, flagsPointer, requestPointer, + &output, &error) + } + } + } + try Self.check(status, error: &error) + guard let value = String(data: Self.take(&output), encoding: .utf8) else { + throw POSIXError(.EIO) + } + return value + } + + func stageWrite(handle: String, offset: UInt64, data: Data) throws -> Int64 { + var generation: Int64 = 0 + var error = vexfs_mount_error() + let request = UUID().uuidString + let status = handle.withCString { handlePointer in + request.withCString { requestPointer in + data.withUnsafeBytes { bytes in + vexfs_mount_handle_stage_write(session, handlePointer, offset, + bytes.baseAddress, UInt64(bytes.count), requestPointer, &generation, &error) + } + } + } + try Self.check(status, error: &error) + return generation + } + + func truncate(handle: String, size: UInt64) throws -> Int64 { + var generation: Int64 = 0 + var error = vexfs_mount_error() + let request = UUID().uuidString + let status = handle.withCString { handlePointer in + request.withCString { requestPointer in + vexfs_mount_handle_truncate(session, handlePointer, size, requestPointer, + &generation, &error) + } + } + try Self.check(status, error: &error) + return generation + } + + func readHandle(handle: String, offset: UInt64, length: UInt64) throws -> Data { + var output = vexfs_mount_bytes() + var error = vexfs_mount_error() + let status = handle.withCString { + vexfs_mount_handle_read(session, $0, offset, length, &output, &error) + } + try Self.check(status, error: &error) + return Self.take(&output) + } + + func publish(handle: String, generation: Int64) throws -> Int64 { + var version: Int64 = 0 + var error = vexfs_mount_error() + let request = UUID().uuidString + let status = handle.withCString { handlePointer in + "data".withCString { durabilityPointer in + request.withCString { requestPointer in + vexfs_mount_handle_publish(session, handlePointer, generation, + durabilityPointer, requestPointer, &version, &error) + } + } + } + try Self.check(status, error: &error) + return version + } + + func closeHandle(_ handle: String, retain: Bool) throws { + var output = vexfs_mount_bytes() + var error = vexfs_mount_error() + let request = UUID().uuidString + let status = handle.withCString { handlePointer in + request.withCString { requestPointer in + vexfs_mount_handle_close(session, handlePointer, retain ? 1 : 0, + requestPointer, &output, &error) + } + } + try Self.check(status, error: &error) + _ = Self.take(&output) + } + + func synchronize() throws { + var published: Int64 = 0 + var error = vexfs_mount_error() + let request = UUID().uuidString + let status = request.withCString { + vexfs_mount_synchronize(session, $0, &published, &error) + } + try Self.check(status, error: &error) + } + + func reclaim() throws { + var reclaimed: Int64 = 0 + var error = vexfs_mount_error() + let request = UUID().uuidString + let status = request.withCString { vexfs_mount_reclaim(session, $0, &reclaimed, &error) } + try Self.check(status, error: &error) + } + + private static func take(_ bytes: inout vexfs_mount_bytes) -> Data { + guard let pointer = bytes.data, bytes.size > 0 else { + vexfs_mount_free(bytes.data) + bytes.data = nil + bytes.size = 0 + return Data() + } + let data = Data(bytes: pointer, count: Int(bytes.size)) + vexfs_mount_free(pointer) + bytes.data = nil + bytes.size = 0 + return data + } + + private static func check(_ status: vexfs_mount_status, + error: inout vexfs_mount_error) throws { + guard status != VEXFS_MOUNT_OK else { return } + let code: POSIXError.Code + switch status { + case VEXFS_MOUNT_NOT_FOUND: code = .ENOENT + case VEXFS_MOUNT_CONFLICT: code = .EBUSY + case VEXFS_MOUNT_READ_ONLY: code = .EROFS + case VEXFS_MOUNT_BUSY: code = .EBUSY + case VEXFS_MOUNT_INVALID_ARGUMENT: code = .EINVAL + case VEXFS_MOUNT_PERMISSION_DENIED: code = .EACCES + case VEXFS_MOUNT_NO_SPACE: code = .ENOSPC + case VEXFS_MOUNT_CORRUPTION: code = .EIO + case VEXFS_MOUNT_UNSUPPORTED: code = .ENOTSUP + default: code = .EIO + } + throw POSIXError(code) + } +} diff --git a/agent_files/macos/VexFSAppEx/VexFSFileSystem.swift b/agent_files/macos/VexFSAppEx/VexFSFileSystem.swift new file mode 100644 index 0000000000..2fd29cc817 --- /dev/null +++ b/agent_files/macos/VexFSAppEx/VexFSFileSystem.swift @@ -0,0 +1,119 @@ +import Foundation +import FSKit +import OSLog +import CryptoKit + +extension Logger { + static let vexfs = Logger(subsystem: "io.vexdb.vexfs", category: "filesystem") +} + +enum VexFSIdentity { + static func uuid(databasePath: String, workspace: String) -> UUID { + let normalized = URL(fileURLWithPath: databasePath).standardizedFileURL.path + let digest = SHA256.hash(data: Data("\(normalized)\u{0}\(workspace)".utf8)) + var bytes = Array(digest.prefix(16)) + bytes[6] = (bytes[6] & 0x0f) | 0x50 + bytes[8] = (bytes[8] & 0x3f) | 0x80 + return UUID(uuid: (bytes[0], bytes[1], bytes[2], bytes[3], + bytes[4], bytes[5], bytes[6], bytes[7], + bytes[8], bytes[9], bytes[10], bytes[11], + bytes[12], bytes[13], bytes[14], bytes[15])) + } +} + +@objc +@available(macOS 26.0, *) +final class VexFSFileSystem: FSUnaryFileSystem & FSUnaryFileSystemOperations { + private static let descriptorName = ".vexfs-volume.json" + private var resource: FSPathURLResource? + private var volume: VexFSVolume? + + private func configuration(for resource: FSPathURLResource) throws + -> (descriptor: VexFSDescriptor, databaseURL: URL) { + let root = resource.url.standardizedFileURL + var isDirectory: ObjCBool = false + guard FileManager.default.fileExists(atPath: root.path, isDirectory: &isDirectory), + isDirectory.boolValue else { + throw POSIXError(.ENOTDIR) + } + let descriptorURL = root.appendingPathComponent(Self.descriptorName, isDirectory: false) + let descriptor = try JSONDecoder().decode( + VexFSDescriptor.self, from: Data(contentsOf: descriptorURL)) + guard descriptor.version == 2, + !descriptor.database_file.isEmpty, + descriptor.database_file != ".", + descriptor.database_file != "..", + URL(fileURLWithPath: descriptor.database_file).lastPathComponent == + descriptor.database_file else { + throw POSIXError(.EINVAL) + } + let databaseURL = root.appendingPathComponent( + descriptor.database_file, isDirectory: false).standardizedFileURL + guard databaseURL.deletingLastPathComponent() == root else { + throw POSIXError(.EACCES) + } + return (descriptor, databaseURL) + } + + func probeResource(resource: FSResource, + replyHandler: @escaping (FSProbeResult?, (any Error)?) -> Void) { + guard let pathResource = resource as? FSPathURLResource else { + return replyHandler(nil, POSIXError(.ENODEV)) + } + do { + let (descriptor, databaseURL) = try configuration(for: pathResource) + let identifier = FSContainerIdentifier(uuid: + VexFSIdentity.uuid(databasePath: databaseURL.path, + workspace: descriptor.workspace)) + replyHandler(.usable(name: "VexFS \(descriptor.workspace)", containerID: identifier), nil) + } catch { + Logger.vexfs.debug("probe rejected resource: \(error.localizedDescription)") + replyHandler(.notRecognized, nil) + } + } + + func loadResource(resource: FSResource, options: FSTaskOptions, + replyHandler: @escaping (FSVolume?, (any Error)?) -> Void) { + guard let pathResource = resource as? FSPathURLResource else { + return replyHandler(nil, POSIXError(.EINVAL)) + } + for option in options.taskOptions where option.contains("-f") { + return replyHandler(nil, POSIXError(.ENOTSUP)) + } + guard pathResource.url.startAccessingSecurityScopedResource() else { + return replyHandler(nil, POSIXError(.EACCES)) + } + do { + let (descriptor, databaseURL) = try configuration(for: pathResource) + let backend = try VexFSBackend(databasePath: databaseURL.path, + workspace: descriptor.workspace) + let identity = VexFSIdentity.uuid(databasePath: databaseURL.path, + workspace: descriptor.workspace) + let loadedVolume = try VexFSVolume(backend: backend, workspace: descriptor.workspace, + volumeID: identity) + containerStatus = .ready + self.resource = pathResource + self.volume = loadedVolume + replyHandler(loadedVolume, nil) + } catch { + Logger.vexfs.error("load rejected resource: \(error.localizedDescription)") + pathResource.url.stopAccessingSecurityScopedResource() + replyHandler(nil, error) + } + } + + func unloadResource(resource: FSResource, options: FSTaskOptions, + replyHandler: @escaping ((any Error)?) -> Void) { + guard let pathResource = resource as? FSPathURLResource, + pathResource.url == self.resource?.url else { + return replyHandler(POSIXError(.EINVAL)) + } + do { try volume?.synchronizeNow() } catch { + Logger.vexfs.error("unload synchronize failed: \(error.localizedDescription)") + } + self.volume = nil + self.resource = nil + pathResource.url.stopAccessingSecurityScopedResource() + replyHandler(nil) + } +} diff --git a/agent_files/macos/VexFSAppEx/VexFSItem.swift b/agent_files/macos/VexFSAppEx/VexFSItem.swift new file mode 100644 index 0000000000..74d76e3a44 --- /dev/null +++ b/agent_files/macos/VexFSAppEx/VexFSItem.swift @@ -0,0 +1,28 @@ +import Foundation +import FSKit + +final class VexFSItem: FSItem { + let lock = NSLock() + var path: String + var record: VexFSStatRecord + var parentID: FSItem.Identifier + var handle: String? + var handleWritable = false + var dirtyGeneration: Int64? + + init(path: String, record: VexFSStatRecord, parentID: FSItem.Identifier) { + self.path = path + self.record = record + self.parentID = parentID + super.init() + } + + var itemType: FSItem.ItemType { + record.kind == "directory" ? .directory : .file + } + + var itemID: FSItem.Identifier { + path == "/" ? .rootDirectory : + (FSItem.Identifier(rawValue: record.inode + 1024) ?? .invalid) + } +} diff --git a/agent_files/macos/VexFSAppEx/VexFSVolume+Operations.swift b/agent_files/macos/VexFSAppEx/VexFSVolume+Operations.swift new file mode 100644 index 0000000000..7e1bc2e469 --- /dev/null +++ b/agent_files/macos/VexFSAppEx/VexFSVolume+Operations.swift @@ -0,0 +1,258 @@ +import Foundation +import FSKit + +extension VexFSVolume: FSVolume.Operations { + var volumeStatistics: FSStatFSResult { + let result = FSStatFSResult(fileSystemTypeName: "vexfs") + result.blockSize = 4096 + result.ioSize = 128 * 1024 + result.fileSystemSubType = 0 + return result + } + + var supportedVolumeCapabilities: FSVolume.SupportedCapabilities { + let capabilities = FSVolume.SupportedCapabilities() + capabilities.supportsPersistentObjectIDs = true + capabilities.supportsJournal = true + capabilities.supportsActiveJournal = true + capabilities.supportsSparseFiles = false + capabilities.supportsZeroRuns = false + capabilities.supportsFastStatFS = false + capabilities.supports2TBFiles = false + capabilities.supports64BitObjectIDs = true + capabilities.supportsSymbolicLinks = false + capabilities.supportsHardLinks = false + capabilities.supportsHiddenFiles = true + capabilities.doesNotSupportVolumeSizes = true + capabilities.doesNotSupportImmutableFiles = true + capabilities.doesNotSupportSettingFilePermissions = true + capabilities.caseFormat = .sensitive + return capabilities + } + + func activate(options: FSTaskOptions, + replyHandler: @escaping (FSItem?, (any Error)?) -> Void) { + replyHandler(rootItem, nil) + } + + func deactivate(options: FSDeactivateOptions = [], + replyHandler: @escaping ((any Error)?) -> Void) { + do { try synchronizeNow(); replyHandler(nil) } catch { replyHandler(error) } + } + + func mount(options: FSTaskOptions, replyHandler: @escaping ((any Error)?) -> Void) { + replyHandler(nil) + } + + func unmount(replyHandler: @escaping () -> Void) { + try? synchronizeNow() + replyHandler() + } + + func synchronize(flags: FSSyncFlags, + replyHandler: @escaping ((any Error)?) -> Void) { + do { try synchronizeNow(); replyHandler(nil) } catch { replyHandler(error) } + } + + func getAttributes(_ desiredAttributes: FSItem.GetAttributesRequest, of item: FSItem, + replyHandler: @escaping (FSItem.Attributes?, (any Error)?) -> Void) { + guard let item = item as? VexFSItem else { + return replyHandler(nil, POSIXError(.EINVAL)) + } + item.lock.lock() + defer { item.lock.unlock() } + do { + let path = try currentPath(for: item) + if item.dirtyGeneration == nil { + item.record = try backend.stat(path: path) + } + replyHandler(attributes(for: item, request: desiredAttributes), nil) + } catch { replyHandler(nil, error) } + } + + func setAttributes(_ newAttributes: FSItem.SetAttributesRequest, on item: FSItem, + replyHandler: @escaping (FSItem.Attributes?, (any Error)?) -> Void) { + guard let item = item as? VexFSItem else { + return replyHandler(nil, POSIXError(.EINVAL)) + } + item.lock.lock() + defer { item.lock.unlock() } + do { + let path = try currentPath(for: item) + if newAttributes.isValid(.size), item.itemType == .file { + let hadHandle = item.handle != nil + if item.handle == nil || !item.handleWritable { + if let old = item.handle { try backend.closeHandle(old, retain: true) } + item.handle = try backend.openHandle(path: path, flags: "rw") + item.handleWritable = true + item.dirtyGeneration = nil + } + let generation = try backend.truncate(handle: item.handle!, size: newAttributes.size) + item.dirtyGeneration = generation + newAttributes.consumedAttributes.insert(.size) + if !hadHandle { + _ = try backend.publish(handle: item.handle!, generation: generation) + try backend.closeHandle(item.handle!, retain: true) + item.handle = nil + item.handleWritable = false + item.dirtyGeneration = nil + item.record = try backend.stat(path: path) + } else { + item.record = VexFSStatRecord(path: path, inode: item.record.inode, + kind: item.record.kind, mode: item.record.mode, size: newAttributes.size, + version: item.record.version, created_at: item.record.created_at, + updated_at: Int64(Date().timeIntervalSince1970 * 1000)) + } + } else { + item.record = try backend.stat(path: path) + } + replyHandler(attributes(for: item, request: nil), nil) + } catch { replyHandler(nil, error) } + } + + func lookupItem(named name: FSFileName, inDirectory directory: FSItem, + replyHandler: @escaping (FSItem?, FSFileName?, (any Error)?) -> Void) { + guard let directory = directory as? VexFSItem, + directory.itemType == .directory, + let nameString = name.string else { + return replyHandler(nil, nil, POSIXError(.ENOTDIR)) + } + if nameString == "." { return replyHandler(directory, name, nil) } + do { + let path = try childPath(parent: directory, name: nameString) + let record = try backend.stat(path: path) + replyHandler(cachedItem(path: path, record: record, parentID: directory.itemID), name, nil) + } catch { replyHandler(nil, nil, error) } + } + + func reclaimItem(_ item: FSItem, replyHandler: @escaping ((any Error)?) -> Void) { + do { try backend.reclaim(); replyHandler(nil) } catch { replyHandler(error) } + } + + func readSymbolicLink(_ item: FSItem, + replyHandler: @escaping (FSFileName?, (any Error)?) -> Void) { + replyHandler(nil, POSIXError(.ENOTSUP)) + } + + func createItem(named name: FSFileName, type: FSItem.ItemType, + inDirectory directory: FSItem, + attributes newAttributes: FSItem.SetAttributesRequest, + replyHandler: @escaping (FSItem?, FSFileName?, (any Error)?) -> Void) { + guard let directory = directory as? VexFSItem, let nameString = name.string else { + return replyHandler(nil, nil, POSIXError(.EINVAL)) + } + do { + let path = try childPath(parent: directory, name: nameString) + switch type { + case .directory: try backend.mkdir(path: path) + case .file: _ = try backend.writeFile(path: path, data: Data()) + default: throw POSIXError(.ENOTSUP) + } + let record = try backend.stat(path: path) + let item = cachedItem(path: path, record: record, parentID: directory.itemID) + replyHandler(item, name, nil) + } catch { replyHandler(nil, nil, error) } + } + + func createSymbolicLink(named name: FSFileName, inDirectory directory: FSItem, + attributes newAttributes: FSItem.SetAttributesRequest, + linkContents contents: FSFileName, + replyHandler: @escaping (FSItem?, FSFileName?, (any Error)?) -> Void) { + replyHandler(nil, nil, POSIXError(.ENOTSUP)) + } + + func createLink(to item: FSItem, named name: FSFileName, inDirectory directory: FSItem, + replyHandler: @escaping (FSFileName?, (any Error)?) -> Void) { + replyHandler(nil, POSIXError(.ENOTSUP)) + } + + func removeItem(_ item: FSItem, named name: FSFileName, fromDirectory directory: FSItem, + replyHandler: @escaping ((any Error)?) -> Void) { + guard let item = item as? VexFSItem else { return replyHandler(POSIXError(.EINVAL)) } + do { + try backend.remove(path: currentPath(for: item)) + dropCachedItems() + replyHandler(nil) + } catch { replyHandler(error) } + } + + func renameItem(_ item: FSItem, inDirectory sourceDirectory: FSItem, + named sourceName: FSFileName, to destinationName: FSFileName, + inDirectory destinationDirectory: FSItem, overItem: FSItem?, + replyHandler: @escaping (FSFileName?, (any Error)?) -> Void) { + guard let source = item as? VexFSItem, + let destinationDirectory = destinationDirectory as? VexFSItem, + let destination = destinationName.string else { + return replyHandler(nil, POSIXError(.EINVAL)) + } + do { + let sourcePath = try currentPath(for: source) + let destinationPath = try childPath(parent: destinationDirectory, name: destination) + try backend.rename(source: sourcePath, destination: destinationPath, + replace: overItem != nil) + source.path = destinationPath + source.record = try backend.stat(path: destinationPath) + source.parentID = destinationDirectory.itemID + dropCachedItems() + replyHandler(destinationName, nil) + } catch { replyHandler(nil, error) } + } + + func enumerateDirectory(_ directory: FSItem, startingAt cookie: FSDirectoryCookie, + verifier: FSDirectoryVerifier, + attributes request: FSItem.GetAttributesRequest?, + packer: FSDirectoryEntryPacker, + replyHandler: @escaping (FSDirectoryVerifier, (any Error)?) -> Void) { + guard let directory = directory as? VexFSItem, directory.itemType == .directory else { + return replyHandler(FSDirectoryVerifier(0), POSIXError(.ENOTDIR)) + } + do { + let directoryPath = try currentPath(for: directory) + directory.record = try backend.stat(path: directoryPath) + let currentVerifier = FSDirectoryVerifier(directory.record.version) + if verifier.rawValue != 0 && verifier.rawValue != currentVerifier.rawValue { + throw FSError(.invalidDirectoryCookie) + } + let entries = try backend.list(path: directoryPath) + let verifiedRecord = try backend.stat(path: directoryPath) + guard verifiedRecord.version == directory.record.version else { + throw FSError(.invalidDirectoryCookie) + } + directory.record = verifiedRecord + let prefixCount = request == nil ? 2 : 0 + let total = prefixCount + entries.count + guard cookie.rawValue <= UInt64(total) else { + throw FSError(.invalidDirectoryCookie) + } + let start = Int(cookie.rawValue) + for index in start.. String { + let path = try backend.path(inode: item.record.inode) + item.path = path + return path + } + + func childPath(parent: VexFSItem, name: String) throws -> String { + let parentPath = try currentPath(for: parent) + return parentPath == "/" ? "/\(name)" : "\(parentPath)/\(name)" + } + + func cachedItem(path: String, record: VexFSStatRecord, + parentID: FSItem.Identifier) -> VexFSItem { + cacheLock.lock() + defer { cacheLock.unlock() } + if let item = itemCache[record.inode] { + item.path = path + item.record = record + item.parentID = parentID + return item + } + let item = VexFSItem(path: path, record: record, parentID: parentID) + itemCache[record.inode] = item + return item + } + + func dropCachedItems(exceptRoot: Bool = true) { + cacheLock.lock() + itemCache.removeAll(keepingCapacity: true) + if exceptRoot { itemCache[rootItem.record.inode] = rootItem } + cacheLock.unlock() + } + + func attributes(for item: VexFSItem, + request: FSItem.GetAttributesRequest?) -> FSItem.Attributes { + let attributes = FSItem.Attributes() + let wanted: (FSItem.Attribute) -> Bool = { attribute in + request == nil || request!.isAttributeWanted(attribute) + } + let created = timespec(tv_sec: Int(item.record.created_at / 1000), + tv_nsec: Int((item.record.created_at % 1000) * 1_000_000)) + let updated = timespec(tv_sec: Int(item.record.updated_at / 1000), + tv_nsec: Int((item.record.updated_at % 1000) * 1_000_000)) + if wanted(.type) { attributes.type = item.itemType } + if wanted(.mode) { attributes.mode = item.record.mode } + if wanted(.linkCount) { attributes.linkCount = item.itemType == .directory ? 2 : 1 } + if wanted(.uid) { attributes.uid = getuid() } + if wanted(.gid) { attributes.gid = getgid() } + if wanted(.flags) { attributes.flags = 0 } + if wanted(.size) { attributes.size = item.record.size } + if wanted(.allocSize) { attributes.allocSize = (item.record.size + 4095) / 4096 * 4096 } + if wanted(.fileID) { attributes.fileID = item.itemID } + if wanted(.parentID) { attributes.parentID = item.parentID } + if wanted(.accessTime) { attributes.accessTime = updated } + if wanted(.modifyTime) { attributes.modifyTime = updated } + if wanted(.changeTime) { attributes.changeTime = updated } + if wanted(.birthTime) { attributes.birthTime = created } + return attributes + } + + func openItem(_ item: FSItem, modes: FSVolume.OpenModes, + replyHandler: @escaping ((any Error)?) -> Void) { + guard let item = item as? VexFSItem else { return replyHandler(POSIXError(.EINVAL)) } + if item.itemType == .directory { return replyHandler(nil) } + item.lock.lock() + defer { item.lock.unlock() } + do { + let writable = modes.contains(.write) + if let oldHandle = item.handle, writable && !item.handleWritable { + try backend.closeHandle(oldHandle, retain: true) + item.handle = nil + item.dirtyGeneration = nil + } + if item.handle == nil { + item.handle = try backend.openHandle(path: currentPath(for: item), + flags: writable ? "rw" : "r") + item.handleWritable = writable + } + replyHandler(nil) + } catch { replyHandler(error) } + } + + func closeItem(_ item: FSItem, modes: FSVolume.OpenModes, + replyHandler: @escaping ((any Error)?) -> Void) { + guard let item = item as? VexFSItem else { return replyHandler(POSIXError(.EINVAL)) } + if !modes.isEmpty || item.itemType == .directory { return replyHandler(nil) } + item.lock.lock() + defer { item.lock.unlock() } + do { + if let handle = item.handle { + if let generation = item.dirtyGeneration { + _ = try backend.publish(handle: handle, generation: generation) + item.record = try backend.stat(path: currentPath(for: item)) + } + try backend.closeHandle(handle, retain: true) + } + item.handle = nil + item.handleWritable = false + item.dirtyGeneration = nil + replyHandler(nil) + } catch { replyHandler(error) } + } + + func read(from item: FSItem, at offset: off_t, length: Int, + into buffer: FSMutableFileDataBuffer, + replyHandler: @escaping (Int, (any Error)?) -> Void) { + guard let item = item as? VexFSItem, offset >= 0, length >= 0 else { + return replyHandler(0, POSIXError(.EINVAL)) + } + item.lock.lock() + defer { item.lock.unlock() } + do { + let data: Data + if let handle = item.handle { + data = try backend.readHandle(handle: handle, offset: UInt64(offset), + length: UInt64(length)) + } else { + let complete = try backend.readFile(path: currentPath(for: item)) + let start = min(Int(offset), complete.count) + data = complete.subdata(in: start.. Void) { + guard let item = item as? VexFSItem, item.itemType == .file, offset >= 0 else { + return replyHandler(0, POSIXError(.EINVAL)) + } + item.lock.lock() + defer { item.lock.unlock() } + do { + if item.handle == nil || !item.handleWritable { + if let old = item.handle { try backend.closeHandle(old, retain: true) } + item.handle = try backend.openHandle(path: currentPath(for: item), flags: "rw") + item.handleWritable = true + item.dirtyGeneration = nil + } + let generation = try backend.stageWrite(handle: item.handle!, offset: UInt64(offset), + data: contents) + item.dirtyGeneration = generation + item.record = VexFSStatRecord(path: item.path, + inode: item.record.inode, kind: item.record.kind, mode: item.record.mode, + size: max(item.record.size, UInt64(offset) + UInt64(contents.count)), + version: item.record.version, created_at: item.record.created_at, + updated_at: Int64(Date().timeIntervalSince1970 * 1000)) + replyHandler(contents.count, nil) + } catch { replyHandler(0, error) } + } + + var maximumLinkCount: Int { 1 } + var maximumNameLength: Int { 255 } + var restrictsOwnershipChanges: Bool { true } + var truncatesLongNames: Bool { false } + var maximumFileSizeInBits: Int { 28 } + var maximumXattrSizeInBits: Int { 0 } +} diff --git a/agent_files/macos/generate_xcode_project.rb b/agent_files/macos/generate_xcode_project.rb new file mode 100644 index 0000000000..e4e5ec1dd0 --- /dev/null +++ b/agent_files/macos/generate_xcode_project.rb @@ -0,0 +1,121 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +# 生成可提交的 Xcode 工程。工程本身不依赖这个脚本运行；脚本用于文件增减后重建。 +require 'xcodeproj' + +root = File.expand_path(__dir__) +project_path = File.join(root, 'VexFS.xcodeproj') +project = Xcodeproj::Project.new(project_path) +project.root_object.attributes['LastSwiftUpdateCheck'] = '2630' +project.root_object.attributes['LastUpgradeCheck'] = '2630' + +app_group = project.main_group.new_group('VexFSApp', 'VexFSApp') +extension_group = project.main_group.new_group('VexFSAppEx', 'VexFSAppEx') + +app = project.new_target(:application, 'VexFSApp', :osx, '26.0') +extension = project.new_target(:app_extension, 'VexFSAppEx', :osx, '26.0') +extension.product_type = 'com.apple.product-type.extensionkit-extension' +extension.product_reference.explicit_file_type = 'wrapper.extensionkit-extension' + +%w[VexFSApp.swift ContentView.swift].each do |name| + app.source_build_phase.add_file_reference(app_group.new_file(name)) +end +app_group.new_file('VexFSApp.entitlements') + +%w[ + VexFSAppEx.swift + VexFSBackend.swift + VexFSFileSystem.swift + VexFSItem.swift + VexFSVolume.swift + VexFSVolume+Operations.swift +].each do |name| + extension.source_build_phase.add_file_reference(extension_group.new_file(name)) +end +extension_group.new_file('VexFS-Bridging-Header.h') +extension_group.new_file('VexFSAppEx.entitlements') +extension_group.new_file('Info.plist') + +fskit = project.frameworks_group.new_file('System/Library/Frameworks/FSKit.framework') +fskit.source_tree = 'SDKROOT' +extension.frameworks_build_phase.add_file_reference(fskit) + +build_core = extension.new_shell_script_build_phase('Build VexFS SQLite Core') +build_core.always_out_of_date = '1' +build_core.shell_script = <<~'SH' + set -e + CMAKE_BIN="$(command -v cmake || true)" + [ -x "$CMAKE_BIN" ] || CMAKE_BIN=/opt/homebrew/bin/cmake + [ -x "$CMAKE_BIN" ] || CMAKE_BIN=/usr/local/bin/cmake + [ -x "$CMAKE_BIN" ] || { echo "cmake is required" >&2; exit 1; } + BUILD_DIR="$SRCROOT/../../vexdb_sqlite/build-fskit" + "$CMAKE_BIN" -S "$SRCROOT/../../vexdb_sqlite" -B "$BUILD_DIR" \ + -DCMAKE_BUILD_TYPE="$CONFIGURATION" -DVEXDB_SQLITE_BUILD_TESTS=OFF + "$CMAKE_BIN" --build "$BUILD_DIR" --target vexfs_mount_contract -j "$(sysctl -n hw.ncpu)" +SH +extension.build_phases.move(build_core, 0) + +embed = app.new_copy_files_build_phase('Embed ExtensionKit Extensions') +embed.dst_subfolder_spec = '16' +embed.dst_path = '$(EXTENSIONS_FOLDER_PATH)' +build_file = embed.add_file_reference(extension.product_reference, true) +build_file.settings = { 'ATTRIBUTES' => %w[RemoveHeadersOnCopy] } +app.add_dependency(extension) + +project.build_configurations.each do |configuration| + configuration.build_settings.merge!( + 'CLANG_ENABLE_MODULES' => 'YES', + 'MACOSX_DEPLOYMENT_TARGET' => '26.0', + 'SDKROOT' => 'macosx', + 'SWIFT_VERSION' => '5.0' + ) +end + +app.build_configurations.each do |configuration| + configuration.build_settings.merge!( + 'CODE_SIGN_ENTITLEMENTS' => 'VexFSApp/VexFSApp.entitlements', + 'CODE_SIGN_STYLE' => 'Automatic', + 'CURRENT_PROJECT_VERSION' => '1', + 'DEVELOPMENT_TEAM' => '', + 'ENABLE_APP_SANDBOX' => 'YES', + 'GENERATE_INFOPLIST_FILE' => 'YES', + 'INFOPLIST_KEY_CFBundleDisplayName' => 'VexFS', + 'LD_RUNPATH_SEARCH_PATHS' => '$(inherited) @executable_path/../Frameworks', + 'MARKETING_VERSION' => '0.1.0', + 'PRODUCT_BUNDLE_IDENTIFIER' => 'io.vexdb.vexfs', + 'PRODUCT_NAME' => 'VexFS' + ) +end + +extension.build_configurations.each do |configuration| + configuration.build_settings.merge!( + 'APPLICATION_EXTENSION_API_ONLY' => 'YES', + 'CODE_SIGN_ENTITLEMENTS' => 'VexFSAppEx/VexFSAppEx.entitlements', + 'CODE_SIGN_STYLE' => 'Automatic', + 'CURRENT_PROJECT_VERSION' => '1', + 'DEVELOPMENT_TEAM' => '', + 'ENABLE_APP_SANDBOX' => 'YES', + 'ENABLE_USER_SCRIPT_SANDBOXING' => 'NO', + 'GENERATE_INFOPLIST_FILE' => 'YES', + 'HEADER_SEARCH_PATHS' => '$(inherited) "$(SRCROOT)/../mount/common/include"', + 'INFOPLIST_FILE' => 'VexFSAppEx/Info.plist', + 'INFOPLIST_KEY_CFBundleDisplayName' => 'VexFS file system', + 'LD_RUNPATH_SEARCH_PATHS' => '$(inherited) @executable_path/../Frameworks @executable_path/../../../../Frameworks', + 'MARKETING_VERSION' => '0.1.0', + 'OTHER_LDFLAGS' => [ + '$(inherited)', + '"$(SRCROOT)/../../vexdb_sqlite/build-fskit/libvexfs_mount_contract.a"', + '"$(SRCROOT)/../../vexdb_sqlite/build-fskit/libvexdb_lite_static.a"', + '-lsqlite3', + '-lc++' + ], + 'PRODUCT_BUNDLE_IDENTIFIER' => 'io.vexdb.vexfs.extension', + 'PRODUCT_NAME' => 'VexFSAppEx', + 'SKIP_INSTALL' => 'YES', + 'SWIFT_OBJC_BRIDGING_HEADER' => 'VexFSAppEx/VexFS-Bridging-Header.h' + ) +end + +project.save +puts project_path diff --git a/agent_files/macos/package/Install.command b/agent_files/macos/package/Install.command new file mode 100755 index 0000000000..bae46cc4cd --- /dev/null +++ b/agent_files/macos/package/Install.command @@ -0,0 +1,6 @@ +#!/bin/bash +set -euo pipefail +ROOT="$(cd "$(dirname "$0")" && pwd)" +/bin/bash "$ROOT/install.sh" +echo "" +read -r -p "安装完成，按回车关闭窗口。" _ diff --git a/agent_files/macos/package/README.md b/agent_files/macos/package/README.md new file mode 100644 index 0000000000..0807bffa58 --- /dev/null +++ b/agent_files/macos/package/README.md @@ -0,0 +1,89 @@ +# VexFS macOS 技术预览 + +VexFS 是由 SQLite 管理的文件工作区。本包提供： + +- `VexFS.app`：包含 FSKit 文件系统扩展； +- `bin/vexfs`：命令行工具； +- `lib/vexdb_lite.dylib`：可由 SQLite 加载的扩展； +- `install.sh`、`Install.command` 和 `uninstall.sh`。 + +## 系统要求 + +- macOS 26.0 或更高版本； +- Apple Silicon 包要求 arm64 Mac； +- 首次尝试真实挂载时，需要在系统设置中启用 VexFS 文件系统扩展。 + +## 重要限制 + +这是 **ad-hoc 签名的技术预览包**，没有 Developer ID 签名和 Apple 公证。 + +- CLI 和 SQLite 文件管理功能可以直接测试； +- 从聊天、邮件或浏览器下载后，macOS 可能阻止打开 App。请在 Finder 中右键 `VexFS.app`，选择“打开”； +- FSKit extension 是否允许在另一台 Mac 上启用，取决于该机器的系统安全设置。正式对外分发必须使用 Apple Developer 证书签名并公证。 + +## 安装 + +双击 `Install.command`，或在终端运行： + +```bash +./install.sh +``` + +默认安装位置： + +- App：`~/Applications/VexFS.app` +- CLI：`~/.local/bin/vexfs` +- SQLite 扩展：`~/.local/lib/vexfs/vexdb_lite.dylib` + +如果 `~/.local/bin` 不在 PATH： + +```bash +echo 'export PATH="$HOME/.local/bin:$PATH"' >> ~/.zshrc +source ~/.zshrc +``` + +## 不挂载也能使用 + +```bash +vexfs setup +printf 'hello VexFS\n' | vexfs write /hello.txt +vexfs cat /hello.txt +vexfs ls / +vexfs history /hello.txt +vexfs doctor --json +``` + +默认数据库位于： + +```text +~/Library/Application Support/VexFS/vexfs.sqlite3 +``` + +## 尝试真实 Bash 挂载 + +1. 打开 `~/Applications/VexFS.app`； +2. 进入“系统设置 → 通用 → 登录项与扩展 → 文件系统扩展”； +3. 启用 VexFS； +4. 运行： + +```bash +vexfs doctor +vexfs mount ~/VexFS +cd ~/VexFS +ls +cat hello.txt +``` + +如果 `doctor` 显示扩展未启用，仍可继续使用上面的直接 CLI 命令。 + +## 卸载 + +```bash +./uninstall.sh +``` + +卸载脚本不会删除数据库。确认不再需要数据后，可自行删除： + +```text +~/Library/Application Support/VexFS/ +``` diff --git a/agent_files/macos/package/install.sh b/agent_files/macos/package/install.sh new file mode 100755 index 0000000000..543c328895 --- /dev/null +++ b/agent_files/macos/package/install.sh @@ -0,0 +1,31 @@ +#!/bin/bash +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")" && pwd)" +APP_DIR="${VEXFS_APP_DIR:-$HOME/Applications}" +BIN_DIR="${VEXFS_BIN_DIR:-$HOME/.local/bin}" +LIB_DIR="${VEXFS_LIB_DIR:-$HOME/.local/lib/vexfs}" + +[ -d "$ROOT/VexFS.app" ] || { echo "安装包缺少 VexFS.app" >&2; exit 1; } +[ -x "$ROOT/bin/vexfs" ] || { echo "安装包缺少 bin/vexfs" >&2; exit 1; } +[ -f "$ROOT/lib/vexdb_lite.dylib" ] || { echo "安装包缺少 SQLite 扩展" >&2; exit 1; } + +mkdir -p "$APP_DIR" "$BIN_DIR" "$LIB_DIR" +ditto "$ROOT/VexFS.app" "$APP_DIR/VexFS.app" +install -m 0755 "$ROOT/bin/vexfs" "$BIN_DIR/vexfs" +install -m 0644 "$ROOT/lib/vexdb_lite.dylib" "$LIB_DIR/vexdb_lite.dylib" + +echo "VexFS 已安装：" +echo " App: $APP_DIR/VexFS.app" +echo " CLI: $BIN_DIR/vexfs" +echo " SQLite: $LIB_DIR/vexdb_lite.dylib" +case ":$PATH:" in + *":$BIN_DIR:"*) ;; + *) + echo "" + echo "请把下面一行加入 ~/.zshrc：" + echo " export PATH=\"$BIN_DIR:\$PATH\"" + ;; +esac +echo "" +echo "下一步：$BIN_DIR/vexfs setup && $BIN_DIR/vexfs doctor" diff --git a/agent_files/macos/package/uninstall.sh b/agent_files/macos/package/uninstall.sh new file mode 100755 index 0000000000..63eef9bca7 --- /dev/null +++ b/agent_files/macos/package/uninstall.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -euo pipefail + +APP_DIR="${VEXFS_APP_DIR:-$HOME/Applications}" +BIN_DIR="${VEXFS_BIN_DIR:-$HOME/.local/bin}" +LIB_DIR="${VEXFS_LIB_DIR:-$HOME/.local/lib/vexfs}" + +rm -rf "$APP_DIR/VexFS.app" +rm -f "$BIN_DIR/vexfs" "$LIB_DIR/vexdb_lite.dylib" +rmdir "$LIB_DIR" 2>/dev/null || true + +echo "VexFS App、CLI 和 SQLite 扩展已卸载。" +echo "数据库未删除：$HOME/Library/Application Support/VexFS/" diff --git a/agent_files/macos/package_preview.sh b/agent_files/macos/package_preview.sh new file mode 100755 index 0000000000..d15f89261f --- /dev/null +++ b/agent_files/macos/package_preview.sh @@ -0,0 +1,122 @@ +#!/bin/bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +SQLITE_DIR="$ROOT/vexdb_sqlite" +PACKAGE_DIR="$SCRIPT_DIR/package" + +VERSION="${VEXFS_PREVIEW_VERSION:-0.1.0-preview.1}" +ARCH="${VEXFS_ARCH:-arm64}" +case "$VERSION" in + *[!A-Za-z0-9._-]*) echo "版本只能包含字母、数字、点、下划线和连字符" >&2; exit 2 ;; +esac +case "$ARCH" in + arm64|x86_64) ;; + *) echo "VEXFS_ARCH 只能是 arm64 或 x86_64" >&2; exit 2 ;; +esac + +CMAKE_BIN="$(command -v cmake || true)" +[ -x "$CMAKE_BIN" ] || CMAKE_BIN=/opt/homebrew/bin/cmake +[ -x "$CMAKE_BIN" ] || CMAKE_BIN=/usr/local/bin/cmake +[ -x "$CMAKE_BIN" ] || { echo "找不到 cmake" >&2; exit 1; } +command -v xcodebuild >/dev/null || { echo "找不到 xcodebuild" >&2; exit 1; } +command -v codesign >/dev/null || { echo "找不到 codesign" >&2; exit 1; } + +NCPU="$(sysctl -n hw.ncpu 2>/dev/null || echo 8)" +BUILD_DIR="$SQLITE_DIR/build-vexfs-package-$ARCH" +DERIVED_DATA="$BUILD_DIR/xcode-derived-data" +DIST_DIR="$ROOT/dist/vexfs" +PACKAGE_NAME="vexfs-${VERSION}-macos-${ARCH}" +STAGE="$DIST_DIR/$PACKAGE_NAME" +ZIP="$DIST_DIR/$PACKAGE_NAME.zip" + +mkdir -p "$BUILD_DIR" "$DIST_DIR" + +echo "=== 构建 VexFS CLI 与 SQLite 扩展 ===" +"$CMAKE_BIN" -S "$SQLITE_DIR" -B "$BUILD_DIR" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_OSX_ARCHITECTURES="$ARCH" \ + -DVEXDB_SQLITE_BUILD_TESTS=OFF +"$CMAKE_BIN" --build "$BUILD_DIR" --target vexfs_cli vexdb_lite_loadable -j "$NCPU" + +echo "=== 构建 VexFS App 与 FSKit extension ===" +xcodebuild \ + -project "$SCRIPT_DIR/VexFS.xcodeproj" \ + -scheme VexFSApp \ + -configuration Release \ + -derivedDataPath "$DERIVED_DATA" \ + ARCHS="$ARCH" ONLY_ACTIVE_ARCH=YES CODE_SIGNING_ALLOWED=NO build + +APP_SOURCE="$DERIVED_DATA/Build/Products/Release/VexFS.app" +[ -d "$APP_SOURCE" ] || { echo "没有找到 VexFS.app" >&2; exit 1; } +[ -x "$BUILD_DIR/vexfs" ] || { echo "没有找到 vexfs CLI" >&2; exit 1; } +[ -f "$BUILD_DIR/vexdb_lite.dylib" ] || { echo "没有找到 vexdb_lite.dylib" >&2; exit 1; } + +rm -rf "$STAGE" +mkdir -p "$STAGE/bin" "$STAGE/lib" +ditto "$APP_SOURCE" "$STAGE/VexFS.app" +install -m 0755 "$BUILD_DIR/vexfs" "$STAGE/bin/vexfs" +install -m 0644 "$BUILD_DIR/vexdb_lite.dylib" "$STAGE/lib/vexdb_lite.dylib" +install -m 0755 "$PACKAGE_DIR/install.sh" "$STAGE/install.sh" +install -m 0755 "$PACKAGE_DIR/Install.command" "$STAGE/Install.command" +install -m 0755 "$PACKAGE_DIR/uninstall.sh" "$STAGE/uninstall.sh" +install -m 0644 "$PACKAGE_DIR/README.md" "$STAGE/README.md" +install -m 0644 "$ROOT/LICENSE" "$STAGE/LICENSE" + +# 没有 Developer ID 时仍做完整的 ad-hoc 签名，避免发送过程中破坏 bundle seal。 +EXTENSION="$STAGE/VexFS.app/Contents/Extensions/VexFSAppEx.appex" +codesign --force --sign - --timestamp=none \ + --entitlements "$SCRIPT_DIR/VexFSAppEx/VexFSAppEx.entitlements" "$EXTENSION" +codesign --force --sign - --timestamp=none \ + --entitlements "$SCRIPT_DIR/VexFSApp/VexFSApp.entitlements" "$STAGE/VexFS.app" +codesign --force --sign - --timestamp=none "$STAGE/bin/vexfs" +codesign --force --sign - --timestamp=none "$STAGE/lib/vexdb_lite.dylib" +codesign --verify --deep --strict "$STAGE/VexFS.app" + +GIT_COMMIT="$(git -C "$ROOT" rev-parse --short HEAD 2>/dev/null || echo unknown)" +GIT_BRANCH="$(git -C "$ROOT" branch --show-current 2>/dev/null || echo unknown)" +if [ -n "$(git -C "$ROOT" status --porcelain 2>/dev/null || true)" ]; then + GIT_DIRTY=true +else + GIT_DIRTY=false +fi +{ + echo "product=VexFS" + echo "preview_version=$VERSION" + echo "contract_version=0.3.0" + echo "mount_abi_version=3" + echo "platform=macOS" + echo "minimum_macos=26.0" + echo "architecture=$ARCH" + echo "source_branch=$GIT_BRANCH" + echo "source_commit=$GIT_COMMIT" + echo "source_dirty=$GIT_DIRTY" + echo "signature=ad-hoc" +} > "$STAGE/MANIFEST.txt" + +( + cd "$STAGE" + shasum -a 256 \ + bin/vexfs \ + lib/vexdb_lite.dylib \ + VexFS.app/Contents/MacOS/VexFS \ + VexFS.app/Contents/Extensions/VexFSAppEx.appex/Contents/MacOS/VexFSAppEx \ + > SHA256SUMS.txt +) + +rm -f "$ZIP" "$ZIP.sha256" +COPYFILE_DISABLE=1 ditto -c -k --norsrc --keepParent "$STAGE" "$ZIP" +if unzip -Z1 "$ZIP" | grep -Eq '(^|/)__MACOSX/|(^|/)\._'; then + echo "压缩包包含 AppleDouble 文件，停止交付" >&2 + exit 1 +fi +( + cd "$DIST_DIR" + shasum -a 256 "$(basename "$ZIP")" > "$(basename "$ZIP").sha256" +) + +echo "" +echo "=== VexFS 技术预览包 ===" +ls -lh "$ZIP" "$ZIP.sha256" +cat "$ZIP.sha256" diff --git a/agent_files/mount/common/include/vexfs_mount_contract.h b/agent_files/mount/common/include/vexfs_mount_contract.h new file mode 100644 index 0000000000..cc7f99bdd0 --- /dev/null +++ b/agent_files/mount/common/include/vexfs_mount_contract.h @@ -0,0 +1,151 @@ +// VexFS 平台挂载合同（C ABI）。 +// +// FSKit、libfuse 和 WinFsp 适配层只依赖这个头。实现通过公开的 vexfs_* SQL +// 函数访问 SQLite，数据库仍然是状态和事务的唯一管理者。 +#ifndef VEXFS_MOUNT_CONTRACT_H +#define VEXFS_MOUNT_CONTRACT_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#define VEXFS_MOUNT_ABI_VERSION 3u +#define VEXFS_MOUNT_OPEN_NO_CREATE 1u +#define VEXFS_MOUNT_EXCLUSIVE_GATEWAY 2u + +typedef struct vexfs_mount_session vexfs_mount_session; + +typedef enum vexfs_mount_status { + VEXFS_MOUNT_OK = 0, + VEXFS_MOUNT_INVALID_ARGUMENT = 1, + VEXFS_MOUNT_NOT_FOUND = 2, + VEXFS_MOUNT_CONFLICT = 3, + VEXFS_MOUNT_READ_ONLY = 4, + VEXFS_MOUNT_BUSY = 5, + VEXFS_MOUNT_DATABASE_ERROR = 6, + VEXFS_MOUNT_INTERNAL_ERROR = 7, + VEXFS_MOUNT_PERMISSION_DENIED = 8, + VEXFS_MOUNT_NO_SPACE = 9, + VEXFS_MOUNT_CORRUPTION = 10, + VEXFS_MOUNT_UNSUPPORTED = 11 +} vexfs_mount_status; + +typedef struct vexfs_mount_error { + vexfs_mount_status status; + int sqlite_code; + char message[512]; +} vexfs_mount_error; + +typedef struct vexfs_mount_config { + uint32_t abi_version; + const char *database_path; + const char *workspace; + uint32_t busy_timeout_ms; + uint32_t flags; +} vexfs_mount_config; + +typedef struct vexfs_mount_bytes { + void *data; + uint64_t size; +} vexfs_mount_bytes; + +// 打开数据库、注册 vexdb_lite/VexFS、初始化内部表并创建（或复用）workspace。 +vexfs_mount_status vexfs_mount_session_open(const vexfs_mount_config *config, + vexfs_mount_session **session, + vexfs_mount_error *error); +void vexfs_mount_session_close(vexfs_mount_session *session); + +// 返回数据库合同、durability 和待恢复 staging 的 JSON 诊断信息。 +vexfs_mount_status vexfs_mount_diagnostics(vexfs_mount_session *session, + vexfs_mount_bytes *json, + vexfs_mount_error *error); + +vexfs_mount_status vexfs_mount_mkdir(vexfs_mount_session *session, const char *path, + vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_write_file(vexfs_mount_session *session, const char *path, + const void *data, uint64_t size, + int64_t *version, vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_read_file(vexfs_mount_session *session, const char *path, + vexfs_mount_bytes *content, + vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_history(vexfs_mount_session *session, const char *path, + vexfs_mount_bytes *json, + vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_history_page(vexfs_mount_session *session, const char *path, + uint32_t limit, int64_t before_version, + vexfs_mount_bytes *json, + vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_read_version(vexfs_mount_session *session, const char *path, + int64_t version, vexfs_mount_bytes *content, + vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_compare_versions(vexfs_mount_session *session, const char *path, + int64_t from_version, int64_t to_version, + vexfs_mount_bytes *json, + vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_restore_version(vexfs_mount_session *session, const char *path, + int64_t target_version, + int64_t expected_version, + int64_t *new_version, + vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_stat(vexfs_mount_session *session, const char *path, + vexfs_mount_bytes *json, vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_path_for_inode(vexfs_mount_session *session, int64_t inode, + vexfs_mount_bytes *path, + vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_list(vexfs_mount_session *session, const char *path, + vexfs_mount_bytes *json, vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_move(vexfs_mount_session *session, const char *source, + const char *destination, vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_rename(vexfs_mount_session *session, const char *source, + const char *destination, int replace, + vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_remove(vexfs_mount_session *session, const char *path, + int recursive, vexfs_mount_error *error); + +vexfs_mount_status vexfs_mount_handle_open(vexfs_mount_session *session, const char *path, + const char *flags, const char *request_id, + vexfs_mount_bytes *handle, + vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_handle_stage_write(vexfs_mount_session *session, + const char *handle, uint64_t offset, + const void *data, uint64_t size, + const char *request_id, + int64_t *generation, + vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_handle_truncate(vexfs_mount_session *session, + const char *handle, uint64_t size, + const char *request_id, + int64_t *generation, + vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_handle_read(vexfs_mount_session *session, const char *handle, + uint64_t offset, uint64_t length, + vexfs_mount_bytes *content, + vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_handle_publish(vexfs_mount_session *session, + const char *handle, int64_t generation, + const char *durability, + const char *request_id, int64_t *version, + vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_handle_close(vexfs_mount_session *session, + const char *handle, int retain_unpublished, + const char *request_id, + vexfs_mount_bytes *state, + vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_synchronize(vexfs_mount_session *session, + const char *request_id, int64_t *published, + vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_reclaim(vexfs_mount_session *session, + const char *request_id, int64_t *reclaimed, + vexfs_mount_error *error); + +// 释放本合同返回的 vexfs_mount_bytes.data。 +void vexfs_mount_free(void *memory); + +#ifdef __cplusplus +} +#endif + +#endif // VEXFS_MOUNT_CONTRACT_H diff --git a/agent_files/mount/common/src/vexfs_mount_contract_sqlite.cpp b/agent_files/mount/common/src/vexfs_mount_contract_sqlite.cpp new file mode 100644 index 0000000000..1e005393b1 --- /dev/null +++ b/agent_files/mount/common/src/vexfs_mount_contract_sqlite.cpp @@ -0,0 +1,812 @@ +#include "vexfs_mount_contract.h" + +#include "sqlite3.h" +#include "vexdb_sqlite.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(_WIN32) +#include +#else +#include +#include +#include +#include +#include +#endif + +struct vexfs_mount_session { + sqlite3 *db = nullptr; + std::string workspace; + std::string session_id; + bool exclusive_gateway = false; + bool session_started = false; + bool synchronous_full = true; + std::recursive_mutex mutex; +}; + +namespace { + +class CallError : public std::runtime_error { + public: + CallError(int code, const std::string &message) : std::runtime_error(message), code(code) {} + int code; +}; + +class Call { + public: + Call(sqlite3 *db, const char *sql) : db_(db) { + const int rc = sqlite3_prepare_v2(db, sql, -1, &statement_, nullptr); + if (rc != SQLITE_OK) Throw(rc); + } + ~Call() { + if (statement_ != nullptr) sqlite3_finalize(statement_); + } + Call(const Call &) = delete; + Call &operator=(const Call &) = delete; + + void Text(int index, const char *value) { + if (value == nullptr) throw CallError(SQLITE_MISUSE, "required text is NULL"); + Check(sqlite3_bind_text(statement_, index, value, -1, SQLITE_TRANSIENT)); + } + void Int64(int index, int64_t value) { + Check(sqlite3_bind_int64(statement_, index, value)); + } + void Int(int index, int value) { Check(sqlite3_bind_int(statement_, index, value)); } + void Blob(int index, const void *data, uint64_t size) { + if (size > static_cast(std::numeric_limits::max())) { + throw CallError(SQLITE_TOOBIG, "buffer is too large"); + } + if (data == nullptr && size != 0) throw CallError(SQLITE_MISUSE, "buffer is NULL"); + Check(sqlite3_bind_blob64(statement_, index, size == 0 ? "" : data, size, SQLITE_TRANSIENT)); + } + void Row() { + const int rc = sqlite3_step(statement_); + if (rc != SQLITE_ROW) Throw(rc); + } + bool MaybeRow() { + const int rc = sqlite3_step(statement_); + if (rc == SQLITE_ROW) return true; + if (rc == SQLITE_DONE) return false; + Throw(rc); + return false; + } + int64_t ResultInt64() const { return sqlite3_column_int64(statement_, 0); } + std::string ResultText() const { + const unsigned char *value = sqlite3_column_text(statement_, 0); + return value == nullptr ? std::string() : + std::string(reinterpret_cast(value)); + } + const void *ResultBlob() const { return sqlite3_column_blob(statement_, 0); } + int ResultBytes() const { return sqlite3_column_bytes(statement_, 0); } + + private: + void Check(int rc) { + if (rc != SQLITE_OK) Throw(rc); + } + [[noreturn]] void Throw(int rc) { + throw CallError(rc, sqlite3_errmsg(db_)); + } + + sqlite3 *db_ = nullptr; + sqlite3_stmt *statement_ = nullptr; +}; + +void ClearError(vexfs_mount_error *error) { + if (error == nullptr) return; + error->status = VEXFS_MOUNT_OK; + error->sqlite_code = SQLITE_OK; + error->message[0] = '\0'; +} + +vexfs_mount_status MapStatus(int sqlite_code, const std::string &) { + const int primary = sqlite_code & 0xff; + if (primary == SQLITE_BUSY || primary == SQLITE_LOCKED) return VEXFS_MOUNT_BUSY; + if (primary == SQLITE_MISUSE || primary == SQLITE_RANGE || primary == SQLITE_TOOBIG || + primary == SQLITE_MISMATCH) return VEXFS_MOUNT_INVALID_ARGUMENT; + if (primary == SQLITE_NOTFOUND) return VEXFS_MOUNT_NOT_FOUND; + if (primary == SQLITE_CONSTRAINT) return VEXFS_MOUNT_CONFLICT; + if (primary == SQLITE_READONLY) return VEXFS_MOUNT_READ_ONLY; + if (primary == SQLITE_AUTH || primary == SQLITE_PERM) return VEXFS_MOUNT_PERMISSION_DENIED; + if (primary == SQLITE_FULL) return VEXFS_MOUNT_NO_SPACE; + if (primary == SQLITE_CORRUPT || primary == SQLITE_NOTADB) return VEXFS_MOUNT_CORRUPTION; + return VEXFS_MOUNT_DATABASE_ERROR; +} + +vexfs_mount_status SetError(vexfs_mount_error *error, int sqlite_code, + const std::string &message) { + const vexfs_mount_status status = MapStatus(sqlite_code, message); + if (error != nullptr) { + error->status = status; + error->sqlite_code = sqlite_code; + std::snprintf(error->message, sizeof(error->message), "%s", message.c_str()); + } + return status; +} + +template +vexfs_mount_status Guard(vexfs_mount_error *error, Function function) { + ClearError(error); + try { + function(); + return VEXFS_MOUNT_OK; + } catch (const CallError &exception) { + return SetError(error, exception.code, exception.what()); + } catch (const std::bad_alloc &) { + return SetError(error, SQLITE_NOMEM, "out of memory"); + } catch (const std::exception &exception) { + return SetError(error, SQLITE_ERROR, exception.what()); + } catch (...) { + return SetError(error, SQLITE_ERROR, "unknown mount contract error"); + } +} + +template +vexfs_mount_status Guard(vexfs_mount_session *session, vexfs_mount_error *error, + Function function) { + return Guard(error, [&] { + if (session == nullptr) throw CallError(SQLITE_MISUSE, "mount session is NULL"); + std::lock_guard lock(session->mutex); + function(); + }); +} + +void RequireSession(vexfs_mount_session *session) { + if (session == nullptr || session->db == nullptr) { + throw CallError(SQLITE_MISUSE, "mount session is NULL"); + } + if (session->exclusive_gateway && session->session_started) { + Call heartbeat(session->db, "SELECT vexfs_mount_session_heartbeat(?1,?2)"); + heartbeat.Text(1, session->workspace.c_str()); + heartbeat.Text(2, session->session_id.c_str()); + heartbeat.Row(); + } +} + +#if !defined(_WIN32) +void PreparePrivateDatabase(const char *path, bool no_create) { + struct stat info {}; + if (stat(path, &info) == 0) { + if (info.st_uid != geteuid()) { + throw CallError(SQLITE_AUTH, "database is not owned by the current user"); + } + if (!no_create && chmod(path, 0600) != 0) { + throw CallError(SQLITE_AUTH, std::string("cannot protect database: ") + std::strerror(errno)); + } + return; + } + if (errno != ENOENT) { + throw CallError(SQLITE_CANTOPEN, std::string("cannot inspect database: ") + std::strerror(errno)); + } + if (no_create) throw CallError(SQLITE_CANTOPEN, "database does not exist"); + const int descriptor = open(path, O_CREAT | O_EXCL | O_RDWR, 0600); + if (descriptor < 0) { + throw CallError(SQLITE_CANTOPEN, std::string("cannot create database: ") + std::strerror(errno)); + } + close(descriptor); +} + +void ProtectSidecar(const std::string &path) { + struct stat info {}; + if (stat(path.c_str(), &info) == 0 && info.st_uid == geteuid()) chmod(path.c_str(), 0600); +} +#else +void PreparePrivateDatabase(const char *, bool) {} +void ProtectSidecar(const std::string &) {} +#endif + +bool LocalProcessIsAlive(const std::string &session_id) { + const size_t separator = session_id.find('-'); + if (separator == std::string::npos) return true; + const std::string process_text = session_id.substr(0, separator); + char *end = nullptr; + const long long process_id = std::strtoll(process_text.c_str(), &end, 10); + if (end == nullptr || *end != '\0' || process_id <= 0) return true; +#if defined(_WIN32) + return true; +#else + if (process_id == static_cast(getpid())) return true; + if (kill(static_cast(process_id), 0) == 0) return true; + return errno != ESRCH; +#endif +} + +void CheckExistingLocalSession(sqlite3 *db, const char *workspace, + const std::string &new_session_id) { + Call current(db, + "SELECT s.session_id FROM _vexfs_mount_sessions s " + "JOIN _vexfs_workspaces w ON w.id=s.workspace_id WHERE w.name=?1"); + current.Text(1, workspace); + if (!current.MaybeRow()) return; + const std::string session_id = current.ResultText(); + if (session_id == new_session_id) return; + if (LocalProcessIsAlive(session_id)) { + throw CallError(SQLITE_BUSY, "workspace already has an active mount process"); + } + Call expire(db, + "UPDATE _vexfs_mount_sessions SET lease_until=0 WHERE session_id=?1 RETURNING 1"); + expire.Text(1, session_id.c_str()); + expire.Row(); +} + +void CopyResult(Call &call, vexfs_mount_bytes *output) { + if (output == nullptr) throw CallError(SQLITE_MISUSE, "output buffer is NULL"); + output->data = nullptr; + output->size = 0; + const int bytes = call.ResultBytes(); + if (bytes < 0) throw CallError(SQLITE_ERROR, "negative result size"); + if (bytes == 0) return; + void *copy = std::malloc(static_cast(bytes) + 1); + if (copy == nullptr) throw std::bad_alloc(); + std::memcpy(copy, call.ResultBlob(), static_cast(bytes)); + static_cast(copy)[bytes] = 0; + output->data = copy; + output->size = static_cast(bytes); +} + +void CopyString(const std::string &value, vexfs_mount_bytes *output) { + if (output == nullptr) throw CallError(SQLITE_MISUSE, "output buffer is NULL"); + output->data = nullptr; + output->size = 0; + if (value.empty()) return; + void *copy = std::malloc(value.size() + 1); + if (copy == nullptr) throw std::bad_alloc(); + std::memcpy(copy, value.data(), value.size()); + static_cast(copy)[value.size()] = '\0'; + output->data = copy; + output->size = static_cast(value.size()); +} + +std::string JsonEscape(const std::string &value) { + std::string output; + output.reserve(value.size() + 8); + for (const unsigned char c : value) { + switch (c) { + case '"': output += "\\\""; break; + case '\\': output += "\\\\"; break; + case '\b': output += "\\b"; break; + case '\f': output += "\\f"; break; + case '\n': output += "\\n"; break; + case '\r': output += "\\r"; break; + case '\t': output += "\\t"; break; + default: + if (c < 0x20) { + char escaped[7] = {}; + std::snprintf(escaped, sizeof(escaped), "\\u%04x", c); + output += escaped; + } else { + output.push_back(static_cast(c)); + } + } + } + return output; +} + +void Exec(sqlite3 *db, const char *sql) { + char *message = nullptr; + const int rc = sqlite3_exec(db, sql, nullptr, nullptr, &message); + if (rc != SQLITE_OK) { + std::string text = message == nullptr ? sqlite3_errmsg(db) : message; + sqlite3_free(message); + throw CallError(rc, text); + } +} + +void UseStagingDurability(vexfs_mount_session *session) { + if (!session->synchronous_full) return; + Exec(session->db, "PRAGMA synchronous=NORMAL"); + session->synchronous_full = false; +} + +void UseFullDurability(vexfs_mount_session *session) { + if (session->synchronous_full) return; + // publish/synchronize 前切回 FULL。这个权威事务会同步同一 WAL 中此前已提交的 + // staging frame；成功返回后，最终文件和它依赖的 staging 都达到 FULL 级别。 + Exec(session->db, "PRAGMA synchronous=FULL"); + session->synchronous_full = true; +} + +std::string RandomSessionId() { + unsigned char bytes[16]; + sqlite3_randomness(sizeof(bytes), bytes); + char output[33]; + for (size_t index = 0; index < sizeof(bytes); ++index) { + std::snprintf(output + index * 2, 3, "%02x", bytes[index]); + } + output[32] = '\0'; + const long long process_id = static_cast( +#if defined(_WIN32) + _getpid() +#else + getpid() +#endif + ); + return std::to_string(process_id) + "-" + output; +} + +} // namespace + +extern "C" vexfs_mount_status vexfs_mount_session_open(const vexfs_mount_config *config, + vexfs_mount_session **output, + vexfs_mount_error *error) { + if (output != nullptr) *output = nullptr; + return Guard(error, [&] { + if (config == nullptr || output == nullptr || config->database_path == nullptr || + config->workspace == nullptr) { + throw CallError(SQLITE_MISUSE, "config, output, database_path and workspace are required"); + } + if (config->abi_version != VEXFS_MOUNT_ABI_VERSION) { + throw CallError(SQLITE_MISUSE, "unsupported mount ABI version"); + } + vexfs_mount_session *session = new vexfs_mount_session(); + session->workspace = config->workspace; + session->session_id = RandomSessionId(); + session->exclusive_gateway = (config->flags & VEXFS_MOUNT_EXCLUSIVE_GATEWAY) != 0; + const bool no_create = (config->flags & VEXFS_MOUNT_OPEN_NO_CREATE) != 0; + PreparePrivateDatabase(config->database_path, no_create); + int rc = sqlite3_open_v2(config->database_path, &session->db, + (no_create ? SQLITE_OPEN_READWRITE : SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE) | + SQLITE_OPEN_FULLMUTEX, + nullptr); + if (rc != SQLITE_OK) { + const std::string message = session->db == nullptr ? "cannot open database" : + sqlite3_errmsg(session->db); + if (session->db != nullptr) sqlite3_close(session->db); + delete session; + throw CallError(rc, message); + } + try { + sqlite3_busy_timeout(session->db, + static_cast(config->busy_timeout_ms == 0 ? 5000 : config->busy_timeout_ms)); + rc = vexdb_sqlite_register(session->db); + if (rc != SQLITE_OK) throw CallError(rc, sqlite3_errmsg(session->db)); + if (no_create) { + // doctor 必须保持只读，不初始化、不迁移，也不在关闭时执行 session_end。 + Exec(session->db, "PRAGMA foreign_keys=ON;"); + } else { + Exec(session->db, + "PRAGMA journal_mode=WAL; PRAGMA synchronous=FULL; PRAGMA foreign_keys=ON;"); + ProtectSidecar(std::string(config->database_path) + "-wal"); + ProtectSidecar(std::string(config->database_path) + "-shm"); + Call init(session->db, "SELECT vexfs_init()"); + init.Row(); + Call workspace(session->db, "SELECT vexfs_workspace_create(?1)"); + workspace.Text(1, config->workspace); + workspace.Row(); + ProtectSidecar(std::string(config->database_path) + "-wal"); + ProtectSidecar(std::string(config->database_path) + "-shm"); + if (session->exclusive_gateway) { + CheckExistingLocalSession(session->db, config->workspace, + session->session_id); + Call start(session->db, "SELECT vexfs_mount_session_start(?1,?2)"); + start.Text(1, config->workspace); + start.Text(2, session->session_id.c_str()); + start.Row(); + session->session_started = true; + } + } + } catch (...) { + sqlite3_close(session->db); + delete session; + throw; + } + *output = session; + }); +} + +extern "C" void vexfs_mount_session_close(vexfs_mount_session *session) { + if (session == nullptr) return; + if (session->db != nullptr) { + std::lock_guard lock(session->mutex); + try { + if (session->session_started) { + UseFullDurability(session); + Call end(session->db, "SELECT vexfs_mount_session_end(?1,?2)"); + end.Text(1, session->workspace.c_str()); + end.Text(2, session->session_id.c_str()); + end.Row(); + session->session_started = false; + } + } catch (...) { + // close 不能抛异常；未发布内容仍留在数据库中，后续可诊断和回收。 + } + sqlite3_close(session->db); + } + delete session; +} + +extern "C" vexfs_mount_status vexfs_mount_diagnostics(vexfs_mount_session *session, + vexfs_mount_bytes *json, + vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + // doctor 以只读方式打开数据库，不能为了诊断偷偷迁移。这里读取数据库自身 + // 的合同版本，并兼容 0.1 staging 列布局。 + Call contract(session->db, + "SELECT value FROM _vexfs_meta WHERE key='contract_version'"); + contract.Row(); + Call journal(session->db, "PRAGMA journal_mode"); + journal.Row(); + Call synchronous(session->db, "PRAGMA synchronous"); + synchronous.Row(); + Call foreign_keys(session->db, "PRAGMA foreign_keys"); + foreign_keys.Row(); + Call workspace_exists(session->db, + "SELECT count(*) FROM _vexfs_workspaces WHERE name=?1"); + workspace_exists.Text(1, session->workspace.c_str()); + workspace_exists.Row(); + Call pending(session->db, + "SELECT count(*) FROM _vexfs_handles WHERE workspace_id=" + "(SELECT id FROM _vexfs_workspaces WHERE name=?1) " + "AND state IN ('open','retained') AND dirty_generation>published_generation"); + pending.Text(1, session->workspace.c_str()); + pending.Row(); + Call retained(session->db, + "SELECT count(*) FROM _vexfs_handles WHERE workspace_id=" + "(SELECT id FROM _vexfs_workspaces WHERE name=?1) AND state='retained'"); + retained.Text(1, session->workspace.c_str()); + retained.Row(); + const bool split_staging = sqlite3_table_column_metadata( + session->db, "main", "_vexfs_staging", "logical_size", + nullptr, nullptr, nullptr, nullptr, nullptr) == SQLITE_OK; + Call staging(session->db, split_staging + ? "SELECT COALESCE(sum(s.logical_size),0) FROM _vexfs_staging s " + "JOIN _vexfs_handles h ON h.id=s.handle_id WHERE h.workspace_id=" + "(SELECT id FROM _vexfs_workspaces WHERE name=?1)" + : "SELECT COALESCE(sum(length(s.content)),0) FROM _vexfs_staging s " + "JOIN _vexfs_handles h ON h.id=s.handle_id WHERE h.workspace_id=" + "(SELECT id FROM _vexfs_workspaces WHERE name=?1)"); + staging.Text(1, session->workspace.c_str()); + staging.Row(); + const char *filename = sqlite3_db_filename(session->db, "main"); + const std::string contract_version = contract.ResultText(); + const bool compatible = contract_version == "0.3.0"; + const bool upgrade_required = contract_version == "0.1.0" || contract_version == "0.2.0"; + const std::string value = + "{\"contract_version\":\"" + JsonEscape(contract_version) + + "\",\"compatible\":" + (compatible ? "true" : "false") + + ",\"upgrade_required\":" + (upgrade_required ? "true" : "false") + + ",\"database_path\":\"" + JsonEscape(filename == nullptr ? "" : filename) + + "\",\"workspace\":\"" + JsonEscape(session->workspace) + + "\",\"workspace_exists\":" + std::to_string(workspace_exists.ResultInt64()) + + ",\"journal_mode\":\"" + JsonEscape(journal.ResultText()) + + "\",\"synchronous\":" + std::to_string(synchronous.ResultInt64()) + + ",\"foreign_keys\":" + std::to_string(foreign_keys.ResultInt64()) + + ",\"pending_handles\":" + std::to_string(pending.ResultInt64()) + + ",\"retained_handles\":" + std::to_string(retained.ResultInt64()) + + ",\"staging_bytes\":" + std::to_string(staging.ResultInt64()) + "}"; + CopyString(value, json); + }); +} + +extern "C" vexfs_mount_status vexfs_mount_mkdir(vexfs_mount_session *session, + const char *path, + vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + UseFullDurability(session); + Call call(session->db, "SELECT vexfs_mkdir(?1,?2)"); + call.Text(1, session->workspace.c_str()); call.Text(2, path); call.Row(); + }); +} + +extern "C" vexfs_mount_status vexfs_mount_write_file(vexfs_mount_session *session, + const char *path, const void *data, + uint64_t size, int64_t *version, + vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + UseFullDurability(session); + if (version == nullptr) throw CallError(SQLITE_MISUSE, "version output is NULL"); + Call call(session->db, "SELECT vexfs_write(?1,?2,?3)"); + call.Text(1, session->workspace.c_str()); call.Text(2, path); call.Blob(3, data, size); + call.Row(); *version = call.ResultInt64(); + }); +} + +extern "C" vexfs_mount_status vexfs_mount_read_file(vexfs_mount_session *session, + const char *path, + vexfs_mount_bytes *content, + vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + Call call(session->db, "SELECT vexfs_read(?1,?2)"); + call.Text(1, session->workspace.c_str()); call.Text(2, path); call.Row(); + CopyResult(call, content); + }); +} + +extern "C" vexfs_mount_status vexfs_mount_history(vexfs_mount_session *session, + const char *path, + vexfs_mount_bytes *json, + vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + Call call(session->db, "SELECT vexfs_history(?1,?2)"); + call.Text(1, session->workspace.c_str()); + call.Text(2, path); + call.Row(); + CopyResult(call, json); + }); +} + +extern "C" vexfs_mount_status vexfs_mount_history_page( + vexfs_mount_session *session, const char *path, uint32_t limit, + int64_t before_version, vexfs_mount_bytes *json, vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + if (limit == 0 || limit > 1000 || before_version < 0) { + throw CallError(SQLITE_RANGE, "history limit must be 1..1000 and before must be non-negative"); + } + Call call(session->db, "SELECT vexfs_history(?1,?2,?3,?4)"); + call.Text(1, session->workspace.c_str()); + call.Text(2, path); + call.Int64(3, static_cast(limit)); + call.Int64(4, before_version); + call.Row(); + CopyResult(call, json); + }); +} + +extern "C" vexfs_mount_status vexfs_mount_read_version(vexfs_mount_session *session, + const char *path, int64_t version, + vexfs_mount_bytes *content, + vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + if (version <= 0) throw CallError(SQLITE_MISUSE, "version must be a positive integer"); + Call call(session->db, "SELECT vexfs_read_version(?1,?2,?3)"); + call.Text(1, session->workspace.c_str()); + call.Text(2, path); + call.Int64(3, version); + call.Row(); + CopyResult(call, content); + }); +} + +extern "C" vexfs_mount_status vexfs_mount_compare_versions( + vexfs_mount_session *session, const char *path, int64_t from_version, + int64_t to_version, vexfs_mount_bytes *json, vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + if (from_version <= 0 || to_version <= 0) { + throw CallError(SQLITE_RANGE, "versions must be positive integers"); + } + Call call(session->db, "SELECT vexfs_compare_versions(?1,?2,?3,?4)"); + call.Text(1, session->workspace.c_str()); + call.Text(2, path); + call.Int64(3, from_version); + call.Int64(4, to_version); + call.Row(); + CopyResult(call, json); + }); +} + +extern "C" vexfs_mount_status vexfs_mount_restore_version( + vexfs_mount_session *session, const char *path, int64_t target_version, + int64_t expected_version, int64_t *new_version, vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + UseFullDurability(session); + if (target_version <= 0 || expected_version <= 0 || new_version == nullptr) { + throw CallError(SQLITE_MISUSE, + "target version, expected version and new version output are required"); + } + Call call(session->db, "SELECT vexfs_restore_version(?1,?2,?3,?4)"); + call.Text(1, session->workspace.c_str()); + call.Text(2, path); + call.Int64(3, target_version); + call.Int64(4, expected_version); + call.Row(); + *new_version = call.ResultInt64(); + }); +} + +extern "C" vexfs_mount_status vexfs_mount_stat(vexfs_mount_session *session, + const char *path, vexfs_mount_bytes *json, + vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + Call call(session->db, "SELECT vexfs_stat(?1,?2)"); + call.Text(1, session->workspace.c_str()); call.Text(2, path); call.Row(); CopyResult(call, json); + }); +} + +extern "C" vexfs_mount_status vexfs_mount_path_for_inode(vexfs_mount_session *session, + int64_t inode, + vexfs_mount_bytes *path, + vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + Call call(session->db, "SELECT vexfs_path(?1,?2)"); + call.Text(1, session->workspace.c_str()); + call.Int64(2, inode); + call.Row(); + CopyResult(call, path); + }); +} + +extern "C" vexfs_mount_status vexfs_mount_list(vexfs_mount_session *session, + const char *path, vexfs_mount_bytes *json, + vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + Call call(session->db, "SELECT vexfs_list(?1,?2)"); + call.Text(1, session->workspace.c_str()); call.Text(2, path); call.Row(); CopyResult(call, json); + }); +} + +extern "C" vexfs_mount_status vexfs_mount_move(vexfs_mount_session *session, + const char *source, const char *destination, + vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + UseFullDurability(session); + Call call(session->db, "SELECT vexfs_move(?1,?2,?3)"); + call.Text(1, session->workspace.c_str()); call.Text(2, source); call.Text(3, destination); call.Row(); + }); +} + +extern "C" vexfs_mount_status vexfs_mount_rename(vexfs_mount_session *session, + const char *source, + const char *destination, int replace, + vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + UseFullDurability(session); + Call call(session->db, "SELECT vexfs_rename(?1,?2,?3,?4)"); + call.Text(1, session->workspace.c_str()); + call.Text(2, source); + call.Text(3, destination); + call.Int(4, replace); + call.Row(); + }); +} + +extern "C" vexfs_mount_status vexfs_mount_remove(vexfs_mount_session *session, + const char *path, int recursive, + vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + UseFullDurability(session); + Call call(session->db, "SELECT vexfs_remove(?1,?2,?3)"); + call.Text(1, session->workspace.c_str()); call.Text(2, path); call.Int(3, recursive); call.Row(); + }); +} + +extern "C" vexfs_mount_status vexfs_mount_handle_open(vexfs_mount_session *session, + const char *path, const char *flags, + const char *request_id, + vexfs_mount_bytes *handle, + vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + UseFullDurability(session); + Call call(session->db, "SELECT vexfs_handle_open(?1,?2,?3,?4,?5)"); + call.Text(1, session->workspace.c_str()); call.Text(2, path); call.Text(3, flags); + call.Text(4, request_id); call.Text(5, session->session_id.c_str()); + call.Row(); CopyResult(call, handle); + }); +} + +extern "C" vexfs_mount_status vexfs_mount_handle_truncate( + vexfs_mount_session *session, const char *handle, uint64_t size, + const char *request_id, int64_t *generation, vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + if (size > static_cast(std::numeric_limits::max()) || + generation == nullptr) { + throw CallError(SQLITE_MISUSE, "size is too large or generation output is NULL"); + } + UseStagingDurability(session); + Call call(session->db, "SELECT vexfs_handle_truncate(?1,?2,?3)"); + call.Text(1, handle); + call.Int64(2, static_cast(size)); + call.Text(3, request_id); + call.Row(); + *generation = call.ResultInt64(); + }); +} + +extern "C" vexfs_mount_status vexfs_mount_handle_stage_write( + vexfs_mount_session *session, const char *handle, uint64_t offset, const void *data, + uint64_t size, const char *request_id, int64_t *generation, vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + if (offset > static_cast(std::numeric_limits::max()) || generation == nullptr) { + throw CallError(SQLITE_MISUSE, "offset is too large or generation output is NULL"); + } + UseStagingDurability(session); + Call call(session->db, "SELECT vexfs_handle_stage_write(?1,?2,?3,?4)"); + call.Text(1, handle); + call.Int64(2, static_cast(offset)); + call.Blob(3, data, size); + call.Text(4, request_id); + call.Row(); + *generation = call.ResultInt64(); + }); +} + +extern "C" vexfs_mount_status vexfs_mount_handle_read(vexfs_mount_session *session, + const char *handle, uint64_t offset, + uint64_t length, + vexfs_mount_bytes *content, + vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + if (offset > static_cast(std::numeric_limits::max()) || + length > static_cast(std::numeric_limits::max())) { + throw CallError(SQLITE_MISUSE, "read range is too large"); + } + Call call(session->db, "SELECT vexfs_handle_read(?1,?2,?3)"); + call.Text(1, handle); call.Int64(2, static_cast(offset)); + call.Int64(3, static_cast(length)); call.Row(); CopyResult(call, content); + }); +} + +extern "C" vexfs_mount_status vexfs_mount_handle_publish( + vexfs_mount_session *session, const char *handle, int64_t generation, + const char *durability, const char *request_id, int64_t *version, + vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + UseFullDurability(session); + if (version == nullptr) throw CallError(SQLITE_MISUSE, "version output is NULL"); + Call call(session->db, "SELECT vexfs_handle_publish(?1,?2,?3,?4)"); + call.Text(1, handle); call.Int64(2, generation); call.Text(3, durability); + call.Text(4, request_id); call.Row(); *version = call.ResultInt64(); + }); +} + +extern "C" vexfs_mount_status vexfs_mount_handle_close( + vexfs_mount_session *session, const char *handle, int retain_unpublished, + const char *request_id, vexfs_mount_bytes *state, vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + UseFullDurability(session); + Call call(session->db, "SELECT vexfs_handle_close(?1,?2,?3)"); + call.Text(1, handle); call.Int(2, retain_unpublished); call.Text(3, request_id); + call.Row(); CopyResult(call, state); + }); +} + +extern "C" vexfs_mount_status vexfs_mount_synchronize(vexfs_mount_session *session, + const char *request_id, + int64_t *published, + vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + UseFullDurability(session); + if (published == nullptr) throw CallError(SQLITE_MISUSE, "published output is NULL"); + Call call(session->db, "SELECT vexfs_mount_synchronize(?1,?2,?3)"); + call.Text(1, session->workspace.c_str()); call.Text(2, request_id); + call.Text(3, session->session_id.c_str()); call.Row(); + *published = call.ResultInt64(); + }); +} + +extern "C" vexfs_mount_status vexfs_mount_reclaim(vexfs_mount_session *session, + const char *request_id, + int64_t *reclaimed, + vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + UseFullDurability(session); + if (reclaimed == nullptr) throw CallError(SQLITE_MISUSE, "reclaimed output is NULL"); + Call call(session->db, "SELECT vexfs_item_reclaim(?1,?2)"); + call.Text(1, session->workspace.c_str()); call.Text(2, request_id); call.Row(); + *reclaimed = call.ResultInt64(); + }); +} + +extern "C" void vexfs_mount_free(void *memory) { + std::free(memory); +} diff --git a/agent_files/mount/common/test/vexfs_mount_contract_smoke.cpp b/agent_files/mount/common/test/vexfs_mount_contract_smoke.cpp new file mode 100644 index 0000000000..20eb595a20 --- /dev/null +++ b/agent_files/mount/common/test/vexfs_mount_contract_smoke.cpp @@ -0,0 +1,228 @@ +#include "vexfs_mount_contract.h" + +#include +#include +#include +#include +#include + +namespace { + +int Fail(const char *operation, const vexfs_mount_error &error) { + std::fprintf(stderr, "VEXFS MOUNT CONTRACT FAIL: %s: status=%d sqlite=%d %s\n", + operation, error.status, error.sqlite_code, error.message); + return 1; +} + +bool Equals(const vexfs_mount_bytes &bytes, const char *expected) { + return bytes.size == std::strlen(expected) && + std::memcmp(bytes.data, expected, static_cast(bytes.size)) == 0; +} + +bool Contains(const vexfs_mount_bytes &bytes, const char *expected) { + if (bytes.data == nullptr) return false; + const std::string value(static_cast(bytes.data), static_cast(bytes.size)); + return value.find(expected) != std::string::npos; +} + +} // namespace + +int main() { + char path[] = "/tmp/vexfs-contract-XXXXXX"; + const int fd = mkstemp(path); + if (fd < 0) return 1; + close(fd); + unlink(path); + + vexfs_mount_config config{}; + config.abi_version = VEXFS_MOUNT_ABI_VERSION; + config.database_path = path; + config.workspace = "default"; + config.busy_timeout_ms = 1000; + vexfs_mount_error error{}; + vexfs_mount_session *session = nullptr; + if (vexfs_mount_session_open(&config, &session, &error) != VEXFS_MOUNT_OK) + return Fail("open", error); + + vexfs_mount_bytes diagnostics{}; + if (vexfs_mount_diagnostics(session, &diagnostics, &error) != VEXFS_MOUNT_OK) + return Fail("diagnostics", error); + if (!Contains(diagnostics, "\"contract_version\":\"0.3.0\"") || + !Contains(diagnostics, "\"workspace_exists\":1")) + return Fail("diagnostics content", error); + vexfs_mount_free(diagnostics.data); + + if (vexfs_mount_mkdir(session, "/agent", &error) != VEXFS_MOUNT_OK) + return Fail("mkdir", error); + int64_t version = 0; + if (vexfs_mount_write_file(session, "/agent/task.txt", "draft", 5, &version, &error) != + VEXFS_MOUNT_OK) return Fail("write", error); + + if (vexfs_mount_write_file(session, "/agent/versioned.txt", "old", 3, &version, &error) != + VEXFS_MOUNT_OK || version != 1) + return Fail("version seed", error); + if (vexfs_mount_write_file(session, "/agent/versioned.txt", "new", 3, &version, &error) != + VEXFS_MOUNT_OK || version != 2) + return Fail("version update", error); + vexfs_mount_bytes history{}; + if (vexfs_mount_history(session, "/agent/versioned.txt", &history, &error) != VEXFS_MOUNT_OK || + !Contains(history, "\"version\":2") || !Contains(history, "\"version\":1")) + return Fail("version history", error); + vexfs_mount_free(history.data); + vexfs_mount_bytes historical{}; + if (vexfs_mount_read_version(session, "/agent/versioned.txt", 1, &historical, &error) != + VEXFS_MOUNT_OK || !Equals(historical, "old")) + return Fail("read historical version", error); + vexfs_mount_free(historical.data); + int64_t restored = 0; + if (vexfs_mount_restore_version(session, "/agent/versioned.txt", 1, 2, &restored, &error) != + VEXFS_MOUNT_OK || restored != 3) + return Fail("restore version", error); + vexfs_mount_bytes restored_content{}; + if (vexfs_mount_read_file(session, "/agent/versioned.txt", &restored_content, &error) != + VEXFS_MOUNT_OK || !Equals(restored_content, "old")) + return Fail("restored content", error); + vexfs_mount_free(restored_content.data); + if (vexfs_mount_restore_version(session, "/agent/versioned.txt", 2, 2, &restored, &error) != + VEXFS_MOUNT_CONFLICT) return Fail("stale restore must conflict", error); + + vexfs_mount_bytes handle{}; + if (vexfs_mount_handle_open(session, "/agent/task.txt", "rw", "open-cabi-1", + &handle, &error) != VEXFS_MOUNT_OK) + return Fail("handle open", error); + std::string handle_id(static_cast(handle.data), static_cast(handle.size)); + vexfs_mount_free(handle.data); + + int64_t generation = 0; + if (vexfs_mount_handle_stage_write(session, handle_id.c_str(), 5, " ready", 6, + "write-cabi-1", &generation, &error) != VEXFS_MOUNT_OK) + return Fail("stage write", error); + if (vexfs_mount_handle_publish(session, handle_id.c_str(), generation, "data", + "publish-cabi-1", &version, &error) != VEXFS_MOUNT_OK) + return Fail("publish", error); + vexfs_mount_bytes state{}; + if (vexfs_mount_handle_close(session, handle_id.c_str(), 1, "close-cabi-1", &state, + &error) != VEXFS_MOUNT_OK) + return Fail("handle close", error); + vexfs_mount_free(state.data); + + // 多次 write 只进入 staging；synchronize 一次发布最终 generation。 + if (vexfs_mount_write_file(session, "/agent/sync.txt", "A", 1, &version, &error) != + VEXFS_MOUNT_OK) return Fail("sync seed", error); + handle = {}; + if (vexfs_mount_handle_open(session, "/agent/sync.txt", "rw", "open-sync-1", + &handle, &error) != VEXFS_MOUNT_OK) + return Fail("sync handle open", error); + handle_id.assign(static_cast(handle.data), static_cast(handle.size)); + vexfs_mount_free(handle.data); + if (vexfs_mount_handle_stage_write(session, handle_id.c_str(), 1, "B", 1, + "write-sync-1", &generation, &error) != VEXFS_MOUNT_OK) + return Fail("sync stage 1", error); + if (vexfs_mount_handle_stage_write(session, handle_id.c_str(), 2, "C", 1, + "write-sync-2", &generation, &error) != VEXFS_MOUNT_OK) + return Fail("sync stage 2", error); + vexfs_mount_bytes content{}; + if (vexfs_mount_read_file(session, "/agent/sync.txt", &content, &error) != VEXFS_MOUNT_OK) + return Fail("read before synchronize", error); + if (!Equals(content, "A")) return Fail("staging must be invisible before synchronize", error); + vexfs_mount_free(content.data); + diagnostics = {}; + if (vexfs_mount_diagnostics(session, &diagnostics, &error) != VEXFS_MOUNT_OK || + !Contains(diagnostics, "\"pending_handles\":1")) + return Fail("pending diagnostics", error); + vexfs_mount_free(diagnostics.data); + int64_t published = 0; + if (vexfs_mount_synchronize(session, "sync-cabi-1", &published, &error) != VEXFS_MOUNT_OK || + published != 1) return Fail("synchronize", error); + content = {}; + if (vexfs_mount_read_file(session, "/agent/sync.txt", &content, &error) != VEXFS_MOUNT_OK) + return Fail("read after synchronize", error); + if (!Equals(content, "ABC")) return Fail("synchronized content", error); + vexfs_mount_free(content.data); + state = {}; + if (vexfs_mount_handle_close(session, handle_id.c_str(), 1, "close-sync-1", &state, + &error) != VEXFS_MOUNT_OK) + return Fail("sync handle close", error); + vexfs_mount_free(state.data); + vexfs_mount_session_close(session); + + // 重开真实数据库文件，确认不是进程内假状态。 + session = nullptr; + config.flags = VEXFS_MOUNT_OPEN_NO_CREATE; + if (vexfs_mount_session_open(&config, &session, &error) != VEXFS_MOUNT_OK) + return Fail("reopen", error); + content = {}; + if (vexfs_mount_read_file(session, "/agent/task.txt", &content, &error) != VEXFS_MOUNT_OK) + return Fail("persistent read", error); + if (!Equals(content, "draft ready")) return Fail("persistent content", error); + vexfs_mount_free(content.data); + content = {}; + if (vexfs_mount_read_file(session, "/agent/sync.txt", &content, &error) != VEXFS_MOUNT_OK || + !Equals(content, "ABC")) return Fail("persistent synchronized content", error); + vexfs_mount_free(content.data); + vexfs_mount_session_close(session); + + // 独占挂载进程崩溃后，新 session 只保留旧 staging，不能自动发布半成品。 + config.flags = VEXFS_MOUNT_EXCLUSIVE_GATEWAY; + session = nullptr; + if (vexfs_mount_session_open(&config, &session, &error) != VEXFS_MOUNT_OK) + return Fail("crash seed open", error); + if (vexfs_mount_write_file(session, "/agent/crash.txt", "old", 3, &version, &error) != + VEXFS_MOUNT_OK) return Fail("crash seed write", error); + vexfs_mount_session *contender = nullptr; + if (vexfs_mount_session_open(&config, &contender, &error) != VEXFS_MOUNT_BUSY) + return Fail("exclusive gateway must reject a live contender", error); + vexfs_mount_session_close(session); + + const pid_t child = fork(); + if (child < 0) return 1; + if (child == 0) { + vexfs_mount_session *crashed = nullptr; + vexfs_mount_error child_error{}; + if (vexfs_mount_session_open(&config, &crashed, &child_error) != VEXFS_MOUNT_OK) + _exit(2); + vexfs_mount_bytes child_handle{}; + if (vexfs_mount_handle_open(crashed, "/agent/crash.txt", "rw", "crash-open-1", + &child_handle, &child_error) != VEXFS_MOUNT_OK) + _exit(3); + std::string child_id(static_cast(child_handle.data), + static_cast(child_handle.size)); + int64_t child_generation = 0; + if (vexfs_mount_handle_stage_write(crashed, child_id.c_str(), 0, "NEW", 3, + "crash-write-1", &child_generation, + &child_error) != VEXFS_MOUNT_OK) + _exit(4); + _exit(0); + } + int child_status = 0; + if (waitpid(child, &child_status, 0) != child || !WIFEXITED(child_status) || + WEXITSTATUS(child_status) != 0) return 1; + + session = nullptr; + if (vexfs_mount_session_open(&config, &session, &error) != VEXFS_MOUNT_OK) + return Fail("crash recovery open", error); + published = -1; + if (vexfs_mount_synchronize(session, "crash-sync-1", &published, &error) != VEXFS_MOUNT_OK || + published != 1) { + std::fprintf(stderr, "crash synchronize published=%lld\n", (long long)published); + return Fail("crash staging recovery publish", error); + } + content = {}; + if (vexfs_mount_read_file(session, "/agent/crash.txt", &content, &error) != VEXFS_MOUNT_OK || + !Equals(content, "NEW")) return Fail("crash content recovered", error); + vexfs_mount_free(content.data); + diagnostics = {}; + if (vexfs_mount_diagnostics(session, &diagnostics, &error) != VEXFS_MOUNT_OK || + !Contains(diagnostics, "\"retained_handles\":0")) + return Fail("crash staging claimed", error); + vexfs_mount_free(diagnostics.data); + vexfs_mount_session_close(session); + + unlink(path); + std::string wal = std::string(path) + "-wal"; + std::string shm = std::string(path) + "-shm"; + unlink(wal.c_str()); + unlink(shm.c_str()); + std::puts("VEXFS MOUNT CONTRACT SMOKE: PASS"); + return 0; +} diff --git a/build_sqlite.sh b/build_sqlite.sh index df6a10246b..93727173f3 100755 --- a/build_sqlite.sh +++ b/build_sqlite.sh @@ -3,7 +3,8 @@ # 仿 vexdb_pg 独立 CMake 模式（根 build.sh 深耦合 DuckDB 源码树，不复用）。 # # bash build_sqlite.sh build # 配置 + 编译双形态 + smoke test 二进制 -# bash build_sqlite.sh test # build + 跑五层测试（smoke ×4 + spec） +# bash build_sqlite.sh test # build + 跑 SQLite/VexFS smoke + spec +# bash build_sqlite.sh eval # build + 跑 VexFS full eval（可传 quick/full/stress） # bash build_sqlite.sh package # Release 编 macOS arm64+x86_64 → dist/ tarball + SHA256 # bash build_sqlite.sh ios # Stage C：device+sim 静态库 → XCFramework（+sim smoke） # bash build_sqlite.sh android # Stage C：NDK arm64-v8a/x86_64 静态库 + loadable .so @@ -34,7 +35,7 @@ case "$CMD" in vendor) bash "$DIR/vendor_sqlite.sh" ;; - build|test) + build|test|eval) [ -f "$DIR/third_party/sqlite/sqlite3ext.h" ] || bash "$DIR/vendor_sqlite.sh" "$CMAKE" -S "$DIR" -B "$BUILD_DIR" -DCMAKE_BUILD_TYPE="${BUILD_TYPE:-Debug}" "$CMAKE" --build "$BUILD_DIR" -j "$NCPU" @@ -51,8 +52,26 @@ case "$CMD" in "$BUILD_DIR/m2_vtab_smoke" echo "=== M3 HNSW 冒烟 ===" "$BUILD_DIR/m3_hnsw_smoke" + echo "=== M3P 并行查询冒烟 ===" + "$BUILD_DIR/m3p_parallel_smoke" + echo "=== VexFS SQLite 合同冒烟 ===" + "$BUILD_DIR/vexfs_static_smoke" + echo "=== VexFS 挂载层 C ABI 冒烟 ===" + "$BUILD_DIR/vexfs_mount_contract_smoke" + echo "=== VexFS CLI 冒烟 ===" + bash "$DIR/../agent_files/cli/test/vexfs_cli_smoke.sh" "$BUILD_DIR/vexfs" echo "=== M4 spec（L2 DSL 渲染 + runner） ===" bash "$DIR/../tests/spec/_lib/docker/run_sqlite.sh" + elif [ "$CMD" = "eval" ]; then + EVAL_MODE="${2:-full}" + case "$EVAL_MODE" in + quick|full|stress) ;; + *) echo "eval mode 只能是 quick、full 或 stress" >&2; exit 2 ;; + esac + echo "" + echo "=== VexFS $EVAL_MODE eval ===" + python3 "$DIR/../tests/eval/vexfs/run.py" \ + --mode "$EVAL_MODE" --build-dir "$BUILD_DIR" fi ;; package) @@ -169,7 +188,7 @@ case "$CMD" in echo "已清理 build 目录" ;; *) - echo "Usage: $0 [build|test|package|ios|android|vendor|clean]" + echo "Usage: $0 [build|test|eval [quick|full|stress]|package|ios|android|vendor|clean]" exit 1 ;; esac diff --git a/tests/eval/vexfs/README.md b/tests/eval/vexfs/README.md new file mode 100644 index 0000000000..7262bfe029 --- /dev/null +++ b/tests/eval/vexfs/README.md @@ -0,0 +1,64 @@ +# VexFS Eval + +这套 eval 使用真实构建产物和磁盘 SQLite 数据库，不使用 mock。每个用例独立记录 +结果、耗时、断言数和性能指标，失败后仍继续执行其他用例。 + +## 运行 + +```bash +# 默认 full：构建后覆盖功能、恢复、并发、备份、性能和 FSKit 编译 +bash build_sqlite.sh eval + +# 日常快速回归 +bash build_sqlite.sh eval quick + +# 最大规模（128 MiB 文件、1 万随机操作、1 万小文件） +bash build_sqlite.sh eval stress + +# 只跑某一类或某个用例 +python3 tests/eval/vexfs/run.py --mode full --filter recovery +python3 tests/eval/vexfs/run.py --mode full --filter performance.staged-overwrite + +# 把性能预算也作为硬 Gate；默认只记录指标，避免不同机器误报 +python3 tests/eval/vexfs/run.py --mode full --enforce-performance +``` + +报告生成在： + +```text +vexdb_sqlite/build/eval/vexfs//report.json +vexdb_sqlite/build/eval/vexfs//report.md +vexdb_sqlite/build/eval/vexfs/latest.json +``` + +随机状态机使用固定 seed，可通过 `--seed` 复现或扩充样本。 + +## 覆盖范围 + +- 现有 Gate：SQL 静态注册、C ABI、CLI smoke、全部 SQLite YAML spec。 +- 文件合同：二进制/Unicode、元数据、路径校验、rename replace、递归删除、公开版本历史和指定版本读取。 +- 版本恢复：SQL/C ABI/CLI restore、expected-version 冲突、事务回滚、dry-run 不写库、文本 diff 和末尾换行差异。 +- 事务：commit、rollback、savepoint、读者快照、目录 verifier。 +- 句柄：flags、稀疏写、truncate、幂等指纹、同步、乐观写冲突。 +- 恢复：进程中途退出、WAL 重开、integrity check、旧 schema 迁移、retained reclaim。 +- 并发：多个真实进程写入、数据库锁和 busy timeout。 +- 备份：运行中的 SQLite online backup、恢复、未发布 staging 可见性。 +- 随机模型：真实数据库状态与 Python 参考文件树持续比对。 +- 性能：大量小文件、大文件顺序读写、分块 staging、随机 4 KiB 覆盖、备份吞吐。 +- 空间：版本历史增长、幂等请求行数、checkpoint/VACUUM 前后的 DB/WAL 体积。 +- 平台：FSKit App/extension 无签名编译；扩展已启用时执行真实 mount 和 bash 命令。 + +## 模式规模 + +| 项目 | quick | full | stress | +|---|---:|---:|---:| +| 随机模型操作 | 300 | 2,000 | 10,000 | +| 小文件 | 250 | 3,000 | 10,000 | +| 顺序文件 | 8 MiB | 100 MiB | 128 MiB | +| staging 文件 | 8 MiB | 100 MiB | 128 MiB | +| 随机 4 KiB 写 | 100 | 1,000 | 5,000 | +| 并发进程 | 4 | 8 | 8 | + +`mount.real-bash` 是环境 Gate。macOS、FSKit 和已启用的 VexFS 系统扩展都满足时， +它会真实挂载并运行 `mkdir`、`cat`、`grep`、`cp`、`mv`、`find`、`rm` 等命令； +条件不满足时报告会明确写 `SKIP` 和原因，不会把未执行伪装成通过。 diff --git a/tests/eval/vexfs/run.py b/tests/eval/vexfs/run.py new file mode 100644 index 0000000000..54f9e1b9d0 --- /dev/null +++ b/tests/eval/vexfs/run.py @@ -0,0 +1,1902 @@ +#!/usr/bin/env python3 +"""VexFS 真实 eval：功能、恢复、并发、备份、性能、CLI 和 macOS mount。 + +每个 case 都使用真实的磁盘 SQLite 数据库和构建产物。默认产出 JSON 与 Markdown， +便于本地回归和 CI 保存；性能数字默认记录，--enforce-performance 才作为硬门槛。 +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import platform +import random +import resource +import shutil +import sqlite3 +import subprocess +import sys +import tempfile +import time +import traceback +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Callable + + +MIB = 1024 * 1024 +MAX_FILE_BYTES = 128 * MIB +CONTRACT_VERSION = "0.3.0" + + +class EvalFailure(AssertionError): + pass + + +class EvalSkip(RuntimeError): + pass + + +@dataclass(frozen=True) +class Mode: + name: str + random_ops: int + small_files: int + sequential_mib: int + staged_mib: int + random_writes: int + parallel_workers: int + files_per_worker: int + include_xcode: bool + + +MODES = { + "quick": Mode("quick", 300, 250, 8, 8, 100, 4, 20, False), + "full": Mode("full", 2_000, 3_000, 100, 100, 1_000, 8, 100, True), + "stress": Mode("stress", 10_000, 10_000, 128, 128, 5_000, 8, 500, True), +} + + +@dataclass +class EvalCase: + case_id: str + category: str + description: str + function: Callable[["Context"], dict[str, Any] | None] + modes: tuple[str, ...] = ("quick", "full", "stress") + + +CASES: list[EvalCase] = [] + + +def case(case_id: str, category: str, description: str, + modes: tuple[str, ...] = ("quick", "full", "stress")): + def register(function: Callable[["Context"], dict[str, Any] | None]): + CASES.append(EvalCase(case_id, category, description, function, modes)) + return function + return register + + +@dataclass +class Context: + root: Path + build_dir: Path + extension: Path + cli: Path + output_dir: Path + mode: Mode + seed: int + enforce_performance: bool + checks: int = 0 + current_case: str = "" + current_artifacts: Path | None = None + + def check(self, condition: bool, message: str) -> None: + self.checks += 1 + if not condition: + raise EvalFailure(message) + + def equal(self, actual: Any, expected: Any, message: str) -> None: + self.checks += 1 + if actual != expected: + raise EvalFailure(f"{message}: expected={expected!r}, actual={actual!r}") + + def expect_error(self, function: Callable[[], Any], contains: str) -> str: + self.checks += 1 + try: + function() + except (sqlite3.Error, RuntimeError) as error: + message = str(error) + if contains not in message: + raise EvalFailure( + f"错误信息不匹配: expected contains={contains!r}, actual={message!r}") + return message + raise EvalFailure(f"预期失败但成功: {contains}") + + def budget(self, metric: str, actual: float, maximum: float) -> None: + """性能预算只有显式开启才阻断；总是检查没有出现灾难性卡死。""" + self.check(actual >= 0, f"{metric} 不能为负数") + if self.enforce_performance: + self.check(actual <= maximum, + f"{metric} 超过预算: {actual:.3f} > {maximum:.3f}") + + +class Database: + def __init__(self, ctx: Context, path: Path | None = None, initialize: bool = True, + timeout: float = 15.0): + self.ctx = ctx + self.temp_dir: tempfile.TemporaryDirectory[str] | None = None + if path is None: + self.temp_dir = tempfile.TemporaryDirectory(prefix="vexfs-eval-") + path = Path(self.temp_dir.name) / "vexfs.sqlite3" + self.path = Path(path) + self.connection = sqlite3.connect( + str(self.path), isolation_level=None, timeout=timeout, check_same_thread=False) + self.connection.enable_load_extension(True) + self.connection.load_extension(str(ctx.extension)) + self.connection.execute(f"PRAGMA busy_timeout={int(timeout * 1000)}") + self.connection.execute("PRAGMA journal_mode=WAL") + self.connection.execute("PRAGMA synchronous=FULL") + self.connection.execute("PRAGMA foreign_keys=ON") + if initialize: + self.scalar("SELECT vexfs_init()") + self.scalar("SELECT vexfs_workspace_create('default')") + + def scalar(self, sql: str, parameters: tuple[Any, ...] = ()) -> Any: + row = self.connection.execute(sql, parameters).fetchone() + if row is None: + raise EvalFailure(f"SQL 没有返回行: {sql}") + return row[0] + + def json(self, sql: str, parameters: tuple[Any, ...] = ()) -> Any: + return json.loads(self.scalar(sql, parameters)) + + def close(self) -> None: + if self.connection is not None: + self.connection.close() + self.connection = None # type: ignore[assignment] + if self.temp_dir is not None: + self.temp_dir.cleanup() + self.temp_dir = None + + def __enter__(self) -> "Database": + return self + + def __exit__(self, *_: Any) -> None: + self.close() + + +def run_process(arguments: list[str], *, input_data: bytes | None = None, + timeout: float = 120.0, check: bool = True, + cwd: Path | None = None) -> subprocess.CompletedProcess[bytes]: + result = subprocess.run(arguments, input=input_data, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, timeout=timeout, cwd=cwd) + if check and result.returncode != 0: + raise EvalFailure( + f"命令失败 ({result.returncode}): {' '.join(arguments)}\n" + f"stdout:\n{result.stdout.decode(errors='replace')}\n" + f"stderr:\n{result.stderr.decode(errors='replace')}") + return result + + +def extension_path(build_dir: Path) -> Path: + for name in ("vexdb_lite.dylib", "vexdb_lite.so"): + candidate = build_dir / name + if candidate.exists(): + return candidate.resolve() + raise SystemExit(f"找不到 SQLite 扩展，请先构建: {build_dir}") + + +def db_storage(path: Path) -> dict[str, int]: + sizes: dict[str, int] = {} + for label, candidate in ( + ("main", path), + ("wal", Path(str(path) + "-wal")), + ("shm", Path(str(path) + "-shm")), + ): + sizes[f"{label}_bytes"] = candidate.stat().st_size if candidate.exists() else 0 + sizes["total_bytes"] = sum(sizes.values()) + return sizes + + +def sha256(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def child_max_rss_bytes() -> int: + value = int(resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss) + return value if platform.system() == "Darwin" else value * 1024 + + +@case("existing.sqlite-smoke", "existing-gates", "静态注册 VexFS SQL 合同冒烟") +def existing_sqlite_smoke(ctx: Context) -> dict[str, Any]: + binary = ctx.build_dir / "vexfs_static_smoke" + ctx.check(binary.exists(), f"缺少 {binary}") + result = run_process([str(binary)]) + ctx.check(b"PASS" in result.stdout, "静态 smoke 没有 PASS") + return {"stdout": result.stdout.decode().strip()} + + +@case("existing.cabi-smoke", "existing-gates", "真实磁盘 C ABI、同步和崩溃 staging 冒烟") +def existing_cabi_smoke(ctx: Context) -> dict[str, Any]: + binary = ctx.build_dir / "vexfs_mount_contract_smoke" + ctx.check(binary.exists(), f"缺少 {binary}") + result = run_process([str(binary)]) + ctx.check(b"PASS" in result.stdout, "C ABI smoke 没有 PASS") + return {"stdout": result.stdout.decode().strip()} + + +@case("existing.cli-smoke", "existing-gates", "CLI 现有端到端冒烟脚本") +def existing_cli_smoke(ctx: Context) -> dict[str, Any]: + script = ctx.root / "agent_files/cli/test/vexfs_cli_smoke.sh" + ctx.check(script.exists(), f"缺少 {script}") + result = run_process(["/bin/bash", str(script), str(ctx.cli)]) + ctx.check(b"PASS" in result.stdout, "CLI smoke 没有 PASS") + return {"stdout": result.stdout.decode().strip()} + + +@case("existing.sqlite-spec", "existing-gates", "渲染并执行全部 SQLite YAML spec") +def existing_sqlite_spec(ctx: Context) -> dict[str, Any]: + script = ctx.root / "tests/spec/_lib/docker/run_sqlite.sh" + ctx.check(script.exists(), f"缺少 {script}") + result = run_process(["/bin/bash", str(script)], timeout=180, cwd=ctx.root) + output = (result.stdout + result.stderr).decode(errors="replace") + ctx.check("0 failed" in output, "SQLite spec 存在失败") + return {"summary": output.strip().splitlines()[-1] if output.strip() else ""} + + +@case("contract.schema-workspaces", "functional", "合同版本、幂等初始化和 workspace 隔离") +def schema_workspaces(ctx: Context) -> dict[str, Any]: + with Database(ctx) as db: + ctx.equal(db.scalar("SELECT vexfs_contract_version()"), CONTRACT_VERSION, "合同版本") + ctx.equal(db.scalar("SELECT vexfs_init()"), 1, "重复初始化") + default_id = db.scalar("SELECT vexfs_workspace_create('default')") + ctx.equal(default_id, 1, "默认 workspace id") + other_id = db.scalar("SELECT vexfs_workspace_create('other')") + ctx.check(other_id != default_id, "workspace id 必须隔离") + db.scalar("SELECT vexfs_mkdir('default','/only-default')") + ctx.expect_error(lambda: db.scalar("SELECT vexfs_stat('other','/only-default')"), + "path not found") + ctx.expect_error(lambda: db.scalar("SELECT vexfs_workspace_create('')"), "1..128") + ctx.expect_error(lambda: db.scalar("SELECT vexfs_workspace_create(?)", ("x" * 129,)), + "1..128") + root = db.json("SELECT vexfs_stat('default','/')") + ctx.equal(root["kind"], "directory", "根节点类型") + ctx.check(root["version"] >= 1, "根目录 verifier 必须非零") + return {"default_workspace": default_id, "other_workspace": other_id, + "root_inode": root["inode"]} + + +@case("contract.binary-unicode-metadata", "functional", "二进制、Unicode 名称和元数据") +def binary_unicode_metadata(ctx: Context) -> dict[str, Any]: + payload = bytes(range(256)) + b"\x00\xff\x00agent\n" + path = "/资料/空 格/\"quote\"-🚀.bin" + with Database(ctx) as db: + db.scalar("SELECT vexfs_mkdir('default','/资料/空 格')") + version = db.scalar("SELECT vexfs_write('default',?,?)", (path, payload)) + ctx.equal(db.scalar("SELECT vexfs_read('default',?)", (path,)), payload, "二进制读取") + stat = db.json("SELECT vexfs_stat('default',?)", (path,)) + ctx.equal(stat["size"], len(payload), "文件大小") + ctx.equal(stat["version"], version, "版本号") + ctx.equal(stat["mode"], 0o644, "文件 mode") + ctx.check(stat["created_at"] > 0 and stat["updated_at"] > 0, "时间戳") + listing = db.json("SELECT vexfs_list('default','/资料/空 格')") + ctx.equal([entry["name"] for entry in listing], ['"quote"-🚀.bin'], "Unicode list") + ctx.equal(db.scalar("SELECT vexfs_path('default',?)", (stat["inode"],)), path, + "inode 反查路径") + return {"bytes": len(payload), "sha256": sha256(payload), "inode": stat["inode"]} + + +@case("contract.transactions", "transaction", "提交、回滚、savepoint 和读者快照") +def transactions(ctx: Context) -> dict[str, Any]: + with Database(ctx) as db: + db.scalar("SELECT vexfs_mkdir('default','/tx')") + db.scalar("SELECT vexfs_write('default','/tx/value','old')") + db.connection.execute("BEGIN") + db.scalar("SELECT vexfs_write('default','/tx/value','rollback')") + db.connection.execute("ROLLBACK") + ctx.equal(db.scalar("SELECT CAST(vexfs_read('default','/tx/value') AS TEXT)"), + "old", "事务回滚") + db.connection.execute("BEGIN") + db.connection.execute("SAVEPOINT nested") + db.scalar("SELECT vexfs_write('default','/tx/value','nested')") + db.connection.execute("ROLLBACK TO nested") + db.scalar("SELECT vexfs_write('default','/tx/value','committed')") + db.connection.execute("COMMIT") + ctx.equal(db.scalar("SELECT CAST(vexfs_read('default','/tx/value') AS TEXT)"), + "committed", "savepoint 后提交") + + reader = sqlite3.connect(str(db.path), isolation_level=None) + reader.enable_load_extension(True) + reader.load_extension(str(ctx.extension)) + reader.execute("BEGIN") + before = reader.execute( + "SELECT CAST(vexfs_read('default','/tx/value') AS TEXT)").fetchone()[0] + db.scalar("SELECT vexfs_write('default','/tx/value','newer')") + during = reader.execute( + "SELECT CAST(vexfs_read('default','/tx/value') AS TEXT)").fetchone()[0] + reader.execute("COMMIT") + after = reader.execute( + "SELECT CAST(vexfs_read('default','/tx/value') AS TEXT)").fetchone()[0] + reader.close() + ctx.equal((before, during, after), ("committed", "committed", "newer"), + "读者快照隔离") + return {"snapshot_values": [before, during, after]} + + +@case("contract.path-validation", "functional", "路径边界、根目录保护和错误信息") +def path_validation(ctx: Context) -> dict[str, Any]: + with Database(ctx) as db: + invalid = [ + ("relative", "path must be absolute"), + ("/a/./b", "must not contain"), + ("/a/../b", "must not contain"), + ("/" + "x" * 256, "longer than 255"), + ] + for path, message in invalid: + ctx.expect_error(lambda p=path: db.scalar( + "SELECT vexfs_mkdir('default',?)", (p,)), message) + ctx.expect_error(lambda: db.scalar("SELECT vexfs_remove('default','/',1)"), + "root cannot be removed") + ctx.expect_error(lambda: db.scalar("SELECT vexfs_move('default','/','/moved')"), + "root cannot be moved") + db.scalar("SELECT vexfs_mkdir('default','/a/b')") + ctx.expect_error(lambda: db.scalar( + "SELECT vexfs_move('default','/a','/a/b/inside')"), "moved into itself") + db.scalar("SELECT vexfs_write('default','/file','x')") + ctx.expect_error(lambda: db.scalar("SELECT vexfs_mkdir('default','/file/child')"), + "not a directory") + return {"invalid_paths": len(invalid), "root_guards": 2} + + +@case("contract.rename-remove", "functional", "移动、替换、目录循环和递归删除") +def rename_remove(ctx: Context) -> dict[str, Any]: + with Database(ctx) as db: + db.scalar("SELECT vexfs_mkdir('default','/src/child')") + db.scalar("SELECT vexfs_mkdir('default','/dst/nonempty')") + db.scalar("SELECT vexfs_write('default','/dst/nonempty/child','occupied')") + db.scalar("SELECT vexfs_write('default','/src/child/a','A')") + db.scalar("SELECT vexfs_write('default','/src/child/live','L')") + db.scalar("SELECT vexfs_write('default','/dst/file','D')") + inode = db.json("SELECT vexfs_stat('default','/src/child/live')")["inode"] + ctx.expect_error(lambda: db.scalar( + "SELECT vexfs_rename('default','/src/child/a','/dst/file',0)"), "already exists") + db.scalar("SELECT vexfs_rename('default','/src/child/a','/dst/file',1)") + ctx.equal(db.scalar("SELECT CAST(vexfs_read('default','/dst/file') AS TEXT)"), + "A", "replace 内容") + ctx.expect_error(lambda: db.scalar( + "SELECT vexfs_rename('default','/src/child','/dst/file',1)"), "types differ") + ctx.expect_error(lambda: db.scalar( + "SELECT vexfs_rename('default','/src/child','/dst/nonempty',1)"), + "destination directory is not empty") + db.scalar("SELECT vexfs_move('default','/src','/moved')") + ctx.equal(db.scalar("SELECT vexfs_path('default',?)", (inode,)), + "/moved/child/live", "祖先移动后的 inode 路径") + ctx.expect_error(lambda: db.scalar( + "SELECT vexfs_remove('default','/moved',0)"), "directory is not empty") + db.scalar("SELECT vexfs_remove('default','/moved',1)") + ctx.expect_error(lambda: db.scalar( + "SELECT vexfs_stat('default','/moved/child/live')"), "path not found") + return {"stable_inode": inode, "replace": True, "recursive_remove": True} + + +@case("contract.version-history", "transaction", "历史查询、指定版本读取和恢复为新版本") +def version_history(ctx: Context) -> dict[str, Any]: + with Database(ctx) as db: + versions = [] + for value in (b"one", b"two", b"three"): + versions.append(db.scalar("SELECT vexfs_write('default','/history',?)", (value,))) + ctx.equal(versions, [1, 2, 3], "文件版本单调增长") + inode = db.json("SELECT vexfs_stat('default','/history')")["inode"] + history = db.connection.execute( + "SELECT version_no,content,size FROM _vexfs_file_versions " + "WHERE inode_id=? ORDER BY version_no", (inode,)).fetchall() + ctx.equal(history, [(1, b"one", 3), (2, b"two", 3), (3, b"three", 5)], + "历史版本内容") + public_history = db.json("SELECT vexfs_history('default','/history')") + ctx.equal([row["version"] for row in public_history], [3, 2, 1], + "公开历史版本倒序") + ctx.equal([row["current"] for row in public_history], [True, False, False], + "公开历史当前标记") + ctx.equal(db.scalar("SELECT vexfs_read_version('default','/history',1)"), + b"one", "读取指定版本") + ctx.expect_error(lambda: db.scalar( + "SELECT vexfs_read_version('default','/history',99)"), "version not found") + commits = db.connection.execute( + "SELECT id,parent_commit,message FROM _vexfs_commits ORDER BY id").fetchall() + ctx.equal([row[1] for row in commits], [None, 1, 2], "commit 父链") + ctx.equal(db.scalar( + "SELECT head_commit FROM _vexfs_workspaces WHERE name='default'"), 3, + "workspace head") + restored = db.scalar( + "SELECT vexfs_restore_version('default','/history',1,3)") + ctx.equal(restored, 4, "恢复生成新版本") + ctx.equal(db.scalar("SELECT vexfs_read('default','/history')"), b"one", + "恢复后的当前内容") + ctx.expect_error(lambda: db.scalar( + "SELECT vexfs_restore_version('default','/history',2,3)"), "version conflict") + restored_history = db.json("SELECT vexfs_history('default','/history')") + ctx.equal([row["version"] for row in restored_history], [4, 3, 2, 1], + "恢复保留旧历史") + ctx.equal(restored_history[0]["message"], "restore version 1", "恢复 commit 消息") + + db.connection.execute("BEGIN") + ctx.equal(db.scalar( + "SELECT vexfs_restore_version('default','/history',2,4)"), 5, + "事务内恢复版本") + db.connection.execute("ROLLBACK") + ctx.equal(db.json("SELECT vexfs_stat('default','/history')")["version"], 4, + "回滚恢复不改变当前版本") + ctx.equal(len(db.json("SELECT vexfs_history('default','/history')")), 4, + "回滚恢复不留下历史") + return {"inode": inode, "versions": 4, "commits": 4, "restored": restored} + + +@case("contract.version-input-validation", "transaction", + "版本参数必须是精确整数，非法 restore 不得留下状态") +def version_input_validation(ctx: Context) -> dict[str, Any]: + with Database(ctx) as db: + db.scalar("SELECT vexfs_write('default','/strict-version','one')") + db.scalar("SELECT vexfs_write('default','/strict-version','two')") + before = db.connection.execute( + "SELECT (SELECT count(*) FROM _vexfs_file_versions)," + "(SELECT count(*) FROM _vexfs_commits)," + "(SELECT head_commit FROM _vexfs_workspaces WHERE name='default')" + ).fetchone() + for invalid in (None, 0, -1, 1.9, "1garbage"): + ctx.expect_error(lambda invalid=invalid: db.scalar( + "SELECT vexfs_read_version('default','/strict-version',?)", (invalid,)), + "positive integer") + ctx.expect_error(lambda invalid=invalid: db.scalar( + "SELECT vexfs_restore_version('default','/strict-version',?,2)", (invalid,)), + "positive integer") + ctx.expect_error(lambda: db.scalar( + "SELECT vexfs_restore_version('default','/strict-version',1,2.9)"), + "positive integer") + after = db.connection.execute( + "SELECT (SELECT count(*) FROM _vexfs_file_versions)," + "(SELECT count(*) FROM _vexfs_commits)," + "(SELECT head_commit FROM _vexfs_workspaces WHERE name='default')" + ).fetchone() + ctx.equal(after, before, "非法版本参数不改变历史和 head") + ctx.equal(db.scalar("SELECT vexfs_read('default','/strict-version')"), b"two", + "非法 restore 不改变内容") + return {"rejected": 11, "state": after} + + +@case("contract.open-unlink", "transaction", "打开文件删除后仍可发布并关闭 handle") +def open_unlink(ctx: Context) -> dict[str, Any]: + with Database(ctx) as db: + db.scalar("SELECT vexfs_write('default','/open-unlink','old')") + inode = db.json("SELECT vexfs_stat('default','/open-unlink')")["inode"] + handle = db.scalar( + "SELECT vexfs_handle_open('default','/open-unlink','rw','unlink-open')") + generation = db.scalar( + "SELECT vexfs_handle_stage_write(?,0,'NEW','unlink-write')", (handle,)) + db.scalar("SELECT vexfs_remove('default','/open-unlink',0)") + ctx.equal(db.scalar( + "SELECT vexfs_handle_publish(?,?,'full','unlink-publish')", (handle, generation)), + 2, "删除后的打开 handle 仍可发布") + ctx.equal(db.scalar("SELECT vexfs_handle_close(?,0,'unlink-close')", (handle,)), + "closed", "删除后的 handle 可关闭") + ctx.expect_error(lambda: db.scalar("SELECT vexfs_stat('default','/open-unlink')"), + "path not found") + ctx.equal(db.scalar( + "SELECT content FROM _vexfs_file_versions WHERE inode_id=? AND version_no=2", (inode,)), + b"NEW", "删除 inode 的已发布历史完整") + return {"inode": inode, "published_version": 2} + + +@case("contract.directory-verifier", "functional", "目录变更 verifier 单调增长") +def directory_verifier(ctx: Context) -> dict[str, Any]: + with Database(ctx) as db: + db.scalar("SELECT vexfs_mkdir('default','/left')") + db.scalar("SELECT vexfs_mkdir('default','/right')") + left_0 = db.json("SELECT vexfs_stat('default','/left')")["version"] + right_0 = db.json("SELECT vexfs_stat('default','/right')")["version"] + db.scalar("SELECT vexfs_write('default','/left/file','x')") + left_1 = db.json("SELECT vexfs_stat('default','/left')")["version"] + db.scalar("SELECT vexfs_move('default','/left/file','/right/file')") + left_2 = db.json("SELECT vexfs_stat('default','/left')")["version"] + right_1 = db.json("SELECT vexfs_stat('default','/right')")["version"] + db.scalar("SELECT vexfs_remove('default','/right/file',0)") + right_2 = db.json("SELECT vexfs_stat('default','/right')")["version"] + ctx.check(left_0 < left_1 < left_2, "源目录 verifier 必须增长") + ctx.check(right_0 < right_1 < right_2, "目标目录 verifier 必须增长") + return {"left": [left_0, left_1, left_2], "right": [right_0, right_1, right_2]} + + +@case("contract.handles", "handles", "句柄 flags、稀疏写、truncate、发布和关闭") +def handles(ctx: Context) -> dict[str, Any]: + with Database(ctx) as db: + db.scalar("SELECT vexfs_mkdir('default','/handles')") + ctx.expect_error(lambda: db.scalar( + "SELECT vexfs_handle_open('default','/handles/missing','r','open-missing')"), + "path not found") + handle = db.scalar( + "SELECT vexfs_handle_open('default','/handles/value','rwc','open-create')") + ctx.equal(len(handle), 32, "handle id 长度") + generation = db.scalar( + "SELECT vexfs_handle_stage_write(?,5,?,'write-sparse')", (handle, b"Z")) + ctx.equal(generation, 1, "首次 generation") + ctx.equal(db.scalar("SELECT vexfs_handle_read(?,0,7)", (handle,)), + b"\x00\x00\x00\x00\x00Z", "稀疏区域补零") + generation = db.scalar( + "SELECT vexfs_handle_truncate(?,3,'truncate-short')", (handle,)) + ctx.equal(generation, 2, "truncate generation") + generation = db.scalar( + "SELECT vexfs_handle_truncate(?,8,'truncate-grow')", (handle,)) + ctx.equal(generation, 3, "regrow generation") + ctx.equal(db.scalar("SELECT vexfs_handle_read(?,0,16)", (handle,)), b"\x00" * 8, + "truncate 后旧数据不能复活") + version = db.scalar( + "SELECT vexfs_handle_publish(?,3,'full','publish-handle')", (handle,)) + ctx.equal(version, 2, "发布版本") + state = db.scalar("SELECT vexfs_handle_close(?,1,'close-handle')", (handle,)) + ctx.equal(state, "closed", "已发布句柄关闭") + ctx.equal(db.scalar("SELECT vexfs_read('default','/handles/value')"), b"\x00" * 8, + "发布内容") + readonly = db.scalar( + "SELECT vexfs_handle_open('default','/handles/value','r','open-readonly')") + ctx.expect_error(lambda: db.scalar( + "SELECT vexfs_handle_stage_write(?,0,'X','write-readonly')", (readonly,)), + "read-only") + ctx.expect_error(lambda: db.scalar( + "SELECT vexfs_handle_open('default','/handles/value','ct','bad-flags')"), + "flags must contain") + return {"generation": generation, "version": version, "sparse_bytes": 8} + + +@case("contract.idempotency", "handles", "所有写请求的幂等重试和参数指纹") +def idempotency(ctx: Context) -> dict[str, Any]: + with Database(ctx) as db: + db.scalar("SELECT vexfs_write('default','/file','abc')") + handle = db.scalar("SELECT vexfs_handle_open('default','/file','rw','open-idem')") + ctx.equal(db.scalar("SELECT vexfs_handle_open('default','/file','rw','open-idem')"), + handle, "open 重试") + ctx.expect_error(lambda: db.scalar( + "SELECT vexfs_handle_open('default','/file','r','open-idem')"), + "different arguments") + write_sql = "SELECT vexfs_handle_stage_write(?,3,'d','write-idem')" + first = db.scalar(write_sql, (handle,)) + ctx.equal(db.scalar(write_sql, (handle,)), first, "write 重试") + ctx.expect_error(lambda: db.scalar( + "SELECT vexfs_handle_stage_write(?,2,'d','write-idem')", (handle,)), + "different arguments") + publish_sql = "SELECT vexfs_handle_publish(?,1,'data','publish-idem')" + version = db.scalar(publish_sql, (handle,)) + ctx.equal(db.scalar(publish_sql, (handle,)), version, "publish 重试") + ctx.expect_error(lambda: db.scalar( + "SELECT vexfs_handle_truncate(?,1,'publish-idem')", (handle,)), + "another operation") + close_sql = "SELECT vexfs_handle_close(?,0,'close-idem')" + ctx.equal(db.scalar(close_sql, (handle,)), "closed", "close") + ctx.equal(db.scalar(close_sql, (handle,)), "closed", "close 重试") + ctx.equal(db.scalar("SELECT CAST(vexfs_read('default','/file') AS TEXT)"), + "abcd", "幂等后内容") + return {"request_rows": db.scalar("SELECT count(*) FROM _vexfs_requests")} + + +@case("contract.write-conflict", "concurrency", "两个写句柄的乐观并发冲突") +def write_conflict(ctx: Context) -> dict[str, Any]: + with Database(ctx) as db: + db.scalar("SELECT vexfs_write('default','/conflict','base')") + first = db.scalar("SELECT vexfs_handle_open('default','/conflict','rw','open-a-eval')") + second = db.scalar("SELECT vexfs_handle_open('default','/conflict','rw','open-b-eval')") + db.scalar("SELECT vexfs_handle_stage_write(?,0,'A','write-a-eval')", (first,)) + db.scalar("SELECT vexfs_handle_stage_write(?,0,'B','write-b-eval')", (second,)) + db.scalar("SELECT vexfs_handle_publish(?,1,'data','publish-a-eval')", (first,)) + ctx.expect_error(lambda: db.scalar( + "SELECT vexfs_handle_publish(?,1,'data','publish-b-eval')", (second,)), + "write conflict") + ctx.equal(db.scalar("SELECT CAST(vexfs_read('default','/conflict') AS TEXT)"), + "Aase", "冲突后已提交内容") + return {"conflicts": 1} + + +@case("contract.session-reclaim", "recovery", "session 隔离、同步、retained 和过期回收") +def session_reclaim(ctx: Context) -> dict[str, Any]: + with Database(ctx) as db: + db.scalar("SELECT vexfs_write('default','/session','old')") + db.scalar("SELECT vexfs_mount_session_start('default','session-a')") + handle = db.scalar( + "SELECT vexfs_handle_open('default','/session','rw','open-session','session-a')") + db.scalar("SELECT vexfs_handle_stage_write(?,0,'NEW','write-session')", (handle,)) + ctx.equal(db.scalar( + "SELECT vexfs_mount_synchronize('default','sync-other','session-b')"), 0, + "其他 session 不能发布") + ctx.equal(db.scalar( + "SELECT vexfs_mount_synchronize('default','sync-owner','session-a')"), 1, + "owner session 发布") + ctx.equal(db.scalar("SELECT CAST(vexfs_read('default','/session') AS TEXT)"), + "NEW", "同步后的内容") + db.scalar("SELECT vexfs_handle_stage_write(?,3,'!','write-retain')", (handle,)) + db.scalar("SELECT vexfs_mount_session_end('default','session-a')") + ctx.equal(db.scalar("SELECT state FROM _vexfs_handles WHERE id=?", (handle,)), + "retained", "未发布 staging 保留") + ctx.equal(db.scalar( + "SELECT vexfs_mount_synchronize('default','sync-no-owner')"), 0, + "owner staging 不能被普通同步接管") + db.scalar("SELECT vexfs_mount_session_start('default','session-b')") + ctx.equal(db.scalar( + "SELECT vexfs_mount_synchronize('default','sync-recovered','session-b')"), 1, + "新 session 接管并发布 retained staging") + ctx.equal(db.scalar("SELECT CAST(vexfs_read('default','/session') AS TEXT)"), + "NEW!", "接管后的暂存内容") + db.scalar("SELECT vexfs_mount_session_end('default','session-b')") + ctx.equal(db.scalar("SELECT vexfs_item_reclaim('default','reclaim-closed')"), 1, + "只回收已关闭句柄") + ctx.equal(db.scalar("SELECT count(*) FROM _vexfs_staging WHERE handle_id=?", (handle,)), + 0, "staging 回收") + return {"published": 2, "reclaimed": 1} + + +@case("migration.contract-010-to-030", "migration", "0.1.0 数据库原地升级并保留 dirty staging") +def migrate_010_to_030(ctx: Context) -> dict[str, Any]: + with tempfile.TemporaryDirectory(prefix="vexfs-migration-") as directory: + path = Path(directory) / "legacy.sqlite3" + db = Database(ctx, path, initialize=False) + db.connection.executescript(""" +CREATE TABLE _vexfs_meta(key TEXT PRIMARY KEY,value TEXT NOT NULL); +INSERT INTO _vexfs_meta VALUES('contract_version','0.1.0'); +CREATE TABLE _vexfs_inodes(kind TEXT,deleted_at INTEGER,current_version INTEGER); +INSERT INTO _vexfs_inodes VALUES('directory',NULL,0); +CREATE TABLE _vexfs_handles(id TEXT PRIMARY KEY,dirty_generation INTEGER NOT NULL); +INSERT INTO _vexfs_handles VALUES('dirty-handle',2); +CREATE TABLE _vexfs_staging(handle_id TEXT,generation INTEGER,content BLOB, + created_at INTEGER,PRIMARY KEY(handle_id,generation)); +INSERT INTO _vexfs_staging VALUES('dirty-handle',1,X'6F6C64',1); +INSERT INTO _vexfs_staging VALUES('dirty-handle',2,X'6E6577',2); +CREATE TABLE _vexfs_requests(request_id TEXT PRIMARY KEY,operation TEXT NOT NULL, + result_integer INTEGER,result_text TEXT,created_at INTEGER); +INSERT INTO _vexfs_requests VALUES('old-request','write',1,NULL,1); +""") + ctx.equal(db.scalar("SELECT vexfs_init()"), 1, "迁移执行") + ctx.equal(db.scalar("SELECT value FROM _vexfs_meta WHERE key='contract_version'"), + CONTRACT_VERSION, "迁移版本") + row = db.connection.execute( + "SELECT s.generation,d.content,s.logical_size,s.capacity " + "FROM _vexfs_staging s JOIN _vexfs_staging_data d USING(handle_id)").fetchone() + ctx.equal(row, (2, b"new", 3, 3), "只保留当前 dirty generation") + ctx.equal(db.scalar("SELECT count(*) FROM _vexfs_requests"), 0, "旧请求缓存清空") + ctx.equal(db.scalar("SELECT current_version FROM _vexfs_inodes"), 1, + "目录 verifier 修复") + columns = [row[1] for row in db.connection.execute( + "PRAGMA table_info(_vexfs_handles)").fetchall()] + ctx.check("owner_session" in columns, "迁移 owner_session 列") + db.close() + return {"from": "0.1.0", "to": CONTRACT_VERSION, "staging_generation": 2} + + +@case("migration.staging-inline-to-split", "migration", + "旧 0.2 inline staging 原地拆表并保留未发布内容") +def migrate_inline_staging_to_split(ctx: Context) -> dict[str, Any]: + with tempfile.TemporaryDirectory(prefix="vexfs-staging-migration-") as directory: + path = Path(directory) / "legacy-020.sqlite3" + db = Database(ctx, path) + db.scalar("SELECT vexfs_write('default','/legacy','old')") + handle = db.scalar( + "SELECT vexfs_handle_open('default','/legacy','rw','legacy-split-open')") + generation = db.scalar( + "SELECT vexfs_handle_stage_write(?,0,'new','legacy-split-write')", (handle,)) + db.connection.executescript(""" +UPDATE _vexfs_staging +SET content=(SELECT content FROM _vexfs_staging_data + WHERE _vexfs_staging_data.handle_id=_vexfs_staging.handle_id); +DROP TABLE _vexfs_staging_data; +DELETE FROM _vexfs_meta WHERE key='staging_layout'; +UPDATE _vexfs_meta SET value='0.2.0' WHERE key='contract_version'; +""") + db.close() + + migrated = Database(ctx, path, initialize=False) + ctx.equal(migrated.scalar("SELECT vexfs_init()"), 1, "0.2 staging 拆表迁移") + ctx.equal(migrated.scalar( + "SELECT value FROM _vexfs_meta WHERE key='staging_layout'"), + "split-v1", "staging 布局标记") + ctx.equal(migrated.scalar( + "SELECT value FROM _vexfs_meta WHERE key='contract_version'"), + CONTRACT_VERSION, "0.2 合同升级") + row = migrated.connection.execute( + "SELECT length(s.content),d.content,s.logical_size,s.capacity " + "FROM _vexfs_staging s JOIN _vexfs_staging_data d USING(handle_id) " + "WHERE s.handle_id=?", (handle,)).fetchone() + ctx.equal(row, (0, b"new" + b"\x00" * (64 * 1024 - 3), 3, 64 * 1024), + "迁移后 metadata 与 BLOB 分离") + migrated.scalar( + "SELECT vexfs_handle_publish(?,?,'full','legacy-split-publish')", + (handle, generation)) + ctx.equal(migrated.scalar("SELECT vexfs_read('default','/legacy')"), b"new", + "迁移后未发布内容可发布") + ctx.equal(migrated.scalar("PRAGMA integrity_check"), "ok", "迁移后完整性") + migrated.close() + return {"layout": "split-v1", "generation": generation, "capacity": 64 * 1024} + + +@case("migration.failure-rollback", "migration", "0.2 迁移失败必须完整回滚并可重试") +def migration_failure_rollback(ctx: Context) -> dict[str, Any]: + with tempfile.TemporaryDirectory(prefix="vexfs-migration-failure-") as directory: + path = Path(directory) / "broken-020.sqlite3" + db = Database(ctx, path) + db.scalar("SELECT vexfs_write('default','/migration','value')") + db.connection.executescript(""" +DROP TABLE _vexfs_staging_data; +CREATE TABLE _vexfs_staging_data(handle_id TEXT PRIMARY KEY,broken BLOB NOT NULL); +DELETE FROM _vexfs_meta WHERE key='staging_layout'; +UPDATE _vexfs_meta SET value='0.2.0' WHERE key='contract_version'; +""") + ctx.expect_error(lambda: db.scalar("SELECT vexfs_init()"), "no column named content") + ctx.equal(db.scalar("SELECT value FROM _vexfs_meta WHERE key='contract_version'"), + "0.2.0", "失败后合同版本回滚") + ctx.equal(db.scalar( + "SELECT count(*) FROM _vexfs_meta WHERE key='staging_layout'"), 0, + "失败后布局标记不残留") + db.connection.execute("DROP TABLE _vexfs_staging_data") + ctx.equal(db.scalar("SELECT vexfs_init()"), 1, "修复条件后可重试") + ctx.equal(db.scalar("SELECT value FROM _vexfs_meta WHERE key='contract_version'"), + CONTRACT_VERSION, "重试后完成迁移") + db.close() + return {"rollback_version": "0.2.0", "retry_version": CONTRACT_VERSION} + + +@case("migration.initialization-failure-rollback", "migration", + "新库初始化空间不足时不得留下半张 schema") +def initialization_failure_rollback(ctx: Context) -> dict[str, Any]: + with tempfile.TemporaryDirectory(prefix="vexfs-init-failure-") as directory: + path = Path(directory) / "full.sqlite3" + db = Database(ctx, path, initialize=False) + db.connection.execute("PRAGMA page_size=512") + db.connection.execute("PRAGMA max_page_count=8") + ctx.expect_error(lambda: db.scalar("SELECT vexfs_init()"), "full") + ctx.equal(db.scalar( + "SELECT count(*) FROM sqlite_master WHERE name LIKE '_vexfs_%'"), 0, + "失败后不残留 VexFS 表") + db.connection.execute("PRAGMA max_page_count=4096") + ctx.equal(db.scalar("SELECT vexfs_init()"), 1, "释放空间后初始化可重试") + ctx.equal(db.scalar("SELECT value FROM _vexfs_meta WHERE key='contract_version'"), + CONTRACT_VERSION, "重试后合同完整") + db.close() + return {"page_size": 512, "failed_pages": 8} + + +@case("limits.invalid-handle-ranges", "limits", "负偏移、负长度、错误 generation 和 durability") +def invalid_handle_ranges(ctx: Context) -> dict[str, Any]: + with Database(ctx) as db: + db.scalar("SELECT vexfs_write('default','/ranges','abc')") + handle = db.scalar("SELECT vexfs_handle_open('default','/ranges','rw','range-open')") + ctx.expect_error(lambda: db.scalar( + "SELECT vexfs_handle_stage_write(?,-1,'x','range-negative')", (handle,)), + "out of range") + ctx.expect_error(lambda: db.scalar( + "SELECT vexfs_handle_read(?,-1,1)", (handle,)), "non-negative") + ctx.expect_error(lambda: db.scalar( + "SELECT vexfs_handle_read(?,0,-1)", (handle,)), "non-negative") + ctx.expect_error(lambda: db.scalar( + "SELECT vexfs_handle_truncate(?,-1,'truncate-negative')", (handle,)), + "out of range") + ctx.expect_error(lambda: db.scalar( + "SELECT vexfs_handle_publish(?,0,'invalid','publish-invalid')", (handle,)), + "durability must be") + db.scalar("SELECT vexfs_handle_stage_write(?,0,'x','range-write')", (handle,)) + ctx.expect_error(lambda: db.scalar( + "SELECT vexfs_handle_publish(?,2,'data','publish-future-generation')", (handle,)), + "generation is stale") + return {"rejected_ranges": 4, "rejected_publish": 2} + + +@case("recovery.reopen-integrity", "recovery", "关闭重开、WAL checkpoint 和 integrity_check") +def reopen_integrity(ctx: Context) -> dict[str, Any]: + with tempfile.TemporaryDirectory(prefix="vexfs-reopen-") as directory: + path = Path(directory) / "db.sqlite3" + db = Database(ctx, path) + payload = os.urandom(2 * MIB) + db.scalar("SELECT vexfs_mkdir('default','/persist')") + db.scalar("SELECT vexfs_write('default','/persist/blob',?)", (payload,)) + db.connection.execute("PRAGMA wal_checkpoint(FULL)").fetchone() + db.close() + reopened = Database(ctx, path) + ctx.equal(reopened.scalar("SELECT vexfs_read('default','/persist/blob')"), payload, + "重开内容") + ctx.equal(reopened.scalar("PRAGMA integrity_check"), "ok", "integrity_check") + fk_rows = reopened.connection.execute("PRAGMA foreign_key_check").fetchall() + ctx.equal(fk_rows, [], "foreign key check") + metrics = {"bytes": len(payload), "sha256": sha256(payload), + "storage": db_storage(path)} + reopened.close() + return metrics + + +@case("recovery.process-crash", "recovery", "进程在事务中退出和提交后立即退出") +def process_crash(ctx: Context) -> dict[str, Any]: + with tempfile.TemporaryDirectory(prefix="vexfs-crash-") as directory: + path = Path(directory) / "db.sqlite3" + with Database(ctx, path) as db: + db.scalar("SELECT vexfs_write('default','/crash','old')") + script = Path(__file__).resolve() + rolled_back = run_process( + [sys.executable, str(script), "--worker", "crash-write", str(path), + str(ctx.extension), "/crash", "uncommitted", "transaction"], + check=False) + ctx.equal(rolled_back.returncode, 17, "事务中崩溃退出码") + with Database(ctx, path) as db: + ctx.equal(db.scalar("SELECT CAST(vexfs_read('default','/crash') AS TEXT)"), + "old", "崩溃事务必须回滚") + ctx.equal(db.scalar("PRAGMA integrity_check"), "ok", "崩溃后完整性") + committed = run_process( + [sys.executable, str(script), "--worker", "crash-write", str(path), + str(ctx.extension), "/crash", "durable", "autocommit"], check=False) + ctx.equal(committed.returncode, 0, "提交后退出码") + with Database(ctx, path) as db: + ctx.equal(db.scalar("SELECT CAST(vexfs_read('default','/crash') AS TEXT)"), + "durable", "提交后立即退出仍持久") + return {"transaction_exit": 17, "autocommit_exit": 0} + + +@case("backup.online-restore", "backup", "在线 SQLite backup、恢复和 staging 可见性") +def online_backup_restore(ctx: Context) -> dict[str, Any]: + with tempfile.TemporaryDirectory(prefix="vexfs-backup-") as directory: + source_path = Path(directory) / "source.sqlite3" + backup_path = Path(directory) / "backup.sqlite3" + source = Database(ctx, source_path) + payload = os.urandom(min(ctx.mode.sequential_mib, 64) * MIB) + source.scalar("SELECT vexfs_write('default','/backup.bin',?)", (payload,)) + handle = source.scalar( + "SELECT vexfs_handle_open('default','/backup.bin','rw','backup-open')") + source.scalar("SELECT vexfs_handle_stage_write(?,0,'DIRTY','backup-write')", (handle,)) + destination = sqlite3.connect(str(backup_path), isolation_level=None) + started = time.perf_counter() + source.connection.backup(destination, pages=256) + elapsed = time.perf_counter() - started + destination.close() + restored = Database(ctx, backup_path) + restored_payload = restored.scalar("SELECT vexfs_read('default','/backup.bin')") + ctx.equal(restored_payload, payload, "备份只能看到已发布内容") + ctx.equal(restored.scalar("SELECT count(*) FROM _vexfs_staging"), 1, + "备份保留完整恢复状态") + ctx.equal(restored.scalar("PRAGMA integrity_check"), "ok", "备份完整性") + storage = db_storage(backup_path) + bytes_copied = storage["total_bytes"] + restored.close() + source.close() + ctx.budget("backup_seconds", elapsed, 30.0) + return {"logical_bytes": len(payload), "backup_bytes": bytes_copied, + "storage": storage, + "seconds": round(elapsed, 6), + "mib_per_second": round(bytes_copied / MIB / max(elapsed, 1e-9), 3)} + + +@case("concurrency.database-lock", "concurrency", "busy_timeout 和数据库锁错误") +def database_lock(ctx: Context) -> dict[str, Any]: + with tempfile.TemporaryDirectory(prefix="vexfs-lock-") as directory: + path = Path(directory) / "db.sqlite3" + owner = Database(ctx, path) + contender = Database(ctx, path, timeout=0.2) + owner.connection.execute("BEGIN IMMEDIATE") + owner.scalar("SELECT vexfs_write('default','/locked','owner')") + started = time.perf_counter() + message = ctx.expect_error(lambda: contender.scalar( + "SELECT vexfs_write('default','/other','contender')"), "locked") + elapsed = time.perf_counter() - started + owner.connection.execute("ROLLBACK") + contender.close() + owner.close() + ctx.check(elapsed >= 0.15, f"busy_timeout 未生效: {elapsed:.3f}s") + return {"wait_seconds": round(elapsed, 6), "error": message} + + +@case("concurrency.multi-process", "concurrency", "多个真实进程同时写不同文件") +def multi_process(ctx: Context) -> dict[str, Any]: + with tempfile.TemporaryDirectory(prefix="vexfs-parallel-") as directory: + path = Path(directory) / "db.sqlite3" + with Database(ctx, path) as db: + db.scalar("SELECT vexfs_mkdir('default','/parallel')") + script = Path(__file__).resolve() + processes: list[subprocess.Popen[bytes]] = [] + started = time.perf_counter() + for worker in range(ctx.mode.parallel_workers): + processes.append(subprocess.Popen( + [sys.executable, str(script), "--worker", "parallel-write", str(path), + str(ctx.extension), str(worker), str(ctx.mode.files_per_worker)], + stdout=subprocess.PIPE, stderr=subprocess.PIPE)) + failures: list[str] = [] + for process in processes: + stdout, stderr = process.communicate(timeout=120) + if process.returncode != 0: + failures.append( + f"rc={process.returncode} stdout={stdout.decode(errors='replace')} " + f"stderr={stderr.decode(errors='replace')}") + ctx.equal(failures, [], "并发 worker") + elapsed = time.perf_counter() - started + expected = ctx.mode.parallel_workers * ctx.mode.files_per_worker + with Database(ctx, path) as db: + listing = db.json("SELECT vexfs_list('default','/parallel')") + ctx.equal(len(listing), expected, "并发文件数") + for worker in range(ctx.mode.parallel_workers): + path_name = f"/parallel/w{worker}-{ctx.mode.files_per_worker - 1}.txt" + expected_value = f"worker={worker};item={ctx.mode.files_per_worker - 1}".encode() + ctx.equal(db.scalar("SELECT vexfs_read('default',?)", (path_name,)), + expected_value, "并发文件内容") + ctx.equal(db.scalar("PRAGMA integrity_check"), "ok", "并发后完整性") + ctx.budget("parallel_seconds", elapsed, 60.0) + return {"workers": ctx.mode.parallel_workers, "files": expected, + "seconds": round(elapsed, 6), + "files_per_second": round(expected / max(elapsed, 1e-9), 3)} + + +@case("concurrency.restore-race", "concurrency", + "两个真实进程同时 restore 只能生成一个版本和一个 commit") +def restore_race(ctx: Context) -> dict[str, Any]: + with tempfile.TemporaryDirectory(prefix="vexfs-restore-race-") as directory: + base = Path(directory) + path = base / "db.sqlite3" + script = Path(__file__).resolve() + with Database(ctx, path) as db: + db.scalar("SELECT vexfs_write('default','/race','one')") + db.scalar("SELECT vexfs_write('default','/race','two')") + inode = db.json("SELECT vexfs_stat('default','/race')")["inode"] + commits_before = db.scalar("SELECT count(*) FROM _vexfs_commits") + barrier = base / "go" + ready_paths = [base / "ready-0", base / "ready-1"] + processes = [subprocess.Popen( + [sys.executable, str(script), "--worker", "restore-race", str(path), + str(ctx.extension), "/race", str(ready_paths[index]), str(barrier)], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) for index in range(2)] + deadline = time.time() + 10 + while not all(candidate.exists() for candidate in ready_paths): + if time.time() >= deadline: + raise EvalFailure("restore worker 未就绪") + time.sleep(0.01) + barrier.touch() + results: list[tuple[int, bytes, bytes]] = [] + for process in processes: + stdout, stderr = process.communicate(timeout=30) + results.append((process.returncode, stdout, stderr)) + ctx.equal(sorted(result[0] for result in results), [0, 5], "restore 并发结果") + with Database(ctx, path) as db: + ctx.equal(db.scalar( + "SELECT count(*) FROM _vexfs_file_versions WHERE inode_id=?", (inode,)), + 3, "并发 restore 只新增一个版本") + ctx.equal(db.scalar("SELECT count(*) FROM _vexfs_commits"), commits_before + 1, + "并发 restore 只新增一个 commit") + head = db.scalar("SELECT head_commit FROM _vexfs_workspaces WHERE name='default'") + version_commit = db.scalar( + "SELECT commit_id FROM _vexfs_file_versions WHERE inode_id=? AND version_no=3", + (inode,)) + ctx.equal(head, version_commit, "workspace head 指向成功版本") + ctx.equal(db.scalar("SELECT vexfs_read('default','/race')"), b"one", + "成功 restore 的内容") + return {"exit_codes": [result[0] for result in results], "versions": 3} + + +@case("model.randomized-state-machine", "model", "确定性随机文件操作与参考模型逐步比对") +def randomized_state_machine(ctx: Context) -> dict[str, Any]: + rng = random.Random(ctx.seed ^ 0x5EED_F5) + model: dict[str, bytes] = {} + next_id = 0 + directories = ["/model/a", "/model/b", "/model/c"] + with Database(ctx) as db: + for directory in directories: + db.scalar("SELECT vexfs_mkdir('default',?)", (directory,)) + + def new_path() -> str: + nonlocal next_id + path = f"{rng.choice(directories)}/f{next_id:06d}.bin" + next_id += 1 + return path + + def verify(sample_all: bool = False) -> None: + for directory in directories: + actual = [entry["name"] for entry in db.json( + "SELECT vexfs_list('default',?)", (directory,))] + expected = sorted(Path(path).name for path in model if + str(Path(path).parent) == directory) + ctx.equal(actual, expected, f"模型目录 {directory}") + paths = list(model) if sample_all else rng.sample( + list(model), min(len(model), 8)) + for path in paths: + ctx.equal(db.scalar("SELECT vexfs_read('default',?)", (path,)), model[path], + f"模型内容 {path}") + + for operation in range(ctx.mode.random_ops): + choice = rng.random() + if not model or choice < 0.45: + path = new_path() + content = rng.randbytes(rng.randint(0, 512)) + db.scalar("SELECT vexfs_write('default',?,?)", (path, content)) + model[path] = content + elif choice < 0.65: + path = rng.choice(list(model)) + content = rng.randbytes(rng.randint(0, 1024)) + db.scalar("SELECT vexfs_write('default',?,?)", (path, content)) + model[path] = content + elif choice < 0.82: + source = rng.choice(list(model)) + destination = new_path() + db.scalar("SELECT vexfs_move('default',?,?)", (source, destination)) + model[destination] = model.pop(source) + elif choice < 0.94: + path = rng.choice(list(model)) + db.scalar("SELECT vexfs_remove('default',?,0)", (path,)) + del model[path] + else: + path = rng.choice(list(model)) + ctx.equal(db.scalar("SELECT vexfs_read('default',?)", (path,)), model[path], + "随机 read") + if operation % 50 == 0: + verify() + verify(sample_all=True) + ctx.equal(db.scalar("PRAGMA integrity_check"), "ok", "随机模型后完整性") + return {"seed": ctx.seed, "operations": ctx.mode.random_ops, + "final_files": len(model)} + + +@case("cli.command-surface", "cli", "常用文件命令、版本命令、descriptor 和 doctor") +def cli_command_surface(ctx: Context) -> dict[str, Any]: + ctx.check(ctx.cli.exists(), f"缺少 CLI: {ctx.cli}") + with tempfile.TemporaryDirectory(prefix="vexfs-cli-eval-") as directory: + base = Path(directory) + database = base / "cli.sqlite3" + descriptor = base / "volume.vexfs" + local_file = base / "payload.bin" + payload = bytes(range(256)) * 8 + b"\x00CLI" + local_file.write_bytes(payload) + prefix = [str(ctx.cli), "--db", str(database), "--workspace", "eval"] + run_process(prefix + ["setup"]) + run_process(prefix + ["mkdir", "/cli/a", "/cli/b"]) + write = run_process(prefix + ["write", "/cli/a/payload.bin", str(local_file)]) + ctx.check(int(write.stdout.strip()) >= 1, "CLI write version") + ctx.equal(run_process(prefix + ["cat", "/cli/a/payload.bin"]).stdout, + payload, "CLI binary cat") + listing = run_process(prefix + ["--json", "ls", "/cli/a"]) + ctx.equal(json.loads(listing.stdout)[0]["name"], "payload.bin", "CLI JSON ls") + stat = json.loads(run_process(prefix + ["stat", "/cli/a/payload.bin"]).stdout) + ctx.equal(stat["size"], len(payload), "CLI stat") + + version_one = base / "version-one.txt" + version_two = base / "version-two.txt" + version_one.write_text("one\n") + version_two.write_text("two\n") + ctx.equal(run_process(prefix + ["write", "/cli/a/version.txt", str(version_one)]).stdout.strip(), + b"1", "CLI 历史版本 1") + ctx.equal(run_process(prefix + ["write", "/cli/a/version.txt", str(version_two)]).stdout.strip(), + b"2", "CLI 历史版本 2") + cli_history = json.loads(run_process( + prefix + ["--json", "history", "/cli/a/version.txt"]).stdout) + ctx.equal([row["version"] for row in cli_history["entries"]], [2, 1], "CLI history") + first_page = json.loads(run_process( + prefix + ["--json", "history", "/cli/a/version.txt", "--limit", "1"]).stdout) + ctx.equal([row["version"] for row in first_page["entries"]], [2], "CLI history 首屏") + ctx.equal(first_page["next_before"], 2, "CLI history cursor") + ctx.equal(run_process(prefix + ["show", "/cli/a/version.txt", "--version", "1"]).stdout, + b"one\n", "CLI show") + diff = run_process(prefix + ["diff", "/cli/a/version.txt", "--from", "1", "--to", "2"], + check=False) + ctx.equal(diff.returncode, 1, "CLI diff 不同退出码") + ctx.check(b"-one" in diff.stdout and b"+two" in diff.stdout, "CLI diff 内容") + missing_diff = run_process( + prefix + ["--json", "diff", "/cli/a/version.txt", "--from", "99", "--to", "2"], + check=False) + ctx.equal(missing_diff.returncode, 3, "CLI 缺失版本退出码") + ctx.equal(json.loads(missing_diff.stderr)["error"]["code"], "VEXFS_NOT_FOUND", + "CLI JSON 错误分类") + dry_run = run_process(prefix + ["restore", "/cli/a/version.txt", "--version", "1", + "--dry-run"]) + ctx.check(b"-two" in dry_run.stdout and b"+one" in dry_run.stdout, + "CLI restore dry-run 预览") + ctx.equal(json.loads(run_process(prefix + ["stat", "/cli/a/version.txt"]).stdout)["version"], + 2, "dry-run 不改变版本") + ctx.equal(run_process(prefix + ["restore", "/cli/a/version.txt", "--version", "1"]).stdout.strip(), + b"3", "CLI restore 新版本") + ctx.equal(run_process(prefix + ["cat", "/cli/a/version.txt"]).stdout, + b"one\n", "CLI restore 内容") + + newline_one = base / "newline-one.txt" + newline_two = base / "newline-two.txt" + newline_one.write_bytes(b"same\n") + newline_two.write_bytes(b"same") + run_process(prefix + ["write", "/cli/a/newline.txt", str(newline_one)]) + run_process(prefix + ["write", "/cli/a/newline.txt", str(newline_two)]) + newline_diff = run_process( + prefix + ["diff", "/cli/a/newline.txt", "--from", "1", "--to", "2"], + check=False) + ctx.equal(newline_diff.returncode, 1, "CLI diff 识别末尾换行差异") + ctx.check(b"No newline at end of file" in newline_diff.stdout, + "CLI diff 末尾换行提示") + run_process(prefix + ["rm", "/cli/a/newline.txt"]) + run_process(prefix + ["rm", "/cli/a/version.txt"]) + run_process(prefix + ["mv", "/cli/a/payload.bin", "/cli/b/moved.bin"]) + run_process(prefix + ["rm", "/cli/a"]) + run_process(prefix + ["rm", "-r", "/cli/b"]) + run_process(prefix + ["descriptor", str(descriptor)]) + descriptor_json = json.loads(descriptor.read_text()) + ctx.equal(descriptor_json["workspace"], "eval", "descriptor workspace") + ctx.equal(Path(descriptor_json["database_path"]).resolve(), database.resolve(), + "descriptor database") + doctor = run_process(prefix + ["--json", "doctor"], check=False) + doctor_json = json.loads(doctor.stdout) + ctx.equal(doctor_json["database"]["contract_version"], CONTRACT_VERSION, + "doctor contract") + ctx.check(doctor.returncode in (0, 1), "doctor 退出码") + invalid = run_process(prefix + ["unknown"], check=False) + ctx.equal(invalid.returncode, 1, "未知命令退出码") + ctx.check(b"unknown command" in invalid.stderr, "未知命令错误信息") + return {"payload_bytes": len(payload), "doctor": doctor_json, + "descriptor": descriptor_json} + + +@case("cli.doctor-readonly-and-permissions", "cli", + "doctor 不迁移数据库，CLI 创建的 SQLite 文件保持私有权限") +def cli_doctor_readonly_and_permissions(ctx: Context) -> dict[str, Any]: + ctx.check(ctx.cli.exists(), f"缺少 CLI: {ctx.cli}") + with tempfile.TemporaryDirectory(prefix="vexfs-cli-security-") as directory: + database = Path(directory) / "private" / "vexfs.sqlite3" + prefix = [str(ctx.cli), "--db", str(database), "--workspace", "secure"] + run_process(prefix + ["setup"]) + run_process(prefix + ["write", "/secret"], input_data=b"secret") + for candidate in (database, Path(str(database) + "-wal"), Path(str(database) + "-shm")): + if candidate.exists(): + ctx.equal(candidate.stat().st_mode & 0o777, 0o600, + f"{candidate.name} 私有权限") + connection = sqlite3.connect(str(database), isolation_level=None) + connection.execute( + "UPDATE _vexfs_meta SET value='0.2.0' WHERE key='contract_version'") + connection.close() + doctor = run_process(prefix + ["--json", "doctor"], check=False) + details = json.loads(doctor.stdout)["database"] + ctx.equal(details["contract_version"], "0.2.0", "doctor 报告真实旧版本") + ctx.equal(details["compatible"], False, "doctor 拒绝把旧版本报为正常") + ctx.equal(details["upgrade_required"], True, "doctor 给出升级状态") + connection = sqlite3.connect(str(database), isolation_level=None) + ctx.equal(connection.execute( + "SELECT value FROM _vexfs_meta WHERE key='contract_version'").fetchone()[0], + "0.2.0", "doctor 结束后没有静默迁移") + connection.close() + readonly = sqlite3.connect(f"file:{database}?mode=ro", uri=True, isolation_level=None) + readonly.enable_load_extension(True) + readonly.load_extension(str(ctx.extension)) + ctx.equal(readonly.execute( + "SELECT vexfs_read('secure','/secret')").fetchone()[0], b"secret", + "0.2 数据库仍可只读访问") + readonly.close() + run_process(prefix + ["setup"]) + connection = sqlite3.connect(str(database), isolation_level=None) + ctx.equal(connection.execute( + "SELECT value FROM _vexfs_meta WHERE key='contract_version'").fetchone()[0], + CONTRACT_VERSION, "显式 setup 才执行升级") + connection.close() + return {"mode": "0600", "doctor_exit": doctor.returncode} + + +@case("performance.small-files", "performance", "大量小文件创建、列表、stat 和读取") +def performance_small_files(ctx: Context) -> dict[str, Any]: + with Database(ctx) as db: + db.scalar("SELECT vexfs_mkdir('default','/small')") + payload = b"small-file-payload" + started = time.perf_counter() + for index in range(ctx.mode.small_files): + db.scalar("SELECT vexfs_write('default',?,?)", + (f"/small/f{index:06d}.txt", payload)) + create_seconds = time.perf_counter() - started + started = time.perf_counter() + listing = db.json("SELECT vexfs_list('default','/small')") + list_seconds = time.perf_counter() - started + ctx.equal(len(listing), ctx.mode.small_files, "小文件列表数量") + ctx.equal([entry["name"] for entry in listing[:3]], + [f"f{i:06d}.txt" for i in range(3)], "列表二进制排序") + started = time.perf_counter() + for index in range(0, ctx.mode.small_files, + max(1, ctx.mode.small_files // 100)): + stat = db.json("SELECT vexfs_stat('default',?)", + (f"/small/f{index:06d}.txt",)) + ctx.equal(stat["size"], len(payload), "小文件 stat") + ctx.equal(db.scalar("SELECT vexfs_read('default',?)", + (f"/small/f{index:06d}.txt",)), payload, "小文件 read") + sample_seconds = time.perf_counter() - started + ctx.budget("small_create_seconds", create_seconds, 60.0) + ctx.budget("small_list_seconds", list_seconds, 5.0) + return {"files": ctx.mode.small_files, + "create_seconds": round(create_seconds, 6), + "create_files_per_second": round( + ctx.mode.small_files / max(create_seconds, 1e-9), 3), + "list_seconds": round(list_seconds, 6), + "sample_stat_read_seconds": round(sample_seconds, 6), + "storage": db_storage(db.path)} + + +@case("performance.sequential-file", "performance", "大文件写入、读取、重开和哈希") +def performance_sequential_file(ctx: Context) -> dict[str, Any]: + payload = (bytes(range(256)) * (ctx.mode.sequential_mib * MIB // 256)) + with tempfile.TemporaryDirectory(prefix="vexfs-sequential-") as directory: + path = Path(directory) / "db.sqlite3" + db = Database(ctx, path) + started = time.perf_counter() + version = db.scalar("SELECT vexfs_write('default','/large.bin',?)", (payload,)) + write_seconds = time.perf_counter() - started + db.close() + reopened = Database(ctx, path) + started = time.perf_counter() + actual = reopened.scalar("SELECT vexfs_read('default','/large.bin')") + read_seconds = time.perf_counter() - started + ctx.equal(sha256(actual), sha256(payload), "大文件哈希") + ctx.equal(len(actual), len(payload), "大文件长度") + ctx.equal(reopened.scalar("PRAGMA integrity_check"), "ok", "大文件 DB 完整性") + storage_before_checkpoint = db_storage(path) + reopened.connection.execute("PRAGMA wal_checkpoint(TRUNCATE)").fetchone() + storage_after_checkpoint = db_storage(path) + reopened.close() + ctx.budget("sequential_write_seconds", write_seconds, 60.0) + ctx.budget("sequential_read_seconds", read_seconds, 30.0) + return {"logical_bytes": len(payload), "version": version, + "sha256": sha256(payload), + "storage_before_checkpoint": storage_before_checkpoint, + "storage_after_checkpoint": storage_after_checkpoint, + "write_seconds": round(write_seconds, 6), + "write_mib_per_second": round(len(payload) / MIB / max(write_seconds, 1e-9), 3), + "read_seconds": round(read_seconds, 6), + "read_mib_per_second": round(len(payload) / MIB / max(read_seconds, 1e-9), 3)} + + +@case("performance.staged-overwrite", "performance", "分块 staging 写入的空间上界和吞吐") +def performance_staged_overwrite(ctx: Context) -> dict[str, Any]: + total = ctx.mode.staged_mib * MIB + chunk = bytes((index * 17) % 251 for index in range(256 * 1024)) + with Database(ctx) as db: + db.scalar("SELECT vexfs_write('default','/staged.bin',X'')") + handle = db.scalar( + "SELECT vexfs_handle_open('default','/staged.bin','rw','staged-open')") + started = time.perf_counter() + generation = 0 + for offset in range(0, total, len(chunk)): + data = chunk[:min(len(chunk), total - offset)] + generation = db.scalar( + "SELECT vexfs_handle_stage_write(?,?,?,?)", + (handle, offset, data, f"staged-{offset}")) + stage_seconds = time.perf_counter() - started + row = db.connection.execute( + "SELECT count(*),s.logical_size,s.capacity,length(d.content) " + "FROM _vexfs_staging s JOIN _vexfs_staging_data d USING(handle_id) " + "WHERE s.handle_id=?", (handle,)).fetchone() + ctx.equal(row[0], 1, "每个 handle 只能有一行 staging") + ctx.equal(row[1], total, "staging logical_size") + ctx.equal(row[2], row[3], "capacity 与 BLOB 长度") + ctx.check(total <= row[2] <= MAX_FILE_BYTES, "staging capacity 边界") + ctx.check(row[2] < max(total * 2, 64 * 1024 + 1), "几何扩容不能超过 2x") + started = time.perf_counter() + version = db.scalar( + "SELECT vexfs_handle_publish(?,?,'full','staged-publish')", + (handle, generation)) + publish_seconds = time.perf_counter() - started + db.scalar("SELECT vexfs_handle_close(?,0,'staged-close')", (handle,)) + ctx.equal(db.scalar("SELECT length(vexfs_read('default','/staged.bin'))"), total, + "staged 发布长度") + sample = db.scalar( + "SELECT substr(vexfs_read('default','/staged.bin'),?,?)", + (total - min(total, len(chunk)) + 1, min(total, len(chunk)))) + ctx.equal(sample, chunk[:len(sample)], "staged 尾部内容") + ctx.budget("staged_seconds", stage_seconds, 120.0) + storage_before_checkpoint = db_storage(db.path) + db.connection.execute("PRAGMA wal_checkpoint(TRUNCATE)").fetchone() + storage_after_checkpoint = db_storage(db.path) + vacuum_started = time.perf_counter() + db.connection.execute("VACUUM") + vacuum_seconds = time.perf_counter() - vacuum_started + db.connection.execute("PRAGMA wal_checkpoint(TRUNCATE)").fetchone() + storage_after_vacuum = db_storage(db.path) + return {"logical_bytes": total, "capacity_bytes": row[2], + "staging_rows": row[0], "generation": generation, "version": version, + "stage_seconds": round(stage_seconds, 6), + "stage_mib_per_second": round(total / MIB / max(stage_seconds, 1e-9), 3), + "publish_seconds": round(publish_seconds, 6), + "storage_before_checkpoint": storage_before_checkpoint, + "storage_after_checkpoint": storage_after_checkpoint, + "vacuum_seconds": round(vacuum_seconds, 6), + "storage_after_vacuum": storage_after_vacuum} + + +@case("performance.random-patches", "performance", "随机 4 KiB 覆盖与最终抽样校验") +def performance_random_patches(ctx: Context) -> dict[str, Any]: + logical_size = min(ctx.mode.staged_mib, 32) * MIB + patch_size = 4096 + rng = random.Random(ctx.seed ^ 0xA11CE) + expected: dict[int, bytes] = {} + with Database(ctx) as db: + db.scalar("SELECT vexfs_write('default','/random.bin',zeroblob(?))", (logical_size,)) + handle = db.scalar( + "SELECT vexfs_handle_open('default','/random.bin','rw','random-open')") + started = time.perf_counter() + generation = 0 + for index in range(ctx.mode.random_writes): + slot = rng.randrange(logical_size // patch_size) + offset = slot * patch_size + data = hashlib.sha256(f"{ctx.seed}:{index}".encode()).digest() * (patch_size // 32) + generation = db.scalar( + "SELECT vexfs_handle_stage_write(?,?,?,?)", + (handle, offset, data, f"random-write-{index}")) + expected[offset] = data + seconds = time.perf_counter() - started + for offset, data in list(expected.items())[-min(len(expected), 50):]: + ctx.equal(db.scalar("SELECT vexfs_handle_read(?,?,?)", + (handle, offset, patch_size)), data, "随机 patch 内容") + row = db.connection.execute( + "SELECT count(*),logical_size,capacity FROM _vexfs_staging WHERE handle_id=?", + (handle,)).fetchone() + ctx.equal(row[0], 1, "随机写 staging 行数") + ctx.equal(row[1], logical_size, "随机写 logical_size") + ctx.check(row[2] <= MAX_FILE_BYTES, "随机写 capacity") + ctx.budget("random_patch_seconds", seconds, 120.0) + storage_before_checkpoint = db_storage(db.path) + db.connection.execute("PRAGMA wal_checkpoint(TRUNCATE)").fetchone() + storage_after_checkpoint = db_storage(db.path) + return {"writes": ctx.mode.random_writes, "unique_slots": len(expected), + "logical_bytes": logical_size, "capacity_bytes": row[2], + "generation": generation, "seconds": round(seconds, 6), + "operations_per_second": round(ctx.mode.random_writes / max(seconds, 1e-9), 3), + "request_rows": db.scalar("SELECT count(*) FROM _vexfs_requests"), + "storage_before_checkpoint": storage_before_checkpoint, + "storage_after_checkpoint": storage_after_checkpoint} + + +@case("performance.version-history", "performance", "重复整文件覆盖的历史版本吞吐和空间") +def performance_version_history(ctx: Context) -> dict[str, Any]: + version_count = {"quick": 5, "full": 25, "stress": 50}[ctx.mode.name] + payload_mib = 1 if ctx.mode.name == "quick" else 4 + payload_size = payload_mib * MIB + with Database(ctx) as db: + started = time.perf_counter() + last_payload = b"" + for index in range(version_count): + marker = index.to_bytes(4, "big") + last_payload = marker + bytes([index % 251]) * (payload_size - len(marker)) + version = db.scalar("SELECT vexfs_write('default','/versions.bin',?)", + (last_payload,)) + ctx.equal(version, index + 1, "历史版本号") + seconds = time.perf_counter() - started + ctx.equal(db.scalar("SELECT vexfs_read('default','/versions.bin')"), last_payload, + "最新历史版本") + inode = db.json("SELECT vexfs_stat('default','/versions.bin')")["inode"] + rows = db.scalar( + "SELECT count(*) FROM _vexfs_file_versions WHERE inode_id=?", (inode,)) + ctx.equal(rows, version_count, "历史版本行数") + history_started = time.perf_counter() + history_page = db.json( + "SELECT vexfs_history('default','/versions.bin',?,0)", (version_count,)) + history_seconds = time.perf_counter() - history_started + ctx.equal(len(history_page["entries"]), version_count, "历史分页返回量") + old_started = time.perf_counter() + oldest = db.scalar("SELECT vexfs_read_version('default','/versions.bin',1)") + old_read_seconds = time.perf_counter() - old_started + ctx.equal(len(oldest), payload_size, "读取最旧版本") + restore_started = time.perf_counter() + restored = db.scalar( + "SELECT vexfs_restore_version('default','/versions.bin',1,?)", (version_count,)) + restore_seconds = time.perf_counter() - restore_started + ctx.equal(restored, version_count + 1, "恢复历史版本") + ctx.equal(db.scalar( + "SELECT count(*) FROM _vexfs_file_versions WHERE inode_id=?", (inode,)), + version_count + 1, "恢复后历史行数") + storage_before_checkpoint = db_storage(db.path) + db.connection.execute("PRAGMA wal_checkpoint(TRUNCATE)").fetchone() + storage_after_checkpoint = db_storage(db.path) + return {"versions": version_count, "bytes_per_version": payload_size, + "logical_written_bytes": version_count * payload_size, + "seconds": round(seconds, 6), + "write_mib_per_second": round( + version_count * payload_mib / max(seconds, 1e-9), 3), + "history_seconds": round(history_seconds, 6), + "old_read_seconds": round(old_read_seconds, 6), + "restore_seconds": round(restore_seconds, 6), + "storage_before_checkpoint": storage_before_checkpoint, + "storage_after_checkpoint": storage_after_checkpoint} + + +@case("limits.maximum-file", "limits", "128 MiB 文件上限和越界拒绝", modes=("full", "stress")) +def maximum_file(ctx: Context) -> dict[str, Any]: + with Database(ctx) as db: + started = time.perf_counter() + version = db.scalar( + "SELECT vexfs_write('default','/max.bin',zeroblob(?))", (MAX_FILE_BYTES,)) + seconds = time.perf_counter() - started + ctx.equal(db.scalar("SELECT length(vexfs_read('default','/max.bin'))"), + MAX_FILE_BYTES, "最大文件长度") + handle = db.scalar( + "SELECT vexfs_handle_open('default','/max.bin','rw','max-open')") + ctx.expect_error(lambda: db.scalar( + "SELECT vexfs_handle_stage_write(?,?,X'00','max-overflow')", + (handle, MAX_FILE_BYTES)), "larger than 128 MiB") + ctx.expect_error(lambda: db.scalar( + "SELECT vexfs_handle_truncate(?,?,'max-truncate')", + (handle, MAX_FILE_BYTES + 1)), "out of range") + db.scalar("SELECT vexfs_handle_close(?,0,'max-close')", (handle,)) + ctx.equal(db.scalar( + "SELECT vexfs_write('default','/max.bin',zeroblob(?))", (MAX_FILE_BYTES,)), + 2, "最大文件第二版本") + diff_rss = None + restore_rss = None + if platform.system() == "Darwin" and ctx.cli.exists(): + prefix = [str(ctx.cli), "--db", str(db.path), "--workspace", "default"] + run_process(prefix + ["diff", "/max.bin", "--from", "1", "--to", "2"], + timeout=60) + diff_rss = child_max_rss_bytes() + ctx.check(diff_rss < 256 * MIB, + f"128 MiB diff 内存过高: {diff_rss / MIB:.1f} MiB") + run_process(prefix + ["restore", "/max.bin", "--version", "1"], timeout=60) + restore_rss = child_max_rss_bytes() + ctx.check(restore_rss < 256 * MIB, + f"128 MiB restore 内存过高: {restore_rss / MIB:.1f} MiB") + ctx.equal(db.scalar( + "SELECT source_version_no FROM _vexfs_file_versions " + "WHERE version_no=3 AND inode_id=(SELECT inode_id FROM _vexfs_dentries " + "WHERE name='max.bin')"), 1, "restore 复用不可变版本内容") + ctx.equal(db.scalar("SELECT length(vexfs_read('default','/max.bin'))"), + MAX_FILE_BYTES, "复用内容的恢复版本可读") + ctx.equal(db.scalar( + "SELECT sum(length(content)) FROM _vexfs_file_versions WHERE inode_id=" + "(SELECT inode_id FROM _vexfs_dentries WHERE name='max.bin')"), + 2 * MAX_FILE_BYTES, "restore 不重复保存 128 MiB BLOB") + storage_before_checkpoint = db_storage(db.path) + db.connection.execute("PRAGMA wal_checkpoint(TRUNCATE)").fetchone() + storage_after_checkpoint = db_storage(db.path) + vacuum_started = time.perf_counter() + db.connection.execute("VACUUM") + vacuum_seconds = time.perf_counter() - vacuum_started + db.connection.execute("PRAGMA wal_checkpoint(TRUNCATE)").fetchone() + storage_after_vacuum = db_storage(db.path) + return {"logical_bytes": MAX_FILE_BYTES, "version": version, + "write_seconds": round(seconds, 6), + "storage_before_checkpoint": storage_before_checkpoint, + "storage_after_checkpoint": storage_after_checkpoint, + "vacuum_seconds": round(vacuum_seconds, 6), + "storage_after_vacuum": storage_after_vacuum, + "diff_max_rss": diff_rss, "restore_max_rss": restore_rss} + + +@case("build.fskit-unsigned", "build", "macOS FSKit App 与扩展无签名编译", + modes=("full", "stress")) +def fskit_unsigned_build(ctx: Context) -> dict[str, Any]: + if platform.system() != "Darwin" or shutil.which("xcodebuild") is None: + raise EvalSkip("只在装有 Xcode 的 macOS 执行") + project = ctx.root / "agent_files/macos/VexFS.xcodeproj" + if not project.exists(): + raise EvalSkip("没有 FSKit Xcode 工程") + mount_output = run_process(["/sbin/mount"], check=False).stdout.decode( + errors="replace").lower() + fskit_mount_markers = ( + "(vexfs,", "(exfat,", "(msdos,", + " type vexfs", " type exfat", " type msdos", + ) + if any(marker in mount_output for marker in fskit_mount_markers): + raise EvalSkip("已有 FSKit 文件系统挂载，跳过会刷新扩展缓存的无签名构建检查") + derived = ctx.output_dir / "xcode-derived-data" + started = time.perf_counter() + app: Path | None = None + try: + result = run_process( + ["xcodebuild", "-project", str(project), "-scheme", "VexFSApp", + "-configuration", "Debug", "-derivedDataPath", str(derived), + "CODE_SIGNING_ALLOWED=NO", "build"], timeout=300, cwd=ctx.root) + seconds = time.perf_counter() - started + output = result.stdout + result.stderr + ctx.check(b"BUILD SUCCEEDED" in output, "Xcode 没有 BUILD SUCCEEDED") + app = next(derived.glob("Build/Products/Debug/VexFS.app"), None) + ctx.check(app is not None, "缺少 VexFS.app") + return {"seconds": round(seconds, 6), "app": str(app)} + finally: + # xcodebuild 会把测试产物登记到 LaunchServices。若不撤销登记，未签名的 + # Debug extension 会和 /Applications 中的正式开发签名版本争用同一标识。 + if app is None: + app = next(derived.glob("Build/Products/Debug/VexFS.app"), None) + lsregister = Path( + "/System/Library/Frameworks/CoreServices.framework/Frameworks/" + "LaunchServices.framework/Support/lsregister") + if app is not None and lsregister.exists(): + # Xcode 返回后，LaunchServices 的构建后登记仍可能延迟一两秒落盘。 + # 等它完成再撤销，否则测试进程结束后会重新出现一条 Debug App 记录。 + time.sleep(2.0) + for attempt in range(3): + run_process([str(lsregister), "-u", str(app)], check=False, timeout=30) + if attempt != 2: + time.sleep(0.5) + # 撤销同标识的测试 App 后，LaunchServices 也可能把 /Applications 中的 + # 正式扩展从 FSKit 的可用列表移除，并让 fskit_agent 保留失效进程缓存。 + # 立即恢复正式版本并刷新空闲代理，保证后续真实 mount 和用户环境不受影响。 + installed_app = Path("/Applications/VexFS.app") + installed_extension = ( + installed_app / "Contents/Extensions/VexFSAppEx.appex") + if (lsregister.exists() and installed_app.exists() and + installed_extension.exists()): + run_process([str(lsregister), "-f", str(installed_app)], + check=False, timeout=30) + if shutil.which("pluginkit") is not None: + run_process(["pluginkit", "-a", str(installed_extension)], + check=False, timeout=30) + refreshed_mount_output = run_process( + ["/sbin/mount"], check=False).stdout.decode(errors="replace").lower() + if not any(marker in refreshed_mount_output + for marker in fskit_mount_markers): + # fskit_agent 会忽略 TERM；没有 FSKit 挂载时用 KILL 让 + # launchd 拉起一个不含无签名测试进程缓存的新代理。 + run_process(["pkill", "-KILL", "-x", "fskit_agent"], + check=False, timeout=30) + time.sleep(2.0) + + +@case("mount.real-bash", "mount", "真实 FSKit mount 后用 bash 常用文件命令操作") +def real_mount_bash(ctx: Context) -> dict[str, Any]: + if platform.system() != "Darwin": + raise EvalSkip("真实 FSKit mount 只在 macOS 执行") + with tempfile.TemporaryDirectory(prefix="vexfs-mount-eval-") as directory: + base = Path(directory) + database = base / "mount.sqlite3" + mount_point = base / "mnt" + prefix = [str(ctx.cli), "--db", str(database), "--workspace", "eval"] + run_process(prefix + ["setup"]) + doctor = run_process(prefix + ["--json", "doctor"], check=False) + details = json.loads(doctor.stdout) + if details.get("extension") != "enabled": + raise EvalSkip( + f"FSKit extension={details.get('extension', 'unknown')}，需安装并在系统设置中启用") + mount_point.mkdir() + expected_target = os.path.realpath(mount_point) + run_process(prefix + ["mount", str(mount_point)], timeout=60) + descriptor: dict[str, Any] = {} + mounts: list[dict[str, Any]] = [] + unmount = None + try: + descriptor = json.loads((base / ".vexfs-volume.json").read_text()) + ctx.equal(descriptor, {"version": 2, "database_file": "mount.sqlite3", + "workspace": "eval"}, "FSKit 目录资源描述文件") + status = run_process(prefix + ["--json", "mount", "status", str(mount_point)]) + mounts = json.loads(status.stdout) + ctx.equal(len(mounts), 1, "挂载状态数量") + ctx.equal(mounts[0]["target"], expected_target, "挂载状态目标") + shell = r''' +set -e +mkdir -p "$1/project/sub" +printf 'alpha\nbeta\nagent\n' > "$1/project/input.txt" +grep agent "$1/project/input.txt" > "$1/project/sub/result.txt" +cp "$1/project/input.txt" "$1/project/copy.txt" +mv "$1/project/copy.txt" "$1/project/moved.txt" +cmp "$1/project/input.txt" "$1/project/moved.txt" +find "$1/project" -type f | sort +rm "$1/project/moved.txt" +test "$(cat "$1/project/sub/result.txt")" = agent +''' + result = run_process(["/bin/bash", "-c", shell, "vexfs-eval", str(mount_point)], + timeout=120) + ctx.check(b"input.txt" in result.stdout and b"result.txt" in result.stdout, + "bash find 输出") + ctx.equal((mount_point / "project/sub/result.txt").read_text(), "agent\n", + "挂载路径结果") + reverse = run_process(prefix + ["cat", "/project/sub/result.txt"]) + ctx.equal(reverse.stdout, b"agent\n", "数据库 CLI 反向读取挂载写入") + finally: + unmount = run_process(prefix + ["unmount", str(mount_point)], + check=False, timeout=60) + ctx.equal(unmount.returncode, 0, "卸载退出码") + after = json.loads(run_process(prefix + ["--json", "doctor"]).stdout) + ctx.equal(after["mount_count"], 0, "卸载后挂载数") + ctx.equal(after["database"]["pending_handles"], 0, "卸载后未发布句柄") + ctx.equal(after["database"]["retained_handles"], 0, "卸载后保留句柄") + ctx.equal(after["database"]["staging_bytes"], 0, "卸载后暂存字节") + return {"doctor_before": details, "doctor_after": after, "mounts": mounts, + "descriptor": descriptor, + "commands": ["mkdir", "printf", "grep", "cp", "mv", "cmp", "find", + "rm", "cat", "vexfs cat"]} + + +@case("mount.performance", "mount", "真实 FSKit 分块顺序写和随机覆盖性能") +def real_mount_performance(ctx: Context) -> dict[str, Any]: + if platform.system() != "Darwin": + raise EvalSkip("真实 FSKit mount 只在 macOS 执行") + sequential_mib = {"quick": 8, "full": 32, "stress": 64}[ctx.mode.name] + random_writes = {"quick": 100, "full": 1_000, "stress": 2_000}[ctx.mode.name] + with tempfile.TemporaryDirectory(prefix="vexfs-mount-perf-") as directory: + base = Path(directory) + database = base / "mount.sqlite3" + mount_point = base / "mnt" + prefix = [str(ctx.cli), "--db", str(database), "--workspace", "perf"] + run_process(prefix + ["setup"]) + details = json.loads(run_process( + prefix + ["--json", "doctor"], check=False).stdout) + if details.get("extension") != "enabled": + raise EvalSkip( + f"FSKit extension={details.get('extension', 'unknown')}，需安装并在系统设置中启用") + mount_point.mkdir() + run_process(prefix + ["mount", str(mount_point)], timeout=60) + fd = -1 + try: + path = mount_point / "bench.bin" + fd = os.open(path, os.O_CREAT | os.O_TRUNC | os.O_RDWR, 0o600) + chunk = b"x" * (256 * 1024) + chunk_count = sequential_mib * MIB // len(chunk) + started = time.perf_counter() + for _ in range(chunk_count): + os.write(fd, chunk) + os.fsync(fd) + sequential_seconds = time.perf_counter() - started + + rng = random.Random(ctx.seed ^ 0xF5A17) + patch = b"p" * 4096 + slots = sequential_mib * MIB // len(patch) + sampled_offsets: list[int] = [] + started = time.perf_counter() + for index in range(random_writes): + offset = rng.randrange(slots) * len(patch) + os.pwrite(fd, patch, offset) + if index >= random_writes - 10: + sampled_offsets.append(offset) + os.fsync(fd) + random_seconds = time.perf_counter() - started + for offset in sampled_offsets: + ctx.equal(os.pread(fd, len(patch), offset), patch, + "FSKit 随机覆盖读回") + os.close(fd) + fd = -1 + ctx.equal(path.stat().st_size, sequential_mib * MIB, "FSKit 性能文件长度") + ctx.budget("mount_sequential_seconds", sequential_seconds, 60.0) + ctx.budget("mount_random_seconds", random_seconds, 120.0) + finally: + if fd >= 0: + os.close(fd) + unmount = run_process(prefix + ["unmount", str(mount_point)], + check=False, timeout=60) + ctx.equal(unmount.returncode, 0, "FSKit 性能测试卸载退出码") + after = json.loads(run_process(prefix + ["--json", "doctor"]).stdout) + ctx.equal(after["mount_count"], 0, "FSKit 性能测试卸载后挂载数") + return { + "sequential_mib": sequential_mib, + "sequential_seconds": round(sequential_seconds, 6), + "sequential_mib_per_second": round( + sequential_mib / max(sequential_seconds, 1e-9), 3), + "random_writes": random_writes, + "random_seconds": round(random_seconds, 6), + "random_operations_per_second": round( + random_writes / max(random_seconds, 1e-9), 3), + "doctor_after": after, + } + + +def worker_main(arguments: list[str]) -> int: + operation = arguments[0] + if operation == "parallel-write": + path, extension, worker_text, count_text = arguments[1:] + worker = int(worker_text) + count = int(count_text) + connection = sqlite3.connect(path, isolation_level=None, timeout=30) + connection.enable_load_extension(True) + connection.load_extension(extension) + connection.execute("PRAGMA busy_timeout=30000") + connection.execute("PRAGMA journal_mode=WAL") + connection.execute("PRAGMA synchronous=FULL") + for index in range(count): + content = f"worker={worker};item={index}".encode() + connection.execute("SELECT vexfs_write('default',?,?)", + (f"/parallel/w{worker}-{index}.txt", content)).fetchone() + connection.close() + return 0 + if operation == "restore-race": + path, extension, file_path, ready_path, barrier_path = arguments[1:] + connection = sqlite3.connect(path, isolation_level=None, timeout=30) + connection.enable_load_extension(True) + connection.load_extension(extension) + connection.execute("PRAGMA busy_timeout=30000") + connection.execute("PRAGMA journal_mode=WAL") + Path(ready_path).touch() + deadline = time.time() + 10 + while not Path(barrier_path).exists(): + if time.time() >= deadline: + connection.close() + return 9 + time.sleep(0.005) + try: + version = connection.execute( + "SELECT vexfs_restore_version('default',?,1,2)", (file_path,)).fetchone()[0] + print(version) + connection.close() + return 0 + except sqlite3.Error as error: + connection.close() + if "conflict" in str(error): + print(str(error), file=sys.stderr) + return 5 + raise + if operation == "crash-write": + path, extension, file_path, value, mode = arguments[1:] + connection = sqlite3.connect(path, isolation_level=None, timeout=30) + connection.enable_load_extension(True) + connection.load_extension(extension) + connection.execute("PRAGMA journal_mode=WAL") + connection.execute("PRAGMA synchronous=FULL") + if mode == "transaction": + connection.execute("BEGIN IMMEDIATE") + connection.execute("SELECT vexfs_write('default',?,?)", + (file_path, value.encode())).fetchone() + os._exit(17) + connection.execute("SELECT vexfs_write('default',?,?)", + (file_path, value.encode())).fetchone() + os._exit(0) + raise SystemExit(f"unknown worker operation: {operation}") + + +def system_details(root: Path, build_dir: Path, extension: Path) -> dict[str, Any]: + git = run_process(["git", "rev-parse", "HEAD"], cwd=root, check=False) + build_type = "unknown" + cache = build_dir / "CMakeCache.txt" + if cache.exists(): + for line in cache.read_text(errors="replace").splitlines(): + if line.startswith("CMAKE_BUILD_TYPE:STRING="): + build_type = line.split("=", 1)[1] or "unset" + break + return { + "platform": platform.platform(), + "python": sys.version.split()[0], + "sqlite": sqlite3.sqlite_version, + "cpu_count": os.cpu_count(), + "git_commit": git.stdout.decode().strip() if git.returncode == 0 else "unknown", + "build_type": build_type, + "extension_bytes": extension.stat().st_size, + } + + +def write_markdown(report: dict[str, Any], path: Path) -> None: + summary = report["summary"] + lines = [ + "# VexFS Eval Report", + "", + f"- run: `{report['run_id']}`", + f"- mode: `{report['mode']}`", + f"- seed: `{report['seed']}`", + f"- build: `{report['environment']['build_type']}`", + f"- result: **{summary['status']}**", + f"- cases: {summary['passed']} passed / {summary['failed']} failed / " + f"{summary['skipped']} skipped", + f"- checks: {summary['checks']}", + f"- duration: {summary['duration_seconds']:.3f}s", + "", + "| Case | Category | Status | Time | Checks |", + "|---|---|---:|---:|---:|", + ] + for result in report["cases"]: + lines.append( + f"| `{result['id']}` | {result['category']} | {result['status']} | " + f"{result['duration_seconds']:.3f}s | {result['checks']} |") + if result.get("error"): + lines.extend(["", f"> `{result['id']}`: {result['error']}", ""]) + lines.extend(["", "## Metrics", ""]) + for result in report["cases"]: + if result.get("metrics"): + lines.extend([f"### {result['id']}", "", "```json", + json.dumps(result["metrics"], ensure_ascii=False, indent=2), + "```", ""]) + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def run_eval(options: argparse.Namespace) -> int: + root = Path(options.root).resolve() + build_dir = Path(options.build_dir).resolve() + ext = extension_path(build_dir) + cli = (build_dir / "vexfs").resolve() + mode = MODES[options.mode] + seed = options.seed + run_id = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S.%fZ") + f"-{mode.name}-{seed}" + output_dir = Path(options.output_dir).resolve() / run_id + output_dir.mkdir(parents=True, exist_ok=False) + ctx = Context(root, build_dir, ext, cli, output_dir, mode, seed, + options.enforce_performance) + selected = [item for item in CASES if mode.name in item.modes] + if options.filter: + selected = [item for item in selected if options.filter in item.case_id or + options.filter in item.category] + started_all = time.perf_counter() + results: list[dict[str, Any]] = [] + print(f"VexFS eval: mode={mode.name} seed={seed} cases={len(selected)}") + print(f"extension: {ext}") + for index, item in enumerate(selected, 1): + before_checks = ctx.checks + ctx.current_case = item.case_id + ctx.current_artifacts = output_dir / "artifacts" / item.case_id + ctx.current_artifacts.mkdir(parents=True, exist_ok=True) + started = time.perf_counter() + status = "PASS" + metrics: dict[str, Any] = {} + error = "" + trace = "" + print(f"[{index:02d}/{len(selected):02d}] {item.case_id} ... ", end="", flush=True) + try: + metrics = item.function(ctx) or {} + except EvalSkip as exception: + status = "SKIP" + error = str(exception) + except Exception as exception: # 每个 case 独立记录，继续跑完全部。 + status = "FAIL" + error = str(exception) + trace = traceback.format_exc() + duration = time.perf_counter() - started + checks = ctx.checks - before_checks + results.append({ + "id": item.case_id, + "category": item.category, + "description": item.description, + "status": status, + "duration_seconds": round(duration, 6), + "checks": checks, + "metrics": metrics, + "error": error, + "traceback": trace, + }) + print(f"{status} ({duration:.3f}s, {checks} checks)") + if error: + print(f" {error}") + if status != "FAIL": + try: + ctx.current_artifacts.rmdir() + except OSError: + pass + duration_all = time.perf_counter() - started_all + passed = sum(item["status"] == "PASS" for item in results) + failed = sum(item["status"] == "FAIL" for item in results) + skipped = sum(item["status"] == "SKIP" for item in results) + report = { + "schema_version": 1, + "run_id": run_id, + "created_at": datetime.now(timezone.utc).isoformat(), + "mode": mode.name, + "seed": seed, + "enforce_performance": options.enforce_performance, + "environment": system_details(root, build_dir, ext), + "extension": str(ext), + "cases": results, + "summary": { + "status": "FAIL" if failed else "PASS", + "passed": passed, + "failed": failed, + "skipped": skipped, + "checks": ctx.checks, + "duration_seconds": round(duration_all, 6), + }, + } + json_path = output_dir / "report.json" + markdown_path = output_dir / "report.md" + json_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8") + write_markdown(report, markdown_path) + latest = output_dir.parent / "latest.json" + latest.write_text(json.dumps({"run_id": run_id, "report": str(json_path), + "status": report["summary"]["status"]}, indent=2) + "\n") + print() + print(f"result: {report['summary']['status']} - {passed} passed, {failed} failed, " + f"{skipped} skipped, {ctx.checks} checks, {duration_all:.3f}s") + print(f"JSON: {json_path}") + print(f"Markdown: {markdown_path}") + return 1 if failed else 0 + + +def parse_arguments(arguments: list[str]) -> argparse.Namespace: + if arguments and arguments[0] == "--worker": + return argparse.Namespace(worker=arguments[1:]) + root = Path(__file__).resolve().parents[3] + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", default=str(root)) + parser.add_argument("--build-dir", default=str(root / "vexdb_sqlite/build")) + parser.add_argument("--output-dir", default=str(root / "vexdb_sqlite/build/eval/vexfs")) + parser.add_argument("--mode", choices=sorted(MODES), default="quick") + parser.add_argument("--seed", type=int, default=20260718) + parser.add_argument("--filter", default="", help="按 case id 或 category 过滤") + parser.add_argument("--enforce-performance", action="store_true") + options = parser.parse_args(arguments) + options.worker = None + return options + + +def main(arguments: list[str] | None = None) -> int: + options = parse_arguments(sys.argv[1:] if arguments is None else arguments) + if options.worker is not None: + return worker_main(options.worker) + return run_eval(options) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/spec/sqlite/agent_files/vexfs_contract.yaml b/tests/spec/sqlite/agent_files/vexfs_contract.yaml new file mode 100644 index 0000000000..dc25faf368 --- /dev/null +++ b/tests/spec/sqlite/agent_files/vexfs_contract.yaml @@ -0,0 +1,111 @@ +name: sqlite__vexfs_contract +tags: [vexfs, sqlite, transaction] +description: VexFS 文件目录、事务、句柄写入、幂等请求和并发冲突合同 +steps: +- query: SELECT vexfs_contract_version(); + expect: + - - 0.3.0 +- query: SELECT vexfs_init(); + expect: + - - 1 +- query: SELECT vexfs_workspace_create('default'); + expect: + - - 1 +- query: SELECT vexfs_mkdir('default', '/notes/2026'); + expect: + - - 3 +- query: SELECT vexfs_write('default', '/notes/2026/hello.txt', X'68656C6C6F'); + expect: + - - 1 +- query: SELECT CAST(vexfs_read('default', '/notes/2026/hello.txt') AS TEXT); + expect: + - - hello +- statement: BEGIN; +- query: SELECT vexfs_write('default', '/notes/2026/hello.txt', 'rolled back'); + expect: + - - 2 +- statement: ROLLBACK; +- query: SELECT CAST(vexfs_read('default', '/notes/2026/hello.txt') AS TEXT); + expect: + - - hello +- query: SELECT length(vexfs_handle_open('default', '/notes/2026/hello.txt', 'rw', 'open-spec')); + expect: + - - 32 +- query: SELECT vexfs_handle_stage_write((SELECT result_text FROM _vexfs_requests WHERE request_id='open-spec'), 5, ' world', 'write-spec'); + expect: + - - 1 +- query: SELECT vexfs_handle_stage_write((SELECT result_text FROM _vexfs_requests WHERE request_id='open-spec'), 5, ' world', 'write-spec'); + expect: + - - 1 +- query: SELECT vexfs_handle_publish((SELECT result_text FROM _vexfs_requests WHERE request_id='open-spec'), 1, 'data', 'publish-spec'); + expect: + - - 2 +- query: SELECT vexfs_handle_close((SELECT result_text FROM _vexfs_requests WHERE request_id='open-spec'), 1, 'close-spec'); + expect: + - - closed +- query: SELECT CAST(vexfs_read('default', '/notes/2026/hello.txt') AS TEXT); + expect: + - - hello world +- query: SELECT length(vexfs_handle_open('default', '/notes/2026/hello.txt', 'rw', 'open-a')); + expect: + - - 32 +- query: SELECT length(vexfs_handle_open('default', '/notes/2026/hello.txt', 'rw', 'open-b')); + expect: + - - 32 +- query: SELECT vexfs_handle_stage_write((SELECT result_text FROM _vexfs_requests WHERE request_id='open-a'), 0, 'A', 'write-a'); + expect: + - - 1 +- query: SELECT vexfs_handle_stage_write((SELECT result_text FROM _vexfs_requests WHERE request_id='open-b'), 0, 'B', 'write-b'); + expect: + - - 1 +- query: SELECT vexfs_handle_publish((SELECT result_text FROM _vexfs_requests WHERE request_id='open-a'), 1, 'data', 'publish-a'); + expect: + - - 3 +- query: SELECT vexfs_handle_publish((SELECT result_text FROM _vexfs_requests WHERE request_id='open-b'), 1, 'data', 'publish-b'); + expect_error: write conflict +- query: SELECT vexfs_handle_close((SELECT result_text FROM _vexfs_requests WHERE request_id='open-a'), 0, 'close-a'); + expect: + - - closed +- query: SELECT vexfs_handle_close((SELECT result_text FROM _vexfs_requests WHERE request_id='open-b'), 0, 'close-b'); + expect: + - - closed +- query: SELECT CAST(vexfs_read('default', '/notes/2026/hello.txt') AS TEXT); + expect: + - - Aello world +- query: SELECT json_extract(value, '$.version'), json_extract(value, '$.current') FROM json_each(vexfs_history('default', '/notes/2026/hello.txt')); + expect: + - - 3 + - 1 + - - 2 + - 0 + - - 1 + - 0 +- query: SELECT json_extract(vexfs_history('default', '/notes/2026/hello.txt', 1, 0), '$.entries[0].version'), json_extract(vexfs_history('default', '/notes/2026/hello.txt', 1, 0), '$.next_before'); + expect: + - - 3 + - 3 +- query: SELECT CAST(vexfs_read_version('default', '/notes/2026/hello.txt', 1) AS TEXT); + expect: + - - hello +- query: SELECT vexfs_restore_version('default', '/notes/2026/hello.txt', 1, 3); + expect: + - - 4 +- query: SELECT CAST(vexfs_read('default', '/notes/2026/hello.txt') AS TEXT); + expect: + - - hello +- query: SELECT source_version_no FROM _vexfs_file_versions WHERE version_no=4; + expect: + - - 1 +- query: SELECT vexfs_move('default', '/notes/2026/hello.txt', '/notes/hello.txt'); + expect: + - - 1 +- query: SELECT vexfs_remove('default', '/notes/2026', 0); + expect: + - - 1 +- query: SELECT json_extract(value, '$.name'), json_extract(value, '$.inode'), json_extract(value, '$.kind'), json_extract(value, '$.size'), json_extract(value, '$.version') FROM json_each(vexfs_list('default', '/notes')); + expect: + - - hello.txt + - 4 + - file + - 5 + - 4 diff --git a/vexdb_sqlite/CMakeLists.txt b/vexdb_sqlite/CMakeLists.txt index 630eac50c2..505c449ede 100644 --- a/vexdb_sqlite/CMakeLists.txt +++ b/vexdb_sqlite/CMakeLists.txt @@ -101,6 +101,7 @@ endif() set(VEXDB_SQLITE_SRCS src/vexdb_sqlite_init.cpp + src/agent_files/vexfs_sqlite.cpp src/vtab/graph_index_vtab.cpp src/functions/distance_functions.cpp src/functions/vector_codec.cpp @@ -207,13 +208,43 @@ target_include_directories(vexdb_lite_static PRIVATE ${VEXDB_SQLITE_INCLUDES}) target_compile_definitions(vexdb_lite_static PRIVATE VEXDB_SQLITE_CORE=1) set_target_properties(vexdb_lite_static PROPERTIES OUTPUT_NAME "vexdb_lite_static") +# 平台挂载适配层共用的稳定 C ABI。FSKit/libfuse/WinFsp 都只接这一层， +# 具体状态仍由上面的 SQLite 插件通过 vexfs_* SQL 管理。 +add_library(vexfs_mount_contract STATIC + ${REPO_ROOT}/agent_files/mount/common/src/vexfs_mount_contract_sqlite.cpp) +target_include_directories(vexfs_mount_contract PUBLIC + ${REPO_ROOT}/agent_files/mount/common/include + ${CMAKE_CURRENT_SOURCE_DIR}/include + ${SQLITE_AMALG_DIR}) +target_compile_definitions(vexfs_mount_contract PRIVATE VEXDB_SQLITE_CORE=1) +target_link_libraries(vexfs_mount_contract PUBLIC vexdb_lite_static) +set_target_properties(vexfs_mount_contract PROPERTIES OUTPUT_NAME "vexfs_mount_contract") + +if(APPLE AND NOT CMAKE_SYSTEM_NAME STREQUAL "iOS") + enable_language(OBJCXX) + find_library(VEXFS_SYSTEM_SQLITE3 sqlite3 REQUIRED) + find_library(VEXFS_FOUNDATION Foundation REQUIRED) + add_executable(vexfs_cli + ${REPO_ROOT}/agent_files/cli/vexfs_main.cpp + ${REPO_ROOT}/agent_files/cli/vexfs_fskit_state.mm) + set_source_files_properties( + ${REPO_ROOT}/agent_files/cli/vexfs_fskit_state.mm + PROPERTIES COMPILE_FLAGS "-fobjc-arc") + target_include_directories(vexfs_cli PRIVATE + ${REPO_ROOT}/agent_files/mount/common/include + ${REPO_ROOT}/agent_files/cli) + target_link_libraries(vexfs_cli PRIVATE vexfs_mount_contract + ${VEXFS_SYSTEM_SQLITE3} ${VEXFS_FOUNDATION} Threads::Threads ${CMAKE_DL_LIBS}) + set_target_properties(vexfs_cli PROPERTIES OUTPUT_NAME "vexfs") +endif() + if(VEXDB_SQLITE_BUILD_TESTS) # 把 amalgamation 编成静态库供 smoke test 链接（模拟移动端宿主自带 sqlite3）。 add_library(sqlite3_amalg STATIC ${SQLITE_AMALG_DIR}/sqlite3.c) target_include_directories(sqlite3_amalg PUBLIC ${SQLITE_AMALG_DIR}) target_compile_definitions(sqlite3_amalg PRIVATE SQLITE_ENABLE_LOAD_EXTENSION=1) - foreach(smoke m0_static_smoke m1_distance_smoke m2_vtab_smoke m3_hnsw_smoke) + foreach(smoke m0_static_smoke m1_distance_smoke m2_vtab_smoke m3_hnsw_smoke vexfs_static_smoke) add_executable(${smoke} test/${smoke}.c) target_include_directories(${smoke} PRIVATE ${VEXDB_SQLITE_INCLUDES}) # 宿主侧静态注册形态：直接用真实 sqlite3 符号（非 ext 间接表）。 @@ -228,4 +259,11 @@ if(VEXDB_SQLITE_BUILD_TESTS) target_compile_definitions(m3p_parallel_smoke PRIVATE VEXDB_SQLITE_CORE=1) target_link_libraries(m3p_parallel_smoke PRIVATE vexdb_lite_static sqlite3_amalg Threads::Threads ${CMAKE_DL_LIBS}) + + add_executable(vexfs_mount_contract_smoke + ${REPO_ROOT}/agent_files/mount/common/test/vexfs_mount_contract_smoke.cpp) + target_include_directories(vexfs_mount_contract_smoke PRIVATE + ${REPO_ROOT}/agent_files/mount/common/include ${SQLITE_AMALG_DIR}) + target_link_libraries(vexfs_mount_contract_smoke PRIVATE + vexfs_mount_contract sqlite3_amalg Threads::Threads ${CMAKE_DL_LIBS}) endif() diff --git a/vexdb_sqlite/include/agent_files/vexfs_sqlite.h b/vexdb_sqlite/include/agent_files/vexfs_sqlite.h new file mode 100644 index 0000000000..812f90edd9 --- /dev/null +++ b/vexdb_sqlite/include/agent_files/vexfs_sqlite.h @@ -0,0 +1,25 @@ +// VexFS SQLite 文件系统合同。 +// +// 这是数据库插件能力，不是独立 SDK：注册后，宿主通过 SQL 操作文件、目录和 +// 挂载写入句柄。所有写操作都参加调用者当前 SQLite 事务，插件不会自行提交。 +#ifndef VEXFS_SQLITE_H +#define VEXFS_SQLITE_H + +#ifdef VEXDB_SQLITE_CORE +#include "sqlite3.h" +#else +#include "sqlite3ext.h" +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +// 注册 vexfs_* SQL 函数。成功返回 SQLITE_OK。 +int vexfs_sqlite_register(sqlite3 *db); + +#ifdef __cplusplus +} +#endif + +#endif // VEXFS_SQLITE_H diff --git a/vexdb_sqlite/src/agent_files/vexfs_sqlite.cpp b/vexdb_sqlite/src/agent_files/vexfs_sqlite.cpp new file mode 100644 index 0000000000..f04fd8ff8e --- /dev/null +++ b/vexdb_sqlite/src/agent_files/vexfs_sqlite.cpp @@ -0,0 +1,2139 @@ +#include "vexdb_sqlite_internal.h" + +#include "agent_files/vexfs_sqlite.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(_WIN32) +#include +#else +#include +#endif + +namespace { + +constexpr const char *kContractVersion = "0.3.0"; +constexpr sqlite3_int64 kMaxStagedBytes = 128LL * 1024LL * 1024LL; +constexpr sqlite3_int64 kInitialStagingCapacity = 64LL * 1024LL; + +class SqlError : public std::runtime_error { + public: + explicit SqlError(const std::string &message, int code = SQLITE_ERROR) + : std::runtime_error(message), code_(code) {} + + int code() const { return code_; } + + private: + int code_; +}; + +class Statement { + public: + Statement(sqlite3 *db, const char *sql) : db_(db) { + const int rc = sqlite3_prepare_v2(db, sql, -1, &stmt_, nullptr); + if (rc != SQLITE_OK) Throw("prepare", rc); + } + + ~Statement() { + if (stmt_ != nullptr) sqlite3_finalize(stmt_); + } + + Statement(const Statement &) = delete; + Statement &operator=(const Statement &) = delete; + + void BindInt64(int index, sqlite3_int64 value) { + Check(sqlite3_bind_int64(stmt_, index, value), "bind integer"); + } + + void BindInt(int index, int value) { + Check(sqlite3_bind_int(stmt_, index, value), "bind integer"); + } + + void BindText(int index, const std::string &value) { + Check(sqlite3_bind_text(stmt_, index, value.data(), static_cast(value.size()), + SQLITE_TRANSIENT), + "bind text"); + } + + void BindBlob(int index, const std::vector &value) { + const void *data = value.empty() ? static_cast("") : value.data(); + Check(sqlite3_bind_blob64(stmt_, index, data, value.size(), SQLITE_STATIC), + "bind blob"); + } + + void BindNull(int index) { Check(sqlite3_bind_null(stmt_, index), "bind NULL"); } + + bool Row() { + const int rc = sqlite3_step(stmt_); + if (rc == SQLITE_ROW) return true; + if (rc == SQLITE_DONE) return false; + Throw("step", rc); + return false; + } + + void Done() { + const int rc = sqlite3_step(stmt_); + if (rc != SQLITE_DONE) Throw("step", rc); + } + + sqlite3_int64 Int64(int column) const { return sqlite3_column_int64(stmt_, column); } + int Int(int column) const { return sqlite3_column_int(stmt_, column); } + int Type(int column) const { return sqlite3_column_type(stmt_, column); } + + std::string Text(int column) const { + const unsigned char *text = sqlite3_column_text(stmt_, column); + const int bytes = sqlite3_column_bytes(stmt_, column); + return text == nullptr ? std::string() : std::string(reinterpret_cast(text), bytes); + } + + std::vector Blob(int column) const { + const auto *data = static_cast(sqlite3_column_blob(stmt_, column)); + const int bytes = sqlite3_column_bytes(stmt_, column); + if (data == nullptr || bytes == 0) return {}; + return std::vector(data, data + bytes); + } + + private: + void Check(int rc, const char *action) { + if (rc != SQLITE_OK) Throw(action, rc); + } + + [[noreturn]] void Throw(const char *action, int rc) { + throw SqlError(std::string(action) + ": " + sqlite3_errmsg(db_) + + " (rc=" + std::to_string(rc) + ")", rc); + } + + sqlite3 *db_ = nullptr; + sqlite3_stmt *stmt_ = nullptr; +}; + +class ReadOnlyBlob { + public: + ReadOnlyBlob(sqlite3 *db, sqlite3_int64 rowid) { + const int rc = sqlite3_blob_open(db, "main", "_vexfs_file_versions", "content", + rowid, 0, &blob_); + if (rc != SQLITE_OK) throw SqlError("cannot open version BLOB", rc); + } + + ~ReadOnlyBlob() { + if (blob_ != nullptr) sqlite3_blob_close(blob_); + } + + int size() const { return sqlite3_blob_bytes(blob_); } + + void Read(void *output, int bytes, int offset) { + const int rc = sqlite3_blob_read(blob_, output, bytes, offset); + if (rc != SQLITE_OK) throw SqlError("cannot read version BLOB", rc); + } + + ReadOnlyBlob(const ReadOnlyBlob &) = delete; + ReadOnlyBlob &operator=(const ReadOnlyBlob &) = delete; + + private: + sqlite3_blob *blob_ = nullptr; +}; + +void Exec(sqlite3 *db, const char *sql) { + char *message = nullptr; + const int rc = sqlite3_exec(db, sql, nullptr, nullptr, &message); + if (rc != SQLITE_OK) { + const std::string error = message == nullptr ? sqlite3_errmsg(db) : message; + sqlite3_free(message); + throw SqlError(error, rc); + } +} + +class Savepoint { + public: + Savepoint(sqlite3 *db, const char *name) : db_(db), name_(name) { + Exec(db_, ("SAVEPOINT " + name_).c_str()); + } + + ~Savepoint() { + if (!active_) return; + try { + Exec(db_, ("ROLLBACK TO SAVEPOINT " + name_).c_str()); + Exec(db_, ("RELEASE SAVEPOINT " + name_).c_str()); + } catch (...) { + } + } + + void Release() { + Exec(db_, ("RELEASE SAVEPOINT " + name_).c_str()); + active_ = false; + } + + Savepoint(const Savepoint &) = delete; + Savepoint &operator=(const Savepoint &) = delete; + + private: + sqlite3 *db_; + std::string name_; + bool active_ = true; +}; + +void AcquireWriteLock(sqlite3 *db) { + // SQLite 从读事务升级为写事务时可能不执行 busy handler。先做一次无副作用 + // UPDATE，让 busy_timeout 在读取业务状态前生效，并在后续 savepoint 内保持写锁。 + Exec(db, "UPDATE _vexfs_meta SET value=value WHERE key='contract_version'"); +} + +bool HasVersionReferences(sqlite3 *db) { + Statement statement(db, + "SELECT 1 FROM pragma_table_info('_vexfs_file_versions') " + "WHERE name='source_version_no' LIMIT 1"); + return statement.Row(); +} + +bool HasVersionTable(sqlite3 *db) { + Statement statement(db, + "SELECT 1 FROM sqlite_master WHERE type='table' " + "AND name='_vexfs_file_versions' LIMIT 1"); + return statement.Row(); +} + +void EnsureVersionReferences(sqlite3 *db) { + // 极早期 0.1 测试库允许只包含 handle/staging 表；没有版本表时不凭空补半张 schema。 + if (HasVersionTable(db) && !HasVersionReferences(db)) { + Exec(db, "ALTER TABLE _vexfs_file_versions ADD COLUMN source_version_no INTEGER"); + } +} + +void MigrateSchema010To020(sqlite3 *db) { + Exec(db, "SAVEPOINT vexfs_migrate_010_020"); + try { + Exec(db, R"SQL( +ALTER TABLE _vexfs_handles ADD COLUMN owner_session TEXT; +ALTER TABLE _vexfs_staging RENAME TO _vexfs_staging_010; +CREATE TABLE _vexfs_staging( + handle_id TEXT PRIMARY KEY, + generation INTEGER NOT NULL, + content BLOB NOT NULL, + logical_size INTEGER NOT NULL, + capacity INTEGER NOT NULL, + created_at INTEGER NOT NULL DEFAULT (CAST(strftime('%s','now') AS INTEGER) * 1000), + updated_at INTEGER NOT NULL DEFAULT (CAST(strftime('%s','now') AS INTEGER) * 1000) +); +INSERT INTO _vexfs_staging(handle_id, generation, content, logical_size, capacity, created_at) +SELECT s.handle_id, s.generation, s.content, length(s.content), length(s.content), s.created_at +FROM _vexfs_staging_010 s +JOIN _vexfs_handles h ON h.id=s.handle_id AND h.dirty_generation=s.generation; +DROP TABLE _vexfs_staging_010; +ALTER TABLE _vexfs_requests ADD COLUMN request_fingerprint BLOB NOT NULL DEFAULT X''; +DELETE FROM _vexfs_requests; +UPDATE _vexfs_inodes SET current_version=1 +WHERE kind='directory' AND deleted_at IS NULL AND current_version<1; +UPDATE _vexfs_meta SET value='0.2.0' WHERE key='contract_version'; +)SQL"); + Exec(db, "RELEASE SAVEPOINT vexfs_migrate_010_020"); + } catch (...) { + try { + Exec(db, "ROLLBACK TO SAVEPOINT vexfs_migrate_010_020"); + Exec(db, "RELEASE SAVEPOINT vexfs_migrate_010_020"); + } catch (...) { + } + throw; + } +} + +void MigrateSchema020To030(sqlite3 *db) { + Savepoint savepoint(db, "vexfs_migrate_020_030"); + Exec(db, R"SQL( +CREATE TABLE IF NOT EXISTS _vexfs_staging_data( + handle_id TEXT PRIMARY KEY, + content BLOB NOT NULL +); +INSERT OR REPLACE INTO _vexfs_staging_data(handle_id, content) +SELECT handle_id, content FROM _vexfs_staging; +UPDATE _vexfs_staging SET content=X'' WHERE length(content)>0; +INSERT OR REPLACE INTO _vexfs_meta(key, value) VALUES('staging_layout', 'split-v1'); +CREATE TABLE IF NOT EXISTS _vexfs_mount_sessions( + workspace_id INTEGER PRIMARY KEY, + session_id TEXT NOT NULL UNIQUE, + lease_until INTEGER NOT NULL, + created_at INTEGER NOT NULL DEFAULT (CAST(strftime('%s','now') AS INTEGER) * 1000), + updated_at INTEGER NOT NULL DEFAULT (CAST(strftime('%s','now') AS INTEGER) * 1000) +); +)SQL"); + EnsureVersionReferences(db); + // 合同标记必须最后写入；前面的布局升级有任何一步失败，整个 savepoint 回滚。 + Exec(db, "UPDATE _vexfs_meta SET value='0.3.0' WHERE key='contract_version'"); + savepoint.Release(); +} + +void EnsureSplitStagingLayout(sqlite3 *db) { + Statement marker(db, + "SELECT 1 FROM _vexfs_meta WHERE key='staging_layout' AND value='split-v1' LIMIT 1"); + if (marker.Row()) return; + + Exec(db, "SAVEPOINT vexfs_split_staging"); + try { + Exec(db, R"SQL( +CREATE TABLE IF NOT EXISTS _vexfs_staging_data( + handle_id TEXT PRIMARY KEY, + content BLOB NOT NULL +); +INSERT OR REPLACE INTO _vexfs_staging_data(handle_id, content) +SELECT handle_id, content FROM _vexfs_staging; +UPDATE _vexfs_staging SET content=X'' WHERE length(content)>0; +INSERT OR REPLACE INTO _vexfs_meta(key, value) VALUES('staging_layout', 'split-v1'); +)SQL"); + Exec(db, "RELEASE SAVEPOINT vexfs_split_staging"); + } catch (...) { + try { + Exec(db, "ROLLBACK TO SAVEPOINT vexfs_split_staging"); + Exec(db, "RELEASE SAVEPOINT vexfs_split_staging"); + } catch (...) { + } + throw; + } +} + +void EnsureRuntimeTables(sqlite3 *db) { + { + Statement exists(db, + "SELECT 1 FROM sqlite_master WHERE type='table' " + "AND name='_vexfs_mount_sessions' LIMIT 1"); + if (!exists.Row()) { + Exec(db, R"SQL( +CREATE TABLE IF NOT EXISTS _vexfs_mount_sessions( + workspace_id INTEGER PRIMARY KEY, + session_id TEXT NOT NULL UNIQUE, + lease_until INTEGER NOT NULL, + created_at INTEGER NOT NULL DEFAULT (CAST(strftime('%s','now') AS INTEGER) * 1000), + updated_at INTEGER NOT NULL DEFAULT (CAST(strftime('%s','now') AS INTEGER) * 1000) +); +)SQL"); + } + } + EnsureVersionReferences(db); +} + +void EnsureSchema(sqlite3 *db) { + bool schema_exists = false; + std::string version; + { + Statement exists(db, + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='_vexfs_meta' LIMIT 1"); + if (exists.Row()) { + schema_exists = true; + Statement check(db, + "SELECT value FROM _vexfs_meta WHERE key='contract_version' LIMIT 1"); + if (!check.Row()) throw SqlError("VexFS schema has no contract version"); + version = check.Text(0); + } + } + if (schema_exists) { + if (version == "0.2.0" && sqlite3_db_readonly(db, "main") == 1) { + // 0.3 的版本读取 API 只依赖 0.2 已有的 file_versions/commits。 + // 只读连接允许兼容读取，但不会创建 session 表或改写合同标记。 + return; + } + if (version == "0.1.0") { + MigrateSchema010To020(db); + version = "0.2.0"; + } + if (version == "0.2.0") { + MigrateSchema020To030(db); + version = "0.3.0"; + } + if (version != kContractVersion) + throw SqlError("unsupported VexFS schema version: " + version, SQLITE_MISMATCH); + EnsureSplitStagingLayout(db); + EnsureRuntimeTables(db); + return; + } + Savepoint savepoint(db, "vexfs_schema_init"); + Exec(db, R"SQL( +CREATE TABLE IF NOT EXISTS _vexfs_meta( + key TEXT PRIMARY KEY, + value TEXT NOT NULL +); +CREATE TABLE IF NOT EXISTS _vexfs_workspaces( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL UNIQUE, + root_inode INTEGER, + head_commit INTEGER, + created_at INTEGER NOT NULL DEFAULT (CAST(strftime('%s','now') AS INTEGER) * 1000) +); +CREATE TABLE IF NOT EXISTS _vexfs_inodes( + id INTEGER PRIMARY KEY, + workspace_id INTEGER NOT NULL, + kind TEXT NOT NULL CHECK(kind IN ('file','directory')), + mode INTEGER NOT NULL, + size INTEGER NOT NULL DEFAULT 0, + current_version INTEGER NOT NULL DEFAULT 0, + created_at INTEGER NOT NULL DEFAULT (CAST(strftime('%s','now') AS INTEGER) * 1000), + updated_at INTEGER NOT NULL DEFAULT (CAST(strftime('%s','now') AS INTEGER) * 1000), + deleted_at INTEGER +); +CREATE INDEX IF NOT EXISTS _vexfs_inodes_workspace_idx + ON _vexfs_inodes(workspace_id, id); +CREATE TABLE IF NOT EXISTS _vexfs_dentries( + workspace_id INTEGER NOT NULL, + parent_inode INTEGER NOT NULL, + name TEXT NOT NULL COLLATE BINARY, + inode_id INTEGER NOT NULL, + PRIMARY KEY(workspace_id, parent_inode, name) +); +CREATE UNIQUE INDEX IF NOT EXISTS _vexfs_dentries_inode_idx + ON _vexfs_dentries(workspace_id, inode_id); +CREATE TABLE IF NOT EXISTS _vexfs_commits( + id INTEGER PRIMARY KEY, + workspace_id INTEGER NOT NULL, + parent_commit INTEGER, + message TEXT NOT NULL, + created_at INTEGER NOT NULL DEFAULT (CAST(strftime('%s','now') AS INTEGER) * 1000) +); +CREATE TABLE IF NOT EXISTS _vexfs_file_versions( + id INTEGER PRIMARY KEY, + inode_id INTEGER NOT NULL, + version_no INTEGER NOT NULL, + commit_id INTEGER NOT NULL, + content BLOB NOT NULL, + size INTEGER NOT NULL, + source_version_no INTEGER, + created_at INTEGER NOT NULL DEFAULT (CAST(strftime('%s','now') AS INTEGER) * 1000), + UNIQUE(inode_id, version_no) +); +CREATE TABLE IF NOT EXISTS _vexfs_handles( + id TEXT PRIMARY KEY, + workspace_id INTEGER NOT NULL, + inode_id INTEGER NOT NULL, + open_flags TEXT NOT NULL, + writable INTEGER NOT NULL, + expected_version INTEGER NOT NULL, + dirty_generation INTEGER NOT NULL DEFAULT 0, + published_generation INTEGER NOT NULL DEFAULT 0, + state TEXT NOT NULL CHECK(state IN ('open','retained','closed')), + lease_until INTEGER, + owner_session TEXT, + opened_at INTEGER NOT NULL DEFAULT (CAST(strftime('%s','now') AS INTEGER) * 1000), + updated_at INTEGER NOT NULL DEFAULT (CAST(strftime('%s','now') AS INTEGER) * 1000) +); +CREATE INDEX IF NOT EXISTS _vexfs_handles_workspace_idx + ON _vexfs_handles(workspace_id, state); +CREATE TABLE IF NOT EXISTS _vexfs_staging( + handle_id TEXT PRIMARY KEY, + generation INTEGER NOT NULL, + content BLOB NOT NULL, + logical_size INTEGER NOT NULL, + capacity INTEGER NOT NULL, + created_at INTEGER NOT NULL DEFAULT (CAST(strftime('%s','now') AS INTEGER) * 1000), + updated_at INTEGER NOT NULL DEFAULT (CAST(strftime('%s','now') AS INTEGER) * 1000) +); +CREATE TABLE IF NOT EXISTS _vexfs_staging_data( + handle_id TEXT PRIMARY KEY, + content BLOB NOT NULL +); +CREATE TABLE IF NOT EXISTS _vexfs_requests( + request_id TEXT PRIMARY KEY, + operation TEXT NOT NULL, + request_fingerprint BLOB NOT NULL, + result_integer INTEGER, + result_text TEXT, + created_at INTEGER NOT NULL DEFAULT (CAST(strftime('%s','now') AS INTEGER) * 1000) +); +CREATE TABLE IF NOT EXISTS _vexfs_mount_sessions( + workspace_id INTEGER PRIMARY KEY, + session_id TEXT NOT NULL UNIQUE, + lease_until INTEGER NOT NULL, + created_at INTEGER NOT NULL DEFAULT (CAST(strftime('%s','now') AS INTEGER) * 1000), + updated_at INTEGER NOT NULL DEFAULT (CAST(strftime('%s','now') AS INTEGER) * 1000) +); +INSERT OR IGNORE INTO _vexfs_meta(key, value) VALUES('contract_version', '0.3.0'); +INSERT OR IGNORE INTO _vexfs_meta(key, value) VALUES('staging_layout', 'split-v1'); +)SQL"); + savepoint.Release(); +} + +std::string RequiredText(sqlite3_value *value, const char *name) { + if (sqlite3_value_type(value) == SQLITE_NULL) { + throw SqlError(std::string(name) + " must not be NULL", SQLITE_MISMATCH); + } + const unsigned char *text = sqlite3_value_text(value); + const int bytes = sqlite3_value_bytes(value); + if (text == nullptr) return {}; + return std::string(reinterpret_cast(text), bytes); +} + +std::vector RequiredBlob(sqlite3_value *value, const char *name) { + if (sqlite3_value_type(value) == SQLITE_NULL) { + throw SqlError(std::string(name) + " must not be NULL", SQLITE_MISMATCH); + } + const auto *data = static_cast(sqlite3_value_blob(value)); + const int bytes = sqlite3_value_bytes(value); + if (bytes <= 0) return {}; + if (data == nullptr) throw SqlError(std::string(name) + " is not readable", SQLITE_MISMATCH); + return std::vector(data, data + bytes); +} + +sqlite3_int64 RequiredPositiveInteger(sqlite3_value *value, const char *name) { + if (sqlite3_value_type(value) != SQLITE_INTEGER) { + throw SqlError(std::string(name) + " must be a positive integer", SQLITE_MISMATCH); + } + const sqlite3_int64 result = sqlite3_value_int64(value); + if (result <= 0) { + throw SqlError(std::string(name) + " must be a positive integer", SQLITE_RANGE); + } + return result; +} + +sqlite3_int64 RequiredNonnegativeInteger(sqlite3_value *value, const char *name) { + if (sqlite3_value_type(value) != SQLITE_INTEGER) { + throw SqlError(std::string(name) + " must be a non-negative integer", SQLITE_MISMATCH); + } + const sqlite3_int64 result = sqlite3_value_int64(value); + if (result < 0) { + throw SqlError(std::string(name) + " must be a non-negative integer", SQLITE_RANGE); + } + return result; +} + +std::vector PathParts(const std::string &path) { + if (path.empty() || path.front() != '/') throw SqlError("path must be absolute", SQLITE_MISMATCH); + if (path.find('\0') != std::string::npos) throw SqlError("path contains NUL", SQLITE_MISMATCH); + + std::vector parts; + size_t start = 1; + while (start <= path.size()) { + const size_t slash = path.find('/', start); + const size_t end = slash == std::string::npos ? path.size() : slash; + if (end > start) { + std::string part = path.substr(start, end - start); + if (part == "." || part == "..") + throw SqlError("path must not contain . or ..", SQLITE_MISMATCH); + if (part.size() > 255) + throw SqlError("path component is longer than 255 bytes", SQLITE_MISMATCH); + parts.push_back(std::move(part)); + } + if (slash == std::string::npos) break; + start = slash + 1; + } + return parts; +} + +struct Workspace { + sqlite3_int64 id = 0; + sqlite3_int64 root_inode = 0; + sqlite3_int64 head_commit = 0; +}; + +Workspace FindWorkspace(sqlite3 *db, const std::string &name) { + Statement statement(db, + "SELECT id, root_inode, COALESCE(head_commit, 0) " + "FROM _vexfs_workspaces WHERE name = ?1"); + statement.BindText(1, name); + if (!statement.Row()) throw SqlError("workspace not found: " + name, SQLITE_NOTFOUND); + return {statement.Int64(0), statement.Int64(1), statement.Int64(2)}; +} + +struct Node { + sqlite3_int64 id = 0; + std::string kind; + sqlite3_int64 size = 0; + sqlite3_int64 version = 0; + int mode = 0; + sqlite3_int64 created_at = 0; + sqlite3_int64 updated_at = 0; +}; + +struct VersionStorage { + sqlite3_int64 rowid = 0; + sqlite3_int64 size = 0; + sqlite3_int64 canonical_version = 0; +}; + +VersionStorage ResolveVersionStorage(sqlite3 *db, sqlite3_int64 inode_id, + sqlite3_int64 version) { + if (!HasVersionReferences(db)) { + Statement statement(db, + "SELECT rowid,size,version_no FROM _vexfs_file_versions " + "WHERE inode_id=?1 AND version_no=?2"); + statement.BindInt64(1, inode_id); + statement.BindInt64(2, version); + if (!statement.Row()) { + throw SqlError("file version not found: " + std::to_string(version), + SQLITE_NOTFOUND); + } + return {statement.Int64(0), statement.Int64(1), statement.Int64(2)}; + } + + Statement statement(db, R"SQL( +SELECT CASE WHEN v.source_version_no IS NULL THEN v.rowid ELSE source.rowid END, + v.size, + COALESCE(v.source_version_no,v.version_no), + CASE WHEN v.source_version_no IS NULL THEN v.size ELSE source.size END +FROM _vexfs_file_versions v +LEFT JOIN _vexfs_file_versions source + ON source.inode_id=v.inode_id AND source.version_no=v.source_version_no +WHERE v.inode_id=?1 AND v.version_no=?2 +)SQL"); + statement.BindInt64(1, inode_id); + statement.BindInt64(2, version); + if (!statement.Row()) { + throw SqlError("file version not found: " + std::to_string(version), SQLITE_NOTFOUND); + } + if (statement.Type(0) == SQLITE_NULL || statement.Int64(1) < 0 || + statement.Int64(1) != statement.Int64(3)) { + throw SqlError("file version content reference is corrupt", SQLITE_CORRUPT); + } + return {statement.Int64(0), statement.Int64(1), statement.Int64(2)}; +} + +std::vector ReadVersionContent(sqlite3 *db, sqlite3_int64 inode_id, + sqlite3_int64 version) { + const VersionStorage storage = ResolveVersionStorage(db, inode_id, version); + if (storage.size > kMaxStagedBytes) { + throw SqlError("file version is larger than 128 MiB", SQLITE_CORRUPT); + } + ReadOnlyBlob blob(db, storage.rowid); + if (blob.size() != storage.size) { + throw SqlError("file version size does not match content", SQLITE_CORRUPT); + } + std::vector content(static_cast(storage.size)); + constexpr int kChunkSize = 64 * 1024; + for (sqlite3_int64 offset = 0; offset < storage.size; offset += kChunkSize) { + const int bytes = static_cast( + std::min(kChunkSize, storage.size - offset)); + blob.Read(content.data() + offset, bytes, static_cast(offset)); + } + return content; +} + +bool FindChild(sqlite3 *db, sqlite3_int64 workspace_id, sqlite3_int64 parent, + const std::string &name, Node *node) { + Statement statement(db, + "SELECT i.id, i.kind, i.size, i.current_version, i.mode, i.created_at, i.updated_at " + "FROM _vexfs_dentries d JOIN _vexfs_inodes i ON i.id = d.inode_id " + "WHERE d.workspace_id = ?1 AND d.parent_inode = ?2 AND d.name = ?3 " + "AND i.deleted_at IS NULL"); + statement.BindInt64(1, workspace_id); + statement.BindInt64(2, parent); + statement.BindText(3, name); + if (!statement.Row()) return false; + *node = {statement.Int64(0), statement.Text(1), statement.Int64(2), + statement.Int64(3), statement.Int(4), statement.Int64(5), statement.Int64(6)}; + return true; +} + +Node RootNode(sqlite3 *db, const Workspace &workspace) { + Statement statement(db, + "SELECT id, kind, size, current_version, mode, created_at, updated_at FROM _vexfs_inodes " + "WHERE id = ?1 AND workspace_id = ?2 AND deleted_at IS NULL"); + statement.BindInt64(1, workspace.root_inode); + statement.BindInt64(2, workspace.id); + if (!statement.Row()) throw SqlError("workspace root is missing", SQLITE_CORRUPT); + return {statement.Int64(0), statement.Text(1), statement.Int64(2), + statement.Int64(3), statement.Int(4), statement.Int64(5), statement.Int64(6)}; +} + +bool TryResolve(sqlite3 *db, const Workspace &workspace, const std::vector &parts, + Node *node) { + Node current = RootNode(db, workspace); + for (const std::string &part : parts) { + if (current.kind != "directory") return false; + if (!FindChild(db, workspace.id, current.id, part, ¤t)) return false; + } + *node = std::move(current); + return true; +} + +Node Resolve(sqlite3 *db, const Workspace &workspace, const std::vector &parts) { + Node node; + if (!TryResolve(db, workspace, parts, &node)) throw SqlError("path not found", SQLITE_NOTFOUND); + return node; +} + +std::pair ResolveParent(sqlite3 *db, const Workspace &workspace, + const std::vector &parts) { + if (parts.empty()) throw SqlError("root path has no parent", SQLITE_MISMATCH); + std::vector parent_parts(parts.begin(), parts.end() - 1); + Node parent = Resolve(db, workspace, parent_parts); + if (parent.kind != "directory") throw SqlError("parent is not a directory", SQLITE_MISMATCH); + return {std::move(parent), parts.back()}; +} + +std::string PathForInode(sqlite3 *db, const Workspace &workspace, sqlite3_int64 inode) { + if (inode == workspace.root_inode) return "/"; + Statement statement(db, R"SQL( +WITH RECURSIVE ancestors(inode_id, parent_inode, path, depth) AS ( + SELECT d.inode_id, d.parent_inode, d.name, 1 + FROM _vexfs_dentries d + JOIN _vexfs_inodes i ON i.id=d.inode_id AND i.deleted_at IS NULL + WHERE d.workspace_id=?1 AND d.inode_id=?2 + UNION ALL + SELECT d.inode_id, d.parent_inode, d.name || '/' || a.path, a.depth+1 + FROM _vexfs_dentries d + JOIN ancestors a ON a.parent_inode=d.inode_id + JOIN _vexfs_inodes i ON i.id=d.inode_id AND i.deleted_at IS NULL + WHERE d.workspace_id=?1 AND a.depth < 1024 +) +SELECT '/' || path FROM ancestors WHERE parent_inode=?3 ORDER BY depth DESC LIMIT 1 +)SQL"); + statement.BindInt64(1, workspace.id); + statement.BindInt64(2, inode); + statement.BindInt64(3, workspace.root_inode); + if (!statement.Row()) throw SqlError("inode path not found", SQLITE_NOTFOUND); + return statement.Text(0); +} + +sqlite3_int64 CreateCommit(sqlite3 *db, const Workspace &workspace, const std::string &message) { + Savepoint savepoint(db, "vexfs_create_commit"); + Statement insert(db, + "INSERT INTO _vexfs_commits(workspace_id, parent_commit, message) " + "SELECT id, NULLIF(head_commit, 0), ?2 FROM _vexfs_workspaces WHERE id=?1"); + insert.BindInt64(1, workspace.id); + insert.BindText(2, message); + insert.Done(); + if (sqlite3_changes64(db) != 1) { + throw SqlError("workspace not found", SQLITE_NOTFOUND); + } + const sqlite3_int64 commit = sqlite3_last_insert_rowid(db); + + Statement update(db, "UPDATE _vexfs_workspaces SET head_commit = ?1 WHERE id = ?2"); + update.BindInt64(1, commit); + update.BindInt64(2, workspace.id); + update.Done(); + savepoint.Release(); + return commit; +} + +sqlite3_int64 CreateInode(sqlite3 *db, sqlite3_int64 workspace_id, const char *kind, int mode) { + Statement insert(db, + "INSERT INTO _vexfs_inodes(workspace_id, kind, mode, current_version) " + "VALUES(?1, ?2, ?3, ?4)"); + insert.BindInt64(1, workspace_id); + insert.BindText(2, kind); + insert.BindInt(3, mode); + insert.BindInt64(4, std::strcmp(kind, "directory") == 0 ? 1 : 0); + insert.Done(); + return sqlite3_last_insert_rowid(db); +} + +void TouchDirectory(sqlite3 *db, sqlite3_int64 workspace_id, sqlite3_int64 inode) { + Statement update(db, + "UPDATE _vexfs_inodes SET current_version=current_version+1, " + "updated_at=CAST(strftime('%s','now') AS INTEGER)*1000 " + "WHERE id=?1 AND workspace_id=?2 AND kind='directory' AND deleted_at IS NULL"); + update.BindInt64(1, inode); + update.BindInt64(2, workspace_id); + update.Done(); +} + +void AddDentry(sqlite3 *db, sqlite3_int64 workspace_id, sqlite3_int64 parent, + const std::string &name, sqlite3_int64 inode) { + Statement insert(db, + "INSERT INTO _vexfs_dentries(workspace_id, parent_inode, name, inode_id) " + "VALUES(?1, ?2, ?3, ?4)"); + insert.BindInt64(1, workspace_id); + insert.BindInt64(2, parent); + insert.BindText(3, name); + insert.BindInt64(4, inode); + insert.Done(); + TouchDirectory(db, workspace_id, parent); +} + +sqlite3_int64 StoreVersion(sqlite3 *db, const Workspace &workspace, const Node &node, + const std::vector &content, + const std::string &message) { + Savepoint savepoint(db, "vexfs_store_version"); + const sqlite3_int64 version = node.version + 1; + Statement advance(db, + "UPDATE _vexfs_inodes SET size=?1, current_version=?2, " + "updated_at=CAST(strftime('%s','now') AS INTEGER)*1000 " + "WHERE id=?3 AND current_version=?4"); + advance.BindInt64(1, static_cast(content.size())); + advance.BindInt64(2, version); + advance.BindInt64(3, node.id); + advance.BindInt64(4, node.version); + advance.Done(); + if (sqlite3_changes64(db) != 1) { + throw SqlError("version conflict: file changed before publish", SQLITE_CONSTRAINT); + } + + const sqlite3_int64 commit = CreateCommit(db, workspace, message); + Statement insert(db, + "INSERT INTO _vexfs_file_versions(inode_id, version_no, commit_id, content, size) " + "VALUES(?1, ?2, ?3, ?4, ?5)"); + insert.BindInt64(1, node.id); + insert.BindInt64(2, version); + insert.BindInt64(3, commit); + insert.BindBlob(4, content); + insert.BindInt64(5, static_cast(content.size())); + insert.Done(); + savepoint.Release(); + return version; +} + +sqlite3_int64 StoreVersionFromVersion(sqlite3 *db, const Workspace &workspace, const Node &node, + sqlite3_int64 source_version, + const std::string &message) { + Savepoint savepoint(db, "vexfs_restore_version"); + AcquireWriteLock(db); + const VersionStorage source = ResolveVersionStorage(db, node.id, source_version); + const sqlite3_int64 version = node.version + 1; + Statement advance(db, + "UPDATE _vexfs_inodes SET size=?1, current_version=?2, " + "updated_at=CAST(strftime('%s','now') AS INTEGER)*1000 " + "WHERE id=?3 AND current_version=?4"); + advance.BindInt64(1, source.size); + advance.BindInt64(2, version); + advance.BindInt64(3, node.id); + advance.BindInt64(4, node.version); + advance.Done(); + if (sqlite3_changes64(db) != 1) { + throw SqlError("version conflict: file changed before restore", SQLITE_CONSTRAINT); + } + const sqlite3_int64 commit = CreateCommit(db, workspace, message); + Statement insert(db, + "INSERT INTO _vexfs_file_versions" + "(inode_id,version_no,commit_id,content,size,source_version_no) " + "VALUES(?1,?2,?3,X'',?4,?5)"); + insert.BindInt64(1, node.id); + insert.BindInt64(2, version); + insert.BindInt64(3, commit); + insert.BindInt64(4, source.size); + insert.BindInt64(5, source.canonical_version); + insert.Done(); + if (sqlite3_changes64(db) != 1) { + throw SqlError("file version not found: " + std::to_string(source_version), SQLITE_NOTFOUND); + } + savepoint.Release(); + return version; +} + +std::vector ReadNode(sqlite3 *db, const Node &node) { + if (node.kind != "file") throw SqlError("path is not a file", SQLITE_MISMATCH); + if (node.version == 0) return {}; + return ReadVersionContent(db, node.id, node.version); +} + +std::vector ReadNodeVersion(sqlite3 *db, const Node &node, + sqlite3_int64 version) { + if (node.kind != "file") throw SqlError("path is not a file", SQLITE_MISMATCH); + if (version <= 0) throw SqlError("version must be a positive integer", SQLITE_RANGE); + return ReadVersionContent(db, node.id, version); +} + +sqlite3_int64 WritePath(sqlite3 *db, const Workspace &workspace, + const std::vector &parts, + const std::vector &content, + const std::string &message) { + Savepoint savepoint(db, "vexfs_write_path"); + AcquireWriteLock(db); + auto [parent, name] = ResolveParent(db, workspace, parts); + Node node; + if (!FindChild(db, workspace.id, parent.id, name, &node)) { + node.id = CreateInode(db, workspace.id, "file", 0644); + node.kind = "file"; + node.mode = 0644; + AddDentry(db, workspace.id, parent.id, name, node.id); + } + if (node.kind != "file") throw SqlError("path is not a file", SQLITE_MISMATCH); + const sqlite3_int64 version = StoreVersion(db, workspace, node, content, message); + savepoint.Release(); + return version; +} + +std::string RandomHandleId() { + unsigned char bytes[16]; + sqlite3_randomness(sizeof(bytes), bytes); + const uint64_t process_id = static_cast( +#if defined(_WIN32) + _getpid() +#else + getpid() +#endif + ); + for (size_t index = 0; index < sizeof(process_id); ++index) { + bytes[sizeof(bytes) - sizeof(process_id) + index] ^= + static_cast(process_id >> (index * 8)); + } + char output[33]; + for (size_t i = 0; i < sizeof(bytes); ++i) { + std::snprintf(output + i * 2, 3, "%02x", bytes[i]); + } + output[32] = '\0'; + return output; +} + +std::string JsonEscape(const std::string &value) { + std::string output; + output.reserve(value.size() + 8); + for (unsigned char c : value) { + switch (c) { + case '"': output += "\\\""; break; + case '\\': output += "\\\\"; break; + case '\b': output += "\\b"; break; + case '\f': output += "\\f"; break; + case '\n': output += "\\n"; break; + case '\r': output += "\\r"; break; + case '\t': output += "\\t"; break; + default: + if (c < 0x20) { + char escaped[7]; + std::snprintf(escaped, sizeof(escaped), "\\u%04x", c); + output += escaped; + } else { + output.push_back(static_cast(c)); + } + } + } + return output; +} + +enum class CachedKind { kMissing, kInteger, kText }; + +struct CachedRequest { + CachedKind kind = CachedKind::kMissing; + sqlite3_int64 integer = 0; + std::string text; +}; + +class RequestFingerprint { + public: + explicit RequestFingerprint(const char *operation) { + states_ = {1469598103934665603ULL, 1099511628211ULL, 7809847782465536322ULL, + 9650029242287828579ULL}; + AddText(operation); + } + + void AddText(const std::string &value) { + AddLength(value.size()); + AddBytes(value.data(), value.size()); + } + + void AddInt64(sqlite3_int64 value) { + const uint64_t bits = static_cast(value); + std::array bytes{}; + for (size_t index = 0; index < bytes.size(); ++index) { + bytes[index] = static_cast(bits >> ((bytes.size() - index - 1) * 8)); + } + AddLength(bytes.size()); + AddBytes(bytes.data(), bytes.size()); + } + + void AddBlob(const std::vector &value) { + AddLength(value.size()); + AddBytes(value.data(), value.size()); + } + + std::vector Finish() const { + std::vector output(sizeof(states_)); + std::memcpy(output.data(), states_.data(), output.size()); + return output; + } + + private: + void AddLength(size_t value) { + const uint64_t length = static_cast(value); + std::array bytes{}; + for (size_t index = 0; index < bytes.size(); ++index) { + bytes[index] = static_cast(length >> ((bytes.size() - index - 1) * 8)); + } + AddBytes(bytes.data(), bytes.size()); + } + + void AddBytes(const void *data, size_t size) { + const auto *bytes = static_cast(data); + for (size_t index = 0; index < size; ++index) { + for (size_t state = 0; state < states_.size(); ++state) { + states_[state] ^= static_cast(bytes[index] + state * 37U); + states_[state] *= 1099511628211ULL + state * 2ULL; + } + } + } + + std::array states_{}; +}; + +CachedRequest FindRequest(sqlite3 *db, const std::string &request_id, const char *operation, + const std::vector &fingerprint) { + if (request_id.empty()) throw SqlError("request_id must not be empty", SQLITE_MISMATCH); + Statement statement(db, + "SELECT operation, request_fingerprint, result_integer, result_text FROM _vexfs_requests " + "WHERE request_id = ?1"); + statement.BindText(1, request_id); + if (!statement.Row()) return {}; + if (statement.Text(0) != operation) { + throw SqlError("request_id was already used by another operation", SQLITE_CONSTRAINT); + } + if (statement.Blob(1) != fingerprint) { + throw SqlError("request_id was reused with different arguments", SQLITE_CONSTRAINT); + } + if (statement.Type(3) != SQLITE_NULL) { + return {CachedKind::kText, 0, statement.Text(3)}; + } + return {CachedKind::kInteger, statement.Int64(2), {}}; +} + +void StoreRequestInteger(sqlite3 *db, const std::string &request_id, const char *operation, + const std::vector &fingerprint, sqlite3_int64 result) { + Statement statement(db, + "INSERT INTO _vexfs_requests(request_id, operation, request_fingerprint, result_integer) " + "VALUES(?1, ?2, ?3, ?4)"); + statement.BindText(1, request_id); + statement.BindText(2, operation); + statement.BindBlob(3, fingerprint); + statement.BindInt64(4, result); + statement.Done(); +} + +void StoreRequestText(sqlite3 *db, const std::string &request_id, const char *operation, + const std::vector &fingerprint, const std::string &result) { + Statement statement(db, + "INSERT INTO _vexfs_requests(request_id, operation, request_fingerprint, result_text) " + "VALUES(?1, ?2, ?3, ?4)"); + statement.BindText(1, request_id); + statement.BindText(2, operation); + statement.BindBlob(3, fingerprint); + statement.BindText(4, result); + statement.Done(); +} + +struct Handle { + std::string id; + sqlite3_int64 workspace_id = 0; + sqlite3_int64 inode_id = 0; + bool writable = false; + sqlite3_int64 expected_version = 0; + sqlite3_int64 dirty_generation = 0; + sqlite3_int64 published_generation = 0; + std::string state; + std::string owner_session; +}; + +Handle FindHandle(sqlite3 *db, const std::string &id) { + Statement statement(db, + "SELECT id, workspace_id, inode_id, writable, expected_version, dirty_generation, " + "published_generation, state, COALESCE(owner_session, '') " + "FROM _vexfs_handles WHERE id = ?1"); + statement.BindText(1, id); + if (!statement.Row()) throw SqlError("handle not found", SQLITE_NOTFOUND); + return {statement.Text(0), statement.Int64(1), statement.Int64(2), statement.Int(3) != 0, + statement.Int64(4), statement.Int64(5), statement.Int64(6), statement.Text(7), + statement.Text(8)}; +} + +struct StagingInfo { + sqlite3_int64 rowid = 0; + sqlite3_int64 generation = 0; + sqlite3_int64 logical_size = 0; + sqlite3_int64 capacity = 0; +}; + +StagingInfo FindStaging(sqlite3 *db, const Handle &handle) { + Statement statement(db, + "SELECT d.rowid, s.generation, s.logical_size, s.capacity " + "FROM _vexfs_staging s JOIN _vexfs_staging_data d ON d.handle_id=s.handle_id " + "WHERE s.handle_id = ?1"); + statement.BindText(1, handle.id); + if (!statement.Row()) throw SqlError("staging area not found", SQLITE_NOTFOUND); + return {statement.Int64(0), statement.Int64(1), statement.Int64(2), statement.Int64(3)}; +} + +void WriteBlob(sqlite3 *db, sqlite3_int64 rowid, sqlite3_int64 offset, + const void *data, sqlite3_int64 size) { + if (size == 0) return; + if (offset < 0 || size < 0 || offset > std::numeric_limits::max() || + size > std::numeric_limits::max() || offset + size > std::numeric_limits::max()) { + throw SqlError("staging BLOB range is out of range", SQLITE_RANGE); + } + sqlite3_blob *blob = nullptr; + int rc = sqlite3_blob_open(db, "main", "_vexfs_staging_data", "content", rowid, 1, &blob); + if (rc != SQLITE_OK) { + throw SqlError("cannot open staging BLOB: " + std::string(sqlite3_errmsg(db))); + } + rc = sqlite3_blob_write(blob, data, static_cast(size), static_cast(offset)); + const int close_rc = sqlite3_blob_close(blob); + if (rc != SQLITE_OK) { + throw SqlError("cannot write staging BLOB: " + std::string(sqlite3_errmsg(db))); + } + if (close_rc != SQLITE_OK) { + throw SqlError("cannot close staging BLOB: " + std::string(sqlite3_errmsg(db))); + } +} + +void ZeroBlobRange(sqlite3 *db, sqlite3_int64 rowid, sqlite3_int64 offset, + sqlite3_int64 size) { + static const std::array zeros{}; + if (size == 0) return; + if (offset < 0 || size < 0 || offset > std::numeric_limits::max() || + size > std::numeric_limits::max() || offset + size > std::numeric_limits::max()) { + throw SqlError("staging zero range is out of range", SQLITE_RANGE); + } + sqlite3_blob *blob = nullptr; + int rc = sqlite3_blob_open(db, "main", "_vexfs_staging_data", "content", rowid, 1, &blob); + if (rc != SQLITE_OK) { + throw SqlError("cannot open staging BLOB: " + std::string(sqlite3_errmsg(db))); + } + while (size > 0) { + const sqlite3_int64 chunk = std::min(size, zeros.size()); + rc = sqlite3_blob_write(blob, zeros.data(), static_cast(chunk), + static_cast(offset)); + if (rc != SQLITE_OK) break; + offset += chunk; + size -= chunk; + } + const int close_rc = sqlite3_blob_close(blob); + if (rc != SQLITE_OK) { + throw SqlError("cannot zero staging BLOB: " + std::string(sqlite3_errmsg(db))); + } + if (close_rc != SQLITE_OK) { + throw SqlError("cannot close staging BLOB: " + std::string(sqlite3_errmsg(db))); + } +} + +StagingInfo EnsureStagingCapacity(sqlite3 *db, const Handle &handle, StagingInfo staging, + sqlite3_int64 required) { + if (required <= staging.capacity) return staging; + if (required > kMaxStagedBytes) + throw SqlError("staged file is larger than 128 MiB", SQLITE_TOOBIG); + sqlite3_int64 capacity = std::max(staging.capacity, kInitialStagingCapacity); + while (capacity < required) capacity = std::min(kMaxStagedBytes, capacity * 2); + Statement grow(db, + "UPDATE _vexfs_staging_data SET content=CAST(content||zeroblob(?1) AS BLOB) " + "WHERE handle_id=?2"); + grow.BindInt64(1, capacity - staging.capacity); + grow.BindText(2, handle.id); + grow.Done(); + Statement update(db, + "UPDATE _vexfs_staging SET capacity=?1, " + "updated_at=CAST(strftime('%s','now') AS INTEGER)*1000 WHERE handle_id=?2"); + update.BindInt64(1, capacity); + update.BindText(2, handle.id); + update.Done(); + staging.capacity = capacity; + return staging; +} + +void CreateStaging(sqlite3 *db, const Handle &handle, + const std::vector &content) { + sqlite3_int64 capacity = static_cast(content.size()); + if (handle.writable && capacity < kInitialStagingCapacity) capacity = kInitialStagingCapacity; + Statement insert(db, + "INSERT INTO _vexfs_staging(handle_id,generation,content,logical_size,capacity) " + "VALUES(?1,?2,X'',?4,?3)"); + insert.BindText(1, handle.id); + insert.BindInt64(2, handle.dirty_generation); + insert.BindInt64(3, capacity); + insert.BindInt64(4, static_cast(content.size())); + insert.Done(); + Statement data(db, + "INSERT INTO _vexfs_staging_data(handle_id,content) VALUES(?1,zeroblob(?2))"); + data.BindText(1, handle.id); + data.BindInt64(2, capacity); + data.Done(); + if (!content.empty()) { + const StagingInfo staging = FindStaging(db, handle); + WriteBlob(db, staging.rowid, 0, content.data(), static_cast(content.size())); + } +} + +sqlite3_int64 StageWrite(sqlite3 *db, const Handle &handle, sqlite3_int64 offset, + const std::vector &patch) { + if (patch.empty()) return handle.dirty_generation; + const sqlite3_int64 required = offset + static_cast(patch.size()); + StagingInfo staging = EnsureStagingCapacity(db, handle, FindStaging(db, handle), required); + if (offset > staging.logical_size) { + ZeroBlobRange(db, staging.rowid, staging.logical_size, offset - staging.logical_size); + } + WriteBlob(db, staging.rowid, offset, patch.data(), static_cast(patch.size())); + const sqlite3_int64 logical_size = std::max(staging.logical_size, required); + const sqlite3_int64 generation = handle.dirty_generation + 1; + Statement stage(db, + "UPDATE _vexfs_staging SET generation=?1, logical_size=?2, " + "updated_at=CAST(strftime('%s','now') AS INTEGER)*1000 WHERE handle_id=?3"); + stage.BindInt64(1, generation); + stage.BindInt64(2, logical_size); + stage.BindText(3, handle.id); + stage.Done(); + Statement update(db, + "UPDATE _vexfs_handles SET dirty_generation=?1, " + "updated_at=CAST(strftime('%s','now') AS INTEGER)*1000 WHERE id=?2"); + update.BindInt64(1, generation); + update.BindText(2, handle.id); + update.Done(); + return generation; +} + +sqlite3_int64 TruncateStaging(sqlite3 *db, const Handle &handle, sqlite3_int64 size) { + if (size < 0 || size > kMaxStagedBytes) + throw SqlError("file size is out of range", SQLITE_RANGE); + StagingInfo staging = FindStaging(db, handle); + if (size == staging.logical_size) return handle.dirty_generation; + staging = EnsureStagingCapacity(db, handle, staging, size); + if (size > staging.logical_size) { + ZeroBlobRange(db, staging.rowid, staging.logical_size, size - staging.logical_size); + } + const sqlite3_int64 generation = handle.dirty_generation + 1; + Statement stage(db, + "UPDATE _vexfs_staging SET generation=?1, logical_size=?2, " + "updated_at=CAST(strftime('%s','now') AS INTEGER)*1000 WHERE handle_id=?3"); + stage.BindInt64(1, generation); + stage.BindInt64(2, size); + stage.BindText(3, handle.id); + stage.Done(); + Statement update(db, + "UPDATE _vexfs_handles SET dirty_generation=?1, " + "updated_at=CAST(strftime('%s','now') AS INTEGER)*1000 WHERE id=?2"); + update.BindInt64(1, generation); + update.BindText(2, handle.id); + update.Done(); + return generation; +} + +std::vector ReadStaging(sqlite3 *db, const Handle &handle, + sqlite3_int64 generation) { + Statement statement(db, + "SELECT substr(d.content,1,s.logical_size) FROM _vexfs_staging s " + "JOIN _vexfs_staging_data d ON d.handle_id=s.handle_id " + "WHERE s.handle_id=?1 AND s.generation=?2"); + statement.BindText(1, handle.id); + statement.BindInt64(2, generation); + if (!statement.Row()) throw SqlError("staged generation is stale or missing", SQLITE_NOTFOUND); + return statement.Blob(0); +} + +std::vector ReadStagingRange(sqlite3 *db, const Handle &handle, + sqlite3_int64 offset, sqlite3_int64 length) { + Statement statement(db, + "SELECT CASE WHEN ?2>=logical_size THEN X'' " + "ELSE substr(d.content,?2+1,min(?3,logical_size-?2)) END " + "FROM _vexfs_staging s JOIN _vexfs_staging_data d ON d.handle_id=s.handle_id " + "WHERE s.handle_id=?1"); + statement.BindText(1, handle.id); + statement.BindInt64(2, offset); + statement.BindInt64(3, length); + if (!statement.Row()) throw SqlError("staging area not found", SQLITE_NOTFOUND); + return statement.Blob(0); +} + +Workspace WorkspaceById(sqlite3 *db, sqlite3_int64 id) { + Statement statement(db, + "SELECT id, root_inode, COALESCE(head_commit, 0) FROM _vexfs_workspaces WHERE id = ?1"); + statement.BindInt64(1, id); + if (!statement.Row()) throw SqlError("workspace not found", SQLITE_NOTFOUND); + return {statement.Int64(0), statement.Int64(1), statement.Int64(2)}; +} + +Node NodeById(sqlite3 *db, sqlite3_int64 id, bool include_deleted = false) { + Statement statement(db, + "SELECT id, kind, size, current_version, mode, created_at, updated_at FROM _vexfs_inodes " + "WHERE id = ?1 AND (?2 OR deleted_at IS NULL)"); + statement.BindInt64(1, id); + statement.BindInt(2, include_deleted ? 1 : 0); + if (!statement.Row()) throw SqlError("inode not found", SQLITE_NOTFOUND); + return {statement.Int64(0), statement.Text(1), statement.Int64(2), + statement.Int64(3), statement.Int(4), statement.Int64(5), statement.Int64(6)}; +} + +sqlite3_int64 PublishHandle(sqlite3 *db, Handle handle, sqlite3_int64 generation) { + if (handle.state == "closed") throw SqlError("handle is closed", SQLITE_NOTFOUND); + if (generation <= handle.published_generation) { + return NodeById(db, handle.inode_id, true).version; + } + if (generation != handle.dirty_generation) { + throw SqlError("generation is stale or has not been staged", SQLITE_CONSTRAINT); + } + + Node node = NodeById(db, handle.inode_id, true); + if (node.version != handle.expected_version) { + throw SqlError("write conflict: file changed after handle_open", SQLITE_CONSTRAINT); + } + const std::vector content = ReadStaging(db, handle, generation); + Workspace workspace = WorkspaceById(db, handle.workspace_id); + Savepoint savepoint(db, "vexfs_publish_handle"); + const sqlite3_int64 version = StoreVersion(db, workspace, node, content, "handle publish"); + + Statement update(db, + "UPDATE _vexfs_handles SET expected_version = ?1, published_generation = ?2, " + "updated_at = CAST(strftime('%s','now') AS INTEGER) * 1000 WHERE id = ?3"); + update.BindInt64(1, version); + update.BindInt64(2, generation); + update.BindText(3, handle.id); + update.Done(); + savepoint.Release(); + return version; +} + +void ResultBlob(sqlite3_context *context, const std::vector &value) { + const void *data = value.empty() ? static_cast("") : value.data(); + sqlite3_result_blob64(context, data, value.size(), SQLITE_TRANSIENT); +} + +template +void Guard(sqlite3_context *context, Function function) { + try { + function(); + } catch (const SqlError &error) { + sqlite3_result_error(context, error.what(), -1); + sqlite3_result_error_code(context, error.code()); + } catch (const std::exception &error) { + sqlite3_result_error(context, error.what(), -1); + } catch (...) { + sqlite3_result_error(context, "unknown VexFS error", -1); + } +} + +void InitFunction(sqlite3_context *context, int, sqlite3_value **) { + Guard(context, [&] { + EnsureSchema(sqlite3_context_db_handle(context)); + sqlite3_result_int(context, 1); + }); +} + +void ContractVersionFunction(sqlite3_context *context, int, sqlite3_value **) { + sqlite3_result_text(context, kContractVersion, -1, SQLITE_STATIC); +} + +void WorkspaceCreateFunction(sqlite3_context *context, int, sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + const std::string name = RequiredText(values[0], "workspace"); + if (name.empty() || name.size() > 128) + throw SqlError("workspace name must be 1..128 bytes", SQLITE_MISMATCH); + + try { + const Workspace existing = FindWorkspace(db, name); + sqlite3_result_int64(context, existing.id); + return; + } catch (const SqlError &) { + // Missing is expected; INSERT below still enforces the unique constraint. + } + + Statement insert(db, "INSERT INTO _vexfs_workspaces(name) VALUES(?1)"); + insert.BindText(1, name); + insert.Done(); + const sqlite3_int64 workspace_id = sqlite3_last_insert_rowid(db); + const sqlite3_int64 root = CreateInode(db, workspace_id, "directory", 0755); + Statement update(db, "UPDATE _vexfs_workspaces SET root_inode = ?1 WHERE id = ?2"); + update.BindInt64(1, root); + update.BindInt64(2, workspace_id); + update.Done(); + sqlite3_result_int64(context, workspace_id); + }); +} + +void MkdirFunction(sqlite3_context *context, int, sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + Workspace workspace = FindWorkspace(db, RequiredText(values[0], "workspace")); + const auto parts = PathParts(RequiredText(values[1], "path")); + Node current = RootNode(db, workspace); + bool changed = false; + for (const std::string &part : parts) { + Node child; + if (!FindChild(db, workspace.id, current.id, part, &child)) { + child.id = CreateInode(db, workspace.id, "directory", 0755); + child.kind = "directory"; + child.mode = 0755; + AddDentry(db, workspace.id, current.id, part, child.id); + changed = true; + } else if (child.kind != "directory") { + throw SqlError("path component is not a directory", SQLITE_MISMATCH); + } + current = std::move(child); + } + if (changed) CreateCommit(db, workspace, "mkdir"); + sqlite3_result_int64(context, current.id); + }); +} + +void WriteFunction(sqlite3_context *context, int, sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + const Workspace workspace = FindWorkspace(db, RequiredText(values[0], "workspace")); + const auto parts = PathParts(RequiredText(values[1], "path")); + const auto content = RequiredBlob(values[2], "content"); + if (content.size() > static_cast(kMaxStagedBytes)) { + throw SqlError("file is larger than the Phase 0 limit (128 MiB)", SQLITE_TOOBIG); + } + sqlite3_result_int64(context, WritePath(db, workspace, parts, content, "write file")); + }); +} + +void ReadFunction(sqlite3_context *context, int, sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + const Workspace workspace = FindWorkspace(db, RequiredText(values[0], "workspace")); + const Node node = Resolve(db, workspace, PathParts(RequiredText(values[1], "path"))); + ResultBlob(context, ReadNode(db, node)); + }); +} + +void HistoryFunction(sqlite3_context *context, int argument_count, sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + const Workspace workspace = FindWorkspace(db, RequiredText(values[0], "workspace")); + const Node node = Resolve(db, workspace, PathParts(RequiredText(values[1], "path"))); + if (node.kind != "file") throw SqlError("path is not a file", SQLITE_MISMATCH); + const sqlite3_int64 limit = argument_count == 2 + ? 100 : RequiredPositiveInteger(values[2], "limit"); + const sqlite3_int64 before_version = argument_count == 2 + ? 0 : RequiredNonnegativeInteger(values[3], "before version"); + if (limit > 1000) throw SqlError("history limit must be at most 1000", SQLITE_RANGE); + + Statement statement(db, + "SELECT v.version_no, v.commit_id, c.parent_commit, v.size, v.created_at, c.message " + "FROM _vexfs_file_versions v JOIN _vexfs_commits c ON c.id=v.commit_id " + "WHERE v.inode_id=?1 AND (?2=0 OR v.version_no(json.size()), SQLITE_TRANSIENT); + }); +} + +void ReadVersionFunction(sqlite3_context *context, int, sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + const Workspace workspace = FindWorkspace(db, RequiredText(values[0], "workspace")); + const Node node = Resolve(db, workspace, PathParts(RequiredText(values[1], "path"))); + ResultBlob(context, ReadNodeVersion(db, node, RequiredPositiveInteger(values[2], "version"))); + }); +} + +void CompareVersionsFunction(sqlite3_context *context, int, sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + const std::string path = RequiredText(values[1], "path"); + const Workspace workspace = FindWorkspace(db, RequiredText(values[0], "workspace")); + const Node node = Resolve(db, workspace, PathParts(path)); + if (node.kind != "file") throw SqlError("path is not a file", SQLITE_MISMATCH); + const sqlite3_int64 from_version = RequiredPositiveInteger(values[2], "from version"); + const sqlite3_int64 to_version = RequiredPositiveInteger(values[3], "to version"); + const VersionStorage from = ResolveVersionStorage(db, node.id, from_version); + const VersionStorage to = ResolveVersionStorage(db, node.id, to_version); + ReadOnlyBlob from_blob(db, from.rowid); + ReadOnlyBlob to_blob(db, to.rowid); + if (from_blob.size() != from.size || to_blob.size() != to.size) { + throw SqlError("file version size does not match content", SQLITE_CORRUPT); + } + std::array from_chunk{}; + std::array to_chunk{}; + bool changed = from.size != to.size; + bool binary = false; + const sqlite3_int64 maximum = std::max(from.size, to.size); + for (sqlite3_int64 offset = 0; offset < maximum; offset += from_chunk.size()) { + const int from_bytes = static_cast(std::min( + from_chunk.size(), std::max(0, from.size - offset))); + const int to_bytes = static_cast(std::min( + to_chunk.size(), std::max(0, to.size - offset))); + if (from_bytes > 0) from_blob.Read(from_chunk.data(), from_bytes, static_cast(offset)); + if (to_bytes > 0) to_blob.Read(to_chunk.data(), to_bytes, static_cast(offset)); + binary = binary || + std::find(from_chunk.begin(), from_chunk.begin() + from_bytes, 0) != + from_chunk.begin() + from_bytes || + std::find(to_chunk.begin(), to_chunk.begin() + to_bytes, 0) != + to_chunk.begin() + to_bytes; + if (!changed && (from_bytes != to_bytes || + !std::equal(from_chunk.begin(), from_chunk.begin() + from_bytes, + to_chunk.begin()))) changed = true; + } + const std::string json = + "{\"path\":\"" + JsonEscape(path) + "\",\"from\":" + + std::to_string(from_version) + ",\"to\":" + std::to_string(to_version) + + ",\"changed\":" + (changed ? "true" : "false") + + ",\"binary\":" + (binary ? "true" : "false") + + ",\"from_size\":" + std::to_string(from.size) + + ",\"to_size\":" + std::to_string(to.size) + "}"; + sqlite3_result_text(context, json.data(), static_cast(json.size()), SQLITE_TRANSIENT); + }); +} + +void RestoreVersionFunction(sqlite3_context *context, int, sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + const Workspace workspace = FindWorkspace(db, RequiredText(values[0], "workspace")); + const Node node = Resolve(db, workspace, PathParts(RequiredText(values[1], "path"))); + const sqlite3_int64 target_version = RequiredPositiveInteger(values[2], "version"); + const sqlite3_int64 expected_version = RequiredPositiveInteger(values[3], "expected version"); + if (node.version != expected_version) { + throw SqlError("version conflict: expected " + std::to_string(expected_version) + + ", current " + std::to_string(node.version), SQLITE_CONSTRAINT); + } + if (target_version == node.version) { + throw SqlError("target version is already current", SQLITE_MISMATCH); + } + const sqlite3_int64 version = StoreVersionFromVersion( + db, workspace, node, target_version, + "restore version " + std::to_string(target_version)); + sqlite3_result_int64(context, version); + }); +} + +void StatFunction(sqlite3_context *context, int, sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + const std::string path = RequiredText(values[1], "path"); + const Workspace workspace = FindWorkspace(db, RequiredText(values[0], "workspace")); + const Node node = Resolve(db, workspace, PathParts(path)); + const std::string json = "{\"path\":\"" + JsonEscape(path) + "\",\"inode\":" + + std::to_string(node.id) + ",\"kind\":\"" + node.kind + "\",\"mode\":" + + std::to_string(node.mode) + ",\"size\":" + std::to_string(node.size) + + ",\"version\":" + std::to_string(node.version) + ",\"created_at\":" + + std::to_string(node.created_at) + ",\"updated_at\":" + + std::to_string(node.updated_at) + "}"; + sqlite3_result_text(context, json.data(), static_cast(json.size()), SQLITE_TRANSIENT); + }); +} + +void PathFunction(sqlite3_context *context, int, sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + const Workspace workspace = FindWorkspace(db, RequiredText(values[0], "workspace")); + const std::string path = PathForInode(db, workspace, sqlite3_value_int64(values[1])); + sqlite3_result_text(context, path.data(), static_cast(path.size()), SQLITE_TRANSIENT); + }); +} + +void ListFunction(sqlite3_context *context, int, sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + const Workspace workspace = FindWorkspace(db, RequiredText(values[0], "workspace")); + const Node directory = Resolve(db, workspace, PathParts(RequiredText(values[1], "path"))); + if (directory.kind != "directory") + throw SqlError("path is not a directory", SQLITE_MISMATCH); + + Statement statement(db, + "SELECT d.name, i.id, i.kind, i.size, i.current_version, i.mode, " + "i.created_at, i.updated_at FROM _vexfs_dentries d " + "JOIN _vexfs_inodes i ON i.id = d.inode_id " + "WHERE d.workspace_id = ?1 AND d.parent_inode = ?2 AND i.deleted_at IS NULL " + "ORDER BY d.name COLLATE BINARY"); + statement.BindInt64(1, workspace.id); + statement.BindInt64(2, directory.id); + std::string json = "["; + bool first = true; + while (statement.Row()) { + if (!first) json += ','; + first = false; + json += "{\"name\":\"" + JsonEscape(statement.Text(0)) + "\",\"inode\":" + + std::to_string(statement.Int64(1)) + ",\"kind\":\"" + statement.Text(2) + + "\",\"size\":" + std::to_string(statement.Int64(3)) + ",\"version\":" + + std::to_string(statement.Int64(4)) + ",\"mode\":" + + std::to_string(statement.Int(5)) + ",\"created_at\":" + + std::to_string(statement.Int64(6)) + ",\"updated_at\":" + + std::to_string(statement.Int64(7)) + "}"; + } + json += ']'; + sqlite3_result_text(context, json.data(), static_cast(json.size()), SQLITE_TRANSIENT); + }); +} + +bool DirectoryIsEmpty(sqlite3 *db, sqlite3_int64 workspace_id, sqlite3_int64 inode) { + Statement children(db, + "SELECT 1 FROM _vexfs_dentries d JOIN _vexfs_inodes i ON i.id=d.inode_id " + "WHERE d.workspace_id=?1 AND d.parent_inode=?2 AND i.deleted_at IS NULL LIMIT 1"); + children.BindInt64(1, workspace_id); + children.BindInt64(2, inode); + return !children.Row(); +} + +void TombstoneTree(sqlite3 *db, sqlite3_int64 workspace_id, sqlite3_int64 inode) { + Statement tombstone(db, R"SQL( +WITH RECURSIVE tree(id) AS ( + SELECT ?1 + UNION ALL + SELECT d.inode_id FROM _vexfs_dentries d JOIN tree ON d.parent_inode = tree.id + WHERE d.workspace_id = ?2 +) +UPDATE _vexfs_inodes SET deleted_at = CAST(strftime('%s','now') AS INTEGER) * 1000 +WHERE workspace_id = ?2 AND id IN (SELECT id FROM tree) +)SQL"); + tombstone.BindInt64(1, inode); + tombstone.BindInt64(2, workspace_id); + tombstone.Done(); +} + +void RenameFunction(sqlite3_context *context, int argument_count, sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + Workspace workspace = FindWorkspace(db, RequiredText(values[0], "workspace")); + const auto source_parts = PathParts(RequiredText(values[1], "source")); + const auto destination_parts = PathParts(RequiredText(values[2], "destination")); + const bool replace = argument_count == 4 && sqlite3_value_int(values[3]) != 0; + if (source_parts.empty() || destination_parts.empty()) + throw SqlError("root cannot be moved", SQLITE_MISMATCH); + if (source_parts == destination_parts) { + sqlite3_result_int(context, 1); + return; + } + if (destination_parts.size() > source_parts.size() && + std::equal(source_parts.begin(), source_parts.end(), destination_parts.begin())) { + throw SqlError("directory cannot be moved into itself", SQLITE_MISMATCH); + } + auto [source_parent, source_name] = ResolveParent(db, workspace, source_parts); + Node source; + if (!FindChild(db, workspace.id, source_parent.id, source_name, &source)) { + throw SqlError("source path not found", SQLITE_NOTFOUND); + } + auto [destination_parent, destination_name] = ResolveParent(db, workspace, destination_parts); + Node existing; + if (FindChild(db, workspace.id, destination_parent.id, destination_name, &existing)) { + if (!replace) throw SqlError("destination already exists", SQLITE_CONSTRAINT); + if (source.kind != existing.kind) + throw SqlError("source and destination types differ", SQLITE_MISMATCH); + if (existing.kind == "directory" && !DirectoryIsEmpty(db, workspace.id, existing.id)) { + throw SqlError("destination directory is not empty", SQLITE_CONSTRAINT); + } + Statement detach(db, + "DELETE FROM _vexfs_dentries WHERE workspace_id=?1 AND parent_inode=?2 AND name=?3"); + detach.BindInt64(1, workspace.id); + detach.BindInt64(2, destination_parent.id); + detach.BindText(3, destination_name); + detach.Done(); + TombstoneTree(db, workspace.id, existing.id); + } + Statement update(db, + "UPDATE _vexfs_dentries SET parent_inode = ?1, name = ?2 " + "WHERE workspace_id = ?3 AND parent_inode = ?4 AND name = ?5"); + update.BindInt64(1, destination_parent.id); + update.BindText(2, destination_name); + update.BindInt64(3, workspace.id); + update.BindInt64(4, source_parent.id); + update.BindText(5, source_name); + update.Done(); + TouchDirectory(db, workspace.id, source_parent.id); + if (destination_parent.id != source_parent.id) { + TouchDirectory(db, workspace.id, destination_parent.id); + } + CreateCommit(db, workspace, replace ? "rename replace" : "move"); + sqlite3_result_int(context, 1); + }); +} + +void RemoveFunction(sqlite3_context *context, int, sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + Workspace workspace = FindWorkspace(db, RequiredText(values[0], "workspace")); + const auto parts = PathParts(RequiredText(values[1], "path")); + const bool recursive = sqlite3_value_int(values[2]) != 0; + if (parts.empty()) throw SqlError("workspace root cannot be removed", SQLITE_MISMATCH); + auto [parent, name] = ResolveParent(db, workspace, parts); + Node node; + if (!FindChild(db, workspace.id, parent.id, name, &node)) + throw SqlError("path not found", SQLITE_NOTFOUND); + if (node.kind == "directory" && !recursive && + !DirectoryIsEmpty(db, workspace.id, node.id)) { + throw SqlError("directory is not empty", SQLITE_CONSTRAINT); + } + Statement detach(db, + "DELETE FROM _vexfs_dentries WHERE workspace_id=?1 AND parent_inode=?2 AND name=?3"); + detach.BindInt64(1, workspace.id); + detach.BindInt64(2, parent.id); + detach.BindText(3, name); + detach.Done(); + TombstoneTree(db, workspace.id, node.id); + TouchDirectory(db, workspace.id, parent.id); + CreateCommit(db, workspace, "remove"); + sqlite3_result_int(context, 1); + }); +} + +void HandleOpenFunction(sqlite3_context *context, int argument_count, sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + const std::string workspace_name = RequiredText(values[0], "workspace"); + const std::string path = RequiredText(values[1], "path"); + const std::string flags = RequiredText(values[2], "flags"); + const std::string request_id = RequiredText(values[3], "request_id"); + const std::string owner_session = argument_count == 5 + ? RequiredText(values[4], "owner_session") : std::string(); + RequestFingerprint fingerprint_builder("handle_open"); + fingerprint_builder.AddText(workspace_name); + fingerprint_builder.AddText(path); + fingerprint_builder.AddText(flags); + fingerprint_builder.AddText(owner_session); + const auto fingerprint = fingerprint_builder.Finish(); + const CachedRequest cached = FindRequest(db, request_id, "handle_open", fingerprint); + if (cached.kind == CachedKind::kText) { + sqlite3_result_text(context, cached.text.data(), static_cast(cached.text.size()), SQLITE_TRANSIENT); + return; + } + + const bool writable = flags.find('w') != std::string::npos; + const bool create = flags.find('c') != std::string::npos; + const bool truncate = flags.find('t') != std::string::npos; + if (flags.find('r') == std::string::npos && !writable) { + throw SqlError("flags must contain r or w", SQLITE_MISMATCH); + } + if ((create || truncate) && !writable) + throw SqlError("create/truncate requires write access", SQLITE_MISMATCH); + + Workspace workspace = FindWorkspace(db, workspace_name); + const auto parts = PathParts(path); + Node node; + if (!TryResolve(db, workspace, parts, &node)) { + if (!create) throw SqlError("path not found", SQLITE_NOTFOUND); + WritePath(db, workspace, parts, {}, "create file"); + workspace = WorkspaceById(db, workspace.id); + node = Resolve(db, workspace, parts); + } + if (node.kind != "file") throw SqlError("path is not a file", SQLITE_MISMATCH); + + const std::string handle_id = RandomHandleId(); + const sqlite3_int64 dirty_generation = truncate ? 1 : 0; + Statement insert(db, + "INSERT INTO _vexfs_handles(id, workspace_id, inode_id, open_flags, writable, " + "expected_version, dirty_generation, state, owner_session) " + "VALUES(?1,?2,?3,?4,?5,?6,?7,'open',?8)"); + insert.BindText(1, handle_id); + insert.BindInt64(2, workspace.id); + insert.BindInt64(3, node.id); + insert.BindText(4, flags); + insert.BindInt(5, writable ? 1 : 0); + insert.BindInt64(6, node.version); + insert.BindInt64(7, dirty_generation); + if (argument_count == 5) insert.BindText(8, owner_session); + else insert.BindNull(8); + insert.Done(); + + Handle handle{handle_id, workspace.id, node.id, writable, node.version, + dirty_generation, 0, "open", owner_session}; + CreateStaging(db, handle, truncate ? std::vector() : ReadNode(db, node)); + StoreRequestText(db, request_id, "handle_open", fingerprint, handle_id); + sqlite3_result_text(context, handle_id.data(), static_cast(handle_id.size()), SQLITE_TRANSIENT); + }); +} + +void HandleStageWriteFunction(sqlite3_context *context, int, sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + const std::string handle_id = RequiredText(values[0], "handle"); + const sqlite3_int64 offset = sqlite3_value_int64(values[1]); + const auto patch = RequiredBlob(values[2], "content"); + const std::string request_id = RequiredText(values[3], "request_id"); + RequestFingerprint fingerprint_builder("handle_stage_write"); + fingerprint_builder.AddText(handle_id); + fingerprint_builder.AddInt64(offset); + fingerprint_builder.AddBlob(patch); + const auto fingerprint = fingerprint_builder.Finish(); + const CachedRequest cached = FindRequest(db, request_id, "handle_stage_write", fingerprint); + if (cached.kind == CachedKind::kInteger) { + sqlite3_result_int64(context, cached.integer); + return; + } + Handle handle = FindHandle(db, handle_id); + if (handle.state != "open") throw SqlError("handle is not open", SQLITE_NOTFOUND); + if (!handle.writable) throw SqlError("handle is read-only", SQLITE_READONLY); + if (offset < 0 || offset > kMaxStagedBytes) + throw SqlError("write offset is out of range", SQLITE_RANGE); + if (patch.size() > static_cast(kMaxStagedBytes - offset)) { + throw SqlError("staged file is larger than 128 MiB", SQLITE_TOOBIG); + } + const sqlite3_int64 generation = StageWrite(db, handle, offset, patch); + StoreRequestInteger(db, request_id, "handle_stage_write", fingerprint, generation); + sqlite3_result_int64(context, generation); + }); +} + +void HandleTruncateFunction(sqlite3_context *context, int, sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + const std::string handle_id = RequiredText(values[0], "handle"); + const sqlite3_int64 size = sqlite3_value_int64(values[1]); + const std::string request_id = RequiredText(values[2], "request_id"); + RequestFingerprint fingerprint_builder("handle_truncate"); + fingerprint_builder.AddText(handle_id); + fingerprint_builder.AddInt64(size); + const auto fingerprint = fingerprint_builder.Finish(); + const CachedRequest cached = FindRequest(db, request_id, "handle_truncate", fingerprint); + if (cached.kind == CachedKind::kInteger) { + sqlite3_result_int64(context, cached.integer); + return; + } + const Handle handle = FindHandle(db, handle_id); + if (handle.state != "open") throw SqlError("handle is not open", SQLITE_NOTFOUND); + if (!handle.writable) throw SqlError("handle is read-only", SQLITE_READONLY); + const sqlite3_int64 generation = TruncateStaging(db, handle, size); + StoreRequestInteger(db, request_id, "handle_truncate", fingerprint, generation); + sqlite3_result_int64(context, generation); + }); +} + +void HandleReadFunction(sqlite3_context *context, int, sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + const Handle handle = FindHandle(db, RequiredText(values[0], "handle")); + if (handle.state == "closed") throw SqlError("handle is closed", SQLITE_NOTFOUND); + const sqlite3_int64 offset = sqlite3_value_int64(values[1]); + const sqlite3_int64 length = sqlite3_value_int64(values[2]); + if (offset < 0 || length < 0) + throw SqlError("offset and length must be non-negative", SQLITE_RANGE); + ResultBlob(context, ReadStagingRange(db, handle, offset, length)); + }); +} + +void HandlePublishFunction(sqlite3_context *context, int, sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + const std::string handle_id = RequiredText(values[0], "handle"); + const sqlite3_int64 generation = sqlite3_value_int64(values[1]); + const std::string durability = RequiredText(values[2], "durability"); + const std::string request_id = RequiredText(values[3], "request_id"); + RequestFingerprint fingerprint_builder("handle_publish"); + fingerprint_builder.AddText(handle_id); + fingerprint_builder.AddInt64(generation); + fingerprint_builder.AddText(durability); + const auto fingerprint = fingerprint_builder.Finish(); + const CachedRequest cached = FindRequest(db, request_id, "handle_publish", fingerprint); + if (cached.kind == CachedKind::kInteger) { + sqlite3_result_int64(context, cached.integer); + return; + } + if (durability != "none" && durability != "data" && durability != "full") { + throw SqlError("durability must be none, data, or full", SQLITE_MISMATCH); + } + const Handle handle = FindHandle(db, handle_id); + const sqlite3_int64 version = PublishHandle(db, handle, generation); + StoreRequestInteger(db, request_id, "handle_publish", fingerprint, version); + sqlite3_result_int64(context, version); + }); +} + +void HandleCloseFunction(sqlite3_context *context, int, sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + const std::string handle_id = RequiredText(values[0], "handle"); + const bool retain_requested = sqlite3_value_int(values[1]) != 0; + const std::string request_id = RequiredText(values[2], "request_id"); + RequestFingerprint fingerprint_builder("handle_close"); + fingerprint_builder.AddText(handle_id); + fingerprint_builder.AddInt64(retain_requested ? 1 : 0); + const auto fingerprint = fingerprint_builder.Finish(); + const CachedRequest cached = FindRequest(db, request_id, "handle_close", fingerprint); + if (cached.kind == CachedKind::kText) { + sqlite3_result_text(context, cached.text.data(), static_cast(cached.text.size()), SQLITE_TRANSIENT); + return; + } + const Handle handle = FindHandle(db, handle_id); + const bool retain = retain_requested && + handle.dirty_generation > handle.published_generation; + const std::string state = retain ? "retained" : "closed"; + Statement update(db, + "UPDATE _vexfs_handles SET state=?1, " + "lease_until=CASE WHEN ?2 THEN (CAST(strftime('%s','now') AS INTEGER)+86400)*1000 " + "ELSE NULL END, updated_at=CAST(strftime('%s','now') AS INTEGER)*1000 WHERE id=?3"); + update.BindText(1, state); + update.BindInt(2, retain ? 1 : 0); + update.BindText(3, handle.id); + update.Done(); + if (!retain) { + Statement clear_data(db, "DELETE FROM _vexfs_staging_data WHERE handle_id=?1"); + clear_data.BindText(1, handle.id); + clear_data.Done(); + Statement clear(db, "DELETE FROM _vexfs_staging WHERE handle_id=?1"); + clear.BindText(1, handle.id); + clear.Done(); + } + StoreRequestText(db, request_id, "handle_close", fingerprint, state); + sqlite3_result_text(context, state.data(), static_cast(state.size()), SQLITE_TRANSIENT); + }); +} + +void RetainOpenHandles(sqlite3 *db, const Workspace &workspace, + const std::string &session_id, bool matching_session) { + const char *predicate = matching_session + ? "owner_session=?2" + : "owner_session IS NOT NULL AND owner_session<>?2"; + const std::string sql = + "UPDATE _vexfs_handles SET " + "state=CASE WHEN dirty_generation>published_generation THEN 'retained' ELSE 'closed' END, " + "lease_until=CASE WHEN dirty_generation>published_generation " + "THEN (CAST(strftime('%s','now') AS INTEGER)+86400)*1000 ELSE NULL END, " + "updated_at=CAST(strftime('%s','now') AS INTEGER)*1000 " + "WHERE workspace_id=?1 AND state='open' AND " + std::string(predicate); + Statement update(db, sql.c_str()); + update.BindInt64(1, workspace.id); + update.BindText(2, session_id); + update.Done(); + Statement clear_data(db, + "DELETE FROM _vexfs_staging_data WHERE handle_id IN " + "(SELECT id FROM _vexfs_handles WHERE workspace_id=?1 AND state='closed')"); + clear_data.BindInt64(1, workspace.id); + clear_data.Done(); + Statement clear(db, + "DELETE FROM _vexfs_staging WHERE handle_id IN " + "(SELECT id FROM _vexfs_handles WHERE workspace_id=?1 AND state='closed')"); + clear.BindInt64(1, workspace.id); + clear.Done(); +} + +void ClaimRetainedHandles(sqlite3 *db, const Workspace &workspace, + const std::string &session_id) { + Statement claim(db, + "UPDATE _vexfs_handles SET state='open',owner_session=?1,lease_until=NULL," + "updated_at=CAST(strftime('%s','now') AS INTEGER)*1000 " + "WHERE workspace_id=?2 AND state='retained' " + "AND dirty_generation>published_generation"); + claim.BindText(1, session_id); + claim.BindInt64(2, workspace.id); + claim.Done(); +} + +void SessionStartFunction(sqlite3_context *context, int, sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + const Workspace workspace = FindWorkspace(db, RequiredText(values[0], "workspace")); + const std::string session_id = RequiredText(values[1], "session_id"); + if (session_id.empty()) throw SqlError("session_id must not be empty", SQLITE_MISMATCH); + Savepoint savepoint(db, "vexfs_session_start"); + Statement current(db, + "SELECT session_id,lease_until>CAST(strftime('%s','now') AS INTEGER)*1000 " + "FROM _vexfs_mount_sessions WHERE workspace_id=?1"); + current.BindInt64(1, workspace.id); + std::string previous_session; + bool active = false; + if (current.Row()) { + previous_session = current.Text(0); + active = current.Int(1) != 0; + } + if (active && previous_session != session_id) { + throw SqlError("workspace already has an active mount session", SQLITE_BUSY); + } + if (!previous_session.empty() && previous_session != session_id) { + RetainOpenHandles(db, workspace, previous_session, true); + } + ClaimRetainedHandles(db, workspace, session_id); + Statement upsert(db, + "INSERT INTO _vexfs_mount_sessions(workspace_id,session_id,lease_until) " + "VALUES(?1,?2,(CAST(strftime('%s','now') AS INTEGER)+30)*1000) " + "ON CONFLICT(workspace_id) DO UPDATE SET session_id=excluded.session_id," + "lease_until=excluded.lease_until," + "updated_at=CAST(strftime('%s','now') AS INTEGER)*1000"); + upsert.BindInt64(1, workspace.id); + upsert.BindText(2, session_id); + upsert.Done(); + savepoint.Release(); + sqlite3_result_int(context, 1); + }); +} + +void SessionHeartbeatFunction(sqlite3_context *context, int, sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + const Workspace workspace = FindWorkspace(db, RequiredText(values[0], "workspace")); + const std::string session_id = RequiredText(values[1], "session_id"); + if (session_id.empty()) throw SqlError("session_id must not be empty", SQLITE_MISMATCH); + Statement update(db, + "UPDATE _vexfs_mount_sessions SET " + "lease_until=(CAST(strftime('%s','now') AS INTEGER)+30)*1000," + "updated_at=CAST(strftime('%s','now') AS INTEGER)*1000 " + "WHERE workspace_id=?1 AND session_id=?2"); + update.BindInt64(1, workspace.id); + update.BindText(2, session_id); + update.Done(); + if (sqlite3_changes64(db) != 1) { + throw SqlError("mount session is stale", SQLITE_BUSY); + } + sqlite3_result_int(context, 1); + }); +} + +void SessionEndFunction(sqlite3_context *context, int, sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + const Workspace workspace = FindWorkspace(db, RequiredText(values[0], "workspace")); + const std::string session_id = RequiredText(values[1], "session_id"); + if (session_id.empty()) throw SqlError("session_id must not be empty", SQLITE_MISMATCH); + Savepoint savepoint(db, "vexfs_session_end"); + RetainOpenHandles(db, workspace, session_id, true); + Statement remove(db, + "DELETE FROM _vexfs_mount_sessions WHERE workspace_id=?1 AND session_id=?2"); + remove.BindInt64(1, workspace.id); + remove.BindText(2, session_id); + remove.Done(); + savepoint.Release(); + sqlite3_result_int(context, 1); + }); +} + +void SynchronizeFunction(sqlite3_context *context, int argument_count, sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + const std::string workspace_name = RequiredText(values[0], "workspace"); + const std::string request_id = RequiredText(values[1], "request_id"); + const std::string owner_session = argument_count == 3 + ? RequiredText(values[2], "owner_session") : std::string(); + RequestFingerprint fingerprint_builder("mount_synchronize"); + fingerprint_builder.AddText(workspace_name); + fingerprint_builder.AddText(owner_session); + const auto fingerprint = fingerprint_builder.Finish(); + const CachedRequest cached = FindRequest(db, request_id, "mount_synchronize", fingerprint); + if (cached.kind == CachedKind::kInteger) { + sqlite3_result_int64(context, cached.integer); + return; + } + const Workspace workspace = FindWorkspace(db, workspace_name); + const char *sql = argument_count == 3 + ? "SELECT id, dirty_generation FROM _vexfs_handles WHERE workspace_id=?1 " + "AND state='open' AND owner_session=?2 " + "AND dirty_generation>published_generation ORDER BY id" + : "SELECT id, dirty_generation FROM _vexfs_handles WHERE workspace_id=?1 " + "AND state IN ('open','retained') AND owner_session IS NULL " + "AND dirty_generation>published_generation ORDER BY id"; + Statement query(db, sql); + query.BindInt64(1, workspace.id); + if (argument_count == 3) query.BindText(2, owner_session); + std::vector> pending; + while (query.Row()) pending.emplace_back(query.Text(0), query.Int64(1)); + for (const auto &item : pending) PublishHandle(db, FindHandle(db, item.first), item.second); + StoreRequestInteger(db, request_id, "mount_synchronize", fingerprint, + static_cast(pending.size())); + sqlite3_result_int64(context, static_cast(pending.size())); + }); +} + +void ReclaimFunction(sqlite3_context *context, int, sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + const std::string workspace_name = RequiredText(values[0], "workspace"); + const std::string request_id = RequiredText(values[1], "request_id"); + RequestFingerprint fingerprint_builder("item_reclaim"); + fingerprint_builder.AddText(workspace_name); + const auto fingerprint = fingerprint_builder.Finish(); + const CachedRequest cached = FindRequest(db, request_id, "item_reclaim", fingerprint); + if (cached.kind == CachedKind::kInteger) { + sqlite3_result_int64(context, cached.integer); + return; + } + const Workspace workspace = FindWorkspace(db, workspace_name); + Statement query(db, + "SELECT id FROM _vexfs_handles WHERE workspace_id=?1 AND state='closed'"); + query.BindInt64(1, workspace.id); + std::vector handles; + while (query.Row()) handles.push_back(query.Text(0)); + for (const std::string &handle : handles) { + Statement clear_data(db, "DELETE FROM _vexfs_staging_data WHERE handle_id=?1"); + clear_data.BindText(1, handle); + clear_data.Done(); + Statement clear(db, "DELETE FROM _vexfs_staging WHERE handle_id=?1"); + clear.BindText(1, handle); + clear.Done(); + Statement remove(db, "DELETE FROM _vexfs_handles WHERE id=?1"); + remove.BindText(1, handle); + remove.Done(); + } + StoreRequestInteger(db, request_id, "item_reclaim", fingerprint, + static_cast(handles.size())); + sqlite3_result_int64(context, static_cast(handles.size())); + }); +} + +struct FunctionDefinition { + const char *name; + int arguments; + void (*function)(sqlite3_context *, int, sqlite3_value **); + int flags; +}; + +} // namespace + +extern "C" int vexfs_sqlite_register(sqlite3 *db) { + static const FunctionDefinition functions[] = { + {"vexfs_init", 0, InitFunction, SQLITE_UTF8}, + {"vexfs_contract_version", 0, ContractVersionFunction, + SQLITE_UTF8 | SQLITE_DETERMINISTIC}, + {"vexfs_workspace_create", 1, WorkspaceCreateFunction, SQLITE_UTF8}, + {"vexfs_mkdir", 2, MkdirFunction, SQLITE_UTF8}, + {"vexfs_write", 3, WriteFunction, SQLITE_UTF8}, + {"vexfs_read", 2, ReadFunction, SQLITE_UTF8}, + {"vexfs_history", 2, HistoryFunction, SQLITE_UTF8}, + {"vexfs_history", 4, HistoryFunction, SQLITE_UTF8}, + {"vexfs_read_version", 3, ReadVersionFunction, SQLITE_UTF8}, + {"vexfs_compare_versions", 4, CompareVersionsFunction, SQLITE_UTF8}, + {"vexfs_restore_version", 4, RestoreVersionFunction, SQLITE_UTF8}, + {"vexfs_stat", 2, StatFunction, SQLITE_UTF8}, + {"vexfs_path", 2, PathFunction, SQLITE_UTF8}, + {"vexfs_list", 2, ListFunction, SQLITE_UTF8}, + {"vexfs_move", 3, RenameFunction, SQLITE_UTF8}, + {"vexfs_rename", 4, RenameFunction, SQLITE_UTF8}, + {"vexfs_remove", 3, RemoveFunction, SQLITE_UTF8}, + {"vexfs_handle_open", 4, HandleOpenFunction, SQLITE_UTF8}, + {"vexfs_handle_open", 5, HandleOpenFunction, SQLITE_UTF8}, + {"vexfs_handle_stage_write", 4, HandleStageWriteFunction, SQLITE_UTF8}, + {"vexfs_handle_truncate", 3, HandleTruncateFunction, SQLITE_UTF8}, + {"vexfs_handle_read", 3, HandleReadFunction, SQLITE_UTF8}, + {"vexfs_handle_publish", 4, HandlePublishFunction, SQLITE_UTF8}, + {"vexfs_handle_close", 3, HandleCloseFunction, SQLITE_UTF8}, + {"vexfs_mount_session_start", 2, SessionStartFunction, SQLITE_UTF8}, + {"vexfs_mount_session_heartbeat", 2, SessionHeartbeatFunction, SQLITE_UTF8}, + {"vexfs_mount_session_end", 2, SessionEndFunction, SQLITE_UTF8}, + {"vexfs_mount_synchronize", 2, SynchronizeFunction, SQLITE_UTF8}, + {"vexfs_mount_synchronize", 3, SynchronizeFunction, SQLITE_UTF8}, + {"vexfs_item_reclaim", 2, ReclaimFunction, SQLITE_UTF8}, + }; + for (const auto &definition : functions) { + const int rc = sqlite3_create_function(db, definition.name, definition.arguments, + definition.flags, nullptr, definition.function, + nullptr, nullptr); + if (rc != SQLITE_OK) return rc; + } + return SQLITE_OK; +} diff --git a/vexdb_sqlite/src/vexdb_sqlite_init.cpp b/vexdb_sqlite/src/vexdb_sqlite_init.cpp index e6ac249301..0f7c3b602e 100644 --- a/vexdb_sqlite/src/vexdb_sqlite_init.cpp +++ b/vexdb_sqlite/src/vexdb_sqlite_init.cpp @@ -5,6 +5,7 @@ SQLITE_EXTENSION_INIT1 #endif #include "vexdb_sqlite.h" +#include "agent_files/vexfs_sqlite.h" #include "functions/distance_functions.h" #include "vtab/graph_index_vtab.h" @@ -46,6 +47,9 @@ extern "C" int vexdb_sqlite_register(sqlite3 *db) { rc = vexdb_sqlite_register_distance_functions(db); if (rc != SQLITE_OK) return rc; + rc = vexfs_sqlite_register(db); + if (rc != SQLITE_OK) return rc; + rc = sqlite3_create_function(db, "vexdb_version", 0, SQLITE_UTF8 | SQLITE_DETERMINISTIC, nullptr, VexVersionFunc, nullptr, nullptr); diff --git a/vexdb_sqlite/test/vexfs_static_smoke.c b/vexdb_sqlite/test/vexfs_static_smoke.c new file mode 100644 index 0000000000..5339589940 --- /dev/null +++ b/vexdb_sqlite/test/vexfs_static_smoke.c @@ -0,0 +1,270 @@ +#include +#include +#include + +#include "sqlite3.h" +#include "vexdb_sqlite.h" + +static int fail(sqlite3 *db, const char *what) { + fprintf(stderr, "VEXFS SMOKE FAIL: %s: %s\n", what, sqlite3_errmsg(db)); + return 1; +} + +static int exec_ok(sqlite3 *db, const char *sql) { + char *message = NULL; + int rc = sqlite3_exec(db, sql, NULL, NULL, &message); + if (rc != SQLITE_OK) { + fprintf(stderr, "SQL failed: %s\n%s\n", sql, message ? message : sqlite3_errmsg(db)); + sqlite3_free(message); + return 0; + } + return 1; +} + +static int exec_fails(sqlite3 *db, const char *sql) { + char *message = NULL; + int rc = sqlite3_exec(db, sql, NULL, NULL, &message); + sqlite3_free(message); + return rc != SQLITE_OK; +} + +static sqlite3_int64 scalar_int(sqlite3 *db, const char *sql, int *ok) { + sqlite3_stmt *statement = NULL; + *ok = 0; + if (sqlite3_prepare_v2(db, sql, -1, &statement, NULL) != SQLITE_OK) return 0; + if (sqlite3_step(statement) != SQLITE_ROW) { + sqlite3_finalize(statement); + return 0; + } + sqlite3_int64 result = sqlite3_column_int64(statement, 0); + sqlite3_finalize(statement); + *ok = 1; + return result; +} + +static char *scalar_text(sqlite3 *db, const char *sql) { + sqlite3_stmt *statement = NULL; + if (sqlite3_prepare_v2(db, sql, -1, &statement, NULL) != SQLITE_OK) return NULL; + if (sqlite3_step(statement) != SQLITE_ROW) { + sqlite3_finalize(statement); + return NULL; + } + const unsigned char *value = sqlite3_column_text(statement, 0); + int bytes = sqlite3_column_bytes(statement, 0); + char *result = (char *)malloc((size_t)bytes + 1); + if (result != NULL) { + memcpy(result, value, (size_t)bytes); + result[bytes] = '\0'; + } + sqlite3_finalize(statement); + return result; +} + +int main(void) { + sqlite3 *db = NULL; + int ok = 0; + if (sqlite3_open(":memory:", &db) != SQLITE_OK) return fail(db, "open"); + if (vexdb_sqlite_register(db) != SQLITE_OK) return fail(db, "register"); + + if (!exec_ok(db, "SELECT vexfs_init(); SELECT vexfs_workspace_create('default');")) + return fail(db, "initialize"); + if (!exec_ok(db, "SELECT vexfs_mkdir('default','/notes/2026');")) + return fail(db, "mkdir"); + if (!exec_ok(db, "SELECT vexfs_write('default','/notes/2026/hello.txt',X'68656C6C6F');")) + return fail(db, "write"); + + char *text = scalar_text(db, "SELECT CAST(vexfs_read('default','/notes/2026/hello.txt') AS TEXT)"); + if (text == NULL || strcmp(text, "hello") != 0) return fail(db, "read after write"); + free(text); + + // SQL 事务回滚必须同时回滚文件内容。 + if (!exec_ok(db, "BEGIN; SELECT vexfs_write('default','/notes/2026/hello.txt','changed'); ROLLBACK;")) + return fail(db, "transaction rollback"); + text = scalar_text(db, "SELECT CAST(vexfs_read('default','/notes/2026/hello.txt') AS TEXT)"); + if (text == NULL || strcmp(text, "hello") != 0) return fail(db, "rollback content"); + free(text); + + char *handle = scalar_text(db, + "SELECT vexfs_handle_open('default','/notes/2026/hello.txt','rw','open-1')"); + if (handle == NULL || strlen(handle) != 32) return fail(db, "handle open"); + + char sql[1024]; + snprintf(sql, sizeof(sql), + "SELECT vexfs_handle_stage_write('%s',5,' world','write-1')", handle); + sqlite3_int64 generation = scalar_int(db, sql, &ok); + if (!ok || generation != 1) return fail(db, "stage write"); + // 同一个 request_id 重试，不得再生成一代。 + sqlite3_int64 repeated = scalar_int(db, sql, &ok); + if (!ok || repeated != generation) return fail(db, "idempotent stage write"); + snprintf(sql, sizeof(sql), + "SELECT vexfs_handle_stage_write('%s',4,'!', 'write-1')", handle); + if (!exec_fails(db, sql)) return fail(db, "request id argument mismatch"); + + snprintf(sql, sizeof(sql), + "SELECT CAST(vexfs_handle_read('%s',0,64) AS TEXT)", handle); + text = scalar_text(db, sql); + if (text == NULL || strcmp(text, "hello world") != 0) return fail(db, "read staged data"); + free(text); + + snprintf(sql, sizeof(sql), + "SELECT vexfs_handle_publish('%s',1,'data','publish-1')", handle); + sqlite3_int64 version = scalar_int(db, sql, &ok); + if (!ok || version != 2) return fail(db, "publish"); + snprintf(sql, sizeof(sql), + "SELECT vexfs_handle_close('%s',1,'close-1')", handle); + text = scalar_text(db, sql); + if (text == NULL || strcmp(text, "closed") != 0) return fail(db, "close"); + free(text); + free(handle); + + // 暂存区每个 handle 只有一行；跨 64 KiB 写入按容量扩展，并补零空洞。 + if (!exec_ok(db, "SELECT vexfs_write('default','/notes/grow.bin',X'');")) + return fail(db, "grow seed"); + handle = scalar_text(db, + "SELECT vexfs_handle_open('default','/notes/grow.bin','rw','open-grow-1')"); + if (handle == NULL) return fail(db, "grow open"); + snprintf(sql, sizeof(sql), + "SELECT vexfs_handle_stage_write('%s',70000,X'5A','write-grow-1')", handle); + generation = scalar_int(db, sql, &ok); + if (!ok || generation != 1) return fail(db, "grow stage"); + snprintf(sql, sizeof(sql), + "SELECT count(*) FROM _vexfs_staging WHERE handle_id='%s'", handle); + if (scalar_int(db, sql, &ok) != 1 || !ok) return fail(db, "bounded staging rows"); + snprintf(sql, sizeof(sql), + "SELECT capacity FROM _vexfs_staging WHERE handle_id='%s'", handle); + if (scalar_int(db, sql, &ok) != 131072 || !ok) return fail(db, "geometric staging capacity"); + snprintf(sql, sizeof(sql), + "SELECT hex(vexfs_handle_read('%s',69999,2))", handle); + text = scalar_text(db, sql); + if (text == NULL || strcmp(text, "005A") != 0) return fail(db, "sparse staging read"); + free(text); + snprintf(sql, sizeof(sql), + "SELECT vexfs_handle_truncate('%s',5,'truncate-grow-1')", handle); + generation = scalar_int(db, sql, &ok); + if (!ok || generation != 2) return fail(db, "handle truncate"); + snprintf(sql, sizeof(sql), + "SELECT length(vexfs_handle_read('%s',0,100))", handle); + if (scalar_int(db, sql, &ok) != 5 || !ok) return fail(db, "truncated staged size"); + snprintf(sql, sizeof(sql), + "SELECT vexfs_handle_truncate('%s',70001,'regrow-grow-1')", handle); + generation = scalar_int(db, sql, &ok); + if (!ok || generation != 3) return fail(db, "handle regrow"); + snprintf(sql, sizeof(sql), + "SELECT hex(vexfs_handle_read('%s',70000,1))", handle); + text = scalar_text(db, sql); + if (text == NULL || strcmp(text, "00") != 0) return fail(db, "truncate must not resurrect data"); + free(text); + snprintf(sql, sizeof(sql), + "SELECT vexfs_handle_truncate('%s',5,'truncate-grow-2')", handle); + generation = scalar_int(db, sql, &ok); + if (!ok || generation != 4) return fail(db, "handle final truncate"); + snprintf(sql, sizeof(sql), + "SELECT vexfs_handle_publish('%s',4,'data','publish-grow-1');" + "SELECT vexfs_handle_close('%s',1,'close-grow-1')", handle, handle); + if (!exec_ok(db, sql)) return fail(db, "publish truncated file"); + free(handle); + + // replace rename 是单个数据库操作，inode 路径会随父目录重命名实时更新。 + if (!exec_ok(db, + "SELECT vexfs_mkdir('default','/rename/source/child');" + "SELECT vexfs_write('default','/rename/source/child/value.txt','source');" + "SELECT vexfs_write('default','/rename/source/child/live.txt','live');" + "SELECT vexfs_write('default','/rename/destination.txt','destination');")) + return fail(db, "rename seed"); + sqlite3_int64 inode = scalar_int(db, + "SELECT json_extract(vexfs_stat('default','/rename/source/child/live.txt'),'$.inode')", &ok); + if (!ok || inode == 0) return fail(db, "rename inode"); + if (!exec_ok(db, + "SELECT vexfs_rename('default','/rename/source/child/value.txt'," + "'/rename/destination.txt',1);" + "SELECT vexfs_rename('default','/rename/source','/rename/moved',0);")) + return fail(db, "atomic rename"); + snprintf(sql, sizeof(sql), "SELECT vexfs_path('default',%lld)", (long long)inode); + text = scalar_text(db, sql); + if (text == NULL || strcmp(text, "/rename/moved/child/live.txt") != 0) { + fprintf(stderr, "resolved inode path: %s\n", text == NULL ? "(null)" : text); + return fail(db, "inode path after ancestor rename"); + } + free(text); + text = scalar_text(db, + "SELECT CAST(vexfs_read('default','/rename/destination.txt') AS TEXT)"); + if (text == NULL || strcmp(text, "source") != 0) return fail(db, "rename replacement content"); + free(text); + + text = scalar_text(db, "SELECT CAST(vexfs_read('default','/notes/2026/hello.txt') AS TEXT)"); + if (text == NULL || strcmp(text, "hello world") != 0) return fail(db, "read published data"); + free(text); + + text = scalar_text(db, "SELECT vexfs_list('default','/notes/2026')"); + if (text == NULL || strstr(text, "hello.txt") == NULL) return fail(db, "list"); + free(text); + + text = scalar_text(db, "SELECT vexfs_history('default','/notes/2026/hello.txt')"); + if (text == NULL || strstr(text, "\"version\":2") == NULL || + strstr(text, "\"version\":1") == NULL) return fail(db, "history"); + free(text); + text = scalar_text(db, + "SELECT vexfs_history('default','/notes/2026/hello.txt',1,0)"); + if (text == NULL || strstr(text, "\"entries\":[{\"version\":2") == NULL || + strstr(text, "\"next_before\":2") == NULL) return fail(db, "paged history"); + free(text); + text = scalar_text(db, + "SELECT CAST(vexfs_read_version('default','/notes/2026/hello.txt',1) AS TEXT)"); + if (text == NULL || strcmp(text, "hello") != 0) return fail(db, "read version"); + free(text); + version = scalar_int(db, + "SELECT vexfs_restore_version('default','/notes/2026/hello.txt',1,2)", &ok); + if (!ok || version != 3) return fail(db, "restore version"); + text = scalar_text(db, + "SELECT CAST(vexfs_read('default','/notes/2026/hello.txt') AS TEXT)"); + if (text == NULL || strcmp(text, "hello") != 0) return fail(db, "restored content"); + free(text); + version = scalar_int(db, + "SELECT source_version_no FROM _vexfs_file_versions " + "WHERE version_no=3 ORDER BY id DESC LIMIT 1", &ok); + if (!ok || version != 1) return fail(db, "restored content reference"); + if (!exec_fails(db, + "SELECT vexfs_restore_version('default','/notes/2026/hello.txt',2,2)")) + return fail(db, "stale restore conflict"); + + if (!exec_ok(db, + "SELECT vexfs_move('default','/notes/2026/hello.txt','/notes/hello.txt');" + "SELECT vexfs_remove('default','/notes/2026',0);")) + return fail(db, "move and remove"); + + scalar_int(db, "SELECT vexfs_item_reclaim('default','reclaim-1')", &ok); + if (!ok) return fail(db, "reclaim"); + + sqlite3_close(db); + + // 0.1.0 数据库原地升级，保留当前 dirty generation，并清空旧幂等缓存。 + db = NULL; + if (sqlite3_open(":memory:", &db) != SQLITE_OK) return fail(db, "migration open"); + if (vexdb_sqlite_register(db) != SQLITE_OK) return fail(db, "migration register"); + if (!exec_ok(db, + "CREATE TABLE _vexfs_meta(key TEXT PRIMARY KEY,value TEXT NOT NULL);" + "INSERT INTO _vexfs_meta VALUES('contract_version','0.1.0');" + "CREATE TABLE _vexfs_inodes(kind TEXT,deleted_at INTEGER,current_version INTEGER);" + "INSERT INTO _vexfs_inodes VALUES('directory',NULL,0);" + "CREATE TABLE _vexfs_handles(id TEXT PRIMARY KEY,dirty_generation INTEGER NOT NULL);" + "INSERT INTO _vexfs_handles VALUES('h',1);" + "CREATE TABLE _vexfs_staging(handle_id TEXT,generation INTEGER,content BLOB," + "created_at INTEGER,PRIMARY KEY(handle_id,generation));" + "INSERT INTO _vexfs_staging VALUES('h',1,X'6162',123);" + "CREATE TABLE _vexfs_requests(request_id TEXT PRIMARY KEY,operation TEXT NOT NULL," + "result_integer INTEGER,result_text TEXT,created_at INTEGER);" + "INSERT INTO _vexfs_requests VALUES('old','write',1,NULL,123);" + "SELECT vexfs_init();")) return fail(db, "0.1 migration"); + text = scalar_text(db, "SELECT value FROM _vexfs_meta WHERE key='contract_version'"); + if (text == NULL || strcmp(text, "0.3.0") != 0) return fail(db, "migration version"); + free(text); + if (scalar_int(db, "SELECT logical_size FROM _vexfs_staging WHERE handle_id='h'", &ok) != 2 || + !ok) return fail(db, "migration staging"); + if (scalar_int(db, "SELECT count(*) FROM _vexfs_requests", &ok) != 0 || !ok) + return fail(db, "migration request cache"); + if (scalar_int(db, "SELECT current_version FROM _vexfs_inodes", &ok) != 1 || !ok) + return fail(db, "migration directory verifier"); + sqlite3_close(db); + printf("VEXFS STATIC SMOKE: PASS\n"); + return 0; +} From 230a91136b9676cc3f68f6ac4450460983bfccbd Mon Sep 17 00:00:00 2001 From: Four <8122332+HungryFour@users.noreply.github.com> Date: Thu, 23 Jul 2026 15:10:57 +0800 Subject: [PATCH 02/28] Add VexFS filesystem integration --- .gitignore | 13 +- README.md | 28 +- README.zh.md | 27 +- agent_files/cli/test/vexdb_unified_smoke.sh | 55 + agent_files/cli/test/vexfs_cli_smoke.sh | 53 +- .../test/vexfs_platform_registry_smoke.cpp | 65 + .../cli/test/vexfs_platform_stub_smoke.cpp | 80 + agent_files/cli/vexdb_main.cpp | 185 + agent_files/cli/vexdb_shell_init.c | 14 + agent_files/cli/vexfs_main.cpp | 695 ++-- agent_files/cli/vexfs_platform.h | 37 + agent_files/cli/vexfs_platform_linux.cpp | 269 ++ agent_files/cli/vexfs_platform_macos.cpp | 342 ++ agent_files/cli/vexfs_platform_registry.cpp | 160 + agent_files/cli/vexfs_platform_registry.h | 17 + agent_files/cli/vexfs_platform_stub.cpp | 101 + agent_files/linux/package/install.sh | 58 + agent_files/linux/package/uninstall.sh | 18 + ...77\347\224\250\350\257\264\346\230\216.md" | 114 + .../macos/ExportOptions-DeveloperID.plist | 14 + .../macos/VexFS.xcodeproj/project.pbxproj | 444 +- agent_files/macos/VexFSApp/ContentView.swift | 15 +- .../macos/VexFSAppEx/VexFS-Bridging-Header.h | 2 +- .../macos/VexFSAppEx/VexFSBackend.swift | 259 +- .../macos/VexFSAppEx/VexFSFileSystem.swift | 15 +- agent_files/macos/VexFSAppEx/VexFSItem.swift | 17 +- .../VexFSAppEx/VexFSVolume+Operations.swift | 278 +- .../macos/VexFSAppEx/VexFSVolume.swift | 214 +- agent_files/macos/generate_xcode_project.rb | 17 +- agent_files/macos/package/Install.command | 2 +- agent_files/macos/package/README.md | 120 +- agent_files/macos/package/install.sh | 192 +- agent_files/macos/package/uninstall.sh | 15 +- ...77\347\224\250\350\257\264\346\230\216.md" | 400 ++ agent_files/macos/package_preview.sh | 388 +- .../mount/common/include/vexfs_checksum.h | 33 + .../common/include/vexfs_mount_contract.h | 151 - .../common/include/vexfs_runtime_admin.h | 79 + .../mount/common/include/vexfs_runtime_io.h | 144 + .../common/include/vexfs_runtime_types.h | 82 + .../mount/common/src/vexfs_checksum.cpp | 157 + .../src/vexfs_mount_contract_sqlite.cpp | 871 +++- .../test/vexfs_mount_contract_smoke.cpp | 580 ++- agent_files/mount/linux/vexfs_fuse.cpp | 895 +++++ build_sqlite.sh | 18 +- common/vtl/allocator | 3 + ...-07-16_agent-files-universal-filesystem.md | 1435 +++++++ ...2026-07-21_vexfs-final-goal-and-roadmap.md | 479 +++ docs/specs/2026-07-16_vexfs-product-spec.md | 1645 ++++++++ scripts/release.sh | 223 +- tests/eval/vexfs/Dockerfile.linux-fuse | 14 + tests/eval/vexfs/README.md | 86 +- tests/eval/vexfs/documentation_smoke.sh | 273 ++ tests/eval/vexfs/run.py | 3273 ++++++++++++++- .../vexfs/run_cross_platform_portability.sh | 68 + ...cross_platform_portability_in_container.sh | 27 + tests/eval/vexfs/run_linux_mount.sh | 20 + .../vexfs/run_linux_mount_in_container.sh | 48 + tests/spec/_lib/docker/run_sqlite.sh | 5 +- tests/spec/_lib/docker/sqlite_spec_runner.py | 16 +- .../sqlite/agent_files/vexfs_contract.yaml | 112 +- tests/spec/sqlite/vtab/vtab_verify.yaml | 2 +- vexdb_sqlite/CMakeLists.txt | 156 +- vexdb_sqlite/README.md | 33 +- .../include/agent_files/vexfs_sqlite.h | 4 + vexdb_sqlite/src/agent_files/vexfs_sqlite.cpp | 3564 +++++++++++++++-- vexdb_sqlite/src/vexdb_sqlite_init.cpp | 6 +- vexdb_sqlite/test/vexfs_static_smoke.c | 135 +- 68 files changed, 17841 insertions(+), 1519 deletions(-) create mode 100755 agent_files/cli/test/vexdb_unified_smoke.sh create mode 100644 agent_files/cli/test/vexfs_platform_registry_smoke.cpp create mode 100644 agent_files/cli/test/vexfs_platform_stub_smoke.cpp create mode 100644 agent_files/cli/vexdb_main.cpp create mode 100644 agent_files/cli/vexdb_shell_init.c create mode 100644 agent_files/cli/vexfs_platform.h create mode 100644 agent_files/cli/vexfs_platform_linux.cpp create mode 100644 agent_files/cli/vexfs_platform_macos.cpp create mode 100644 agent_files/cli/vexfs_platform_registry.cpp create mode 100644 agent_files/cli/vexfs_platform_registry.h create mode 100644 agent_files/cli/vexfs_platform_stub.cpp create mode 100755 agent_files/linux/package/install.sh create mode 100755 agent_files/linux/package/uninstall.sh create mode 100644 "agent_files/linux/package/\344\275\277\347\224\250\350\257\264\346\230\216.md" create mode 100644 agent_files/macos/ExportOptions-DeveloperID.plist create mode 100644 "agent_files/macos/package/\344\275\277\347\224\250\350\257\264\346\230\216.md" create mode 100644 agent_files/mount/common/include/vexfs_checksum.h delete mode 100644 agent_files/mount/common/include/vexfs_mount_contract.h create mode 100644 agent_files/mount/common/include/vexfs_runtime_admin.h create mode 100644 agent_files/mount/common/include/vexfs_runtime_io.h create mode 100644 agent_files/mount/common/include/vexfs_runtime_types.h create mode 100644 agent_files/mount/common/src/vexfs_checksum.cpp create mode 100644 agent_files/mount/linux/vexfs_fuse.cpp create mode 100644 docs/design/2026-07-16_agent-files-universal-filesystem.md create mode 100644 docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md create mode 100644 docs/specs/2026-07-16_vexfs-product-spec.md create mode 100644 tests/eval/vexfs/Dockerfile.linux-fuse create mode 100755 tests/eval/vexfs/documentation_smoke.sh create mode 100755 tests/eval/vexfs/run_cross_platform_portability.sh create mode 100755 tests/eval/vexfs/run_cross_platform_portability_in_container.sh create mode 100755 tests/eval/vexfs/run_linux_mount.sh create mode 100755 tests/eval/vexfs/run_linux_mount_in_container.sh diff --git a/.gitignore b/.gitignore index 9319d6fb50..b6db5cefab 100644 --- a/.gitignore +++ b/.gitignore @@ -406,8 +406,17 @@ vexdb_duckdb/test/sql/vex/ scripts/env.local.sh scripts/*.local.sh -# 内部文档目录（本地保留，不入仓库） -docs/ +# 本地分析和生成报告不入仓库；当前 VexFS 产品基线必须随代码版本管理。 +docs/* +!docs/specs/ +docs/specs/* +!docs/specs/2026-07-16_vexfs-product-spec.md +!docs/plans/ +docs/plans/* +!docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md +!docs/design/ +docs/design/* +!docs/design/2026-07-16_agent-files-universal-filesystem.md CHANGELOG.md # SIFT benchmark data (large, generated from hdf5; not in repo) diff --git a/README.md b/README.md index f916eaa3a0..12b1494d90 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ **English** | **[中文](README.zh.md)** -`VexDB-Lite` is a vector similarity search engine for PostgreSQL (`vexdb_lite` extension) and DuckDB (`vexdb_lite` extension). Both backends share the same graph index algorithm, SIMD distance dispatch, and quantization kernel. +`VexDB-Lite` extends PostgreSQL, DuckDB, and SQLite with a shared graph index, SIMD distance dispatch, and quantization kernel. The SQLite edition also includes VexFS, so one database can hold regular tables, vector indexes, and database-managed files. > See [vexdb_duckdb/README.md](vexdb_duckdb/README.md) for the DuckDB extension docs. > This root README is a project-level overview and build guide. @@ -46,6 +46,32 @@ Current functionality: - Vector buffer cache and parallel index build - Runtime settings: `vexdb_ef_search`, `vexdb_brute_force_threshold`, `vexdb_pq_search_mode`, `vexdb_pq_refine_k_factor` +### 1.3 SQLite: unified VexDB-Lite + +The unified macOS package installs: + +- `vexdb`, with SQLite 3.45.3 and automatic extension registration; +- `GRAPH_INDEX` and vector distance functions; +- VexFS files, directories, history, restore, SHA-256, and read-only integrity checks; +- `vexdb fs ...`, including database-batched `grep` and an optional FTS5 trigram index, with `vexfs ...` kept as a compatible shortcut; +- a VexFS FSKit extension for native `ls`, `cat`, `grep`, `cp`, and `mv` after enablement. + +```bash +vexdb agent.db +vexdb agent.db "SELECT vexdb_l2_distance('[1,2]', '[4,6]');" +vexdb fs --db agent.db write /notes/hello.txt +vexdb fs --db agent.db grep -n hello /notes +vexdb fs --db agent.db check +vexdb fs --db agent.db index enable +vexdb fs --db agent.db mount ~/VexDB +``` + +The text index is disabled by default, so ordinary writes have no index overhead. Without the +index, or when the SQLite host has no FTS5 support, `grep` falls back to a single-connection +database scan. + +Build the unified macOS preview with `bash agent_files/macos/package_preview.sh`. + --- ## 2. Capability Matrix diff --git a/README.zh.md b/README.zh.md index e2cba24e8f..d696455e97 100644 --- a/README.zh.md +++ b/README.zh.md @@ -2,7 +2,7 @@ **[English](README.md)** | **中文** -`VexDB-Lite` 是一个高性能向量检索系统，提供 PostgreSQL（`vexdb_lite` 扩展）和 DuckDB（`vexdb_lite` 扩展）两种适配形式，共享同一套 graph_index 图索引算法、SIMD 距离分发和量化器内核。 +`VexDB-Lite` 是一个数据库能力扩展，提供 PostgreSQL、DuckDB 和 SQLite 三种适配形式，共享同一套 graph_index 图索引算法、SIMD 距离分发和量化器内核。SQLite 版还内置 VexFS 文件能力，让同一个数据库同时保存普通表、向量索引和文件。 > DuckDB 扩展详见 [vexdb_duckdb/README.md](vexdb_duckdb/README.md)。 > 本根 README 只做项目级综述与构建总览。 @@ -45,6 +45,31 @@ - 向量缓存、并行建索引 - 运行参数：`vexdb_ef_search`、`vexdb_brute_force_threshold`、`vexdb_pq_search_mode`、`vexdb_pq_refine_k_factor` +### 1.3 SQLite：统一的 VexDB-Lite + +macOS 统一包一次安装即可获得： + +- `vexdb`：自带 SQLite 3.45.3，无需手动 `.load`； +- `GRAPH_INDEX` 和向量距离函数； +- VexFS 文件、目录、历史版本、恢复、SHA-256 和只读完整性检查； +- `vexdb fs ...` 文件命令，含数据库批量 `grep` 和可选 FTS5 trigram 索引，兼容入口为 `vexfs ...`； +- VexFS FSKit extension，启用后可用原生 `ls`、`cat`、`grep`、`cp`、`mv`。 + +```bash +vexdb agent.db +vexdb agent.db "SELECT vexdb_l2_distance('[1,2]', '[4,6]');" +vexdb fs --db agent.db write /notes/hello.txt +vexdb fs --db agent.db grep -n hello /notes +vexdb fs --db agent.db check +vexdb fs --db agent.db index enable +vexdb fs --db agent.db mount ~/VexDB +``` + +文本索引默认关闭，不增加普通写入开销；未开启索引或宿主没有 FTS5 时，`grep` 自动使用 +单连接数据库扫描。 + +统一 macOS 技术预览包通过 `bash agent_files/macos/package_preview.sh` 构建。 + --- ## 2. 产品能力矩阵 diff --git a/agent_files/cli/test/vexdb_unified_smoke.sh b/agent_files/cli/test/vexdb_unified_smoke.sh new file mode 100755 index 0000000000..d2ed4343be --- /dev/null +++ b/agent_files/cli/test/vexdb_unified_smoke.sh @@ -0,0 +1,55 @@ +#!/bin/bash +set -euo pipefail + +VEXDB="${1:?usage: vexdb_unified_smoke.sh /path/to/vexdb}" +TMP_DIR="$(mktemp -d -t vexdb-unified-smoke)" +DB="$TMP_DIR/agent.sqlite3" +COPY="$TMP_DIR/agent-copy.sqlite3" +BACKUP_LINK_TARGET="$TMP_DIR/backup-link-target" +BACKUP_LINK="$TMP_DIR/backup-link.sqlite3" +VEXFS_EXE="$TMP_DIR/vexfs.exe" +cleanup() { + rm -f "$DB" "$DB-wal" "$DB-shm" "$COPY" "$COPY-wal" "$COPY-shm" \ + "$BACKUP_LINK" "$BACKUP_LINK_TARGET" "$VEXFS_EXE" + rmdir "$TMP_DIR" 2>/dev/null || true +} +trap cleanup EXIT + +"$VEXDB" --version | grep -q 'vexdb-lite' +cp "$VEXDB" "$VEXFS_EXE" +"$VEXFS_EXE" --help | grep -q 'Commands:' + +RESULT="$("$VEXDB" "$DB" \ + "SELECT vexfs_contract_version(), vexdb_l2_distance('[1,2]','[4,6]');")" +[ "$RESULT" = "0.7.0|5.0" ] + +"$VEXDB" "$DB" \ + "CREATE VIRTUAL TABLE idx USING GRAPH_INDEX(embedding FLOAT[2], metric=l2); \ + INSERT INTO idx(rowid, embedding) VALUES (1, '[1,1]'), (2, '[9,9]');" >/dev/null +[ "$("$VEXDB" "$DB" \ + "SELECT rowid FROM idx WHERE embedding MATCH '[1,1]' AND k=1;")" = "1" ] + +"$VEXDB" fs --db "$DB" --workspace smoke setup >/dev/null +printf 'one product' | "$VEXDB" fs --db "$DB" --workspace smoke write /note.txt >/dev/null +[ "$("$VEXDB" fs --db "$DB" --workspace smoke cat /note.txt)" = "one product" ] + +"$VEXDB" backup "$DB" "$COPY" >/dev/null +[ "$("$VEXDB" "$COPY" "SELECT count(*) FROM idx;")" = "2" ] +[ "$("$VEXDB" fs --db "$COPY" --workspace smoke cat /note.txt)" = "one product" ] +COPY_MODE="$(stat -f '%Lp' "$COPY" 2>/dev/null || stat -c '%a' "$COPY")" +[ "$COPY_MODE" = "600" ] + +if [ "$(uname -s)" != Windows_NT ]; then + printf 'must-not-change' > "$BACKUP_LINK_TARGET" + chmod 0644 "$BACKUP_LINK_TARGET" + ln -s "$BACKUP_LINK_TARGET" "$BACKUP_LINK" + if "$VEXDB" backup "$DB" "$BACKUP_LINK" >/dev/null 2>&1; then + echo "backup unexpectedly followed a destination symlink" >&2 + exit 1 + fi + [ "$(cat "$BACKUP_LINK_TARGET")" = "must-not-change" ] + TARGET_MODE="$(stat -f '%Lp' "$BACKUP_LINK_TARGET" 2>/dev/null || stat -c '%a' "$BACKUP_LINK_TARGET")" + [ "$TARGET_MODE" = "644" ] +fi + +echo "VEXDB UNIFIED CLI SMOKE: PASS" diff --git a/agent_files/cli/test/vexfs_cli_smoke.sh b/agent_files/cli/test/vexfs_cli_smoke.sh index 2ed08e5acf..18771dae75 100755 --- a/agent_files/cli/test/vexfs_cli_smoke.sh +++ b/agent_files/cli/test/vexfs_cli_smoke.sh @@ -18,6 +18,12 @@ set -e [ ! -e "$DB" ] printf '%s' "$MISSING_DOCTOR" | /usr/bin/python3 -c \ 'import json,sys; value=json.load(sys.stdin); assert "error" in value["database"]' +set +e +"$VEXFS" --db "$DB" --workspace smoke --json check >/dev/null 2>&1 +MISSING_CHECK_STATUS=$? +set -e +[ "$MISSING_CHECK_STATUS" -eq 7 ] +[ ! -e "$DB" ] "$VEXFS" --db "$DB" --workspace smoke setup >/dev/null "$VEXFS" --db "$DB" --workspace smoke mkdir /agent @@ -28,13 +34,27 @@ printf 'beta' | "$VEXFS" --db "$DB" --workspace smoke write /agent/version.txt > CONTENT="$("$VEXFS" --db "$DB" --workspace smoke cat /agent/task.txt)" [ "$CONTENT" = "hello from cli" ] +"$VEXFS" --db "$DB" --workspace smoke ln /agent/task.txt /agent/task-copy.txt +LINK_STAT="$("$VEXFS" --db "$DB" --workspace smoke stat /agent/task-copy.txt)" +printf '%s' "$LINK_STAT" | /usr/bin/python3 -c \ + 'import json,sys; value=json.load(sys.stdin); assert value["link_count"] == 2' +"$VEXFS" --db "$DB" --workspace smoke chown -:1234 /agent/task.txt +printf '%s' "$("$VEXFS" --db "$DB" --workspace smoke stat /agent/task.txt)" | /usr/bin/python3 -c \ + 'import json,sys; assert json.load(sys.stdin)["gid"] == 1234' +printf '[{"principal":"alice","effect":"allow","permissions":"read,write","inherit":1}]' | + "$VEXFS" --db "$DB" --workspace smoke setfacl /agent/task.txt +ACL="$("$VEXFS" --db "$DB" --workspace smoke getfacl /agent/task.txt)" +printf '%s' "$ACL" | /usr/bin/python3 -c \ + 'import json,sys; value=json.load(sys.stdin); assert value[0]["principal"] == "alice"' + LIST="$("$VEXFS" --db "$DB" --workspace smoke --json ls /agent)" printf '%s' "$LIST" | /usr/bin/python3 -c \ - 'import json,sys; rows=json.load(sys.stdin); assert rows[0]["name"] == "task.txt"' + 'import json,sys; rows=json.load(sys.stdin); assert any(row["name"] == "task.txt" for row in rows)' HISTORY="$("$VEXFS" --db "$DB" --workspace smoke --json history /agent/version.txt)" printf '%s' "$HISTORY" | /usr/bin/python3 -c \ - 'import json,sys; page=json.load(sys.stdin); rows=page["entries"]; assert [row["version"] for row in rows] == [2,1]; assert rows[0]["current"] is True; assert page["next_before"] is None' + 'import json,sys; page=json.load(sys.stdin); rows=page["entries"]; assert [row["version"] for row in rows] == [2,1]; assert all(len(row["checksum"]) == 64 for row in rows); assert rows[0]["current"] is True; assert page["next_before"] is None' +[ "$("$VEXFS" --db "$DB" --workspace smoke stat /agent/version.txt | /usr/bin/python3 -c 'import json,sys; print(json.load(sys.stdin)["checksum"])')" = "f44e64e75f3948e9f73f8dfa94721c4ce8cbb4f265c4790c702b2d41cfbf2753" ] [ "$("$VEXFS" --db "$DB" --workspace smoke show /agent/version.txt --version 1)" = "alpha" ] set +e DIFF="$("$VEXFS" --db "$DB" --workspace smoke diff /agent/version.txt --from 1 --to 2)" @@ -52,13 +72,40 @@ set -e [ "$("$VEXFS" --db "$DB" --workspace smoke restore /agent/version.txt --version 1)" = "3" ] [ "$("$VEXFS" --db "$DB" --workspace smoke cat /agent/version.txt)" = "alpha" ] +SNAPSHOT_COMMIT="$("$VEXFS" --db "$DB" --workspace smoke snapshot create cli-baseline)" +[ "$SNAPSHOT_COMMIT" -gt 0 ] +printf 'changed after snapshot' | "$VEXFS" --db "$DB" --workspace smoke write /agent/task.txt >/dev/null +set +e +SNAPSHOT_DIFF="$("$VEXFS" --db "$DB" --workspace smoke snapshot diff cli-baseline)" +SNAPSHOT_DIFF_STATUS=$? +set -e +[ "$SNAPSHOT_DIFF_STATUS" -eq 1 ] +printf '%s' "$SNAPSHOT_DIFF" | /usr/bin/python3 -c \ + 'import json,sys; value=json.load(sys.stdin); assert any(row["path"] == "/agent/task.txt" for row in value["changes"])' +"$VEXFS" --db "$DB" --workspace smoke snapshot restore cli-baseline --dry-run >/dev/null +"$VEXFS" --db "$DB" --workspace smoke snapshot restore cli-baseline >/dev/null +[ "$("$VEXFS" --db "$DB" --workspace smoke cat /agent/task.txt)" = "hello from cli" ] +[ "$("$VEXFS" --db "$DB" --workspace smoke cat /agent/task-copy.txt)" = "hello from cli" ] +"$VEXFS" --db "$DB" --workspace smoke snapshot diff cli-baseline >/dev/null +SNAPSHOT_LIST="$("$VEXFS" --db "$DB" --workspace smoke --json snapshot list)" +printf '%s' "$SNAPSHOT_LIST" | /usr/bin/python3 -c \ + 'import json,sys; value=json.load(sys.stdin); assert value[0]["name"] == "cli-baseline"' +"$VEXFS" --db "$DB" --workspace smoke snapshot show cli-baseline | /usr/bin/python3 -c \ + 'import json,sys; value=json.load(sys.stdin); assert value["name"] == "cli-baseline"' +"$VEXFS" --db "$DB" --workspace smoke snapshot drop cli-baseline + +CHECK="$("$VEXFS" --db "$DB" --workspace smoke --json check)" +printf '%s' "$CHECK" | /usr/bin/python3 -c \ + 'import json,sys; value=json.load(sys.stdin); assert value["ok"] is True; assert value["mode"] == "deep"; assert value["checked"]["versions"] >= 5' +"$VEXFS" --db "$DB" --workspace smoke check --quick | grep -q '^OK ' + set +e DOCTOR="$("$VEXFS" --db "$DB" --workspace smoke --json doctor)" DOCTOR_STATUS=$? set -e [ "$DOCTOR_STATUS" -eq 0 ] || [ "$DOCTOR_STATUS" -eq 1 ] printf '%s' "$DOCTOR" | /usr/bin/python3 -c \ - 'import json,sys; value=json.load(sys.stdin); assert value["database"]["contract_version"] == "0.3.0"' + 'import json,sys; value=json.load(sys.stdin); assert value["database"]["schema_version"] == "0.7.0"' MOUNTS="$("$VEXFS" --json mount status)" printf '%s' "$MOUNTS" | /usr/bin/python3 -c 'import json,sys; assert isinstance(json.load(sys.stdin), list)' diff --git a/agent_files/cli/test/vexfs_platform_registry_smoke.cpp b/agent_files/cli/test/vexfs_platform_registry_smoke.cpp new file mode 100644 index 0000000000..f61a8db4e4 --- /dev/null +++ b/agent_files/cli/test/vexfs_platform_registry_smoke.cpp @@ -0,0 +1,65 @@ +#include "vexfs_platform_registry.h" + +#include +#include +#include +#include + +namespace { + +int Fail(const std::string &message) { + std::cerr << "VEXFS MOUNT REGISTRY SMOKE: FAIL: " << message << '\n'; + return 1; +} + +} // namespace + +int main() { + char directory_template[] = "/tmp/vexfs-mount-registry-XXXXXX"; + const char *created = mkdtemp(directory_template); + if (created == nullptr) return Fail("mkdtemp"); + const std::filesystem::path root(created); + const std::filesystem::path registry = root / "registry"; + const std::filesystem::path source = root / "source dir"; + const std::filesystem::path target = root / "mount point"; + const std::filesystem::path database = root / "workspace.sqlite3"; + std::filesystem::create_directories(source); + std::filesystem::create_directories(target); + + try { + VexFSPlatformRememberMount(registry, + {source.string(), target.string(), "vexfs", database.string(), "agent"}); + std::string encoded_source = source.string(); + const size_t space = encoded_source.find(' '); + if (space != std::string::npos) encoded_source.replace(space, 1, "%20"); + auto attached = VexFSPlatformAttachMountRegistry( + {{"file://" + encoded_source + "/", target.string(), "vexfs"}}, registry); + if (attached.size() != 1 || + attached[0].database != std::filesystem::weakly_canonical(database).string() || + attached[0].workspace != "agent") { + const std::string details = attached.empty() ? "empty" : + "database=" + attached[0].database + ", workspace=" + attached[0].workspace; + std::filesystem::remove_all(root); + return Fail("matching live mount identity: " + details); + } + attached = VexFSPlatformAttachMountRegistry( + {{"file:///different/source", target.string(), "vexfs"}}, registry); + if (!attached[0].database.empty() || !attached[0].workspace.empty()) { + std::filesystem::remove_all(root); + return Fail("source mismatch must not be trusted"); + } + VexFSPlatformForgetMount(registry, target.string()); + attached = VexFSPlatformAttachMountRegistry( + {{source.string(), target.string(), "vexfs"}}, registry); + if (!attached[0].database.empty()) { + std::filesystem::remove_all(root); + return Fail("forgotten mount identity"); + } + } catch (const std::exception &error) { + std::filesystem::remove_all(root); + return Fail(error.what()); + } + std::filesystem::remove_all(root); + std::cout << "VEXFS MOUNT REGISTRY SMOKE: PASS\n"; + return 0; +} diff --git a/agent_files/cli/test/vexfs_platform_stub_smoke.cpp b/agent_files/cli/test/vexfs_platform_stub_smoke.cpp new file mode 100644 index 0000000000..349436cc32 --- /dev/null +++ b/agent_files/cli/test/vexfs_platform_stub_smoke.cpp @@ -0,0 +1,80 @@ +#include "vexfs_platform.h" + +#include +#include +#include +#include +#include + +namespace { + +void SetEnvironment(const char *name, const char *value) { +#if defined(_WIN32) + if (_putenv_s(name, value) != 0) throw std::runtime_error("_putenv_s failed"); +#else + if (setenv(name, value, 1) != 0) throw std::runtime_error("setenv failed"); +#endif +} + +void UnsetEnvironment(const char *name) { +#if defined(_WIN32) + if (_putenv_s(name, "") != 0) throw std::runtime_error("_putenv_s failed"); +#else + if (unsetenv(name) != 0) throw std::runtime_error("unsetenv failed"); +#endif +} + +int Fail(const std::string &message) { + std::cerr << "VEXFS PLATFORM STUB SMOKE: FAIL: " << message << '\n'; + return 1; +} + +} // namespace + +int main() { + try { + UnsetEnvironment("VEXDB_LITE_DATABASE"); + UnsetEnvironment("VEXFS_DATABASE"); +#if defined(VEXFS_PLATFORM_EXPECT_LINUX) + SetEnvironment("XDG_DATA_HOME", "/tmp/vexfs-platform-data"); + const std::string expected_database = + (std::filesystem::path("/tmp/vexfs-platform-data") / + "vexdb-lite/default.sqlite3").string(); + const std::string expected_platform = "linux"; + const std::string expected_driver = "libfuse3"; +#elif defined(VEXFS_PLATFORM_EXPECT_WINDOWS) + SetEnvironment("LOCALAPPDATA", "C:/Users/VexFS/AppData/Local"); + const std::string expected_database = + (std::filesystem::path("C:/Users/VexFS/AppData/Local") / + "VexDB-Lite/default.sqlite3").string(); + const std::string expected_platform = "windows"; + const std::string expected_driver = "WinFsp"; +#else +#error "expected platform is required" +#endif + if (VexFSPlatformDefaultDatabasePath() != expected_database) + return Fail("default database path"); + SetEnvironment("VEXDB_LITE_DATABASE", "/tmp/override.sqlite3"); + if (VexFSPlatformDefaultDatabasePath() != "/tmp/override.sqlite3") + return Fail("database environment override"); + UnsetEnvironment("VEXDB_LITE_DATABASE"); + + const VexFSPlatformState state = VexFSPlatformInspect(); + if (state.platform != expected_platform || state.mount_driver != expected_driver || + state.mount_driver_available || state.mount_ready || + state.extension_state != "not-implemented") { + return Fail("platform state"); + } + try { + VexFSPlatformMount("workspace.sqlite3", "default", "/tmp/vexfs-mount"); + return Fail("unsupported mount succeeded"); + } catch (const std::runtime_error &error) { + if (std::string(error.what()).find(expected_driver) == std::string::npos) + return Fail("unsupported mount error"); + } + std::cout << "VEXFS PLATFORM STUB SMOKE: PASS (" << expected_platform << ")\n"; + return 0; + } catch (const std::exception &error) { + return Fail(error.what()); + } +} diff --git a/agent_files/cli/vexdb_main.cpp b/agent_files/cli/vexdb_main.cpp new file mode 100644 index 0000000000..16ced2d135 --- /dev/null +++ b/agent_files/cli/vexdb_main.cpp @@ -0,0 +1,185 @@ +#include "sqlite3.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(_WIN32) +#include +#else +#include +#include +#include +#include +#endif + +#ifndef VEXDB_LITE_VERSION +#define VEXDB_LITE_VERSION "0.1.0-dev" +#endif +#ifndef VEXDB_SQLITE_GIT_HASH +#define VEXDB_SQLITE_GIT_HASH "unknown" +#endif + +extern "C" int vexdb_sqlite_shell_main(int argc, char **argv); +int VexFsMain(int argc, char **argv); + +namespace { + +std::string ProgramName(const char *argument) { + if (argument == nullptr || *argument == '\0') return "vexdb"; + const std::filesystem::path path(argument); + std::string extension = path.extension().string(); + std::transform(extension.begin(), extension.end(), extension.begin(), + [](unsigned char value) { return static_cast(std::tolower(value)); }); + return extension == ".exe" ? path.stem().string() : path.filename().string(); +} + +int RunWithArguments(int (*entry)(int, char **), const std::string &program_name, + int argc, char **argv, int first_argument) { + std::vector forwarded; + forwarded.reserve(static_cast(argc - first_argument + 2)); + forwarded.push_back(const_cast(program_name.c_str())); + for (int index = first_argument; index < argc; ++index) forwarded.push_back(argv[index]); + forwarded.push_back(nullptr); + return entry(static_cast(forwarded.size() - 1), forwarded.data()); +} + +void Usage(std::ostream &output) { + output << + "VexDB-Lite: SQLite, vector search and database-managed files\n" + "\n" + "Usage:\n" + " vexdb [SQLITE_OPTIONS] [DATABASE [SQL]]\n" + " vexdb sql [SQLITE_OPTIONS] [DATABASE [SQL]]\n" + " vexdb fs [--db DATABASE] COMMAND [ARGS]\n" + " vexdb backup SOURCE_DATABASE DESTINATION_DATABASE\n" + " vexfs [--db DATABASE] COMMAND [ARGS]\n" + "\n" + "Examples:\n" + " vexdb agent.db\n" + " vexdb agent.db \"SELECT vexdb_version();\"\n" + " vexdb fs --db agent.db ls /\n" + " vexdb fs --db agent.db mount ~/VexDB\n" + "\n" + "Run 'vexdb sql --help' for SQLite shell options or\n" + "'vexdb fs --help' for file commands.\n"; +} + +void CreatePrivateBackupDestination(const std::filesystem::path &destination) { +#if defined(_WIN32) + std::ofstream output(destination, std::ios::binary | std::ios::out); + if (!output) throw std::runtime_error("cannot create backup destination"); +#else + int flags = O_CREAT | O_EXCL | O_WRONLY; +#if defined(O_NOFOLLOW) + flags |= O_NOFOLLOW; +#endif + const int descriptor = open(destination.c_str(), flags, 0600); + if (descriptor < 0) { + throw std::runtime_error("cannot create private backup destination: " + + std::string(std::strerror(errno))); + } + if (close(descriptor) != 0) { + const std::string message = std::strerror(errno); + std::error_code ignored; + std::filesystem::remove(destination, ignored); + throw std::runtime_error("cannot close backup destination: " + message); + } +#endif +} + +int BackupDatabase(const std::filesystem::path &source, + const std::filesystem::path &destination) { + if (source.empty() || destination.empty()) + throw std::runtime_error("source and destination database paths are required"); + if (std::filesystem::absolute(source).lexically_normal() == + std::filesystem::absolute(destination).lexically_normal()) + throw std::runtime_error("source and destination database must be different"); + if (!std::filesystem::is_regular_file(source)) + throw std::runtime_error("source database does not exist: " + source.string()); + if (std::filesystem::exists(destination)) + throw std::runtime_error("destination database already exists: " + destination.string()); + if (!destination.parent_path().empty()) + std::filesystem::create_directories(destination.parent_path()); + CreatePrivateBackupDestination(destination); + + sqlite3 *input = nullptr; + sqlite3 *output = nullptr; + sqlite3_backup *backup = nullptr; + const std::string source_utf8 = source.u8string(); + const std::string destination_utf8 = destination.u8string(); + int source_flags = SQLITE_OPEN_READONLY; + int destination_flags = SQLITE_OPEN_READWRITE; + int rc = sqlite3_open_v2(source_utf8.c_str(), &input, source_flags, nullptr); + if (rc == SQLITE_OK) + rc = sqlite3_open_v2(destination_utf8.c_str(), &output, destination_flags, nullptr); + if (rc == SQLITE_OK) { + backup = sqlite3_backup_init(output, "main", input, "main"); + if (backup == nullptr) rc = sqlite3_errcode(output); + } + if (backup != nullptr) { + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(30); + do { + rc = sqlite3_backup_step(backup, 256); + if (rc == SQLITE_BUSY || rc == SQLITE_LOCKED) { + if (std::chrono::steady_clock::now() >= deadline) break; + sqlite3_sleep(25); + } + } while (rc == SQLITE_OK || rc == SQLITE_BUSY || rc == SQLITE_LOCKED); + const int finish_rc = sqlite3_backup_finish(backup); + if (rc == SQLITE_DONE) rc = finish_rc; + } + const std::string error = output != nullptr ? sqlite3_errmsg(output) : + (input != nullptr ? sqlite3_errmsg(input) : sqlite3_errstr(rc)); + if (output != nullptr) sqlite3_close(output); + if (input != nullptr) sqlite3_close(input); + if (rc != SQLITE_OK) { + std::error_code ignored; + std::filesystem::remove(destination, ignored); + throw std::runtime_error("database backup failed: " + error); + } + std::cout << destination << '\n'; + return 0; +} + +} // namespace + +int main(int argc, char **argv) { + const std::string invoked_as = ProgramName(argc > 0 ? argv[0] : nullptr); + if (invoked_as == "vexfs") return VexFsMain(argc, argv); + + if (argc >= 2) { + const std::string command = argv[1]; + if (command == "fs") return RunWithArguments(VexFsMain, "vexdb fs", argc, argv, 2); + if (command == "sql") + return RunWithArguments(vexdb_sqlite_shell_main, "vexdb", argc, argv, 2); + if (command == "backup") { + if (argc != 4) { + std::cerr << "vexdb: backup needs SOURCE_DATABASE DESTINATION_DATABASE\n"; + return 2; + } + try { + return BackupDatabase(argv[2], argv[3]); + } catch (const std::exception &error) { + std::cerr << "vexdb: " << error.what() << '\n'; + return 1; + } + } + if (command == "help" || command == "--help" || command == "-h") { + Usage(std::cout); + return 0; + } + if (command == "--version") { + std::cout << "vexdb-lite " << VEXDB_LITE_VERSION + << " (" << VEXDB_SQLITE_GIT_HASH << "), SQLite " + << sqlite3_libversion() << '\n'; + return 0; + } + } + return vexdb_sqlite_shell_main(argc, argv); +} diff --git a/agent_files/cli/vexdb_shell_init.c b/agent_files/cli/vexdb_shell_init.c new file mode 100644 index 0000000000..c811501d58 --- /dev/null +++ b/agent_files/cli/vexdb_shell_init.c @@ -0,0 +1,14 @@ +#include "sqlite3.h" +#include "vexdb_sqlite.h" + +#include +#include + +void vexdb_shell_init(void) { + const int rc = sqlite3_auto_extension((void (*)(void))sqlite3_vexdblite_init); + if (rc != SQLITE_OK) { + fprintf(stderr, "vexdb: cannot register VexDB-Lite SQLite extension: %s\n", + sqlite3_errstr(rc)); + exit(1); + } +} diff --git a/agent_files/cli/vexfs_main.cpp b/agent_files/cli/vexfs_main.cpp index db5e1cabe1..6ea60aae55 100644 --- a/agent_files/cli/vexfs_main.cpp +++ b/agent_files/cli/vexfs_main.cpp @@ -1,15 +1,9 @@ -#include "vexfs_mount_contract.h" -#include "vexfs_fskit_state.h" - -#include -#include -#include -#include -#include -#include +#include "vexfs_runtime_admin.h" +#include "vexfs_platform.h" #include #include +#include #include #include #include @@ -20,6 +14,7 @@ #include #include #include +#include #include #include @@ -33,21 +28,11 @@ struct Options { std::vector arguments; }; -std::string HomeDirectory() { - const char *home = std::getenv("HOME"); - return home == nullptr ? std::string(".") : std::string(home); -} - -std::string DefaultDatabasePath() { - if (const char *configured = std::getenv("VEXFS_DATABASE")) return configured; - const std::filesystem::path directory = std::filesystem::path(HomeDirectory()) / - "Library/Application Support/VexFS"; - return (directory / "vexfs.sqlite3").string(); -} +std::string g_program_name = "vexfs"; void Usage(std::ostream &output) { output << - "Usage: vexfs [--db PATH] [--workspace NAME] COMMAND [ARGS]\n" + "Usage: " << g_program_name << " [--db PATH] [--workspace NAME] COMMAND [ARGS]\n" "\n" "Commands:\n" " setup [--mount PATH] Initialize and optionally mount a workspace\n" @@ -56,6 +41,12 @@ void Usage(std::ostream &output) { " write PATH [LOCAL_FILE] Write a local file or stdin\n" " cat PATH Print a file\n" " ls [PATH] [--json] List a directory\n" + " grep [-i] [-l] [-n] [--max-results N] PATTERN [PATH]\n" + " Search current text files inside SQLite\n" + " index status|enable|rebuild|disable\n" + " Manage the optional trigram text index\n" + " check [--quick] Verify workspace metadata and content\n" + " --quick skips SHA-256 BLOB hashing\n" " stat PATH Print file metadata as JSON\n" " history PATH [--limit N] [--before N] [--json]\n" " List one page of file versions\n" @@ -63,18 +54,32 @@ void Usage(std::ostream &output) { " diff PATH --from N [--to N] Compare two versions (default: current)\n" " restore PATH --version N [--dry-run]\n" " Restore as a new version\n" + " snapshot create NAME [--committed-only]\n" + " Snapshot all published data; default refuses\n" + " while another mount has unpublished writes\n" + " snapshot list List workspace snapshots\n" + " snapshot show NAME Show the complete historical tree as JSON\n" + " snapshot diff FROM [--to TO] Compare snapshots (default TO: HEAD)\n" + " snapshot restore NAME [--dry-run] [--force-unmount]\n" + " Restore the complete tree as a new commit\n" + " snapshot drop NAME Delete a snapshot name, not its history\n" " mv SOURCE DESTINATION Move a file or directory\n" + " ln SOURCE DESTINATION Create a hard link to a regular file\n" + " chown UID:GID PATH Store owner IDs (- keeps an ID)\n" + " getfacl PATH Print the portable ACL JSON\n" + " setfacl PATH [LOCAL_FILE] Replace ACL JSON from file or stdin\n" " rm [-r] PATH Remove a file or directory\n" - " descriptor OUTPUT Write an FSKit .vexfs descriptor\n" - " mount MOUNT_POINT Mount through the enabled FSKit extension\n" + " descriptor OUTPUT Write a portable workspace descriptor\n" + " mount MOUNT_POINT Mount through the platform adapter\n" " mount status [MOUNT_POINT] Show active VexFS mounts\n" - " unmount MOUNT_POINT Unmount a VexFS mount\n" - " doctor [--json] Check macOS, extension and SQLite state\n"; + " unmount [--force] MOUNT_POINT\n" + " Unmount; --force detaches a stale mount\n" + " doctor [--json] Check platform adapter and SQLite state\n"; } Options ParseOptions(int argc, char **argv) { Options options; - options.database = DefaultDatabasePath(); + options.database = VexFSPlatformDefaultDatabasePath(); for (int index = 1; index < argc; ++index) { std::string argument = argv[index]; if (argument == "--db" && index + 1 < argc) { @@ -110,6 +115,7 @@ const char *ErrorCode(vexfs_mount_status status) { case VEXFS_MOUNT_NO_SPACE: return "VEXFS_NO_SPACE"; case VEXFS_MOUNT_CORRUPTION: return "VEXFS_CORRUPTION"; case VEXFS_MOUNT_UNSUPPORTED: return "VEXFS_UNSUPPORTED"; + case VEXFS_MOUNT_NOT_EMPTY: return "VEXFS_NOT_EMPTY"; case VEXFS_MOUNT_DATABASE_ERROR: return "VEXFS_DATABASE_ERROR"; case VEXFS_MOUNT_INTERNAL_ERROR: return "VEXFS_INTERNAL_ERROR"; default: return "VEXFS_ERROR"; @@ -128,6 +134,7 @@ int ExitCode(vexfs_mount_status status) { case VEXFS_MOUNT_DATABASE_ERROR: return 7; case VEXFS_MOUNT_CORRUPTION: return 8; case VEXFS_MOUNT_UNSUPPORTED: return 9; + case VEXFS_MOUNT_NOT_EMPTY: return 5; default: return 1; } } @@ -150,15 +157,16 @@ class Session { if (initialize && path.has_parent_path()) { const bool existed = std::filesystem::exists(path.parent_path()); std::filesystem::create_directories(path.parent_path()); - if (!existed && chmod(path.parent_path().c_str(), 0700) != 0) - throw std::runtime_error(std::strerror(errno)); + if (!existed) VexFSPlatformProtectDirectory(path.parent_path()); } vexfs_mount_config config{}; - config.abi_version = VEXFS_MOUNT_ABI_VERSION; - config.database_path = options.database.c_str(); + config.abi_version = VEXFS_RUNTIME_ABI_VERSION; + config.backend = VEXFS_RUNTIME_BACKEND_SQLITE; + config.connection = options.database.c_str(); config.workspace = options.workspace.c_str(); - config.busy_timeout_ms = 5000; - config.flags = initialize ? 0 : VEXFS_MOUNT_OPEN_NO_CREATE; + config.principal = "local"; + config.operation_timeout_ms = 5000; + config.flags = initialize ? 0 : VEXFS_RUNTIME_OPEN_NO_CREATE; vexfs_mount_error error{}; const auto status = vexfs_mount_session_open(&config, &session_, &error); if (status != VEXFS_MOUNT_OK) throw CliError(status, ErrorMessage(error)); @@ -229,6 +237,36 @@ void PrintNames(const std::string &json) { } } +void PrintGrep(const std::string &json, bool files_only, bool show_line) { + const std::string path_marker = "\"path\":\""; + const std::string line_marker = "\"line\":"; + const std::string text_marker = "\"text\":\""; + size_t position = 0; + while ((position = json.find(path_marker, position)) != std::string::npos) { + position += path_marker.size(); + const std::string path = JsonUnescape(json, &position); + const size_t line_position = json.find(line_marker, position); + const size_t text_position = json.find(text_marker, position); + if (line_position == std::string::npos || text_position == std::string::npos) { + throw std::runtime_error("invalid grep JSON"); + } + char *end = nullptr; + const long long line = std::strtoll( + json.c_str() + line_position + line_marker.size(), &end, 10); + if (end == json.c_str() + line_position + line_marker.size()) { + throw std::runtime_error("invalid grep line number"); + } + position = text_position + text_marker.size(); + const std::string text = JsonUnescape(json, &position); + std::cout << path; + if (!files_only) { + if (show_line) std::cout << ':' << line; + std::cout << ':' << text; + } + std::cout << '\n'; + } +} + struct ParsedCommand { std::vector positional; std::vector> values; @@ -304,6 +342,19 @@ int64_t NonnegativeInteger(const std::string &value, const std::string &name) { return static_cast(result); } +int64_t OwnerId(const std::string &value, const std::string &name) { + if (value == "-") return -1; + char *end = nullptr; + errno = 0; + const long long result = std::strtoll(value.c_str(), &end, 10); + if (errno != 0 || end == value.c_str() || *end != '\0' || result < 0 || + static_cast(result) > 0xffffffffULL) { + throw CliError(VEXFS_MOUNT_INVALID_ARGUMENT, + name + " must be - or an integer in 0..4294967295"); + } + return static_cast(result); +} + int64_t JsonInteger(const std::string &json, const std::string &name) { const std::string marker = "\"" + name + "\":"; size_t position = json.find(marker); @@ -334,6 +385,29 @@ bool JsonBoolean(const std::string &json, const std::string &name) { return json.compare(position + marker.size(), 4, "true") == 0; } +void PrintCheck(const std::string &json) { + const bool ok = JsonBoolean(json, "ok"); + std::cout << (ok ? "OK" : "CORRUPT") + << " workspace=" << JsonString(json, "workspace") + << " mode=" << JsonString(json, "mode") + << " issues=" << JsonInteger(json, "issue_count") + << " versions=" << JsonInteger(json, "versions") + << " content_bytes=" << JsonInteger(json, "content_bytes") + << " elapsed_ms=" << JsonInteger(json, "elapsed_ms") << '\n'; + const std::string code_marker = "\"code\":\""; + size_t position = 0; + while ((position = json.find(code_marker, position)) != std::string::npos) { + const size_t row_end = json.find('}', position); + if (row_end == std::string::npos) throw std::runtime_error("invalid check JSON"); + const std::string row = json.substr(position, row_end - position + 1); + std::cout << JsonString(row, "code") << '\t' + << JsonString(row, "object") << '\t' + << JsonString(row, "message") << '\t' + << JsonString(row, "suggestion") << '\n'; + position = row_end + 1; + } +} + void PrintHistory(const std::string &json) { std::cout << "VERSION\tCOMMIT\tSIZE\tCREATED_AT\tCURRENT\tMESSAGE\n"; const std::string entries_marker = "\"entries\":["; @@ -474,166 +548,6 @@ void WriteDescriptor(const Options &options, const std::filesystem::path &path) << "\",\n \"workspace\": \"" << JsonEscape(options.workspace) << "\"\n}\n"; } -std::filesystem::path WriteMountResourceDescriptor(const Options &options) { - const std::filesystem::path database = - std::filesystem::absolute(options.database).lexically_normal(); - const std::filesystem::path directory = database.parent_path(); - if (directory.empty() || database.filename().empty()) { - throw std::runtime_error("database path must include a file name"); - } - std::filesystem::create_directories(directory); - const std::filesystem::path descriptor = directory / ".vexfs-volume.json"; - std::ofstream output(descriptor, std::ios::binary | std::ios::trunc); - if (!output) throw std::runtime_error("cannot write mount resource: " + descriptor.string()); - output << "{\n \"version\": 2,\n \"database_file\": \"" - << JsonEscape(database.filename().string()) - << "\",\n \"workspace\": \"" << JsonEscape(options.workspace) << "\"\n}\n"; - return directory; -} - -int RunProcess(const char *program, const std::vector &arguments) { - const pid_t child = fork(); - if (child < 0) throw std::runtime_error(std::strerror(errno)); - if (child == 0) { - std::vector values; - values.push_back(const_cast(program)); - for (const std::string &argument : arguments) - values.push_back(const_cast(argument.c_str())); - values.push_back(nullptr); - execv(program, values.data()); - _exit(127); - } - int status = 0; - if (waitpid(child, &status, 0) < 0) throw std::runtime_error(std::strerror(errno)); - return WIFEXITED(status) ? WEXITSTATUS(status) : 1; -} - -struct ProcessOutput { - int exit_code = 1; - std::string output; -}; - -ProcessOutput CaptureProcess(const char *program, const std::vector &arguments) { - int descriptors[2] = {-1, -1}; - if (pipe(descriptors) != 0) throw std::runtime_error(std::strerror(errno)); - const pid_t child = fork(); - if (child < 0) { - close(descriptors[0]); - close(descriptors[1]); - throw std::runtime_error(std::strerror(errno)); - } - if (child == 0) { - close(descriptors[0]); - dup2(descriptors[1], STDOUT_FILENO); - dup2(descriptors[1], STDERR_FILENO); - close(descriptors[1]); - std::vector values; - values.push_back(const_cast(program)); - for (const std::string &argument : arguments) - values.push_back(const_cast(argument.c_str())); - values.push_back(nullptr); - execv(program, values.data()); - _exit(127); - } - close(descriptors[1]); - ProcessOutput result; - char buffer[4096]; - for (;;) { - const ssize_t count = read(descriptors[0], buffer, sizeof(buffer)); - if (count > 0) result.output.append(buffer, static_cast(count)); - else if (count == 0) break; - else if (errno != EINTR) break; - } - close(descriptors[0]); - int status = 0; - if (waitpid(child, &status, 0) < 0) throw std::runtime_error(std::strerror(errno)); - result.exit_code = WIFEXITED(status) ? WEXITSTATUS(status) : 1; - return result; -} - -enum class ExtensionState { kMissing, kEnabled, kDisabled, kRegistered }; - -ExtensionState VexFSExtensionState() { - char module_url[4096]{}; - const int fskit_state = vexfs_fskit_extension_state( - "io.vexdb.vexfs.extension", module_url, sizeof(module_url)); - if (std::getenv("VEXFS_DEBUG") != nullptr) { - std::cerr << "vexfs: FSKit state=" << fskit_state - << " module=" << module_url << '\n'; - } - if (fskit_state == 2) return ExtensionState::kEnabled; - if (fskit_state == 1) return ExtensionState::kDisabled; - - // FSKit filters out modules whose signature or provisioning it cannot accept. - // pluginkit can still distinguish that state from a package that is not installed. - // Its '+' marker is registration/election, not the FSKit per-user enable switch, - // so it must never claim that mounting is ready. - const ProcessOutput result = CaptureProcess("/usr/bin/pluginkit", - {"-m", "-A", "-D", "-i", "io.vexdb.vexfs.extension"}); - if (result.exit_code != 0 || result.output.empty()) return ExtensionState::kMissing; - for (size_t position = 0; position < result.output.size();) { - const size_t end = result.output.find('\n', position); - const std::string line = result.output.substr(position, - end == std::string::npos ? std::string::npos : end - position); - const size_t marker = line.find_first_not_of(" \t"); - if (marker != std::string::npos) { - if (line[marker] == '+' || line[marker] == '!') return ExtensionState::kRegistered; - if (line[marker] == '-') return ExtensionState::kDisabled; - } - if (end == std::string::npos) break; - position = end + 1; - } - return ExtensionState::kRegistered; -} - -const char *ExtensionStateName(ExtensionState state) { - switch (state) { - case ExtensionState::kEnabled: return "enabled"; - case ExtensionState::kDisabled: return "disabled"; - case ExtensionState::kRegistered: return "registered"; - case ExtensionState::kMissing: return "missing"; - } - return "missing"; -} - -struct MountedVolume { - std::string source; - std::string target; - std::string type; -}; - -std::string NormalizedPath(const std::string &path) { - std::error_code error; - const auto canonical = std::filesystem::weakly_canonical(path, error); - if (!error) return canonical.string(); - return std::filesystem::absolute(path).lexically_normal().string(); -} - -std::vector VexFSMounts() { - struct statfs *entries = nullptr; - const int count = getmntinfo(&entries, MNT_NOWAIT); - std::vector mounts; - for (int index = 0; index < count; ++index) { - if (std::strcmp(entries[index].f_fstypename, "vexfs") != 0) continue; - mounts.push_back({entries[index].f_mntfromname, entries[index].f_mntonname, - entries[index].f_fstypename}); - } - return mounts; -} - -bool ProductVersionSupported(const std::string &version) { - char *end = nullptr; - const long major = std::strtol(version.c_str(), &end, 10); - return end != version.c_str() && major >= 26; -} - -std::string ProductVersion() { - char buffer[128] = {}; - size_t size = sizeof(buffer); - if (sysctlbyname("kern.osproductversion", buffer, &size, nullptr, 0) != 0) return "unknown"; - return buffer; -} - std::string DatabaseDiagnostics(Session &session) { vexfs_mount_bytes json{}; vexfs_mount_error error{}; @@ -701,11 +615,30 @@ bool RunVersionDiff(Session &session, const std::string &path, int64_t from_vers return PrintDiff(path, from_version, from, to_version, to, false); } +std::string NormalizedPath(const std::string &path) { + std::error_code error; + const auto canonical = std::filesystem::weakly_canonical(path, error); + if (!error) return canonical.string(); + return std::filesystem::absolute(path).lexically_normal().string(); +} + +std::vector WorkspaceMounts(const Options &options) { + const std::string database = NormalizedPath(options.database); + std::vector matches; + for (const auto &mount : VexFSPlatformInspect().mounts) { + if (!mount.database.empty() && NormalizedPath(mount.database) == database && + mount.workspace == options.workspace) { + matches.push_back(mount); + } + } + return matches; +} + int PrintMountStatus(const Options &options, const std::string &requested_path) { - const std::string normalized = requested_path.empty() ? "" : NormalizedPath(requested_path); - std::vector mounts; - for (const auto &mount : VexFSMounts()) { - if (normalized.empty() || NormalizedPath(mount.target) == normalized) mounts.push_back(mount); + const std::string requested = requested_path.empty() ? "" : NormalizedPath(requested_path); + std::vector mounts; + for (const auto &mount : VexFSPlatformInspect().mounts) { + if (requested.empty() || NormalizedPath(mount.target) == requested) mounts.push_back(mount); } if (options.json) { std::cout << '['; @@ -713,7 +646,10 @@ int PrintMountStatus(const Options &options, const std::string &requested_path) if (index != 0) std::cout << ','; std::cout << "{\"source\":\"" << JsonEscape(mounts[index].source) << "\",\"target\":\"" << JsonEscape(mounts[index].target) - << "\",\"type\":\"" << JsonEscape(mounts[index].type) << "\"}"; + << "\",\"type\":\"" << JsonEscape(mounts[index].type) + << "\",\"database\":\"" << JsonEscape(mounts[index].database) + << "\",\"workspace\":\"" << JsonEscape(mounts[index].workspace) + << "\"}"; } std::cout << "]\n"; } else if (mounts.empty()) { @@ -725,71 +661,53 @@ int PrintMountStatus(const Options &options, const std::string &requested_path) return requested_path.empty() || !mounts.empty() ? 0 : 1; } -void PrepareMountPoint(const std::string &path) { - const std::filesystem::path mount_point(path); - if (std::filesystem::exists(mount_point) && !std::filesystem::is_directory(mount_point)) { - throw std::runtime_error("mount point is not a directory: " + path); - } - std::filesystem::create_directories(mount_point); - if (chmod(mount_point.c_str(), 0700) != 0) throw std::runtime_error(std::strerror(errno)); - if (std::filesystem::directory_iterator(mount_point) != std::filesystem::directory_iterator()) { - throw std::runtime_error("mount point must be empty: " + path); - } +int MountWorkspace(const Options &options, const std::string &mount_point) { + return VexFSPlatformMount(options.database, options.workspace, mount_point); } -std::string MountedFileSystemAt(const std::string &path) { - const std::string normalized = NormalizedPath(path); - struct statfs *entries = nullptr; - const int count = getmntinfo(&entries, MNT_NOWAIT); - for (int index = 0; index < count; ++index) { - if (NormalizedPath(entries[index].f_mntonname) == normalized) - return entries[index].f_fstypename; - } - return {}; +int UnmountWorkspace(const std::string &mount_point, bool force) { + return VexFSPlatformUnmount(mount_point, force); } -int MountWorkspace(const Options &options, const std::string &mount_point) { - const std::string mounted_type = MountedFileSystemAt(mount_point); - if (mounted_type == "vexfs") { - std::cout << mount_point << " is already mounted\n"; - return 0; - } - if (!mounted_type.empty()) { - throw std::runtime_error("mount point is already used by " + mounted_type); - } - const ExtensionState extension = VexFSExtensionState(); - if (extension != ExtensionState::kEnabled) { - throw std::runtime_error(std::string("VexFS extension is ") + - ExtensionStateName(extension) + - "; install VexFS.app and enable its file system extension in System Settings"); - } - PrepareMountPoint(mount_point); - const std::filesystem::path resource_directory = WriteMountResourceDescriptor(options); - int result = 1; - for (int attempt = 0; attempt < 3; ++attempt) { - result = RunProcess("/sbin/mount", - {"-F", "-t", "vexfs", resource_directory.string(), mount_point}); - if (MountedFileSystemAt(mount_point) == "vexfs") { - result = 0; - break; +void RemountWorkspace(const Options &options, const std::string &mount_point) { + std::string last_error = "mount returned a non-zero status"; + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(8); + do { + try { + const int result = MountWorkspace(options, mount_point); + if (result == 0) return; + last_error = "mount exited with status " + std::to_string(result); + } catch (const std::exception &error) { + last_error = error.what(); } - if (result == 0) break; - if (attempt != 2) usleep(500 * 1000); - } - if (result == 0 && MountedFileSystemAt(mount_point) != "vexfs") { - throw std::runtime_error("mount command returned success but VexFS is not mounted"); + std::this_thread::sleep_for(std::chrono::milliseconds(400)); + } while (std::chrono::steady_clock::now() < deadline); + throw std::runtime_error(last_error); +} + +int64_t RestoreSnapshotAfterUnmount(Session &session, const std::string &name, + int64_t head, bool wait_for_mount_shutdown, + std::chrono::seconds shutdown_timeout = + std::chrono::seconds(35)) { + const auto deadline = std::chrono::steady_clock::now() + shutdown_timeout; + while (true) { + int64_t commit = 0; + vexfs_mount_error error{}; + const vexfs_mount_status status = vexfs_mount_snapshot_restore( + session.get(), name.c_str(), head, &commit, &error); + if (status == VEXFS_MOUNT_OK) return commit; + const std::string message = ErrorMessage(error); + if (!wait_for_mount_shutdown || status != VEXFS_MOUNT_BUSY || + message.find("active mount session") == std::string::npos || + std::chrono::steady_clock::now() >= deadline) { + Check(status, error); + } + std::this_thread::sleep_for(std::chrono::milliseconds(100)); } - return result; } int RunDoctor(const Options &options) { - const std::string version = ProductVersion(); - const bool supported_macos = ProductVersionSupported(version); - void *framework = dlopen("/System/Library/Frameworks/FSKit.framework/FSKit", RTLD_LAZY); - const bool fskit = framework != nullptr; - if (framework != nullptr) dlclose(framework); - const ExtensionState extension = VexFSExtensionState(); - const auto mounts = VexFSMounts(); + const VexFSPlatformState platform = VexFSPlatformInspect(); std::string database_details; std::string database_error; try { @@ -799,29 +717,52 @@ int RunDoctor(const Options &options) { database_error = error.what(); } const bool database_readable = !database_details.empty(); - const bool database_ok = database_readable && JsonBoolean(database_details, "compatible"); + const bool database_ok = database_readable && JsonBoolean(database_details, "schema_ready"); if (options.json) { - std::cout << "{\"macos\":\"" << JsonEscape(version) - << "\",\"macos_supported\":" << (supported_macos ? "true" : "false") - << ",\"fskit\":" << (fskit ? "true" : "false") - << ",\"extension\":\"" << ExtensionStateName(extension) << "\"" + std::cout << "{\"platform\":\"" << JsonEscape(platform.platform) + << "\",\"runtime_abi\":" << VEXFS_RUNTIME_ABI_VERSION + << ",\"platform_version\":\"" << JsonEscape(platform.version) + << "\",\"platform_supported\":" + << (platform.platform_supported ? "true" : "false") + << ",\"mount_driver\":\"" << JsonEscape(platform.mount_driver) + << "\",\"mount_driver_available\":" + << (platform.mount_driver_available ? "true" : "false") + << ",\"mount_ready\":" << (platform.mount_ready ? "true" : "false") + << ",\"extension\":\"" << JsonEscape(platform.extension_state) << "\"" + << ",\"extension_path\":\"" << JsonEscape(platform.extension_path) << "\"" + << ",\"extension_path_matches\":" + << (platform.extension_path_matches ? "true" : "false"); + if (platform.platform == "macos") { + std::cout << ",\"macos\":\"" << JsonEscape(platform.version) + << "\",\"macos_supported\":" + << (platform.platform_supported ? "true" : "false") + << ",\"fskit\":" + << (platform.mount_driver_available ? "true" : "false"); + } + std::cout << ",\"database\":"; if (database_readable) std::cout << database_details; else std::cout << "{\"path\":\"" << JsonEscape(options.database) << "\",\"error\":\"" << JsonEscape(database_error) << "\"}"; - std::cout << ",\"mount_count\":" << mounts.size() << "}\n"; + std::cout << ",\"mount_count\":" << platform.mounts.size() << "}\n"; } else { - std::cout << "macOS: " << version << (supported_macos ? " (supported)" : " (requires 26+)") << '\n' - << "FSKit: " << (fskit ? "available" : "missing") << '\n' - << "extension: " << ExtensionStateName(extension) << '\n' + std::cout << "platform: " << platform.platform << ' ' << platform.version + << (platform.platform_supported ? " (supported)" : " (unsupported)") << '\n' + << "mount driver: " << platform.mount_driver << ' ' + << (platform.mount_driver_available ? "(available)" : "(unavailable)") << '\n' + << "mount state: " << platform.extension_state << '\n' + << (platform.extension_path.empty() ? std::string() : + "extension path: " + platform.extension_path + "\n") + << (platform.extension_path_matches ? std::string() : + "extension path: does not match installed App\n") << "database: " << options.database << '\n' << "workspace: " << options.workspace << '\n' - << "SQLite contract: " << (database_ok ? "ok" : - (database_readable ? "upgrade required or incompatible" : database_error)) << '\n' - << "active mounts: " << mounts.size() << '\n'; + << "SQLite schema: " << (database_ok ? "ok" : + (database_readable ? "version mismatch" : database_error)) << '\n' + << "active mounts: " << platform.mounts.size() << '\n'; if (database_readable) std::cout << "database details: " << database_details << '\n'; } - return supported_macos && fskit && extension == ExtensionState::kEnabled && database_ok ? 0 : 1; + return platform.mount_ready && database_ok ? 0 : 1; } int Run(const Options &options) { @@ -847,15 +788,19 @@ int Run(const Options &options) { return MountWorkspace(options, options.arguments[1]); } if (command == "unmount") { - if (options.arguments.size() != 2) throw std::runtime_error("unmount needs MOUNT_POINT"); - return RunProcess("/sbin/umount", {options.arguments[1]}); + if (options.arguments.size() == 2) + return UnmountWorkspace(options.arguments[1], false); + if (options.arguments.size() == 3 && options.arguments[1] == "--force") + return UnmountWorkspace(options.arguments[2], true); + throw std::runtime_error("unmount needs [--force] MOUNT_POINT"); } if (command == "doctor") { if (options.arguments.size() != 1) throw std::runtime_error("doctor accepts only global options"); return RunDoctor(options); } - Session session(options); + // check 必须保持只读边界：不创建数据库、workspace、WAL/SHM，也不发布 staging。 + Session session(options, command != "check"); vexfs_mount_error error{}; if (command == "init" || command == "setup") { if (options.arguments.size() != 1) throw std::runtime_error(command + " accepts only global options"); @@ -885,6 +830,47 @@ int Run(const Options &options) { Check(vexfs_mount_list(session.get(), path.c_str(), &json, &error), error); const std::string value = BytesToString(&json); if (options.json) std::cout << value << '\n'; else PrintNames(value); + } else if (command == "grep") { + const ParsedCommand parsed = ParseCommand( + options.arguments, {"--max-results"}, {"-i", "-l", "-n"}); + if (parsed.positional.empty() || parsed.positional.size() > 2) { + throw std::runtime_error("grep needs PATTERN and optional PATH"); + } + const std::string limit_value = parsed.Value("--max-results", false); + const int64_t limit = limit_value.empty() ? 1000 : PositiveInteger(limit_value, "max-results"); + if (limit > 10240) { + throw CliError(VEXFS_MOUNT_INVALID_ARGUMENT, "max-results must be at most 10240"); + } + uint32_t flags = 0; + if (parsed.Flag("-i")) flags |= VEXFS_MOUNT_GREP_IGNORE_CASE; + if (parsed.Flag("-l")) flags |= VEXFS_MOUNT_GREP_FILES_ONLY; + const std::string path = parsed.positional.size() == 2 ? parsed.positional[1] : "/"; + vexfs_mount_bytes json{}; + Check(vexfs_mount_grep(session.get(), path.c_str(), parsed.positional[0].c_str(), + flags, static_cast(limit), &json, &error), error); + const std::string value = BytesToString(&json); + if (options.json) std::cout << value << '\n'; + else PrintGrep(value, parsed.Flag("-l"), parsed.Flag("-n")); + return JsonInteger(value, "match_count") == 0 ? 1 : 0; + } else if (command == "index") { + if (options.arguments.size() != 2) { + throw std::runtime_error("index needs status, enable, rebuild or disable"); + } + vexfs_mount_bytes json{}; + Check(vexfs_mount_grep_index(session.get(), options.arguments[1].c_str(), + &json, &error), error); + std::cout << BytesToString(&json) << '\n'; + } else if (command == "check") { + const ParsedCommand parsed = ParseCommand(options.arguments, {}, {"--quick"}); + if (!parsed.positional.empty()) { + throw std::runtime_error("check accepts only --quick and global --json"); + } + vexfs_mount_bytes json{}; + const uint32_t flags = parsed.Flag("--quick") ? VEXFS_MOUNT_CHECK_QUICK : 0; + Check(vexfs_mount_check(session.get(), flags, &json, &error), error); + const std::string value = BytesToString(&json); + if (options.json) std::cout << value << '\n'; else PrintCheck(value); + return JsonBoolean(value, "ok") ? 0 : ExitCode(VEXFS_MOUNT_CORRUPTION); } else if (command == "stat") { if (options.arguments.size() != 2) throw std::runtime_error("stat needs PATH"); vexfs_mount_bytes json{}; @@ -942,10 +928,170 @@ int Run(const Options &options) { std::cout << new_version << '\n'; } } + } else if (command == "snapshot") { + if (options.arguments.size() < 2) + throw std::runtime_error("snapshot needs create, list, show, diff, restore or drop"); + const std::string &action = options.arguments[1]; + if (action == "create") { + const ParsedCommand parsed = ParseCommand(options.arguments, {}, {"--committed-only"}); + if (parsed.positional.size() != 2 || parsed.positional[0] != "create") + throw std::runtime_error("snapshot create needs NAME and optional --committed-only"); + const uint32_t flags = parsed.Flag("--committed-only") + ? VEXFS_SNAPSHOT_COMMITTED_ONLY : 0; + int64_t commit = 0; + Check(vexfs_mount_snapshot_create(session.get(), parsed.positional[1].c_str(), + flags, &commit, &error), error); + if (options.json) { + std::cout << "{\"name\":\"" << JsonEscape(parsed.positional[1]) + << "\",\"commit\":" << commit + << ",\"consistency\":\"" + << (flags == 0 ? "consistent" : "committed-only") << "\"}\n"; + } else { + std::cout << commit << '\n'; + } + } else if (action == "list") { + if (options.arguments.size() != 2) + throw std::runtime_error("snapshot list accepts no arguments"); + vexfs_mount_bytes json{}; + Check(vexfs_mount_snapshot_list(session.get(), &json, &error), error); + const std::string value = BytesToString(&json); + if (options.json) std::cout << value << '\n'; else PrintNames(value); + } else if (action == "show") { + if (options.arguments.size() != 3) + throw std::runtime_error("snapshot show needs NAME"); + vexfs_mount_bytes json{}; + Check(vexfs_mount_snapshot_show(session.get(), options.arguments[2].c_str(), + &json, &error), error); + std::cout << BytesToString(&json) << '\n'; + } else if (action == "diff") { + const ParsedCommand parsed = ParseCommand(options.arguments, {"--to"}, {}); + if (parsed.positional.size() != 2 || parsed.positional[0] != "diff") + throw std::runtime_error("snapshot diff needs FROM and optional --to TO"); + const std::string to = parsed.Value("--to", false).empty() ? + "HEAD" : parsed.Value("--to"); + vexfs_mount_bytes json{}; + Check(vexfs_mount_snapshot_diff(session.get(), parsed.positional[1].c_str(), + to.c_str(), &json, &error), error); + const std::string value = BytesToString(&json); + std::cout << value << '\n'; + return value.find("\"changes\":[]") == std::string::npos ? 1 : 0; + } else if (action == "restore") { + const ParsedCommand parsed = ParseCommand( + options.arguments, {}, {"--dry-run", "--force-unmount"}); + if (parsed.positional.size() != 2 || parsed.positional[0] != "restore") + throw std::runtime_error( + "snapshot restore needs NAME and optional --dry-run/--force-unmount"); + if (parsed.Flag("--dry-run") && parsed.Flag("--force-unmount")) + throw std::runtime_error("--force-unmount cannot be combined with --dry-run"); + const std::string &name = parsed.positional[1]; + int64_t head = 0; + Check(vexfs_mount_workspace_head(session.get(), &head, &error), error); + if (parsed.Flag("--dry-run")) { + vexfs_mount_bytes json{}; + Check(vexfs_mount_snapshot_diff(session.get(), "HEAD", name.c_str(), + &json, &error), error); + std::cout << BytesToString(&json) << '\n'; + } else { + const std::vector mounts = WorkspaceMounts(options); + if (mounts.size() > 1) { + throw CliError(VEXFS_MOUNT_BUSY, + "workspace has more than one active mount; unmount them before restore"); + } + const std::string mount_point = mounts.empty() ? "" : mounts.front().target; + if (!mount_point.empty()) { + const int unmount = UnmountWorkspace( + mount_point, parsed.Flag("--force-unmount")); + if (unmount != 0) { + throw CliError(VEXFS_MOUNT_BUSY, + "cannot unmount active workspace; snapshot restore was not started"); + } + } + + int64_t commit = 0; + try { + commit = RestoreSnapshotAfterUnmount( + session, name, head, !mount_point.empty(), + std::chrono::seconds(35)); + } catch (const std::exception &restore_error) { + const std::string restore_message = restore_error.what(); + if (!mount_point.empty()) { + try { + RemountWorkspace(options, mount_point); + } catch (const std::exception &remount_error) { + throw std::runtime_error( + "snapshot restore failed: " + restore_message + + "; workspace remount also failed: " + remount_error.what()); + } + } + throw; + } + if (!mount_point.empty()) { + try { + RemountWorkspace(options, mount_point); + } catch (const std::exception &remount_error) { + throw std::runtime_error( + "snapshot was restored at commit " + std::to_string(commit) + + " but workspace remount failed: " + remount_error.what() + + "; mount it again at " + mount_point); + } + } + if (options.json) { + std::cout << "{\"name\":\"" << JsonEscape(name) + << "\",\"previous_head\":" << head + << ",\"commit\":" << commit + << ",\"remounted\":" + << (mount_point.empty() ? "false" : "true"); + if (!mount_point.empty()) + std::cout << ",\"mount_point\":\"" << JsonEscape(mount_point) << "\""; + std::cout << "}\n"; + } else { + std::cout << commit << '\n'; + } + } + } else if (action == "drop") { + if (options.arguments.size() != 3) + throw std::runtime_error("snapshot drop needs NAME"); + Check(vexfs_mount_snapshot_drop(session.get(), options.arguments[2].c_str(), + &error), error); + } else { + throw std::runtime_error("unknown snapshot command: " + action); + } } else if (command == "mv") { if (options.arguments.size() != 3) throw std::runtime_error("mv needs SOURCE DESTINATION"); Check(vexfs_mount_move(session.get(), options.arguments[1].c_str(), options.arguments[2].c_str(), &error), error); + } else if (command == "ln") { + if (options.arguments.size() != 3) throw std::runtime_error("ln needs SOURCE DESTINATION"); + Check(vexfs_mount_link(session.get(), options.arguments[1].c_str(), + options.arguments[2].c_str(), &error), error); + } else if (command == "chown") { + if (options.arguments.size() != 3) throw std::runtime_error("chown needs UID:GID PATH"); + const std::string &owner = options.arguments[1]; + const size_t separator = owner.find(':'); + if (separator == std::string::npos || owner.find(':', separator + 1) != std::string::npos) + throw std::runtime_error("chown owner must be UID:GID"); + const int64_t uid = OwnerId(owner.substr(0, separator), "uid"); + const int64_t gid = OwnerId(owner.substr(separator + 1), "gid"); + vexfs_mount_bytes stat{}; + Check(vexfs_mount_stat(session.get(), options.arguments[2].c_str(), &stat, &error), error); + const int64_t inode = JsonInteger(BytesToString(&stat), "inode"); + Check(vexfs_mount_chown(session.get(), inode, uid, gid, &error), error); + } else if (command == "getfacl") { + if (options.arguments.size() != 2) throw std::runtime_error("getfacl needs PATH"); + vexfs_mount_bytes stat{}; + Check(vexfs_mount_stat(session.get(), options.arguments[1].c_str(), &stat, &error), error); + const int64_t inode = JsonInteger(BytesToString(&stat), "inode"); + vexfs_mount_bytes acl{}; + Check(vexfs_mount_acl_get(session.get(), inode, &acl, &error), error); + std::cout << BytesToString(&acl) << '\n'; + } else if (command == "setfacl") { + if (options.arguments.size() < 2 || options.arguments.size() > 3) + throw std::runtime_error("setfacl needs PATH and optional LOCAL_FILE"); + vexfs_mount_bytes stat{}; + Check(vexfs_mount_stat(session.get(), options.arguments[1].c_str(), &stat, &error), error); + const int64_t inode = JsonInteger(BytesToString(&stat), "inode"); + const auto acl = ReadInput(options.arguments, 2); + Check(vexfs_mount_acl_set(session.get(), inode, acl.data(), acl.size(), &error), error); } else if (command == "rm") { bool recursive = false; size_t path_index = 1; @@ -976,13 +1122,16 @@ void PrintError(const std::string &code, const std::string &message, int exit_co << "\",\"message\":\"" << JsonEscape(message) << "\",\"exit_code\":" << exit_code << "}}\n"; } else { - std::cerr << "vexfs: " << message << '\n'; + std::cerr << g_program_name << ": " << message << '\n'; } } } // namespace -int main(int argc, char **argv) { +int VexFsMain(int argc, char **argv) { + if (argc > 0 && argv[0] != nullptr && *argv[0] != '\0') { + g_program_name = std::filesystem::path(argv[0]).filename().string(); + } try { return Run(ParseOptions(argc, argv)); } catch (const CliError &error) { @@ -993,3 +1142,7 @@ int main(int argc, char **argv) { return 1; } } + +#ifndef VEXFS_EMBEDDED_MAIN +int main(int argc, char **argv) { return VexFsMain(argc, argv); } +#endif diff --git a/agent_files/cli/vexfs_platform.h b/agent_files/cli/vexfs_platform.h new file mode 100644 index 0000000000..7b99665aee --- /dev/null +++ b/agent_files/cli/vexfs_platform.h @@ -0,0 +1,37 @@ +#pragma once + +#include +#include +#include + +struct VexFSPlatformMountEntry { + std::string source; + std::string target; + std::string type; + // Filled only when the mount was created by VexDB-Lite and its live mount + // table entry still matches the saved identity record. + std::string database; + std::string workspace; +}; + +struct VexFSPlatformState { + std::string platform; + std::string version; + bool platform_supported = false; + std::string mount_driver; + bool mount_driver_available = false; + std::string extension_state; + std::string extension_path; + bool extension_path_matches = true; + bool mount_ready = false; + std::vector mounts; +}; + +// These functions are the only operating-system boundary used by the shared CLI. +// Database operations use vexfs_runtime_admin.h; platform adapters stay independent. +std::string VexFSPlatformDefaultDatabasePath(); +void VexFSPlatformProtectDirectory(const std::filesystem::path &path); +VexFSPlatformState VexFSPlatformInspect(); +int VexFSPlatformMount(const std::string &database, const std::string &workspace, + const std::string &mount_point); +int VexFSPlatformUnmount(const std::string &mount_point, bool force); diff --git a/agent_files/cli/vexfs_platform_linux.cpp b/agent_files/cli/vexfs_platform_linux.cpp new file mode 100644 index 0000000000..fc3d6710ba --- /dev/null +++ b/agent_files/cli/vexfs_platform_linux.cpp @@ -0,0 +1,269 @@ +#include "vexfs_platform.h" +#include "vexfs_platform_registry.h" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +std::filesystem::path UserDataDirectory() { + if (const char *xdg = std::getenv("XDG_DATA_HOME")) return xdg; + if (const char *home = std::getenv("HOME")) return std::filesystem::path(home) / ".local/share"; + return std::filesystem::current_path(); +} + +std::string DecodeMountField(const std::string &value) { + std::string output; + output.reserve(value.size()); + for (size_t index = 0; index < value.size(); ++index) { + if (value[index] == '\\' && index + 3 < value.size()) { + const std::string code = value.substr(index + 1, 3); + if (code == "040") output.push_back(' '); + else if (code == "011") output.push_back('\t'); + else if (code == "012") output.push_back('\n'); + else if (code == "134") output.push_back('\\'); + else { + output.push_back(value[index]); + continue; + } + index += 3; + } else { + output.push_back(value[index]); + } + } + return output; +} + +std::string NormalizedPath(const std::string &path) { + std::error_code error; + const auto canonical = std::filesystem::weakly_canonical(path, error); + if (!error) return canonical.string(); + return std::filesystem::absolute(path).lexically_normal().string(); +} + +std::vector MountTable(bool vexfs_only) { + std::ifstream input("/proc/self/mountinfo"); + std::vector mounts; + std::string line; + while (std::getline(input, line)) { + std::istringstream fields(line); + std::vector values; + std::string value; + while (fields >> value) values.push_back(value); + if (values.size() < 10) continue; + size_t separator = 0; + while (separator < values.size() && values[separator] != "-") ++separator; + if (separator + 3 >= values.size() || separator < 6) continue; + const std::string type = DecodeMountField(values[separator + 1]); + const std::string source = DecodeMountField(values[separator + 2]); + const bool is_vexfs = type == "fuse.vexfs" || (type == "fuse" && source == "vexfs"); + if (vexfs_only && !is_vexfs) continue; + mounts.push_back({source, DecodeMountField(values[4]), type}); + } + return mounts; +} + +std::vector MountedVolumes() { + std::filesystem::path registry; + if (const char *runtime = std::getenv("XDG_RUNTIME_DIR")) + registry = std::filesystem::path(runtime) / "vexdb-lite/mounts"; + else + registry = UserDataDirectory() / "vexdb-lite/mounts"; + return VexFSPlatformAttachMountRegistry(MountTable(true), registry); +} + +std::filesystem::path MountRegistryDirectory() { + if (const char *runtime = std::getenv("XDG_RUNTIME_DIR")) + return std::filesystem::path(runtime) / "vexdb-lite/mounts"; + return UserDataDirectory() / "vexdb-lite/mounts"; +} + +std::string MountedFileSystemAt(const std::string &path) { + const std::string normalized = NormalizedPath(path); + for (const auto &mount : MountTable(false)) { + if (NormalizedPath(mount.target) == normalized) { + if (mount.type == "fuse" && mount.source == "vexfs") return "fuse.vexfs"; + return mount.type; + } + } + return {}; +} + +std::string FindExecutable(const std::string &name) { + if (name.find('/') != std::string::npos) + return access(name.c_str(), X_OK) == 0 ? name : std::string(); + const char *path_value = std::getenv("PATH"); + if (path_value == nullptr) return {}; + std::istringstream paths(path_value); + std::string directory; + while (std::getline(paths, directory, ':')) { + if (directory.empty()) directory = "."; + const std::filesystem::path candidate = std::filesystem::path(directory) / name; + if (access(candidate.c_str(), X_OK) == 0) return candidate.string(); + } + return {}; +} + +std::string FuseHelper() { + if (const char *configured = std::getenv("VEXFS_FUSE_HELPER")) + return FindExecutable(configured); + char executable[4096]{}; + const ssize_t size = readlink("/proc/self/exe", executable, sizeof(executable) - 1); + if (size > 0) { + executable[size] = '\0'; + const std::filesystem::path sibling = + std::filesystem::path(executable).parent_path() / "vexfs-fuse"; + if (access(sibling.c_str(), X_OK) == 0) return sibling.string(); + } + return FindExecutable("vexfs-fuse"); +} + +int RunProcess(const std::string &program, const std::vector &arguments) { + const pid_t child = fork(); + if (child < 0) throw std::runtime_error(std::strerror(errno)); + if (child == 0) { + std::vector values; + values.push_back(const_cast(program.c_str())); + for (const std::string &argument : arguments) + values.push_back(const_cast(argument.c_str())); + values.push_back(nullptr); + execv(program.c_str(), values.data()); + _exit(127); + } + int status = 0; + if (waitpid(child, &status, 0) < 0) throw std::runtime_error(std::strerror(errno)); + return WIFEXITED(status) ? WEXITSTATUS(status) : 1; +} + +void PrepareMountPoint(const std::string &path) { + const std::filesystem::path mount_point(path); + if (std::filesystem::exists(mount_point) && !std::filesystem::is_directory(mount_point)) + throw std::runtime_error("mount point is not a directory: " + path); + std::filesystem::create_directories(mount_point); + VexFSPlatformProtectDirectory(mount_point); + if (std::filesystem::directory_iterator(mount_point) != std::filesystem::directory_iterator()) + throw std::runtime_error("mount point must be empty: " + path); +} + +std::string KernelVersion() { + struct utsname information{}; + return uname(&information) == 0 ? information.release : "unknown"; +} + +} // namespace + +std::string VexFSPlatformDefaultDatabasePath() { + if (const char *configured = std::getenv("VEXDB_LITE_DATABASE")) return configured; + if (const char *configured = std::getenv("VEXFS_DATABASE")) return configured; + return (UserDataDirectory() / "vexdb-lite/default.sqlite3").string(); +} + +void VexFSPlatformProtectDirectory(const std::filesystem::path &path) { + if (chmod(path.c_str(), 0700) != 0) throw std::runtime_error(std::strerror(errno)); +} + +VexFSPlatformState VexFSPlatformInspect() { + VexFSPlatformState state; + state.platform = "linux"; + state.version = KernelVersion(); + state.platform_supported = true; + state.mount_driver = "libfuse3"; + const bool helper = !FuseHelper().empty(); + const bool fusermount = !FindExecutable("fusermount3").empty(); + const bool device = access("/dev/fuse", F_OK) == 0; + const bool device_accessible = access("/dev/fuse", R_OK | W_OK) == 0; + state.mount_driver_available = helper && fusermount; + if (!helper) state.extension_state = "helper-missing"; + else if (!fusermount) state.extension_state = "fusermount3-missing"; + else if (!device) state.extension_state = "fuse-device-missing"; + else if (!device_accessible) state.extension_state = "fuse-device-permission-denied"; + else state.extension_state = "ready"; + state.mount_ready = state.mount_driver_available && device_accessible; + state.mounts = MountedVolumes(); + return state; +} + +int VexFSPlatformMount(const std::string &database, const std::string &workspace, + const std::string &mount_point) { + const std::string mounted_type = MountedFileSystemAt(mount_point); + if (mounted_type == "fuse.vexfs") { + const std::string requested_database = NormalizedPath(database); + const std::string requested_target = NormalizedPath(mount_point); + for (const auto &mount : MountedVolumes()) { + if (NormalizedPath(mount.target) != requested_target) continue; + if (mount.database == requested_database && mount.workspace == workspace) return 0; + throw std::runtime_error( + "mount point already contains a VexFS workspace with different or unknown identity"); + } + throw std::runtime_error("mount point is VexFS but its identity cannot be verified"); + } + if (!mounted_type.empty()) + throw std::runtime_error("mount point is already used by " + mounted_type); + const VexFSPlatformState state = VexFSPlatformInspect(); + if (!state.mount_ready) + throw std::runtime_error("VexFS libfuse3 mount is not ready: " + state.extension_state); + PrepareMountPoint(mount_point); + const std::filesystem::path absolute_database = + std::filesystem::absolute(database).lexically_normal(); + if (absolute_database.has_parent_path()) { + std::filesystem::create_directories(absolute_database.parent_path()); + VexFSPlatformProtectDirectory(absolute_database.parent_path()); + } + const std::string helper = FuseHelper(); + const int result = RunProcess(helper, + {"--db", absolute_database.string(), "--workspace", workspace, mount_point}); + for (int attempt = 0; attempt < 50; ++attempt) { + if (!MountedFileSystemAt(mount_point).empty()) { + try { + VexFSPlatformRememberMount(MountRegistryDirectory(), + {"vexfs", NormalizedPath(mount_point), "fuse.vexfs", + NormalizedPath(database), workspace}); + } catch (...) { + const std::string fusermount = FindExecutable("fusermount3"); + if (!fusermount.empty()) RunProcess(fusermount, {"-u", mount_point}); + throw; + } + return 0; + } + usleep(100 * 1000); + } + if (result == 0) + throw std::runtime_error("vexfs-fuse returned success but the mount is not active"); + return result; +} + +int VexFSPlatformUnmount(const std::string &mount_point, bool force) { + const std::string mounted_type = MountedFileSystemAt(mount_point); + if (mounted_type.empty()) { + VexFSPlatformForgetMount(MountRegistryDirectory(), mount_point); + return 0; + } + if (mounted_type != "fuse.vexfs") + throw std::runtime_error("mount point is not VexFS: " + mounted_type); + const std::string fusermount = FindExecutable("fusermount3"); + if (fusermount.empty()) throw std::runtime_error("fusermount3 is not installed"); + int result = 1; + for (int attempt = 0; attempt < 5; ++attempt) { + result = RunProcess(fusermount, + force ? std::vector{"-u", "-z", mount_point} + : std::vector{"-u", mount_point}); + if (MountedFileSystemAt(mount_point).empty()) { + VexFSPlatformForgetMount(MountRegistryDirectory(), mount_point); + return 0; + } + usleep(200 * 1000); + } + return result; +} diff --git a/agent_files/cli/vexfs_platform_macos.cpp b/agent_files/cli/vexfs_platform_macos.cpp new file mode 100644 index 0000000000..622b0368ca --- /dev/null +++ b/agent_files/cli/vexfs_platform_macos.cpp @@ -0,0 +1,342 @@ +#include "vexfs_platform.h" +#include "vexfs_platform_registry.h" +#include "vexfs_fskit_state.h" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +enum class ExtensionState { kMissing, kEnabled, kDisabled, kRegistered }; + +std::string HomeDirectory() { + const char *home = std::getenv("HOME"); + return home == nullptr ? std::string(".") : std::string(home); +} + +std::string JsonEscape(const std::string &value) { + std::string output; + output.reserve(value.size() + 8); + for (const unsigned char c : value) { + switch (c) { + case '"': output += "\\\""; break; + case '\\': output += "\\\\"; break; + case '\b': output += "\\b"; break; + case '\f': output += "\\f"; break; + case '\n': output += "\\n"; break; + case '\r': output += "\\r"; break; + case '\t': output += "\\t"; break; + default: + if (c < 0x20) { + char escaped[7] = {}; + std::snprintf(escaped, sizeof(escaped), "\\u%04x", c); + output += escaped; + } else { + output.push_back(static_cast(c)); + } + } + } + return output; +} + +struct ProcessOutput { + int exit_code = 1; + std::string output; +}; + +int RunProcess(const char *program, const std::vector &arguments) { + const pid_t child = fork(); + if (child < 0) throw std::runtime_error(std::strerror(errno)); + if (child == 0) { + std::vector values; + values.push_back(const_cast(program)); + for (const std::string &argument : arguments) + values.push_back(const_cast(argument.c_str())); + values.push_back(nullptr); + execv(program, values.data()); + _exit(127); + } + int status = 0; + if (waitpid(child, &status, 0) < 0) throw std::runtime_error(std::strerror(errno)); + return WIFEXITED(status) ? WEXITSTATUS(status) : 1; +} + +ProcessOutput CaptureProcess(const char *program, const std::vector &arguments) { + int descriptors[2] = {-1, -1}; + if (pipe(descriptors) != 0) throw std::runtime_error(std::strerror(errno)); + const pid_t child = fork(); + if (child < 0) { + close(descriptors[0]); + close(descriptors[1]); + throw std::runtime_error(std::strerror(errno)); + } + if (child == 0) { + close(descriptors[0]); + dup2(descriptors[1], STDOUT_FILENO); + dup2(descriptors[1], STDERR_FILENO); + close(descriptors[1]); + std::vector values; + values.push_back(const_cast(program)); + for (const std::string &argument : arguments) + values.push_back(const_cast(argument.c_str())); + values.push_back(nullptr); + execv(program, values.data()); + _exit(127); + } + close(descriptors[1]); + ProcessOutput result; + char buffer[4096]; + for (;;) { + const ssize_t count = read(descriptors[0], buffer, sizeof(buffer)); + if (count > 0) result.output.append(buffer, static_cast(count)); + else if (count == 0) break; + else if (errno != EINTR) break; + } + close(descriptors[0]); + int status = 0; + if (waitpid(child, &status, 0) < 0) throw std::runtime_error(std::strerror(errno)); + result.exit_code = WIFEXITED(status) ? WEXITSTATUS(status) : 1; + return result; +} + +ExtensionState GetExtensionState(std::string *module_path) { + char module_url[4096]{}; + const int fskit_state = vexfs_fskit_extension_state( + "io.vexdb.vexfs.extension", module_url, sizeof(module_url)); + if (std::getenv("VEXFS_DEBUG") != nullptr) { + std::fprintf(stderr, "vexfs: FSKit state=%d module=%s\n", fskit_state, module_url); + } + if (module_path != nullptr) *module_path = module_url; + if (fskit_state == 2) return ExtensionState::kEnabled; + if (fskit_state == 1) return ExtensionState::kDisabled; + + const ProcessOutput result = CaptureProcess("/usr/bin/pluginkit", + {"-m", "-A", "-D", "-i", "io.vexdb.vexfs.extension"}); + if (result.exit_code != 0 || result.output.empty()) return ExtensionState::kMissing; + for (size_t position = 0; position < result.output.size();) { + const size_t end = result.output.find('\n', position); + const std::string line = result.output.substr(position, + end == std::string::npos ? std::string::npos : end - position); + const size_t marker = line.find_first_not_of(" \t"); + if (marker != std::string::npos) { + if (line[marker] == '+' || line[marker] == '!') return ExtensionState::kRegistered; + if (line[marker] == '-') return ExtensionState::kDisabled; + } + if (end == std::string::npos) break; + position = end + 1; + } + return ExtensionState::kRegistered; +} + +const char *ExtensionStateName(ExtensionState state) { + switch (state) { + case ExtensionState::kEnabled: return "enabled"; + case ExtensionState::kDisabled: return "disabled"; + case ExtensionState::kRegistered: return "registered"; + case ExtensionState::kMissing: return "missing"; + } + return "missing"; +} + +std::string NormalizedPath(const std::string &path) { + std::error_code error; + const auto canonical = std::filesystem::weakly_canonical(path, error); + if (!error) return canonical.string(); + return std::filesystem::absolute(path).lexically_normal().string(); +} + +std::filesystem::path MountRegistryDirectory() { + return std::filesystem::path(HomeDirectory()) / + "Library/Application Support/VexDB-Lite/mounts"; +} + +std::vector MountedVolumes() { + struct statfs *entries = nullptr; + const int count = getmntinfo(&entries, MNT_NOWAIT); + std::vector mounts; + for (int index = 0; index < count; ++index) { + if (std::strcmp(entries[index].f_fstypename, "vexfs") != 0) continue; + mounts.push_back({entries[index].f_mntfromname, entries[index].f_mntonname, + entries[index].f_fstypename}); + } + return VexFSPlatformAttachMountRegistry(std::move(mounts), MountRegistryDirectory()); +} + +std::string ProductVersion() { + char buffer[128] = {}; + size_t size = sizeof(buffer); + if (sysctlbyname("kern.osproductversion", buffer, &size, nullptr, 0) != 0) return "unknown"; + return buffer; +} + +bool ProductVersionSupported(const std::string &version) { + char *end = nullptr; + const long major = std::strtol(version.c_str(), &end, 10); + return end != version.c_str() && major >= 26; +} + +std::string MountedFileSystemAt(const std::string &path) { + const std::string normalized = NormalizedPath(path); + struct statfs *entries = nullptr; + const int count = getmntinfo(&entries, MNT_NOWAIT); + for (int index = 0; index < count; ++index) { + if (NormalizedPath(entries[index].f_mntonname) == normalized) + return entries[index].f_fstypename; + } + return {}; +} + +void PrepareMountPoint(const std::string &path) { + const std::filesystem::path mount_point(path); + if (std::filesystem::exists(mount_point) && !std::filesystem::is_directory(mount_point)) { + throw std::runtime_error("mount point is not a directory: " + path); + } + std::filesystem::create_directories(mount_point); + VexFSPlatformProtectDirectory(mount_point); + if (std::filesystem::directory_iterator(mount_point) != std::filesystem::directory_iterator()) { + throw std::runtime_error("mount point must be empty: " + path); + } +} + +std::filesystem::path WriteMountResourceDescriptor(const std::string &database_path, + const std::string &workspace) { + const std::filesystem::path database = + std::filesystem::absolute(database_path).lexically_normal(); + const std::filesystem::path directory = database.parent_path(); + if (directory.empty() || database.filename().empty()) { + throw std::runtime_error("database path must include a file name"); + } + std::filesystem::create_directories(directory); + const std::filesystem::path descriptor = directory / ".vexfs-volume.json"; + std::ofstream output(descriptor, std::ios::binary | std::ios::trunc); + if (!output) throw std::runtime_error("cannot write mount resource: " + descriptor.string()); + output << "{\n \"version\": 2,\n \"database_file\": \"" + << JsonEscape(database.filename().string()) + << "\",\n \"workspace\": \"" << JsonEscape(workspace) << "\"\n}\n"; + return directory; +} + +} // namespace + +std::string VexFSPlatformDefaultDatabasePath() { + if (const char *configured = std::getenv("VEXDB_LITE_DATABASE")) return configured; + if (const char *configured = std::getenv("VEXFS_DATABASE")) return configured; + const std::filesystem::path home = HomeDirectory(); + const std::filesystem::path current = home / + "Library/Application Support/VexDB-Lite/default.sqlite3"; + return current.string(); +} + +void VexFSPlatformProtectDirectory(const std::filesystem::path &path) { + if (chmod(path.c_str(), 0700) != 0) throw std::runtime_error(std::strerror(errno)); +} + +VexFSPlatformState VexFSPlatformInspect() { + VexFSPlatformState state; + state.platform = "macos"; + state.version = ProductVersion(); + state.platform_supported = ProductVersionSupported(state.version); + state.mount_driver = "FSKit"; + void *framework = dlopen("/System/Library/Frameworks/FSKit.framework/FSKit", RTLD_LAZY); + state.mount_driver_available = framework != nullptr; + if (framework != nullptr) dlclose(framework); + const ExtensionState extension = GetExtensionState(&state.extension_path); + state.extension_state = ExtensionStateName(extension); + if (const char *expected = std::getenv("VEXFS_EXPECTED_EXTENSION_PATH")) { + state.extension_path_matches = !state.extension_path.empty() && + NormalizedPath(state.extension_path) == NormalizedPath(expected); + } + state.mount_ready = state.platform_supported && state.mount_driver_available && + extension == ExtensionState::kEnabled && state.extension_path_matches; + state.mounts = MountedVolumes(); + return state; +} + +int VexFSPlatformMount(const std::string &database, const std::string &workspace, + const std::string &mount_point) { + const std::string mounted_type = MountedFileSystemAt(mount_point); + if (mounted_type == "vexfs") { + const std::string requested_database = NormalizedPath(database); + const std::string requested_target = NormalizedPath(mount_point); + for (const auto &mount : MountedVolumes()) { + if (NormalizedPath(mount.target) != requested_target) continue; + if (mount.database == requested_database && mount.workspace == workspace) return 0; + throw std::runtime_error( + "mount point already contains a VexFS workspace with different or unknown identity"); + } + throw std::runtime_error("mount point is VexFS but its identity cannot be verified"); + } + if (!mounted_type.empty()) { + throw std::runtime_error("mount point is already used by " + mounted_type); + } + const VexFSPlatformState state = VexFSPlatformInspect(); + if (!state.mount_ready) { + throw std::runtime_error("VexFS FSKit extension is " + state.extension_state + + "; install VexDB Lite.app and enable the VexFS file system extension in System Settings"); + } + PrepareMountPoint(mount_point); + const std::filesystem::path resource_directory = + WriteMountResourceDescriptor(database, workspace); + int result = 1; + for (int attempt = 0; attempt < 3; ++attempt) { + result = RunProcess("/sbin/mount", + {"-F", "-t", "vexfs", resource_directory.string(), mount_point}); + if (MountedFileSystemAt(mount_point) == "vexfs") { + try { + VexFSPlatformRememberMount(MountRegistryDirectory(), + {resource_directory.string(), NormalizedPath(mount_point), "vexfs", + NormalizedPath(database), workspace}); + } catch (...) { + RunProcess("/sbin/umount", {mount_point}); + throw; + } + return 0; + } + if (result == 0) break; + if (attempt != 2) usleep(500 * 1000); + } + if (result == 0) { + throw std::runtime_error("mount command returned success but VexFS is not mounted"); + } + return result; +} + +int VexFSPlatformUnmount(const std::string &mount_point, bool force) { + const std::string mounted_type = MountedFileSystemAt(mount_point); + if (mounted_type.empty()) { + VexFSPlatformForgetMount(MountRegistryDirectory(), mount_point); + return 0; + } + if (mounted_type != "vexfs") + throw std::runtime_error("mount point is not VexFS: " + mounted_type); + int result = 1; + // fseventsd/Spotlight can briefly hold a newly mounted FSKit volume even + // when no user process has an open file. Keep this a normal (non-force) + // unmount, but allow that short system handoff to finish. + constexpr int kUnmountAttempts = 12; + for (int attempt = 0; attempt < kUnmountAttempts; ++attempt) { + result = RunProcess("/sbin/umount", + force ? std::vector{"-f", mount_point} + : std::vector{mount_point}); + if (MountedFileSystemAt(mount_point).empty()) { + VexFSPlatformForgetMount(MountRegistryDirectory(), mount_point); + return 0; + } + if (attempt + 1 != kUnmountAttempts) usleep(500 * 1000); + } + return result; +} diff --git a/agent_files/cli/vexfs_platform_registry.cpp b/agent_files/cli/vexfs_platform_registry.cpp new file mode 100644 index 0000000000..b28df7316d --- /dev/null +++ b/agent_files/cli/vexfs_platform_registry.cpp @@ -0,0 +1,160 @@ +#include "vexfs_platform_registry.h" + +#include +#include +#include +#include +#include +#include +#include + +#if !defined(_WIN32) +#include +#include +#endif + +namespace { + +constexpr const char *kRegistryVersion = "VEXFS_MOUNT_REGISTRY_V1"; + +std::string PercentDecode(std::string value) { + const auto hex = [](char character) -> int { + if (character >= '0' && character <= '9') return character - '0'; + if (character >= 'a' && character <= 'f') return character - 'a' + 10; + if (character >= 'A' && character <= 'F') return character - 'A' + 10; + return -1; + }; + std::string output; + output.reserve(value.size()); + for (size_t index = 0; index < value.size(); ++index) { + if (value[index] == '%' && index + 2 < value.size()) { + const int high = hex(value[index + 1]); + const int low = hex(value[index + 2]); + if (high >= 0 && low >= 0) { + output.push_back(static_cast((high << 4) | low)); + index += 2; + continue; + } + } + output.push_back(value[index]); + } + return output; +} + +std::string NormalizePath(const std::string &path) { + if (path.empty()) return {}; + std::error_code error; + const auto canonical = std::filesystem::weakly_canonical(path, error); + if (!error) return canonical.string(); + return std::filesystem::absolute(path).lexically_normal().string(); +} + +std::string NormalizeSource(std::string source) { + if (source.rfind("file://", 0) == 0) source = PercentDecode(source.substr(7)); + if (!source.empty() && source.front() == '/') return NormalizePath(source); + return source; +} + +uint64_t StableHash(const std::string &value) { + uint64_t hash = 1469598103934665603ULL; + for (const unsigned char byte : value) { + hash ^= byte; + hash *= 1099511628211ULL; + } + return hash; +} + +std::filesystem::path RecordPath(const std::filesystem::path &directory, + const std::string &target) { + std::ostringstream name; + name << "mount-" << std::hex << std::setw(16) << std::setfill('0') + << StableHash(NormalizePath(target)) << ".record"; + return directory / name.str(); +} + +void ProtectDirectory(const std::filesystem::path &directory) { + std::filesystem::create_directories(directory); +#if !defined(_WIN32) + if (chmod(directory.c_str(), 0700) != 0) + throw std::runtime_error("cannot protect VexFS mount registry"); +#endif +} + +bool ReadRecord(const std::filesystem::path &path, VexFSPlatformMountEntry *entry) { + std::ifstream input(path, std::ios::binary); + std::string version; + if (!std::getline(input, version) || version != kRegistryVersion) return false; + return static_cast(input >> std::quoted(entry->source) + >> std::quoted(entry->target) >> std::quoted(entry->type) + >> std::quoted(entry->database) >> std::quoted(entry->workspace)); +} + +} // namespace + +std::vector VexFSPlatformAttachMountRegistry( + std::vector mounts, + const std::filesystem::path ®istry_directory) { + std::error_code error; + if (!std::filesystem::is_directory(registry_directory, error)) return mounts; + for (const auto &file : std::filesystem::directory_iterator(registry_directory, error)) { + if (error) break; + if (!file.is_regular_file() || file.path().extension() != ".record") continue; + VexFSPlatformMountEntry saved; + if (!ReadRecord(file.path(), &saved)) continue; + for (auto &live : mounts) { + if (NormalizePath(live.target) != NormalizePath(saved.target) || + NormalizeSource(live.source) != NormalizeSource(saved.source)) continue; + live.database = NormalizePath(saved.database); + live.workspace = saved.workspace; + break; + } + } + return mounts; +} + +void VexFSPlatformRememberMount(const std::filesystem::path ®istry_directory, + const VexFSPlatformMountEntry &mount) { + if (mount.target.empty() || mount.source.empty() || mount.database.empty() || + mount.workspace.empty()) { + throw std::runtime_error("incomplete VexFS mount identity"); + } + ProtectDirectory(registry_directory); + const std::filesystem::path destination = RecordPath(registry_directory, mount.target); + const std::filesystem::path temporary = destination.string() + ".tmp-" + + std::to_string(static_cast( + std::chrono::steady_clock::now().time_since_epoch().count())); + std::error_code error; + { + std::ofstream output(temporary, std::ios::binary | std::ios::trunc); + if (!output) throw std::runtime_error("cannot write VexFS mount registry"); + output << kRegistryVersion << '\n' + << std::quoted(mount.source) << '\n' + << std::quoted(NormalizePath(mount.target)) << '\n' + << std::quoted(mount.type) << '\n' + << std::quoted(NormalizePath(mount.database)) << '\n' + << std::quoted(mount.workspace) << '\n'; + if (!output) throw std::runtime_error("cannot finish VexFS mount registry"); + } +#if !defined(_WIN32) + if (chmod(temporary.c_str(), 0600) != 0) { + std::filesystem::remove(temporary, error); + throw std::runtime_error("cannot protect VexFS mount registry record"); + } +#endif + std::filesystem::rename(temporary, destination, error); + if (error) { + std::filesystem::remove(destination, error); + error.clear(); + std::filesystem::rename(temporary, destination, error); + } + if (error) { + std::filesystem::remove(temporary, error); + throw std::runtime_error("cannot publish VexFS mount registry record"); + } +} + +void VexFSPlatformForgetMount(const std::filesystem::path ®istry_directory, + const std::string &target) { + std::error_code ignored; + std::filesystem::remove(RecordPath(registry_directory, target), ignored); +} diff --git a/agent_files/cli/vexfs_platform_registry.h b/agent_files/cli/vexfs_platform_registry.h new file mode 100644 index 0000000000..b4baf5da4d --- /dev/null +++ b/agent_files/cli/vexfs_platform_registry.h @@ -0,0 +1,17 @@ +#pragma once + +#include "vexfs_platform.h" + +#include +#include + +// The kernel mount table does not carry VexFS database/workspace options. Keep +// a small per-mount identity record and only trust it while target+source still +// match a real kernel mount entry. +std::vector VexFSPlatformAttachMountRegistry( + std::vector mounts, + const std::filesystem::path ®istry_directory); +void VexFSPlatformRememberMount(const std::filesystem::path ®istry_directory, + const VexFSPlatformMountEntry &mount); +void VexFSPlatformForgetMount(const std::filesystem::path ®istry_directory, + const std::string &target); diff --git a/agent_files/cli/vexfs_platform_stub.cpp b/agent_files/cli/vexfs_platform_stub.cpp new file mode 100644 index 0000000000..46ce63c104 --- /dev/null +++ b/agent_files/cli/vexfs_platform_stub.cpp @@ -0,0 +1,101 @@ +#include "vexfs_platform.h" + +#include +#include +#include +#include + +#if defined(_WIN32) +#include +#else +#include +#include +#include +#endif + +namespace { + +std::filesystem::path UserDataDirectory() { +#if defined(VEXFS_PLATFORM_STUB_LINUX) + if (const char *xdg = std::getenv("XDG_DATA_HOME")) return xdg; + if (const char *home = std::getenv("HOME")) return std::filesystem::path(home) / ".local/share"; + return std::filesystem::current_path(); +#elif defined(VEXFS_PLATFORM_STUB_WINDOWS) || defined(_WIN32) + if (const char *local = std::getenv("LOCALAPPDATA")) return local; + if (const char *profile = std::getenv("USERPROFILE")) return profile; + return std::filesystem::current_path(); +#else + if (const char *xdg = std::getenv("XDG_DATA_HOME")) return xdg; + if (const char *home = std::getenv("HOME")) return std::filesystem::path(home) / ".local/share"; + return std::filesystem::current_path(); +#endif +} + +const char *PlatformName() { +#if defined(VEXFS_PLATFORM_STUB_LINUX) + return "linux"; +#elif defined(VEXFS_PLATFORM_STUB_WINDOWS) || defined(_WIN32) + return "windows"; +#elif defined(__linux__) + return "linux"; +#else + return "unsupported"; +#endif +} + +const char *MountDriverName() { +#if defined(VEXFS_PLATFORM_STUB_LINUX) + return "libfuse3"; +#elif defined(VEXFS_PLATFORM_STUB_WINDOWS) || defined(_WIN32) + return "WinFsp"; +#elif defined(__linux__) + return "libfuse3"; +#else + return "none"; +#endif +} + +} // namespace + +std::string VexFSPlatformDefaultDatabasePath() { + if (const char *configured = std::getenv("VEXDB_LITE_DATABASE")) return configured; + if (const char *configured = std::getenv("VEXFS_DATABASE")) return configured; +#if defined(VEXFS_PLATFORM_STUB_LINUX) + return (UserDataDirectory() / "vexdb-lite/default.sqlite3").string(); +#elif defined(VEXFS_PLATFORM_STUB_WINDOWS) || defined(_WIN32) + return (UserDataDirectory() / "VexDB-Lite/default.sqlite3").string(); +#else + return (UserDataDirectory() / "vexdb-lite/default.sqlite3").string(); +#endif +} + +void VexFSPlatformProtectDirectory(const std::filesystem::path &path) { +#if defined(_WIN32) + (void)path; +#else + if (chmod(path.c_str(), 0700) != 0) throw std::runtime_error(std::strerror(errno)); +#endif +} + +VexFSPlatformState VexFSPlatformInspect() { + VexFSPlatformState state; + state.platform = PlatformName(); + state.version = "unknown"; + state.platform_supported = true; + state.mount_driver = MountDriverName(); + state.mount_driver_available = false; + state.extension_state = "not-implemented"; + state.mount_ready = false; + return state; +} + +int VexFSPlatformMount(const std::string &, const std::string &, + const std::string &) { + throw std::runtime_error(std::string("VexFS ") + MountDriverName() + + " mount adapter is not implemented yet"); +} + +int VexFSPlatformUnmount(const std::string &, bool) { + throw std::runtime_error(std::string("VexFS ") + MountDriverName() + + " mount adapter is not implemented yet"); +} diff --git a/agent_files/linux/package/install.sh b/agent_files/linux/package/install.sh new file mode 100755 index 0000000000..43deedb46d --- /dev/null +++ b/agent_files/linux/package/install.sh @@ -0,0 +1,58 @@ +#!/bin/bash +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")" && pwd)" +BIN_DIR="${VEXDB_LITE_BIN_DIR:-$HOME/.local/bin}" +LIB_DIR="${VEXDB_LITE_LIB_DIR:-$HOME/.local/lib/vexdb-lite}" +EXPECTED_ARCH="$(sed -n 's/^architecture=//p' "$ROOT/MANIFEST.txt")" +ACTUAL_ARCH="$(uname -m)" + +[ "$(uname -s)" = Linux ] || { echo "这个安装包只支持 Linux" >&2; exit 1; } +[ "$ACTUAL_ARCH" = "$EXPECTED_ARCH" ] || { + echo "安装包架构不匹配：当前 $ACTUAL_ARCH，安装包 $EXPECTED_ARCH" >&2 + exit 1 +} +[ -x "$ROOT/bin/vexdb" ] || { echo "安装包缺少 bin/vexdb" >&2; exit 1; } +[ -x "$ROOT/bin/vexfs-fuse" ] || { echo "安装包缺少 bin/vexfs-fuse" >&2; exit 1; } +[ -f "$ROOT/lib/vexdb_lite.so" ] || { echo "安装包缺少 lib/vexdb_lite.so" >&2; exit 1; } +( + cd "$ROOT" + sha256sum -c SHA256SUMS.txt >/dev/null +) || { echo "安装包文件哈希校验失败" >&2; exit 1; } + +mkdir -p "$BIN_DIR" "$LIB_DIR" +install -m 0755 "$ROOT/bin/vexdb" "$BIN_DIR/vexdb" +install -m 0755 "$ROOT/bin/vexfs-fuse" "$BIN_DIR/vexfs-fuse" +ln -sfn vexdb "$BIN_DIR/vexfs" +install -m 0644 "$ROOT/lib/vexdb_lite.so" "$LIB_DIR/vexdb_lite.so" + +echo "VexDB-Lite 已安装：" +echo " CLI: $BIN_DIR/vexdb" +echo " 文件快捷命令: $BIN_DIR/vexfs" +echo " FUSE helper: $BIN_DIR/vexfs-fuse" +echo " SQLite 扩展: $LIB_DIR/vexdb_lite.so" + +if ! command -v fusermount3 >/dev/null 2>&1; then + echo "" + echo "还不能挂载：系统缺少 fuse3。" + echo " Debian/Ubuntu: sudo apt install fuse3" + echo " Fedora/RHEL: sudo dnf install fuse3" +elif [ ! -e /dev/fuse ]; then + echo "" + echo "还不能挂载：系统没有 /dev/fuse，请加载 fuse 内核模块或检查容器权限。" +elif [ ! -r /dev/fuse ] || [ ! -w /dev/fuse ]; then + echo "" + echo "还不能挂载：当前用户没有读写 /dev/fuse 的权限。" +fi + +case ":$PATH:" in + *":$BIN_DIR:"*) ;; + *) + echo "" + echo "请把下面一行加入 shell 配置：" + echo " export PATH=\"$BIN_DIR:\$PATH\"" + ;; +esac + +echo "" +echo "下一步：$BIN_DIR/vexdb --version && $BIN_DIR/vexdb fs setup && $BIN_DIR/vexdb fs doctor" diff --git a/agent_files/linux/package/uninstall.sh b/agent_files/linux/package/uninstall.sh new file mode 100755 index 0000000000..401dfe675b --- /dev/null +++ b/agent_files/linux/package/uninstall.sh @@ -0,0 +1,18 @@ +#!/bin/bash +set -euo pipefail + +BIN_DIR="${VEXDB_LITE_BIN_DIR:-$HOME/.local/bin}" +LIB_DIR="${VEXDB_LITE_LIB_DIR:-$HOME/.local/lib/vexdb-lite}" + +if awk '$0 ~ / - fuse(\.vexfs)? vexfs / { found=1 } END { exit !found }' \ + /proc/self/mountinfo 2>/dev/null; then + echo "仍有 VexFS 目录挂载。请先执行 vexdb fs unmount，必要时加 --force。" >&2 + exit 1 +fi + +rm -f "$BIN_DIR/vexfs" "$BIN_DIR/vexfs-fuse" "$BIN_DIR/vexdb" +rm -f "$LIB_DIR/vexdb_lite.so" +rmdir "$LIB_DIR" 2>/dev/null || true + +echo "VexDB-Lite 程序已卸载。数据库没有删除：" +echo " ${XDG_DATA_HOME:-$HOME/.local/share}/vexdb-lite/" diff --git "a/agent_files/linux/package/\344\275\277\347\224\250\350\257\264\346\230\216.md" "b/agent_files/linux/package/\344\275\277\347\224\250\350\257\264\346\230\216.md" new file mode 100644 index 0000000000..292d06406c --- /dev/null +++ "b/agent_files/linux/package/\344\275\277\347\224\250\350\257\264\346\230\216.md" @@ -0,0 +1,114 @@ +# VexDB-Lite Linux SQLite + Files 使用说明 + +这个包提供四部分：SQLite 命令行、向量能力、VexFS 文件能力，以及把数据库工作区挂载成 +普通目录的 libfuse3 helper。安装不需要 root；真实挂载需要系统已经安装 `fuse3`，并允许 +当前用户访问 `/dev/fuse`。 + +## 1. 安装 + +先确认下载的包与机器架构一致： + +```bash +uname -m +# x86_64 下载 linux-x86_64；aarch64 下载 linux-aarch64 +``` + +解压后运行： + +```bash +bash install.sh +export PATH="$HOME/.local/bin:$PATH" +vexdb --version +``` + +如果 `vexdb fs doctor` 提示缺少 fuse3： + +```bash +# Debian / Ubuntu +sudo apt install fuse3 + +# Fedora / RHEL +sudo dnf install fuse3 +``` + +容器中还需要把 `/dev/fuse` 传入容器，并授予挂载需要的能力。 + +## 2. SQLite、向量和文件命令 + +```bash +vexdb agent.sqlite3 "SELECT sqlite_version(), vexdb_version();" + +vexdb fs setup +vexdb fs mkdir /docs +printf 'hello\n' | vexdb fs write /docs/hello.txt +vexdb fs ls /docs +vexdb fs cat /docs/hello.txt +``` + +默认数据库位于： + +```text +${XDG_DATA_HOME:-$HOME/.local/share}/vexdb-lite/default.sqlite3 +``` + +## 3. 挂载后直接使用 Bash + +```bash +vexdb fs doctor +mkdir -p "$HOME/VexDB" +vexdb fs mount "$HOME/VexDB" + +cd "$HOME/VexDB" +ls -la +grep -R "TODO" . +git init +``` + +普通卸载： + +```bash +vexdb fs unmount "$HOME/VexDB" +``` + +进程异常退出或目录一直被占用时： + +```bash +vexdb fs unmount --force "$HOME/VexDB" +``` + +## 4. 版本、快照和备份 + +```bash +vexdb fs history /docs/hello.txt +vexdb fs snapshot create before-agent +vexdb fs snapshot diff before-agent +vexdb fs snapshot restore before-agent +vexdb fs check +``` + +`vexdb fs check` 是只读深度检查，会核对目录、版本、快照、暂存区和内容 SHA-256。只想快速 +检查结构时使用 `vexdb fs check --quick`。发现损坏时命令输出问题和建议，返回退出码 8， +不会自动改写数据库。 + +用 SQLite Backup API 复制整个数据库： + +```bash +vexdb backup \ + "${XDG_DATA_HOME:-$HOME/.local/share}/vexdb-lite/default.sqlite3" \ + "$HOME/vexdb-backup.sqlite3" +``` + +## 5. 跨 macOS 和 Linux + +数据库文件、内容、Unicode、mode、可执行权限、symlink、hardlink、xattr、便携 ACL、历史和 +快照可在 macOS 与 Linux 间往返。数字 UID/GID 原样保存在数据库中。Linux 挂载按数字 ID +显示；macOS 普通用户 FSKit 挂载使用系统强制的 `noowners`，Bash `stat` 显示挂载用户， +`vexdb fs stat PATH` 仍能看到数据库保存的原始 UID/GID。 + +## 6. 卸载 + +```bash +bash uninstall.sh +``` + +卸载脚本只删除程序，不删除数据库。 diff --git a/agent_files/macos/ExportOptions-DeveloperID.plist b/agent_files/macos/ExportOptions-DeveloperID.plist new file mode 100644 index 0000000000..af1b04cb88 --- /dev/null +++ b/agent_files/macos/ExportOptions-DeveloperID.plist @@ -0,0 +1,14 @@ + + + + + method + developer-id + signingStyle + automatic + stripSwiftSymbols + + teamID + BB5VK42K87 + + diff --git a/agent_files/macos/VexFS.xcodeproj/project.pbxproj b/agent_files/macos/VexFS.xcodeproj/project.pbxproj index 21adaca682..8b06dcbe54 100644 --- a/agent_files/macos/VexFS.xcodeproj/project.pbxproj +++ b/agent_files/macos/VexFS.xcodeproj/project.pbxproj @@ -7,38 +7,38 @@ objects = { /* Begin PBXBuildFile section */ - 0BC68A3B319449C6CEE91C8D /* VexFSAppEx.appex in Embed ExtensionKit Extensions */ = {isa = PBXBuildFile; fileRef = 6CCDB74FA186F6FCD7A300FB /* VexFSAppEx.appex */; settings = {ATTRIBUTES = (RemoveHeadersOnCopy, ); }; }; - 23DDC1E282C33F2F633E7872 /* VexFSBackend.swift in Sources */ = {isa = PBXBuildFile; fileRef = F36BCE283AA3A5947E3A75BC /* VexFSBackend.swift */; }; - 3B11C6F78EA62D945AF6E165 /* FSKit.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 24ABADDA7C830A7635A906E5 /* FSKit.framework */; }; - A6C51020C5D7076193EA157A /* VexFSVolume+Operations.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0157B8EFB8B783DBB2A4B15D /* VexFSVolume+Operations.swift */; }; - A89A6A7839DECBD962C9FA02 /* Cocoa.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = D48085DD8AFB52024CD9512B /* Cocoa.framework */; }; - B383BA27662F9E6DD90C2C81 /* VexFSFileSystem.swift in Sources */ = {isa = PBXBuildFile; fileRef = 612EC9965B79CDECF771CFE7 /* VexFSFileSystem.swift */; }; - B6B84C145BFAEA0C42E15916 /* ContentView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 339D10E1B23713660842B8ED /* ContentView.swift */; }; - BD1E4AE3BF6880BE89877841 /* Cocoa.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = D48085DD8AFB52024CD9512B /* Cocoa.framework */; }; - BF65C046A873125AC5174F58 /* VexFSVolume.swift in Sources */ = {isa = PBXBuildFile; fileRef = 12A53F9C5F028FC3DD4BE3E3 /* VexFSVolume.swift */; }; - C9D129F35E11C5025B10D5F6 /* VexFSItem.swift in Sources */ = {isa = PBXBuildFile; fileRef = 79162BBEE0BBD0A628DBB6BD /* VexFSItem.swift */; }; - CCA1446D1EA1AA3AD47E37D0 /* VexFSAppEx.swift in Sources */ = {isa = PBXBuildFile; fileRef = 94204BA19A4C73FA79C190C9 /* VexFSAppEx.swift */; }; - D180822993DA1907CE7B6A10 /* VexFSApp.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0D63C4F4452011ECF470D69E /* VexFSApp.swift */; }; + 1C608A9018A4422EA1952C9C /* VexFSFileSystem.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2F39B3A14283D310941712A9 /* VexFSFileSystem.swift */; }; + 1EA761C45020F997EB84C888 /* VexFSVolume.swift in Sources */ = {isa = PBXBuildFile; fileRef = 44C93C9A924C2FE4F6908B8B /* VexFSVolume.swift */; }; + 3780F8276301D1A2E0BFA83A /* VexFSItem.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7E0E0BBFF61E43527CEFB2D3 /* VexFSItem.swift */; }; + 3C4A509CFA9740CED0FA6288 /* VexFSAppEx.appex in Embed ExtensionKit Extensions */ = {isa = PBXBuildFile; fileRef = B9426785DC8CB247B8D17175 /* VexFSAppEx.appex */; settings = {ATTRIBUTES = (RemoveHeadersOnCopy, ); }; }; + 42FDB23ACE4332A35F1EC106 /* ContentView.swift in Sources */ = {isa = PBXBuildFile; fileRef = ACD925958EF883345CE46C7D /* ContentView.swift */; }; + 45F16BD430300F6F029AE478 /* Cocoa.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 7462B11A9FB54192939A31FB /* Cocoa.framework */; }; + 5E2138B682A54BE885D561E1 /* FSKit.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 8B6A0DDE2868EC68E6A66B4B /* FSKit.framework */; }; + 6C5EE8691215FE2119F96681 /* Cocoa.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 7462B11A9FB54192939A31FB /* Cocoa.framework */; }; + 94CD0A2563F9AFB2FD7F6D7F /* VexFSApp.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6D28655136F6C3AE749EC51A /* VexFSApp.swift */; }; + B5409F3CA8A4250DE5EC350D /* VexFSAppEx.swift in Sources */ = {isa = PBXBuildFile; fileRef = AE67026F8EE22D6D7FDDF80B /* VexFSAppEx.swift */; }; + EFE51E86624B1AAEE193E04A /* VexFSVolume+Operations.swift in Sources */ = {isa = PBXBuildFile; fileRef = 61EE820F7CF3E4F8A29B0443 /* VexFSVolume+Operations.swift */; }; + FC4015CAE58C58A33AB7D39B /* VexFSBackend.swift in Sources */ = {isa = PBXBuildFile; fileRef = ADC6AEF286D1A4ABBB5D64FC /* VexFSBackend.swift */; }; /* End PBXBuildFile section */ /* Begin PBXContainerItemProxy section */ - 83549E04744B8CA7BF0D59B9 /* PBXContainerItemProxy */ = { + 27172AE9F45A0ED82BF173F0 /* PBXContainerItemProxy */ = { isa = PBXContainerItemProxy; - containerPortal = AC4A42524ED5CBEE19926577 /* Project object */; + containerPortal = ED84B2349ED06A80AABDC7EC /* Project object */; proxyType = 1; - remoteGlobalIDString = 1A5D042A492F7D6E68921B31; + remoteGlobalIDString = 755E955E75926D9A199A34AE; remoteInfo = VexFSAppEx; }; /* End PBXContainerItemProxy section */ /* Begin PBXCopyFilesBuildPhase section */ - C9410A46AA6747711A591D0A /* Embed ExtensionKit Extensions */ = { + AE3E23C090B6C56902DD5920 /* Embed ExtensionKit Extensions */ = { isa = PBXCopyFilesBuildPhase; buildActionMask = 2147483647; dstPath = "$(EXTENSIONS_FOLDER_PATH)"; dstSubfolderSpec = 16; files = ( - 0BC68A3B319449C6CEE91C8D /* VexFSAppEx.appex in Embed ExtensionKit Extensions */, + 3C4A509CFA9740CED0FA6288 /* VexFSAppEx.appex in Embed ExtensionKit Extensions */, ); name = "Embed ExtensionKit Extensions"; runOnlyForDeploymentPostprocessing = 0; @@ -46,120 +46,120 @@ /* End PBXCopyFilesBuildPhase section */ /* Begin PBXFileReference section */ - 0157B8EFB8B783DBB2A4B15D /* VexFSVolume+Operations.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = "VexFSVolume+Operations.swift"; sourceTree = ""; }; - 0D63C4F4452011ECF470D69E /* VexFSApp.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = VexFSApp.swift; sourceTree = ""; }; - 12A53F9C5F028FC3DD4BE3E3 /* VexFSVolume.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = VexFSVolume.swift; sourceTree = ""; }; - 24ABADDA7C830A7635A906E5 /* FSKit.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = FSKit.framework; path = System/Library/Frameworks/FSKit.framework; sourceTree = SDKROOT; }; - 339D10E1B23713660842B8ED /* ContentView.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = ContentView.swift; sourceTree = ""; }; - 4A51E7C4A79392F1B486CBDC /* VexFSApp.entitlements */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.plist.entitlements; path = VexFSApp.entitlements; sourceTree = ""; }; - 5B9E7D35E13C5EDA8EB0EC02 /* VexFS-Bridging-Header.h */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.c.h; path = "VexFS-Bridging-Header.h"; sourceTree = ""; }; - 612EC9965B79CDECF771CFE7 /* VexFSFileSystem.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = VexFSFileSystem.swift; sourceTree = ""; }; - 6CCDB74FA186F6FCD7A300FB /* VexFSAppEx.appex */ = {isa = PBXFileReference; explicitFileType = "wrapper.extensionkit-extension"; includeInIndex = 0; path = VexFSAppEx.appex; sourceTree = BUILT_PRODUCTS_DIR; }; - 79162BBEE0BBD0A628DBB6BD /* VexFSItem.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = VexFSItem.swift; sourceTree = ""; }; - 8B79A91CD5F1353E7C4DBC40 /* VexFSApp.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = VexFSApp.app; sourceTree = BUILT_PRODUCTS_DIR; }; - 94204BA19A4C73FA79C190C9 /* VexFSAppEx.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = VexFSAppEx.swift; sourceTree = ""; }; - C9358B6392411F7B776722E1 /* VexFSAppEx.entitlements */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.plist.entitlements; path = VexFSAppEx.entitlements; sourceTree = ""; }; - D48085DD8AFB52024CD9512B /* Cocoa.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = Cocoa.framework; path = Platforms/MacOSX.platform/Developer/SDKs/MacOSX15.0.sdk/System/Library/Frameworks/Cocoa.framework; sourceTree = DEVELOPER_DIR; }; - E1821EECDA039CD17708C8F8 /* Info.plist */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; - F36BCE283AA3A5947E3A75BC /* VexFSBackend.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = VexFSBackend.swift; sourceTree = ""; }; + 2B548BA0C8FC4FE83BDADFAA /* VexFSApp.entitlements */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.plist.entitlements; path = VexFSApp.entitlements; sourceTree = ""; }; + 2F39B3A14283D310941712A9 /* VexFSFileSystem.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = VexFSFileSystem.swift; sourceTree = ""; }; + 44C93C9A924C2FE4F6908B8B /* VexFSVolume.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = VexFSVolume.swift; sourceTree = ""; }; + 50A67298EECEEE0C501FD3D7 /* VexFS-Bridging-Header.h */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.c.h; path = "VexFS-Bridging-Header.h"; sourceTree = ""; }; + 61EE820F7CF3E4F8A29B0443 /* VexFSVolume+Operations.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = "VexFSVolume+Operations.swift"; sourceTree = ""; }; + 6D28655136F6C3AE749EC51A /* VexFSApp.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = VexFSApp.swift; sourceTree = ""; }; + 7462B11A9FB54192939A31FB /* Cocoa.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = Cocoa.framework; path = Platforms/MacOSX.platform/Developer/SDKs/MacOSX15.0.sdk/System/Library/Frameworks/Cocoa.framework; sourceTree = DEVELOPER_DIR; }; + 7E0E0BBFF61E43527CEFB2D3 /* VexFSItem.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = VexFSItem.swift; sourceTree = ""; }; + 8B6A0DDE2868EC68E6A66B4B /* FSKit.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = FSKit.framework; path = System/Library/Frameworks/FSKit.framework; sourceTree = SDKROOT; }; + 92DD930797D5751B88DD0B4C /* VexFSAppEx.entitlements */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.plist.entitlements; path = VexFSAppEx.entitlements; sourceTree = ""; }; + 955CCBFC4ADDE7F3B2AB6171 /* VexFSApp.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = VexFSApp.app; sourceTree = BUILT_PRODUCTS_DIR; }; + ACD925958EF883345CE46C7D /* ContentView.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = ContentView.swift; sourceTree = ""; }; + ADC6AEF286D1A4ABBB5D64FC /* VexFSBackend.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = VexFSBackend.swift; sourceTree = ""; }; + AE67026F8EE22D6D7FDDF80B /* VexFSAppEx.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = VexFSAppEx.swift; sourceTree = ""; }; + B9426785DC8CB247B8D17175 /* VexFSAppEx.appex */ = {isa = PBXFileReference; explicitFileType = "wrapper.extensionkit-extension"; includeInIndex = 0; path = VexFSAppEx.appex; sourceTree = BUILT_PRODUCTS_DIR; }; + C0B85E0FADEEE8DCCF88E2F2 /* Info.plist */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ - 537E5BFB297AFC43B476E8BA /* Frameworks */ = { + A44DD1638B2FEDF54E7E6D9A /* Frameworks */ = { isa = PBXFrameworksBuildPhase; buildActionMask = 2147483647; files = ( - A89A6A7839DECBD962C9FA02 /* Cocoa.framework in Frameworks */, - 3B11C6F78EA62D945AF6E165 /* FSKit.framework in Frameworks */, + 6C5EE8691215FE2119F96681 /* Cocoa.framework in Frameworks */, ); runOnlyForDeploymentPostprocessing = 0; }; - F9E2F58F5023DE4798CEA89E /* Frameworks */ = { + FB3B1CE2358B294D0A3F2843 /* Frameworks */ = { isa = PBXFrameworksBuildPhase; buildActionMask = 2147483647; files = ( - BD1E4AE3BF6880BE89877841 /* Cocoa.framework in Frameworks */, + 45F16BD430300F6F029AE478 /* Cocoa.framework in Frameworks */, + 5E2138B682A54BE885D561E1 /* FSKit.framework in Frameworks */, ); runOnlyForDeploymentPostprocessing = 0; }; /* End PBXFrameworksBuildPhase section */ /* Begin PBXGroup section */ - 1398F869C8324455467B1337 /* VexFSAppEx */ = { + 5249ACF4B47C0DD1B5FCBCFE /* VexFSApp */ = { isa = PBXGroup; children = ( - 94204BA19A4C73FA79C190C9 /* VexFSAppEx.swift */, - F36BCE283AA3A5947E3A75BC /* VexFSBackend.swift */, - 612EC9965B79CDECF771CFE7 /* VexFSFileSystem.swift */, - 79162BBEE0BBD0A628DBB6BD /* VexFSItem.swift */, - 12A53F9C5F028FC3DD4BE3E3 /* VexFSVolume.swift */, - 0157B8EFB8B783DBB2A4B15D /* VexFSVolume+Operations.swift */, - 5B9E7D35E13C5EDA8EB0EC02 /* VexFS-Bridging-Header.h */, - C9358B6392411F7B776722E1 /* VexFSAppEx.entitlements */, - E1821EECDA039CD17708C8F8 /* Info.plist */, + 6D28655136F6C3AE749EC51A /* VexFSApp.swift */, + ACD925958EF883345CE46C7D /* ContentView.swift */, + 2B548BA0C8FC4FE83BDADFAA /* VexFSApp.entitlements */, ); - name = VexFSAppEx; - path = VexFSAppEx; + name = VexFSApp; + path = VexFSApp; sourceTree = ""; }; - 43A182FDE5C8742E9D66F9DE /* OS X */ = { + 9940EB58D403ACAE6F0D63D1 /* VexFSAppEx */ = { isa = PBXGroup; children = ( - D48085DD8AFB52024CD9512B /* Cocoa.framework */, + AE67026F8EE22D6D7FDDF80B /* VexFSAppEx.swift */, + ADC6AEF286D1A4ABBB5D64FC /* VexFSBackend.swift */, + 2F39B3A14283D310941712A9 /* VexFSFileSystem.swift */, + 7E0E0BBFF61E43527CEFB2D3 /* VexFSItem.swift */, + 44C93C9A924C2FE4F6908B8B /* VexFSVolume.swift */, + 61EE820F7CF3E4F8A29B0443 /* VexFSVolume+Operations.swift */, + 50A67298EECEEE0C501FD3D7 /* VexFS-Bridging-Header.h */, + 92DD930797D5751B88DD0B4C /* VexFSAppEx.entitlements */, + C0B85E0FADEEE8DCCF88E2F2 /* Info.plist */, ); - name = "OS X"; + name = VexFSAppEx; + path = VexFSAppEx; sourceTree = ""; }; - 5C3B6513C200006C7D3ED342 /* Products */ = { + AFB77C8A39FAFE054703C88A /* Products */ = { isa = PBXGroup; children = ( - 8B79A91CD5F1353E7C4DBC40 /* VexFSApp.app */, - 6CCDB74FA186F6FCD7A300FB /* VexFSAppEx.appex */, + 955CCBFC4ADDE7F3B2AB6171 /* VexFSApp.app */, + B9426785DC8CB247B8D17175 /* VexFSAppEx.appex */, ); name = Products; sourceTree = ""; }; - 667505ECC9BC5B1D8210D9F4 /* Frameworks */ = { + C33214477C261251221288EE = { isa = PBXGroup; children = ( - 43A182FDE5C8742E9D66F9DE /* OS X */, - 24ABADDA7C830A7635A906E5 /* FSKit.framework */, + AFB77C8A39FAFE054703C88A /* Products */, + D07AD92FC8225E0F250C0546 /* Frameworks */, + 5249ACF4B47C0DD1B5FCBCFE /* VexFSApp */, + 9940EB58D403ACAE6F0D63D1 /* VexFSAppEx */, ); - name = Frameworks; sourceTree = ""; }; - 975A9C6292155E57E5284D04 = { + D07AD92FC8225E0F250C0546 /* Frameworks */ = { isa = PBXGroup; children = ( - 5C3B6513C200006C7D3ED342 /* Products */, - 667505ECC9BC5B1D8210D9F4 /* Frameworks */, - D6D01A997E4D4C079763432B /* VexFSApp */, - 1398F869C8324455467B1337 /* VexFSAppEx */, + FAA5BD3C573B5E15140317D1 /* OS X */, + 8B6A0DDE2868EC68E6A66B4B /* FSKit.framework */, ); + name = Frameworks; sourceTree = ""; }; - D6D01A997E4D4C079763432B /* VexFSApp */ = { + FAA5BD3C573B5E15140317D1 /* OS X */ = { isa = PBXGroup; children = ( - 0D63C4F4452011ECF470D69E /* VexFSApp.swift */, - 339D10E1B23713660842B8ED /* ContentView.swift */, - 4A51E7C4A79392F1B486CBDC /* VexFSApp.entitlements */, + 7462B11A9FB54192939A31FB /* Cocoa.framework */, ); - name = VexFSApp; - path = VexFSApp; + name = "OS X"; sourceTree = ""; }; /* End PBXGroup section */ /* Begin PBXNativeTarget section */ - 1A5D042A492F7D6E68921B31 /* VexFSAppEx */ = { + 755E955E75926D9A199A34AE /* VexFSAppEx */ = { isa = PBXNativeTarget; - buildConfigurationList = 81F32C156A3B0AF8EA9C65C0 /* Build configuration list for PBXNativeTarget "VexFSAppEx" */; + buildConfigurationList = 016567E2D1ACC4AB4EFB697A /* Build configuration list for PBXNativeTarget "VexFSAppEx" */; buildPhases = ( - C4F2425D1C43106F2CDA1C6C /* Build VexFS SQLite Core */, - FE25A55875F5753214F89004 /* Sources */, - 537E5BFB297AFC43B476E8BA /* Frameworks */, - 3F8ACBA139F4993B190B48B6 /* Resources */, + 5A328AF2F8ACEA4AC8F29B3C /* Build VexFS SQLite Core */, + 8FB42330433A1B49BFEDDE6F /* Sources */, + FB3B1CE2358B294D0A3F2843 /* Frameworks */, + 3028435B675F4CF744C89BC5 /* Resources */, ); buildRules = ( ); @@ -167,38 +167,38 @@ ); name = VexFSAppEx; productName = VexFSAppEx; - productReference = 6CCDB74FA186F6FCD7A300FB /* VexFSAppEx.appex */; + productReference = B9426785DC8CB247B8D17175 /* VexFSAppEx.appex */; productType = "com.apple.product-type.extensionkit-extension"; }; - 6FF435D5C1BC036A4F7B4BA6 /* VexFSApp */ = { + BAD729588513F880EAA2F59F /* VexFSApp */ = { isa = PBXNativeTarget; - buildConfigurationList = C235E4ED94D989172CFFDA17 /* Build configuration list for PBXNativeTarget "VexFSApp" */; + buildConfigurationList = 6581806B5669A7CF0E93B968 /* Build configuration list for PBXNativeTarget "VexFSApp" */; buildPhases = ( - 046F460A8B6C8F5B35D1F22D /* Sources */, - F9E2F58F5023DE4798CEA89E /* Frameworks */, - 99F228151C9C8F18C28D2EBB /* Resources */, - C9410A46AA6747711A591D0A /* Embed ExtensionKit Extensions */, + E1E929D517CB3B3CFC12756C /* Sources */, + A44DD1638B2FEDF54E7E6D9A /* Frameworks */, + 456B33A1846ED212B12332AD /* Resources */, + AE3E23C090B6C56902DD5920 /* Embed ExtensionKit Extensions */, ); buildRules = ( ); dependencies = ( - 479D729D1D8E7D50EEF80968 /* PBXTargetDependency */, + 8A050C6C471C700BA18BBEF1 /* PBXTargetDependency */, ); name = VexFSApp; productName = VexFSApp; - productReference = 8B79A91CD5F1353E7C4DBC40 /* VexFSApp.app */; + productReference = 955CCBFC4ADDE7F3B2AB6171 /* VexFSApp.app */; productType = "com.apple.product-type.application"; }; /* End PBXNativeTarget section */ /* Begin PBXProject section */ - AC4A42524ED5CBEE19926577 /* Project object */ = { + ED84B2349ED06A80AABDC7EC /* Project object */ = { isa = PBXProject; attributes = { LastSwiftUpdateCheck = 2630; LastUpgradeCheck = 2630; }; - buildConfigurationList = 3DCF428E65687F1FDC0726AD /* Build configuration list for PBXProject "VexFS" */; + buildConfigurationList = 759F70EA6018D5E6B193E205 /* Build configuration list for PBXProject "VexFS" */; compatibilityVersion = "Xcode 3.2"; developmentRegion = en; hasScannedForEncodings = 0; @@ -206,28 +206,28 @@ en, Base, ); - mainGroup = 975A9C6292155E57E5284D04; + mainGroup = C33214477C261251221288EE; minimizedProjectReferenceProxies = 0; preferredProjectObjectVersion = 77; - productRefGroup = 5C3B6513C200006C7D3ED342 /* Products */; + productRefGroup = AFB77C8A39FAFE054703C88A /* Products */; projectDirPath = ""; projectRoot = ""; targets = ( - 6FF435D5C1BC036A4F7B4BA6 /* VexFSApp */, - 1A5D042A492F7D6E68921B31 /* VexFSAppEx */, + BAD729588513F880EAA2F59F /* VexFSApp */, + 755E955E75926D9A199A34AE /* VexFSAppEx */, ); }; /* End PBXProject section */ /* Begin PBXResourcesBuildPhase section */ - 3F8ACBA139F4993B190B48B6 /* Resources */ = { + 3028435B675F4CF744C89BC5 /* Resources */ = { isa = PBXResourcesBuildPhase; buildActionMask = 2147483647; files = ( ); runOnlyForDeploymentPostprocessing = 0; }; - 99F228151C9C8F18C28D2EBB /* Resources */ = { + 456B33A1846ED212B12332AD /* Resources */ = { isa = PBXResourcesBuildPhase; buildActionMask = 2147483647; files = ( @@ -237,7 +237,7 @@ /* End PBXResourcesBuildPhase section */ /* Begin PBXShellScriptBuildPhase section */ - C4F2425D1C43106F2CDA1C6C /* Build VexFS SQLite Core */ = { + 5A328AF2F8ACEA4AC8F29B3C /* Build VexFS SQLite Core */ = { isa = PBXShellScriptBuildPhase; alwaysOutOfDate = 1; buildActionMask = 2147483647; @@ -254,46 +254,46 @@ ); runOnlyForDeploymentPostprocessing = 0; shellPath = /bin/sh; - shellScript = "set -e\nCMAKE_BIN=\"$(command -v cmake || true)\"\n[ -x \"$CMAKE_BIN\" ] || CMAKE_BIN=/opt/homebrew/bin/cmake\n[ -x \"$CMAKE_BIN\" ] || CMAKE_BIN=/usr/local/bin/cmake\n[ -x \"$CMAKE_BIN\" ] || { echo \"cmake is required\" >&2; exit 1; }\nBUILD_DIR=\"$SRCROOT/../../vexdb_sqlite/build-fskit\"\n\"$CMAKE_BIN\" -S \"$SRCROOT/../../vexdb_sqlite\" -B \"$BUILD_DIR\" \\\n -DCMAKE_BUILD_TYPE=\"$CONFIGURATION\" -DVEXDB_SQLITE_BUILD_TESTS=OFF\n\"$CMAKE_BIN\" --build \"$BUILD_DIR\" --target vexfs_mount_contract -j \"$(sysctl -n hw.ncpu)\"\n"; + shellScript = "set -e\nCMAKE_BIN=\"$(command -v cmake || true)\"\n[ -x \"$CMAKE_BIN\" ] || CMAKE_BIN=/opt/homebrew/bin/cmake\n[ -x \"$CMAKE_BIN\" ] || CMAKE_BIN=/usr/local/bin/cmake\n[ -x \"$CMAKE_BIN\" ] || { echo \"cmake is required\" >&2; exit 1; }\nBUILD_DIR=\"$SRCROOT/../../vexdb_sqlite/build-fskit\"\nBUILD_JOBS=\"${VEXDB_LITE_BUILD_JOBS:-4}\"\ncase \"$BUILD_JOBS\" in *[!0-9]*|0) echo \"VEXDB_LITE_BUILD_JOBS must be a positive integer\" >&2; exit 2;; esac\n[ \"$BUILD_JOBS\" -le 4 ] || BUILD_JOBS=4\n\"$CMAKE_BIN\" -S \"$SRCROOT/../../vexdb_sqlite\" -B \"$BUILD_DIR\" \\\n -DCMAKE_BUILD_TYPE=\"$CONFIGURATION\" -DVEXDB_SQLITE_BUILD_TESTS=OFF\n\"$CMAKE_BIN\" --build \"$BUILD_DIR\" --target vexfs_runtime -j \"$BUILD_JOBS\"\n"; }; /* End PBXShellScriptBuildPhase section */ /* Begin PBXSourcesBuildPhase section */ - 046F460A8B6C8F5B35D1F22D /* Sources */ = { + 8FB42330433A1B49BFEDDE6F /* Sources */ = { isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; files = ( - D180822993DA1907CE7B6A10 /* VexFSApp.swift in Sources */, - B6B84C145BFAEA0C42E15916 /* ContentView.swift in Sources */, + B5409F3CA8A4250DE5EC350D /* VexFSAppEx.swift in Sources */, + FC4015CAE58C58A33AB7D39B /* VexFSBackend.swift in Sources */, + 1C608A9018A4422EA1952C9C /* VexFSFileSystem.swift in Sources */, + 3780F8276301D1A2E0BFA83A /* VexFSItem.swift in Sources */, + 1EA761C45020F997EB84C888 /* VexFSVolume.swift in Sources */, + EFE51E86624B1AAEE193E04A /* VexFSVolume+Operations.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; - FE25A55875F5753214F89004 /* Sources */ = { + E1E929D517CB3B3CFC12756C /* Sources */ = { isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; files = ( - CCA1446D1EA1AA3AD47E37D0 /* VexFSAppEx.swift in Sources */, - 23DDC1E282C33F2F633E7872 /* VexFSBackend.swift in Sources */, - B383BA27662F9E6DD90C2C81 /* VexFSFileSystem.swift in Sources */, - C9D129F35E11C5025B10D5F6 /* VexFSItem.swift in Sources */, - BF65C046A873125AC5174F58 /* VexFSVolume.swift in Sources */, - A6C51020C5D7076193EA157A /* VexFSVolume+Operations.swift in Sources */, + 94CD0A2563F9AFB2FD7F6D7F /* VexFSApp.swift in Sources */, + 42FDB23ACE4332A35F1EC106 /* ContentView.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; /* End PBXSourcesBuildPhase section */ /* Begin PBXTargetDependency section */ - 479D729D1D8E7D50EEF80968 /* PBXTargetDependency */ = { + 8A050C6C471C700BA18BBEF1 /* PBXTargetDependency */ = { isa = PBXTargetDependency; name = VexFSAppEx; - target = 1A5D042A492F7D6E68921B31 /* VexFSAppEx */; - targetProxy = 83549E04744B8CA7BF0D59B9 /* PBXContainerItemProxy */; + target = 755E955E75926D9A199A34AE /* VexFSAppEx */; + targetProxy = 27172AE9F45A0ED82BF173F0 /* PBXContainerItemProxy */; }; /* End PBXTargetDependency section */ /* Begin XCBuildConfiguration section */ - 09491CF975FB59BD73BDAA62 /* Release */ = { + 00F1949ED702892DE84D34BC /* Debug */ = { isa = XCBuildConfiguration; buildSettings = { ALWAYS_SEARCH_USER_PATHS = NO; @@ -327,11 +327,17 @@ CLANG_WARN_UNREACHABLE_CODE = YES; CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; COPY_PHASE_STRIP = NO; - DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; - ENABLE_NS_ASSERTIONS = NO; + DEBUG_INFORMATION_FORMAT = dwarf; ENABLE_STRICT_OBJC_MSGSEND = YES; + ENABLE_TESTABILITY = YES; GCC_C_LANGUAGE_STANDARD = gnu11; + GCC_DYNAMIC_NO_PIC = NO; GCC_NO_COMMON_BLOCKS = YES; + GCC_OPTIMIZATION_LEVEL = 0; + GCC_PREPROCESSOR_DEFINITIONS = ( + "DEBUG=1", + "$(inherited)", + ); GCC_WARN_64_TO_32_BIT_CONVERSION = YES; GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; GCC_WARN_UNDECLARED_SELECTOR = YES; @@ -339,96 +345,18 @@ GCC_WARN_UNUSED_FUNCTION = YES; GCC_WARN_UNUSED_VARIABLE = YES; MACOSX_DEPLOYMENT_TARGET = 26.0; - MTL_ENABLE_DEBUG_INFO = NO; + MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; MTL_FAST_MATH = YES; + ONLY_ACTIVE_ARCH = YES; PRODUCT_NAME = "$(TARGET_NAME)"; SDKROOT = macosx; - SWIFT_COMPILATION_MODE = wholemodule; - SWIFT_OPTIMIZATION_LEVEL = "-O"; + SWIFT_ACTIVE_COMPILATION_CONDITIONS = DEBUG; + SWIFT_OPTIMIZATION_LEVEL = "-Onone"; SWIFT_VERSION = 5.0; }; - name = Release; - }; - 31A0A5719E39E7BB401D99FE /* Debug */ = { - isa = XCBuildConfiguration; - buildSettings = { - ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; - ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; - CODE_SIGN_ENTITLEMENTS = VexFSApp/VexFSApp.entitlements; - CODE_SIGN_STYLE = Automatic; - COMBINE_HIDPI_IMAGES = YES; - CURRENT_PROJECT_VERSION = 1; - DEVELOPMENT_TEAM = ""; - ENABLE_APP_SANDBOX = YES; - GENERATE_INFOPLIST_FILE = YES; - INFOPLIST_KEY_CFBundleDisplayName = VexFS; - LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/../Frameworks"; - MACOSX_DEPLOYMENT_TARGET = 26.0; - MARKETING_VERSION = 0.1.0; - PRODUCT_BUNDLE_IDENTIFIER = io.vexdb.vexfs; - PRODUCT_NAME = VexFS; - SDKROOT = macosx; - }; name = Debug; }; - A6AECEA8884F34F863EE1397 /* Release */ = { - isa = XCBuildConfiguration; - buildSettings = { - ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; - ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; - CODE_SIGN_ENTITLEMENTS = VexFSApp/VexFSApp.entitlements; - CODE_SIGN_STYLE = Automatic; - COMBINE_HIDPI_IMAGES = YES; - CURRENT_PROJECT_VERSION = 1; - DEVELOPMENT_TEAM = ""; - ENABLE_APP_SANDBOX = YES; - GENERATE_INFOPLIST_FILE = YES; - INFOPLIST_KEY_CFBundleDisplayName = VexFS; - LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/../Frameworks"; - MACOSX_DEPLOYMENT_TARGET = 26.0; - MARKETING_VERSION = 0.1.0; - PRODUCT_BUNDLE_IDENTIFIER = io.vexdb.vexfs; - PRODUCT_NAME = VexFS; - SDKROOT = macosx; - }; - name = Release; - }; - B5F77B8B9328312919A0E4EA /* Release */ = { - isa = XCBuildConfiguration; - buildSettings = { - APPLICATION_EXTENSION_API_ONLY = YES; - CODE_SIGN_ENTITLEMENTS = VexFSAppEx/VexFSAppEx.entitlements; - CODE_SIGN_STYLE = Automatic; - CURRENT_PROJECT_VERSION = 1; - DEVELOPMENT_TEAM = ""; - ENABLE_APP_SANDBOX = YES; - ENABLE_USER_SCRIPT_SANDBOXING = NO; - GENERATE_INFOPLIST_FILE = YES; - HEADER_SEARCH_PATHS = ( - "$(inherited)", - "\"$(SRCROOT)/../mount/common/include\"", - ); - INFOPLIST_FILE = VexFSAppEx/Info.plist; - INFOPLIST_KEY_CFBundleDisplayName = "VexFS file system"; - LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/../Frameworks @executable_path/../../../../Frameworks"; - MACOSX_DEPLOYMENT_TARGET = 26.0; - MARKETING_VERSION = 0.1.0; - OTHER_LDFLAGS = ( - "$(inherited)", - "\"$(SRCROOT)/../../vexdb_sqlite/build-fskit/libvexfs_mount_contract.a\"", - "\"$(SRCROOT)/../../vexdb_sqlite/build-fskit/libvexdb_lite_static.a\"", - "-lsqlite3", - "-lc++", - ); - PRODUCT_BUNDLE_IDENTIFIER = io.vexdb.vexfs.extension; - PRODUCT_NAME = VexFSAppEx; - SDKROOT = macosx; - SKIP_INSTALL = YES; - SWIFT_OBJC_BRIDGING_HEADER = "VexFSAppEx/VexFS-Bridging-Header.h"; - }; - name = Release; - }; - B6EE28519513696CE1930079 /* Debug */ = { + 2B05C13542B6725C575D64A9 /* Release */ = { isa = XCBuildConfiguration; buildSettings = { ALWAYS_SEARCH_USER_PATHS = NO; @@ -462,17 +390,11 @@ CLANG_WARN_UNREACHABLE_CODE = YES; CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; COPY_PHASE_STRIP = NO; - DEBUG_INFORMATION_FORMAT = dwarf; + DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; + ENABLE_NS_ASSERTIONS = NO; ENABLE_STRICT_OBJC_MSGSEND = YES; - ENABLE_TESTABILITY = YES; GCC_C_LANGUAGE_STANDARD = gnu11; - GCC_DYNAMIC_NO_PIC = NO; GCC_NO_COMMON_BLOCKS = YES; - GCC_OPTIMIZATION_LEVEL = 0; - GCC_PREPROCESSOR_DEFINITIONS = ( - "DEBUG=1", - "$(inherited)", - ); GCC_WARN_64_TO_32_BIT_CONVERSION = YES; GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; GCC_WARN_UNDECLARED_SELECTOR = YES; @@ -480,26 +402,108 @@ GCC_WARN_UNUSED_FUNCTION = YES; GCC_WARN_UNUSED_VARIABLE = YES; MACOSX_DEPLOYMENT_TARGET = 26.0; - MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; + MTL_ENABLE_DEBUG_INFO = NO; MTL_FAST_MATH = YES; - ONLY_ACTIVE_ARCH = YES; PRODUCT_NAME = "$(TARGET_NAME)"; SDKROOT = macosx; - SWIFT_ACTIVE_COMPILATION_CONDITIONS = DEBUG; - SWIFT_OPTIMIZATION_LEVEL = "-Onone"; + SWIFT_COMPILATION_MODE = wholemodule; + SWIFT_OPTIMIZATION_LEVEL = "-O"; SWIFT_VERSION = 5.0; }; + name = Release; + }; + 3D1A46E7C3D9E7486EA7515F /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + APPLICATION_EXTENSION_API_ONLY = YES; + CODE_SIGN_ENTITLEMENTS = VexFSAppEx/VexFSAppEx.entitlements; + CODE_SIGN_STYLE = Automatic; + CURRENT_PROJECT_VERSION = 1; + DEVELOPMENT_TEAM = BB5VK42K87; + ENABLE_APP_SANDBOX = YES; + ENABLE_HARDENED_RUNTIME = YES; + ENABLE_USER_SCRIPT_SANDBOXING = NO; + GENERATE_INFOPLIST_FILE = YES; + HEADER_SEARCH_PATHS = ( + "$(inherited)", + "\"$(SRCROOT)/../mount/common/include\"", + ); + INFOPLIST_FILE = VexFSAppEx/Info.plist; + INFOPLIST_KEY_CFBundleDisplayName = "VexFS file system"; + LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/../Frameworks @executable_path/../../../../Frameworks"; + MACOSX_DEPLOYMENT_TARGET = 26.0; + MARKETING_VERSION = 0.1.0; + OTHER_LDFLAGS = ( + "$(inherited)", + "\"$(SRCROOT)/../../vexdb_sqlite/build-fskit/libvexfs_runtime.a\"", + "\"$(SRCROOT)/../../vexdb_sqlite/build-fskit/libvexdb_lite_static.a\"", + "-lsqlite3", + "-lc++", + ); + PRODUCT_BUNDLE_IDENTIFIER = io.vexdb.vexfs.extension; + PRODUCT_NAME = VexFSAppEx; + SDKROOT = macosx; + SKIP_INSTALL = YES; + SWIFT_OBJC_BRIDGING_HEADER = "VexFSAppEx/VexFS-Bridging-Header.h"; + }; + name = Release; + }; + 4C1189A9B0E0C89A11F8F389 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; + CODE_SIGN_ENTITLEMENTS = VexFSApp/VexFSApp.entitlements; + CODE_SIGN_STYLE = Automatic; + COMBINE_HIDPI_IMAGES = YES; + CURRENT_PROJECT_VERSION = 1; + DEVELOPMENT_TEAM = BB5VK42K87; + ENABLE_APP_SANDBOX = YES; + ENABLE_HARDENED_RUNTIME = YES; + GENERATE_INFOPLIST_FILE = YES; + INFOPLIST_KEY_CFBundleDisplayName = "VexDB Lite"; + LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/../Frameworks"; + MACOSX_DEPLOYMENT_TARGET = 26.0; + MARKETING_VERSION = 0.1.0; + PRODUCT_BUNDLE_IDENTIFIER = io.vexdb.vexfs; + PRODUCT_NAME = "VexDB Lite"; + SDKROOT = macosx; + }; + name = Release; + }; + 5B6CE2F7E21FBD659FA20DBF /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; + CODE_SIGN_ENTITLEMENTS = VexFSApp/VexFSApp.entitlements; + CODE_SIGN_STYLE = Automatic; + COMBINE_HIDPI_IMAGES = YES; + CURRENT_PROJECT_VERSION = 1; + DEVELOPMENT_TEAM = BB5VK42K87; + ENABLE_APP_SANDBOX = YES; + ENABLE_HARDENED_RUNTIME = YES; + GENERATE_INFOPLIST_FILE = YES; + INFOPLIST_KEY_CFBundleDisplayName = "VexDB Lite"; + LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/../Frameworks"; + MACOSX_DEPLOYMENT_TARGET = 26.0; + MARKETING_VERSION = 0.1.0; + PRODUCT_BUNDLE_IDENTIFIER = io.vexdb.vexfs; + PRODUCT_NAME = "VexDB Lite"; + SDKROOT = macosx; + }; name = Debug; }; - E8620A8E6E1735705B0CE3C8 /* Debug */ = { + D4767CC85F241F3F0711CBC2 /* Debug */ = { isa = XCBuildConfiguration; buildSettings = { APPLICATION_EXTENSION_API_ONLY = YES; CODE_SIGN_ENTITLEMENTS = VexFSAppEx/VexFSAppEx.entitlements; CODE_SIGN_STYLE = Automatic; CURRENT_PROJECT_VERSION = 1; - DEVELOPMENT_TEAM = ""; + DEVELOPMENT_TEAM = BB5VK42K87; ENABLE_APP_SANDBOX = YES; + ENABLE_HARDENED_RUNTIME = YES; ENABLE_USER_SCRIPT_SANDBOXING = NO; GENERATE_INFOPLIST_FILE = YES; HEADER_SEARCH_PATHS = ( @@ -513,7 +517,7 @@ MARKETING_VERSION = 0.1.0; OTHER_LDFLAGS = ( "$(inherited)", - "\"$(SRCROOT)/../../vexdb_sqlite/build-fskit/libvexfs_mount_contract.a\"", + "\"$(SRCROOT)/../../vexdb_sqlite/build-fskit/libvexfs_runtime.a\"", "\"$(SRCROOT)/../../vexdb_sqlite/build-fskit/libvexdb_lite_static.a\"", "-lsqlite3", "-lc++", @@ -529,34 +533,34 @@ /* End XCBuildConfiguration section */ /* Begin XCConfigurationList section */ - 3DCF428E65687F1FDC0726AD /* Build configuration list for PBXProject "VexFS" */ = { + 016567E2D1ACC4AB4EFB697A /* Build configuration list for PBXNativeTarget "VexFSAppEx" */ = { isa = XCConfigurationList; buildConfigurations = ( - B6EE28519513696CE1930079 /* Debug */, - 09491CF975FB59BD73BDAA62 /* Release */, + 3D1A46E7C3D9E7486EA7515F /* Release */, + D4767CC85F241F3F0711CBC2 /* Debug */, ); defaultConfigurationIsVisible = 0; defaultConfigurationName = Release; }; - 81F32C156A3B0AF8EA9C65C0 /* Build configuration list for PBXNativeTarget "VexFSAppEx" */ = { + 6581806B5669A7CF0E93B968 /* Build configuration list for PBXNativeTarget "VexFSApp" */ = { isa = XCConfigurationList; buildConfigurations = ( - B5F77B8B9328312919A0E4EA /* Release */, - E8620A8E6E1735705B0CE3C8 /* Debug */, + 4C1189A9B0E0C89A11F8F389 /* Release */, + 5B6CE2F7E21FBD659FA20DBF /* Debug */, ); defaultConfigurationIsVisible = 0; defaultConfigurationName = Release; }; - C235E4ED94D989172CFFDA17 /* Build configuration list for PBXNativeTarget "VexFSApp" */ = { + 759F70EA6018D5E6B193E205 /* Build configuration list for PBXProject "VexFS" */ = { isa = XCConfigurationList; buildConfigurations = ( - A6AECEA8884F34F863EE1397 /* Release */, - 31A0A5719E39E7BB401D99FE /* Debug */, + 00F1949ED702892DE84D34BC /* Debug */, + 2B05C13542B6725C575D64A9 /* Release */, ); defaultConfigurationIsVisible = 0; defaultConfigurationName = Release; }; /* End XCConfigurationList section */ }; - rootObject = AC4A42524ED5CBEE19926577 /* Project object */; + rootObject = ED84B2349ED06A80AABDC7EC /* Project object */; } diff --git a/agent_files/macos/VexFSApp/ContentView.swift b/agent_files/macos/VexFSApp/ContentView.swift index 2b80016c04..f3323c731b 100644 --- a/agent_files/macos/VexFSApp/ContentView.swift +++ b/agent_files/macos/VexFSApp/ContentView.swift @@ -3,20 +3,20 @@ import SwiftUI struct ContentView: View { private var databasePath: String { FileManager.default.homeDirectoryForCurrentUser - .appendingPathComponent("Library/Application Support/VexFS/vexfs.sqlite3").path + .appendingPathComponent("Library/Application Support/VexDB-Lite/default.sqlite3").path } var body: some View { VStack(alignment: .leading, spacing: 18) { - Label("VexFS", systemImage: "externaldrive.connected.to.line.below") + Label("VexDB-Lite", systemImage: "externaldrive.connected.to.line.below") .font(.system(size: 32, weight: .semibold)) - Text("由 SQLite 管理、可直接通过 Bash 使用的文件系统。") + Text("一个安装包，提供 SQLite、向量检索和 VexFS 文件能力。") .font(.title3) GroupBox("启用步骤") { VStack(alignment: .leading, spacing: 10) { - Text("1. 在 系统设置 → 通用 → 登录项与扩展 → 文件系统扩展 中启用 VexFS。") - Text("2. 运行 vexfs setup 初始化数据库，再用 vexfs doctor 检查状态。") - Text("3. 运行 vexfs mount ~/VexFS 挂载，然后直接使用 ls、cat、grep、cp、mv 等命令。") + Text("1. 打开 系统设置 → 通用 → 登录项与扩展，滚动到扩展区域，切换到“按类别”，点“文件系统扩展”右侧 ⓘ，再打开 VexDB Lite。") + Text("2. 运行 vexdb fs setup 初始化数据库，再用 vexdb fs doctor 检查状态。") + Text("3. 运行 vexdb fs mount ~/VexDB 挂载，然后直接使用 ls、cat、grep、cp、mv 等命令。") } .frame(maxWidth: .infinity, alignment: .leading) .padding(4) @@ -25,6 +25,9 @@ struct ContentView: View { .font(.system(.footnote, design: .monospaced)) .textSelection(.enabled) .foregroundStyle(.secondary) + Text("如果看到的是“VexDB Lite / FSKit Modules”只读详情，说明当前在“按 App”视图。返回并切换到“按类别”后启用。") + .font(.footnote) + .foregroundStyle(.secondary) Spacer() } .padding(28) diff --git a/agent_files/macos/VexFSAppEx/VexFS-Bridging-Header.h b/agent_files/macos/VexFSAppEx/VexFS-Bridging-Header.h index 1e1a311632..4555a60478 100644 --- a/agent_files/macos/VexFSAppEx/VexFS-Bridging-Header.h +++ b/agent_files/macos/VexFSAppEx/VexFS-Bridging-Header.h @@ -1,6 +1,6 @@ #ifndef VEXFS_BRIDGING_HEADER_H #define VEXFS_BRIDGING_HEADER_H -#include "vexfs_mount_contract.h" +#include "vexfs_runtime_io.h" #endif diff --git a/agent_files/macos/VexFSAppEx/VexFSBackend.swift b/agent_files/macos/VexFSAppEx/VexFSBackend.swift index 6b007583d6..537890ae4f 100644 --- a/agent_files/macos/VexFSAppEx/VexFSBackend.swift +++ b/agent_files/macos/VexFSAppEx/VexFSBackend.swift @@ -14,7 +14,31 @@ struct VexFSStatRecord: Codable { let size: UInt64 let version: UInt64 let created_at: Int64 + let accessed_at: Int64? let updated_at: Int64 + let changed_at: Int64? + let uid: UInt32? + let gid: UInt32? + let link_count: UInt32? + + init(path: String, inode: UInt64, kind: String, mode: UInt32, size: UInt64, + version: UInt64, created_at: Int64, accessed_at: Int64? = nil, + updated_at: Int64, changed_at: Int64? = nil, + uid: UInt32? = nil, gid: UInt32? = nil, link_count: UInt32? = nil) { + self.path = path + self.inode = inode + self.kind = kind + self.mode = mode + self.size = size + self.version = version + self.created_at = created_at + self.accessed_at = accessed_at + self.updated_at = updated_at + self.changed_at = changed_at + self.uid = uid + self.gid = gid + self.link_count = link_count + } } struct VexFSDirectoryEntry: Codable { @@ -25,24 +49,68 @@ struct VexFSDirectoryEntry: Codable { let version: UInt64 let mode: UInt32 let created_at: Int64 + let accessed_at: Int64? let updated_at: Int64 + let changed_at: Int64? + let uid: UInt32? + let gid: UInt32? + let link_count: UInt32? + + init(name: String, inode: UInt64, kind: String, size: UInt64, version: UInt64, + mode: UInt32, created_at: Int64, accessed_at: Int64? = nil, + updated_at: Int64, changed_at: Int64? = nil, + uid: UInt32? = nil, gid: UInt32? = nil, link_count: UInt32? = nil) { + self.name = name + self.inode = inode + self.kind = kind + self.size = size + self.version = version + self.mode = mode + self.created_at = created_at + self.accessed_at = accessed_at + self.updated_at = updated_at + self.changed_at = changed_at + self.uid = uid + self.gid = gid + self.link_count = link_count + } +} + +struct VexFSDirectoryListing: Codable { + let version: UInt64 + let entries: [VexFSDirectoryEntry] } final class VexFSBackend { private var session: OpaquePointer? + private let databasePath: String + private let workspace: String init(databasePath: String, workspace: String) throws { + self.databasePath = databasePath + self.workspace = workspace + try reopen() + } + + func reopen() throws { + guard session == nil else { return } var config = vexfs_mount_config() - config.abi_version = UInt32(VEXFS_MOUNT_ABI_VERSION) - config.busy_timeout_ms = 5_000 - config.flags = UInt32(VEXFS_MOUNT_EXCLUSIVE_GATEWAY) + config.abi_version = UInt32(VEXFS_RUNTIME_ABI_VERSION) + config.operation_timeout_ms = 5_000 + config.flags = UInt32(VEXFS_RUNTIME_EXCLUSIVE_GATEWAY) var opened: OpaquePointer? var error = vexfs_mount_error() - let status = databasePath.withCString { databasePointer in - workspace.withCString { workspacePointer in - config.database_path = databasePointer - config.workspace = workspacePointer - return vexfs_mount_session_open(&config, &opened, &error) + let status = "sqlite".withCString { backendPointer in + databasePath.withCString { databasePointer in + workspace.withCString { workspacePointer in + "local".withCString { principalPointer in + config.backend = backendPointer + config.connection = databasePointer + config.workspace = workspacePointer + config.principal = principalPointer + return vexfs_mount_session_open(&config, &opened, &error) + } + } } } try Self.check(status, error: &error) @@ -50,7 +118,13 @@ final class VexFSBackend { } deinit { - vexfs_mount_session_close(session) + close() + } + + func close() { + guard let activeSession = session else { return } + session = nil + vexfs_mount_session_close(activeSession) } func stat(path: String) throws -> VexFSStatRecord { @@ -80,12 +154,89 @@ final class VexFSBackend { return try JSONDecoder().decode([VexFSDirectoryEntry].self, from: Self.take(&output)) } + func listVersioned(path: String) throws -> VexFSDirectoryListing { + var output = vexfs_mount_bytes() + var error = vexfs_mount_error() + let status = path.withCString { + vexfs_mount_list_versioned(session, $0, &output, &error) + } + try Self.check(status, error: &error) + return try JSONDecoder().decode(VexFSDirectoryListing.self, from: Self.take(&output)) + } + func mkdir(path: String) throws { var error = vexfs_mount_error() let status = path.withCString { vexfs_mount_mkdir(session, $0, &error) } try Self.check(status, error: &error) } + func create(path: String, kind: String, mode: UInt32) throws { + var error = vexfs_mount_error() + let status = path.withCString { pathPointer in + kind.withCString { kindPointer in + vexfs_mount_create(session, pathPointer, kindPointer, mode, &error) + } + } + do { + try Self.check(status, error: &error) + } catch let failure as POSIXError where failure.code == .EBUSY { + throw POSIXError(.EEXIST) + } + } + + func setMode(inode: UInt64, mode: UInt32) throws { + var error = vexfs_mount_error() + let status = vexfs_mount_set_mode(session, Int64(bitPattern: inode), mode, &error) + try Self.check(status, error: &error) + } + + func setTimes(inode: UInt64, accessedAt: Int64, modifiedAt: Int64, + mask: UInt32) throws { + var error = vexfs_mount_error() + let status = vexfs_mount_set_times(session, Int64(bitPattern: inode), accessedAt, + modifiedAt, mask, &error) + try Self.check(status, error: &error) + } + + func symlink(path: String, target: Data) throws { + var error = vexfs_mount_error() + let status = path.withCString { pathPointer in + target.withUnsafeBytes { bytes in + vexfs_mount_symlink(session, pathPointer, bytes.baseAddress, + UInt64(bytes.count), &error) + } + } + do { + try Self.check(status, error: &error) + } catch let failure as POSIXError where failure.code == .EBUSY { + throw POSIXError(.EEXIST) + } + } + + func link(source: String, destination: String) throws { + var error = vexfs_mount_error() + let status = source.withCString { sourcePointer in + destination.withCString { destinationPointer in + vexfs_mount_link(session, sourcePointer, destinationPointer, &error) + } + } + try Self.check(status, error: &error) + } + + func chown(inode: UInt64, uid: Int64, gid: Int64) throws { + var error = vexfs_mount_error() + let status = vexfs_mount_chown(session, Int64(bitPattern: inode), uid, gid, &error) + try Self.check(status, error: &error) + } + + func readlink(inode: UInt64) throws -> Data { + var output = vexfs_mount_bytes() + var error = vexfs_mount_error() + let status = vexfs_mount_readlink(session, Int64(bitPattern: inode), &output, &error) + try Self.check(status, error: &error) + return Self.take(&output) + } + @discardableResult func writeFile(path: String, data: Data) throws -> Int64 { var version: Int64 = 0 @@ -137,6 +288,52 @@ final class VexFSBackend { try Self.check(status, error: &error) } + func getXattr(inode: UInt64, name: String) throws -> Data { + var output = vexfs_mount_bytes() + var error = vexfs_mount_error() + let status = name.withCString { + vexfs_mount_xattr_get(session, Int64(bitPattern: inode), $0, &output, &error) + } + do { + try Self.check(status, error: &error) + } catch let failure as POSIXError where failure.code == .ENOENT { + throw POSIXError(.ENOATTR) + } + return Self.take(&output) + } + + func listXattrs(inode: UInt64) throws -> [String] { + var output = vexfs_mount_bytes() + var error = vexfs_mount_error() + let status = vexfs_mount_xattr_list(session, Int64(bitPattern: inode), &output, &error) + try Self.check(status, error: &error) + return try JSONDecoder().decode([String].self, from: Self.take(&output)) + } + + func setXattr(inode: UInt64, name: String, value: Data?, policy: Int) throws { + var error = vexfs_mount_error() + let status = name.withCString { namePointer in + if let value { + return value.withUnsafeBytes { bytes in + vexfs_mount_xattr_set(session, Int64(bitPattern: inode), namePointer, + bytes.baseAddress, UInt64(bytes.count), Int32(policy), + &error) + } + } + return vexfs_mount_xattr_set(session, Int64(bitPattern: inode), namePointer, + nil, 0, Int32(policy), &error) + } + do { + try Self.check(status, error: &error) + } catch let failure as POSIXError { + if policy == 1 && failure.code == .EBUSY { throw POSIXError(.EEXIST) } + if (policy == 2 || policy == 3) && failure.code == .ENOENT { + throw POSIXError(.ENOATTR) + } + throw failure + } + } + func openHandle(path: String, flags: String) throws -> String { var output = vexfs_mount_bytes() var error = vexfs_mount_error() @@ -156,6 +353,27 @@ final class VexFSBackend { return value } + func createFileHandle(path: String, mode: UInt32) throws -> String { + var output = vexfs_mount_bytes() + var error = vexfs_mount_error() + let request = UUID().uuidString + let status = path.withCString { pathPointer in + request.withCString { requestPointer in + vexfs_mount_handle_create(session, pathPointer, mode, requestPointer, + &output, &error) + } + } + do { + try Self.check(status, error: &error) + } catch let failure as POSIXError where failure.code == .EBUSY { + throw POSIXError(.EEXIST) + } + guard let value = String(data: Self.take(&output), encoding: .utf8) else { + throw POSIXError(.EIO) + } + return value + } + func stageWrite(handle: String, offset: UInt64, data: Data) throws -> Int64 { var generation: Int64 = 0 var error = vexfs_mount_error() @@ -212,6 +430,19 @@ final class VexFSBackend { return version } + func publishAndClose(handle: String, generation: Int64) throws -> Int64 { + var version: Int64 = 0 + var error = vexfs_mount_error() + let status = handle.withCString { handlePointer in + "data".withCString { durabilityPointer in + vexfs_mount_handle_publish_close(session, handlePointer, generation, + durabilityPointer, &version, &error) + } + } + try Self.check(status, error: &error) + return version + } + func closeHandle(_ handle: String, retain: Bool) throws { var output = vexfs_mount_bytes() var error = vexfs_mount_error() @@ -236,6 +467,15 @@ final class VexFSBackend { try Self.check(status, error: &error) } + func refreshVisibility() throws -> (head: Int64, generation: UInt64, external: Bool) { + var visibility = vexfs_mount_visibility() + var error = vexfs_mount_error() + let status = vexfs_mount_refresh_visibility(session, &visibility, &error) + try Self.check(status, error: &error) + return (visibility.workspace_head, visibility.cache_generation, + visibility.external_commit != 0) + } + func reclaim() throws { var reclaimed: Int64 = 0 var error = vexfs_mount_error() @@ -272,6 +512,7 @@ final class VexFSBackend { case VEXFS_MOUNT_NO_SPACE: code = .ENOSPC case VEXFS_MOUNT_CORRUPTION: code = .EIO case VEXFS_MOUNT_UNSUPPORTED: code = .ENOTSUP + case VEXFS_MOUNT_NOT_EMPTY: code = .ENOTEMPTY default: code = .EIO } throw POSIXError(code) diff --git a/agent_files/macos/VexFSAppEx/VexFSFileSystem.swift b/agent_files/macos/VexFSAppEx/VexFSFileSystem.swift index 2fd29cc817..f44f5e9d60 100644 --- a/agent_files/macos/VexFSAppEx/VexFSFileSystem.swift +++ b/agent_files/macos/VexFSAppEx/VexFSFileSystem.swift @@ -104,16 +104,25 @@ final class VexFSFileSystem: FSUnaryFileSystem & FSUnaryFileSystemOperations { func unloadResource(resource: FSResource, options: FSTaskOptions, replyHandler: @escaping ((any Error)?) -> Void) { - guard let pathResource = resource as? FSPathURLResource, - pathResource.url == self.resource?.url else { + guard resource is FSPathURLResource else { return replyHandler(POSIXError(.EINVAL)) } + // FSUnaryFileSystem has exactly one loaded resource. FSKit may give the + // unload callback a new FSPathURLResource object whose URL spelling is + // not byte-for-byte identical to the load callback (for example + // /tmp versus /private/tmp). Comparing URL objects here rejected a real + // unmount and left the database mount lease alive until its 30s expiry. + guard let loadedResource = self.resource else { + return replyHandler(nil) + } + Logger.vexfs.info("unloading VexFS resource") do { try volume?.synchronizeNow() } catch { Logger.vexfs.error("unload synchronize failed: \(error.localizedDescription)") } + volume?.closeBackend() self.volume = nil self.resource = nil - pathResource.url.stopAccessingSecurityScopedResource() + loadedResource.url.stopAccessingSecurityScopedResource() replyHandler(nil) } } diff --git a/agent_files/macos/VexFSAppEx/VexFSItem.swift b/agent_files/macos/VexFSAppEx/VexFSItem.swift index 74d76e3a44..8781a909db 100644 --- a/agent_files/macos/VexFSAppEx/VexFSItem.swift +++ b/agent_files/macos/VexFSAppEx/VexFSItem.swift @@ -5,20 +5,33 @@ final class VexFSItem: FSItem { let lock = NSLock() var path: String var record: VexFSStatRecord + var recordValidatedAt: TimeInterval + var cacheGeneration: UInt64 var parentID: FSItem.Identifier var handle: String? var handleWritable = false var dirtyGeneration: Int64? + // 小型只读文件在一次 open/close 生命周期内保留内容快照，避免文本搜索为 + // 每个文件创建和关闭数据库 handle。写打开和大文件仍使用数据库 handle。 + var readCache: Data? + var isUnlinked = false - init(path: String, record: VexFSStatRecord, parentID: FSItem.Identifier) { + init(path: String, record: VexFSStatRecord, parentID: FSItem.Identifier, + cacheGeneration: UInt64) { self.path = path self.record = record + self.recordValidatedAt = ProcessInfo.processInfo.systemUptime + self.cacheGeneration = cacheGeneration self.parentID = parentID super.init() } var itemType: FSItem.ItemType { - record.kind == "directory" ? .directory : .file + switch record.kind { + case "directory": return .directory + case "symlink": return .symlink + default: return .file + } } var itemID: FSItem.Identifier { diff --git a/agent_files/macos/VexFSAppEx/VexFSVolume+Operations.swift b/agent_files/macos/VexFSAppEx/VexFSVolume+Operations.swift index 7e1bc2e469..ff95190187 100644 --- a/agent_files/macos/VexFSAppEx/VexFSVolume+Operations.swift +++ b/agent_files/macos/VexFSAppEx/VexFSVolume+Operations.swift @@ -20,24 +20,32 @@ extension VexFSVolume: FSVolume.Operations { capabilities.supportsFastStatFS = false capabilities.supports2TBFiles = false capabilities.supports64BitObjectIDs = true - capabilities.supportsSymbolicLinks = false - capabilities.supportsHardLinks = false + capabilities.supportsSymbolicLinks = true + capabilities.supportsHardLinks = true capabilities.supportsHiddenFiles = true capabilities.doesNotSupportVolumeSizes = true capabilities.doesNotSupportImmutableFiles = true - capabilities.doesNotSupportSettingFilePermissions = true + capabilities.doesNotSupportSettingFilePermissions = false capabilities.caseFormat = .sensitive return capabilities } func activate(options: FSTaskOptions, replyHandler: @escaping (FSItem?, (any Error)?) -> Void) { - replyHandler(rootItem, nil) + do { + try reactivateBackend() + replyHandler(rootItem, nil) + } catch { + replyHandler(nil, error) + } } func deactivate(options: FSDeactivateOptions = [], replyHandler: @escaping ((any Error)?) -> Void) { - do { try synchronizeNow(); replyHandler(nil) } catch { replyHandler(error) } + // FSKit calls synchronize and unmount before deactivate. Resource + // teardown must not start another I/O round here. + closeBackend() + replyHandler(nil) } func mount(options: FSTaskOptions, replyHandler: @escaping ((any Error)?) -> Void) { @@ -46,6 +54,7 @@ extension VexFSVolume: FSVolume.Operations { func unmount(replyHandler: @escaping () -> Void) { try? synchronizeNow() + closeBackend() replyHandler() } @@ -59,15 +68,35 @@ extension VexFSVolume: FSVolume.Operations { guard let item = item as? VexFSItem else { return replyHandler(nil, POSIXError(.EINVAL)) } + let generation: UInt64 + do { generation = try currentCacheGeneration() } + catch { return replyHandler(nil, error) } item.lock.lock() defer { item.lock.unlock() } + if item.isUnlinked { + return replyHandler(attributes(for: item, request: desiredAttributes), nil) + } do { - let path = try currentPath(for: item) - if item.dirtyGeneration == nil { + let now = ProcessInfo.processInfo.systemUptime + if item.dirtyGeneration == nil && + (item.cacheGeneration != generation || now - item.recordValidatedAt >= 0.25) { + let path = try currentPath(for: item) item.record = try backend.stat(path: path) + item.recordValidatedAt = now + item.cacheGeneration = generation } replyHandler(attributes(for: item, request: desiredAttributes), nil) - } catch { replyHandler(nil, error) } + } catch { + let nsError = error as NSError + if nsError.domain == NSPOSIXErrorDomain && nsError.code == Int(ENOENT) { + // rename 覆盖或 unlink 后，内核仍可能持有旧 vnode，并继续查询 + // 它的属性。目录项已经消失不代表这个 FSItem 立刻失效；返回 + // 最后一次已知属性，直到 reclaimItem 回收它。 + replyHandler(attributes(for: item, request: desiredAttributes), nil) + } else { + replyHandler(nil, error) + } + } } func setAttributes(_ newAttributes: FSItem.SetAttributesRequest, on item: FSItem, @@ -79,7 +108,12 @@ extension VexFSVolume: FSVolume.Operations { defer { item.lock.unlock() } do { let path = try currentPath(for: item) + var localMode = item.record.mode + var localSize = item.record.size if newAttributes.isValid(.size), item.itemType == .file { + // truncate 会改变所有已打开描述符随后看到的文件内容。丢弃只读 + // 快照，避免同一个 vnode 上的旧读描述符继续返回截断前的数据。 + clearReadCache(item) let hadHandle = item.handle != nil if item.handle == nil || !item.handleWritable { if let old = item.handle { try backend.closeHandle(old, retain: true) } @@ -89,22 +123,67 @@ extension VexFSVolume: FSVolume.Operations { } let generation = try backend.truncate(handle: item.handle!, size: newAttributes.size) item.dirtyGeneration = generation + localSize = newAttributes.size newAttributes.consumedAttributes.insert(.size) if !hadHandle { - _ = try backend.publish(handle: item.handle!, generation: generation) - try backend.closeHandle(item.handle!, retain: true) + _ = try backend.publishAndClose(handle: item.handle!, generation: generation) item.handle = nil item.handleWritable = false item.dirtyGeneration = nil item.record = try backend.stat(path: path) } else { item.record = VexFSStatRecord(path: path, inode: item.record.inode, - kind: item.record.kind, mode: item.record.mode, size: newAttributes.size, + kind: item.record.kind, mode: localMode, size: localSize, version: item.record.version, created_at: item.record.created_at, updated_at: Int64(Date().timeIntervalSince1970 * 1000)) } - } else { + } + if newAttributes.isValid(.mode), item.itemType != .symlink { + localMode = newAttributes.mode & 0o777 + try backend.setMode(inode: item.record.inode, mode: localMode) + newAttributes.consumedAttributes.insert(.mode) + } + if newAttributes.isValid(.uid) || newAttributes.isValid(.gid) { + let uid: Int64 = newAttributes.isValid(.uid) ? Int64(newAttributes.uid) : -1 + let gid: Int64 = newAttributes.isValid(.gid) ? Int64(newAttributes.gid) : -1 + try backend.chown(inode: item.record.inode, uid: uid, gid: gid) + if newAttributes.isValid(.uid) { newAttributes.consumedAttributes.insert(.uid) } + if newAttributes.isValid(.gid) { newAttributes.consumedAttributes.insert(.gid) } + } + if newAttributes.isValid(.accessTime) || newAttributes.isValid(.modifyTime) { + var mask: UInt32 = 0 + var accessedAt = item.record.accessed_at ?? item.record.updated_at + var modifiedAt = item.record.updated_at + if newAttributes.isValid(.accessTime) { + let value = newAttributes.accessTime + guard value.tv_sec >= 0, value.tv_nsec >= 0, + value.tv_nsec < 1_000_000_000 else { + throw POSIXError(.EINVAL) + } + accessedAt = Int64(value.tv_sec) * 1000 + Int64(value.tv_nsec) / 1_000_000 + mask |= UInt32(VEXFS_MOUNT_TIME_ACCESS) + newAttributes.consumedAttributes.insert(.accessTime) + } + if newAttributes.isValid(.modifyTime) { + let value = newAttributes.modifyTime + guard value.tv_sec >= 0, value.tv_nsec >= 0, + value.tv_nsec < 1_000_000_000 else { + throw POSIXError(.EINVAL) + } + modifiedAt = Int64(value.tv_sec) * 1000 + Int64(value.tv_nsec) / 1_000_000 + mask |= UInt32(VEXFS_MOUNT_TIME_MODIFY) + newAttributes.consumedAttributes.insert(.modifyTime) + } + try backend.setTimes(inode: item.record.inode, accessedAt: accessedAt, + modifiedAt: modifiedAt, mask: mask) + } + if item.dirtyGeneration == nil { item.record = try backend.stat(path: path) + } else { + item.record = VexFSStatRecord(path: path, inode: item.record.inode, + kind: item.record.kind, mode: localMode, size: localSize, + version: item.record.version, created_at: item.record.created_at, + updated_at: Int64(Date().timeIntervalSince1970 * 1000)) } replyHandler(attributes(for: item, request: nil), nil) } catch { replyHandler(nil, error) } @@ -126,12 +205,24 @@ extension VexFSVolume: FSVolume.Operations { } func reclaimItem(_ item: FSItem, replyHandler: @escaping ((any Error)?) -> Void) { - do { try backend.reclaim(); replyHandler(nil) } catch { replyHandler(error) } + guard let item = item as? VexFSItem else { + return replyHandler(POSIXError(.EINVAL)) + } + do { + try backend.reclaim() + reclaimCachedItem(item) + replyHandler(nil) + } catch { replyHandler(error) } } func readSymbolicLink(_ item: FSItem, replyHandler: @escaping (FSFileName?, (any Error)?) -> Void) { - replyHandler(nil, POSIXError(.ENOTSUP)) + guard let item = item as? VexFSItem, item.itemType == .symlink else { + return replyHandler(nil, POSIXError(.EINVAL)) + } + do { + replyHandler(FSFileName(data: try backend.readlink(inode: item.record.inode)), nil) + } catch { replyHandler(nil, error) } } func createItem(named name: FSFileName, type: FSItem.ItemType, @@ -143,13 +234,26 @@ extension VexFSVolume: FSVolume.Operations { } do { let path = try childPath(parent: directory, name: nameString) + let defaultMode: UInt32 = type == .directory ? 0o755 : 0o644 + let mode = newAttributes.isValid(.mode) ? newAttributes.mode & 0o777 : defaultMode + var createdHandle: String? switch type { - case .directory: try backend.mkdir(path: path) - case .file: _ = try backend.writeFile(path: path, data: Data()) + case .directory: try backend.create(path: path, kind: "directory", mode: mode) + case .file: createdHandle = try backend.createFileHandle(path: path, mode: mode) default: throw POSIXError(.ENOTSUP) } + if newAttributes.isValid(.mode) { newAttributes.consumedAttributes.insert(.mode) } let record = try backend.stat(path: path) let item = cachedItem(path: path, record: record, parentID: directory.itemID) + if let createdHandle { + // The atomic create leaves an empty generation staged. Reuse + // that handle for the first write and publish one version on + // close/synchronize, instead of an empty version plus content. + item.handle = createdHandle + item.handleWritable = true + item.dirtyGeneration = 1 + } + recordDirectoryMutation(directory) replyHandler(item, name, nil) } catch { replyHandler(nil, nil, error) } } @@ -158,20 +262,89 @@ extension VexFSVolume: FSVolume.Operations { attributes newAttributes: FSItem.SetAttributesRequest, linkContents contents: FSFileName, replyHandler: @escaping (FSItem?, FSFileName?, (any Error)?) -> Void) { - replyHandler(nil, nil, POSIXError(.ENOTSUP)) + guard let directory = directory as? VexFSItem, let nameString = name.string else { + return replyHandler(nil, nil, POSIXError(.EINVAL)) + } + do { + let path = try childPath(parent: directory, name: nameString) + try backend.symlink(path: path, target: contents.data) + if newAttributes.isValid(.mode) { + newAttributes.consumedAttributes.insert(.mode) + } + let record = try backend.stat(path: path) + let item = cachedItem(path: path, record: record, parentID: directory.itemID) + recordDirectoryMutation(directory) + replyHandler(item, name, nil) + } catch { replyHandler(nil, nil, error) } } func createLink(to item: FSItem, named name: FSFileName, inDirectory directory: FSItem, replyHandler: @escaping (FSFileName?, (any Error)?) -> Void) { - replyHandler(nil, POSIXError(.ENOTSUP)) + guard let item = item as? VexFSItem, + let directory = directory as? VexFSItem, + let name = name.string else { + return replyHandler(nil, POSIXError(.EINVAL)) + } + do { + let source = try currentPath(for: item) + let destination = try childPath(parent: directory, name: name) + try backend.link(source: source, destination: destination) + // hardlink 改变的是同一个 inode。立即刷新源 vnode，确保 link(2) + // 返回后紧接着的 stat(2) 能看到新的 st_nlink。 + try refreshRecord(item, at: source) + recordDirectoryMutation(directory) + replyHandler(FSFileName(string: name), nil) + } catch { replyHandler(nil, error) } + } + + func getXattr(named name: FSFileName, of item: FSItem, + replyHandler: @escaping (Data?, (any Error)?) -> Void) { + guard let item = item as? VexFSItem, let name = name.string else { + return replyHandler(nil, POSIXError(.EINVAL)) + } + do { + replyHandler(try backend.getXattr(inode: item.record.inode, name: name), nil) + } catch { replyHandler(nil, error) } + } + + func setXattr(named name: FSFileName, to value: Data?, on item: FSItem, + policy: FSVolume.SetXattrPolicy, + replyHandler: @escaping ((any Error)?) -> Void) { + guard let item = item as? VexFSItem, let name = name.string else { + return replyHandler(POSIXError(.EINVAL)) + } + do { + try backend.setXattr(inode: item.record.inode, name: name, value: value, + policy: Int(policy.rawValue)) + replyHandler(nil) + } catch { replyHandler(error) } + } + + func listXattrs(of item: FSItem, + replyHandler: @escaping ([FSFileName]?, (any Error)?) -> Void) { + guard let item = item as? VexFSItem else { + return replyHandler(nil, POSIXError(.EINVAL)) + } + do { + let names = try backend.listXattrs(inode: item.record.inode) + .map { FSFileName(string: $0) } + replyHandler(names, nil) + } catch { replyHandler(nil, error) } } func removeItem(_ item: FSItem, named name: FSFileName, fromDirectory directory: FSItem, replyHandler: @escaping ((any Error)?) -> Void) { - guard let item = item as? VexFSItem else { return replyHandler(POSIXError(.EINVAL)) } + guard let item = item as? VexFSItem, + let directory = directory as? VexFSItem, + let name = name.string else { return replyHandler(POSIXError(.EINVAL)) } do { - try backend.remove(path: currentPath(for: item)) - dropCachedItems() + let path = try childPath(parent: directory, name: name) + try backend.remove(path: path) + item.lock.lock() + item.isUnlinked = true + item.recordValidatedAt = 0 + item.lock.unlock() + recordDirectoryMutation(directory) replyHandler(nil) } catch { replyHandler(error) } } @@ -181,21 +354,39 @@ extension VexFSVolume: FSVolume.Operations { inDirectory destinationDirectory: FSItem, overItem: FSItem?, replyHandler: @escaping (FSFileName?, (any Error)?) -> Void) { guard let source = item as? VexFSItem, + let sourceDirectory = sourceDirectory as? VexFSItem, let destinationDirectory = destinationDirectory as? VexFSItem, + let sourceName = sourceName.string, let destination = destinationName.string else { return replyHandler(nil, POSIXError(.EINVAL)) } do { - let sourcePath = try currentPath(for: source) + // FSKit 给出的目录和名称代表这次 rename 请求看到的目录项。 + // 不要通过 inode 缓存反查源路径：Git 的 config.lock -> config + // 原子替换会快速创建、关闭并重命名文件，缓存路径可能已经失效。 + let sourcePath = try childPath(parent: sourceDirectory, name: sourceName) let destinationPath = try childPath(parent: destinationDirectory, name: destination) try backend.rename(source: sourcePath, destination: destinationPath, replace: overItem != nil) - source.path = destinationPath - source.record = try backend.stat(path: destinationPath) + try refreshRecord(source, at: destinationPath) + source.lock.lock() source.parentID = destinationDirectory.itemID - dropCachedItems() + source.isUnlinked = false + source.lock.unlock() + if let replaced = overItem as? VexFSItem { + replaced.lock.lock() + replaced.isUnlinked = true + replaced.recordValidatedAt = 0 + replaced.lock.unlock() + } + recordDirectoryMutation(sourceDirectory) + if destinationDirectory !== sourceDirectory { + recordDirectoryMutation(destinationDirectory) + } replyHandler(destinationName, nil) - } catch { replyHandler(nil, error) } + } catch { + replyHandler(nil, error) + } } func enumerateDirectory(_ directory: FSItem, startingAt cookie: FSDirectoryCookie, @@ -207,18 +398,29 @@ extension VexFSVolume: FSVolume.Operations { return replyHandler(FSDirectoryVerifier(0), POSIXError(.ENOTDIR)) } do { - let directoryPath = try currentPath(for: directory) - directory.record = try backend.stat(path: directoryPath) - let currentVerifier = FSDirectoryVerifier(directory.record.version) + let generation = try currentCacheGeneration() + let now = ProcessInfo.processInfo.systemUptime + let directoryPath = directory.cacheGeneration == generation && + now - directory.recordValidatedAt < 0.25 + ? directory.path : try currentPath(for: directory) + let listing = try backend.listVersioned(path: directoryPath) + let currentVerifier = FSDirectoryVerifier(listing.version) if verifier.rawValue != 0 && verifier.rawValue != currentVerifier.rawValue { throw FSError(.invalidDirectoryCookie) } - let entries = try backend.list(path: directoryPath) - let verifiedRecord = try backend.stat(path: directoryPath) - guard verifiedRecord.version == directory.record.version else { - throw FSError(.invalidDirectoryCookie) - } - directory.record = verifiedRecord + directory.record = VexFSStatRecord( + path: directoryPath, inode: directory.record.inode, + kind: directory.record.kind, mode: directory.record.mode, + size: directory.record.size, version: listing.version, + created_at: directory.record.created_at, + accessed_at: directory.record.accessed_at, + updated_at: directory.record.updated_at, + changed_at: directory.record.changed_at, + uid: directory.record.uid, gid: directory.record.gid, + link_count: directory.record.link_count) + directory.recordValidatedAt = now + directory.cacheGeneration = generation + let entries = listing.entries let prefixCount = request == nil ? 2 : 0 let total = prefixCount + entries.count guard cookie.rawValue <= UInt64(total) else { @@ -244,7 +446,11 @@ extension VexFSVolume: FSVolume.Operations { mode: entry.mode, size: entry.size, version: entry.version, created_at: entry.created_at, - updated_at: entry.updated_at) + accessed_at: entry.accessed_at, + updated_at: entry.updated_at, + changed_at: entry.changed_at, + uid: entry.uid, gid: entry.gid, + link_count: entry.link_count) let item = cachedItem(path: path, record: record, parentID: directory.itemID) let packed = packer.packEntry(name: FSFileName(string: entry.name), itemType: item.itemType, itemID: item.itemID, diff --git a/agent_files/macos/VexFSAppEx/VexFSVolume.swift b/agent_files/macos/VexFSAppEx/VexFSVolume.swift index 87ab22c3a7..17d33a186a 100644 --- a/agent_files/macos/VexFSAppEx/VexFSVolume.swift +++ b/agent_files/macos/VexFSAppEx/VexFSVolume.swift @@ -3,18 +3,27 @@ import FSKit final class VexFSVolume: FSVolume, FSVolume.ReadWriteOperations, - FSVolume.OpenCloseOperations { + FSVolume.OpenCloseOperations, + FSVolume.XattrOperations { let backend: VexFSBackend let workspace: String let rootItem: VexFSItem private let cacheLock = NSLock() private var itemCache: [UInt64: VexFSItem] = [:] + private let maximumReadCacheBytes: UInt64 = 1024 * 1024 + private let maximumTotalReadCacheBytes: UInt64 = 64 * 1024 * 1024 + private var readCacheBytes: UInt64 = 0 + private var observedCacheGeneration: UInt64 init(backend: VexFSBackend, workspace: String, volumeID: UUID) throws { self.backend = backend self.workspace = workspace + let visibility = try backend.refreshVisibility() + self.observedCacheGeneration = visibility.generation let rootRecord = try backend.stat(path: "/") - self.rootItem = VexFSItem(path: "/", record: rootRecord, parentID: .parentOfRoot) + self.rootItem = VexFSItem(path: "/", record: rootRecord, + parentID: .parentOfRoot, + cacheGeneration: visibility.generation) super.init(volumeID: FSVolume.Identifier(uuid: volumeID), volumeName: FSFileName(string: "VexFS \(workspace)")) itemCache[rootRecord.inode] = rootItem @@ -22,6 +31,65 @@ final class VexFSVolume: FSVolume, func synchronizeNow() throws { try backend.synchronize() } + func closeBackend() { backend.close() } + + func reactivateBackend() throws { + try backend.reopen() + let visibility = try backend.refreshVisibility() + let rootRecord = try backend.stat(path: "/") + + rootItem.lock.lock() + rootItem.record = rootRecord + rootItem.recordValidatedAt = ProcessInfo.processInfo.systemUptime + rootItem.cacheGeneration = visibility.generation + rootItem.readCache = nil + rootItem.isUnlinked = false + rootItem.lock.unlock() + + cacheLock.lock() + for item in itemCache.values { item.readCache = nil } + itemCache.removeAll(keepingCapacity: true) + itemCache[rootRecord.inode] = rootItem + readCacheBytes = 0 + observedCacheGeneration = visibility.generation + cacheLock.unlock() + } + + // PRAGMA data_version 只在其他连接提交后变化。runtime 仅在变化时读取 + // workspace HEAD；这里保存一个 generation，已有 vnode 在下次访问时懒刷新。 + func currentCacheGeneration() throws -> UInt64 { + let visibility = try backend.refreshVisibility() + cacheLock.lock() + observedCacheGeneration = visibility.generation + let generation = observedCacheGeneration + cacheLock.unlock() + return generation + } + + func clearReadCache(_ item: VexFSItem) { + cacheLock.lock() + if let cached = item.readCache { + readCacheBytes -= min(readCacheBytes, UInt64(cached.count)) + item.readCache = nil + } + cacheLock.unlock() + } + + func installReadCache(_ data: Data, for item: VexFSItem) -> Bool { + cacheLock.lock() + defer { cacheLock.unlock() } + if let previous = item.readCache { + readCacheBytes -= min(readCacheBytes, UInt64(previous.count)) + item.readCache = nil + } + guard UInt64(data.count) <= maximumTotalReadCacheBytes - readCacheBytes else { + return false + } + item.readCache = data + readCacheBytes += UInt64(data.count) + return true + } + func currentPath(for item: VexFSItem) throws -> String { let path = try backend.path(inode: item.record.inode) item.path = path @@ -40,19 +108,66 @@ final class VexFSVolume: FSVolume, if let item = itemCache[record.inode] { item.path = path item.record = record + item.recordValidatedAt = ProcessInfo.processInfo.systemUptime + item.cacheGeneration = observedCacheGeneration item.parentID = parentID + item.isUnlinked = false return item } - let item = VexFSItem(path: path, record: record, parentID: parentID) + let item = VexFSItem(path: path, record: record, parentID: parentID, + cacheGeneration: observedCacheGeneration) itemCache[record.inode] = item return item } - func dropCachedItems(exceptRoot: Bool = true) { + func reclaimCachedItem(_ item: VexFSItem) { + clearReadCache(item) cacheLock.lock() - itemCache.removeAll(keepingCapacity: true) - if exceptRoot { itemCache[rootItem.record.inode] = rootItem } - cacheLock.unlock() + defer { cacheLock.unlock() } + guard item !== rootItem, + let cached = itemCache[item.record.inode], + cached === item else { return } + itemCache.removeValue(forKey: item.record.inode) + } + + // record cache 只负责减少重复 stat。任何已经提交到数据库的元数据变更都要 + // 精确失效相关 vnode，不能等待 250 ms TTL，否则 Bash 会在 mutation 返回 + // 成功后立刻看到旧的 nlink、mtime 或 ctime。 + func invalidateRecord(_ item: VexFSItem) { + item.lock.lock() + item.recordValidatedAt = 0 + item.lock.unlock() + } + + func recordDirectoryMutation(_ directory: VexFSItem) { + directory.lock.lock() + let wallClock = Int64(Date().timeIntervalSince1970 * 1000) + // SQLite 时间戳是毫秒级。同一毫秒内连续 mutation 时仍要让调用方 + // 观察到单调推进，随后 TTL 刷新会与数据库权威记录重新对齐。 + let modified = max(wallClock, directory.record.updated_at + 1) + let changed = max(wallClock, (directory.record.changed_at ?? + directory.record.updated_at) + 1) + directory.record = VexFSStatRecord( + path: directory.path, inode: directory.record.inode, + kind: directory.record.kind, mode: directory.record.mode, + size: directory.record.size, version: directory.record.version, + created_at: directory.record.created_at, + accessed_at: directory.record.accessed_at, + updated_at: modified, changed_at: changed, + uid: directory.record.uid, gid: directory.record.gid, + link_count: directory.record.link_count) + directory.recordValidatedAt = ProcessInfo.processInfo.systemUptime + directory.lock.unlock() + } + + func refreshRecord(_ item: VexFSItem, at path: String) throws { + let record = try backend.stat(path: path) + item.lock.lock() + item.path = path + item.record = record + item.recordValidatedAt = ProcessInfo.processInfo.systemUptime + item.cacheGeneration = observedCacheGeneration + item.lock.unlock() } func attributes(for item: VexFSItem, @@ -65,19 +180,27 @@ final class VexFSVolume: FSVolume, tv_nsec: Int((item.record.created_at % 1000) * 1_000_000)) let updated = timespec(tv_sec: Int(item.record.updated_at / 1000), tv_nsec: Int((item.record.updated_at % 1000) * 1_000_000)) + let accessedMilliseconds = item.record.accessed_at ?? item.record.updated_at + let changedMilliseconds = item.record.changed_at ?? item.record.updated_at + let accessed = timespec(tv_sec: Int(accessedMilliseconds / 1000), + tv_nsec: Int((accessedMilliseconds % 1000) * 1_000_000)) + let changed = timespec(tv_sec: Int(changedMilliseconds / 1000), + tv_nsec: Int((changedMilliseconds % 1000) * 1_000_000)) if wanted(.type) { attributes.type = item.itemType } if wanted(.mode) { attributes.mode = item.record.mode } - if wanted(.linkCount) { attributes.linkCount = item.itemType == .directory ? 2 : 1 } - if wanted(.uid) { attributes.uid = getuid() } - if wanted(.gid) { attributes.gid = getgid() } + if wanted(.linkCount) { + attributes.linkCount = item.record.link_count ?? (item.itemType == .directory ? 2 : 1) + } + if wanted(.uid) { attributes.uid = item.record.uid ?? getuid() } + if wanted(.gid) { attributes.gid = item.record.gid ?? getgid() } if wanted(.flags) { attributes.flags = 0 } if wanted(.size) { attributes.size = item.record.size } if wanted(.allocSize) { attributes.allocSize = (item.record.size + 4095) / 4096 * 4096 } if wanted(.fileID) { attributes.fileID = item.itemID } if wanted(.parentID) { attributes.parentID = item.parentID } - if wanted(.accessTime) { attributes.accessTime = updated } + if wanted(.accessTime) { attributes.accessTime = accessed } if wanted(.modifyTime) { attributes.modifyTime = updated } - if wanted(.changeTime) { attributes.changeTime = updated } + if wanted(.changeTime) { attributes.changeTime = changed } if wanted(.birthTime) { attributes.birthTime = created } return attributes } @@ -86,19 +209,44 @@ final class VexFSVolume: FSVolume, replyHandler: @escaping ((any Error)?) -> Void) { guard let item = item as? VexFSItem else { return replyHandler(POSIXError(.EINVAL)) } if item.itemType == .directory { return replyHandler(nil) } + guard item.itemType == .file else { return replyHandler(POSIXError(.EINVAL)) } + let generation: UInt64 + do { generation = try currentCacheGeneration() } + catch { return replyHandler(error) } item.lock.lock() defer { item.lock.unlock() } do { let writable = modes.contains(.write) + if writable { + clearReadCache(item) + } + if item.cacheGeneration != generation && item.dirtyGeneration == nil { + clearReadCache(item) + let path = try currentPath(for: item) + item.record = try backend.stat(path: path) + item.recordValidatedAt = ProcessInfo.processInfo.systemUptime + item.cacheGeneration = generation + } if let oldHandle = item.handle, writable && !item.handleWritable { try backend.closeHandle(oldHandle, retain: true) item.handle = nil item.dirtyGeneration = nil } if item.handle == nil { - item.handle = try backend.openHandle(path: currentPath(for: item), - flags: writable ? "rw" : "r") - item.handleWritable = writable + let path = try currentPath(for: item) + if !writable && item.record.size <= maximumReadCacheBytes { + let data = try backend.readFile(path: path) + if installReadCache(data, for: item) { + item.handleWritable = false + } else { + item.handle = try backend.openHandle(path: path, flags: "r") + item.handleWritable = false + } + } else { + item.handle = try backend.openHandle(path: path, + flags: writable ? "rw" : "r") + item.handleWritable = writable + } } replyHandler(nil) } catch { replyHandler(error) } @@ -113,12 +261,29 @@ final class VexFSVolume: FSVolume, do { if let handle = item.handle { if let generation = item.dirtyGeneration { - _ = try backend.publish(handle: handle, generation: generation) - item.record = try backend.stat(path: currentPath(for: item)) + let version = try backend.publishAndClose(handle: handle, + generation: generation) + item.handle = nil + item.dirtyGeneration = nil + if item.isUnlinked { + // unlink 后目录项已经不存在，但打开的 vnode 仍必须能正常 + // publish 和 close。此时不能再通过路径刷新属性。 + item.record = VexFSStatRecord( + path: item.path, inode: item.record.inode, + kind: item.record.kind, mode: item.record.mode, + size: item.record.size, version: UInt64(version), + created_at: item.record.created_at, + updated_at: Int64(Date().timeIntervalSince1970 * 1000), + uid: item.record.uid, gid: item.record.gid, link_count: 0) + } else { + item.record = try backend.stat(path: currentPath(for: item)) + } + } else { + try backend.closeHandle(handle, retain: true) } - try backend.closeHandle(handle, retain: true) } item.handle = nil + clearReadCache(item) item.handleWritable = false item.dirtyGeneration = nil replyHandler(nil) @@ -128,14 +293,18 @@ final class VexFSVolume: FSVolume, func read(from item: FSItem, at offset: off_t, length: Int, into buffer: FSMutableFileDataBuffer, replyHandler: @escaping (Int, (any Error)?) -> Void) { - guard let item = item as? VexFSItem, offset >= 0, length >= 0 else { + guard let item = item as? VexFSItem, item.itemType == .file, + offset >= 0, length >= 0 else { return replyHandler(0, POSIXError(.EINVAL)) } item.lock.lock() defer { item.lock.unlock() } do { let data: Data - if let handle = item.handle { + if let cached = item.readCache, item.handle == nil { + let start = min(Int(offset), cached.count) + data = cached.subdata(in: start..&2; exit 1; } BUILD_DIR="$SRCROOT/../../vexdb_sqlite/build-fskit" + BUILD_JOBS="${VEXDB_LITE_BUILD_JOBS:-4}" + case "$BUILD_JOBS" in *[!0-9]*|0) echo "VEXDB_LITE_BUILD_JOBS must be a positive integer" >&2; exit 2;; esac + [ "$BUILD_JOBS" -le 4 ] || BUILD_JOBS=4 "$CMAKE_BIN" -S "$SRCROOT/../../vexdb_sqlite" -B "$BUILD_DIR" \ -DCMAKE_BUILD_TYPE="$CONFIGURATION" -DVEXDB_SQLITE_BUILD_TESTS=OFF - "$CMAKE_BIN" --build "$BUILD_DIR" --target vexfs_mount_contract -j "$(sysctl -n hw.ncpu)" + "$CMAKE_BIN" --build "$BUILD_DIR" --target vexfs_runtime -j "$BUILD_JOBS" SH extension.build_phases.move(build_core, 0) @@ -77,14 +80,15 @@ 'CODE_SIGN_ENTITLEMENTS' => 'VexFSApp/VexFSApp.entitlements', 'CODE_SIGN_STYLE' => 'Automatic', 'CURRENT_PROJECT_VERSION' => '1', - 'DEVELOPMENT_TEAM' => '', + 'DEVELOPMENT_TEAM' => 'BB5VK42K87', 'ENABLE_APP_SANDBOX' => 'YES', + 'ENABLE_HARDENED_RUNTIME' => 'YES', 'GENERATE_INFOPLIST_FILE' => 'YES', - 'INFOPLIST_KEY_CFBundleDisplayName' => 'VexFS', + 'INFOPLIST_KEY_CFBundleDisplayName' => 'VexDB Lite', 'LD_RUNPATH_SEARCH_PATHS' => '$(inherited) @executable_path/../Frameworks', 'MARKETING_VERSION' => '0.1.0', 'PRODUCT_BUNDLE_IDENTIFIER' => 'io.vexdb.vexfs', - 'PRODUCT_NAME' => 'VexFS' + 'PRODUCT_NAME' => 'VexDB Lite' ) end @@ -94,8 +98,9 @@ 'CODE_SIGN_ENTITLEMENTS' => 'VexFSAppEx/VexFSAppEx.entitlements', 'CODE_SIGN_STYLE' => 'Automatic', 'CURRENT_PROJECT_VERSION' => '1', - 'DEVELOPMENT_TEAM' => '', + 'DEVELOPMENT_TEAM' => 'BB5VK42K87', 'ENABLE_APP_SANDBOX' => 'YES', + 'ENABLE_HARDENED_RUNTIME' => 'YES', 'ENABLE_USER_SCRIPT_SANDBOXING' => 'NO', 'GENERATE_INFOPLIST_FILE' => 'YES', 'HEADER_SEARCH_PATHS' => '$(inherited) "$(SRCROOT)/../mount/common/include"', @@ -105,7 +110,7 @@ 'MARKETING_VERSION' => '0.1.0', 'OTHER_LDFLAGS' => [ '$(inherited)', - '"$(SRCROOT)/../../vexdb_sqlite/build-fskit/libvexfs_mount_contract.a"', + '"$(SRCROOT)/../../vexdb_sqlite/build-fskit/libvexfs_runtime.a"', '"$(SRCROOT)/../../vexdb_sqlite/build-fskit/libvexdb_lite_static.a"', '-lsqlite3', '-lc++' diff --git a/agent_files/macos/package/Install.command b/agent_files/macos/package/Install.command index bae46cc4cd..b4ac978811 100755 --- a/agent_files/macos/package/Install.command +++ b/agent_files/macos/package/Install.command @@ -3,4 +3,4 @@ set -euo pipefail ROOT="$(cd "$(dirname "$0")" && pwd)" /bin/bash "$ROOT/install.sh" echo "" -read -r -p "安装完成，按回车关闭窗口。" _ +read -r -p "VexDB-Lite 安装完成，按回车关闭窗口。" _ diff --git a/agent_files/macos/package/README.md b/agent_files/macos/package/README.md index 0807bffa58..3949679d13 100644 --- a/agent_files/macos/package/README.md +++ b/agent_files/macos/package/README.md @@ -1,25 +1,21 @@ -# VexFS macOS 技术预览 +# VexDB-Lite macOS 技术预览 -VexFS 是由 SQLite 管理的文件工作区。本包提供： +完整的安装、SQLite、向量、文件版本、挂载、备份和卸载说明见同目录 +[`使用说明.md`](使用说明.md)。 -- `VexFS.app`：包含 FSKit 文件系统扩展； -- `bin/vexfs`：命令行工具； -- `lib/vexdb_lite.dylib`：可由 SQLite 加载的扩展； -- `install.sh`、`Install.command` 和 `uninstall.sh`。 +VexDB-Lite 在一个安装包中提供： + +- `VexDB Lite.app`：包含 VexFS FSKit 文件系统扩展； +- `bin/vexdb`：内置 SQLite、向量检索和文件 SQL 能力的统一命令； +- `bin/vexfs`：`vexdb fs` 的兼容快捷入口； +- `lib/vexdb_lite.dylib`：供现有 SQLite 程序加载的扩展； +- 安装和卸载脚本。 ## 系统要求 - macOS 26.0 或更高版本； - Apple Silicon 包要求 arm64 Mac； -- 首次尝试真实挂载时，需要在系统设置中启用 VexFS 文件系统扩展。 - -## 重要限制 - -这是 **ad-hoc 签名的技术预览包**，没有 Developer ID 签名和 Apple 公证。 - -- CLI 和 SQLite 文件管理功能可以直接测试； -- 从聊天、邮件或浏览器下载后，macOS 可能阻止打开 App。请在 Finder 中右键 `VexFS.app`，选择“打开”； -- FSKit extension 是否允许在另一台 Mac 上启用，取决于该机器的系统安全设置。正式对外分发必须使用 Apple Developer 证书签名并公证。 +- 只有真实挂载需要 macOS 允许 `VexDB Lite` 文件系统扩展；系统显示的是宿主 App 名，内部文件系统名才是 VexFS。 ## 安装 @@ -29,11 +25,19 @@ VexFS 是由 SQLite 管理的文件工作区。本包提供： ./install.sh ``` +正式包必须是 `signature=developer-id`。本地构建的 `signature=ad-hoc` 包默认不能安装； +仅在开发测试时显式运行 `VEXDB_LITE_ALLOW_ADHOC_INSTALL=1 ./install.sh`。 + 默认安装位置： -- App：`~/Applications/VexFS.app` -- CLI：`~/.local/bin/vexfs` -- SQLite 扩展：`~/.local/lib/vexfs/vexdb_lite.dylib` +- App：`~/Applications/VexDB Lite.app` +- CLI：`~/.local/bin/vexdb` +- 文件快捷命令：`~/.local/bin/vexfs` +- SQLite 扩展：`~/.local/lib/vexdb-lite/vexdb_lite.dylib` +- 默认数据库：`~/Library/Application Support/VexDB-Lite/default.sqlite3` + +安装程序会在后台启动一次 VexDB Lite App，让 macOS 立即发现新安装或刚替换的 FSKit 模块。 +它不会替用户打开扩展开关；首次安装仍需在系统设置中允许。 如果 `~/.local/bin` 不在 PATH： @@ -42,39 +46,75 @@ echo 'export PATH="$HOME/.local/bin:$PATH"' >> ~/.zshrc source ~/.zshrc ``` -## 不挂载也能使用 +## SQLite 和向量 + +`vexdb` 自带经过验证的 SQLite，不需要手动 `.load`： ```bash -vexfs setup -printf 'hello VexFS\n' | vexfs write /hello.txt -vexfs cat /hello.txt -vexfs ls / -vexfs history /hello.txt -vexfs doctor --json +vexdb ~/data/agent.db +vexdb ~/data/agent.db "SELECT vexdb_version();" +vexdb ~/data/agent.db "SELECT vexdb_l2_distance('[1,2]', '[4,6]');" +``` + +## 文件管理 + +不挂载也能使用： + +```bash +vexdb fs setup +printf 'hello VexFS\n' | vexdb fs write /hello.txt +vexdb fs cat /hello.txt +vexdb fs ls / +vexdb fs grep -n hello / +vexdb fs index enable # 可选的 FTS5 trigram 文本索引 +vexdb fs history /hello.txt +vexdb fs snapshot create before-agent +vexdb fs snapshot diff before-agent +vexdb fs snapshot restore before-agent --dry-run +vexdb fs snapshot restore before-agent +vexdb fs check # 深度检查结构、历史和内容 SHA-256 +vexdb fs check --quick # 只检查结构和引用 ``` -默认数据库位于： +已挂载 workspace 会在恢复时自动安全卸载并挂回原目录；正常卸载失败时不会开始恢复。 +`--force-unmount` 只用于用户明确接受中断打开文件的场景。 + +旧命令继续可用： -```text -~/Library/Application Support/VexFS/vexfs.sqlite3 +```bash +vexfs ls / ``` -## 尝试真实 Bash 挂载 +## 真实 Bash 挂载 -1. 打开 `~/Applications/VexFS.app`； -2. 进入“系统设置 → 通用 → 登录项与扩展 → 文件系统扩展”； -3. 启用 VexFS； -4. 运行： +1. 打开 `~/Applications/VexDB Lite.app`； +2. 进入“系统设置 → 通用 → 登录项与扩展”，滚动到扩展区域； +3. 切换到“按类别”视图； +4. 在“文件系统扩展”这一行点右侧 ⓘ，找到 `VexDB Lite` 并打开它的开关； +5. 如果看到的是 `VexDB Lite / FSKit Modules` 只读详情，说明仍在“按 App”视图，返回上一层切换视图； +6. 运行： ```bash -vexfs doctor -vexfs mount ~/VexFS -cd ~/VexFS +vexdb fs doctor +vexdb fs mount ~/VexDB +cd ~/VexDB ls cat hello.txt ``` -如果 `doctor` 显示扩展未启用，仍可继续使用上面的直接 CLI 命令。 +## 签名状态 + +打开 `MANIFEST.txt` 查看： + +- `signature=developer-id`：App、extension、CLI 和 dylib 使用 Developer ID 签名； +- `notarization=accepted`：包已通过 Apple 公证； +- `signature=ad-hoc`：仅供本机开发测试。 + +默认拒绝安装任何脏源码 Developer ID 包。只有真机开发验证时才可显式运行 +`VEXDB_LITE_ALLOW_DIRTY_INSTALL=1 ./install.sh`；正式发布和普通安装禁止使用这个开关。 + +FSKit 首次使用需要 macOS 允许该文件系统扩展。安装程序不能静默启用； +请在“按类别”视图的“文件系统扩展”中完成这一步。 ## 卸载 @@ -82,8 +122,4 @@ cat hello.txt ./uninstall.sh ``` -卸载脚本不会删除数据库。确认不再需要数据后，可自行删除： - -```text -~/Library/Application Support/VexFS/ -``` +卸载不会删除默认数据库。 diff --git a/agent_files/macos/package/install.sh b/agent_files/macos/package/install.sh index 543c328895..aadee6251c 100755 --- a/agent_files/macos/package/install.sh +++ b/agent_files/macos/package/install.sh @@ -2,22 +2,194 @@ set -euo pipefail ROOT="$(cd "$(dirname "$0")" && pwd)" -APP_DIR="${VEXFS_APP_DIR:-$HOME/Applications}" -BIN_DIR="${VEXFS_BIN_DIR:-$HOME/.local/bin}" -LIB_DIR="${VEXFS_LIB_DIR:-$HOME/.local/lib/vexfs}" +APP_DIR_OVERRIDE="${VEXDB_LITE_APP_DIR:-${VEXFS_APP_DIR:-}}" +APP_DIR="${APP_DIR_OVERRIDE:-$HOME/Applications}" +BIN_DIR="${VEXDB_LITE_BIN_DIR:-${VEXFS_BIN_DIR:-$HOME/.local/bin}}" +LIB_DIR="${VEXDB_LITE_LIB_DIR:-${VEXFS_LIB_DIR:-$HOME/.local/lib/vexdb-lite}}" +LSREGISTER="/System/Library/Frameworks/CoreServices.framework/Frameworks/LaunchServices.framework/Support/lsregister" +OFFICIAL_TEAM_ID="BB5VK42K87" -[ -d "$ROOT/VexFS.app" ] || { echo "安装包缺少 VexFS.app" >&2; exit 1; } +[ -d "$ROOT/VexDB Lite.app" ] || { echo "安装包缺少 VexDB Lite.app" >&2; exit 1; } +[ -x "$ROOT/bin/vexdb" ] || { echo "安装包缺少 bin/vexdb" >&2; exit 1; } [ -x "$ROOT/bin/vexfs" ] || { echo "安装包缺少 bin/vexfs" >&2; exit 1; } [ -f "$ROOT/lib/vexdb_lite.dylib" ] || { echo "安装包缺少 SQLite 扩展" >&2; exit 1; } +[ -f "$ROOT/MANIFEST.txt" ] || { echo "安装包缺少 MANIFEST.txt" >&2; exit 1; } +[ -f "$ROOT/SHA256SUMS.txt" ] || { echo "安装包缺少 SHA256SUMS.txt" >&2; exit 1; } + +(cd "$ROOT" && /usr/bin/shasum -a 256 -c SHA256SUMS.txt >/dev/null) || { + echo "安装包文件哈希校验失败" >&2 + exit 1 +} + +PACKAGE_SIGNATURE="$(sed -n 's/^signature=//p' "$ROOT/MANIFEST.txt")" +EXPECTED_TEAM="$(sed -n 's/^signing_team=//p' "$ROOT/MANIFEST.txt")" +EXPECTED_CONTRACT="$(sed -n 's/^contract_version=//p' "$ROOT/MANIFEST.txt")" +EXPECTED_RUNTIME_ABI="$(sed -n 's/^mount_abi_version=//p' "$ROOT/MANIFEST.txt")" +SOURCE_DIRTY="$(sed -n 's/^source_dirty=//p' "$ROOT/MANIFEST.txt")" +case "$PACKAGE_SIGNATURE" in + developer-id) + [ "$EXPECTED_TEAM" = "$OFFICIAL_TEAM_ID" ] || { + echo "Developer ID 包发布者无效：${EXPECTED_TEAM:-missing}" >&2 + exit 1 + } + if [ "$SOURCE_DIRTY" != false ]; then + [ "${VEXDB_LITE_ALLOW_DIRTY_INSTALL:-0}" = 1 ] || { + echo "拒绝安装来自脏工作树的 Developer ID 包" >&2 + exit 1 + } + echo "警告：正在安装脏源码测试包；此开关不得用于正式发布。" >&2 + fi + ;; + ad-hoc) + [ "${VEXDB_LITE_ALLOW_ADHOC_INSTALL:-0}" = 1 ] || { + echo "ad-hoc 包仅供本地测试；如确需安装，请设置 VEXDB_LITE_ALLOW_ADHOC_INSTALL=1" >&2 + exit 1 + } + ;; + *) echo "安装包签名类型无效：${PACKAGE_SIGNATURE:-missing}" >&2; exit 1 ;; +esac + +ACTUAL_RUNTIME_ABI="$("$ROOT/bin/vexdb" fs --json doctor 2>/dev/null \ + | sed -n 's/.*"runtime_abi":\([0-9][0-9]*\).*/\1/p' || true)" +[ -n "$ACTUAL_RUNTIME_ABI" ] && [ "$ACTUAL_RUNTIME_ABI" = "$EXPECTED_RUNTIME_ABI" ] || { + echo "安装包 runtime ABI 不一致：实际 ${ACTUAL_RUNTIME_ABI:-missing}，清单 ${EXPECTED_RUNTIME_ABI:-missing}" >&2 + exit 1 +} +ACTUAL_CONTRACT="$("$ROOT/bin/vexdb" :memory: "SELECT vexfs_contract_version();" 2>/dev/null || true)" +[ -n "$ACTUAL_CONTRACT" ] && [ "$ACTUAL_CONTRACT" = "$EXPECTED_CONTRACT" ] || { + echo "安装包文件合同不一致：实际 ${ACTUAL_CONTRACT:-missing}，清单 ${EXPECTED_CONTRACT:-missing}" >&2 + exit 1 +} + +verify_component() { + local item="$1" + shift + /usr/bin/codesign --verify --strict "$@" "$item" || { + echo "安装包签名校验失败：$item" >&2 + exit 1 + } + if [ "$PACKAGE_SIGNATURE" = developer-id ]; then + local actual + actual="$(/usr/bin/codesign -d --verbose=4 "$item" 2>&1 | sed -n 's/^TeamIdentifier=//p')" + [ "$actual" = "$EXPECTED_TEAM" ] || { + echo "安装包发布者错误：$item（实际 ${actual:-none}，期望 $EXPECTED_TEAM）" >&2 + exit 1 + } + fi +} + +verify_component "$ROOT/VexDB Lite.app" --deep +verify_component "$ROOT/VexDB Lite.app/Contents/Extensions/VexFSAppEx.appex" +verify_component "$ROOT/bin/vexdb" +verify_component "$ROOT/lib/vexdb_lite.dylib" mkdir -p "$APP_DIR" "$BIN_DIR" "$LIB_DIR" -ditto "$ROOT/VexFS.app" "$APP_DIR/VexFS.app" -install -m 0755 "$ROOT/bin/vexfs" "$BIN_DIR/vexfs" + +# 旧 VexFS.app 与新 App 沿用同一个 extension ID。两者同时注册时 macOS 可能继续 +# 调用旧扩展，表现为 mount "Loading resource: Input/output error"。正常安装时先 +# 撤销旧注册，并在权限允许时把旧 App 改成不会被 LaunchServices 扫描的备份。 +# 自定义 APP_DIR 用于测试或集成，不触碰系统中已有的 App 注册。 +LEGACY_FOUND=false +LEGACY_INDEX=0 +if [ -z "$APP_DIR_OVERRIDE" ]; then + for LEGACY_APP in "$HOME/Applications/VexFS.app" "/Applications/VexFS.app"; do + [ -d "$LEGACY_APP" ] || continue + LEGACY_FOUND=true + LEGACY_INDEX=$((LEGACY_INDEX + 1)) + if [ -x "$LSREGISTER" ]; then + "$LSREGISTER" -u "$LEGACY_APP" >/dev/null 2>&1 || true + fi + LEGACY_BACKUP="$APP_DIR/VexFS Legacy $(date +%Y%m%d-%H%M%S)-$LEGACY_INDEX.app.disabled" + if mv "$LEGACY_APP" "$LEGACY_BACKUP" 2>/dev/null; then + echo "旧 VexFS App 已停用并保留：$LEGACY_BACKUP" + else + echo "警告：无法移动旧 App，请手动删除：$LEGACY_APP" >&2 + fi + done +fi + +INSTALL_ID="$(date +%Y%m%d-%H%M%S)-$$" +INSTALLED_APP="$APP_DIR/VexDB Lite.app" +STAGED_APP="$APP_DIR/.VexDB Lite $INSTALL_ID.app" +PREVIOUS_APP="" + +# ditto 会合并已有目录，不能直接覆盖已安装 App。旧 bundle 中多出的文件（例如旧 +# embedded.provisionprofile）会留在新签名封装内，导致 codesign 立即失效。先复制到 +# 新目录并验签，再整体替换；失败时保留旧 App。 +ditto "$ROOT/VexDB Lite.app" "$STAGED_APP" +if ! /usr/bin/codesign --verify --deep --strict "$STAGED_APP"; then + FAILED_APP="$APP_DIR/VexDB Lite Failed $INSTALL_ID.app.disabled" + mv "$STAGED_APP" "$FAILED_APP" 2>/dev/null || true + echo "安装包 App 签名校验失败，未替换当前版本：$FAILED_APP" >&2 + exit 1 +fi +if [ -d "$INSTALLED_APP" ]; then + if [ -x "$LSREGISTER" ] && [ -z "$APP_DIR_OVERRIDE" ]; then + "$LSREGISTER" -u "$INSTALLED_APP" >/dev/null 2>&1 || true + fi + PREVIOUS_APP="$APP_DIR/VexDB Lite Previous $INSTALL_ID.app.disabled" + mv "$INSTALLED_APP" "$PREVIOUS_APP" +fi +if ! mv "$STAGED_APP" "$INSTALLED_APP"; then + [ -n "$PREVIOUS_APP" ] && [ -d "$PREVIOUS_APP" ] && \ + mv "$PREVIOUS_APP" "$INSTALLED_APP" 2>/dev/null || true + echo "无法替换 VexDB Lite.app，已经尝试恢复旧版本" >&2 + exit 1 +fi +install -m 0755 "$ROOT/bin/vexdb" "$BIN_DIR/vexdb" +ln -sfn vexdb "$BIN_DIR/vexfs" install -m 0644 "$ROOT/lib/vexdb_lite.dylib" "$LIB_DIR/vexdb_lite.dylib" -echo "VexFS 已安装：" -echo " App: $APP_DIR/VexFS.app" -echo " CLI: $BIN_DIR/vexfs" +if [ -z "$APP_DIR_OVERRIDE" ]; then + INSTALLED_EXTENSION="$INSTALLED_APP/Contents/Extensions/VexFSAppEx.appex" + if [ -x "$LSREGISTER" ]; then + "$LSREGISTER" -f "$INSTALLED_APP" >/dev/null 2>&1 || true + fi + if command -v pluginkit >/dev/null; then + pluginkit -a "$INSTALLED_EXTENSION" >/dev/null 2>&1 || true + fi + # ExtensionKit may not publish a replaced FSKit module until its containing + # App has been launched once. Launch it hidden so doctor does not keep + # reporting `extension: missing` after a valid upgrade. + /usr/bin/open -gj "$INSTALLED_APP" >/dev/null 2>&1 || true + MOUNTS="$(/sbin/mount 2>/dev/null | tr '[:upper:]' '[:lower:]')" + case "$MOUNTS" in + *"(vexfs,"*|*"(exfat,"*|*"(msdos,"*|*" type vexfs"*|*" type exfat"*|*" type msdos"*) ;; + *) pkill -KILL -x fskit_agent >/dev/null 2>&1 || true ;; + esac + + # enabled 只表示同 bundle ID 的扩展已经获准使用，不保证 FSKit 当前解析到 + # 新安装的 App。用签名 CLI 读取 FSKit 的真实 module URL，避免安装成功但 + # mount 实际卡在 Xcode archive 或旧副本上。 + EXTENSION_CHECK="" + for ATTEMPT in 1 2 3 4 5; do + EXTENSION_CHECK="$(VEXFS_EXPECTED_EXTENSION_PATH="$INSTALLED_EXTENSION" \ + "$BIN_DIR/vexdb" fs --json doctor 2>/dev/null || true)" + case "$EXTENSION_CHECK" in + *'"extension_path_matches":true'*) break ;; + esac + "$LSREGISTER" -f "$INSTALLED_APP" >/dev/null 2>&1 || true + pluginkit -a "$INSTALLED_EXTENSION" >/dev/null 2>&1 || true + sleep 1 + done + case "$EXTENSION_CHECK" in + *'"extension_path_matches":true'*) ;; + *'"extension_path":""'*) + echo "警告：系统尚未返回 FSKit module 路径；启用扩展后请运行 vexdb fs doctor。" >&2 + ;; + *) + echo "FSKit 当前没有加载新安装的扩展，安装未完成：" >&2 + echo "$EXTENSION_CHECK" >&2 + echo "期望路径：$INSTALLED_EXTENSION" >&2 + exit 1 + ;; + esac +fi + +echo "VexDB-Lite 已安装：" +echo " App: $APP_DIR/VexDB Lite.app" +echo " CLI: $BIN_DIR/vexdb" +echo " 文件快捷命令: $BIN_DIR/vexfs" echo " SQLite: $LIB_DIR/vexdb_lite.dylib" case ":$PATH:" in *":$BIN_DIR:"*) ;; @@ -28,4 +200,4 @@ case ":$PATH:" in ;; esac echo "" -echo "下一步：$BIN_DIR/vexfs setup && $BIN_DIR/vexfs doctor" +echo "下一步：$BIN_DIR/vexdb --version && $BIN_DIR/vexdb fs setup" diff --git a/agent_files/macos/package/uninstall.sh b/agent_files/macos/package/uninstall.sh index 63eef9bca7..2e0e03361b 100755 --- a/agent_files/macos/package/uninstall.sh +++ b/agent_files/macos/package/uninstall.sh @@ -1,13 +1,14 @@ #!/bin/bash set -euo pipefail -APP_DIR="${VEXFS_APP_DIR:-$HOME/Applications}" -BIN_DIR="${VEXFS_BIN_DIR:-$HOME/.local/bin}" -LIB_DIR="${VEXFS_LIB_DIR:-$HOME/.local/lib/vexfs}" +APP_DIR="${VEXDB_LITE_APP_DIR:-${VEXFS_APP_DIR:-$HOME/Applications}}" +BIN_DIR="${VEXDB_LITE_BIN_DIR:-${VEXFS_BIN_DIR:-$HOME/.local/bin}}" +LIB_DIR="${VEXDB_LITE_LIB_DIR:-${VEXFS_LIB_DIR:-$HOME/.local/lib/vexdb-lite}}" -rm -rf "$APP_DIR/VexFS.app" -rm -f "$BIN_DIR/vexfs" "$LIB_DIR/vexdb_lite.dylib" +rm -rf "$APP_DIR/VexDB Lite.app" +rm -f "$BIN_DIR/vexdb" "$BIN_DIR/vexfs" "$LIB_DIR/vexdb_lite.dylib" rmdir "$LIB_DIR" 2>/dev/null || true -echo "VexFS App、CLI 和 SQLite 扩展已卸载。" -echo "数据库未删除：$HOME/Library/Application Support/VexFS/" +echo "VexDB-Lite App、CLI 和 SQLite 扩展已卸载。" +echo "数据库未删除：" +echo " $HOME/Library/Application Support/VexDB-Lite/" diff --git "a/agent_files/macos/package/\344\275\277\347\224\250\350\257\264\346\230\216.md" "b/agent_files/macos/package/\344\275\277\347\224\250\350\257\264\346\230\216.md" new file mode 100644 index 0000000000..8207c5b00e --- /dev/null +++ "b/agent_files/macos/package/\344\275\277\347\224\250\350\257\264\346\230\216.md" @@ -0,0 +1,400 @@ +# VexDB-Lite macOS 使用说明 + +适用版本：`0.1.0-preview` 系列；准确版本见同目录 `MANIFEST.txt`。 +适用平台：Apple Silicon Mac、macOS 26.0 或更高版本 + +VexDB-Lite 把三种能力放进同一个 SQLite 数据库： + +- SQLite 表和 SQL； +- 向量距离计算与向量索引； +- 由数据库管理、带版本记录的 VexFS 文件。 + +主命令是 `vexdb`。`vexfs` 是文件命令的兼容快捷入口。 + +## 1. 安装 + +解压后双击 `Install.command`，或者在终端进入解压目录运行： + +```bash +./install.sh +``` + +正式下载包应在 `MANIFEST.txt` 中显示 `signature=developer-id`，可直接安装。 +`signature=ad-hoc` 只用于源码目录里的本机开发测试，安装器默认拒绝；开发者确认后使用： + +```bash +VEXDB_LITE_ALLOW_ADHOC_INSTALL=1 ./install.sh +``` + +默认安装位置： + +- App：`~/Applications/VexDB Lite.app` +- 主命令：`~/.local/bin/vexdb` +- 兼容命令：`~/.local/bin/vexfs` +- SQLite 扩展：`~/.local/lib/vexdb-lite/vexdb_lite.dylib` +- 默认数据库：`~/Library/Application Support/VexDB-Lite/default.sqlite3` + +安装程序会在后台启动一次 VexDB Lite App，让 macOS 立即发现新安装或刚替换的 FSKit 模块。 +首次安装仍需由用户在系统设置中打开文件系统扩展开关。 + +如果终端提示 `command not found: vexdb`，运行： + +```bash +echo 'export PATH="$HOME/.local/bin:$PATH"' >> ~/.zshrc +source ~/.zshrc +``` + +检查安装： + +```bash +vexdb --version +vexdb fs setup +vexdb fs doctor +``` + +Developer ID 签名和 Apple 公证状态以同目录 `MANIFEST.txt` 为准。正式分发包应同时显示 +`signature=developer-id` 和 `notarization=accepted`，macOS Gatekeeper 可以直接验证。 +如果状态不是这两个值，请把它当作本机开发包，不要转发给其他用户，也不要关闭 +Gatekeeper 绕过检查。 + +## 2. 使用 SQLite + +打开或新建数据库： + +```bash +vexdb ~/data/agent.db +``` + +进入交互界面后可以直接使用 SQLite 命令： + +```sql +.tables +CREATE TABLE notes(id INTEGER PRIMARY KEY, content TEXT); +INSERT INTO notes(content) VALUES ('hello'); +SELECT * FROM notes; +.quit +``` + +也可以直接执行一条 SQL： + +```bash +vexdb ~/data/agent.db "SELECT sqlite_version(), vexdb_version();" +``` + +`vexdb` 已经自动加载向量和 VexFS 能力，不需要手动执行 `.load`。 + +## 3. 使用向量能力 + +计算两个向量的 L2 距离： + +```bash +vexdb ~/data/agent.db \ + "SELECT vexdb_l2_distance('[1,2]', '[4,6]');" +``` + +创建向量索引并查询最近邻： + +```bash +vexdb ~/data/agent.db \ + "CREATE VIRTUAL TABLE vectors USING GRAPH_INDEX(embedding FLOAT[2], metric=l2);" + +vexdb ~/data/agent.db \ + "INSERT INTO vectors(rowid, embedding) VALUES (1, '[1,1]'), (2, '[9,9]');" + +vexdb ~/data/agent.db \ + "SELECT rowid FROM vectors WHERE embedding MATCH '[1,1]' AND k=1;" +``` + +## 4. 不挂载也能管理文件 + +初始化默认文件空间： + +```bash +vexdb fs setup +``` + +常用操作： + +```bash +vexdb fs mkdir /docs +printf 'hello VexFS\n' | vexdb fs write /docs/hello.txt +vexdb fs write /docs/photo.png ~/Pictures/photo.png +vexdb fs ls /docs +vexdb fs cat /docs/hello.txt +vexdb fs stat /docs/hello.txt +vexdb fs mv /docs/hello.txt /docs/readme.txt +vexdb fs ln /docs/readme.txt /docs/readme-copy.txt +vexdb fs chown 501:20 /docs/readme.txt +vexdb fs getfacl /docs/readme.txt +vexdb fs rm /docs/readme.txt +``` + +操作指定数据库和文件空间： + +```bash +vexdb fs --db ~/data/agent.db --workspace project-a setup +vexdb fs --db ~/data/agent.db --workspace project-a ls / +``` + +兼容命令也可以继续使用： + +```bash +vexfs ls / +vexfs cat /docs/hello.txt +``` + +### 4.1 数据库内搜索和可选文本索引 + +不挂载也可以搜索当前文件内容： + +```bash +vexdb fs grep -n keyword /docs +vexdb fs grep -i -l keyword /docs +vexdb fs --json grep keyword /docs +``` + +当前是 UTF-8 字面匹配，不是正则或语义搜索；二进制文件会跳过。`-i` 对 ASCII 字母 +忽略大小写，`-l` 只输出文件名，`-n` 输出行号。 + +默认搜索一次连接内批量扫描文件，不增加日常写入成本。大工作区可以显式开启数据库级 +trigram 文本索引： + +```bash +vexdb fs index status +vexdb fs index enable +vexdb fs grep -n keyword /docs +vexdb fs index rebuild +vexdb fs index disable +``` + +索引对数据库中的全部 workspace 生效。少于 3 个字符的查询会自动退回批量扫描。开启 +索引后，文件写入、单文件版本恢复和完整工作区恢复会同步更新索引。宿主 SQLite 没有 +FTS5 时仍能扫描搜索，不会影响基本文件能力。 + +## 5. 文件版本 + +每次写入和恢复都会生成新版本： + +```bash +vexdb fs history /docs/plan.md +vexdb fs show /docs/plan.md --version 1 +vexdb fs diff /docs/plan.md --from 1 +vexdb fs restore /docs/plan.md --version 1 --dry-run +vexdb fs restore /docs/plan.md --version 1 +``` + +恢复不会覆盖历史记录，而是写入一个新版本。 + +## 6. 完整工作区快照 + +单文件 `history` 适合恢复一个文件；workspace snapshot 会记录整个目录树的时间点，包括 +文件内容、目录移动和删除、可执行权限、符号链接以及扩展属性： + +```bash +vexdb fs snapshot create before-agent +vexdb fs snapshot list +vexdb fs snapshot show before-agent +``` + +Agent 工作后查看完整差异： + +```bash +vexdb fs snapshot diff before-agent +``` + +存在差异时 `snapshot diff` 返回退出码 1，没有差异时返回 0，方便脚本判断。先预览再一键 +还原： + +```bash +vexdb fs snapshot restore before-agent --dry-run +vexdb fs snapshot restore before-agent +``` + +workspace restore 会生成一个新 commit，不会删除或重写旧历史。为了防止静默覆盖，它会 +检查当前 workspace head；并发修改后旧 restore 会返回冲突。存在打开或崩溃保留的文件 +句柄时也会拒绝还原。如果 workspace 当前已挂载，命令会先正常卸载，完成数据库恢复后再挂回 +原来的目录，避免应用继续看到恢复前的内核缓存。若目录正被进程占用、正常卸载失败，恢复不会 +开始。只有确认可以中断所有打开文件时，才显式使用： + +```bash +vexdb fs snapshot restore before-agent --force-unmount +``` + +确认不再需要某个快照后删除名称： + +```bash +vexdb fs snapshot drop before-agent +``` + +删除快照名称不会删除 workspace 历史。当前技术预览还没有自动历史清理和 GC。快照用于 +任务检查点，不是独立备份；机器或数据库损坏的恢复仍应使用第 8 节的 SQLite Backup。 + +### 检查工作区 + +```bash +vexdb fs check +vexdb fs check --quick +vexdb fs --json check +``` + +默认命令只读检查目录、版本、提交、快照、暂存区、BLOB 长度和内容 SHA-256。`--quick` +跳过内容哈希，适合频繁检查结构；它不能发现同长度内容被改写。发现损坏时命令仍输出完整 +报告并返回退出码 8，不会自动修复或用空内容补文件。先保存数据库副本，再按报告处理。 + +## 7. 挂载后用普通 Bash 命令 + +只有这一步需要启用 FSKit 文件系统扩展。SQLite、向量和上一节的 `vexdb fs` +命令不依赖挂载。 + +第一次使用： + +1. 打开 `~/Applications/VexDB Lite.app`； +2. 打开“系统设置 → 通用 → 登录项与扩展”，滚动到扩展区域； +3. 切换到“按类别”视图； +4. 在“文件系统扩展”这一行点右侧 ⓘ，找到 `VexDB Lite` 并打开它的开关； +5. 如果看到的是 `VexDB Lite / FSKit Modules` 只读详情，说明仍在“按 App”视图，返回上一层切换视图； +6. 回到终端检查并挂载。 + +```bash +vexdb fs doctor +vexdb fs mount ~/VexDB +vexdb fs mount status +``` + +挂载成功后，可以直接使用模型和开发者熟悉的 Bash 命令： + +```bash +ls -la ~/VexDB +cat ~/VexDB/docs/plan.md +cp ./local.txt ~/VexDB/docs/ +grep -R "keyword" ~/VexDB +mv ~/VexDB/docs/local.txt ~/VexDB/docs/archive.txt +rm ~/VexDB/docs/archive.txt +``` + +常用开发操作也可以直接工作： + +```bash +chmod +x ~/VexDB/scripts/build.sh +ln -s scripts/build.sh ~/VexDB/build +ln ~/VexDB/scripts/build.sh ~/VexDB/scripts/build-copy.sh +chown 501:20 ~/VexDB/scripts/build.sh +getfacl ~/VexDB/scripts/build.sh +readlink ~/VexDB/build +git -C ~/VexDB init +``` + +可执行权限位、符号链接、硬链接、owner ID 和便携 ACL 都会保存在 SQLite 中，卸载并重新挂载后仍然存在。 +Git 可以正常使用 `core.filemode=true` 和 `core.symlinks=true`；普通文件可用 `ln` 建立硬链接， +`chown UID:GID` 修改 owner ID，`getfacl`/`setfacl` 读取和替换 ACL JSON。ACL 是由数据库管理的 +跨平台表示，适配层可以在目标平台支持时映射到原生安全描述符。setuid/setgid/sticky bit、FIFO、 +字符/块设备和 socket 等特殊文件仍明确不支持，不会在数据库里伪造。符号链接 target 按 POSIX 规则原样 +保存，可以指向挂载目录外；VexFS 本身不是安全沙箱。 + +`chown` 设置的数字 UID/GID 会原样保存在数据库和快照中。Linux 挂载会按数字 ID 显示；macOS +普通用户的 FSKit 挂载由系统强制使用 `noowners`，所以 Bash `stat` 会显示当前挂载用户。此时可用 +`vexdb fs stat PATH` 查看数据库保存的原始 UID/GID。这是显示映射，不会改写数据库里的 owner。 + +卸载目录： + +```bash +vexdb fs unmount ~/VexDB +``` + +如果普通卸载因为仍有终端或程序占用目录而失败，可以明确要求强制卸载： + +```bash +vexdb fs unmount --force ~/VexDB +``` + +强制卸载会让现有文件句柄失效，适合进程崩溃或无法正常退出时使用。正常情况下仍应先关闭 +正在访问挂载目录的程序，再执行普通卸载。 + +macOS 要求用户授权文件系统扩展，安装程序不能静默跳过这一步。 +授权开关在“按类别”视图的“文件系统扩展”详情中，不在 `VexDB Lite` +的“按 App”只读详情中。 + +## 8. 备份和迁移 + +使用 SQLite Backup API 在线复制整个数据库，包括普通表、向量索引和 VexFS 文件： + +```bash +vexdb backup \ + "$HOME/Library/Application Support/VexDB-Lite/default.sqlite3" \ + "$HOME/Desktop/vexdb-backup.sqlite3" +``` + +为了防止误覆盖，目标文件已经存在时命令会拒绝执行。源数据库不会被删除。 + +验证备份： + +```bash +vexdb "$HOME/Desktop/vexdb-backup.sqlite3" "PRAGMA integrity_check;" +vexdb fs --db "$HOME/Desktop/vexdb-backup.sqlite3" ls / +``` + +如果检测到旧版 `VexFS.app`，安装程序会先撤销旧文件系统扩展的注册，再把旧 App +改名为 `.disabled` 备份，避免 macOS 继续加载旧扩展。这个过程不会删除任何数据库。 + +## 9. 在其他 SQLite 程序中加载 + +最稳妥的方式是使用 `vexdb`。macOS 自带的 `/usr/bin/sqlite3` 通常关闭了动态扩展 +加载，不能执行 `.load`。 + +如果使用 Homebrew、Anaconda 或应用程序自己提供的 SQLite，并且该程序允许动态 +扩展，可以手动加载： + +```bash +sqlite3 ~/data/agent.db +``` + +```sql +.load ~/.local/lib/vexdb-lite/vexdb_lite.dylib +SELECT vexdb_version(); +``` + +可以先运行 `sqlite3 :memory: ".help"`，确认帮助中存在 `.load`。是否允许动态加载 +取决于该 SQLite 程序自己的编译设置；不支持时请直接使用 `vexdb`。 + +## 10. 卸载 + +在解压目录运行： + +```bash +./uninstall.sh +``` + +卸载只移除 App、命令和 SQLite 扩展，不会删除： + +- `~/Library/Application Support/VexDB-Lite/` +- `~/Library/Application Support/VexFS/` + +需要删除数据时请先备份，再由用户手动处理。 + +## 11. 常见问题 + +### `vexdb` 找不到 + +把 `~/.local/bin` 加入 PATH，参见安装章节。 + +### `vexdb fs` 能用，但 `mount` 失败 + +先运行 `vexdb fs doctor`。如果输出是 `extension=disabled`，进入系统设置的 +“登录项与扩展”，切换到“按类别”，然后在“文件系统扩展”详情中打开 +`VexDB Lite`。“按 App”视图里的 `FSKit Modules` 详情是只读的。 + +### 首次打开被 macOS 阻止 + +先检查 `MANIFEST.txt`、压缩包 SHA-256 和系统时间。公证包应显示 +`notarization=accepted`。如果公证包仍被阻止，请保留错误信息并联系维护者，不要关闭 +Gatekeeper 或使用删除隔离属性的方式绕过系统检查。 + +### 是否必须挂载 + +不是。Agent 可以直接使用 `vexdb fs` 完成文件管理。只有需要让系统 `ls`、`cat`、 +`grep`、`cp` 等命令直接访问数据库文件时才需要挂载。 + +### 文件究竟存在哪里 + +文件内容、目录、版本和权限信息都由 SQLite 数据库管理。挂载目录只是访问入口， +不是另一份真实数据。 diff --git a/agent_files/macos/package_preview.sh b/agent_files/macos/package_preview.sh index d15f89261f..02d6bad7ed 100755 --- a/agent_files/macos/package_preview.sh +++ b/agent_files/macos/package_preview.sh @@ -1,20 +1,58 @@ #!/bin/bash set -euo pipefail +# 本机预览：bash agent_files/macos/package_preview.sh +# Developer ID：VEXDB_LITE_SIGN_MODE=developer-id bash agent_files/macos/package_preview.sh +# 签名并公证：VEXDB_LITE_SIGN_MODE=developer-id VEXDB_LITE_NOTARY_PROFILE=vexdb-lite-notary bash agent_files/macos/package_preview.sh + SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" SQLITE_DIR="$ROOT/vexdb_sqlite" PACKAGE_DIR="$SCRIPT_DIR/package" +RUNTIME_HEADER="$ROOT/agent_files/mount/common/include/vexfs_runtime_types.h" +RUNTIME_ABI="$(sed -n 's/^#define VEXFS_RUNTIME_ABI_VERSION \([0-9][0-9]*\)u$/\1/p' "$RUNTIME_HEADER")" +[ -n "$RUNTIME_ABI" ] || { echo "无法从 $RUNTIME_HEADER 读取 runtime ABI" >&2; exit 1; } -VERSION="${VEXFS_PREVIEW_VERSION:-0.1.0-preview.1}" -ARCH="${VEXFS_ARCH:-arm64}" +VERSION="${VEXDB_LITE_PREVIEW_VERSION:-${VEXFS_PREVIEW_VERSION:-0.1.0-preview.1}}" +ARCH="${VEXDB_LITE_ARCH:-${VEXFS_ARCH:-arm64}}" +SIGN_MODE="${VEXDB_LITE_SIGN_MODE:-${VEXFS_SIGN_MODE:-ad-hoc}}" +TEAM_ID="${VEXDB_LITE_TEAM_ID:-${VEXFS_TEAM_ID:-BB5VK42K87}}" +EXPORT_OPTIONS="${VEXDB_LITE_EXPORT_OPTIONS:-${VEXFS_EXPORT_OPTIONS:-$SCRIPT_DIR/ExportOptions-DeveloperID.plist}}" +NOTARY_PROFILE="${VEXDB_LITE_NOTARY_PROFILE:-${VEXFS_NOTARY_PROFILE:-}}" +GIT_COMMIT="$(git -C "$ROOT" rev-parse --short HEAD 2>/dev/null || echo unknown)" +GIT_BRANCH="$(git -C "$ROOT" branch --show-current 2>/dev/null || echo unknown)" +if [ -n "$(git -C "$ROOT" status --porcelain 2>/dev/null || true)" ]; then + GIT_DIRTY=true +else + GIT_DIRTY=false +fi case "$VERSION" in *[!A-Za-z0-9._-]*) echo "版本只能包含字母、数字、点、下划线和连字符" >&2; exit 2 ;; esac case "$ARCH" in arm64|x86_64) ;; - *) echo "VEXFS_ARCH 只能是 arm64 或 x86_64" >&2; exit 2 ;; + *) echo "VEXDB_LITE_ARCH 只能是 arm64 或 x86_64" >&2; exit 2 ;; +esac +case "$SIGN_MODE" in + ad-hoc|developer-id) ;; + *) echo "VEXDB_LITE_SIGN_MODE 只能是 ad-hoc 或 developer-id" >&2; exit 2 ;; esac +if [ "$SIGN_MODE" = developer-id ]; then + [ -n "$TEAM_ID" ] || { echo "Developer ID 签名需要 VEXDB_LITE_TEAM_ID" >&2; exit 2; } + [ -f "$EXPORT_OPTIONS" ] || { echo "找不到导出配置：$EXPORT_OPTIONS" >&2; exit 1; } + command -v openssl >/dev/null || { echo "Developer ID 签名需要 openssl" >&2; exit 1; } + if [ "$GIT_DIRTY" = true ] && [ -n "$NOTARY_PROFILE" ]; then + echo "拒绝公证脏工作树：请先提交本次源码，确保公证包可追溯" >&2 + exit 1 + fi + if [ "$GIT_DIRTY" = true ] && [ "${VEXDB_LITE_ALLOW_DIRTY_SIGNING:-0}" != 1 ]; then + echo "拒绝签名脏工作树；仅本地验证可显式设置 VEXDB_LITE_ALLOW_DIRTY_SIGNING=1" >&2 + exit 1 + fi + if [ "$GIT_DIRTY" = true ]; then + export VEXDB_LITE_ALLOW_DIRTY_PACKAGE_TEST=1 + fi +fi CMAKE_BIN="$(command -v cmake || true)" [ -x "$CMAKE_BIN" ] || CMAKE_BIN=/opt/homebrew/bin/cmake @@ -23,100 +61,306 @@ CMAKE_BIN="$(command -v cmake || true)" command -v xcodebuild >/dev/null || { echo "找不到 xcodebuild" >&2; exit 1; } command -v codesign >/dev/null || { echo "找不到 codesign" >&2; exit 1; } -NCPU="$(sysctl -n hw.ncpu 2>/dev/null || echo 8)" -BUILD_DIR="$SQLITE_DIR/build-vexfs-package-$ARCH" +NCPU="$(sysctl -n hw.ncpu 2>/dev/null || echo 4)" +BUILD_JOBS="${VEXDB_LITE_BUILD_JOBS:-$NCPU}" +XCODE_JOBS="${VEXDB_LITE_XCODE_JOBS:-2}" +case "$BUILD_JOBS:$XCODE_JOBS" in + *[!0-9:]*|0:*|*:0) echo "构建并行度必须是正整数" >&2; exit 2 ;; +esac +# 打包是交付步骤，不需要占满开发机。默认最多四个 C/C++ 编译任务，Xcode 最多两个。 +[ "$BUILD_JOBS" -le 4 ] || BUILD_JOBS=4 +BUILD_DIR="$SQLITE_DIR/build-vexdb-lite-package-$ARCH" DERIVED_DATA="$BUILD_DIR/xcode-derived-data" -DIST_DIR="$ROOT/dist/vexfs" -PACKAGE_NAME="vexfs-${VERSION}-macos-${ARCH}" +ARCHIVE_PATH="$BUILD_DIR/VexDB-Lite.xcarchive" +EXPORT_DIR="$BUILD_DIR/developer-id-export" +DIST_DIR="$ROOT/dist/vexdb-lite" +PACKAGE_NAME="vexdb-lite-${VERSION}-macos-${ARCH}" STAGE="$DIST_DIR/$PACKAGE_NAME" ZIP="$DIST_DIR/$PACKAGE_NAME.zip" +GUIDE="$DIST_DIR/$PACKAGE_NAME-使用说明.md" +APP_NAME="VexDB Lite.app" +LSREGISTER="/System/Library/Frameworks/CoreServices.framework/Frameworks/LaunchServices.framework/Support/lsregister" + +unregister_temporary_fskit_apps() { + [ -x "$LSREGISTER" ] || return 0 + local app + for app in \ + "$ARCHIVE_PATH/Products/Applications/$APP_NAME" \ + "$EXPORT_DIR/$APP_NAME" \ + "$DERIVED_DATA/Build/Intermediates.noindex/ArchiveIntermediates/VexFSApp/InstallationBuildProductsLocation/Applications/$APP_NAME" \ + "$DERIVED_DATA/Build/Intermediates.noindex/ArchiveIntermediates/VexFSApp/BuildProductsPath/Release/$APP_NAME"; do + [ -d "$app" ] || continue + # xcodebuild 的登记可能延迟落盘。重复撤销只作用于这两个构建路径， + # 不会清除用户已经批准的 extension 启用状态。 + "$LSREGISTER" -u "$app" >/dev/null 2>&1 || true + done +} + +restore_installed_fskit_app() { + [ -x "$LSREGISTER" ] || return 0 + local installed="" + local candidate + for candidate in \ + "$HOME/Applications/$APP_NAME" \ + "/Applications/$APP_NAME"; do + if [ -d "$candidate" ]; then + installed="$candidate" + break + fi + done + [ -n "$installed" ] || return 0 + "$LSREGISTER" -f "$installed" >/dev/null 2>&1 || true + if command -v pluginkit >/dev/null; then + pluginkit -a "$installed/Contents/Extensions/VexFSAppEx.appex" >/dev/null 2>&1 || true + fi + local mounts + mounts="$(/sbin/mount 2>/dev/null | tr '[:upper:]' '[:lower:]')" + case "$mounts" in + *"(vexfs,"*|*"(exfat,"*|*"(msdos,"*|*" type vexfs"*|*" type exfat"*|*" type msdos"*) ;; + *) pkill -KILL -x fskit_agent >/dev/null 2>&1 || true ;; + esac +} + +cleanup_fskit_build_registration() { + unregister_temporary_fskit_apps + restore_installed_fskit_app +} + +# archive/export 即使在后续步骤失败也不能留在 FSKit 的候选模块列表里。 +trap cleanup_fskit_build_registration EXIT mkdir -p "$BUILD_DIR" "$DIST_DIR" -echo "=== 构建 VexFS CLI 与 SQLite 扩展 ===" +echo "=== 构建 VexDB-Lite CLI 与 SQLite 扩展 ===" "$CMAKE_BIN" -S "$SQLITE_DIR" -B "$BUILD_DIR" \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_OSX_ARCHITECTURES="$ARCH" \ + -DVEXDB_LITE_VERSION="$VERSION" \ -DVEXDB_SQLITE_BUILD_TESTS=OFF -"$CMAKE_BIN" --build "$BUILD_DIR" --target vexfs_cli vexdb_lite_loadable -j "$NCPU" - -echo "=== 构建 VexFS App 与 FSKit extension ===" -xcodebuild \ - -project "$SCRIPT_DIR/VexFS.xcodeproj" \ - -scheme VexFSApp \ - -configuration Release \ - -derivedDataPath "$DERIVED_DATA" \ - ARCHS="$ARCH" ONLY_ACTIVE_ARCH=YES CODE_SIGNING_ALLOWED=NO build - -APP_SOURCE="$DERIVED_DATA/Build/Products/Release/VexFS.app" -[ -d "$APP_SOURCE" ] || { echo "没有找到 VexFS.app" >&2; exit 1; } -[ -x "$BUILD_DIR/vexfs" ] || { echo "没有找到 vexfs CLI" >&2; exit 1; } +"$CMAKE_BIN" --build "$BUILD_DIR" --target vexdb_cli vexdb_lite_loadable -j "$BUILD_JOBS" + +echo "=== 构建 VexDB Lite App 与 VexFS extension ===" +if [ "$SIGN_MODE" = developer-id ]; then + rm -rf "$ARCHIVE_PATH" "$EXPORT_DIR" + xcodebuild \ + -jobs "$XCODE_JOBS" \ + -project "$SCRIPT_DIR/VexFS.xcodeproj" \ + -scheme VexFSApp \ + -configuration Release \ + -derivedDataPath "$DERIVED_DATA" \ + -archivePath "$ARCHIVE_PATH" \ + -destination 'generic/platform=macOS' \ + ARCHS="$ARCH" ONLY_ACTIVE_ARCH=YES \ + DEVELOPMENT_TEAM="$TEAM_ID" CODE_SIGN_STYLE=Automatic \ + -allowProvisioningUpdates archive + # InstallationBuildProductsLocation 可能在 exportArchive 期间被 Xcode 删除； + # 必须趁临时 App 仍存在时撤销登记，否则会留下无法按路径清除的 LS 记录。 + sleep 2 + unregister_temporary_fskit_apps + xcodebuild \ + -jobs "$XCODE_JOBS" \ + -exportArchive \ + -archivePath "$ARCHIVE_PATH" \ + -exportPath "$EXPORT_DIR" \ + -exportOptionsPlist "$EXPORT_OPTIONS" \ + -allowProvisioningUpdates + APP_SOURCE="$EXPORT_DIR/$APP_NAME" +else + xcodebuild \ + -jobs "$XCODE_JOBS" \ + -project "$SCRIPT_DIR/VexFS.xcodeproj" \ + -scheme VexFSApp \ + -configuration Release \ + -derivedDataPath "$DERIVED_DATA" \ + ARCHS="$ARCH" ONLY_ACTIVE_ARCH=YES CODE_SIGNING_ALLOWED=NO build + APP_SOURCE="$DERIVED_DATA/Build/Products/Release/$APP_NAME" +fi +[ -d "$APP_SOURCE" ] || { echo "没有找到 $APP_NAME" >&2; exit 1; } +[ -x "$BUILD_DIR/vexdb" ] || { echo "没有找到 vexdb CLI" >&2; exit 1; } [ -f "$BUILD_DIR/vexdb_lite.dylib" ] || { echo "没有找到 vexdb_lite.dylib" >&2; exit 1; } rm -rf "$STAGE" mkdir -p "$STAGE/bin" "$STAGE/lib" -ditto "$APP_SOURCE" "$STAGE/VexFS.app" -install -m 0755 "$BUILD_DIR/vexfs" "$STAGE/bin/vexfs" +ditto "$APP_SOURCE" "$STAGE/$APP_NAME" +if [ "$SIGN_MODE" = developer-id ]; then + # 先复制交付 App，再撤销并删除 Xcode 的临时同 ID App。这样 LaunchServices + # 后续不会重新发现 .xcarchive 并让 FSKit 加载到已经失效的构建路径。 + sleep 2 + unregister_temporary_fskit_apps + rm -rf "$ARCHIVE_PATH" "$EXPORT_DIR" "$DERIVED_DATA" + restore_installed_fskit_app +fi +install -m 0755 "$BUILD_DIR/vexdb" "$STAGE/bin/vexdb" +ln -s vexdb "$STAGE/bin/vexfs" install -m 0644 "$BUILD_DIR/vexdb_lite.dylib" "$STAGE/lib/vexdb_lite.dylib" install -m 0755 "$PACKAGE_DIR/install.sh" "$STAGE/install.sh" install -m 0755 "$PACKAGE_DIR/Install.command" "$STAGE/Install.command" install -m 0755 "$PACKAGE_DIR/uninstall.sh" "$STAGE/uninstall.sh" install -m 0644 "$PACKAGE_DIR/README.md" "$STAGE/README.md" +install -m 0644 "$PACKAGE_DIR/使用说明.md" "$STAGE/使用说明.md" +install -m 0644 "$PACKAGE_DIR/使用说明.md" "$GUIDE" install -m 0644 "$ROOT/LICENSE" "$STAGE/LICENSE" -# 没有 Developer ID 时仍做完整的 ad-hoc 签名，避免发送过程中破坏 bundle seal。 -EXTENSION="$STAGE/VexFS.app/Contents/Extensions/VexFSAppEx.appex" -codesign --force --sign - --timestamp=none \ - --entitlements "$SCRIPT_DIR/VexFSAppEx/VexFSAppEx.entitlements" "$EXTENSION" -codesign --force --sign - --timestamp=none \ - --entitlements "$SCRIPT_DIR/VexFSApp/VexFSApp.entitlements" "$STAGE/VexFS.app" -codesign --force --sign - --timestamp=none "$STAGE/bin/vexfs" -codesign --force --sign - --timestamp=none "$STAGE/lib/vexdb_lite.dylib" -codesign --verify --deep --strict "$STAGE/VexFS.app" +EXTENSION="$STAGE/$APP_NAME/Contents/Extensions/VexFSAppEx.appex" +SIGNATURE=ad-hoc +SIGNING_IDENTITY=- +SIGNING_TEAM=none +CERTIFICATE_SHA1=none +if [ "$SIGN_MODE" = developer-id ]; then + CERT_PREFIX="$BUILD_DIR/vexfs-app-certificate-" + rm -f "${CERT_PREFIX}"* + codesign -d --extract-certificates="$CERT_PREFIX" "$STAGE/$APP_NAME" + CERTIFICATE_SHA1="$(openssl x509 -inform DER -in "${CERT_PREFIX}0" -noout -fingerprint -sha1 | cut -d= -f2 | tr -d ':')" + [ -n "$CERTIFICATE_SHA1" ] || { echo "无法读取 App 的签名证书" >&2; exit 1; } + SIGNING_IDENTITY="$CERTIFICATE_SHA1" + SIGNING_TEAM="$TEAM_ID" + SIGNATURE=developer-id + codesign --force --options runtime --timestamp --sign "$SIGNING_IDENTITY" "$STAGE/bin/vexdb" + codesign --force --options runtime --timestamp --sign "$SIGNING_IDENTITY" "$STAGE/lib/vexdb_lite.dylib" -GIT_COMMIT="$(git -C "$ROOT" rev-parse --short HEAD 2>/dev/null || echo unknown)" -GIT_BRANCH="$(git -C "$ROOT" branch --show-current 2>/dev/null || echo unknown)" -if [ -n "$(git -C "$ROOT" status --porcelain 2>/dev/null || true)" ]; then - GIT_DIRTY=true + verify_team_identifier() { + local item="$1" + local actual + actual="$(codesign -d --verbose=4 "$item" 2>&1 | sed -n 's/^TeamIdentifier=//p')" + if [ "$actual" != "$TEAM_ID" ]; then + echo "Developer ID 签名团队错误：$item（实际 ${actual:-none}，期望 $TEAM_ID）" >&2 + exit 1 + fi + } + verify_team_identifier "$STAGE/$APP_NAME" + verify_team_identifier "$EXTENSION" + verify_team_identifier "$STAGE/bin/vexdb" + verify_team_identifier "$STAGE/lib/vexdb_lite.dylib" else - GIT_DIRTY=false + # 没有 Developer ID 时仍做完整的 ad-hoc 签名，避免发送过程中破坏 bundle seal。 + codesign --force --sign - --timestamp=none \ + --entitlements "$SCRIPT_DIR/VexFSAppEx/VexFSAppEx.entitlements" "$EXTENSION" + codesign --force --sign - --timestamp=none \ + --entitlements "$SCRIPT_DIR/VexFSApp/VexFSApp.entitlements" "$STAGE/$APP_NAME" + codesign --force --sign - --timestamp=none "$STAGE/bin/vexdb" + codesign --force --sign - --timestamp=none "$STAGE/lib/vexdb_lite.dylib" fi -{ - echo "product=VexFS" - echo "preview_version=$VERSION" - echo "contract_version=0.3.0" - echo "mount_abi_version=3" - echo "platform=macOS" - echo "minimum_macos=26.0" - echo "architecture=$ARCH" - echo "source_branch=$GIT_BRANCH" - echo "source_commit=$GIT_COMMIT" - echo "source_dirty=$GIT_DIRTY" - echo "signature=ad-hoc" -} > "$STAGE/MANIFEST.txt" - -( - cd "$STAGE" - shasum -a 256 \ - bin/vexfs \ - lib/vexdb_lite.dylib \ - VexFS.app/Contents/MacOS/VexFS \ - VexFS.app/Contents/Extensions/VexFSAppEx.appex/Contents/MacOS/VexFSAppEx \ - > SHA256SUMS.txt -) - -rm -f "$ZIP" "$ZIP.sha256" -COPYFILE_DISABLE=1 ditto -c -k --norsrc --keepParent "$STAGE" "$ZIP" -if unzip -Z1 "$ZIP" | grep -Eq '(^|/)__MACOSX/|(^|/)\._'; then - echo "压缩包包含 AppleDouble 文件，停止交付" >&2 - exit 1 +codesign --verify --deep --strict --verbose=2 "$STAGE/$APP_NAME" +codesign --verify --strict --verbose=2 "$STAGE/bin/vexdb" +codesign --verify --strict --verbose=2 "$STAGE/lib/vexdb_lite.dylib" + +echo "=== 验证统一 SQLite、向量和文件入口 ===" +bash "$ROOT/agent_files/cli/test/vexdb_unified_smoke.sh" "$STAGE/bin/vexdb" + +NOTARIZATION_STATUS=not-submitted +NOTARIZATION_SUBMISSION_ID=none + +write_manifest() { + { + echo "product=VexDB-Lite" + echo "filesystem=VexFS" + echo "preview_version=$VERSION" + echo "contract_version=0.7.0" + echo "mount_abi_version=$RUNTIME_ABI" + echo "platform=macOS" + echo "minimum_macos=26.0" + echo "architecture=$ARCH" + echo "source_branch=$GIT_BRANCH" + echo "source_commit=$GIT_COMMIT" + echo "source_dirty=$GIT_DIRTY" + echo "signature=$SIGNATURE" + echo "signing_team=$SIGNING_TEAM" + echo "certificate_sha1=$CERTIFICATE_SHA1" + echo "notarization=$NOTARIZATION_STATUS" + echo "notarization_submission_id=$NOTARIZATION_SUBMISSION_ID" + } > "$STAGE/MANIFEST.txt" +} + +write_hashes() { + ( + cd "$STAGE" + shasum -a 256 \ + MANIFEST.txt \ + install.sh \ + Install.command \ + uninstall.sh \ + README.md \ + LICENSE \ + bin/vexdb \ + lib/vexdb_lite.dylib \ + 使用说明.md \ + "$APP_NAME/Contents/MacOS/VexDB Lite" \ + "$APP_NAME/Contents/Extensions/VexFSAppEx.appex/Contents/MacOS/VexFSAppEx" \ + > SHA256SUMS.txt + ) +} + +write_zip() { + rm -f "$ZIP" "$ZIP.sha256" + COPYFILE_DISABLE=1 ditto -c -k --norsrc --keepParent "$STAGE" "$ZIP" + if unzip -Z1 "$ZIP" | grep -Eq '(^|/)__MACOSX/|(^|/)\._'; then + echo "压缩包包含 AppleDouble 文件，停止交付" >&2 + exit 1 + fi + ( + cd "$DIST_DIR" + shasum -a 256 "$(basename "$ZIP")" > "$(basename "$ZIP").sha256" + ) +} + +verify_zip() { + local verify_dir verify_root + verify_dir="$(mktemp -d "$BUILD_DIR/zip-verify.XXXXXX")" + verify_root="$verify_dir/$PACKAGE_NAME" + + ( + cd "$DIST_DIR" + shasum -a 256 -c "$(basename "$ZIP").sha256" + ) + ditto -x -k "$ZIP" "$verify_dir" + [ -d "$verify_root/$APP_NAME" ] || { echo "压缩包缺少 $APP_NAME" >&2; return 1; } + [ -x "$verify_root/bin/vexdb" ] || { echo "压缩包缺少 bin/vexdb" >&2; return 1; } + [ -L "$verify_root/bin/vexfs" ] || { echo "压缩包中的 bin/vexfs 不是兼容链接" >&2; return 1; } + [ -f "$verify_root/lib/vexdb_lite.dylib" ] || { echo "压缩包缺少 SQLite 扩展" >&2; return 1; } + [ -f "$verify_root/使用说明.md" ] || { echo "压缩包缺少使用说明" >&2; return 1; } + + codesign --verify --deep --strict --verbose=2 "$verify_root/$APP_NAME" + codesign --verify --strict --verbose=2 "$verify_root/bin/vexdb" + codesign --verify --strict --verbose=2 "$verify_root/lib/vexdb_lite.dylib" + bash "$ROOT/agent_files/cli/test/vexdb_unified_smoke.sh" "$verify_root/bin/vexdb" + rm -rf "$verify_dir" +} + +write_manifest +write_hashes +bash "$ROOT/tests/eval/vexfs/documentation_smoke.sh" "$STAGE" +VEXDB_LITE_PACKAGE_STAGE="$STAGE" python3 "$ROOT/tests/eval/vexfs/run.py" \ + --mode quick --filter package.unified-install --include-package --fail-on-skip \ + --build-dir "$BUILD_DIR" +write_zip +verify_zip + +if [ "$SIGN_MODE" = developer-id ] && [ -n "$NOTARY_PROFILE" ]; then + echo "=== 提交 Apple 公证 ===" + NOTARY_RESULT="$BUILD_DIR/notary-result.json" + xcrun notarytool submit "$ZIP" \ + --keychain-profile "$NOTARY_PROFILE" \ + --output-format json \ + --wait > "$NOTARY_RESULT" + cat "$NOTARY_RESULT" + NOTARIZATION_STATUS="$(plutil -extract status raw -o - "$NOTARY_RESULT")" + NOTARIZATION_SUBMISSION_ID="$(plutil -extract id raw -o - "$NOTARY_RESULT")" + if [ "$NOTARIZATION_STATUS" != Accepted ]; then + echo "Apple 公证未通过：$NOTARIZATION_STATUS（$NOTARIZATION_SUBMISSION_ID）" >&2 + exit 1 + fi + NOTARIZATION_STATUS=accepted + xcrun stapler staple "$STAGE/$APP_NAME" + xcrun stapler validate "$STAGE/$APP_NAME" + spctl --assess --type execute --verbose=4 "$STAGE/$APP_NAME" + write_manifest + write_hashes + bash "$ROOT/tests/eval/vexfs/documentation_smoke.sh" "$STAGE" + write_zip + verify_zip fi -( - cd "$DIST_DIR" - shasum -a 256 "$(basename "$ZIP")" > "$(basename "$ZIP").sha256" -) echo "" -echo "=== VexFS 技术预览包 ===" -ls -lh "$ZIP" "$ZIP.sha256" +echo "=== VexDB-Lite 技术预览包 ===" +ls -lh "$ZIP" "$ZIP.sha256" "$GUIDE" cat "$ZIP.sha256" diff --git a/agent_files/mount/common/include/vexfs_checksum.h b/agent_files/mount/common/include/vexfs_checksum.h new file mode 100644 index 0000000000..861d24b8bf --- /dev/null +++ b/agent_files/mount/common/include/vexfs_checksum.h @@ -0,0 +1,33 @@ +#ifndef VEXFS_CHECKSUM_H +#define VEXFS_CHECKSUM_H + +#include +#include +#include +#include + +namespace vexfs { + +class Sha256 { + public: + Sha256(); + + void Update(const void *data, size_t size); + std::array Finish(); + + private: + void Transform(const unsigned char block[64]); + + std::array state_{}; + std::array buffer_{}; + uint64_t total_bytes_ = 0; + size_t buffered_bytes_ = 0; + bool finished_ = false; +}; + +std::string Sha256Hex(const void *data, size_t size); +std::string Hex(const std::array &digest); + +} // namespace vexfs + +#endif // VEXFS_CHECKSUM_H diff --git a/agent_files/mount/common/include/vexfs_mount_contract.h b/agent_files/mount/common/include/vexfs_mount_contract.h deleted file mode 100644 index cc7f99bdd0..0000000000 --- a/agent_files/mount/common/include/vexfs_mount_contract.h +++ /dev/null @@ -1,151 +0,0 @@ -// VexFS 平台挂载合同（C ABI）。 -// -// FSKit、libfuse 和 WinFsp 适配层只依赖这个头。实现通过公开的 vexfs_* SQL -// 函数访问 SQLite，数据库仍然是状态和事务的唯一管理者。 -#ifndef VEXFS_MOUNT_CONTRACT_H -#define VEXFS_MOUNT_CONTRACT_H - -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -#define VEXFS_MOUNT_ABI_VERSION 3u -#define VEXFS_MOUNT_OPEN_NO_CREATE 1u -#define VEXFS_MOUNT_EXCLUSIVE_GATEWAY 2u - -typedef struct vexfs_mount_session vexfs_mount_session; - -typedef enum vexfs_mount_status { - VEXFS_MOUNT_OK = 0, - VEXFS_MOUNT_INVALID_ARGUMENT = 1, - VEXFS_MOUNT_NOT_FOUND = 2, - VEXFS_MOUNT_CONFLICT = 3, - VEXFS_MOUNT_READ_ONLY = 4, - VEXFS_MOUNT_BUSY = 5, - VEXFS_MOUNT_DATABASE_ERROR = 6, - VEXFS_MOUNT_INTERNAL_ERROR = 7, - VEXFS_MOUNT_PERMISSION_DENIED = 8, - VEXFS_MOUNT_NO_SPACE = 9, - VEXFS_MOUNT_CORRUPTION = 10, - VEXFS_MOUNT_UNSUPPORTED = 11 -} vexfs_mount_status; - -typedef struct vexfs_mount_error { - vexfs_mount_status status; - int sqlite_code; - char message[512]; -} vexfs_mount_error; - -typedef struct vexfs_mount_config { - uint32_t abi_version; - const char *database_path; - const char *workspace; - uint32_t busy_timeout_ms; - uint32_t flags; -} vexfs_mount_config; - -typedef struct vexfs_mount_bytes { - void *data; - uint64_t size; -} vexfs_mount_bytes; - -// 打开数据库、注册 vexdb_lite/VexFS、初始化内部表并创建（或复用）workspace。 -vexfs_mount_status vexfs_mount_session_open(const vexfs_mount_config *config, - vexfs_mount_session **session, - vexfs_mount_error *error); -void vexfs_mount_session_close(vexfs_mount_session *session); - -// 返回数据库合同、durability 和待恢复 staging 的 JSON 诊断信息。 -vexfs_mount_status vexfs_mount_diagnostics(vexfs_mount_session *session, - vexfs_mount_bytes *json, - vexfs_mount_error *error); - -vexfs_mount_status vexfs_mount_mkdir(vexfs_mount_session *session, const char *path, - vexfs_mount_error *error); -vexfs_mount_status vexfs_mount_write_file(vexfs_mount_session *session, const char *path, - const void *data, uint64_t size, - int64_t *version, vexfs_mount_error *error); -vexfs_mount_status vexfs_mount_read_file(vexfs_mount_session *session, const char *path, - vexfs_mount_bytes *content, - vexfs_mount_error *error); -vexfs_mount_status vexfs_mount_history(vexfs_mount_session *session, const char *path, - vexfs_mount_bytes *json, - vexfs_mount_error *error); -vexfs_mount_status vexfs_mount_history_page(vexfs_mount_session *session, const char *path, - uint32_t limit, int64_t before_version, - vexfs_mount_bytes *json, - vexfs_mount_error *error); -vexfs_mount_status vexfs_mount_read_version(vexfs_mount_session *session, const char *path, - int64_t version, vexfs_mount_bytes *content, - vexfs_mount_error *error); -vexfs_mount_status vexfs_mount_compare_versions(vexfs_mount_session *session, const char *path, - int64_t from_version, int64_t to_version, - vexfs_mount_bytes *json, - vexfs_mount_error *error); -vexfs_mount_status vexfs_mount_restore_version(vexfs_mount_session *session, const char *path, - int64_t target_version, - int64_t expected_version, - int64_t *new_version, - vexfs_mount_error *error); -vexfs_mount_status vexfs_mount_stat(vexfs_mount_session *session, const char *path, - vexfs_mount_bytes *json, vexfs_mount_error *error); -vexfs_mount_status vexfs_mount_path_for_inode(vexfs_mount_session *session, int64_t inode, - vexfs_mount_bytes *path, - vexfs_mount_error *error); -vexfs_mount_status vexfs_mount_list(vexfs_mount_session *session, const char *path, - vexfs_mount_bytes *json, vexfs_mount_error *error); -vexfs_mount_status vexfs_mount_move(vexfs_mount_session *session, const char *source, - const char *destination, vexfs_mount_error *error); -vexfs_mount_status vexfs_mount_rename(vexfs_mount_session *session, const char *source, - const char *destination, int replace, - vexfs_mount_error *error); -vexfs_mount_status vexfs_mount_remove(vexfs_mount_session *session, const char *path, - int recursive, vexfs_mount_error *error); - -vexfs_mount_status vexfs_mount_handle_open(vexfs_mount_session *session, const char *path, - const char *flags, const char *request_id, - vexfs_mount_bytes *handle, - vexfs_mount_error *error); -vexfs_mount_status vexfs_mount_handle_stage_write(vexfs_mount_session *session, - const char *handle, uint64_t offset, - const void *data, uint64_t size, - const char *request_id, - int64_t *generation, - vexfs_mount_error *error); -vexfs_mount_status vexfs_mount_handle_truncate(vexfs_mount_session *session, - const char *handle, uint64_t size, - const char *request_id, - int64_t *generation, - vexfs_mount_error *error); -vexfs_mount_status vexfs_mount_handle_read(vexfs_mount_session *session, const char *handle, - uint64_t offset, uint64_t length, - vexfs_mount_bytes *content, - vexfs_mount_error *error); -vexfs_mount_status vexfs_mount_handle_publish(vexfs_mount_session *session, - const char *handle, int64_t generation, - const char *durability, - const char *request_id, int64_t *version, - vexfs_mount_error *error); -vexfs_mount_status vexfs_mount_handle_close(vexfs_mount_session *session, - const char *handle, int retain_unpublished, - const char *request_id, - vexfs_mount_bytes *state, - vexfs_mount_error *error); -vexfs_mount_status vexfs_mount_synchronize(vexfs_mount_session *session, - const char *request_id, int64_t *published, - vexfs_mount_error *error); -vexfs_mount_status vexfs_mount_reclaim(vexfs_mount_session *session, - const char *request_id, int64_t *reclaimed, - vexfs_mount_error *error); - -// 释放本合同返回的 vexfs_mount_bytes.data。 -void vexfs_mount_free(void *memory); - -#ifdef __cplusplus -} -#endif - -#endif // VEXFS_MOUNT_CONTRACT_H diff --git a/agent_files/mount/common/include/vexfs_runtime_admin.h b/agent_files/mount/common/include/vexfs_runtime_admin.h new file mode 100644 index 0000000000..a5c09f2c00 --- /dev/null +++ b/agent_files/mount/common/include/vexfs_runtime_admin.h @@ -0,0 +1,79 @@ +// CLI 和管理工具允许调用的管理面接口。文件 I/O 来自 runtime_io。 +#ifndef VEXFS_RUNTIME_ADMIN_H +#define VEXFS_RUNTIME_ADMIN_H + +#include "vexfs_runtime_io.h" + +#ifdef __cplusplus +extern "C" { +#endif + +vexfs_mount_status vexfs_mount_diagnostics(vexfs_mount_session *session, + vexfs_mount_bytes *json, + vexfs_mount_error *error); +// flags=0 performs a deep checksum scan; VEXFS_MOUNT_CHECK_QUICK skips BLOB hashing. +#define VEXFS_MOUNT_CHECK_QUICK 1u +vexfs_mount_status vexfs_mount_check(vexfs_mount_session *session, uint32_t flags, + vexfs_mount_bytes *json, + vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_grep(vexfs_mount_session *session, const char *path, + const char *pattern, uint32_t flags, + uint32_t limit, vexfs_mount_bytes *json, + vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_grep_index(vexfs_mount_session *session, + const char *action, + vexfs_mount_bytes *json, + vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_history(vexfs_mount_session *session, const char *path, + vexfs_mount_bytes *json, + vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_history_page(vexfs_mount_session *session, const char *path, + uint32_t limit, int64_t before_version, + vexfs_mount_bytes *json, + vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_read_version(vexfs_mount_session *session, const char *path, + int64_t version, + vexfs_mount_bytes *content, + vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_compare_versions(vexfs_mount_session *session, + const char *path, + int64_t from_version, + int64_t to_version, + vexfs_mount_bytes *json, + vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_restore_version(vexfs_mount_session *session, const char *path, + int64_t target_version, + int64_t expected_version, + int64_t *new_version, + vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_workspace_head(vexfs_mount_session *session, + int64_t *head_commit, + vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_snapshot_create(vexfs_mount_session *session, + const char *name, uint32_t flags, + int64_t *commit, + vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_snapshot_list(vexfs_mount_session *session, + vexfs_mount_bytes *json, + vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_snapshot_show(vexfs_mount_session *session, + const char *name, + vexfs_mount_bytes *json, + vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_snapshot_diff(vexfs_mount_session *session, + const char *from, const char *to, + vexfs_mount_bytes *json, + vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_snapshot_drop(vexfs_mount_session *session, + const char *name, + vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_snapshot_restore(vexfs_mount_session *session, + const char *name, int64_t expected_head, + int64_t *new_commit, + vexfs_mount_error *error); + +#ifdef __cplusplus +} +#endif + +#endif // VEXFS_RUNTIME_ADMIN_H diff --git a/agent_files/mount/common/include/vexfs_runtime_io.h b/agent_files/mount/common/include/vexfs_runtime_io.h new file mode 100644 index 0000000000..2e96be51f6 --- /dev/null +++ b/agent_files/mount/common/include/vexfs_runtime_io.h @@ -0,0 +1,144 @@ +// 平台文件系统 adapter 允许调用的数据面接口。 +#ifndef VEXFS_RUNTIME_IO_H +#define VEXFS_RUNTIME_IO_H + +#include "vexfs_runtime_types.h" + +#ifdef __cplusplus +extern "C" { +#endif + +vexfs_mount_status vexfs_mount_session_open(const vexfs_mount_config *config, + vexfs_mount_session **session, + vexfs_mount_error *error); +void vexfs_mount_session_close(vexfs_mount_session *session); + +vexfs_mount_status vexfs_mount_mkdir(vexfs_mount_session *session, const char *path, + vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_create(vexfs_mount_session *session, const char *path, + const char *kind, uint32_t mode, + vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_set_mode(vexfs_mount_session *session, int64_t inode, + uint32_t mode, vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_set_times(vexfs_mount_session *session, int64_t inode, + int64_t accessed_at_ms, int64_t modified_at_ms, + uint32_t mask, vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_symlink(vexfs_mount_session *session, const char *path, + const void *target, uint64_t target_size, + vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_link(vexfs_mount_session *session, const char *source, + const char *destination, vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_chown(vexfs_mount_session *session, int64_t inode, + int64_t uid, int64_t gid, vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_readlink(vexfs_mount_session *session, int64_t inode, + vexfs_mount_bytes *target, + vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_write_file(vexfs_mount_session *session, const char *path, + const void *data, uint64_t size, + int64_t *version, vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_append_file(vexfs_mount_session *session, int64_t inode, + const void *data, uint64_t size, + int64_t *version, vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_read_file(vexfs_mount_session *session, const char *path, + vexfs_mount_bytes *content, + vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_stat(vexfs_mount_session *session, const char *path, + vexfs_mount_bytes *json, vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_path_for_inode(vexfs_mount_session *session, int64_t inode, + vexfs_mount_bytes *path, + vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_list(vexfs_mount_session *session, const char *path, + vexfs_mount_bytes *json, vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_list_versioned(vexfs_mount_session *session, + const char *path, + vexfs_mount_bytes *json, + vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_move(vexfs_mount_session *session, const char *source, + const char *destination, vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_rename(vexfs_mount_session *session, const char *source, + const char *destination, int replace, + vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_remove(vexfs_mount_session *session, const char *path, + int recursive, vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_xattr_get(vexfs_mount_session *session, int64_t inode, + const char *name, vexfs_mount_bytes *value, + vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_xattr_list(vexfs_mount_session *session, int64_t inode, + vexfs_mount_bytes *json, + vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_xattr_set(vexfs_mount_session *session, int64_t inode, + const char *name, const void *value, + uint64_t size, int policy, + vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_acl_get(vexfs_mount_session *session, int64_t inode, + vexfs_mount_bytes *json, + vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_acl_set(vexfs_mount_session *session, int64_t inode, + const void *json, uint64_t size, + vexfs_mount_error *error); + +// 快速检测其他数据库连接是否已经提交。只有 data_version 变化时才读取 HEAD； +// adapter 用 cache_generation 对已有 vnode 做懒失效，不扫描整个缓存。 +vexfs_mount_status vexfs_mount_refresh_visibility(vexfs_mount_session *session, + vexfs_mount_visibility *visibility, + vexfs_mount_error *error); + +vexfs_mount_status vexfs_mount_handle_open(vexfs_mount_session *session, const char *path, + const char *flags, const char *request_id, + vexfs_mount_bytes *handle, + vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_handle_create(vexfs_mount_session *session, + const char *path, uint32_t mode, + const char *request_id, + vexfs_mount_bytes *handle, + vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_handle_stage_write(vexfs_mount_session *session, + const char *handle, uint64_t offset, + const void *data, uint64_t size, + const char *request_id, + int64_t *generation, + vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_handle_append(vexfs_mount_session *session, + const char *handle, const void *data, + uint64_t size, const char *request_id, + int64_t *version, + vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_handle_truncate(vexfs_mount_session *session, + const char *handle, uint64_t size, + const char *request_id, + int64_t *generation, + vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_handle_read(vexfs_mount_session *session, const char *handle, + uint64_t offset, uint64_t length, + vexfs_mount_bytes *content, + vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_handle_publish(vexfs_mount_session *session, + const char *handle, int64_t generation, + const char *durability, + const char *request_id, int64_t *version, + vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_handle_publish_close(vexfs_mount_session *session, + const char *handle, + int64_t generation, + const char *durability, + int64_t *version, + vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_handle_close(vexfs_mount_session *session, + const char *handle, int retain_unpublished, + const char *request_id, + vexfs_mount_bytes *state, + vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_synchronize(vexfs_mount_session *session, + const char *request_id, int64_t *published, + vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_reclaim(vexfs_mount_session *session, + const char *request_id, int64_t *reclaimed, + vexfs_mount_error *error); + +void vexfs_mount_free(void *memory); + +#ifdef __cplusplus +} +#endif + +#endif // VEXFS_RUNTIME_IO_H diff --git a/agent_files/mount/common/include/vexfs_runtime_types.h b/agent_files/mount/common/include/vexfs_runtime_types.h new file mode 100644 index 0000000000..f1c4d644c4 --- /dev/null +++ b/agent_files/mount/common/include/vexfs_runtime_types.h @@ -0,0 +1,82 @@ +// VexFS runtime 的稳定 C 类型。数据库原生细节只能出现在 diagnostic 字段中。 +#ifndef VEXFS_RUNTIME_TYPES_H +#define VEXFS_RUNTIME_TYPES_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#define VEXFS_RUNTIME_ABI_VERSION 1u +#define VEXFS_RUNTIME_OPEN_NO_CREATE 1u +#define VEXFS_RUNTIME_EXCLUSIVE_GATEWAY 2u +#define VEXFS_RUNTIME_BACKEND_SQLITE "sqlite" +#define VEXFS_MOUNT_SYMLINK_MAX 4096u +#define VEXFS_MOUNT_TIME_ACCESS 1u +#define VEXFS_MOUNT_TIME_MODIFY 2u +#define VEXFS_MOUNT_GREP_IGNORE_CASE 1u +#define VEXFS_MOUNT_GREP_FILES_ONLY 2u +#define VEXFS_SNAPSHOT_COMMITTED_ONLY 1u + +typedef enum vexfs_mount_xattr_policy { + VEXFS_MOUNT_XATTR_ALWAYS_SET = 0, + VEXFS_MOUNT_XATTR_MUST_CREATE = 1, + VEXFS_MOUNT_XATTR_MUST_REPLACE = 2, + VEXFS_MOUNT_XATTR_DELETE = 3 +} vexfs_mount_xattr_policy; + +typedef struct vexfs_mount_session vexfs_mount_session; + +typedef enum vexfs_mount_status { + VEXFS_MOUNT_OK = 0, + VEXFS_MOUNT_INVALID_ARGUMENT = 1, + VEXFS_MOUNT_NOT_FOUND = 2, + VEXFS_MOUNT_CONFLICT = 3, + VEXFS_MOUNT_READ_ONLY = 4, + VEXFS_MOUNT_BUSY = 5, + VEXFS_MOUNT_DATABASE_ERROR = 6, + VEXFS_MOUNT_INTERNAL_ERROR = 7, + VEXFS_MOUNT_PERMISSION_DENIED = 8, + VEXFS_MOUNT_NO_SPACE = 9, + VEXFS_MOUNT_CORRUPTION = 10, + VEXFS_MOUNT_UNSUPPORTED = 11, + VEXFS_MOUNT_NOT_EMPTY = 12 +} vexfs_mount_status; + +typedef struct vexfs_mount_error { + vexfs_mount_status status; + int native_code; + char backend[16]; + char message[512]; +} vexfs_mount_error; + +// connection 的格式由 backend 决定。SQLite 使用数据库文件路径；后续 PG +// 使用 DSN。principal 是 runtime 的请求身份，不再由平台层自己猜测。 +typedef struct vexfs_mount_config { + uint32_t abi_version; + const char *backend; + const char *connection; + const char *workspace; + const char *principal; + uint32_t operation_timeout_ms; + uint32_t flags; +} vexfs_mount_config; + +typedef struct vexfs_mount_bytes { + void *data; + uint64_t size; +} vexfs_mount_bytes; + +typedef struct vexfs_mount_visibility { + int64_t workspace_head; + uint64_t cache_generation; + int external_commit; +} vexfs_mount_visibility; + +#ifdef __cplusplus +} +#endif + +#endif // VEXFS_RUNTIME_TYPES_H diff --git a/agent_files/mount/common/src/vexfs_checksum.cpp b/agent_files/mount/common/src/vexfs_checksum.cpp new file mode 100644 index 0000000000..d32ac359db --- /dev/null +++ b/agent_files/mount/common/src/vexfs_checksum.cpp @@ -0,0 +1,157 @@ +#include "vexfs_checksum.h" + +#include +#include +#include + +namespace vexfs { +namespace { + +constexpr std::array kRoundConstants = { + 0x428a2f98u, 0x71374491u, 0xb5c0fbcfu, 0xe9b5dba5u, 0x3956c25bu, 0x59f111f1u, + 0x923f82a4u, 0xab1c5ed5u, 0xd807aa98u, 0x12835b01u, 0x243185beu, 0x550c7dc3u, + 0x72be5d74u, 0x80deb1feu, 0x9bdc06a7u, 0xc19bf174u, 0xe49b69c1u, 0xefbe4786u, + 0x0fc19dc6u, 0x240ca1ccu, 0x2de92c6fu, 0x4a7484aau, 0x5cb0a9dcu, 0x76f988dau, + 0x983e5152u, 0xa831c66du, 0xb00327c8u, 0xbf597fc7u, 0xc6e00bf3u, 0xd5a79147u, + 0x06ca6351u, 0x14292967u, 0x27b70a85u, 0x2e1b2138u, 0x4d2c6dfcu, 0x53380d13u, + 0x650a7354u, 0x766a0abbu, 0x81c2c92eu, 0x92722c85u, 0xa2bfe8a1u, 0xa81a664bu, + 0xc24b8b70u, 0xc76c51a3u, 0xd192e819u, 0xd6990624u, 0xf40e3585u, 0x106aa070u, + 0x19a4c116u, 0x1e376c08u, 0x2748774cu, 0x34b0bcb5u, 0x391c0cb3u, 0x4ed8aa4au, + 0x5b9cca4fu, 0x682e6ff3u, 0x748f82eeu, 0x78a5636fu, 0x84c87814u, 0x8cc70208u, + 0x90befffau, 0xa4506cebu, 0xbef9a3f7u, 0xc67178f2u}; + +uint32_t RotateRight(uint32_t value, unsigned bits) { + return (value >> bits) | (value << (32 - bits)); +} + +uint32_t LoadBigEndian(const unsigned char *input) { + return (static_cast(input[0]) << 24) | + (static_cast(input[1]) << 16) | + (static_cast(input[2]) << 8) | + static_cast(input[3]); +} + +void StoreBigEndian(uint32_t value, unsigned char *output) { + output[0] = static_cast(value >> 24); + output[1] = static_cast(value >> 16); + output[2] = static_cast(value >> 8); + output[3] = static_cast(value); +} + +} // namespace + +Sha256::Sha256() + : state_{0x6a09e667u, 0xbb67ae85u, 0x3c6ef372u, 0xa54ff53au, + 0x510e527fu, 0x9b05688cu, 0x1f83d9abu, 0x5be0cd19u} {} + +void Sha256::Transform(const unsigned char block[64]) { + std::array words{}; + for (size_t index = 0; index < 16; ++index) { + words[index] = LoadBigEndian(block + index * 4); + } + for (size_t index = 16; index < words.size(); ++index) { + const uint32_t s0 = RotateRight(words[index - 15], 7) ^ + RotateRight(words[index - 15], 18) ^ + (words[index - 15] >> 3); + const uint32_t s1 = RotateRight(words[index - 2], 17) ^ + RotateRight(words[index - 2], 19) ^ + (words[index - 2] >> 10); + words[index] = words[index - 16] + s0 + words[index - 7] + s1; + } + + uint32_t a = state_[0]; + uint32_t b = state_[1]; + uint32_t c = state_[2]; + uint32_t d = state_[3]; + uint32_t e = state_[4]; + uint32_t f = state_[5]; + uint32_t g = state_[6]; + uint32_t h = state_[7]; + for (size_t index = 0; index < words.size(); ++index) { + const uint32_t sigma1 = RotateRight(e, 6) ^ RotateRight(e, 11) ^ RotateRight(e, 25); + const uint32_t choose = (e & f) ^ (~e & g); + const uint32_t first = h + sigma1 + choose + kRoundConstants[index] + words[index]; + const uint32_t sigma0 = RotateRight(a, 2) ^ RotateRight(a, 13) ^ RotateRight(a, 22); + const uint32_t majority = (a & b) ^ (a & c) ^ (b & c); + const uint32_t second = sigma0 + majority; + h = g; + g = f; + f = e; + e = d + first; + d = c; + c = b; + b = a; + a = first + second; + } + state_[0] += a; + state_[1] += b; + state_[2] += c; + state_[3] += d; + state_[4] += e; + state_[5] += f; + state_[6] += g; + state_[7] += h; +} + +void Sha256::Update(const void *data, size_t size) { + if (finished_) throw std::logic_error("SHA-256 is already finished"); + if (size != 0 && data == nullptr) throw std::invalid_argument("SHA-256 input is NULL"); + const auto *input = static_cast(data); + if (size > UINT64_MAX / 8 - total_bytes_) { + throw std::overflow_error("SHA-256 input is too large"); + } + total_bytes_ += static_cast(size); + + while (size != 0) { + const size_t copied = std::min(size, buffer_.size() - buffered_bytes_); + std::memcpy(buffer_.data() + buffered_bytes_, input, copied); + buffered_bytes_ += copied; + input += copied; + size -= copied; + if (buffered_bytes_ == buffer_.size()) { + Transform(buffer_.data()); + buffered_bytes_ = 0; + } + } +} + +std::array Sha256::Finish() { + if (finished_) throw std::logic_error("SHA-256 is already finished"); + finished_ = true; + const uint64_t bit_length = total_bytes_ * 8; + buffer_[buffered_bytes_++] = 0x80; + if (buffered_bytes_ > 56) { + std::fill(buffer_.begin() + buffered_bytes_, buffer_.end(), 0); + Transform(buffer_.data()); + buffered_bytes_ = 0; + } + std::fill(buffer_.begin() + buffered_bytes_, buffer_.begin() + 56, 0); + for (size_t index = 0; index < 8; ++index) { + buffer_[63 - index] = static_cast(bit_length >> (index * 8)); + } + Transform(buffer_.data()); + + std::array digest{}; + for (size_t index = 0; index < state_.size(); ++index) { + StoreBigEndian(state_[index], digest.data() + index * 4); + } + return digest; +} + +std::string Hex(const std::array &digest) { + static constexpr char kHex[] = "0123456789abcdef"; + std::string result(64, '0'); + for (size_t index = 0; index < digest.size(); ++index) { + result[index * 2] = kHex[digest[index] >> 4]; + result[index * 2 + 1] = kHex[digest[index] & 0x0f]; + } + return result; +} + +std::string Sha256Hex(const void *data, size_t size) { + Sha256 hash; + hash.Update(data, size); + return Hex(hash.Finish()); +} + +} // namespace vexfs diff --git a/agent_files/mount/common/src/vexfs_mount_contract_sqlite.cpp b/agent_files/mount/common/src/vexfs_mount_contract_sqlite.cpp index 1e005393b1..5b0922b7ec 100644 --- a/agent_files/mount/common/src/vexfs_mount_contract_sqlite.cpp +++ b/agent_files/mount/common/src/vexfs_mount_contract_sqlite.cpp @@ -1,9 +1,11 @@ -#include "vexfs_mount_contract.h" +#include "vexfs_runtime_admin.h" #include "sqlite3.h" +#include "agent_files/vexfs_sqlite.h" #include "vexdb_sqlite.h" #include +#include #include #include #include @@ -26,15 +28,31 @@ struct vexfs_mount_session { sqlite3 *db = nullptr; std::string workspace; + std::string principal; std::string session_id; bool exclusive_gateway = false; bool session_started = false; + bool no_create = false; bool synchronous_full = true; + std::chrono::steady_clock::time_point next_heartbeat{}; + uint64_t contract_calls = 0; + uint64_t ordinary_mutation_calls = 0; + uint64_t full_boundary_calls = 0; + uint64_t synchronous_mode_switches = 0; + uint64_t durability_barriers = 0; + int64_t observed_data_version = -1; + int64_t observed_workspace_head = 0; + uint64_t cache_generation = 1; + uint64_t mutation_epoch = 0; + uint64_t observed_mutation_epoch = 0; + bool visibility_initialized = false; std::recursive_mutex mutex; }; namespace { +constexpr int kUnsupportedBackend = -1001; + class CallError : public std::runtime_error { public: CallError(int code, const std::string &message) : std::runtime_error(message), code(code) {} @@ -61,6 +79,7 @@ class Call { Check(sqlite3_bind_int64(statement_, index, value)); } void Int(int index, int value) { Check(sqlite3_bind_int(statement_, index, value)); } + void Null(int index) { Check(sqlite3_bind_null(statement_, index)); } void Blob(int index, const void *data, uint64_t size) { if (size > static_cast(std::numeric_limits::max())) { throw CallError(SQLITE_TOOBIG, "buffer is too large"); @@ -103,11 +122,15 @@ class Call { void ClearError(vexfs_mount_error *error) { if (error == nullptr) return; error->status = VEXFS_MOUNT_OK; - error->sqlite_code = SQLITE_OK; + error->native_code = SQLITE_OK; + std::snprintf(error->backend, sizeof(error->backend), "%s", + VEXFS_RUNTIME_BACKEND_SQLITE); error->message[0] = '\0'; } vexfs_mount_status MapStatus(int sqlite_code, const std::string &) { + if (sqlite_code == kUnsupportedBackend) return VEXFS_MOUNT_UNSUPPORTED; + if (sqlite_code == VEXFS_SQLITE_CONSTRAINT_NOT_EMPTY) return VEXFS_MOUNT_NOT_EMPTY; const int primary = sqlite_code & 0xff; if (primary == SQLITE_BUSY || primary == SQLITE_LOCKED) return VEXFS_MOUNT_BUSY; if (primary == SQLITE_MISUSE || primary == SQLITE_RANGE || primary == SQLITE_TOOBIG || @@ -126,7 +149,9 @@ vexfs_mount_status SetError(vexfs_mount_error *error, int sqlite_code, const vexfs_mount_status status = MapStatus(sqlite_code, message); if (error != nullptr) { error->status = status; - error->sqlite_code = sqlite_code; + error->native_code = sqlite_code; + std::snprintf(error->backend, sizeof(error->backend), "%s", + VEXFS_RUNTIME_BACKEND_SQLITE); std::snprintf(error->message, sizeof(error->message), "%s", message.c_str()); } return status; @@ -145,7 +170,7 @@ vexfs_mount_status Guard(vexfs_mount_error *error, Function function) { } catch (const std::exception &exception) { return SetError(error, SQLITE_ERROR, exception.what()); } catch (...) { - return SetError(error, SQLITE_ERROR, "unknown mount contract error"); + return SetError(error, SQLITE_ERROR, "unknown VexFS runtime error"); } } @@ -155,6 +180,7 @@ vexfs_mount_status Guard(vexfs_mount_session *session, vexfs_mount_error *error, return Guard(error, [&] { if (session == nullptr) throw CallError(SQLITE_MISUSE, "mount session is NULL"); std::lock_guard lock(session->mutex); + ++session->contract_calls; function(); }); } @@ -164,39 +190,94 @@ void RequireSession(vexfs_mount_session *session) { throw CallError(SQLITE_MISUSE, "mount session is NULL"); } if (session->exclusive_gateway && session->session_started) { + const auto now = std::chrono::steady_clock::now(); + if (now < session->next_heartbeat) return; Call heartbeat(session->db, "SELECT vexfs_mount_session_heartbeat(?1,?2)"); heartbeat.Text(1, session->workspace.c_str()); heartbeat.Text(2, session->session_id.c_str()); heartbeat.Row(); + // The database lease is 30 seconds. Refreshing every 10 seconds keeps + // the safety margin without turning every filesystem callback into an + // extra SQLite write transaction. + session->next_heartbeat = now + std::chrono::seconds(10); } } #if !defined(_WIN32) -void PreparePrivateDatabase(const char *path, bool no_create) { +int OpenPrivateFile(const char *path, bool writable) { + int flags = (writable ? O_RDWR : O_RDONLY) | O_NONBLOCK; +#if defined(O_CLOEXEC) + flags |= O_CLOEXEC; +#endif +#if defined(O_NOFOLLOW) + flags |= O_NOFOLLOW; +#endif + return open(path, flags); +} + +void ValidatePrivateFileDescriptor(int descriptor, const std::string &path, + bool protect_mode) { struct stat info {}; - if (stat(path, &info) == 0) { - if (info.st_uid != geteuid()) { - throw CallError(SQLITE_AUTH, "database is not owned by the current user"); - } - if (!no_create && chmod(path, 0600) != 0) { - throw CallError(SQLITE_AUTH, std::string("cannot protect database: ") + std::strerror(errno)); + if (fstat(descriptor, &info) != 0) { + throw CallError(SQLITE_CANTOPEN, + "cannot inspect opened database file: " + path); + } + if (!S_ISREG(info.st_mode)) { + throw CallError(SQLITE_AUTH, "database path must be a regular file"); + } + if (info.st_uid != geteuid()) { + throw CallError(SQLITE_AUTH, "database is not owned by the current user"); + } + if (protect_mode && fchmod(descriptor, 0600) != 0) { + throw CallError(SQLITE_AUTH, + "cannot protect database file: " + path + ": " + + std::strerror(errno)); + } +} + +void PreparePrivateDatabase(const char *path, bool no_create) { + int descriptor = OpenPrivateFile(path, !no_create); + if (descriptor >= 0) { + try { + ValidatePrivateFileDescriptor(descriptor, path, !no_create); + } catch (...) { + close(descriptor); + throw; } + close(descriptor); return; } + if (errno == ELOOP) { + throw CallError(SQLITE_AUTH, "database path must not be a symbolic link"); + } if (errno != ENOENT) { throw CallError(SQLITE_CANTOPEN, std::string("cannot inspect database: ") + std::strerror(errno)); } if (no_create) throw CallError(SQLITE_CANTOPEN, "database does not exist"); - const int descriptor = open(path, O_CREAT | O_EXCL | O_RDWR, 0600); - if (descriptor < 0) { + int flags = O_CREAT | O_EXCL | O_RDWR; +#if defined(O_NOFOLLOW) + flags |= O_NOFOLLOW; +#endif + const int created_descriptor = open(path, flags, 0600); + if (created_descriptor < 0) { throw CallError(SQLITE_CANTOPEN, std::string("cannot create database: ") + std::strerror(errno)); } - close(descriptor); + close(created_descriptor); } void ProtectSidecar(const std::string &path) { - struct stat info {}; - if (stat(path.c_str(), &info) == 0 && info.st_uid == geteuid()) chmod(path.c_str(), 0600); + const int descriptor = OpenPrivateFile(path.c_str(), true); + if (descriptor < 0) { + if (errno == ENOENT) return; + throw CallError(SQLITE_AUTH, "SQLite sidecar is not a private regular file: " + path); + } + try { + ValidatePrivateFileDescriptor(descriptor, path, true); + } catch (...) { + close(descriptor); + throw; + } + close(descriptor); } #else void PreparePrivateDatabase(const char *, bool) {} @@ -300,18 +381,42 @@ void Exec(sqlite3 *db, const char *sql) { } } -void UseStagingDurability(vexfs_mount_session *session) { +void UseOrdinaryDurability(vexfs_mount_session *session) { + ++session->ordinary_mutation_calls; + ++session->mutation_epoch; if (!session->synchronous_full) return; Exec(session->db, "PRAGMA synchronous=NORMAL"); session->synchronous_full = false; + ++session->synchronous_mode_switches; } void UseFullDurability(vexfs_mount_session *session) { + ++session->full_boundary_calls; if (session->synchronous_full) return; - // publish/synchronize 前切回 FULL。这个权威事务会同步同一 WAL 中此前已提交的 - // staging frame；成功返回后，最终文件和它依赖的 staging 都达到 FULL 级别。 + // fsync/synchronize/snapshot 前切回 FULL。随后必须执行真实写事务；仅修改 + // PRAGMA 不会形成持久化屏障。 Exec(session->db, "PRAGMA synchronous=FULL"); session->synchronous_full = true; + ++session->synchronous_mode_switches; +} + +void DurabilityBarrier(vexfs_mount_session *session) { + Call barrier(session->db, + "INSERT INTO _vexfs_meta(key,value) VALUES('durability_epoch','1') " + "ON CONFLICT(key) DO UPDATE SET value=CAST(CAST(value AS INTEGER)+1 AS TEXT) " + "RETURNING value"); + barrier.Row(); + ++session->durability_barriers; +} + +const char *EffectiveRequestId(vexfs_mount_session *session, const char *request_id, + bool safe_without_persistent_replay) { + if (request_id == nullptr) throw CallError(SQLITE_MISUSE, "request_id is NULL"); + // FSKit/libfuse generate a fresh UUID for every callback, so persisting + // stage/publish/sync UUIDs cannot deduplicate an OS retry. These operations + // are generation-idempotent. Keep open/close/append requests persisted. + if (session->exclusive_gateway && safe_without_persistent_replay) return ""; + return request_id; } std::string RandomSessionId() { @@ -339,23 +444,35 @@ extern "C" vexfs_mount_status vexfs_mount_session_open(const vexfs_mount_config vexfs_mount_error *error) { if (output != nullptr) *output = nullptr; return Guard(error, [&] { - if (config == nullptr || output == nullptr || config->database_path == nullptr || - config->workspace == nullptr) { - throw CallError(SQLITE_MISUSE, "config, output, database_path and workspace are required"); + if (config == nullptr || output == nullptr || config->backend == nullptr || + config->connection == nullptr || config->workspace == nullptr || + config->principal == nullptr) { + throw CallError(SQLITE_MISUSE, + "config, output, backend, connection, workspace and principal are required"); + } + if (config->abi_version != VEXFS_RUNTIME_ABI_VERSION) { + throw CallError(SQLITE_MISUSE, "unsupported runtime ABI version"); + } + if (std::strcmp(config->backend, VEXFS_RUNTIME_BACKEND_SQLITE) != 0) { + throw CallError(kUnsupportedBackend, + "this adapter only supports the sqlite backend"); } - if (config->abi_version != VEXFS_MOUNT_ABI_VERSION) { - throw CallError(SQLITE_MISUSE, "unsupported mount ABI version"); + if (std::strcmp(config->principal, "local") != 0) { + throw CallError(SQLITE_AUTH, + "sqlite backend is single-user and only accepts the local principal"); } vexfs_mount_session *session = new vexfs_mount_session(); session->workspace = config->workspace; + session->principal = config->principal; session->session_id = RandomSessionId(); - session->exclusive_gateway = (config->flags & VEXFS_MOUNT_EXCLUSIVE_GATEWAY) != 0; - const bool no_create = (config->flags & VEXFS_MOUNT_OPEN_NO_CREATE) != 0; - PreparePrivateDatabase(config->database_path, no_create); - int rc = sqlite3_open_v2(config->database_path, &session->db, + session->exclusive_gateway = (config->flags & VEXFS_RUNTIME_EXCLUSIVE_GATEWAY) != 0; + const bool no_create = (config->flags & VEXFS_RUNTIME_OPEN_NO_CREATE) != 0; + session->no_create = no_create; + PreparePrivateDatabase(config->connection, no_create); + int sqlite_open_flags = (no_create ? SQLITE_OPEN_READWRITE : SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE) | - SQLITE_OPEN_FULLMUTEX, - nullptr); + SQLITE_OPEN_FULLMUTEX; + int rc = sqlite3_open_v2(config->connection, &session->db, sqlite_open_flags, nullptr); if (rc != SQLITE_OK) { const std::string message = session->db == nullptr ? "cannot open database" : sqlite3_errmsg(session->db); @@ -364,25 +481,28 @@ extern "C" vexfs_mount_status vexfs_mount_session_open(const vexfs_mount_config throw CallError(rc, message); } try { + rc = sqlite3_extended_result_codes(session->db, 1); + if (rc != SQLITE_OK) throw CallError(rc, sqlite3_errmsg(session->db)); sqlite3_busy_timeout(session->db, - static_cast(config->busy_timeout_ms == 0 ? 5000 : config->busy_timeout_ms)); + static_cast(config->operation_timeout_ms == 0 + ? 5000 : config->operation_timeout_ms)); rc = vexdb_sqlite_register(session->db); if (rc != SQLITE_OK) throw CallError(rc, sqlite3_errmsg(session->db)); if (no_create) { - // doctor 必须保持只读，不初始化、不迁移，也不在关闭时执行 session_end。 + // doctor/check 必须保持只读，不初始化、不迁移，也不在关闭时执行 session_end。 Exec(session->db, "PRAGMA foreign_keys=ON;"); } else { Exec(session->db, "PRAGMA journal_mode=WAL; PRAGMA synchronous=FULL; PRAGMA foreign_keys=ON;"); - ProtectSidecar(std::string(config->database_path) + "-wal"); - ProtectSidecar(std::string(config->database_path) + "-shm"); + ProtectSidecar(std::string(config->connection) + "-wal"); + ProtectSidecar(std::string(config->connection) + "-shm"); Call init(session->db, "SELECT vexfs_init()"); init.Row(); Call workspace(session->db, "SELECT vexfs_workspace_create(?1)"); workspace.Text(1, config->workspace); workspace.Row(); - ProtectSidecar(std::string(config->database_path) + "-wal"); - ProtectSidecar(std::string(config->database_path) + "-shm"); + ProtectSidecar(std::string(config->connection) + "-wal"); + ProtectSidecar(std::string(config->connection) + "-shm"); if (session->exclusive_gateway) { CheckExistingLocalSession(session->db, config->workspace, session->session_id); @@ -391,9 +511,34 @@ extern "C" vexfs_mount_status vexfs_mount_session_open(const vexfs_mount_config start.Text(2, session->session_id.c_str()); start.Row(); session->session_started = true; + // A previous gateway may have died after staging writes but before + // close. SessionStart claims those retained handles for this new + // session; publish them before exposing the remounted workspace. + Call recover(session->db, + "SELECT vexfs_mount_synchronize(?1,?2,?3)"); + recover.Text(1, config->workspace); + recover.Text(2, ""); + recover.Text(3, session->session_id.c_str()); + recover.Row(); + session->next_heartbeat = + std::chrono::steady_clock::now() + std::chrono::seconds(10); } + // Ordinary filesystem callbacks use WAL/NORMAL. Explicit + // synchronize, fsync, snapshot and safe close switch to FULL. + Exec(session->db, "PRAGMA synchronous=NORMAL"); + session->synchronous_full = false; + ++session->synchronous_mode_switches; } } catch (...) { + if (session->session_started) { + try { + Call end(session->db, "SELECT vexfs_mount_session_end(?1,?2)"); + end.Text(1, session->workspace.c_str()); + end.Text(2, session->session_id.c_str()); + end.Row(); + } catch (...) { + } + } sqlite3_close(session->db); delete session; throw; @@ -407,13 +552,24 @@ extern "C" void vexfs_mount_session_close(vexfs_mount_session *session) { if (session->db != nullptr) { std::lock_guard lock(session->mutex); try { - if (session->session_started) { + if (!session->no_create) { UseFullDurability(session); - Call end(session->db, "SELECT vexfs_mount_session_end(?1,?2)"); - end.Text(1, session->workspace.c_str()); - end.Text(2, session->session_id.c_str()); - end.Row(); - session->session_started = false; + Call synchronize(session->db, session->session_started + ? "SELECT vexfs_mount_synchronize(?1,'',?2)" + : "SELECT vexfs_mount_synchronize(?1,'')"); + synchronize.Text(1, session->workspace.c_str()); + if (session->session_started) { + synchronize.Text(2, session->session_id.c_str()); + } + synchronize.Row(); + DurabilityBarrier(session); + if (session->session_started) { + Call end(session->db, "SELECT vexfs_mount_session_end(?1,?2)"); + end.Text(1, session->workspace.c_str()); + end.Text(2, session->session_id.c_str()); + end.Row(); + session->session_started = false; + } } } catch (...) { // close 不能抛异常；未发布内容仍留在数据库中，后续可诊断和回收。 @@ -428,8 +584,7 @@ extern "C" vexfs_mount_status vexfs_mount_diagnostics(vexfs_mount_session *sessi vexfs_mount_error *error) { return Guard(session, error, [&] { RequireSession(session); - // doctor 以只读方式打开数据库，不能为了诊断偷偷迁移。这里读取数据库自身 - // 的合同版本，并兼容 0.1 staging 列布局。 + // doctor 以只读方式打开数据库，不初始化也不改写 schema。 Call contract(session->db, "SELECT value FROM _vexfs_meta WHERE key='contract_version'"); contract.Row(); @@ -468,21 +623,33 @@ extern "C" vexfs_mount_status vexfs_mount_diagnostics(vexfs_mount_session *sessi staging.Row(); const char *filename = sqlite3_db_filename(session->db, "main"); const std::string contract_version = contract.ResultText(); - const bool compatible = contract_version == "0.3.0"; - const bool upgrade_required = contract_version == "0.1.0" || contract_version == "0.2.0"; + const bool schema_ready = contract_version == "0.7.0"; const std::string value = - "{\"contract_version\":\"" + JsonEscape(contract_version) + - "\",\"compatible\":" + (compatible ? "true" : "false") + - ",\"upgrade_required\":" + (upgrade_required ? "true" : "false") + - ",\"database_path\":\"" + JsonEscape(filename == nullptr ? "" : filename) + + "{\"schema_version\":\"" + JsonEscape(contract_version) + + "\",\"schema_ready\":" + (schema_ready ? "true" : "false") + + ",\"backend\":\"sqlite\",\"connection\":\"" + + JsonEscape(filename == nullptr ? "" : filename) + "\",\"workspace\":\"" + JsonEscape(session->workspace) + + "\",\"security_mode\":\"single-user\",\"principal\":\"" + + JsonEscape(session->principal) + "\",\"workspace_exists\":" + std::to_string(workspace_exists.ResultInt64()) + ",\"journal_mode\":\"" + JsonEscape(journal.ResultText()) + "\",\"synchronous\":" + std::to_string(synchronous.ResultInt64()) + ",\"foreign_keys\":" + std::to_string(foreign_keys.ResultInt64()) + ",\"pending_handles\":" + std::to_string(pending.ResultInt64()) + ",\"retained_handles\":" + std::to_string(retained.ResultInt64()) + - ",\"staging_bytes\":" + std::to_string(staging.ResultInt64()) + "}"; + ",\"staging_bytes\":" + std::to_string(staging.ResultInt64()) + + ",\"contract_calls\":" + std::to_string(session->contract_calls) + + ",\"ordinary_mutation_calls\":" + + std::to_string(session->ordinary_mutation_calls) + + ",\"full_boundary_calls\":" + + std::to_string(session->full_boundary_calls) + + ",\"synchronous_mode_switches\":" + + std::to_string(session->synchronous_mode_switches) + + ",\"durability_barriers\":" + + std::to_string(session->durability_barriers) + + ",\"sqlite_total_changes\":" + + std::to_string(sqlite3_total_changes64(session->db)) + "}"; CopyString(value, json); }); } @@ -492,19 +659,155 @@ extern "C" vexfs_mount_status vexfs_mount_mkdir(vexfs_mount_session *session, vexfs_mount_error *error) { return Guard(session, error, [&] { RequireSession(session); - UseFullDurability(session); + UseOrdinaryDurability(session); Call call(session->db, "SELECT vexfs_mkdir(?1,?2)"); call.Text(1, session->workspace.c_str()); call.Text(2, path); call.Row(); }); } +extern "C" vexfs_mount_status vexfs_mount_create(vexfs_mount_session *session, + const char *path, const char *kind, + uint32_t mode, + vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + UseOrdinaryDurability(session); + if (path == nullptr || kind == nullptr || mode > 0777) { + throw CallError(SQLITE_MISUSE, "path, kind and mode 0..0777 are required"); + } + Call call(session->db, "SELECT vexfs_create(?1,?2,?3,?4)"); + call.Text(1, session->workspace.c_str()); + call.Text(2, path); + call.Text(3, kind); + call.Int(4, static_cast(mode)); + call.Row(); + }); +} + +extern "C" vexfs_mount_status vexfs_mount_set_mode(vexfs_mount_session *session, + int64_t inode, uint32_t mode, + vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + UseOrdinaryDurability(session); + if (inode <= 0 || mode > 0777) { + throw CallError(SQLITE_MISUSE, "positive inode and mode 0..0777 are required"); + } + Call call(session->db, "SELECT vexfs_set_mode(?1,?2,?3)"); + call.Text(1, session->workspace.c_str()); + call.Int64(2, inode); + call.Int(3, static_cast(mode)); + call.Row(); + }); +} + +extern "C" vexfs_mount_status vexfs_mount_set_times(vexfs_mount_session *session, + int64_t inode, + int64_t accessed_at_ms, + int64_t modified_at_ms, + uint32_t mask, + vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + UseOrdinaryDurability(session); + if (inode <= 0 || accessed_at_ms < 0 || modified_at_ms < 0 || + mask == 0 || (mask & ~(VEXFS_MOUNT_TIME_ACCESS | VEXFS_MOUNT_TIME_MODIFY)) != 0) { + throw CallError(SQLITE_MISUSE, + "positive inode, non-negative times and a valid mask are required"); + } + Call call(session->db, "SELECT vexfs_set_times(?1,?2,?3,?4,?5)"); + call.Text(1, session->workspace.c_str()); + call.Int64(2, inode); + call.Int64(3, accessed_at_ms); + call.Int64(4, modified_at_ms); + call.Int(5, static_cast(mask)); + call.Row(); + }); +} + +extern "C" vexfs_mount_status vexfs_mount_symlink(vexfs_mount_session *session, + const char *path, const void *target, + uint64_t target_size, + vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + UseOrdinaryDurability(session); + if (path == nullptr || target == nullptr || target_size == 0 || + target_size > VEXFS_MOUNT_SYMLINK_MAX) { + throw CallError(SQLITE_MISUSE, + "path and symlink target of 1..4096 bytes are required"); + } + Call call(session->db, "SELECT vexfs_symlink(?1,?2,?3)"); + call.Text(1, session->workspace.c_str()); + call.Text(2, path); + call.Blob(3, target, target_size); + call.Row(); + }); +} + +extern "C" vexfs_mount_status vexfs_mount_readlink(vexfs_mount_session *session, + int64_t inode, + vexfs_mount_bytes *target, + vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + if (inode <= 0 || target == nullptr) { + throw CallError(SQLITE_MISUSE, "positive inode and target output are required"); + } + Call call(session->db, "SELECT vexfs_readlink(?1,?2)"); + call.Text(1, session->workspace.c_str()); + call.Int64(2, inode); + call.Row(); + CopyResult(call, target); + }); +} + +extern "C" vexfs_mount_status vexfs_mount_link(vexfs_mount_session *session, + const char *source, + const char *destination, + vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + UseOrdinaryDurability(session); + if (source == nullptr || destination == nullptr) { + throw CallError(SQLITE_MISUSE, "source and destination are required"); + } + Call call(session->db, "SELECT vexfs_link(?1,?2,?3)"); + call.Text(1, session->workspace.c_str()); + call.Text(2, source); + call.Text(3, destination); + call.Row(); + }); +} + +extern "C" vexfs_mount_status vexfs_mount_chown(vexfs_mount_session *session, + int64_t inode, int64_t uid, int64_t gid, + vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + UseOrdinaryDurability(session); + if (inode <= 0 || uid < -1 || gid < -1 || + uid > static_cast(UINT32_MAX) || + gid > static_cast(UINT32_MAX)) { + throw CallError(SQLITE_MISUSE, + "positive inode and uid/gid in -1..4294967295 are required"); + } + Call call(session->db, "SELECT vexfs_chown(?1,?2,?3,?4)"); + call.Text(1, session->workspace.c_str()); + call.Int64(2, inode); + call.Int64(3, uid); + call.Int64(4, gid); + call.Row(); + }); +} + extern "C" vexfs_mount_status vexfs_mount_write_file(vexfs_mount_session *session, const char *path, const void *data, uint64_t size, int64_t *version, vexfs_mount_error *error) { return Guard(session, error, [&] { RequireSession(session); - UseFullDurability(session); + UseOrdinaryDurability(session); if (version == nullptr) throw CallError(SQLITE_MISUSE, "version output is NULL"); Call call(session->db, "SELECT vexfs_write(?1,?2,?3)"); call.Text(1, session->workspace.c_str()); call.Text(2, path); call.Blob(3, data, size); @@ -512,6 +815,26 @@ extern "C" vexfs_mount_status vexfs_mount_write_file(vexfs_mount_session *sessio }); } +extern "C" vexfs_mount_status vexfs_mount_append_file(vexfs_mount_session *session, + int64_t inode, const void *data, + uint64_t size, int64_t *version, + vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + UseOrdinaryDurability(session); + if (inode <= 0 || version == nullptr || (size > 0 && data == nullptr)) { + throw CallError(SQLITE_MISUSE, + "positive inode, content and version output are required"); + } + Call call(session->db, "SELECT vexfs_append(?1,?2,?3)"); + call.Text(1, session->workspace.c_str()); + call.Int64(2, inode); + call.Blob(3, data, size); + call.Row(); + *version = call.ResultInt64(); + }); +} + extern "C" vexfs_mount_status vexfs_mount_read_file(vexfs_mount_session *session, const char *path, vexfs_mount_bytes *content, @@ -524,6 +847,63 @@ extern "C" vexfs_mount_status vexfs_mount_read_file(vexfs_mount_session *session }); } +extern "C" vexfs_mount_status vexfs_mount_grep(vexfs_mount_session *session, + const char *path, const char *pattern, + uint32_t flags, uint32_t limit, + vexfs_mount_bytes *json, + vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + if (path == nullptr || pattern == nullptr || pattern[0] == '\0' || limit == 0 || + limit > 10240 || (flags & ~(VEXFS_MOUNT_GREP_IGNORE_CASE | + VEXFS_MOUNT_GREP_FILES_ONLY)) != 0) { + throw CallError(SQLITE_RANGE, + "grep needs path, a non-empty pattern, valid flags and limit 1..10240"); + } + Call call(session->db, "SELECT vexfs_grep(?1,?2,?3,?4,?5)"); + call.Text(1, session->workspace.c_str()); + call.Text(2, path); + call.Text(3, pattern); + call.Int64(4, static_cast(flags)); + call.Int64(5, static_cast(limit)); + call.Row(); + CopyResult(call, json); + }); +} + +extern "C" vexfs_mount_status vexfs_mount_check(vexfs_mount_session *session, + uint32_t flags, + vexfs_mount_bytes *json, + vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + if ((flags & ~VEXFS_MOUNT_CHECK_QUICK) != 0) { + throw CallError(SQLITE_RANGE, "check flags are invalid"); + } + Call call(session->db, "SELECT vexfs_check(?1,?2)"); + call.Text(1, session->workspace.c_str()); + call.Int(2, (flags & VEXFS_MOUNT_CHECK_QUICK) == 0 ? 1 : 0); + call.Row(); + CopyResult(call, json); + }); +} + +extern "C" vexfs_mount_status vexfs_mount_grep_index(vexfs_mount_session *session, + const char *action, + vexfs_mount_bytes *json, + vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + if (action == nullptr || action[0] == '\0') { + throw CallError(SQLITE_MISUSE, "index action is required"); + } + Call call(session->db, "SELECT vexfs_grep_index(?1)"); + call.Text(1, action); + call.Row(); + CopyResult(call, json); + }); +} + extern "C" vexfs_mount_status vexfs_mount_history(vexfs_mount_session *session, const char *path, vexfs_mount_bytes *json, @@ -644,12 +1024,31 @@ extern "C" vexfs_mount_status vexfs_mount_list(vexfs_mount_session *session, }); } +extern "C" vexfs_mount_status vexfs_mount_list_versioned( + vexfs_mount_session *session, const char *path, vexfs_mount_bytes *json, + vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + if (path == nullptr || json == nullptr) { + throw CallError(SQLITE_MISUSE, "path and output are required"); + } + Call call(session->db, + "SELECT json_object('version'," + "json_extract(vexfs_stat(?1,?2),'$.version')," + "'entries',json(vexfs_list(?1,?2)))"); + call.Text(1, session->workspace.c_str()); + call.Text(2, path); + call.Row(); + CopyResult(call, json); + }); +} + extern "C" vexfs_mount_status vexfs_mount_move(vexfs_mount_session *session, const char *source, const char *destination, vexfs_mount_error *error) { return Guard(session, error, [&] { RequireSession(session); - UseFullDurability(session); + UseOrdinaryDurability(session); Call call(session->db, "SELECT vexfs_move(?1,?2,?3)"); call.Text(1, session->workspace.c_str()); call.Text(2, source); call.Text(3, destination); call.Row(); }); @@ -661,7 +1060,7 @@ extern "C" vexfs_mount_status vexfs_mount_rename(vexfs_mount_session *session, vexfs_mount_error *error) { return Guard(session, error, [&] { RequireSession(session); - UseFullDurability(session); + UseOrdinaryDurability(session); Call call(session->db, "SELECT vexfs_rename(?1,?2,?3,?4)"); call.Text(1, session->workspace.c_str()); call.Text(2, source); @@ -676,12 +1075,273 @@ extern "C" vexfs_mount_status vexfs_mount_remove(vexfs_mount_session *session, vexfs_mount_error *error) { return Guard(session, error, [&] { RequireSession(session); - UseFullDurability(session); + UseOrdinaryDurability(session); Call call(session->db, "SELECT vexfs_remove(?1,?2,?3)"); call.Text(1, session->workspace.c_str()); call.Text(2, path); call.Int(3, recursive); call.Row(); }); } +extern "C" vexfs_mount_status vexfs_mount_xattr_get(vexfs_mount_session *session, + int64_t inode, const char *name, + vexfs_mount_bytes *value, + vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + if (inode <= 0 || name == nullptr || value == nullptr) { + throw CallError(SQLITE_MISUSE, "inode, xattr name and output are required"); + } + Call call(session->db, "SELECT vexfs_xattr_get(?1,?2,?3)"); + call.Text(1, session->workspace.c_str()); + call.Int64(2, inode); + call.Text(3, name); + call.Row(); + CopyResult(call, value); + }); +} + +extern "C" vexfs_mount_status vexfs_mount_xattr_list(vexfs_mount_session *session, + int64_t inode, + vexfs_mount_bytes *json, + vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + if (inode <= 0 || json == nullptr) { + throw CallError(SQLITE_MISUSE, "inode and output are required"); + } + Call call(session->db, "SELECT vexfs_xattr_list(?1,?2)"); + call.Text(1, session->workspace.c_str()); + call.Int64(2, inode); + call.Row(); + CopyResult(call, json); + }); +} + +extern "C" vexfs_mount_status vexfs_mount_xattr_set(vexfs_mount_session *session, + int64_t inode, const char *name, + const void *value, uint64_t size, + int policy, + vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + UseOrdinaryDurability(session); + if (inode <= 0 || name == nullptr || policy < 0 || policy > 3 || + (policy != 3 && value == nullptr && size != 0)) { + throw CallError(SQLITE_MISUSE, "invalid inode, xattr name, value or policy"); + } + Call call(session->db, "SELECT vexfs_xattr_set(?1,?2,?3,?4,?5)"); + call.Text(1, session->workspace.c_str()); + call.Int64(2, inode); + call.Text(3, name); + if (policy == 3) call.Null(4); else call.Blob(4, value, size); + call.Int(5, policy); + call.Row(); + }); +} + +extern "C" vexfs_mount_status vexfs_mount_acl_get(vexfs_mount_session *session, + int64_t inode, vexfs_mount_bytes *json, + vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + if (inode <= 0 || json == nullptr) { + throw CallError(SQLITE_MISUSE, "inode and output are required"); + } + Call call(session->db, "SELECT vexfs_acl_get(?1,?2)"); + call.Text(1, session->workspace.c_str()); + call.Int64(2, inode); + call.Row(); + CopyResult(call, json); + }); +} + +extern "C" vexfs_mount_status vexfs_mount_acl_set(vexfs_mount_session *session, + int64_t inode, const void *json, + uint64_t size, vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + UseOrdinaryDurability(session); + if (inode <= 0 || json == nullptr || size == 0) { + throw CallError(SQLITE_MISUSE, "inode and non-empty ACL JSON are required"); + } + Call call(session->db, "SELECT vexfs_acl_set(?1,?2,?3)"); + call.Text(1, session->workspace.c_str()); + call.Int64(2, inode); + call.Blob(3, json, size); + call.Row(); + }); +} + +extern "C" vexfs_mount_status vexfs_mount_workspace_head(vexfs_mount_session *session, + int64_t *head_commit, + vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + if (head_commit == nullptr) throw CallError(SQLITE_MISUSE, "head output is NULL"); + Call call(session->db, + "SELECT COALESCE(head_commit,0) FROM _vexfs_workspaces WHERE name=?1"); + call.Text(1, session->workspace.c_str()); + call.Row(); + *head_commit = call.ResultInt64(); + }); +} + +extern "C" vexfs_mount_status vexfs_mount_refresh_visibility( + vexfs_mount_session *session, vexfs_mount_visibility *visibility, + vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + if (visibility == nullptr) { + throw CallError(SQLITE_MISUSE, "visibility output is NULL"); + } + Call data_version(session->db, "PRAGMA data_version"); + data_version.Row(); + const int64_t current_data_version = data_version.ResultInt64(); + const bool external_database_commit = session->visibility_initialized && + current_data_version != session->observed_data_version; + const bool local_mutation = session->visibility_initialized && + session->mutation_epoch != session->observed_mutation_epoch; + int external_workspace_commit = 0; + if (!session->visibility_initialized || external_database_commit || local_mutation) { + Call head(session->db, + "SELECT COALESCE(head_commit,0) FROM _vexfs_workspaces WHERE name=?1"); + head.Text(1, session->workspace.c_str()); + head.Row(); + const int64_t current_head = head.ResultInt64(); + if (session->visibility_initialized && + current_head != session->observed_workspace_head) { + ++session->cache_generation; + external_workspace_commit = external_database_commit ? 1 : 0; + } + session->observed_workspace_head = current_head; + session->observed_data_version = current_data_version; + session->observed_mutation_epoch = session->mutation_epoch; + session->visibility_initialized = true; + } + visibility->workspace_head = session->observed_workspace_head; + visibility->cache_generation = session->cache_generation; + visibility->external_commit = external_workspace_commit; + }); +} + +extern "C" vexfs_mount_status vexfs_mount_snapshot_create(vexfs_mount_session *session, + const char *name, + uint32_t flags, + int64_t *commit, + vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + UseFullDurability(session); + if (name == nullptr || commit == nullptr) { + throw CallError(SQLITE_MISUSE, "snapshot name and commit output are required"); + } + if ((flags & ~VEXFS_SNAPSHOT_COMMITTED_ONLY) != 0) { + throw CallError(SQLITE_MISUSE, "unsupported snapshot flags"); + } + const bool committed_only = (flags & VEXFS_SNAPSHOT_COMMITTED_ONLY) != 0; + // Only the owning gateway may publish its live handles. A management + // session first publishes its own handles, then the database-level + // snapshot barrier rejects any dirty handle owned by another session. + // This prevents a successful snapshot from silently missing writes. + if (!committed_only) { + Call synchronize(session->db, + "SELECT vexfs_mount_synchronize(?1,'',?2)"); + synchronize.Text(1, session->workspace.c_str()); + synchronize.Text(2, session->session_id.c_str()); + synchronize.Row(); + } + Call call(session->db, committed_only + ? "SELECT vexfs_snapshot_create(?1,?2,NULL,'committed-only')" + : "SELECT vexfs_snapshot_create(?1,?2)"); + call.Text(1, session->workspace.c_str()); + call.Text(2, name); + call.Row(); + *commit = call.ResultInt64(); + }); +} + +extern "C" vexfs_mount_status vexfs_mount_snapshot_list(vexfs_mount_session *session, + vexfs_mount_bytes *json, + vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + if (json == nullptr) throw CallError(SQLITE_MISUSE, "output is NULL"); + Call call(session->db, "SELECT vexfs_snapshot_list(?1)"); + call.Text(1, session->workspace.c_str()); + call.Row(); + CopyResult(call, json); + }); +} + +extern "C" vexfs_mount_status vexfs_mount_snapshot_show(vexfs_mount_session *session, + const char *name, + vexfs_mount_bytes *json, + vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + if (name == nullptr || json == nullptr) { + throw CallError(SQLITE_MISUSE, "snapshot name and output are required"); + } + Call call(session->db, "SELECT vexfs_snapshot_show(?1,?2)"); + call.Text(1, session->workspace.c_str()); + call.Text(2, name); + call.Row(); + CopyResult(call, json); + }); +} + +extern "C" vexfs_mount_status vexfs_mount_snapshot_diff(vexfs_mount_session *session, + const char *from, const char *to, + vexfs_mount_bytes *json, + vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + if (from == nullptr || to == nullptr || json == nullptr) { + throw CallError(SQLITE_MISUSE, "from, to and output are required"); + } + Call call(session->db, "SELECT vexfs_snapshot_diff(?1,?2,?3)"); + call.Text(1, session->workspace.c_str()); + call.Text(2, from); + call.Text(3, to); + call.Row(); + CopyResult(call, json); + }); +} + +extern "C" vexfs_mount_status vexfs_mount_snapshot_drop(vexfs_mount_session *session, + const char *name, + vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + UseFullDurability(session); + if (name == nullptr) throw CallError(SQLITE_MISUSE, "snapshot name is required"); + Call call(session->db, "SELECT vexfs_snapshot_drop(?1,?2)"); + call.Text(1, session->workspace.c_str()); + call.Text(2, name); + call.Row(); + }); +} + +extern "C" vexfs_mount_status vexfs_mount_snapshot_restore(vexfs_mount_session *session, + const char *name, + int64_t expected_head, + int64_t *new_commit, + vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + UseFullDurability(session); + if (name == nullptr || expected_head <= 0 || new_commit == nullptr) { + throw CallError(SQLITE_MISUSE, + "snapshot name, positive expected head and commit output are required"); + } + Call call(session->db, "SELECT vexfs_snapshot_restore(?1,?2,?3)"); + call.Text(1, session->workspace.c_str()); + call.Text(2, name); + call.Int64(3, expected_head); + call.Row(); + *new_commit = call.ResultInt64(); + }); +} + extern "C" vexfs_mount_status vexfs_mount_handle_open(vexfs_mount_session *session, const char *path, const char *flags, const char *request_id, @@ -689,14 +1349,36 @@ extern "C" vexfs_mount_status vexfs_mount_handle_open(vexfs_mount_session *sessi vexfs_mount_error *error) { return Guard(session, error, [&] { RequireSession(session); - UseFullDurability(session); + UseOrdinaryDurability(session); Call call(session->db, "SELECT vexfs_handle_open(?1,?2,?3,?4,?5)"); call.Text(1, session->workspace.c_str()); call.Text(2, path); call.Text(3, flags); - call.Text(4, request_id); call.Text(5, session->session_id.c_str()); + call.Text(4, EffectiveRequestId(session, request_id, true)); + call.Text(5, session->session_id.c_str()); call.Row(); CopyResult(call, handle); }); } +extern "C" vexfs_mount_status vexfs_mount_handle_create( + vexfs_mount_session *session, const char *path, uint32_t mode, + const char *request_id, vexfs_mount_bytes *handle, vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + UseOrdinaryDurability(session); + if (path == nullptr || request_id == nullptr || handle == nullptr || mode > 0777) { + throw CallError(SQLITE_MISUSE, + "path, mode 0..0777, request_id and handle output are required"); + } + Call call(session->db, "SELECT vexfs_handle_create(?1,?2,?3,?4,?5)"); + call.Text(1, session->workspace.c_str()); + call.Text(2, path); + call.Int(3, static_cast(mode)); + call.Text(4, EffectiveRequestId(session, request_id, true)); + call.Text(5, session->session_id.c_str()); + call.Row(); + CopyResult(call, handle); + }); +} + extern "C" vexfs_mount_status vexfs_mount_handle_truncate( vexfs_mount_session *session, const char *handle, uint64_t size, const char *request_id, int64_t *generation, vexfs_mount_error *error) { @@ -706,11 +1388,11 @@ extern "C" vexfs_mount_status vexfs_mount_handle_truncate( generation == nullptr) { throw CallError(SQLITE_MISUSE, "size is too large or generation output is NULL"); } - UseStagingDurability(session); + UseOrdinaryDurability(session); Call call(session->db, "SELECT vexfs_handle_truncate(?1,?2,?3)"); call.Text(1, handle); call.Int64(2, static_cast(size)); - call.Text(3, request_id); + call.Text(3, EffectiveRequestId(session, request_id, true)); call.Row(); *generation = call.ResultInt64(); }); @@ -724,17 +1406,37 @@ extern "C" vexfs_mount_status vexfs_mount_handle_stage_write( if (offset > static_cast(std::numeric_limits::max()) || generation == nullptr) { throw CallError(SQLITE_MISUSE, "offset is too large or generation output is NULL"); } - UseStagingDurability(session); + UseOrdinaryDurability(session); Call call(session->db, "SELECT vexfs_handle_stage_write(?1,?2,?3,?4)"); call.Text(1, handle); call.Int64(2, static_cast(offset)); call.Blob(3, data, size); - call.Text(4, request_id); + call.Text(4, EffectiveRequestId(session, request_id, true)); call.Row(); *generation = call.ResultInt64(); }); } +extern "C" vexfs_mount_status vexfs_mount_handle_append( + vexfs_mount_session *session, const char *handle, const void *data, uint64_t size, + const char *request_id, int64_t *version, vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + if (handle == nullptr || request_id == nullptr || version == nullptr || + (size > 0 && data == nullptr)) { + throw CallError(SQLITE_MISUSE, + "handle, content, request_id and version output are required"); + } + UseOrdinaryDurability(session); + Call call(session->db, "SELECT vexfs_handle_append(?1,?2,?3)"); + call.Text(1, handle); + call.Blob(2, data, size); + call.Text(3, request_id); + call.Row(); + *version = call.ResultInt64(); + }); +} + extern "C" vexfs_mount_status vexfs_mount_handle_read(vexfs_mount_session *session, const char *handle, uint64_t offset, uint64_t length, @@ -758,11 +1460,36 @@ extern "C" vexfs_mount_status vexfs_mount_handle_publish( vexfs_mount_error *error) { return Guard(session, error, [&] { RequireSession(session); - UseFullDurability(session); + if (durability == nullptr) { + throw CallError(SQLITE_MISUSE, "durability is NULL"); + } + if (std::strcmp(durability, "full") == 0) UseFullDurability(session); + else UseOrdinaryDurability(session); if (version == nullptr) throw CallError(SQLITE_MISUSE, "version output is NULL"); Call call(session->db, "SELECT vexfs_handle_publish(?1,?2,?3,?4)"); call.Text(1, handle); call.Int64(2, generation); call.Text(3, durability); - call.Text(4, request_id); call.Row(); *version = call.ResultInt64(); + call.Text(4, EffectiveRequestId(session, request_id, true)); + call.Row(); *version = call.ResultInt64(); + }); +} + +extern "C" vexfs_mount_status vexfs_mount_handle_publish_close( + vexfs_mount_session *session, const char *handle, int64_t generation, + const char *durability, int64_t *version, vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + if (handle == nullptr || durability == nullptr || version == nullptr) { + throw CallError(SQLITE_MISUSE, + "handle, durability and version output are required"); + } + if (std::strcmp(durability, "full") == 0) UseFullDurability(session); + else UseOrdinaryDurability(session); + Call call(session->db, "SELECT vexfs_handle_publish_close(?1,?2,?3)"); + call.Text(1, handle); + call.Int64(2, generation); + call.Text(3, durability); + call.Row(); + *version = call.ResultInt64(); }); } @@ -771,9 +1498,10 @@ extern "C" vexfs_mount_status vexfs_mount_handle_close( const char *request_id, vexfs_mount_bytes *state, vexfs_mount_error *error) { return Guard(session, error, [&] { RequireSession(session); - UseFullDurability(session); + UseOrdinaryDurability(session); Call call(session->db, "SELECT vexfs_handle_close(?1,?2,?3)"); - call.Text(1, handle); call.Int(2, retain_unpublished); call.Text(3, request_id); + call.Text(1, handle); call.Int(2, retain_unpublished); + call.Text(3, EffectiveRequestId(session, request_id, true)); call.Row(); CopyResult(call, state); }); } @@ -787,9 +1515,12 @@ extern "C" vexfs_mount_status vexfs_mount_synchronize(vexfs_mount_session *sessi UseFullDurability(session); if (published == nullptr) throw CallError(SQLITE_MISUSE, "published output is NULL"); Call call(session->db, "SELECT vexfs_mount_synchronize(?1,?2,?3)"); - call.Text(1, session->workspace.c_str()); call.Text(2, request_id); + call.Text(1, session->workspace.c_str()); + call.Text(2, EffectiveRequestId(session, request_id, true)); call.Text(3, session->session_id.c_str()); call.Row(); *published = call.ResultInt64(); + if (*published > 0) ++session->mutation_epoch; + DurabilityBarrier(session); }); } @@ -799,10 +1530,12 @@ extern "C" vexfs_mount_status vexfs_mount_reclaim(vexfs_mount_session *session, vexfs_mount_error *error) { return Guard(session, error, [&] { RequireSession(session); - UseFullDurability(session); + UseOrdinaryDurability(session); if (reclaimed == nullptr) throw CallError(SQLITE_MISUSE, "reclaimed output is NULL"); Call call(session->db, "SELECT vexfs_item_reclaim(?1,?2)"); - call.Text(1, session->workspace.c_str()); call.Text(2, request_id); call.Row(); + call.Text(1, session->workspace.c_str()); + call.Text(2, EffectiveRequestId(session, request_id, true)); + call.Row(); *reclaimed = call.ResultInt64(); }); } diff --git a/agent_files/mount/common/test/vexfs_mount_contract_smoke.cpp b/agent_files/mount/common/test/vexfs_mount_contract_smoke.cpp index 20eb595a20..157b2f5df4 100644 --- a/agent_files/mount/common/test/vexfs_mount_contract_smoke.cpp +++ b/agent_files/mount/common/test/vexfs_mount_contract_smoke.cpp @@ -1,16 +1,21 @@ -#include "vexfs_mount_contract.h" +#include "vexfs_runtime_admin.h" +#include "sqlite3.h" +#include +#include #include +#include #include #include +#include #include #include namespace { int Fail(const char *operation, const vexfs_mount_error &error) { - std::fprintf(stderr, "VEXFS MOUNT CONTRACT FAIL: %s: status=%d sqlite=%d %s\n", - operation, error.status, error.sqlite_code, error.message); + std::fprintf(stderr, "VEXFS RUNTIME FAIL: %s: status=%d backend=%s native=%d %s\n", + operation, error.status, error.backend, error.native_code, error.message); return 1; } @@ -25,9 +30,159 @@ bool Contains(const vexfs_mount_bytes &bytes, const char *expected) { return value.find(expected) != std::string::npos; } +int64_t JsonInteger(const vexfs_mount_bytes &bytes, const char *key) { + const std::string value(static_cast(bytes.data), + static_cast(bytes.size)); + const std::string marker = std::string("\"") + key + "\":"; + const size_t start = value.find(marker); + if (start == std::string::npos) return -1; + return std::strtoll(value.c_str() + start + marker.size(), nullptr, 10); +} + +int64_t Scalar(const char *path, const char *sql) { + sqlite3 *db = nullptr; + if (sqlite3_open_v2(path, &db, SQLITE_OPEN_READONLY, nullptr) != SQLITE_OK) { + if (db != nullptr) sqlite3_close(db); + return -1; + } + sqlite3_stmt *statement = nullptr; + int64_t value = -1; + if (sqlite3_prepare_v2(db, sql, -1, &statement, nullptr) == SQLITE_OK && + sqlite3_step(statement) == SQLITE_ROW) { + value = sqlite3_column_int64(statement, 0); + } + if (statement != nullptr) sqlite3_finalize(statement); + sqlite3_close(db); + return value; +} + +bool Execute(const char *path, const char *sql) { + sqlite3 *db = nullptr; + if (sqlite3_open_v2(path, &db, SQLITE_OPEN_READWRITE, nullptr) != SQLITE_OK) { + if (db != nullptr) sqlite3_close(db); + return false; + } + const int rc = sqlite3_exec(db, sql, nullptr, nullptr, nullptr); + sqlite3_close(db); + return rc == SQLITE_OK; +} + +int RunBenchmark(int file_count) { + if (file_count <= 0 || file_count > 1000) { + std::fprintf(stderr, "benchmark file count must be 1..1000\n"); + return 2; + } + char path[] = "/tmp/vexfs-contract-benchmark-XXXXXX"; + const int fd = mkstemp(path); + if (fd < 0) return 1; + close(fd); + unlink(path); + + vexfs_mount_config config{}; + config.abi_version = VEXFS_RUNTIME_ABI_VERSION; + config.backend = VEXFS_RUNTIME_BACKEND_SQLITE; + config.connection = path; + config.workspace = "benchmark"; + config.principal = "local"; + config.operation_timeout_ms = 1000; + config.flags = VEXFS_RUNTIME_EXCLUSIVE_GATEWAY; + vexfs_mount_error error{}; + vexfs_mount_session *session = nullptr; + if (vexfs_mount_session_open(&config, &session, &error) != VEXFS_MOUNT_OK) + return Fail("benchmark open", error); + if (vexfs_mount_mkdir(session, "/files", &error) != VEXFS_MOUNT_OK) + return Fail("benchmark mkdir", error); + + const int64_t commits_before = Scalar(path, "SELECT count(*) FROM _vexfs_commits"); + const int64_t versions_before = Scalar(path, "SELECT count(*) FROM _vexfs_file_versions"); + const int64_t requests_before = Scalar(path, "SELECT count(*) FROM _vexfs_requests"); + const char payload[] = "vexfs benchmark payload\n"; + const auto started = std::chrono::steady_clock::now(); + for (int index = 0; index < file_count; ++index) { + char file_path[64]; + char request[64]; + std::snprintf(file_path, sizeof(file_path), "/files/f%06d.txt", index); + std::snprintf(request, sizeof(request), "create-%06d", index); + vexfs_mount_bytes handle{}; + if (vexfs_mount_handle_create(session, file_path, 0644, request, + &handle, &error) != VEXFS_MOUNT_OK) + return Fail("benchmark create", error); + const std::string handle_id(static_cast(handle.data), + static_cast(handle.size)); + vexfs_mount_free(handle.data); + vexfs_mount_bytes item_stat{}; + if (vexfs_mount_stat(session, file_path, &item_stat, &error) != VEXFS_MOUNT_OK) + return Fail("benchmark stat", error); + const int64_t inode = JsonInteger(item_stat, "inode"); + vexfs_mount_free(item_stat.data); + if (vexfs_mount_xattr_set(session, inode, "com.apple.provenance", "p", 1, + VEXFS_MOUNT_XATTR_ALWAYS_SET, &error) != VEXFS_MOUNT_OK) + return Fail("benchmark xattr", error); + int64_t generation = 0; + std::snprintf(request, sizeof(request), "write-%06d", index); + if (vexfs_mount_handle_stage_write( + session, handle_id.c_str(), 0, payload, sizeof(payload) - 1, + request, &generation, &error) != VEXFS_MOUNT_OK) + return Fail("benchmark stage", error); + int64_t version = 0; + if (vexfs_mount_handle_publish_close(session, handle_id.c_str(), generation, + "data", &version, &error) != + VEXFS_MOUNT_OK || version != 1) + return Fail("benchmark publish close", error); + } + const double create_seconds = std::chrono::duration( + std::chrono::steady_clock::now() - started).count(); + const int64_t commits = Scalar(path, "SELECT count(*) FROM _vexfs_commits") - commits_before; + const int64_t versions = Scalar(path, "SELECT count(*) FROM _vexfs_file_versions") - versions_before; + const int64_t requests = Scalar(path, "SELECT count(*) FROM _vexfs_requests") - requests_before; + + const auto sync_started = std::chrono::steady_clock::now(); + int64_t published = -1; + if (vexfs_mount_synchronize(session, "benchmark-sync", &published, &error) != + VEXFS_MOUNT_OK || published != 0) + return Fail("benchmark synchronize", error); + const double sync_seconds = std::chrono::duration( + std::chrono::steady_clock::now() - sync_started).count(); + vexfs_mount_bytes diagnostics{}; + if (vexfs_mount_diagnostics(session, &diagnostics, &error) != VEXFS_MOUNT_OK) + return Fail("benchmark diagnostics", error); + const int64_t ordinary_calls = JsonInteger(diagnostics, "ordinary_mutation_calls"); + const int64_t full_calls = JsonInteger(diagnostics, "full_boundary_calls"); + const int64_t barriers = JsonInteger(diagnostics, "durability_barriers"); + vexfs_mount_free(diagnostics.data); + vexfs_mount_session_close(session); + + struct stat info {}; + const int64_t database_bytes = stat(path, &info) == 0 ? info.st_size : -1; + std::printf( + "{\"files\":%d,\"create_seconds\":%.6f,\"files_per_second\":%.3f," + "\"sync_seconds\":%.6f,\"commit_rows\":%lld,\"version_rows\":%lld," + "\"request_rows\":%lld,\"commits_per_file\":%.3f," + "\"versions_per_file\":%.3f,\"requests_per_file\":%.3f," + "\"ordinary_mutation_calls\":%lld,\"full_boundary_calls\":%lld," + "\"durability_barriers\":%lld,\"database_bytes\":%lld}\n", + file_count, create_seconds, file_count / std::max(create_seconds, 1e-9), + sync_seconds, static_cast(commits), static_cast(versions), + static_cast(requests), static_cast(commits) / file_count, + static_cast(versions) / file_count, + static_cast(requests) / file_count, + static_cast(ordinary_calls), static_cast(full_calls), + static_cast(barriers), static_cast(database_bytes)); + unlink(path); + std::string wal = std::string(path) + "-wal"; + std::string shm = std::string(path) + "-shm"; + unlink(wal.c_str()); + unlink(shm.c_str()); + return commits == file_count && versions == file_count && + requests <= static_cast(file_count) * 2 ? 0 : 1; +} + } // namespace -int main() { +int main(int argc, char **argv) { + if (argc == 3 && std::strcmp(argv[1], "--benchmark") == 0) { + return RunBenchmark(std::atoi(argv[2])); + } char path[] = "/tmp/vexfs-contract-XXXXXX"; const int fd = mkstemp(path); if (fd < 0) return 1; @@ -35,19 +190,33 @@ int main() { unlink(path); vexfs_mount_config config{}; - config.abi_version = VEXFS_MOUNT_ABI_VERSION; - config.database_path = path; + config.abi_version = VEXFS_RUNTIME_ABI_VERSION; + config.backend = VEXFS_RUNTIME_BACKEND_SQLITE; + config.connection = path; config.workspace = "default"; - config.busy_timeout_ms = 1000; + config.principal = "local"; + config.operation_timeout_ms = 1000; vexfs_mount_error error{}; vexfs_mount_session *session = nullptr; + vexfs_mount_config unsupported_config = config; + unsupported_config.backend = "postgres"; + vexfs_mount_session *unsupported_session = nullptr; + if (vexfs_mount_session_open(&unsupported_config, &unsupported_session, &error) != + VEXFS_MOUNT_UNSUPPORTED || unsupported_session != nullptr) + return Fail("sqlite rejects unsupported backend", error); + vexfs_mount_config rejected_config = config; + rejected_config.principal = "alice"; + vexfs_mount_session *rejected_session = nullptr; + if (vexfs_mount_session_open(&rejected_config, &rejected_session, &error) != + VEXFS_MOUNT_PERMISSION_DENIED || rejected_session != nullptr) + return Fail("sqlite rejects non-local principal", error); if (vexfs_mount_session_open(&config, &session, &error) != VEXFS_MOUNT_OK) return Fail("open", error); vexfs_mount_bytes diagnostics{}; if (vexfs_mount_diagnostics(session, &diagnostics, &error) != VEXFS_MOUNT_OK) return Fail("diagnostics", error); - if (!Contains(diagnostics, "\"contract_version\":\"0.3.0\"") || + if (!Contains(diagnostics, "\"schema_version\":\"0.7.0\"") || !Contains(diagnostics, "\"workspace_exists\":1")) return Fail("diagnostics content", error); vexfs_mount_free(diagnostics.data); @@ -58,6 +227,237 @@ int main() { if (vexfs_mount_write_file(session, "/agent/task.txt", "draft", 5, &version, &error) != VEXFS_MOUNT_OK) return Fail("write", error); + vexfs_mount_visibility visibility{}; + if (vexfs_mount_refresh_visibility(session, &visibility, &error) != VEXFS_MOUNT_OK || + visibility.workspace_head <= 0 || visibility.cache_generation != 1 || + visibility.external_commit != 0) + return Fail("initial cache visibility", error); + vexfs_mount_config peer_config = config; + vexfs_mount_session *peer_session = nullptr; + if (vexfs_mount_session_open(&peer_config, &peer_session, &error) != VEXFS_MOUNT_OK) + return Fail("visibility peer open", error); + int64_t peer_version = 0; + if (vexfs_mount_write_file(peer_session, "/agent/external.txt", "peer", 4, + &peer_version, &error) != VEXFS_MOUNT_OK) + return Fail("visibility peer write", error); + if (vexfs_mount_refresh_visibility(session, &visibility, &error) != VEXFS_MOUNT_OK || + visibility.cache_generation != 2 || visibility.external_commit != 1) + return Fail("external commit visibility", error); + const int64_t external_head = visibility.workspace_head; + if (vexfs_mount_refresh_visibility(session, &visibility, &error) != VEXFS_MOUNT_OK || + visibility.workspace_head != external_head || visibility.cache_generation != 2 || + visibility.external_commit != 0) + return Fail("stable cache visibility", error); + vexfs_mount_session_close(peer_session); + + vexfs_mount_bytes versioned_list{}; + if (vexfs_mount_list_versioned(session, "/agent", &versioned_list, &error) != + VEXFS_MOUNT_OK || JsonInteger(versioned_list, "version") <= 0 || + !Contains(versioned_list, "\"name\":\"task.txt\"")) + return Fail("versioned directory list", error); + vexfs_mount_free(versioned_list.data); + vexfs_mount_bytes grep{}; + if (vexfs_mount_grep(session, "/agent", "RAF", VEXFS_MOUNT_GREP_IGNORE_CASE, + 100, &grep, &error) != VEXFS_MOUNT_OK || + !Contains(grep, "\"path\":\"/agent/task.txt\"") || + JsonInteger(grep, "match_count") != 1) + return Fail("database grep", error); + vexfs_mount_free(grep.data); + if (vexfs_mount_grep(session, "/agent", "draft", 8, 100, &grep, &error) != + VEXFS_MOUNT_INVALID_ARGUMENT) return Fail("database grep flags", error); + vexfs_mount_bytes grep_index{}; + if (vexfs_mount_grep_index(session, "enable", &grep_index, &error) != VEXFS_MOUNT_OK || + !Contains(grep_index, "\"available\":true") || + !Contains(grep_index, "\"backend\":\"fts5-trigram\"")) + return Fail("database grep index enable", error); + vexfs_mount_free(grep_index.data); + grep = {}; + if (vexfs_mount_grep(session, "/agent", "draft", 0, 100, &grep, &error) != + VEXFS_MOUNT_OK || !Contains(grep, "\"index_used\":true") || + JsonInteger(grep, "match_count") != 1) + return Fail("indexed database grep", error); + vexfs_mount_free(grep.data); + if (vexfs_mount_remove(session, "/agent", 0, &error) != VEXFS_MOUNT_NOT_EMPTY) + return Fail("non-empty directory", error); + vexfs_mount_bytes task_stat{}; + if (vexfs_mount_stat(session, "/agent/task.txt", &task_stat, &error) != VEXFS_MOUNT_OK) + return Fail("xattr stat", error); + const int64_t task_inode = JsonInteger(task_stat, "inode"); + vexfs_mount_free(task_stat.data); + if (task_inode <= 0) return Fail("xattr inode", error); + if (vexfs_mount_set_mode(session, task_inode, 0750, &error) != VEXFS_MOUNT_OK) + return Fail("set mode", error); + task_stat = {}; + if (vexfs_mount_stat(session, "/agent/task.txt", &task_stat, &error) != VEXFS_MOUNT_OK || + JsonInteger(task_stat, "mode") != 0750) + return Fail("set mode stat", error); + vexfs_mount_free(task_stat.data); + if (vexfs_mount_set_mode(session, task_inode, 01000, &error) != + VEXFS_MOUNT_INVALID_ARGUMENT) return Fail("invalid mode", error); + if (vexfs_mount_set_times(session, task_inode, 1700000000123LL, 1700000100456LL, + VEXFS_MOUNT_TIME_ACCESS | VEXFS_MOUNT_TIME_MODIFY, + &error) != VEXFS_MOUNT_OK) + return Fail("set times", error); + task_stat = {}; + if (vexfs_mount_stat(session, "/agent/task.txt", &task_stat, &error) != VEXFS_MOUNT_OK || + JsonInteger(task_stat, "accessed_at") != 1700000000123LL || + JsonInteger(task_stat, "updated_at") != 1700000100456LL) + return Fail("set times stat", error); + vexfs_mount_free(task_stat.data); + if (vexfs_mount_set_times(session, task_inode, -1, 0, VEXFS_MOUNT_TIME_ACCESS, + &error) != VEXFS_MOUNT_INVALID_ARGUMENT) + return Fail("invalid times", error); + if (vexfs_mount_symlink(session, "/agent/task-link", "task.txt", 8, &error) != + VEXFS_MOUNT_OK) return Fail("symlink", error); + if (vexfs_mount_symlink(session, "/agent/task-link", "duplicate", 9, &error) != + VEXFS_MOUNT_CONFLICT) return Fail("duplicate symlink", error); + vexfs_mount_bytes link_stat{}; + if (vexfs_mount_stat(session, "/agent/task-link", &link_stat, &error) != VEXFS_MOUNT_OK || + !Contains(link_stat, "\"kind\":\"symlink\"") || JsonInteger(link_stat, "size") != 8) + return Fail("symlink stat", error); + const int64_t link_inode = JsonInteger(link_stat, "inode"); + vexfs_mount_free(link_stat.data); + vexfs_mount_bytes target{}; + if (vexfs_mount_readlink(session, link_inode, &target, &error) != VEXFS_MOUNT_OK || + !Equals(target, "task.txt")) return Fail("readlink", error); + vexfs_mount_free(target.data); + target = {}; + if (vexfs_mount_readlink(session, task_inode, &target, &error) != + VEXFS_MOUNT_INVALID_ARGUMENT) return Fail("readlink regular file", error); + if (vexfs_mount_xattr_set(session, task_inode, "com.vexfs.test", "alpha", 5, + VEXFS_MOUNT_XATTR_MUST_CREATE, &error) != VEXFS_MOUNT_OK) + return Fail("xattr create", error); + if (vexfs_mount_xattr_set(session, task_inode, "com.vexfs.test", "duplicate", 9, + VEXFS_MOUNT_XATTR_MUST_CREATE, &error) != VEXFS_MOUNT_CONFLICT) + return Fail("xattr duplicate create", error); + vexfs_mount_bytes xattr{}; + if (vexfs_mount_xattr_get(session, task_inode, "com.vexfs.test", &xattr, &error) != + VEXFS_MOUNT_OK || !Equals(xattr, "alpha")) + return Fail("xattr get", error); + vexfs_mount_free(xattr.data); + if (vexfs_mount_xattr_set(session, task_inode, "com.vexfs.test", "beta", 4, + VEXFS_MOUNT_XATTR_MUST_REPLACE, &error) != VEXFS_MOUNT_OK) + return Fail("xattr replace", error); + vexfs_mount_bytes xattr_names{}; + if (vexfs_mount_xattr_list(session, task_inode, &xattr_names, &error) != VEXFS_MOUNT_OK || + !Contains(xattr_names, "com.vexfs.test")) + return Fail("xattr list", error); + vexfs_mount_free(xattr_names.data); + + if (vexfs_mount_link(session, "/agent/task.txt", "/agent/task-copy.txt", &error) != + VEXFS_MOUNT_OK) return Fail("hard link", error); + vexfs_mount_bytes hardlink_stat{}; + if (vexfs_mount_stat(session, "/agent/task-copy.txt", &hardlink_stat, &error) != + VEXFS_MOUNT_OK || JsonInteger(hardlink_stat, "inode") != task_inode || + JsonInteger(hardlink_stat, "link_count") != 2) + return Fail("hard link stat", error); + vexfs_mount_free(hardlink_stat.data); + if (vexfs_mount_chown(session, task_inode, -1, 1234, &error) != VEXFS_MOUNT_OK) + return Fail("chown keep uid", error); + task_stat = {}; + if (vexfs_mount_stat(session, "/agent/task.txt", &task_stat, &error) != VEXFS_MOUNT_OK || + JsonInteger(task_stat, "gid") != 1234) + return Fail("chown stat", error); + vexfs_mount_free(task_stat.data); + const char acl_json[] = + "[{\"principal\":\"alice\",\"effect\":\"allow\"," + "\"permissions\":\"read,write\",\"inherit\":1}]"; + if (vexfs_mount_acl_set(session, task_inode, acl_json, sizeof(acl_json) - 1, &error) != + VEXFS_MOUNT_OK) return Fail("acl set", error); + vexfs_mount_bytes acl{}; + if (vexfs_mount_acl_get(session, task_inode, &acl, &error) != VEXFS_MOUNT_OK || + !Contains(acl, "alice") || !Contains(acl, "read,write")) + return Fail("acl get", error); + vexfs_mount_free(acl.data); + + int64_t snapshot_commit = 0; + if (vexfs_mount_snapshot_create(session, "before-change", 0, &snapshot_commit, &error) != + VEXFS_MOUNT_OK || snapshot_commit <= 0) + return Fail("snapshot create", error); + vexfs_mount_bytes snapshots{}; + if (vexfs_mount_snapshot_list(session, &snapshots, &error) != VEXFS_MOUNT_OK || + !Contains(snapshots, "before-change")) return Fail("snapshot list", error); + vexfs_mount_free(snapshots.data); + if (vexfs_mount_write_file(session, "/agent/task.txt", "changed", 7, &version, &error) != + VEXFS_MOUNT_OK) return Fail("snapshot mutation write", error); + if (vexfs_mount_set_mode(session, task_inode, 0600, &error) != VEXFS_MOUNT_OK) + return Fail("snapshot mutation mode", error); + if (vexfs_mount_remove(session, "/agent/task-link", 0, &error) != VEXFS_MOUNT_OK) + return Fail("snapshot mutation remove", error); + if (vexfs_mount_xattr_set(session, task_inode, "com.vexfs.test", "gamma", 5, + VEXFS_MOUNT_XATTR_MUST_REPLACE, &error) != VEXFS_MOUNT_OK) + return Fail("snapshot mutation xattr", error); + vexfs_mount_bytes snapshot_diff{}; + if (vexfs_mount_snapshot_diff(session, "before-change", "HEAD", &snapshot_diff, &error) != + VEXFS_MOUNT_OK || !Contains(snapshot_diff, "\"changes\":[{") || + !Contains(snapshot_diff, "/agent/task-link")) + return Fail("snapshot diff", error); + vexfs_mount_free(snapshot_diff.data); + vexfs_mount_bytes snapshot_show{}; + if (vexfs_mount_snapshot_show(session, "before-change", &snapshot_show, &error) != + VEXFS_MOUNT_OK || !Contains(snapshot_show, "/agent/task.txt")) + return Fail("snapshot show", error); + vexfs_mount_free(snapshot_show.data); + int64_t head_commit = 0; + if (vexfs_mount_workspace_head(session, &head_commit, &error) != VEXFS_MOUNT_OK || + head_commit <= snapshot_commit) return Fail("workspace head", error); + int64_t restored_commit = 0; + vexfs_mount_config mounted_config = config; + mounted_config.flags = VEXFS_RUNTIME_EXCLUSIVE_GATEWAY; + vexfs_mount_session *mounted_session = nullptr; + if (vexfs_mount_session_open(&mounted_config, &mounted_session, &error) != VEXFS_MOUNT_OK) + return Fail("mounted restore guard open", error); + if (vexfs_mount_snapshot_restore(session, "before-change", head_commit, + &restored_commit, &error) != VEXFS_MOUNT_BUSY || + std::strstr(error.message, "active mount session") == nullptr) + return Fail("mounted snapshot restore must be busy", error); + vexfs_mount_session_close(mounted_session); + if (vexfs_mount_snapshot_restore(session, "before-change", head_commit, + &restored_commit, &error) != VEXFS_MOUNT_OK || + restored_commit <= head_commit) return Fail("snapshot restore", error); + vexfs_mount_bytes snapshot_content{}; + if (vexfs_mount_read_file(session, "/agent/task.txt", &snapshot_content, &error) != + VEXFS_MOUNT_OK || !Equals(snapshot_content, "draft")) + return Fail("snapshot restored content", error); + vexfs_mount_free(snapshot_content.data); + grep = {}; + if (vexfs_mount_grep(session, "/agent", "draft", 0, 100, &grep, &error) != + VEXFS_MOUNT_OK || !Contains(grep, "\"index_used\":true") || + JsonInteger(grep, "match_count") != 2 || Contains(grep, "changed")) + return Fail("snapshot restored grep index", error); + vexfs_mount_free(grep.data); + task_stat = {}; + if (vexfs_mount_stat(session, "/agent/task.txt", &task_stat, &error) != VEXFS_MOUNT_OK || + JsonInteger(task_stat, "mode") != 0750) return Fail("snapshot restored mode", error); + vexfs_mount_free(task_stat.data); + target = {}; + if (vexfs_mount_readlink(session, link_inode, &target, &error) != VEXFS_MOUNT_OK || + !Equals(target, "task.txt")) return Fail("snapshot restored symlink", error); + vexfs_mount_free(target.data); + hardlink_stat = {}; + if (vexfs_mount_stat(session, "/agent/task-copy.txt", &hardlink_stat, &error) != + VEXFS_MOUNT_OK || JsonInteger(hardlink_stat, "inode") != task_inode || + JsonInteger(hardlink_stat, "link_count") != 2) + return Fail("snapshot restored hard link", error); + vexfs_mount_free(hardlink_stat.data); + task_stat = {}; + if (vexfs_mount_stat(session, "/agent/task.txt", &task_stat, &error) != VEXFS_MOUNT_OK || + JsonInteger(task_stat, "gid") != 1234) + return Fail("snapshot restored owner", error); + vexfs_mount_free(task_stat.data); + acl = {}; + if (vexfs_mount_acl_get(session, task_inode, &acl, &error) != VEXFS_MOUNT_OK || + !Contains(acl, "alice")) return Fail("snapshot restored acl", error); + vexfs_mount_free(acl.data); + xattr = {}; + if (vexfs_mount_xattr_get(session, task_inode, "com.vexfs.test", &xattr, &error) != + VEXFS_MOUNT_OK || !Equals(xattr, "beta")) + return Fail("snapshot restored xattr", error); + vexfs_mount_free(xattr.data); + if (vexfs_mount_snapshot_restore(session, "before-change", head_commit, + &restored_commit, &error) != VEXFS_MOUNT_CONFLICT) + return Fail("stale snapshot restore", error); + if (vexfs_mount_write_file(session, "/agent/versioned.txt", "old", 3, &version, &error) != VEXFS_MOUNT_OK || version != 1) return Fail("version seed", error); @@ -148,7 +548,7 @@ int main() { // 重开真实数据库文件，确认不是进程内假状态。 session = nullptr; - config.flags = VEXFS_MOUNT_OPEN_NO_CREATE; + config.flags = VEXFS_RUNTIME_OPEN_NO_CREATE; if (vexfs_mount_session_open(&config, &session, &error) != VEXFS_MOUNT_OK) return Fail("reopen", error); content = {}; @@ -156,17 +556,134 @@ int main() { return Fail("persistent read", error); if (!Equals(content, "draft ready")) return Fail("persistent content", error); vexfs_mount_free(content.data); + task_stat = {}; + if (vexfs_mount_stat(session, "/agent/task.txt", &task_stat, &error) != VEXFS_MOUNT_OK || + JsonInteger(task_stat, "mode") != 0750) + return Fail("persistent mode", error); + vexfs_mount_free(task_stat.data); + target = {}; + if (vexfs_mount_readlink(session, link_inode, &target, &error) != VEXFS_MOUNT_OK || + !Equals(target, "task.txt")) return Fail("persistent symlink", error); + vexfs_mount_free(target.data); + xattr = {}; + if (vexfs_mount_xattr_get(session, task_inode, "com.vexfs.test", &xattr, &error) != + VEXFS_MOUNT_OK || !Equals(xattr, "beta")) + return Fail("persistent xattr", error); + vexfs_mount_free(xattr.data); + if (vexfs_mount_xattr_set(session, task_inode, "com.vexfs.test", nullptr, 0, + VEXFS_MOUNT_XATTR_DELETE, &error) != VEXFS_MOUNT_OK) + return Fail("xattr delete", error); + xattr = {}; + if (vexfs_mount_xattr_get(session, task_inode, "com.vexfs.test", &xattr, &error) != + VEXFS_MOUNT_NOT_FOUND) return Fail("deleted xattr must be missing", error); content = {}; if (vexfs_mount_read_file(session, "/agent/sync.txt", &content, &error) != VEXFS_MOUNT_OK || !Equals(content, "ABC")) return Fail("persistent synchronized content", error); vexfs_mount_free(content.data); vexfs_mount_session_close(session); - // 独占挂载进程崩溃后，新 session 只保留旧 staging，不能自动发布半成品。 - config.flags = VEXFS_MOUNT_EXCLUSIVE_GATEWAY; + // 独占挂载进程崩溃后，新 session 自动认领并发布该 session 已提交的 staging。 + // 这是 mount helper 重启恢复合同；再次 synchronize 必须保持幂等。 + config.flags = VEXFS_RUNTIME_EXCLUSIVE_GATEWAY; session = nullptr; if (vexfs_mount_session_open(&config, &session, &error) != VEXFS_MOUNT_OK) return Fail("crash seed open", error); + + handle = {}; + if (vexfs_mount_handle_create(session, "/agent/pending-snapshot.txt", 0644, + "create-pending-snapshot", &handle, &error) != + VEXFS_MOUNT_OK) + return Fail("pending snapshot create", error); + handle_id.assign(static_cast(handle.data), static_cast(handle.size)); + vexfs_mount_free(handle.data); + if (vexfs_mount_handle_stage_write(session, handle_id.c_str(), 0, "snap", 4, + "write-pending-snapshot", &generation, &error) != + VEXFS_MOUNT_OK) + return Fail("pending snapshot stage", error); + // A management session must not silently snapshot stale HEAD while the + // exclusive gateway owns dirty staging. It may explicitly request the + // committed-only view, which must not contain the pending file. + vexfs_mount_config admin_config = config; + admin_config.flags = 0; + vexfs_mount_session *admin_session = nullptr; + if (vexfs_mount_session_open(&admin_config, &admin_session, &error) != VEXFS_MOUNT_OK) + return Fail("admin session open", error); + if (vexfs_mount_snapshot_create(admin_session, "must-not-be-stale", 0, + &snapshot_commit, &error) != VEXFS_MOUNT_BUSY) + return Fail("cross-session snapshot must be busy", error); + if (vexfs_mount_snapshot_create(admin_session, "committed-only", + VEXFS_SNAPSHOT_COMMITTED_ONLY, + &snapshot_commit, &error) != VEXFS_MOUNT_OK) + return Fail("committed-only snapshot", error); + vexfs_mount_bytes committed_tree{}; + if (vexfs_mount_snapshot_show(admin_session, "committed-only", &committed_tree, + &error) != VEXFS_MOUNT_OK || + Contains(committed_tree, "/agent/pending-snapshot.txt")) + return Fail("committed-only snapshot excludes staging", error); + vexfs_mount_free(committed_tree.data); + vexfs_mount_session_close(admin_session); + + if (vexfs_mount_snapshot_create(session, "with-pending-handle", 0, + &snapshot_commit, &error) != VEXFS_MOUNT_OK) + return Fail("pending snapshot boundary", error); + content = {}; + if (vexfs_mount_read_file(session, "/agent/pending-snapshot.txt", &content, &error) != + VEXFS_MOUNT_OK || !Equals(content, "snap")) + return Fail("pending snapshot published content", error); + vexfs_mount_free(content.data); + state = {}; + if (vexfs_mount_handle_close(session, handle_id.c_str(), 0, + "close-pending-snapshot", &state, &error) != + VEXFS_MOUNT_OK) + return Fail("pending snapshot close", error); + vexfs_mount_free(state.data); + + // The mount-only create-and-open path must produce one history commit and + // one content version, even when macOS metadata arrives before content. + const int64_t commits_before = Scalar(path, "SELECT count(*) FROM _vexfs_commits"); + const int64_t versions_before = Scalar(path, "SELECT count(*) FROM _vexfs_file_versions"); + const int64_t requests_before = Scalar(path, "SELECT count(*) FROM _vexfs_requests"); + handle = {}; + if (vexfs_mount_handle_create(session, "/agent/coalesced.txt", 0640, + "create-coalesced-1", &handle, &error) != VEXFS_MOUNT_OK) + return Fail("coalesced create handle", error); + handle_id.assign(static_cast(handle.data), static_cast(handle.size)); + vexfs_mount_free(handle.data); + vexfs_mount_bytes coalesced_stat{}; + if (vexfs_mount_stat(session, "/agent/coalesced.txt", &coalesced_stat, &error) != + VEXFS_MOUNT_OK || JsonInteger(coalesced_stat, "version") != 0) + return Fail("coalesced create has no empty version", error); + const int64_t coalesced_inode = JsonInteger(coalesced_stat, "inode"); + vexfs_mount_free(coalesced_stat.data); + if (vexfs_mount_xattr_set(session, coalesced_inode, "com.apple.provenance", "p", 1, + VEXFS_MOUNT_XATTR_ALWAYS_SET, &error) != VEXFS_MOUNT_OK) + return Fail("coalesced provenance", error); + if (vexfs_mount_set_mode(session, coalesced_inode, 0600, &error) != VEXFS_MOUNT_OK) + return Fail("coalesced mode", error); + if (vexfs_mount_handle_stage_write(session, handle_id.c_str(), 0, "content", 7, + "write-coalesced-1", &generation, &error) != + VEXFS_MOUNT_OK) + return Fail("coalesced stage", error); + if (vexfs_mount_handle_publish_close(session, handle_id.c_str(), generation, "data", + &version, &error) != + VEXFS_MOUNT_OK || version != 1) + return Fail("coalesced publish close", error); + if (Scalar(path, "SELECT count(*) FROM _vexfs_commits") - commits_before != 1 || + Scalar(path, "SELECT count(*) FROM _vexfs_file_versions") - versions_before != 1 || + Scalar(path, "SELECT count(*) FROM _vexfs_requests") - requests_before != 0) + return Fail("coalesced write amplification", error); + content = {}; + if (vexfs_mount_read_file(session, "/agent/coalesced.txt", &content, &error) != + VEXFS_MOUNT_OK || !Equals(content, "content")) + return Fail("coalesced content", error); + vexfs_mount_free(content.data); + diagnostics = {}; + if (vexfs_mount_diagnostics(session, &diagnostics, &error) != VEXFS_MOUNT_OK || + JsonInteger(diagnostics, "ordinary_mutation_calls") < 5 || + JsonInteger(diagnostics, "synchronous_mode_switches") < 1) + return Fail("performance diagnostics", error); + vexfs_mount_free(diagnostics.data); + if (vexfs_mount_write_file(session, "/agent/crash.txt", "old", 3, &version, &error) != VEXFS_MOUNT_OK) return Fail("crash seed write", error); vexfs_mount_session *contender = nullptr; @@ -201,21 +718,48 @@ int main() { session = nullptr; if (vexfs_mount_session_open(&config, &session, &error) != VEXFS_MOUNT_OK) return Fail("crash recovery open", error); + content = {}; + if (vexfs_mount_read_file(session, "/agent/crash.txt", &content, &error) != VEXFS_MOUNT_OK || + !Equals(content, "NEW")) return Fail("crash content auto recovered", error); + vexfs_mount_free(content.data); published = -1; if (vexfs_mount_synchronize(session, "crash-sync-1", &published, &error) != VEXFS_MOUNT_OK || - published != 1) { + published != 0) { std::fprintf(stderr, "crash synchronize published=%lld\n", (long long)published); - return Fail("crash staging recovery publish", error); + return Fail("crash recovery synchronize idempotent", error); } - content = {}; - if (vexfs_mount_read_file(session, "/agent/crash.txt", &content, &error) != VEXFS_MOUNT_OK || - !Equals(content, "NEW")) return Fail("crash content recovered", error); - vexfs_mount_free(content.data); diagnostics = {}; if (vexfs_mount_diagnostics(session, &diagnostics, &error) != VEXFS_MOUNT_OK || !Contains(diagnostics, "\"retained_handles\":0")) return Fail("crash staging claimed", error); vexfs_mount_free(diagnostics.data); + + vexfs_mount_bytes check{}; + if (vexfs_mount_check(session, 0, &check, &error) != VEXFS_MOUNT_OK || + !Contains(check, "\"ok\":true") || !Contains(check, "\"mode\":\"deep\"")) + return Fail("deep integrity check", error); + vexfs_mount_free(check.data); + if (!Execute(path, + "UPDATE _vexfs_file_versions SET content=X'424144' " + "WHERE inode_id=(SELECT inode_id FROM _vexfs_dentries WHERE name='crash.txt') " + "AND version_no=(SELECT current_version FROM _vexfs_inodes WHERE id=inode_id)")) + return Fail("inject checksum corruption", error); + check = {}; + if (vexfs_mount_check(session, VEXFS_MOUNT_CHECK_QUICK, &check, &error) != + VEXFS_MOUNT_OK || !Contains(check, "\"ok\":true") || + !Contains(check, "\"mode\":\"quick\"")) + return Fail("quick integrity check", error); + vexfs_mount_free(check.data); + check = {}; + if (vexfs_mount_check(session, 0, &check, &error) != VEXFS_MOUNT_OK || + !Contains(check, "\"ok\":false") || + !Contains(check, "VEXFS_CHECKSUM_MISMATCH")) + return Fail("checksum corruption report", error); + vexfs_mount_free(check.data); + content = {}; + if (vexfs_mount_read_file(session, "/agent/crash.txt", &content, &error) != + VEXFS_MOUNT_CORRUPTION) + return Fail("corrupt content must not be returned", error); vexfs_mount_session_close(session); unlink(path); @@ -223,6 +767,6 @@ int main() { std::string shm = std::string(path) + "-shm"; unlink(wal.c_str()); unlink(shm.c_str()); - std::puts("VEXFS MOUNT CONTRACT SMOKE: PASS"); + std::puts("VEXFS RUNTIME SMOKE: PASS"); return 0; } diff --git a/agent_files/mount/linux/vexfs_fuse.cpp b/agent_files/mount/linux/vexfs_fuse.cpp new file mode 100644 index 0000000000..b4aa59fe25 --- /dev/null +++ b/agent_files/mount/linux/vexfs_fuse.cpp @@ -0,0 +1,895 @@ +#define FUSE_USE_VERSION 35 + +#include + +#include "vexfs_runtime_io.h" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef RENAME_NOREPLACE +#define RENAME_NOREPLACE (1u << 0) +#endif +#ifndef RENAME_EXCHANGE +#define RENAME_EXCHANGE (1u << 1) +#endif + +namespace { + +struct FuseState { + vexfs_mount_session *session = nullptr; + std::string database; + std::string request_prefix; + std::recursive_mutex mutex; + std::atomic request_sequence{1}; +}; + +struct FileHandle { + std::string id; + int64_t generation = 0; + bool dirty = false; + bool append = false; +}; + +FuseState *State() { + const fuse_context *context = fuse_get_context(); + return context == nullptr ? nullptr : static_cast(context->private_data); +} + +std::string RequestId(FuseState *state, const char *operation) { + return state->request_prefix + "-" + operation + "-" + + std::to_string(state->request_sequence.fetch_add(1)); +} + +std::string NewRequestPrefix() { + const uint64_t now = static_cast( + std::chrono::high_resolution_clock::now().time_since_epoch().count()); + std::random_device random; + const uint64_t entropy = + (static_cast(random()) << 32) ^ static_cast(random()); + return std::string("fuse-") + std::to_string(static_cast(getpid())) + "-" + + std::to_string(now) + "-" + std::to_string(entropy); +} + +int Errno(vexfs_mount_status status) { + switch (status) { + case VEXFS_MOUNT_OK: return 0; + case VEXFS_MOUNT_INVALID_ARGUMENT: return EINVAL; + case VEXFS_MOUNT_NOT_FOUND: return ENOENT; + case VEXFS_MOUNT_CONFLICT: return EEXIST; + case VEXFS_MOUNT_READ_ONLY: return EROFS; + case VEXFS_MOUNT_BUSY: return EBUSY; + case VEXFS_MOUNT_PERMISSION_DENIED: return EACCES; + case VEXFS_MOUNT_NO_SPACE: return ENOSPC; + case VEXFS_MOUNT_UNSUPPORTED: return ENOTSUP; + case VEXFS_MOUNT_NOT_EMPTY: return ENOTEMPTY; + case VEXFS_MOUNT_CORRUPTION: return EIO; + case VEXFS_MOUNT_DATABASE_ERROR: return EIO; + case VEXFS_MOUNT_INTERNAL_ERROR: return EIO; + } + return EIO; +} + +int Result(vexfs_mount_status status) { + return status == VEXFS_MOUNT_OK ? 0 : -Errno(status); +} + +std::string TakeBytes(vexfs_mount_bytes *bytes) { + std::string result; + if (bytes->data != nullptr) { + result.assign(static_cast(bytes->data), static_cast(bytes->size)); + vexfs_mount_free(bytes->data); + } + bytes->data = nullptr; + bytes->size = 0; + return result; +} + +std::string JsonUnescape(const std::string &json, size_t *position) { + std::string value; + while (*position < json.size()) { + const char character = json[(*position)++]; + if (character == '"') break; + if (character != '\\' || *position >= json.size()) { + value.push_back(character); + continue; + } + const char escaped = json[(*position)++]; + switch (escaped) { + case 'n': value.push_back('\n'); break; + case 'r': value.push_back('\r'); break; + case 't': value.push_back('\t'); break; + case 'b': value.push_back('\b'); break; + case 'f': value.push_back('\f'); break; + case '"': value.push_back('"'); break; + case '\\': value.push_back('\\'); break; + default: value.push_back('?'); break; + } + } + return value; +} + +int64_t JsonInteger(const std::string &json, const std::string &name) { + const std::string marker = "\"" + name + "\":"; + size_t position = json.find(marker); + if (position == std::string::npos) throw std::runtime_error("missing JSON integer: " + name); + position += marker.size(); + char *end = nullptr; + const long long value = std::strtoll(json.c_str() + position, &end, 10); + if (end == json.c_str() + position) throw std::runtime_error("invalid JSON integer: " + name); + return static_cast(value); +} + +std::string JsonString(const std::string &json, const std::string &name) { + const std::string marker = "\"" + name + "\":\""; + size_t position = json.find(marker); + if (position == std::string::npos) throw std::runtime_error("missing JSON string: " + name); + position += marker.size(); + return JsonUnescape(json, &position); +} + +std::vector JsonObjects(const std::string &json) { + std::vector objects; + bool in_string = false; + bool escaped = false; + int depth = 0; + size_t start = 0; + for (size_t position = 0; position < json.size(); ++position) { + const char character = json[position]; + if (in_string) { + if (escaped) escaped = false; + else if (character == '\\') escaped = true; + else if (character == '"') in_string = false; + continue; + } + if (character == '"') in_string = true; + else if (character == '{') { + if (depth++ == 0) start = position; + } else if (character == '}' && depth > 0) { + if (--depth == 0) objects.push_back(json.substr(start, position - start + 1)); + } + } + return objects; +} + +std::vector JsonStrings(const std::string &json) { + std::vector values; + size_t position = 0; + while (position < json.size()) { + position = json.find('"', position); + if (position == std::string::npos) break; + ++position; + values.push_back(JsonUnescape(json, &position)); + } + return values; +} + +int StatPath(FuseState *state, const char *path, std::string *json) { + vexfs_mount_bytes bytes{}; + vexfs_mount_error error{}; + const vexfs_mount_status status = vexfs_mount_stat(state->session, path, &bytes, &error); + if (status != VEXFS_MOUNT_OK) return -Errno(status); + *json = TakeBytes(&bytes); + return 0; +} + +int ChownCreatedPath(FuseState *state, const char *path) { + const fuse_context *context = fuse_get_context(); + if (context == nullptr) return -EIO; + try { + std::string json; + const int stat_result = StatPath(state, path, &json); + if (stat_result != 0) return stat_result; + vexfs_mount_error error{}; + return Result(vexfs_mount_chown(state->session, JsonInteger(json, "inode"), + static_cast(context->uid), + static_cast(context->gid), &error)); + } catch (...) { + return -EIO; + } +} + +void FillStat(const std::string &json, struct stat *attributes) { + std::memset(attributes, 0, sizeof(*attributes)); + const std::string kind = JsonString(json, "kind"); + const mode_t permissions = static_cast(JsonInteger(json, "mode") & 0777); + if (kind == "directory") attributes->st_mode = S_IFDIR | permissions; + else if (kind == "symlink") attributes->st_mode = S_IFLNK | 0777; + else attributes->st_mode = S_IFREG | permissions; + attributes->st_ino = static_cast(JsonInteger(json, "inode")); + attributes->st_nlink = static_cast(JsonInteger(json, "link_count")); + attributes->st_size = static_cast(JsonInteger(json, "size")); + attributes->st_uid = static_cast(JsonInteger(json, "uid")); + attributes->st_gid = static_cast(JsonInteger(json, "gid")); + attributes->st_blksize = 4096; + attributes->st_blocks = (attributes->st_size + 511) / 512; + const int64_t created_ms = JsonInteger(json, "created_at"); + const int64_t accessed_ms = JsonInteger(json, "accessed_at"); + const int64_t updated_ms = JsonInteger(json, "updated_at"); + const int64_t changed_ms = JsonInteger(json, "changed_at"); + (void)created_ms; // Linux stat 没有独立 birthtime 字段。 + attributes->st_ctim = {changed_ms / 1000, (changed_ms % 1000) * 1000000}; + attributes->st_mtim = {updated_ms / 1000, (updated_ms % 1000) * 1000000}; + attributes->st_atim = {accessed_ms / 1000, (accessed_ms % 1000) * 1000000}; +} + +int GetAttr(const char *path, struct stat *attributes, fuse_file_info *) { + FuseState *state = State(); + if (state == nullptr) return -EIO; + std::lock_guard guard(state->mutex); + try { + std::string json; + const int result = StatPath(state, path, &json); + if (result != 0) return result; + FillStat(json, attributes); + return 0; + } catch (...) { + return -EIO; + } +} + +int ReadLink(const char *path, char *buffer, size_t size) { + if (size == 0) return -EINVAL; + FuseState *state = State(); + if (state == nullptr) return -EIO; + std::lock_guard guard(state->mutex); + try { + std::string json; + const int stat_result = StatPath(state, path, &json); + if (stat_result != 0) return stat_result; + vexfs_mount_bytes target{}; + vexfs_mount_error error{}; + const auto status = vexfs_mount_readlink( + state->session, JsonInteger(json, "inode"), &target, &error); + if (status != VEXFS_MOUNT_OK) return -Errno(status); + const std::string value = TakeBytes(&target); + const size_t count = std::min(value.size(), size - 1); + std::memcpy(buffer, value.data(), count); + buffer[count] = '\0'; + return 0; + } catch (...) { + return -EIO; + } +} + +int MakeNode(const char *path, mode_t mode, dev_t) { + if (!S_ISREG(mode)) return -ENOTSUP; + FuseState *state = State(); + if (state == nullptr) return -EIO; + std::lock_guard guard(state->mutex); + vexfs_mount_error error{}; + const int create_result = + Result(vexfs_mount_create(state->session, path, "file", mode & 0777, &error)); + return create_result == 0 ? ChownCreatedPath(state, path) : create_result; +} + +int MakeDirectory(const char *path, mode_t mode) { + FuseState *state = State(); + if (state == nullptr) return -EIO; + std::lock_guard guard(state->mutex); + vexfs_mount_error error{}; + const auto status = vexfs_mount_mkdir(state->session, path, &error); + if (status != VEXFS_MOUNT_OK) return -Errno(status); + std::string json; + const int stat_result = StatPath(state, path, &json); + if (stat_result != 0) return stat_result; + const int mode_result = Result(vexfs_mount_set_mode( + state->session, JsonInteger(json, "inode"), mode & 0777, &error)); + return mode_result == 0 ? ChownCreatedPath(state, path) : mode_result; +} + +int RemovePath(const char *path) { + FuseState *state = State(); + if (state == nullptr) return -EIO; + std::lock_guard guard(state->mutex); + vexfs_mount_error error{}; + return Result(vexfs_mount_remove(state->session, path, 0, &error)); +} + +int CreateSymlink(const char *target, const char *path) { + FuseState *state = State(); + if (state == nullptr) return -EIO; + std::lock_guard guard(state->mutex); + vexfs_mount_error error{}; + const int create_result = Result(vexfs_mount_symlink( + state->session, path, target, std::strlen(target), &error)); + return create_result == 0 ? ChownCreatedPath(state, path) : create_result; +} + +int RenamePath(const char *source, const char *destination, unsigned int flags) { + if ((flags & RENAME_EXCHANGE) != 0 || (flags & ~(RENAME_EXCHANGE | RENAME_NOREPLACE)) != 0) + return -ENOTSUP; + FuseState *state = State(); + if (state == nullptr) return -EIO; + std::lock_guard guard(state->mutex); + vexfs_mount_error error{}; + const int replace = (flags & RENAME_NOREPLACE) == 0 ? 1 : 0; + return Result(vexfs_mount_rename(state->session, source, destination, replace, &error)); +} + +int LinkPath(const char *source, const char *destination) { + FuseState *state = State(); + if (state == nullptr) return -EIO; + std::lock_guard guard(state->mutex); + vexfs_mount_error error{}; + return Result(vexfs_mount_link(state->session, source, destination, &error)); +} + +int ChmodPath(const char *path, mode_t mode, fuse_file_info *) { + FuseState *state = State(); + if (state == nullptr) return -EIO; + std::lock_guard guard(state->mutex); + try { + std::string json; + const int stat_result = StatPath(state, path, &json); + if (stat_result != 0) return stat_result; + vexfs_mount_error error{}; + return Result(vexfs_mount_set_mode(state->session, JsonInteger(json, "inode"), + mode & 0777, &error)); + } catch (...) { + return -EIO; + } +} + +int ChownPath(const char *path, uid_t uid, gid_t gid, fuse_file_info *) { + FuseState *state = State(); + if (state == nullptr) return -EIO; + std::lock_guard guard(state->mutex); + try { + std::string json; + const int stat_result = StatPath(state, path, &json); + if (stat_result != 0) return stat_result; + vexfs_mount_error error{}; + const int64_t owner = uid == static_cast(-1) ? -1 : static_cast(uid); + const int64_t group = gid == static_cast(-1) ? -1 : static_cast(gid); + return Result(vexfs_mount_chown(state->session, JsonInteger(json, "inode"), + owner, group, &error)); + } catch (...) { + return -EIO; + } +} + +const char *OpenMode(int flags) { + switch (flags & O_ACCMODE) { + case O_WRONLY: return "w"; + case O_RDWR: return "rw"; + default: return "r"; + } +} + +int OpenPath(const char *path, fuse_file_info *info) { + FuseState *state = State(); + if (state == nullptr) return -EIO; + std::lock_guard guard(state->mutex); + auto handle = std::make_unique(); + vexfs_mount_bytes identifier{}; + vexfs_mount_error error{}; + const std::string request = RequestId(state, "open"); + auto status = vexfs_mount_handle_open(state->session, path, OpenMode(info->flags), + request.c_str(), &identifier, &error); + if (status != VEXFS_MOUNT_OK) return -Errno(status); + handle->id = TakeBytes(&identifier); + handle->append = (info->flags & O_APPEND) != 0; + if ((info->flags & O_TRUNC) != 0) { + const std::string truncate_request = RequestId(state, "truncate-open"); + status = vexfs_mount_handle_truncate(state->session, handle->id.c_str(), 0, + truncate_request.c_str(), &handle->generation, + &error); + if (status != VEXFS_MOUNT_OK) { + vexfs_mount_bytes close_state{}; + vexfs_mount_handle_close(state->session, handle->id.c_str(), 0, + RequestId(state, "close-open-failure").c_str(), + &close_state, + &error); + TakeBytes(&close_state); + return -Errno(status); + } + handle->dirty = true; + } + info->fh = reinterpret_cast(handle.release()); + info->direct_io = 1; + return 0; +} + +FileHandle *Handle(fuse_file_info *info) { + return info == nullptr ? nullptr : reinterpret_cast(info->fh); +} + +int CreatePath(const char *path, mode_t mode, fuse_file_info *info) { + FuseState *state = State(); + if (state == nullptr) return -EIO; + std::lock_guard guard(state->mutex); + auto handle = std::make_unique(); + vexfs_mount_bytes identifier{}; + vexfs_mount_error error{}; + const std::string request = RequestId(state, "create-open"); + const auto status = vexfs_mount_handle_create( + state->session, path, mode & 0777, request.c_str(), &identifier, &error); + if (status != VEXFS_MOUNT_OK) return -Errno(status); + handle->id = TakeBytes(&identifier); + handle->append = (info->flags & O_APPEND) != 0; + handle->generation = 1; + handle->dirty = true; + info->fh = reinterpret_cast(handle.release()); + info->direct_io = 1; + return 0; +} + +int ReadPath(const char *path, char *buffer, size_t size, off_t offset, + fuse_file_info *info) { + if (offset < 0) return -EINVAL; + FuseState *state = State(); + if (state == nullptr) return -EIO; + std::lock_guard guard(state->mutex); + vexfs_mount_bytes content{}; + vexfs_mount_error error{}; + vexfs_mount_status status; + FileHandle *handle = Handle(info); + if (handle != nullptr) { + status = vexfs_mount_handle_read(state->session, handle->id.c_str(), + static_cast(offset), size, &content, &error); + } else { + status = vexfs_mount_read_file(state->session, path, &content, &error); + } + if (status != VEXFS_MOUNT_OK) return -Errno(status); + std::string value = TakeBytes(&content); + if (handle == nullptr) { + if (static_cast(offset) >= value.size()) return 0; + value = value.substr(static_cast(offset), size); + } + const size_t count = std::min(size, value.size()); + std::memcpy(buffer, value.data(), count); + return static_cast(count); +} + +int WritePath(const char *, const char *buffer, size_t size, off_t offset, + fuse_file_info *info) { + if (offset < 0 || Handle(info) == nullptr) return -EINVAL; + FuseState *state = State(); + if (state == nullptr) return -EIO; + std::lock_guard guard(state->mutex); + FileHandle *handle = Handle(info); + vexfs_mount_error error{}; + const std::string request = RequestId(state, "write"); + if (handle->append) { + int64_t version = 0; + const auto status = vexfs_mount_handle_append( + state->session, handle->id.c_str(), buffer, size, request.c_str(), &version, &error); + return status == VEXFS_MOUNT_OK ? static_cast(size) : -Errno(status); + } + const auto status = vexfs_mount_handle_stage_write( + state->session, handle->id.c_str(), static_cast(offset), buffer, size, + request.c_str(), &handle->generation, &error); + if (status != VEXFS_MOUNT_OK) return -Errno(status); + handle->dirty = true; + return static_cast(size); +} + +int Publish(FuseState *state, FileHandle *handle, const char *durability) { + if (handle == nullptr || !handle->dirty) return 0; + vexfs_mount_error error{}; + int64_t version = 0; + const std::string request = RequestId(state, "publish"); + const auto status = vexfs_mount_handle_publish( + state->session, handle->id.c_str(), handle->generation, durability, + request.c_str(), &version, &error); + if (status != VEXFS_MOUNT_OK) return -Errno(status); + handle->dirty = false; + return 0; +} + +int FlushPath(const char *, fuse_file_info *info) { + FuseState *state = State(); + if (state == nullptr) return -EIO; + std::lock_guard guard(state->mutex); + return Publish(state, Handle(info), "data"); +} + +int FsyncPath(const char *, int, fuse_file_info *info) { + FuseState *state = State(); + if (state == nullptr) return -EIO; + std::lock_guard guard(state->mutex); + int result = Publish(state, Handle(info), "full"); + if (result != 0) return result; + int64_t published = 0; + vexfs_mount_error error{}; + const std::string request = RequestId(state, "fsync"); + return Result(vexfs_mount_synchronize(state->session, request.c_str(), &published, &error)); +} + +int ReleasePath(const char *, fuse_file_info *info) { + FuseState *state = State(); + FileHandle *handle = Handle(info); + if (state == nullptr || handle == nullptr) return 0; + std::lock_guard guard(state->mutex); + vexfs_mount_error error{}; + vexfs_mount_status status = VEXFS_MOUNT_OK; + if (handle->dirty) { + int64_t version = 0; + status = vexfs_mount_handle_publish_close( + state->session, handle->id.c_str(), handle->generation, "data", &version, &error); + if (status != VEXFS_MOUNT_OK) { + vexfs_mount_bytes close_state{}; + const std::string request = RequestId(state, "close-retain"); + vexfs_mount_handle_close(state->session, handle->id.c_str(), 1, + request.c_str(), &close_state, &error); + TakeBytes(&close_state); + } + } else { + vexfs_mount_bytes close_state{}; + const std::string request = RequestId(state, "close"); + status = vexfs_mount_handle_close(state->session, handle->id.c_str(), 0, + request.c_str(), &close_state, &error); + TakeBytes(&close_state); + } + delete handle; + info->fh = 0; + return Result(status); +} + +int TruncatePath(const char *path, off_t size, fuse_file_info *info) { + if (size < 0) return -EINVAL; + FuseState *state = State(); + if (state == nullptr) return -EIO; + std::lock_guard guard(state->mutex); + bool temporary = Handle(info) == nullptr; + fuse_file_info local{}; + if (temporary) { + local.flags = O_RDWR; + const int open_result = OpenPath(path, &local); + if (open_result != 0) return open_result; + info = &local; + } + FileHandle *handle = Handle(info); + vexfs_mount_error error{}; + const std::string request = RequestId(state, "truncate"); + const auto status = vexfs_mount_handle_truncate( + state->session, handle->id.c_str(), static_cast(size), request.c_str(), + &handle->generation, &error); + int result = Result(status); + if (result == 0) { + handle->dirty = true; + result = Publish(state, handle, "full"); + } + if (temporary) { + const int close_result = ReleasePath(path, info); + if (result == 0) result = close_result; + } + return result; +} + +int ReadDirectory(const char *path, void *buffer, fuse_fill_dir_t filler, off_t, + fuse_file_info *, fuse_readdir_flags) { + FuseState *state = State(); + if (state == nullptr) return -EIO; + std::lock_guard guard(state->mutex); + vexfs_mount_bytes listing{}; + vexfs_mount_error error{}; + const auto status = vexfs_mount_list(state->session, path, &listing, &error); + if (status != VEXFS_MOUNT_OK) return -Errno(status); + if (filler(buffer, ".", nullptr, 0, static_cast(0)) != 0 || + filler(buffer, "..", nullptr, 0, static_cast(0)) != 0) { + TakeBytes(&listing); + return 0; + } + try { + for (const std::string &entry : JsonObjects(TakeBytes(&listing))) { + struct stat attributes{}; + const std::string kind = JsonString(entry, "kind"); + if (kind == "directory") attributes.st_mode = S_IFDIR; + else if (kind == "symlink") attributes.st_mode = S_IFLNK; + else attributes.st_mode = S_IFREG; + attributes.st_ino = static_cast(JsonInteger(entry, "inode")); + const std::string name = JsonString(entry, "name"); + if (filler(buffer, name.c_str(), &attributes, 0, + static_cast(0)) != 0) break; + } + return 0; + } catch (...) { + return -EIO; + } +} + +int StatFs(const char *, struct statvfs *output) { + FuseState *state = State(); + if (state == nullptr) return -EIO; + const std::filesystem::path database(state->database); + const std::filesystem::path directory = database.has_parent_path() ? + database.parent_path() : std::filesystem::current_path(); + return statvfs(directory.c_str(), output) == 0 ? 0 : -errno; +} + +int PathInode(FuseState *state, const char *path, int64_t *inode) { + try { + std::string json; + const int result = StatPath(state, path, &json); + if (result != 0) return result; + *inode = JsonInteger(json, "inode"); + return 0; + } catch (...) { + return -EIO; + } +} + +int SetXattr(const char *path, const char *name, const char *value, size_t size, int flags) { + FuseState *state = State(); + if (state == nullptr) return -EIO; + std::lock_guard guard(state->mutex); + int64_t inode = 0; + const int stat_result = PathInode(state, path, &inode); + if (stat_result != 0) return stat_result; + int policy = VEXFS_MOUNT_XATTR_ALWAYS_SET; + if ((flags & XATTR_CREATE) != 0) policy = VEXFS_MOUNT_XATTR_MUST_CREATE; + else if ((flags & XATTR_REPLACE) != 0) policy = VEXFS_MOUNT_XATTR_MUST_REPLACE; + vexfs_mount_error error{}; + return Result(vexfs_mount_xattr_set(state->session, inode, name, value, size, + policy, &error)); +} + +int GetXattr(const char *path, const char *name, char *value, size_t size) { + FuseState *state = State(); + if (state == nullptr) return -EIO; + std::lock_guard guard(state->mutex); + int64_t inode = 0; + const int stat_result = PathInode(state, path, &inode); + if (stat_result != 0) return stat_result; + vexfs_mount_bytes bytes{}; + vexfs_mount_error error{}; + const auto status = vexfs_mount_xattr_get(state->session, inode, name, &bytes, &error); + if (status != VEXFS_MOUNT_OK) return -Errno(status); + const std::string result = TakeBytes(&bytes); + if (size == 0) return static_cast(result.size()); + if (size < result.size()) return -ERANGE; + std::memcpy(value, result.data(), result.size()); + return static_cast(result.size()); +} + +int ListXattr(const char *path, char *list, size_t size) { + FuseState *state = State(); + if (state == nullptr) return -EIO; + std::lock_guard guard(state->mutex); + int64_t inode = 0; + const int stat_result = PathInode(state, path, &inode); + if (stat_result != 0) return stat_result; + vexfs_mount_bytes json{}; + vexfs_mount_error error{}; + const auto status = vexfs_mount_xattr_list(state->session, inode, &json, &error); + if (status != VEXFS_MOUNT_OK) return -Errno(status); + std::string packed; + for (const std::string &name : JsonStrings(TakeBytes(&json))) { + packed.append(name); + packed.push_back('\0'); + } + if (size == 0) return static_cast(packed.size()); + if (size < packed.size()) return -ERANGE; + std::memcpy(list, packed.data(), packed.size()); + return static_cast(packed.size()); +} + +int RemoveXattr(const char *path, const char *name) { + FuseState *state = State(); + if (state == nullptr) return -EIO; + std::lock_guard guard(state->mutex); + int64_t inode = 0; + const int stat_result = PathInode(state, path, &inode); + if (stat_result != 0) return stat_result; + vexfs_mount_error error{}; + return Result(vexfs_mount_xattr_set(state->session, inode, name, nullptr, 0, + VEXFS_MOUNT_XATTR_DELETE, &error)); +} + +int SyncDirectory(const char *, int, fuse_file_info *) { + FuseState *state = State(); + if (state == nullptr) return -EIO; + std::lock_guard guard(state->mutex); + int64_t published = 0; + vexfs_mount_error error{}; + const std::string request = RequestId(state, "fsyncdir"); + return Result(vexfs_mount_synchronize(state->session, request.c_str(), &published, &error)); +} + +int SetTimes(const char *path, const struct timespec times[2], fuse_file_info *) { + FuseState *state = State(); + if (state == nullptr || times == nullptr) return -EIO; + std::lock_guard guard(state->mutex); + try { + std::string json; + const int stat_result = StatPath(state, path, &json); + if (stat_result != 0) return stat_result; + const int64_t inode = JsonInteger(json, "inode"); + int64_t accessed_at = JsonInteger(json, "accessed_at"); + int64_t modified_at = JsonInteger(json, "updated_at"); + uint32_t mask = 0; + const int64_t now = std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()).count(); + auto convert = [&](const struct timespec &value, int64_t *target, + uint32_t field) -> int { + if (value.tv_nsec == UTIME_OMIT) return 0; + if (value.tv_nsec == UTIME_NOW) { + *target = now; + } else { + if (value.tv_sec < 0 || value.tv_nsec < 0 || value.tv_nsec >= 1000000000L) + return -EINVAL; + *target = static_cast(value.tv_sec) * 1000 + value.tv_nsec / 1000000; + } + mask |= field; + return 0; + }; + int result = convert(times[0], &accessed_at, VEXFS_MOUNT_TIME_ACCESS); + if (result != 0) return result; + result = convert(times[1], &modified_at, VEXFS_MOUNT_TIME_MODIFY); + if (result != 0) return result; + if (mask == 0) return 0; + vexfs_mount_error error{}; + return Result(vexfs_mount_set_times(state->session, inode, accessed_at, modified_at, + mask, &error)); + } catch (...) { + return -EIO; + } +} + +void *Initialize(fuse_conn_info *, fuse_config *config) { + config->kernel_cache = 0; + config->use_ino = 1; + config->entry_timeout = 0; + config->attr_timeout = 0; + config->negative_timeout = 0; + return State(); +} + +fuse_operations Operations() { + fuse_operations operations{}; + operations.getattr = GetAttr; + operations.readlink = ReadLink; + operations.mknod = MakeNode; + operations.mkdir = MakeDirectory; + operations.unlink = RemovePath; + operations.rmdir = RemovePath; + operations.symlink = CreateSymlink; + operations.rename = RenamePath; + operations.link = LinkPath; + operations.chmod = ChmodPath; + operations.chown = ChownPath; + operations.utimens = SetTimes; + operations.truncate = TruncatePath; + operations.open = OpenPath; + operations.read = ReadPath; + operations.write = WritePath; + operations.statfs = StatFs; + operations.flush = FlushPath; + operations.release = ReleasePath; + operations.fsync = FsyncPath; + operations.setxattr = SetXattr; + operations.getxattr = GetXattr; + operations.listxattr = ListXattr; + operations.removexattr = RemoveXattr; + operations.readdir = ReadDirectory; + operations.fsyncdir = SyncDirectory; + operations.create = CreatePath; + operations.init = Initialize; + return operations; +} + +struct Options { + std::string database; + std::string workspace = "default"; + std::string mount_point; + bool foreground = false; + bool self_test = false; +}; + +Options ParseOptions(int argc, char **argv) { + Options options; + for (int index = 1; index < argc; ++index) { + const std::string argument = argv[index]; + if (argument == "--db" && index + 1 < argc) options.database = argv[++index]; + else if (argument == "--workspace" && index + 1 < argc) options.workspace = argv[++index]; + else if (argument == "--foreground" || argument == "-f") options.foreground = true; + else if (argument == "--self-test") options.self_test = true; + else if (!argument.empty() && argument.front() == '-') + throw std::runtime_error("unknown option: " + argument); + else if (options.mount_point.empty()) options.mount_point = argument; + else throw std::runtime_error("only one mount point is allowed"); + } + if (options.database.empty()) throw std::runtime_error("--db DATABASE is required"); + if (!options.self_test && options.mount_point.empty()) + throw std::runtime_error("MOUNT_POINT is required"); + return options; +} + +int SelfTest(FuseState *state) { + vexfs_mount_error error{}; + const char *path = "/.vexfs-fuse-self-test"; + const char content[] = "linux-fuse"; + vexfs_mount_remove(state->session, path, 0, &error); + int64_t version = 0; + auto status = vexfs_mount_write_file(state->session, path, content, sizeof(content) - 1, + &version, &error); + if (status != VEXFS_MOUNT_OK) return -Errno(status); + vexfs_mount_bytes value{}; + status = vexfs_mount_read_file(state->session, path, &value, &error); + if (status != VEXFS_MOUNT_OK) return -Errno(status); + const bool matches = TakeBytes(&value) == content; + status = vexfs_mount_remove(state->session, path, 0, &error); + if (status != VEXFS_MOUNT_OK) return -Errno(status); + if (!matches) return -EIO; + std::cout << "VEXFS FUSE SELF TEST: PASS\n"; + return 0; +} + +void AdoptDefaultRootOwnership(FuseState *state) { + std::string json; + if (StatPath(state, "/", &json) != 0) return; + if (JsonInteger(json, "uid") != 0 || JsonInteger(json, "gid") != 0) return; + const uid_t uid = getuid(); + const gid_t gid = getgid(); + if (uid == 0 && gid == 0) return; + vexfs_mount_error error{}; + const auto status = vexfs_mount_chown(state->session, JsonInteger(json, "inode"), + static_cast(uid), + static_cast(gid), &error); + if (status != VEXFS_MOUNT_OK) throw std::runtime_error(error.message); +} + +} // namespace + +int main(int argc, char **argv) { + try { + const Options options = ParseOptions(argc, argv); + FuseState state; + state.database = options.database; + state.request_prefix = NewRequestPrefix(); + vexfs_mount_config config{}; + config.abi_version = VEXFS_RUNTIME_ABI_VERSION; + config.backend = VEXFS_RUNTIME_BACKEND_SQLITE; + config.connection = options.database.c_str(); + config.workspace = options.workspace.c_str(); + config.principal = "local"; + config.operation_timeout_ms = 5000; + config.flags = options.self_test ? 0 : VEXFS_RUNTIME_EXCLUSIVE_GATEWAY; + vexfs_mount_error error{}; + const auto status = vexfs_mount_session_open(&config, &state.session, &error); + if (status != VEXFS_MOUNT_OK) throw std::runtime_error(error.message); + AdoptDefaultRootOwnership(&state); + + int result = 0; + if (options.self_test) { + result = SelfTest(&state); + } else { + std::vector arguments = {argv[0], "-s", "-o", + "default_permissions,fsname=vexfs,subtype=vexfs"}; + if (options.foreground) arguments.push_back("-f"); + arguments.push_back(options.mount_point); + std::vector values; + for (std::string &argument : arguments) values.push_back(argument.data()); + values.push_back(nullptr); + const fuse_operations operations = Operations(); + result = fuse_main(static_cast(arguments.size()), values.data(), + &operations, &state); + } + vexfs_mount_session_close(state.session); + return result < 0 ? 1 : result; + } catch (const std::exception &error) { + std::cerr << "vexfs-fuse: " << error.what() << '\n'; + return 1; + } +} diff --git a/build_sqlite.sh b/build_sqlite.sh index 93727173f3..a0642e1fa5 100755 --- a/build_sqlite.sh +++ b/build_sqlite.sh @@ -56,10 +56,24 @@ case "$CMD" in "$BUILD_DIR/m3p_parallel_smoke" echo "=== VexFS SQLite 合同冒烟 ===" "$BUILD_DIR/vexfs_static_smoke" - echo "=== VexFS 挂载层 C ABI 冒烟 ===" - "$BUILD_DIR/vexfs_mount_contract_smoke" + echo "=== VexFS runtime C ABI 冒烟 ===" + "$BUILD_DIR/vexfs_runtime_smoke" + echo "=== VexFS Linux/Windows 平台边界冒烟 ===" + "$BUILD_DIR/vexfs_platform_linux_smoke" + "$BUILD_DIR/vexfs_platform_windows_smoke" + "$BUILD_DIR/vexfs_platform_registry_smoke" + if [ -x "$BUILD_DIR/vexfs-fuse" ]; then + echo "=== VexFS Linux FUSE helper 冒烟 ===" + FUSE_SMOKE_DB="${TMPDIR:-/tmp}/vexfs-fuse-helper-smoke-$$.sqlite3" + trap 'rm -f "$FUSE_SMOKE_DB" "$FUSE_SMOKE_DB-wal" "$FUSE_SMOKE_DB-shm"' EXIT + "$BUILD_DIR/vexfs-fuse" --db "$FUSE_SMOKE_DB" --workspace smoke --self-test + rm -f "$FUSE_SMOKE_DB" "$FUSE_SMOKE_DB-wal" "$FUSE_SMOKE_DB-shm" + trap - EXIT + fi echo "=== VexFS CLI 冒烟 ===" bash "$DIR/../agent_files/cli/test/vexfs_cli_smoke.sh" "$BUILD_DIR/vexfs" + echo "=== VexDB-Lite 统一 CLI 冒烟 ===" + bash "$DIR/../agent_files/cli/test/vexdb_unified_smoke.sh" "$BUILD_DIR/vexdb" echo "=== M4 spec（L2 DSL 渲染 + runner） ===" bash "$DIR/../tests/spec/_lib/docker/run_sqlite.sh" elif [ "$CMD" = "eval" ]; then diff --git a/common/vtl/allocator b/common/vtl/allocator index 22c2e91a15..e418939941 100644 --- a/common/vtl/allocator +++ b/common/vtl/allocator @@ -31,6 +31,9 @@ inline void pfree(void *ptr) { std::free(ptr); } /* MemoryContextAlloc 由 vexdb-duck/include/duck_pg_shim.hpp:56 提供 (避免重定义). * 这里补 MemoryContextAllocZero — b042c2c827 引入, duck_pg_shim.hpp 未提供, 否则 * 编 graph_index_algorithm.h:148 的 Vector temp 时找不到. */ +#if defined(PG_VEXDB_TARGET_SQLITE) +inline void *MemoryContextAlloc(MemoryContext /*ctx*/, size_t size) { return palloc(size); } +#endif inline void *MemoryContextAllocZero(MemoryContext /*ctx*/, size_t size) { return palloc0(size); } inline void *palloc_aligned(size_t size, size_t align, int /*flags*/) { void *ptr = nullptr; diff --git a/docs/design/2026-07-16_agent-files-universal-filesystem.md b/docs/design/2026-07-16_agent-files-universal-filesystem.md new file mode 100644 index 0000000000..b4b2150fee --- /dev/null +++ b/docs/design/2026-07-16_agent-files-universal-filesystem.md @@ -0,0 +1,1435 @@ +# VexFS：数据库原生文件管理扩展设计 + +- 日期：2026-07-16 +- 分支：`feature/agent_files` +- 文档版本：1.4 +- 状态：macOS `preview.17` 已完成 Developer ID 签名、Apple 公证、staple、本机安装和真实 FSKit 回归，但它不是当前工作树的发行证明；当前统一 runtime ABI 为 1，已修复包 Gate、ABI 清单、安装校验和脏源码发行拦截，待从干净 commit 重新签名、公证并跑真机挂载；Linux libfuse3 双架构预览包已完成；1 万文件、900 秒稳定性与真实工具链已验证；干净 Mac、10 万完整 mount 复测和真实 Coding Agent 待完成 +- 产品名：暂定 `VexFS` +- 产品规格：`docs/specs/2026-07-16_vexfs-product-spec.md` +- 最终目标和阶段顺序：`docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md` + +## 1. 最终定义 + +VexFS 是由 PostgreSQL、DuckDB 或 SQLite 加载和管理的数据库扩展。三个宿主共享逻辑合同，但首个闭环固定为 macOS + SQLite，不要求第一版同时完成其他数据库和操作系统。 + +它给数据库增加一套分层文件管理能力： + +- 工作区； +- 文件和目录； +- 路径； +- 文件内容； +- 文件属性； +- 版本； +- 快照和恢复； +- 权限； +- 审计； +- 完整性检查； +- 工作区逻辑导入导出。 + +VexFS 的权威核心是数据库扩展。首个产品提供 macOS FSKit App Extension，把 SQLite 中的工作区显示为真实操作系统目录。FSKit extension 是非权威 mount adapter：它通过 SQLite 连接调用公开 VexFS SQL/handle 合同，不保存最终状态，不绕过数据库事务，也不管理 SQLite 的 WAL、锁或物理文件。 + +用户通过数据库连接和 SQL 使用它： + +```sql +SELECT vexfs_workspace_create('agent-workspace'); +SELECT vexfs_mkdir('agent-workspace', '/reports', true); +SELECT vexfs_write('agent-workspace', '/reports/result.md', '# Result'); +SELECT vexfs_read('agent-workspace', '/reports/result.md'); +SELECT * FROM vexfs_list('agent-workspace', '/reports'); +``` + +安装和配置后，用户优先直接进入挂载目录： + +```bash +vexfs setup --engine sqlite \ + --workspace agent-workspace \ + --mount ~/VexFS/agent-workspace + +cd ~/VexFS/agent-workspace +ls reports +rg 'Result' reports +python scripts/generate.py +``` + +不能挂载时，也可以用配套 CLI 执行同样的操作： + +```bash +vexfs --workspace agent-workspace ls /reports +vexfs --workspace agent-workspace cat /reports/result.md +vexfs --workspace agent-workspace grep -R "Result" /reports +``` + +一句话定位： + +> **VexFS 给 PostgreSQL、DuckDB 和 SQLite 增加数据库原生的文件管理系统。** + +首个可挂载实现从 macOS 26.0+、Apple Silicon 和 SQLite 的垂直闭环开始，Linux libfuse3 预览现已接入同一合同并通过真实普通用户挂载。FSKit 基础接口从 macOS 15.4 提供，但数据库目录资源所需的 `FSPathURLResource` 属于 V2、从 macOS 26.0 提供，因此 15.4–15.x 只保留数据库与 CLI 能力，不承诺无缝挂载。当前阶段先把 macOS/Linux + SQLite 收口为本地开发者预览；PostgreSQL 用于下一阶段的跨电脑共享和权限验证，Windows 后续使用 WinFsp 接入同一 mount runtime。 + +### 1.1 适配工作量必须按 N+M 拆分 + +禁止按“3 个数据库 × 3 个操作系统”开发 9 套 VexFS。数据库适配和操作系统挂载适配必须相互独立： + +```text +SQLite adapter ----+ +DuckDB adapter ----+--> VexFS 统一合同和 mount runtime +PostgreSQL adapter-+ | + +--> macOS FSKit + +--> Linux libfuse3 + +--> Windows WinFsp + +--> 跨平台 CLI +``` + +数据库 adapter 只处理宿主差异：事务、锁、BLOB、内部表、SQL 注册、权限身份和备份。mount adapter 只处理操作系统差异：inode/vnode 回调、错误码、缓存失效、原生权限和挂载生命周期。路径、文件规则、版本、快照、冲突、硬链接、ACL 和 handle 语义只能定义一次。 + +因此完整目标是一个共享合同、三个数据库 adapter、三个 mount adapter 和一个 CLI，不是九个数据库与操作系统组合。新增第四个数据库或第四个操作系统时，只新增一侧 adapter。 + +数据库仍然是管理者和权威状态。统一 runtime 只是非权威翻译层，不保存最终文件状态，也不代替数据库事务、权限、备份和恢复。 + +交付顺序以路线图为准，当前固定为： + +1. 收口 SQLite + macOS/Linux 本地开发者预览； +2. 实现 PostgreSQL adapter，提供 macOS/Linux 跨电脑、多用户和共享 workspace； +3. 实现 Windows WinFsp，并验证三种操作系统上的 SQLite 本地和 PostgreSQL 远程工作区； +4. 实现 DuckDB adapter，仅用于本地分析和单写者场景，不把 DuckDB 当多用户文件服务器； +5. 完成统一合同、发布、治理和规模验收，进入 v1。 + +截至 2026-07-21，第一轮平台拆分和 Linux libfuse3 预览已经落地： + +- `agent_files/cli/vexfs_main.cpp` 只保留跨平台文件命令和公共 doctor 输出； +- `agent_files/cli/vexfs_platform.h` 是 CLI 唯一操作系统边界； +- `agent_files/cli/vexfs_platform_macos.cpp` 独占 FSKit、系统 mount 表和 macOS 版本检测； +- `agent_files/cli/vexfs_platform_linux.cpp` 检查 helper、`fusermount3`、`/dev/fuse` 和 + `/proc/self/mountinfo`，负责启动、检测和卸载 Linux mount； +- `agent_files/mount/linux/vexfs_fuse.cpp` 用 FUSE3 high-level API 把 POSIX 回调翻译到 + 同一 C ABI；最终文件、版本和 handle staging 仍只存入 SQLite； +- `agent_files/cli/vexfs_platform_stub.cpp` 只继续承担 Windows/WinFsp 未实现边界和跨平台编译 smoke； +- CMake 在 Linux 找到 fuse3 开发包时生成 `vexfs-fuse`，并让统一 `vexdb`/`vexfs` + 与 helper 一起构建；缺少 fuse3 时 CLI 仍能构建，但 doctor 明确报告 helper 不可用； +- Alpine Linux GCC 15 已完成全量 SQLite + CLI + helper 构建；helper C ABI 自测通过； + 真实 `/dev/fuse` 下以 root 和 uid 1000 普通用户两次执行 Bash/Git Eval 均通过。 + 覆盖脚本执行、mode、uid/gid、hardlink、symlink、`git init/add/commit/status`、 + 卸载和重挂载；普通用户一轮为 1 passed、13 checks； +- Linux x86_64 与 AArch64 发行物已改为 manylinux_2_28 原生构建，并静态链接 + libstdc++/libgcc；三个产物均通过 `GLIBCXX_3.4.22` 上限、ELF、RPATH 和 libfuse3 检查； +- 两个架构的压缩包均包含普通用户安装/卸载脚本、中文说明、版本清单和哈希，并已在对应 + 架构真实 Linux 机器的全新 HOME 中完成安装、SQLite 扩展加载、VexFS 文件读写和卸载回归； +- FUSE request id 使用每个 mount 会话的随机前缀和递增序号，避免重挂载后与数据库 + 中已记录的幂等请求冲突； +- MinGW x86_64 已编译 Windows 的公共 CLI 和平台 stub，但 WinFsp mount 尚未实现。 + +因此当前 Linux 是可真实挂载的预览实现，Windows 仍只是平台边界。Linux doctor 只有在 +helper、`fusermount3` 和可读写 `/dev/fuse` 都存在时才返回 `mount_ready=true`；不能把 +缺少运行条件的环境报告为可挂载。 + +## 2. 所有权关系 + +数据库是宿主、管理者和权威状态所有者。 + +```text +PostgreSQL / DuckDB / SQLite + ├── 加载、升级和卸载 VexFS + ├── 管理扩展内存和执行线程 + ├── 管理事务、锁和并发 + ├── 管理 WAL、checkpoint 和崩溃恢复 + ├── 管理权限、配额和资源限制 + ├── 管理数据库备份、复制和恢复 + └── 保存 VexFS 内部表、文件块和审计记录 + | + v + VexFS 受管扩展 + ├── 注册 SQL API + ├── 执行文件规则 + └── 调用宿主数据库能力 + ^ + | + 公开 VexFS SQL / handle 合同 + | + +-------------+-------------+ + | | + macOS FSKit mount adapter CLI/worktree + 真实 Bash 和任意程序 受限环境回退 +``` + +VexFS 禁止： + +- 直接读写 PostgreSQL、DuckDB 或 SQLite 数据库文件； +- 自己实现 WAL、buffer pool、锁或崩溃恢复； +- 私自修改数据库全局配置； +- 启动宿主不允许的后台线程； +- 使用不受宿主管理的长期内存； +- 接管数据库备份、复制、恢复、启动或停止； +- 把文件最终状态放到数据库之外； +- 让 mount gateway 自己决定身份、权限、版本或提交结果。 + +## 3. 产品边界 + +### VexFS 是什么 + +- 数据库 extension； +- 数据库内部的分层 BLOB 文件管理器； +- 可以参加用户数据库事务的文件操作系统； +- 三个数据库共享同一份逻辑合同； +- 通过 SQL 查询和修改； +- 通过非权威 mount gateway 向操作系统提供普通文件接口。 + +### VexFS 不是什么 + +- 不是 ext4/APFS/NTFS 的替代品； +- mount gateway 不是独立权威文件服务器； +- 核心扩展不是数据库客户端； +- 不是数据库管理工具； +- 不是把任意业务表自动显示成文件； +- 不做内容理解、Embedding 或语义检索； +- 不提供设备文件、socket 或 FIFO。 + +### 模型终端兼容层 + +目标不是只模仿几个命令，而是让模型完成日常文件工作：浏览、读取、查找、批量修改、运行脚本、打包、生成代码和提交结果。 + +终端兼容层分为三种模式，按产品优先级排列。 + +#### 模式一：真实挂载目录（P0） + +macOS 用户安装并启用 VexFS FSKit File System Extension 后，通过 `vexfs mount` 请求系统挂载。系统程序调用 `open/read/write/rename/fsync` 时，FSKit 把操作交给 adapter，adapter 再转换为公开 VexFS 合同，由 SQLite 扩展完成权限、版本和提交。 + +```text +Bash / Git / Python / 编译器 + | + macOS VFS + | + Apple FSKit + | + VexFS FileSystem.appex + | + SQLite connection + principal + | + VexFS SQLite 扩展和内部表 +``` + +一个 mount 只代表一个数据库 principal。MVP mount point 只允许当前 macOS 用户访问。操作系统用户只控制谁能进入本地 mount point，不能替代数据库身份。 + +FSKit extension 的进程生命周期由 macOS 管理，CLI 只负责配置、mount/unmount 和诊断。SQLite 是嵌入式数据库，所以 FSKit extension 会在自己的进程中打开 SQLite 连接并加载 VexFS 扩展；但 FSKit callback 不能由 SQLite 扩展初始化函数启动，VexFS SQL 回调也不能控制 FSKit 生命周期。进入 PostgreSQL 阶段后，mount adapter 改为通过正常数据库连接访问服务器端扩展。 + +#### 模式二：直接命令（回退和诊断） + +#### 直接命令模式 + +常见文件操作直接翻译成 VexFS SQL，每个写命令默认是一个数据库事务： + +```bash +vexfs ls /src +vexfs tree /src +vexfs find /src --name '*.cpp' +vexfs cat /src/main.cpp +vexfs head -n 20 /src/main.cpp +vexfs tail -n 100 /logs/run.log +vexfs grep -R -n 'TODO' /src +vexfs stat /src/main.cpp +vexfs wc -l /src/main.cpp +vexfs du -h /src + +vexfs touch /src/new.cpp +vexfs mkdir -p /src/lib +vexfs cp /src/a.cpp /src/lib/a.cpp +vexfs mv /src/old.cpp /src/new.cpp +vexfs rm -r /tmp/output +vexfs truncate -s 0 /logs/run.log +``` + +第一组直接命令包括： + +- 浏览：`ls`、`tree`、`find`； +- 读取：`cat`、`head`、`tail`、`stat`、`file`； +- 查找和统计：`grep`、`rg`、`wc`、`du`； +- 修改：`touch`、`mkdir`、`put`、`get`、`cp`、`mv`、`rm`、`rmdir`、`truncate`、`ln`、`readlink`； +- 权限：`chmod`、`chown`、`acl`； +- 版本：`history`、`diff`、`snapshot`、`restore`； +- 运维：`check`、`gc`、`export`、`import`。 + +普通终端使用 `vexfs ls`，避免覆盖系统命令。正常 macOS 安装优先使用真实 mount；直接命令用于脚本、诊断和不能挂载的环境。 + +#### 模式三：任意命令 worktree（受限环境回退） + +`sed -i`、`awk`、`jq`、`python`、`tar`、`git`、编译器和 `apply_patch` 都会调用操作系统文件接口。macOS FSKit mount 中它们可以直接工作；没有挂载能力时，使用受控 worktree。 + +VexFS 为此提供受控 worktree。`vexfs shell` 会创建 worktree、进入目录并启动真正的系统 shell，因此模型可以直接使用系统中的全部文件工具： + +```bash +# 创建一个基于数据库版本 847 的临时工作区 +vexfs worktree create agent-workspace --path ./work + +cd ./work +rg 'TODO' . +sed -i.bak 's/old/new/g' src/main.cpp +python scripts/generate.py +tar -czf output.tar.gz build/ + +# 查看差异并事务性提交回数据库 +vexfs worktree status +vexfs worktree diff +vexfs worktree commit -m 'update generated files' +``` + +交互使用可以简化为： + +```bash +vexfs shell agent-workspace +# 现在可直接使用 cd、ls、cat、rg、sed、python、git、make 等系统命令 +``` + +也支持一次执行： + +```bash +vexfs exec agent-workspace -- sh -lc \ + "python scripts/generate.py && rg 'ERROR' logs/" +``` + +`worktree create` 执行以下步骤： + +1. 使用数据库连接身份认证； +2. 读取用户有权访问的目录和文件； +3. 固定工作区起始 commit； +4. 把普通文件和目录写入受控临时目录； +5. 保存路径、inode、版本和 checksum 清单。 + +创建完成后不保持数据库事务和锁。外部命令无论运行几秒还是几小时，都不会占用一个长期数据库事务。 + +`worktree commit` 执行以下步骤： + +1. 比较起始清单和本地目录，得到新增、修改、改名和删除； +2. 拒绝越界路径、设备文件、socket、FIFO 和不安全链接；只允许指向工作区内部的相对符号链接； +3. 检查每个路径的写入、删除、改名和管理权限； +4. 检查文件大小、工作区配额和保留规则； +5. 检查起始版本之后是否存在并发修改； +6. 把全部变更作为一个 changeset 放进单个数据库事务； +7. 生成文件版本、工作区 commit 和审计记录； +8. 数据库提交成功后才报告 worktree 已提交。 + +如果权限、配额或并发检查失败，数据库不产生部分文件修改，本地 worktree 保留用于处理问题。 + +#### Mount 和 CLI 都不改变所有权 + +- 不直接读取数据库物理文件； +- 不保存权威文件状态； +- 不自行决定权限、版本或冲突结果； +- mount adapter 不能由数据库扩展初始化或 SQL callback 启动；SQLite 的 FSKit extension 可以在系统管理的自身进程中打开 SQLite 连接，PostgreSQL adapter 则保持为外部客户端； +- 不建立新网络协议； +- 临时 worktree 不是备份； +- 移除 gateway 和 CLI 后，全部文件仍可通过 SQL 管理。 + +因此，VexFS 支持的是“模型常用的普通文件和目录工作流”。P0 mount 支持明确的 POSIX 子集，不支持设备文件、socket、FIFO、setuid 或全部特殊对象。 + +## 4. 核心价值：SQL 事务和 Bash 入口 + +VexFS 的关键价值不是模仿 `ls` 和 `cat`，而是同时提供两种明确分开的入口： + +1. SQL 文件函数参加调用者当前数据库事务，可以和业务表原子提交； +2. mount 让 Bash 和普通程序无感使用文件，但每个文件操作使用 gateway 自己的短数据库事务。 + +mount 不能加入应用已经打开的数据库事务，多条 Bash 命令也不是一个事务。需要多文件原子提交时使用 changeset；未来的 `vexfs txn run` 通过隔离工作视图生成一个 changeset。 + +```sql +BEGIN; + +UPDATE tasks +SET status = 'completed' +WHERE id = 42; + +SELECT vexfs_write( + 'task-files', + '/tasks/42/report.md', + '# Task 42 report' +); + +COMMIT; +``` + +结果只能是： + +- 任务状态和报告文件同时提交；或 +- 两者同时回滚。 + +插件不能在内部偷偷开启一个独立事务，也不能在数据库事务回滚后保留文件修改。 + +### 4.1 平台无关句柄和提交边界 + +mount 不能把一次 `open()` 到 `close()` 变成长期数据库事务。编辑器、编译器和 Agent 可能持有文件几分钟，长期事务会占连接、持锁并阻止 SQLite checkpoint 或 PostgreSQL 正常回收。 + +共享核心不使用 `flush/release` 等单个平台名称，而定义五个动作： + +```text +mount_open(path, flags) + └── 读取 inode、权限和 expected version，创建带租约的 handle + +mount_stage_write(handle, offset, bytes, request_id) + └── 短事务写入未发布 staging；成功后才向系统返回 write 成功 + +mount_publish(handle, generation, durability, request_id) + └── 幂等复查 principal、quota 和 expected version + 生成 manifest、file version、workspace commit 和审计 + +mount_close(handle, retained_modes) + └── 最终 close 时发布 dirty generation；错误可返回给平台 adapter + +mount_reclaim(handle/item) + └── 释放 adapter 引用；不能删除未确认 staging 或已发布版本 +``` + +macOS FSKit 映射如下： + +- `openItem` → `mount_open`； +- `write` → `mount_stage_write`； +- 没有 retained open mode 的最终 `closeItem` → `mount_publish` 后 `mount_close`； +- `synchronize` → 发布该 volume 的全部 dirty generation，再按该 mount adapter 声明的 + durability 策略完成数据库提交； +- `reclaimItem` → 释放 FSItem 映射，只做可恢复清理，不作为唯一发布点。 + +Phase 0 必须用真实 `fsync(2)`、`close(2)` 和 FSKit callback 日志确认系统调用到 `synchronize/closeItem` 的实际映射。如果当前稳定 SDK 不提供可报告的 durability 路径，MVP Gate 失败，不能用猜测代替。 + +句柄状态机固定为： + +```text +OPEN_CLEAN + └─ write ─> DIRTY + ├─ publish/synchronize ─> PUBLISHING ─> COMMITTED ─> OPEN_CLEAN + └─ 失败 ─> DIRTY_FAILED + +final close + ├─ clean/committed ─> CLOSED + └─ dirty ─> publish ─> CLOSED 或 DIRTY_FAILED + +reclaim with dirty/failed + └─ RECOVERABLE_STAGING ─> retry 或过期回收 +``` + +所有 publish/synchronize 请求都带稳定 request id。连接结果不确定时先查询请求结果，不能盲目重复生成 commit。每次 write 增加 `dirty_generation`；成功发布后记录 `published_generation` 和 commit id，没有新 write 的重复 synchronize 直接返回同一个结果。已完成请求至少保留最近 65,536 条，并每 4,096 条批量清理一次更早记录；清理只影响超过重试窗口的结果，不触碰 handle、staging、文件版本或 workspace commit。 + +SQLite schema 检查属于连接初始化工作。`vexfs_init()` 每次都执行完整验证和迁移，失败后可在同一连接修复并重试；普通 `stat/read/write/xattr` 等热路径复用本连接已经验证的状态。连接关闭时扩展自动清除缓存，不依赖 SQLite 3.44 才增加的 client-data API，因此仍保持 SQLite 3.24 的宿主门槛。 + +成功 `write` 表示数据库 staging 已保存，但不表示新版本已经对其他 handle 可见；成功 +`synchronize` 表示版本已发布，并按 adapter 声明的 durability 策略完成提交。普通 SQL +函数遵守调用者连接配置；macOS SQLite mount 使用自己的专用连接，明确采用 +`WAL + NORMAL staging + FULL publish/synchronize/close`。它不修改其他业务连接，也不 +修改 PostgreSQL `synchronous_commit`。doctor 必须显示该连接的实际配置和风险。 + +FSKit extension 被终止时，已发布版本保持完整，未发布 staging 不出现在目录树中。重新挂载只检查和恢复 staging，只有超过保留期且确认未发布后才能回收。adapter 还必须限制单次 write、dirty handle 和 staging 字节数，对 SQLite 单 writer 施加背压。 + +`mkdir`、`unlink`、`rename` 和 `truncate` 各自使用一个短数据库事务。多条 Bash 命令默认不是同一个数据库事务；多文件或“文件 + 业务表”原子提交仍使用 SQL 事务或 changeset。 + +### 4.2 FSKit 调用集 + +最初 MVP 使用当前稳定 Xcode SDK 中的 `FSUnaryFileSystem`、`FSVolume.Operations`、`FSVolume.OpenCloseOperations` 和 `FSVolume.ReadWriteOperations`。macOS adapter 单独封装这些 API，避免共享核心依赖 Apple 类型，并为后续 `Handler` API 迁移保留边界。 + +MVP 支持： + +- volume：probe、load/unload resource、activate/deactivate、mount/unmount、synchronize、statfs； +- 查找和属性：lookupItem、getAttributes、enumerateDirectory； +- 文件：openItem、closeItem、createItem、read、write、truncate、reclaimItem； +- 目录：createItem、removeItem、renameItem； +- flags：O_CREAT、O_EXCL、O_TRUNC、O_APPEND、只读和读写。 + +最初 MVP 必须覆盖原子 rename、open 后 unlink、最终 close、O_APPEND 和版本冲突。当前合同已经继续加入 chmod、chown、symlink、hardlink 和 xattr，并由各 mount adapter 按能力矩阵接入；资源 fork、共享可写 mmap、设备文件、socket 和 FIFO 仍不支持。任何尚未接入的平台能力返回稳定 `EPERM` 或 `ENOTSUP`，不能假装成功。 + +数据库 `inode_id` 稳定映射为 FSKit `FSItem`： + +- lookup 返回或复用同一个 inode 的 FSItem 表示； +- removeItem 只移除 dentry，已打开 item 继续引用原 inode 版本； +- rename replace 在一个数据库事务中替换目标 dentry，目标 item 延迟到 reclaim 后释放； +- reclaimItem 只释放 adapter 对象，数据库 inode 必须在没有 dentry、handle、版本或快照引用后才能进入 GC。 + +FSKit 的 open-unlink emulation、advisory lock、Finder xattr 和文件协调行为必须通过真实回调测试确认；未确认前不进入 MVP 承诺。 + +### 4.3 缓存与一致性 + +MVP 采用 close-to-open 一致性： + +- 同一 handle 读取时叠加自己的 staged writes； +- 其他已打开 handle 固定读取其打开时版本； +- publish/synchronize 成功后，新 open 读取新版本； +- 多 mount 主动失效完成前不承诺立即看到新版本，但发布时必须检测 expected version 冲突。 + +缓存策略以权限和正确性为先： + +- MVP 不启用 FSKit kernel data cache； +- 在提供 `DataCacheHandler` 的 SDK 上显式协商 no-cache；稳定旧 SDK 通过 ReadWriteOperations 让读写进入 extension； +- 每次 publish 前重新检查数据库版本、权限和配额； +- SQLite MVP 只有一个 mount，不实现跨 mount 主动失效； +- Phase 0 记录属性、目录和数据缓存的真实 callback 行为，发现无法避免的陈旧权限时必须阻断 MVP。 + +当前 FSKit 的已实现优化不使用 `DataCacheHandler`，也不把缓存变成权威状态： + +- 不超过 1 MiB 的只读普通文件在 FSItem 的 open/close 生命周期内保存完整内容快照； +- 同一 vnode 出现可写打开或 truncate 时立即清除快照； +- rename/unlink 后快照继续代表原 inode，使已打开文件仍可读； +- 超过 1 MiB 的文件和所有可写文件继续使用数据库 handle； +- close/reclaim 清除进程内快照，重挂载后重新从数据库读取； +- 该策略只解决本地单 mount 文本搜索的 handle 开销，不解决多 mount、远程权限失效或 mmap。 + +同一 mount 内对同一 vnode 的本地写和 truncate 会主动失效快照，所以已有只读描述符会看到 +本地已提交变更；独立数据库 handle 和未来其他 mount 仍按 close-to-open 与版本冲突规则处理。 + +只读 kernel cache、主动 invalidation 和 mmap 在后续性能数据表明必要时再设计。启用前必须先证明缓存不会绕过权限、版本和 close-to-open 合同。 + +### 4.4 身份与沙箱模型 + +当前 FSKit 本地预览只服务当前 macOS 用户，并绑定一个 SQLite context、workspace 和数据库 principal： + +- 默认 SQLite 数据库位于 `~/Library/Application Support/VexFS/`； +- CLI 把数据库所在目录作为 security-scoped `FSPathURLResource` 交给 FSKit； +- extension 只访问该目录内经过描述文件校验的数据库、WAL 和 SHM； +- mount point 只允许当前用户访问； +- SQLite principal 固定为 workspace owner，不接受 CLI 自报用户； +- 文件和目录保存并报告数据库中的 mode 和 owner/group 元数据；便携 ACL 可以保存和恢复，但本阶段尚未实现完整路径 ACL 授权和继承； +- App sandbox、entitlement、签名和公证配置必须进入安装回归。 + +新版 FSKit `FSContext` 可以提供调用者 uid/gid，但 MVP 不据此实现多用户权限。PostgreSQL 阶段增加数据库 role、路径 ACL，以及每次 read/write/publish 所需的权限复查规则。操作系统身份负责保护本地入口，数据库 principal 负责决定数据库文件权限，两者不能互相替代。 + +## 5. 三端统一 SQL 合同 + +三端注册相同函数名、参数顺序和错误分类。返回表的具体声明按宿主扩展 API 适配。 + +### 5.1 工作区 + +```sql +SELECT vexfs_workspace_create('workspace'); +SELECT * FROM vexfs_workspace_list(); +SELECT vexfs_workspace_stat('workspace'); +SELECT vexfs_workspace_freeze('workspace'); +SELECT vexfs_workspace_drop('workspace'); +``` + +工作区是最高隔离单位，拥有独立根目录、版本序列、权限和空间统计。 + +### 5.2 文件和目录 + +```sql +SELECT vexfs_mkdir('workspace', '/a/b', true); +SELECT vexfs_write('workspace', '/a/b/file.txt', 'hello'); +SELECT vexfs_write_range('workspace', '/a/b/file.txt', 5, ' world'); +SELECT vexfs_read('workspace', '/a/b/file.txt'); +SELECT vexfs_read_range('workspace', '/a/b/file.txt', 0, 5); +SELECT vexfs_move('workspace', '/a/b/file.txt', '/a/file.txt'); +SELECT vexfs_copy('workspace', '/a/file.txt', '/a/file-copy.txt'); +SELECT vexfs_remove('workspace', '/a/file-copy.txt', false); +``` + +### 5.3 查询 + +```sql +SELECT * FROM vexfs_list('workspace', '/a'); +SELECT * FROM vexfs_stat('workspace', '/a/file.txt'); +SELECT * FROM vexfs_history('workspace', '/a/file.txt'); +SELECT * FROM vexfs_diff('workspace', 100, 120); +SELECT vexfs_grep('workspace', '/a', 'error', 0, 1000); +SELECT vexfs_grep_index('enable'); +SELECT vexfs_grep_index('status'); +``` + +`vexfs_grep(workspace, path, pattern, flags, limit)` 是普通文本查找，不是语义检索。 +`flags=1` 表示 ASCII 忽略大小写，`flags=2` 表示每个文件只返回一次。它保证 UTF-8 +字面匹配、行号和递归目录查询，二进制文件跳过；正则表达式不在当前合同内。 + +默认实现用一个 SQLite 连接批量扫描当前文件版本，不经过 mount 的逐文件回调。用户可 +显式启用数据库级 FTS5 trigram 索引；索引关闭时没有额外写放大，开启后由文件写入和 +恢复事务同步维护。短查询或不支持 FTS5 的宿主自动回退批量扫描。 + +### 5.4 快照和恢复 + +```sql +SELECT vexfs_snapshot_create('workspace', 'before-change'); +SELECT * FROM vexfs_snapshot_list('workspace'); +SELECT vexfs_restore_file('workspace', '/a/file.txt', 'before-change'); +SELECT vexfs_restore_workspace('workspace', 'before-change'); +SELECT vexfs_snapshot_drop('workspace', 'before-change'); +``` + +### 5.5 权限和审计 + +```sql +SELECT vexfs_grant('workspace', '/reports', 'report_writer', 'read,write'); +SELECT vexfs_revoke('workspace', '/reports', 'report_writer', 'write'); +SELECT * FROM vexfs_audit('workspace', since_commit => 100); +``` + +PG 中的 principal 是数据库 role。DuckDB 和 SQLite 使用宿主数据库能提供的身份边界；不能假装三个数据库都有 PG 级别的角色系统。 + +### 5.6 检查和维护 + +```sql +SELECT vexfs_check('workspace', 1); -- deep: 结构、引用、长度和 SHA-256 +SELECT vexfs_check('workspace', 0); -- quick: 不读取 BLOB 计算 SHA-256 +SELECT * FROM vexfs_repair_plan('workspace'); +SELECT vexfs_gc('workspace', 1000); +SELECT vexfs_retention_set('workspace', keep_versions => 20, keep_days => 30); +``` + +repair 必须先给出计划。缺失文件块时不能填空内容后报告成功。 + +SQLite 当前实现只读 check，没有 repair。默认 CLI 使用 deep；每个 canonical 完整 BLOB 按 +64 KiB 流式读取。版本恢复生成的 alias 必须直接引用 canonical 版本，并复制相同的 size 和 +SHA-256。当前仍是 `full-blob-v1`，chunk/manifest 要等后续内容模型落地后再加入检查项。 + +### 5.7 Worktree changeset + +CLI 不能直接写内部表。worktree 差异必须通过数据库公开的 changeset 接口提交： + +```sql +BEGIN; + +SELECT vexfs_changeset_open('workspace', 847); +SELECT vexfs_changeset_put('/src/main.cpp', :content, :expected_version); +SELECT vexfs_changeset_move('/src/old.cpp', '/src/new.cpp', :expected_version); +SELECT vexfs_changeset_remove('/tmp/output', :expected_version, true); +SELECT vexfs_changeset_apply('update generated files'); + +COMMIT; +``` + +这是概念合同，三个数据库可以使用事务级临时表或流式接口承载大文件，但必须满足： + +- `open` 固定 workspace 和 base commit，并锁定最终发布条件； +- `put/move/remove` 只写当前事务的暂存 changeset，不立即发布目录树； +- `apply` 一次完成权限、配额、冲突、版本和审计检查； +- 未执行 `apply`、数据库 rollback 或连接中断时，不产生可见变更； +- 一个 changeset 只产生一个 workspace commit； +- 内容太大时使用分块流，不要求 CLI 把整个工作区装进一个 SQL 参数。 + +### 5.8 Mount handle 合同 + +gateway 不能直接写内部表。它使用一组窄的、可版本化的数据库合同： + +```text +vexfs_handle_open(workspace, path, flags, request_id) + -> handle_id, inode_id, expected_version, size, mode + +vexfs_handle_read(handle_id, offset, length) + -> bytes, inode_version + +vexfs_handle_stage_write(handle_id, offset, bytes, request_id) + -> staged_size + +vexfs_handle_setattr(handle_id, size?, times?, request_id) +vexfs_handle_publish(handle_id, generation, durability, request_id) + -> publish_state, new_inode_version, workspace_commit, durable +vexfs_mount_synchronize(workspace, request_id) + -> published_handles, workspace_commit?, durable +vexfs_handle_close(handle_id, retained_modes, request_id) + -> close_state, recoverable_staging_id? +vexfs_item_reclaim(workspace, inode_id, request_id) +vexfs_handle_request_status(handle_id, request_id) + -> request_state, result +vexfs_handle_renew(handle_id) +vexfs_handle_abort(handle_id) +``` + +目录操作继续复用 `lookup/list/mkdir/move/remove` 合同。每个写入和发布请求带稳定 request id，连接结果不确定时先查询 request status，而不是盲目重复写。`publish` 和 `synchronize` 对同一个 dirty generation 必须返回同一个 commit 结果。close/reclaim 不接受绕过 publish 状态机的快捷参数。 + +handle 保存在数据库的未发布区，带 owner principal、expected version、租约和状态；它不是数据库事务，也不能越过重新认证后的权限检查。gateway 只保存数据库 handle id 和短期读取缓存，进程退出后不拥有任何已提交状态。 + +## 6. 外部 API 要保持小 + +上面的 SQL 是概念合同，最终不应把每个内部步骤都公开。 + +建议公开： + +- 工作区管理； +- 目录管理； +- 文件整体和范围读写； +- move/copy/remove； +- list/stat/history； +- snapshot/restore； +- grant/revoke/audit； +- check/GC； +- export/import； +- mount handle 的 open/read/stage/publish/synchronize/close/reclaim/request-status； +- worktree changeset 暂存和原子提交。 + +以下只作为内部函数： + +- inode 分配； +- dentry 版本更新； +- chunk 插入； +- manifest 发布； +- commit id 分配； +- 引用计数更新； +- 审计 hash 链更新。 + +内部表和内部函数必须由数据库权限系统保护。 + +## 7. 统一核心模块 + +### NamespaceCore + +- 路径解析； +- 文件和目录创建； +- move、copy、remove； +- 名称校验； +- 目录唯一性； +- 安全的工作区内相对 symlink； +- hardlink 的 dentry 共享 inode 和引用关系； +- 循环和越界检查。 + +### ContentCore + +- BLOB 整体读写； +- 范围读写； +- chunk 切分； +- 稀疏区间； +- manifest； +- checksum； +- 小文件内联； +- 大文件分块。 + +### TransactionCore + +- 工作区 head 锁； +- inode generation； +- 唯一冲突； +- 幂等 request id； +- 后端错误转统一错误； +- 所有写入服从当前数据库事务； +- rollback 后无残留状态。 + +### VersionCore + +- inode 和 dentry 显式版本； +- 文件历史； +- 工作区 commit 序列； +- 快照； +- 单文件恢复； +- 目录恢复； +- 工作区恢复； +- diff； +- 版本保留规则。 + +### AccessCore + +- 数据库 principal 映射； +- workspace/path 权限； +- read/write/delete/admin 权限； +- 权限继承； +- 空间限制； +- 内部对象保护。 + +### AuditCore + +- 当前数据库用户； +- 事务和 commit id； +- 操作类型； +- 路径和 inode； +- 修改前后版本； +- 审计 hash 链； +- 审计查询。 + +### IntegrityCore + +- chunk checksum； +- manifest checksum； +- inode/dentry 引用； +- hardlink 启用后的引用计数； +- 孤立对象； +- 缺失 chunk； +- commit 链； +- 审计链； +- 修复计划。 + +### MaintenanceCore + +- 不可达版本回收； +- 不可达 chunk 回收； +- 保留策略； +- 批量 GC； +- 统计信息； +- schema 版本； +- 扩展升级检查。 + +## 8. 数据模型 + +内部对象使用宿主数据库正常管理的表、索引和 BLOB，不建立私有磁盘格式。 + +### 8.1 工作区 + +```text +vexfs_workspaces + workspace_id + name + root_inode + head_commit + next_inode + frozen + owner_principal + created_at +``` + +所有修改先锁定 workspace head，再分配 commit id。这样同一工作区的版本顺序稳定。 + +### 8.2 inode 版本 + +```text +vexfs_inode_versions + workspace_id + inode_id + version_id + valid_from_commit + valid_to_commit + kind + mode + owner_principal + size + manifest_id + deleted + created_at + modified_at +``` + +版本记录不可原地改写。`valid_from_commit <= snapshot_commit < valid_to_commit` 表示该快照看到的版本。 + +### 8.3 目录项版本 + +```text +vexfs_dentry_versions + workspace_id + parent_inode + raw_name + child_inode + valid_from_commit + valid_to_commit + tombstone +``` + +文件名按字节比较，不使用语言区域排序。禁止 NUL 和 `/`。 + +### 8.4 文件 manifest 和块 + +```text +vexfs_manifests + workspace_id + manifest_id + file_size + block_size + checksum + +vexfs_chunks + workspace_id + manifest_id + chunk_no + data_blob + checksum +``` + +规则： + +- 默认块大小建议 64 KiB； +- 小文件允许内联； +- manifest 发布后不可修改； +- 随机写只产生受影响的新块； +- 第一版不做跨文件去重； +- 文件内容全部由数据库 BLOB 管理。 + +### 8.5 快照 + +```text +vexfs_snapshots + workspace_id + snapshot_name + commit_id + created_by + created_at +``` + +快照只记录 commit id，接近 O(1)。长期历史由显式版本表保证，不依赖数据库 MVCC 旧版本保留。 + +### 8.6 权限和审计 + +```text +vexfs_acl + workspace_id + inode_id + principal + permissions + +vexfs_audit_events + workspace_id + commit_id + seq + principal + operation + inode_id + before_version + after_version + previous_hash + event_hash +``` + +权限不只分为 read/write。统一权限集合为： + +```text +traverse 经过目录解析下级路径 +list 查看目录成员 +read 读取文件内容和基本属性 +create 在目录中创建文件或子目录 +write 修改已有文件内容和属性 +delete 删除目录项 +rename 移动或改名目录项 +history 查看历史版本和 diff +snapshot 创建和恢复快照 +share 修改 ACL +admin 修改 owner、配额和保留规则 +``` + +规则： + +- 每次 SQL 操作、mount open/发布和 worktree commit 都在数据库内重新检查权限； +- checkout 时只能导出当前用户可读的路径； +- worktree 中的本地 chmod 不能改变数据库权限； +- `vexfs chmod/chown/acl` 必须调用数据库 SQL，并要求 `share` 或 `admin`； +- 删除和改名同时检查源目录、目标目录及目标文件权限； +- 快照恢复不能绕过当前权限； +- owner 和 ACL 的每次变化都写入审计记录； +- PostgreSQL principal 对应数据库 role；DuckDB 和 SQLite 使用宿主可证明的连接身份，不能接受 CLI 自报身份。 + +### 8.7 Mount handle 和未发布暂存 + +```text +vexfs_open_handles + handle_id + workspace_id + inode_id + principal + expected_version + open_flags + state # OPEN_CLEAN / DIRTY / PUBLISHING / DIRTY_FAILED / RELEASED + dirty_generation + published_generation + published_commit_id + lease_until + last_error_code + last_request_id + created_at + +vexfs_staging + handle_id + generation + logical_size + capacity + created_at + updated_at + +vexfs_staging_data + handle_id + content_blob +``` + +handle 和 staging 都是数据库受管的临时状态：普通 list/read 看不到，备份恢复后也不会被当成已发布文件。`publish/synchronize` 在短事务中把 staging 组装成不可变 manifest 并发布；close/reclaim 只结束 adapter 引用并记录仍需恢复的 staging。 + +SQLite Phase 0 的物理表为 `_vexfs_handles`、`_vexfs_staging` 和 +`_vexfs_staging_data`。小而频繁变化的 generation、logical size 和 capacity 与大 BLOB +分开；每个 handle 在数据表中只有一行，使用 SQLite incremental BLOB I/O 做分块和 +随机写。`_vexfs_meta.staging_layout=split-v1` 标记该布局。打开旧的 0.2 内联布局时， +插件在 savepoint 内把 BLOB 迁移到数据表；旧元数据表里的 `content` 兼容列保留为空。 +后续 PostgreSQL 或 DuckDB 可以按宿主特点使用 chunk 表，但不能重新把热点元数据和 +大内容绑在同一条反复改写的记录中。 + +重挂载时，gateway 先扫描自己 principal 和 workspace 下的 `DIRTY_FAILED`、过期 `DIRTY` 和未确认 `PUBLISHING`： + +1. 有 request result 的 `PUBLISHING` 直接恢复为对应 committed 状态； +2. 完整且未冲突的 dirty staging 可以由显式 `vexfs recover` 重试； +3. 自动清理必须等待保留期，并再次确认没有已发布 manifest 引用； +4. 回收未发布块不能删除任何已发布版本。 + +## 9. 事务规则 + +### 参加用户事务 + +VexFS 函数必须使用当前数据库事务。 + +```sql +BEGIN; +SELECT vexfs_write(...); +SELECT vexfs_move(...); +ROLLBACK; +``` + +rollback 后，write 和 move 都不可见。 + +### 每次修改 + +1. 检查数据库用户权限； +2. 锁定 workspace head； +3. 解析当前路径和 inode 版本； +4. 校验冲突和配额； +5. 写新 chunk/manifest； +6. 关闭旧 inode/dentry 版本； +7. 写新版本； +8. 增加 head commit； +9. 写审计记录； +10. 返回结果，等待数据库事务 commit/rollback。 + +插件不能自行 commit。 + +### 并发 + +- PG 使用数据库行锁、唯一约束和 MVCC； +- DuckDB 使用宿主事务和 optimistic conflict； +- SQLite 使用宿主事务、单 writer 和 busy 处理； +- 三端统一返回 `VEXFS_CONFLICT`、`VEXFS_BUSY` 等错误分类。 + +## 10. 数据库原生备份与 VexFS 导出 + +VexFS 的保护分为四层： + +1. **文件版本**：每次提交保留旧版本，用于查看和恢复单个文件； +2. **工作区快照**：固定一个 commit，用于恢复目录或整个工作区； +3. **数据库原生备份**：灾难恢复的完整备份，包含业务数据和 VexFS； +4. **工作区逻辑导出**：用于单工作区离线保存和跨数据库迁移。 + +数据库原生备份是第一灾难恢复机制： + +- PG 的原生备份必须包含 VexFS schema、表、索引和 BLOB； +- DuckDB 数据库备份必须包含 VexFS 对象； +- SQLite backup API 或安全数据库备份必须包含 VexFS shadow tables； +- VexFS 不控制备份进程，不建立自己的物理备份系统。 + +VexFS 可以额外提供**工作区逻辑导出**，用途只有： + +- 导出单个 workspace； +- 恢复单个 workspace； +- PG、DuckDB、SQLite 之间迁移； +- 验证工作区级内容。 + +概念 API： + +```sql +SELECT * FROM vexfs_export('workspace', 'snapshot-name'); +SELECT vexfs_import_begin('new-workspace'); +SELECT vexfs_import_record('new-workspace', record_type, record_key, payload, checksum); +SELECT vexfs_import_finish('new-workspace'); +``` + +CLI 可以提供： + +```bash +vexfs snapshot create workspace before-change +vexfs snapshot list workspace +vexfs restore workspace --snapshot before-change --path /reports +vexfs export workspace --snapshot before-change --output workspace.vexfs +vexfs import workspace.vexfs --workspace restored-workspace +vexfs backup verify workspace +``` + +`backup verify` 只检查当前数据库、快照和逻辑导出的完整性，不假装替数据库执行物理备份。 + +实际大数据导出使用数据库自己的 COPY、结果流或应用接口。VexFS 不上传 S3、不调度定时任务、不保存密钥。备份调度、保存位置、加密和异地复制仍由数据库运维系统负责。 + +## 11. 三个宿主适配器 + +### PostgreSQL + +- 随 PG extension 生命周期安装、升级和卸载； +- 使用 PG memory context； +- 不跨 PG boundary 抛 C++ 异常； +- 使用 PG table、index、BLOB/TOAST、事务、锁、WAL 和权限； +- SQL 函数默认 `SECURITY INVOKER`； +- 内部 schema 不授予普通用户直接写权限； +- PostgreSQL 数据库进程不启动 FSKit/FUSE/WinFsp loop 或网络线程；外部 mount adapter 使用正常 PostgreSQL 连接和 role。 + +### DuckDB + +- 随 DuckDB extension 加载和 catalog 管理； +- 使用 DuckDB allocator、Value、transaction 和 storage API； +- 使用普通表和 BLOB； +- optimistic conflict 转统一错误； +- 不直接读写 `.duckdb` 文件； +- 不自建 WAL 或后台服务。 +- 首个 DuckDB 版本只承诺 SQL、CLI 和 worktree； +- 可写 mount 必须让 gateway 成为唯一 DuckDB 写进程，或者等待宿主提供并验证可用的多进程写入能力； +- 不允许 gateway 和外部应用进程同时以普通方式写同一个 `.duckdb` 文件。 + +### SQLite + +- 随 SQLite extension 连接生命周期注册； +- 使用 SQLite allocator、transaction、BLOB 和 VFS； +- 使用受保护的 shadow tables； +- 不直接读写 `.db`、`-wal` 或 `-shm`； +- `SQLITE_BUSY` 转统一错误； +- 普通 SQL 函数不改变调用者的 journal mode 或 synchronous； +- macOS mount 使用独立读写连接，打开时固定 `journal_mode=WAL`、`foreign_keys=ON`、 + `synchronous=FULL`；未发布 stage write/truncate 临时使用 NORMAL，任何发布、sync、 + close、目录修改和 session 结束前切回 FULL； +- NORMAL 阶段允许电源故障丢失最近的未发布 staging，但不得产生半个已发布版本； + publish/synchronize 成功返回时，权威事务和它依赖的 WAL frame 必须达到 FULL； +- 宿主负责 SQLite 版本和文件系统安全，busy timeout 和实际 PRAGMA 值由 doctor 显示。 + +## 12. Mount 平台适配器 + +### macOS FSKit（MVP） + +首个 mount adapter 由 `VexFS.app` 内的 FSKit File System Extension 提供： + +```text +macOS VFS / mount(8) + | + FSKit + | +VexFSFileSystem.appex (Swift) + | +MountContract C ABI + | +SQLite C API + VexFS SQLite extension + | +security-scoped 目录中的数据库 +``` + +- Swift 层只负责 FSKit 类型、callback、result 和 POSIXError 映射； +- 平台无关 mount contract 使用窄 C ABI，不让共享 C++ 核心依赖 Swift 或 FSKit； +- adapter 通过 SQLite C API 调用公开 VexFS SQL/handle 合同，不直接写 shadow table； +- 一个串行 writer connection 处理 staging 和 publish，有限 read connection 处理读取； +- SQLite WAL、busy_timeout、连接数和重试上限由 context 配置并在 doctor 中显示； +- 每个 mount 把数据库父目录作为 security-scoped `FSPathURLResource`，不使用 App Group entitlement； +- 父目录内的 `.vexfs-volume.json` 只保存合同版本、数据库文件名和 workspace，不保存密码、绝对路径或文件数据；FSKit `loadResource` 校验文件名不能逃逸该目录后再打开数据库； +- MVP 要求一个数据库文件独占一个父目录，避免两个挂载改写同一个描述文件； +- 当前稳定 SDK 的 `FSVolume.Operations` 封装在单独 compatibility layer；新版 `FSVolume.Handler` 稳定后只替换该层； +- App Extension 由 macOS 启停，重启后通过数据库 handle/staging 状态恢复，不依赖进程内存。 +- 小文件只读缓存只存在 extension 内存中，最大 1 MiB；14 项生命周期测试覆盖 rename、 + unlink、同名重建、并发改写、truncate 和大文件 handle 回退。 + +### Linux libfuse3（预览已实现） + +- 当前预览使用 FUSE3 high-level path API，并把数据库 inode_id 返回为稳定 inode number； +- 已处理 getattr/readdir/open/create/read/write/truncate/rename/remove、chmod/chown、 + hardlink、symlink、xattr、flush/fsync/release、direct I/O、statfs 和 `fusermount3`； +- 新建节点采用发起请求的 uid/gid；默认 root inode 在首次普通用户挂载时接管默认的 + 0:0 owner，避免 `default_permissions` 阻止用户在根目录创建文件； +- 复用同一 mount contract，不把 FUSE 语义或最终文件状态写到数据库之外； +- 四类时间戳、最小 `flock`/`fcntl`、原子 append、打开文件 rename/unlink、强制卸载和 + helper 崩溃恢复已接入公共合同并通过真机 eval； +- 进入稳定版前仍需补 low-level lookup/forget、内核缓存和 TTL 调优、 + fallocate/copy_file_range、更多 Linux 发行版、长时间运行和规模测试。 + +### Windows WinFsp（后续） + +- 使用 WinFsp user-mode file system API； +- 单独映射 CreateFile share mode、delete-pending、盘符、ACL/SID、reparse point 和 attributes； +- 复用同一 mount contract，不把 NT 内核类型写入数据库核心。 + +## 13. HostStore 合同 + +共享核心通过小接口调用宿主能力： + +```text +CurrentTransaction() +CurrentPrincipal() + +LockWorkspace(workspace_id) +GetWorkspace(workspace_id) +UpdateWorkspaceHead(workspace_id, expected_head, new_head) + +GetInodeAt(workspace_id, inode_id, commit_id) +PutInodeVersion(version) +LookupDentryAt(workspace_id, parent_inode, raw_name, commit_id) +ListDentriesAt(workspace_id, parent_inode, commit_id, cursor, limit) +PutDentryVersion(version) + +PutManifest(manifest) +PutChunk(chunk) +GetChunk(manifest_id, chunk_no) + +PutSnapshot(snapshot) +PutAudit(events) +``` + +适配器只负责把这些调用映射到宿主 API。它不拥有事务，也不能改变核心文件规则。 + +## 14. 错误合同 + +```text +VEXFS_NOT_FOUND +VEXFS_ALREADY_EXISTS +VEXFS_NOT_DIRECTORY +VEXFS_IS_DIRECTORY +VEXFS_NOT_EMPTY +VEXFS_PERMISSION_DENIED +VEXFS_READ_ONLY +VEXFS_QUOTA_EXCEEDED +VEXFS_CONFLICT +VEXFS_BUSY +VEXFS_NO_SPACE +VEXFS_CORRUPTION +VEXFS_INVALID_PATH +VEXFS_UNSAFE_LINK +VEXFS_UNSUPPORTED +VEXFS_VERSION_MISMATCH +VEXFS_UNAVAILABLE +VEXFS_TIMEOUT +VEXFS_STALE_HANDLE +``` + +数据库适配器保留底层诊断信息，但用户 API 返回统一错误类型和通俗错误消息。macOS FSKit adapter 在实现前冻结以下 POSIXError 映射；Linux 和 Windows adapter 再做平台专项映射： + +| VexFS 错误 | errno | +|---|---:| +| `NOT_FOUND` | `ENOENT` | +| `ALREADY_EXISTS` | `EEXIST` | +| `NOT_DIRECTORY` | `ENOTDIR` | +| `IS_DIRECTORY` | `EISDIR` | +| `NOT_EMPTY` | `ENOTEMPTY` | +| `PERMISSION_DENIED` | `EACCES` | +| `READ_ONLY` | `EROFS` | +| `QUOTA_EXCEEDED` | `EDQUOT` | +| `NO_SPACE` | `ENOSPC` | +| `CONFLICT` | `EAGAIN` | +| `BUSY` | `EBUSY` | +| `CORRUPTION`、`UNAVAILABLE` | `EIO` | +| `INVALID_PATH` | `EINVAL` | +| `UNSAFE_LINK` | `ELOOP` | +| `UNSUPPORTED` | `ENOTSUP` | +| `VERSION_MISMATCH` | `EPROTO` | +| `TIMEOUT` | `ETIMEDOUT` | +| `STALE_HANDLE` | `ESTALE` | + +当前平台已接入的 chmod/chown 调用数据库合同；尚未接入时返回 `EPERM` 或 `ENOTSUP`。尚未支持的跨 workspace rename 返回 `EXDEV`。同一错误不能因为数据库宿主不同而改变 errno。 + +## 15. 仓库结构 + +```text +vexdb_lite/ +├── common/ # 原向量算法，不改职责 +├── vexdb_pg/ +├── vexdb_duckdb/ +├── vexdb_sqlite/ +├── agent_files/ +│ ├── include/ +│ │ ├── vexfs_types.h +│ │ ├── vexfs_error.h +│ │ └── host_store.h +│ ├── core/ +│ │ ├── namespace.cpp +│ │ ├── content.cpp +│ │ ├── transaction.cpp +│ │ ├── version.cpp +│ │ ├── access.cpp +│ │ ├── audit.cpp +│ │ └── integrity.cpp +│ ├── adapters/ +│ │ ├── pg/ +│ │ ├── duckdb/ +│ │ └── sqlite/ +│ └── tests/ +├── tools/ +│ ├── vexfs_cli/ # 安装、mount、直接命令和 worktree 入口 +│ └── vexfs_mount/ +│ ├── common/ +│ │ ├── mount_contract/ # 平台无关 open/read/stage/publish/sync/close/reclaim +│ │ ├── db_client/ # 公开 handle/SQL 合同客户端 +│ │ ├── handle_manager/ # handle、租约、generation 和恢复 +│ │ └── error_map/ # VexFS 错误到平台错误 +│ ├── macos/ +│ │ ├── VexFSApp/ # macOS App 和首次启用引导 +│ │ ├── FileSystemExtension/ # FSKit App Extension +│ │ ├── FSKitCompatibility/ # Operations/Handler API 隔离 +│ │ └── MountContractBridge/ # Swift 到 C ABI +│ ├── linux/fuse_gateway/ # libfuse3 预览 adapter +│ └── windows/winfsp_gateway/ # 后续 WinFsp adapter +├── packaging/vexfs/ +│ ├── macos/ # App、appex、CLI、SQLite 扩展、签名和公证 +│ ├── linux/ # FUSE 组合包（待补正式打包） +│ └── windows/ # 后续 WinFsp 组合包 +└── tests/spec/agent_files/ +``` + +仓库不增加远程文件协议、SDK 或独立权威服务。macOS FSKit extension 通过自己的 SQLite 连接加载数据库扩展并调用公开合同；PostgreSQL 阶段使用正常客户端连接。任何 adapter 启动前都必须与扩展交换合同版本和能力位，主版本不兼容时拒绝可写挂载。 + +## 16. 验收测试 + +### SQL 合同 + +- 三端相同函数名和参数顺序； +- 相同操作序列得到相同目录树； +- 内容 checksum 一致； +- 错误分类一致； +- list/stat/history 结果字段一致。 + +### CLI + +- `vexfs setup/mount/unmount/mount status/doctor` 生命周期正确； +- `vexfs ls/cat/grep/stat` 与对应 SQL 结果一致； +- 支持 PostgreSQL、DuckDB 和 SQLite 连接配置； +- 默认一个写命令使用一个数据库事务； +- 权限错误、文件不存在和连接失败返回不同退出码； +- 标准输出可以安全用于管道，诊断信息写入标准错误； +- worktree 只导出有权读取的普通文件和安全链接； +- 系统 `sed/python/tar/git/apply_patch` 修改可以正确生成 changeset； +- worktree 运行期间不持有长期数据库事务或锁； +- commit 时重新检查权限、配额和起始版本； +- changeset 全部成功或全部回滚； +- 命令失败或提交冲突时保留本地差异； +- 越界链接、设备文件、socket 和 FIFO 不能提交； +- `vexfs exec` 只有在命令退出成功且数据库提交成功时返回 0。 + +### macOS FSKit mount + +- 真实 coreutils：ls、cat、cp、mv、rm、mkdir、truncate； +- 真实文本工具：rg、grep、sed、awk、jq； +- MVP 真实程序：Python、基础 Git、apply_patch；编译器和大型构建目录只记录兼容性，不作为 MVP SLA； +- flags：O_CREAT、O_EXCL、O_TRUNC、O_APPEND； +- FSKit probe/load/activate/mount/unmount/deactivate 生命周期； +- lookup/get attributes/enumerate/open/close/create/read/write/remove/rename/set attributes/synchronize/reclaim； +- 原子 rename、open-unlink 和最终 close； +- 同一 handle read-your-writes、旧 handle 版本固定、新 open 读取已发布版本； +- synchronize 重复调用只产生一个 commit；真实 fsync(2) 到 FSKit callback 的映射有证据； +- close/reclaim 兜底失败后 staging 可检查、可恢复且不会被发布为半文件； +- 终止 FSKit extension、SQLite reopen 和重新挂载； +- MVP 不启用 FSKit kernel data cache； +- SQLite busy、freeze、quota、空间不足、stale handle 和固定 errno； +- mount point 当前用户私有，security-scoped 数据库目录权限正确； +- 开发签名，以及 Developer ID + 公证或 Mac App Store/TestFlight 至少一条真实分发路径；完成 extension 启用和卸载回归； +- App/appex/CLI 与扩展版本不兼容时拒绝可写挂载； +- 不启用 FSKit extension 时 SQL 和 CLI 仍然完整可用。 + +### 事务 + +- 文件写入和业务表更新同时 commit； +- 文件写入和业务表更新同时 rollback； +- 每个内部写步骤故障注入； +- 不出现半个 dentry 或缺块 manifest； +- savepoint/rollback-to 行为明确并一致。 + +### 并发 + +- 同名创建； +- 同文件双写； +- move 与 remove； +- workspace freeze 与写入； +- PG deadlock； +- DuckDB optimistic conflict； +- SQLite busy。 + +### 版本和恢复 + +- 快照不随当前状态变化； +- 单文件恢复； +- 目录恢复； +- 工作区恢复； +- retention 后受保护快照仍可读取； +- GC 不删除活动版本引用的 chunk。 + +### 数据库管理 + +- extension 安装、升级和卸载； +- 数据库原生备份恢复后 VexFS 文件完整； +- 数据库权限阻止内部表直写； +- 数据库 crash/reopen； +- 数据库只读模式； +- 无宿主允许时不创建线程或外部文件。 + +## 17. 实施顺序 + +详细阶段范围、状态和完成门槛只在 +`docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md` 维护。技术实现顺序与之对应： + +### Phase 0：统一合同与垂直闭环 — 已完成 + +- SQLite schema、SQL、C ABI、CLI 和 mount runtime 形成单一合同； +- macOS FSKit 跑通数据库权威状态、真实 Bash、Git、同步和恢复； +- Linux libfuse3 使用同一 runtime 跑通 root 和普通用户真实挂载； +- version、workspace commit、snapshot、restore 和基础 POSIX metadata 进入自动测试。 + +### Phase 1：SQLite 本地开发者预览 — 进行中 + +- 把 macOS FSKit 与 Linux FUSE 组合到同一 mount 一致性 eval； +- 补齐 utimens、O_APPEND 并发、flock/fcntl 最小合同、open-unlink、崩溃和强制卸载； +- 冻结缓存、TTL、owner/group 和便携 ACL 的跨平台规则； +- 验证 macOS 与 Linux 交换同一个 backup/export 后的内容、元数据、历史和快照； +- 完成 macOS 公证、Linux x86_64/AArch64 包、干净机器安装和卸载； +- 完成真实项目、Coding Agent、10 万文件和长时间运行 eval； +- 完成开发者预览所需的 retention、GC、quota、check 和 export/import 最小集合。 + +### Phase 2：PostgreSQL 共享工作区 Alpha — 未开始 + +- 实现 PG HostStore，不重新定义文件语义； +- 数据库 role 映射 principal、owner、ACL、审计和配额； +- 多 gateway 的锁、缓存失效、冲突、断线重连和数据库重启； +- 文件与普通业务 SQL 原子提交； +- 电脑 A 写入后，电脑 B 通过 macOS/Linux mount 继承同一 workspace、历史和快照； +- pg_dump/pg_restore、物理备份和逻辑导出通过恢复测试。 + +### Phase 3：Windows 与跨系统预览 — 未开始 + +- 实现 WinFsp adapter，复用 mount runtime； +- 单独处理盘符、CreateFile share mode、delete-pending、ACL/SID 和 reparse point； +- 验证 Windows SQLite 本地工作区和 PostgreSQL 远程工作区； +- 运行 macOS/Linux/Windows 三端合同和跨系统可移植性 eval。 + +### Phase 4：DuckDB 本地分析工作区 Beta — 未开始 + +- 实现 DuckDB HostStore 和相同 SQL/C ABI 能力矩阵； +- 验证事务冲突、checkpoint、reopen、backup/copy/export； +- 默认先提供 SQL、CLI 和 worktree； +- 只有 gateway 可以成为唯一写进程时才开放可写 mount。 + +### Phase 5：VexDB-Lite Files v1 — 未开始 + +- 稳定 SQL、C ABI、CLI、schema、能力协商和逻辑导出格式； +- 完成 retention、GC、quota、audit、check、安全和故障修复工具； +- 完成正式平台/数据库能力矩阵、升级保护、规模和长期回归； +- 形成同一版本的发行物、用户文档、管理员文档和故障处理手册。 + +当前编码优先级固定为：共用 mount eval → 时间戳/锁/append/生命周期 → macOS/Linux 可移植性 → 正式安装包 → 真实项目和规模基线。除关键设计验证外，不应跳过这些门槛提前展开 PostgreSQL、Windows 或 DuckDB 大规模实现。 + +## 18. 安装发布与可观测性 + +Phase 1 本地预览发行物是一组经过同版本测试的组合，而不是互不校验的二进制： + +```text +VexFS.app + └── VexFSFileSystem.appex +vexfs CLI +vexfs-sqlite extension +contract/schema version manifest + +Linux package + ├── vexdb/vexfs CLI + ├── vexfs-fuse helper + ├── vexdb_lite.so SQLite extension + ├── install/uninstall + 中文说明 + └── contract/schema/source version manifest + SHA256SUMS +``` + +macOS 可挂载预览支持 macOS 26.0+ Apple Silicon。Phase 1 必须验证 Developer ID + 公证或 Mac App Store/TestFlight 至少一条真实分发路径；CI 检查 App、appex、CLI 的签名链、FSKit entitlement 和系统信任状态。首次启动必须引导用户在系统设置中启用文件系统扩展，不能尝试绕过系统授权。Linux 包必须检查 libfuse3、`fusermount3` 和 `/dev/fuse`，普通用户不能依赖 root helper 才能工作。 + +默认 SQLite 数据库位于 `~/Library/Application Support/VexFS/vexfs.sqlite3`。每个 mount 创建不含密码的 `.vexfs-volume.json`，再把数据库父目录作为 security-scoped FSKit source resource 传给系统；extension 只能打开描述文件指定的同目录数据库。用户选择其他位置时沿用同一目录授权模型。卸载 App 不能自动删除数据库，数据删除必须是独立且明确的动作。 + +每次 mount 的启动顺序固定为： + +1. 检查 macOS、CPU 架构、FSKit、App/appex 签名和 extension 启用状态； +2. 在数据库父目录创建并解析 `.vexfs-volume.json`，把该目录作为 security-scoped resource； +3. 打开 SQLite，加载 VexFS 扩展并认证 principal； +4. 查询 schema、合同版本和能力位； +5. 检查或恢复本 workspace 的未发布 handle； +6. 请求 FSKit mount，成功后才报告可写目录。 + +主合同版本不兼容时拒绝可写挂载。安全卸载先停止新 open，再执行 synchronize，列出未发布 staging，最后调用 FSKit unmount/unload；不能为了卸载干净而删除未确认数据。 + +gateway 至少公开以下状态和指标： + +- 每类 FSKit callback 的次数、P50/P95/P99 和 POSIXError； +- 数据库调用耗时、SQLite busy 次数和重试次数； +- 活动 handle、dirty handle、recoverable staging 数量和字节数； +- publish/synchronize/close 成功、失败和结果不确定次数； +- expected version 冲突、配额失败和空间失败； +- 当前 principal、workspace、扩展版本、gateway 版本和合同版本； +- 最近一次成功 mount、unmount、数据库断线和恢复时间。 + +`vexfs doctor` 汇总上述关键状态，但日志和指标不能包含密码、文件正文或完整敏感路径。 + +## 19. 最终判断 + +VexFS 的产品本体是数据库 extension： + +> **数据库管理 VexFS，VexFS 管理数据库中的文件。** + +用户安装并完成对应系统授权后，优先通过 macOS FSKit、Linux libfuse3 或后续 Windows WinFsp mount 直接使用 Bash 和现有程序；SQL 是完整权威合同，CLI/worktree 是回退。mount adapter 只是访问路径，最终变更仍服从数据库事务、权限、版本、审计、备份和恢复规则。 + +这个设计同时满足两件事：用户获得真实文件路径，数据库继续拥有全部已提交权威状态。VexDB-Lite 的差异点不是单独做一个文件服务，而是让数据库扩展提供受管文件能力，再用 mount 把它交给 Bash。 + +实施上不按“三数据库 × 三操作系统”开发九套文件系统。当前门槛是把已经跑通的 macOS/Linux + SQLite 闭环收口为可安装、可恢复、可跑真实项目的本地开发者预览；达到路线图中的一致性、分发和规模条件后，再把主要开发重心转到 PostgreSQL 多用户共享工作区。 diff --git a/docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md b/docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md new file mode 100644 index 0000000000..9088876283 --- /dev/null +++ b/docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md @@ -0,0 +1,479 @@ +# VexFS 最终目标与阶段路线图 + +- 文档类型：唯一有效的目标和开发顺序基线 +- 所属产品：VexDB-Lite 的文件管理能力 VexFS +- 日期：2026-07-22 +- 分支：`feature/agent_files` +- 文档版本：2.0 +- 当前阶段：Phase 1，SQLite 本地开发者预览 + +## 1. 这份文档解决什么问题 + +VexFS 已经从 SQLite SQL/CLI 发展到 macOS FSKit 和 Linux libfuse3 真实挂载。如果继续按单个功能推进，很容易出现三个问题： + +1. 做了很多 POSIX 功能，但没有形成用户可以安装和验证的产品； +2. 数据库、操作系统和 CLI 各自扩展，最后变成多套不同实现； +3. 新功能不断加入，核心目标“数据库管理的无缝 Agent 工作区”反而被冲淡。 + +从本版本开始，文档职责固定如下： + +- 产品规格定义“产品是什么、用户需要什么”； +- 技术设计定义“统一合同和各适配器怎么实现”； +- **本路线图定义“最终做到哪里、现在在哪、下一步按什么顺序做”。** + +如果其他文档中的阶段顺序、实现状态与本路线图冲突，以本路线图为准。历史分析文档只保留当时的分析价值，不再决定当前开发顺序。 + +## 2. 不变的最终目标 + +> 用户下载并安装 VexDB-Lite 后，可以在 macOS、Linux 或 Windows 上把数据库工作区挂载成普通目录。人、Coding Agent 和现有工具直接使用 Bash、Git、编辑器和编译器，不需要改命令，也不需要学习另一套文件协议。文件、目录、权限、版本、快照、审计、配额和备份都由数据库管理，挂载进程不保存权威数据。 + +VexFS 是 VexDB-Lite 内的一项文件管理能力，不是独立 SDK、独立客户端或反过来管理数据库的文件服务器。 + +### 2.1 最终用户体验 + +```bash +# 安装 VexDB-Lite 后完成一次初始化和授权 +vexdb fs setup + +# 把数据库工作区挂载为普通目录 +vexdb fs --workspace workspace mount ~/Work/agent-workspace + +# 后续全部使用原生命令和现有 Agent +cd ~/Work/agent-workspace +ls +rg "TODO" +git status +opencode + +# 查看和恢复数据库保存的工作区版本 +vexdb fs --workspace workspace snapshot list +vexdb fs --workspace workspace snapshot restore before-refactor +``` + +产品可以继续保留 `vexfs` 兼容命令，但对外安装物和主入口应属于 VexDB-Lite，避免用户把向量、SQLite 基础能力和文件能力理解成三个无关产品。 + +### 2.2 最终能力边界 + +最终版本必须同时满足以下条件： + +1. **统一安装**：同一份 VexDB-Lite 发行物包含数据库基础能力、向量能力和文件能力；安装过程会检查并引导系统挂载依赖。 +2. **无缝终端**：挂载后可直接运行常用 Bash、Git、Coding Agent、编辑器、编译器和构建工具。 +3. **数据库权威**：已提交文件及元数据只以数据库为准；mount adapter 崩溃、重启或被删除不能破坏已提交数据。 +4. **本地和远程**：SQLite 提供单机工作区；PostgreSQL 提供跨电脑、多用户和共享工作区；DuckDB 提供本地分析场景。 +5. **版本和恢复**：支持文件历史、完整工作区快照、一键恢复、备份恢复、保留策略和垃圾回收。 +6. **管理能力**：支持身份、权限、ACL、审计、配额、完整性检查和安全导入导出。 +7. **跨平台一致**：macOS、Linux、Windows 共享同一逻辑合同；平台不支持的能力必须明确返回错误，不能静默丢失数据或权限。 +8. **可发布质量**：有干净机器安装、升级、卸载、故障、规模、安全和性能证据，不以“能编译”或“单个 demo 能运行”代替完成。 + +## 3. 不能改变的架构规则 + +以下规则用于判断一个需求是否会让项目偏离目标： + +1. **数据库管理 VexFS**。数据库扩展是权威核心，挂载进程只是协议翻译层。 +2. **核心不做语义层**。Embedding、向量搜索、BM25 和 Agent 记忆不属于文件系统核心；它们可以读取 VexFS 文件，但不能改变文件合同。 +3. **不是 3 × 3 九套实现**。三个数据库 adapter 加三个操作系统 mount adapter，通过统一 SQL/C ABI 和测试合同组合。 +4. **mount 不拥有最终状态**。它只保留 handle、缓存和未发布写入；所有提交、冲突、权限和版本判断回到数据库合同。 +5. **SQL 和 Bash 都是一等入口**。没有 mount 时数据仍可通过 SQL/CLI 使用；挂载不能成为读取数据库文件的唯一办法。 +6. **快照不等于备份**。快照用于版本检查点和误操作恢复；数据库原生备份或逻辑导出用于灾难恢复和跨机器迁移。 +7. **正确性先于索引加速**。原生 `grep`、`rg` 先按普通文件工作；索引搜索是后续显式能力，不伪装成透明的原生命令加速。 + +## 4. 明确不做什么 + +这些内容不是 VexFS 的目标，除非路线图经过正式修改： + +- 不替代 APFS、ext4、NTFS 等系统根文件系统； +- 不追求所有 POSIX 特殊文件和内核设备语义； +- 不替代 Git 的分支、合并和代码协作能力； +- 不把大视频、模型文件和数据湖对象作为第一目标； +- 不新建脱离数据库权限和事务的文件服务器协议； +- 不让 DuckDB 承担多用户远程文件服务器职责； +- 不为了宣传“更快 grep”而阻塞挂载正确性、备份和跨机器工作区； +- 不在核心文件合同中加入 Agent 专用的 prompt、memory 或语义字段。 + +## 5. 当前基线：2026-07-22 + +| 范围 | 当前事实 | 主要缺口 | +|---|---|---| +| 统一合同 | SQLite 合同 `0.7.0`，runtime ABI 1；SQL、C ABI、CLI、mount runtime 已连通；已增加跨平台一致性合同和结构化错误状态 | PostgreSQL、DuckDB 尚未实现 VexFS adapter | +| 文件语义 | 文件、目录、四类时间戳、并发 append、进程锁、mode、symlink、xattr、hardlink、owner/group 元数据、便携 ACL 已进入数据库合同和测试 | ACL 还没有完整授权执行和继承；特殊文件不支持 | +| 版本恢复 | 单文件版本、workspace commit、snapshot、diff、expected-head restore、SQLite Backup 已实现 | retention、GC、配额、审计、逻辑导入导出未完成 | +| 长期校验 | 每个文件版本持久化 SHA-256；只读 `vexfs_check` / `vexdb fs check` 已覆盖结构、历史、快照、staging、BLOB 长度和流式内容校验 | 自动 repair 不在当前范围；chunk/manifest 尚未落地 | +| macOS | preview.17 已通过 Apple 公证并 staple；本机安装后 Gatekeeper、FSKit 注册、Bash、Git、Python、Node.js、Go、Rust、并发 append、进程锁、打开文件生命周期和强制卸载全部通过 | 尚缺另一台没有源码和 Xcode 构建记录的干净 Mac 验证 | +| Linux | libfuse3 真实挂载已通过 root 和普通用户；时间戳、并发 append、进程锁、打开文件生命周期、强制卸载和 helper 崩溃恢复已验证；x86_64/AArch64 manylinux 安装包已在对应架构真机完成无 root 安装回归 | 缓存/TTL、更多干净发行版、长期运行和真实挂载安装回归仍不足 | +| Windows | 只有平台边界和规划 | WinFsp adapter、安装签名、路径/ACL/SID 合同均未实现 | +| 远程共享 | 统一合同为 PostgreSQL 预留 | 还不能让另一台电脑挂载同一个远程工作区 | +| 性能规模 | SQLite 直连已完成 10 万文件；preview.16 综合 quick 为 300.590 files/s，1 千文件普通 `rg` 为 418.559 ms、索引 grep 为 7.842 ms；固定 full 负载为 31.859 MiB/s 顺序写、215.884 次/s 随机覆盖 | 小文件创建仍比同轮 APFS 慢 40.433 倍，普通 `rg` 慢 23.396 倍；后续仍需重跑 1 万/10 万和真实 Coding Agent | + +“合同中已保存”不等于“所有平台已经完整执行”。例如 owner/group 和便携 ACL 已经可以保存、读取和恢复，但身份认证、权限判断和各系统的原生 ACL 映射仍属于后续阶段。 + +### 5.1 当前收口 Gate + +2026-07-22 产品对齐审查后，Phase 1 先执行收口 Gate，再继续长期存储能力： + +1. **已完成代码修复**：零用例失败、`PASS_WITH_SKIPS`、`--fail-on-skip`、显式 package/mount CLI 绑定、runtime ABI 单一来源、旧 schema 明确拒绝、数据库符号链接拒绝、备份目标 0600、安装哈希/签名/ABI/合同校验； +2. **已完成本地交付验证**：当前源码 ad-hoc 包通过 64 项文档 smoke 和 2 个 package Gate；这只证明本地包结构与功能，不替代 Developer ID、公证和真实 FSKit； +3. **待完成发行证明**：提交当前实现后，从干净 commit 重新 Developer ID 签名、公证，并在启用 FSKit 的 Mac 上用本包 CLI 运行 `--fail-on-skip` 的真实 mount Gate； +4. **随后进入长期安全**：checksum/check 已完成；继续 retention/GC、配额、逻辑 export/import，再做 chunk + manifest 和 append 合并发布。 + +当前不提前开发 PostgreSQL、DuckDB 或 Windows adapter。 + +### 5.2 checksum/check 完成证据 + +2026-07-22 SQLite 合同升到 `0.7.0`：普通写入保存 canonical BLOB 和 SHA-256；单文件恢复 +与 workspace 恢复只创建直接别名，并复制源版本的 size 和 checksum。默认读取、历史读取、 +版本比较和数据库 grep 都会拒绝返回校验失败的内容。 + +- `vexdb fs check` 默认深度检查，`--quick` 只检查结构；不存在的数据库不会被 check 创建； +- 损坏注入覆盖同长度 BLOB 改写、目录父引用、commit 父引用、snapshot、version alias 和 + staging BLOB 缺失；深度内容损坏返回 `VEXFS_CHECKSUM_MISMATCH`，CLI 退出码为 8； +- 最终 quick eval：52 passed、0 failed、18 个平台性 skipped、3121 checks； +- 4 MiB / 4 文件深度扫描为 0.081381 秒、49.152 MiB/s，quick 为 0.001861 秒；检查阶段 + 观测到的额外 RSS 峰值增长为 0； +- 报告：`vexdb_sqlite/build/eval/vexfs/20260722T104155.646993Z-quick-20260718/report.json`。 + +这不是 repair，也不是 chunk 校验。当前内容模型明确报告为 `full-blob-v1`；下一项是 +retention/GC，之后才是 quota、export/import 和 chunk/manifest。 + +## 6. 阶段路线图 + +每个阶段必须同时写清范围、暂时不做的内容和完成证据。没有达到完成条件，不得把阶段标为完成。 + +### Phase 0：统一合同与垂直闭环 — 已完成 + +目标：证明“数据库是权威状态，同时 Bash 可以操作真实挂载目录”在技术上成立。 + +已完成范围： + +- SQLite 内部 schema、SQL API、C ABI、CLI 和 mount handle 合同； +- 写入暂存、publish/sync/close/reclaim、request-id 和崩溃恢复； +- macOS FSKit 真实挂载和 SQLite 垂直闭环； +- 文件版本、workspace commit、快照和恢复； +- mode、symlink、xattr、hardlink、owner/group 和便携 ACL 基础合同； +- Linux libfuse3 的真实普通用户挂载验证。 + +完成证据：数据库重开、挂载重开、基础 Bash/Git、快照恢复、备份恢复和错误注入测试已经形成 eval。 + +### Phase 1：SQLite 本地开发者预览（macOS + Linux）— 进行中 + +目标：把当前原型变成其他开发者可以安装、可以日常运行、出现故障可以恢复的本地产品。 + +本阶段必须完成： + +1. 建立 macOS FSKit 和 Linux FUSE 共用的 mount 一致性 eval，防止两个 adapter 行为分叉； +2. 补齐真实项目最容易碰到的正确性：时间戳更新、`O_APPEND` 并发、`flock`/`fcntl` 最小锁合同、rename/unlink 打开文件、崩溃和强制卸载； +3. 明确缓存和 TTL 行为，验证重新挂载、两个进程和数据库重开后不会读到错误内容或权限； +4. 做 macOS 与 Linux 的 SQLite 工作区互换测试，覆盖 Unicode、大小写边界、mode、symlink、hardlink、xattr/ACL 和快照恢复； +5. 跑真实 Python、Node.js、Go、Rust、Git 和至少一种 Coding Agent 工作区； +6. 完成 1 万、10 万文件以及小文件/大文件/混合读写的正确性和性能基线； +7. macOS 完成 Apple 公证和干净机器安装；Linux 完成 x86_64、AArch64 安装包和普通用户安装说明； +8. 补齐 check、retention、GC、quota、export/import 中保证长期试用不会失控的最小集合。 + +暂时不做： + +- PostgreSQL 多用户产品能力； +- Windows adapter； +- DuckDB adapter； +- 透明加速原生 `grep`； +- 完整 POSIX 特殊文件。 + +完成条件： + +- 两台干净机器只按使用文档即可安装、挂载、运行真实项目、卸载和恢复； +- 普通用户可以持续使用，不要求源码构建或 root 运行 mount helper； +- macOS/Linux 共用一致性 eval 全通过，没有静默丢文件、版本、权限或链接； +- 10 万文件和长时间运行测试达到文档中的性能目标； +- 数据库备份恢复、进程崩溃、断电等价故障和错误卸载有可重复证据； +- 发布包、版本清单、使用文档和已知边界放在一起。 + +### Phase 2：PostgreSQL 共享工作区 Alpha（macOS + Linux）— 未开始 + +目标：让电脑 A 创建的 Agent 工作区可以由电脑 B 通过 PostgreSQL 直接挂载，并由数据库角色管理并发和权限。 + +本阶段必须完成： + +- PostgreSQL HostStore 和与 SQLite 等价的 SQL/C ABI 合同； +- 每个 mount 绑定真实 PostgreSQL role，完成 principal、ACL、审计和配额； +- 多 gateway、多 Agent、文件锁、expected-head 和并发冲突； +- 网络断开、重连、数据库重启、缓存失效和慢网络行为； +- 文件变更与普通业务 SQL 在同一数据库事务提交或回滚； +- pg_dump/pg_restore、物理备份和逻辑导入导出； +- 电脑 A 写入、电脑 B 挂载并继承同一版本历史和快照。 + +完成条件： + +- 两台电脑可以同时操作一个远程 workspace，权限隔离和冲突行为可重复； +- 任一 gateway 或数据库重启后不出现静默覆盖； +- PostgreSQL 原生备份恢复后文件、历史、权限和业务表处于同一时间点； +- macOS 和 Linux 使用同一 mount 一致性测试，不增加 PostgreSQL 专用文件语义。 + +### Phase 3：Windows 与跨系统预览 — 未开始 + +目标：通过 WinFsp 接入同一 mount runtime，使 Windows 既能使用本地 SQLite 工作区，也能挂载 PostgreSQL 远程工作区。 + +本阶段必须完成： + +- WinFsp adapter、盘符/目录挂载、安装、签名和卸载； +- Windows 路径、保留名、大小写、Unicode、share mode 和 delete-pending； +- SID 与 VexFS principal/owner/ACL 的明确映射； +- symlink/reparse point、hardlink、可执行属性和 xattr 的跨系统降级规则； +- macOS、Linux、Windows 三端读写同一逻辑工作区的兼容测试。 + +完成条件：Windows 上常用终端、Git、编辑器、构建工具和 Coding Agent 可以直接使用；跨系统移动工作区不会静默改变内容、版本或权限。 + +### Phase 4：DuckDB 本地分析工作区 Beta — 未开始 + +目标：让 DuckDB 用户在本地分析流程中使用相同文件合同、版本和 CLI，而不是新建一套文件系统。 + +本阶段必须完成： + +- DuckDB HostStore 和相同 SQL/C ABI 能力矩阵； +- 事务冲突、checkpoint、reopen、backup/copy/export； +- SQLite、PostgreSQL、DuckDB 之间的逻辑导入导出； +- 默认提供 SQL、CLI 和 worktree； +- 只有 gateway 能成为唯一写进程时，才开放 DuckDB 可写 mount。 + +完成条件：DuckDB 不产生独有文件语义；相同 eval 可以验证支持的公共能力；不把单写者数据库宣传成多用户共享文件服务器。 + +### Phase 5：VexDB-Lite Files v1 稳定版 — 未开始 + +目标：把已验证能力固化为长期兼容、可升级和可支持的正式产品。 + +本阶段必须完成： + +- 稳定 SQL、C ABI、CLI、schema 和逻辑导出格式； +- retention、GC、quota、audit、check 和修复工具； +- 三个数据库与三个操作系统的正式能力矩阵和版本协商； +- 安装、升级、降级保护、卸载、备份、灾难恢复和安全评审； +- 10 万文件以上规模、长时间运行、并发、故障和性能回归； +- 用户文档、管理员文档、Agent 使用文档和故障处理手册； +- 可选的显式文本索引和搜索命令，前提是不改变普通文件语义。 + +完成条件：所有正式承诺的平台和数据库都有可下载产物、干净机器证据、自动回归和清楚的已知边界。 + +## 7. 现在立即做什么 + +下一轮开发固定按以下顺序推进： + +### 当前固定顺序（2026-07-23 M1 快照恢复复测后） + +1. **P0（已完成）：修复 FSKit 元数据缓存失效。** hardlink 后立即刷新源 inode；create、symlink、 + hardlink、remove、rename 后刷新或失效相关父目录。250 ms TTL 只做兜底，不能代替 + mutation 后精确失效。 +2. **P0（已完成）：修复 macOS 打包注册。** `package_preview.sh` 固定独立 DerivedData， + 在 export 前撤销 InstallationBuildProductsLocation，结束时删除 archive/export/DerivedData； + 安装完成后验证 FSKit module URL 必须指向已安装 App，不能只看 `enabled`。 +3. **P0（已完成）：构建 preview.16 并重跑真实 Gate。** `mount.cross-platform-conformance`、 + `mount.timestamps` 必须恢复全绿；同时重跑 Bash、Git、工具链、强制卸载和 1 千文件 + 性能，确认正确性修复没有明显性能回退。 +4. **P0（代码和 M1 实测已完成）：修复挂载状态快照恢复。** 数据库拒绝活动 mount 的直接 + restore；CLI 正常卸载、恢复并原位重挂载；FSKit 在 `unmount()` / `deactivate()` 关闭 + session，在 `activate()` 重开并清理旧缓存。M1 实测 lease 约 2.832 ms 释放，恢复后内容 + 正确且无残留挂载。 +5. **P0（下一项）：从已提交的干净源码重新签名并公证当前 preview。** 使用已经验证可用的 + `vexdb-lite-notary` profile，要求 Apple `Accepted`、staple、Gatekeeper、安装后 doctor 和 + 快照恢复真机回归全部通过；旧 preview.17 的公证不能替代当前实现的公证。 +6. **P0（紧随公证）：把本次真实挂载快照恢复脚本变成正式 eval。** 覆盖活动 mount 直接 + restore 被拒、一键卸载/恢复/重挂载、lease 立即释放、旧缓存清理、失败回滚、多 workspace + 定位和最终无残留挂载，避免以后只靠 `/tmp` 手工脚本发现回归。 +7. **P1：完成干净 Mac 安装。** 在没有源码、Xcode DerivedData 和历史 VexDB Lite 注册记录的 + Mac 上，仅按压缩包文档完成安装、启用、挂载、Bash/Git、快照恢复和卸载。 +8. **P1：完成长期管理能力。** `check` 已完成，下一项固定为 retention/GC，随后是 quota 和 + export/import，先保证长期使用不会无限增长且可以迁移、恢复。 +9. **P1：恢复规模和 Agent 验证。** 先 1 万文件，再决定是否重跑 10 万；最后执行真实 + OpenCode 工作区。所有大测试继续保留 RSS 上限和低并发。 + +当前 1–4 已完成。接下来严格按 5（当前代码公证）→ 6（永久回归）→ 7（干净 Mac）推进， +然后再做 retention/GC。以上完成前，不把主要开发重心切到 PostgreSQL、DuckDB 或 Windows； +三种数据库和三端适配仍是最终目标，但必须建立在稳定的文件系统合同和发行 Gate 上。 + +### N1. 共用 mount 一致性 eval + +先把同一套文件操作分别跑在 macOS FSKit 和 Linux FUSE 上，覆盖 errno、mode、link、xattr、snapshot、重挂载和故障。它是后续增加 PostgreSQL、Windows 时防止出现九套行为的基础。 + +状态（2026-07-22）：**已完成**。 + +- 已建立 `mount.cross-platform-conformance`，macOS FSKit 与 Linux libfuse3 共用同一测试函数和 50 项检查； +- Linux AArch64 已在真实 `/dev/fuse` 上分别以 root 和 uid 1000 通过； +- 测试发现并修复了非空目录错误由 `EEXIST` 错映射的问题，数据库合同现返回结构化 `VEXFS_MOUNT_NOT_EMPTY`，平台映射为 `ENOTEMPTY`； +- macOS 已使用 `Developer ID Application: Jian ming Wu (BB5VK42K87)` 重新签名并启用 FSKit，在真实挂载、快照恢复和重挂载上通过同一套 50 项检查； +- Developer ID `preview.5` 统一安装回归通过 13 项检查，App、extension、CLI 和 SQLite dylib 的 TeamIdentifier 均为 `BB5VK42K87`。 +- preview.15 的同一合同真实重跑曾发现 hardlink 即时 `st_nlink=1`，而数据库中为 2； + preview.16 在 hardlink 成功后立即刷新源 inode，并把真实 module URL 纳入自动验收， + 最终 macOS 真实挂载 52 项检查全部通过。 + +证据位置： + +- Linux root：`vexdb_sqlite/build/eval/vexfs-linux-mount/root/latest.json` +- Linux uid 1000：`vexdb_sqlite/build/eval/vexfs-linux-mount/uid-1000/latest.json` +- macOS FSKit：`vexdb_sqlite/build/eval/vexfs/20260721T052213.276765Z-quick-20260718/report.json` +- macOS Developer ID 安装：`vexdb_sqlite/build/eval/vexfs/20260721T051850.511156Z-quick-20260718/report.json` +- macOS preview.16：`vexdb_sqlite/build/eval/vexfs/20260722T061930.005789Z-quick-20260718/report.json` +- 测试入口：`tests/eval/vexfs/run_linux_mount.sh` + +### N2. 时间戳、锁、append 和挂载生命周期 + +补 `utimens`、最小 `flock`/`fcntl`、并发 `O_APPEND`、打开文件 rename/unlink、强制卸载和 helper 异常退出。这些问题比增加更多 CLI 命令更容易破坏真实项目。 + +状态（2026-07-22）：**已完成**。 + +- 数据库保存 birth/access/modify/change 四类时间戳，macOS FSKit 与 Linux FUSE 均可读取并通过 `utimens` 更新； +- `O_APPEND` 使用数据库原子 append，四个并发进程各写 100 条记录时没有丢失、重复或覆盖； +- macOS 和 Linux 均通过最小 `flock`/`fcntl` 进程锁合同； +- 打开文件被 rename 或 unlink 后，已有句柄仍可读写并正常关闭； +- CLI 增加 `vexdb fs unmount --force MOUNT_POINT`，macOS 和 Linux 的占用挂载强制卸载均通过； +- Linux helper 被 `SIGKILL` 后，重新挂载会回收过期 session 并把保留的暂存写入发布到数据库；root 与 uid 1000 均通过； +- Developer ID `preview.8` 已包含 mount ABI 7。最终 macOS quick eval 为 62 passed、0 failed、3 个平台性 skipped，共 1027 项检查。 +- preview.15 真实 `mount.timestamps` 曾显示创建子项后父目录即时 mtime 没有推进； + preview.16 在 mutation 提交后直接更新父目录内存时间戳，不增加每个文件一次数据库 + stat。最终时间戳 10 项、并发 append 8 项、open/rename/unlink 14 项、锁 9 项、强制 + 卸载 11 项均通过。 + +证据位置： + +- macOS 最终全量：`vexdb_sqlite/build/eval/vexfs/20260721T061518.496228Z-quick-20260718/report.json` +- macOS 强制卸载单项：`vexdb_sqlite/build/eval/vexfs/20260721T061217.566816Z-quick-20260718/report.json` +- macOS preview.16 mount quick：`vexdb_sqlite/build/eval/vexfs/20260722T062028.825614Z-quick-20260718/report.json` +- Linux root 强制卸载：`vexdb_sqlite/build/eval/vexfs-linux-mount/root/20260721T060850.055986Z-quick-20260718/report.json` +- Linux root 崩溃恢复：`vexdb_sqlite/build/eval/vexfs-linux-mount/root/20260721T060850.232851Z-quick-20260718/report.json` +- Linux uid 1000 强制卸载：`vexdb_sqlite/build/eval/vexfs-linux-mount/uid-1000/20260721T060850.416919Z-quick-20260718/report.json` +- Linux uid 1000 崩溃恢复：`vexdb_sqlite/build/eval/vexfs-linux-mount/uid-1000/20260721T060850.564144Z-quick-20260718/report.json` +- 签名包：`dist/vexdb-lite/vexdb-lite-0.1.0-preview.8-macos-arm64.zip` +- SHA-256：`0e0c10f8711950eed9558aa57369a0461baa6f4ee008990850089698926e7c44` + +### N3. macOS ↔ Linux 可移植性 + +在两端打开同一个 SQLite Backup 或 export，验证内容、目录、Unicode、mode、symlink、hardlink、xattr/ACL、历史和快照；同时明确 owner/group 在另一台机器上的映射规则。 + +状态（2026-07-21）：**已完成**。同一数据库文件已完成 macOS → Linux → macOS 真机往返。 +数字 UID/GID 在数据库和快照中原样保存；Linux 挂载原样显示，macOS +普通用户 FSKit 挂载被系统强制设为 `noowners`，Bash `stat` 显示挂载用户，原始值通过 +`vexdb fs stat` 读取。该行为必须作为明确的跨平台规则，不能误报为 owner 丢失。 + +已验证内容：目录、Unicode、mode、可执行脚本、symlink、hardlink、xattr、便携 ACL、数字 +owner、文件历史、macOS 快照在 Linux 恢复、Linux 快照在 macOS 恢复，以及最终 +`PRAGMA integrity_check`。 + +证据位置： + +- 汇总目录：`vexdb_sqlite/build/eval/vexfs-portability/20260721T062755Z-72014` +- macOS 创建：`mac-create/20260721T062755.952108Z-quick-20260718/report.json`（6 checks） +- Linux 往返：`linux-roundtrip/20260721T062841.965486Z-quick-20260718/report.json`（36 checks） +- macOS 回读：`mac-verify/20260721T062842.499861Z-quick-20260718/report.json`（37 checks） +- 测试入口：`tests/eval/vexfs/run_cross_platform_portability.sh` + +### N4. 真正可交付的安装包 + +macOS 完成 Apple 公证、干净机器安装和 extension 启用流程；Linux 完成普通用户依赖检查、x86_64/AArch64 包和卸载。安装说明必须与压缩包或安装包一起交付。 + +状态（2026-07-23）：**进行中**。历史 `preview.17` 已公证；当前 `preview.19-test` 已完成签名和 +M1 真机快照恢复复测，但仍需从本次干净 commit 重新打包、公证，之后再做额外干净 Mac 分发验证。 + +- macOS 最新 `preview.17` 已使用 `Developer ID Application: Jian ming Wu (BB5VK42K87)` 签名； + App、FSKit extension、CLI 和 SQLite dylib 的签名链、统一安装回归与文档 smoke 均通过； +- 打包会在 Xcode export 前撤销临时 App 注册并删除专用 archive/export/DerivedData；真实 + mount eval 会拒绝 module URL 指向 `.xcarchive`、DerivedData 或 dist 副本； +- `preview.17` 已由 Apple 公证接受，submission ID 为 + `57118d6b-12d9-4fb1-9c5d-147eb4fc4bca`；App 已 staple，`spctl` 返回 + `source=Notarized Developer ID`；最终包安装后的真实 FSKit quick 为 14 passed、 + 0 failed、3 skipped、229 checks； +- 当前发行脚本已经禁止公证脏工作树，Developer ID 包默认禁止脏源码；安装器核对官方 + TeamIdentifier、SHA-256、文件合同和 runtime ABI，ad-hoc 包必须显式打开本地测试开关； +- `preview.19-test` 的 M1 真机普通卸载后 lease 约 `2.832 ms` 消失；CLI 一键恢复自动重挂载， + 旧内容恢复、快照后新增文件消失，最终 `mount status` 为 `[]`。当前 ZIP SHA-256 为 + `3e45f0684dc34118450f55887a3f75ef5f10eef6aa7e21cf48b32b585cd050ef`，但它尚未公证， + 不能作为最终对外交付包； +- package eval 不再接受零用例或必跑 SKIP，并要求显式指定本次 stage；当前工作树生成的 + `0.1.0-gate0-local` ad-hoc 包已通过 64 项文档 smoke、统一安装以及 2 个 package Gate， + 但它不作为正式发行物； +- Linux x86_64 和 AArch64 均在对应架构的 manylinux_2_28 环境原生构建；CLI、libfuse3 + helper 和 SQLite loadable extension 均无超出 `GLIBCXX_3.4.22` 的依赖； +- 两个 Linux 最终压缩包均带安装、卸载、中文使用说明、MANIFEST 和文件哈希； +- 两台对应架构的真实 Linux 机器均在全新临时 HOME 中通过无 root 安装、内置 SQLite 查询、 + 独立 `.so` 加载、FUSE helper 自检、VexFS 文件读写、卸载和数据库保留测试； +- 发行脚本增加可重复入口：`bash scripts/release.sh test sqlite all`。 + +交付物和证据： + +- macOS：`dist/vexdb-lite/vexdb-lite-0.1.0-preview.17-macos-arm64.zip` + （SHA-256 `08bd4f3dc3c3f896b51e92422a4a33ae941d5eab90d986f360a7919cf2f7c2d6`）； +- Linux x86_64：`dist/release/vexdb-lite-sqlite-files-linux-x86_64.tar.gz` + （SHA-256 `7a736d020cb523740b6e5763d27d96dcb7a7124442b851f2f1c4d389b31e2085`）； +- Linux AArch64：`dist/release/vexdb-lite-sqlite-files-linux-aarch64.tar.gz` + （SHA-256 `8455773a2808e60dfb3ba60366f634a1bd55b560825722c9bb1aa49e074ef6fe`）； +- 详细报告：`docs/reports/2026-07-21_vexfs-preview9-cross-platform-release.md`。 +- preview.16 修复报告：`docs/reports/2026-07-22_vexfs-preview16-p0-fixes.md`。 +- preview.17 公证报告：`docs/reports/2026-07-22_vexfs-preview17-apple-notarization.md`。 + +### N5. 真实项目和规模基线 + +持续运行 Git、Python、Node.js、Go、Rust 和 Coding Agent；补 1 万/10 万文件、大小文件混合、并发和长时间运行。结果全部进入 eval，不只保留手工日志。 + +状态（2026-07-22）：**进行中**。 + +- `mount.real-toolchain-projects` 已在真实 FSKit 挂载中运行 Python 3.12、Node.js 26、 + Go 1.22、Cargo 1.93 和 Git 2.50，卸载、重挂载后全部再次通过； +- SQLite 直连已通过 100,000 文件：123.327 秒创建，810.854 文件/秒； +- `preview.13` FSKit 已通过 10,000 文件创建、遍历、`rg`、快照恢复和重挂载： + 总耗时 177.224 秒；创建 114.828 秒，`rg` 23.895 秒； +- schema 热路径缓存让 10,000 文件创建较 `preview.12` 快约 37%，有界幂等日志至少 + 保留最近 65,536 条并已由 69,632 行 eval 验证批量清理； +- 100,000 文件真实 mount 完成创建、遍历和抽样读取，但 `rg` 在 3,600 秒内未完成， + 用例总耗时 13,002.565 秒后 FAIL；快照恢复和重挂载因旧 eval 提前退出尚未验证； +- 60 秒混合稳定性通过 59,904 次操作、299 次数据库重开、29 个快照和 11 次 checkpoint， + 没有未发布 staging； +- 900 秒长测通过 594,000 次操作、2,970 次数据库重开、297 个快照和 118 次 checkpoint， + 共 102,516 项检查，最终没有未发布 staging； +- `preview.13` 全量 quick eval 为 68 passed、0 failed、5 skipped、3,639 checks； +- 数据库批量 `grep` 和显式可选 FTS5 trigram 索引已完成。1 万文件无索引扫描为 + 6.416 秒，索引构建 0.024 秒，索引查询 0.013 秒，只读取 10 个候选文件； +- 规模 eval 已增加 1/1.5/2 GiB 峰值 RSS 保护线、64 MiB SQLite 页缓存和磁盘临时表； +- 2026-07-22 完整 quick eval 为 68 passed、0 failed、5 skipped、3,414 checks； +- handle open/close 降低 durability 的实验没有带来有效写入收益，最终包已恢复 FULL； +- 真实 OpenCode 已形成显式 eval，但审批要求用户看到具体外发风险后再次明确授权， + 不能写成已通过； +- preview.17 公证最终包在真实 FSKit 上完成 14 passed、0 failed、3 skipped、229 checks； + 1000 文件创建 333.971 files/s，同轮比 APFS 慢 38.954 倍； +- 尚需完成 100,000 文件完整复测、真实 Coding Agent 和写入优化。 + +证据位置： + +- 详细报告：`docs/reports/2026-07-21_vexfs-scale-and-toolchain-eval.md` +- 10 万 SQLite：`vexdb_sqlite/build/eval/vexfs/20260721T074800.216004Z-stress-20260718/report.json` +- 1 万 FSKit：`vexdb_sqlite/build/eval/vexfs/20260721T124346.009574Z-full-20260718/report.json` +- 10 万 FSKit 失败基线：`vexdb_sqlite/build/eval/vexfs/20260721T085806.189675Z-stress-20260718/report.json` +- 60 秒稳定性：`vexdb_sqlite/build/eval/vexfs/20260721T074017.731725Z-full-20260718/report.json` +- 900 秒稳定性：`vexdb_sqlite/build/eval/vexfs/20260721T084114.256428Z-stress-20260718/report.json` +- 最终 quick：`vexdb_sqlite/build/eval/vexfs/20260721T124750.583778Z-quick-20260718/report.json` +- 索引优化：`docs/reports/2026-07-22_vexfs-indexed-grep-optimization.md` +- 最新 quick：`vexdb_sqlite/build/eval/vexfs/20260722T021515.322756Z-quick-20260718/report.json` +- preview.17 最终公证包 mount quick：`vexdb_sqlite/build/eval/vexfs/20260722T065031.225854Z-quick-20260718/report.json` + +只有 N1–N5 达到 Phase 1 完成条件，才把主要开发重心转到 PostgreSQL。可以提前做 PostgreSQL 设计验证，但不能用它代替 SQLite 本地预览的发布闭环。 + +## 8. 防偏离规则 + +新增需求进入开发前，必须回答四个问题： + +1. 它服务于哪个最终目标和哪个当前阶段？ +2. 如果现在不做，会阻塞本阶段哪条完成条件？ +3. 它应该进入统一核心、数据库 adapter、mount adapter，还是只进入 CLI？ +4. 用什么自动测试或真机证据证明完成？ + +如果无法回答，默认放入后续列表，不进入当前阶段。 + +阶段状态只能使用“未开始、进行中、已完成”。更新状态时必须同时更新： + +- 当前事实和缺口； +- 完成证据或 eval 报告； +- 产品规格中的当前实现状态； +- 技术设计中的对应 adapter 状态。 + +不允许把“schema 中有字段”“函数能单独调用”“代码可以编译”写成完整产品能力。必须区分合同已实现、平台已接入、权限已执行、真机已验证和发行物已交付。 + +## 9. 相关文档 + +- 产品规格：`docs/specs/2026-07-16_vexfs-product-spec.md` +- 技术设计：`docs/design/2026-07-16_agent-files-universal-filesystem.md` +- Workspace 场景分析：`docs/analysis/2026-07-20_vexfs-claude-code-workspace-scenario.md` +- POSIX 与快照边界：`docs/analysis/2026-07-20_vexfs-remaining-posix-snapshot-feasibility.md` +- Eval 说明：`agent_files/eval/README.md` diff --git a/docs/specs/2026-07-16_vexfs-product-spec.md b/docs/specs/2026-07-16_vexfs-product-spec.md new file mode 100644 index 0000000000..5382ba003d --- /dev/null +++ b/docs/specs/2026-07-16_vexfs-product-spec.md @@ -0,0 +1,1645 @@ +# VexFS 完整产品需求文档 + +- 文档类型：产品需求基线 +- 产品名：`VexFS`（暂定） +- 所属项目：VexDB-Lite +- 日期：2026-07-16 +- 分支：`feature/agent_files` +- 文档版本：1.3 +- 状态：SQLite 合同、macOS FSKit 垂直闭环和 Linux libfuse3 双架构预览包已实现；当前继续本地开发者预览收口 +- 适用范围：VexFS 完整产品定义；开发顺序以 `docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md` 为准 + +## 0. 阅读结论 + +VexFS 是由 PostgreSQL、DuckDB 或 SQLite 加载和管理的数据库文件管理扩展。三个数据库共享文件合同，但不要求同时完成。macOS + SQLite 的首个闭环已经跑通，Linux + SQLite 的 libfuse3 真实挂载预览也已经跑通；当前先把这两个本地平台收口成可安装的开发者预览，再进入 PostgreSQL 远程共享工作区。Windows 使用 WinFsp 接入同一 mount runtime；DuckDB 后续先提供 SQL、CLI 和 worktree。 + +它在数据库中提供文件、目录、路径、版本、快照、权限、审计、配额、完整性检查和逻辑导入导出。文件修改可以和普通业务 SQL 在同一个事务中提交或回滚。 + +用户安装并完成一次数据库连接配置后，VexFS 把工作区挂载为真实操作系统目录。人和模型进入该目录后，可以直接使用 Bash、`ls`、`cat`、`rg`、`sed`、`python`、`git`、编译器和 `apply_patch`，不需要给命令增加 `vexfs` 前缀。 + +VexFS 同时保留两种回退入口： + +1. 用 `vexfs ls/cat/find/grep/cp/mv/rm` 等直接命令操作数据库中的文件； +2. 在容器、端侧或不能挂载的环境中，用 `vexfs shell` 或 `vexfs exec` 建立临时 worktree，最后把差异事务性提交回数据库。 + +最重要的边界是： + +> **数据库管理 VexFS，VexFS 管理数据库中的文件；mount gateway 只把这些文件展示给 Bash。** + +数据库扩展是权威核心。mount gateway 是随产品安装的非权威访问进程：它可以保存当前 handle 和未发布写入的临时状态，但不能把这些状态当作已经提交的文件，也不能绕过数据库事务、权限和版本检查。 + +VexFS 有两种不同的事务入口，文档和产品界面必须明确区分： + +1. SQL 调用可以参加调用者当前数据库事务，实现“文件 + 业务表”原子提交； +2. mount 中的普通 Bash 操作由 gateway 建立短事务，不能自动加入应用已经打开的数据库事务，多条 Bash 命令也不会自动合成一个事务。 + +macOS 可挂载技术闭环支持 macOS 26.0+、Apple Silicon 和 FSKit。FSKit 基础接口从 macOS 15.4 提供，但本产品用来传递数据库目录资源的 `FSPathURLResource` 从 macOS 26.0 才提供。Linux 已有 libfuse3 预览，x86_64 和 AArch64 都已有 manylinux 安装包和对应架构真机安装证据，真实挂载已覆盖 root 与普通用户；Windows、较旧 macOS 和不能启用文件系统扩展的环境仍先使用数据库函数与 CLI。 + +### 0.1 最终目标和开发顺序 + +VexFS 的不变最终目标、阶段边界、完成条件和当前下一步统一记录在： + +`docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md` + +本产品规格负责定义“产品做什么”，不再单独维护另一套阶段顺序。其他文档与路线图冲突时，以路线图为准。 + +## 1. 优先级定义 + +| 优先级 | 含义 | +|---|---| +| MVP Gate | 首个 macOS + SQLite 垂直闭环必须具备，用来决定是否继续投入 | +| P0 | 完整产品没有该能力则核心价值不成立，但不要求全部进入首个技术闭环 | +| P1 | 产品可用和可推广需要，允许在 P0 后完成 | +| P2 | 增强能力，不阻塞首个可用版本 | + +文档中的“必须”表示对应阶段的验收硬要求；“应该”表示默认实现；“可以”表示可选能力。需求表中的 P0 是产品优先级，不等于全部进入 Phase 1；每个阶段的实际范围和完成条件以路线图为准。 + +### 1.1 CLI 评审保留项和阶段边界 + +2026-07-17 CLI 评审中保留以下长期合同： + +- v1 的 workspace 使用线性历史，不引入 branch、DAG 或 detached HEAD； +- file version、workspace commit、snapshot 和 worktree 是不同对象； +- restore 产生新 commit，不删除、不重写旧历史； +- restore、worktree commit 和 changeset publish 使用 expected-head/base version 防止静默覆盖； +- request-id 用于结果不确定时的安全重试； +- commit 创建后不可修改，只允许追加 annotation； +- 原生 `grep`/`rg` 经过 mount 时按文件扫描，不能透明改写成数据库查询。 + +这些是完整产品合同，不全部进入当前阶段。功能依赖顺序仍是： + +1. 先保证真实挂载、基础 Bash、同步、重开和故障正确性； +2. 再保证可重建的 workspace 历史、snapshot、restore 和 expected-head； +3. 再补数据库身份、ACL、审计、配额、保留策略和备份恢复； +4. 最后增加 worktree、`txn run` 和可选的普通文本索引。 + +这只是能力依赖，不是另一套发布阶段。当前实施顺序以路线图中的 Phase 1–5 为准。 + +文本索引不是 VexFS 核心 schema 的前置条件。BM25、向量索引、Embedding 和语义检索不进入 VexFS 文件系统核心，也不阻塞任何文件系统阶段。 + +SQLite MVP 没有数据库用户认证。该阶段的 principal 只能来自受签名 App/extension、当前 macOS 用户、security-scoped 数据库目录和数据库文件权限共同建立的本地身份边界；不能把 PostgreSQL 角色语义直接套到 SQLite。 + +### 1.2 当前实现状态（2026-07-21） + +SQLite 合同为 `0.7.0`，runtime ABI 为 1。除单文件版本外，当前实现已经提供完整 workspace 时间点恢复： + +- 每个 workspace commit 增量保存 inode、dentry 和 xattr 状态；文件内容继续复用不可变 file version； +- snapshot 是“名称 → 固定 workspace commit”的不可变书签，创建不复制目录树或文件内容； +- `snapshot create/list/show/diff/drop/restore` 已同时进入 SQL、C ABI 和 CLI； +- restore 覆盖目录创建、删除和移动，以及文件内容、mode、symlink target 和 xattr，并生成新的 workspace commit； +- restore 必须带 expected-head；旧 head、并发 restore 和打开/retained handle 都会被拒绝，不能静默覆盖； +- 产品尚未正式发版，当前只接受 `0.7.0` schema；其他版本明确拒绝且不改写数据库，不保留历史迁移代码； +- SQLite Online Backup 会同时保存 snapshot、历史树和未发布 staging；数据库重开后仍可 diff 和 restore； +- 3,000 文件实测 snapshot create 约 0.16 ms、单文件变化 diff 约 0.65 s、完整 restore 约 1.47 s；10,000 文件压力档分别约 0.17 ms、6.83 s 和 15.42 s。 + +现有数据库合同和已接入平台的 POSIX 兼容能力包括： + +- 文件和目录创建时保存 `0000..0777` mode，`chmod` 后重新挂载仍保留，可执行脚本可以直接运行； +- 符号链接使用独立 inode，target 原样保存，支持相对链接、绝对链接和 dangling link； +- FSKit、C ABI、SQL 和 SQLite schema 对 mode/symlink 使用同一套合同； +- Git 真机探测为 `core.filemode=true`、`core.symlinks=true`，init/add/commit/checkout/gc 已通过； +- xattr 已由数据库保存，不需要把它退化为普通文件内容； +- hardlink、owner/group 元数据和便携 ACL 已进入 SQLite schema、SQL/C ABI/CLI 和 eval； +- Linux libfuse3 已接入 hardlink、chown 和 xattr，并通过 root、普通用户、Git 工作区和重挂载真机测试。 +- macOS FSKit 与 Linux libfuse3 已共用 mount 一致性 eval；四类时间戳、原子并发 + `O_APPEND`、最小 `flock`/`fcntl`、打开文件 rename/unlink、强制卸载和 helper 崩溃恢复已验证； +- 同一 SQLite 数据库已完成 macOS → Linux → macOS 往返，内容、Unicode、mode、链接、 + xattr、便携 ACL、数字 UID/GID、历史和跨系统快照恢复均通过； +- macOS `preview.17` 已完成 Developer ID 签名、Apple 公证、staple、本机安装和真实 FSKit + 回归；尚缺另一台没有源码和 Xcode 构建记录的干净 Mac 验证。Linux x86_64/AArch64 + `preview.9` 包已完成原生无 root 安装、扩展加载、自检、文件读写和卸载回归。 +- Python、Node.js、Go、Rust 和 Git 已在 FSKit 挂载盘中运行并通过重挂载复测；SQLite + 直连已通过 10 万文件，FSKit 已通过 1 万文件；10 万文件已完成创建/遍历/抽样读取， + 但普通 `rg` 在 3,600 秒内未完成；900 秒混合重开稳定性完成 594,000 + 次操作、2,970 次重开、297 个快照和 118 次 checkpoint，未留下 staging； +- FSKit 对不超过 1 MiB 的只读普通文件使用打开期内容快照，但不启用 kernel data + cache；写打开和 truncate 会失效快照，大文件继续使用数据库 handle。 +- 幂等结果至少保留最近 65,536 条，每 4,096 条完成记录批量清理一次；显式 + `vexfs_init()` 始终完整检查 schema，普通热路径在同一连接内复用已验证状态。 +- `vexdb fs grep` 已提供数据库内批量扫描；可显式开启 FTS5 trigram 索引。索引默认关闭， + 不增加普通写入成本；开启后写入、版本恢复和工作区快照恢复都维护索引。少于 3 个字符 + 的查询自动回退扫描，宿主没有 FTS5 时也回退扫描并标记索引待重建。 +- 1 万文件索引查询为 0.013 秒，构建为 0.024 秒，峰值 RSS 50.9 MiB。10 万完整 + mount 流程尚未用新索引包复测，不能写成通过。 + +已有的单文件版本能力包括： + +- 普通写入保存不可变完整内容并生成递增版本； +- `vexfs_history` 支持有上限的分页查询，`vexfs_read_version` 读取指定版本； +- `vexfs_restore_version` 创建新版本，但复用目标版本的不可变内容，不复制大 BLOB； +- 四参数恢复带 `expected_version`，当前版本已变化时返回冲突； +- CLI 已提供 `history`、`show`、`diff`、`restore` 和 `restore --dry-run`； +- SQL 事务回滚会同时撤销版本、commit 和 workspace head；两个进程并发恢复时只有一个成功； +- mount session 使用独占租约，崩溃遗留 staging 只能显式恢复，不能被新进程静默发布； +- `doctor` 不创建、不迁移旧数据库，并能报告合同兼容性和升级需要； +- SQLite 主库、WAL 和 SHM 默认限制为当前用户读写，CLI 提供稳定错误码和退出码。 + +当前已经是完整 workspace 时间点恢复，但还不是完整 POSIX 文件系统。SQLite 文件版本 SHA-256 和只读 check 已实现；principal 权限执行、ACL 授权、审计、配额、保留策略、GC、分块和去重仍未实现。四类时间戳和本地最小文件锁已经验证，但跨 gateway 锁、setuid/setgid/sticky bit 和特殊文件没有形成跨平台承诺。hardlink、owner/group 和便携 ACL 当前只能称为“数据库合同已实现并在部分平台接入”，不能称为完整身份权限系统。当前历史只追加、不自动回收；长期运行前必须补保留策略和 GC。snapshot 解决误改和任务检查点，SQLite Backup 才是独立备份，两者不能混为一谈。 + +当前符号链接采用 POSIX 行为：target 是原样字符串，可以指向挂载目录外部。它解决兼容性，但不提供路径隔离；需要隔离时仍由 Agent sandbox 负责。未来可以增加可选的 `internal` 策略，但当前版本没有实现该策略。 + +## 2. 背景与问题 + +业务系统和 Agent 经常同时保存结构化记录与文件： + +- 任务状态和生成报告； +- 客户记录和合同附件； +- Agent 输入资料和中间结果； +- 分析任务和导出文件； +- 配置、日志、代码和构建结果； +- 历史版本和审计证据。 + +常见做法是数据库保存记录，文件系统或对象存储保存文件。这会产生以下问题： + +1. 业务记录提交成功，但文件写入失败。 +2. 文件写入成功，但数据库事务回滚。 +3. 数据库和文件服务维护两套身份与权限。 +4. 数据库备份没有包含文件，恢复时无法对齐。 +5. 文件版本和业务记录版本无法对应。 +6. 多个 Agent 同时修改时容易静默覆盖。 +7. Agent 需要学习另一套服务协议和工具。 +8. SQLite、DuckDB 和 PostgreSQL 使用三套文件方案。 + +VexFS 的目标是把普通文件管理放进数据库事务、权限、版本和备份边界，同时保留模型熟悉的终端工作方式。 + +## 3. 产品定义 + +### 3.1 一句话定义 + +> **VexFS 是数据库原生的版本化 Agent 文件工作区。它让模型使用正常终端工具工作，并把最终文件变更按数据库权限和事务提交。** + +### 3.2 产品组成 + +VexFS 由三部分组成： + +1. **数据库扩展**：权威状态、SQL 合同、事务、权限、版本、快照、审计和备份兼容。 +2. **mount gateway**：通过数据库公开合同把 workspace 显示为真实操作系统目录；macOS 使用 FSKit App Extension，Linux 使用 libfuse3，Windows 后续使用 WinFsp。 +3. **配套 CLI**：安装、连接、mount/unmount、直接文件命令、worktree、shell、exec、导入导出和运维入口。 + +数据库扩展是权威核心。没有 mount gateway 或 CLI 时，全部数据能力仍可通过 SQL 使用；mount gateway 崩溃或卸载不能损坏数据库中已经提交的文件。 + +### 3.3 与 VexDB-Lite 的关系 + +- VexFS 位于 VexDB-Lite 仓库和产品品牌下； +- VexFS 与向量索引共享多数据库适配、构建、测试和发布经验； +- VexFS 文件核心和现有向量图核心相互独立； +- 不安装或不启用向量索引时，VexFS 必须完整可用； +- VexFS 不做 Embedding、语义搜索或内容理解。 + +## 4. 产品目标 + +### 4.1 MVP Gate 目标 + +- 只支持 macOS 26.0+、Apple Silicon、SQLite、单 gateway、单 mount 和单数据库 principal； +- 采用 FSKit `FSUnaryFileSystem` 和 `FSVolume`，把一个 workspace 挂载为真实目录； +- 支持普通文件和目录的最小调用集，以及 O_CREAT、O_EXCL、O_TRUNC 和 O_APPEND； +- 明确采用 close-to-open 一致性，不宣传完整 POSIX 强一致； +- MVP 不启用 FSKit kernel data cache，权限与正确性优先； +- 允许在 adapter 内对小型只读文件做有边界的打开期缓存，但必须覆盖并通过写打开、 + truncate、rename、unlink、同名重建和大文件回退测试； +- FSKit `write`、`synchronize`、`closeItem` 和 `reclaimItem` 的成功含义及故障恢复规则可测试； +- `synchronize` 成功后的内容在 FSKit extension 被终止和数据库 reopen 后仍完整； +- 完成签名并公证的 macOS App、FSKit extension、SQLite 扩展和 CLI 组合包，以及版本握手、doctor 和卸载流程； +- Bash、Python 和基础 Git 流程不需要 VexFS 专用命令； +- 1,000 个文件、1 GiB 当前内容和 100 MiB 单文件通过正确性测试。 + +MVP Gate 不等于完整产品。它只回答三个问题：数据库能否保持唯一权威、真实 Bash 是否可用、故障后是否不产生伪成功和半文件。 + +### 4.2 P0 目标 + +- 在数据库中提供稳定的文件和目录模型； +- 文件修改参加当前数据库事务； +- 文件和普通业务表可以原子提交； +- 提供不可变文件版本、工作区 commit 和快照； +- 提供数据库身份驱动的路径权限； +- 数据库原生备份包含全部 VexFS 状态； +- macOS 用户完成一次系统扩展授权和 context 配置后，可以把 workspace 挂载为真实目录； +- 模型可以在挂载目录中直接用普通终端工具完成文件任务； +- mount 的每次文件操作都经过数据库身份、权限、配额、版本和审计检查； +- 单个文件系统操作不能产生半状态； +- 多文件 changeset 和业务 SQL 可以显式原子提交； +- 不能挂载时，worktree 差异可以全部提交或全部回滚； +- 并发冲突不能静默覆盖； +- 三端共享相同的逻辑文件合同。 + +### 4.3 P1 目标 + +- 工作区逻辑导入导出； +- 路径权限继承、审计和配额； +- 保留策略和分批垃圾回收； +- 跨 SQLite、PostgreSQL 和 DuckDB 迁移； +- 完整的检查和安全修复流程； +- 增量 checkout 和大目录分页。 +- Linux libfuse3 预览收口和 Windows WinFsp 原生挂载入口； +- mount 缓存失效通知和只读离线浏览优化。 + +### 4.4 非目标 + +- MVP Gate 不承诺 Linux、Windows、macOS 26.0 以下版本和所有容器都能真实挂载； +- 不把 mount gateway 做成另一套权威文件服务器； +- 不建立另一套网络协议； +- 不提供必须接入的 SDK； +- 不替代 ext4、APFS、NTFS 或 Git； +- 不支持设备文件、socket 或 FIFO； +- 不作为视频、模型文件或数据湖的无限对象存储； +- 不管理数据库启动、停止、WAL、复制或物理备份任务； +- 不做跨数据库分布式事务； +- 不把任意已有数据库表自动投影成文件； +- 不做语义检索、Embedding 或 Agent 内容理解。 +- MVP Gate 不支持多 gateway、跨 gateway 锁和主动缓存失效； +- 最初 MVP Gate 不要求 chown、路径 ACL 继承、hardlink、共享可写 mmap 和特殊文件；当前 hardlink、owner/group 元数据和便携 ACL 已进入合同，但仍不做完整 POSIX 权限判定； +- 当前已支持 symlink 和 xattr，但不承诺编译器全量产物、`node_modules` 或大型构建缓存适合作为长期版本化数据； +- DuckDB 可写 mount 不进入 MVP Gate；除非 gateway 是唯一写进程，否则不承诺与外部 DuckDB 进程共享写入。 + +## 5. 名词定义 + +| 名词 | 定义 | +|---|---| +| workspace | 最高文件隔离单位，拥有根目录、owner、head commit、权限、配额和保留规则 | +| inode | 文件、目录或符号链接的稳定身份 | +| dentry | 父目录下的名称到 inode 的映射 | +| file version | 某个 inode 在一个 commit 中的不可变状态 | +| workspace commit | 一次成功 changeset 产生的工作区版本号 | +| snapshot | 指向固定 workspace commit 的不可变名称 | +| changeset | 一组新增、修改、复制、移动和删除操作 | +| mount gateway | 使用数据库连接和公开 VexFS 合同，把 workspace 显示为操作系统目录的非权威进程 | +| mount point | 用户可以直接 `cd` 进入的真实操作系统挂载目录 | +| file handle | mount gateway 为一次 open 保存的短期句柄；不代表长期数据库事务 | +| worktree | 从固定 commit 导出的临时本地普通文件目录 | +| principal | 数据库连接能够证明的用户或角色身份 | +| native backup | PostgreSQL、DuckDB 或 SQLite 自己提供的完整数据库备份 | +| logical export | VexFS 提供的单工作区、跨数据库可交换格式 | + +## 6. 目标用户 + +### 6.1 Agent 平台开发者 + +希望 Agent 使用正常终端工具，但文件最终进入数据库权限、版本和审计边界。 + +### 6.2 数据库应用开发者 + +需要在业务事务中同时保存记录和附件、报告、配置或生成物。 + +### 6.3 企业数据平台团队 + +需要多用户共享、数据库角色、审计、备份恢复和统一运维。 + +### 6.4 本地与端侧应用开发者 + +希望 SQLite 或 DuckDB 单文件数据库同时保存业务数据和版本化文件工作区。 + +## 7. 核心场景 + +### 场景 A：业务状态与文件原子提交 + +```sql +BEGIN; + +UPDATE tasks +SET status = 'completed' +WHERE id = 42; + +SELECT vexfs_write( + 'task-files', + '/tasks/42/report.md', + CAST('# Task report' AS BLOB) +); + +COMMIT; +``` + +任何一步失败，任务状态和报告都必须回滚。 + +### 场景 B：Agent 使用直接命令 + +```bash +vexfs --workspace task-files ls /tasks/42 +vexfs --workspace task-files cat /tasks/42/report.md +vexfs --workspace task-files grep -R -n 'TODO' /tasks +vexfs --workspace task-files stat /tasks/42/report.md +``` + +### 场景 C：不能挂载时使用任意终端工具 + +```bash +vexfs shell task-files + +find . -name '*.md' +rg 'TODO' . +sed -i.bak 's/draft/final/g' tasks/42/report.md +python scripts/generate.py +tar -czf output.tar.gz output/ + +vexfs worktree status +vexfs worktree commit -m 'finish task 42' +``` + +### 场景 D：并发冲突 + +Agent A 和 Agent B 从 commit 100 checkout 同一文件。Agent A 先提交 commit 101。Agent B 提交时必须收到明确冲突，不能覆盖 commit 101；Agent B 的本地差异必须保留。 + +### 场景 E：权限控制 + +报告作者可以在 `/reports/team-a` 创建和修改文件，但不能读取 `/reports/team-b`，不能修改 ACL，也不能恢复整个工作区。 + +### 场景 F：误操作恢复 + +用户创建快照后批量修改。出现问题时，可以恢复单文件、目录或整个工作区。恢复动作本身产生新 commit，不删除历史。 + +### 场景 G：数据库灾难恢复 + +数据库管理员使用数据库原生工具备份和恢复。恢复后业务表、VexFS 文件、版本、快照、权限和审计必须一致。 + +### 场景 H：跨数据库迁移 + +用户把 SQLite 工作区导出为 VexFS 逻辑包，再导入 PostgreSQL。目录、内容、版本、快照和 checksum 必须一致；数据库特有身份需要显式映射。 + +### 场景 I：安装后无缝进入 Bash 工作区 + +数据库扩展安装和初始化完成后，用户只需要配置一次数据库连接： + +```bash +vexfs context add local-db --engine postgres --dsn postgres://localhost/mydb +vexfs context use local-db +vexfs workspace use task-files +vexfs mount ~/VexFS/task-files +cd ~/VexFS/task-files +``` + +挂载完成后，目录是数据库 workspace 的实时视图。用户直接使用系统原生命令： + +```bash +pwd +ls +cat tasks/42/report.md +rg 'TODO' . +sed -i.bak 's/draft/final/g' tasks/42/report.md +python scripts/generate.py +git diff +vexfs mount status +``` + +用户不需要给 `ls/cat/rg/sed/python` 添加 `vexfs` 前缀。文件在平台 adapter 成功 publish/sync 后提交为新版本；macOS MVP 由 FSKit `synchronize` 和可报告结果的 close 路径触发。需要把多个文件和业务 SQL 一起原子提交时，必须使用显式 changeset/事务入口，不能假设多条 Bash 命令天然属于同一个数据库事务。 + +## 8. 系统边界和所有权 + +```text +模型 / 人 / 自动化 + | + +--------------------+---------------------+ + | | | + v v v + 操作系统文件 API VexFS CLI VexFS SQL + | | + v +---- 直接命令 / worktree + mount gateway + | + +---------- 公开 VexFS SQL/文件合同 --------+ + | + v + PostgreSQL / DuckDB / SQLite + ├── 管理扩展生命周期 + ├── 管理内存、线程和连接 + ├── 管理事务、锁和并发 + ├── 管理 WAL 和崩溃恢复 + ├── 管理数据库身份和外层权限 + ├── 管理数据库备份和复制 + └── 保存 VexFS 全部权威状态 +``` + +VexFS 禁止： + +- 直接读写数据库物理文件； +- 自己实现 WAL、buffer pool 或数据库锁； +- 自行提交用户事务； +- 把权威文件状态保存在数据库之外； +- 接受 CLI 自报身份替代数据库身份； +- 让 mount gateway 自报身份替代数据库连接身份； +- 让 mount gateway 保存数据库之外的最终文件状态； +- 让 worktree 直接修改内部表； +- 从数据库扩展初始化函数或 SQL callback 启动 FSKit/FUSE/WinFsp loop、网络服务或未受宿主管理的长期线程。SQLite 场景允许系统管理的 FSKit extension 在自身进程中打开 SQLite 连接。 + +## 9. 产品原则 + +### 9.1 数据库优先 + +数据库负责生命周期、事务、身份、权限上限、备份和恢复。VexFS 只实现文件规则。 + +### 9.2 SQL 是权威合同 + +所有数据能力必须存在对应 SQL 合同。mount gateway 和 CLI 只能调用公开合同；worktree 只能通过 changeset SQL 提交。 + +### 9.3 事务优先 + +文件操作必须参加当前数据库事务。完整旧状态和完整新状态之间不能出现半状态。 + +### 9.4 模型终端优先 + +安装后优先提供真实挂载目录，不重新实现完整 shell。任意现成程序直接通过操作系统文件 API 使用 VexFS；不能挂载时，再使用 CLI 或真实临时 worktree。 + +### 9.5 原生备份优先 + +数据库原生备份是灾难恢复方式。文件版本和快照不是数据库备份的替代品。 + +### 9.6 逻辑一致，能力分级 + +三端统一文件结果、错误和版本行为；身份强度、并发上限和运维方式按宿主能力分级。 + +## 10. 功能需求 + +### 10.1 扩展生命周期 + +| ID | 优先级 | 需求 | +|---|---|---| +| FR-LC-001 | P0 | VexFS 必须由宿主数据库扩展机制加载和卸载。 | +| FR-LC-002 | P0 | 初始化必须创建受保护的内部对象，并且可重复执行。 | +| FR-LC-003 | P0 | 必须提供扩展版本、内部 schema 版本和能力查询。 | +| FR-LC-004 | P0 | 升级必须使用数据库支持的升级路径，失败时不能留下半升级状态。 | +| FR-LC-005 | P0 | 卸载不得静默删除已有文件数据；破坏性卸载需要显式确认步骤。 | +| FR-LC-006 | P0 | 普通用户不能直接写内部表或调用内部函数。 | +| FR-LC-007 | P1 | 必须提供只读升级前检查和升级后完整性检查。 | + +### 10.2 工作区 + +| ID | 优先级 | 需求 | +|---|---|---| +| FR-WS-001 | P0 | 支持创建、列出、查看、冻结、解冻和删除工作区。 | +| FR-WS-002 | P0 | 每个工作区必须拥有独立根目录和稳定 workspace id。 | +| FR-WS-003 | P0 | 每个工作区必须维护单调递增的 head commit。 | +| FR-WS-004 | P0 | 工作区必须拥有 owner principal。 | +| FR-WS-005 | P0 | 冻结后允许读取，不允许产生新文件 commit。 | +| FR-WS-006 | P0 | 删除非空工作区必须显式指定，并写审计记录。 | +| FR-WS-007 | P1 | 工作区必须提供文件数、目录数、当前字节、历史字节和快照数统计。 | +| FR-WS-008 | P1 | 支持工作区级配额和保留策略。 | +| FR-WS-009 | P1 | 支持只 checkout 某个子目录。 | + +### 10.3 文件、目录和路径 + +| ID | 优先级 | 需求 | +|---|---|---| +| FR-FS-001 | P0 | 支持普通文件和目录。 | +| FR-FS-002 | P0 | 路径必须从工作区根开始解析，不能越过根目录。 | +| FR-FS-003 | P0 | 文件名禁止 NUL 和 `/`，同目录名称必须唯一。 | +| FR-FS-004 | P0 | 支持 mkdir、create、read、write、range read、range write 和 truncate。 | +| FR-FS-005 | P0 | 支持 copy、move、rename、replace、remove 和递归删除。 | +| FR-FS-006 | P0 | move、replace 和 remove 必须原子执行。 | +| FR-FS-007 | P0 | 支持 stat、分页 list 和稳定排序。 | +| FR-FS-008 | P0 | 支持 size、checksum、创建时间、修改时间、owner 和 mode 元数据。 | +| FR-FS-009 | P0 | 文件内容必须使用分块 BLOB，不能要求一次加载完整大文件。 | +| FR-FS-010 | P0 | 发布的 manifest 不能缺少任何 chunk。 | +| FR-FS-011 | P1 | 符号链接 target 按 POSIX 原样保存，支持相对、绝对、dangling 和越界目标；VexFS 不把它当作 Agent 沙箱。 | +| FR-FS-012 | P1 | 后续可增加显式 `internal` 策略，启用后拒绝绝对和越界目标；不得静默改变已有 POSIX 合同。 | +| FR-FS-013 | P1 | 支持 hardlink；当前 SQLite 合同和 Linux adapter 已接入，其他平台按统一合同补齐。 | +| FR-FS-014 | P0 | 不支持设备文件、socket 和 FIFO。 | +| FR-FS-015 | P0 | 默认单文件上限为 100 MiB，可由管理员调低或调高。 | +| FR-FS-016 | P0 | 默认单工作区当前内容上限为 10 GiB，可由管理员配置。 | + +### 10.4 事务和 changeset + +| ID | 优先级 | 需求 | +|---|---|---| +| FR-TX-001 | P0 | VexFS 写操作必须参加当前数据库事务。 | +| FR-TX-002 | P0 | VexFS 不得在内部自行 commit 用户事务。 | +| FR-TX-003 | P0 | 数据库 rollback 后不能留下可见文件修改。 | +| FR-TX-004 | P0 | 文件操作可以和普通业务表更新原子提交。 | +| FR-TX-005 | P0 | savepoint 和 rollback-to 的行为必须明确并通过三端合同测试。 | +| FR-TX-006 | P0 | worktree 必须通过单个 changeset 提交全部差异。 | +| FR-TX-007 | P0 | 一个成功 changeset 只产生一个 workspace commit。 | +| FR-TX-008 | P0 | changeset 必须携带 base commit 和被修改对象的 expected version。 | +| FR-TX-009 | P0 | 发现并发修改时必须返回冲突，不能静默覆盖。 | +| FR-TX-010 | P0 | changeset 中任意操作失败时，全部操作必须回滚。 | +| FR-TX-011 | P1 | 写操作支持可选 request id，重复请求不能产生重复 commit；至少保留最近 65,536 条完成结果，并以固定批次清理更早记录，避免长期 mount 无限增长。 | + +### 10.5 版本、历史和快照 + +| ID | 优先级 | 需求 | +|---|---|---| +| FR-VS-001 | P0 | 每次成功修改必须产生不可变文件版本。 | +| FR-VS-002 | P0 | 文件版本必须记录 inode、version id、commit id、checksum、size、principal 和时间。 | +| FR-VS-003 | P0 | 支持查看文件历史和读取指定版本。 | +| FR-VS-004 | P0 | 支持比较两个 workspace commit 的目录和文件变化。 | +| FR-VS-005 | P0 | 快照必须指向固定 commit，创建快照不能复制全部文件内容。 | +| FR-VS-006 | P0 | 快照创建后保持不可变。 | +| FR-VS-007 | P0 | 支持恢复单文件、目录和整个工作区。 | +| FR-VS-008 | P0 | 恢复必须产生新 commit 和审计记录，不覆盖历史。 | +| FR-VS-009 | P0 | 快照引用的版本和 chunk 不能被保留策略或 GC 删除。 | +| FR-VS-010 | P1 | 支持给 commit 添加消息和外部业务引用。 | + +### 10.6 权限和身份 + +统一权限集合： + +| 权限 | 含义 | +|---|---| +| traverse | 经过目录解析下级路径 | +| list | 查看目录成员 | +| read | 读取文件内容和基本属性 | +| create | 在目录中创建文件或子目录 | +| write | 修改已有文件内容和属性 | +| delete | 删除目录项 | +| rename | 移动或改名目录项 | +| history | 查看历史版本和 diff | +| snapshot | 创建和恢复快照 | +| share | 修改 ACL | +| admin | 修改 owner、配额和保留规则 | + +| ID | 优先级 | 需求 | +|---|---|---| +| FR-AC-001 | P0 | 公开 SQL 函数默认使用当前数据库连接身份。 | +| FR-AC-002 | P0 | 有效权限必须同时满足数据库外层权限和 VexFS 路径 ACL。 | +| FR-AC-003 | P0 | workspace owner 默认拥有 admin。 | +| FR-AC-004 | P0 | ACL 支持工作区根和任意目录或文件。 | +| FR-AC-005 | P0 | 子路径默认继承最近祖先 ACL；显式 deny 优先于 allow。 | +| FR-AC-006 | P0 | 删除、覆盖和改名必须检查源对象、源目录、目标目录和目标对象权限。 | +| FR-AC-007 | P0 | checkout 只能导出当前 principal 有 read 权限的内容。 | +| FR-AC-008 | P0 | worktree commit 必须重新检查全部权限。 | +| FR-AC-009 | P0 | worktree 本地 chmod、owner 或 ACL 不能直接成为数据库权限。 | +| FR-AC-010 | P0 | CLI 不能通过参数伪造 principal。 | +| FR-AC-011 | P0 | PostgreSQL principal 对应数据库 role。 | +| FR-AC-012 | P0 | DuckDB 和 SQLite 只能使用宿主能够证明的连接身份，不能假装拥有 PG 级角色系统。 | +| FR-AC-013 | P1 | chmod 作为兼容命令映射到 VexFS ACL；普通文件 execute 只作为元数据，不代表数据库执行文件。 | +| FR-AC-014 | P1 | chown 和 ACL 修改要求 share 或 admin。 | + +### 10.7 审计 + +| ID | 优先级 | 需求 | +|---|---|---| +| FR-AU-001 | P0 | 所有文件写入、删除、改名、恢复和权限修改必须写审计记录。 | +| FR-AU-002 | P0 | 审计必须记录 principal、workspace、commit、操作、对象和前后版本。 | +| FR-AU-003 | P0 | 审计不能保存密码、连接串或文件正文。 | +| FR-AU-004 | P0 | 审计记录不能由普通用户修改或删除。 | +| FR-AU-005 | P1 | 审计支持按 commit、principal、路径和时间查询。 | +| FR-AU-006 | P1 | 支持可选审计 hash 链，用于发现记录被改写。 | +| FR-AU-007 | P1 | 工作区可以开启读取审计，默认关闭以控制数据量。 | + +### 10.8 配额、保留和垃圾回收 + +| ID | 优先级 | 需求 | +|---|---|---| +| FR-QT-001 | P0 | 必须支持最大单文件、当前内容字节和文件数限制。 | +| FR-QT-002 | P0 | changeset 发布前必须计算新增空间并检查配额。 | +| FR-QT-003 | P0 | 配额失败不能产生部分 commit。 | +| FR-QT-004 | P1 | 必须分别统计 live bytes 和 retained history bytes。 | +| FR-QT-005 | P1 | 保留策略支持保留最近版本数和保留天数。 | +| FR-QT-006 | P1 | GC 只能删除没有活动版本、快照或导出任务引用的对象。 | +| FR-QT-007 | P1 | GC 必须分批执行，避免不可控长事务。 | +| FR-QT-008 | P1 | 工作区冻结和备份验证期间可以暂停 GC。 | + +### 10.9 数据保护、备份和恢复 + +VexFS 使用四层数据保护： + +1. 文件版本； +2. 工作区快照； +3. 数据库原生备份； +4. 工作区逻辑导出。 + +| ID | 优先级 | 需求 | +|---|---|---| +| FR-BK-001 | P0 | 数据库原生备份必须包含全部 VexFS 元数据、内容块、版本、快照、ACL 和审计。 | +| FR-BK-002 | P0 | 数据库恢复后，VexFS 和普通业务表必须处于同一恢复点。 | +| FR-BK-003 | P0 | VexFS 不得自己调度或冒充数据库物理备份。 | +| FR-BK-004 | P0 | 必须提供备份后完整性验证入口。 | +| FR-BK-005 | P0 | 快照和文件版本不能被描述为灾难恢复备份。 | +| FR-BK-006 | P1 | 支持从固定快照导出单个工作区。 | +| FR-BK-007 | P1 | 逻辑导出必须携带格式版本、记录 checksum、内容 checksum 和源 commit。 | +| FR-BK-008 | P1 | 导入必须先进入隐藏状态，全部校验成功后原子发布。 | +| FR-BK-009 | P1 | 导入失败不能发布半个工作区。 | +| FR-BK-010 | P1 | 跨数据库导入必须验证目录结构和内容 hash 一致。 | +| FR-BK-011 | P1 | principal 映射必须显式提供，不能自动扩大权限。 | +| FR-BK-012 | P1 | 逻辑导出不得保存数据库密码或外部密钥。 | + +### 10.10 完整性检查和修复 + +| ID | 优先级 | 需求 | +|---|---|---| +| FR-CK-001 | P0 | 提供默认只读的 workspace check。 | +| FR-CK-002 | P0 | check 必须验证 inode、dentry、manifest、chunk、checksum、commit 和 snapshot 引用。 | +| FR-CK-003 | P0 | 缺失 chunk 不能用空数据替代后报告成功。 | +| FR-CK-004 | P0 | 发现损坏时必须返回对象、类型和建议动作。 | +| FR-CK-005 | P1 | repair 必须先生成计划，用户确认后执行。 | +| FR-CK-006 | P1 | repair 不能静默删除用户文件或历史。 | +| FR-CK-007 | P1 | 修复动作必须产生审计记录。 | + +### 10.11 CLI 直接命令 + +| ID | 优先级 | 需求 | +|---|---|---| +| FR-CLI-001 | P0 | CLI 必须使用 PostgreSQL、DuckDB 或 SQLite 正常连接方式。 | +| FR-CLI-002 | P0 | CLI 必须支持显式 workspace 和当前 workspace 配置。 | +| FR-CLI-003 | P0 | 直接读命令必须把正常结果写 stdout，诊断写 stderr。 | +| FR-CLI-004 | P0 | 直接写命令默认各自使用一个数据库事务。 | +| FR-CLI-005 | P0 | 文件不存在、权限不足、冲突、空间不足和连接失败必须返回不同退出码。 | +| FR-CLI-006 | P0 | CLI 不能直接修改内部表。 | +| FR-CLI-007 | P0 | CLI 不得把密码写入命令历史、worktree 清单或日志。 | +| FR-CLI-008 | P0 | cat/get/export 等大内容操作必须流式处理。 | +| FR-CLI-009 | P1 | 支持 JSON 输出，供 Agent 稳定解析。 | +| FR-CLI-010 | P1 | 支持安静模式、详细模式和无颜色模式。 | +| FR-CLI-011 | P1 | 支持命令超时、连接超时和 busy 重试参数。 | +| FR-CLI-012 | P0 | 支持保存不含密码的数据库 context，并切换当前 context。 | +| FR-CLI-013 | P0 | 支持保存当前 workspace，避免每条命令重复传参。 | +| FR-CLI-014 | P0 | 提供 doctor 命令检查 CLI、数据库连接、扩展版本、初始化状态和权限。 | +| FR-CLI-015 | P1 | vexfs shell 必须默认使用用户当前 shell，并把工作目录设置为 worktree 根。 | +| FR-CLI-016 | P1 | shell 退出时如果存在未提交修改，必须提示、自动提交或保留，不能静默丢弃。 | +| FR-CLI-017 | P1 | 非交互 Agent 使用 vexfs exec 时，可以在命令成功后自动提交。 | +| FR-CLI-018 | P1 | CLI 和数据库扩展版本不兼容时必须在执行写操作前拒绝并给出升级说明。 | + +### 10.12 Worktree、shell 和 exec（不能挂载时的回退） + +| ID | 优先级 | 需求 | +|---|---|---| +| FR-WT-001 | P1 | worktree create 必须固定 base commit。 | +| FR-WT-002 | P1 | worktree 清单必须记录 workspace、base commit、路径、inode、version、size 和 checksum。 | +| FR-WT-003 | P1 | worktree 只导出用户有权读取的普通文件、目录和安全链接。 | +| FR-WT-004 | P1 | 创建完成后不得长期保持数据库事务或锁。 | +| FR-WT-005 | P1 | worktree 必须使用权限受限的本地目录。 | +| FR-WT-006 | P1 | worktree 元数据不能包含数据库密码或访问令牌。 | +| FR-WT-007 | P1 | status 必须识别新增、修改、删除、改名和类型变化。 | +| FR-WT-008 | P1 | diff 必须支持文本摘要和机器可读结果。 | +| FR-WT-009 | P1 | commit 必须重新连接数据库并重新确认 principal。 | +| FR-WT-010 | P1 | commit 必须检查权限、配额、base commit 和 expected version。 | +| FR-WT-011 | P1 | commit 必须通过公开 changeset SQL 原子提交。 | +| FR-WT-012 | P1 | 提交失败必须保留本地 worktree 和差异。 | +| FR-WT-013 | P1 | discard 必须只删除受管 worktree，不能误删普通目录。 | +| FR-WT-014 | P1 | worktree 不能提交设备文件、socket、FIFO、绝对链接或越界链接。 | +| FR-WT-015 | P1 | vexfs shell 必须在 worktree 根目录启动真实系统 shell。 | +| FR-WT-016 | P1 | VexFS 不承诺限制 shell 子进程访问其他本地路径；进程沙箱由 Agent 运行环境负责。 | +| FR-WT-017 | P1 | vexfs exec 在命令退出非零时默认不能提交。 | +| FR-WT-018 | P1 | vexfs exec 只有命令成功且数据库提交成功时返回 0。 | +| FR-WT-019 | P1 | 支持只 checkout 子目录以及 include/exclude 规则。 | +| FR-WT-020 | P1 | 支持增量 refresh，并在覆盖本地修改前拒绝执行。 | +| FR-WT-021 | P1 | 支持 worktree 过期提示和安全清理。 | +| FR-WT-022 | P1 | 支持 commit message 和外部任务 id。 | + +### 10.13 Mount gateway 与 POSIX 行为 + +#### 10.13.1 安装与生命周期 + +| ID | 优先级 | 需求 | +|---|---|---| +| FR-MNT-001 | P0 | macOS 发行物必须包含受系统信任的 VexFS App、FSKit App Extension、SQLite 扩展和 CLI；Phase 1 必须验证 Developer ID 公证或 Mac App Store/TestFlight 中至少一条真实分发路径。 | +| FR-MNT-002 | P0 | `vexfs mount <目录>` 必须使用当前 context 和 workspace 完成挂载。 | +| FR-MNT-003 | P0 | 必须提供 `mount`、`unmount`、`mount list`、`mount status` 和 `doctor`，并显示 FSKit extension 的安装、启用和版本状态。 | +| FR-MNT-004 | P0 | mount point 必须是当前操作系统用户控制的空目录，默认权限为 0700。 | +| FR-MNT-005 | P0 | 本地 mount 默认只服务当前操作系统用户，不向其他本地用户共享；Linux `allow_other` 默认关闭。 | +| FR-MNT-006 | P0 | FSKit extension 由 macOS 管理生命周期；CLI 不能自行启动长期 root 进程，数据库扩展初始化和 SQL callback 也不能启动文件系统 loop。FSKit extension 可以在自身进程中打开 SQLite 连接。 | +| FR-MNT-007 | P1 | 登录后自动挂载必须显式开启，并使用 macOS 允许的用户级生命周期机制。 | +| FR-MNT-008 | P0 | mount gateway 停止后，数据库扩展、SQL 和已提交文件必须继续正常工作。 | +| FR-MNT-009 | P0 | FSKit 不可用、extension 未启用、签名不可信或系统版本过低时，doctor 必须给出明确原因、系统设置入口及 CLI/worktree 回退方法。 | + +#### 10.13.2 文件与目录操作 + +| ID | 优先级 | 需求 | +|---|---|---| +| FR-MNT-010 | P0 | FSKit adapter 支持 lookupItem、getAttributes、enumerateDirectory、openItem、closeItem、read、write 和 createItem，并映射到平台无关核心合同。 | +| FR-MNT-011 | P0 | FSKit adapter 支持 removeItem、renameItem、setAttributes、synchronize、reclaimItem 和 statfs 等 MVP 所需操作。 | +| FR-MNT-012 | P0 | 支持 lseek、statfs 和 utimens；chmod、symlink、readlink、hardlink 和 chown 必须按能力矩阵接入。当前 Linux 尚需补齐 utimens 的正式验收。 | +| FR-MNT-013 | P0 | 正确处理 O_CREAT、O_EXCL、O_TRUNC、O_APPEND、只读和读写打开模式。 | +| FR-MNT-014 | P0 | 同一目录内及跨目录 rename 必须原子生效，不能短暂出现两个名称都不存在。 | +| FR-MNT-015 | P0 | open 后 unlink 必须允许已打开句柄继续使用；最后一个句柄关闭后再回收无引用版本。 | +| FR-MNT-016 | P0 | 路径规范化、`.`、`..` 和越界规则必须与 SQL 合同一致；当前符号链接明确采用 POSIX target 原样保存合同，未来的 workspace 内限制策略必须作为独立可选合同。 | +| FR-MNT-017 | P0 | 不支持设备文件、socket、FIFO 和 setuid/setgid 文件；创建时返回明确错误。 | +| FR-MNT-018 | P1 | MVP Gate 不承诺 advisory flock；Phase 0 记录 macOS/FSKit 实际锁行为，跨 gateway 锁在 PostgreSQL 阶段映射为数据库租约或锁。 | +| FR-MNT-019 | P1 | 支持 hardlink，并在支持前对 link 调用返回稳定的不支持错误。 | +| FR-MNT-020 | P1 | 已支持扩展属性，并把值持久化到 SQLite；未知或不存在的 xattr 返回稳定错误。 | + +#### 10.13.3 写入、版本与事务 + +| ID | 优先级 | 需求 | +|---|---|---| +| FR-MNT-021 | P0 | 普通文件写入先进入带 request id 的未发布 staging；平台无关 `publish/sync` 以幂等短事务发布完整新版本。macOS adapter 主要由 `synchronize` 和 `closeItem` 触发。 | +| FR-MNT-022 | P0 | 不能在文件 open 到 close 的整个期间持有长期数据库事务。 | +| FR-MNT-023 | P0 | `write` 成功表示对应数据库 staging 已保存，但不表示新版本已发布。macOS SQLite mount 的未发布 staging 使用 WAL + NORMAL；`synchronize/publish/close` 前必须切回 FULL。电源故障可以丢失最近的未发布 staging，但不能产生半个已发布文件；`synchronize` 成功后完整版本必须达到 FULL。普通 SQL 连接仍使用调用者自己的 durability 配置。 | +| FR-MNT-024 | P0 | create、mkdir、unlink、rename 和 truncate 等单个文件系统操作各自在数据库事务内原子执行；chmod/chown 在进入支持范围后遵守同一规则。 | +| FR-MNT-025 | P0 | 多条 Bash 命令和多个 file handle 默认不是一个数据库事务，文档和错误信息不能暗示它们天然原子。 | +| FR-MNT-026 | P0 | 多文件或“文件 + 业务 SQL”原子提交必须使用公开 changeset/数据库事务合同。 | +| FR-MNT-027 | P1 | 提供 `vexfs txn run -- `，在隔离工作视图中运行命令，并把全部差异作为一个 changeset 提交。 | +| FR-MNT-028 | P0 | 每次成功发布文件内容都产生不可变 file version 和可审计 workspace commit。 | +| FR-MNT-029 | P0 | 并发写同一 expected version 时必须检测冲突，不能静默覆盖。 | + +#### 10.13.4 身份与权限 + +| ID | 优先级 | 需求 | +|---|---|---| +| FR-MNT-030 | P0 | 一个 mount 实例只使用一个数据库 principal；principal 来自真实数据库认证。 | +| FR-MNT-031 | P0 | 操作系统 uid 不能替代或提升数据库 principal。 | +| FR-MNT-032 | P0 | 每次 open、目录修改和内容发布都必须在数据库内重新检查权限。 | +| FR-MNT-033 | P0 | 数据库权限变化后，后续文件操作必须生效；已有 handle 不能绕过写入和发布权限。 | +| FR-MNT-034 | P0 | 凭证不能写入命令参数、mount 元数据、日志或审计；使用数据库密码文件、环境变量、系统凭证或交互输入。 | +| FR-MNT-035 | P0 | 其他本地用户读取 mount point 不等于获得数据库权限；MVP 使用当前用户私有 mount point，不开放本地多用户共享。 | + +#### 10.13.5 缓存、故障与恢复 + +| ID | 优先级 | 需求 | +|---|---|---| +| FR-MNT-036 | P0 | 正确性优先：MVP Gate 不启用 FSKit `DataCacheHandler` 的 kernel data cache；如 SDK 要求协商缓存，选择 no-cache 模式。后续启用缓存前必须具备版本校验和权限失效方案。 | +| FR-MNT-037 | P0 | 数据库连接中断时，未确认提交的操作必须返回 I/O 错误，不能报告成功。 | +| FR-MNT-038 | P0 | gateway 崩溃后，数据库中只能存在完整旧版本、完整新版本或可识别的未发布暂存对象。 | +| FR-MNT-039 | P0 | 重新挂载后必须清理或恢复过期暂存对象，不能把半写文件发布给用户。 | +| FR-MNT-040 | P0 | stale handle、数据库重启、workspace freeze、配额不足和冲突必须映射为稳定 errno，并保留详细诊断。 | +| FR-MNT-041 | P0 | mount gateway 不能成为数据库备份的一部分；数据库原生备份必须独立包含全部已提交状态。 | +| FR-MNT-042 | P1 | 多个 gateway 同时访问时支持主动失效通知；在此能力完成前，不承诺跨 gateway 实时可见，只保证 close-to-open、冲突检测和不静默覆盖。 | +| FR-MNT-043 | P0 | 同一 handle 必须读到自己的 staged writes；其他已打开 handle 继续读取其打开时版本；成功 publish/synchronize 后的新 open 读取新版本。 | +| FR-MNT-044 | P0 | closeItem/reclaimItem 时如果仍有 dirty staging，可以执行可恢复的兜底发布，但失败不能删除 staging、伪造成功或产生半版本。 | + +### 10.14 多数据库适配 + +| ID | 优先级 | 需求 | +|---|---|---| +| FR-DB-001 | P0 | 已进入支持范围的数据库必须共享路径、版本、快照、changeset 和错误分类合同；不要求三个适配器同时发布。 | +| FR-DB-002 | P0 | 三端不得共享或模拟数据库内部实现。 | +| FR-DB-003 | P0 | 每个适配器必须使用宿主 allocator、事务和存储 API。 | +| FR-DB-004 | P0 | 适配器不得直接读写数据库物理文件。 | +| FR-DB-005 | P0 | 宿主不允许时，不得创建线程、后台任务或外部文件。 | +| FR-DB-006 | P0 | PostgreSQL 必须使用 role、MVCC、WAL 和数据库锁能力。 | +| FR-DB-007 | P0 | SQLite 必须正确处理单 writer、busy、savepoint、WAL/reopen 和 Backup API。mount staging 的热点元数据必须与大内容 BLOB 分开；旧内联布局要能原子迁移。 | +| FR-DB-008 | P1 | DuckDB 必须正确处理单进程写入、乐观冲突、checkpoint 和 reopen；默认先支持 SQL/CLI/worktree，可写 mount 只允许 gateway 成为唯一写进程或宿主提供可验证的多进程写入能力。 | +| FR-DB-009 | P1 | 同一组合同 spec 必须在所有已支持引擎运行。 | + +## 11. CLI 命令范围 + +### 11.1 直接命令 + +| 分类 | 命令 | P0/P1 | +|---|---|---| +| 安装诊断 | setup、doctor、version | P0 | +| 挂载 | mount、unmount、mount list、mount status | P0 | +| 工作区 | workspace create/list/stat/freeze/unfreeze/drop/use | P0 | +| 浏览 | ls、tree、find | P0 | +| 读取 | cat、head、tail、stat、file | P0 | +| 查找统计 | grep、rg、wc、du | P0 | +| 修改 | touch、mkdir、put、get、cp、mv、rm、rmdir、truncate | P0 | +| 链接 | ln -s、readlink | P1 | +| 权限 | acl、chmod、chown | P1 | +| 版本 | history、show、diff、restore | P0 | +| 快照 | snapshot create/list/drop、restore | P0 | +| 运维 | check、repair plan、gc、quota、retention | P1 | +| 迁移 | export、import | P1 | +| worktree | create、status、diff、refresh、commit、discard、list | P1 | +| 运行 | shell、exec、txn run | P1 | +| 机器输出 | `--json`、`--no-color`、稳定退出码 | P1 | + +### 11.2 命令行为 + +- 首选路径是 `vexfs mount ~/VexFS/`，挂载后直接使用系统 `cd/ls/rg/sed/python/git`； +- `vexfs ls` 等直接命令用于脚本、诊断和不能挂载的环境，不覆盖系统命令； +- `vexfs shell` 是不能挂载时的 worktree 回退入口； +- 直接 `grep/rg` 是普通文本查找，不是语义搜索； +- 二进制文件默认不作为文本输出，除非用户显式指定； +- 直接命令的路径表示 VexFS workspace 路径； +- mount point 内的路径表示数据库 workspace 的实时视图； +- worktree 内的路径表示本地普通文件路径； +- CLI 帮助必须清楚说明当前命令操作 mount、数据库路径还是本地 worktree。 + +### 11.3 直接命令与 SQL 映射 + +| CLI | SQL 合同 | +|---|---| +| ls/tree/find | vexfs_list / vexfs_walk | +| cat/head/tail/get | vexfs_read / vexfs_read_range | +| grep/rg | vexfs_grep | +| stat/file/du/wc | vexfs_stat / vexfs_usage | +| touch/mkdir/put | vexfs_create / vexfs_mkdir / vexfs_write | +| cp/mv/rm | vexfs_copy / vexfs_move / vexfs_remove | +| history/show | vexfs_history / vexfs_read_version | +| diff | 读取两个版本后由 CLI 比较；workspace commit diff 后续使用 vexfs_diff | +| restore | vexfs_restore_version；snapshot 恢复后续使用 vexfs_restore_* | +| acl/chmod/chown | vexfs_grant / vexfs_revoke / vexfs_owner_set | +| mount 文件操作 | vexfs_handle_* / vexfs_read_range / vexfs_changeset_* | +| worktree commit | vexfs_changeset_* | +| check/gc | vexfs_check / vexfs_gc | +| export/import | vexfs_export_* / vexfs_import_* | + +## 12. SQL 合同范围 + +以下是产品级函数族，不在本文锁定每个数据库的最终声明语法。 + +### 12.1 初始化和能力 + +```sql +SELECT vexfs_init(); +SELECT * FROM vexfs_version(); +SELECT * FROM vexfs_capabilities(); +``` + +### 12.2 工作区 + +```sql +SELECT vexfs_workspace_create('workspace'); +SELECT * FROM vexfs_workspace_list(); +SELECT * FROM vexfs_workspace_stat('workspace'); +SELECT vexfs_workspace_freeze('workspace'); +SELECT vexfs_workspace_drop('workspace', false); +``` + +### 12.3 文件和目录 + +```sql +SELECT vexfs_mkdir('workspace', '/reports', true); +SELECT vexfs_write('workspace', '/reports/a.md', :content); +SELECT vexfs_write_range('workspace', '/reports/a.md', 100, :content); +SELECT vexfs_read('workspace', '/reports/a.md'); +SELECT vexfs_read_range('workspace', '/reports/a.md', 0, 4096); +SELECT vexfs_copy('workspace', '/reports/a.md', '/archive/a.md'); +SELECT vexfs_move('workspace', '/reports/a.md', '/reports/final.md'); +SELECT vexfs_remove('workspace', '/archive', true); +SELECT * FROM vexfs_list('workspace', '/reports', 1000, :cursor); +SELECT * FROM vexfs_stat('workspace', '/reports/final.md'); +``` + +### 12.4 版本和快照 + +SQLite `0.7.0` 当前已经实现文件版本和 workspace 快照： + +```sql +SELECT vexfs_history('workspace', '/reports/final.md'); +SELECT vexfs_history('workspace', '/reports/final.md', 100, :before_version); +SELECT vexfs_read_version('workspace', '/reports/final.md', 2); +SELECT vexfs_compare_versions('workspace', '/reports/final.md', 2, 3); +SELECT vexfs_restore_version('workspace', '/reports/final.md', 2, :expected_version); +``` + +四参数 `vexfs_history` 返回 `{"entries": [...], "next_before": ...}`，按版本倒序；每页最多 1000 条，CLI 默认 100 条。两参数形式为兼容入口，返回 JSON 数组并固定最多 100 条。每项包含 `version`、`commit`、`parent_commit`、`size`、`checksum`、`created_at`、`message` 和 `current`。`checksum` 是小写 SHA-256；文件和符号链接的 `vexfs_stat` 也返回当前版本的 `checksum`，目录返回 `null`。 + +恢复成功返回新版本号。恢复必须使用带 `expected_version` 的四参数接口，不能省略并发检查。恢复版本只引用目标版本的不可变内容，历史中仍保留一条新的 version 和 commit；这不是通用内容去重，普通写入仍保存完整内容。 + +workspace 快照当前公开合同： + +```sql +SELECT vexfs_snapshot_create('workspace', 'before-change'); +SELECT * FROM vexfs_snapshot_list('workspace'); +SELECT vexfs_snapshot_diff('workspace', 'before-change', 'HEAD'); +SELECT vexfs_snapshot_restore('workspace', 'before-change', :expected_head); +``` + +### 12.5 权限和审计 + +```sql +SELECT vexfs_grant('workspace', '/reports', 'report_writer', 'read,write'); +SELECT vexfs_revoke('workspace', '/reports', 'report_writer', 'write'); +SELECT vexfs_owner_set('workspace', '/reports', 'report_admin'); +SELECT * FROM vexfs_audit('workspace', :since_commit, :limit, :cursor); +``` + +### 12.6 Changeset + +```sql +BEGIN; + +SELECT vexfs_changeset_open('workspace', 847, :request_id); +SELECT vexfs_changeset_put('/src/main.cpp', :content, :expected_version); +SELECT vexfs_changeset_move('/src/old.cpp', '/src/new.cpp', :expected_version); +SELECT vexfs_changeset_remove('/tmp/output', :expected_version, true); +SELECT vexfs_changeset_apply('update generated files', :external_task_id); + +COMMIT; +``` + +大内容必须允许分块暂存，不能要求一个 SQL 参数携带完整工作区。 + +### 12.7 检查、配额和迁移 + +```sql +SELECT vexfs_check('workspace', 1); -- 深度检查，包含所有 canonical BLOB 的 SHA-256 +SELECT vexfs_check('workspace', 0); -- 快速检查，只检查结构、引用和 BLOB 长度 +SELECT * FROM vexfs_repair_plan('workspace'); +SELECT vexfs_gc('workspace', 1000); +SELECT vexfs_quota_set('workspace', :max_bytes, :max_files, :max_file_bytes); +SELECT vexfs_retention_set('workspace', :keep_versions, :keep_days); +SELECT * FROM vexfs_export_begin('workspace', 'snapshot-name'); +SELECT vexfs_import_begin('new-workspace', :format_version); +``` + +SQLite `0.7.0` 已实现只读的 `vexfs_check`。它返回 JSON，检查 SQLite 自身、workspace root、 +inode、dentry、可达性、commit 父链、snapshot、当前和历史版本引用、版本别名、BLOB 长度、 +staging 和 handle。深度模式额外按 64 KiB 流式计算 SHA-256，不把全部历史内容载入内存。 +每个问题包含稳定的 `code`、对象、类型、说明和建议动作，最多返回 1000 个问题并报告完整 +问题数。当前内容模型是 `full-blob-v1`，尚未存在的 chunk/manifest 不会被假装成已检查。 + +对应 CLI 是 `vexdb fs check`，默认深度检查；`--quick` 跳过内容哈希，`--json` 输出原始 +JSON。发现损坏时仍输出报告并返回退出码 8。当前没有自动 repair；检查不能修改或填补内容。 + +## 13. 权限规则 + +### 13.1 两层权限 + +有效权限取以下两层的交集： + +1. 数据库是否允许当前 principal 调用 VexFS 公开函数； +2. VexFS ACL 是否允许 principal 操作目标 workspace/path。 + +VexFS ACL 不能扩大数据库拒绝的权限。 + +### 13.2 ACL 继承 + +- 工作区根必须有 owner ACL； +- 子对象默认继承最近祖先 ACL； +- 显式 deny 优先于 allow； +- owner 变更需要 admin； +- ACL 变更需要 share 或 admin； +- 恢复旧版本不能恢复旧 ACL 来绕过当前权限； +- 删除和改名必须检查父目录权限。 + +### 13.3 三端身份 + +| 数据库 | principal 来源 | 权限等级 | +|---|---|---| +| PostgreSQL | 当前数据库 role | 完整多用户权限 | +| DuckDB | 当前连接和宿主可证明身份 | 应用内权限，不宣传为服务器级身份 | +| SQLite | 连接、authorizer 或宿主注入的受信身份 | 应用内权限，不宣传为服务器级身份 | + +CLI 不能接受 `--as-user` 之类参数绕过数据库身份。 + +## 14. Worktree 详细流程 + +### 14.1 创建 + +1. CLI 使用标准数据库连接认证。 +2. 查询 workspace head commit。 +3. 检查目标路径的 traverse/list/read 权限。 +4. 创建权限受限的临时目录。 +5. 流式导出允许读取的普通文件和安全链接。 +6. 保存 base commit、inode、version、size 和 checksum 清单。 +7. 关闭读取事务和数据库锁。 + +worktree 清单不保存凭证，也不能被数据库扩展信任为权限依据。 + +### 14.2 工作 + +模型可以使用运行环境中已有的任何普通文件工具。VexFS 只设置工作目录，不承诺提供 OS 进程沙箱。禁止进程访问主机其他路径属于 Agent 沙箱或容器职责。 + +### 14.3 检测差异 + +必须识别： + +- 新文件和目录； +- 内容修改； +- 元数据修改； +- 删除； +- 可证明的改名； +- 文件、目录、链接之间的类型变化； +- 不支持的特殊文件。 + +### 14.4 提交 + +1. 重新认证数据库身份。 +2. 读取当前 workspace head。 +3. 校验 base commit 和 expected version。 +4. 校验路径和链接安全。 +5. 重新检查所有读取、创建、写入、删除、改名和管理权限。 +6. 计算新增空间并检查配额。 +7. 把差异写入事务级 changeset。 +8. 一次发布文件版本、目录版本、workspace commit 和审计记录。 +9. 数据库 commit 成功后才更新本地 worktree 基线。 + +### 14.5 冲突和失败 + +- 权限失败：不提交，保留本地差异； +- 配额失败：不提交，返回超限对象和所需空间； +- 并发冲突：不覆盖，返回冲突路径、base version 和 current version； +- 网络中断：数据库事务回滚，本地 worktree 保留； +- 命令退出非零：`vexfs exec` 默认不提交； +- 数据库提交成功但本地状态更新失败：下次 status 必须能从数据库 commit 恢复基线。 + +### 14.6 清理 + +- discard 只能删除带有效 VexFS worktree 标记的目录； +- 非空且有未提交变更时，需要显式 `--force`； +- 过期 worktree 只提示，不自动删除用户未提交内容； +- 自动清理不得删除普通目录； +- worktree 不是备份，长期保存需要 commit、snapshot 或 export。 + +## 15. 数据保护与备份 + +### 15.1 四层保护 + +| 层次 | 解决问题 | 不解决问题 | +|---|---|---| +| 文件版本 | 单文件误改、查看历史 | 整个数据库丢失 | +| 工作区快照 | 批量误操作、目录恢复 | 数据库介质损坏 | +| 数据库原生备份 | 灾难恢复、业务数据一致性 | 跨数据库格式迁移 | +| 逻辑导出 | 单工作区离线保存和迁移 | 数据库全量灾难恢复 | + +### 15.2 PostgreSQL + +- pg_dump、物理备份和恢复测试必须包含 VexFS 内部对象和内容； +- extension 升级与 restore 顺序必须有文档； +- 权限和 ACL 需要验证恢复结果； +- VexFS 不调用或调度 pg_dump、base backup 或复制。 + +### 15.3 SQLite + +- 使用 SQLite Online Backup API 或安全数据库备份方式； +- 禁止在活跃写入时直接复制 `.db` 而忽略 `-wal`； +- 恢复测试必须覆盖 WAL、busy、crash/reopen； +- 备份后必须运行 VexFS check。 + +### 15.4 DuckDB + +- 使用 DuckDB 支持的数据库复制、导出或安全文件备份流程； +- 备份时遵守 DuckDB 连接、锁和 checkpoint 规则； +- 恢复后验证 catalog、内容和 checksum； +- 不把普通文件复制包装成无条件安全的在线备份。 + +### 15.5 逻辑导出格式 + +逻辑包至少包含： + +- format version； +- workspace metadata； +- source engine 和 source commit； +- inode、dentry、file version、manifest 和 chunk； +- snapshot； +- ACL principal 映射表； +- 每条记录 checksum； +- 整包 checksum； +- 导出完成标记。 + +## 16. 三端能力矩阵 + +| 能力 | PostgreSQL | SQLite | DuckDB | +|---|---|---|---| +| 扩展形态 | CREATE/ALTER EXTENSION | loadable/static extension | loadable extension | +| 主要场景 | 共享、多 Agent、企业服务 | 端侧、单文件、离线 | 本地分析、单进程工作区 | +| 用户身份 | 完整 role | 连接/authorizer | 连接/宿主边界 | +| 多写者 | 支持 | 单 writer | 主要是单进程内 | +| 冲突方式 | 锁、MVCC、约束 | busy、单 writer | 乐观冲突 | +| WAL/恢复 | PostgreSQL 管理 | SQLite 管理 | DuckDB 管理 | +| 原生备份 | PG 工具 | Online Backup API | DB copy/export/安全文件流程 | +| 路径 ACL | 完整 | 应用内 | 应用内 | +| P0 合同 | Phase 2 实现 | Phase 1 收口 | Phase 4 实现 | +| 可写 macOS mount | Phase 2 | 已有 SQLite 预览 | Phase 4 后按单写者条件决定 | +| 可写 Linux mount | Phase 2 | 已有 libfuse3 预览 | Phase 4 后按单写者条件决定 | +| 可写 Windows mount | Phase 3 | Phase 3 | Phase 4 后按单写者条件决定 | + +版本约束： + +- PostgreSQL：16、17、18、19； +- DuckDB：仓库当前约束 v1.5.2； +- SQLite：不低于 3.24.0，并由构建和发布矩阵固定具体版本。 + +## 17. 核心数据模型要求 + +### 17.1 工作区 + +必须保存 workspace id、名称、根 inode、head commit、owner、冻结状态、配额、保留规则和时间。 + +### 17.2 inode 版本 + +必须保存 inode id、version id、有效 commit 范围、类型、mode、owner、size、manifest、checksum、删除标记和时间。 + +### 17.3 目录项版本 + +必须保存 workspace、parent inode、原始名称、child inode、有效 commit 范围和 tombstone。 + +### 17.4 文件内容 + +必须使用不可变 manifest 和 chunk；manifest 发布后不能原地修改；chunk 必须带 checksum。 + +### 17.5 快照 + +快照只保存名称、workspace、commit、创建人和创建时间。 + +### 17.6 ACL 与审计 + +ACL 必须绑定 workspace/inode/principal；审计必须绑定 workspace/commit/sequence/principal/operation/before/after。 + +## 18. 非功能需求 + +### 18.1 正确性 + +| ID | 要求 | +|---|---| +| NFR-COR-001 | 任何写入只能出现完整旧状态或完整新状态。 | +| NFR-COR-002 | 三端对相同操作序列返回相同目录树、内容 hash、版本结果和错误分类。 | +| NFR-COR-003 | 每个内部写步骤必须支持故障注入测试。 | +| NFR-COR-004 | 数据库 crash/reopen 后，已提交文件完整，未提交修改不可见。 | +| NFR-COR-005 | worktree changeset 不能产生半个目录项、半个 manifest 或缺块文件。 | +| NFR-COR-006 | mount 的每个成功操作必须符合文档中的 POSIX 子集和数据库提交语义。 | +| NFR-COR-007 | mount adapter 崩溃、进程终止或断网后，已报告 publish/sync 成功的文件版本必须存在；只报告 write 成功的内容至少保留为可识别的 staging。电源故障发生在 SQLite NORMAL staging 阶段时允许丢失最近未发布内容，但不能损坏最后一次 FULL publish 的版本。 | + +### 18.2 可靠性 + +| ID | 要求 | +|---|---| +| NFR-REL-001 | 提交结果不确定时，CLI 必须通过 request id 或 commit 查询确认，不能盲目重试。 | +| NFR-REL-002 | 数据库连接中断不能导致本地未提交内容丢失。 | +| NFR-REL-003 | GC、check、export 和 backup verify 必须可中断并安全重试。 | +| NFR-REL-004 | 恢复和 repair 也必须生成版本和审计记录。 | +| NFR-REL-005 | mount gateway 可重启、可重新挂载，不能成为已提交数据恢复的前置条件。 | +| NFR-REL-006 | 数据库重启后 gateway 必须明确断开旧 handle，并在重新连接后恢复新操作。 | + +### 18.3 安全 + +| ID | 要求 | +|---|---| +| NFR-SEC-001 | 所有外部输入按不可信处理，包括路径、导入包、BLOB、正则模式和 worktree。 | +| NFR-SEC-002 | 路径规范化后必须再次检查工作区边界。 | +| NFR-SEC-003 | worktree 使用当前用户私有目录，默认权限不得允许其他本地用户读取。 | +| NFR-SEC-004 | 密码不得出现在命令参数、日志、审计或 worktree 元数据中。 | +| NFR-SEC-005 | 传输加密使用数据库连接能力，静态加密使用数据库或磁盘能力。 | +| NFR-SEC-006 | VexFS 不承诺提供进程级沙箱；文档必须明确这一点。 | +| NFR-SEC-007 | 正则、递归目录和导入大小必须有限制，避免资源耗尽。 | +| NFR-SEC-008 | mount point 默认只允许当前操作系统用户访问，一个 mount 只能代表一个数据库 principal。 | +| NFR-SEC-009 | gateway 必须以普通用户运行；安装流程不能要求长期 root 进程。 | + +### 18.4 资源和规模 + +MVP Gate 验收数据集： + +- 1,000 个普通文件； +- 总当前内容 1 GiB； +- 单文件最大 100 MiB； +- 单目录最大 1,000 个直接子项； +- 100 个 workspace commit； +- 10 个快照。 + +完整 P0 产品验收数据集： + +- 10 万个普通文件； +- 总当前内容 10 GiB； +- 单文件最大 100 MiB； +- 单目录最大 1 万个直接子项； +- 1,000 个 workspace commit； +- 100 个快照。 + +| ID | 要求 | +|---|---| +| NFR-RES-001 | 扩展使用宿主 allocator，不使用不受管长期内存。 | +| NFR-RES-002 | 大内容读写和导出必须流式或分块。 | +| NFR-RES-003 | list、history 和 audit 必须分页。 | +| NFR-RES-004 | GC 必须分批并可配置最大处理量。 | +| NFR-RES-005 | worktree 必须支持空间上限和最小导出范围。 | +| NFR-RES-006 | 版本、快照和历史空间必须可观测。 | + +### 18.5 初始性能目标 + +以下是完整 P0 的本地参考环境目标，不是公开 SLA。参考环境为 8 核 CPU、16 GiB 内存、NVMe、本机数据库连接。MVP Gate 只要求正确完成 1,000 文件和 1 GiB 数据集，并记录每个调用的基线；在语义稳定前不为原型设公开性能承诺。 + +| 操作 | 目标 | +|---|---| +| stat 单文件 SQL P95 | 小于 50 ms | +| list 1,000 项首屏 P95 | 小于 300 ms | +| 读取 64 KiB 小文件 SQL P95 | 小于 100 ms | +| 写入 64 KiB 小文件 SQL P95 | 小于 200 ms | +| CLI 单次只读命令 P95 | 小于 300 ms | +| mount stat 单文件 P95 | 小于 100 ms | +| mount 读取 64 KiB 小文件 P95 | 小于 200 ms | +| mount 写入并 close 64 KiB 小文件 P95 | 小于 400 ms | +| mount 顺序读取 100 MiB | 不低于参考环境本地数据库 SQL 流式读取吞吐的 60% | +| 10 万文件 worktree status | 小于 30 秒 | +| 创建快照 | 不随工作区总内容线性复制 | + +2026-07-21 当前参考机实测：SQLite 直连创建 10 万文件为 123.327 秒；`preview.13` +FSKit 创建 1 万文件为 114.828 秒，遍历为 0.462 秒，`rg` 为 23.895 秒，快照恢复为 +21.465 秒。这些是当前基线，不是公开 SLA。100,000 文件真实 mount 已完成创建、遍历 +和抽样读取，但 `rg` 在 3,600 秒内未完成，快照/恢复后半段尚未验收。 + +### 18.6 可运维性 + +- 提供扩展版本、能力、schema 版本和初始化状态； +- 提供 mount 列表、连接状态、principal、workspace、缓存和未发布 handle 数量； +- 提供 workspace usage、quota、retention 和 GC 状态； +- 提供只读 check 和 backup verify； +- 错误消息使用通俗语言，并保留底层数据库诊断； +- 日志不能包含密码或文件正文； +- 升级、备份、恢复、迁移和故障处理必须有独立文档。 + +### 18.7 兼容性 + +- macOS 26.0+、Apple Silicon 是 mount gateway 的首个验证平台； +- MVP 使用当前稳定 Xcode SDK 中可用的 FSKit API，并通过薄适配层隔离后续 `Operations` 到 `Handler` 的迁移； +- macOS 技术预览需要签名、公证，并提供 FSKit extension 启用检查； +- CLI 首轮支持 macOS；Linux 和 Windows 在 mount adapter 完成前使用直接命令或 worktree； +- PostgreSQL 和 DuckDB 数据库扩展仍按各自服务器支持平台构建，不因 macOS gateway 改变宿主约束； +- 文件名按字节稳定比较，不依赖系统语言区域； +- UTF-8 文本命令提供稳定行为，二进制内容按字节保存； +- 路径分隔符在 VexFS 合同中统一使用 `/`； +- worktree 在宿主平台映射文件名时必须发现并拒绝无法安全表示的冲突。 + +## 19. 错误与退出码 + +### 19.1 统一错误分类 + +```text +VEXFS_NOT_FOUND +VEXFS_ALREADY_EXISTS +VEXFS_NOT_DIRECTORY +VEXFS_IS_DIRECTORY +VEXFS_NOT_EMPTY +VEXFS_PERMISSION_DENIED +VEXFS_READ_ONLY +VEXFS_QUOTA_EXCEEDED +VEXFS_CONFLICT +VEXFS_BUSY +VEXFS_NO_SPACE +VEXFS_CORRUPTION +VEXFS_INVALID_PATH +VEXFS_UNSAFE_LINK +VEXFS_UNSUPPORTED +VEXFS_VERSION_MISMATCH +VEXFS_BACKUP_INCOMPLETE +VEXFS_UNAVAILABLE +VEXFS_TIMEOUT +VEXFS_STALE_HANDLE +``` + +### 19.2 Mount POSIX errno 映射 + +首个 FSKit 原型开始前必须冻结以下基础映射；FSKit adapter 使用 `POSIXError` 或等价结果返回，底层数据库诊断写入日志和 `doctor`，不能把不同数据库错误直接泄露成不同 errno。 + +| VexFS 错误 | errno | 说明 | +|---|---:|---| +| `VEXFS_NOT_FOUND` | `ENOENT` | 文件、目录或 workspace 不存在 | +| `VEXFS_ALREADY_EXISTS` | `EEXIST` | O_EXCL 或同名创建冲突 | +| `VEXFS_NOT_DIRECTORY` | `ENOTDIR` | 路径中间项不是目录 | +| `VEXFS_IS_DIRECTORY` | `EISDIR` | 对目录执行普通文件操作 | +| `VEXFS_NOT_EMPTY` | `ENOTEMPTY` | 删除非空目录 | +| `VEXFS_PERMISSION_DENIED` | `EACCES` | 数据库 principal 无权操作 | +| `VEXFS_READ_ONLY` | `EROFS` | workspace 或数据库只读 | +| `VEXFS_QUOTA_EXCEEDED` | `EDQUOT` | workspace 配额不足 | +| `VEXFS_NO_SPACE` | `ENOSPC` | 数据库或宿主空间不足 | +| `VEXFS_CONFLICT` | `EAGAIN` | expected version 已变化，可重新打开后重试 | +| `VEXFS_BUSY` | `EBUSY` | SQLite busy、workspace freeze 或资源正被占用 | +| `VEXFS_CORRUPTION` | `EIO` | 校验失败或内部状态损坏 | +| `VEXFS_INVALID_PATH` | `EINVAL` | 路径或参数不合法 | +| `VEXFS_UNSAFE_LINK` | `ELOOP` | 链接循环或越界 | +| `VEXFS_UNSUPPORTED` | `ENOTSUP` | 当前阶段不支持的调用 | +| `VEXFS_VERSION_MISMATCH` | `EPROTO` | gateway 与扩展合同版本不兼容 | +| `VEXFS_UNAVAILABLE` | `EIO` | 数据库连接已断开 | +| `VEXFS_TIMEOUT` | `ETIMEDOUT` | 数据库调用超时 | +| `VEXFS_STALE_HANDLE` | `ESTALE` | handle 已过期或数据库重启后失效 | + +chmod/chown 等已明确不支持的权限修改返回 `EPERM`；跨 workspace rename 在尚未支持时返回 `EXDEV`。同一个错误在重试前后必须保持稳定，不能根据 PostgreSQL、SQLite 或 DuckDB 的原始错误文案变化。 + +### 19.3 CLI 退出码 + +| 退出码 | 含义 | +|---|---| +| 0 | 命令和数据库提交成功 | +| 1 | 普通命令失败 | +| 2 | 参数或路径格式错误 | +| 3 | 文件或工作区不存在 | +| 4 | 权限不足 | +| 5 | 并发冲突 | +| 6 | 配额或空间不足 | +| 7 | 数据库连接、busy 或超时 | +| 8 | 数据损坏或校验失败 | +| 9 | 不支持的文件类型或能力 | +| 10 | 外部命令退出非零，未提交 | + +`--json` 输出必须同时提供稳定错误 code、通俗 message、可选 path 和底层 detail。 + +## 20. 验收标准 + +### 20.1 MVP Gate 验收 + +| ID | 验收场景 | +|---|---| +| AC-MVP-001 | 在干净的 macOS 26.0+ Apple Silicon 环境安装并启用 VexFS FSKit extension 后，`vexfs doctor` 通过，一条 `vexfs mount` 命令得到可 `cd` 的真实目录。 | +| AC-MVP-002 | `ls/cat/cp/mv/rm/mkdir/find/grep/sed`、Python 读写和基础 Git `init/add/commit/checkout` 直接运行。 | +| AC-MVP-003 | FSKit lookup/get attributes/enumerate/open/close/create/read/write/remove/rename/set attributes/synchronize/reclaim 和 statfs 通过核心合同测试。 | +| AC-MVP-004 | 同一 handle 读到自己的未发布写入；其他 handle 看到打开时版本；成功 publish/synchronize 后的新 open 看到新版本。 | +| AC-MVP-005 | `synchronize` 重复调用不产生重复 commit；close/reclaim 失败不能丢弃 dirty staging 或伪造提交成功。 | +| AC-MVP-006 | `synchronize` 成功后终止 FSKit extension、重启 SQLite 并重新挂载，内容和 checksum 保持正确。 | +| AC-MVP-007 | SQLite busy、连接中断、空间不足、版本冲突和 stale handle 返回冻结后的稳定 errno。 | +| AC-MVP-008 | MVP 不启用 FSKit kernel data cache，mount point 为当前用户私有，其他本地用户不能访问。 | +| AC-MVP-009 | 1,000 个文件、1 GiB 当前内容、100 MiB 单文件完成写入、重挂载、读取和校验，没有半文件或丢失目录项。 | +| AC-MVP-010 | gateway 与扩展合同版本不兼容时拒绝可写挂载；卸载 gateway 后，SQL 仍可读写全部已提交文件。 | + +当前验收已经额外覆盖 `chmod` 的 `0000..0777` mode、可执行脚本、symlink、readlink、xattr、hardlink、owner/group 元数据和便携 ACL；Linux 已覆盖 chown 和 hardlink 的 mount 映射。仍不验收多 gateway、完整 ACL 授权、跨 gateway flock、共享可写 mmap、10 万文件和 DuckDB mount。 + +### 20.2 P0 产品验收 + +| ID | 验收场景 | +|---|---| +| AC-P0-001 | 业务表更新和文件写入在同一事务中同时 commit。 | +| AC-P0-002 | 事务 rollback 后业务表和文件都保持旧状态。 | +| AC-P0-003 | 两个 Agent 修改同一文件时，后提交者得到冲突且本地差异保留。 | +| AC-P0-004 | 无 read 权限的用户不能通过 SQL、CLI、checkout 或历史读取内容。 | +| AC-P0-005 | 无 write/delete/rename 权限的 changeset 整体失败。 | +| AC-P0-006 | macOS 用户挂载 workspace 后，可以直接运行 ls、cat、rg、sed、python、tar、git、编译器和 apply_patch。 | +| AC-P0-007 | create、write、rename、unlink、truncate、symlink、flock、fsync 和 open-unlink 行为通过 POSIX 子集测试。 | +| AC-P0-008 | 快照后批量修改，恢复单文件、目录和工作区均正确。 | +| AC-P0-009 | 数据库原生备份恢复后，业务表、文件、版本、快照、ACL 和审计一致。 | +| AC-P0-010 | crash 注入后只出现完整旧状态或完整新状态。 | +| AC-P0-011 | 缺失 chunk 时 check 报告损坏，read 不返回伪造内容。 | +| AC-P0-012 | 10 万文件和 10 GiB 数据集通过正确性与初始性能目标。 | +| AC-P0-013 | 完成一次 context 和 workspace 配置后，用户可以用一条 vexfs mount 命令得到真实目录并直接 cd 进入。 | +| AC-P0-014 | FSKit extension 被终止、数据库重启或连接中断后，已成功 synchronize 的文件仍完整，未成功发布不被伪装成成功。 | +| AC-P0-015 | vexfs doctor 能准确报告 App/extension 未安装、FSKit extension 未启用、系统版本过低、签名或版本不兼容、未初始化、连接失败和权限不足。 | +| AC-P0-016 | mount point 默认仅当前操作系统用户可访问，数据库 ACL 变化会影响后续文件操作。 | +| AC-P0-017 | 两个 mount 同时修改同一版本时至少一个得到冲突，不能静默覆盖。 | +| AC-P0-018 | 不安装或不运行 gateway 时，SQL 和 CLI 数据能力仍然可用。 | +| AC-P0-019 | 发行评测没有匹配用例、选中用例全部跳过或必跑用例出现跳过时，必须非零退出。 | +| AC-P0-020 | manifest、CLI、SQLite 扩展和 mount adapter 的合同版本、runtime ABI、源码 commit 与哈希必须来自同一次构建。 | +| AC-P0-021 | Developer ID 和公证包必须来自干净 commit；安装器拒绝脏源码正式包、错误 TeamIdentifier、哈希损坏和 ABI/合同不一致。 | +| AC-P0-022 | ad-hoc 包只能通过显式本地测试开关安装，不能伪装成用户可分发的正式包。 | + +### 20.3 合同测试 + +- 路径规范化和非法路径； +- 同名创建和原子 replace； +- range read/write 和 truncate； +- 目录分页和稳定排序； +- savepoint 和 rollback-to； +- 同文件双写； +- move 与 remove 并发； +- workspace freeze 与写入； +- 版本、快照、恢复和 retention； +- ACL 继承、deny 和 owner； +- quota、GC 和快照保护； +- export/import checksum； +- mount lookup/getattr/readdir/open/read/write/close/reclaim； +- mount create/mkdir/rmdir/unlink/rename/truncate/publish/sync； +- O_EXCL、O_TRUNC、O_APPEND、open-unlink、symlink 和 flock； +- mount principal、ACL 变更、freeze、quota 和冲突； +- gateway kill、数据库重启、断网、重新挂载和暂存清理； +- worktree create/status/diff/commit/discard； +- 安全链接和越界链接； +- 数据库 backup/restore 和 crash/reopen。 + +### 20.4 数据库专项测试 + +#### SQLite + +- transaction、savepoint、busy 和单 writer； +- WAL 模式 crash/reopen； +- Online Backup API； +- shadow/internal tables 保护； +- 扩展加载和 SQLite 最低版本检查。 + +#### PostgreSQL + +- PG16、17、18、19； +- role、权限和内部 schema； +- 多连接并发和 deadlock； +- WAL/restart； +- pg_dump/pg_restore 和物理备份恢复； +- extension 安装和升级。 + +#### DuckDB + +- v1.5.2 加载； +- 单进程多连接和乐观冲突； +- checkpoint/reopen； +- 数据库 copy/export 恢复； +- catalog 和内部对象保护。 + +## 21. 发布阶段 + +发布阶段的详细范围、状态、完成条件和下一步只在路线图维护： + +`docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md` + +当前阶段摘要如下： + +| 阶段 | 目标 | 状态 | +|---|---|---| +| Phase 0 | 统一合同、macOS + SQLite 垂直闭环、Linux 真实挂载验证 | 已完成 | +| Phase 1 | SQLite 本地开发者预览：macOS + Linux 可安装、可恢复、可跑真实项目 | 进行中 | +| Phase 2 | PostgreSQL 共享工作区：跨电脑、多用户、数据库权限和并发 | 未开始 | +| Phase 3 | Windows WinFsp 和 macOS/Linux/Windows 跨系统预览 | 未开始 | +| Phase 4 | DuckDB 本地分析工作区 Beta | 未开始 | +| Phase 5 | VexDB-Lite Files v1 稳定版 | 未开始 | + +阶段不能只因代码合并或单个 demo 通过而完成。必须达到路线图中对应的干净机器安装、真机、一致性、故障、备份和规模验收条件。 + +## 22. 成功指标 + +### 22.1 产品通过条件 + +在继续投入完整三端 v1 前，必须满足： + +1. 至少三个真实流程需要业务数据和文件原子提交； +2. 真实模型可以在 macOS FSKit 挂载目录中完成任务，不需要为工具逐个适配； +3. 并发修改不会静默丢失； +4. 权限、配额和命令失败不会产生部分提交； +5. 数据库原生备份恢复后状态一致； +6. 10 万文件规模下达到正确性和性能目标； +7. 历史空间可以通过 retention 和 GC 控制。 + +### 22.2 不适用判断 + +- 只有单机源码编辑需求：优先使用 Git 和普通文件系统； +- 主要保存超大视频、模型或数据湖对象：优先使用对象存储加数据库元数据； +- 不需要事务、共享、权限、版本或审计：VexFS 价值不足。 + +## 23. 主要风险与应对 + +| 风险 | 影响 | 应对 | +|---|---|---| +| 范围过大 | 三端长期无法交付 | 用 MVP Gate 隔离首个闭环；SQLite 技术验证，PG 产品验证，DuckDB 后接 | +| POSIX 范围无限扩大 | 无法交付稳定挂载 | MVP 只做普通文件和目录最小调用集；其余能力逐项用真实程序数据决定 | +| gateway 变成权威服务 | 数据和数据库事务失去控制 | 只调用公开数据库合同，不保存最终状态，SQL 独立可用 | +| open 到 close 长事务 | 锁、膨胀和连接耗尽 | handle staging 使用短事务；publish/synchronize 幂等发布，close/reclaim 只做可恢复兜底 | +| 多条 Bash 命令被误认为一个事务 | 多文件任务出现部分结果 | 明确每操作事务；多文件使用 changeset/txn run | +| 缓存返回旧权限或旧内容 | 越权和覆盖 | MVP 不启用 FSKit kernel data cache；后续缓存必须带版本校验和主动失效 | +| FSKit extension 未启用 | 用户无法挂载 | 首次启动引导到系统设置，doctor 显示准确状态，CLI/worktree 可回退 | +| FSKit API 快速变化 | 新 SDK 编译失败或行为变化 | 独立 macOS adapter；MVP 使用稳定 SDK API；新版 Handler 接口通过兼容层迁移 | +| 签名、entitlement 或分发路径错误 | App 能构建但 extension 无法安装或加载 | Phase 1 验证 Developer ID 或 App Store 路径；CI 检查签名、entitlement 和系统信任状态，并在干净 macOS 环境做安装回归 | +| gateway 崩溃留下暂存对象 | 空间泄露或半文件 | 未发布状态不可见、租约/句柄 id、重挂载清理、GC | +| 临时文件泄露 | 敏感内容暴露 | 私有目录、最小导出、无凭证、清理提示、可选加密盘 | +| 长任务并发冲突 | 模型工作无法直接提交 | 不长期锁库，提交时检测并保留本地差异 | +| 大文件放大 WAL 和备份 | 成本不可控 | 默认 100 MiB 单文件、10 GiB 工作区、分块和配额 | +| 历史无限增长 | 数据库膨胀 | retention、快照保护、分批 GC 和空间统计 | +| 三端权限不对等 | 宣传和行为不一致 | 统一权限合同，明确 PostgreSQL 完整、嵌入式应用内 | +| arbitrary exec 被误认为沙箱 | 主机安全风险 | 明确进程沙箱不在 VexFS 范围，交给 Agent 环境 | +| 数据库备份职责混淆 | 灾难恢复失败 | 四层保护模型和真实 restore 演练 | +| CLI 与 SQL 漂移 | 行为不一致 | CLI 合同测试必须对照 SQL 结果 | +| gateway 与扩展版本不兼容 | 写入协议错误或数据损坏 | 挂载前能力握手；不兼容时拒绝可写挂载并给出升级步骤 | +| DuckDB 多进程写入受限 | mount 与应用互相阻塞或失败 | DuckDB 先做 SQL/CLI/worktree；可写 mount 默认要求 gateway 是唯一写进程 | + +## 24. 已确定决策 + +- 产品名暂定 VexFS； +- 产品属于 VexDB-Lite 仓库和品牌； +- VexFS 是数据库 extension； +- 数据库管理 VexFS； +- VexFS 只管理数据库中的文件； +- SQL 是完整权威合同； +- “安装后无缝 Bash”是第一产品目标； +- 产品 P0 与首个 MVP Gate 分开管理，P0 能力不要求全部进入 SQLite 首个闭环； +- mount gateway 是第一等入口，但不是权威核心； +- macOS FSKit mount 是 MVP 和 P0 主入口； +- 可挂载 MVP 支持 macOS 26.0+ Apple Silicon，使用 FSUnaryFileSystem、FSVolume 稳定接口和 FSKit V2 的 FSPathURLResource； +- MVP Gate 不启用 FSKit kernel data cache，先保证权限与正确性； +- mount 采用 close-to-open 一致性，不承诺完整跨 gateway POSIX 强一致； +- 平台无关 publish/sync 是可报告的幂等发布边界；macOS 由 synchronize 映射，close/reclaim 只做可恢复兜底； +- 一个 mount 绑定一个数据库 principal，mount point 默认当前用户私有； +- mount 的单个文件操作分别使用短数据库事务； +- 多文件和业务 SQL 原子性通过显式 changeset/事务提供； +- CLI 是配套工具，不保存权威状态； +- 常用命令直接映射 SQL； +- 任意普通终端程序优先通过挂载目录运行； +- worktree 是不能挂载时的回退，不是权威存储或备份； +- mount gateway 不建立新网络协议，不提供 SDK，不管理数据库； +- 通过 SQL 调用的文件修改参加调用者当前数据库事务；mount 操作使用 gateway 短事务； +- 权限来自数据库可证明身份； +- 使用文件版本、快照、数据库原生备份和逻辑导出四层保护； +- 数据库原生备份必须包含 VexFS； +- 技术原型已完成 macOS + SQLite，并已跑通 Linux + SQLite libfuse3 预览； +- 当前优先收口 macOS/Linux + SQLite 本地开发者预览，然后用 PostgreSQL 验证跨电脑共享； +- Windows 使用 WinFsp 接入同一 mount runtime； +- DuckDB 在产品价值验证后适配，默认先做 SQL/CLI/worktree，可写 mount 不与外部写进程并存； +- 不做内容检索层。 + +## 25. 技术评审前需要确定 + +这些问题不改变产品需求，但必须在对应阶段开始前确定： + +1. VexFS 编入现有 `vexdb_lite` 扩展，还是同仓库独立扩展并随同发布； +2. SQLite 和 DuckDB principal 的宿主注入接口； +3. changeset 大内容分块暂存的三端 SQL 形式； +4. 逻辑导出容器格式和版本升级策略； +5. Phase 1 是否启用 FSKit DataCacheHandler、只读缓存、主动失效和 mmap；当前正确性基线不启用 kernel data cache； +6. PostgreSQL 内容存储使用普通 BLOB/TOAST、Large Object 或分块表的最终选择； +7. 首轮性能参考机和自动化基准环境； +8. hardlink 在 macOS、Linux、Windows 上的最终降级和跨系统导出规则； +9. mount gateway 使用独立连接池还是每 handle 短连接，以及连接上限； +10. SQLite P0 暂存对象和 crash 清理的具体表结构； +11. FSKit `Operations` 到新版 `Handler` API 的兼容层和最低 Xcode 版本； +12. dirty staging 的保留时间、手动恢复命令和自动回收门槛； +13. SQLite gateway 每次 write 的最大批量、连接 busy_timeout 和背压上限； +14. PostgreSQL 多 gateway 场景采用通知、轮询还是租约来做缓存失效与 flock； +15. FSKit extension、签名 CLI、security-scoped 数据库目录和 `.vexfs-volume.json` 的升级方案； +16. Finder 产生的 `.DS_Store`、xattr、资源 fork 和文件协调调用在技术预览中的支持边界。 + +## 26. 相关文档 + +- 最终目标与阶段路线图：`docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md` +- 技术设计：`docs/design/2026-07-16_agent-files-universal-filesystem.md` +- 可行性判断：`docs/analysis/2026-07-16_vexfs-requirement-feasibility.md` +- 文章研究：`docs/research/2026-07-16_agent-filesystem-article-research.md` +- 输入文章：`/Users/Four/.codex/attachments/90dd9c19-5911-4cbb-8751-e71298816b37/pasted-text.txt` + +## 27. 安装与首次使用体验 + +### 27.1 用户需要安装什么 + +macOS 用户安装一个签名并公证的 VexDB-Lite 发行包，包内包含四个产物： + +1. `VexDB Lite.app`； +2. App 内的 FSKit File System Extension，它承担首个 mount gateway； +3. SQLite VexFS 数据库扩展； +4. `vexdb` CLI，以及指向 `vexdb fs` 的 `vexfs` 兼容入口。 + +macOS 本地预览首先支持 macOS 26.0+ Apple Silicon。Phase 1 必须证明至少一条真实分发路径：优先 Developer ID 签名与公证，如果 FSKit entitlement 或系统策略不允许，则使用 Mac App Store/TestFlight。首次打开 App 后，用户需要在“系统设置 → 通用 → 登录项与扩展 → 文件系统扩展”中启用 VexFS；这是系统安全步骤，产品不能静默绕过。App 必须显示准确引导，`vexfs doctor` 必须检测是否已经启用。 + +默认 SQLite 数据库放在 `~/Library/Application Support/VexDB-Lite/default.sqlite3`。每个 mount 把数据库所在的独立 0700 资源目录作为 security-scoped `FSPathURLResource` 交给系统；目录内的非权威 `.vexfs-volume.json` 只保存合同版本、数据库文件名和 workspace，不保存密码或绝对路径。extension 必须拒绝 `.`、`..`、子路径和任何逃逸该目录的文件名。一个数据库文件独占一个父目录，目录只放 descriptor、数据库、WAL 和 SHM。 + +CLI 与 FSKit extension 必须在挂载前读取数据库扩展的合同版本和能力位。主版本不兼容时拒绝可写挂载；卸载时先停止新操作、执行 synchronize、报告仍未发布的 handle，再执行 unmount。安装、升级和卸载都不得删除数据库中的已提交文件。 + +运行时边界保持不变：数据库扩展保存全部权威状态；mount gateway 只把数据库文件展示为本地路径；CLI 管理连接和挂载。gateway 不保存最终文件，不管理数据库。 + +### 27.2 数据库初始化 + +```sql +-- PostgreSQL +CREATE EXTENSION vexdb_lite; +SELECT vexfs_init(); + +-- DuckDB +LOAD vexdb_lite; +SELECT vexfs_init(); + +-- SQLite +.load ./vexdb_lite +SELECT vexfs_init(); +``` + +最终扩展名由技术评审决定，但三端必须提供一致的初始化和能力检查。 + +### 27.3 一次性配置并挂载 + +```bash +vexdb fs --workspace agent-workspace setup \ + --mount ~/VexDB/agent-workspace +``` + +未传 `--db` 时使用 `~/Library/Application Support/VexDB-Lite/default.sqlite3`。`setup` 初始化 SQLite 和 workspace；传入 `--mount` 时再检查 macOS 和 FSKit、App/extension、签名和合同版本，创建私有 mount point 并请求系统挂载。用户没有启用 extension 时必须给出准确的系统设置位置，不能假装挂载成功。 + +context 配置不能保存明文密码。认证使用数据库支持的密码文件、环境变量、系统凭证或交互提示。需要登录后自动挂载时，用户显式增加 `--persist`。 + +### 27.4 日常 Bash 使用 + +```bash +cd ~/VexFS/agent-workspace +ls +rg 'TODO' . +python scripts/generate.py +git status +``` + +这是真实挂载目录，不是 checkout 副本。程序通过普通文件 API 读写；FSKit +`synchronize` 映射到平台无关 sync。macOS mount 的专用 SQLite 连接使用 +`WAL + NORMAL staging + FULL publish`：普通 write 返回时内容仍未发布，sync 成功时 +完整版本已经发布并达到 FULL。这个策略不修改其他业务 SQLite 连接，`doctor` 必须显示 +实际配置和风险。`closeItem/reclaimItem` 只负责句柄结束和可恢复兜底。另一个 handle +在重新 open 后看到已发布版本；MVP 不承诺多个 gateway 实时同步。 + +常用生命周期命令： + +- `vexfs mount ~/VexFS/agent-workspace`：挂载当前 workspace； +- `vexfs mount status`：查看连接、principal、workspace 和待处理 handle； +- `vexfs unmount ~/VexFS/agent-workspace`：安全卸载； +- `vexfs doctor`：检查 App、FSKit extension、签名、SQLite 扩展、连接、权限和版本。 + +### 27.5 Agent 使用 + +Agent 的默认工作目录直接设置为 mount point： + +```bash +cd ~/VexFS/agent-workspace +python scripts/generate.py +rg 'ERROR' logs/ +``` + +无需让模型学习 VexFS 专用文件命令。Agent 运行环境仍负责进程沙箱，VexFS 只限制挂载目录内的数据库文件权限。 + +如果一个任务要求“多个文件要么全部成功，要么全部失败”，使用显式事务入口，而不是连续执行普通 Bash 命令： + +```bash +vexfs txn run --workspace agent-workspace -- \ + bash -lc "python scripts/generate.py && rg 'ERROR' logs/" +``` + +`txn run` 使用隔离工作视图，在命令成功后把差异作为单个 changeset 提交；该能力为 P1。P0 中每个文件系统操作分别原子，多个命令不自动合成一个事务。 + +### 27.6 “无缝”的准确含义 + +VexFS 承诺： + +- macOS 完成一次 FSKit extension 启用和 setup 后，workspace 是可以直接 `cd` 进入的真实路径； +- Bash、ls、cat、rg、sed、python、git、编译器和 apply_patch 直接可用； +- 不需要为每个系统命令开发 VexFS 版本； +- 每个文件操作仍由数据库统一处理事务、权限、版本、配额和审计； +- gateway 停止不会使数据库中的已提交文件失效。 + +VexFS 不承诺： + +- MVP 覆盖 Linux、Windows、macOS 26.0 以下版本、普通容器和不能启用 FSKit extension 环境的真实挂载； +- 把多条 Bash 命令自动变成一个数据库事务； +- 支持设备文件、socket、FIFO、setuid 或全部 POSIX 特殊行为； +- 通过 mount point 获得高于数据库 principal 的权限。 + +不能挂载时，用户使用 `vexfs` 直接命令或 `vexfs shell/exec` worktree 回退。回退不改变数据库权威地位，也不作为安装后无缝 Bash 的主宣传路径。 + +## 28. 参考资料 + +- [PostgreSQL Large Objects](https://www.postgresql.org/docs/current/lo-intro.html) +- [PostgreSQL Backup and Restore](https://www.postgresql.org/docs/current/backup.html) +- [SQLite Limits](https://www.sqlite.org/limits.html) +- [SQLite Online Backup API](https://www.sqlite.org/backup.html) +- [SQLite Atomic Commit](https://www.sqlite.org/atomiccommit.html) +- [DuckDB Transactions](https://duckdb.org/docs/current/sql/statements/transactions) +- [DuckDB Concurrency](https://duckdb.org/docs/current/connect/concurrency) +- [DuckDB Securing DuckDB](https://duckdb.org/docs/current/operations_manual/securing_duckdb/overview) +- [Apple FSKit](https://developer.apple.com/documentation/fskit) +- [Apple Building a passthrough file system](https://developer.apple.com/documentation/fskit/building-a-passthrough-file-system) +- [Apple FSKit updates](https://developer.apple.com/documentation/updates/fskit) +- [WinFsp Documentation](https://winfsp.dev/doc/) +- [libfuse fuse_operations](https://libfuse.github.io/doxygen/structfuse__operations.html) +- [libfuse low-level operations](https://libfuse.github.io/doxygen/structfuse__lowlevel__ops.html) +- [libfuse FAQ](https://github.com/libfuse/libfuse/wiki/FAQ) +- [Linux FUSE I/O modes](https://docs.kernel.org/filesystems/fuse-io.html) diff --git a/scripts/release.sh b/scripts/release.sh index 60a02e0806..7f482af0c0 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -4,15 +4,17 @@ # 用法: # bash scripts/release.sh build [target] [arch] # 远程 build + 拉回 dist/ (默认动作) # bash scripts/release.sh package # 把 dist// 打成 tarball + SHA256SUMS +# bash scripts/release.sh test [target] [arch] # 原生 Linux 验证（target: duck|sqlite|all） # bash scripts/release.sh upload # 上传 tarballs 到 GitHub Release # -# build target: duck | pg | all (default: all) +# build target: duck | pg | sqlite | all (default: all) # build arch: x86 | arm | all (default: all) # # 示例: # bash scripts/release.sh # build 全量 (4 个产物) # bash scripts/release.sh build duck arm # 只 build ARM DuckDB # bash scripts/release.sh package # 4 个 tarball 进 dist/release/ +# bash scripts/release.sh test sqlite all # 两个架构安装并验证 SQLite + VexFS 包 # bash scripts/release.sh upload v0.1.0 # 推送到 GitHub Release # # 凭证: 通过 SERVERS_FILE 环境变量指定凭证文件，或直接覆盖以下 *_HOST/*_USER/*_PASS。 @@ -35,6 +37,15 @@ REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" # nested duckdb/ wrapper retired 2026-05-13 — repo root is now flat. PG_VEXDB_ROOT="${PG_VEXDB_ROOT:-$REPO_ROOT}" DIST_DIR="$REPO_ROOT/dist" +VEXDB_LITE_RELEASE_VERSION="${VEXDB_LITE_RELEASE_VERSION:-0.1.0-dev}" +SOURCE_COMMIT="$(git -C "$REPO_ROOT" rev-parse --short HEAD 2>/dev/null || echo unknown)" +SOURCE_DIRTY=false +[[ -n "$(git -C "$REPO_ROOT" status --porcelain 2>/dev/null || true)" ]] && SOURCE_DIRTY=true +SOURCE_REVISION="$SOURCE_COMMIT" +[[ "$SOURCE_DIRTY" == true ]] && SOURCE_REVISION="${SOURCE_REVISION}-dirty" +VEXFS_RUNTIME_HEADER="$REPO_ROOT/agent_files/mount/common/include/vexfs_runtime_types.h" +VEXFS_RUNTIME_ABI="$(sed -n 's/^#define VEXFS_RUNTIME_ABI_VERSION \([0-9][0-9]*\)u$/\1/p' "$VEXFS_RUNTIME_HEADER")" +[[ -n "$VEXFS_RUNTIME_ABI" ]] || { echo "无法读取 VexFS runtime ABI" >&2; exit 1; } # 远程 GitHub 不可达，cmake / DuckDB src 必须本地下载后 scp CMAKE_VER="3.28.6" @@ -72,7 +83,7 @@ ARM_BOOST="${ARM_BOOST:-/path/to/boost}" CMD="${1:-build}" case "$CMD" in build|package|upload|test) shift ;; - duck|pg|all) CMD="build" ;; + duck|pg|sqlite|all) CMD="build" ;; *) CMD="build" ;; esac TARGET="${1:-all}" @@ -101,6 +112,12 @@ ok() { printf '%s[release]%s %s\n' "$GRN" "$NC" "$*"; } fail() { printf '%s[release]%s %s\n' "$RED" "$NC" "$*" >&2; exit 1; } section() { printf '\n%s━━━ %s ━━━%s\n' "$BLU" "$*" "$NC"; } +require_clean_release_source() { + if [[ "$SOURCE_DIRTY" == true && "${VEXDB_LITE_ALLOW_DIRTY_RELEASE:-0}" != 1 ]]; then + fail "拒绝从脏工作树生成发行物；仅本地验证可设置 VEXDB_LITE_ALLOW_DIRTY_RELEASE=1" + fi +} + load_credentials() { local arch="${1:-all}" # 只加载用到的架构凭证 local f="${SERVERS_FILE:-}" @@ -162,12 +179,12 @@ rrsync_up() { prepare_local() { section "本地准备" for cmd in sshpass scp curl tar; do - command -v "$cmd" >/dev/null || fail "缺少命令: $cmd（brew install $cmd）" + command -v "$cmd" >/dev/null || fail "缺少命令: ${cmd}（brew install ${cmd}）" done [[ -d "$REPO_ROOT" ]] || fail "找不到 vexdb_lite 仓库: $REPO_ROOT" - load_credentials + load_credentials "$ARCH" mkdir -p "$DIST_DIR/x86_64-linux" "$DIST_DIR/aarch64-linux" [[ -f "$CMAKE_TARBALL" ]] || { @@ -616,6 +633,102 @@ validate_pg() { ok "$arch vexdb_lite.so: $(ls -lh "$outdir/vexdb_lite.so" | awk '{print $5}')" } +# SQLite + VexFS Linux 发行物。和 DuckDB/PG 一样在 manylinux_2_28 容器中构建， +# 避免绑定远程构建机的 glibc/libstdc++。fuse3-devel 只用于编译；用户机器运行时 +# 需要系统提供 libfuse3.so.3、fusermount3 和 /dev/fuse。 +build_sqlite_files() { + local arch=$1 image docker_pfx boost outarch outdir host_uid host_gid remote_home + if [[ "$arch" == "x86" ]]; then + image="quay.io/pypa/manylinux_2_28_x86_64" + # 不把 sudo 密码拼进远程命令。构建账号必须能直接运行 Docker，或只对 + # Docker 配置免交互 sudo；否则这里明确失败。 + docker_pfx="sudo -n docker" + boost="$X86_BOOST" + outarch=x86_64 + else + image="quay.io/pypa/manylinux_2_28_aarch64" + docker_pfx="docker" + boost="$ARM_BOOST" + outarch=aarch64 + fi + outdir="$DIST_DIR/${outarch}-linux/sqlite-files" + mkdir -p "$outdir/bin" "$outdir/lib" + + section "$arch build SQLite + VexFS Files (manylinux container)" + rssh "$arch" "$docker_pfx image inspect $image >/dev/null 2>&1" \ + || fail "$arch manylinux 镜像 $image 不在" + host_uid=$(rssh "$arch" 'id -u') + host_gid=$(rssh "$arch" 'id -g') + remote_home=$(rssh "$arch" 'printf %s "$HOME"') + + rssh "$arch" "$docker_pfx run --rm \ + -v \$HOME/$REMOTE_DIR/vexdb_lite:/work \ + -v $boost:/opt/boost:ro \ + -w /work \ + $image \ + bash -c 'set -e + export PATH=/opt/python/cp310-cp310/bin:\$PATH + dnf install -y fuse3-devel pkgconf-pkg-config >/dev/null + pip install --quiet -i https://mirrors.aliyun.com/pypi/simple cmake==3.29.6 2>/dev/null || \ + pip install --quiet cmake==3.29.6 + ${CLEAN_SQLITE_BUILD:+rm -rf build/sqlite-files-ml;} + cmake -S vexdb_sqlite -B build/sqlite-files-ml \ + -DCMAKE_BUILD_TYPE=Release -DVEXDB_SQLITE_BUILD_TESTS=OFF \ + -DVEXDB_BOOST_INC=/opt/boost \ + -DVEXDB_LITE_VERSION=$VEXDB_LITE_RELEASE_VERSION \ + -DVEXDB_SQLITE_GIT_HASH=$SOURCE_REVISION \ + -DCMAKE_EXE_LINKER_FLAGS=\"-static-libstdc++ -static-libgcc\" \ + -DCMAKE_SHARED_LINKER_FLAGS=\"-static-libstdc++ -static-libgcc\" + cmake --build build/sqlite-files-ml \ + --target vexdb_cli vexfs_fuse vexdb_lite_loadable -j\$(nproc) + build/sqlite-files-ml/vexdb --version + DB=\$(mktemp /tmp/vexfs-release-smoke.XXXXXX.sqlite3) + build/sqlite-files-ml/vexfs-fuse --db \$DB --workspace release --self-test + rm -f \$DB \$DB-wal \$DB-shm + chown -R $host_uid:$host_gid build/sqlite-files-ml + '" || fail "$arch SQLite + VexFS manylinux build 失败" + + local remote_build="$remote_home/$REMOTE_DIR/vexdb_lite/build/sqlite-files-ml" + for binary in vexdb vexfs-fuse vexdb_lite.so; do + rssh "$arch" "test -f '$remote_build/$binary'" \ + || fail "$arch SQLite + VexFS 缺少产物 $binary" + _assert_glibcxx_remote "$arch" "$remote_build/$binary" + done + rssh "$arch" "set -e + file '$remote_build/vexdb' | grep -q ELF + file '$remote_build/vexfs-fuse' | grep -q ELF + file '$remote_build/vexdb_lite.so' | grep -q ELF + ldd '$remote_build/vexfs-fuse' | grep -q 'libfuse3.so.3' + ! readelf -d '$remote_build/vexdb' | grep -E '(RPATH|RUNPATH)' >/dev/null + ! readelf -d '$remote_build/vexfs-fuse' | grep -E '(RPATH|RUNPATH)' >/dev/null + " || fail "$arch SQLite + VexFS ELF/依赖校验失败" + + rscp_down "$arch" "~/$REMOTE_DIR/vexdb_lite/build/sqlite-files-ml/vexdb" "$outdir/bin/vexdb" + rscp_down "$arch" "~/$REMOTE_DIR/vexdb_lite/build/sqlite-files-ml/vexfs-fuse" "$outdir/bin/vexfs-fuse" + rscp_down "$arch" "~/$REMOTE_DIR/vexdb_lite/build/sqlite-files-ml/vexdb_lite.so" "$outdir/lib/vexdb_lite.so" + chmod 0755 "$outdir/bin/vexdb" "$outdir/bin/vexfs-fuse" + ln -sfn vexdb "$outdir/bin/vexfs" + install -m 0755 "$REPO_ROOT/agent_files/linux/package/install.sh" "$outdir/install.sh" + install -m 0755 "$REPO_ROOT/agent_files/linux/package/uninstall.sh" "$outdir/uninstall.sh" + install -m 0644 "$REPO_ROOT/agent_files/linux/package/使用说明.md" "$outdir/使用说明.md" + { + echo "product=vexdb-lite" + echo "component=sqlite-files" + echo "version=$VEXDB_LITE_RELEASE_VERSION" + echo "architecture=$outarch" + echo "mount_abi=$VEXFS_RUNTIME_ABI" + echo "source_commit=$SOURCE_COMMIT" + echo "source_dirty=$SOURCE_DIRTY" + echo "build_environment=manylinux_2_28" + echo "runtime_dependency=fuse3" + } > "$outdir/MANIFEST.txt" + ( + cd "$outdir" + shasum -a 256 bin/vexdb bin/vexfs-fuse lib/vexdb_lite.so > SHA256SUMS.txt + ) + ok "$arch SQLite + VexFS Files 已拉回 $outdir" +} + # ============================================================ # Main # ============================================================ @@ -626,7 +739,7 @@ ARCHES=() cmd_build() { info "build 目标=$TARGET 架构=${ARCHES[*]}" - [[ "$TARGET" =~ ^(duck|pg|all)$ ]] || fail "target 必须为 duck/pg/all" + [[ "$TARGET" =~ ^(duck|pg|sqlite|all)$ ]] || fail "target 必须为 duck/pg/sqlite/all" [[ "$ARCH" =~ ^(x86|arm|all)$ ]] || fail "arch 必须为 x86/arm/all" prepare_local @@ -650,8 +763,15 @@ cmd_build() { done fi + # SQLite + VexFS：统一 CLI、SQLite loadable extension 和 libfuse3 helper。 + if [[ "$TARGET" == "sqlite" || "$TARGET" == "all" ]]; then + section "SQLite + VexFS Files" + for arch in "${ARCHES[@]}"; do build_sqlite_files "$arch"; done + fi + section "build 完成" - find "$DIST_DIR" -maxdepth 3 -type f \( -name "*.duckdb_extension" -o -name "vexdb_lite.so" \ + find "$DIST_DIR" -maxdepth 4 -type f \( -name "*.duckdb_extension" -o -name "vexdb_lite.so" \ + -o -name "vexdb_lite" -o -name "vexdb" -o -name "vexfs-fuse" \ -o -name "*.control" -o -name "*.sql" \) -exec ls -lh {} \; } @@ -661,6 +781,7 @@ cmd_build() { cmd_package() { section "Package" + require_clean_release_source # Pre-flight: 扫 dist 下所有 .duckdb_extension,校验 GLIBCXX 上限。 # 即使上游 build 漏了 MANYLINUX=1 / 产物从别处拉来 / validate 被跳过, @@ -741,6 +862,18 @@ cmd_package() { vexdb_lite.so.debug fi done + + # SQLite + VexFS：统一 CLI、libfuse3 helper、loadable extension 和普通用户安装说明。 + local filesdir="$archdir/sqlite-files" + if [[ -x "$filesdir/bin/vexdb" && -x "$filesdir/bin/vexfs-fuse" && + -f "$filesdir/lib/vexdb_lite.so" ]]; then + ( + cd "$filesdir" + shasum -a 256 -c SHA256SUMS.txt >/dev/null + ) || fail "$arch SQLite + VexFS stage 哈希校验失败" + pkg_tar "$RELEASE_DIR/vexdb-lite-sqlite-files-linux-${arch}.tar.gz" \ + "$filesdir" bin lib install.sh uninstall.sh 使用说明.md MANIFEST.txt SHA256SUMS.txt + fi done # 防回归: 每个 tarball 都扫一遍,有 AppleDouble 直接 fail。 @@ -789,7 +922,15 @@ with tarfile.open(sys.argv[1]) as t: # 进制可用。 cmd_test() { - local arch_arg="${1:-all}" + # 兼容旧用法 `test x86`（默认跑 DuckDB），同时支持 + # `test sqlite x86` 和 `test all all`。 + local test_target="duck" arch_arg="all" + if [[ "${1:-}" =~ ^(duck|sqlite|all)$ ]]; then + test_target="$1" + arch_arg="${2:-all}" + else + arch_arg="${1:-all}" + fi [[ "$arch_arg" =~ ^(x86|arm|all)$ ]] || fail "test arch 必须为 x86/arm/all" load_credentials "$arch_arg" @@ -798,14 +939,67 @@ cmd_test() { [[ "$arch_arg" == "all" || "$arch_arg" == "x86" ]] && arches+=("x86") [[ "$arch_arg" == "all" || "$arch_arg" == "arm" ]] && arches+=("arm") - # 每个 DuckDB 版本各跑一遍 spec（用对应版本的 build/duck//build 树）。 - for dver in $DUCKDB_VERSIONS; do + if [[ "$test_target" == "duck" || "$test_target" == "all" ]]; then + # 每个 DuckDB 版本各跑一遍 spec（用对应版本的 build/duck//build 树）。 + for dver in $DUCKDB_VERSIONS; do + for arch in "${arches[@]}"; do + section "$arch spec test $dver (manylinux container)" + _spec_test_manylinux "$arch" "$dver" || fail "$arch $dver spec test 失败" + done + done + ok "DuckDB spec test 全过" + fi + + if [[ "$test_target" == "sqlite" || "$test_target" == "all" ]]; then for arch in "${arches[@]}"; do - section "$arch spec test $dver (manylinux container)" - _spec_test_manylinux "$arch" "$dver" || fail "$arch $dver spec test 失败" + section "$arch SQLite + VexFS 安装包测试（原生 Linux）" + _sqlite_files_package_test "$arch" || fail "$arch SQLite + VexFS 安装包测试失败" done - done - ok "spec test 全过" + ok "SQLite + VexFS 安装包测试全过" + fi +} + +_sqlite_files_package_test() { + local arch=$1 outarch pkg pkg_name + [[ "$arch" == "x86" ]] && outarch="x86_64" || outarch="aarch64" + pkg="$RELEASE_DIR/vexdb-lite-sqlite-files-linux-${outarch}.tar.gz" + pkg_name="$(basename "$pkg")" + [[ -f "$pkg" ]] || fail "找不到 $pkg；请先运行 package" + + # 上传到对应架构的真实 Linux 机器，在全新的 HOME 中走完整安装和卸载流程。 + rscp_up "$arch" "$pkg" + rssh "$arch" "set -euo pipefail + package=\"\$HOME/$pkg_name\" + tmp=\$(mktemp -d \"\$HOME/vexdb-sqlite-files-package-test.XXXXXX\") + cleanup() { rm -rf \"\$tmp\" \"\$package\"; } + trap cleanup EXIT + mkdir -p \"\$tmp/package\" \"\$tmp/home\" \"\$tmp/data\" + tar xzf \"\$package\" -C \"\$tmp/package\" + + export HOME=\"\$tmp/home\" + export XDG_DATA_HOME=\"\$tmp/data\" + export VEXDB_LITE_BIN_DIR=\"\$HOME/.local/bin\" + export VEXDB_LITE_LIB_DIR=\"\$HOME/.local/lib/vexdb-lite\" + bash \"\$tmp/package/install.sh\" + + bin=\"\$VEXDB_LITE_BIN_DIR/vexdb\" + helper=\"\$VEXDB_LITE_BIN_DIR/vexfs-fuse\" + extension=\"\$VEXDB_LITE_LIB_DIR/vexdb_lite.so\" + db=\"\$XDG_DATA_HOME/package-test.sqlite3\" + \"\$bin\" --version + \"\$bin\" \"\$db\" 'SELECT sqlite_version(), vexdb_version();' + python3 -c 'import sqlite3,sys; db=sqlite3.connect(\":memory:\"); db.enable_load_extension(True); db.load_extension(sys.argv[1]); print(db.execute(\"SELECT vexdb_version()\").fetchone()[0])' \"\$extension\" + \"\$helper\" --db \"\$db\" --workspace package-test --self-test + \"\$bin\" fs --db \"\$db\" mkdir /package-test + printf 'package-smoke\\n' | \"\$bin\" fs --db \"\$db\" write /package-test/hello.txt + test \"\$(\"\$bin\" fs --db \"\$db\" cat /package-test/hello.txt)\" = package-smoke + + bash \"\$tmp/package/uninstall.sh\" + test ! -e \"\$bin\" + test ! -e \"\$helper\" + test ! -e \"\$extension\" + test -f \"\$db\" + " || return 1 } _spec_test_manylinux() { @@ -848,6 +1042,7 @@ _spec_test_manylinux() { cmd_upload() { local tag="${1:-}" + [[ "$SOURCE_DIRTY" == false ]] || fail "拒绝从脏工作树上传 GitHub Release" [[ -n "$tag" ]] || fail "upload 必须提供 tag: bash scripts/release.sh upload v0.0.1" command -v gh >/dev/null || fail "缺少 gh CLI (brew install gh)" [[ -d "$RELEASE_DIR" ]] || fail "$RELEASE_DIR 不存在；先跑 package" @@ -885,5 +1080,5 @@ case "$CMD" in package) cmd_package ;; upload) cmd_upload "$@" ;; test) cmd_test "$@" ;; - *) fail "unknown command: $CMD（build|package|upload|test）" ;; + *) fail "unknown command: ${CMD}（build|package|upload|test）" ;; esac diff --git a/tests/eval/vexfs/Dockerfile.linux-fuse b/tests/eval/vexfs/Dockerfile.linux-fuse new file mode 100644 index 0000000000..66654ae003 --- /dev/null +++ b/tests/eval/vexfs/Dockerfile.linux-fuse @@ -0,0 +1,14 @@ +FROM python:3.11-slim + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + build-essential \ + ca-certificates \ + cmake \ + fuse3 \ + git \ + libfuse3-dev \ + pkg-config \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /workspace diff --git a/tests/eval/vexfs/README.md b/tests/eval/vexfs/README.md index 7262bfe029..6ea0405447 100644 --- a/tests/eval/vexfs/README.md +++ b/tests/eval/vexfs/README.md @@ -12,15 +12,43 @@ bash build_sqlite.sh eval # 日常快速回归 bash build_sqlite.sh eval quick +# 指定已有构建目录（评测器会从这里查找 vexdb、扩展和测试产物） +python3 tests/eval/vexfs/run.py --mode quick --build-dir vexdb_sqlite/build + # 最大规模（128 MiB 文件、1 万随机操作、1 万小文件） bash build_sqlite.sh eval stress # 只跑某一类或某个用例 python3 tests/eval/vexfs/run.py --mode full --filter recovery python3 tests/eval/vexfs/run.py --mode full --filter performance.staged-overwrite +python3 tests/eval/vexfs/run.py --mode full --filter performance.mount-contract-small-files # 把性能预算也作为硬 Gate；默认只记录指标，避免不同机器误报 python3 tests/eval/vexfs/run.py --mode full --enforce-performance + +# 发版前重新打包和签名后，再显式运行交付包测试 +VEXDB_LITE_PACKAGE_STAGE=/absolute/path/to/stage \ + python3 tests/eval/vexfs/run.py --mode full --include-package --fail-on-skip + +# 真实挂载和跨平台 Gate 必须显式指定本次要验证的挂载 CLI；任何 SKIP 都失败 +python3 tests/eval/vexfs/run.py --mode full --filter mount \ + --mount-cli /absolute/path/to/vexfs --fail-on-skip + +# Linux AArch64/x86_64：在真实 /dev/fuse 上分别以 root 和 uid 1000 +# 运行与 macOS 完全相同的 mount 合同（需要 Docker Desktop 或 Docker Engine） +bash tests/eval/vexfs/run_linux_mount.sh + +# 用同一个 SQLite 文件完成 macOS FSKit → Linux FUSE → macOS FSKit 往返 +bash tests/eval/vexfs/run_cross_platform_portability.sh + +# 显式调用真实 OpenCode 模型，在挂载项目中自主改代码并跑测试 +# 默认不运行，避免日常回归自动消耗模型额度 +VEXFS_EVAL_OPENCODE=1 \ + python3 tests/eval/vexfs/run.py --mode quick --filter mount.real-opencode-project + +# 可选择模型，默认 openai/gpt-5.4-mini +VEXFS_EVAL_OPENCODE=1 VEXFS_EVAL_OPENCODE_MODEL=openai/gpt-5.4-mini \ + python3 tests/eval/vexfs/run.py --mode quick --filter mount.real-opencode-project ``` 报告生成在： @@ -29,24 +57,56 @@ python3 tests/eval/vexfs/run.py --mode full --enforce-performance vexdb_sqlite/build/eval/vexfs//report.json vexdb_sqlite/build/eval/vexfs//report.md vexdb_sqlite/build/eval/vexfs/latest.json +vexdb_sqlite/build/eval/vexfs-linux-mount/{root,uid-1000}//report.json ``` 随机状态机使用固定 seed，可通过 `--seed` 复现或扩充样本。 +评测结果有三种：`PASS` 表示所有选中用例都执行并通过；`PASS_WITH_SKIPS` +表示已执行用例通过，但有环境不满足的用例未执行；`FAIL` 表示至少一个用例失败， +或在 `--fail-on-skip` 下出现了 `SKIP`。没有匹配到任何用例会直接以非零状态退出。 + ## 覆盖范围 - 现有 Gate：SQL 静态注册、C ABI、CLI smoke、全部 SQLite YAML spec。 - 文件合同：二进制/Unicode、元数据、路径校验、rename replace、递归删除、公开版本历史和指定版本读取。 - 版本恢复：SQL/C ABI/CLI restore、expected-version 冲突、事务回滚、dry-run 不写库、文本 diff 和末尾换行差异。 - 事务：commit、rollback、savepoint、读者快照、目录 verifier。 -- 句柄：flags、稀疏写、truncate、幂等指纹、同步、乐观写冲突。 -- 恢复：进程中途退出、WAL 重开、integrity check、旧 schema 迁移、retained reclaim。 +- 句柄：flags、稀疏写、truncate、幂等指纹、同步、乐观写冲突；幂等结果至少 + 保留最近 65,536 条，并以 4,096 条为一批清理更早的完成记录。 +- 时间戳和生命周期：birth/access/modify/change、`utimens`、并发原子 append、 + `flock`/`fcntl`、打开文件 rename/unlink、普通/强制卸载。 +- 恢复：进程中途退出、WAL 重开、integrity check、retained reclaim； + Linux helper 被 `SIGKILL` 后重挂载会自动发布保留的暂存写入。 +- 长期校验：版本 SHA-256、单文件恢复和 workspace 恢复别名、commit/snapshot/history/dentry + 引用、staging 缺失、同长度内容损坏注入、损坏读取拒绝、CLI 退出码 8；性能用例分别记录 + quick 元数据检查和 64 KiB 流式 deep hash 的吞吐与 RSS 增长。 +- 数据库版本：当前尚未发版，只支持当前 schema，不运行旧 schema 迁移测试。 - 并发：多个真实进程写入、数据库锁和 busy timeout。 - 备份：运行中的 SQLite online backup、恢复、未发布 staging 可见性。 - 随机模型：真实数据库状态与 Python 参考文件树持续比对。 -- 性能：大量小文件、大文件顺序读写、分块 staging、随机 4 KiB 覆盖、备份吞吐。 -- 空间：版本历史增长、幂等请求行数、checkpoint/VACUUM 前后的 DB/WAL 体积。 -- 平台：FSKit App/extension 无签名编译；扩展已启用时执行真实 mount 和 bash 命令。 +- 性能：大量小文件、大文件顺序读写、分块 staging、随机 4 KiB 覆盖、备份吞吐； + `performance.mount-contract-small-files` 额外模拟原子创建、macOS provenance xattr、 + 内容发布和最终关闭，检查每文件 commit/version/request 写放大、FULL 屏障和峰值 RSS。 +- 文本搜索：`vexdb fs grep` 的 UTF-8 字面匹配、二进制跳过、大小写、文件名/行号输出、 + 短查询回退；可选 FTS5 trigram 索引的启用、维护、快照恢复和候选文件性能。 +- 规模与稳定性：数据库直连和真实 mount 分别覆盖 1 千/1 万/10 万文件；混合负载 + 按 quick/full/stress 持续 5/60/900 秒，期间反复重开、快照和 checkpoint。 +- 真实项目：Python、Node.js、Go、Rust 与 Git 在挂载盘中构建和测试，并在重挂载后 + 重跑；`mount.real-opencode-project` 显式启用后由真实 OpenCode 自主改代码和跑测试。 +- 空间：版本历史增长、幂等请求保留水位、规模树的提交/版本/ACL/xattr 写放大、 + checkpoint/VACUUM 前后的 DB/WAL 体积。 +- 平台：Linux/libfuse3 helper 独立 C ABI smoke，并在有 `/dev/fuse` 时执行真实 + mount、普通用户 uid/gid、Bash、可执行脚本、hardlink/symlink、Git 提交、卸载和 + 重挂载；Windows/WinFsp 边界独立编译和状态 smoke；FSKit App/extension 无签名 + 编译，扩展已启用时执行真实 mount 和 bash 命令。 +- 共用挂载合同：`mount.cross-platform-conformance` 不按平台分叉测试步骤；在 + macOS FSKit 与 Linux libfuse3 上统一验证独占创建、范围读写、fsync、append、 + truncate、rename replace、mode/执行、hardlink、symlink、xattr、Unicode、标准 + errno、snapshot restore、卸载与重挂载。新平台 adapter 必须复用这套用例。 +- 跨系统往返：`portability.cross-os-roundtrip` 由专用脚本分三段执行，验证同一个 + SQLite 工作区在 macOS 和 Linux 之间保留内容、Unicode、mode、可执行文件、链接、 + xattr、ACL、数字 owner、历史和双向快照恢复。 ## 模式规模 @@ -54,11 +114,27 @@ vexdb_sqlite/build/eval/vexfs/latest.json |---|---:|---:|---:| | 随机模型操作 | 300 | 2,000 | 10,000 | | 小文件 | 250 | 3,000 | 10,000 | +| 挂载合同小文件 | 250 | 1,000 | 1,000 | | 顺序文件 | 8 MiB | 100 MiB | 128 MiB | | staging 文件 | 8 MiB | 100 MiB | 128 MiB | | 随机 4 KiB 写 | 100 | 1,000 | 5,000 | | 并发进程 | 4 | 8 | 8 | +| 规模目录文件数 | 1,000 | 10,000 | 100,000 | +| 混合重开稳定性 | 5 秒 | 60 秒 | 900 秒 | + +`performance.scale-tree` 还会记录无索引数据库 grep、trigram 索引构建和索引搜索。 +quick/full/stress 的峰值 RSS 保护线分别为 1 GiB、1.5 GiB、2 GiB；SQLite 临时表使用 +磁盘，页缓存限制为 64 MiB。越过保护线会立即失败，不继续扩大测试规模。 `mount.real-bash` 是环境 Gate。macOS、FSKit 和已启用的 VexFS 系统扩展都满足时， 它会真实挂载并运行 `mkdir`、`cat`、`grep`、`cp`、`mv`、`find`、`rm` 等命令； 条件不满足时报告会明确写 `SKIP` 和原因，不会把未执行伪装成通过。 + +`mount.real-linux-bash-git` 是 Linux 环境 Gate。构建中存在 `vexfs-fuse`，且 +`/dev/fuse`、`fusermount3` 和 Git 可用时，它会以真实挂载目录验证完整 Git +workspace，并在卸载和重挂载后再次运行 `git status`。 + +`mount.cross-platform-conformance` 是平台一致性 Gate。Linux 可通过 +`run_linux_mount.sh` 自动准备 libfuse3 环境并验证 root/普通用户；macOS 必须先在 +“系统设置 → 通用 → 登录项与扩展 → 文件系统扩展”中启用 VexDB Lite，再运行该 +用例。缺少真实挂载条件时结果是 `SKIP`，不会用 C ABI smoke 冒充平台通过。 diff --git a/tests/eval/vexfs/documentation_smoke.sh b/tests/eval/vexfs/documentation_smoke.sh new file mode 100755 index 0000000000..6e6506068f --- /dev/null +++ b/tests/eval/vexfs/documentation_smoke.sh @@ -0,0 +1,273 @@ +#!/bin/bash +set -euo pipefail + +STAGE="${1:?usage: documentation_smoke.sh /path/to/package-stage}" +STAGE="$(cd "$STAGE" && pwd)" +TMP_DIR="$(mktemp -d -t vexdb-documentation-smoke)" +TEST_HOME="$TMP_DIR/home" +APP_DIR="$TEST_HOME/Applications" +BIN_DIR="$TEST_HOME/.local/bin" +LIB_DIR="$TEST_HOME/.local/lib/vexdb-lite" +CHECKS=0 +PACKAGE_VERSION="$(sed -n 's/^preview_version=//p' "$STAGE/MANIFEST.txt")" +[ -n "$PACKAGE_VERSION" ] || { echo "DOCUMENTATION SMOKE: FAIL: manifest has no preview_version" >&2; exit 1; } +MANIFEST_RUNTIME_ABI="$(sed -n 's/^mount_abi_version=//p' "$STAGE/MANIFEST.txt")" +[ -n "$MANIFEST_RUNTIME_ABI" ] || { echo "DOCUMENTATION SMOKE: FAIL: manifest has no runtime ABI" >&2; exit 1; } +PACKAGE_SIGNATURE="$(sed -n 's/^signature=//p' "$STAGE/MANIFEST.txt")" +SOURCE_DIRTY="$(sed -n 's/^source_dirty=//p' "$STAGE/MANIFEST.txt")" +ALLOW_ADHOC=0 +[ "$PACKAGE_SIGNATURE" != ad-hoc ] || ALLOW_ADHOC=1 +ALLOW_DIRTY=0 +if [ "$SOURCE_DIRTY" = true ] && [ "${VEXDB_LITE_ALLOW_DIRTY_PACKAGE_TEST:-0}" = 1 ]; then + ALLOW_DIRTY=1 +fi +EXPECTED_PRODUCT_VERSION="vexdb-lite $PACKAGE_VERSION" + +cleanup() { + rm -rf "$TMP_DIR" +} +trap cleanup EXIT + +fail() { + echo "DOCUMENTATION SMOKE: FAIL: $*" >&2 + exit 1 +} + +check() { + CHECKS=$((CHECKS + 1)) + "$@" || fail "check $CHECKS failed: $*" +} + +equal() { + CHECKS=$((CHECKS + 1)) + [ "$1" = "$2" ] || fail "check $CHECKS expected '$1', got '$2'${3:+ ($3)}" +} + +contains() { + CHECKS=$((CHECKS + 1)) + case "$1" in + *"$2"*) ;; + *) fail "check $CHECKS did not find '$2'${3:+ ($3)}" ;; + esac +} + +expect_fail() { + CHECKS=$((CHECKS + 1)) + if "$@" >/dev/null 2>&1; then + fail "check $CHECKS expected failure: $*" + fi +} + +mkdir -p "$TEST_HOME" + +# 交付物和使用说明。 +check test -d "$STAGE/VexDB Lite.app" +check test -x "$STAGE/bin/vexdb" +check test -L "$STAGE/bin/vexfs" +check test -f "$STAGE/lib/vexdb_lite.dylib" +check test -f "$STAGE/使用说明.md" +check grep -q 'Developer ID' "$STAGE/使用说明.md" +check grep -q '按类别' "$STAGE/使用说明.md" +check grep -q '文件系统扩展' "$STAGE/使用说明.md" +expect_fail grep -q '没有提供可用的授权入口' "$STAGE/使用说明.md" +check grep -Eq '^signature=(developer-id|ad-hoc)$' "$STAGE/MANIFEST.txt" +check grep -Eq '^notarization=(not-submitted|accepted)$' "$STAGE/MANIFEST.txt" +NOTARIZATION_STATUS="$(sed -n 's/^notarization=//p' "$STAGE/MANIFEST.txt")" +case "$NOTARIZATION_STATUS" in + accepted) + check grep -Eq '^notarization_submission_id=[0-9a-fA-F-]{36}$' \ + "$STAGE/MANIFEST.txt" + ;; + not-submitted) + check grep -q '^notarization_submission_id=none$' "$STAGE/MANIFEST.txt" + ;; +esac +( + cd "$STAGE" + shasum -a 256 -c SHA256SUMS.txt >/dev/null +) || fail "package SHA256SUMS verification failed" +CHECKS=$((CHECKS + 1)) + +# 按文档执行无污染安装。所有安装位置和 HOME 都在临时目录。 +HOME="$TEST_HOME" \ +VEXDB_LITE_APP_DIR="$APP_DIR" \ +VEXDB_LITE_BIN_DIR="$BIN_DIR" \ +VEXDB_LITE_LIB_DIR="$LIB_DIR" \ +VEXDB_LITE_ALLOW_ADHOC_INSTALL="$ALLOW_ADHOC" \ +VEXDB_LITE_ALLOW_DIRTY_INSTALL="$ALLOW_DIRTY" \ + "$STAGE/install.sh" >"$TMP_DIR/install.out" + +VEXDB="$BIN_DIR/vexdb" +VEXFS="$BIN_DIR/vexfs" +DYLIB="$LIB_DIR/vexdb_lite.dylib" +check test -d "$APP_DIR/VexDB Lite.app" +check test -x "$VEXDB" +check test -L "$VEXFS" +check test -f "$DYLIB" +contains "$("$VEXDB" --version)" "$EXPECTED_PRODUCT_VERSION" "version" +contains "$(env HOME="$TEST_HOME" PATH="$BIN_DIR:/usr/bin:/bin:/usr/sbin:/sbin" vexdb --version)" \ + "vexdb-lite" "PATH invocation" + +# SQLite 交互和直接 SQL。 +SQL_DB="$TEST_HOME/data/agent.db" +mkdir -p "$(dirname "$SQL_DB")" +INTERACTIVE_OUT="$(printf '%s\n' \ + 'CREATE TABLE notes(id INTEGER PRIMARY KEY, content TEXT);' \ + "INSERT INTO notes(content) VALUES ('hello');" \ + 'SELECT content FROM notes;' \ + '.quit' | "$VEXDB" "$SQL_DB")" +contains "$INTERACTIVE_OUT" "hello" "interactive SQLite" +contains "$("$VEXDB" "$SQL_DB" "SELECT sqlite_version(), vexdb_version();")" \ + "vexdb-lite" "direct SQL" + +# 向量距离和 GRAPH_INDEX。 +equal "5.0" "$("$VEXDB" "$SQL_DB" \ + "SELECT vexdb_l2_distance('[1,2]', '[4,6]');")" "L2 distance" +"$VEXDB" "$SQL_DB" \ + "CREATE VIRTUAL TABLE vectors USING GRAPH_INDEX(embedding FLOAT[2], metric=l2);" >/dev/null +"$VEXDB" "$SQL_DB" \ + "INSERT INTO vectors(rowid, embedding) VALUES (1, '[1,1]'), (2, '[9,9]');" >/dev/null +equal "1" "$("$VEXDB" "$SQL_DB" \ + "SELECT rowid FROM vectors WHERE embedding MATCH '[1,1]' AND k=1;")" \ + "nearest neighbor" + +# 默认数据库的文件管理命令。 +HOME="$TEST_HOME" "$VEXDB" fs setup >/dev/null +DOCTOR_JSON="$(HOME="$TEST_HOME" "$VEXDB" fs --json doctor || true)" +CLI_RUNTIME_ABI="$(printf '%s' "$DOCTOR_JSON" | /usr/bin/python3 -c \ + 'import json,sys; print(json.load(sys.stdin)["runtime_abi"])')" +equal "$MANIFEST_RUNTIME_ABI" "$CLI_RUNTIME_ABI" "manifest runtime ABI" +HOME="$TEST_HOME" "$VEXDB" fs mkdir /docs +printf 'hello VexFS\n' | HOME="$TEST_HOME" "$VEXDB" fs write /docs/hello.txt >/dev/null +printf 'fake-png-data\x00tail' >"$TMP_DIR/photo.png" +HOME="$TEST_HOME" "$VEXDB" fs write /docs/photo.png "$TMP_DIR/photo.png" >/dev/null +LISTING="$(HOME="$TEST_HOME" "$VEXDB" fs ls /docs)" +contains "$LISTING" "hello.txt" "file listing" +contains "$LISTING" "photo.png" "local file write" +equal "hello VexFS" "$(HOME="$TEST_HOME" "$VEXDB" fs cat /docs/hello.txt)" "file cat" +contains "$(HOME="$TEST_HOME" "$VEXDB" fs stat /docs/hello.txt)" '"kind":"file"' "file stat" +HOME="$TEST_HOME" "$VEXDB" fs mv /docs/hello.txt /docs/readme.txt +equal "hello VexFS" "$(HOME="$TEST_HOME" "$VEXDB" fs cat /docs/readme.txt)" "file move" +expect_fail env HOME="$TEST_HOME" "$VEXDB" fs stat /docs/hello.txt +HOME="$TEST_HOME" "$VEXDB" fs rm /docs/readme.txt +expect_fail env HOME="$TEST_HOME" "$VEXDB" fs stat /docs/readme.txt + +# 指定数据库、workspace 和 vexfs 兼容入口。 +HOME="$TEST_HOME" "$VEXDB" fs --db "$SQL_DB" --workspace project-a setup >/dev/null +printf 'project only' | HOME="$TEST_HOME" "$VEXDB" fs --db "$SQL_DB" \ + --workspace project-a write /project.txt >/dev/null +equal "project only" "$(HOME="$TEST_HOME" "$VEXDB" fs --db "$SQL_DB" \ + --workspace project-a cat /project.txt)" "workspace file" +equal "project only" "$(HOME="$TEST_HOME" "$VEXFS" --db "$SQL_DB" \ + --workspace project-a cat /project.txt)" "vexfs compatibility" +HOME="$TEST_HOME" "$VEXDB" fs --db "$SQL_DB" --workspace default setup >/dev/null +expect_fail env HOME="$TEST_HOME" "$VEXDB" fs --db "$SQL_DB" \ + --workspace default stat /project.txt + +# 文件版本、diff、dry-run 和恢复。 +printf 'first\n' | HOME="$TEST_HOME" "$VEXDB" fs write /docs/plan.md >"$TMP_DIR/v1" +printf 'second\n' | HOME="$TEST_HOME" "$VEXDB" fs write /docs/plan.md >"$TMP_DIR/v2" +V1="$(tr -d '\n' <"$TMP_DIR/v1")" +V2="$(tr -d '\n' <"$TMP_DIR/v2")" +contains "$(HOME="$TEST_HOME" "$VEXDB" fs history /docs/plan.md)" "$V1" "history" +equal "first" "$(HOME="$TEST_HOME" "$VEXDB" fs show /docs/plan.md --version "$V1")" \ + "historical content" +set +e +DIFF_OUT="$(HOME="$TEST_HOME" "$VEXDB" fs diff /docs/plan.md --from "$V1" 2>&1)" +DIFF_RC=$? +set -e +equal "1" "$DIFF_RC" "diff exit code" +contains "$DIFF_OUT" "first" "diff old content" +contains "$DIFF_OUT" "second" "diff new content" +HOME="$TEST_HOME" "$VEXDB" fs restore /docs/plan.md --version "$V1" --dry-run >/dev/null +RESTORED_VERSION="$(HOME="$TEST_HOME" "$VEXDB" fs restore /docs/plan.md --version "$V1")" +equal "first" "$(HOME="$TEST_HOME" "$VEXDB" fs cat /docs/plan.md)" "restored content" +check test "$RESTORED_VERSION" -gt "$V2" + +# 完整 workspace 快照、差异、dry-run 和一键还原。 +SNAPSHOT_COMMIT="$(HOME="$TEST_HOME" "$VEXDB" fs snapshot create before-agent)" +check test "$SNAPSHOT_COMMIT" -gt 0 +contains "$(HOME="$TEST_HOME" "$VEXDB" fs snapshot list)" "before-agent" \ + "workspace snapshot list" +contains "$(HOME="$TEST_HOME" "$VEXDB" fs snapshot show before-agent)" \ + '"path":"/docs/plan.md"' "workspace snapshot tree" +printf 'third\n' | HOME="$TEST_HOME" "$VEXDB" fs write /docs/plan.md >/dev/null +HOME="$TEST_HOME" "$VEXDB" fs mkdir /after-snapshot +printf 'new file\n' | HOME="$TEST_HOME" "$VEXDB" fs write /after-snapshot/new.txt >/dev/null +set +e +SNAPSHOT_DIFF="$(HOME="$TEST_HOME" "$VEXDB" fs snapshot diff before-agent 2>&1)" +SNAPSHOT_DIFF_RC=$? +set -e +equal "1" "$SNAPSHOT_DIFF_RC" "workspace snapshot diff exit code" +contains "$SNAPSHOT_DIFF" '"path":"/docs/plan.md"' "workspace modified file" +contains "$SNAPSHOT_DIFF" '"path":"/after-snapshot"' "workspace added directory" +HOME="$TEST_HOME" "$VEXDB" fs snapshot restore before-agent --dry-run >/dev/null +SNAPSHOT_RESTORE_COMMIT="$(HOME="$TEST_HOME" "$VEXDB" fs snapshot restore before-agent)" +check test "$SNAPSHOT_RESTORE_COMMIT" -gt "$SNAPSHOT_COMMIT" +equal "first" "$(HOME="$TEST_HOME" "$VEXDB" fs cat /docs/plan.md)" \ + "workspace restored content" +expect_fail env HOME="$TEST_HOME" "$VEXDB" fs stat /after-snapshot/new.txt +HOME="$TEST_HOME" "$VEXDB" fs snapshot diff before-agent >/dev/null + +# doctor 必须如实报告当前机器的 FSKit 状态。临时 HOME 只隔离数据库， +# 不会隔离系统已经注册的扩展，因此测试机可能是 missing、registered 或 enabled。 +set +e +DOCTOR_JSON="$(HOME="$TEST_HOME" "$VEXDB" fs doctor --json 2>/dev/null)" +DOCTOR_RC=$? +set -e +DOCTOR_STATE="$(printf '%s\n' "$DOCTOR_JSON" | sed -n 's/.*"extension":"\([^"]*\)".*/\1/p')" +case "$DOCTOR_STATE" in + enabled) + equal "0" "$DOCTOR_RC" "doctor enabled extension exit code" + ;; + missing|registered) + equal "1" "$DOCTOR_RC" "doctor unavailable extension exit code" + ;; + *) + fail "unexpected doctor extension state: ${DOCTOR_STATE:-empty}" + ;; +esac +contains "$DOCTOR_JSON" "\"extension\":\"$DOCTOR_STATE\"" "doctor extension state" +contains "$DOCTOR_JSON" '"schema_ready":true' "doctor database state" + +# 数据库备份、拒绝覆盖和备份验证。 +DEFAULT_DB="$TEST_HOME/Library/Application Support/VexDB-Lite/default.sqlite3" +BACKUP_DB="$TEST_HOME/Desktop/vexdb-backup.sqlite3" +"$VEXDB" backup "$DEFAULT_DB" "$BACKUP_DB" >/dev/null +check test -f "$DEFAULT_DB" +check test -f "$BACKUP_DB" +equal "ok" "$("$VEXDB" "$BACKUP_DB" "PRAGMA integrity_check;")" "backup integrity" +equal "first" "$(HOME="$TEST_HOME" "$VEXDB" fs --db "$BACKUP_DB" cat /docs/plan.md)" \ + "backup VexFS content" +contains "$(HOME="$TEST_HOME" "$VEXDB" fs --db "$BACKUP_DB" snapshot list)" \ + "before-agent" "backup workspace snapshot" +HOME="$TEST_HOME" "$VEXDB" fs --db "$BACKUP_DB" snapshot diff before-agent >/dev/null +expect_fail "$VEXDB" backup "$DEFAULT_DB" "$BACKUP_DB" +HOME="$TEST_HOME" "$VEXDB" fs snapshot drop before-agent + +# macOS 系统 sqlite3 通常关闭 .load；另有支持动态加载的 sqlite3 时验证 dylib。 +if /usr/bin/sqlite3 :memory: ".help" | grep -q '^\.load'; then + fail "documentation must be reviewed: /usr/bin/sqlite3 unexpectedly supports .load" +fi +CHECKS=$((CHECKS + 1)) +LOADABLE_SQLITE="$(command -v sqlite3 || true)" +if [ -n "$LOADABLE_SQLITE" ] && [ "$LOADABLE_SQLITE" != "/usr/bin/sqlite3" ] && \ + "$LOADABLE_SQLITE" :memory: ".help" | grep -q '^\.load'; then + contains "$("$LOADABLE_SQLITE" "$SQL_DB" ".load $DYLIB" "SELECT vexdb_version();")" \ + "$EXPECTED_PRODUCT_VERSION" "third-party sqlite loadable extension" +fi + +# 卸载只移除程序，不删除数据库。 +HOME="$TEST_HOME" \ +VEXDB_LITE_APP_DIR="$APP_DIR" \ +VEXDB_LITE_BIN_DIR="$BIN_DIR" \ +VEXDB_LITE_LIB_DIR="$LIB_DIR" \ + "$STAGE/uninstall.sh" >"$TMP_DIR/uninstall.out" +check test ! -e "$APP_DIR/VexDB Lite.app" +check test ! -e "$VEXDB" +check test ! -e "$VEXFS" +check test ! -e "$DYLIB" +check test -f "$DEFAULT_DB" +check test -f "$BACKUP_DB" + +echo "VEXDB DOCUMENTATION SMOKE: PASS ($CHECKS checks)" diff --git a/tests/eval/vexfs/run.py b/tests/eval/vexfs/run.py index 54f9e1b9d0..4ff79ff68e 100644 --- a/tests/eval/vexfs/run.py +++ b/tests/eval/vexfs/run.py @@ -8,12 +8,16 @@ from __future__ import annotations import argparse +import ctypes +import errno +import fcntl import hashlib import json import os import platform import random import resource +import signal import shutil import sqlite3 import subprocess @@ -29,13 +33,19 @@ MIB = 1024 * 1024 MAX_FILE_BYTES = 128 * MIB -CONTRACT_VERSION = "0.3.0" +CONTRACT_VERSION = "0.7.0" class EvalFailure(AssertionError): pass +class EvalFailureWithMetrics(EvalFailure): + def __init__(self, message: str, metrics: dict[str, Any]): + super().__init__(message) + self.metrics = metrics + + class EvalSkip(RuntimeError): pass @@ -90,6 +100,8 @@ class Context: mode: Mode seed: int enforce_performance: bool + fail_on_skip: bool = False + mount_cli_override: Path | None = None checks: int = 0 current_case: str = "" current_artifacts: Path | None = None @@ -171,9 +183,10 @@ def __exit__(self, *_: Any) -> None: def run_process(arguments: list[str], *, input_data: bytes | None = None, timeout: float = 120.0, check: bool = True, - cwd: Path | None = None) -> subprocess.CompletedProcess[bytes]: + cwd: Path | None = None, + env: dict[str, str] | None = None) -> subprocess.CompletedProcess[bytes]: result = subprocess.run(arguments, input=input_data, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, timeout=timeout, cwd=cwd) + stderr=subprocess.PIPE, timeout=timeout, cwd=cwd, env=env) if check and result.returncode != 0: raise EvalFailure( f"命令失败 ({result.returncode}): {' '.join(arguments)}\n" @@ -211,6 +224,11 @@ def child_max_rss_bytes() -> int: return value if platform.system() == "Darwin" else value * 1024 +def self_max_rss_bytes() -> int: + value = int(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss) + return value if platform.system() == "Darwin" else value * 1024 + + @case("existing.sqlite-smoke", "existing-gates", "静态注册 VexFS SQL 合同冒烟") def existing_sqlite_smoke(ctx: Context) -> dict[str, Any]: binary = ctx.build_dir / "vexfs_static_smoke" @@ -220,15 +238,46 @@ def existing_sqlite_smoke(ctx: Context) -> dict[str, Any]: return {"stdout": result.stdout.decode().strip()} -@case("existing.cabi-smoke", "existing-gates", "真实磁盘 C ABI、同步和崩溃 staging 冒烟") +@case("existing.cabi-smoke", "existing-gates", + "真实 runtime、跨连接 cache generation、同步和崩溃 staging 冒烟") def existing_cabi_smoke(ctx: Context) -> dict[str, Any]: - binary = ctx.build_dir / "vexfs_mount_contract_smoke" + binary = ctx.build_dir / "vexfs_runtime_smoke" ctx.check(binary.exists(), f"缺少 {binary}") result = run_process([str(binary)]) ctx.check(b"PASS" in result.stdout, "C ABI smoke 没有 PASS") return {"stdout": result.stdout.decode().strip()} +@case("existing.platform-boundary-smoke", "existing-gates", + "Linux libfuse3 和 Windows WinFsp 平台边界可独立编译并明确报告状态") +def existing_platform_boundary_smoke(ctx: Context) -> dict[str, Any]: + outputs = {} + for platform_name in ("linux", "windows"): + binary = ctx.build_dir / f"vexfs_platform_{platform_name}_smoke" + ctx.check(binary.exists(), f"缺少 {binary}") + result = run_process([str(binary)]) + ctx.check(b"PASS" in result.stdout, f"{platform_name} 平台边界 smoke 没有 PASS") + outputs[platform_name] = result.stdout.decode().strip() + return outputs + + +@case("existing.linux-fuse-helper-smoke", "existing-gates", + "Linux FUSE3 helper 通过真实 SQLite C ABI 完成写入、读取和清理") +def existing_linux_fuse_helper_smoke(ctx: Context) -> dict[str, Any]: + binary = ctx.build_dir / "vexfs-fuse" + if platform.system() != "Linux" or not binary.exists(): + raise EvalSkip("当前构建没有 Linux vexfs-fuse helper") + with tempfile.TemporaryDirectory(prefix="vexfs-fuse-helper-eval-") as directory: + database = Path(directory) / "helper.sqlite3" + result = run_process( + [str(binary), "--db", str(database), "--workspace", "eval", "--self-test"]) + ctx.check(b"VEXFS FUSE SELF TEST: PASS" in result.stdout, + "Linux FUSE helper self-test 没有 PASS") + ctx.check(database.exists(), "Linux FUSE helper 没有创建 SQLite 数据库") + return {"stdout": result.stdout.decode().strip(), + "database_bytes": database.stat().st_size} + + @case("existing.cli-smoke", "existing-gates", "CLI 现有端到端冒烟脚本") def existing_cli_smoke(ctx: Context) -> dict[str, Any]: script = ctx.root / "agent_files/cli/test/vexfs_cli_smoke.sh" @@ -238,11 +287,139 @@ def existing_cli_smoke(ctx: Context) -> dict[str, Any]: return {"stdout": result.stdout.decode().strip()} +@case("existing.unified-cli-smoke", "existing-gates", + "vexdb 内置 SQLite、向量、文件和数据库备份统一入口") +def existing_unified_cli_smoke(ctx: Context) -> dict[str, Any]: + script = ctx.root / "agent_files/cli/test/vexdb_unified_smoke.sh" + binary = ctx.build_dir / "vexdb" + ctx.check(script.exists(), f"缺少 {script}") + ctx.check(binary.exists(), f"缺少 {binary}") + result = run_process(["/bin/bash", str(script), str(binary)], timeout=120) + ctx.check(b"PASS" in result.stdout, "统一 CLI smoke 没有 PASS") + return {"stdout": result.stdout.decode().strip()} + + +def package_stage(ctx: Context) -> Path: + configured_stage = os.environ.get("VEXDB_LITE_PACKAGE_STAGE", "") + if configured_stage: + stage = Path(configured_stage).resolve() + ctx.check(stage.is_dir(), f"指定的 VexDB-Lite stage 不存在: {stage}") + return stage + if ctx.fail_on_skip: + raise EvalFailure( + "发行 package Gate 必须通过 VEXDB_LITE_PACKAGE_STAGE 显式绑定本次构建") + dist = ctx.root / "dist/vexdb-lite" + candidates = [path for path in dist.glob("vexdb-lite-*-macos-*") + if path.is_dir()] + if not candidates: + raise EvalSkip("没有已构建的 VexDB-Lite macOS stage") + return max(candidates, key=lambda path: path.stat().st_mtime) + + +@case("package.unified-install", "package", + "统一包全新安装、重复安装、文件命令和保留数据卸载") +def package_unified_install(ctx: Context) -> dict[str, Any]: + if platform.system() != "Darwin": + raise EvalSkip("统一 macOS 安装包只在 macOS 执行") + stage = package_stage(ctx) + packaged_cli = stage / "bin/vexdb" + ctx.check(packaged_cli.exists(), "统一包缺少 bin/vexdb") + ctx.check((stage / "bin/vexfs").is_symlink(), "vexfs 必须是兼容链接") + ctx.check((stage / "VexDB Lite.app").is_dir(), "统一包缺少 VexDB Lite.app") + ctx.check((stage / "lib/vexdb_lite.dylib").exists(), "统一包缺少 SQLite 扩展") + manifest = dict( + line.split("=", 1) for line in (stage / "MANIFEST.txt").read_text().splitlines() + if "=" in line) + ctx.equal(manifest.get("contract_version"), CONTRACT_VERSION, "manifest schema 版本") + ctx.check(manifest.get("source_commit") not in (None, "", "unknown"), + "manifest 必须记录源码 commit") + if manifest.get("signature") == "developer-id": + if os.environ.get("VEXDB_LITE_ALLOW_DIRTY_PACKAGE_TEST") == "1": + ctx.equal(manifest.get("source_dirty"), "true", + "显式脏包测试必须真实标记 source_dirty") + else: + ctx.equal(manifest.get("source_dirty"), "false", + "Developer ID 包不能来自脏工作树") + + with tempfile.TemporaryDirectory(prefix="vexdb-package-eval-") as directory: + home = Path(directory) + current = home / "Library/Application Support/VexDB-Lite/default.sqlite3" + install_env = dict(os.environ) + install_env.update({ + "HOME": str(home), + "VEXDB_LITE_APP_DIR": str(home / "Applications"), + "VEXDB_LITE_BIN_DIR": str(home / "bin"), + "VEXDB_LITE_LIB_DIR": str(home / "lib"), + }) + if manifest.get("signature") == "ad-hoc": + install_env["VEXDB_LITE_ALLOW_ADHOC_INSTALL"] = "1" + if manifest.get("source_dirty") == "true" and \ + os.environ.get("VEXDB_LITE_ALLOW_DIRTY_PACKAGE_TEST") == "1": + install_env["VEXDB_LITE_ALLOW_DIRTY_INSTALL"] = "1" + run_process([str(stage / "install.sh")], env=install_env) + installed = home / "bin/vexdb" + installed_app = home / "Applications/VexDB Lite.app" + ctx.check(installed.exists(), "安装后缺少 vexdb") + ctx.check((home / "bin/vexfs").is_symlink(), "安装后 vexfs 不是兼容链接") + # Developer ID 导出的 extension 合法包含 embedded.provisionprofile，不能把它 + # 当作残留。注入一个发行包绝不会包含的额外资源，模拟旧 bundle 合并污染。 + stale_resource = installed_app / "Contents/vexdb-stale-install-regression.txt" + stale_resource.write_bytes(b"stale-resource-from-previous-version") + invalid = run_process( + ["/usr/bin/codesign", "--verify", "--deep", "--strict", str(installed_app)], + check=False) + ctx.check(invalid.returncode != 0, "测试残留必须先破坏旧 App 签名") + run_process([str(stage / "install.sh")], env=install_env) + ctx.check(not stale_resource.exists(), "重复安装必须删除旧 bundle 的多余资源") + verified = run_process( + ["/usr/bin/codesign", "--verify", "--deep", "--strict", str(installed_app)], + check=False) + ctx.equal(verified.returncode, 0, "整体替换后 App 签名有效") + run_process([str(installed), "fs", "--db", str(current), "setup"]) + doctor = json.loads(run_process( + [str(installed), "fs", "--db", str(current), "--json", "doctor"], + check=False).stdout) + ctx.equal(str(doctor["runtime_abi"]), manifest.get("mount_abi_version"), + "manifest 与 CLI runtime ABI") + run_process([str(installed), "fs", "--db", str(current), + "write", "/kept.txt"], input_data=b"kept file") + ctx.equal(run_process([str(installed), str(current), + "SELECT vexfs_contract_version();"]).stdout.strip(), + CONTRACT_VERSION.encode(), "全新数据库 schema") + ctx.equal(run_process([str(installed), "fs", "--db", str(current), + "cat", "/kept.txt"]).stdout.strip(), + b"kept file", "新建 VexFS 文件") + run_process([str(stage / "uninstall.sh")], env=install_env) + ctx.check(current.exists(), "卸载不能删除新数据库") + return {"stage": str(stage), + "atomic_reinstall": True, + "version": run_process([str(packaged_cli), "--version"]) + .stdout.decode().strip()} + + +@case("package.documentation-smoke", "package", + "按交付使用说明验证安装、SQLite、向量、文件、版本、备份、动态加载和卸载") +def package_documentation_smoke(ctx: Context) -> dict[str, Any]: + if platform.system() != "Darwin": + raise EvalSkip("统一 macOS 安装包只在 macOS 执行") + stage = package_stage(ctx) + script = ctx.root / "tests/eval/vexfs/documentation_smoke.sh" + ctx.check(script.exists(), f"缺少 {script}") + result = run_process(["/bin/bash", str(script), str(stage)], timeout=180) + output = (result.stdout + result.stderr).decode(errors="replace") + ctx.check("VEXDB DOCUMENTATION SMOKE: PASS" in output, + "使用说明端到端测试没有 PASS") + return {"stage": str(stage), "summary": output.strip().splitlines()[-1]} + + @case("existing.sqlite-spec", "existing-gates", "渲染并执行全部 SQLite YAML spec") def existing_sqlite_spec(ctx: Context) -> dict[str, Any]: script = ctx.root / "tests/spec/_lib/docker/run_sqlite.sh" ctx.check(script.exists(), f"缺少 {script}") - result = run_process(["/bin/bash", str(script)], timeout=180, cwd=ctx.root) + environment = os.environ.copy() + environment["VEXDB_SQLITE_EXTENSION"] = str(ctx.extension) + result = run_process(["/bin/bash", str(script)], timeout=180, cwd=ctx.root, + env=environment) output = (result.stdout + result.stderr).decode(errors="replace") ctx.check("0 failed" in output, "SQLite spec 存在失败") return {"summary": output.strip().splitlines()[-1] if output.strip() else ""} @@ -290,6 +467,199 @@ def binary_unicode_metadata(ctx: Context) -> dict[str, Any]: return {"bytes": len(payload), "sha256": sha256(payload), "inode": stat["inode"]} +@case("contract.posix-mode", "functional", "可执行权限持久化、事务回滚且不干扰打开句柄") +def posix_mode(ctx: Context) -> dict[str, Any]: + with Database(ctx) as db: + inode = db.scalar("SELECT vexfs_create('default','/run.sh','file',?)", (0o750,)) + initial = db.json("SELECT vexfs_stat('default','/run.sh')") + ctx.equal(initial["mode"], 0o750, "创建时 mode") + ctx.equal(initial["version"], 1, "空文件初始内容版本") + + head_before = db.scalar( + "SELECT head_commit FROM _vexfs_workspaces WHERE name='default'") + ctx.equal(db.scalar("SELECT vexfs_set_mode('default',?,?)", (inode, 0o750)), + 0o750, "相同 mode 幂等") + ctx.equal(db.scalar( + "SELECT head_commit FROM _vexfs_workspaces WHERE name='default'"), + head_before, "相同 mode 不产生提交") + + ctx.equal(db.scalar("SELECT vexfs_set_mode('default',?,?)", (inode, 0o700)), + 0o700, "修改 mode") + changed = db.json("SELECT vexfs_stat('default','/run.sh')") + ctx.equal(changed["mode"], 0o700, "mode 已修改") + ctx.equal(changed["version"], 1, "chmod 不修改内容版本") + + db.connection.execute("BEGIN") + db.scalar("SELECT vexfs_set_mode('default',?,?)", (inode, 0o600)) + db.connection.execute("ROLLBACK") + ctx.equal(db.json("SELECT vexfs_stat('default','/run.sh')")["mode"], + 0o700, "chmod 事务回滚") + ctx.expect_error( + lambda: db.scalar("SELECT vexfs_set_mode('default',?,-1)", (inode,)), + "0..0777") + ctx.expect_error( + lambda: db.scalar("SELECT vexfs_set_mode('default',?,512)", (inode,)), + "0..0777") + + handle = db.scalar( + "SELECT vexfs_handle_open('default','/run.sh','rw','mode-open')") + generation = db.scalar( + "SELECT vexfs_handle_stage_write(?,0,'#!/bin/sh\necho ok\n','mode-write')", + (handle,)) + db.scalar("SELECT vexfs_set_mode('default',?,?)", (inode, 0o755)) + published = db.scalar( + "SELECT vexfs_handle_publish(?,?,'full','mode-publish')", + (handle, generation)) + db.scalar("SELECT vexfs_handle_close(?,1,'mode-close')", (handle,)) + final = db.json("SELECT vexfs_stat('default','/run.sh')") + ctx.equal((final["mode"], final["version"]), (0o755, published), + "打开句柄期间 chmod 后仍可发布") + return {"inode": inode, "mode": final["mode"], "version": final["version"]} + + +@case("contract.timestamps", "functional", + "独立 atime/mtime/ctime、元数据变更和 workspace 快照恢复") +def contract_timestamps(ctx: Context) -> dict[str, Any]: + with Database(ctx) as db: + inode = db.scalar("SELECT vexfs_create('default','/times.txt','file',420)") + initial = db.json("SELECT vexfs_stat('default','/times.txt')") + for field in ("created_at", "accessed_at", "updated_at", "changed_at"): + ctx.check(initial[field] > 0, f"{field} 必须存在") + + requested_atime = 1_700_000_000_123 + requested_mtime = 1_700_000_100_456 + ctx.equal(db.scalar("SELECT vexfs_set_times('default',?,?,?,3)", + (inode, requested_atime, requested_mtime)), 3, + "同时设置 atime/mtime") + changed = db.json("SELECT vexfs_stat('default','/times.txt')") + ctx.equal(changed["accessed_at"], requested_atime, "atime 独立持久化") + ctx.equal(changed["updated_at"], requested_mtime, "mtime 独立持久化") + ctx.check(changed["changed_at"] >= initial["changed_at"], "ctime 不能倒退") + ctx.equal(changed["version"], initial["version"], "时间戳不修改内容版本") + + db.scalar("SELECT vexfs_snapshot_create('default','before-time-change')") + second_atime = requested_atime + 10_000 + db.scalar("SELECT vexfs_set_times('default',?,?,?,1)", + (inode, second_atime, 0)) + access_only = db.json("SELECT vexfs_stat('default','/times.txt')") + ctx.equal(access_only["accessed_at"], second_atime, "只更新 atime") + ctx.equal(access_only["updated_at"], requested_mtime, "只更新 atime 不改 mtime") + + root_before = db.json("SELECT vexfs_stat('default','/')") + time.sleep(0.01) + db.scalar("SELECT vexfs_create('default','/child.txt','file',420)") + root_after = db.json("SELECT vexfs_stat('default','/')") + ctx.check(root_after["updated_at"] > root_before["updated_at"], + "目录项变化必须以毫秒精度推进目录 mtime") + + head = db.scalar("SELECT head_commit FROM _vexfs_workspaces WHERE name='default'") + db.scalar("SELECT vexfs_snapshot_restore('default','before-time-change',?)", (head,)) + restored = db.json("SELECT vexfs_stat('default','/times.txt')") + ctx.equal(restored["accessed_at"], requested_atime, "快照恢复 atime") + ctx.equal(restored["updated_at"], requested_mtime, "快照恢复 mtime") + ctx.expect_error( + lambda: db.scalar("SELECT vexfs_set_times('default',?,?,?,0)", + (inode, requested_atime, requested_mtime)), + "time mask") + ctx.expect_error( + lambda: db.scalar("SELECT vexfs_set_times('default',?,-1,0,1)", (inode,)), + "non-negative") + return {"inode": inode, "atime": restored["accessed_at"], + "mtime": restored["updated_at"], "ctime": restored["changed_at"]} + + +@case("contract.owner-acl", "functional", "owner uid/gid、chown 和数据库 ACL 元数据持久化") +def owner_acl(ctx: Context) -> dict[str, Any]: + with Database(ctx) as db: + inode = db.scalar("SELECT vexfs_create('default','/owned','file',?)", (0o640,)) + db.scalar("SELECT vexfs_chown('default',?,?,?)", (inode, 1234, 2345)) + stat = db.json("SELECT vexfs_stat('default','/owned')") + ctx.equal((stat["uid"], stat["gid"]), (1234, 2345), "chown uid/gid") + ctx.equal(db.scalar("SELECT vexfs_chown('default',?,?,?)", (inode, -1, 3456)), 1234, + "chown -1 保留 uid") + ctx.expect_error(lambda: db.scalar( + "SELECT vexfs_create('default','/fifo','fifo',?)", (0o644,)), + "special files") + db.scalar("SELECT vexfs_mkdir('default','/owned-dir')") + ctx.expect_error(lambda: db.scalar( + "SELECT vexfs_link('default','/owned-dir','/owned-link')"), + "regular files") + acl = '[{"principal":"alice","effect":"allow","permissions":"read,write","inherit":1}]' + ctx.equal(db.scalar("SELECT vexfs_acl_set('default',?,?)", (inode, acl)), 1, + "设置 ACL") + ctx.equal(db.json("SELECT vexfs_acl_get('default',?)", (inode,))[0]["principal"], + "alice", "读取 ACL") + db.scalar("SELECT vexfs_snapshot_create('default','owner-acl')") + db.scalar("SELECT vexfs_chown('default',?,?,?)", (inode, 9, 9)) + db.scalar("SELECT vexfs_acl_delete('default',?)", (inode,)) + head = db.scalar("SELECT head_commit FROM _vexfs_workspaces WHERE name='default'") + db.scalar("SELECT vexfs_snapshot_restore('default','owner-acl',?)", (head,)) + restored = db.json("SELECT vexfs_stat('default','/owned')") + ctx.equal((restored["uid"], restored["gid"]), (1234, 3456), "快照恢复 owner") + ctx.equal(db.json("SELECT vexfs_acl_get('default',?)", (inode,))[0]["principal"], + "alice", "快照恢复 ACL") + return {"inode": inode, "uid": restored["uid"], "gid": restored["gid"]} + + +@case("contract.symbolic-links", "functional", "相对、绝对、悬空符号链接和边界输入") +def symbolic_links(ctx: Context) -> dict[str, Any]: + with tempfile.TemporaryDirectory(prefix="vexfs-symlink-") as directory: + path = Path(directory) / "links.sqlite3" + db = Database(ctx, path) + db.scalar("SELECT vexfs_mkdir('default','/workspace/bin')") + db.scalar("SELECT vexfs_write('default','/workspace/bin/run.sh','echo ok')") + targets = { + "/workspace/run": b"bin/run.sh", + "/workspace/dangling": b"missing/target", + "/workspace/absolute": b"/workspace/bin/run.sh", + } + inodes: dict[str, int] = {} + for link_path, target in targets.items(): + inode = db.scalar("SELECT vexfs_symlink('default',?,?)", (link_path, target)) + inodes[link_path] = inode + stat = db.json("SELECT vexfs_stat('default',?)", (link_path,)) + ctx.equal((stat["kind"], stat["mode"], stat["size"]), + ("symlink", 0o777, len(target)), "符号链接元数据") + ctx.equal(db.scalar("SELECT vexfs_readlink('default',?)", (inode,)), + target, "readlink 原样返回 target") + + listing = db.json("SELECT vexfs_list('default','/workspace')") + listed = {entry["name"]: entry["kind"] for entry in listing} + ctx.equal(listed["run"], "symlink", "list 暴露 symlink 类型") + ctx.expect_error(lambda: db.scalar( + "SELECT vexfs_symlink('default','/workspace/run','other')"), + "already exists") + regular = db.json("SELECT vexfs_stat('default','/workspace/bin/run.sh')") + ctx.expect_error(lambda: db.scalar( + "SELECT vexfs_readlink('default',?)", (regular["inode"],)), + "not a symbolic link") + ctx.expect_error(lambda: db.scalar( + "SELECT vexfs_set_mode('default',?,448)", (inodes["/workspace/run"],)), + "cannot be changed") + for invalid_target, message in ((b"", "must not be empty"), + (b"bad\x00target", "contains NUL"), + (b"x" * 4097, "longer than 4096")): + ctx.expect_error(lambda target=invalid_target: db.scalar( + "SELECT vexfs_symlink('default','/workspace/invalid',?)", (target,)), + message) + + moved_inode = inodes["/workspace/run"] + db.scalar("SELECT vexfs_move('default','/workspace/run','/workspace/moved')") + ctx.equal(db.scalar("SELECT vexfs_readlink('default',?)", (moved_inode,)), + b"bin/run.sh", "重命名不修改 target") + db.scalar("SELECT vexfs_remove('default','/workspace/moved',0)") + ctx.equal(db.scalar("SELECT CAST(vexfs_read('default','/workspace/bin/run.sh') AS TEXT)"), + "echo ok", "删除链接不删除目标") + db.close() + + reopened = Database(ctx, path) + dangling = reopened.json("SELECT vexfs_stat('default','/workspace/dangling')") + ctx.equal(reopened.scalar("SELECT vexfs_readlink('default',?)", (dangling["inode"],)), + b"missing/target", "重开数据库后 target 持久化") + reopened.close() + return {"links": len(targets), "persisted": True} + + @case("contract.transactions", "transaction", "提交、回滚、savepoint 和读者快照") def transactions(ctx: Context) -> dict[str, Any]: with Database(ctx) as db: @@ -407,9 +777,9 @@ def version_history(ctx: Context) -> dict[str, Any]: "SELECT vexfs_read_version('default','/history',99)"), "version not found") commits = db.connection.execute( "SELECT id,parent_commit,message FROM _vexfs_commits ORDER BY id").fetchall() - ctx.equal([row[1] for row in commits], [None, 1, 2], "commit 父链") + ctx.equal([row[1] for row in commits], [None, 1, 2, 3], "commit 父链") ctx.equal(db.scalar( - "SELECT head_commit FROM _vexfs_workspaces WHERE name='default'"), 3, + "SELECT head_commit FROM _vexfs_workspaces WHERE name='default'"), 4, "workspace head") restored = db.scalar( "SELECT vexfs_restore_version('default','/history',1,3)") @@ -432,7 +802,165 @@ def version_history(ctx: Context) -> dict[str, Any]: "回滚恢复不改变当前版本") ctx.equal(len(db.json("SELECT vexfs_history('default','/history')")), 4, "回滚恢复不留下历史") - return {"inode": inode, "versions": 4, "commits": 4, "restored": restored} + return {"inode": inode, "versions": 4, "commits": 5, "restored": restored} + + +@case("contract.workspace-snapshot", "transaction", + "完整 workspace 快照覆盖目录、内容、权限、符号链接和扩展属性") +def workspace_snapshot(ctx: Context) -> dict[str, Any]: + with Database(ctx) as db: + db.scalar("SELECT vexfs_mkdir('default','/project/src')") + script = db.scalar( + "SELECT vexfs_create('default','/project/run.sh','file',?)", (0o755,)) + db.scalar("SELECT vexfs_write('default','/project/run.sh','alpha')") + db.scalar("SELECT vexfs_write('default','/project/src/keep.txt','keep')") + link = db.scalar( + "SELECT vexfs_symlink('default','/project/current',X'737263')") + db.scalar( + "SELECT vexfs_xattr_set('default',?,'user.vexfs.eval','old',0)", (script,)) + snapshot_head = db.scalar( + "SELECT head_commit FROM _vexfs_workspaces WHERE name='default'") + ctx.equal(db.scalar( + "SELECT vexfs_snapshot_create('default','before',?)", (snapshot_head,)), + snapshot_head, "创建快照") + ctx.expect_error(lambda: db.scalar( + "SELECT vexfs_snapshot_create('default','before')"), "UNIQUE") + shown = db.json("SELECT vexfs_snapshot_show('default','before')") + ctx.equal(shown["commit"], snapshot_head, "快照绑定不可变 commit") + ctx.equal({entry["path"] for entry in shown["entries"]}, + {"/", "/project", "/project/current", "/project/run.sh", + "/project/src", "/project/src/keep.txt"}, + "快照包含完整目录树") + + db.scalar("SELECT vexfs_write('default','/project/run.sh','beta')") + db.scalar("SELECT vexfs_set_mode('default',?,?)", (script, 0o600)) + db.scalar( + "SELECT vexfs_xattr_set('default',?,'user.vexfs.eval','new',0)", (script,)) + db.scalar("SELECT vexfs_move('default','/project/src','/project/lib')") + db.scalar("SELECT vexfs_remove('default','/project/current',0)") + db.scalar("SELECT vexfs_symlink('default','/project/current',X'6C6962')") + db.scalar("SELECT vexfs_write('default','/project/new.txt','new')") + changed_head = db.scalar( + "SELECT head_commit FROM _vexfs_workspaces WHERE name='default'") + difference = db.json( + "SELECT vexfs_snapshot_diff('default','before','HEAD')") + changes = {(entry["path"], entry["change"]) + for entry in difference["changes"]} + ctx.check(("/project/run.sh", "modify") in changes, "快照识别内容、mode 和 xattr") + ctx.check(("/project/src", "delete") in changes and + ("/project/lib", "add") in changes, "快照识别目录移动") + ctx.check(("/project/new.txt", "add") in changes, "快照识别新增文件") + ctx.expect_error(lambda: db.scalar( + "SELECT vexfs_snapshot_restore('default','before',?)", (snapshot_head,)), + "workspace head conflict") + + workspace_id = db.scalar( + "SELECT id FROM _vexfs_workspaces WHERE name='default'") + db.connection.execute( + "INSERT INTO _vexfs_mount_sessions(workspace_id,session_id,lease_until) " + "VALUES(?, 'eval-live-mount', (CAST(strftime('%s','now') AS INTEGER)+30)*1000)", + (workspace_id,)) + ctx.expect_error(lambda: db.scalar( + "SELECT vexfs_snapshot_restore('default','before',?)", (changed_head,)), + "active mount session") + db.connection.execute( + "DELETE FROM _vexfs_mount_sessions WHERE workspace_id=?", (workspace_id,)) + + handle = db.scalar( + "SELECT vexfs_handle_open('default','/project/run.sh','r','snapshot-open')") + ctx.expect_error(lambda: db.scalar( + "SELECT vexfs_snapshot_restore('default','before',?)", (changed_head,)), + "open or retained") + db.scalar("SELECT vexfs_handle_close(?,0,'snapshot-close')", (handle,)) + restored_head = db.scalar( + "SELECT vexfs_snapshot_restore('default','before',?)", (changed_head,)) + ctx.check(restored_head > changed_head, "还原产生新的 workspace commit") + ctx.equal(db.scalar( + "SELECT CAST(vexfs_read('default','/project/run.sh') AS TEXT)"), + "alpha", "还原文件内容") + ctx.equal(db.json("SELECT vexfs_stat('default','/project/run.sh')")["mode"], + 0o755, "还原可执行权限") + ctx.equal(db.scalar("SELECT CAST(vexfs_readlink('default',?) AS TEXT)", (link,)), + "src", "还原符号链接及 inode") + ctx.equal(db.scalar( + "SELECT CAST(vexfs_xattr_get('default',?,'user.vexfs.eval') AS TEXT)", + (script,)), "old", "还原扩展属性") + ctx.equal(db.scalar( + "SELECT CAST(vexfs_read('default','/project/src/keep.txt') AS TEXT)"), + "keep", "还原被移动的目录") + ctx.expect_error(lambda: db.scalar( + "SELECT vexfs_stat('default','/project/new.txt')"), "path not found") + ctx.equal(db.json( + "SELECT vexfs_snapshot_diff('default','before','HEAD')")["changes"], [], + "还原后逻辑状态和快照完全一致") + ctx.expect_error(lambda: db.scalar( + "SELECT vexfs_snapshot_restore('default','before',?)", (restored_head,)), + "already matches") + + maximum_version = db.scalar( + "SELECT max(version_no) FROM _vexfs_file_versions WHERE inode_id=?", (script,)) + new_version = db.scalar( + "SELECT vexfs_write('default','/project/run.sh','after-restore')") + ctx.check(new_version > maximum_version, "还原后的再次写入版本继续单调增长") + listing = db.json("SELECT vexfs_snapshot_list('default')") + ctx.equal([(entry["name"], entry["commit"]) for entry in listing], + [("before", snapshot_head)], "列出快照") + ctx.equal(db.scalar("SELECT vexfs_snapshot_drop('default','before')"), 1, + "删除快照") + ctx.expect_error(lambda: db.scalar( + "SELECT vexfs_snapshot_show('default','before')"), "snapshot not found") + ctx.equal(db.scalar("PRAGMA integrity_check"), "ok", "快照还原后完整性") + ctx.equal(db.connection.execute("PRAGMA foreign_key_check").fetchall(), [], + "快照还原后外键完整性") + return {"snapshot_commit": snapshot_head, "changed_commit": changed_head, + "restored_commit": restored_head, "changes": len(changes), + "next_version": new_version} + + +@case("contract.workspace-snapshot-hardlinks", "transaction", + "workspace 快照保留普通文件的多路径硬链接关系并可完整还原") +def workspace_snapshot_hardlinks(ctx: Context) -> dict[str, Any]: + with Database(ctx) as db: + db.scalar("SELECT vexfs_mkdir('default','/links')") + db.scalar("SELECT vexfs_mkdir('default','/links/left')") + db.scalar("SELECT vexfs_mkdir('default','/links/right')") + db.scalar("SELECT vexfs_write('default','/links/left/shared.txt','before')") + inode = db.json("SELECT vexfs_stat('default','/links/left/shared.txt')")["inode"] + db.scalar("SELECT vexfs_link('default','/links/left/shared.txt'," + "'/links/right/shared.txt')") + right = db.json("SELECT vexfs_stat('default','/links/right/shared.txt')") + ctx.equal(right["inode"], inode, "硬链接复用原 inode") + + snapshot_commit = db.scalar( + "SELECT vexfs_snapshot_create('default','hardlinks')") + shown = db.json("SELECT vexfs_snapshot_show('default','hardlinks')") + entries = {entry["path"]: entry["state"] for entry in shown["entries"]} + ctx.equal(entries["/links/left/shared.txt"]["inode"], inode, + "快照记录第一条硬链接") + ctx.equal(entries["/links/right/shared.txt"]["inode"], inode, + "快照记录第二条硬链接") + + # Mutate through one name and remove the other. Restore must put back + # the deleted dentry and the content/metadata observed at the snapshot. + db.scalar("SELECT vexfs_write('default','/links/left/shared.txt','after')") + db.scalar("SELECT vexfs_remove('default','/links/right/shared.txt',0)") + changed_head = db.scalar( + "SELECT head_commit FROM _vexfs_workspaces WHERE name='default'") + restored_head = db.scalar( + "SELECT vexfs_snapshot_restore('default','hardlinks',?)", (changed_head,)) + ctx.check(restored_head > changed_head, "硬链接还原生成新 commit") + left = db.json("SELECT vexfs_stat('default','/links/left/shared.txt')") + right = db.json("SELECT vexfs_stat('default','/links/right/shared.txt')") + ctx.equal(left["inode"], inode, "还原保留第一条硬链接 inode") + ctx.equal(right["inode"], inode, "还原补回第二条硬链接 inode") + ctx.equal(db.scalar( + "SELECT CAST(vexfs_read('default','/links/right/shared.txt') AS TEXT)"), + "before", "硬链接还原文件内容") + ctx.equal(db.json( + "SELECT vexfs_snapshot_diff('default','hardlinks','HEAD')")["changes"], + [], "硬链接还原后快照无差异") + return {"snapshot_commit": snapshot_commit, "changed_commit": changed_head, + "restored_commit": restored_head, "inode": inode} @case("contract.version-input-validation", "transaction", @@ -550,6 +1078,34 @@ def handles(ctx: Context) -> dict[str, Any]: return {"generation": generation, "version": version, "sparse_bytes": 8} +@case("contract.atomic-append", "concurrency", + "多个打开句柄按最新 inode 版本原子追加并刷新自身视图") +def atomic_append(ctx: Context) -> dict[str, Any]: + with Database(ctx) as db: + db.scalar("SELECT vexfs_write('default','/append','base')") + first = db.scalar("SELECT vexfs_handle_open('default','/append','rw','append-open-a')") + second = db.scalar("SELECT vexfs_handle_open('default','/append','rw','append-open-b')") + first_version = db.scalar( + "SELECT vexfs_handle_append(?,'-a','append-a')", (first,)) + ctx.equal(db.scalar("SELECT vexfs_handle_read(?,0,64)", (first,)), b"base-a", + "追加后刷新当前句柄") + second_version = db.scalar( + "SELECT vexfs_handle_append(?,'-b','append-b')", (second,)) + ctx.check(second_version > first_version, "第二个句柄基于最新版本追加") + ctx.equal(db.scalar("SELECT vexfs_handle_read(?,0,64)", (second,)), b"base-a-b", + "第二个句柄看到完整追加结果") + final_version = db.scalar( + "SELECT vexfs_handle_append(?,'-c','append-c')", (first,)) + ctx.equal(db.scalar("SELECT vexfs_handle_append(?,'-c','append-c')", (first,)), + final_version, "原子 append 请求可幂等重试") + ctx.equal(db.scalar("SELECT vexfs_read('default','/append')"), b"base-a-b-c", + "多句柄追加不丢失") + db.scalar("SELECT vexfs_handle_close(?,0,'append-close-a')", (first,)) + db.scalar("SELECT vexfs_handle_close(?,0,'append-close-b')", (second,)) + return {"first_version": first_version, "second_version": second_version, + "final_version": final_version} + + @case("contract.idempotency", "handles", "所有写请求的幂等重试和参数指纹") def idempotency(ctx: Context) -> dict[str, Any]: with Database(ctx) as db: @@ -580,6 +1136,48 @@ def idempotency(ctx: Context) -> dict[str, Any]: return {"request_rows": db.scalar("SELECT count(*) FROM _vexfs_requests")} +@case("maintenance.request-retention", "handles", + "幂等请求日志有上限，并保留最近请求的重试结果") +def request_retention(ctx: Context) -> dict[str, Any]: + retention_rows = 64 * 1024 + prune_interval = 4 * 1024 + with Database(ctx) as db: + maximum = int(db.scalar("SELECT COALESCE(max(rowid),0) FROM _vexfs_requests")) + trigger_rowid = ((maximum + retention_rows + prune_interval) // prune_interval + * prune_interval) + seed_count = trigger_rowid - maximum - 1 + db.connection.execute("BEGIN") + try: + db.connection.executemany( + "INSERT INTO _vexfs_requests(request_id,operation,request_fingerprint," + "result_integer) VALUES(?, 'retention-seed', X'', 0)", + ((f"retention-seed-{index}",) for index in range(seed_count))) + db.connection.execute("COMMIT") + except Exception: + db.connection.execute("ROLLBACK") + raise + + result = db.scalar( + "SELECT vexfs_item_reclaim('default','retention-trigger')") + ctx.equal(db.scalar("SELECT max(rowid) FROM _vexfs_requests"), trigger_rowid, + "清理触发行号") + ctx.equal(db.scalar("SELECT count(*) FROM _vexfs_requests"), retention_rows, + "幂等请求保留水位") + ctx.equal(db.scalar( + "SELECT count(*) FROM _vexfs_requests WHERE request_id='retention-seed-0'"), + 0, "最旧请求被清理") + ctx.equal(db.scalar( + "SELECT vexfs_item_reclaim('default','retention-trigger')"), result, + "最近请求仍可幂等重试") + ctx.equal(db.scalar( + "SELECT count(*) FROM _vexfs_requests WHERE request_id='retention-trigger'"), + 1, "幂等重试不会新增请求") + return {"retention_rows": retention_rows, + "prune_interval": prune_interval, + "seed_rows": seed_count, + "retained_rows": db.scalar("SELECT count(*) FROM _vexfs_requests")} + + @case("contract.write-conflict", "concurrency", "两个写句柄的乐观并发冲突") def write_conflict(ctx: Context) -> dict[str, Any]: with Database(ctx) as db: @@ -634,115 +1232,162 @@ def session_reclaim(ctx: Context) -> dict[str, Any]: return {"published": 2, "reclaimed": 1} -@case("migration.contract-010-to-030", "migration", "0.1.0 数据库原地升级并保留 dirty staging") -def migrate_010_to_030(ctx: Context) -> dict[str, Any]: - with tempfile.TemporaryDirectory(prefix="vexfs-migration-") as directory: - path = Path(directory) / "legacy.sqlite3" - db = Database(ctx, path, initialize=False) - db.connection.executescript(""" -CREATE TABLE _vexfs_meta(key TEXT PRIMARY KEY,value TEXT NOT NULL); -INSERT INTO _vexfs_meta VALUES('contract_version','0.1.0'); -CREATE TABLE _vexfs_inodes(kind TEXT,deleted_at INTEGER,current_version INTEGER); -INSERT INTO _vexfs_inodes VALUES('directory',NULL,0); -CREATE TABLE _vexfs_handles(id TEXT PRIMARY KEY,dirty_generation INTEGER NOT NULL); -INSERT INTO _vexfs_handles VALUES('dirty-handle',2); -CREATE TABLE _vexfs_staging(handle_id TEXT,generation INTEGER,content BLOB, - created_at INTEGER,PRIMARY KEY(handle_id,generation)); -INSERT INTO _vexfs_staging VALUES('dirty-handle',1,X'6F6C64',1); -INSERT INTO _vexfs_staging VALUES('dirty-handle',2,X'6E6577',2); -CREATE TABLE _vexfs_requests(request_id TEXT PRIMARY KEY,operation TEXT NOT NULL, - result_integer INTEGER,result_text TEXT,created_at INTEGER); -INSERT INTO _vexfs_requests VALUES('old-request','write',1,NULL,1); -""") - ctx.equal(db.scalar("SELECT vexfs_init()"), 1, "迁移执行") - ctx.equal(db.scalar("SELECT value FROM _vexfs_meta WHERE key='contract_version'"), - CONTRACT_VERSION, "迁移版本") - row = db.connection.execute( - "SELECT s.generation,d.content,s.logical_size,s.capacity " - "FROM _vexfs_staging s JOIN _vexfs_staging_data d USING(handle_id)").fetchone() - ctx.equal(row, (2, b"new", 3, 3), "只保留当前 dirty generation") - ctx.equal(db.scalar("SELECT count(*) FROM _vexfs_requests"), 0, "旧请求缓存清空") - ctx.equal(db.scalar("SELECT current_version FROM _vexfs_inodes"), 1, - "目录 verifier 修复") - columns = [row[1] for row in db.connection.execute( - "PRAGMA table_info(_vexfs_handles)").fetchall()] - ctx.check("owner_session" in columns, "迁移 owner_session 列") - db.close() - return {"from": "0.1.0", "to": CONTRACT_VERSION, "staging_generation": 2} - - -@case("migration.staging-inline-to-split", "migration", - "旧 0.2 inline staging 原地拆表并保留未发布内容") -def migrate_inline_staging_to_split(ctx: Context) -> dict[str, Any]: - with tempfile.TemporaryDirectory(prefix="vexfs-staging-migration-") as directory: - path = Path(directory) / "legacy-020.sqlite3" - db = Database(ctx, path) - db.scalar("SELECT vexfs_write('default','/legacy','old')") +@case("contract.cross-session-snapshot-barrier", "concurrency", + "管理会话不能静默快照另一个挂载会话尚未发布的写入") +def cross_session_snapshot_barrier(ctx: Context) -> dict[str, Any]: + with Database(ctx) as db: + db.scalar("SELECT vexfs_mkdir('default','/snapshot')") + db.scalar("SELECT vexfs_write('default','/snapshot/value.txt','base')") + db.scalar("SELECT vexfs_mount_session_start('default','mount-owner')") handle = db.scalar( - "SELECT vexfs_handle_open('default','/legacy','rw','legacy-split-open')") - generation = db.scalar( - "SELECT vexfs_handle_stage_write(?,0,'new','legacy-split-write')", (handle,)) - db.connection.executescript(""" -UPDATE _vexfs_staging -SET content=(SELECT content FROM _vexfs_staging_data - WHERE _vexfs_staging_data.handle_id=_vexfs_staging.handle_id); -DROP TABLE _vexfs_staging_data; -DELETE FROM _vexfs_meta WHERE key='staging_layout'; -UPDATE _vexfs_meta SET value='0.2.0' WHERE key='contract_version'; -""") - db.close() - - migrated = Database(ctx, path, initialize=False) - ctx.equal(migrated.scalar("SELECT vexfs_init()"), 1, "0.2 staging 拆表迁移") - ctx.equal(migrated.scalar( - "SELECT value FROM _vexfs_meta WHERE key='staging_layout'"), - "split-v1", "staging 布局标记") - ctx.equal(migrated.scalar( - "SELECT value FROM _vexfs_meta WHERE key='contract_version'"), - CONTRACT_VERSION, "0.2 合同升级") - row = migrated.connection.execute( - "SELECT length(s.content),d.content,s.logical_size,s.capacity " - "FROM _vexfs_staging s JOIN _vexfs_staging_data d USING(handle_id) " - "WHERE s.handle_id=?", (handle,)).fetchone() - ctx.equal(row, (0, b"new" + b"\x00" * (64 * 1024 - 3), 3, 64 * 1024), - "迁移后 metadata 与 BLOB 分离") - migrated.scalar( - "SELECT vexfs_handle_publish(?,?,'full','legacy-split-publish')", - (handle, generation)) - ctx.equal(migrated.scalar("SELECT vexfs_read('default','/legacy')"), b"new", - "迁移后未发布内容可发布") - ctx.equal(migrated.scalar("PRAGMA integrity_check"), "ok", "迁移后完整性") - migrated.close() - return {"layout": "split-v1", "generation": generation, "capacity": 64 * 1024} - + "SELECT vexfs_handle_open('default','/snapshot/value.txt','rw'," + "'snapshot-open','mount-owner')") + db.scalar("SELECT vexfs_handle_stage_write(?,0,'dirty','snapshot-stage')", + (handle,)) + ctx.expect_error(lambda: db.scalar( + "SELECT vexfs_snapshot_create('default','must-block')"), + "unpublished file handles") + + prefix = [str(ctx.cli), "--db", str(db.path), "--workspace", "default", + "--json", "snapshot", "create"] + blocked = run_process(prefix + ["cli-must-block"], check=False) + ctx.check(blocked.returncode != 0, "CLI consistent snapshot 必须失败") + ctx.equal(json.loads(blocked.stderr)["error"]["code"], "VEXFS_BUSY", + "CLI 明确报告 BUSY") + committed = json.loads(run_process( + prefix + ["published-only", "--committed-only"]).stdout) + ctx.equal(committed["consistency"], "committed-only", + "CLI 明确标记 committed-only") + names = {item["name"] for item in db.json( + "SELECT vexfs_snapshot_list('default')")} + ctx.check("cli-must-block" not in names and "must-block" not in names, + "失败的快照没有留下名字") + ctx.check("published-only" in names, "committed-only 快照已创建") -@case("migration.failure-rollback", "migration", "0.2 迁移失败必须完整回滚并可重试") -def migration_failure_rollback(ctx: Context) -> dict[str, Any]: - with tempfile.TemporaryDirectory(prefix="vexfs-migration-failure-") as directory: - path = Path(directory) / "broken-020.sqlite3" - db = Database(ctx, path) - db.scalar("SELECT vexfs_write('default','/migration','value')") - db.connection.executescript(""" -DROP TABLE _vexfs_staging_data; -CREATE TABLE _vexfs_staging_data(handle_id TEXT PRIMARY KEY,broken BLOB NOT NULL); -DELETE FROM _vexfs_meta WHERE key='staging_layout'; -UPDATE _vexfs_meta SET value='0.2.0' WHERE key='contract_version'; -""") - ctx.expect_error(lambda: db.scalar("SELECT vexfs_init()"), "no column named content") - ctx.equal(db.scalar("SELECT value FROM _vexfs_meta WHERE key='contract_version'"), - "0.2.0", "失败后合同版本回滚") ctx.equal(db.scalar( - "SELECT count(*) FROM _vexfs_meta WHERE key='staging_layout'"), 0, - "失败后布局标记不残留") - db.connection.execute("DROP TABLE _vexfs_staging_data") - ctx.equal(db.scalar("SELECT vexfs_init()"), 1, "修复条件后可重试") - ctx.equal(db.scalar("SELECT value FROM _vexfs_meta WHERE key='contract_version'"), - CONTRACT_VERSION, "重试后完成迁移") - db.close() - return {"rollback_version": "0.2.0", "retry_version": CONTRACT_VERSION} + "SELECT vexfs_mount_synchronize('default','snapshot-sync','mount-owner')"), + 1, "owner 发布 staging") + changes = db.json( + "SELECT vexfs_snapshot_diff('default','published-only','HEAD')")["changes"] + ctx.equal([item["path"] for item in changes], ["/snapshot/value.txt"], + "committed-only 与发布后 HEAD 的差异明确") + db.scalar("SELECT vexfs_mount_session_end('default','mount-owner')") + return {"blocked_status": "VEXFS_BUSY", + "committed_snapshot": committed["commit"], + "changes_after_publish": len(changes)} + + +@case("integrity.checksum-and-aliases", "integrity", + "SHA-256 覆盖普通版本、单文件恢复和 workspace 快照恢复，损坏内容不得返回") +def integrity_checksum_and_aliases(ctx: Context) -> dict[str, Any]: + with Database(ctx) as db: + ctx.equal(db.scalar("SELECT vexfs_write('default','/checked.txt','alpha')"), 1, + "写入第一个版本") + ctx.equal(db.json("SELECT vexfs_stat('default','/checked.txt')")["checksum"], + sha256(b"alpha"), "stat 返回真实 SHA-256") + ctx.equal(db.scalar("SELECT vexfs_write('default','/checked.txt','beta')"), 2, + "写入第二个版本") + ctx.equal(db.scalar( + "SELECT vexfs_restore_version('default','/checked.txt',1,2)"), 3, + "单文件恢复创建别名版本") + snapshot_commit = db.scalar( + "SELECT vexfs_snapshot_create('default','checked-baseline')") + ctx.equal(db.scalar("SELECT vexfs_write('default','/checked.txt','gamma')"), 4, + "快照后写入新版本") + head = db.scalar( + "SELECT head_commit FROM _vexfs_workspaces WHERE name='default'") + restored_commit = db.scalar( + "SELECT vexfs_snapshot_restore('default','checked-baseline',?)", (head,)) + ctx.equal(db.scalar("SELECT CAST(vexfs_read('default','/checked.txt') AS TEXT)"), + "alpha", "快照恢复后的内容") + inode = db.json("SELECT vexfs_stat('default','/checked.txt')")["inode"] + ctx.equal(db.scalar( + "SELECT count(*) FROM _vexfs_file_versions alias " + "JOIN _vexfs_file_versions source ON source.inode_id=alias.inode_id " + "AND source.version_no=alias.source_version_no " + "WHERE alias.inode_id=? AND alias.source_version_no IS NOT NULL " + "AND (alias.checksum<>source.checksum OR alias.size<>source.size " + "OR source.source_version_no IS NOT NULL)", (inode,)), 0, + "所有恢复别名直接引用 canonical 版本并复制校验和") + aliases = db.scalar( + "SELECT count(*) FROM _vexfs_file_versions WHERE inode_id=? " + "AND source_version_no IS NOT NULL", (inode,)) + ctx.check(aliases >= 2, "单文件恢复和快照恢复都产生版本别名") + + deep = db.json("SELECT vexfs_check('default',1)") + quick = db.json("SELECT vexfs_check('default',0)") + ctx.check(deep["ok"] and quick["ok"], "干净 workspace 深度和快速检查通过") + ctx.equal(deep["content_model"], "full-blob-v1", "检查明确当前内容模型") + ctx.equal(deep["checked"]["snapshots"], 1, "检查覆盖快照引用") + + db.connection.execute( + "UPDATE _vexfs_file_versions SET content=zeroblob(size) " + "WHERE inode_id=? AND version_no=1", (inode,)) + quick_after = db.json("SELECT vexfs_check('default',0)") + ctx.check(quick_after["ok"], "快速检查明确不读取 BLOB 校验和") + deep_after = db.json("SELECT vexfs_check('default',1)") + codes = {issue["code"] for issue in deep_after["issues"]} + ctx.check(not deep_after["ok"] and "VEXFS_CHECKSUM_MISMATCH" in codes, + "深度检查发现同长度内容损坏") + ctx.expect_error( + lambda: db.scalar("SELECT vexfs_read('default','/checked.txt')"), + "checksum does not match") + + cli = run_process( + [str(ctx.cli), "--db", str(db.path), "--workspace", "default", + "--json", "check"], check=False) + ctx.equal(cli.returncode, 8, "CLI 对损坏返回稳定的 corruption 退出码") + cli_report = json.loads(cli.stdout) + ctx.check(not cli_report["ok"], "CLI 仍把完整检查报告写到 stdout") + return {"snapshot_commit": snapshot_commit, + "restored_commit": restored_commit, + "aliases": aliases, + "clean_versions": deep["checked"]["versions"], + "corruption_codes": sorted(codes), + "cli_exit_code": cli.returncode} + + +@case("integrity.structural-corruption", "integrity", + "目录、提交、快照、版本别名和 staging 引用损坏均被只读检查发现") +def integrity_structural_corruption(ctx: Context) -> dict[str, Any]: + with Database(ctx) as db: + db.scalar("SELECT vexfs_mkdir('default','/tree')") + db.scalar("SELECT vexfs_write('default','/tree/value.txt','one')") + db.scalar("SELECT vexfs_write('default','/tree/value.txt','two')") + db.scalar("SELECT vexfs_restore_version('default','/tree/value.txt',1,2)") + db.scalar("SELECT vexfs_snapshot_create('default','broken-snapshot')") + handle = db.scalar( + "SELECT vexfs_handle_open('default','/tree/value.txt','rw','check-staging')") + db.scalar("SELECT vexfs_handle_stage_write(?,0,'dirty','check-stage')", (handle,)) + + db.connection.execute( + "UPDATE _vexfs_dentries SET parent_inode=999999 WHERE name='value.txt'") + db.connection.execute( + "UPDATE _vexfs_commits SET parent_commit=999998 " + "WHERE id=(SELECT max(id) FROM _vexfs_commits)") + db.connection.execute( + "UPDATE _vexfs_snapshots SET commit_id=999997 WHERE name='broken-snapshot'") + db.connection.execute( + "UPDATE _vexfs_file_versions SET source_version_no=999996 " + "WHERE source_version_no IS NOT NULL") + db.connection.execute("DELETE FROM _vexfs_staging_data WHERE handle_id=?", (handle,)) + + report = db.json("SELECT vexfs_check('default',0)") + codes = {issue["code"] for issue in report["issues"]} + expected = { + "VEXFS_DENTRY_PARENT_INVALID", "VEXFS_INODE_UNREACHABLE", + "VEXFS_COMMIT_PARENT_MISSING", "VEXFS_SNAPSHOT_COMMIT_MISSING", + "VEXFS_VERSION_SOURCE_INVALID", "VEXFS_STAGING_MISSING", + } + ctx.check(not report["ok"], "结构损坏必须失败") + ctx.equal(expected - codes, set(), "所有结构损坏均有稳定问题码") + ctx.check(all(issue["object"] and issue["type"] and issue["suggestion"] + for issue in report["issues"]), + "每个问题都包含对象、类型和建议动作") + return {"issue_count": report["issue_count"], "codes": sorted(codes)} -@case("migration.initialization-failure-rollback", "migration", +@case("recovery.initialization-failure-rollback", "recovery", "新库初始化空间不足时不得留下半张 schema") def initialization_failure_rollback(ctx: Context) -> dict[str, Any]: with tempfile.TemporaryDirectory(prefix="vexfs-init-failure-") as directory: @@ -795,11 +1440,21 @@ def reopen_integrity(ctx: Context) -> dict[str, Any]: payload = os.urandom(2 * MIB) db.scalar("SELECT vexfs_mkdir('default','/persist')") db.scalar("SELECT vexfs_write('default','/persist/blob',?)", (payload,)) + snapshot_commit = db.scalar( + "SELECT vexfs_snapshot_create('default','before-reopen')") + db.scalar("SELECT vexfs_write('default','/persist/blob','changed')") db.connection.execute("PRAGMA wal_checkpoint(FULL)").fetchone() db.close() reopened = Database(ctx, path) + ctx.equal(reopened.json( + "SELECT vexfs_snapshot_list('default')")[0]["commit"], + snapshot_commit, "重开后快照仍存在") + current_head = reopened.scalar( + "SELECT head_commit FROM _vexfs_workspaces WHERE name='default'") + reopened.scalar( + "SELECT vexfs_snapshot_restore('default','before-reopen',?)", (current_head,)) ctx.equal(reopened.scalar("SELECT vexfs_read('default','/persist/blob')"), payload, - "重开内容") + "重开后可从快照还原内容") ctx.equal(reopened.scalar("PRAGMA integrity_check"), "ok", "integrity_check") fk_rows = reopened.connection.execute("PRAGMA foreign_key_check").fetchall() ctx.equal(fk_rows, [], "foreign key check") @@ -843,6 +1498,8 @@ def online_backup_restore(ctx: Context) -> dict[str, Any]: source = Database(ctx, source_path) payload = os.urandom(min(ctx.mode.sequential_mib, 64) * MIB) source.scalar("SELECT vexfs_write('default','/backup.bin',?)", (payload,)) + snapshot_commit = source.scalar( + "SELECT vexfs_snapshot_create('default','backup-point')") handle = source.scalar( "SELECT vexfs_handle_open('default','/backup.bin','rw','backup-open')") source.scalar("SELECT vexfs_handle_stage_write(?,0,'DIRTY','backup-write')", (handle,)) @@ -856,6 +1513,12 @@ def online_backup_restore(ctx: Context) -> dict[str, Any]: ctx.equal(restored_payload, payload, "备份只能看到已发布内容") ctx.equal(restored.scalar("SELECT count(*) FROM _vexfs_staging"), 1, "备份保留完整恢复状态") + ctx.equal(restored.json( + "SELECT vexfs_snapshot_list('default')")[0]["commit"], + snapshot_commit, "备份保留 workspace 快照") + ctx.equal(restored.json( + "SELECT vexfs_snapshot_diff('default','backup-point','HEAD')")["changes"], + [], "未发布 staging 不污染快照 head") ctx.equal(restored.scalar("PRAGMA integrity_check"), "ok", "备份完整性") storage = db_storage(backup_path) bytes_copied = storage["total_bytes"] @@ -971,6 +1634,54 @@ def restore_race(ctx: Context) -> dict[str, Any]: return {"exit_codes": [result[0] for result in results], "versions": 3} +@case("concurrency.workspace-snapshot-restore-race", "concurrency", + "两个真实进程用同一 workspace head 还原快照时只能有一个成功") +def workspace_snapshot_restore_race(ctx: Context) -> dict[str, Any]: + with tempfile.TemporaryDirectory(prefix="vexfs-snapshot-race-") as directory: + base = Path(directory) + path = base / "db.sqlite3" + script = Path(__file__).resolve() + with Database(ctx, path) as db: + db.scalar("SELECT vexfs_write('default','/race','before')") + snapshot_commit = db.scalar( + "SELECT vexfs_snapshot_create('default','race-point')") + db.scalar("SELECT vexfs_write('default','/race','after')") + expected_head = db.scalar( + "SELECT head_commit FROM _vexfs_workspaces WHERE name='default'") + commits_before = db.scalar("SELECT count(*) FROM _vexfs_commits") + barrier = base / "go" + ready_paths = [base / "ready-snapshot-0", base / "ready-snapshot-1"] + processes = [subprocess.Popen( + [sys.executable, str(script), "--worker", "snapshot-restore-race", str(path), + str(ctx.extension), str(expected_head), str(ready_paths[index]), str(barrier)], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) for index in range(2)] + deadline = time.time() + 10 + while not all(candidate.exists() for candidate in ready_paths): + if time.time() >= deadline: + raise EvalFailure("snapshot restore worker 未就绪") + time.sleep(0.01) + barrier.touch() + results: list[tuple[int, bytes, bytes]] = [] + for process in processes: + stdout, stderr = process.communicate(timeout=30) + results.append((process.returncode, stdout, stderr)) + ctx.equal(sorted(result[0] for result in results), [0, 5], + "workspace snapshot restore 并发结果") + with Database(ctx, path) as db: + ctx.equal(db.scalar("SELECT count(*) FROM _vexfs_commits"), commits_before + 1, + "并发 workspace restore 只新增一个 commit") + ctx.equal(db.scalar("SELECT CAST(vexfs_read('default','/race') AS TEXT)"), + "before", "成功还原完整 workspace") + ctx.equal(db.json( + "SELECT vexfs_snapshot_diff('default','race-point','HEAD')")["changes"], + [], "成功还原后的 workspace 与快照一致") + ctx.equal(db.json( + "SELECT vexfs_snapshot_show('default','race-point')")["commit"], + snapshot_commit, "并发还原不移动快照") + return {"exit_codes": [result[0] for result in results], + "snapshot_commit": snapshot_commit, "expected_head": expected_head} + + @case("model.randomized-state-machine", "model", "确定性随机文件操作与参考模型逐步比对") def randomized_state_machine(ctx: Context) -> dict[str, Any]: rng = random.Random(ctx.seed ^ 0x5EED_F5) @@ -1092,6 +1803,27 @@ def cli_command_surface(ctx: Context) -> dict[str, Any]: b"3", "CLI restore 新版本") ctx.equal(run_process(prefix + ["cat", "/cli/a/version.txt"]).stdout, b"one\n", "CLI restore 内容") + grep_json = json.loads(run_process( + prefix + ["--json", "grep", "ONE", "/cli/a", "-i", "-n"]).stdout) + ctx.equal(grep_json["match_count"], 1, "CLI 数据库 grep 命中数") + ctx.equal(grep_json["matches"][0], + {"path": "/cli/a/version.txt", "line": 1, "text": "one"}, + "CLI 数据库 grep 当前版本") + ctx.equal(grep_json["binary_files_skipped"], 1, "CLI 数据库 grep 跳过二进制") + index_before = json.loads(run_process(prefix + ["index", "status"]).stdout) + ctx.equal(index_before["enabled"], False, "CLI 文本索引默认关闭") + index_enabled = json.loads(run_process(prefix + ["index", "enable"]).stdout) + ctx.equal(index_enabled["available"], True, "CLI 启用 trigram 文本索引") + indexed_grep = json.loads(run_process( + prefix + ["--json", "grep", "one", "/cli/a"]).stdout) + ctx.equal(indexed_grep["index_used"], True, "CLI grep 使用 trigram 索引") + ctx.equal(indexed_grep["files_scanned"], 1, "CLI 索引只读取候选文件") + grep_text = run_process(prefix + ["grep", "-n", "one", "/cli/a"]) + ctx.equal(grep_text.stdout, b"/cli/a/version.txt:1:one\n", "CLI grep 文本输出") + grep_files = run_process(prefix + ["grep", "-i", "-l", "ONE", "/cli/a"]) + ctx.equal(grep_files.stdout, b"/cli/a/version.txt\n", "CLI grep 文件名输出") + ctx.equal(run_process(prefix + ["grep", "absent", "/cli/a"], check=False).returncode, + 1, "CLI grep 无匹配退出码") newline_one = base / "newline-one.txt" newline_two = base / "newline-two.txt" @@ -1105,11 +1837,50 @@ def cli_command_surface(ctx: Context) -> dict[str, Any]: ctx.equal(newline_diff.returncode, 1, "CLI diff 识别末尾换行差异") ctx.check(b"No newline at end of file" in newline_diff.stdout, "CLI diff 末尾换行提示") + snapshot = json.loads(run_process( + prefix + ["--json", "snapshot", "create", "cli-stable"]).stdout) + ctx.check(snapshot["commit"] > 0, "CLI 创建 workspace 快照") run_process(prefix + ["rm", "/cli/a/newline.txt"]) run_process(prefix + ["rm", "/cli/a/version.txt"]) run_process(prefix + ["mv", "/cli/a/payload.bin", "/cli/b/moved.bin"]) run_process(prefix + ["rm", "/cli/a"]) + snapshot_diff = run_process( + prefix + ["--json", "snapshot", "diff", "cli-stable"], check=False) + ctx.equal(snapshot_diff.returncode, 1, "CLI workspace diff 不同退出码") + ctx.check(json.loads(snapshot_diff.stdout)["changes"], "CLI workspace diff 内容") + preview = json.loads(run_process( + prefix + ["--json", "snapshot", "restore", "cli-stable", "--dry-run"]).stdout) + ctx.check(preview["changes"], "CLI workspace restore dry-run") + restored_snapshot = json.loads(run_process( + prefix + ["--json", "snapshot", "restore", "cli-stable"]).stdout) + ctx.check(restored_snapshot["commit"] > restored_snapshot["previous_head"], + "CLI workspace restore 新 commit") + ctx.equal(run_process(prefix + ["cat", "/cli/a/payload.bin"]).stdout, + payload, "CLI workspace restore 文件树") + restored_grep = json.loads(run_process( + prefix + ["--json", "grep", "one", "/cli/a"]).stdout) + ctx.equal(restored_grep["index_used"], True, "CLI workspace restore 后索引仍启用") + ctx.equal(restored_grep["match_count"], 1, "CLI workspace restore 后索引内容正确") + ctx.equal(run_process( + prefix + ["snapshot", "diff", "cli-stable"], check=False).returncode, + 0, "CLI workspace restore 后无差异") + snapshot_list = json.loads(run_process( + prefix + ["--json", "snapshot", "list"]).stdout) + ctx.equal(snapshot_list[0]["name"], "cli-stable", "CLI snapshot list") + shown_snapshot = json.loads(run_process( + prefix + ["snapshot", "show", "cli-stable"]).stdout) + ctx.equal(shown_snapshot["commit"], snapshot["commit"], "CLI snapshot show") + run_process(prefix + ["snapshot", "drop", "cli-stable"]) + run_process(prefix + ["rm", "/cli/a/newline.txt"]) + run_process(prefix + ["rm", "/cli/a/version.txt"]) + run_process(prefix + ["rm", "/cli/a/payload.bin"]) + run_process(prefix + ["rm", "/cli/a"]) run_process(prefix + ["rm", "-r", "/cli/b"]) + index_after_remove = json.loads(run_process(prefix + ["index", "status"]).stdout) + ctx.equal(index_after_remove["indexed_files"], 0, + "CLI 删除文件后同步回收文本索引") + index_disabled = json.loads(run_process(prefix + ["index", "disable"]).stdout) + ctx.equal(index_disabled["enabled"], False, "CLI 关闭文本索引") run_process(prefix + ["descriptor", str(descriptor)]) descriptor_json = json.loads(descriptor.read_text()) ctx.equal(descriptor_json["workspace"], "eval", "descriptor workspace") @@ -1117,8 +1888,15 @@ def cli_command_surface(ctx: Context) -> dict[str, Any]: "descriptor database") doctor = run_process(prefix + ["--json", "doctor"], check=False) doctor_json = json.loads(doctor.stdout) - ctx.equal(doctor_json["database"]["contract_version"], CONTRACT_VERSION, - "doctor contract") + expected_platform = {"Darwin": "macos", "Linux": "linux", + "Windows": "windows"}.get(platform.system(), "unsupported") + expected_driver = {"Darwin": "FSKit", "Linux": "libfuse3", + "Windows": "WinFsp"}.get(platform.system(), "none") + ctx.equal(doctor_json["platform"], expected_platform, "doctor platform") + ctx.equal(doctor_json["mount_driver"], expected_driver, "doctor mount driver") + ctx.check(isinstance(doctor_json["mount_ready"], bool), "doctor mount ready") + ctx.equal(doctor_json["database"]["schema_version"], CONTRACT_VERSION, + "doctor schema") ctx.check(doctor.returncode in (0, 1), "doctor 退出码") invalid = run_process(prefix + ["unknown"], check=False) ctx.equal(invalid.returncode, 1, "未知命令退出码") @@ -1146,28 +1924,33 @@ def cli_doctor_readonly_and_permissions(ctx: Context) -> dict[str, Any]: connection.close() doctor = run_process(prefix + ["--json", "doctor"], check=False) details = json.loads(doctor.stdout)["database"] - ctx.equal(details["contract_version"], "0.2.0", "doctor 报告真实旧版本") - ctx.equal(details["compatible"], False, "doctor 拒绝把旧版本报为正常") - ctx.equal(details["upgrade_required"], True, "doctor 给出升级状态") + ctx.equal(details["schema_version"], "0.2.0", "doctor 报告真实 schema 版本") + ctx.equal(details["schema_ready"], False, "doctor 拒绝把错误版本报为正常") connection = sqlite3.connect(str(database), isolation_level=None) ctx.equal(connection.execute( "SELECT value FROM _vexfs_meta WHERE key='contract_version'").fetchone()[0], "0.2.0", "doctor 结束后没有静默迁移") connection.close() - readonly = sqlite3.connect(f"file:{database}?mode=ro", uri=True, isolation_level=None) - readonly.enable_load_extension(True) - readonly.load_extension(str(ctx.extension)) - ctx.equal(readonly.execute( - "SELECT vexfs_read('secure','/secret')").fetchone()[0], b"secret", - "0.2 数据库仍可只读访问") - readonly.close() - run_process(prefix + ["setup"]) + setup = run_process(prefix + ["setup"], check=False) + ctx.check(setup.returncode != 0, "错误 schema 版本不做自动升级") connection = sqlite3.connect(str(database), isolation_level=None) ctx.equal(connection.execute( "SELECT value FROM _vexfs_meta WHERE key='contract_version'").fetchone()[0], - CONTRACT_VERSION, "显式 setup 才执行升级") + "0.2.0", "setup 失败后仍不改写 schema") connection.close() - return {"mode": "0600", "doctor_exit": doctor.returncode} + if os.name != "nt": + target = Path(directory) / "symlink-target" + target.write_bytes(b"must-not-change") + target.chmod(0o644) + linked_database = Path(directory) / "linked.sqlite3" + linked_database.symlink_to(target) + rejected = run_process( + [str(ctx.cli), "--db", str(linked_database), "setup"], check=False) + ctx.check(rejected.returncode != 0, "数据库符号链接必须被拒绝") + ctx.equal(target.read_bytes(), b"must-not-change", "符号链接目标内容不变") + ctx.equal(target.stat().st_mode & 0o777, 0o644, "符号链接目标权限不变") + return {"mode": "0600", "doctor_exit": doctor.returncode, + "symlink_rejected": os.name != "nt"} @case("performance.small-files", "performance", "大量小文件创建、列表、stat 和读取") @@ -1195,17 +1978,279 @@ def performance_small_files(ctx: Context) -> dict[str, Any]: ctx.equal(db.scalar("SELECT vexfs_read('default',?)", (f"/small/f{index:06d}.txt",)), payload, "小文件 read") sample_seconds = time.perf_counter() - started + started = time.perf_counter() + snapshot_commit = db.scalar( + "SELECT vexfs_snapshot_create('default','small-files')") + snapshot_create_seconds = time.perf_counter() - started + db.scalar("SELECT vexfs_write('default','/small/f000000.txt','changed')") + started = time.perf_counter() + snapshot_changes = db.json( + "SELECT vexfs_snapshot_diff('default','small-files','HEAD')")["changes"] + snapshot_diff_seconds = time.perf_counter() - started + ctx.equal([entry["path"] for entry in snapshot_changes], + ["/small/f000000.txt"], "大量小文件 workspace diff") + restore_head = db.scalar( + "SELECT head_commit FROM _vexfs_workspaces WHERE name='default'") + started = time.perf_counter() + db.scalar( + "SELECT vexfs_snapshot_restore('default','small-files',?)", (restore_head,)) + snapshot_restore_seconds = time.perf_counter() - started + ctx.equal(db.scalar("SELECT vexfs_read('default','/small/f000000.txt')"), payload, + "大量小文件 workspace restore") ctx.budget("small_create_seconds", create_seconds, 60.0) ctx.budget("small_list_seconds", list_seconds, 5.0) + ctx.budget("snapshot_create_seconds", snapshot_create_seconds, 1.0) + ctx.budget("snapshot_diff_seconds", snapshot_diff_seconds, 15.0) + ctx.budget("snapshot_restore_seconds", snapshot_restore_seconds, 30.0) return {"files": ctx.mode.small_files, "create_seconds": round(create_seconds, 6), "create_files_per_second": round( ctx.mode.small_files / max(create_seconds, 1e-9), 3), "list_seconds": round(list_seconds, 6), "sample_stat_read_seconds": round(sample_seconds, 6), + "snapshot_commit": snapshot_commit, + "snapshot_create_seconds": round(snapshot_create_seconds, 6), + "snapshot_diff_seconds": round(snapshot_diff_seconds, 6), + "snapshot_restore_seconds": round(snapshot_restore_seconds, 6), "storage": db_storage(db.path)} +@case("performance.mount-contract-small-files", "performance", + "挂载合同原子创建、macOS xattr、内容发布和同步写放大") +def performance_mount_contract_small_files(ctx: Context) -> dict[str, Any]: + executable = ctx.build_dir / "vexfs_runtime_smoke" + if not executable.exists(): + raise EvalSkip(f"缺少挂载合同 benchmark: {executable}") + file_count = 250 if ctx.mode.name == "quick" else 1_000 + rss_before = child_max_rss_bytes() + result = run_process([str(executable), "--benchmark", str(file_count)], timeout=60) + metrics = json.loads(result.stdout) + rss_after = child_max_rss_bytes() + ctx.equal(metrics["files"], file_count, "挂载合同 benchmark 文件数") + ctx.equal(metrics["commit_rows"], file_count, "每个新文件一个历史 commit") + ctx.equal(metrics["version_rows"], file_count, "每个新文件一个内容版本") + ctx.check(metrics["request_rows"] <= file_count * 2, + "本地挂载每个文件最多保留 create/close 两条请求") + ctx.check(metrics["durability_barriers"] >= 1, "synchronize 执行真实 FULL 屏障") + ctx.check(rss_after < 1024 * MIB, "挂载合同 benchmark 峰值内存低于 1 GiB") + ctx.budget("mount_contract_create_seconds", metrics["create_seconds"], 30.0) + ctx.budget("mount_contract_sync_seconds", metrics["sync_seconds"], 5.0) + metrics["child_max_rss_before_bytes"] = rss_before + metrics["child_max_rss_after_bytes"] = rss_after + return metrics + + +@case("performance.scale-tree", "performance", + "1 千/1 万/10 万文件目录树的创建、重开、遍历、抽样读取和完整性") +def performance_scale_tree(ctx: Context) -> dict[str, Any]: + file_count = {"quick": 1_000, "full": 10_000, "stress": 100_000}[ctx.mode.name] + directory_count = min(1_000, max(10, file_count // 100)) + plain_payload = b"vexfs-scale-payload marker=plain\n" + needle_payload = b"vexfs-scale-payload marker=needle\n" + memory_budget = {"quick": 1 * 1024 * MIB, "full": 1536 * MIB, + "stress": 2 * 1024 * MIB}[ctx.mode.name] + rss_before = self_max_rss_bytes() + with tempfile.TemporaryDirectory(prefix="vexfs-scale-tree-") as directory: + database_path = Path(directory) / "scale.sqlite3" + db = Database(ctx, database_path) + db.scalar("SELECT vexfs_mkdir('default','/scale')") + for index in range(directory_count): + db.scalar("SELECT vexfs_mkdir('default',?)", (f"/scale/d{index:04d}",)) + + started = time.perf_counter() + for index in range(file_count): + bucket = index % directory_count + payload = needle_payload if index % 1_000 == 0 else plain_payload + db.scalar("SELECT vexfs_write('default',?,?)", + (f"/scale/d{bucket:04d}/f{index:06d}.txt", payload)) + if index % 1_000 == 0 and self_max_rss_bytes() > memory_budget: + raise EvalFailure( + f"规模测试峰值内存超过保护线: {self_max_rss_bytes() / MIB:.1f} MiB") + create_seconds = time.perf_counter() - started + db.close() + + reopened = Database(ctx, database_path) + # 大规模目录和 FTS 构建优先使用临时文件，并把 SQLite 页缓存限制在 + # 64 MiB，避免测试为了追求速度挤占整机内存。 + reopened.connection.execute("PRAGMA temp_store=FILE") + reopened.connection.execute("PRAGMA cache_size=-65536") + started = time.perf_counter() + stored_files = reopened.scalar( + "SELECT count(*) FROM _vexfs_dentries d " + "JOIN _vexfs_inodes i ON i.id=d.inode_id " + "JOIN _vexfs_workspaces w ON w.id=d.workspace_id " + "WHERE w.name='default' AND i.kind='file' AND i.deleted_at IS NULL") + count_seconds = time.perf_counter() - started + ctx.equal(stored_files, file_count, "规模目录树文件数") + + sample_count = min(200, file_count) + started = time.perf_counter() + for sample in range(sample_count): + index = sample * max(1, file_count // sample_count) + if index >= file_count: + index = file_count - 1 + bucket = index % directory_count + path = f"/scale/d{bucket:04d}/f{index:06d}.txt" + payload = needle_payload if index % 1_000 == 0 else plain_payload + ctx.equal(reopened.scalar("SELECT vexfs_read('default',?)", (path,)), + payload, "规模目录树抽样读取") + sample_seconds = time.perf_counter() - started + started = time.perf_counter() + grep = reopened.json( + "SELECT vexfs_grep('default','/scale','needle',2,10240)") + grep_seconds = time.perf_counter() - started + expected_matches = (file_count + 999) // 1_000 + ctx.equal(grep["match_count"], expected_matches, "数据库批量 grep 命中数") + ctx.equal(grep["files_scanned"], file_count, "数据库批量 grep 扫描文件数") + ctx.equal(grep["binary_files_skipped"], 0, "数据库批量 grep 文本识别") + ctx.equal(grep["index_used"], False, "数据库批量 grep 默认不增加索引写放大") + started = time.perf_counter() + index_status = reopened.json("SELECT vexfs_grep_index('enable')") + index_build_seconds = time.perf_counter() - started + ctx.equal(index_status["available"], True, "trigram 索引可用") + ctx.equal(index_status["indexed_files"], file_count, "trigram 索引文件数") + started = time.perf_counter() + indexed_grep = reopened.json( + "SELECT vexfs_grep('default','/scale','needle',2,10240)") + indexed_grep_seconds = time.perf_counter() - started + ctx.equal(indexed_grep["index_used"], True, "数据库 grep 使用 trigram 索引") + ctx.equal(indexed_grep["match_count"], expected_matches, "trigram grep 命中数") + ctx.equal(indexed_grep["files_scanned"], expected_matches, + "trigram grep 只读取候选文件") + rss_after_index = self_max_rss_bytes() + ctx.check(rss_after_index <= memory_budget, + f"规模测试峰值内存不超过 {memory_budget / MIB:.0f} MiB") + ctx.equal(reopened.scalar("PRAGMA integrity_check"), "ok", "规模目录树完整性") + storage = db_storage(database_path) + reopened.close() + + create_budget = {"quick": 120.0, "full": 900.0, "stress": 7_200.0}[ + ctx.mode.name] + ctx.budget("scale_tree_create_seconds", create_seconds, create_budget) + ctx.budget("scale_tree_count_seconds", count_seconds, 60.0) + ctx.budget("scale_tree_grep_seconds", grep_seconds, + {"quick": 5.0, "full": 30.0, "stress": 300.0}[ctx.mode.name]) + ctx.budget("scale_tree_index_build_seconds", index_build_seconds, + {"quick": 10.0, "full": 60.0, "stress": 600.0}[ctx.mode.name]) + ctx.budget("scale_tree_indexed_grep_seconds", indexed_grep_seconds, + {"quick": 1.0, "full": 5.0, "stress": 30.0}[ctx.mode.name]) + return { + "files": file_count, + "directories": directory_count, + "create_seconds": round(create_seconds, 6), + "create_files_per_second": round(file_count / max(create_seconds, 1e-9), 3), + "count_seconds": round(count_seconds, 6), + "sample_reads": sample_count, + "sample_seconds": round(sample_seconds, 6), + "database_grep_seconds": round(grep_seconds, 6), + "database_grep_matches": grep["match_count"], + "database_grep_files_per_second": round( + file_count / max(grep_seconds, 1e-9), 3), + "index_build_seconds": round(index_build_seconds, 6), + "indexed_grep_seconds": round(indexed_grep_seconds, 6), + "indexed_grep_matches": indexed_grep["match_count"], + "indexed_grep_candidates": indexed_grep["files_scanned"], + "max_rss_before_bytes": rss_before, + "max_rss_after_index_bytes": rss_after_index, + "memory_budget_bytes": memory_budget, + "storage": storage, + } + + +@case("stability.mixed-reopen-soak", "stability", + "持续混合读写、元数据、快照、checkpoint 和数据库重开") +def stability_mixed_reopen_soak(ctx: Context) -> dict[str, Any]: + default_seconds = {"quick": 5, "full": 60, "stress": 900}[ctx.mode.name] + duration_seconds = max(1, int(os.environ.get( + "VEXFS_EVAL_SOAK_SECONDS", str(default_seconds)))) + rng = random.Random(ctx.seed ^ 0x50A4) + with tempfile.TemporaryDirectory(prefix="vexfs-soak-") as directory: + database_path = Path(directory) / "soak.sqlite3" + db = Database(ctx, database_path) + db.scalar("SELECT vexfs_mkdir('default','/soak')") + expected: dict[int, bytes] = {} + for index in range(128): + value = f"seed-{index}".encode() + db.scalar("SELECT vexfs_write('default',?,?)", + (f"/soak/f{index:03d}.txt", value)) + expected[index] = value + + operations = 0 + reopen_count = 0 + checkpoint_count = 0 + snapshot_count = 0 + started = time.perf_counter() + deadline = started + duration_seconds + while time.perf_counter() < deadline: + index = rng.randrange(128) + path = f"/soak/f{index:03d}.txt" + operation = operations % 6 + if operation in (0, 1): + value = hashlib.sha256(f"{ctx.seed}:{operations}:{index}".encode()).digest() + db.scalar("SELECT vexfs_write('default',?,?)", (path, value)) + expected[index] = value + elif operation == 2: + ctx.equal(db.scalar("SELECT vexfs_read('default',?)", (path,)), + expected[index], "soak 混合读取") + elif operation == 3: + inode = db.json("SELECT vexfs_stat('default',?)", (path,))["inode"] + mode = 0o600 | (operations & 0o77) + db.scalar("SELECT vexfs_set_mode('default',?,?)", (inode, mode)) + elif operation == 4: + inode = db.json("SELECT vexfs_stat('default',?)", (path,))["inode"] + db.scalar("SELECT vexfs_xattr_set('default',?,'user.soak',?,0)", + (inode, str(operations).encode())) + else: + temporary = f"/soak/.tmp-{index:03d}.txt" + db.scalar("SELECT vexfs_rename('default',?,?,0)", (path, temporary)) + db.scalar("SELECT vexfs_rename('default',?,?,0)", (temporary, path)) + operations += 1 + + if operations % 200 == 0: + db.close() + db = Database(ctx, database_path) + reopen_count += 1 + sample = rng.randrange(128) + ctx.equal(db.scalar("SELECT vexfs_read('default',?)", + (f"/soak/f{sample:03d}.txt",)), expected[sample], + "soak 重开抽样") + if operations % 2_000 == 0: + name = f"soak-{snapshot_count:06d}" + db.scalar("SELECT vexfs_snapshot_create('default',?)", (name,)) + ctx.equal(db.json( + "SELECT vexfs_snapshot_diff('default',?,'HEAD')", (name,))["changes"], + [], "soak 快照创建后无差异") + db.scalar("SELECT vexfs_snapshot_drop('default',?)", (name,)) + snapshot_count += 1 + if operations % 5_000 == 0: + db.connection.execute("PRAGMA wal_checkpoint(PASSIVE)").fetchone() + ctx.equal(db.scalar("PRAGMA integrity_check"), "ok", "soak 中途完整性") + checkpoint_count += 1 + + elapsed = time.perf_counter() - started + for index in range(128): + ctx.equal(db.scalar("SELECT vexfs_read('default',?)", + (f"/soak/f{index:03d}.txt",)), expected[index], + "soak 最终内容") + ctx.equal(db.scalar("PRAGMA integrity_check"), "ok", "soak 最终完整性") + pending = db.scalar("SELECT count(*) FROM _vexfs_staging") + ctx.equal(pending, 0, "soak 没有未发布 staging") + storage = db_storage(database_path) + db.close() + ctx.check(elapsed >= duration_seconds * 0.95, "soak 达到目标时长") + return { + "target_seconds": duration_seconds, + "elapsed_seconds": round(elapsed, 6), + "operations": operations, + "operations_per_second": round(operations / max(elapsed, 1e-9), 3), + "reopens": reopen_count, + "snapshots": snapshot_count, + "checkpoints": checkpoint_count, + "pending_staging": pending, + "storage": storage, + } + + @case("performance.sequential-file", "performance", "大文件写入、读取、重开和哈希") def performance_sequential_file(ctx: Context) -> dict[str, Any]: payload = (bytes(range(256)) * (ctx.mode.sequential_mib * MIB // 256)) @@ -1239,6 +2284,71 @@ def performance_sequential_file(ctx: Context) -> dict[str, Any]: "read_mib_per_second": round(len(payload) / MIB / max(read_seconds, 1e-9), 3)} +@case("performance.metadata-hardlinks", "performance", + "owner、ACL 和硬链接批量操作性能") +def performance_metadata_hardlinks(ctx: Context) -> dict[str, Any]: + file_count = {"quick": 100, "full": 500, "stress": 1_000}[ctx.mode.name] + with Database(ctx) as db: + db.scalar("SELECT vexfs_mkdir('default','/metadata')") + db.scalar("SELECT vexfs_mkdir('default','/metadata/links')") + for index in range(file_count): + db.scalar("SELECT vexfs_write('default',?,X'7061796C6F6164')", + (f"/metadata/f{index:05d}.txt",)) + + started = time.perf_counter() + for index in range(file_count): + inode = db.json( + "SELECT vexfs_stat('default',?)", + (f"/metadata/f{index:05d}.txt",))["inode"] + db.scalar("SELECT vexfs_chown('default',?,?,?)", (inode, 1000 + index, 2000)) + chown_seconds = time.perf_counter() - started + + started = time.perf_counter() + for index in range(file_count): + inode = db.json( + "SELECT vexfs_stat('default',?)", + (f"/metadata/f{index:05d}.txt",))["inode"] + db.scalar("SELECT vexfs_acl_grant('default',?,?,?)", + (inode, "agent", "read,write")) + acl_seconds = time.perf_counter() - started + + started = time.perf_counter() + for index in range(file_count): + db.scalar("SELECT vexfs_link('default',?,?)", + (f"/metadata/f{index:05d}.txt", + f"/metadata/links/f{index:05d}.txt")) + hardlink_seconds = time.perf_counter() - started + + ctx.equal(db.scalar( + "SELECT json_array_length(vexfs_list('default','/metadata/links'))"), + file_count, + "批量硬链接数量") + sample_path = "/metadata/f00000.txt" + sample = db.json("SELECT vexfs_stat('default',?)", (sample_path,)) + ctx.equal((sample["link_count"], sample["uid"], sample["gid"]), + (2, 1000, 2000), "批量 metadata 结果") + acl = db.json("SELECT vexfs_acl_get('default',?)", (sample["inode"],)) + ctx.equal(acl[0]["principal"], "agent", "批量 ACL 结果") + + started = time.perf_counter() + snapshot_commit = db.scalar( + "SELECT vexfs_snapshot_create('default','metadata-hardlinks')") + snapshot_seconds = time.perf_counter() - started + + ctx.budget("metadata_chown_seconds", chown_seconds, 60.0) + ctx.budget("metadata_acl_seconds", acl_seconds, 60.0) + ctx.budget("metadata_hardlink_seconds", hardlink_seconds, 60.0) + ctx.budget("metadata_snapshot_seconds", snapshot_seconds, 30.0) + return {"files": file_count, "snapshot_commit": snapshot_commit, + "chown_seconds": round(chown_seconds, 6), + "acl_seconds": round(acl_seconds, 6), + "hardlink_seconds": round(hardlink_seconds, 6), + "snapshot_seconds": round(snapshot_seconds, 6), + "chown_ops_per_second": round(file_count / max(chown_seconds, 1e-9), 3), + "acl_ops_per_second": round(file_count / max(acl_seconds, 1e-9), 3), + "hardlink_ops_per_second": round(file_count / max(hardlink_seconds, 1e-9), 3)} + + @case("performance.staged-overwrite", "performance", "分块 staging 写入的空间上界和吞吐") def performance_staged_overwrite(ctx: Context) -> dict[str, Any]: total = ctx.mode.staged_mib * MIB @@ -1392,6 +2502,44 @@ def performance_version_history(ctx: Context) -> dict[str, Any]: "storage_after_checkpoint": storage_after_checkpoint} +@case("performance.integrity-check", "performance", + "完整性检查按 64 KiB 流式扫描，记录快速检查与 SHA-256 吞吐和内存上界") +def performance_integrity_check(ctx: Context) -> dict[str, Any]: + file_count = {"quick": 4, "full": 32, "stress": 64}[ctx.mode.name] + payload = bytes((index * 31 + 7) % 251 for index in range(MIB)) + logical_bytes = file_count * len(payload) + with Database(ctx) as db: + for index in range(file_count): + db.scalar("SELECT vexfs_write('default',?,?)", + (f"/check-{index:04d}.bin", payload)) + before_rss = self_max_rss_bytes() + quick_started = time.perf_counter() + quick = db.json("SELECT vexfs_check('default',0)") + quick_seconds = time.perf_counter() - quick_started + deep_started = time.perf_counter() + deep = db.json("SELECT vexfs_check('default',1)") + deep_seconds = time.perf_counter() - deep_started + after_rss = self_max_rss_bytes() + ctx.check(quick["ok"] and deep["ok"], "性能数据集检查通过") + ctx.equal(deep["checked"]["content_bytes"], logical_bytes, + "深度检查扫描所有 canonical BLOB") + ctx.equal(deep["checked"]["versions"], file_count, "版本计数") + ctx.budget("integrity_check_seconds", deep_seconds, + {"quick": 10.0, "full": 45.0, "stress": 90.0}[ctx.mode.name]) + # ru_maxrss 是进程历史峰值；这里只约束检查阶段没有出现灾难性额外增长。 + rss_growth = max(0, after_rss - before_rss) + ctx.check(rss_growth <= 64 * MIB, + f"流式检查阶段 RSS 峰值增长过大: {rss_growth} bytes") + return {"files": file_count, + "logical_bytes": logical_bytes, + "quick_seconds": round(quick_seconds, 6), + "deep_seconds": round(deep_seconds, 6), + "deep_mib_per_second": round( + logical_bytes / MIB / max(deep_seconds, 1e-9), 3), + "reported_elapsed_ms": deep["elapsed_ms"], + "rss_growth_bytes": rss_growth} + + @case("limits.maximum-file", "limits", "128 MiB 文件上限和越界拒绝", modes=("full", "stress")) def maximum_file(ctx: Context) -> dict[str, Any]: with Database(ctx) as db: @@ -1470,24 +2618,27 @@ def fskit_unsigned_build(ctx: Context) -> dict[str, Any]: if any(marker in mount_output for marker in fskit_mount_markers): raise EvalSkip("已有 FSKit 文件系统挂载，跳过会刷新扩展缓存的无签名构建检查") derived = ctx.output_dir / "xcode-derived-data" + build_env = dict(os.environ) + build_env["CCACHE_DIR"] = str(ctx.output_dir / "ccache") started = time.perf_counter() app: Path | None = None try: result = run_process( ["xcodebuild", "-project", str(project), "-scheme", "VexFSApp", "-configuration", "Debug", "-derivedDataPath", str(derived), - "CODE_SIGNING_ALLOWED=NO", "build"], timeout=300, cwd=ctx.root) + "ARCHS=arm64", "ONLY_ACTIVE_ARCH=YES", "CODE_SIGNING_ALLOWED=NO", "build"], + timeout=300, cwd=ctx.root, env=build_env) seconds = time.perf_counter() - started output = result.stdout + result.stderr ctx.check(b"BUILD SUCCEEDED" in output, "Xcode 没有 BUILD SUCCEEDED") - app = next(derived.glob("Build/Products/Debug/VexFS.app"), None) - ctx.check(app is not None, "缺少 VexFS.app") + app = next(derived.glob("Build/Products/Debug/VexDB Lite.app"), None) + ctx.check(app is not None, "缺少 VexDB Lite.app") return {"seconds": round(seconds, 6), "app": str(app)} finally: # xcodebuild 会把测试产物登记到 LaunchServices。若不撤销登记，未签名的 # Debug extension 会和 /Applications 中的正式开发签名版本争用同一标识。 if app is None: - app = next(derived.glob("Build/Products/Debug/VexFS.app"), None) + app = next(derived.glob("Build/Products/Debug/VexDB Lite.app"), None) lsregister = Path( "/System/Library/Frameworks/CoreServices.framework/Frameworks/" "LaunchServices.framework/Support/lsregister") @@ -1502,11 +2653,15 @@ def fskit_unsigned_build(ctx: Context) -> dict[str, Any]: # 撤销同标识的测试 App 后，LaunchServices 也可能把 /Applications 中的 # 正式扩展从 FSKit 的可用列表移除，并让 fskit_agent 保留失效进程缓存。 # 立即恢复正式版本并刷新空闲代理，保证后续真实 mount 和用户环境不受影响。 - installed_app = Path("/Applications/VexFS.app") - installed_extension = ( - installed_app / "Contents/Extensions/VexFSAppEx.appex") - if (lsregister.exists() and installed_app.exists() and - installed_extension.exists()): + installed_apps = [ + Path.home() / "Applications/VexDB Lite.app", + Path("/Applications/VexDB Lite.app"), + Path("/Applications/VexFS.app"), + ] + installed_app = next((candidate for candidate in installed_apps + if candidate.exists()), installed_apps[0]) + installed_extension = installed_app / "Contents/Extensions/VexFSAppEx.appex" + if (lsregister.exists() and installed_extension.exists()): run_process([str(lsregister), "-f", str(installed_app)], check=False, timeout=30) if shutil.which("pluginkit") is not None: @@ -1523,6 +2678,966 @@ def fskit_unsigned_build(ctx: Context) -> dict[str, Any]: time.sleep(2.0) +def mount_cli(ctx: Context) -> Path: + """FSKit 只向受信任调用方公开扩展；发行 Gate 必须显式绑定被测 CLI。""" + if ctx.mount_cli_override is not None: + return ctx.mount_cli_override + installed = Path.home() / ".local/bin/vexfs" + return installed if installed.exists() else ctx.cli + + +def filesystem_cli_prefix(cli: Path, database: Path, workspace: str) -> list[str]: + """同时支持独立 vexfs 名称和统一 vexdb fs 入口。""" + name = cli.stem.lower() if cli.suffix.lower() == ".exe" else cli.name.lower() + command = [str(cli)] if name == "vexfs" else [str(cli), "fs"] + return command + ["--db", str(database), "--workspace", workspace] + + +MOUNT_CONFORMANCE_VERSION = 1 +MOUNT_CONFORMANCE_CAPABILITIES = ( + "create-exclusive", + "range-io", + "fsync", + "append", + "truncate", + "rename-replace", + "chmod-exec", + "hardlink", + "symlink", + "xattr", + "unicode-name", + "stable-errno", + "snapshot-restore", + "remount", +) + + +@dataclass(frozen=True) +class RealMountAdapter: + name: str + cli: Path + database: Path + mount_point: Path + prefix: list[str] + doctor_before: dict[str, Any] + + +def prepare_real_mount_adapter(ctx: Context, base: Path, + workspace: str, + database: Path | None = None) -> RealMountAdapter: + """为当前系统准备真实 mount；测试主体不包含平台分支。""" + system = platform.system() + if system == "Darwin": + name = "fskit" + cli = mount_cli(ctx) + elif system == "Linux": + name = "libfuse3" + cli = ctx.cli + helper = ctx.build_dir / "vexfs-fuse" + if not helper.exists(): + raise EvalSkip("当前构建没有 vexfs-fuse helper") + if not Path("/dev/fuse").exists() or not os.access( + "/dev/fuse", os.R_OK | os.W_OK): + raise EvalSkip("当前环境没有可读写的 /dev/fuse") + if shutil.which("fusermount3") is None: + raise EvalSkip("当前环境没有 fusermount3") + else: + raise EvalSkip("共享 mount 一致性 eval 当前支持 macOS 和 Linux") + + database = database or base / "mount.sqlite3" + mount_point = base / "mnt" + prefix = filesystem_cli_prefix(cli, database, workspace) + run_process(prefix + ["setup"]) + doctor = run_process(prefix + ["--json", "doctor"], check=False) + try: + details = json.loads(doctor.stdout) + except json.JSONDecodeError as error: + raise EvalFailure( + f"{name} doctor 没有返回 JSON: {doctor.stdout.decode(errors='replace')}") from error + if system == "Darwin" and details.get("extension") != "enabled": + raise EvalSkip( + f"FSKit extension={details.get('extension', 'unknown')}，需安装并在系统设置中启用") + if system == "Darwin": + module_path = details.get("extension_path", "") + installed_extensions = { + str(path.resolve()) + for path in ( + Path.home() / "Applications/VexDB Lite.app/Contents/Extensions/VexFSAppEx.appex", + Path("/Applications/VexDB Lite.app/Contents/Extensions/VexFSAppEx.appex"), + ) + if path.exists() + } + ctx.check(bool(module_path), "doctor 必须返回 FSKit 真实 module URL") + ctx.check(str(Path(module_path).resolve()) in installed_extensions, + "FSKit module URL 必须指向正式安装 App，不能指向 archive/DerivedData") + if system == "Linux" and not details.get("mount_ready"): + raise EvalSkip(f"libfuse3 adapter={details.get('extension', 'unknown')}") + mount_point.mkdir() + return RealMountAdapter(name, cli, database, mount_point, prefix, details) + + +def expect_mount_errno(ctx: Context, operation: Callable[[], Any], expected: int, + message: str) -> None: + ctx.checks += 1 + try: + operation() + except OSError as error: + if error.errno != expected: + raise EvalFailure( + f"{message}: expected errno={expected}, actual={error.errno} ({error})") + return + raise EvalFailure(f"{message}: 预期 errno={expected} 但操作成功") + + +def system_setxattr(path: Path, name: str, value: bytes) -> None: + if hasattr(os, "setxattr"): + os.setxattr(path, name, value) + return + libc = ctypes.CDLL(None, use_errno=True) + function = libc.setxattr + if platform.system() == "Darwin": + function.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_void_p, + ctypes.c_size_t, ctypes.c_uint32, ctypes.c_int] + arguments = (os.fsencode(path), os.fsencode(name), + ctypes.create_string_buffer(value), len(value), 0, 0) + else: + function.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_void_p, + ctypes.c_size_t, ctypes.c_int] + arguments = (os.fsencode(path), os.fsencode(name), + ctypes.create_string_buffer(value), len(value), 0) + function.restype = ctypes.c_int + if function(*arguments) != 0: + code = ctypes.get_errno() + raise OSError(code, os.strerror(code), os.fspath(path)) + + +def system_getxattr(path: Path, name: str) -> bytes: + if hasattr(os, "getxattr"): + return os.getxattr(path, name) + libc = ctypes.CDLL(None, use_errno=True) + function = libc.getxattr + common = (os.fsencode(path), os.fsencode(name)) + if platform.system() == "Darwin": + function.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_void_p, + ctypes.c_size_t, ctypes.c_uint32, ctypes.c_int] + suffix = (0, 0) + else: + function.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_void_p, + ctypes.c_size_t] + suffix = () + function.restype = ctypes.c_ssize_t + size = function(*common, None, 0, *suffix) + if size < 0: + code = ctypes.get_errno() + raise OSError(code, os.strerror(code), os.fspath(path)) + buffer = ctypes.create_string_buffer(max(size, 1)) + result = function(*common, buffer, size, *suffix) + if result < 0: + code = ctypes.get_errno() + raise OSError(code, os.strerror(code), os.fspath(path)) + return buffer.raw[:result] + + +def system_listxattr(path: Path) -> list[str]: + if hasattr(os, "listxattr"): + return os.listxattr(path) + libc = ctypes.CDLL(None, use_errno=True) + function = libc.listxattr + if platform.system() == "Darwin": + function.argtypes = [ctypes.c_char_p, ctypes.c_void_p, ctypes.c_size_t, + ctypes.c_int] + suffix = (0,) + else: + function.argtypes = [ctypes.c_char_p, ctypes.c_void_p, ctypes.c_size_t] + suffix = () + function.restype = ctypes.c_ssize_t + encoded_path = os.fsencode(path) + size = function(encoded_path, None, 0, *suffix) + if size < 0: + code = ctypes.get_errno() + raise OSError(code, os.strerror(code), os.fspath(path)) + buffer = ctypes.create_string_buffer(max(size, 1)) + result = function(encoded_path, buffer, size, *suffix) + if result < 0: + code = ctypes.get_errno() + raise OSError(code, os.strerror(code), os.fspath(path)) + return [os.fsdecode(item) for item in buffer.raw[:result].split(b"\0") if item] + + +def mount_adapter_mount(ctx: Context, adapter: RealMountAdapter) -> list[dict[str, Any]]: + run_process(adapter.prefix + ["mount", str(adapter.mount_point)], timeout=60) + status = run_process( + adapter.prefix + ["--json", "mount", "status", str(adapter.mount_point)]) + mounts = json.loads(status.stdout) + ctx.equal(len(mounts), 1, f"{adapter.name} 挂载状态数量") + ctx.equal(mounts[0]["target"], os.path.realpath(adapter.mount_point), + f"{adapter.name} 挂载目标") + return mounts + + +def mount_adapter_unmount(ctx: Context, adapter: RealMountAdapter, + label: str) -> None: + result = run_process(adapter.prefix + ["unmount", str(adapter.mount_point)], + check=False, timeout=60) + ctx.equal(result.returncode, 0, f"{adapter.name} {label}卸载退出码") + if adapter.name == "fskit": + # FSKit 在 umount 返回后仍会异步释放 volume identity；重挂载前给系统收尾时间。 + time.sleep(2.0) + + +def run_shared_mount_operations(ctx: Context, adapter: RealMountAdapter) -> dict[str, Any]: + """只使用跨平台系统调用；macOS/Linux 必须执行完全相同的断言。""" + root = adapter.mount_point / "conformance" + left = root / "left" + right = root / "right" + left.mkdir(parents=True) + right.mkdir() + + data = left / "data.txt" + descriptor = os.open(data, os.O_CREAT | os.O_EXCL | os.O_RDWR, 0o600) + try: + ctx.equal(os.write(descriptor, b"alpha\nbeta\n"), 11, "共享合同顺序写") + ctx.equal(os.pwrite(descriptor, b"AGENT", 6), 5, "共享合同 range write") + os.fsync(descriptor) + ctx.equal(os.pread(descriptor, 11, 0), b"alpha\nAGENT", "共享合同 range read") + finally: + os.close(descriptor) + data.chmod(0o640) + expect_mount_errno( + ctx, lambda: os.close(os.open(data, os.O_CREAT | os.O_EXCL | os.O_WRONLY)), + errno.EEXIST, "O_EXCL 已存在文件") + + descriptor = os.open(data, os.O_WRONLY | os.O_APPEND) + try: + ctx.equal(os.write(descriptor, b"\nagent\n"), 7, "共享合同 append") + os.fsync(descriptor) + finally: + os.close(descriptor) + ctx.equal(data.read_bytes(), b"alpha\nAGENT\nagent\n", "共享合同 close-to-open") + + copied = left / "copied.txt" + shutil.copyfile(data, copied) + moved = right / "moved.txt" + copied.rename(moved) + replacement = left / "replacement.txt" + replacement.write_bytes(b"replacement") + os.replace(replacement, moved) + ctx.equal(moved.read_bytes(), b"replacement", "共享合同 rename replace") + os.truncate(moved, 4) + ctx.equal(moved.read_bytes(), b"repl", "共享合同 truncate") + + hardlink = right / "data-hard.txt" + os.link(data, hardlink) + data_stat = data.stat() + link_stat = hardlink.stat() + ctx.equal(data_stat.st_ino, link_stat.st_ino, "共享合同 hardlink inode") + ctx.equal(data_stat.st_nlink, 2, "共享合同 hardlink link count") + with hardlink.open("ab") as stream: + stream.write(b"hard\n") + stream.flush() + os.fsync(stream.fileno()) + ctx.check(data.read_bytes().endswith(b"hard\n"), "共享合同 hardlink 共享内容") + + symlink = right / "data-link.txt" + symlink.symlink_to("../left/data.txt") + dangling = right / "dangling.txt" + dangling.symlink_to("missing.txt") + ctx.equal(os.readlink(symlink), "../left/data.txt", "共享合同 symlink target") + ctx.equal(symlink.read_bytes(), data.read_bytes(), "共享合同 symlink 读取") + ctx.check(dangling.is_symlink() and not dangling.exists(), "共享合同 dangling symlink") + + unicode_file = left / "你好-agent-🚀.txt" + unicode_file.write_text("跨平台\n", encoding="utf-8") + ctx.equal(unicode_file.read_text(encoding="utf-8"), "跨平台\n", "共享合同 Unicode") + + xattr_name = "user.vexfs.conformance" + xattr_value = b"stored-in-database\x00binary" + system_setxattr(data, xattr_name, xattr_value) + ctx.equal(system_getxattr(data, xattr_name), xattr_value, "共享合同 xattr 读取") + ctx.check(xattr_name in system_listxattr(data), "共享合同 xattr 列表") + + script = root / "run.sh" + script.write_text("#!/bin/sh\nprintf 'vexfs-conformance\\n'\n", encoding="utf-8") + script.chmod(0o755) + ctx.equal(script.stat().st_mode & 0o777, 0o755, "共享合同 executable mode") + ctx.equal(run_process([str(script)], cwd=root).stdout, b"vexfs-conformance\n", + "共享合同直接执行") + + ctx.equal(data.stat().st_uid, os.getuid(), "共享合同 uid") + ctx.equal(data.stat().st_gid, os.getgid(), "共享合同 gid") + ctx.check(os.statvfs(adapter.mount_point).f_bsize > 0, "共享合同 statvfs") + expect_mount_errno(ctx, lambda: (root / "missing.txt").read_bytes(), + errno.ENOENT, "不存在路径") + expect_mount_errno(ctx, lambda: left.mkdir(), errno.EEXIST, "重复目录") + expect_mount_errno(ctx, lambda: left.rmdir(), errno.ENOTEMPTY, "非空目录删除") + + return { + "data": data.read_bytes(), + "mode": data.stat().st_mode & 0o777, + "hardlink_inode_equal": data.stat().st_ino == hardlink.stat().st_ino, + "hardlink_count": data.stat().st_nlink, + "symlink_target": os.readlink(symlink), + "xattr_name": xattr_name, + "xattr_value": system_getxattr(data, xattr_name), + "unicode": unicode_file.read_text(encoding="utf-8"), + "script_output": run_process([str(script)], cwd=root).stdout, + "entries": sorted(path.name for path in root.iterdir()), + } + + +def verify_shared_mount_after_restore(ctx: Context, adapter: RealMountAdapter, + expected: dict[str, Any]) -> dict[str, Any]: + root = adapter.mount_point / "conformance" + data = root / "left/data.txt" + hardlink = root / "right/data-hard.txt" + symlink = root / "right/data-link.txt" + unicode_file = root / "left/你好-agent-🚀.txt" + script = root / "run.sh" + ctx.equal(data.read_bytes(), expected["data"], "恢复后内容") + ctx.equal(data.stat().st_mode & 0o777, expected["mode"], "恢复后 mode") + ctx.equal(data.stat().st_ino, hardlink.stat().st_ino, "恢复后 hardlink inode") + ctx.equal(data.stat().st_nlink, expected["hardlink_count"], "恢复后 hardlink count") + ctx.equal(os.readlink(symlink), expected["symlink_target"], "恢复后 symlink") + ctx.equal(system_getxattr(data, expected["xattr_name"]), expected["xattr_value"], + "恢复后 xattr") + ctx.equal(unicode_file.read_text(encoding="utf-8"), expected["unicode"], + "恢复后 Unicode") + ctx.equal(run_process([str(script)], cwd=root).stdout, expected["script_output"], + "恢复后可执行文件") + ctx.equal(sorted(path.name for path in root.iterdir()), expected["entries"], + "恢复后目录树") + return { + "bytes": len(expected["data"]), + "mode": expected["mode"], + "hardlink_count": data.stat().st_nlink, + "symlink_target": os.readlink(symlink), + "xattrs": system_listxattr(data), + "unicode_name": unicode_file.name, + } + + +@case("mount.cross-platform-conformance", "mount", + "macOS FSKit 与 Linux FUSE 执行同一份挂载、元数据、快照和重挂载合同") +def cross_platform_mount_conformance(ctx: Context) -> dict[str, Any]: + with tempfile.TemporaryDirectory(prefix="vexfs-mount-conformance-") as directory: + base = Path(directory) + adapter = prepare_real_mount_adapter(ctx, base, "conformance") + mounts = mount_adapter_mount(ctx, adapter) + expected: dict[str, Any] + try: + expected = run_shared_mount_operations(ctx, adapter) + reverse = run_process(adapter.prefix + ["cat", "/conformance/left/data.txt"]) + ctx.equal(reverse.stdout, expected["data"], "挂载写入可由数据库 CLI 读取") + finally: + mount_adapter_unmount(ctx, adapter, "首次") + + snapshot = run_process(adapter.prefix + ["snapshot", "create", "baseline"]) + snapshot_commit = int(snapshot.stdout.strip()) + ctx.check(snapshot_commit > 0, "共享合同 snapshot commit") + run_process(adapter.prefix + ["write", "/conformance/left/data.txt"], + input_data=b"mutated-after-snapshot") + run_process(adapter.prefix + ["rm", "/conformance/left/你好-agent-🚀.txt"]) + diff = run_process(adapter.prefix + ["snapshot", "diff", "baseline"], check=False) + ctx.equal(diff.returncode, 1, "共享合同 snapshot diff 有变化") + changes = json.loads(diff.stdout)["changes"] + changed_paths = {change["path"] for change in changes} + ctx.check("/conformance/left/data.txt" in changed_paths, "snapshot diff 内容变化") + ctx.check("/conformance/left/你好-agent-🚀.txt" in changed_paths, + "snapshot diff Unicode 删除") + mount_adapter_mount(ctx, adapter) + try: + ctx.equal((adapter.mount_point / "conformance/left/data.txt").read_bytes(), + b"mutated-after-snapshot", "恢复前挂载视图包含修改内容") + restore = json.loads(run_process( + adapter.prefix + ["--json", "snapshot", "restore", "baseline"], + timeout=120).stdout) + ctx.equal(restore["remounted"], True, "挂载状态恢复自动重挂载") + ctx.equal(restore["mount_point"], os.path.realpath(adapter.mount_point), + "快照恢复保持原挂载点") + status = json.loads(run_process( + adapter.prefix + ["--json", "mount", "status", + str(adapter.mount_point)]).stdout) + ctx.equal(len(status), 1, "快照恢复后挂载仍然存在") + ctx.equal(status[0]["database"], os.path.realpath(adapter.database), + "挂载身份记录数据库") + ctx.equal(status[0]["workspace"], "conformance", "挂载身份记录 workspace") + clean_diff = run_process(adapter.prefix + ["snapshot", "diff", "baseline"]) + ctx.equal(json.loads(clean_diff.stdout)["changes"], [], + "snapshot restore 后无差异") + restored = verify_shared_mount_after_restore(ctx, adapter, expected) + finally: + mount_adapter_unmount(ctx, adapter, "第二次") + + after = json.loads(run_process(adapter.prefix + ["--json", "doctor"]).stdout) + ctx.equal(after["mount_count"], 0, "共享合同卸载后无挂载") + ctx.equal(after["database"]["pending_handles"], 0, "共享合同无未发布句柄") + ctx.equal(after["database"]["retained_handles"], 0, "共享合同无保留句柄") + ctx.equal(after["database"]["staging_bytes"], 0, "共享合同无暂存字节") + return { + "contract_version": MOUNT_CONFORMANCE_VERSION, + "adapter": adapter.name, + "capabilities": list(MOUNT_CONFORMANCE_CAPABILITIES), + "doctor_before": adapter.doctor_before, + "doctor_after": after, + "mounts": mounts, + "snapshot_commit": snapshot_commit, + "restored": restored, + } + + +PORTABILITY_WORKSPACE = "cross-os" +PORTABILITY_ACL = [ + {"principal": "agent-portable", "effect": "allow", + "permissions": "read,write", "inherit": 1}, +] +PORTABILITY_XATTR = "user.vexfs.portable" +PORTABILITY_XATTR_VALUE = b"mac-linux-mac\x00metadata" + + +def verify_portable_tree(ctx: Context, adapter: RealMountAdapter, + plan_content: bytes, linux_file: bool) -> None: + root = adapter.mount_point / "portable" + unicode_dir = root / "你好-🚀" + plan = unicode_dir / "plan.txt" + hardlink = root / "plan-hard.txt" + symlink = unicode_dir / "plan-link.txt" + script = root / "build.sh" + ctx.equal(plan.read_bytes(), plan_content, "跨系统 plan 内容") + ctx.equal(plan.stat().st_ino, hardlink.stat().st_ino, "跨系统 hardlink inode") + ctx.equal(plan.stat().st_nlink, 2, "跨系统 hardlink 数量") + ctx.equal(os.readlink(symlink), "plan.txt", "跨系统 symlink target") + ctx.equal(symlink.read_bytes(), plan_content, "跨系统 symlink 读取") + ctx.equal(plan.stat().st_mode & 0o777, 0o640, "跨系统文件 mode") + visible_owner = ((os.getuid(), os.getgid()) if adapter.name == "fskit" + else (4242, 4343)) + ctx.equal((plan.stat().st_uid, plan.stat().st_gid), visible_owner, + "跨系统 owner 映射") + ctx.equal(system_getxattr(plan, PORTABILITY_XATTR), PORTABILITY_XATTR_VALUE, + "跨系统 xattr") + ctx.equal(script.stat().st_mode & 0o777, 0o755, "跨系统可执行 mode") + ctx.equal(run_process([str(script)], cwd=root).stdout, b"portable-script\n", + "跨系统执行脚本") + ctx.equal((root / "from-linux.txt").exists(), linux_file, + "Linux 新文件可见性") + if linux_file: + ctx.equal((root / "from-linux.txt").read_text(encoding="utf-8"), + "created on linux\n", "Linux 新文件内容") + + +def verify_portable_acl(ctx: Context, adapter: RealMountAdapter) -> None: + acl = json.loads(run_process( + adapter.prefix + ["getfacl", "/portable/你好-🚀/plan.txt"]).stdout) + ctx.equal(acl, PORTABILITY_ACL, "跨系统便携 ACL") + + +def verify_portable_database_owner(ctx: Context, adapter: RealMountAdapter) -> None: + record = json.loads(run_process( + adapter.prefix + ["stat", "/portable/你好-🚀/plan.txt"]).stdout) + ctx.equal((record["uid"], record["gid"]), (4242, 4343), + "数据库原样保存数字 owner") + + +@case("portability.cross-os-roundtrip", "portability", + "同一 SQLite 工作区按 macOS 创建、Linux 修改、macOS 回读三个阶段往返") +def cross_os_portability_roundtrip(ctx: Context) -> dict[str, Any]: + phase = os.environ.get("VEXFS_PORTABILITY_PHASE", "") + database_value = os.environ.get("VEXFS_PORTABILITY_DB", "") + if not phase or not database_value: + raise EvalSkip( + "需通过 run_cross_platform_portability.sh 设置跨系统往返阶段和共享数据库") + if phase not in {"mac-create", "linux-roundtrip", "mac-verify"}: + raise EvalFailure(f"未知跨系统往返阶段：{phase}") + expected_system = "Linux" if phase == "linux-roundtrip" else "Darwin" + if platform.system() != expected_system: + raise EvalFailure( + f"阶段 {phase} 必须在 {expected_system} 执行，当前为 {platform.system()}") + + database = Path(database_value).resolve() + database.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix=f"vexfs-portability-{phase}-") as directory: + adapter = prepare_real_mount_adapter( + ctx, Path(directory), PORTABILITY_WORKSPACE, database=database) + + if phase == "mac-create": + mount_adapter_mount(ctx, adapter) + try: + root = adapter.mount_point / "portable" + unicode_dir = root / "你好-🚀" + unicode_dir.mkdir(parents=True) + plan = unicode_dir / "plan.txt" + plan.write_bytes(b"mac-v1\n") + plan.chmod(0o640) + os.link(plan, root / "plan-hard.txt") + (unicode_dir / "plan-link.txt").symlink_to("plan.txt") + system_setxattr(plan, PORTABILITY_XATTR, PORTABILITY_XATTR_VALUE) + script = root / "build.sh" + script.write_text( + "#!/bin/sh\nprintf 'portable-script\\n'\n", encoding="utf-8") + script.chmod(0o755) + finally: + mount_adapter_unmount(ctx, adapter, "macOS 创建") + + run_process(adapter.prefix + [ + "chown", "4242:4343", "/portable/你好-🚀/plan.txt"]) + run_process(adapter.prefix + ["setfacl", "/portable/你好-🚀/plan.txt"], + input_data=json.dumps(PORTABILITY_ACL).encode()) + verify_portable_acl(ctx, adapter) + verify_portable_database_owner(ctx, adapter) + snapshot = int(run_process( + adapter.prefix + ["snapshot", "create", "mac-baseline"] + ).stdout.strip()) + ctx.check(snapshot > 0, "macOS baseline snapshot") + return {"phase": phase, "adapter": adapter.name, + "database": str(database), "snapshot": snapshot} + + if phase == "linux-roundtrip": + mount_adapter_mount(ctx, adapter) + try: + verify_portable_tree(ctx, adapter, b"mac-v1\n", linux_file=False) + plan = adapter.mount_point / "portable/你好-🚀/plan.txt" + plan.write_bytes(b"temporary-linux-change\n") + (adapter.mount_point / "portable/你好-🚀/plan-link.txt").unlink() + finally: + mount_adapter_unmount(ctx, adapter, "Linux 首次") + verify_portable_acl(ctx, adapter) + verify_portable_database_owner(ctx, adapter) + diff = run_process( + adapter.prefix + ["snapshot", "diff", "mac-baseline"], check=False) + ctx.equal(diff.returncode, 1, "Linux 识别 macOS snapshot 差异") + changes = {row["path"] for row in json.loads(diff.stdout)["changes"]} + ctx.check("/portable/你好-🚀/plan.txt" in changes, "Linux snapshot 内容差异") + ctx.check("/portable/你好-🚀/plan-link.txt" in changes, + "Linux snapshot 链接差异") + run_process(adapter.prefix + ["snapshot", "restore", "mac-baseline"]) + + mount_adapter_mount(ctx, adapter) + try: + verify_portable_tree(ctx, adapter, b"mac-v1\n", linux_file=False) + plan = adapter.mount_point / "portable/你好-🚀/plan.txt" + plan.write_bytes(b"mac-v1\nlinux-v2\n") + (adapter.mount_point / "portable/from-linux.txt").write_text( + "created on linux\n", encoding="utf-8") + finally: + mount_adapter_unmount(ctx, adapter, "Linux 第二次") + verify_portable_acl(ctx, adapter) + verify_portable_database_owner(ctx, adapter) + snapshot = int(run_process( + adapter.prefix + ["snapshot", "create", "linux-baseline"] + ).stdout.strip()) + ctx.check(snapshot > 0, "Linux baseline snapshot") + return {"phase": phase, "adapter": adapter.name, + "database": str(database), "snapshot": snapshot} + + mount_adapter_mount(ctx, adapter) + try: + verify_portable_tree( + ctx, adapter, b"mac-v1\nlinux-v2\n", linux_file=True) + finally: + mount_adapter_unmount(ctx, adapter, "macOS 回读") + verify_portable_acl(ctx, adapter) + verify_portable_database_owner(ctx, adapter) + + history = json.loads(run_process( + adapter.prefix + ["--json", "history", "/portable/你好-🚀/plan.txt"] + ).stdout)["entries"] + historical = [run_process( + adapter.prefix + ["show", "/portable/你好-🚀/plan.txt", + "--version", str(entry["version"])] + ).stdout for entry in history] + ctx.check(b"mac-v1\n" in historical, "macOS 可读 Linux 保留的旧文件版本") + snapshots = json.loads(run_process( + adapter.prefix + ["--json", "snapshot", "list"]).stdout) + snapshot_names = {entry["name"] for entry in snapshots} + ctx.check({"mac-baseline", "linux-baseline"}.issubset(snapshot_names), + "macOS 可见两端 snapshot") + run_process(adapter.prefix + ["write", "/portable/你好-🚀/plan.txt"], + input_data=b"mac-temporary\n") + run_process(adapter.prefix + ["snapshot", "restore", "linux-baseline"]) + clean = run_process(adapter.prefix + ["snapshot", "diff", "linux-baseline"]) + ctx.equal(json.loads(clean.stdout)["changes"], [], + "macOS 恢复 Linux snapshot 后无差异") + + mount_adapter_mount(ctx, adapter) + try: + verify_portable_tree( + ctx, adapter, b"mac-v1\nlinux-v2\n", linux_file=True) + finally: + mount_adapter_unmount(ctx, adapter, "macOS 恢复 Linux snapshot") + unified_cli = adapter.cli.parent / "vexdb" + ctx.check(unified_cli.exists(), "跨系统测试缺少统一 vexdb 入口") + integrity = run_process( + [str(unified_cli), str(database), "PRAGMA integrity_check;"]).stdout.strip() + ctx.equal(integrity, b"ok", "跨系统往返后 SQLite 完整性") + return {"phase": phase, "adapter": adapter.name, + "database": str(database), "history_versions": len(history), + "snapshots": sorted(snapshot_names)} + + +@case("mount.timestamps", "mount", + "真实挂载的 utimens、写入和目录变更时间戳语义") +def mount_timestamps(ctx: Context) -> dict[str, Any]: + with tempfile.TemporaryDirectory(prefix="vexfs-mount-timestamps-") as directory: + adapter = prepare_real_mount_adapter(ctx, Path(directory), "timestamps") + mount_adapter_mount(ctx, adapter) + try: + root = adapter.mount_point / "times" + root.mkdir() + file = root / "value.txt" + file.write_bytes(b"before") + requested_atime = 1_700_000_000_123_000_000 + requested_mtime = 1_700_000_100_456_000_000 + os.utime(file, ns=(requested_atime, requested_mtime)) + updated = file.stat() + ctx.check(abs(updated.st_atime_ns - requested_atime) <= 1_000_000, + "utimens access time 必须达到毫秒精度: " + f"requested={requested_atime}, actual={updated.st_atime_ns}") + ctx.check(abs(updated.st_mtime_ns - requested_mtime) <= 1_000_000, + "utimens modify time 必须达到毫秒精度: " + f"requested={requested_mtime}, actual={updated.st_mtime_ns}") + + before_write = file.stat() + time.sleep(0.01) + with file.open("ab") as stream: + stream.write(b"-after") + stream.flush() + os.fsync(stream.fileno()) + after_write = file.stat() + ctx.check(after_write.st_mtime_ns > before_write.st_mtime_ns, + "写入必须推进 modify time") + ctx.check(after_write.st_ctime_ns >= before_write.st_ctime_ns, + "写入不能倒退 change time") + + before_child = root.stat() + time.sleep(0.01) + (root / "child.txt").write_text("child", encoding="utf-8") + after_child = root.stat() + ctx.check(after_child.st_mtime_ns > before_child.st_mtime_ns, + "目录项变化必须推进目录 modify time") + return { + "adapter": adapter.name, + "requested_atime_ns": requested_atime, + "actual_atime_ns": updated.st_atime_ns, + "requested_mtime_ns": requested_mtime, + "actual_mtime_ns": updated.st_mtime_ns, + "write_mtime_delta_ns": after_write.st_mtime_ns - before_write.st_mtime_ns, + "directory_mtime_delta_ns": after_child.st_mtime_ns - before_child.st_mtime_ns, + } + finally: + mount_adapter_unmount(ctx, adapter, "时间戳") + + +@case("mount.concurrent-append", "mount", + "多个真实进程通过 O_APPEND 写同一文件且不丢失、不覆盖") +def mount_concurrent_append(ctx: Context) -> dict[str, Any]: + with tempfile.TemporaryDirectory(prefix="vexfs-mount-append-") as directory: + adapter = prepare_real_mount_adapter(ctx, Path(directory), "append") + mount_adapter_mount(ctx, adapter) + try: + target = adapter.mount_point / "append.log" + target.write_bytes(b"") + workers = 4 + records_per_worker = 100 + child = ( + "import os,sys\n" + "path,worker,count=sys.argv[1],int(sys.argv[2]),int(sys.argv[3])\n" + "fd=os.open(path,os.O_WRONLY|os.O_APPEND)\n" + "try:\n" + " for index in range(count):\n" + " value=f'{worker:02d}:{index:04d}\\n'.encode()\n" + " if os.write(fd,value)!=len(value): raise RuntimeError('short append')\n" + " os.fsync(fd)\n" + "finally:\n" + " os.close(fd)\n" + ) + started = time.perf_counter() + processes = [subprocess.Popen( + [sys.executable, "-c", child, str(target), str(worker), + str(records_per_worker)], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + for worker in range(workers)] + failures: list[str] = [] + for process in processes: + stdout, stderr = process.communicate(timeout=60) + if process.returncode != 0: + failures.append( + f"exit={process.returncode} stdout={stdout!r} stderr={stderr!r}") + ctx.equal(failures, [], "并发 O_APPEND 子进程") + elapsed = time.perf_counter() - started + lines = target.read_text(encoding="utf-8").splitlines() + expected = {f"{worker:02d}:{index:04d}" + for worker in range(workers) + for index in range(records_per_worker)} + ctx.equal(len(lines), len(expected), "并发 append 行数") + ctx.equal(set(lines), expected, "并发 append 不能丢失或重复记录") + return { + "adapter": adapter.name, + "workers": workers, + "records": len(lines), + "elapsed_seconds": elapsed, + "records_per_second": len(lines) / elapsed, + } + finally: + mount_adapter_unmount(ctx, adapter, "append") + + +@case("mount.open-rename-unlink", "mount", + "打开文件在 rename/unlink 后保持可读写，且同名新文件不复用旧 inode") +def mount_open_rename_unlink(ctx: Context) -> dict[str, Any]: + with tempfile.TemporaryDirectory(prefix="vexfs-mount-open-life-") as directory: + adapter = prepare_real_mount_adapter(ctx, Path(directory), "open-life") + mount_adapter_mount(ctx, adapter) + try: + original = adapter.mount_point / "original.txt" + renamed = adapter.mount_point / "renamed.txt" + original.write_bytes(b"abcdef") + descriptor = os.open(original, os.O_RDWR) + try: + original.rename(renamed) + ctx.equal(os.pwrite(descriptor, b"XY", 2), 2, "rename 后旧 fd 写入") + os.fsync(descriptor) + ctx.equal(os.pread(descriptor, 6, 0), b"abXYef", "rename 后旧 fd 读取") + finally: + os.close(descriptor) + ctx.equal(renamed.read_bytes(), b"abXYef", "rename 后路径内容") + + descriptor = os.open(renamed, os.O_RDWR) + old_inode = os.fstat(descriptor).st_ino + try: + renamed.unlink() + ctx.check(not renamed.exists(), "unlink 后目录项立即消失") + ctx.equal(os.pwrite(descriptor, b"OLD", 0), 3, "unlink 后旧 fd 写入") + os.fsync(descriptor) + ctx.equal(os.pread(descriptor, 6, 0), b"OLDYef", "unlink 后旧 fd 读取") + renamed.write_bytes(b"new-file") + ctx.check(renamed.stat().st_ino != old_inode, "同名新文件必须使用新 inode") + ctx.equal(renamed.read_bytes(), b"new-file", "同名新文件内容独立") + finally: + os.close(descriptor) + ctx.equal(renamed.read_bytes(), b"new-file", "旧 fd 关闭不覆盖同名新文件") + return {"adapter": adapter.name, "old_inode": old_inode, + "new_inode": renamed.stat().st_ino} + finally: + mount_adapter_unmount(ctx, adapter, "打开文件生命周期") + + +@case("mount.read-only-open-lifecycle", "mount", + "小文件只读快速路径和大文件 handle 在 rename/unlink/并发改写后保持正确") +def mount_read_only_open_lifecycle(ctx: Context) -> dict[str, Any]: + with tempfile.TemporaryDirectory(prefix="vexfs-mount-read-life-") as directory: + adapter = prepare_real_mount_adapter(ctx, Path(directory), "read-life") + mount_adapter_mount(ctx, adapter) + try: + root = adapter.mount_point + + small = root / "small.txt" + renamed_small = root / "small-renamed.txt" + small_payload = b"small-read-only-payload" + small.write_bytes(small_payload) + descriptor = os.open(small, os.O_RDONLY) + old_inode = os.fstat(descriptor).st_ino + try: + small.rename(renamed_small) + ctx.equal(os.pread(descriptor, len(small_payload), 0), small_payload, + "小文件只读 fd 在 rename 后读取") + renamed_small.unlink() + ctx.equal(os.pread(descriptor, len(small_payload), 0), small_payload, + "小文件只读 fd 在 unlink 后读取") + renamed_small.write_bytes(b"replacement") + ctx.check(renamed_small.stat().st_ino != old_inode, + "小文件同名重建不能复用已 unlink inode") + finally: + os.close(descriptor) + ctx.equal(renamed_small.read_bytes(), b"replacement", + "小文件旧只读 fd 关闭不影响同名新文件") + + shared = root / "shared.txt" + shared.write_bytes(b"before") + reader = os.open(shared, os.O_RDONLY) + writer = os.open(shared, os.O_WRONLY) + try: + ctx.equal(os.pwrite(writer, b"after!", 0), 6, "并发写描述符写入") + os.fsync(writer) + ctx.equal(os.pread(reader, 6, 0), b"after!", + "同 vnode 只读描述符看到已提交改写") + os.truncate(shared, 3) + ctx.equal(os.pread(reader, 8, 0), b"aft", + "同 vnode 只读描述符看到路径截断") + finally: + os.close(writer) + os.close(reader) + + large = root / "large.bin" + renamed_large = root / "large-renamed.bin" + large_payload = b"L" * (1024 * 1024 + 257) + large.write_bytes(large_payload) + descriptor = os.open(large, os.O_RDONLY) + large_inode = os.fstat(descriptor).st_ino + try: + large.rename(renamed_large) + renamed_large.unlink() + ctx.equal(os.pread(descriptor, 32, 0), large_payload[:32], + "大文件只读 handle 在 unlink 后读取开头") + ctx.equal(os.pread(descriptor, 257, 1024 * 1024), large_payload[-257:], + "大文件只读 handle 在 unlink 后读取结尾") + renamed_large.write_bytes(b"new-large") + ctx.check(renamed_large.stat().st_ino != large_inode, + "大文件同名重建不能复用已 unlink inode") + finally: + os.close(descriptor) + ctx.equal(renamed_large.read_bytes(), b"new-large", + "大文件旧只读 fd 关闭不影响同名新文件") + return { + "adapter": adapter.name, + "small_bytes": len(small_payload), + "large_bytes": len(large_payload), + } + finally: + mount_adapter_unmount(ctx, adapter, "只读打开生命周期") + + +def run_lock_probe(path: Path, operation: str, start: int = 0, + length: int = 0) -> subprocess.CompletedProcess[bytes]: + child = ( + "import fcntl,os,sys\n" + "fd=os.open(sys.argv[1],os.O_RDWR)\n" + "try:\n" + " try:\n" + " if sys.argv[2]=='flock': fcntl.flock(fd,fcntl.LOCK_EX|fcntl.LOCK_NB)\n" + " else: fcntl.lockf(fd,fcntl.LOCK_EX|fcntl.LOCK_NB,int(sys.argv[4])," + "int(sys.argv[3]),os.SEEK_SET)\n" + " except BlockingIOError: sys.exit(10)\n" + "finally:\n" + " os.close(fd)\n" + ) + return run_process([sys.executable, "-c", child, str(path), operation, + str(start), str(length)], check=False) + + +@case("mount.process-locks", "mount", + "真实进程间 flock 与 fcntl byte-range 最小锁语义") +def mount_process_locks(ctx: Context) -> dict[str, Any]: + with tempfile.TemporaryDirectory(prefix="vexfs-mount-locks-") as directory: + adapter = prepare_real_mount_adapter(ctx, Path(directory), "locks") + mount_adapter_mount(ctx, adapter) + try: + target = adapter.mount_point / "lock.bin" + target.write_bytes(b"0" * 64) + descriptor = os.open(target, os.O_RDWR) + try: + fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB) + ctx.equal(run_lock_probe(target, "flock").returncode, 10, + "flock 冲突必须返回 would-block") + fcntl.flock(descriptor, fcntl.LOCK_UN) + ctx.equal(run_lock_probe(target, "flock").returncode, 0, + "flock 释放后可再次获取") + + fcntl.lockf(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB, 16, 0, os.SEEK_SET) + ctx.equal(run_lock_probe(target, "lockf", 0, 16).returncode, 10, + "重叠 byte-range lock 必须冲突") + ctx.equal(run_lock_probe(target, "lockf", 32, 16).returncode, 0, + "不重叠 byte-range lock 可以并存") + fcntl.lockf(descriptor, fcntl.LOCK_UN, 16, 0, os.SEEK_SET) + finally: + os.close(descriptor) + return {"adapter": adapter.name, "flock": True, "fcntl_range": True} + finally: + mount_adapter_unmount(ctx, adapter, "进程锁") + + +@case("mount.force-unmount", "mount", + "强制卸载后挂载状态消失，重新挂载仍可读取已提交内容") +def mount_force_unmount(ctx: Context) -> dict[str, Any]: + with tempfile.TemporaryDirectory(prefix="vexfs-mount-force-") as directory: + adapter = prepare_real_mount_adapter(ctx, Path(directory), "force-unmount") + mount_adapter_mount(ctx, adapter) + target = adapter.mount_point / "durable.txt" + target.write_bytes(b"force-durable") + with target.open("rb") as stream: + os.fsync(stream.fileno()) + forced = run_process( + adapter.prefix + ["unmount", "--force", str(adapter.mount_point)], + check=False, timeout=60) + ctx.equal(forced.returncode, 0, f"{adapter.name} 强制卸载退出码") + status_result = run_process( + adapter.prefix + ["--json", "mount", "status", str(adapter.mount_point)], + check=False) + ctx.equal(status_result.returncode, 1, "强制卸载后指定路径状态退出码") + status = json.loads(status_result.stdout) + ctx.equal(status, [], "强制卸载后状态必须消失") + if adapter.name == "fskit": + time.sleep(2.0) + mount_adapter_mount(ctx, adapter) + try: + ctx.equal(target.read_bytes(), b"force-durable", "强制卸载后已提交内容") + return {"adapter": adapter.name} + finally: + mount_adapter_unmount(ctx, adapter, "强制卸载重挂载") + + +@case("mount.helper-crash-recovery", "mount", + "Linux FUSE helper 异常退出后保留 staging，并在新 session 自动发布") +def mount_helper_crash_recovery(ctx: Context) -> dict[str, Any]: + if platform.system() != "Linux": + raise EvalSkip("helper 异常退出恢复当前只在 Linux libfuse3 执行") + with tempfile.TemporaryDirectory(prefix="vexfs-mount-helper-crash-") as directory: + adapter = prepare_real_mount_adapter(ctx, Path(directory), "helper-crash") + mount_adapter_mount(ctx, adapter) + target = adapter.mount_point / "staged.txt" + target.write_bytes(b"baseline") + child_code = ( + "import os,sys,time\n" + "fd=os.open(sys.argv[1],os.O_RDWR)\n" + "os.pwrite(fd,b'CRASH',0)\n" + "print('staged',flush=True)\n" + "time.sleep(120)\n" + ) + child = subprocess.Popen([sys.executable, "-c", child_code, str(target)], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + remounted = False + try: + ready = child.stdout.readline().decode(errors="replace").strip() + ctx.equal(ready, "staged", "子进程已写入但未关闭 fd") + process_table = run_process(["ps", "-eo", "pid=,args="]).stdout.decode() + helper_pids = [] + for line in process_table.splitlines(): + fields = line.strip().split(maxsplit=1) + if len(fields) != 2: + continue + if ("vexfs-fuse" in fields[1] and str(adapter.database) in fields[1] + and str(adapter.mount_point) in fields[1]): + helper_pids.append(int(fields[0])) + ctx.equal(len(helper_pids), 1, "定位唯一 FUSE helper") + os.kill(helper_pids[0], signal.SIGKILL) + child.terminate() + child.wait(timeout=10) + + forced = run_process( + adapter.prefix + ["unmount", "--force", str(adapter.mount_point)], + check=False, timeout=60) + ctx.equal(forced.returncode, 0, "helper 崩溃后强制卸载") + with sqlite3.connect(adapter.database) as connection: + dirty = connection.execute( + "SELECT count(*) FROM _vexfs_handles " + "WHERE dirty_generation>published_generation").fetchone()[0] + ctx.equal(dirty, 1, "helper 崩溃后 staging 仍在数据库") + connection.execute("UPDATE _vexfs_mount_sessions SET lease_until=0") + + mount_adapter_mount(ctx, adapter) + remounted = True + ctx.equal(target.read_bytes(), b"CRASHine", "新 session 自动发布崩溃前 staging") + with sqlite3.connect(adapter.database) as connection: + dirty_after = connection.execute( + "SELECT count(*) FROM _vexfs_handles " + "WHERE dirty_generation>published_generation").fetchone()[0] + ctx.equal(dirty_after, 0, "恢复后没有未发布 staging") + return {"adapter": adapter.name, "helper_pid": helper_pids[0]} + finally: + if child.poll() is None: + child.terminate() + child.wait(timeout=10) + if remounted: + mount_adapter_unmount(ctx, adapter, "helper 崩溃恢复") + + @case("mount.real-bash", "mount", "真实 FSKit mount 后用 bash 常用文件命令操作") def real_mount_bash(ctx: Context) -> dict[str, Any]: if platform.system() != "Darwin": @@ -1531,7 +3646,8 @@ def real_mount_bash(ctx: Context) -> dict[str, Any]: base = Path(directory) database = base / "mount.sqlite3" mount_point = base / "mnt" - prefix = [str(ctx.cli), "--db", str(database), "--workspace", "eval"] + cli = mount_cli(ctx) + prefix = filesystem_cli_prefix(cli, database, "eval") run_process(prefix + ["setup"]) doctor = run_process(prefix + ["--json", "doctor"], check=False) details = json.loads(doctor.stdout) @@ -1560,9 +3676,10 @@ def real_mount_bash(ctx: Context) -> dict[str, Any]: cp "$1/project/input.txt" "$1/project/copy.txt" mv "$1/project/copy.txt" "$1/project/moved.txt" cmp "$1/project/input.txt" "$1/project/moved.txt" +/bin/ls -la "$1/project" find "$1/project" -type f | sort rm "$1/project/moved.txt" -test "$(cat "$1/project/sub/result.txt")" = agent +test "$(/bin/cat "$1/project/sub/result.txt")" = agent ''' result = run_process(["/bin/bash", "-c", shell, "vexfs-eval", str(mount_point)], timeout=120) @@ -1581,10 +3698,202 @@ def real_mount_bash(ctx: Context) -> dict[str, Any]: ctx.equal(after["database"]["pending_handles"], 0, "卸载后未发布句柄") ctx.equal(after["database"]["retained_handles"], 0, "卸载后保留句柄") ctx.equal(after["database"]["staging_bytes"], 0, "卸载后暂存字节") - return {"doctor_before": details, "doctor_after": after, "mounts": mounts, + return {"cli": str(cli), "doctor_before": details, "doctor_after": after, + "mounts": mounts, "descriptor": descriptor, - "commands": ["mkdir", "printf", "grep", "cp", "mv", "cmp", "find", - "rm", "cat", "vexfs cat"]} + "commands": ["mkdir", "printf", "grep", "cp", "mv", "cmp", "ls", + "find", "rm", "cat", "vexfs cat"]} + + +@case("mount.real-linux-bash-git", "mount", + "真实 libfuse3 mount 后运行 Bash、可执行脚本、链接和 Git 工作区") +def real_linux_mount_bash_git(ctx: Context) -> dict[str, Any]: + if platform.system() != "Linux": + raise EvalSkip("真实 libfuse3 mount 只在 Linux 执行") + helper = ctx.build_dir / "vexfs-fuse" + if not helper.exists(): + raise EvalSkip("当前构建没有 vexfs-fuse helper") + if not Path("/dev/fuse").exists() or not os.access("/dev/fuse", os.R_OK | os.W_OK): + raise EvalSkip("当前环境没有可读写的 /dev/fuse") + if shutil.which("fusermount3") is None: + raise EvalSkip("当前环境没有 fusermount3") + if shutil.which("git") is None: + raise EvalSkip("当前环境没有 git") + + with tempfile.TemporaryDirectory(prefix="vexfs-linux-mount-eval-") as directory: + base = Path(directory) + database = base / "mount.sqlite3" + mount_point = base / "mnt" + prefix = [str(ctx.cli), "--db", str(database), "--workspace", "linux-eval"] + run_process(prefix + ["setup"]) + details = json.loads(run_process(prefix + ["--json", "doctor"], check=False).stdout) + if not details.get("mount_ready"): + raise EvalSkip(f"libfuse3 adapter={details.get('extension', 'unknown')}") + mount_point.mkdir() + run_process(prefix + ["mount", str(mount_point)], timeout=60) + first_unmount = None + try: + shell = r''' +set -e +mkdir -p "$1/project/src" +printf '#!/bin/sh\nprintf agent-linux\n' > "$1/project/run.sh" +chmod 755 "$1/project/run.sh" +"$1/project/run.sh" > "$1/project/output.txt" +printf 'alpha\nagent\nomega\n' > "$1/project/src/input.txt" +grep agent "$1/project/src/input.txt" > "$1/project/src/result.txt" +cp "$1/project/src/input.txt" "$1/project/src/copy.txt" +mv "$1/project/src/copy.txt" "$1/project/src/moved.txt" +ln "$1/project/run.sh" "$1/project/run-hard.sh" +ln -s run.sh "$1/project/run-link.sh" +git -C "$1/project" init -q +git -C "$1/project" config user.name VexFS-Eval +git -C "$1/project" config user.email vexfs-eval@example.invalid +git -C "$1/project" add . +git -C "$1/project" commit -qm initial +test -z "$(git -C "$1/project" status --porcelain)" +test "$(cat "$1/project/output.txt")" = agent-linux +''' + run_process(["/bin/sh", "-c", shell, "vexfs-linux-eval", str(mount_point)], + timeout=180) + script = mount_point / "project/run.sh" + hardlink = mount_point / "project/run-hard.sh" + symlink = mount_point / "project/run-link.sh" + ctx.equal(script.stat().st_ino, hardlink.stat().st_ino, "Linux hardlink inode") + ctx.equal(script.stat().st_mode & 0o777, 0o755, "Linux executable mode") + ctx.equal(script.stat().st_uid, os.getuid(), "Linux created-file uid") + ctx.equal(script.stat().st_gid, os.getgid(), "Linux created-file gid") + ctx.equal(os.readlink(symlink), "run.sh", "Linux symbolic link target") + ctx.equal((mount_point / "project/src/result.txt").read_text(), "agent\n", + "Linux grep result") + reverse = run_process(prefix + ["cat", "/project/output.txt"]) + ctx.equal(reverse.stdout, b"agent-linux", "Linux mount 写入可由数据库 CLI 读取") + finally: + first_unmount = run_process(prefix + ["unmount", str(mount_point)], + check=False, timeout=60) + ctx.equal(first_unmount.returncode, 0, "Linux 首次卸载") + + run_process(prefix + ["mount", str(mount_point)], timeout=60) + second_unmount = None + try: + status = run_process( + ["git", "-C", str(mount_point / "project"), "status", "--porcelain"], + timeout=120) + ctx.equal(status.stdout, b"", "Linux 重挂载后 Git workspace 完整") + ctx.equal((mount_point / "project/output.txt").read_bytes(), b"agent-linux", + "Linux 重挂载内容") + finally: + second_unmount = run_process(prefix + ["unmount", str(mount_point)], + check=False, timeout=60) + ctx.equal(second_unmount.returncode, 0, "Linux 第二次卸载") + after = json.loads(run_process(prefix + ["--json", "doctor"]).stdout) + ctx.equal(after["mount_count"], 0, "Linux 卸载后无挂载") + ctx.equal(after["database"]["pending_handles"], 0, "Linux 卸载后无未发布句柄") + return {"doctor_before": details, "doctor_after": after, + "commands": ["mkdir", "printf", "chmod", "exec", "grep", "cp", "mv", + "hardlink", "symlink", "git init", "git add", "git commit", + "git status", "unmount", "remount"]} + + +@case("mount.posix-metadata", "mount", + "真实 FSKit mount 的可执行权限、符号链接、重命名和重挂载") +def real_mount_posix_metadata(ctx: Context) -> dict[str, Any]: + if platform.system() != "Darwin": + raise EvalSkip("真实 FSKit mount 只在 macOS 执行") + with tempfile.TemporaryDirectory(prefix="vexfs-mount-posix-") as directory: + base = Path(directory) + database = base / "mount.sqlite3" + mount_point = base / "mnt" + cli = mount_cli(ctx) + prefix = filesystem_cli_prefix(cli, database, "posix") + run_process(prefix + ["setup"]) + details = json.loads(run_process( + prefix + ["--json", "doctor"], check=False).stdout) + if details.get("extension") != "enabled": + raise EvalSkip( + f"FSKit extension={details.get('extension', 'unknown')}，需安装并在系统设置中启用") + + mount_point.mkdir() + run_process(prefix + ["mount", str(mount_point)], timeout=60) + first_unmount = None + script = mount_point / "project/run.sh" + link = mount_point / "project/run-link" + dangling = mount_point / "project/dangling" + try: + script.parent.mkdir() + script.write_text("#!/bin/sh\nprintf 'vexfs-exec\\n'\n", encoding="utf-8") + script.chmod(0o755) + ctx.equal(script.stat().st_mode & 0o777, 0o755, "chmod 后可执行 mode") + executed = run_process([str(script)], cwd=script.parent, timeout=30) + ctx.equal(executed.stdout, b"vexfs-exec\n", "直接执行挂载盘脚本") + + link.symlink_to("run.sh") + dangling.symlink_to("missing.txt") + ctx.check(link.is_symlink() and dangling.is_symlink(), "符号链接类型") + ctx.equal(os.readlink(link), "run.sh", "相对符号链接 target") + ctx.equal(link.read_text(encoding="utf-8"), script.read_text(encoding="utf-8"), + "通过符号链接读取目标") + ctx.check(not dangling.exists() and dangling.is_symlink(), "悬空符号链接") + + temporary = mount_point / "project/temp-link" + temporary.symlink_to("run.sh") + renamed = mount_point / "project/renamed-link" + temporary.rename(renamed) + ctx.equal(os.readlink(renamed), "run.sh", "重命名链接不改 target") + renamed.unlink() + ctx.check(script.exists(), "删除链接不删除目标") + finally: + first_unmount = run_process(prefix + ["unmount", str(mount_point)], + check=False, timeout=60) + ctx.equal(first_unmount.returncode, 0, "POSIX 首次卸载退出码") + + # FSKit 的 unload 在 umount 返回后仍可能异步收尾；等待旧 volume 释放同一 + # container identity，避免把系统生命周期竞态误判成持久化失败。 + time.sleep(2.0) + run_process(prefix + ["mount", str(mount_point)], timeout=60) + second_unmount = None + try: + ctx.equal(script.stat().st_mode & 0o777, 0o755, "重挂载后 mode 持久化") + ctx.equal(os.readlink(link), "run.sh", "重挂载后链接 target 持久化") + ctx.equal(run_process([str(link)], cwd=script.parent).stdout, + b"vexfs-exec\n", "重挂载后通过链接执行") + finally: + # mount 返回后 FSKit 仍会完成少量异步激活工作；这个回归用例操作很少， + # 需要给 activate/deactivate 留出正常交接时间。 + time.sleep(2.0) + second_unmount = run_process(prefix + ["unmount", str(mount_point)], + check=False, timeout=60) + if second_unmount.returncode != 0: + run_process(["/sbin/umount", "-f", str(mount_point)], + check=False, timeout=60) + ctx.equal(second_unmount.returncode, 0, "POSIX 第二次卸载退出码") + + connection = sqlite3.connect(database, isolation_level=None) + connection.enable_load_extension(True) + connection.load_extension(str(ctx.extension)) + try: + row = connection.execute( + "SELECT i.id,i.kind,i.mode,i.size FROM _vexfs_dentries d " + "JOIN _vexfs_inodes i ON i.id=d.inode_id " + "JOIN _vexfs_workspaces w ON w.id=d.workspace_id " + "WHERE w.name='posix' AND d.name='run-link' AND i.deleted_at IS NULL" + ).fetchone() + ctx.check(row is not None, "SQLite 中存在符号链接 inode") + assert row is not None + ctx.equal((row[1], row[2], row[3]), ("symlink", 0o777, len("run.sh")), + "SQLite 中的链接元数据") + target = connection.execute( + "SELECT vexfs_readlink('posix',?)", (row[0],)).fetchone()[0] + ctx.equal(target, b"run.sh", "SQLite 中保存链接 target") + stored_mode = connection.execute( + "SELECT i.mode FROM _vexfs_dentries d JOIN _vexfs_inodes i ON i.id=d.inode_id " + "JOIN _vexfs_workspaces w ON w.id=d.workspace_id " + "WHERE w.name='posix' AND d.name='run.sh' AND i.deleted_at IS NULL" + ).fetchone()[0] + ctx.equal(stored_mode, 0o755, "SQLite 中保存可执行权限") + finally: + connection.close() + return {"cli": str(cli), "mode": 0o755, "target": "run.sh", + "remounted": True, "database_verified": True} @case("mount.performance", "mount", "真实 FSKit 分块顺序写和随机覆盖性能") @@ -1597,7 +3906,8 @@ def real_mount_performance(ctx: Context) -> dict[str, Any]: base = Path(directory) database = base / "mount.sqlite3" mount_point = base / "mnt" - prefix = [str(ctx.cli), "--db", str(database), "--workspace", "perf"] + cli = mount_cli(ctx) + prefix = filesystem_cli_prefix(cli, database, "perf") run_process(prefix + ["setup"]) details = json.loads(run_process( prefix + ["--json", "doctor"], check=False).stdout) @@ -1647,6 +3957,7 @@ def real_mount_performance(ctx: Context) -> dict[str, Any]: after = json.loads(run_process(prefix + ["--json", "doctor"]).stdout) ctx.equal(after["mount_count"], 0, "FSKit 性能测试卸载后挂载数") return { + "cli": str(cli), "sequential_mib": sequential_mib, "sequential_seconds": round(sequential_seconds, 6), "sequential_mib_per_second": round( @@ -1659,6 +3970,603 @@ def real_mount_performance(ctx: Context) -> dict[str, Any]: } +@case("mount.git-workspace", "mount", "真实 FSKit mount 中运行完整 Git 工作区流程") +def real_mount_git_workspace(ctx: Context) -> dict[str, Any]: + if platform.system() != "Darwin": + raise EvalSkip("真实 FSKit mount 只在 macOS 执行") + git = shutil.which("git") + if git is None: + raise EvalSkip("系统没有 git") + xattr = shutil.which("xattr") + if xattr is None: + raise EvalSkip("系统没有 xattr") + with tempfile.TemporaryDirectory(prefix="vexfs-mount-git-") as directory: + base = Path(directory) + database = base / "mount.sqlite3" + mount_point = base / "mnt" + cli = mount_cli(ctx) + prefix = filesystem_cli_prefix(cli, database, "git-eval") + run_process(prefix + ["setup"]) + details = json.loads(run_process( + prefix + ["--json", "doctor"], check=False).stdout) + if details.get("extension") != "enabled": + raise EvalSkip( + f"FSKit extension={details.get('extension', 'unknown')}，需安装并在系统设置中启用") + mount_point.mkdir() + run_process(prefix + ["mount", str(mount_point)], timeout=60) + project = mount_point / "project" + unmount = None + try: + project.mkdir() + run_process([xattr, "-w", "com.vexfs.eval", "stored-in-sqlite", str(project)]) + xattr_value = run_process( + [xattr, "-p", "com.vexfs.eval", str(project)], timeout=120) + ctx.equal(xattr_value.stdout.strip(), b"stored-in-sqlite", + "挂载盘扩展属性读写") + environment = os.environ.copy() + environment.update({ + "GIT_CONFIG_NOSYSTEM": "1", + "HOME": str(base / "home"), + }) + (base / "home").mkdir() + commands = [ + [git, "init", "-b", "main"], + [git, "config", "user.name", "VexFS Eval"], + [git, "config", "user.email", "vexfs-eval@example.invalid"], + [git, "config", "core.ignorecase", "false"], + ] + for command in commands: + run_process(command, cwd=project, env=environment, timeout=120) + + filemode = run_process( + [git, "config", "--type=bool", "--default=true", "core.filemode"], + cwd=project, env=environment) + symlinks = run_process( + [git, "config", "--type=bool", "--default=true", "core.symlinks"], + cwd=project, env=environment) + ctx.equal(filemode.stdout.strip(), b"true", "Git 识别可执行权限") + ctx.equal(symlinks.stdout.strip(), b"true", "Git 识别符号链接") + + (project / "README.md").write_text("main\n", encoding="utf-8") + executable = project / "run.sh" + executable.write_text("#!/bin/sh\nprintf 'main-exec\\n'\n", encoding="utf-8") + executable.chmod(0o755) + current = project / "current" + current.symlink_to("run.sh") + run_process([git, "add", "."], cwd=project, env=environment) + run_process([git, "commit", "-m", "initial"], cwd=project, + env=environment, timeout=120) + run_process([git, "checkout", "-b", "feature"], cwd=project, + env=environment, timeout=120) + (project / "README.md").write_text("feature\n", encoding="utf-8") + executable.chmod(0o644) + current.unlink() + current.symlink_to("README.md") + run_process([git, "add", "-A"], cwd=project, env=environment) + run_process([git, "commit", "-m", "feature"], cwd=project, + env=environment, timeout=120) + run_process([git, "checkout", "main"], cwd=project, + env=environment, timeout=120) + status = run_process([git, "status", "--porcelain"], cwd=project, + env=environment, timeout=120) + ctx.equal(status.stdout, b"", "Git checkout 后工作区状态") + apple_double = sorted( + str(path.relative_to(project)) for path in project.rglob("._*") + ) + ctx.equal(apple_double, [], "Git 工作区没有 AppleDouble 垃圾文件") + ctx.equal((project / "README.md").read_text(encoding="utf-8"), + "main\n", "Git checkout 恢复主分支内容") + ctx.equal(executable.stat().st_mode & 0o777, 0o755, + "Git checkout 恢复可执行权限") + ctx.equal(os.readlink(current), "run.sh", "Git checkout 恢复符号链接") + ctx.equal(run_process([str(current)], cwd=project, env=environment).stdout, + b"main-exec\n", "Git checkout 后通过链接执行") + commits = run_process([git, "rev-list", "--all", "--count"], cwd=project, + env=environment, timeout=120) + ctx.equal(commits.stdout.strip(), b"2", "Git 提交数量") + run_process([git, "gc"], cwd=project, env=environment, timeout=120) + ctx.equal(run_process(prefix + ["cat", "/project/README.md"]).stdout, + b"main\n", "数据库 CLI 反向读取 Git 工作区") + finally: + unmount = run_process(prefix + ["unmount", str(mount_point)], + check=False, timeout=60) + ctx.equal(unmount.returncode, 0, "Git 测试卸载退出码") + after = json.loads(run_process(prefix + ["--json", "doctor"]).stdout) + ctx.equal(after["mount_count"], 0, "Git 测试卸载后挂载数") + ctx.equal(after["database"]["pending_handles"], 0, "Git 测试未发布句柄") + ctx.equal(after["database"]["retained_handles"], 0, "Git 测试保留句柄") + ctx.equal(after["database"]["staging_bytes"], 0, "Git 测试暂存字节") + connection = sqlite3.connect(database) + try: + stored_xattr = connection.execute( + "SELECT value FROM _vexfs_xattrs WHERE name='com.vexfs.eval'" + ).fetchone() + finally: + connection.close() + ctx.equal(None if stored_xattr is None else stored_xattr[0], b"stored-in-sqlite", + "扩展属性持久化到 SQLite") + return { + "cli": str(cli), + "git": run_process([git, "--version"]).stdout.decode().strip(), + "filemode": True, + "symlinks": True, + "commands": ["xattr", "init", "config", "chmod", "symlink", "add", + "commit", "checkout", "status", "gc"], + "doctor_after": after, + } + + +@case("mount.real-toolchain-projects", "mount", + "真实挂载目录中运行 Python、Node.js、Go、Rust 和 Git 项目并重挂载复测") +def real_mount_toolchain_projects(ctx: Context) -> dict[str, Any]: + tool_paths = {name: shutil.which(name) for name in + ("python3", "node", "go", "cargo", "git")} + missing = [name for name, path in tool_paths.items() if path is None] + if missing: + raise EvalSkip(f"当前环境缺少真实项目工具链: {', '.join(missing)}") + + with tempfile.TemporaryDirectory(prefix="vexfs-toolchain-projects-") as directory: + base = Path(directory) + adapter = prepare_real_mount_adapter(ctx, base, "toolchains") + project = adapter.mount_point / "workspace" + home = base / "home" + cache = base / "cache" + home.mkdir() + cache.mkdir() + host_home = Path.home() + environment = os.environ.copy() + environment.update({ + "HOME": str(home), + "GIT_CONFIG_NOSYSTEM": "1", + "GOCACHE": str(cache / "go-build"), + "GOMODCACHE": str(cache / "go-mod"), + "GOPATH": str(cache / "gopath"), + # cargo/rustc 可能是 rustup shim；HOME 隔离后仍要指向已经安装好的 + # toolchain。项目 target 继续写在 VexFS 工作区，真实覆盖编译器文件操作。 + "CARGO_HOME": os.environ.get("CARGO_HOME", str(host_home / ".cargo")), + "RUSTUP_HOME": os.environ.get("RUSTUP_HOME", str(host_home / ".rustup")), + "CARGO_NET_OFFLINE": "true", + }) + + def run_toolchains() -> dict[str, float]: + timings: dict[str, float] = {} + commands = [ + ("python", [tool_paths["python3"], "-m", "unittest", "-v"], + project / "python"), + ("node", [tool_paths["node"], "--test"], project / "node"), + ("go", [tool_paths["go"], "test", "./..."], project / "go"), + ("rust", [tool_paths["cargo"], "test", "--offline"], project / "rust"), + ] + for label, command, cwd in commands: + started = time.perf_counter() + run_process([str(part) for part in command], cwd=cwd, + env=environment, timeout=600) + timings[f"{label}_seconds"] = round(time.perf_counter() - started, 6) + return timings + + mount_adapter_mount(ctx, adapter) + first_timings: dict[str, float] = {} + first_unmounted = False + try: + project.mkdir() + (project / ".gitignore").write_text( + "__pycache__/\n*.pyc\ntarget/\n", encoding="utf-8") + + python_project = project / "python" + python_project.mkdir() + (python_project / "calc.py").write_text( + "def add(left, right):\n return left + right\n", encoding="utf-8") + (python_project / "test_calc.py").write_text( + "import unittest\nfrom calc import add\n\n" + "class CalcTest(unittest.TestCase):\n" + " def test_add(self):\n self.assertEqual(add(20, 22), 42)\n\n" + "if __name__ == '__main__':\n unittest.main()\n", encoding="utf-8") + + node_project = project / "node" + node_project.mkdir() + (node_project / "package.json").write_text( + '{"name":"vexfs-node-eval","version":"1.0.0","type":"commonjs"}\n', + encoding="utf-8") + (node_project / "calc.js").write_text( + "exports.add = (left, right) => left + right;\n", encoding="utf-8") + (node_project / "calc.test.js").write_text( + "const test = require('node:test');\n" + "const assert = require('node:assert/strict');\n" + "const { add } = require('./calc');\n" + "test('add', () => assert.equal(add(20, 22), 42));\n", encoding="utf-8") + + go_project = project / "go" + go_project.mkdir() + (go_project / "go.mod").write_text( + "module example.invalid/vexfs-eval\n\ngo 1.20\n", encoding="utf-8") + (go_project / "calc.go").write_text( + "package calc\n\nfunc Add(left, right int) int { return left + right }\n", + encoding="utf-8") + (go_project / "calc_test.go").write_text( + "package calc\n\nimport \"testing\"\n\n" + "func TestAdd(t *testing.T) { if Add(20, 22) != 42 { t.Fatal(\"bad sum\") } }\n", + encoding="utf-8") + + rust_project = project / "rust" + (rust_project / "src").mkdir(parents=True) + (rust_project / "tests").mkdir() + (rust_project / "Cargo.toml").write_text( + "[package]\nname = \"vexfs-rust-eval\"\nversion = \"0.1.0\"\n" + "edition = \"2021\"\n\n[dependencies]\n", encoding="utf-8") + (rust_project / "src/lib.rs").write_text( + "pub fn add(left: i32, right: i32) -> i32 { left + right }\n", + encoding="utf-8") + (rust_project / "tests/integration.rs").write_text( + "use vexfs_rust_eval::add;\n\n#[test]\nfn adds() { assert_eq!(add(20, 22), 42); }\n", + encoding="utf-8") + environment["CARGO_TARGET_DIR"] = str(rust_project / "target") + + first_timings = run_toolchains() + git = str(tool_paths["git"]) + run_process([git, "init", "-b", "main"], cwd=project, env=environment) + run_process([git, "config", "user.name", "VexFS Toolchain Eval"], + cwd=project, env=environment) + run_process([git, "config", "user.email", "eval@example.invalid"], + cwd=project, env=environment) + run_process([git, "add", "."], cwd=project, env=environment, timeout=300) + run_process([git, "commit", "-m", "toolchain projects"], cwd=project, + env=environment, timeout=300) + ctx.equal(run_process([git, "status", "--porcelain"], cwd=project, + env=environment).stdout, b"", "工具链项目 Git 状态") + finally: + mount_adapter_unmount(ctx, adapter, "工具链首次") + first_unmounted = True + + connection = sqlite3.connect(adapter.database) + try: + ctx.equal(connection.execute("PRAGMA integrity_check").fetchone()[0], "ok", + "工具链项目数据库完整性") + stored_files = connection.execute( + "SELECT count(*) FROM _vexfs_inodes WHERE kind='file' AND deleted_at IS NULL" + ).fetchone()[0] + finally: + connection.close() + ctx.check(stored_files >= 15, "工具链项目产物进入数据库") + + mount_adapter_mount(ctx, adapter) + second_timings: dict[str, float] = {} + try: + second_timings = run_toolchains() + git = str(tool_paths["git"]) + ctx.equal(run_process([git, "status", "--porcelain"], cwd=project, + env=environment).stdout, b"", "重挂载后 Git 状态") + finally: + mount_adapter_unmount(ctx, adapter, "工具链第二次") + ctx.check(first_unmounted, "工具链首次卸载完成") + after = json.loads(run_process(adapter.prefix + ["--json", "doctor"]).stdout) + ctx.equal(after["database"]["pending_handles"], 0, "工具链无未发布句柄") + ctx.equal(after["database"]["retained_handles"], 0, "工具链无保留句柄") + return { + "adapter": adapter.name, + "tools": { + "python": run_process([str(tool_paths["python3"]), "--version"]).stdout.decode().strip(), + "node": run_process([str(tool_paths["node"]), "--version"]).stdout.decode().strip(), + "go": run_process([str(tool_paths["go"]), "version"]).stdout.decode().strip(), + "cargo": run_process([str(tool_paths["cargo"]), "--version"]).stdout.decode().strip(), + "git": run_process([str(tool_paths["git"]), "--version"]).stdout.decode().strip(), + }, + "first_run": first_timings, + "after_remount": second_timings, + "stored_files": stored_files, + "doctor_after": after, + } + + +@case("mount.real-opencode-project", "mount", + "OpenCode 在真实挂载项目中自主查看、修改、测试，并在重挂载后保留结果") +def real_mount_opencode_project(ctx: Context) -> dict[str, Any]: + if os.environ.get("VEXFS_EVAL_OPENCODE") != "1": + raise EvalSkip("设置 VEXFS_EVAL_OPENCODE=1 后执行真实模型调用") + opencode = shutil.which("opencode") + python = shutil.which("python3") + git = shutil.which("git") + if opencode is None or python is None or git is None: + raise EvalSkip("真实 OpenCode 用例需要 opencode、python3 和 git") + model = os.environ.get("VEXFS_EVAL_OPENCODE_MODEL", "openai/gpt-5.4-mini") + + with tempfile.TemporaryDirectory(prefix="vexfs-opencode-project-") as directory: + base = Path(directory) + adapter = prepare_real_mount_adapter(ctx, base, "opencode") + project = adapter.mount_point / "agent-project" + environment = os.environ.copy() + environment["GIT_CONFIG_NOSYSTEM"] = "1" + + mount_adapter_mount(ctx, adapter) + agent_seconds = 0.0 + changed: list[str] = [] + try: + project.mkdir() + (project / "calc.py").write_text( + "def total_even(values):\n" + " # TODO: return the sum of only the even integers.\n" + " raise NotImplementedError\n", + encoding="utf-8") + (project / "test_calc.py").write_text( + "import unittest\n" + "from calc import total_even\n\n" + "class CalcTest(unittest.TestCase):\n" + " def test_mixed_values(self):\n" + " self.assertEqual(total_even([1, 2, 3, 4, -6]), 0)\n\n" + " def test_empty(self):\n" + " self.assertEqual(total_even([]), 0)\n\n" + "if __name__ == '__main__':\n" + " unittest.main()\n", + encoding="utf-8") + (project / "README.md").write_text( + "Complete the TODO in calc.py and keep the public function name.\n", + encoding="utf-8") + run_process([git, "init", "-b", "main"], cwd=project, env=environment) + run_process([git, "config", "user.name", "VexFS Agent Eval"], + cwd=project, env=environment) + run_process([git, "config", "user.email", "eval@example.invalid"], + cwd=project, env=environment) + run_process([git, "add", "."], cwd=project, env=environment) + run_process([git, "commit", "-m", "agent task fixture"], + cwd=project, env=environment) + + prompt = ( + "Inspect this repository and complete the TODO in calc.py. " + "Edit only calc.py. Run `python3 -m unittest -v` and do not stop " + "until the tests pass. Do not ask questions and do not commit." + ) + started = time.perf_counter() + result = run_process( + [opencode, "run", "--auto", "--pure", "--format", "json", + "--model", model, "--dir", str(project), prompt], + cwd=project, env=environment, timeout=900) + agent_seconds = time.perf_counter() - started + ctx.check(len(result.stdout) > 0, "OpenCode 必须返回执行事件") + run_process([python, "-m", "unittest", "-v"], cwd=project, + env=environment, timeout=120) + changed = run_process([git, "diff", "--name-only"], cwd=project, + env=environment).stdout.decode().splitlines() + ctx.equal(changed, ["calc.py"], "OpenCode 只修改任务要求的文件") + implementation = (project / "calc.py").read_text(encoding="utf-8") + ctx.check("NotImplementedError" not in implementation, + "OpenCode 已完成 TODO") + finally: + mount_adapter_unmount(ctx, adapter, "OpenCode 首次") + + stored = run_process(adapter.prefix + ["cat", "/agent-project/calc.py"]) + ctx.check(b"NotImplementedError" not in stored.stdout, + "OpenCode 修改已经进入数据库权威内容") + connection = sqlite3.connect(adapter.database) + try: + ctx.equal(connection.execute("PRAGMA integrity_check").fetchone()[0], "ok", + "OpenCode 项目数据库完整性") + finally: + connection.close() + + mount_adapter_mount(ctx, adapter) + try: + run_process([python, "-m", "unittest", "-v"], cwd=project, + env=environment, timeout=120) + ctx.equal(run_process([git, "diff", "--name-only"], cwd=project, + env=environment).stdout.decode().splitlines(), + ["calc.py"], "重挂载后保留 OpenCode 工作区差异") + finally: + mount_adapter_unmount(ctx, adapter, "OpenCode 第二次") + return { + "adapter": adapter.name, + "opencode": run_process([opencode, "--version"]).stdout.decode().strip(), + "model": model, + "agent_seconds": round(agent_seconds, 6), + "changed_files": changed, + } + + +@case("mount.scale-tree", "mount", + "真实挂载目录创建并遍历 1 千/1 万/10 万文件，执行文本搜索和快照恢复") +def real_mount_scale_tree(ctx: Context) -> dict[str, Any]: + file_count = {"quick": 1_000, "full": 10_000, "stress": 100_000}[ctx.mode.name] + directory_count = min(1_000, max(20, file_count // 100)) + with tempfile.TemporaryDirectory(prefix="vexfs-mount-scale-") as directory: + base = Path(directory) + adapter = prepare_real_mount_adapter(ctx, base, "mount-scale") + root = adapter.mount_point / "scale" + search = shutil.which("rg") or shutil.which("grep") + if search is None: + raise EvalSkip("系统没有 rg 或 grep") + + def create_tree(target: Path) -> None: + target.mkdir() + for directory_index in range(directory_count): + (target / f"d{directory_index:04d}").mkdir() + for file_index in range(file_count): + bucket = file_index % directory_count + marker = "needle" if file_index % 1_000 == 0 else "plain" + (target / f"d{bucket:04d}/f{file_index:06d}.txt").write_text( + f"file={file_index}; marker={marker}\n", encoding="utf-8") + + def search_tree(target: Path) -> tuple[float, int, bool]: + started = time.perf_counter() + try: + if Path(search).name == "rg": + result = run_process( + [search, "-l", "needle", str(target)], timeout=3_600) + else: + result = run_process( + [search, "-R", "-l", "needle", str(target)], timeout=3_600) + except subprocess.TimeoutExpired: + return time.perf_counter() - started, 0, True + return time.perf_counter() - started, len( + [line for line in result.stdout.splitlines() if line]), False + + native_root = base / "native-scale" + native_started = time.perf_counter() + create_tree(native_root) + native_create_seconds = time.perf_counter() - native_started + native_walk_started = time.perf_counter() + native_files = sum(len(files) for _, _, files in os.walk(native_root)) + native_walk_seconds = time.perf_counter() - native_walk_started + native_search_seconds, native_matches, native_search_timed_out = search_tree(native_root) + ctx.check(not native_search_timed_out, "原生目录对照搜索不能超时") + ctx.equal(native_files, file_count, "原生目录对照文件数") + expected_matches = (file_count + 999) // 1_000 + ctx.equal(native_matches, expected_matches, "原生目录对照搜索命中数") + + mount_adapter_mount(ctx, adapter) + create_seconds = 0.0 + walk_seconds = 0.0 + search_seconds = 0.0 + search_timed_out = False + matches = 0 + try: + started = time.perf_counter() + create_tree(root) + create_seconds = time.perf_counter() - started + + started = time.perf_counter() + walked_files = sum(len(files) for _, _, files in os.walk(root)) + walk_seconds = time.perf_counter() - started + ctx.equal(walked_files, file_count, "真实规模目录遍历文件数") + for index in (0, file_count // 2, file_count - 1): + bucket = index % directory_count + text = (root / f"d{bucket:04d}/f{index:06d}.txt").read_text() + ctx.check(f"file={index};" in text, "真实规模目录抽样内容") + + search_seconds, matches, search_timed_out = search_tree(root) + if not search_timed_out: + ctx.equal(matches, expected_matches, "真实规模目录文本搜索命中数") + finally: + mount_adapter_unmount(ctx, adapter, "规模首次") + + database_search_started = time.perf_counter() + database_search = json.loads(run_process( + [str(ctx.cli), "--db", str(adapter.database), "--workspace", "mount-scale", + "--json", "grep", "-l", "needle", "/scale", "--max-results", "10240"], + timeout=3_600).stdout) + database_search_seconds = time.perf_counter() - database_search_started + ctx.equal(database_search["match_count"], expected_matches, + "数据库快路径搜索命中数") + ctx.equal(database_search["files_scanned"], file_count, + "数据库快路径搜索扫描完整目录树") + index_build_started = time.perf_counter() + index_status = json.loads(run_process( + [str(ctx.cli), "--db", str(adapter.database), "--workspace", "mount-scale", + "index", "enable"], timeout=3_600).stdout) + index_build_seconds = time.perf_counter() - index_build_started + ctx.check(index_status["indexed_files"] >= file_count, + "数据库 trigram 索引覆盖业务文件和 FSKit 系统文件") + indexed_search_started = time.perf_counter() + indexed_search = json.loads(run_process( + [str(ctx.cli), "--db", str(adapter.database), "--workspace", "mount-scale", + "--json", "grep", "-l", "needle", "/scale", "--max-results", "10240"], + timeout=3_600).stdout) + indexed_search_seconds = time.perf_counter() - indexed_search_started + ctx.equal(indexed_search["index_used"], True, "真实挂载数据使用 trigram 索引") + ctx.equal(indexed_search["match_count"], expected_matches, + "trigram 索引搜索命中数") + # 当前真实 FSKit helper 可能来自已安装的上一预览包。性能数据取完后关闭 + # 新索引，避免旧 helper 后续修改临时数据库时无法维护这个新合同。 + run_process( + [str(ctx.cli), "--db", str(adapter.database), "--workspace", "mount-scale", + "index", "disable"], timeout=300) + + snapshot_started = time.perf_counter() + run_process(adapter.prefix + ["snapshot", "create", "scale-baseline"], + timeout=3_600) + snapshot_seconds = time.perf_counter() - snapshot_started + original = b"file=0; marker=needle\n" + run_process(adapter.prefix + ["write", "/scale/d0000/f000000.txt"], + input_data=b"changed\n", timeout=300) + diff = run_process(adapter.prefix + ["snapshot", "diff", "scale-baseline"], + check=False, timeout=3_600) + ctx.equal(diff.returncode, 1, "真实规模目录快照检测变化") + restore_started = time.perf_counter() + run_process(adapter.prefix + ["snapshot", "restore", "scale-baseline"], + timeout=3_600) + restore_seconds = time.perf_counter() - restore_started + + mount_adapter_mount(ctx, adapter) + try: + ctx.equal((root / "d0000/f000000.txt").read_bytes(), original, + "真实规模目录快照恢复内容") + remounted_files = sum(len(files) for _, _, files in os.walk(root)) + ctx.equal(remounted_files, file_count, "真实规模目录重挂载文件数") + finally: + mount_adapter_unmount(ctx, adapter, "规模第二次") + + connection = sqlite3.connect(adapter.database) + storage_rows: dict[str, int] = {} + try: + ctx.equal(connection.execute("PRAGMA integrity_check").fetchone()[0], "ok", + "真实规模目录数据库完整性") + storage_rows = { + label: int(connection.execute(f"SELECT count(*) FROM {table}").fetchone()[0]) + for label, table in ( + ("inode_rows", "_vexfs_inodes"), + ("commit_rows", "_vexfs_commits"), + ("file_version_rows", "_vexfs_file_versions"), + ("request_rows", "_vexfs_requests"), + ("inode_history_rows", "_vexfs_inode_states"), + ("dentry_history_rows", "_vexfs_dentry_states"), + ("acl_rows", "_vexfs_acl_entries"), + ("xattr_rows", "_vexfs_xattrs"), + ) + } + finally: + connection.close() + create_budget = {"quick": 600.0, "full": 3_600.0, "stress": 14_400.0}[ + ctx.mode.name] + ctx.check(create_seconds >= 0, "mount_scale_create_seconds 不能为负数") + deferred_failures: list[str] = [] + if ctx.enforce_performance: + ctx.checks += 1 + if create_seconds > create_budget: + deferred_failures.append( + "mount_scale_create_seconds 超过预算: " + f"{create_seconds:.3f} > {create_budget:.3f}") + if search_timed_out: + ctx.checks += 1 + deferred_failures.append("真实规模目录文本搜索在 3600 秒内未完成") + metrics = { + "adapter": adapter.name, + "files": file_count, + "directories": directory_count, + "create_seconds": round(create_seconds, 6), + "create_files_per_second": round(file_count / max(create_seconds, 1e-9), 3), + "walk_seconds": round(walk_seconds, 6), + "search_seconds": round(search_seconds, 6), + "search_timed_out": search_timed_out, + "search_matches": matches, + "database_search_seconds": round(database_search_seconds, 6), + "database_search_matches": database_search["match_count"], + "database_search_files_per_second": round( + file_count / max(database_search_seconds, 1e-9), 3), + "index_build_seconds": round(index_build_seconds, 6), + "indexed_search_seconds": round(indexed_search_seconds, 6), + "indexed_search_matches": indexed_search["match_count"], + "indexed_search_candidates": indexed_search["files_scanned"], + "native_create_seconds": round(native_create_seconds, 6), + "native_create_files_per_second": round( + file_count / max(native_create_seconds, 1e-9), 3), + "native_walk_seconds": round(native_walk_seconds, 6), + "native_search_seconds": round(native_search_seconds, 6), + "create_slowdown_vs_native": round( + create_seconds / max(native_create_seconds, 1e-9), 3), + "search_slowdown_vs_native": round( + search_seconds / max(native_search_seconds, 1e-9), 3), + "snapshot_seconds": round(snapshot_seconds, 6), + "restore_seconds": round(restore_seconds, 6), + "database_bytes": adapter.database.stat().st_size, + "database_bytes_per_file": round( + adapter.database.stat().st_size / max(file_count, 1), 3), + **storage_rows, + "commits_per_file": round(storage_rows["commit_rows"] / file_count, 3), + "versions_per_file": round(storage_rows["file_version_rows"] / file_count, 3), + "requests_per_file": round(storage_rows["request_rows"] / file_count, 3), + } + if deferred_failures: + raise EvalFailureWithMetrics("; ".join(deferred_failures), metrics) + return metrics + + def worker_main(arguments: list[str]) -> int: operation = arguments[0] if operation == "parallel-write": @@ -1703,6 +4611,33 @@ def worker_main(arguments: list[str]) -> int: print(str(error), file=sys.stderr) return 5 raise + if operation == "snapshot-restore-race": + path, extension, expected_head_text, ready_path, barrier_path = arguments[1:] + connection = sqlite3.connect(path, isolation_level=None, timeout=30) + connection.enable_load_extension(True) + connection.load_extension(extension) + connection.execute("PRAGMA busy_timeout=30000") + connection.execute("PRAGMA journal_mode=WAL") + Path(ready_path).touch() + deadline = time.time() + 10 + while not Path(barrier_path).exists(): + if time.time() >= deadline: + connection.close() + return 9 + time.sleep(0.005) + try: + commit = connection.execute( + "SELECT vexfs_snapshot_restore('default','race-point',?)", + (int(expected_head_text),)).fetchone()[0] + print(commit) + connection.close() + return 0 + except sqlite3.Error as error: + connection.close() + if "conflict" in str(error): + print(str(error), file=sys.stderr) + return 5 + raise if operation == "crash-write": path, extension, file_path, value, mode = arguments[1:] connection = sqlite3.connect(path, isolation_level=None, timeout=30) @@ -1778,18 +4713,35 @@ def run_eval(options: argparse.Namespace) -> int: root = Path(options.root).resolve() build_dir = Path(options.build_dir).resolve() ext = extension_path(build_dir) - cli = (build_dir / "vexfs").resolve() + # 保留 vexfs 符号链接本身，统一二进制通过 argv[0] 进入兼容文件命令。 + cli = build_dir / "vexfs" + if not cli.exists(): + raise SystemExit(f"找不到 VexFS CLI，请先构建: {cli}") mode = MODES[options.mode] seed = options.seed run_id = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S.%fZ") + f"-{mode.name}-{seed}" output_dir = Path(options.output_dir).resolve() / run_id output_dir.mkdir(parents=True, exist_ok=False) + mount_cli_override = Path(options.mount_cli).expanduser().absolute() if options.mount_cli else None + if mount_cli_override is not None and not mount_cli_override.exists(): + raise SystemExit(f"指定的 mount CLI 不存在: {mount_cli_override}") ctx = Context(root, build_dir, ext, cli, output_dir, mode, seed, - options.enforce_performance) - selected = [item for item in CASES if mode.name in item.modes] + options.enforce_performance, options.fail_on_skip, + mount_cli_override) + selected = [item for item in CASES + if mode.name in item.modes and + (item.category != "package" or options.include_package)] if options.filter: selected = [item for item in selected if options.filter in item.case_id or options.filter in item.category] + if not selected: + raise SystemExit( + f"没有匹配的 eval 用例: filter={options.filter!r}, " + f"include_package={options.include_package}") + if (options.fail_on_skip and platform.system() == "Darwin" and + any(item.category == "mount" for item in selected) and + mount_cli_override is None): + raise SystemExit("macOS 发行 mount Gate 必须用 --mount-cli 显式绑定签名 CLI") started_all = time.perf_counter() results: list[dict[str, Any]] = [] print(f"VexFS eval: mode={mode.name} seed={seed} cases={len(selected)}") @@ -1810,6 +4762,11 @@ def run_eval(options: argparse.Namespace) -> int: except EvalSkip as exception: status = "SKIP" error = str(exception) + except EvalFailureWithMetrics as exception: + status = "FAIL" + metrics = exception.metrics + error = str(exception) + trace = traceback.format_exc() except Exception as exception: # 每个 case 独立记录，继续跑完全部。 status = "FAIL" error = str(exception) @@ -1839,6 +4796,15 @@ def run_eval(options: argparse.Namespace) -> int: passed = sum(item["status"] == "PASS" for item in results) failed = sum(item["status"] == "FAIL" for item in results) skipped = sum(item["status"] == "SKIP" for item in results) + gate_failed = failed > 0 or (options.fail_on_skip and skipped > 0) + if gate_failed: + overall_status = "FAIL" + elif skipped > 0: + overall_status = "PASS_WITH_SKIPS" + else: + overall_status = "PASS" + selected_mount_cli = mount_cli(ctx) + mount_cli_sha256 = hashlib.sha256(selected_mount_cli.read_bytes()).hexdigest() report = { "schema_version": 1, "run_id": run_id, @@ -1846,11 +4812,14 @@ def run_eval(options: argparse.Namespace) -> int: "mode": mode.name, "seed": seed, "enforce_performance": options.enforce_performance, + "fail_on_skip": options.fail_on_skip, "environment": system_details(root, build_dir, ext), "extension": str(ext), + "mount_cli": str(selected_mount_cli), + "mount_cli_sha256": mount_cli_sha256, "cases": results, "summary": { - "status": "FAIL" if failed else "PASS", + "status": overall_status, "passed": passed, "failed": failed, "skipped": skipped, @@ -1871,7 +4840,7 @@ def run_eval(options: argparse.Namespace) -> int: f"{skipped} skipped, {ctx.checks} checks, {duration_all:.3f}s") print(f"JSON: {json_path}") print(f"Markdown: {markdown_path}") - return 1 if failed else 0 + return 1 if gate_failed else 0 def parse_arguments(arguments: list[str]) -> argparse.Namespace: @@ -1885,6 +4854,12 @@ def parse_arguments(arguments: list[str]) -> argparse.Namespace: parser.add_argument("--mode", choices=sorted(MODES), default="quick") parser.add_argument("--seed", type=int, default=20260718) parser.add_argument("--filter", default="", help="按 case id 或 category 过滤") + parser.add_argument("--include-package", action="store_true", + help="额外运行需要预先重打包和签名的交付包测试") + parser.add_argument("--fail-on-skip", action="store_true", + help="任何 SKIP 都视为 Gate 失败；发行和真机 wrapper 必须开启") + parser.add_argument("--mount-cli", default=os.environ.get("VEXFS_EVAL_MOUNT_CLI", ""), + help="真实 macOS Gate 使用的签名 vexdb/vexfs；不再猜测 dist 最新包") parser.add_argument("--enforce-performance", action="store_true") options = parser.parse_args(arguments) options.worker = None diff --git a/tests/eval/vexfs/run_cross_platform_portability.sh b/tests/eval/vexfs/run_cross_platform_portability.sh new file mode 100755 index 0000000000..b787d8c02c --- /dev/null +++ b/tests/eval/vexfs/run_cross_platform_portability.sh @@ -0,0 +1,68 @@ +#!/bin/bash +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" +IMAGE="${VEXFS_LINUX_EVAL_IMAGE:-vexdb-lite-vexfs-linux-eval:debian}" +HOST_BUILD_DIR="${VEXFS_MACOS_BUILD_DIR:-$ROOT/vexdb_sqlite/build}" +MOUNT_CLI="${VEXFS_EVAL_MOUNT_CLI:?set VEXFS_EVAL_MOUNT_CLI to the signed CLI from the current build}" +RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)-$$" +OUTPUT_ROOT="$ROOT/vexdb_sqlite/build/eval/vexfs-portability/$RUN_ID" +DATABASE="$OUTPUT_ROOT/shared.sqlite3" +RELATIVE_DATABASE="${DATABASE#"$ROOT/"}" + +mkdir -p "$OUTPUT_ROOT/mac-create" "$OUTPUT_ROOT/linux-roundtrip" "$OUTPUT_ROOT/mac-verify" + +require_pass() { + local report="$1" + python3 -c \ + 'import json,sys; from pathlib import Path; latest=Path(sys.argv[1]); pointer=json.load(open(latest)); reports=sorted(latest.parent.glob("*/report.json"), key=lambda path: path.stat().st_mtime); assert reports, latest; value=json.load(open(reports[-1])); summary=value["summary"]; assert pointer["status"] == "PASS" and summary["failed"] == 0 and summary["passed"] == 1 and summary["skipped"] == 0, summary' \ + "$report" +} + +echo "=== macOS FSKit 创建共享工作区 ===" +VEXFS_PORTABILITY_PHASE=mac-create \ +VEXFS_PORTABILITY_DB="$DATABASE" \ +python3 "$ROOT/tests/eval/vexfs/run.py" \ + --root "$ROOT" \ + --build-dir "$HOST_BUILD_DIR" \ + --output-dir "$OUTPUT_ROOT/mac-create" \ + --mode quick \ + --fail-on-skip \ + --mount-cli "$MOUNT_CLI" \ + --filter portability.cross-os-roundtrip +require_pass "$OUTPUT_ROOT/mac-create/latest.json" + +echo "=== Linux libfuse3 读取、恢复并修改同一数据库 ===" +docker build \ + --file "$ROOT/tests/eval/vexfs/Dockerfile.linux-fuse" \ + --tag "$IMAGE" \ + "$ROOT/tests/eval/vexfs" +docker run --rm \ + --cap-add SYS_ADMIN \ + --device /dev/fuse \ + --security-opt apparmor=unconfined \ + --env VEXFS_PORTABILITY_PHASE=linux-roundtrip \ + --env "VEXFS_PORTABILITY_DB=/workspace/$RELATIVE_DATABASE" \ + --env "VEXFS_PORTABILITY_OUTPUT=/workspace/${OUTPUT_ROOT#"$ROOT/"}/linux-roundtrip" \ + --volume "$ROOT:/workspace" \ + --workdir /workspace \ + "$IMAGE" \ + /bin/bash /workspace/tests/eval/vexfs/run_cross_platform_portability_in_container.sh +require_pass "$OUTPUT_ROOT/linux-roundtrip/latest.json" + +echo "=== macOS FSKit 回读并恢复 Linux 快照 ===" +VEXFS_PORTABILITY_PHASE=mac-verify \ +VEXFS_PORTABILITY_DB="$DATABASE" \ +python3 "$ROOT/tests/eval/vexfs/run.py" \ + --root "$ROOT" \ + --build-dir "$HOST_BUILD_DIR" \ + --output-dir "$OUTPUT_ROOT/mac-verify" \ + --mode quick \ + --fail-on-skip \ + --mount-cli "$MOUNT_CLI" \ + --filter portability.cross-os-roundtrip +require_pass "$OUTPUT_ROOT/mac-verify/latest.json" + +echo "VEXFS CROSS-PLATFORM PORTABILITY: PASS" +echo " database: $DATABASE" +echo " reports: $OUTPUT_ROOT" diff --git a/tests/eval/vexfs/run_cross_platform_portability_in_container.sh b/tests/eval/vexfs/run_cross_platform_portability_in_container.sh new file mode 100755 index 0000000000..639f0da29e --- /dev/null +++ b/tests/eval/vexfs/run_cross_platform_portability_in_container.sh @@ -0,0 +1,27 @@ +#!/bin/bash +set -euo pipefail + +BUILD_DIR=/tmp/vexfs-portability-build +BUILD_LOG=/tmp/vexfs-portability-build.log +OUTPUT_DIR="${VEXFS_PORTABILITY_OUTPUT:?missing VEXFS_PORTABILITY_OUTPUT}" +: "${VEXFS_PORTABILITY_DB:?missing VEXFS_PORTABILITY_DB}" +: "${VEXFS_PORTABILITY_PHASE:?missing VEXFS_PORTABILITY_PHASE}" + +if ! cmake -S /workspace/vexdb_sqlite -B "$BUILD_DIR" \ + -DCMAKE_BUILD_TYPE=Release \ + -DVEXDB_SQLITE_BUILD_TESTS=ON >"$BUILD_LOG" 2>&1; then + tail -n 200 "$BUILD_LOG" + exit 1 +fi +if ! cmake --build "$BUILD_DIR" -j "$(nproc)" >>"$BUILD_LOG" 2>&1; then + tail -n 200 "$BUILD_LOG" + exit 1 +fi + +python3 /workspace/tests/eval/vexfs/run.py \ + --root /workspace \ + --build-dir "$BUILD_DIR" \ + --output-dir "$OUTPUT_DIR" \ + --mode quick \ + --fail-on-skip \ + --filter portability.cross-os-roundtrip diff --git a/tests/eval/vexfs/run_linux_mount.sh b/tests/eval/vexfs/run_linux_mount.sh new file mode 100755 index 0000000000..fc2dfcb8b7 --- /dev/null +++ b/tests/eval/vexfs/run_linux_mount.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" +IMAGE="${VEXFS_LINUX_EVAL_IMAGE:-vexdb-lite-vexfs-linux-eval:debian}" + +docker build \ + --file "$ROOT/tests/eval/vexfs/Dockerfile.linux-fuse" \ + --tag "$IMAGE" \ + "$ROOT/tests/eval/vexfs" + +docker run --rm \ + --cap-add SYS_ADMIN \ + --device /dev/fuse \ + --security-opt apparmor=unconfined \ + --env "VEXFS_LINUX_EVAL_CASES=${VEXFS_LINUX_EVAL_CASES:-}" \ + --volume "$ROOT:/workspace" \ + --workdir /workspace \ + "$IMAGE" \ + /bin/bash /workspace/tests/eval/vexfs/run_linux_mount_in_container.sh diff --git a/tests/eval/vexfs/run_linux_mount_in_container.sh b/tests/eval/vexfs/run_linux_mount_in_container.sh new file mode 100755 index 0000000000..7f5ace4990 --- /dev/null +++ b/tests/eval/vexfs/run_linux_mount_in_container.sh @@ -0,0 +1,48 @@ +#!/bin/bash +set -euo pipefail + +BUILD_DIR=/tmp/vexfs-linux-build +OUTPUT_BASE=/workspace/vexdb_sqlite/build/eval/vexfs-linux-mount +ROOT_OUTPUT="$OUTPUT_BASE/root" +USER_OUTPUT="$OUTPUT_BASE/uid-1000" +BUILD_LOG=/tmp/vexfs-linux-build.log + +if ! cmake -S /workspace/vexdb_sqlite -B "$BUILD_DIR" \ + -DCMAKE_BUILD_TYPE=Release \ + -DVEXDB_SQLITE_BUILD_TESTS=ON >"$BUILD_LOG" 2>&1; then + tail -n 200 "$BUILD_LOG" + exit 1 +fi +if ! cmake --build "$BUILD_DIR" -j "$(nproc)" >>"$BUILD_LOG" 2>&1; then + tail -n 200 "$BUILD_LOG" + exit 1 +fi + +read -r -a MOUNT_CASES <<< "${VEXFS_LINUX_EVAL_CASES:-mount.cross-platform-conformance mount.timestamps mount.concurrent-append mount.open-rename-unlink mount.process-locks mount.force-unmount mount.helper-crash-recovery}" + +for test_case in "${MOUNT_CASES[@]}"; do + python3 /workspace/tests/eval/vexfs/run.py \ + --root /workspace \ + --build-dir "$BUILD_DIR" \ + --output-dir "$ROOT_OUTPUT" \ + --mode quick \ + --fail-on-skip \ + --filter "$test_case" +done + +useradd --create-home --uid 1000 vexeval +chmod 0666 /dev/fuse +mkdir -p "$ROOT_OUTPUT" "$USER_OUTPUT" +chown vexeval:vexeval "$USER_OUTPUT" +for test_case in "${MOUNT_CASES[@]}"; do + su --shell /bin/bash vexeval --command \ + "python3 /workspace/tests/eval/vexfs/run.py \ + --root /workspace \ + --build-dir $BUILD_DIR \ + --output-dir $USER_OUTPUT \ + --mode quick \ + --fail-on-skip \ + --filter $test_case" +done + +echo "VEXFS LINUX MOUNT CONFORMANCE: PASS" diff --git a/tests/spec/_lib/docker/run_sqlite.sh b/tests/spec/_lib/docker/run_sqlite.sh index 3c447bb2d3..d8be384765 100644 --- a/tests/spec/_lib/docker/run_sqlite.sh +++ b/tests/spec/_lib/docker/run_sqlite.sh @@ -3,6 +3,8 @@ # # bash run_sqlite.sh # 渲染 + 全量跑 # bash run_sqlite.sh # 只跑名字含 的 spec +# VEXDB_SQLITE_EXTENSION=/abs/vexdb_lite.dylib bash run_sqlite.sh +# # 显式绑定本次构建，避免误测旧产物 # # 协议： # - 渲染产物 build/spec/sqlite/sql/.sql + expected/.out @@ -16,4 +18,5 @@ ROOT_DIR="$(cd "$(dirname "$0")/../../../.." && pwd)" FILTER="${1:-}" python3 "$ROOT_DIR/tests/spec/_lib/render.py" --engine sqlite --out build/spec >/dev/null || exit 1 -exec python3 "$ROOT_DIR/tests/spec/_lib/docker/sqlite_spec_runner.py" "$ROOT_DIR" "$FILTER" +exec python3 "$ROOT_DIR/tests/spec/_lib/docker/sqlite_spec_runner.py" \ + "$ROOT_DIR" "$FILTER" "${VEXDB_SQLITE_EXTENSION:-}" diff --git a/tests/spec/_lib/docker/sqlite_spec_runner.py b/tests/spec/_lib/docker/sqlite_spec_runner.py index dec7fae42c..3127a9f673 100644 --- a/tests/spec/_lib/docker/sqlite_spec_runner.py +++ b/tests/spec/_lib/docker/sqlite_spec_runner.py @@ -103,10 +103,18 @@ def main() -> int: for old in results.glob("*.diff"): old.unlink() - ext = str(root / "vexdb_sqlite/build/vexdb_lite") - if not (Path(ext + ".dylib").exists() or Path(ext + ".so").exists()): - print("loadable 扩展不存在，先 bash build_sqlite.sh build", file=sys.stderr) - return 1 + configured_ext = sys.argv[3] if len(sys.argv) > 3 else "" + if configured_ext: + ext_path = Path(configured_ext).resolve() + if not ext_path.is_file(): + print(f"显式指定的 loadable 扩展不存在: {ext_path}", file=sys.stderr) + return 1 + ext = str(ext_path) + else: + ext = str(root / "vexdb_sqlite/build/vexdb_lite") + if not (Path(ext + ".dylib").exists() or Path(ext + ".so").exists()): + print("loadable 扩展不存在，先 bash build_sqlite.sh build", file=sys.stderr) + return 1 compare = root / "tests/spec/_lib/docker/compare.py" passed, failed, failed_names = 0, 0, [] diff --git a/tests/spec/sqlite/agent_files/vexfs_contract.yaml b/tests/spec/sqlite/agent_files/vexfs_contract.yaml index dc25faf368..18c483aa3c 100644 --- a/tests/spec/sqlite/agent_files/vexfs_contract.yaml +++ b/tests/spec/sqlite/agent_files/vexfs_contract.yaml @@ -4,7 +4,7 @@ description: VexFS 文件目录、事务、句柄写入、幂等请求和并发 steps: - query: SELECT vexfs_contract_version(); expect: - - - 0.3.0 + - - 0.7.0 - query: SELECT vexfs_init(); expect: - - 1 @@ -20,6 +20,39 @@ steps: - query: SELECT CAST(vexfs_read('default', '/notes/2026/hello.txt') AS TEXT); expect: - - hello +- query: SELECT vexfs_create('default', '/notes/run.sh', 'file', 488); + expect: + - - 5 +- query: SELECT json_extract(vexfs_stat('default', '/notes/run.sh'), '$.mode'); + expect: + - - 488 +- query: SELECT vexfs_chown('default', 5, 123, 456); + expect: + - - 123 +- query: SELECT json_extract(vexfs_stat('default', '/notes/run.sh'), '$.uid'), json_extract(vexfs_stat('default', '/notes/run.sh'), '$.gid'); + expect: + - - 123 + - 456 +- query: SELECT vexfs_acl_set('default', 5, '[{"principal":"alice","effect":"allow","permissions":"read,write","inherit":1}]'); + expect: + - - 1 +- query: SELECT json_extract(vexfs_acl_get('default', 5), '$[0].principal'), json_extract(vexfs_acl_get('default', 5), '$[0].permissions'); + expect: + - - alice + - read,write +- query: SELECT vexfs_set_mode('default', 5, 448); + expect: + - - 448 +- query: SELECT vexfs_symlink('default', '/notes/run-link', X'72756E2E7368'); + expect: + - - 6 +- query: SELECT CAST(vexfs_readlink('default', 6) AS TEXT); + expect: + - - run.sh +- query: SELECT json_extract(vexfs_stat('default', '/notes/run-link'), '$.kind'), json_extract(vexfs_stat('default', '/notes/run-link'), '$.mode'); + expect: + - - symlink + - 511 - statement: BEGIN; - query: SELECT vexfs_write('default', '/notes/2026/hello.txt', 'rolled back'); expect: @@ -109,3 +142,80 @@ steps: - file - 5 - 4 + - - run-link + - 6 + - symlink + - 6 + - 1 + - - run.sh + - 5 + - file + - 0 + - 1 +- query: SELECT vexfs_xattr_set('default', 5, 'user.spec', 'old', 0); + expect: + - - 1 +- query: SELECT vexfs_snapshot_create('default', 'spec-baseline') = (SELECT head_commit FROM _vexfs_workspaces WHERE name='default'); + expect: + - - 1 +- query: SELECT json_extract(vexfs_snapshot_show('default', 'spec-baseline'), '$.name'); + expect: + - - spec-baseline +- query: SELECT vexfs_write('default', '/notes/hello.txt', 'changed'); + expect: + - - 5 +- query: SELECT vexfs_set_mode('default', 5, 384); + expect: + - - 384 +- query: SELECT vexfs_xattr_set('default', 5, 'user.spec', 'new', 0); + expect: + - - 1 +- query: SELECT vexfs_chown('default', 5, 789, 987); + expect: + - - 789 +- query: SELECT vexfs_acl_set('default', 5, '[]'); + expect: + - - 0 +- query: SELECT vexfs_remove('default', '/notes/run-link', 0); + expect: + - - 1 +- query: SELECT vexfs_write('default', '/notes/extra.txt', 'extra'); + expect: + - - 1 +- query: SELECT json_array_length(json_extract(vexfs_snapshot_diff('default', 'spec-baseline', 'HEAD'), '$.changes')) > 0; + expect: + - - 1 +- query: SELECT vexfs_snapshot_restore('default', 'spec-baseline', (SELECT head_commit FROM _vexfs_workspaces WHERE name='default')) > 0; + expect: + - - 1 +- query: SELECT CAST(vexfs_read('default', '/notes/hello.txt') AS TEXT), json_extract(vexfs_stat('default', '/notes/run.sh'), '$.mode'), json_extract(vexfs_stat('default', '/notes/run.sh'), '$.uid'), json_extract(vexfs_stat('default', '/notes/run.sh'), '$.gid'), json_extract(vexfs_acl_get('default', 5), '$[0].principal'), CAST(vexfs_readlink('default', 6) AS TEXT), CAST(vexfs_xattr_get('default', 5, 'user.spec') AS TEXT); + expect: + - - hello + - 448 + - 123 + - 456 + - alice + - run.sh + - old +- query: SELECT vexfs_stat('default', '/notes/extra.txt'); + expect_error: path not found +- query: SELECT json_array_length(json_extract(vexfs_snapshot_diff('default', 'spec-baseline', 'HEAD'), '$.changes')); + expect: + - - 0 +- query: SELECT json_extract(vexfs_stat('default', '/notes/hello.txt'), '$.checksum'); + expect: + - - 2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824 +- query: SELECT json_extract(vexfs_check('default', 1), '$.ok'), json_extract(vexfs_check('default', 1), '$.mode'), json_extract(vexfs_check('default', 1), '$.issue_count'); + expect: + - - 1 + - deep + - 0 +- query: SELECT json_extract(vexfs_check('default', 0), '$.ok'), json_extract(vexfs_check('default', 0), '$.mode'); + expect: + - - 1 + - quick +- query: SELECT vexfs_snapshot_restore('default', 'spec-baseline', (SELECT head_commit FROM _vexfs_workspaces WHERE name='default')); + expect_error: already matches +- query: SELECT vexfs_snapshot_drop('default', 'spec-baseline'); + expect: + - - 1 diff --git a/tests/spec/sqlite/vtab/vtab_verify.yaml b/tests/spec/sqlite/vtab/vtab_verify.yaml index 21d521c317..de93a4ce0c 100644 --- a/tests/spec/sqlite/vtab/vtab_verify.yaml +++ b/tests/spec/sqlite/vtab/vtab_verify.yaml @@ -2,7 +2,7 @@ name: sqlite__vtab_verify tags: [vex, sqlite] description: 扩展自省——版本函数可调、全部函数已注册、shadow 表三件套与 vtab 共存于 sqlite_master（对照 duck graph_index_verify 行为面） steps: -- query: SELECT vexdb_version() LIKE 'vexdb_lite sqlite extension%'; +- query: SELECT vexdb_version() LIKE 'vexdb-lite % sqlite extension%'; expect: - - 1 # 六个标量函数全部注册（任一缺失会直接报 no such function） diff --git a/vexdb_sqlite/CMakeLists.txt b/vexdb_sqlite/CMakeLists.txt index 505c449ede..28c31552c5 100644 --- a/vexdb_sqlite/CMakeLists.txt +++ b/vexdb_sqlite/CMakeLists.txt @@ -14,6 +14,7 @@ set(CMAKE_POSITION_INDEPENDENT_CODE ON) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) option(VEXDB_SQLITE_BUILD_TESTS "Build M0 smoke tests" ON) +set(VEXDB_LITE_VERSION "0.1.0-dev" CACHE STRING "VexDB-Lite product version") find_program(CCACHE_PROGRAM ccache) if(CCACHE_PROGRAM) @@ -102,6 +103,7 @@ endif() set(VEXDB_SQLITE_SRCS src/vexdb_sqlite_init.cpp src/agent_files/vexfs_sqlite.cpp + ${REPO_ROOT}/agent_files/mount/common/src/vexfs_checksum.cpp src/vtab/graph_index_vtab.cpp src/functions/distance_functions.cpp src/functions/vector_codec.cpp @@ -112,6 +114,7 @@ set(VEXDB_SQLITE_SRCS ) set(VEXDB_SQLITE_INCLUDES + ${REPO_ROOT}/agent_files/mount/common/include ${CMAKE_CURRENT_SOURCE_DIR}/include ${CMAKE_CURRENT_SOURCE_DIR}/include/store ${SQLITE_AMALG_DIR} @@ -123,15 +126,20 @@ set(VEXDB_SQLITE_INCLUDES ${VEXDB_BOOST_INC} ) -# 版本信息注入（vexdb_version() 输出；对齐 duck 端做法） -execute_process(COMMAND git rev-parse --short HEAD - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} - OUTPUT_VARIABLE VEXDB_SQLITE_GIT_HASH OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET) +# 版本信息注入（vexdb_version() 输出；对齐 duck 端做法）。远程发行构建不带 +# .git，允许发布脚本把本机提交号显式传进来，避免产物显示 unknown。 +set(VEXDB_SQLITE_GIT_HASH "" CACHE STRING "VexDB-Lite source revision") +if(NOT VEXDB_SQLITE_GIT_HASH) + execute_process(COMMAND git rev-parse --short HEAD + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + OUTPUT_VARIABLE VEXDB_SQLITE_GIT_HASH OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET) +endif() if(NOT VEXDB_SQLITE_GIT_HASH) set(VEXDB_SQLITE_GIT_HASH "unknown") endif() string(TIMESTAMP VEXDB_SQLITE_BUILD_TIME "%Y-%m-%d %H:%M") add_compile_definitions( + VEXDB_LITE_VERSION="${VEXDB_LITE_VERSION}" VEXDB_SQLITE_GIT_HASH="${VEXDB_SQLITE_GIT_HASH}" VEXDB_SQLITE_BUILD_TIME="${VEXDB_SQLITE_BUILD_TIME}") @@ -208,42 +216,104 @@ target_include_directories(vexdb_lite_static PRIVATE ${VEXDB_SQLITE_INCLUDES}) target_compile_definitions(vexdb_lite_static PRIVATE VEXDB_SQLITE_CORE=1) set_target_properties(vexdb_lite_static PROPERTIES OUTPUT_NAME "vexdb_lite_static") -# 平台挂载适配层共用的稳定 C ABI。FSKit/libfuse/WinFsp 都只接这一层， -# 具体状态仍由上面的 SQLite 插件通过 vexfs_* SQL 管理。 -add_library(vexfs_mount_contract STATIC +# 平台和 CLI 共用的 runtime。平台只能包含 runtime_io，CLI 额外包含 +# runtime_admin；SQLite 只是当前 backend adapter。 +add_library(vexfs_runtime STATIC ${REPO_ROOT}/agent_files/mount/common/src/vexfs_mount_contract_sqlite.cpp) -target_include_directories(vexfs_mount_contract PUBLIC +target_include_directories(vexfs_runtime PUBLIC ${REPO_ROOT}/agent_files/mount/common/include ${CMAKE_CURRENT_SOURCE_DIR}/include ${SQLITE_AMALG_DIR}) -target_compile_definitions(vexfs_mount_contract PRIVATE VEXDB_SQLITE_CORE=1) -target_link_libraries(vexfs_mount_contract PUBLIC vexdb_lite_static) -set_target_properties(vexfs_mount_contract PROPERTIES OUTPUT_NAME "vexfs_mount_contract") +target_compile_definitions(vexfs_runtime PRIVATE VEXDB_SQLITE_CORE=1) +target_link_libraries(vexfs_runtime PUBLIC vexdb_lite_static) +set_target_properties(vexfs_runtime PROPERTIES OUTPUT_NAME "vexfs_runtime") -if(APPLE AND NOT CMAKE_SYSTEM_NAME STREQUAL "iOS") - enable_language(OBJCXX) - find_library(VEXFS_SYSTEM_SQLITE3 sqlite3 REQUIRED) - find_library(VEXFS_FOUNDATION Foundation REQUIRED) - add_executable(vexfs_cli - ${REPO_ROOT}/agent_files/cli/vexfs_main.cpp - ${REPO_ROOT}/agent_files/cli/vexfs_fskit_state.mm) - set_source_files_properties( - ${REPO_ROOT}/agent_files/cli/vexfs_fskit_state.mm - PROPERTIES COMPILE_FLAGS "-fobjc-arc") - target_include_directories(vexfs_cli PRIVATE - ${REPO_ROOT}/agent_files/mount/common/include - ${REPO_ROOT}/agent_files/cli) - target_link_libraries(vexfs_cli PRIVATE vexfs_mount_contract - ${VEXFS_SYSTEM_SQLITE3} ${VEXFS_FOUNDATION} Threads::Threads ${CMAKE_DL_LIBS}) - set_target_properties(vexfs_cli PROPERTIES OUTPUT_NAME "vexfs") +if(NOT CMAKE_SYSTEM_NAME STREQUAL "iOS" AND NOT ANDROID) + set(VEXDB_SQLITE_DESKTOP ON) endif() -if(VEXDB_SQLITE_BUILD_TESTS) - # 把 amalgamation 编成静态库供 smoke test 链接（模拟移动端宿主自带 sqlite3）。 +if(VEXDB_SQLITE_BUILD_TESTS OR VEXDB_SQLITE_DESKTOP) + # 自带 SQLite 的 shell 和 smoke test 共用同一份锁定版本 amalgamation。 add_library(sqlite3_amalg STATIC ${SQLITE_AMALG_DIR}/sqlite3.c) target_include_directories(sqlite3_amalg PUBLIC ${SQLITE_AMALG_DIR}) - target_compile_definitions(sqlite3_amalg PRIVATE SQLITE_ENABLE_LOAD_EXTENSION=1) + target_compile_definitions(sqlite3_amalg PRIVATE + SQLITE_ENABLE_LOAD_EXTENSION=1 SQLITE_ENABLE_FTS5=1) +endif() +if(VEXDB_SQLITE_DESKTOP) + set_source_files_properties(${SQLITE_AMALG_DIR}/shell.c PROPERTIES + COMPILE_DEFINITIONS "main=vexdb_sqlite_shell_main;SQLITE_SHELL_INIT_PROC=vexdb_shell_init") + set(VEXDB_CLI_SOURCES + ${REPO_ROOT}/agent_files/cli/vexdb_main.cpp + ${REPO_ROOT}/agent_files/cli/vexdb_shell_init.c + ${REPO_ROOT}/agent_files/cli/vexfs_main.cpp + ${REPO_ROOT}/agent_files/cli/vexfs_platform_registry.cpp + ${SQLITE_AMALG_DIR}/shell.c) + set(VEXDB_CLI_PLATFORM_LIBS) + if(APPLE) + enable_language(OBJCXX) + find_library(VEXFS_FOUNDATION Foundation REQUIRED) + list(APPEND VEXDB_CLI_SOURCES + ${REPO_ROOT}/agent_files/cli/vexfs_platform_macos.cpp + ${REPO_ROOT}/agent_files/cli/vexfs_fskit_state.mm) + set_source_files_properties( + ${REPO_ROOT}/agent_files/cli/vexfs_fskit_state.mm + PROPERTIES COMPILE_FLAGS "-fobjc-arc") + list(APPEND VEXDB_CLI_PLATFORM_LIBS ${VEXFS_FOUNDATION}) + elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux") + list(APPEND VEXDB_CLI_SOURCES + ${REPO_ROOT}/agent_files/cli/vexfs_platform_linux.cpp) + else() + list(APPEND VEXDB_CLI_SOURCES + ${REPO_ROOT}/agent_files/cli/vexfs_platform_stub.cpp) + endif() + add_executable(vexdb_cli ${VEXDB_CLI_SOURCES}) + target_include_directories(vexdb_cli PRIVATE + ${REPO_ROOT}/agent_files/mount/common/include + ${REPO_ROOT}/agent_files/cli + ${CMAKE_CURRENT_SOURCE_DIR}/include + ${SQLITE_AMALG_DIR}) + target_compile_definitions(vexdb_cli PRIVATE VEXFS_EMBEDDED_MAIN=1 VEXDB_SQLITE_CORE=1) + target_link_libraries(vexdb_cli PRIVATE vexfs_runtime sqlite3_amalg + ${VEXDB_CLI_PLATFORM_LIBS} Threads::Threads ${CMAKE_DL_LIBS}) + set_target_properties(vexdb_cli PROPERTIES OUTPUT_NAME "vexdb" LINKER_LANGUAGE CXX) + if(WIN32) + add_custom_command(TARGET vexdb_cli POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different + $ + $/vexfs${CMAKE_EXECUTABLE_SUFFIX} + COMMENT "Create vexfs compatibility entry") + else() + add_custom_command(TARGET vexdb_cli POST_BUILD + COMMAND ${CMAKE_COMMAND} -E remove -f $/vexfs + COMMAND ${CMAKE_COMMAND} -E create_symlink + $ $/vexfs + COMMENT "Create vexfs compatibility entry") + endif() + add_custom_target(vexfs_cli DEPENDS vexdb_cli) + + if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + find_package(PkgConfig QUIET) + if(PkgConfig_FOUND) + pkg_check_modules(FUSE3 QUIET IMPORTED_TARGET fuse3) + endif() + if(TARGET PkgConfig::FUSE3) + add_executable(vexfs_fuse + ${REPO_ROOT}/agent_files/mount/linux/vexfs_fuse.cpp) + target_include_directories(vexfs_fuse PRIVATE + ${REPO_ROOT}/agent_files/mount/common/include) + target_link_libraries(vexfs_fuse PRIVATE vexfs_runtime sqlite3_amalg + PkgConfig::FUSE3 Threads::Threads ${CMAKE_DL_LIBS}) + set_target_properties(vexfs_fuse PROPERTIES OUTPUT_NAME "vexfs-fuse") + add_dependencies(vexfs_cli vexfs_fuse) + else() + message(WARNING + "libfuse3 development files were not found; vexfs-fuse will not be built") + endif() + endif() +endif() + +if(VEXDB_SQLITE_BUILD_TESTS) foreach(smoke m0_static_smoke m1_distance_smoke m2_vtab_smoke m3_hnsw_smoke vexfs_static_smoke) add_executable(${smoke} test/${smoke}.c) target_include_directories(${smoke} PRIVATE ${VEXDB_SQLITE_INCLUDES}) @@ -260,10 +330,28 @@ if(VEXDB_SQLITE_BUILD_TESTS) target_link_libraries(m3p_parallel_smoke PRIVATE vexdb_lite_static sqlite3_amalg Threads::Threads ${CMAKE_DL_LIBS}) - add_executable(vexfs_mount_contract_smoke + add_executable(vexfs_runtime_smoke ${REPO_ROOT}/agent_files/mount/common/test/vexfs_mount_contract_smoke.cpp) - target_include_directories(vexfs_mount_contract_smoke PRIVATE + target_include_directories(vexfs_runtime_smoke PRIVATE ${REPO_ROOT}/agent_files/mount/common/include ${SQLITE_AMALG_DIR}) - target_link_libraries(vexfs_mount_contract_smoke PRIVATE - vexfs_mount_contract sqlite3_amalg Threads::Threads ${CMAKE_DL_LIBS}) + target_link_libraries(vexfs_runtime_smoke PRIVATE + vexfs_runtime sqlite3_amalg Threads::Threads ${CMAKE_DL_LIBS}) + + add_executable(vexfs_platform_registry_smoke + ${REPO_ROOT}/agent_files/cli/test/vexfs_platform_registry_smoke.cpp + ${REPO_ROOT}/agent_files/cli/vexfs_platform_registry.cpp) + target_include_directories(vexfs_platform_registry_smoke PRIVATE + ${REPO_ROOT}/agent_files/cli) + + foreach(platform linux windows) + add_executable(vexfs_platform_${platform}_smoke + ${REPO_ROOT}/agent_files/cli/test/vexfs_platform_stub_smoke.cpp + ${REPO_ROOT}/agent_files/cli/vexfs_platform_stub.cpp) + target_include_directories(vexfs_platform_${platform}_smoke PRIVATE + ${REPO_ROOT}/agent_files/cli) + string(TOUPPER "${platform}" platform_upper) + target_compile_definitions(vexfs_platform_${platform}_smoke PRIVATE + VEXFS_PLATFORM_STUB_${platform_upper}=1 + VEXFS_PLATFORM_EXPECT_${platform_upper}=1) + endforeach() endif() diff --git a/vexdb_sqlite/README.md b/vexdb_sqlite/README.md index 3a6e66b0d7..337e019ad6 100644 --- a/vexdb_sqlite/README.md +++ b/vexdb_sqlite/README.md @@ -1,11 +1,33 @@ # vexdb_sqlite — VexDB-Lite 的 SQLite 适配层 -把 VexDB 自研图索引（GRAPH_INDEX / HNSW）以 **SQLite 虚拟表 + Shadow Table** 形态外放，补齐三态同源的 **Edge** 一极。对标 sqlite-vec（有过滤、无 ANN），目标是 SQLite 生态第一个「持久化 ANN 索引 + 过滤查询 + 端侧 + 宽松许可」的向量扩展。 +SQLite 版在同一个扩展中提供 VexDB 自研图索引和 VexFS 文件能力。普通表、向量索引和文件可以保存在同一个数据库中，并由 SQLite 事务和 Backup API 统一管理。 > 完整计划：`docs/plans/2026-06-10_sqlite-adapter-v1-plan.md` > 范围调研：`docs/research/2026-06-10_sqlite-v1-scope-reinvestigation.md` -## 当前进度：M0 骨架 ✅ → M1 距离层 ✅ → M2 虚拟表+暴力 KNN ✅ +## 统一命令 + +macOS 构建会生成自带 SQLite 的 `vexdb`，无需手动加载扩展： + +```bash +vexdb agent.db +vexdb agent.db "SELECT vexdb_version();" +vexdb fs --db agent.db ls / +vexdb fs --db agent.db grep -n error / +vexdb fs --db agent.db index enable +vexdb fs --db agent.db check +``` + +`vexfs` 是同一个可执行文件的兼容入口，等价于 `vexdb fs`。 + +`grep` 默认通过单个数据库连接批量搜索，不需要逐个打开挂载文件。大工作区可显式开启 +FTS5 trigram 索引；索引默认关闭，关闭时不增加文件写入开销。宿主 SQLite 没有 FTS5 时 +仍可使用批量扫描，不会让文件不可读。 + +每个文件版本保存 SHA-256。`vexdb fs check` 默认只读检查目录、提交、历史、快照、暂存区 +和全部内容；`vexdb fs check --quick` 跳过 BLOB 哈希，只检查结构和引用。 + +现有 SQLite 程序仍然可以加载 `vexdb_lite.dylib`： ```sql .load ./vexdb_lite -- 桌面；移动端走静态注册 @@ -14,9 +36,11 @@ SELECT vexdb_l2_distance('[1,2,3]', '[4,5,6]'); -- 4 个距离函数 + vexd CREATE VIRTUAL TABLE idx USING GRAPH_INDEX(embedding FLOAT[128], metric=cosine, m=16); INSERT INTO idx(rowid, embedding) VALUES (1, :blob_or_json); -SELECT rowid, distance FROM idx WHERE embedding MATCH :query AND k = 10; -- 暴力 KNN（M3 切 HNSW） +SELECT rowid, distance FROM idx WHERE embedding MATCH :query AND k = 10; -- 持久化 ANN ``` +## 当前进度 + | 里程碑 | 验证 | 状态 | |---|---|---| | M0 双形态注册链路 | `m0_static_smoke` + CLI `.load` | ✅ arm64 + x86_64 | @@ -24,7 +48,8 @@ SELECT rowid, distance FROM idx WHERE embedding MATCH :query AND k = 10; -- 暴 | M2 虚拟表（shadow table 持久化 + 暴力 KNN） | `m2_vtab_smoke`（KNN 正确性/事务回滚/关库重开/错误路径） | ✅ arm64 + x86_64 | | M3 HNSW（共享算法 × SQLite store，>64 行走图，`%_graph` blob 持久化） | `m3_hnsw_smoke`（recall@10=1.000、增量、重开 blob 还原、DELETE/ROLLBACK） | ✅ arm64 + x86_64 | | M3+ 并行建图（rebuild 预读后多线程，publish fence，TSan 零 race） | `m3p_parallel_smoke`（N=40000、8 线程 ×3 轮 recall==串行 baseline） | ✅ | -| M4 spec 落地 / M5 桌面发版 | — | 下一步 | +| M4 spec 落地 | 多引擎 YAML spec | ✅ | +| M5 macOS 统一包 | `vexdb` + `vexfs` + dylib + App + FSKit | ✅ 技术预览 | 距离语义三 metric 统一 **lower = closer**（L2=sqrt、cosine=1-sim、ip=负内积），`ORDER BY distance ASC` 即最近优先。跨 ISA（NEON/SSE）允许 ~1e-6 级 float32 重排序分歧。 diff --git a/vexdb_sqlite/include/agent_files/vexfs_sqlite.h b/vexdb_sqlite/include/agent_files/vexfs_sqlite.h index 812f90edd9..6c7769d67a 100644 --- a/vexdb_sqlite/include/agent_files/vexfs_sqlite.h +++ b/vexdb_sqlite/include/agent_files/vexfs_sqlite.h @@ -15,6 +15,10 @@ extern "C" { #endif +// VexFS 私有扩展错误码。低 8 位保留 SQLite 主错误码，使不了解该扩展码的 +// 调用方仍可按 SQLITE_CONSTRAINT 处理；挂载合同则可还原成准确的 POSIX 错误。 +#define VEXFS_SQLITE_CONSTRAINT_NOT_EMPTY (SQLITE_CONSTRAINT | (200 << 8)) + // 注册 vexfs_* SQL 函数。成功返回 SQLITE_OK。 int vexfs_sqlite_register(sqlite3 *db); diff --git a/vexdb_sqlite/src/agent_files/vexfs_sqlite.cpp b/vexdb_sqlite/src/agent_files/vexfs_sqlite.cpp index f04fd8ff8e..ea4b109def 100644 --- a/vexdb_sqlite/src/agent_files/vexfs_sqlite.cpp +++ b/vexdb_sqlite/src/agent_files/vexfs_sqlite.cpp @@ -1,15 +1,21 @@ #include "vexdb_sqlite_internal.h" #include "agent_files/vexfs_sqlite.h" +#include "vexfs_checksum.h" #include #include +#include #include #include #include #include +#include +#include #include #include +#include +#include #include #include @@ -21,9 +27,23 @@ namespace { -constexpr const char *kContractVersion = "0.3.0"; +constexpr const char *kContractVersion = "0.7.0"; constexpr sqlite3_int64 kMaxStagedBytes = 128LL * 1024LL * 1024LL; constexpr sqlite3_int64 kInitialStagingCapacity = 64LL * 1024LL; +constexpr sqlite3_int64 kEmptyFileStagingCapacity = 4LL * 1024LL; +constexpr sqlite3_int64 kMaxXattrBytes = 64LL * 1024LL; +constexpr sqlite3_int64 kMaxSymlinkBytes = 4096; +constexpr sqlite3_int64 kRequestRetentionRows = 64LL * 1024LL; +constexpr sqlite3_int64 kRequestPruneInterval = 4LL * 1024LL; +constexpr sqlite3_int64 kMaxGrepResults = 10LL * 1024LL; +constexpr size_t kMaxGrepPatternBytes = 4u * 1024u; + +std::mutex g_schema_ready_mutex; +std::unordered_set g_schema_ready_connections; +enum class GrepIndexState { kDisabled, kAvailable, kUnavailable }; +std::unordered_map g_grep_index_connections; + +bool IsSha256(const std::string &value); class SqlError : public std::runtime_error { public: @@ -187,138 +207,297 @@ void AcquireWriteLock(sqlite3 *db) { Exec(db, "UPDATE _vexfs_meta SET value=value WHERE key='contract_version'"); } -bool HasVersionReferences(sqlite3 *db) { +bool HasTable(sqlite3 *db, const char *table) { Statement statement(db, - "SELECT 1 FROM pragma_table_info('_vexfs_file_versions') " - "WHERE name='source_version_no' LIMIT 1"); + "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?1 LIMIT 1"); + statement.BindText(1, table); return statement.Row(); } -bool HasVersionTable(sqlite3 *db) { - Statement statement(db, - "SELECT 1 FROM sqlite_master WHERE type='table' " - "AND name='_vexfs_file_versions' LIMIT 1"); - return statement.Row(); -} +void CreateHistorySchema(sqlite3 *db) { + Exec(db, R"SQL( +CREATE TABLE IF NOT EXISTS _vexfs_inode_states( + workspace_id INTEGER NOT NULL, + inode_id INTEGER NOT NULL, + commit_id INTEGER NOT NULL, + kind TEXT NOT NULL CHECK(kind IN ('file','directory','symlink')), + mode INTEGER NOT NULL, + owner_principal TEXT NOT NULL DEFAULT 'local', + uid INTEGER NOT NULL DEFAULT 0, + gid INTEGER NOT NULL DEFAULT 0, + size INTEGER NOT NULL, + current_version INTEGER NOT NULL, + created_at INTEGER NOT NULL, + accessed_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + changed_at INTEGER NOT NULL, + deleted_at INTEGER, + PRIMARY KEY(workspace_id, inode_id, commit_id) +); +CREATE INDEX IF NOT EXISTS _vexfs_inode_states_commit_idx + ON _vexfs_inode_states(workspace_id, commit_id, inode_id); +CREATE TABLE IF NOT EXISTS _vexfs_dentry_states( + workspace_id INTEGER NOT NULL, + parent_inode INTEGER NOT NULL, + name TEXT NOT NULL COLLATE BINARY, + commit_id INTEGER NOT NULL, + inode_id INTEGER NOT NULL, + deleted INTEGER NOT NULL CHECK(deleted IN (0,1)), + PRIMARY KEY(workspace_id, parent_inode, name, commit_id) +); +CREATE INDEX IF NOT EXISTS _vexfs_dentry_states_commit_idx + ON _vexfs_dentry_states(workspace_id, commit_id, parent_inode, name); +CREATE TABLE IF NOT EXISTS _vexfs_xattr_states( + workspace_id INTEGER NOT NULL, + inode_id INTEGER NOT NULL, + name TEXT NOT NULL COLLATE BINARY, + commit_id INTEGER NOT NULL, + value BLOB NOT NULL, + deleted INTEGER NOT NULL CHECK(deleted IN (0,1)), + PRIMARY KEY(workspace_id, inode_id, name, commit_id) +); +CREATE INDEX IF NOT EXISTS _vexfs_xattr_states_commit_idx + ON _vexfs_xattr_states(workspace_id, commit_id, inode_id, name); +CREATE TABLE IF NOT EXISTS _vexfs_acl_states( + workspace_id INTEGER NOT NULL, + inode_id INTEGER NOT NULL, + principal_id TEXT NOT NULL COLLATE BINARY, + effect TEXT NOT NULL, + commit_id INTEGER NOT NULL, + permissions TEXT NOT NULL, + inherit_flags INTEGER NOT NULL DEFAULT 0, + deleted INTEGER NOT NULL CHECK(deleted IN (0,1)), + PRIMARY KEY(workspace_id,inode_id,principal_id,effect,commit_id) +); +CREATE INDEX IF NOT EXISTS _vexfs_acl_states_commit_idx + ON _vexfs_acl_states(workspace_id,commit_id,inode_id,principal_id,effect); +CREATE TABLE IF NOT EXISTS _vexfs_snapshots( + id INTEGER PRIMARY KEY, + workspace_id INTEGER NOT NULL, + name TEXT NOT NULL COLLATE BINARY, + commit_id INTEGER NOT NULL, + created_at INTEGER NOT NULL DEFAULT (CAST(strftime('%s','now') AS INTEGER) * 1000), + UNIQUE(workspace_id, name) +); +CREATE INDEX IF NOT EXISTS _vexfs_snapshots_commit_idx + ON _vexfs_snapshots(workspace_id, commit_id); -void EnsureVersionReferences(sqlite3 *db) { - // 极早期 0.1 测试库允许只包含 handle/staging 表；没有版本表时不凭空补半张 schema。 - if (HasVersionTable(db) && !HasVersionReferences(db)) { - Exec(db, "ALTER TABLE _vexfs_file_versions ADD COLUMN source_version_no INTEGER"); - } +CREATE TABLE IF NOT EXISTS _vexfs_dirty_inodes( + workspace_id INTEGER NOT NULL, + inode_id INTEGER NOT NULL, + PRIMARY KEY(workspace_id, inode_id) +); +CREATE TABLE IF NOT EXISTS _vexfs_dirty_dentries( + workspace_id INTEGER NOT NULL, + parent_inode INTEGER NOT NULL, + name TEXT NOT NULL COLLATE BINARY, + inode_id INTEGER NOT NULL, + PRIMARY KEY(workspace_id, parent_inode, name) +); +CREATE TABLE IF NOT EXISTS _vexfs_dirty_xattrs( + workspace_id INTEGER NOT NULL, + inode_id INTEGER NOT NULL, + name TEXT NOT NULL COLLATE BINARY, + PRIMARY KEY(workspace_id, inode_id, name) +); +CREATE TABLE IF NOT EXISTS _vexfs_dirty_acl( + workspace_id INTEGER NOT NULL, + inode_id INTEGER NOT NULL, + principal_id TEXT NOT NULL COLLATE BINARY, + effect TEXT NOT NULL, + PRIMARY KEY(workspace_id,inode_id,principal_id,effect) +); + +CREATE TRIGGER IF NOT EXISTS _vexfs_history_inode_insert +AFTER INSERT ON _vexfs_inodes BEGIN + INSERT INTO _vexfs_dirty_inodes(workspace_id,inode_id) + VALUES(NEW.workspace_id,NEW.id) + ON CONFLICT(workspace_id,inode_id) DO NOTHING; +END; +CREATE TRIGGER IF NOT EXISTS _vexfs_history_inode_update +AFTER UPDATE ON _vexfs_inodes BEGIN + INSERT INTO _vexfs_dirty_inodes(workspace_id,inode_id) + VALUES(NEW.workspace_id,NEW.id) + ON CONFLICT(workspace_id,inode_id) DO NOTHING; +END; +CREATE TRIGGER IF NOT EXISTS _vexfs_history_dentry_insert +AFTER INSERT ON _vexfs_dentries BEGIN + INSERT INTO _vexfs_dirty_dentries(workspace_id,parent_inode,name,inode_id) + VALUES(NEW.workspace_id,NEW.parent_inode,NEW.name,NEW.inode_id) + ON CONFLICT(workspace_id,parent_inode,name) DO UPDATE SET inode_id=excluded.inode_id; +END; +CREATE TRIGGER IF NOT EXISTS _vexfs_history_dentry_update_old +AFTER UPDATE ON _vexfs_dentries BEGIN + INSERT INTO _vexfs_dirty_dentries(workspace_id,parent_inode,name,inode_id) + VALUES(OLD.workspace_id,OLD.parent_inode,OLD.name,OLD.inode_id) + ON CONFLICT(workspace_id,parent_inode,name) DO UPDATE SET inode_id=excluded.inode_id; +END; +CREATE TRIGGER IF NOT EXISTS _vexfs_history_dentry_update_new +AFTER UPDATE ON _vexfs_dentries BEGIN + INSERT INTO _vexfs_dirty_dentries(workspace_id,parent_inode,name,inode_id) + VALUES(NEW.workspace_id,NEW.parent_inode,NEW.name,NEW.inode_id) + ON CONFLICT(workspace_id,parent_inode,name) DO UPDATE SET inode_id=excluded.inode_id; +END; +CREATE TRIGGER IF NOT EXISTS _vexfs_history_dentry_delete +AFTER DELETE ON _vexfs_dentries BEGIN + INSERT INTO _vexfs_dirty_dentries(workspace_id,parent_inode,name,inode_id) + VALUES(OLD.workspace_id,OLD.parent_inode,OLD.name,OLD.inode_id) + ON CONFLICT(workspace_id,parent_inode,name) DO UPDATE SET inode_id=excluded.inode_id; +END; +CREATE TRIGGER IF NOT EXISTS _vexfs_history_xattr_insert +AFTER INSERT ON _vexfs_xattrs BEGIN + INSERT INTO _vexfs_dirty_xattrs(workspace_id,inode_id,name) + SELECT workspace_id,NEW.inode_id,NEW.name FROM _vexfs_inodes WHERE id=NEW.inode_id + ON CONFLICT(workspace_id,inode_id,name) DO NOTHING; +END; +CREATE TRIGGER IF NOT EXISTS _vexfs_history_xattr_update +AFTER UPDATE ON _vexfs_xattrs BEGIN + INSERT INTO _vexfs_dirty_xattrs(workspace_id,inode_id,name) + SELECT workspace_id,NEW.inode_id,NEW.name FROM _vexfs_inodes WHERE id=NEW.inode_id + ON CONFLICT(workspace_id,inode_id,name) DO NOTHING; +END; +CREATE TRIGGER IF NOT EXISTS _vexfs_history_xattr_delete +AFTER DELETE ON _vexfs_xattrs BEGIN + INSERT INTO _vexfs_dirty_xattrs(workspace_id,inode_id,name) + SELECT workspace_id,OLD.inode_id,OLD.name FROM _vexfs_inodes WHERE id=OLD.inode_id + ON CONFLICT(workspace_id,inode_id,name) DO NOTHING; +END; +)SQL"); } -void MigrateSchema010To020(sqlite3 *db) { - Exec(db, "SAVEPOINT vexfs_migrate_010_020"); - try { - Exec(db, R"SQL( -ALTER TABLE _vexfs_handles ADD COLUMN owner_session TEXT; -ALTER TABLE _vexfs_staging RENAME TO _vexfs_staging_010; -CREATE TABLE _vexfs_staging( - handle_id TEXT PRIMARY KEY, - generation INTEGER NOT NULL, - content BLOB NOT NULL, - logical_size INTEGER NOT NULL, - capacity INTEGER NOT NULL, - created_at INTEGER NOT NULL DEFAULT (CAST(strftime('%s','now') AS INTEGER) * 1000), - updated_at INTEGER NOT NULL DEFAULT (CAST(strftime('%s','now') AS INTEGER) * 1000) +void EnsureAclHistoryTriggers(sqlite3 *db) { + Exec(db, R"SQL( +CREATE TABLE IF NOT EXISTS _vexfs_dirty_acl( + workspace_id INTEGER NOT NULL, + inode_id INTEGER NOT NULL, + principal_id TEXT NOT NULL COLLATE BINARY, + effect TEXT NOT NULL, + PRIMARY KEY(workspace_id,inode_id,principal_id,effect) ); -INSERT INTO _vexfs_staging(handle_id, generation, content, logical_size, capacity, created_at) -SELECT s.handle_id, s.generation, s.content, length(s.content), length(s.content), s.created_at -FROM _vexfs_staging_010 s -JOIN _vexfs_handles h ON h.id=s.handle_id AND h.dirty_generation=s.generation; -DROP TABLE _vexfs_staging_010; -ALTER TABLE _vexfs_requests ADD COLUMN request_fingerprint BLOB NOT NULL DEFAULT X''; -DELETE FROM _vexfs_requests; -UPDATE _vexfs_inodes SET current_version=1 -WHERE kind='directory' AND deleted_at IS NULL AND current_version<1; -UPDATE _vexfs_meta SET value='0.2.0' WHERE key='contract_version'; +CREATE TRIGGER IF NOT EXISTS _vexfs_history_acl_insert +AFTER INSERT ON _vexfs_acl_entries BEGIN + INSERT INTO _vexfs_dirty_acl(workspace_id,inode_id,principal_id,effect) + VALUES(NEW.workspace_id,NEW.inode_id,NEW.principal_id,NEW.effect) + ON CONFLICT(workspace_id,inode_id,principal_id,effect) DO NOTHING; +END; +CREATE TRIGGER IF NOT EXISTS _vexfs_history_acl_update +AFTER UPDATE ON _vexfs_acl_entries BEGIN + INSERT INTO _vexfs_dirty_acl(workspace_id,inode_id,principal_id,effect) + VALUES(NEW.workspace_id,NEW.inode_id,NEW.principal_id,NEW.effect) + ON CONFLICT(workspace_id,inode_id,principal_id,effect) DO NOTHING; +END; +CREATE TRIGGER IF NOT EXISTS _vexfs_history_acl_delete +AFTER DELETE ON _vexfs_acl_entries BEGIN + INSERT INTO _vexfs_dirty_acl(workspace_id,inode_id,principal_id,effect) + VALUES(OLD.workspace_id,OLD.inode_id,OLD.principal_id,OLD.effect) + ON CONFLICT(workspace_id,inode_id,principal_id,effect) DO NOTHING; +END; )SQL"); - Exec(db, "RELEASE SAVEPOINT vexfs_migrate_010_020"); - } catch (...) { - try { - Exec(db, "ROLLBACK TO SAVEPOINT vexfs_migrate_010_020"); - Exec(db, "RELEASE SAVEPOINT vexfs_migrate_010_020"); - } catch (...) { - } - throw; - } } -void MigrateSchema020To030(sqlite3 *db) { - Savepoint savepoint(db, "vexfs_migrate_020_030"); +bool GrepIndexMarker(sqlite3 *db) { + Statement marker(db, + "SELECT 1 FROM _vexfs_meta WHERE key='grep_index' AND value='trigram-v1' LIMIT 1"); + return marker.Row(); +} + +bool GrepIndexDirty(sqlite3 *db) { + Statement marker(db, + "SELECT 1 FROM _vexfs_meta WHERE key='grep_index_dirty' AND value='1' LIMIT 1"); + return marker.Row(); +} + +void SetGrepIndexState(sqlite3 *db, GrepIndexState state) { + std::lock_guard lock(g_schema_ready_mutex); + g_grep_index_connections[db] = state; +} + +GrepIndexState GetGrepIndexState(sqlite3 *db) { + std::lock_guard lock(g_schema_ready_mutex); + const auto found = g_grep_index_connections.find(db); + return found == g_grep_index_connections.end() ? GrepIndexState::kDisabled : found->second; +} + +void RebuildGrepIndex(sqlite3 *db) { + Savepoint savepoint(db, "vexfs_grep_index_rebuild"); Exec(db, R"SQL( -CREATE TABLE IF NOT EXISTS _vexfs_staging_data( - handle_id TEXT PRIMARY KEY, - content BLOB NOT NULL -); -INSERT OR REPLACE INTO _vexfs_staging_data(handle_id, content) -SELECT handle_id, content FROM _vexfs_staging; -UPDATE _vexfs_staging SET content=X'' WHERE length(content)>0; -INSERT OR REPLACE INTO _vexfs_meta(key, value) VALUES('staging_layout', 'split-v1'); -CREATE TABLE IF NOT EXISTS _vexfs_mount_sessions( - workspace_id INTEGER PRIMARY KEY, - session_id TEXT NOT NULL UNIQUE, - lease_until INTEGER NOT NULL, - created_at INTEGER NOT NULL DEFAULT (CAST(strftime('%s','now') AS INTEGER) * 1000), - updated_at INTEGER NOT NULL DEFAULT (CAST(strftime('%s','now') AS INTEGER) * 1000) -); +DELETE FROM _vexfs_content_fts; +INSERT INTO _vexfs_content_fts(rowid,workspace_id,inode_id,version_no,content) +SELECT inode.id,inode.workspace_id,inode.id,inode.current_version, + CAST(CASE WHEN version.source_version_no IS NULL + THEN version.content ELSE source.content END AS TEXT) +FROM _vexfs_inodes inode +JOIN _vexfs_file_versions version + ON version.inode_id=inode.id AND version.version_no=inode.current_version +LEFT JOIN _vexfs_file_versions source + ON source.inode_id=version.inode_id AND source.version_no=version.source_version_no +WHERE inode.kind='file' AND inode.deleted_at IS NULL; +INSERT OR REPLACE INTO _vexfs_meta(key,value) VALUES('grep_index_dirty','0'); )SQL"); - EnsureVersionReferences(db); - // 合同标记必须最后写入；前面的布局升级有任何一步失败，整个 savepoint 回滚。 - Exec(db, "UPDATE _vexfs_meta SET value='0.3.0' WHERE key='contract_version'"); savepoint.Release(); } -void EnsureSplitStagingLayout(sqlite3 *db) { - Statement marker(db, - "SELECT 1 FROM _vexfs_meta WHERE key='staging_layout' AND value='split-v1' LIMIT 1"); - if (marker.Row()) return; +void DetectGrepIndex(sqlite3 *db) { + if (!GrepIndexMarker(db)) { + SetGrepIndexState(db, GrepIndexState::kDisabled); + return; + } + if (sqlite3_compileoption_used("ENABLE_FTS5") == 0 || + !HasTable(db, "_vexfs_content_fts")) { + SetGrepIndexState(db, GrepIndexState::kUnavailable); + return; + } + if (GrepIndexDirty(db)) { + if (sqlite3_db_readonly(db, "main") == 1) { + SetGrepIndexState(db, GrepIndexState::kUnavailable); + return; + } + RebuildGrepIndex(db); + } + SetGrepIndexState(db, GrepIndexState::kAvailable); +} - Exec(db, "SAVEPOINT vexfs_split_staging"); +void EnableGrepIndex(sqlite3 *db) { + if (sqlite3_compileoption_used("ENABLE_FTS5") == 0) { + throw SqlError("this SQLite host does not provide FTS5", SQLITE_MISMATCH); + } + const GrepIndexState previous_state = GetGrepIndexState(db); + Savepoint savepoint(db, "vexfs_grep_index_enable"); try { Exec(db, R"SQL( -CREATE TABLE IF NOT EXISTS _vexfs_staging_data( - handle_id TEXT PRIMARY KEY, - content BLOB NOT NULL +CREATE VIRTUAL TABLE IF NOT EXISTS _vexfs_content_fts USING fts5( + workspace_id UNINDEXED, + inode_id UNINDEXED, + version_no UNINDEXED, + content, + tokenize='trigram' ); -INSERT OR REPLACE INTO _vexfs_staging_data(handle_id, content) -SELECT handle_id, content FROM _vexfs_staging; -UPDATE _vexfs_staging SET content=X'' WHERE length(content)>0; -INSERT OR REPLACE INTO _vexfs_meta(key, value) VALUES('staging_layout', 'split-v1'); +INSERT OR REPLACE INTO _vexfs_meta(key,value) VALUES('grep_index','trigram-v1'); +INSERT OR REPLACE INTO _vexfs_meta(key,value) VALUES('grep_index_dirty','1'); )SQL"); - Exec(db, "RELEASE SAVEPOINT vexfs_split_staging"); + SetGrepIndexState(db, GrepIndexState::kAvailable); + RebuildGrepIndex(db); + savepoint.Release(); } catch (...) { - try { - Exec(db, "ROLLBACK TO SAVEPOINT vexfs_split_staging"); - Exec(db, "RELEASE SAVEPOINT vexfs_split_staging"); - } catch (...) { - } + SetGrepIndexState(db, previous_state); throw; } } -void EnsureRuntimeTables(sqlite3 *db) { - { - Statement exists(db, - "SELECT 1 FROM sqlite_master WHERE type='table' " - "AND name='_vexfs_mount_sessions' LIMIT 1"); - if (!exists.Row()) { - Exec(db, R"SQL( -CREATE TABLE IF NOT EXISTS _vexfs_mount_sessions( - workspace_id INTEGER PRIMARY KEY, - session_id TEXT NOT NULL UNIQUE, - lease_until INTEGER NOT NULL, - created_at INTEGER NOT NULL DEFAULT (CAST(strftime('%s','now') AS INTEGER) * 1000), - updated_at INTEGER NOT NULL DEFAULT (CAST(strftime('%s','now') AS INTEGER) * 1000) -); -)SQL"); - } +void DisableGrepIndex(sqlite3 *db) { + if (GrepIndexMarker(db) && sqlite3_compileoption_used("ENABLE_FTS5") == 0) { + throw SqlError("cannot remove the FTS5 index from this SQLite host", SQLITE_MISMATCH); } - EnsureVersionReferences(db); + Savepoint savepoint(db, "vexfs_grep_index_disable"); + Exec(db, R"SQL( +DROP TABLE IF EXISTS _vexfs_content_fts; +DELETE FROM _vexfs_meta WHERE key IN ('grep_index','grep_index_dirty'); +)SQL"); + savepoint.Release(); + SetGrepIndexState(db, GrepIndexState::kDisabled); } -void EnsureSchema(sqlite3 *db) { +void EnsureSchemaSlow(sqlite3 *db) { bool schema_exists = false; std::string version; { @@ -333,23 +512,9 @@ void EnsureSchema(sqlite3 *db) { } } if (schema_exists) { - if (version == "0.2.0" && sqlite3_db_readonly(db, "main") == 1) { - // 0.3 的版本读取 API 只依赖 0.2 已有的 file_versions/commits。 - // 只读连接允许兼容读取，但不会创建 session 表或改写合同标记。 - return; - } - if (version == "0.1.0") { - MigrateSchema010To020(db); - version = "0.2.0"; - } - if (version == "0.2.0") { - MigrateSchema020To030(db); - version = "0.3.0"; - } if (version != kContractVersion) throw SqlError("unsupported VexFS schema version: " + version, SQLITE_MISMATCH); - EnsureSplitStagingLayout(db); - EnsureRuntimeTables(db); + DetectGrepIndex(db); return; } Savepoint savepoint(db, "vexfs_schema_init"); @@ -363,17 +528,23 @@ CREATE TABLE IF NOT EXISTS _vexfs_workspaces( name TEXT NOT NULL UNIQUE, root_inode INTEGER, head_commit INTEGER, + history_floor_commit INTEGER, created_at INTEGER NOT NULL DEFAULT (CAST(strftime('%s','now') AS INTEGER) * 1000) ); CREATE TABLE IF NOT EXISTS _vexfs_inodes( id INTEGER PRIMARY KEY, workspace_id INTEGER NOT NULL, - kind TEXT NOT NULL CHECK(kind IN ('file','directory')), + kind TEXT NOT NULL CHECK(kind IN ('file','directory','symlink')), mode INTEGER NOT NULL, + owner_principal TEXT NOT NULL DEFAULT 'local', + uid INTEGER NOT NULL DEFAULT 0, + gid INTEGER NOT NULL DEFAULT 0, size INTEGER NOT NULL DEFAULT 0, current_version INTEGER NOT NULL DEFAULT 0, created_at INTEGER NOT NULL DEFAULT (CAST(strftime('%s','now') AS INTEGER) * 1000), + accessed_at INTEGER NOT NULL DEFAULT (CAST(strftime('%s','now') AS INTEGER) * 1000), updated_at INTEGER NOT NULL DEFAULT (CAST(strftime('%s','now') AS INTEGER) * 1000), + changed_at INTEGER NOT NULL DEFAULT (CAST(strftime('%s','now') AS INTEGER) * 1000), deleted_at INTEGER ); CREATE INDEX IF NOT EXISTS _vexfs_inodes_workspace_idx @@ -385,7 +556,7 @@ CREATE TABLE IF NOT EXISTS _vexfs_dentries( inode_id INTEGER NOT NULL, PRIMARY KEY(workspace_id, parent_inode, name) ); -CREATE UNIQUE INDEX IF NOT EXISTS _vexfs_dentries_inode_idx +CREATE INDEX IF NOT EXISTS _vexfs_dentries_inode_idx ON _vexfs_dentries(workspace_id, inode_id); CREATE TABLE IF NOT EXISTS _vexfs_commits( id INTEGER PRIMARY KEY, @@ -401,10 +572,31 @@ CREATE TABLE IF NOT EXISTS _vexfs_file_versions( commit_id INTEGER NOT NULL, content BLOB NOT NULL, size INTEGER NOT NULL, + checksum TEXT NOT NULL CHECK(length(checksum)=64), source_version_no INTEGER, created_at INTEGER NOT NULL DEFAULT (CAST(strftime('%s','now') AS INTEGER) * 1000), UNIQUE(inode_id, version_no) ); +CREATE TABLE IF NOT EXISTS _vexfs_xattrs( + inode_id INTEGER NOT NULL, + name TEXT NOT NULL COLLATE BINARY, + value BLOB NOT NULL, + updated_at INTEGER NOT NULL DEFAULT (CAST(strftime('%s','now') AS INTEGER) * 1000), + PRIMARY KEY(inode_id, name) +); +CREATE TABLE IF NOT EXISTS _vexfs_acl_entries( + workspace_id INTEGER NOT NULL, + inode_id INTEGER NOT NULL, + principal_id TEXT NOT NULL COLLATE BINARY, + effect TEXT NOT NULL CHECK(effect IN ('allow','deny')), + permissions TEXT NOT NULL, + inherit_flags INTEGER NOT NULL DEFAULT 0 CHECK(inherit_flags >= 0), + created_at INTEGER NOT NULL DEFAULT (CAST(strftime('%s','now') AS INTEGER) * 1000), + updated_at INTEGER NOT NULL DEFAULT (CAST(strftime('%s','now') AS INTEGER) * 1000), + PRIMARY KEY(workspace_id,inode_id,principal_id,effect) +); +CREATE INDEX IF NOT EXISTS _vexfs_acl_entries_inode_idx + ON _vexfs_acl_entries(workspace_id,inode_id,principal_id,effect); CREATE TABLE IF NOT EXISTS _vexfs_handles( id TEXT PRIMARY KEY, workspace_id INTEGER NOT NULL, @@ -450,10 +642,46 @@ CREATE TABLE IF NOT EXISTS _vexfs_mount_sessions( created_at INTEGER NOT NULL DEFAULT (CAST(strftime('%s','now') AS INTEGER) * 1000), updated_at INTEGER NOT NULL DEFAULT (CAST(strftime('%s','now') AS INTEGER) * 1000) ); -INSERT OR IGNORE INTO _vexfs_meta(key, value) VALUES('contract_version', '0.3.0'); +INSERT OR IGNORE INTO _vexfs_meta(key, value) VALUES('contract_version', '0.7.0'); INSERT OR IGNORE INTO _vexfs_meta(key, value) VALUES('staging_layout', 'split-v1'); )SQL"); + CreateHistorySchema(db); + EnsureAclHistoryTriggers(db); savepoint.Release(); + DetectGrepIndex(db); +} + +void MarkSchemaReady(sqlite3 *db) { + // A mounted workspace owns one long-lived SQLite connection. Schema validation and + // migration are connection setup work; repeating the sqlite_master/pragma checks for + // every stat, write and xattr callback becomes dominant on large trees. + std::lock_guard lock(g_schema_ready_mutex); + g_schema_ready_connections.insert(db); +} + +bool IsSchemaReady(sqlite3 *db) { + std::lock_guard lock(g_schema_ready_mutex); + return g_schema_ready_connections.find(db) != g_schema_ready_connections.end(); +} + +void ForgetSchemaReady(void *pointer) { + auto *db = static_cast(pointer); + std::lock_guard lock(g_schema_ready_mutex); + g_schema_ready_connections.erase(db); + g_grep_index_connections.erase(db); +} + +void EnsureSchema(sqlite3 *db) { + if (IsSchemaReady(db)) return; + EnsureSchemaSlow(db); + MarkSchemaReady(db); +} + +void EnsureSchemaForced(sqlite3 *db) { + // vexfs_init 是显式校验边界。项目尚未发版，不做旧 schema 原地迁移；版本 + // 不一致直接失败，避免把历史 SQLite 分支带入其他 backend。 + EnsureSchemaSlow(db); + MarkSchemaReady(db); } std::string RequiredText(sqlite3_value *value, const char *name) { @@ -499,6 +727,45 @@ sqlite3_int64 RequiredNonnegativeInteger(sqlite3_value *value, const char *name) return result; } +sqlite3_int64 RequiredOwnerId(sqlite3_value *value, const char *name) { + if (sqlite3_value_type(value) != SQLITE_INTEGER) { + throw SqlError(std::string(name) + " must be an integer", SQLITE_MISMATCH); + } + const sqlite3_int64 result = sqlite3_value_int64(value); + if (result < -1 || result > 0xffffffffLL) { + throw SqlError(std::string(name) + " must be -1 or fit unsigned 32-bit values", + SQLITE_RANGE); + } + return result; +} + +int RequiredMode(sqlite3_value *value) { + if (sqlite3_value_type(value) != SQLITE_INTEGER) { + throw SqlError("mode must be an integer", SQLITE_MISMATCH); + } + const sqlite3_int64 mode = sqlite3_value_int64(value); + if (mode < 0 || mode > 0777) { + throw SqlError("mode must contain only rwx permission bits (0..0777)", SQLITE_RANGE); + } + return static_cast(mode); +} + +sqlite3_int64 CurrentUid() { +#if defined(_WIN32) + return 0; +#else + return static_cast(getuid()); +#endif +} + +sqlite3_int64 CurrentGid() { +#if defined(_WIN32) + return 0; +#else + return static_cast(getgid()); +#endif +} + std::vector PathParts(const std::string &path) { if (path.empty() || path.front() != '/') throw SqlError("path must be absolute", SQLITE_MISMATCH); if (path.find('\0') != std::string::npos) throw SqlError("path contains NUL", SQLITE_MISMATCH); @@ -543,36 +810,31 @@ struct Node { sqlite3_int64 size = 0; sqlite3_int64 version = 0; int mode = 0; + std::string owner_principal = "local"; + sqlite3_int64 uid = 0; + sqlite3_int64 gid = 0; sqlite3_int64 created_at = 0; + sqlite3_int64 accessed_at = 0; sqlite3_int64 updated_at = 0; + sqlite3_int64 changed_at = 0; }; struct VersionStorage { sqlite3_int64 rowid = 0; sqlite3_int64 size = 0; sqlite3_int64 canonical_version = 0; + std::string checksum; }; VersionStorage ResolveVersionStorage(sqlite3 *db, sqlite3_int64 inode_id, sqlite3_int64 version) { - if (!HasVersionReferences(db)) { - Statement statement(db, - "SELECT rowid,size,version_no FROM _vexfs_file_versions " - "WHERE inode_id=?1 AND version_no=?2"); - statement.BindInt64(1, inode_id); - statement.BindInt64(2, version); - if (!statement.Row()) { - throw SqlError("file version not found: " + std::to_string(version), - SQLITE_NOTFOUND); - } - return {statement.Int64(0), statement.Int64(1), statement.Int64(2)}; - } - Statement statement(db, R"SQL( SELECT CASE WHEN v.source_version_no IS NULL THEN v.rowid ELSE source.rowid END, v.size, COALESCE(v.source_version_no,v.version_no), - CASE WHEN v.source_version_no IS NULL THEN v.size ELSE source.size END + CASE WHEN v.source_version_no IS NULL THEN v.size ELSE source.size END, + v.checksum, + CASE WHEN v.source_version_no IS NULL THEN v.checksum ELSE source.checksum END FROM _vexfs_file_versions v LEFT JOIN _vexfs_file_versions source ON source.inode_id=v.inode_id AND source.version_no=v.source_version_no @@ -584,10 +846,11 @@ WHERE v.inode_id=?1 AND v.version_no=?2 throw SqlError("file version not found: " + std::to_string(version), SQLITE_NOTFOUND); } if (statement.Type(0) == SQLITE_NULL || statement.Int64(1) < 0 || - statement.Int64(1) != statement.Int64(3)) { + statement.Int64(1) != statement.Int64(3) || statement.Text(4).size() != 64 || + statement.Text(4) != statement.Text(5)) { throw SqlError("file version content reference is corrupt", SQLITE_CORRUPT); } - return {statement.Int64(0), statement.Int64(1), statement.Int64(2)}; + return {statement.Int64(0), statement.Int64(1), statement.Int64(2), statement.Text(4)}; } std::vector ReadVersionContent(sqlite3 *db, sqlite3_int64 inode_id, @@ -607,13 +870,17 @@ std::vector ReadVersionContent(sqlite3 *db, sqlite3_int64 inode_i std::min(kChunkSize, storage.size - offset)); blob.Read(content.data() + offset, bytes, static_cast(offset)); } + if (vexfs::Sha256Hex(content.data(), content.size()) != storage.checksum) { + throw SqlError("file version checksum does not match content", SQLITE_CORRUPT); + } return content; } bool FindChild(sqlite3 *db, sqlite3_int64 workspace_id, sqlite3_int64 parent, const std::string &name, Node *node) { Statement statement(db, - "SELECT i.id, i.kind, i.size, i.current_version, i.mode, i.created_at, i.updated_at " + "SELECT i.id, i.kind, i.size, i.current_version, i.mode, i.owner_principal, i.uid, i.gid, " + "i.created_at, i.accessed_at, i.updated_at, i.changed_at " "FROM _vexfs_dentries d JOIN _vexfs_inodes i ON i.id = d.inode_id " "WHERE d.workspace_id = ?1 AND d.parent_inode = ?2 AND d.name = ?3 " "AND i.deleted_at IS NULL"); @@ -622,19 +889,24 @@ bool FindChild(sqlite3 *db, sqlite3_int64 workspace_id, sqlite3_int64 parent, statement.BindText(3, name); if (!statement.Row()) return false; *node = {statement.Int64(0), statement.Text(1), statement.Int64(2), - statement.Int64(3), statement.Int(4), statement.Int64(5), statement.Int64(6)}; + statement.Int64(3), statement.Int(4), statement.Text(5), statement.Int64(6), + statement.Int64(7), statement.Int64(8), statement.Int64(9), + statement.Int64(10), statement.Int64(11)}; return true; } Node RootNode(sqlite3 *db, const Workspace &workspace) { Statement statement(db, - "SELECT id, kind, size, current_version, mode, created_at, updated_at FROM _vexfs_inodes " + "SELECT id, kind, size, current_version, mode, owner_principal, uid, gid, " + "created_at, accessed_at, updated_at, changed_at FROM _vexfs_inodes " "WHERE id = ?1 AND workspace_id = ?2 AND deleted_at IS NULL"); statement.BindInt64(1, workspace.root_inode); statement.BindInt64(2, workspace.id); if (!statement.Row()) throw SqlError("workspace root is missing", SQLITE_CORRUPT); return {statement.Int64(0), statement.Text(1), statement.Int64(2), - statement.Int64(3), statement.Int(4), statement.Int64(5), statement.Int64(6)}; + statement.Int64(3), statement.Int(4), statement.Text(5), statement.Int64(6), + statement.Int64(7), statement.Int64(8), statement.Int64(9), + statement.Int64(10), statement.Int64(11)}; } bool TryResolve(sqlite3 *db, const Workspace &workspace, const std::vector &parts, @@ -687,6 +959,99 @@ SELECT '/' || path FROM ancestors WHERE parent_inode=?3 ORDER BY depth DESC LIMI return statement.Text(0); } +sqlite3_int64 LinkCount(sqlite3 *db, const Workspace &workspace, sqlite3_int64 inode) { + Statement statement(db, R"SQL( +SELECT i.kind, + CASE WHEN i.kind='directory' THEN + 2 + (SELECT count(*) FROM _vexfs_dentries d + JOIN _vexfs_inodes child ON child.id=d.inode_id + WHERE d.workspace_id=i.workspace_id AND d.parent_inode=i.id + AND child.kind='directory' AND child.deleted_at IS NULL) + ELSE + (SELECT count(*) FROM _vexfs_dentries d + WHERE d.workspace_id=i.workspace_id AND d.inode_id=i.id) + END +FROM _vexfs_inodes i WHERE i.workspace_id=?1 AND i.id=?2 +)SQL"); + statement.BindInt64(1, workspace.id); + statement.BindInt64(2, inode); + if (!statement.Row()) throw SqlError("inode link count unavailable", SQLITE_CORRUPT); + return statement.Int64(1); +} + +void FlushHistoryChanges(sqlite3 *db, const Workspace &workspace, sqlite3_int64 commit) { + Statement inodes(db, R"SQL( +INSERT OR REPLACE INTO _vexfs_inode_states( + workspace_id,inode_id,commit_id,kind,mode,owner_principal,uid,gid,size,current_version, + created_at,accessed_at,updated_at,changed_at,deleted_at) +SELECT i.workspace_id,i.id,?2,i.kind,i.mode,i.owner_principal,i.uid,i.gid,i.size,i.current_version, + i.created_at,i.accessed_at,i.updated_at,i.changed_at,i.deleted_at +FROM _vexfs_dirty_inodes dirty +JOIN _vexfs_inodes i ON i.workspace_id=dirty.workspace_id AND i.id=dirty.inode_id +WHERE dirty.workspace_id=?1 +)SQL"); + inodes.BindInt64(1, workspace.id); + inodes.BindInt64(2, commit); + inodes.Done(); + + Statement dentries(db, R"SQL( +INSERT OR REPLACE INTO _vexfs_dentry_states( + workspace_id,parent_inode,name,commit_id,inode_id,deleted) +SELECT dirty.workspace_id,dirty.parent_inode,dirty.name,?2, + COALESCE(d.inode_id,dirty.inode_id),CASE WHEN d.inode_id IS NULL THEN 1 ELSE 0 END +FROM _vexfs_dirty_dentries dirty +LEFT JOIN _vexfs_dentries d + ON d.workspace_id=dirty.workspace_id AND d.parent_inode=dirty.parent_inode + AND d.name=dirty.name +WHERE dirty.workspace_id=?1 +)SQL"); + dentries.BindInt64(1, workspace.id); + dentries.BindInt64(2, commit); + dentries.Done(); + + Statement xattrs(db, R"SQL( +INSERT OR REPLACE INTO _vexfs_xattr_states( + workspace_id,inode_id,name,commit_id,value,deleted) +SELECT dirty.workspace_id,dirty.inode_id,dirty.name,?2, + COALESCE(x.value,X''),CASE WHEN x.inode_id IS NULL THEN 1 ELSE 0 END +FROM _vexfs_dirty_xattrs dirty +LEFT JOIN _vexfs_xattrs x ON x.inode_id=dirty.inode_id AND x.name=dirty.name +WHERE dirty.workspace_id=?1 +)SQL"); + xattrs.BindInt64(1, workspace.id); + xattrs.BindInt64(2, commit); + xattrs.Done(); + + Statement acl(db, R"SQL( +INSERT OR REPLACE INTO _vexfs_acl_states( + workspace_id,inode_id,principal_id,effect,commit_id,permissions,inherit_flags,deleted) +SELECT dirty.workspace_id,dirty.inode_id,dirty.principal_id,dirty.effect,?2, + COALESCE(a.permissions,''),COALESCE(a.inherit_flags,0), + CASE WHEN a.inode_id IS NULL THEN 1 ELSE 0 END +FROM _vexfs_dirty_acl dirty +LEFT JOIN _vexfs_acl_entries a + ON a.workspace_id=dirty.workspace_id AND a.inode_id=dirty.inode_id + AND a.principal_id=dirty.principal_id AND a.effect=dirty.effect +WHERE dirty.workspace_id=?1 +)SQL"); + acl.BindInt64(1, workspace.id); + acl.BindInt64(2, commit); + acl.Done(); + + Statement clear_inodes(db, "DELETE FROM _vexfs_dirty_inodes WHERE workspace_id=?1"); + clear_inodes.BindInt64(1, workspace.id); + clear_inodes.Done(); + Statement clear_dentries(db, "DELETE FROM _vexfs_dirty_dentries WHERE workspace_id=?1"); + clear_dentries.BindInt64(1, workspace.id); + clear_dentries.Done(); + Statement clear_xattrs(db, "DELETE FROM _vexfs_dirty_xattrs WHERE workspace_id=?1"); + clear_xattrs.BindInt64(1, workspace.id); + clear_xattrs.Done(); + Statement clear_acl(db, "DELETE FROM _vexfs_dirty_acl WHERE workspace_id=?1"); + clear_acl.BindInt64(1, workspace.id); + clear_acl.Done(); +} + sqlite3_int64 CreateCommit(sqlite3 *db, const Workspace &workspace, const std::string &message) { Savepoint savepoint(db, "vexfs_create_commit"); Statement insert(db, @@ -700,7 +1065,11 @@ sqlite3_int64 CreateCommit(sqlite3 *db, const Workspace &workspace, const std::s } const sqlite3_int64 commit = sqlite3_last_insert_rowid(db); - Statement update(db, "UPDATE _vexfs_workspaces SET head_commit = ?1 WHERE id = ?2"); + FlushHistoryChanges(db, workspace, commit); + + Statement update(db, + "UPDATE _vexfs_workspaces SET head_commit=?1, " + "history_floor_commit=COALESCE(history_floor_commit,?1) WHERE id=?2"); update.BindInt64(1, commit); update.BindInt64(2, workspace.id); update.Done(); @@ -710,12 +1079,14 @@ sqlite3_int64 CreateCommit(sqlite3 *db, const Workspace &workspace, const std::s sqlite3_int64 CreateInode(sqlite3 *db, sqlite3_int64 workspace_id, const char *kind, int mode) { Statement insert(db, - "INSERT INTO _vexfs_inodes(workspace_id, kind, mode, current_version) " - "VALUES(?1, ?2, ?3, ?4)"); + "INSERT INTO _vexfs_inodes(workspace_id, kind, mode, owner_principal, uid, gid, current_version) " + "VALUES(?1, ?2, ?3, 'local', ?4, ?5, ?6)"); insert.BindInt64(1, workspace_id); insert.BindText(2, kind); insert.BindInt(3, mode); - insert.BindInt64(4, std::strcmp(kind, "directory") == 0 ? 1 : 0); + insert.BindInt64(4, CurrentUid()); + insert.BindInt64(5, CurrentGid()); + insert.BindInt64(6, std::strcmp(kind, "directory") == 0 ? 1 : 0); insert.Done(); return sqlite3_last_insert_rowid(db); } @@ -723,7 +1094,8 @@ sqlite3_int64 CreateInode(sqlite3 *db, sqlite3_int64 workspace_id, const char *k void TouchDirectory(sqlite3 *db, sqlite3_int64 workspace_id, sqlite3_int64 inode) { Statement update(db, "UPDATE _vexfs_inodes SET current_version=current_version+1, " - "updated_at=CAST(strftime('%s','now') AS INTEGER)*1000 " + "updated_at=CAST(unixepoch('subsec')*1000 AS INTEGER), " + "changed_at=CAST(unixepoch('subsec')*1000 AS INTEGER) " "WHERE id=?1 AND workspace_id=?2 AND kind='directory' AND deleted_at IS NULL"); update.BindInt64(1, inode); update.BindInt64(2, workspace_id); @@ -740,9 +1112,83 @@ void AddDentry(sqlite3 *db, sqlite3_int64 workspace_id, sqlite3_int64 parent, insert.BindText(3, name); insert.BindInt64(4, inode); insert.Done(); + // ACLs marked inheritable are copied at creation time. Existing inode ACLs + // win on hard-link creation, so this is an INSERT OR IGNORE rather than a + // replacement of the inode's access policy. + Statement inherit(db, R"SQL( +INSERT OR IGNORE INTO _vexfs_acl_entries( + workspace_id,inode_id,principal_id,effect,permissions,inherit_flags) +SELECT workspace_id,?3,principal_id,effect,permissions,inherit_flags +FROM _vexfs_acl_entries +WHERE workspace_id=?1 AND inode_id=?2 AND inherit_flags<>0 +)SQL"); + inherit.BindInt64(1, workspace_id); + inherit.BindInt64(2, parent); + inherit.BindInt64(3, inode); + inherit.Done(); TouchDirectory(db, workspace_id, parent); } +void MarkGrepIndexDirty(sqlite3 *db) { + if (GetGrepIndexState(db) == GrepIndexState::kDisabled) return; + Exec(db, + "INSERT OR REPLACE INTO _vexfs_meta(key,value) VALUES('grep_index_dirty','1')"); +} + +void UpdateGrepIndexFromVersion(sqlite3 *db, sqlite3_int64 workspace_id, + sqlite3_int64 inode_id, sqlite3_int64 version) { + const GrepIndexState state = GetGrepIndexState(db); + if (state == GrepIndexState::kDisabled) return; + if (state == GrepIndexState::kUnavailable) { + MarkGrepIndexDirty(db); + return; + } + Statement remove(db, "DELETE FROM _vexfs_content_fts WHERE rowid=?1"); + remove.BindInt64(1, inode_id); + remove.Done(); + Statement insert(db, R"SQL( +INSERT INTO _vexfs_content_fts(rowid,workspace_id,inode_id,version_no,content) +SELECT inode.id,inode.workspace_id,inode.id,version.version_no, + CAST(CASE WHEN version.source_version_no IS NULL + THEN version.content ELSE source.content END AS TEXT) +FROM _vexfs_inodes inode +JOIN _vexfs_file_versions version + ON version.inode_id=inode.id AND version.version_no=?3 +LEFT JOIN _vexfs_file_versions source + ON source.inode_id=version.inode_id AND source.version_no=version.source_version_no +WHERE inode.id=?2 AND inode.workspace_id=?1 AND inode.kind='file' + AND inode.deleted_at IS NULL +)SQL"); + insert.BindInt64(1, workspace_id); + insert.BindInt64(2, inode_id); + insert.BindInt64(3, version); + insert.Done(); +} + +void RefreshGrepIndexAfterTreeChange(sqlite3 *db) { + const GrepIndexState state = GetGrepIndexState(db); + if (state == GrepIndexState::kAvailable) RebuildGrepIndex(db); + else if (state == GrepIndexState::kUnavailable) MarkGrepIndexDirty(db); +} + +void RemoveDeletedFilesFromGrepIndex(sqlite3 *db, sqlite3_int64 workspace_id) { + const GrepIndexState state = GetGrepIndexState(db); + if (state == GrepIndexState::kDisabled) return; + if (state == GrepIndexState::kUnavailable) { + MarkGrepIndexDirty(db); + return; + } + Statement remove(db, R"SQL( +DELETE FROM _vexfs_content_fts +WHERE rowid IN ( + SELECT id FROM _vexfs_inodes + WHERE workspace_id=?1 AND kind='file' AND deleted_at IS NOT NULL +) +)SQL"); + remove.BindInt64(1, workspace_id); + remove.Done(); +} + sqlite3_int64 StoreVersion(sqlite3 *db, const Workspace &workspace, const Node &node, const std::vector &content, const std::string &message) { @@ -750,7 +1196,8 @@ sqlite3_int64 StoreVersion(sqlite3 *db, const Workspace &workspace, const Node & const sqlite3_int64 version = node.version + 1; Statement advance(db, "UPDATE _vexfs_inodes SET size=?1, current_version=?2, " - "updated_at=CAST(strftime('%s','now') AS INTEGER)*1000 " + "updated_at=CAST(unixepoch('subsec')*1000 AS INTEGER), " + "changed_at=CAST(unixepoch('subsec')*1000 AS INTEGER) " "WHERE id=?3 AND current_version=?4"); advance.BindInt64(1, static_cast(content.size())); advance.BindInt64(2, version); @@ -762,15 +1209,19 @@ sqlite3_int64 StoreVersion(sqlite3 *db, const Workspace &workspace, const Node & } const sqlite3_int64 commit = CreateCommit(db, workspace, message); + const std::string checksum = vexfs::Sha256Hex(content.data(), content.size()); Statement insert(db, - "INSERT INTO _vexfs_file_versions(inode_id, version_no, commit_id, content, size) " - "VALUES(?1, ?2, ?3, ?4, ?5)"); + "INSERT INTO _vexfs_file_versions" + "(inode_id,version_no,commit_id,content,size,checksum) " + "VALUES(?1,?2,?3,?4,?5,?6)"); insert.BindInt64(1, node.id); insert.BindInt64(2, version); insert.BindInt64(3, commit); insert.BindBlob(4, content); insert.BindInt64(5, static_cast(content.size())); + insert.BindText(6, checksum); insert.Done(); + UpdateGrepIndexFromVersion(db, workspace.id, node.id, version); savepoint.Release(); return version; } @@ -784,7 +1235,8 @@ sqlite3_int64 StoreVersionFromVersion(sqlite3 *db, const Workspace &workspace, c const sqlite3_int64 version = node.version + 1; Statement advance(db, "UPDATE _vexfs_inodes SET size=?1, current_version=?2, " - "updated_at=CAST(strftime('%s','now') AS INTEGER)*1000 " + "updated_at=CAST(unixepoch('subsec')*1000 AS INTEGER), " + "changed_at=CAST(unixepoch('subsec')*1000 AS INTEGER) " "WHERE id=?3 AND current_version=?4"); advance.BindInt64(1, source.size); advance.BindInt64(2, version); @@ -797,17 +1249,19 @@ sqlite3_int64 StoreVersionFromVersion(sqlite3 *db, const Workspace &workspace, c const sqlite3_int64 commit = CreateCommit(db, workspace, message); Statement insert(db, "INSERT INTO _vexfs_file_versions" - "(inode_id,version_no,commit_id,content,size,source_version_no) " - "VALUES(?1,?2,?3,X'',?4,?5)"); + "(inode_id,version_no,commit_id,content,size,checksum,source_version_no) " + "VALUES(?1,?2,?3,X'',?4,?5,?6)"); insert.BindInt64(1, node.id); insert.BindInt64(2, version); insert.BindInt64(3, commit); insert.BindInt64(4, source.size); - insert.BindInt64(5, source.canonical_version); + insert.BindText(5, source.checksum); + insert.BindInt64(6, source.canonical_version); insert.Done(); if (sqlite3_changes64(db) != 1) { throw SqlError("file version not found: " + std::to_string(source_version), SQLITE_NOTFOUND); } + UpdateGrepIndexFromVersion(db, workspace.id, node.id, version); savepoint.Release(); return version; } @@ -825,6 +1279,63 @@ std::vector ReadNodeVersion(sqlite3 *db, const Node &node, return ReadVersionContent(db, node.id, version); } +std::string CanonicalPath(const std::vector &parts) { + if (parts.empty()) return "/"; + std::string path; + for (const std::string &part : parts) { + path.push_back('/'); + path += part; + } + return path; +} + +bool IsUtf8Text(const std::vector &content) { + for (size_t index = 0; index < content.size();) { + const unsigned char first = content[index]; + if (first == 0) return false; + if (first < 0x80) { + ++index; + continue; + } + size_t trailing = 0; + uint32_t codepoint = 0; + uint32_t minimum = 0; + if ((first & 0xe0) == 0xc0) { + trailing = 1; codepoint = first & 0x1f; minimum = 0x80; + } else if ((first & 0xf0) == 0xe0) { + trailing = 2; codepoint = first & 0x0f; minimum = 0x800; + } else if ((first & 0xf8) == 0xf0) { + trailing = 3; codepoint = first & 0x07; minimum = 0x10000; + } else { + return false; + } + if (index + trailing >= content.size()) return false; + for (size_t offset = 1; offset <= trailing; ++offset) { + const unsigned char next = content[index + offset]; + if ((next & 0xc0) != 0x80) return false; + codepoint = (codepoint << 6) | (next & 0x3f); + } + if (codepoint < minimum || codepoint > 0x10ffff || + (codepoint >= 0xd800 && codepoint <= 0xdfff)) { + return false; + } + index += trailing + 1; + } + return true; +} + +unsigned char FoldAscii(unsigned char value) { + return value >= 'A' && value <= 'Z' ? static_cast(value + ('a' - 'A')) : value; +} + +bool LineContains(const unsigned char *begin, const unsigned char *end, + const std::string &pattern, bool ignore_case) { + const auto compare = [ignore_case](unsigned char left, unsigned char right) { + return ignore_case ? FoldAscii(left) == FoldAscii(right) : left == right; + }; + return std::search(begin, end, pattern.begin(), pattern.end(), compare) != end; +} + sqlite3_int64 WritePath(sqlite3 *db, const Workspace &workspace, const std::vector &parts, const std::vector &content, @@ -959,7 +1470,11 @@ class RequestFingerprint { CachedRequest FindRequest(sqlite3 *db, const std::string &request_id, const char *operation, const std::vector &fingerprint) { - if (request_id.empty()) throw SqlError("request_id must not be empty", SQLITE_MISMATCH); + // An empty request ID is the in-process mount fast path. Generation checks + // make stage/truncate/publish/synchronize replay-safe without growing the + // persistent request table. Public callers still pass IDs and retain the + // cross-process retry contract. + if (request_id.empty()) return {}; Statement statement(db, "SELECT operation, request_fingerprint, result_integer, result_text FROM _vexfs_requests " "WHERE request_id = ?1"); @@ -977,8 +1492,23 @@ CachedRequest FindRequest(sqlite3 *db, const std::string &request_id, const char return {CachedKind::kInteger, statement.Int64(2), {}}; } +void MaybePruneRequests(sqlite3 *db) { + const sqlite3_int64 newest_rowid = sqlite3_last_insert_rowid(db); + if (newest_rowid <= kRequestRetentionRows || + newest_rowid % kRequestPruneInterval != 0) { + return; + } + // request_id protects retries, but keeping every completed request forever makes a + // long-lived mount grow without a bound. Delete in batches so ordinary writes do + // not pay for a count/sort query; the newest 65,536 results remain replayable. + Statement prune(db, "DELETE FROM _vexfs_requests WHERE rowid <= ?1"); + prune.BindInt64(1, newest_rowid - kRequestRetentionRows); + prune.Done(); +} + void StoreRequestInteger(sqlite3 *db, const std::string &request_id, const char *operation, const std::vector &fingerprint, sqlite3_int64 result) { + if (request_id.empty()) return; Statement statement(db, "INSERT INTO _vexfs_requests(request_id, operation, request_fingerprint, result_integer) " "VALUES(?1, ?2, ?3, ?4)"); @@ -987,10 +1517,12 @@ void StoreRequestInteger(sqlite3 *db, const std::string &request_id, const char statement.BindBlob(3, fingerprint); statement.BindInt64(4, result); statement.Done(); + MaybePruneRequests(db); } void StoreRequestText(sqlite3 *db, const std::string &request_id, const char *operation, const std::vector &fingerprint, const std::string &result) { + if (request_id.empty()) return; Statement statement(db, "INSERT INTO _vexfs_requests(request_id, operation, request_fingerprint, result_text) " "VALUES(?1, ?2, ?3, ?4)"); @@ -999,6 +1531,7 @@ void StoreRequestText(sqlite3 *db, const std::string &request_id, const char *op statement.BindBlob(3, fingerprint); statement.BindText(4, result); statement.Done(); + MaybePruneRequests(db); } struct Handle { @@ -1099,7 +1632,9 @@ StagingInfo EnsureStagingCapacity(sqlite3 *db, const Handle &handle, StagingInfo if (required <= staging.capacity) return staging; if (required > kMaxStagedBytes) throw SqlError("staged file is larger than 128 MiB", SQLITE_TOOBIG); - sqlite3_int64 capacity = std::max(staging.capacity, kInitialStagingCapacity); + sqlite3_int64 capacity = staging.capacity == 0 + ? kEmptyFileStagingCapacity + : std::max(staging.capacity, kInitialStagingCapacity); while (capacity < required) capacity = std::min(kMaxStagedBytes, capacity * 2); Statement grow(db, "UPDATE _vexfs_staging_data SET content=CAST(content||zeroblob(?1) AS BLOB) " @@ -1120,7 +1655,9 @@ StagingInfo EnsureStagingCapacity(sqlite3 *db, const Handle &handle, StagingInfo void CreateStaging(sqlite3 *db, const Handle &handle, const std::vector &content) { sqlite3_int64 capacity = static_cast(content.size()); - if (handle.writable && capacity < kInitialStagingCapacity) capacity = kInitialStagingCapacity; + if (handle.writable && capacity > 0 && capacity < kInitialStagingCapacity) { + capacity = kInitialStagingCapacity; + } Statement insert(db, "INSERT INTO _vexfs_staging(handle_id,generation,content,logical_size,capacity) " "VALUES(?1,?2,X'',?4,?3)"); @@ -1229,13 +1766,16 @@ Workspace WorkspaceById(sqlite3 *db, sqlite3_int64 id) { Node NodeById(sqlite3 *db, sqlite3_int64 id, bool include_deleted = false) { Statement statement(db, - "SELECT id, kind, size, current_version, mode, created_at, updated_at FROM _vexfs_inodes " + "SELECT id, kind, size, current_version, mode, owner_principal, uid, gid, " + "created_at, accessed_at, updated_at, changed_at FROM _vexfs_inodes " "WHERE id = ?1 AND (?2 OR deleted_at IS NULL)"); statement.BindInt64(1, id); statement.BindInt(2, include_deleted ? 1 : 0); if (!statement.Row()) throw SqlError("inode not found", SQLITE_NOTFOUND); return {statement.Int64(0), statement.Text(1), statement.Int64(2), - statement.Int64(3), statement.Int(4), statement.Int64(5), statement.Int64(6)}; + statement.Int64(3), statement.Int(4), statement.Text(5), statement.Int64(6), + statement.Int64(7), statement.Int64(8), statement.Int64(9), + statement.Int64(10), statement.Int64(11)}; } sqlite3_int64 PublishHandle(sqlite3 *db, Handle handle, sqlite3_int64 generation) { @@ -1267,6 +1807,27 @@ sqlite3_int64 PublishHandle(sqlite3 *db, Handle handle, sqlite3_int64 generation return version; } +bool HasPendingContentPublish(sqlite3 *db, sqlite3_int64 workspace_id, + sqlite3_int64 inode_id) { + Statement statement(db, + "SELECT 1 FROM _vexfs_handles WHERE workspace_id=?1 AND inode_id=?2 " + "AND writable=1 AND state IN ('open','retained') " + "AND dirty_generation>published_generation LIMIT 1"); + statement.BindInt64(1, workspace_id); + statement.BindInt64(2, inode_id); + return statement.Row(); +} + +void CommitMetadataChange(sqlite3 *db, const Workspace &workspace, + sqlite3_int64 inode_id, const char *message) { + // FSKit commonly sets mode/times/com.apple.provenance between create and + // the first content publish. Leave those trigger rows dirty so the content + // publish flushes one coherent history commit instead of three. + if (!HasPendingContentPublish(db, workspace.id, inode_id)) { + CreateCommit(db, workspace, message); + } +} + void ResultBlob(sqlite3_context *context, const std::vector &value) { const void *data = value.empty() ? static_cast("") : value.data(); sqlite3_result_blob64(context, data, value.size(), SQLITE_TRANSIENT); @@ -1288,7 +1849,7 @@ void Guard(sqlite3_context *context, Function function) { void InitFunction(sqlite3_context *context, int, sqlite3_value **) { Guard(context, [&] { - EnsureSchema(sqlite3_context_db_handle(context)); + EnsureSchemaForced(sqlite3_context_db_handle(context)); sqlite3_result_int(context, 1); }); } @@ -1322,6 +1883,13 @@ void WorkspaceCreateFunction(sqlite3_context *context, int, sqlite3_value **valu update.BindInt64(1, root); update.BindInt64(2, workspace_id); update.Done(); + Statement acl(db, + "INSERT INTO _vexfs_acl_entries(workspace_id,inode_id,principal_id,effect,permissions,inherit_flags) " + "VALUES(?1,?2,'local','allow','traverse,list,read,write,create,delete,rename,history,snapshot,share,admin',1)"); + acl.BindInt64(1, workspace_id); + acl.BindInt64(2, root); + acl.Done(); + CreateCommit(db, Workspace{workspace_id, root, 0}, "create workspace"); sqlite3_result_int64(context, workspace_id); }); } @@ -1352,224 +1920,2344 @@ void MkdirFunction(sqlite3_context *context, int, sqlite3_value **values) { }); } -void WriteFunction(sqlite3_context *context, int, sqlite3_value **values) { +void CreateFunction(sqlite3_context *context, int, sqlite3_value **values) { Guard(context, [&] { sqlite3 *db = sqlite3_context_db_handle(context); EnsureSchema(db); const Workspace workspace = FindWorkspace(db, RequiredText(values[0], "workspace")); const auto parts = PathParts(RequiredText(values[1], "path")); - const auto content = RequiredBlob(values[2], "content"); - if (content.size() > static_cast(kMaxStagedBytes)) { - throw SqlError("file is larger than the Phase 0 limit (128 MiB)", SQLITE_TOOBIG); + const std::string kind = RequiredText(values[2], "kind"); + const int mode = RequiredMode(values[3]); + if (parts.empty()) throw SqlError("workspace root already exists", SQLITE_CONSTRAINT); + if (kind != "file" && kind != "directory") { + throw SqlError("special files (fifo, device and socket) are not supported", + SQLITE_MISMATCH); } - sqlite3_result_int64(context, WritePath(db, workspace, parts, content, "write file")); - }); -} -void ReadFunction(sqlite3_context *context, int, sqlite3_value **values) { - Guard(context, [&] { - sqlite3 *db = sqlite3_context_db_handle(context); - EnsureSchema(db); - const Workspace workspace = FindWorkspace(db, RequiredText(values[0], "workspace")); + Savepoint savepoint(db, "vexfs_create_item"); + AcquireWriteLock(db); + auto [parent, name] = ResolveParent(db, workspace, parts); + Node existing; + if (FindChild(db, workspace.id, parent.id, name, &existing)) { + throw SqlError("destination already exists", SQLITE_CONSTRAINT); + } + Node node; + node.id = CreateInode(db, workspace.id, kind.c_str(), mode); + node.kind = kind; + node.mode = mode; + AddDentry(db, workspace.id, parent.id, name, node.id); + if (kind == "file") { + StoreVersion(db, workspace, node, {}, "create file"); + } else { + CreateCommit(db, workspace, "create directory"); + } + savepoint.Release(); + sqlite3_result_int64(context, node.id); + }); +} + +void SymlinkFunction(sqlite3_context *context, int, sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + const Workspace workspace = FindWorkspace(db, RequiredText(values[0], "workspace")); + const auto parts = PathParts(RequiredText(values[1], "path")); + const auto target = RequiredBlob(values[2], "symlink target"); + if (parts.empty()) throw SqlError("workspace root already exists", SQLITE_CONSTRAINT); + if (target.empty()) throw SqlError("symlink target must not be empty", SQLITE_MISMATCH); + if (static_cast(target.size()) > kMaxSymlinkBytes) { + throw SqlError("symlink target is longer than 4096 bytes", SQLITE_TOOBIG); + } + if (std::find(target.begin(), target.end(), 0) != target.end()) { + throw SqlError("symlink target contains NUL", SQLITE_MISMATCH); + } + + Savepoint savepoint(db, "vexfs_create_symlink"); + AcquireWriteLock(db); + auto [parent, name] = ResolveParent(db, workspace, parts); + Node existing; + if (FindChild(db, workspace.id, parent.id, name, &existing)) { + throw SqlError("destination already exists", SQLITE_CONSTRAINT); + } + Node node; + node.id = CreateInode(db, workspace.id, "symlink", 0777); + node.kind = "symlink"; + node.mode = 0777; + AddDentry(db, workspace.id, parent.id, name, node.id); + StoreVersion(db, workspace, node, target, "create symlink"); + savepoint.Release(); + sqlite3_result_int64(context, node.id); + }); +} + +void HardlinkFunction(sqlite3_context *context, int, sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + const Workspace workspace = FindWorkspace(db, RequiredText(values[0], "workspace")); + const auto source_parts = PathParts(RequiredText(values[1], "source")); + const auto destination_parts = PathParts(RequiredText(values[2], "destination")); + if (source_parts.empty() || destination_parts.empty()) { + throw SqlError("hard links cannot target the workspace root", SQLITE_MISMATCH); + } + Savepoint savepoint(db, "vexfs_hardlink"); + AcquireWriteLock(db); + const Node source = Resolve(db, workspace, source_parts); + if (source.kind != "file") { + throw SqlError("hard links are supported only for regular files", SQLITE_MISMATCH); + } + auto [parent, name] = ResolveParent(db, workspace, destination_parts); + Node existing; + if (FindChild(db, workspace.id, parent.id, name, &existing)) { + throw SqlError("destination already exists", SQLITE_CONSTRAINT); + } + AddDentry(db, workspace.id, parent.id, name, source.id); + CreateCommit(db, workspace, "create hard link"); + savepoint.Release(); + sqlite3_result_int64(context, source.id); + }); +} + +void RequireWorkspaceInode(sqlite3 *db, const Workspace &workspace, sqlite3_int64 inode); + +void WriteFunction(sqlite3_context *context, int, sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + const Workspace workspace = FindWorkspace(db, RequiredText(values[0], "workspace")); + const auto parts = PathParts(RequiredText(values[1], "path")); + const auto content = RequiredBlob(values[2], "content"); + if (content.size() > static_cast(kMaxStagedBytes)) { + throw SqlError("file is larger than the Phase 0 limit (128 MiB)", SQLITE_TOOBIG); + } + sqlite3_result_int64(context, WritePath(db, workspace, parts, content, "write file")); + }); +} + +void AppendFunction(sqlite3_context *context, int, sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + const Workspace workspace = FindWorkspace(db, RequiredText(values[0], "workspace")); + const sqlite3_int64 inode = RequiredPositiveInteger(values[1], "inode"); + const auto suffix = RequiredBlob(values[2], "content"); + Savepoint savepoint(db, "vexfs_append"); + AcquireWriteLock(db); + RequireWorkspaceInode(db, workspace, inode); + const Node node = NodeById(db, inode, true); + if (node.kind != "file") throw SqlError("inode is not a file", SQLITE_MISMATCH); + if (node.size < 0 || node.size > kMaxStagedBytes || + suffix.size() > static_cast(kMaxStagedBytes - node.size)) { + throw SqlError("file is larger than the Phase 0 limit (128 MiB)", SQLITE_TOOBIG); + } + std::vector content = ReadNode(db, node); + content.insert(content.end(), suffix.begin(), suffix.end()); + const sqlite3_int64 version = StoreVersion( + db, workspace, node, content, "append file"); + savepoint.Release(); + sqlite3_result_int64(context, version); + }); +} + +void ReadFunction(sqlite3_context *context, int, sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + const Workspace workspace = FindWorkspace(db, RequiredText(values[0], "workspace")); const Node node = Resolve(db, workspace, PathParts(RequiredText(values[1], "path"))); ResultBlob(context, ReadNode(db, node)); }); } -void HistoryFunction(sqlite3_context *context, int argument_count, sqlite3_value **values) { +void GrepFunction(sqlite3_context *context, int, sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + const Workspace workspace = FindWorkspace(db, RequiredText(values[0], "workspace")); + const auto parts = PathParts(RequiredText(values[1], "path")); + const Node root = Resolve(db, workspace, parts); + const std::string pattern = RequiredText(values[2], "pattern"); + const sqlite3_int64 flags = RequiredNonnegativeInteger(values[3], "flags"); + const sqlite3_int64 limit = RequiredPositiveInteger(values[4], "limit"); + if (pattern.empty() || pattern.size() > kMaxGrepPatternBytes) { + throw SqlError("grep pattern must be 1..4096 bytes", SQLITE_RANGE); + } + if ((flags & ~3LL) != 0) throw SqlError("unsupported grep flags", SQLITE_RANGE); + if (limit > kMaxGrepResults) { + throw SqlError("grep limit must be at most 10240", SQLITE_RANGE); + } + const bool ignore_case = (flags & 1) != 0; + const bool files_only = (flags & 2) != 0; + const std::vector pattern_bytes(pattern.begin(), pattern.end()); + size_t pattern_characters = 0; + if (IsUtf8Text(pattern_bytes)) { + pattern_characters = static_cast(std::count_if( + pattern_bytes.begin(), pattern_bytes.end(), + [](unsigned char value) { return (value & 0xc0) != 0x80; })); + } + const bool index_used = GetGrepIndexState(db) == GrepIndexState::kAvailable && + pattern_characters >= 3; + constexpr const char *scan_query = R"SQL( +WITH RECURSIVE tree(inode_id,path,kind,current_version,size) AS ( + SELECT i.id,?2,i.kind,i.current_version,i.size + FROM _vexfs_inodes i WHERE i.id=?3 AND i.workspace_id=?1 AND i.deleted_at IS NULL + UNION ALL + SELECT child.id, + CASE WHEN tree.path='/' THEN '/'||d.name ELSE tree.path||'/'||d.name END, + child.kind,child.current_version,child.size + FROM tree + JOIN _vexfs_dentries d ON d.workspace_id=?1 AND d.parent_inode=tree.inode_id + JOIN _vexfs_inodes child ON child.id=d.inode_id AND child.deleted_at IS NULL + WHERE tree.kind='directory' +) +SELECT tree.path,tree.inode_id,tree.current_version,tree.size, + CASE WHEN version.source_version_no IS NULL THEN version.content ELSE source.content END, + version.checksum +FROM tree +JOIN _vexfs_file_versions version + ON version.inode_id=tree.inode_id AND version.version_no=tree.current_version +LEFT JOIN _vexfs_file_versions source + ON source.inode_id=version.inode_id AND source.version_no=version.source_version_no +WHERE tree.kind='file' +)SQL"; + constexpr const char *indexed_query = R"SQL( +WITH RECURSIVE +candidates(inode_id) AS MATERIALIZED ( + SELECT rowid FROM _vexfs_content_fts WHERE _vexfs_content_fts MATCH ?4 +), +ancestors(inode_id,parent_inode,path,depth) AS ( + SELECT d.inode_id,d.parent_inode,d.name,1 + FROM candidates candidate + JOIN _vexfs_dentries d ON d.inode_id=candidate.inode_id AND d.workspace_id=?1 + UNION ALL + SELECT ancestor.inode_id,d.parent_inode,d.name||'/'||ancestor.path,ancestor.depth+1 + FROM ancestors ancestor + JOIN _vexfs_dentries d ON d.inode_id=ancestor.parent_inode AND d.workspace_id=?1 + WHERE ancestor.parent_inode<>?3 AND ancestor.depth<1024 +), +paths(inode_id,path) AS MATERIALIZED ( + SELECT inode_id,'/'||path FROM ancestors WHERE parent_inode=?3 +) +SELECT paths.path,inode.id,inode.current_version,inode.size, + CASE WHEN version.source_version_no IS NULL THEN version.content ELSE source.content END, + version.checksum +FROM paths +JOIN _vexfs_inodes inode ON inode.id=paths.inode_id AND inode.deleted_at IS NULL +JOIN _vexfs_file_versions version + ON version.inode_id=inode.id AND version.version_no=inode.current_version +LEFT JOIN _vexfs_file_versions source + ON source.inode_id=version.inode_id AND source.version_no=version.source_version_no +WHERE inode.kind='file' AND + (?2='/' OR paths.path=?2 OR substr(paths.path,1,length(?2)+1)=?2||'/') +)SQL"; + Statement files(db, index_used ? indexed_query : scan_query); + files.BindInt64(1, workspace.id); + files.BindText(2, CanonicalPath(parts)); + files.BindInt64(3, index_used ? workspace.root_inode : root.id); + if (index_used) { + std::string quoted_pattern = "\""; + for (const char value : pattern) { + if (value == '"') quoted_pattern += "\"\""; + else quoted_pattern.push_back(value); + } + quoted_pattern.push_back('"'); + files.BindText(4, quoted_pattern); + } + + std::string matches = "["; + bool first_match = true; + bool truncated = false; + sqlite3_int64 match_count = 0; + sqlite3_int64 files_scanned = 0; + sqlite3_int64 bytes_scanned = 0; + sqlite3_int64 binary_files_skipped = 0; + while (files.Row()) { + ++files_scanned; + const sqlite3_int64 expected_size = files.Int64(3); + if (expected_size < 0 || expected_size > kMaxStagedBytes || + files.Type(4) == SQLITE_NULL) { + throw SqlError("current file content reference is corrupt", SQLITE_CORRUPT); + } + const std::vector content = files.Blob(4); + if (static_cast(content.size()) != expected_size) { + throw SqlError("current file size does not match content", SQLITE_CORRUPT); + } + if (!IsSha256(files.Text(5)) || + vexfs::Sha256Hex(content.data(), content.size()) != files.Text(5)) { + throw SqlError("current file checksum does not match content", SQLITE_CORRUPT); + } + bytes_scanned += expected_size; + if (!IsUtf8Text(content)) { + ++binary_files_skipped; + continue; + } + const std::string path = files.Text(0); + size_t line_start = 0; + sqlite3_int64 line_number = 1; + while (line_start < content.size()) { + const auto begin = content.begin() + static_cast(line_start); + const auto newline = std::find(begin, content.end(), static_cast('\n')); + const size_t line_end = static_cast(newline - content.begin()); + if (LineContains(content.data() + line_start, content.data() + line_end, + pattern, ignore_case)) { + if (!first_match) matches.push_back(','); + first_match = false; + const std::string line(reinterpret_cast(content.data() + line_start), + line_end - line_start); + matches += "{\"path\":\"" + JsonEscape(path) + + "\",\"line\":" + std::to_string(line_number) + + ",\"text\":\"" + JsonEscape(line) + "\"}"; + ++match_count; + if (match_count == limit) { + truncated = true; + break; + } + if (files_only) break; + } + if (newline == content.end()) break; + line_start = line_end + 1; + ++line_number; + } + if (truncated) break; + } + matches.push_back(']'); + const std::string json = "{\"matches\":" + matches + + ",\"match_count\":" + std::to_string(match_count) + + ",\"files_scanned\":" + std::to_string(files_scanned) + + ",\"bytes_scanned\":" + std::to_string(bytes_scanned) + + ",\"binary_files_skipped\":" + std::to_string(binary_files_skipped) + + ",\"index_used\":" + (index_used ? "true" : "false") + + ",\"truncated\":" + (truncated ? "true" : "false") + "}"; + sqlite3_result_text(context, json.data(), static_cast(json.size()), SQLITE_TRANSIENT); + }); +} + +std::string GrepIndexStatus(sqlite3 *db) { + const GrepIndexState state = GetGrepIndexState(db); + sqlite3_int64 indexed_files = 0; + if (state == GrepIndexState::kAvailable) { + Statement count(db, "SELECT count(*) FROM _vexfs_content_fts"); + if (count.Row()) indexed_files = count.Int64(0); + } + return "{\"enabled\":" + std::string(state == GrepIndexState::kDisabled ? "false" : "true") + + ",\"available\":" + (state == GrepIndexState::kAvailable ? "true" : "false") + + ",\"dirty\":" + (GrepIndexDirty(db) ? "true" : "false") + + ",\"backend\":\"" + + (state == GrepIndexState::kAvailable ? "fts5-trigram" : "scan") + + "\",\"indexed_files\":" + std::to_string(indexed_files) + "}"; +} + +void GrepIndexFunction(sqlite3_context *context, int, sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + const std::string action = RequiredText(values[0], "action"); + if (action == "enable") { + EnableGrepIndex(db); + } else if (action == "rebuild") { + if (GetGrepIndexState(db) != GrepIndexState::kAvailable) { + throw SqlError("grep index is not enabled or available", SQLITE_MISMATCH); + } + RebuildGrepIndex(db); + } else if (action == "disable") { + DisableGrepIndex(db); + } else if (action != "status") { + throw SqlError("index action must be status, enable, rebuild or disable", + SQLITE_MISMATCH); + } + const std::string json = GrepIndexStatus(db); + sqlite3_result_text(context, json.data(), static_cast(json.size()), SQLITE_TRANSIENT); + }); +} + +void HistoryFunction(sqlite3_context *context, int argument_count, sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + const Workspace workspace = FindWorkspace(db, RequiredText(values[0], "workspace")); + const Node node = Resolve(db, workspace, PathParts(RequiredText(values[1], "path"))); + if (node.kind != "file") throw SqlError("path is not a file", SQLITE_MISMATCH); + const sqlite3_int64 limit = argument_count == 2 + ? 100 : RequiredPositiveInteger(values[2], "limit"); + const sqlite3_int64 before_version = argument_count == 2 + ? 0 : RequiredNonnegativeInteger(values[3], "before version"); + if (limit > 1000) throw SqlError("history limit must be at most 1000", SQLITE_RANGE); + + Statement statement(db, + "SELECT v.version_no, v.commit_id, c.parent_commit, v.size, v.created_at, c.message, " + "v.checksum " + "FROM _vexfs_file_versions v JOIN _vexfs_commits c ON c.id=v.commit_id " + "WHERE v.inode_id=?1 AND (?2=0 OR v.version_no(json.size()), SQLITE_TRANSIENT); + }); +} + +void ReadVersionFunction(sqlite3_context *context, int, sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + const Workspace workspace = FindWorkspace(db, RequiredText(values[0], "workspace")); + const Node node = Resolve(db, workspace, PathParts(RequiredText(values[1], "path"))); + ResultBlob(context, ReadNodeVersion(db, node, RequiredPositiveInteger(values[2], "version"))); + }); +} + +void CompareVersionsFunction(sqlite3_context *context, int, sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + const std::string path = RequiredText(values[1], "path"); + const Workspace workspace = FindWorkspace(db, RequiredText(values[0], "workspace")); + const Node node = Resolve(db, workspace, PathParts(path)); + if (node.kind != "file") throw SqlError("path is not a file", SQLITE_MISMATCH); + const sqlite3_int64 from_version = RequiredPositiveInteger(values[2], "from version"); + const sqlite3_int64 to_version = RequiredPositiveInteger(values[3], "to version"); + const VersionStorage from = ResolveVersionStorage(db, node.id, from_version); + const VersionStorage to = ResolveVersionStorage(db, node.id, to_version); + ReadOnlyBlob from_blob(db, from.rowid); + ReadOnlyBlob to_blob(db, to.rowid); + if (from_blob.size() != from.size || to_blob.size() != to.size) { + throw SqlError("file version size does not match content", SQLITE_CORRUPT); + } + std::array from_chunk{}; + std::array to_chunk{}; + vexfs::Sha256 from_hash; + vexfs::Sha256 to_hash; + bool changed = from.size != to.size; + bool binary = false; + const sqlite3_int64 maximum = std::max(from.size, to.size); + for (sqlite3_int64 offset = 0; offset < maximum; offset += from_chunk.size()) { + const int from_bytes = static_cast(std::min( + from_chunk.size(), std::max(0, from.size - offset))); + const int to_bytes = static_cast(std::min( + to_chunk.size(), std::max(0, to.size - offset))); + if (from_bytes > 0) from_blob.Read(from_chunk.data(), from_bytes, static_cast(offset)); + if (to_bytes > 0) to_blob.Read(to_chunk.data(), to_bytes, static_cast(offset)); + from_hash.Update(from_chunk.data(), static_cast(from_bytes)); + to_hash.Update(to_chunk.data(), static_cast(to_bytes)); + binary = binary || + std::find(from_chunk.begin(), from_chunk.begin() + from_bytes, 0) != + from_chunk.begin() + from_bytes || + std::find(to_chunk.begin(), to_chunk.begin() + to_bytes, 0) != + to_chunk.begin() + to_bytes; + if (!changed && (from_bytes != to_bytes || + !std::equal(from_chunk.begin(), from_chunk.begin() + from_bytes, + to_chunk.begin()))) changed = true; + } + if (vexfs::Hex(from_hash.Finish()) != from.checksum || + vexfs::Hex(to_hash.Finish()) != to.checksum) { + throw SqlError("file version checksum does not match content", SQLITE_CORRUPT); + } + const std::string json = + "{\"path\":\"" + JsonEscape(path) + "\",\"from\":" + + std::to_string(from_version) + ",\"to\":" + std::to_string(to_version) + + ",\"changed\":" + (changed ? "true" : "false") + + ",\"binary\":" + (binary ? "true" : "false") + + ",\"from_size\":" + std::to_string(from.size) + + ",\"to_size\":" + std::to_string(to.size) + "}"; + sqlite3_result_text(context, json.data(), static_cast(json.size()), SQLITE_TRANSIENT); + }); +} + +void RestoreVersionFunction(sqlite3_context *context, int, sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + const Workspace workspace = FindWorkspace(db, RequiredText(values[0], "workspace")); + const Node node = Resolve(db, workspace, PathParts(RequiredText(values[1], "path"))); + const sqlite3_int64 target_version = RequiredPositiveInteger(values[2], "version"); + const sqlite3_int64 expected_version = RequiredPositiveInteger(values[3], "expected version"); + if (node.version != expected_version) { + throw SqlError("version conflict: expected " + std::to_string(expected_version) + + ", current " + std::to_string(node.version), SQLITE_CONSTRAINT); + } + if (target_version == node.version) { + throw SqlError("target version is already current", SQLITE_MISMATCH); + } + const sqlite3_int64 version = StoreVersionFromVersion( + db, workspace, node, target_version, + "restore version " + std::to_string(target_version)); + sqlite3_result_int64(context, version); + }); +} + +void StatFunction(sqlite3_context *context, int, sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + const std::string path = RequiredText(values[1], "path"); + const Workspace workspace = FindWorkspace(db, RequiredText(values[0], "workspace")); + const Node node = Resolve(db, workspace, PathParts(path)); + std::string checksum; + if (node.kind != "directory" && node.version > 0) { + Statement version(db, + "SELECT checksum FROM _vexfs_file_versions WHERE inode_id=?1 AND version_no=?2"); + version.BindInt64(1, node.id); + version.BindInt64(2, node.version); + if (!version.Row()) throw SqlError("current file version is missing", SQLITE_CORRUPT); + checksum = version.Text(0); + } + const std::string json = "{\"path\":\"" + JsonEscape(path) + "\",\"inode\":" + + std::to_string(node.id) + ",\"kind\":\"" + node.kind + "\",\"mode\":" + + std::to_string(node.mode) + ",\"owner_principal\":\"" + + JsonEscape(node.owner_principal) + "\",\"uid\":" + std::to_string(node.uid) + + ",\"gid\":" + std::to_string(node.gid) + ",\"size\":" + std::to_string(node.size) + + ",\"link_count\":" + std::to_string(LinkCount(db, workspace, node.id)) + + ",\"version\":" + std::to_string(node.version) + ",\"checksum\":" + + (checksum.empty() ? "null" : "\"" + JsonEscape(checksum) + "\"") + + ",\"created_at\":" + + std::to_string(node.created_at) + ",\"accessed_at\":" + + std::to_string(node.accessed_at) + ",\"updated_at\":" + + std::to_string(node.updated_at) + ",\"changed_at\":" + + std::to_string(node.changed_at) + "}"; + sqlite3_result_text(context, json.data(), static_cast(json.size()), SQLITE_TRANSIENT); + }); +} + +struct CheckIssue { + std::string code; + std::string object; + std::string type; + std::string message; + std::string suggestion; +}; + +struct CheckReport { + static constexpr size_t kMaxReportedIssues = 1000; + + std::vector issues; + sqlite3_int64 total_issues = 0; + sqlite3_int64 inodes = 0; + sqlite3_int64 dentries = 0; + sqlite3_int64 versions = 0; + sqlite3_int64 content_bytes = 0; + sqlite3_int64 commits = 0; + sqlite3_int64 snapshots = 0; + sqlite3_int64 handles = 0; + sqlite3_int64 staging_objects = 0; + sqlite3_int64 history_rows = 0; + + void Add(std::string code, std::string object, std::string type, + std::string message, std::string suggestion) { + ++total_issues; + if (issues.size() < kMaxReportedIssues) { + issues.push_back({std::move(code), std::move(object), std::move(type), + std::move(message), std::move(suggestion)}); + } + } +}; + +bool IsSha256(const std::string &value) { + if (value.size() != 64) return false; + return std::all_of(value.begin(), value.end(), [](unsigned char character) { + return (character >= '0' && character <= '9') || + (character >= 'a' && character <= 'f'); + }); +} + +std::string HashVersionBlob(sqlite3 *db, sqlite3_int64 rowid, sqlite3_int64 expected_size) { + ReadOnlyBlob blob(db, rowid); + if (blob.size() != expected_size) { + throw SqlError("file version size does not match content", SQLITE_CORRUPT); + } + vexfs::Sha256 hash; + std::array chunk{}; + for (sqlite3_int64 offset = 0; offset < expected_size; offset += chunk.size()) { + const int bytes = static_cast(std::min( + static_cast(chunk.size()), expected_size - offset)); + blob.Read(chunk.data(), bytes, static_cast(offset)); + hash.Update(chunk.data(), static_cast(bytes)); + } + return vexfs::Hex(hash.Finish()); +} + +std::string CheckReportJson(const CheckReport &report, const std::string &workspace, + bool deep, sqlite3_int64 elapsed_ms) { + std::string issues = "["; + bool first = true; + for (const CheckIssue &issue : report.issues) { + if (!first) issues += ','; + first = false; + issues += "{\"code\":\"" + JsonEscape(issue.code) + + "\",\"object\":\"" + JsonEscape(issue.object) + + "\",\"type\":\"" + JsonEscape(issue.type) + + "\",\"message\":\"" + JsonEscape(issue.message) + + "\",\"suggestion\":\"" + JsonEscape(issue.suggestion) + "\"}"; + } + issues += ']'; + return "{\"ok\":" + std::string(report.total_issues == 0 ? "true" : "false") + + ",\"workspace\":\"" + JsonEscape(workspace) + + "\",\"mode\":\"" + (deep ? "deep" : "quick") + + "\",\"content_model\":\"full-blob-v1\",\"checked\":{" + + "\"inodes\":" + std::to_string(report.inodes) + + ",\"dentries\":" + std::to_string(report.dentries) + + ",\"versions\":" + std::to_string(report.versions) + + ",\"content_bytes\":" + std::to_string(report.content_bytes) + + ",\"commits\":" + std::to_string(report.commits) + + ",\"snapshots\":" + std::to_string(report.snapshots) + + ",\"handles\":" + std::to_string(report.handles) + + ",\"staging_objects\":" + std::to_string(report.staging_objects) + + ",\"history_rows\":" + std::to_string(report.history_rows) + + "},\"issue_count\":" + std::to_string(report.total_issues) + + ",\"reported_issue_count\":" + std::to_string(report.issues.size()) + + ",\"truncated\":" + + (report.total_issues > static_cast(report.issues.size()) ? "true" : "false") + + ",\"elapsed_ms\":" + std::to_string(elapsed_ms) + + ",\"issues\":" + issues + "}"; +} + +void CheckFunction(sqlite3_context *context, int, sqlite3_value **values) { + Guard(context, [&] { + const auto started = std::chrono::steady_clock::now(); + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + const std::string workspace_name = RequiredText(values[0], "workspace"); + if (sqlite3_value_type(values[1]) != SQLITE_INTEGER) { + throw SqlError("deep must be 0 or 1", SQLITE_MISMATCH); + } + const int deep_value = sqlite3_value_int(values[1]); + if (deep_value != 0 && deep_value != 1) { + throw SqlError("deep must be 0 or 1", SQLITE_RANGE); + } + const bool deep = deep_value != 0; + const Workspace workspace = FindWorkspace(db, workspace_name); + CheckReport report; + + try { + Statement quick_check(db, "PRAGMA quick_check(100)"); + while (quick_check.Row()) { + const std::string result = quick_check.Text(0); + if (result != "ok") { + report.Add("VEXFS_SQLITE_CORRUPTION", "database", "sqlite", + result, "restore the SQLite database from a verified backup"); + } + } + } catch (const SqlError &error) { + report.Add("VEXFS_SQLITE_CHECK_FAILED", "database", "sqlite", error.what(), + "run SQLite integrity_check offline and restore a verified backup"); + } + + std::unordered_map commits; + { + Statement rows(db, + "SELECT id,COALESCE(parent_commit,0) FROM _vexfs_commits " + "WHERE workspace_id=?1 ORDER BY id"); + rows.BindInt64(1, workspace.id); + while (rows.Row()) { + ++report.commits; + commits[rows.Int64(0)] = rows.Int64(1); + } + } + if (workspace.head_commit != 0 && commits.count(workspace.head_commit) == 0) { + report.Add("VEXFS_HEAD_COMMIT_MISSING", "workspace:" + workspace_name, + "commit", "workspace head_commit does not exist", + "restore a verified backup; do not create a replacement commit by hand"); + } + for (const auto &entry : commits) { + const sqlite3_int64 id = entry.first; + const sqlite3_int64 parent = entry.second; + if (parent != 0 && commits.count(parent) == 0) { + report.Add("VEXFS_COMMIT_PARENT_MISSING", "commit:" + std::to_string(id), + "commit", "parent commit does not exist in this workspace", + "restore commit history from a verified backup"); + } else if (parent >= id && parent != 0) { + report.Add("VEXFS_COMMIT_CHAIN_INVALID", "commit:" + std::to_string(id), + "commit", "parent commit is not older than its child", + "restore commit history from a verified backup"); + } + } + std::unordered_set commit_chain; + sqlite3_int64 chain_commit = workspace.head_commit; + while (chain_commit != 0 && commits.count(chain_commit) != 0 && + commit_chain.insert(chain_commit).second) { + chain_commit = commits.at(chain_commit); + } + if (chain_commit != 0 && commit_chain.count(chain_commit) != 0) { + report.Add("VEXFS_COMMIT_CYCLE", "commit:" + std::to_string(chain_commit), + "commit", "workspace commit chain contains a cycle", + "restore commit history from a verified backup"); + } + for (const auto &entry : commits) { + if (commit_chain.count(entry.first) == 0) { + report.Add("VEXFS_COMMIT_UNREACHABLE", "commit:" + + std::to_string(entry.first), "commit", + "commit is not reachable from workspace head", + "restore commit history from a verified backup"); + } + } + + struct InodeCheck { + std::string kind; + sqlite3_int64 size = 0; + sqlite3_int64 version = 0; + bool deleted = false; + }; + std::unordered_map inodes; + { + Statement rows(db, + "SELECT id,kind,size,current_version,deleted_at IS NOT NULL " + "FROM _vexfs_inodes WHERE workspace_id=?1"); + rows.BindInt64(1, workspace.id); + while (rows.Row()) { + ++report.inodes; + inodes[rows.Int64(0)] = {rows.Text(1), rows.Int64(2), rows.Int64(3), + rows.Int(4) != 0}; + } + } + const auto root = inodes.find(workspace.root_inode); + if (root == inodes.end() || root->second.deleted || root->second.kind != "directory") { + report.Add("VEXFS_ROOT_INVALID", "inode:" + std::to_string(workspace.root_inode), + "inode", "workspace root is missing, deleted, or not a directory", + "restore the workspace from a verified snapshot or backup"); + } + + std::unordered_map> children; + std::unordered_map directory_links; + { + Statement rows(db, + "SELECT parent_inode,name,inode_id FROM _vexfs_dentries WHERE workspace_id=?1"); + rows.BindInt64(1, workspace.id); + while (rows.Row()) { + ++report.dentries; + const sqlite3_int64 parent = rows.Int64(0); + const std::string name = rows.Text(1); + const sqlite3_int64 child = rows.Int64(2); + const auto parent_inode = inodes.find(parent); + const auto child_inode = inodes.find(child); + const std::string object = "dentry:" + std::to_string(parent) + "/" + name; + const bool valid_parent = parent_inode != inodes.end() && + !parent_inode->second.deleted && parent_inode->second.kind == "directory"; + if (name.empty() || name == "." || name == ".." || name.size() > 255 || + name.find('/') != std::string::npos || name.find('\0') != std::string::npos) { + report.Add("VEXFS_DENTRY_NAME_INVALID", object, "dentry", + "directory entry name is invalid", + "restore the directory entry from a verified snapshot"); + } + if (!valid_parent) { + report.Add("VEXFS_DENTRY_PARENT_INVALID", object, "dentry", + "parent inode is missing, deleted, or not a directory", + "restore the directory tree from a verified snapshot"); + } + if (child_inode == inodes.end() || child_inode->second.deleted) { + report.Add("VEXFS_DENTRY_CHILD_INVALID", object, "dentry", + "child inode is missing or deleted", + "restore the directory tree from a verified snapshot"); + } else { + if (child_inode->second.kind == "directory") ++directory_links[child]; + if (valid_parent) children[parent].push_back(child); + } + if (child == workspace.root_inode) { + report.Add("VEXFS_ROOT_HAS_PARENT", object, "dentry", + "workspace root appears as a child", + "restore the directory tree from a verified snapshot"); + } + } + } + for (const auto &entry : directory_links) { + if (entry.second > 1) { + report.Add("VEXFS_DIRECTORY_HARDLINK", "inode:" + std::to_string(entry.first), + "dentry", "directory has more than one parent", + "restore the directory tree from a verified snapshot"); + } + } + std::unordered_set reachable; + std::vector pending; + if (root != inodes.end() && !root->second.deleted) { + reachable.insert(workspace.root_inode); + pending.push_back(workspace.root_inode); + } + while (!pending.empty()) { + const sqlite3_int64 parent = pending.back(); + pending.pop_back(); + for (const sqlite3_int64 child : children[parent]) { + if (reachable.insert(child).second) pending.push_back(child); + } + } + for (const auto &entry : inodes) { + if (!entry.second.deleted && reachable.count(entry.first) == 0) { + report.Add("VEXFS_INODE_UNREACHABLE", "inode:" + std::to_string(entry.first), + "inode", "active inode is not reachable from the workspace root", + "restore the directory tree from a verified snapshot"); + } + } + + struct VersionCheck { + sqlite3_int64 rowid = 0; + sqlite3_int64 commit = 0; + sqlite3_int64 size = 0; + sqlite3_int64 content_size = 0; + sqlite3_int64 source = 0; + std::string checksum; + }; + std::map, VersionCheck> versions; + { + Statement rows(db, R"SQL( +SELECT v.rowid,v.inode_id,v.version_no,v.commit_id,v.size,length(v.content), + COALESCE(v.source_version_no,0),v.checksum +FROM _vexfs_file_versions v +JOIN _vexfs_inodes i ON i.id=v.inode_id +WHERE i.workspace_id=?1 +ORDER BY v.inode_id,v.version_no +)SQL"); + rows.BindInt64(1, workspace.id); + while (rows.Row()) { + ++report.versions; + VersionCheck version{rows.Int64(0), rows.Int64(3), rows.Int64(4), + rows.Int64(5), rows.Int64(6), rows.Text(7)}; + versions[{rows.Int64(1), rows.Int64(2)}] = std::move(version); + } + } + { + Statement orphan(db, + "SELECT v.id,v.inode_id FROM _vexfs_file_versions v " + "LEFT JOIN _vexfs_inodes i ON i.id=v.inode_id WHERE i.id IS NULL"); + while (orphan.Row()) { + report.Add("VEXFS_VERSION_INODE_MISSING", + "version-row:" + std::to_string(orphan.Int64(0)), "content", + "file version references an unknown inode", + "restore version history from a verified backup"); + } + } + for (const auto &entry : versions) { + const sqlite3_int64 inode = entry.first.first; + const sqlite3_int64 number = entry.first.second; + const VersionCheck &version = entry.second; + const std::string object = "version:" + std::to_string(inode) + "/" + + std::to_string(number); + if (version.size < 0 || version.size > kMaxStagedBytes) { + report.Add("VEXFS_VERSION_SIZE_INVALID", object, "content", + "file version size is outside the supported range", + "restore this file version from a verified snapshot or backup"); + } + if (!IsSha256(version.checksum)) { + report.Add("VEXFS_CHECKSUM_INVALID", object, "content", + "stored SHA-256 is not a lowercase 64-character digest", + "restore this file version from a verified snapshot or backup"); + } + if (commits.count(version.commit) == 0) { + report.Add("VEXFS_VERSION_COMMIT_MISSING", object, "commit", + "file version commit does not exist in this workspace", + "restore version history from a verified backup"); + } + if (version.source == 0) { + report.content_bytes += std::max(version.size, 0); + if (version.content_size != version.size) { + report.Add("VEXFS_CONTENT_SIZE_MISMATCH", object, "content", + "stored BLOB length does not match version size", + "restore this file version from a verified snapshot or backup"); + } else if (deep && IsSha256(version.checksum)) { + try { + const std::string actual = HashVersionBlob(db, version.rowid, version.size); + if (actual != version.checksum) { + report.Add("VEXFS_CHECKSUM_MISMATCH", object, "content", + "stored BLOB does not match its SHA-256", + "restore this file version from a verified snapshot or backup"); + } + } catch (const SqlError &error) { + report.Add("VEXFS_CONTENT_UNREADABLE", object, "content", error.what(), + "restore this file version from a verified snapshot or backup"); + } + } + } else { + const auto source = versions.find({inode, version.source}); + if (version.content_size != 0) { + report.Add("VEXFS_ALIAS_HAS_CONTENT", object, "content", + "version alias unexpectedly stores its own BLOB", + "restore version history from a verified backup"); + } + if (source == versions.end() || source->second.source != 0) { + report.Add("VEXFS_VERSION_SOURCE_INVALID", object, "content", + "version alias does not point directly to a canonical version", + "restore version history from a verified backup"); + } else if (source->second.size != version.size || + source->second.checksum != version.checksum) { + report.Add("VEXFS_VERSION_SOURCE_MISMATCH", object, "content", + "version alias metadata differs from its source", + "restore version history from a verified backup"); + } + } + } + for (const auto &entry : inodes) { + if (entry.second.deleted || entry.second.kind == "directory") continue; + const auto version = versions.find({entry.first, entry.second.version}); + bool pending_first_publish = false; + if (entry.second.version == 0 && entry.second.size == 0) { + Statement handle(db, + "SELECT 1 FROM _vexfs_handles WHERE workspace_id=?1 AND inode_id=?2 " + "AND state IN ('open','retained') LIMIT 1"); + handle.BindInt64(1, workspace.id); + handle.BindInt64(2, entry.first); + pending_first_publish = handle.Row(); + } + if (!pending_first_publish && + (entry.second.version <= 0 || version == versions.end())) { + report.Add("VEXFS_CURRENT_VERSION_MISSING", "inode:" + std::to_string(entry.first), + "inode", "active file or symlink has no current version", + "restore this inode from a verified snapshot or backup"); + } else if (!pending_first_publish && version->second.size != entry.second.size) { + report.Add("VEXFS_CURRENT_SIZE_MISMATCH", "inode:" + std::to_string(entry.first), + "inode", "inode size differs from its current version", + "restore this inode from a verified snapshot or backup"); + } + } + + { + Statement rows(db, + "SELECT name,commit_id FROM _vexfs_snapshots WHERE workspace_id=?1"); + rows.BindInt64(1, workspace.id); + while (rows.Row()) { + ++report.snapshots; + if (commits.count(rows.Int64(1)) == 0) { + report.Add("VEXFS_SNAPSHOT_COMMIT_MISSING", "snapshot:" + rows.Text(0), + "snapshot", "snapshot commit does not exist in this workspace", + "drop the broken snapshot name or restore history from backup"); + } + } + } + { + Statement rows(db, + "SELECT inode_id,commit_id,kind,size,current_version,deleted_at IS NOT NULL " + "FROM _vexfs_inode_states WHERE workspace_id=?1"); + rows.BindInt64(1, workspace.id); + while (rows.Row()) { + ++report.history_rows; + const sqlite3_int64 inode = rows.Int64(0); + const sqlite3_int64 commit = rows.Int64(1); + const std::string object = "inode-state:" + std::to_string(inode) + "@" + + std::to_string(commit); + if (commits.count(commit) == 0) { + report.Add("VEXFS_HISTORY_COMMIT_MISSING", object, + "history", "history row references a missing commit", + "restore workspace history from a verified backup"); + } + if (inodes.count(inode) == 0) { + report.Add("VEXFS_HISTORY_INODE_MISSING", object, "history", + "inode state references an unknown inode", + "restore workspace history from a verified backup"); + } + if (rows.Int(5) == 0 && rows.Text(2) != "directory") { + const auto version = versions.find({inode, rows.Int64(4)}); + if (version == versions.end()) { + report.Add("VEXFS_HISTORY_VERSION_MISSING", object, "history", + "inode state references a missing file version", + "restore workspace history from a verified backup"); + } else if (version->second.size != rows.Int64(3)) { + report.Add("VEXFS_HISTORY_SIZE_MISMATCH", object, "history", + "inode state size differs from its file version", + "restore workspace history from a verified backup"); + } + } + } + } + { + Statement rows(db, + "SELECT parent_inode,name,commit_id,inode_id FROM _vexfs_dentry_states " + "WHERE workspace_id=?1"); + rows.BindInt64(1, workspace.id); + while (rows.Row()) { + ++report.history_rows; + const std::string object = "dentry-state:" + std::to_string(rows.Int64(0)) + + "/" + rows.Text(1) + "@" + + std::to_string(rows.Int64(2)); + if (commits.count(rows.Int64(2)) == 0) { + report.Add("VEXFS_HISTORY_COMMIT_MISSING", object, "history", + "history row references a missing commit", + "restore workspace history from a verified backup"); + } + if (inodes.count(rows.Int64(0)) == 0 || inodes.count(rows.Int64(3)) == 0) { + report.Add("VEXFS_HISTORY_DENTRY_INVALID", object, "history", + "historical directory entry references an unknown inode", + "restore workspace history from a verified backup"); + } + } + } + for (const char *table : {"_vexfs_xattr_states", "_vexfs_acl_states"}) { + const std::string sql = "SELECT inode_id,commit_id FROM " + std::string(table) + + " WHERE workspace_id=?1"; + Statement rows(db, sql.c_str()); + rows.BindInt64(1, workspace.id); + while (rows.Row()) { + ++report.history_rows; + const std::string object = std::string("table:") + table + "@" + + std::to_string(rows.Int64(1)); + if (commits.count(rows.Int64(1)) == 0) { + report.Add("VEXFS_HISTORY_COMMIT_MISSING", object, "history", + "history row references a missing commit", + "restore workspace history from a verified backup"); + } + if (inodes.count(rows.Int64(0)) == 0) { + report.Add("VEXFS_HISTORY_INODE_MISSING", object, "history", + "history row references an unknown inode", + "restore workspace history from a verified backup"); + } + } + } + + { + Statement rows(db, R"SQL( +SELECT h.id,h.inode_id,h.state,h.dirty_generation,h.published_generation, + s.generation,s.logical_size,s.capacity,length(d.content) +FROM _vexfs_handles h +LEFT JOIN _vexfs_staging s ON s.handle_id=h.id +LEFT JOIN _vexfs_staging_data d ON d.handle_id=h.id +WHERE h.workspace_id=?1 +)SQL"); + rows.BindInt64(1, workspace.id); + while (rows.Row()) { + ++report.handles; + const std::string id = rows.Text(0); + const std::string object = "handle:" + id; + const bool active = rows.Text(2) == "open" || rows.Text(2) == "retained"; + if (inodes.count(rows.Int64(1)) == 0) { + report.Add("VEXFS_HANDLE_INODE_MISSING", object, "staging", + "handle inode does not exist in this workspace", + "close the handle and restore the inode from backup if needed"); + } + if (rows.Int64(3) < rows.Int64(4)) { + report.Add("VEXFS_HANDLE_GENERATION_INVALID", object, "staging", + "published generation is newer than dirty generation", + "discard the handle and reopen the file"); + } + if (active) { + ++report.staging_objects; + if (rows.Type(5) == SQLITE_NULL || rows.Type(8) == SQLITE_NULL) { + report.Add("VEXFS_STAGING_MISSING", object, "staging", + "open or retained handle has no complete staging object", + "discard the handle; recover unpublished work from backup if available"); + } else if (rows.Int64(5) != rows.Int64(3) || rows.Int64(6) < 0 || + rows.Int64(7) < rows.Int64(6) || + rows.Int64(7) > kMaxStagedBytes || rows.Int64(8) != rows.Int64(7)) { + report.Add("VEXFS_STAGING_INVALID", object, "staging", + "staging generation, size, or capacity is inconsistent", + "discard the handle; recover unpublished work from backup if available"); + } + } else if (rows.Type(5) != SQLITE_NULL || rows.Type(8) != SQLITE_NULL) { + report.Add("VEXFS_CLOSED_HANDLE_STAGING", object, "staging", + "closed handle still owns staging data", + "run a future repair or garbage-collection command after backing up"); + } + } + } + { + Statement orphan(db, + "SELECT s.handle_id FROM _vexfs_staging s LEFT JOIN _vexfs_handles h " + "ON h.id=s.handle_id WHERE h.id IS NULL"); + while (orphan.Row()) { + report.Add("VEXFS_ORPHAN_STAGING", "handle:" + orphan.Text(0), "staging", + "staging metadata has no owning handle", + "run a future repair or garbage-collection command after backing up"); + } + Statement orphan_data(db, + "SELECT d.handle_id FROM _vexfs_staging_data d LEFT JOIN _vexfs_staging s " + "ON s.handle_id=d.handle_id WHERE s.handle_id IS NULL"); + while (orphan_data.Row()) { + report.Add("VEXFS_ORPHAN_STAGING_DATA", "handle:" + orphan_data.Text(0), + "staging", "staging BLOB has no metadata row", + "run a future repair or garbage-collection command after backing up"); + } + } + + const sqlite3_int64 elapsed_ms = std::chrono::duration_cast( + std::chrono::steady_clock::now() - started).count(); + const std::string json = CheckReportJson(report, workspace_name, deep, elapsed_ms); + sqlite3_result_text(context, json.data(), static_cast(json.size()), SQLITE_TRANSIENT); + }); +} + +void PathFunction(sqlite3_context *context, int, sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + const Workspace workspace = FindWorkspace(db, RequiredText(values[0], "workspace")); + const std::string path = PathForInode(db, workspace, sqlite3_value_int64(values[1])); + sqlite3_result_text(context, path.data(), static_cast(path.size()), SQLITE_TRANSIENT); + }); +} + +void ListFunction(sqlite3_context *context, int, sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + const Workspace workspace = FindWorkspace(db, RequiredText(values[0], "workspace")); + const Node directory = Resolve(db, workspace, PathParts(RequiredText(values[1], "path"))); + if (directory.kind != "directory") + throw SqlError("path is not a directory", SQLITE_MISMATCH); + + Statement statement(db, + "SELECT d.name, i.id, i.kind, i.size, i.current_version, i.mode, " + "i.owner_principal, i.uid, i.gid, i.created_at, i.accessed_at, " + "i.updated_at, i.changed_at FROM _vexfs_dentries d " + "JOIN _vexfs_inodes i ON i.id = d.inode_id " + "WHERE d.workspace_id = ?1 AND d.parent_inode = ?2 AND i.deleted_at IS NULL " + "ORDER BY d.name COLLATE BINARY"); + statement.BindInt64(1, workspace.id); + statement.BindInt64(2, directory.id); + std::string json = "["; + bool first = true; + while (statement.Row()) { + if (!first) json += ','; + first = false; + json += "{\"name\":\"" + JsonEscape(statement.Text(0)) + "\",\"inode\":" + + std::to_string(statement.Int64(1)) + ",\"kind\":\"" + statement.Text(2) + + "\",\"size\":" + std::to_string(statement.Int64(3)) + ",\"version\":" + + std::to_string(statement.Int64(4)) + ",\"mode\":" + + std::to_string(statement.Int(5)) + ",\"owner_principal\":\"" + + JsonEscape(statement.Text(6)) + "\",\"uid\":" + + std::to_string(statement.Int64(7)) + ",\"gid\":" + + std::to_string(statement.Int64(8)) + ",\"created_at\":" + + std::to_string(statement.Int64(9)) + ",\"accessed_at\":" + + std::to_string(statement.Int64(10)) + ",\"updated_at\":" + + std::to_string(statement.Int64(11)) + ",\"changed_at\":" + + std::to_string(statement.Int64(12)) + ",\"link_count\":" + + std::to_string(LinkCount(db, workspace, statement.Int64(1))) + "}"; + } + json += ']'; + sqlite3_result_text(context, json.data(), static_cast(json.size()), SQLITE_TRANSIENT); + }); +} + +void RequireWorkspaceInode(sqlite3 *db, const Workspace &workspace, sqlite3_int64 inode) { + Statement statement(db, + "SELECT 1 FROM _vexfs_inodes WHERE id=?1 AND workspace_id=?2 LIMIT 1"); + statement.BindInt64(1, inode); + statement.BindInt64(2, workspace.id); + if (!statement.Row()) throw SqlError("inode not found", SQLITE_NOTFOUND); +} + +void ReadlinkFunction(sqlite3_context *context, int, sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + const Workspace workspace = FindWorkspace(db, RequiredText(values[0], "workspace")); + const sqlite3_int64 inode = RequiredPositiveInteger(values[1], "inode"); + RequireWorkspaceInode(db, workspace, inode); + const Node node = NodeById(db, inode, true); + if (node.kind != "symlink") { + throw SqlError("inode is not a symbolic link", SQLITE_MISMATCH); + } + if (node.version <= 0) throw SqlError("symlink target is missing", SQLITE_CORRUPT); + ResultBlob(context, ReadVersionContent(db, node.id, node.version)); + }); +} + +void SetModeFunction(sqlite3_context *context, int, sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + const Workspace workspace = FindWorkspace(db, RequiredText(values[0], "workspace")); + const sqlite3_int64 inode = RequiredPositiveInteger(values[1], "inode"); + const int mode = RequiredMode(values[2]); + Savepoint savepoint(db, "vexfs_set_mode"); + AcquireWriteLock(db); + RequireWorkspaceInode(db, workspace, inode); + const Node node = NodeById(db, inode, true); + if (node.kind == "symlink") { + throw SqlError("symbolic link mode cannot be changed", SQLITE_MISMATCH); + } + if (node.mode != mode) { + Statement update(db, + "UPDATE _vexfs_inodes SET mode=?1, " + "changed_at=CAST(unixepoch('subsec')*1000 AS INTEGER) " + "WHERE id=?2 AND workspace_id=?3"); + update.BindInt(1, mode); + update.BindInt64(2, inode); + update.BindInt64(3, workspace.id); + update.Done(); + if (sqlite3_changes64(db) != 1) { + throw SqlError("inode not found", SQLITE_NOTFOUND); + } + CommitMetadataChange(db, workspace, inode, "chmod"); + } + savepoint.Release(); + sqlite3_result_int(context, mode); + }); +} + +void SetTimesFunction(sqlite3_context *context, int, sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + const Workspace workspace = FindWorkspace(db, RequiredText(values[0], "workspace")); + const sqlite3_int64 inode = RequiredPositiveInteger(values[1], "inode"); + const sqlite3_int64 accessed_at = RequiredNonnegativeInteger(values[2], "access time"); + const sqlite3_int64 updated_at = RequiredNonnegativeInteger(values[3], "modify time"); + const int mask = sqlite3_value_int(values[4]); + if ((mask & ~3) != 0 || mask == 0) { + throw SqlError("time mask must select access time, modify time, or both", + SQLITE_MISMATCH); + } + Savepoint savepoint(db, "vexfs_set_times"); + AcquireWriteLock(db); + RequireWorkspaceInode(db, workspace, inode); + Statement update(db, + "UPDATE _vexfs_inodes SET " + "accessed_at=CASE WHEN (?1&1)<>0 THEN ?2 ELSE accessed_at END," + "updated_at=CASE WHEN (?1&2)<>0 THEN ?3 ELSE updated_at END," + "changed_at=CAST(unixepoch('subsec')*1000 AS INTEGER) " + "WHERE id=?4 AND workspace_id=?5 AND deleted_at IS NULL"); + update.BindInt(1, mask); + update.BindInt64(2, accessed_at); + update.BindInt64(3, updated_at); + update.BindInt64(4, inode); + update.BindInt64(5, workspace.id); + update.Done(); + if (sqlite3_changes64(db) != 1) throw SqlError("inode not found", SQLITE_NOTFOUND); + CommitMetadataChange(db, workspace, inode, "set times"); + savepoint.Release(); + sqlite3_result_int(context, mask); + }); +} + +struct AclEntry { + std::string principal; + std::string effect; + std::string permissions; + sqlite3_int64 inherit_flags = 0; + + bool operator==(const AclEntry &other) const { + return principal == other.principal && effect == other.effect && + permissions == other.permissions && inherit_flags == other.inherit_flags; + } +}; + +std::vector ReadAclEntries(sqlite3 *db, const Workspace &workspace, + sqlite3_int64 inode) { + Statement statement(db, + "SELECT principal_id,effect,permissions,inherit_flags FROM _vexfs_acl_entries " + "WHERE workspace_id=?1 AND inode_id=?2 ORDER BY principal_id COLLATE BINARY,effect"); + statement.BindInt64(1, workspace.id); + statement.BindInt64(2, inode); + std::vector entries; + while (statement.Row()) { + entries.push_back({statement.Text(0), statement.Text(1), statement.Text(2), + statement.Int64(3)}); + } + return entries; +} + +std::string AclJson(const std::vector &entries) { + std::string json = "["; + bool first = true; + for (const AclEntry &entry : entries) { + if (!first) json += ','; + first = false; + json += "{\"principal\":\"" + JsonEscape(entry.principal) + + "\",\"effect\":\"" + JsonEscape(entry.effect) + + "\",\"permissions\":\"" + JsonEscape(entry.permissions) + + "\",\"inherit\":" + std::to_string(entry.inherit_flags) + "}"; + } + json += ']'; + return json; +} + +std::vector ParseAclJson(sqlite3 *db, sqlite3_value *value) { + const std::string json = RequiredText(value, "acl"); + Statement root(db, "SELECT json_type(?1)"); + root.BindText(1, json); + if (!root.Row() || root.Type(0) == SQLITE_NULL || root.Text(0) != "array") { + throw SqlError("acl must be a JSON array", SQLITE_MISMATCH); + } + Statement statement(db, R"SQL( +SELECT COALESCE(json_extract(value,'$.principal'),json_extract(value,'$.principal_id')), + COALESCE(json_extract(value,'$.effect'),'allow'), + json_extract(value,'$.permissions'), + COALESCE(json_extract(value,'$.inherit'),json_extract(value,'$.inherit_flags'),0) +FROM json_each(?1) +)SQL"); + statement.BindText(1, json); + std::vector entries; + while (statement.Row()) { + const std::string principal = statement.Text(0); + const std::string effect = statement.Text(1); + const std::string permissions = statement.Text(2); + const sqlite3_int64 inherit = statement.Int64(3); + if (principal.empty() || principal.size() > 255) + throw SqlError("acl principal must be 1..255 bytes", SQLITE_RANGE); + if (effect != "allow" && effect != "deny") + throw SqlError("acl effect must be allow or deny", SQLITE_MISMATCH); + if (permissions.empty() || permissions.size() > 1024) + throw SqlError("acl permissions must be 1..1024 bytes", SQLITE_RANGE); + if (inherit < 0 || inherit > 255) + throw SqlError("acl inherit flags must be 0..255", SQLITE_RANGE); + entries.push_back({principal, effect, permissions, inherit}); + if (entries.size() > 1024) throw SqlError("acl has too many entries", SQLITE_TOOBIG); + } + return entries; +} + +void AclGetFunction(sqlite3_context *context, int, sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + const Workspace workspace = FindWorkspace(db, RequiredText(values[0], "workspace")); + const sqlite3_int64 inode = RequiredPositiveInteger(values[1], "inode"); + RequireWorkspaceInode(db, workspace, inode); + const std::string json = AclJson(ReadAclEntries(db, workspace, inode)); + sqlite3_result_text(context, json.data(), static_cast(json.size()), SQLITE_TRANSIENT); + }); +} + +void AclSetFunction(sqlite3_context *context, int, sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + const Workspace workspace = FindWorkspace(db, RequiredText(values[0], "workspace")); + const sqlite3_int64 inode = RequiredPositiveInteger(values[1], "inode"); + RequireWorkspaceInode(db, workspace, inode); + const std::vector entries = ParseAclJson(db, values[2]); + Savepoint savepoint(db, "vexfs_acl_set"); + AcquireWriteLock(db); + Statement clear(db, "DELETE FROM _vexfs_acl_entries WHERE workspace_id=?1 AND inode_id=?2"); + clear.BindInt64(1, workspace.id); + clear.BindInt64(2, inode); + clear.Done(); + for (const AclEntry &entry : entries) { + Statement insert(db, + "INSERT INTO _vexfs_acl_entries(workspace_id,inode_id,principal_id,effect,permissions,inherit_flags) " + "VALUES(?1,?2,?3,?4,?5,?6)"); + insert.BindInt64(1, workspace.id); + insert.BindInt64(2, inode); + insert.BindText(3, entry.principal); + insert.BindText(4, entry.effect); + insert.BindText(5, entry.permissions); + insert.BindInt64(6, entry.inherit_flags); + insert.Done(); + } + Statement touch(db, + "UPDATE _vexfs_inodes SET changed_at=CAST(unixepoch('subsec')*1000 AS INTEGER) " + "WHERE id=?1 AND workspace_id=?2"); + touch.BindInt64(1, inode); + touch.BindInt64(2, workspace.id); + touch.Done(); + CommitMetadataChange(db, workspace, inode, "set acl"); + savepoint.Release(); + sqlite3_result_int(context, static_cast(entries.size())); + }); +} + +void AclDeleteFunction(sqlite3_context *context, int, sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + const Workspace workspace = FindWorkspace(db, RequiredText(values[0], "workspace")); + const sqlite3_int64 inode = RequiredPositiveInteger(values[1], "inode"); + RequireWorkspaceInode(db, workspace, inode); + Savepoint savepoint(db, "vexfs_acl_delete"); + AcquireWriteLock(db); + Statement remove(db, "DELETE FROM _vexfs_acl_entries WHERE workspace_id=?1 AND inode_id=?2"); + remove.BindInt64(1, workspace.id); + remove.BindInt64(2, inode); + remove.Done(); + const sqlite3_int64 removed = sqlite3_changes64(db); + if (removed > 0) CommitMetadataChange(db, workspace, inode, "delete acl"); + savepoint.Release(); + sqlite3_result_int64(context, removed); + }); +} + +void AclGrantFunction(sqlite3_context *context, int argument_count, sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + const Workspace workspace = FindWorkspace(db, RequiredText(values[0], "workspace")); + const sqlite3_int64 inode = RequiredPositiveInteger(values[1], "inode"); + const std::string principal = RequiredText(values[2], "principal"); + const std::string permissions = RequiredText(values[3], "permissions"); + if (principal.empty() || principal.size() > 255 || permissions.empty() || permissions.size() > 1024) + throw SqlError("invalid ACL principal or permissions", SQLITE_RANGE); + RequireWorkspaceInode(db, workspace, inode); + const std::string effect = (argument_count < 5 || sqlite3_value_type(values[4]) == SQLITE_NULL) + ? "allow" : RequiredText(values[4], "effect"); + if (effect != "allow" && effect != "deny") + throw SqlError("acl effect must be allow or deny", SQLITE_MISMATCH); + Savepoint savepoint(db, "vexfs_acl_grant"); + AcquireWriteLock(db); + Statement upsert(db, + "INSERT INTO _vexfs_acl_entries(workspace_id,inode_id,principal_id,effect,permissions) " + "VALUES(?1,?2,?3,?4,?5) ON CONFLICT(workspace_id,inode_id,principal_id,effect) " + "DO UPDATE SET permissions=excluded.permissions,updated_at=CAST(strftime('%s','now') AS INTEGER)*1000"); + upsert.BindInt64(1, workspace.id); + upsert.BindInt64(2, inode); + upsert.BindText(3, principal); + upsert.BindText(4, effect); + upsert.BindText(5, permissions); + upsert.Done(); + CommitMetadataChange(db, workspace, inode, "grant acl"); + savepoint.Release(); + sqlite3_result_int(context, 1); + }); +} + +void AclRevokeFunction(sqlite3_context *context, int argument_count, sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + const Workspace workspace = FindWorkspace(db, RequiredText(values[0], "workspace")); + const sqlite3_int64 inode = RequiredPositiveInteger(values[1], "inode"); + const std::string principal = RequiredText(values[2], "principal"); + RequireWorkspaceInode(db, workspace, inode); + const std::string effect = (argument_count < 4 || sqlite3_value_type(values[3]) == SQLITE_NULL) + ? std::string() : RequiredText(values[3], "effect"); + if (!effect.empty() && effect != "allow" && effect != "deny") + throw SqlError("acl effect must be allow or deny", SQLITE_MISMATCH); + Savepoint savepoint(db, "vexfs_acl_revoke"); + AcquireWriteLock(db); + Statement remove(db, + "DELETE FROM _vexfs_acl_entries WHERE workspace_id=?1 AND inode_id=?2 " + "AND principal_id=?3 AND (?4='' OR effect=?4)"); + remove.BindInt64(1, workspace.id); + remove.BindInt64(2, inode); + remove.BindText(3, principal); + remove.BindText(4, effect); + remove.Done(); + const sqlite3_int64 removed = sqlite3_changes64(db); + if (removed > 0) CommitMetadataChange(db, workspace, inode, "revoke acl"); + savepoint.Release(); + sqlite3_result_int64(context, removed); + }); +} + +void ChownFunction(sqlite3_context *context, int, sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + const Workspace workspace = FindWorkspace(db, RequiredText(values[0], "workspace")); + const sqlite3_int64 inode = RequiredPositiveInteger(values[1], "inode"); + const sqlite3_int64 requested_uid = RequiredOwnerId(values[2], "uid"); + const sqlite3_int64 requested_gid = RequiredOwnerId(values[3], "gid"); + RequireWorkspaceInode(db, workspace, inode); + const Node current = NodeById(db, inode, true); + const sqlite3_int64 uid = requested_uid < 0 ? current.uid : requested_uid; + const sqlite3_int64 gid = requested_gid < 0 ? current.gid : requested_gid; + Savepoint savepoint(db, "vexfs_chown"); + AcquireWriteLock(db); + Statement update(db, + "UPDATE _vexfs_inodes SET uid=?1,gid=?2,changed_at=CAST(unixepoch('subsec')*1000 AS INTEGER) " + "WHERE workspace_id=?3 AND id=?4"); + update.BindInt64(1, uid); + update.BindInt64(2, gid); + update.BindInt64(3, workspace.id); + update.BindInt64(4, inode); + update.Done(); + if (sqlite3_changes64(db) != 1) throw SqlError("inode not found", SQLITE_NOTFOUND); + CommitMetadataChange(db, workspace, inode, "chown"); + savepoint.Release(); + sqlite3_result_int64(context, uid); + }); +} + +void OwnerSetFunction(sqlite3_context *context, int, sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + const Workspace workspace = FindWorkspace(db, RequiredText(values[0], "workspace")); + const std::string path = RequiredText(values[1], "path"); + const std::string owner = RequiredText(values[2], "owner principal"); + if (owner.empty() || owner.size() > 255) throw SqlError("owner principal must be 1..255 bytes", SQLITE_RANGE); + const Node node = Resolve(db, workspace, PathParts(path)); + Savepoint savepoint(db, "vexfs_owner_set"); + AcquireWriteLock(db); + Statement update(db, + "UPDATE _vexfs_inodes SET owner_principal=?1,changed_at=CAST(unixepoch('subsec')*1000 AS INTEGER) " + "WHERE workspace_id=?2 AND id=?3"); + update.BindText(1, owner); + update.BindInt64(2, workspace.id); + update.BindInt64(3, node.id); + update.Done(); + if (sqlite3_changes64(db) != 1) throw SqlError("inode not found", SQLITE_NOTFOUND); + CommitMetadataChange(db, workspace, node.id, "set owner"); + savepoint.Release(); + sqlite3_result_text(context, owner.data(), static_cast(owner.size()), SQLITE_TRANSIENT); + }); +} + +void GetXattrFunction(sqlite3_context *context, int, sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + const Workspace workspace = FindWorkspace(db, RequiredText(values[0], "workspace")); + const sqlite3_int64 inode = RequiredPositiveInteger(values[1], "inode"); + const std::string name = RequiredText(values[2], "xattr name"); + if (name.empty() || name.size() > 255) + throw SqlError("xattr name must be 1..255 bytes", SQLITE_RANGE); + RequireWorkspaceInode(db, workspace, inode); + Statement statement(db, + "SELECT value FROM _vexfs_xattrs WHERE inode_id=?1 AND name=?2"); + statement.BindInt64(1, inode); + statement.BindText(2, name); + if (!statement.Row()) throw SqlError("xattr not found", SQLITE_NOTFOUND); + const std::vector value = statement.Blob(0); + sqlite3_result_blob64(context, value.empty() ? static_cast("") : value.data(), + value.size(), SQLITE_TRANSIENT); + }); +} + +void ListXattrsFunction(sqlite3_context *context, int, sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + const Workspace workspace = FindWorkspace(db, RequiredText(values[0], "workspace")); + const sqlite3_int64 inode = RequiredPositiveInteger(values[1], "inode"); + RequireWorkspaceInode(db, workspace, inode); + Statement statement(db, + "SELECT name FROM _vexfs_xattrs WHERE inode_id=?1 ORDER BY name COLLATE BINARY"); + statement.BindInt64(1, inode); + std::string json = "["; + bool first = true; + while (statement.Row()) { + if (!first) json += ','; + first = false; + json += "\"" + JsonEscape(statement.Text(0)) + "\""; + } + json += ']'; + sqlite3_result_text(context, json.data(), static_cast(json.size()), SQLITE_TRANSIENT); + }); +} + +void SetXattrFunction(sqlite3_context *context, int, sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + const Workspace workspace = FindWorkspace(db, RequiredText(values[0], "workspace")); + const sqlite3_int64 inode = RequiredPositiveInteger(values[1], "inode"); + const std::string name = RequiredText(values[2], "xattr name"); + const int policy = sqlite3_value_int(values[4]); + if (name.empty() || name.size() > 255) + throw SqlError("xattr name must be 1..255 bytes", SQLITE_RANGE); + if (policy < 0 || policy > 3) + throw SqlError("xattr policy must be 0..3", SQLITE_RANGE); + RequireWorkspaceInode(db, workspace, inode); + + Statement exists(db, + "SELECT 1 FROM _vexfs_xattrs WHERE inode_id=?1 AND name=?2"); + exists.BindInt64(1, inode); + exists.BindText(2, name); + const bool found = exists.Row(); + if (policy == 1 && found) throw SqlError("xattr already exists", SQLITE_CONSTRAINT); + if ((policy == 2 || policy == 3) && !found) + throw SqlError("xattr not found", SQLITE_NOTFOUND); + + Savepoint savepoint(db, "vexfs_set_xattr"); + if (policy == 3) { + Statement remove(db, + "DELETE FROM _vexfs_xattrs WHERE inode_id=?1 AND name=?2"); + remove.BindInt64(1, inode); + remove.BindText(2, name); + remove.Done(); + } else { + const std::vector value = RequiredBlob(values[3], "xattr value"); + if (static_cast(value.size()) > kMaxXattrBytes) + throw SqlError("xattr is larger than 64 KiB", SQLITE_TOOBIG); + Statement store(db, + "INSERT INTO _vexfs_xattrs(inode_id,name,value) VALUES(?1,?2,?3) " + "ON CONFLICT(inode_id,name) DO UPDATE SET value=excluded.value, " + "updated_at=CAST(strftime('%s','now') AS INTEGER)*1000"); + store.BindInt64(1, inode); + store.BindText(2, name); + store.BindBlob(3, value); + store.Done(); + } + Statement touch(db, + "UPDATE _vexfs_inodes SET changed_at=CAST(unixepoch('subsec')*1000 AS INTEGER) " + "WHERE id=?1 AND workspace_id=?2"); + touch.BindInt64(1, inode); + touch.BindInt64(2, workspace.id); + touch.Done(); + CommitMetadataChange(db, workspace, inode, + policy == 3 ? "remove xattr" : "set xattr"); + savepoint.Release(); + sqlite3_result_int(context, 1); + }); +} + +struct SnapshotInfo { + sqlite3_int64 id = 0; + std::string name; + sqlite3_int64 commit = 0; + sqlite3_int64 created_at = 0; +}; + +struct SnapshotTreeNode { + sqlite3_int64 inode = 0; + std::string kind; + int mode = 0; + std::string owner_principal = "local"; + sqlite3_int64 uid = 0; + sqlite3_int64 gid = 0; + sqlite3_int64 size = 0; + sqlite3_int64 version = 0; + sqlite3_int64 canonical_version = 0; + sqlite3_int64 created_at = 0; + sqlite3_int64 accessed_at = 0; + sqlite3_int64 updated_at = 0; + sqlite3_int64 changed_at = 0; + std::vector acl; + std::map> xattrs; +}; + +using SnapshotTree = std::map; + +void ResolveSnapshotCanonicalVersions(sqlite3 *db, SnapshotTree &tree) { + std::string requested = "["; + bool first = true; + size_t expected = 0; + std::map, bool> unique; + for (const auto &[path, node] : tree) { + (void)path; + if ((node.kind != "file" && node.kind != "symlink") || node.version <= 0) continue; + if (!unique.emplace(std::make_pair(node.inode, node.version), true).second) continue; + if (!first) requested += ','; + first = false; + ++expected; + requested += "{\"inode\":" + std::to_string(node.inode) + + ",\"version\":" + std::to_string(node.version) + "}"; + } + requested += ']'; + if (expected == 0) return; + + Statement statement(db, R"SQL( +WITH RECURSIVE +requested(inode_id,requested_version) AS ( + SELECT CAST(json_extract(value,'$.inode') AS INTEGER), + CAST(json_extract(value,'$.version') AS INTEGER) + FROM json_each(?1) +), +chain(inode_id,requested_version,version_no,source_version_no,depth) AS ( + SELECT r.inode_id,r.requested_version,v.version_no,v.source_version_no,0 + FROM requested r JOIN _vexfs_file_versions v + ON v.inode_id=r.inode_id AND v.version_no=r.requested_version + UNION ALL + SELECT c.inode_id,c.requested_version,v.version_no,v.source_version_no,c.depth+1 + FROM chain c JOIN _vexfs_file_versions v + ON v.inode_id=c.inode_id AND v.version_no=c.source_version_no + WHERE c.source_version_no IS NOT NULL AND c.depth<1024 +) +SELECT inode_id,requested_version,version_no FROM chain +WHERE source_version_no IS NULL +)SQL"); + statement.BindText(1, requested); + std::map, sqlite3_int64> resolved; + while (statement.Row()) { + resolved[{statement.Int64(0), statement.Int64(1)}] = statement.Int64(2); + } + if (resolved.size() != expected) { + throw SqlError("workspace history references a missing or cyclic file version", + SQLITE_CORRUPT); + } + for (auto &[path, node] : tree) { + (void)path; + if ((node.kind == "file" || node.kind == "symlink") && node.version > 0) { + node.canonical_version = resolved.at({node.inode, node.version}); + } else { + node.canonical_version = node.version; + } + } +} + +sqlite3_int64 CurrentHead(sqlite3 *db, const Workspace &workspace) { + Statement statement(db, + "SELECT COALESCE(head_commit,0) FROM _vexfs_workspaces WHERE id=?1"); + statement.BindInt64(1, workspace.id); + if (!statement.Row()) throw SqlError("workspace not found", SQLITE_NOTFOUND); + return statement.Int64(0); +} + +void ValidateHistoryCommit(sqlite3 *db, const Workspace &workspace, sqlite3_int64 commit) { + Statement statement(db, + "SELECT COALESCE(history_floor_commit,0)," + "EXISTS(SELECT 1 FROM _vexfs_commits WHERE id=?2 AND workspace_id=?1) " + "FROM _vexfs_workspaces WHERE id=?1"); + statement.BindInt64(1, workspace.id); + statement.BindInt64(2, commit); + if (!statement.Row() || statement.Int(1) != 1) { + throw SqlError("workspace commit not found: " + std::to_string(commit), SQLITE_NOTFOUND); + } + const sqlite3_int64 floor = statement.Int64(0); + if (floor == 0 || commit < floor) { + throw SqlError("workspace history before commit " + std::to_string(floor) + + " is not restorable", SQLITE_NOTFOUND); + } +} + +SnapshotInfo FindSnapshot(sqlite3 *db, const Workspace &workspace, const std::string &name) { + Statement statement(db, + "SELECT id,name,commit_id,created_at FROM _vexfs_snapshots " + "WHERE workspace_id=?1 AND name=?2"); + statement.BindInt64(1, workspace.id); + statement.BindText(2, name); + if (!statement.Row()) throw SqlError("snapshot not found: " + name, SQLITE_NOTFOUND); + SnapshotInfo result{statement.Int64(0), statement.Text(1), statement.Int64(2), + statement.Int64(3)}; + ValidateHistoryCommit(db, workspace, result.commit); + return result; +} + +sqlite3_int64 ResolveSnapshotReference(sqlite3 *db, const Workspace &workspace, + const std::string &reference) { + if (reference == "HEAD") { + const sqlite3_int64 head = CurrentHead(db, workspace); + ValidateHistoryCommit(db, workspace, head); + return head; + } + return FindSnapshot(db, workspace, reference).commit; +} + +SnapshotTree CurrentTree(sqlite3 *db, const Workspace &workspace) { + Statement statement(db, R"SQL( +WITH RECURSIVE tree(path,inode_id,depth) AS ( + SELECT '/',?2,0 + UNION ALL + SELECT CASE WHEN tree.path='/' THEN '/'||d.name ELSE tree.path||'/'||d.name END, + d.inode_id,tree.depth+1 + FROM tree JOIN _vexfs_dentries d + ON d.workspace_id=?1 AND d.parent_inode=tree.inode_id + JOIN _vexfs_inodes child ON child.id=d.inode_id AND child.deleted_at IS NULL + WHERE tree.depth<1024 +) +SELECT tree.path,i.id,i.kind,i.mode,i.owner_principal,i.uid,i.gid, + i.size,i.current_version,i.created_at,i.accessed_at,i.updated_at,i.changed_at +FROM tree JOIN _vexfs_inodes i ON i.id=tree.inode_id AND i.deleted_at IS NULL +ORDER BY tree.path COLLATE BINARY +)SQL"); + statement.BindInt64(1, workspace.id); + statement.BindInt64(2, workspace.root_inode); + SnapshotTree tree; + std::map> paths_by_inode; + while (statement.Row()) { + const std::string path = statement.Text(0); + SnapshotTreeNode node{statement.Int64(1), statement.Text(2), statement.Int(3), + statement.Text(4), statement.Int64(5), statement.Int64(6), + statement.Int64(7), statement.Int64(8), 0, + statement.Int64(9), statement.Int64(10), + statement.Int64(11), statement.Int64(12), {}, {}}; + tree.emplace(path, std::move(node)); + paths_by_inode[statement.Int64(1)].push_back(path); + } + if (tree.find("/") == tree.end()) { + throw SqlError("workspace root is missing", SQLITE_CORRUPT); + } + // A regular file (or symlink) may have more than one dentry when the + // workspace contains a hard link. Directories may not: a repeated + // directory inode would make the path graph cyclic/ambiguous and would + // make a snapshot impossible to restore safely. Keep this check after + // materialising the CTE so it also catches cycles in a manually-corrupted + // current tree (the depth guard in the CTE merely truncates those cycles). + for (const auto &[inode, paths] : paths_by_inode) { + const auto first = tree.find(paths.front()); + if (first != tree.end() && first->second.kind == "directory" && paths.size() > 1) { + throw SqlError("workspace contains a directory cycle or duplicate inode", + SQLITE_CORRUPT); + } + (void)inode; + } + ResolveSnapshotCanonicalVersions(db, tree); + Statement xattrs(db, R"SQL( +SELECT x.inode_id,x.name,x.value FROM _vexfs_xattrs x +JOIN _vexfs_inodes i ON i.id=x.inode_id +WHERE i.workspace_id=?1 AND i.deleted_at IS NULL +ORDER BY x.inode_id,x.name COLLATE BINARY +)SQL"); + xattrs.BindInt64(1, workspace.id); + while (xattrs.Row()) { + const auto paths = paths_by_inode.find(xattrs.Int64(0)); + if (paths == paths_by_inode.end()) continue; + for (const std::string &path : paths->second) { + tree[path].xattrs.emplace(xattrs.Text(1), xattrs.Blob(2)); + } + } + Statement acl(db, + "SELECT inode_id,principal_id,effect,permissions,inherit_flags FROM _vexfs_acl_entries " + "WHERE workspace_id=?1 ORDER BY inode_id,principal_id COLLATE BINARY,effect"); + acl.BindInt64(1, workspace.id); + while (acl.Row()) { + const auto paths = paths_by_inode.find(acl.Int64(0)); + if (paths == paths_by_inode.end()) continue; + const AclEntry entry{acl.Text(1), acl.Text(2), acl.Text(3), acl.Int64(4)}; + for (const std::string &path : paths->second) tree[path].acl.push_back(entry); + } + return tree; +} + +SnapshotTree TreeAtCommit(sqlite3 *db, const Workspace &workspace, sqlite3_int64 commit) { + ValidateHistoryCommit(db, workspace, commit); + Statement inode_statement(db, R"SQL( +WITH +ranked_inode AS MATERIALIZED ( + SELECT s.*,ROW_NUMBER() OVER( + PARTITION BY s.inode_id ORDER BY s.commit_id DESC) AS vexfs_rank + FROM _vexfs_inode_states s WHERE s.workspace_id=?1 AND s.commit_id<=?2 +) +SELECT inode_id,kind,mode,owner_principal,uid,gid,size,current_version, + created_at,accessed_at,updated_at,changed_at +FROM ranked_inode WHERE vexfs_rank=1 AND deleted_at IS NULL ORDER BY inode_id +)SQL"); + inode_statement.BindInt64(1, workspace.id); + inode_statement.BindInt64(2, commit); + std::map nodes; + while (inode_statement.Row()) { + SnapshotTreeNode node{inode_statement.Int64(0), inode_statement.Text(1), + inode_statement.Int(2), inode_statement.Text(3), + inode_statement.Int64(4), inode_statement.Int64(5), + inode_statement.Int64(6), inode_statement.Int64(7), 0, + inode_statement.Int64(8), inode_statement.Int64(9), + inode_statement.Int64(10), inode_statement.Int64(11), {}, {}}; + nodes.emplace(node.inode, std::move(node)); + } + if (nodes.find(workspace.root_inode) == nodes.end()) { + throw SqlError("workspace root is missing from history", SQLITE_CORRUPT); + } + + Statement dentry_statement(db, R"SQL( +WITH +ranked_dentry AS MATERIALIZED ( + SELECT s.*,ROW_NUMBER() OVER( + PARTITION BY s.parent_inode,s.name ORDER BY s.commit_id DESC) AS vexfs_rank + FROM _vexfs_dentry_states s WHERE s.workspace_id=?1 AND s.commit_id<=?2 +) +SELECT parent_inode,name,inode_id FROM ranked_dentry +WHERE vexfs_rank=1 AND deleted=0 ORDER BY parent_inode,name COLLATE BINARY +)SQL"); + dentry_statement.BindInt64(1, workspace.id); + dentry_statement.BindInt64(2, commit); + std::map>> children; + while (dentry_statement.Row()) { + children[dentry_statement.Int64(0)].emplace_back( + dentry_statement.Text(1), dentry_statement.Int64(2)); + } + + struct PendingNode { + sqlite3_int64 inode; + std::string path; + int depth; + }; + std::vector pending{{workspace.root_inode, "/", 0}}; + // Files and symlinks can legitimately be reached through several + // dentries (hard links). Only directory inodes participate in traversal, + // so only they need a visited set. Treating every inode as globally + // visited incorrectly rejected a valid snapshot containing two hard links + // to the same file. + std::map visited_directories; + SnapshotTree tree; + std::map> paths_by_inode; + for (size_t index = 0; index < pending.size(); ++index) { + const PendingNode current = pending[index]; + const auto node = nodes.find(current.inode); + if (node == nodes.end()) continue; + if (node->second.kind == "directory" && + !visited_directories.emplace(current.inode, true).second) { + throw SqlError("workspace history contains a directory cycle or duplicate inode", + SQLITE_CORRUPT); + } + tree.emplace(current.path, node->second); + paths_by_inode[current.inode].push_back(current.path); + if (node->second.kind != "directory") continue; + const auto child_entries = children.find(current.inode); + if (child_entries == children.end()) continue; + if (current.depth >= 1024) { + throw SqlError("workspace history exceeds 1024 path components", SQLITE_TOOBIG); + } + for (const auto &[name, child_inode] : child_entries->second) { + if (nodes.find(child_inode) == nodes.end()) continue; + const std::string child_path = current.path == "/" ? "/" + name : + current.path + "/" + name; + pending.push_back({child_inode, child_path, current.depth + 1}); + } + } + ResolveSnapshotCanonicalVersions(db, tree); + + Statement xattrs(db, R"SQL( +WITH ranked AS MATERIALIZED ( + SELECT s.*,ROW_NUMBER() OVER( + PARTITION BY s.inode_id,s.name ORDER BY s.commit_id DESC) AS vexfs_rank + FROM _vexfs_xattr_states s WHERE s.workspace_id=?1 AND s.commit_id<=?2) +SELECT inode_id,name,value FROM ranked +WHERE vexfs_rank=1 AND deleted=0 ORDER BY inode_id,name COLLATE BINARY +)SQL"); + xattrs.BindInt64(1, workspace.id); + xattrs.BindInt64(2, commit); + while (xattrs.Row()) { + const auto paths = paths_by_inode.find(xattrs.Int64(0)); + if (paths == paths_by_inode.end()) continue; + for (const std::string &path : paths->second) { + tree[path].xattrs.emplace(xattrs.Text(1), xattrs.Blob(2)); + } + } + Statement acl(db, R"SQL( +WITH ranked AS MATERIALIZED ( + SELECT s.*,ROW_NUMBER() OVER( + PARTITION BY s.inode_id,s.principal_id,s.effect ORDER BY s.commit_id DESC) AS vexfs_rank + FROM _vexfs_acl_states s WHERE s.workspace_id=?1 AND s.commit_id<=?2) +SELECT inode_id,principal_id,effect,permissions,inherit_flags FROM ranked +WHERE vexfs_rank=1 AND deleted=0 +ORDER BY inode_id,principal_id COLLATE BINARY,effect +)SQL"); + acl.BindInt64(1, workspace.id); + acl.BindInt64(2, commit); + while (acl.Row()) { + const auto paths = paths_by_inode.find(acl.Int64(0)); + if (paths == paths_by_inode.end()) continue; + const AclEntry entry{acl.Text(1), acl.Text(2), acl.Text(3), acl.Int64(4)}; + for (const std::string &path : paths->second) tree[path].acl.push_back(entry); + } + return tree; +} + +bool SnapshotNodesEqual(const SnapshotTreeNode &left, const SnapshotTreeNode &right) { + return left.inode == right.inode && left.kind == right.kind && left.mode == right.mode && + left.size == right.size && left.canonical_version == right.canonical_version && + left.owner_principal == right.owner_principal && left.uid == right.uid && + left.gid == right.gid && left.created_at == right.created_at && + left.accessed_at == right.accessed_at && left.updated_at == right.updated_at && + left.changed_at == right.changed_at && left.acl == right.acl && + left.xattrs == right.xattrs; +} + +std::string SnapshotNodeJson(const SnapshotTreeNode &node) { + std::string xattrs = "["; + bool first = true; + for (const auto &entry : node.xattrs) { + if (!first) xattrs += ','; + first = false; + xattrs += "\"" + JsonEscape(entry.first) + "\""; + } + xattrs += ']'; + std::string acl = "["; + first = true; + for (const AclEntry &entry : node.acl) { + if (!first) acl += ','; + first = false; + acl += "{\"principal\":\"" + JsonEscape(entry.principal) + + "\",\"effect\":\"" + JsonEscape(entry.effect) + + "\",\"permissions\":\"" + JsonEscape(entry.permissions) + + "\",\"inherit\":" + std::to_string(entry.inherit_flags) + "}"; + } + acl += ']'; + return "{\"inode\":" + std::to_string(node.inode) + + ",\"kind\":\"" + JsonEscape(node.kind) + "\",\"mode\":" + + std::to_string(node.mode) + ",\"owner_principal\":\"" + + JsonEscape(node.owner_principal) + "\",\"uid\":" + std::to_string(node.uid) + + ",\"gid\":" + std::to_string(node.gid) + ",\"size\":" + std::to_string(node.size) + + ",\"version\":" + std::to_string(node.version) + + ",\"content_version\":" + std::to_string(node.canonical_version) + + ",\"created_at\":" + std::to_string(node.created_at) + + ",\"accessed_at\":" + std::to_string(node.accessed_at) + + ",\"updated_at\":" + std::to_string(node.updated_at) + + ",\"changed_at\":" + std::to_string(node.changed_at) + + ",\"acl\":" + acl + ",\"xattrs\":" + xattrs + "}"; +} + +std::string SnapshotTreeJson(const SnapshotTree &tree) { + std::string json = "["; + bool first = true; + for (const auto &[path, node] : tree) { + if (!first) json += ','; + first = false; + json += "{\"path\":\"" + JsonEscape(path) + "\",\"state\":" + + SnapshotNodeJson(node) + "}"; + } + json += ']'; + return json; +} + +std::string SnapshotDiffJson(const SnapshotTree &before, const SnapshotTree &after) { + std::string json = "["; + bool first = true; + auto append = [&](const std::string &path, const char *change, + const SnapshotTreeNode *old_node, const SnapshotTreeNode *new_node) { + if (!first) json += ','; + first = false; + json += "{\"path\":\"" + JsonEscape(path) + "\",\"change\":\"" + change + + "\",\"before\":" + (old_node == nullptr ? "null" : SnapshotNodeJson(*old_node)) + + ",\"after\":" + (new_node == nullptr ? "null" : SnapshotNodeJson(*new_node)) + "}"; + }; + auto left = before.begin(); + auto right = after.begin(); + while (left != before.end() || right != after.end()) { + if (right == after.end() || (left != before.end() && left->first < right->first)) { + append(left->first, "delete", &left->second, nullptr); + ++left; + } else if (left == before.end() || right->first < left->first) { + append(right->first, "add", nullptr, &right->second); + ++right; + } else { + if (!SnapshotNodesEqual(left->second, right->second)) { + append(left->first, "modify", &left->second, &right->second); + } + ++left; + ++right; + } + } + json += ']'; + return json; +} + +void SnapshotCreateFunction(sqlite3_context *context, int argument_count, + sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + const Workspace workspace = FindWorkspace(db, RequiredText(values[0], "workspace")); + const std::string name = RequiredText(values[1], "snapshot name"); + if (name.empty() || name.size() > 128 || name == "HEAD") { + throw SqlError("snapshot name must be 1..128 bytes and not HEAD", SQLITE_MISMATCH); + } + const std::string mode = argument_count == 4 + ? RequiredText(values[3], "snapshot mode") : "consistent"; + if (mode != "consistent" && mode != "committed-only") { + throw SqlError("snapshot mode must be consistent or committed-only", SQLITE_MISMATCH); + } + Savepoint savepoint(db, "vexfs_snapshot_create"); + AcquireWriteLock(db); + const sqlite3_int64 head = CurrentHead(db, workspace); + if (argument_count >= 3 && sqlite3_value_type(values[2]) != SQLITE_NULL) { + const sqlite3_int64 expected = RequiredPositiveInteger(values[2], "expected head"); + if (expected != head) throw SqlError("workspace head conflict", SQLITE_CONSTRAINT); + } + if (mode == "consistent") { + Statement dirty(db, + "SELECT count(*) FROM _vexfs_handles WHERE workspace_id=?1 " + "AND state IN ('open','retained') " + "AND dirty_generation>published_generation"); + dirty.BindInt64(1, workspace.id); + dirty.Row(); + if (dirty.Int64(0) != 0) { + throw SqlError( + "workspace has unpublished file handles; synchronize the owning mount " + "or use committed-only", SQLITE_BUSY); + } + } + ValidateHistoryCommit(db, workspace, head); + Statement insert(db, + "INSERT INTO _vexfs_snapshots(workspace_id,name,commit_id) VALUES(?1,?2,?3)"); + insert.BindInt64(1, workspace.id); + insert.BindText(2, name); + insert.BindInt64(3, head); + insert.Done(); + savepoint.Release(); + sqlite3_result_int64(context, head); + }); +} + +void SnapshotListFunction(sqlite3_context *context, int, sqlite3_value **values) { Guard(context, [&] { sqlite3 *db = sqlite3_context_db_handle(context); EnsureSchema(db); const Workspace workspace = FindWorkspace(db, RequiredText(values[0], "workspace")); - const Node node = Resolve(db, workspace, PathParts(RequiredText(values[1], "path"))); - if (node.kind != "file") throw SqlError("path is not a file", SQLITE_MISMATCH); - const sqlite3_int64 limit = argument_count == 2 - ? 100 : RequiredPositiveInteger(values[2], "limit"); - const sqlite3_int64 before_version = argument_count == 2 - ? 0 : RequiredNonnegativeInteger(values[3], "before version"); - if (limit > 1000) throw SqlError("history limit must be at most 1000", SQLITE_RANGE); - Statement statement(db, - "SELECT v.version_no, v.commit_id, c.parent_commit, v.size, v.created_at, c.message " - "FROM _vexfs_file_versions v JOIN _vexfs_commits c ON c.id=v.commit_id " - "WHERE v.inode_id=?1 AND (?2=0 OR v.version_no(json.size()), SQLITE_TRANSIENT); }); } -void ReadVersionFunction(sqlite3_context *context, int, sqlite3_value **values) { +void SnapshotShowFunction(sqlite3_context *context, int, sqlite3_value **values) { Guard(context, [&] { sqlite3 *db = sqlite3_context_db_handle(context); EnsureSchema(db); const Workspace workspace = FindWorkspace(db, RequiredText(values[0], "workspace")); - const Node node = Resolve(db, workspace, PathParts(RequiredText(values[1], "path"))); - ResultBlob(context, ReadNodeVersion(db, node, RequiredPositiveInteger(values[2], "version"))); + const SnapshotInfo snapshot = FindSnapshot( + db, workspace, RequiredText(values[1], "snapshot name")); + const std::string json = "{\"name\":\"" + JsonEscape(snapshot.name) + + "\",\"commit\":" + std::to_string(snapshot.commit) + + ",\"created_at\":" + std::to_string(snapshot.created_at) + + ",\"entries\":" + SnapshotTreeJson(TreeAtCommit(db, workspace, snapshot.commit)) + "}"; + sqlite3_result_text(context, json.data(), static_cast(json.size()), SQLITE_TRANSIENT); }); } -void CompareVersionsFunction(sqlite3_context *context, int, sqlite3_value **values) { +void SnapshotDiffFunction(sqlite3_context *context, int, sqlite3_value **values) { Guard(context, [&] { sqlite3 *db = sqlite3_context_db_handle(context); EnsureSchema(db); - const std::string path = RequiredText(values[1], "path"); const Workspace workspace = FindWorkspace(db, RequiredText(values[0], "workspace")); - const Node node = Resolve(db, workspace, PathParts(path)); - if (node.kind != "file") throw SqlError("path is not a file", SQLITE_MISMATCH); - const sqlite3_int64 from_version = RequiredPositiveInteger(values[2], "from version"); - const sqlite3_int64 to_version = RequiredPositiveInteger(values[3], "to version"); - const VersionStorage from = ResolveVersionStorage(db, node.id, from_version); - const VersionStorage to = ResolveVersionStorage(db, node.id, to_version); - ReadOnlyBlob from_blob(db, from.rowid); - ReadOnlyBlob to_blob(db, to.rowid); - if (from_blob.size() != from.size || to_blob.size() != to.size) { - throw SqlError("file version size does not match content", SQLITE_CORRUPT); - } - std::array from_chunk{}; - std::array to_chunk{}; - bool changed = from.size != to.size; - bool binary = false; - const sqlite3_int64 maximum = std::max(from.size, to.size); - for (sqlite3_int64 offset = 0; offset < maximum; offset += from_chunk.size()) { - const int from_bytes = static_cast(std::min( - from_chunk.size(), std::max(0, from.size - offset))); - const int to_bytes = static_cast(std::min( - to_chunk.size(), std::max(0, to.size - offset))); - if (from_bytes > 0) from_blob.Read(from_chunk.data(), from_bytes, static_cast(offset)); - if (to_bytes > 0) to_blob.Read(to_chunk.data(), to_bytes, static_cast(offset)); - binary = binary || - std::find(from_chunk.begin(), from_chunk.begin() + from_bytes, 0) != - from_chunk.begin() + from_bytes || - std::find(to_chunk.begin(), to_chunk.begin() + to_bytes, 0) != - to_chunk.begin() + to_bytes; - if (!changed && (from_bytes != to_bytes || - !std::equal(from_chunk.begin(), from_chunk.begin() + from_bytes, - to_chunk.begin()))) changed = true; - } - const std::string json = - "{\"path\":\"" + JsonEscape(path) + "\",\"from\":" + - std::to_string(from_version) + ",\"to\":" + std::to_string(to_version) + - ",\"changed\":" + (changed ? "true" : "false") + - ",\"binary\":" + (binary ? "true" : "false") + - ",\"from_size\":" + std::to_string(from.size) + - ",\"to_size\":" + std::to_string(to.size) + "}"; + const std::string from = RequiredText(values[1], "from snapshot"); + const std::string to = RequiredText(values[2], "to snapshot"); + const sqlite3_int64 from_commit = ResolveSnapshotReference(db, workspace, from); + const sqlite3_int64 to_commit = ResolveSnapshotReference(db, workspace, to); + const SnapshotTree from_tree = from == "HEAD" ? CurrentTree(db, workspace) : + TreeAtCommit(db, workspace, from_commit); + const SnapshotTree to_tree = to == "HEAD" ? CurrentTree(db, workspace) : + TreeAtCommit(db, workspace, to_commit); + const std::string changes = SnapshotDiffJson(from_tree, to_tree); + const std::string json = "{\"from\":\"" + JsonEscape(from) + + "\",\"from_commit\":" + std::to_string(from_commit) + + ",\"to\":\"" + JsonEscape(to) + "\",\"to_commit\":" + + std::to_string(to_commit) + ",\"changes\":" + changes + "}"; sqlite3_result_text(context, json.data(), static_cast(json.size()), SQLITE_TRANSIENT); }); } -void RestoreVersionFunction(sqlite3_context *context, int, sqlite3_value **values) { +void SnapshotDropFunction(sqlite3_context *context, int, sqlite3_value **values) { Guard(context, [&] { sqlite3 *db = sqlite3_context_db_handle(context); EnsureSchema(db); const Workspace workspace = FindWorkspace(db, RequiredText(values[0], "workspace")); - const Node node = Resolve(db, workspace, PathParts(RequiredText(values[1], "path"))); - const sqlite3_int64 target_version = RequiredPositiveInteger(values[2], "version"); - const sqlite3_int64 expected_version = RequiredPositiveInteger(values[3], "expected version"); - if (node.version != expected_version) { - throw SqlError("version conflict: expected " + std::to_string(expected_version) + - ", current " + std::to_string(node.version), SQLITE_CONSTRAINT); - } - if (target_version == node.version) { - throw SqlError("target version is already current", SQLITE_MISMATCH); + const std::string name = RequiredText(values[1], "snapshot name"); + Statement remove(db, + "DELETE FROM _vexfs_snapshots WHERE workspace_id=?1 AND name=?2"); + remove.BindInt64(1, workspace.id); + remove.BindText(2, name); + remove.Done(); + if (sqlite3_changes64(db) != 1) { + throw SqlError("snapshot not found: " + name, SQLITE_NOTFOUND); } - const sqlite3_int64 version = StoreVersionFromVersion( - db, workspace, node, target_version, - "restore version " + std::to_string(target_version)); - sqlite3_result_int64(context, version); + sqlite3_result_int(context, 1); }); } -void StatFunction(sqlite3_context *context, int, sqlite3_value **values) { - Guard(context, [&] { - sqlite3 *db = sqlite3_context_db_handle(context); - EnsureSchema(db); - const std::string path = RequiredText(values[1], "path"); - const Workspace workspace = FindWorkspace(db, RequiredText(values[0], "workspace")); - const Node node = Resolve(db, workspace, PathParts(path)); - const std::string json = "{\"path\":\"" + JsonEscape(path) + "\",\"inode\":" + - std::to_string(node.id) + ",\"kind\":\"" + node.kind + "\",\"mode\":" + - std::to_string(node.mode) + ",\"size\":" + std::to_string(node.size) + - ",\"version\":" + std::to_string(node.version) + ",\"created_at\":" + - std::to_string(node.created_at) + ",\"updated_at\":" + - std::to_string(node.updated_at) + "}"; - sqlite3_result_text(context, json.data(), static_cast(json.size()), SQLITE_TRANSIENT); - }); +void RestoreWorkspaceTree(sqlite3 *db, const Workspace &workspace, sqlite3_int64 commit) { + Statement clear_acl(db, + "DELETE FROM _vexfs_acl_entries WHERE workspace_id=?1"); + clear_acl.BindInt64(1, workspace.id); + clear_acl.Done(); + Statement clear_xattrs(db, R"SQL( +DELETE FROM _vexfs_xattrs WHERE inode_id IN ( + SELECT id FROM _vexfs_inodes WHERE workspace_id=?1) +)SQL"); + clear_xattrs.BindInt64(1, workspace.id); + clear_xattrs.Done(); + Statement clear_dentries(db, "DELETE FROM _vexfs_dentries WHERE workspace_id=?1"); + clear_dentries.BindInt64(1, workspace.id); + clear_dentries.Done(); + Statement tombstone(db, + "UPDATE _vexfs_inodes SET deleted_at=CAST(strftime('%s','now') AS INTEGER)*1000 " + "WHERE workspace_id=?1"); + tombstone.BindInt64(1, workspace.id); + tombstone.Done(); + + Statement restore_inodes(db, R"SQL( +WITH ranked AS MATERIALIZED ( + SELECT s.*,ROW_NUMBER() OVER( + PARTITION BY s.inode_id ORDER BY s.commit_id DESC) AS vexfs_rank + FROM _vexfs_inode_states s WHERE s.workspace_id=?1 AND s.commit_id<=?2), +latest AS MATERIALIZED (SELECT * FROM ranked WHERE vexfs_rank=1) +UPDATE _vexfs_inodes AS inode SET + kind=(SELECT kind FROM latest WHERE inode_id=inode.id), + mode=(SELECT mode FROM latest WHERE inode_id=inode.id), + owner_principal=(SELECT owner_principal FROM latest WHERE inode_id=inode.id), + uid=(SELECT uid FROM latest WHERE inode_id=inode.id), + gid=(SELECT gid FROM latest WHERE inode_id=inode.id), + size=(SELECT size FROM latest WHERE inode_id=inode.id), + current_version=(SELECT current_version FROM latest WHERE inode_id=inode.id), + created_at=(SELECT created_at FROM latest WHERE inode_id=inode.id), + accessed_at=(SELECT accessed_at FROM latest WHERE inode_id=inode.id), + updated_at=(SELECT updated_at FROM latest WHERE inode_id=inode.id), + changed_at=(SELECT changed_at FROM latest WHERE inode_id=inode.id), + deleted_at=NULL +WHERE inode.workspace_id=?1 AND EXISTS( + SELECT 1 FROM latest WHERE inode_id=inode.id AND deleted_at IS NULL) +)SQL"); + restore_inodes.BindInt64(1, workspace.id); + restore_inodes.BindInt64(2, commit); + restore_inodes.Done(); + + Statement restore_dentries(db, R"SQL( +WITH +ranked_inode AS MATERIALIZED ( + SELECT s.*,ROW_NUMBER() OVER( + PARTITION BY s.inode_id ORDER BY s.commit_id DESC) AS vexfs_rank + FROM _vexfs_inode_states s WHERE s.workspace_id=?1 AND s.commit_id<=?2), +latest_inode AS MATERIALIZED (SELECT * FROM ranked_inode WHERE vexfs_rank=1), +active_inode AS MATERIALIZED (SELECT * FROM latest_inode WHERE deleted_at IS NULL), +ranked_dentry AS MATERIALIZED ( + SELECT s.*,ROW_NUMBER() OVER( + PARTITION BY s.parent_inode,s.name ORDER BY s.commit_id DESC) AS vexfs_rank + FROM _vexfs_dentry_states s WHERE s.workspace_id=?1 AND s.commit_id<=?2), +latest_dentry AS MATERIALIZED (SELECT * FROM ranked_dentry WHERE vexfs_rank=1) +INSERT INTO _vexfs_dentries(workspace_id,parent_inode,name,inode_id) +SELECT d.workspace_id,d.parent_inode,d.name,d.inode_id FROM latest_dentry d +JOIN active_inode child ON child.inode_id=d.inode_id +JOIN active_inode parent ON parent.inode_id=d.parent_inode +WHERE d.deleted=0 +)SQL"); + restore_dentries.BindInt64(1, workspace.id); + restore_dentries.BindInt64(2, commit); + restore_dentries.Done(); + + Statement restore_xattrs(db, R"SQL( +WITH +ranked_inode AS MATERIALIZED ( + SELECT s.*,ROW_NUMBER() OVER( + PARTITION BY s.inode_id ORDER BY s.commit_id DESC) AS vexfs_rank + FROM _vexfs_inode_states s WHERE s.workspace_id=?1 AND s.commit_id<=?2), +latest_inode AS MATERIALIZED (SELECT * FROM ranked_inode WHERE vexfs_rank=1), +active_inode AS MATERIALIZED (SELECT * FROM latest_inode WHERE deleted_at IS NULL), +ranked_xattr AS MATERIALIZED ( + SELECT s.*,ROW_NUMBER() OVER( + PARTITION BY s.inode_id,s.name ORDER BY s.commit_id DESC) AS vexfs_rank + FROM _vexfs_xattr_states s WHERE s.workspace_id=?1 AND s.commit_id<=?2), +latest_xattr AS MATERIALIZED (SELECT * FROM ranked_xattr WHERE vexfs_rank=1) +INSERT INTO _vexfs_xattrs(inode_id,name,value) +SELECT x.inode_id,x.name,x.value FROM latest_xattr x +JOIN active_inode inode ON inode.inode_id=x.inode_id +WHERE x.deleted=0 +)SQL"); + restore_xattrs.BindInt64(1, workspace.id); + restore_xattrs.BindInt64(2, commit); + restore_xattrs.Done(); + + Statement restore_acl(db, R"SQL( +WITH +ranked_inode AS MATERIALIZED ( + SELECT s.*,ROW_NUMBER() OVER( + PARTITION BY s.inode_id ORDER BY s.commit_id DESC) AS vexfs_rank + FROM _vexfs_inode_states s WHERE s.workspace_id=?1 AND s.commit_id<=?2), +active_inode AS MATERIALIZED (SELECT * FROM ranked_inode WHERE vexfs_rank=1 AND deleted_at IS NULL), +ranked_acl AS MATERIALIZED ( + SELECT s.*,ROW_NUMBER() OVER( + PARTITION BY s.inode_id,s.principal_id,s.effect ORDER BY s.commit_id DESC) AS vexfs_rank + FROM _vexfs_acl_states s WHERE s.workspace_id=?1 AND s.commit_id<=?2), +latest_acl AS MATERIALIZED (SELECT * FROM ranked_acl WHERE vexfs_rank=1) +INSERT INTO _vexfs_acl_entries(workspace_id,inode_id,principal_id,effect,permissions,inherit_flags) +SELECT ?1,a.inode_id,a.principal_id,a.effect,a.permissions,a.inherit_flags +FROM latest_acl a JOIN active_inode inode ON inode.inode_id=a.inode_id +WHERE a.deleted=0 +)SQL"); + restore_acl.BindInt64(1, workspace.id); + restore_acl.BindInt64(2, commit); + restore_acl.Done(); } -void PathFunction(sqlite3_context *context, int, sqlite3_value **values) { - Guard(context, [&] { - sqlite3 *db = sqlite3_context_db_handle(context); - EnsureSchema(db); - const Workspace workspace = FindWorkspace(db, RequiredText(values[0], "workspace")); - const std::string path = PathForInode(db, workspace, sqlite3_value_int64(values[1])); - sqlite3_result_text(context, path.data(), static_cast(path.size()), SQLITE_TRANSIENT); - }); +void PrepareRestoredVersionAliases(sqlite3 *db, const Workspace &workspace, + sqlite3_int64 target_commit) { + Statement update(db, R"SQL( +WITH +ranked AS MATERIALIZED ( + SELECT s.*,ROW_NUMBER() OVER( + PARTITION BY s.inode_id ORDER BY s.commit_id DESC) AS vexfs_rank + FROM _vexfs_inode_states s WHERE s.workspace_id=?1 AND s.commit_id<=?2), +latest AS MATERIALIZED (SELECT * FROM ranked WHERE vexfs_rank=1), +maximum AS ( + SELECT inode_id,MAX(version_no) AS version_no FROM _vexfs_file_versions GROUP BY inode_id) +UPDATE _vexfs_inodes AS inode SET + current_version=(SELECT maximum.version_no+1 FROM maximum WHERE maximum.inode_id=inode.id) +WHERE inode.workspace_id=?1 AND inode.deleted_at IS NULL + AND inode.kind IN ('file','symlink') + AND EXISTS( + SELECT 1 FROM latest JOIN maximum ON maximum.inode_id=latest.inode_id + WHERE latest.inode_id=inode.id AND latest.deleted_at IS NULL + AND maximum.version_no>latest.current_version) +)SQL"); + update.BindInt64(1, workspace.id); + update.BindInt64(2, target_commit); + update.Done(); } -void ListFunction(sqlite3_context *context, int, sqlite3_value **values) { +void StoreRestoredVersionAliases(sqlite3 *db, const Workspace &workspace, + sqlite3_int64 target_commit, sqlite3_int64 restore_commit) { + Statement insert(db, R"SQL( +WITH ranked AS MATERIALIZED ( + SELECT s.*,ROW_NUMBER() OVER( + PARTITION BY s.inode_id ORDER BY s.commit_id DESC) AS vexfs_rank + FROM _vexfs_inode_states s WHERE s.workspace_id=?1 AND s.commit_id<=?2), +latest AS MATERIALIZED (SELECT * FROM ranked WHERE vexfs_rank=1) +INSERT INTO _vexfs_file_versions( + inode_id,version_no,commit_id,content,size,checksum,source_version_no) +SELECT inode.id,inode.current_version,?3,X'',target.size,source.checksum, + COALESCE(source.source_version_no,source.version_no) +FROM _vexfs_inodes inode +JOIN latest target ON target.inode_id=inode.id AND target.deleted_at IS NULL +JOIN _vexfs_file_versions source + ON source.inode_id=target.inode_id AND source.version_no=target.current_version +WHERE inode.workspace_id=?1 AND inode.deleted_at IS NULL + AND inode.kind IN ('file','symlink') + AND inode.current_version<>target.current_version +)SQL"); + insert.BindInt64(1, workspace.id); + insert.BindInt64(2, target_commit); + insert.BindInt64(3, restore_commit); + insert.Done(); +} + +void SnapshotRestoreFunction(sqlite3_context *context, int, sqlite3_value **values) { Guard(context, [&] { sqlite3 *db = sqlite3_context_db_handle(context); EnsureSchema(db); - const Workspace workspace = FindWorkspace(db, RequiredText(values[0], "workspace")); - const Node directory = Resolve(db, workspace, PathParts(RequiredText(values[1], "path"))); - if (directory.kind != "directory") - throw SqlError("path is not a directory", SQLITE_MISMATCH); - - Statement statement(db, - "SELECT d.name, i.id, i.kind, i.size, i.current_version, i.mode, " - "i.created_at, i.updated_at FROM _vexfs_dentries d " - "JOIN _vexfs_inodes i ON i.id = d.inode_id " - "WHERE d.workspace_id = ?1 AND d.parent_inode = ?2 AND i.deleted_at IS NULL " - "ORDER BY d.name COLLATE BINARY"); - statement.BindInt64(1, workspace.id); - statement.BindInt64(2, directory.id); - std::string json = "["; - bool first = true; - while (statement.Row()) { - if (!first) json += ','; - first = false; - json += "{\"name\":\"" + JsonEscape(statement.Text(0)) + "\",\"inode\":" + - std::to_string(statement.Int64(1)) + ",\"kind\":\"" + statement.Text(2) + - "\",\"size\":" + std::to_string(statement.Int64(3)) + ",\"version\":" + - std::to_string(statement.Int64(4)) + ",\"mode\":" + - std::to_string(statement.Int(5)) + ",\"created_at\":" + - std::to_string(statement.Int64(6)) + ",\"updated_at\":" + - std::to_string(statement.Int64(7)) + "}"; + Workspace workspace = FindWorkspace(db, RequiredText(values[0], "workspace")); + const std::string name = RequiredText(values[1], "snapshot name"); + const sqlite3_int64 expected_head = RequiredPositiveInteger(values[2], "expected head"); + Savepoint savepoint(db, "vexfs_snapshot_restore"); + AcquireWriteLock(db); + workspace = FindWorkspace(db, RequiredText(values[0], "workspace")); + if (workspace.head_commit != expected_head) { + throw SqlError("workspace head conflict: expected " + std::to_string(expected_head) + + ", current " + std::to_string(workspace.head_commit), + SQLITE_CONSTRAINT); } - json += ']'; - sqlite3_result_text(context, json.data(), static_cast(json.size()), SQLITE_TRANSIENT); + Statement active_mount(db, + "SELECT 1 FROM _vexfs_mount_sessions WHERE workspace_id=?1 " + "AND lease_until>CAST(strftime('%s','now') AS INTEGER)*1000 LIMIT 1"); + active_mount.BindInt64(1, workspace.id); + if (active_mount.Row()) { + throw SqlError( + "workspace has an active mount session; unmount before snapshot restore", + SQLITE_BUSY); + } + const SnapshotInfo snapshot = FindSnapshot(db, workspace, name); + Statement handles(db, + "SELECT 1 FROM _vexfs_handles WHERE workspace_id=?1 " + "AND state IN ('open','retained') LIMIT 1"); + handles.BindInt64(1, workspace.id); + if (handles.Row()) { + throw SqlError("workspace has open or retained file handles", SQLITE_BUSY); + } + const SnapshotTree current = CurrentTree(db, workspace); + const SnapshotTree target = TreeAtCommit(db, workspace, snapshot.commit); + if (SnapshotDiffJson(current, target) == "[]") { + throw SqlError("snapshot already matches current workspace", SQLITE_MISMATCH); + } + RestoreWorkspaceTree(db, workspace, snapshot.commit); + PrepareRestoredVersionAliases(db, workspace, snapshot.commit); + const sqlite3_int64 commit = CreateCommit(db, workspace, "restore snapshot " + name); + StoreRestoredVersionAliases(db, workspace, snapshot.commit, commit); + RefreshGrepIndexAfterTreeChange(db); + savepoint.Release(); + sqlite3_result_int64(context, commit); }); } @@ -1596,6 +4284,22 @@ WHERE workspace_id = ?2 AND id IN (SELECT id FROM tree) tombstone.BindInt64(1, inode); tombstone.BindInt64(2, workspace_id); tombstone.Done(); + RemoveDeletedFilesFromGrepIndex(db, workspace_id); +} + +void TombstoneIfUnreferenced(sqlite3 *db, sqlite3_int64 workspace_id, + sqlite3_int64 inode) { + Statement tombstone(db, R"SQL( +UPDATE _vexfs_inodes SET deleted_at=CAST(strftime('%s','now') AS INTEGER)*1000 +WHERE workspace_id=?1 AND id=?2 AND kind<>'directory' + AND NOT EXISTS( + SELECT 1 FROM _vexfs_dentries + WHERE workspace_id=?1 AND inode_id=?2) +)SQL"); + tombstone.BindInt64(1, workspace_id); + tombstone.BindInt64(2, inode); + tombstone.Done(); + RemoveDeletedFilesFromGrepIndex(db, workspace_id); } void RenameFunction(sqlite3_context *context, int argument_count, sqlite3_value **values) { @@ -1628,7 +4332,8 @@ void RenameFunction(sqlite3_context *context, int argument_count, sqlite3_value if (source.kind != existing.kind) throw SqlError("source and destination types differ", SQLITE_MISMATCH); if (existing.kind == "directory" && !DirectoryIsEmpty(db, workspace.id, existing.id)) { - throw SqlError("destination directory is not empty", SQLITE_CONSTRAINT); + throw SqlError("destination directory is not empty", + VEXFS_SQLITE_CONSTRAINT_NOT_EMPTY); } Statement detach(db, "DELETE FROM _vexfs_dentries WHERE workspace_id=?1 AND parent_inode=?2 AND name=?3"); @@ -1636,7 +4341,11 @@ void RenameFunction(sqlite3_context *context, int argument_count, sqlite3_value detach.BindInt64(2, destination_parent.id); detach.BindText(3, destination_name); detach.Done(); - TombstoneTree(db, workspace.id, existing.id); + if (existing.kind == "directory") { + TombstoneTree(db, workspace.id, existing.id); + } else { + TombstoneIfUnreferenced(db, workspace.id, existing.id); + } } Statement update(db, "UPDATE _vexfs_dentries SET parent_inode = ?1, name = ?2 " @@ -1670,7 +4379,7 @@ void RemoveFunction(sqlite3_context *context, int, sqlite3_value **values) { throw SqlError("path not found", SQLITE_NOTFOUND); if (node.kind == "directory" && !recursive && !DirectoryIsEmpty(db, workspace.id, node.id)) { - throw SqlError("directory is not empty", SQLITE_CONSTRAINT); + throw SqlError("directory is not empty", VEXFS_SQLITE_CONSTRAINT_NOT_EMPTY); } Statement detach(db, "DELETE FROM _vexfs_dentries WHERE workspace_id=?1 AND parent_inode=?2 AND name=?3"); @@ -1678,13 +4387,76 @@ void RemoveFunction(sqlite3_context *context, int, sqlite3_value **values) { detach.BindInt64(2, parent.id); detach.BindText(3, name); detach.Done(); - TombstoneTree(db, workspace.id, node.id); + if (node.kind == "directory") { + TombstoneTree(db, workspace.id, node.id); + } else { + TombstoneIfUnreferenced(db, workspace.id, node.id); + } TouchDirectory(db, workspace.id, parent.id); CreateCommit(db, workspace, "remove"); sqlite3_result_int(context, 1); }); } +void HandleCreateFunction(sqlite3_context *context, int argument_count, + sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + const std::string workspace_name = RequiredText(values[0], "workspace"); + const std::string path = RequiredText(values[1], "path"); + const int mode = RequiredMode(values[2]); + const std::string request_id = RequiredText(values[3], "request_id"); + const std::string owner_session = argument_count == 5 + ? RequiredText(values[4], "owner_session") : std::string(); + RequestFingerprint fingerprint_builder("handle_create"); + fingerprint_builder.AddText(workspace_name); + fingerprint_builder.AddText(path); + fingerprint_builder.AddInt64(mode); + fingerprint_builder.AddText(owner_session); + const auto fingerprint = fingerprint_builder.Finish(); + const CachedRequest cached = FindRequest(db, request_id, "handle_create", fingerprint); + if (cached.kind == CachedKind::kText) { + sqlite3_result_text(context, cached.text.data(), + static_cast(cached.text.size()), SQLITE_TRANSIENT); + return; + } + + const Workspace workspace = FindWorkspace(db, workspace_name); + const auto parts = PathParts(path); + if (parts.empty()) throw SqlError("workspace root already exists", SQLITE_CONSTRAINT); + Savepoint savepoint(db, "vexfs_handle_create"); + AcquireWriteLock(db); + auto [parent, name] = ResolveParent(db, workspace, parts); + Node existing; + if (FindChild(db, workspace.id, parent.id, name, &existing)) { + throw SqlError("destination already exists", SQLITE_CONSTRAINT); + } + const sqlite3_int64 inode = CreateInode(db, workspace.id, "file", mode); + AddDentry(db, workspace.id, parent.id, name, inode); + + const std::string handle_id = RandomHandleId(); + Statement insert(db, + "INSERT INTO _vexfs_handles(id, workspace_id, inode_id, open_flags, writable, " + "expected_version, dirty_generation, state, owner_session) " + "VALUES(?1,?2,?3,'rw',1,0,1,'open',?4)"); + insert.BindText(1, handle_id); + insert.BindInt64(2, workspace.id); + insert.BindInt64(3, inode); + if (argument_count == 5) insert.BindText(4, owner_session); + else insert.BindNull(4); + insert.Done(); + + Handle handle{handle_id, workspace.id, inode, true, 0, 1, 0, + "open", owner_session}; + CreateStaging(db, handle, {}); + StoreRequestText(db, request_id, "handle_create", fingerprint, handle_id); + savepoint.Release(); + sqlite3_result_text(context, handle_id.data(), + static_cast(handle_id.size()), SQLITE_TRANSIENT); + }); +} + void HandleOpenFunction(sqlite3_context *context, int argument_count, sqlite3_value **values) { Guard(context, [&] { sqlite3 *db = sqlite3_context_db_handle(context); @@ -1784,6 +4556,66 @@ void HandleStageWriteFunction(sqlite3_context *context, int, sqlite3_value **val }); } +void HandleAppendFunction(sqlite3_context *context, int, sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + const std::string handle_id = RequiredText(values[0], "handle"); + const auto suffix = RequiredBlob(values[1], "content"); + const std::string request_id = RequiredText(values[2], "request_id"); + RequestFingerprint fingerprint_builder("handle_append"); + fingerprint_builder.AddText(handle_id); + fingerprint_builder.AddBlob(suffix); + const auto fingerprint = fingerprint_builder.Finish(); + const CachedRequest cached = FindRequest(db, request_id, "handle_append", fingerprint); + if (cached.kind == CachedKind::kInteger) { + sqlite3_result_int64(context, cached.integer); + return; + } + Handle handle = FindHandle(db, handle_id); + if (handle.state != "open") throw SqlError("handle is not open", SQLITE_NOTFOUND); + if (!handle.writable) throw SqlError("handle is read-only", SQLITE_READONLY); + if (handle.dirty_generation != handle.published_generation) { + throw SqlError("append requires a clean handle", SQLITE_CONSTRAINT); + } + Workspace workspace = WorkspaceById(db, handle.workspace_id); + Savepoint savepoint(db, "vexfs_handle_append"); + AcquireWriteLock(db); + const Node node = NodeById(db, handle.inode_id, true); + if (node.kind != "file") throw SqlError("inode is not a file", SQLITE_MISMATCH); + if (node.size < 0 || node.size > kMaxStagedBytes || + suffix.size() > static_cast(kMaxStagedBytes - node.size)) { + throw SqlError("file is larger than the Phase 0 limit (128 MiB)", SQLITE_TOOBIG); + } + std::vector content = ReadNode(db, node); + content.insert(content.end(), suffix.begin(), suffix.end()); + const sqlite3_int64 version = StoreVersion( + db, workspace, node, content, "append handle"); + + StagingInfo staging = EnsureStagingCapacity( + db, handle, FindStaging(db, handle), static_cast(content.size())); + if (!content.empty()) { + WriteBlob(db, staging.rowid, 0, content.data(), + static_cast(content.size())); + } + Statement stage(db, + "UPDATE _vexfs_staging SET logical_size=?1," + "updated_at=CAST(unixepoch('subsec')*1000 AS INTEGER) WHERE handle_id=?2"); + stage.BindInt64(1, static_cast(content.size())); + stage.BindText(2, handle.id); + stage.Done(); + Statement update(db, + "UPDATE _vexfs_handles SET expected_version=?1," + "updated_at=CAST(unixepoch('subsec')*1000 AS INTEGER) WHERE id=?2"); + update.BindInt64(1, version); + update.BindText(2, handle.id); + update.Done(); + StoreRequestInteger(db, request_id, "handle_append", fingerprint, version); + savepoint.Release(); + sqlite3_result_int64(context, version); + }); +} + void HandleTruncateFunction(sqlite3_context *context, int, sqlite3_value **values) { Guard(context, [&] { sqlite3 *db = sqlite3_context_db_handle(context); @@ -1851,6 +4683,34 @@ void HandlePublishFunction(sqlite3_context *context, int, sqlite3_value **values }); } +void HandlePublishCloseFunction(sqlite3_context *context, int, sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + const std::string handle_id = RequiredText(values[0], "handle"); + const sqlite3_int64 generation = sqlite3_value_int64(values[1]); + const std::string durability = RequiredText(values[2], "durability"); + if (durability != "none" && durability != "data" && durability != "full") { + throw SqlError("durability must be none, data, or full", SQLITE_MISMATCH); + } + Savepoint savepoint(db, "vexfs_handle_publish_close"); + const sqlite3_int64 version = PublishHandle(db, FindHandle(db, handle_id), generation); + Statement update(db, + "UPDATE _vexfs_handles SET state='closed',lease_until=NULL," + "updated_at=CAST(strftime('%s','now') AS INTEGER)*1000 WHERE id=?1"); + update.BindText(1, handle_id); + update.Done(); + Statement clear_data(db, "DELETE FROM _vexfs_staging_data WHERE handle_id=?1"); + clear_data.BindText(1, handle_id); + clear_data.Done(); + Statement clear(db, "DELETE FROM _vexfs_staging WHERE handle_id=?1"); + clear.BindText(1, handle_id); + clear.Done(); + savepoint.Release(); + sqlite3_result_int64(context, version); + }); +} + void HandleCloseFunction(sqlite3_context *context, int, sqlite3_value **values) { Guard(context, [&] { sqlite3 *db = sqlite3_context_db_handle(context); @@ -2086,6 +4946,10 @@ void ReclaimFunction(sqlite3_context *context, int, sqlite3_value **values) { }); } +void ConnectionGuardFunction(sqlite3_context *context, int, sqlite3_value **) { + sqlite3_result_int(context, 1); +} + struct FunctionDefinition { const char *name; int arguments; @@ -2102,25 +4966,60 @@ extern "C" int vexfs_sqlite_register(sqlite3 *db) { SQLITE_UTF8 | SQLITE_DETERMINISTIC}, {"vexfs_workspace_create", 1, WorkspaceCreateFunction, SQLITE_UTF8}, {"vexfs_mkdir", 2, MkdirFunction, SQLITE_UTF8}, + {"vexfs_create", 4, CreateFunction, SQLITE_UTF8}, + {"vexfs_symlink", 3, SymlinkFunction, SQLITE_UTF8}, + {"vexfs_link", 3, HardlinkFunction, SQLITE_UTF8}, {"vexfs_write", 3, WriteFunction, SQLITE_UTF8}, + {"vexfs_append", 3, AppendFunction, SQLITE_UTF8}, {"vexfs_read", 2, ReadFunction, SQLITE_UTF8}, + {"vexfs_grep", 5, GrepFunction, SQLITE_UTF8}, + {"vexfs_grep_index", 1, GrepIndexFunction, SQLITE_UTF8}, {"vexfs_history", 2, HistoryFunction, SQLITE_UTF8}, {"vexfs_history", 4, HistoryFunction, SQLITE_UTF8}, {"vexfs_read_version", 3, ReadVersionFunction, SQLITE_UTF8}, {"vexfs_compare_versions", 4, CompareVersionsFunction, SQLITE_UTF8}, {"vexfs_restore_version", 4, RestoreVersionFunction, SQLITE_UTF8}, {"vexfs_stat", 2, StatFunction, SQLITE_UTF8}, + {"vexfs_check", 2, CheckFunction, SQLITE_UTF8}, {"vexfs_path", 2, PathFunction, SQLITE_UTF8}, {"vexfs_list", 2, ListFunction, SQLITE_UTF8}, + {"vexfs_readlink", 2, ReadlinkFunction, SQLITE_UTF8}, + {"vexfs_set_mode", 3, SetModeFunction, SQLITE_UTF8}, + {"vexfs_set_times", 5, SetTimesFunction, SQLITE_UTF8}, + {"vexfs_chown", 4, ChownFunction, SQLITE_UTF8}, + {"vexfs_owner_set", 3, OwnerSetFunction, SQLITE_UTF8}, + {"vexfs_acl_get", 2, AclGetFunction, SQLITE_UTF8}, + {"vexfs_acl_list", 2, AclGetFunction, SQLITE_UTF8}, + {"vexfs_acl_set", 3, AclSetFunction, SQLITE_UTF8}, + {"vexfs_acl_delete", 2, AclDeleteFunction, SQLITE_UTF8}, + {"vexfs_acl_grant", 4, AclGrantFunction, SQLITE_UTF8}, + {"vexfs_acl_grant", 5, AclGrantFunction, SQLITE_UTF8}, + {"vexfs_acl_revoke", 3, AclRevokeFunction, SQLITE_UTF8}, + {"vexfs_acl_revoke", 4, AclRevokeFunction, SQLITE_UTF8}, + {"vexfs_xattr_get", 3, GetXattrFunction, SQLITE_UTF8}, + {"vexfs_xattr_list", 2, ListXattrsFunction, SQLITE_UTF8}, + {"vexfs_xattr_set", 5, SetXattrFunction, SQLITE_UTF8}, + {"vexfs_snapshot_create", 2, SnapshotCreateFunction, SQLITE_UTF8}, + {"vexfs_snapshot_create", 3, SnapshotCreateFunction, SQLITE_UTF8}, + {"vexfs_snapshot_create", 4, SnapshotCreateFunction, SQLITE_UTF8}, + {"vexfs_snapshot_list", 1, SnapshotListFunction, SQLITE_UTF8}, + {"vexfs_snapshot_show", 2, SnapshotShowFunction, SQLITE_UTF8}, + {"vexfs_snapshot_diff", 3, SnapshotDiffFunction, SQLITE_UTF8}, + {"vexfs_snapshot_drop", 2, SnapshotDropFunction, SQLITE_UTF8}, + {"vexfs_snapshot_restore", 3, SnapshotRestoreFunction, SQLITE_UTF8}, {"vexfs_move", 3, RenameFunction, SQLITE_UTF8}, {"vexfs_rename", 4, RenameFunction, SQLITE_UTF8}, {"vexfs_remove", 3, RemoveFunction, SQLITE_UTF8}, + {"vexfs_handle_create", 4, HandleCreateFunction, SQLITE_UTF8}, + {"vexfs_handle_create", 5, HandleCreateFunction, SQLITE_UTF8}, {"vexfs_handle_open", 4, HandleOpenFunction, SQLITE_UTF8}, {"vexfs_handle_open", 5, HandleOpenFunction, SQLITE_UTF8}, {"vexfs_handle_stage_write", 4, HandleStageWriteFunction, SQLITE_UTF8}, + {"vexfs_handle_append", 3, HandleAppendFunction, SQLITE_UTF8}, {"vexfs_handle_truncate", 3, HandleTruncateFunction, SQLITE_UTF8}, {"vexfs_handle_read", 3, HandleReadFunction, SQLITE_UTF8}, {"vexfs_handle_publish", 4, HandlePublishFunction, SQLITE_UTF8}, + {"vexfs_handle_publish_close", 3, HandlePublishCloseFunction, SQLITE_UTF8}, {"vexfs_handle_close", 3, HandleCloseFunction, SQLITE_UTF8}, {"vexfs_mount_session_start", 2, SessionStartFunction, SQLITE_UTF8}, {"vexfs_mount_session_heartbeat", 2, SessionHeartbeatFunction, SQLITE_UTF8}, @@ -2129,10 +5028,17 @@ extern "C" int vexfs_sqlite_register(sqlite3 *db) { {"vexfs_mount_synchronize", 3, SynchronizeFunction, SQLITE_UTF8}, {"vexfs_item_reclaim", 2, ReclaimFunction, SQLITE_UTF8}, }; + // sqlite3_create_function_v2 exists throughout the supported SQLite range. Its + // destructor removes the pointer-keyed schema cache before a connection address can + // be reused; this avoids requiring sqlite3 client-data APIs added only in 3.44. + int rc = sqlite3_create_function_v2( + db, "_vexfs_connection_guard", 0, SQLITE_UTF8 | SQLITE_DETERMINISTIC, db, + ConnectionGuardFunction, nullptr, nullptr, ForgetSchemaReady); + if (rc != SQLITE_OK) return rc; for (const auto &definition : functions) { - const int rc = sqlite3_create_function(db, definition.name, definition.arguments, - definition.flags, nullptr, definition.function, - nullptr, nullptr); + rc = sqlite3_create_function(db, definition.name, definition.arguments, + definition.flags, nullptr, definition.function, + nullptr, nullptr); if (rc != SQLITE_OK) return rc; } return SQLITE_OK; diff --git a/vexdb_sqlite/src/vexdb_sqlite_init.cpp b/vexdb_sqlite/src/vexdb_sqlite_init.cpp index 0f7c3b602e..d6a7d67b27 100644 --- a/vexdb_sqlite/src/vexdb_sqlite_init.cpp +++ b/vexdb_sqlite/src/vexdb_sqlite_init.cpp @@ -15,13 +15,17 @@ SQLITE_EXTENSION_INIT1 #ifndef VEXDB_SQLITE_BUILD_TIME #define VEXDB_SQLITE_BUILD_TIME "unknown" #endif +#ifndef VEXDB_LITE_VERSION +#define VEXDB_LITE_VERSION "0.1.0-dev" +#endif namespace { void VexVersionFunc(sqlite3_context *ctx, int /*argc*/, sqlite3_value ** /*argv*/) { sqlite3_result_text( ctx, - "vexdb_lite sqlite extension " VEXDB_SQLITE_GIT_HASH " (" VEXDB_SQLITE_BUILD_TIME ")", + "vexdb-lite " VEXDB_LITE_VERSION " sqlite extension " VEXDB_SQLITE_GIT_HASH + " (" VEXDB_SQLITE_BUILD_TIME ")", -1, SQLITE_STATIC); } diff --git a/vexdb_sqlite/test/vexfs_static_smoke.c b/vexdb_sqlite/test/vexfs_static_smoke.c index 5339589940..7fd524e48a 100644 --- a/vexdb_sqlite/test/vexfs_static_smoke.c +++ b/vexdb_sqlite/test/vexfs_static_smoke.c @@ -28,6 +28,14 @@ static int exec_fails(sqlite3 *db, const char *sql) { return rc != SQLITE_OK; } +static int exec_fails_with(sqlite3 *db, const char *sql, const char *expected) { + char *message = NULL; + int rc = sqlite3_exec(db, sql, NULL, NULL, &message); + int matches = rc != SQLITE_OK && message != NULL && strstr(message, expected) != NULL; + sqlite3_free(message); + return matches; +} + static sqlite3_int64 scalar_int(sqlite3 *db, const char *sql, int *ok) { sqlite3_stmt *statement = NULL; *ok = 0; @@ -77,6 +85,36 @@ int main(void) { if (text == NULL || strcmp(text, "hello") != 0) return fail(db, "read after write"); free(text); + text = scalar_text(db, + "SELECT vexfs_grep('default','/notes','ell',0,100)"); + if (text == NULL || strstr(text, "\"path\":\"/notes/2026/hello.txt\"") == NULL || + strstr(text, "\"line\":1") == NULL || strstr(text, "\"match_count\":1") == NULL) + return fail(db, "database grep"); + free(text); + text = scalar_text(db, + "SELECT vexfs_grep('default','/notes','HEL',1,100)"); + if (text == NULL || strstr(text, "\"match_count\":1") == NULL) + return fail(db, "database grep ignore case"); + free(text); + text = scalar_text(db, "SELECT vexfs_grep_index('enable')"); + if (text == NULL || strstr(text, "\"enabled\":true") == NULL || + strstr(text, "\"available\":true") == NULL || + strstr(text, "\"backend\":\"fts5-trigram\"") == NULL) + return fail(db, "enable database grep index"); + free(text); + text = scalar_text(db, + "SELECT vexfs_grep('default','/notes','ell',0,100)"); + if (text == NULL || strstr(text, "\"index_used\":true") == NULL || + strstr(text, "\"match_count\":1") == NULL) + return fail(db, "indexed database grep"); + free(text); + text = scalar_text(db, + "SELECT vexfs_grep('default','/notes','el',0,100)"); + if (text == NULL || strstr(text, "\"index_used\":false") == NULL || + strstr(text, "\"match_count\":1") == NULL) + return fail(db, "short database grep fallback"); + free(text); + // SQL 事务回滚必须同时回滚文件内容。 if (!exec_ok(db, "BEGIN; SELECT vexfs_write('default','/notes/2026/hello.txt','changed'); ROLLBACK;")) return fail(db, "transaction rollback"); @@ -232,38 +270,93 @@ int main(void) { "SELECT vexfs_remove('default','/notes/2026',0);")) return fail(db, "move and remove"); + // POSIX mode 与符号链接必须通过静态注册入口工作。 + inode = scalar_int(db, + "SELECT vexfs_create('default','/notes/run.sh','file',488)", &ok); + if (!ok || inode == 0) return fail(db, "create file with mode"); + snprintf(sql, sizeof(sql), "SELECT vexfs_set_mode('default',%lld,493)", + (long long)inode); + if (scalar_int(db, sql, &ok) != 493 || !ok) return fail(db, "set executable mode"); + if (scalar_int(db, + "SELECT json_extract(vexfs_stat('default','/notes/run.sh'),'$.mode')", &ok) != 493 || + !ok) return fail(db, "persist executable mode"); + sqlite3_int64 link_inode = scalar_int(db, + "SELECT vexfs_symlink('default','/notes/run-link','run.sh')", &ok); + if (!ok || link_inode == 0) return fail(db, "create symlink"); + snprintf(sql, sizeof(sql), "SELECT CAST(vexfs_readlink('default',%lld) AS TEXT)", + (long long)link_inode); + text = scalar_text(db, sql); + if (text == NULL || strcmp(text, "run.sh") != 0) return fail(db, "read symlink"); + free(text); + + if (!exec_ok(db, + "SELECT vexfs_xattr_set('default'," + "json_extract(vexfs_stat('default','/notes/run.sh'),'$.inode')," + "'user.snapshot','old',0);" + "SELECT vexfs_snapshot_create('default','before-tree');" + "SELECT vexfs_write('default','/notes/run.sh','changed');" + "SELECT vexfs_set_mode('default'," + "json_extract(vexfs_stat('default','/notes/run.sh'),'$.inode'),420);" + "SELECT vexfs_rename('default','/notes/run.sh','/notes/renamed.sh',0);" + "SELECT vexfs_remove('default','/notes/run-link',0);" + "SELECT vexfs_xattr_set('default'," + "json_extract(vexfs_stat('default','/notes/renamed.sh'),'$.inode')," + "'user.snapshot','new',0);")) return fail(db, "snapshot mutations"); + text = scalar_text(db, + "SELECT vexfs_snapshot_diff('default','before-tree','HEAD')"); + if (text == NULL || strstr(text, "renamed.sh") == NULL || + strstr(text, "run-link") == NULL) return fail(db, "snapshot diff"); + free(text); + version = scalar_int(db, + "SELECT vexfs_snapshot_restore('default','before-tree'," + "(SELECT head_commit FROM _vexfs_workspaces WHERE name='default'))", &ok); + if (!ok || version <= 0) return fail(db, "snapshot restore"); + if (scalar_int(db, + "SELECT json_extract(vexfs_stat('default','/notes/run.sh'),'$.mode')", &ok) != 493 || + !ok) return fail(db, "snapshot restored mode"); + if (!exec_fails(db, "SELECT vexfs_stat('default','/notes/renamed.sh')")) + return fail(db, "snapshot removed later rename"); + snprintf(sql, sizeof(sql), "SELECT CAST(vexfs_readlink('default',%lld) AS TEXT)", + (long long)link_inode); + text = scalar_text(db, sql); + if (text == NULL || strcmp(text, "run.sh") != 0) + return fail(db, "snapshot restored symlink"); + free(text); + text = scalar_text(db, + "SELECT CAST(vexfs_xattr_get('default'," + "json_extract(vexfs_stat('default','/notes/run.sh'),'$.inode')," + "'user.snapshot') AS TEXT)"); + if (text == NULL || strcmp(text, "old") != 0) return fail(db, "snapshot restored xattr"); + free(text); + text = scalar_text(db, "SELECT vexfs_snapshot_list('default')"); + if (text == NULL || strstr(text, "before-tree") == NULL) return fail(db, "snapshot list"); + free(text); + scalar_int(db, "SELECT vexfs_item_reclaim('default','reclaim-1')", &ok); if (!ok) return fail(db, "reclaim"); sqlite3_close(db); - // 0.1.0 数据库原地升级，保留当前 dirty generation，并清空旧幂等缓存。 + // 尚未正式发版，不保留旧 schema 迁移。版本不匹配必须明确拒绝且不能改库。 db = NULL; - if (sqlite3_open(":memory:", &db) != SQLITE_OK) return fail(db, "migration open"); - if (vexdb_sqlite_register(db) != SQLITE_OK) return fail(db, "migration register"); + if (sqlite3_open(":memory:", &db) != SQLITE_OK) return fail(db, "version mismatch open"); + if (vexdb_sqlite_register(db) != SQLITE_OK) return fail(db, "version mismatch register"); if (!exec_ok(db, "CREATE TABLE _vexfs_meta(key TEXT PRIMARY KEY,value TEXT NOT NULL);" "INSERT INTO _vexfs_meta VALUES('contract_version','0.1.0');" - "CREATE TABLE _vexfs_inodes(kind TEXT,deleted_at INTEGER,current_version INTEGER);" - "INSERT INTO _vexfs_inodes VALUES('directory',NULL,0);" - "CREATE TABLE _vexfs_handles(id TEXT PRIMARY KEY,dirty_generation INTEGER NOT NULL);" - "INSERT INTO _vexfs_handles VALUES('h',1);" - "CREATE TABLE _vexfs_staging(handle_id TEXT,generation INTEGER,content BLOB," - "created_at INTEGER,PRIMARY KEY(handle_id,generation));" - "INSERT INTO _vexfs_staging VALUES('h',1,X'6162',123);" - "CREATE TABLE _vexfs_requests(request_id TEXT PRIMARY KEY,operation TEXT NOT NULL," - "result_integer INTEGER,result_text TEXT,created_at INTEGER);" - "INSERT INTO _vexfs_requests VALUES('old','write',1,NULL,123);" - "SELECT vexfs_init();")) return fail(db, "0.1 migration"); + "CREATE TABLE legacy_marker(value TEXT NOT NULL);" + "INSERT INTO legacy_marker VALUES('unchanged');")) + return fail(db, "version mismatch fixture"); + if (!exec_fails_with(db, "SELECT vexfs_init();", "unsupported VexFS schema version: 0.1.0")) + return fail(db, "old schema must be rejected"); text = scalar_text(db, "SELECT value FROM _vexfs_meta WHERE key='contract_version'"); - if (text == NULL || strcmp(text, "0.3.0") != 0) return fail(db, "migration version"); + if (text == NULL || strcmp(text, "0.1.0") != 0) return fail(db, "old schema changed"); + free(text); + text = scalar_text(db, "SELECT value FROM legacy_marker"); + if (text == NULL || strcmp(text, "unchanged") != 0) return fail(db, "old data changed"); free(text); - if (scalar_int(db, "SELECT logical_size FROM _vexfs_staging WHERE handle_id='h'", &ok) != 2 || - !ok) return fail(db, "migration staging"); - if (scalar_int(db, "SELECT count(*) FROM _vexfs_requests", &ok) != 0 || !ok) - return fail(db, "migration request cache"); - if (scalar_int(db, "SELECT current_version FROM _vexfs_inodes", &ok) != 1 || !ok) - return fail(db, "migration directory verifier"); + if (scalar_int(db, "SELECT count(*) FROM sqlite_master WHERE name LIKE '_vexfs_%'", &ok) != 1 || + !ok) return fail(db, "old schema was partially initialized"); sqlite3_close(db); printf("VEXFS STATIC SMOKE: PASS\n"); return 0; From 4f535778843d36dc4f1ee5302f8fa2b563f0a364 Mon Sep 17 00:00:00 2001 From: Four <8122332+HungryFour@users.noreply.github.com> Date: Thu, 23 Jul 2026 17:00:00 +0800 Subject: [PATCH 03/28] Add chunked VexFS storage and archive v2 --- agent_files/cli/test/vexdb_unified_smoke.sh | 2 +- agent_files/cli/test/vexfs_cli_smoke.sh | 47 +- agent_files/cli/vexfs_archive.cpp | 1434 +++++++++++++++ agent_files/cli/vexfs_archive.h | 18 + agent_files/cli/vexfs_main.cpp | 125 ++ ...77\347\224\250\350\257\264\346\230\216.md" | 23 + agent_files/macos/package/README.md | 5 + ...77\347\224\250\350\257\264\346\230\216.md" | 34 +- agent_files/macos/package_preview.sh | 2 +- .../common/include/vexfs_runtime_admin.h | 23 + .../src/vexfs_mount_contract_sqlite.cpp | 100 +- .../test/vexfs_mount_contract_smoke.cpp | 79 +- ...-07-16_agent-files-universal-filesystem.md | 27 +- ...2026-07-21_vexfs-final-goal-and-roadmap.md | 55 +- .../2026-07-23_vexfs-chunk-manifest-v2.md | 87 + docs/specs/2026-07-16_vexfs-product-spec.md | 81 +- tests/eval/vexfs/run.py | 427 ++++- .../sqlite/agent_files/vexfs_contract.yaml | 61 +- vexdb_sqlite/CMakeLists.txt | 1 + vexdb_sqlite/src/agent_files/vexfs_sqlite.cpp | 1554 +++++++++++++++-- 20 files changed, 3943 insertions(+), 242 deletions(-) create mode 100644 agent_files/cli/vexfs_archive.cpp create mode 100644 agent_files/cli/vexfs_archive.h create mode 100644 docs/reports/2026-07-23_vexfs-chunk-manifest-v2.md diff --git a/agent_files/cli/test/vexdb_unified_smoke.sh b/agent_files/cli/test/vexdb_unified_smoke.sh index d2ed4343be..ba7490336c 100755 --- a/agent_files/cli/test/vexdb_unified_smoke.sh +++ b/agent_files/cli/test/vexdb_unified_smoke.sh @@ -21,7 +21,7 @@ cp "$VEXDB" "$VEXFS_EXE" RESULT="$("$VEXDB" "$DB" \ "SELECT vexfs_contract_version(), vexdb_l2_distance('[1,2]','[4,6]');")" -[ "$RESULT" = "0.7.0|5.0" ] +[ "$RESULT" = "0.9.0|5.0" ] "$VEXDB" "$DB" \ "CREATE VIRTUAL TABLE idx USING GRAPH_INDEX(embedding FLOAT[2], metric=l2); \ diff --git a/agent_files/cli/test/vexfs_cli_smoke.sh b/agent_files/cli/test/vexfs_cli_smoke.sh index 18771dae75..bcb668c25a 100755 --- a/agent_files/cli/test/vexfs_cli_smoke.sh +++ b/agent_files/cli/test/vexfs_cli_smoke.sh @@ -4,8 +4,10 @@ set -euo pipefail VEXFS="${1:?usage: vexfs_cli_smoke.sh /path/to/vexfs}" TMP_DIR="$(mktemp -d -t vexfs-cli-smoke)" DB="$TMP_DIR/vexfs.sqlite3" +DEST_DB="$TMP_DIR/restored.sqlite3" +ARCHIVE="$TMP_DIR/workspace.vexfs" cleanup() { - rm -f "$DB" "$DB-wal" "$DB-shm" + rm -f "$DB" "$DB-wal" "$DB-shm" "$DEST_DB" "$DEST_DB-wal" "$DEST_DB-shm" "$ARCHIVE" rmdir "$TMP_DIR" 2>/dev/null || true } trap cleanup EXIT @@ -99,13 +101,54 @@ printf '%s' "$CHECK" | /usr/bin/python3 -c \ 'import json,sys; value=json.load(sys.stdin); assert value["ok"] is True; assert value["mode"] == "deep"; assert value["checked"]["versions"] >= 5' "$VEXFS" --db "$DB" --workspace smoke check --quick | grep -q '^OK ' +"$VEXFS" --db "$DB" --workspace smoke quota show | /usr/bin/python3 -c \ + 'import json,sys; value=json.load(sys.stdin); assert value["max_bytes"] is None; assert value["live_files"] > 0' +"$VEXFS" --db "$DB" --workspace smoke quota set \ + --max-bytes unlimited --max-files unlimited --max-file-bytes 4 >/dev/null +set +e +printf '12345' | "$VEXFS" --db "$DB" --workspace smoke write /agent/version.txt >/dev/null 2>&1 +QUOTA_STATUS=$? +set -e +[ "$QUOTA_STATUS" -eq 6 ] +[ "$("$VEXFS" --db "$DB" --workspace smoke cat /agent/version.txt)" = "alpha" ] +"$VEXFS" --db "$DB" --workspace smoke quota set \ + --max-bytes unlimited --max-files unlimited --max-file-bytes unlimited >/dev/null + +"$VEXFS" --db "$DB" --workspace smoke retention set --keep-versions 1 --keep-days 0 | + /usr/bin/python3 -c 'import json,sys; assert json.load(sys.stdin)["reclaimable_versions"] > 0' +"$VEXFS" --db "$DB" --workspace smoke gc pause | + /usr/bin/python3 -c 'import json,sys; assert json.load(sys.stdin)["gc_paused"] is True' +set +e +"$VEXFS" --db "$DB" --workspace smoke gc --batch 1 >/dev/null 2>&1 +PAUSED_GC_STATUS=$? +set -e +[ "$PAUSED_GC_STATUS" -eq 7 ] +"$VEXFS" --db "$DB" --workspace smoke gc resume | + /usr/bin/python3 -c 'import json,sys; assert json.load(sys.stdin)["gc_paused"] is False' +"$VEXFS" --db "$DB" --workspace smoke gc --batch 1 | + /usr/bin/python3 -c 'import json,sys; value=json.load(sys.stdin); assert value["deleted_versions"] == 1; assert value["has_more"] is True' +while "$VEXFS" --db "$DB" --workspace smoke gc --batch 1000 | + /usr/bin/python3 -c 'import json,sys; raise SystemExit(0 if json.load(sys.stdin)["has_more"] else 1)' +do :; done +"$VEXFS" --db "$DB" --workspace smoke --json check | + /usr/bin/python3 -c 'import json,sys; assert json.load(sys.stdin)["ok"] is True' + +"$VEXFS" --db "$DB" --workspace smoke export --output "$ARCHIVE" | + /usr/bin/python3 -c 'import json,sys; value=json.load(sys.stdin); assert value["format_version"] == 2; assert len(value["package_checksum"]) == 64' +"$VEXFS" archive verify "$ARCHIVE" | + /usr/bin/python3 -c 'import json,sys; assert json.load(sys.stdin)["ok"] is True' +"$VEXFS" --db "$DEST_DB" --workspace restored import "$ARCHIVE" >/dev/null +[ "$("$VEXFS" --db "$DEST_DB" --workspace restored cat /agent/task.txt)" = "hello from cli" ] +"$VEXFS" --db "$DEST_DB" --workspace restored --json check | + /usr/bin/python3 -c 'import json,sys; assert json.load(sys.stdin)["ok"] is True' + set +e DOCTOR="$("$VEXFS" --db "$DB" --workspace smoke --json doctor)" DOCTOR_STATUS=$? set -e [ "$DOCTOR_STATUS" -eq 0 ] || [ "$DOCTOR_STATUS" -eq 1 ] printf '%s' "$DOCTOR" | /usr/bin/python3 -c \ - 'import json,sys; value=json.load(sys.stdin); assert value["database"]["schema_version"] == "0.7.0"' + 'import json,sys; value=json.load(sys.stdin); assert value["database"]["schema_version"] == "0.9.0"' MOUNTS="$("$VEXFS" --json mount status)" printf '%s' "$MOUNTS" | /usr/bin/python3 -c 'import json,sys; assert isinstance(json.load(sys.stdin), list)' diff --git a/agent_files/cli/vexfs_archive.cpp b/agent_files/cli/vexfs_archive.cpp new file mode 100644 index 0000000000..4a6455f2cc --- /dev/null +++ b/agent_files/cli/vexfs_archive.cpp @@ -0,0 +1,1434 @@ +#include "vexfs_archive.h" + +#include "vexdb_sqlite.h" +#include "vexfs_checksum.h" + +#include "sqlite3.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace vexfs_cli { +namespace { + +constexpr int kArchiveFormatVersion = 2; +constexpr int kBlobVerifyChunk = 1024 * 1024; + +class Error : public std::runtime_error { + public: + explicit Error(const std::string &message) : std::runtime_error(message) {} +}; + +struct DatabaseCloser { + void operator()(sqlite3 *db) const { + if (db != nullptr) sqlite3_close_v2(db); + } +}; +using Database = std::unique_ptr; + +class NewDatabaseCleanup { + public: + NewDatabaseCleanup(std::string path, bool active) + : path_(std::move(path)), active_(active) {} + ~NewDatabaseCleanup() { + if (!active_) return; + std::error_code ignored; + std::filesystem::remove(path_, ignored); + std::filesystem::remove(path_ + "-wal", ignored); + std::filesystem::remove(path_ + "-shm", ignored); + } + void Release() { active_ = false; } + + private: + std::string path_; + bool active_ = false; +}; + +class Statement { + public: + Statement(sqlite3 *db, const char *sql) : db_(db) { + const int rc = sqlite3_prepare_v2(db, sql, -1, &statement_, nullptr); + if (rc != SQLITE_OK) Throw("prepare", rc); + } + ~Statement() { sqlite3_finalize(statement_); } + Statement(const Statement &) = delete; + Statement &operator=(const Statement &) = delete; + + void Text(int index, const std::string &value) { + Check("bind text", sqlite3_bind_text(statement_, index, value.data(), + static_cast(value.size()), SQLITE_TRANSIENT)); + } + void Int64(int index, sqlite3_int64 value) { + Check("bind integer", sqlite3_bind_int64(statement_, index, value)); + } + void Null(int index) { Check("bind NULL", sqlite3_bind_null(statement_, index)); } + bool Row() { + const int rc = sqlite3_step(statement_); + if (rc == SQLITE_ROW) return true; + if (rc == SQLITE_DONE) return false; + Throw("step", rc); + return false; + } + void Done() { + const int rc = sqlite3_step(statement_); + if (rc != SQLITE_DONE) Throw("step", rc); + } + sqlite3_int64 Int64(int column) const { return sqlite3_column_int64(statement_, column); } + int Int(int column) const { return sqlite3_column_int(statement_, column); } + int Type(int column) const { return sqlite3_column_type(statement_, column); } + std::string Text(int column) const { + const auto *value = sqlite3_column_text(statement_, column); + const int size = sqlite3_column_bytes(statement_, column); + return value == nullptr ? std::string() : + std::string(reinterpret_cast(value), size); + } + + private: + void Check(const char *action, int rc) { + if (rc != SQLITE_OK) Throw(action, rc); + } + [[noreturn]] void Throw(const char *action, int rc) const { + throw Error(std::string(action) + ": " + sqlite3_errmsg(db_) + + " (rc=" + std::to_string(rc) + ")"); + } + sqlite3 *db_ = nullptr; + sqlite3_stmt *statement_ = nullptr; +}; + +void Exec(sqlite3 *db, const char *sql) { + char *message = nullptr; + const int rc = sqlite3_exec(db, sql, nullptr, nullptr, &message); + if (rc == SQLITE_OK) return; + const std::string detail = message == nullptr ? sqlite3_errmsg(db) : message; + sqlite3_free(message); + throw Error(detail + " (rc=" + std::to_string(rc) + ")"); +} + +Database Open(const std::string &path, int flags) { + sqlite3 *raw = nullptr; + const int rc = sqlite3_open_v2(path.c_str(), &raw, flags, nullptr); + Database db(raw); + if (rc != SQLITE_OK) { + throw Error("cannot open " + path + ": " + + (raw == nullptr ? std::string(sqlite3_errstr(rc)) : sqlite3_errmsg(raw))); + } + sqlite3_extended_result_codes(raw, 1); + sqlite3_busy_timeout(raw, 5000); + return db; +} + +void RegisterVexDB(sqlite3 *db) { + const int rc = vexdb_sqlite_register(db); + if (rc != SQLITE_OK) { + throw Error("cannot register VexDB: " + std::string(sqlite3_errmsg(db))); + } +} + +void HashFunction(sqlite3_context *context, int, sqlite3_value **values) { + const void *data = sqlite3_value_blob(values[0]); + const int size = sqlite3_value_bytes(values[0]); + const std::string result = vexfs::Sha256Hex(size == 0 ? "" : data, + static_cast(size)); + sqlite3_result_text(context, result.c_str(), static_cast(result.size()), + SQLITE_TRANSIENT); +} + +void RegisterHash(sqlite3 *db) { + const int rc = sqlite3_create_function_v2(db, "vexfs_archive_sha256", 1, + SQLITE_UTF8 | SQLITE_DETERMINISTIC | SQLITE_INNOCUOUS, nullptr, + HashFunction, nullptr, nullptr, nullptr); + if (rc != SQLITE_OK) throw Error("cannot register archive checksum function"); +} + +void Attach(sqlite3 *db, const std::string &path, const char *name) { + const std::string sql = std::string("ATTACH DATABASE ?1 AS ") + name; + Statement attach(db, sql.c_str()); + attach.Text(1, path); + attach.Done(); +} + +std::string ReadOnlyUri(const std::string &path) { + const std::string absolute = std::filesystem::absolute(path).generic_string(); + static constexpr char hex[] = "0123456789ABCDEF"; + std::string uri = "file:"; + for (unsigned char value : absolute) { + const bool safe = (value >= 'a' && value <= 'z') || + (value >= 'A' && value <= 'Z') || (value >= '0' && value <= '9') || + value == '/' || value == ':' || value == '-' || value == '_' || + value == '.' || value == '~'; + if (safe) uri.push_back(static_cast(value)); + else { + uri.push_back('%'); + uri.push_back(hex[value >> 4]); + uri.push_back(hex[value & 15]); + } + } + return uri + "?mode=ro"; +} + +void AttachReadOnly(sqlite3 *db, const std::string &path, const char *name) { + Attach(db, ReadOnlyUri(path), name); +} + +std::string JsonEscape(const std::string &input) { + std::string output; + output.reserve(input.size() + 8); + for (unsigned char value : input) { + switch (value) { + case '\\': output += "\\\\"; break; + case '"': output += "\\\""; break; + case '\b': output += "\\b"; break; + case '\f': output += "\\f"; break; + case '\n': output += "\\n"; break; + case '\r': output += "\\r"; break; + case '\t': output += "\\t"; break; + default: + if (value < 0x20) { + static constexpr char hex[] = "0123456789abcdef"; + output += "\\u00"; + output.push_back(hex[value >> 4]); + output.push_back(hex[value & 15]); + } else { + output.push_back(static_cast(value)); + } + } + } + return output; +} + +sqlite3_int64 ScalarInt64(sqlite3 *db, const char *sql) { + Statement statement(db, sql); + if (!statement.Row()) throw Error("query returned no row"); + return statement.Int64(0); +} + +std::string ComputePackageChecksum(sqlite3 *db) { + vexfs::Sha256 hash; + { + Statement manifest(db, R"SQL( +SELECT json_array(format_version,source_engine,source_workspace,source_commit, + source_snapshot,root_source_inode,history_floor_source_commit,retention_keep_versions, + retention_keep_days,quota_max_bytes,quota_max_files,quota_max_file_bytes,created_at,complete) +FROM package.manifest +)SQL"); + if (!manifest.Row()) throw Error("archive manifest is missing"); + const std::string metadata = manifest.Text(0); + hash.Update("manifest\0", 9); + hash.Update(metadata.data(), metadata.size()); + hash.Update("\n", 1); + if (manifest.Row()) throw Error("archive has more than one manifest row"); + } + constexpr std::array tables = { + "commits", "inodes", "dentries", "file_versions", "manifests", "chunks", "inode_states", + "dentry_states", "xattr_states", "acl_states", "snapshots", "xattrs", + "acl_entries", "principals" + }; + for (const char *table : tables) { + const std::string sql = "SELECT record_key,record_checksum FROM package." + + std::string(table) + " ORDER BY record_key"; + Statement rows(db, sql.c_str()); + while (rows.Row()) { + const std::string key = rows.Text(0); + const std::string checksum = rows.Text(1); + hash.Update(table, std::strlen(table)); + hash.Update("\0", 1); + hash.Update(key.data(), key.size()); + hash.Update("\0", 1); + hash.Update(checksum.data(), checksum.size()); + hash.Update("\n", 1); + } + } + return vexfs::Hex(hash.Finish()); +} + +void CreateArchiveSchema(sqlite3 *db) { + Exec(db, R"SQL( +CREATE TABLE package.manifest( + format_version INTEGER NOT NULL,source_engine TEXT NOT NULL,source_workspace TEXT NOT NULL, + source_commit INTEGER NOT NULL,source_snapshot TEXT,root_source_inode INTEGER NOT NULL, + history_floor_source_commit INTEGER NOT NULL,retention_keep_versions INTEGER NOT NULL, + retention_keep_days INTEGER NOT NULL,quota_max_bytes INTEGER,quota_max_files INTEGER, + quota_max_file_bytes INTEGER,created_at INTEGER NOT NULL,package_checksum TEXT, + complete INTEGER NOT NULL CHECK(complete IN (0,1))); +CREATE TABLE package.commits( + record_key TEXT PRIMARY KEY,source_id INTEGER UNIQUE NOT NULL,parent_source_id INTEGER, + message TEXT NOT NULL,created_at INTEGER NOT NULL,record_checksum TEXT NOT NULL); +CREATE TABLE package.inodes( + record_key TEXT PRIMARY KEY,source_id INTEGER UNIQUE NOT NULL,kind TEXT NOT NULL,mode INTEGER NOT NULL, + owner_principal TEXT NOT NULL,uid INTEGER NOT NULL,gid INTEGER NOT NULL,size INTEGER NOT NULL, + current_version INTEGER NOT NULL,created_at INTEGER NOT NULL,accessed_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL,changed_at INTEGER NOT NULL,deleted_at INTEGER, + record_checksum TEXT NOT NULL); +CREATE TABLE package.dentries( + record_key TEXT PRIMARY KEY,parent_source_inode INTEGER NOT NULL,name TEXT NOT NULL, + inode_source_id INTEGER NOT NULL,record_checksum TEXT NOT NULL); +CREATE TABLE package.file_versions( + id INTEGER PRIMARY KEY,record_key TEXT UNIQUE NOT NULL,source_inode INTEGER NOT NULL, + version_no INTEGER NOT NULL,source_commit INTEGER NOT NULL,source_manifest INTEGER, + size INTEGER NOT NULL,checksum TEXT NOT NULL,source_version_no INTEGER,created_at INTEGER NOT NULL, + record_checksum TEXT NOT NULL,UNIQUE(source_inode,version_no)); +CREATE TABLE package.manifests( + id INTEGER PRIMARY KEY,record_key TEXT UNIQUE NOT NULL,source_id INTEGER UNIQUE NOT NULL, + file_size INTEGER NOT NULL,chunk_size INTEGER NOT NULL,chunk_count INTEGER NOT NULL, + checksum TEXT NOT NULL,created_at INTEGER NOT NULL,record_checksum TEXT NOT NULL); +CREATE TABLE package.chunks( + id INTEGER PRIMARY KEY,record_key TEXT UNIQUE NOT NULL,source_manifest INTEGER NOT NULL, + chunk_no INTEGER NOT NULL,content BLOB NOT NULL,size INTEGER NOT NULL,checksum TEXT NOT NULL, + record_checksum TEXT NOT NULL,UNIQUE(source_manifest,chunk_no)); +CREATE TABLE package.inode_states( + record_key TEXT PRIMARY KEY,source_inode INTEGER NOT NULL,source_commit INTEGER NOT NULL, + kind TEXT NOT NULL,mode INTEGER NOT NULL,owner_principal TEXT NOT NULL,uid INTEGER NOT NULL, + gid INTEGER NOT NULL,size INTEGER NOT NULL,current_version INTEGER NOT NULL, + created_at INTEGER NOT NULL,accessed_at INTEGER NOT NULL,updated_at INTEGER NOT NULL, + changed_at INTEGER NOT NULL,deleted_at INTEGER,record_checksum TEXT NOT NULL); +CREATE TABLE package.dentry_states( + record_key TEXT PRIMARY KEY,parent_source_inode INTEGER NOT NULL,name TEXT NOT NULL, + source_commit INTEGER NOT NULL,inode_source_id INTEGER NOT NULL,deleted INTEGER NOT NULL, + record_checksum TEXT NOT NULL); +CREATE TABLE package.xattr_states( + record_key TEXT PRIMARY KEY,source_inode INTEGER NOT NULL,name TEXT NOT NULL, + source_commit INTEGER NOT NULL,value BLOB NOT NULL,deleted INTEGER NOT NULL, + record_checksum TEXT NOT NULL); +CREATE TABLE package.acl_states( + record_key TEXT PRIMARY KEY,source_inode INTEGER NOT NULL,principal_id TEXT NOT NULL, + effect TEXT NOT NULL,source_commit INTEGER NOT NULL,permissions TEXT NOT NULL, + inherit_flags INTEGER NOT NULL,deleted INTEGER NOT NULL,record_checksum TEXT NOT NULL); +CREATE TABLE package.snapshots( + record_key TEXT PRIMARY KEY,name TEXT UNIQUE NOT NULL,source_commit INTEGER NOT NULL, + created_at INTEGER NOT NULL,record_checksum TEXT NOT NULL); +CREATE TABLE package.xattrs( + record_key TEXT PRIMARY KEY,source_inode INTEGER NOT NULL,name TEXT NOT NULL,value BLOB NOT NULL, + updated_at INTEGER NOT NULL,record_checksum TEXT NOT NULL); +CREATE TABLE package.acl_entries( + record_key TEXT PRIMARY KEY,source_inode INTEGER NOT NULL,principal_id TEXT NOT NULL, + effect TEXT NOT NULL,permissions TEXT NOT NULL,inherit_flags INTEGER NOT NULL, + created_at INTEGER NOT NULL,updated_at INTEGER NOT NULL,record_checksum TEXT NOT NULL); +CREATE TABLE package.principals( + record_key TEXT PRIMARY KEY,source_principal TEXT UNIQUE NOT NULL, + record_checksum TEXT NOT NULL); +)SQL"); +} + +void CopyArchiveChunks(sqlite3 *db) { + struct BlobCopy { + sqlite3_int64 package_rowid; + sqlite3_int64 source_rowid; + sqlite3_int64 size; + std::string checksum; + }; + std::vector copies; + { + Statement rows(db, R"SQL( +SELECT package_chunk.id,source_chunk.rowid,package_chunk.size,package_chunk.checksum +FROM package.chunks package_chunk +JOIN _vexfs_manifest_chunks source_entry + ON source_entry.manifest_id=package_chunk.source_manifest + AND source_entry.chunk_no=package_chunk.chunk_no +JOIN _vexfs_chunks source_chunk ON source_chunk.id=source_entry.chunk_id +ORDER BY package_chunk.id +)SQL"); + while (rows.Row()) { + copies.push_back({rows.Int64(0), rows.Int64(1), rows.Int64(2), rows.Text(3)}); + } + } + std::vector buffer(kBlobVerifyChunk); + for (const BlobCopy © : copies) { + const sqlite3_int64 package_rowid = copy.package_rowid; + const sqlite3_int64 source_rowid = copy.source_rowid; + const sqlite3_int64 expected_size = copy.size; + const std::string &expected_checksum = copy.checksum; + if (expected_size < 0 || expected_size > std::numeric_limits::max()) { + throw Error("source chunk size is outside archive format limits"); + } + sqlite3_blob *raw_source = nullptr; + sqlite3_blob *raw_target = nullptr; + if (sqlite3_blob_open(db, "main", "_vexfs_chunks", "content", + source_rowid, 0, &raw_source) != SQLITE_OK) { + throw Error("cannot open source chunk BLOB"); + } + std::unique_ptr source( + raw_source, sqlite3_blob_close); + if (sqlite3_blob_open(db, "package", "chunks", "content", + package_rowid, 1, &raw_target) != SQLITE_OK) { + throw Error("cannot open archive chunk BLOB"); + } + std::unique_ptr target( + raw_target, sqlite3_blob_close); + const int bytes = sqlite3_blob_bytes(source.get()); + if (bytes != expected_size || sqlite3_blob_bytes(target.get()) != bytes) { + throw Error("source chunk size does not match metadata"); + } + vexfs::Sha256 hash; + for (int offset = 0; offset < bytes;) { + const int amount = std::min(static_cast(buffer.size()), bytes - offset); + if (sqlite3_blob_read(source.get(), buffer.data(), amount, offset) != SQLITE_OK || + sqlite3_blob_write(target.get(), buffer.data(), amount, offset) != SQLITE_OK) { + throw Error("cannot stream archive chunk BLOB"); + } + hash.Update(buffer.data(), static_cast(amount)); + offset += amount; + } + if (vexfs::Hex(hash.Finish()) != expected_checksum) { + throw Error("source chunk checksum mismatch during export"); + } + } +} + +void CopyImportedChunks(sqlite3 *db) { + struct BlobCopy { + sqlite3_int64 target_rowid; + sqlite3_int64 package_rowid; + sqlite3_int64 size; + std::string checksum; + }; + std::vector copies; + { + Statement rows(db, R"SQL( +SELECT target.rowid,package_chunk.id,package_chunk.size,package_chunk.checksum +FROM package.chunks package_chunk +JOIN import_chunk_map mapped + ON mapped.source_manifest=package_chunk.source_manifest + AND mapped.chunk_no=package_chunk.chunk_no +JOIN _vexfs_chunks target ON target.id=mapped.local_id +ORDER BY package_chunk.id +)SQL"); + while (rows.Row()) { + copies.push_back({rows.Int64(0), rows.Int64(1), rows.Int64(2), rows.Text(3)}); + } + } + std::vector buffer(kBlobVerifyChunk); + for (const BlobCopy © : copies) { + sqlite3_blob *raw_source = nullptr; + sqlite3_blob *raw_target = nullptr; + if (sqlite3_blob_open(db, "package", "chunks", "content", + copy.package_rowid, 0, &raw_source) != SQLITE_OK) { + throw Error("cannot open package chunk BLOB during import"); + } + std::unique_ptr source( + raw_source, sqlite3_blob_close); + if (sqlite3_blob_open(db, "main", "_vexfs_chunks", "content", + copy.target_rowid, 1, &raw_target) != SQLITE_OK) { + throw Error("cannot open target chunk BLOB during import"); + } + std::unique_ptr target( + raw_target, sqlite3_blob_close); + const int bytes = sqlite3_blob_bytes(source.get()); + if (copy.size < 0 || copy.size > std::numeric_limits::max() || + bytes != copy.size || sqlite3_blob_bytes(target.get()) != bytes) { + throw Error("package chunk size changed during import"); + } + vexfs::Sha256 hash; + for (int offset = 0; offset < bytes;) { + const int amount = std::min(static_cast(buffer.size()), bytes - offset); + if (sqlite3_blob_read(source.get(), buffer.data(), amount, offset) != SQLITE_OK || + sqlite3_blob_write(target.get(), buffer.data(), amount, offset) != SQLITE_OK) { + throw Error("cannot stream package chunk BLOB during import"); + } + hash.Update(buffer.data(), static_cast(amount)); + offset += amount; + } + if (vexfs::Hex(hash.Finish()) != copy.checksum) { + throw Error("package chunk checksum changed during import"); + } + } +} + +} // namespace + +std::string ExportArchive(const std::string &database, const std::string &workspace, + const std::string &snapshot, const std::string &output) { + if (database.empty() || database == ":memory:") { + throw Error("export needs a persistent SQLite database path"); + } + if (!std::filesystem::is_regular_file(database)) { + throw Error("source database file not found: " + database); + } + if (output.empty()) throw Error("export output path is empty"); + if (std::filesystem::exists(output)) { + throw Error("export output already exists: " + output); + } + + // SQLite applies the connection open mode to newly attached databases. Open the + // source read-write so the new package can be created; the source stays in a read + // transaction and is never modified by the export statements below. + Database db = Open(database, SQLITE_OPEN_READWRITE | SQLITE_OPEN_URI); + RegisterVexDB(db.get()); + RegisterHash(db.get()); + bool owns_output = false; + try { + { + Database package = Open(output, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | + SQLITE_OPEN_EXCLUSIVE); + Exec(package.get(), "PRAGMA journal_mode=DELETE; PRAGMA synchronous=FULL;"); + } + owns_output = true; + std::filesystem::permissions( + output, std::filesystem::perms::owner_read | + std::filesystem::perms::owner_write, + std::filesystem::perm_options::replace); + Exec(db.get(), "BEGIN"); + Statement source(db.get(), R"SQL( +SELECT id,root_inode,head_commit,history_floor_commit,retention_keep_versions, + retention_keep_days,quota_max_bytes,quota_max_files,quota_max_file_bytes +FROM _vexfs_workspaces WHERE name=?1 AND state='active' +)SQL"); + source.Text(1, workspace); + if (!source.Row()) throw Error("workspace not found: " + workspace); + const sqlite3_int64 workspace_id = source.Int64(0); + const sqlite3_int64 root_inode = source.Int64(1); + sqlite3_int64 source_commit = source.Int64(2); + const sqlite3_int64 history_floor = source.Int64(3); + const sqlite3_int64 keep_versions = source.Int64(4); + const sqlite3_int64 keep_days = source.Int64(5); + const bool max_bytes_null = source.Type(6) == SQLITE_NULL; + const bool max_files_null = source.Type(7) == SQLITE_NULL; + const bool max_file_bytes_null = source.Type(8) == SQLITE_NULL; + const sqlite3_int64 max_bytes = source.Int64(6); + const sqlite3_int64 max_files = source.Int64(7); + const sqlite3_int64 max_file_bytes = source.Int64(8); + + if (!snapshot.empty() && snapshot != "HEAD") { + Statement selected(db.get(), + "SELECT commit_id FROM _vexfs_snapshots WHERE workspace_id=?1 AND name=?2"); + selected.Int64(1, workspace_id); + selected.Text(2, snapshot); + if (!selected.Row()) throw Error("snapshot not found: " + snapshot); + source_commit = selected.Int64(0); + } else { + Statement dirty(db.get(), + "SELECT 1 FROM _vexfs_handles WHERE workspace_id=?1 " + "AND state IN ('open','retained') " + "AND dirty_generation>published_generation LIMIT 1"); + dirty.Int64(1, workspace_id); + if (dirty.Row()) { + throw Error("workspace has unpublished file handles; synchronize or export " + "an existing snapshot"); + } + } + if (source_commit <= 0 || history_floor <= 0 || source_commit < history_floor) { + throw Error("selected commit is outside restorable workspace history"); + } + + Attach(db.get(), output, "package"); + CreateArchiveSchema(db.get()); + { + Statement manifest(db.get(), R"SQL( +INSERT INTO package.manifest( + format_version,source_engine,source_workspace,source_commit,source_snapshot, + root_source_inode,history_floor_source_commit,retention_keep_versions, + retention_keep_days,quota_max_bytes,quota_max_files,quota_max_file_bytes, + created_at,package_checksum,complete) +VALUES(?1,'sqlite',?2,?3,?4,?5,?6,?7,?8,?9,?10,?11, + CAST(unixepoch('subsec')*1000 AS INTEGER),NULL,0) +)SQL"); + manifest.Int64(1, kArchiveFormatVersion); + manifest.Text(2, workspace); + manifest.Int64(3, source_commit); + if (snapshot.empty() || snapshot == "HEAD") manifest.Null(4); + else manifest.Text(4, snapshot); + manifest.Int64(5, root_inode); + manifest.Int64(6, history_floor); + manifest.Int64(7, keep_versions); + manifest.Int64(8, keep_days); + if (max_bytes_null) manifest.Null(9); else manifest.Int64(9, max_bytes); + if (max_files_null) manifest.Null(10); else manifest.Int64(10, max_files); + if (max_file_bytes_null) manifest.Null(11); else manifest.Int64(11, max_file_bytes); + manifest.Done(); + } + + Exec(db.get(), R"SQL( +CREATE TEMP TABLE selected_commits(id INTEGER PRIMARY KEY,parent_commit INTEGER); +)SQL"); + { + Statement commits(db.get(), R"SQL( +INSERT INTO selected_commits(id,parent_commit) +WITH RECURSIVE chain(id,parent_commit,depth) AS ( + SELECT id,parent_commit,0 FROM _vexfs_commits WHERE id=?1 AND workspace_id=?2 + UNION ALL + SELECT parent.id,parent.parent_commit,chain.depth+1 + FROM chain JOIN _vexfs_commits parent ON parent.id=chain.parent_commit + WHERE parent.workspace_id=?2 AND chain.depth<1000000) +SELECT id,parent_commit FROM chain +)SQL"); + commits.Int64(1, source_commit); + commits.Int64(2, workspace_id); + commits.Done(); + } + if (ScalarInt64(db.get(), "SELECT count(*) FROM selected_commits") == 0) { + throw Error("selected source commit does not exist"); + } + + Exec(db.get(), R"SQL( +INSERT INTO package.commits +SELECT printf('%020lld',c.id),c.id,c.parent_commit,c.message,c.created_at, + vexfs_archive_sha256(CAST(json_array(c.id,c.parent_commit,c.message,c.created_at) AS BLOB)) +FROM _vexfs_commits c JOIN selected_commits selected ON selected.id=c.id; + +INSERT INTO package.inodes +WITH ranked AS ( + SELECT state.*,ROW_NUMBER() OVER(PARTITION BY state.inode_id ORDER BY state.commit_id DESC) rank + FROM _vexfs_inode_states state JOIN selected_commits selected ON selected.id=state.commit_id + WHERE state.workspace_id=(SELECT id FROM _vexfs_workspaces + WHERE name=(SELECT source_workspace FROM package.manifest))) +SELECT printf('%020lld',inode_id),inode_id,kind,mode,owner_principal,uid,gid,size, + current_version,created_at,accessed_at,updated_at,changed_at,deleted_at, + vexfs_archive_sha256(CAST(json_array(inode_id,kind,mode,owner_principal,uid,gid,size, + current_version,created_at,accessed_at,updated_at,changed_at,deleted_at) AS BLOB)) +FROM ranked WHERE rank=1; + +INSERT INTO package.dentries +WITH ranked AS ( + SELECT state.*,ROW_NUMBER() OVER( + PARTITION BY state.parent_inode,state.name ORDER BY state.commit_id DESC) rank + FROM _vexfs_dentry_states state JOIN selected_commits selected ON selected.id=state.commit_id + WHERE state.workspace_id=(SELECT id FROM _vexfs_workspaces + WHERE name=(SELECT source_workspace FROM package.manifest))) +SELECT printf('%020lld:%s',parent_inode,hex(name)),parent_inode,name,inode_id, + vexfs_archive_sha256(CAST(json_array(parent_inode,name,inode_id) AS BLOB)) +FROM ranked WHERE rank=1 AND deleted=0; + +INSERT INTO package.file_versions( + record_key,source_inode,version_no,source_commit,source_manifest,size,checksum, + source_version_no,created_at,record_checksum) +WITH RECURSIVE needed(source_inode,version_no) AS ( + SELECT state.inode_id,state.current_version + FROM _vexfs_inode_states state JOIN selected_commits selected ON selected.id=state.commit_id + WHERE state.workspace_id=(SELECT id FROM _vexfs_workspaces + WHERE name=(SELECT source_workspace FROM package.manifest)) + AND state.kind<>'directory' AND state.current_version>0 + UNION + SELECT version.inode_id,version.source_version_no + FROM needed JOIN _vexfs_file_versions version + ON version.inode_id=needed.source_inode AND version.version_no=needed.version_no + WHERE version.source_version_no IS NOT NULL) +SELECT printf('%020lld:%020lld',version.inode_id,version.version_no),version.inode_id, + version.version_no,version.commit_id,version.manifest_id,version.size,version.checksum, + version.source_version_no,version.created_at, + vexfs_archive_sha256(CAST(json_array(version.inode_id,version.version_no, + version.commit_id,version.manifest_id,version.size,version.checksum,version.source_version_no, + version.created_at) AS BLOB)) +FROM _vexfs_file_versions version JOIN needed + ON needed.source_inode=version.inode_id AND needed.version_no=version.version_no; + +INSERT INTO package.manifests( + record_key,source_id,file_size,chunk_size,chunk_count,checksum,created_at,record_checksum) +SELECT printf('%020lld',manifest.id),manifest.id,manifest.file_size,manifest.chunk_size, + manifest.chunk_count,manifest.checksum,manifest.created_at, + vexfs_archive_sha256(CAST(json_array(manifest.id,manifest.file_size, + manifest.chunk_size,manifest.chunk_count,manifest.checksum,manifest.created_at) AS BLOB)) +FROM _vexfs_manifests manifest +JOIN (SELECT DISTINCT source_manifest FROM package.file_versions + WHERE source_manifest IS NOT NULL) needed + ON needed.source_manifest=manifest.id; + +INSERT INTO package.chunks( + record_key,source_manifest,chunk_no,content,size,checksum,record_checksum) +SELECT printf('%020lld:%020lld',entry.manifest_id,entry.chunk_no),entry.manifest_id, + entry.chunk_no,zeroblob(chunk.size),chunk.size,chunk.checksum, + vexfs_archive_sha256(CAST(json_array(entry.manifest_id,entry.chunk_no, + chunk.size,chunk.checksum) AS BLOB)) +FROM _vexfs_manifest_chunks entry +JOIN _vexfs_chunks chunk ON chunk.id=entry.chunk_id +JOIN package.manifests manifest ON manifest.source_id=entry.manifest_id; + +INSERT INTO package.inode_states +SELECT printf('%020lld:%020lld',state.inode_id,state.commit_id),state.inode_id,state.commit_id, + state.kind,state.mode,state.owner_principal,state.uid,state.gid,state.size, + state.current_version,state.created_at,state.accessed_at,state.updated_at,state.changed_at, + state.deleted_at, + vexfs_archive_sha256(CAST(json_array(state.inode_id,state.commit_id,state.kind,state.mode, + state.owner_principal,state.uid,state.gid,state.size,state.current_version, + state.created_at,state.accessed_at,state.updated_at,state.changed_at,state.deleted_at) + AS BLOB)) +FROM _vexfs_inode_states state JOIN selected_commits selected ON selected.id=state.commit_id +WHERE state.workspace_id=(SELECT id FROM _vexfs_workspaces + WHERE name=(SELECT source_workspace FROM package.manifest)); + +INSERT INTO package.dentry_states +SELECT printf('%020lld:%s:%020lld',state.parent_inode,hex(state.name),state.commit_id), + state.parent_inode,state.name,state.commit_id,state.inode_id,state.deleted, + vexfs_archive_sha256(CAST(json_array(state.parent_inode,state.name,state.commit_id, + state.inode_id,state.deleted) AS BLOB)) +FROM _vexfs_dentry_states state JOIN selected_commits selected ON selected.id=state.commit_id +WHERE state.workspace_id=(SELECT id FROM _vexfs_workspaces + WHERE name=(SELECT source_workspace FROM package.manifest)); + +INSERT INTO package.xattr_states +SELECT printf('%020lld:%s:%020lld',state.inode_id,hex(state.name),state.commit_id), + state.inode_id,state.name,state.commit_id,state.value,state.deleted, + vexfs_archive_sha256(CAST(json_array(state.inode_id,state.name,state.commit_id, + hex(state.value),state.deleted) AS BLOB)) +FROM _vexfs_xattr_states state JOIN selected_commits selected ON selected.id=state.commit_id +WHERE state.workspace_id=(SELECT id FROM _vexfs_workspaces + WHERE name=(SELECT source_workspace FROM package.manifest)); + +INSERT INTO package.acl_states +SELECT printf('%020lld:%s:%s:%020lld',state.inode_id,hex(state.principal_id),state.effect, + state.commit_id),state.inode_id,state.principal_id,state.effect,state.commit_id, + state.permissions,state.inherit_flags,state.deleted, + vexfs_archive_sha256(CAST(json_array(state.inode_id,state.principal_id,state.effect, + state.commit_id,state.permissions,state.inherit_flags,state.deleted) AS BLOB)) +FROM _vexfs_acl_states state JOIN selected_commits selected ON selected.id=state.commit_id +WHERE state.workspace_id=(SELECT id FROM _vexfs_workspaces + WHERE name=(SELECT source_workspace FROM package.manifest)); + +INSERT INTO package.snapshots +SELECT hex(snapshot.name),snapshot.name,snapshot.commit_id,snapshot.created_at, + vexfs_archive_sha256(CAST(json_array(snapshot.name,snapshot.commit_id, + snapshot.created_at) AS BLOB)) +FROM _vexfs_snapshots snapshot JOIN selected_commits selected ON selected.id=snapshot.commit_id +WHERE snapshot.workspace_id=(SELECT id FROM _vexfs_workspaces + WHERE name=(SELECT source_workspace FROM package.manifest)); + +INSERT INTO package.xattrs +WITH ranked AS ( + SELECT state.*,ROW_NUMBER() OVER(PARTITION BY state.inode_id,state.name + ORDER BY state.commit_id DESC) rank + FROM _vexfs_xattr_states state JOIN selected_commits selected ON selected.id=state.commit_id + WHERE state.workspace_id=(SELECT id FROM _vexfs_workspaces + WHERE name=(SELECT source_workspace FROM package.manifest))) +SELECT printf('%020lld:%s',inode_id,hex(name)),inode_id,name,value, + (SELECT created_at FROM package.manifest), + vexfs_archive_sha256(CAST(json_array(inode_id,name,hex(value), + (SELECT created_at FROM package.manifest)) AS BLOB)) +FROM ranked WHERE rank=1 AND deleted=0; + +INSERT INTO package.acl_entries +WITH ranked AS ( + SELECT state.*,ROW_NUMBER() OVER(PARTITION BY state.inode_id,state.principal_id,state.effect + ORDER BY state.commit_id DESC) rank + FROM _vexfs_acl_states state JOIN selected_commits selected ON selected.id=state.commit_id + WHERE state.workspace_id=(SELECT id FROM _vexfs_workspaces + WHERE name=(SELECT source_workspace FROM package.manifest))) +SELECT printf('%020lld:%s:%s',inode_id,hex(principal_id),effect),inode_id,principal_id,effect, + permissions,inherit_flags,(SELECT created_at FROM package.manifest), + (SELECT created_at FROM package.manifest), + vexfs_archive_sha256(CAST(json_array(inode_id,principal_id,effect,permissions, + inherit_flags,(SELECT created_at FROM package.manifest), + (SELECT created_at FROM package.manifest)) AS BLOB)) +FROM ranked WHERE rank=1 AND deleted=0; + +INSERT INTO package.principals +SELECT hex(principal),principal, + vexfs_archive_sha256(CAST(json_array(principal) AS BLOB)) +FROM ( + SELECT owner_principal principal FROM package.inodes + UNION + SELECT principal_id FROM package.acl_states + UNION + SELECT principal_id FROM package.acl_entries) +WHERE principal<>''; +)SQL"); + + CopyArchiveChunks(db.get()); + Exec(db.get(), "UPDATE package.manifest SET complete=1"); + const std::string checksum = ComputePackageChecksum(db.get()); + { + Statement finish(db.get(), + "UPDATE package.manifest SET package_checksum=?1"); + finish.Text(1, checksum); + finish.Done(); + } + Exec(db.get(), "COMMIT"); + + const auto bytes = std::filesystem::file_size(output); + return "{\"format_version\":" + std::to_string(kArchiveFormatVersion) + + ",\"workspace\":\"" + JsonEscape(workspace) + + "\",\"source_commit\":" + std::to_string(source_commit) + + ",\"package_checksum\":\"" + checksum + + "\",\"bytes\":" + std::to_string(bytes) + "}"; + } catch (...) { + try { Exec(db.get(), "ROLLBACK"); } catch (...) {} + db.reset(); + if (owns_output) { + std::error_code ignored; + std::filesystem::remove(output, ignored); + } + throw; + } +} + +namespace { + +struct ArchiveInfo { + std::string workspace; + sqlite3_int64 source_commit = 0; + sqlite3_int64 versions = 0; + sqlite3_int64 content_bytes = 0; + std::string checksum; +}; + +void RequireZero(sqlite3 *db, const char *sql, const char *message) { + if (ScalarInt64(db, sql) != 0) throw Error(message); +} + +ArchiveInfo VerifyAttachedArchive(sqlite3 *db) { + Exec(db, "PRAGMA trusted_schema=OFF"); + { + Statement quick(db, "PRAGMA package.quick_check"); + if (!quick.Row() || quick.Text(0) != "ok") { + throw Error("archive SQLite container is corrupt"); + } + } + RequireZero(db, R"SQL( +SELECT count(*) FROM package.sqlite_master +WHERE type IN ('view','trigger') OR (type='table' AND name NOT IN ( + 'manifest','commits','inodes','dentries','file_versions','manifests','chunks','inode_states', + 'dentry_states','xattr_states','acl_states','snapshots','xattrs','acl_entries','principals')) +)SQL", "archive has an unexpected schema object"); + if (ScalarInt64(db, R"SQL( +SELECT count(*) FROM package.sqlite_master WHERE type='table' AND name IN ( + 'manifest','commits','inodes','dentries','file_versions','manifests','chunks','inode_states', + 'dentry_states','xattr_states','acl_states','snapshots','xattrs','acl_entries','principals') +)SQL") != 15) { + throw Error("archive schema is incomplete"); + } + + ArchiveInfo info; + { + Statement manifest(db, R"SQL( +SELECT format_version,source_workspace,source_commit,package_checksum,complete +FROM package.manifest +)SQL"); + if (!manifest.Row()) throw Error("archive manifest is missing"); + if (manifest.Int(0) != kArchiveFormatVersion) { + throw Error("unsupported archive format version: " + + std::to_string(manifest.Int(0))); + } + info.workspace = manifest.Text(1); + info.source_commit = manifest.Int64(2); + info.checksum = manifest.Text(3); + if (manifest.Int(4) != 1 || info.checksum.size() != 64) { + throw Error("archive was not completed"); + } + if (manifest.Row()) throw Error("archive has more than one manifest row"); + } + + RequireZero(db, R"SQL( +SELECT sum(bad) FROM ( + SELECT count(*) bad FROM package.commits WHERE record_checksum<> + vexfs_archive_sha256(CAST(json_array(source_id,parent_source_id,message,created_at) AS BLOB)) + UNION ALL SELECT count(*) FROM package.inodes WHERE record_checksum<> + vexfs_archive_sha256(CAST(json_array(source_id,kind,mode,owner_principal,uid,gid,size, + current_version,created_at,accessed_at,updated_at,changed_at,deleted_at) AS BLOB))) +)SQL", "archive commit or inode record checksum mismatch"); + RequireZero(db, R"SQL( +SELECT sum(bad) FROM ( + SELECT count(*) bad FROM package.dentries WHERE record_checksum<> + vexfs_archive_sha256(CAST(json_array(parent_source_inode,name,inode_source_id) AS BLOB)) + UNION ALL SELECT count(*) FROM package.file_versions WHERE record_checksum<> + vexfs_archive_sha256(CAST(json_array(source_inode,version_no,source_commit,source_manifest, + size,checksum,source_version_no,created_at) AS BLOB)) + UNION ALL SELECT count(*) FROM package.manifests WHERE record_checksum<> + vexfs_archive_sha256(CAST(json_array(source_id,file_size,chunk_size,chunk_count,checksum, + created_at) AS BLOB)) + UNION ALL SELECT count(*) FROM package.chunks WHERE record_checksum<> + vexfs_archive_sha256(CAST(json_array(source_manifest,chunk_no,size,checksum) AS BLOB)) + UNION ALL SELECT count(*) FROM package.inode_states WHERE record_checksum<> + vexfs_archive_sha256(CAST(json_array(source_inode,source_commit,kind,mode,owner_principal, + uid,gid,size,current_version,created_at,accessed_at,updated_at,changed_at,deleted_at) AS BLOB)) + UNION ALL SELECT count(*) FROM package.dentry_states WHERE record_checksum<> + vexfs_archive_sha256(CAST(json_array(parent_source_inode,name,source_commit,inode_source_id, + deleted) AS BLOB)) + UNION ALL SELECT count(*) FROM package.xattr_states WHERE record_checksum<> + vexfs_archive_sha256(CAST(json_array(source_inode,name,source_commit,hex(value),deleted) AS BLOB)) + UNION ALL SELECT count(*) FROM package.acl_states WHERE record_checksum<> + vexfs_archive_sha256(CAST(json_array(source_inode,principal_id,effect,source_commit, + permissions,inherit_flags,deleted) AS BLOB)) + UNION ALL SELECT count(*) FROM package.snapshots WHERE record_checksum<> + vexfs_archive_sha256(CAST(json_array(name,source_commit,created_at) AS BLOB)) + UNION ALL SELECT count(*) FROM package.xattrs WHERE record_checksum<> + vexfs_archive_sha256(CAST(json_array(source_inode,name,hex(value),updated_at) AS BLOB)) + UNION ALL SELECT count(*) FROM package.acl_entries WHERE record_checksum<> + vexfs_archive_sha256(CAST(json_array(source_inode,principal_id,effect,permissions, + inherit_flags,created_at,updated_at) AS BLOB)) + UNION ALL SELECT count(*) FROM package.principals WHERE record_checksum<> + vexfs_archive_sha256(CAST(json_array(source_principal) AS BLOB))) +)SQL", "archive record checksum mismatch"); + + RequireZero(db, R"SQL( +SELECT count(*) FROM package.manifest manifest +WHERE NOT EXISTS(SELECT 1 FROM package.commits WHERE source_id=manifest.source_commit) + OR NOT EXISTS(SELECT 1 FROM package.commits + WHERE source_id=manifest.history_floor_source_commit) + OR NOT EXISTS(SELECT 1 FROM package.inodes + WHERE source_id=manifest.root_source_inode AND kind='directory' + AND deleted_at IS NULL) +)SQL", "archive manifest references a missing object"); + RequireZero(db, R"SQL( +SELECT sum(bad) FROM ( + SELECT count(*) bad FROM package.commits child + WHERE child.parent_source_id IS NOT NULL AND NOT EXISTS( + SELECT 1 FROM package.commits parent WHERE parent.source_id=child.parent_source_id) + UNION ALL SELECT count(*) FROM package.dentries entry + WHERE NOT EXISTS(SELECT 1 FROM package.inodes inode + WHERE inode.source_id=entry.parent_source_inode AND inode.kind='directory' + AND inode.deleted_at IS NULL) + OR NOT EXISTS(SELECT 1 FROM package.inodes inode + WHERE inode.source_id=entry.inode_source_id AND inode.deleted_at IS NULL) + UNION ALL SELECT count(*) FROM package.file_versions version + WHERE NOT EXISTS(SELECT 1 FROM package.inodes inode + WHERE inode.source_id=version.source_inode) + OR NOT EXISTS(SELECT 1 FROM package.commits commit_row + WHERE commit_row.source_id=version.source_commit) + OR (version.source_version_no IS NULL AND + (version.source_manifest IS NULL OR NOT EXISTS( + SELECT 1 FROM package.manifests manifest + WHERE manifest.source_id=version.source_manifest + AND manifest.file_size=version.size AND manifest.checksum=version.checksum))) + OR (version.source_version_no IS NOT NULL AND version.source_manifest IS NOT NULL) + UNION ALL SELECT count(*) FROM package.manifests manifest + WHERE manifest.chunk_size<>65536 OR manifest.file_size<0 OR manifest.chunk_count<0 + OR manifest.chunk_count<>(manifest.file_size+65535)/65536 + OR (SELECT count(*) FROM package.file_versions version + WHERE version.source_manifest=manifest.source_id + AND version.source_version_no IS NULL)<>1 + UNION ALL SELECT count(*) FROM package.chunks chunk + WHERE NOT EXISTS(SELECT 1 FROM package.manifests manifest + WHERE manifest.source_id=chunk.source_manifest) + OR chunk.chunk_no<0 OR chunk.size<=0 OR chunk.size>65536 + UNION ALL SELECT count(*) FROM package.inodes inode + WHERE inode.kind<>'directory' AND inode.current_version>0 AND NOT EXISTS( + SELECT 1 FROM package.file_versions version + WHERE version.source_inode=inode.source_id AND version.version_no=inode.current_version) + UNION ALL SELECT count(*) FROM package.snapshots snapshot + WHERE NOT EXISTS(SELECT 1 FROM package.commits commit_row + WHERE commit_row.source_id=snapshot.source_commit)) +)SQL", "archive contains a broken object reference"); + RequireZero(db, R"SQL( +SELECT count(*) FROM package.file_versions alias +WHERE alias.source_version_no IS NOT NULL AND ( + alias.source_manifest IS NOT NULL OR NOT EXISTS( + SELECT 1 FROM package.file_versions source + WHERE source.source_inode=alias.source_inode + AND source.version_no=alias.source_version_no + AND source.source_version_no IS NULL + AND source.size=alias.size AND source.checksum=alias.checksum)) +)SQL", "archive contains a broken content alias"); + RequireZero(db, R"SQL( +SELECT sum(bad) FROM ( + SELECT count(*) bad FROM package.inode_states state + WHERE NOT EXISTS(SELECT 1 FROM package.inodes inode + WHERE inode.source_id=state.source_inode) + OR NOT EXISTS(SELECT 1 FROM package.commits commit_row + WHERE commit_row.source_id=state.source_commit) + OR (state.kind<>'directory' AND state.current_version>0 AND NOT EXISTS( + SELECT 1 FROM package.file_versions version + WHERE version.source_inode=state.source_inode + AND version.version_no=state.current_version)) + UNION ALL SELECT count(*) FROM package.dentry_states state + WHERE NOT EXISTS(SELECT 1 FROM package.inodes inode + WHERE inode.source_id=state.parent_source_inode AND inode.kind='directory') + OR NOT EXISTS(SELECT 1 FROM package.inodes inode + WHERE inode.source_id=state.inode_source_id) + OR NOT EXISTS(SELECT 1 FROM package.commits commit_row + WHERE commit_row.source_id=state.source_commit) + UNION ALL SELECT count(*) FROM package.xattr_states state + WHERE NOT EXISTS(SELECT 1 FROM package.inodes inode + WHERE inode.source_id=state.source_inode) + OR NOT EXISTS(SELECT 1 FROM package.commits commit_row + WHERE commit_row.source_id=state.source_commit) + UNION ALL SELECT count(*) FROM package.acl_states state + WHERE NOT EXISTS(SELECT 1 FROM package.inodes inode + WHERE inode.source_id=state.source_inode) + OR NOT EXISTS(SELECT 1 FROM package.commits commit_row + WHERE commit_row.source_id=state.source_commit) + UNION ALL SELECT count(*) FROM package.xattrs attribute + WHERE NOT EXISTS(SELECT 1 FROM package.inodes inode + WHERE inode.source_id=attribute.source_inode AND inode.deleted_at IS NULL) + UNION ALL SELECT count(*) FROM package.acl_entries entry + WHERE NOT EXISTS(SELECT 1 FROM package.inodes inode + WHERE inode.source_id=entry.source_inode AND inode.deleted_at IS NULL) + UNION ALL SELECT count(*) FROM package.inodes inode + WHERE NOT EXISTS(SELECT 1 FROM package.principals principal + WHERE principal.source_principal=inode.owner_principal) + UNION ALL SELECT count(*) FROM package.acl_states entry + WHERE NOT EXISTS(SELECT 1 FROM package.principals principal + WHERE principal.source_principal=entry.principal_id)) +)SQL", "archive history or metadata references a missing object"); + RequireZero(db, R"SQL( +WITH RECURSIVE reachable(source_id) AS ( + SELECT root_source_inode FROM package.manifest + UNION + SELECT entry.inode_source_id FROM package.dentries entry + JOIN reachable parent ON parent.source_id=entry.parent_source_inode +), counts AS ( + SELECT (SELECT count(*) FROM package.inodes WHERE deleted_at IS NULL) live_count, + (SELECT count(*) FROM reachable) reachable_count) +SELECT (live_count<>reachable_count) + OR EXISTS(SELECT 1 FROM package.dentries entry + JOIN package.manifest manifest + ON entry.inode_source_id=manifest.root_source_inode) + OR EXISTS( + SELECT 1 FROM package.inodes directory + WHERE directory.kind='directory' AND directory.deleted_at IS NULL + AND directory.source_id<>(SELECT root_source_inode FROM package.manifest) + AND (SELECT count(*) FROM package.dentries entry + WHERE entry.inode_source_id=directory.source_id)<>1) +FROM counts +)SQL", "archive directory tree is cyclic or unreachable"); + + Statement manifests(db, R"SQL( +SELECT source_id,file_size,chunk_count,checksum FROM package.manifests ORDER BY source_id +)SQL"); + std::vector buffer(kBlobVerifyChunk); + while (manifests.Row()) { + const sqlite3_int64 source_manifest = manifests.Int64(0); + const sqlite3_int64 expected_size = manifests.Int64(1); + const sqlite3_int64 expected_chunks = manifests.Int64(2); + const std::string expected_checksum = manifests.Text(3); + if (expected_size < 0 || expected_chunks < 0 || expected_checksum.size() != 64) { + throw Error("archive manifest content metadata is invalid"); + } + Statement chunks(db, R"SQL( +SELECT id,chunk_no,size,checksum,length(content) FROM package.chunks +WHERE source_manifest=?1 ORDER BY chunk_no +)SQL"); + chunks.Int64(1, source_manifest); + vexfs::Sha256 file_hash; + sqlite3_int64 total = 0; + sqlite3_int64 chunk_no = 0; + while (chunks.Row()) { + const sqlite3_int64 rowid = chunks.Int64(0); + const sqlite3_int64 stored_no = chunks.Int64(1); + const sqlite3_int64 chunk_size = chunks.Int64(2); + const std::string chunk_checksum = chunks.Text(3); + const sqlite3_int64 expected_chunk_size = std::min( + 65536, expected_size - total); + if (stored_no != chunk_no || expected_chunk_size <= 0 || + chunk_size != expected_chunk_size || chunks.Int64(4) != chunk_size || + chunk_checksum.size() != 64 || chunk_size > std::numeric_limits::max()) { + throw Error("archive chunk size or order is invalid"); + } + sqlite3_blob *raw_blob = nullptr; + if (sqlite3_blob_open(db, "package", "chunks", "content", rowid, 0, + &raw_blob) != SQLITE_OK) { + throw Error("cannot open archive chunk BLOB"); + } + std::unique_ptr blob( + raw_blob, sqlite3_blob_close); + vexfs::Sha256 chunk_hash; + int offset = 0; + const int bytes = sqlite3_blob_bytes(blob.get()); + while (offset < bytes) { + const int amount = std::min(static_cast(buffer.size()), bytes - offset); + if (sqlite3_blob_read(blob.get(), buffer.data(), amount, offset) != SQLITE_OK) { + throw Error("cannot read archive chunk BLOB"); + } + chunk_hash.Update(buffer.data(), static_cast(amount)); + file_hash.Update(buffer.data(), static_cast(amount)); + offset += amount; + } + if (vexfs::Hex(chunk_hash.Finish()) != chunk_checksum) { + throw Error("archive chunk checksum mismatch"); + } + total += chunk_size; + ++chunk_no; + } + if (total != expected_size || chunk_no != expected_chunks || + vexfs::Hex(file_hash.Finish()) != expected_checksum) { + throw Error("archive content checksum mismatch"); + } + info.content_bytes += expected_size; + } + info.versions = ScalarInt64(db, "SELECT count(*) FROM package.file_versions"); + + const std::string actual_checksum = ComputePackageChecksum(db); + if (actual_checksum != info.checksum) throw Error("archive package checksum mismatch"); + return info; +} + +} // namespace + +std::string VerifyArchive(const std::string &input) { + if (!std::filesystem::is_regular_file(input)) { + throw Error("archive file not found: " + input); + } + Database db = Open(":memory:", SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_URI); + RegisterHash(db.get()); + AttachReadOnly(db.get(), input, "package"); + Exec(db.get(), "PRAGMA query_only=ON"); + const ArchiveInfo info = VerifyAttachedArchive(db.get()); + return "{\"ok\":true,\"format_version\":" + + std::to_string(kArchiveFormatVersion) + ",\"workspace\":\"" + + JsonEscape(info.workspace) + "\",\"source_commit\":" + + std::to_string(info.source_commit) + ",\"versions\":" + + std::to_string(info.versions) + ",\"content_bytes\":" + + std::to_string(info.content_bytes) + ",\"package_checksum\":\"" + + info.checksum + "\"}"; +} + +std::string ImportArchive(const std::string &database, const std::string &workspace, + const std::string &input) { + if (database.empty() || database == ":memory:") { + throw Error("import needs a persistent SQLite database path"); + } + if (workspace.empty() || workspace.size() > 128) { + throw Error("workspace name must be 1..128 bytes"); + } + if (!std::filesystem::is_regular_file(input)) { + throw Error("archive file not found: " + input); + } + + const bool database_exists = std::filesystem::exists(database); + if (!database_exists) { + Database created = Open(database, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | + SQLITE_OPEN_EXCLUSIVE); + created.reset(); + std::filesystem::permissions( + database, std::filesystem::perms::owner_read | + std::filesystem::perms::owner_write, + std::filesystem::perm_options::replace); + } + NewDatabaseCleanup new_database(database, !database_exists); + Database db = Open(database, SQLITE_OPEN_READWRITE | SQLITE_OPEN_URI); + RegisterVexDB(db.get()); + RegisterHash(db.get()); + AttachReadOnly(db.get(), input, "package"); + Exec(db.get(), "SELECT vexfs_init()"); + + Exec(db.get(), "BEGIN IMMEDIATE"); + try { + // Verify and import from the same attached-database snapshot. This closes + // the window where another process could change the archive after + // verification but before rows are copied. + const ArchiveInfo info = VerifyAttachedArchive(db.get()); + { + Statement existing(db.get(), + "SELECT 1 FROM _vexfs_workspaces WHERE name=?1 LIMIT 1"); + existing.Text(1, workspace); + if (existing.Row()) throw Error("workspace already exists: " + workspace); + } + Statement source_manifest(db.get(), R"SQL( +SELECT root_source_inode,source_commit,history_floor_source_commit, + retention_keep_versions,retention_keep_days,quota_max_bytes,quota_max_files, + quota_max_file_bytes,created_at +FROM package.manifest +)SQL"); + if (!source_manifest.Row()) throw Error("archive manifest is missing"); + const sqlite3_int64 source_root = source_manifest.Int64(0); + const sqlite3_int64 source_head = source_manifest.Int64(1); + const sqlite3_int64 source_floor = source_manifest.Int64(2); + + Statement workspace_insert(db.get(), R"SQL( +INSERT INTO _vexfs_workspaces( + name,state,root_inode,head_commit,history_floor_commit,retention_keep_versions, + retention_keep_days,quota_max_bytes,quota_max_files,quota_max_file_bytes,created_at) +VALUES(?1,'importing',NULL,NULL,NULL,?2,?3,?4,?5,?6,?7) +)SQL"); + workspace_insert.Text(1, workspace); + workspace_insert.Int64(2, source_manifest.Int64(3)); + workspace_insert.Int64(3, source_manifest.Int64(4)); + if (source_manifest.Type(5) == SQLITE_NULL) workspace_insert.Null(4); + else workspace_insert.Int64(4, source_manifest.Int64(5)); + if (source_manifest.Type(6) == SQLITE_NULL) workspace_insert.Null(5); + else workspace_insert.Int64(5, source_manifest.Int64(6)); + if (source_manifest.Type(7) == SQLITE_NULL) workspace_insert.Null(6); + else workspace_insert.Int64(6, source_manifest.Int64(7)); + workspace_insert.Int64(7, source_manifest.Int64(8)); + workspace_insert.Done(); + const sqlite3_int64 workspace_id = sqlite3_last_insert_rowid(db.get()); + + Exec(db.get(), R"SQL( +CREATE TEMP TABLE import_commit_map(source_id INTEGER PRIMARY KEY,local_id INTEGER UNIQUE NOT NULL); +CREATE TEMP TABLE import_inode_map(source_id INTEGER PRIMARY KEY,local_id INTEGER UNIQUE NOT NULL); +CREATE TEMP TABLE import_manifest_map(source_id INTEGER PRIMARY KEY,local_id INTEGER UNIQUE NOT NULL); +CREATE TEMP TABLE import_chunk_map( + source_manifest INTEGER NOT NULL,chunk_no INTEGER NOT NULL,local_id INTEGER NOT NULL, + PRIMARY KEY(source_manifest,chunk_no)); +)SQL"); + std::unordered_map commit_map; + { + Statement rows(db.get(), + "SELECT source_id,parent_source_id,message,created_at " + "FROM package.commits ORDER BY source_id"); + while (rows.Row()) { + const sqlite3_int64 source_id = rows.Int64(0); + const bool has_parent = rows.Type(1) != SQLITE_NULL; + sqlite3_int64 local_parent = 0; + if (has_parent) { + const auto parent = commit_map.find(rows.Int64(1)); + if (parent == commit_map.end()) { + throw Error("archive commit order is not a valid parent chain"); + } + local_parent = parent->second; + } + Statement insert(db.get(), R"SQL( +INSERT INTO _vexfs_commits(workspace_id,parent_commit,message,created_at) +VALUES(?1,?2,?3,?4) +)SQL"); + insert.Int64(1, workspace_id); + if (has_parent) insert.Int64(2, local_parent); else insert.Null(2); + insert.Text(3, rows.Text(2)); + insert.Int64(4, rows.Int64(3)); + insert.Done(); + const sqlite3_int64 local_id = sqlite3_last_insert_rowid(db.get()); + commit_map[source_id] = local_id; + Statement map_row(db.get(), + "INSERT INTO import_commit_map(source_id,local_id) VALUES(?1,?2)"); + map_row.Int64(1, source_id); + map_row.Int64(2, local_id); + map_row.Done(); + } + } + + std::unordered_map inode_map; + { + Statement rows(db.get(), R"SQL( +SELECT source_id,kind,mode,owner_principal,uid,gid,size,current_version, + created_at,accessed_at,updated_at,changed_at,deleted_at +FROM package.inodes ORDER BY source_id +)SQL"); + while (rows.Row()) { + Statement insert(db.get(), R"SQL( +INSERT INTO _vexfs_inodes( + workspace_id,kind,mode,owner_principal,uid,gid,size,current_version, + created_at,accessed_at,updated_at,changed_at,deleted_at) +VALUES(?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11,?12,?13) +)SQL"); + insert.Int64(1, workspace_id); + insert.Text(2, rows.Text(1)); + for (int source_column = 2, target_parameter = 3; + source_column <= 11; ++source_column, ++target_parameter) { + if (source_column == 3) insert.Text(target_parameter, rows.Text(source_column)); + else insert.Int64(target_parameter, rows.Int64(source_column)); + } + if (rows.Type(12) == SQLITE_NULL) insert.Null(13); + else insert.Int64(13, rows.Int64(12)); + insert.Done(); + const sqlite3_int64 local_id = sqlite3_last_insert_rowid(db.get()); + const sqlite3_int64 source_id = rows.Int64(0); + inode_map[source_id] = local_id; + Statement map_row(db.get(), + "INSERT INTO import_inode_map(source_id,local_id) VALUES(?1,?2)"); + map_row.Int64(1, source_id); + map_row.Int64(2, local_id); + map_row.Done(); + } + } + const auto root = inode_map.find(source_root); + const auto head = commit_map.find(source_head); + const auto floor = commit_map.find(source_floor); + if (root == inode_map.end() || head == commit_map.end() || floor == commit_map.end()) { + throw Error("archive manifest mapping is incomplete"); + } + + { + Statement rows(db.get(), R"SQL( +SELECT source_id,file_size,chunk_size,chunk_count,checksum,created_at +FROM package.manifests ORDER BY source_id +)SQL"); + while (rows.Row()) { + Statement insert(db.get(), R"SQL( +INSERT INTO _vexfs_manifests( + workspace_id,file_size,chunk_size,chunk_count,checksum,created_at) +VALUES(?1,?2,?3,?4,?5,?6) +)SQL"); + insert.Int64(1, workspace_id); + insert.Int64(2, rows.Int64(1)); + insert.Int64(3, rows.Int64(2)); + insert.Int64(4, rows.Int64(3)); + insert.Text(5, rows.Text(4)); + insert.Int64(6, rows.Int64(5)); + insert.Done(); + Statement map_row(db.get(), + "INSERT INTO import_manifest_map(source_id,local_id) VALUES(?1,?2)"); + map_row.Int64(1, rows.Int64(0)); + map_row.Int64(2, sqlite3_last_insert_rowid(db.get())); + map_row.Done(); + } + } + { + std::unordered_map reusable_chunks; + Statement rows(db.get(), R"SQL( +SELECT chunk.source_manifest,chunk.chunk_no,chunk.size,chunk.checksum,inode.local_id +FROM package.chunks chunk +JOIN package.file_versions version + ON version.source_manifest=chunk.source_manifest AND version.source_version_no IS NULL +JOIN import_inode_map inode ON inode.source_id=version.source_inode +ORDER BY chunk.source_manifest,chunk.chunk_no +)SQL"); + while (rows.Row()) { + const std::string reuse_key = std::to_string(rows.Int64(4)) + ":" + + std::to_string(rows.Int64(2)) + ":" + rows.Text(3); + sqlite3_int64 local_chunk = 0; + const auto reusable = reusable_chunks.find(reuse_key); + if (reusable != reusable_chunks.end()) { + local_chunk = reusable->second; + } else { + Statement insert(db.get(), R"SQL( +INSERT INTO _vexfs_chunks(workspace_id,inode_id,content,size,checksum) +VALUES(?1,?2,zeroblob(?3),?3,?4) +)SQL"); + insert.Int64(1, workspace_id); + insert.Int64(2, rows.Int64(4)); + insert.Int64(3, rows.Int64(2)); + insert.Text(4, rows.Text(3)); + insert.Done(); + local_chunk = sqlite3_last_insert_rowid(db.get()); + reusable_chunks.emplace(reuse_key, local_chunk); + } + Statement map_row(db.get(), R"SQL( +INSERT INTO import_chunk_map(source_manifest,chunk_no,local_id) +VALUES(?1,?2,?3) +)SQL"); + map_row.Int64(1, rows.Int64(0)); + map_row.Int64(2, rows.Int64(1)); + map_row.Int64(3, local_chunk); + map_row.Done(); + } + } + Exec(db.get(), R"SQL( +INSERT INTO _vexfs_manifest_chunks(manifest_id,chunk_no,chunk_id) +SELECT manifest.local_id,chunk.chunk_no,chunk.local_id +FROM import_chunk_map chunk +JOIN import_manifest_map manifest ON manifest.source_id=chunk.source_manifest; +)SQL"); + CopyImportedChunks(db.get()); + + Exec(db.get(), R"SQL( +INSERT INTO _vexfs_file_versions( + inode_id,version_no,commit_id,manifest_id,size,checksum,source_version_no,created_at) +SELECT inode.local_id,version.version_no,commit_row.local_id,manifest.local_id, + version.size,version.checksum,version.source_version_no,version.created_at +FROM package.file_versions version +JOIN import_inode_map inode ON inode.source_id=version.source_inode +JOIN import_commit_map commit_row ON commit_row.source_id=version.source_commit +LEFT JOIN import_manifest_map manifest ON manifest.source_id=version.source_manifest; + +INSERT INTO _vexfs_inode_states( + workspace_id,inode_id,commit_id,kind,mode,owner_principal,uid,gid,size,current_version, + created_at,accessed_at,updated_at,changed_at,deleted_at) +SELECT (SELECT id FROM _vexfs_workspaces WHERE state='importing'),inode.local_id, + commit_row.local_id,state.kind,state.mode,state.owner_principal,state.uid,state.gid, + state.size,state.current_version,state.created_at,state.accessed_at,state.updated_at, + state.changed_at,state.deleted_at +FROM package.inode_states state +JOIN import_inode_map inode ON inode.source_id=state.source_inode +JOIN import_commit_map commit_row ON commit_row.source_id=state.source_commit; + +INSERT INTO _vexfs_dentries(workspace_id,parent_inode,name,inode_id) +SELECT (SELECT id FROM _vexfs_workspaces WHERE state='importing'),parent.local_id, + entry.name,child.local_id +FROM package.dentries entry +JOIN import_inode_map parent ON parent.source_id=entry.parent_source_inode +JOIN import_inode_map child ON child.source_id=entry.inode_source_id; + +INSERT INTO _vexfs_dentry_states( + workspace_id,parent_inode,name,commit_id,inode_id,deleted) +SELECT (SELECT id FROM _vexfs_workspaces WHERE state='importing'),parent.local_id, + state.name,commit_row.local_id,child.local_id,state.deleted +FROM package.dentry_states state +JOIN import_inode_map parent ON parent.source_id=state.parent_source_inode +JOIN import_inode_map child ON child.source_id=state.inode_source_id +JOIN import_commit_map commit_row ON commit_row.source_id=state.source_commit; + +INSERT INTO _vexfs_xattrs(inode_id,name,value,updated_at) +SELECT inode.local_id,attribute.name,attribute.value,attribute.updated_at +FROM package.xattrs attribute +JOIN import_inode_map inode ON inode.source_id=attribute.source_inode; + +INSERT INTO _vexfs_xattr_states(workspace_id,inode_id,name,commit_id,value,deleted) +SELECT (SELECT id FROM _vexfs_workspaces WHERE state='importing'),inode.local_id, + state.name,commit_row.local_id,state.value,state.deleted +FROM package.xattr_states state +JOIN import_inode_map inode ON inode.source_id=state.source_inode +JOIN import_commit_map commit_row ON commit_row.source_id=state.source_commit; + +INSERT INTO _vexfs_acl_entries( + workspace_id,inode_id,principal_id,effect,permissions,inherit_flags,created_at,updated_at) +SELECT (SELECT id FROM _vexfs_workspaces WHERE state='importing'),inode.local_id, + entry.principal_id,entry.effect,entry.permissions,entry.inherit_flags, + entry.created_at,entry.updated_at +FROM package.acl_entries entry +JOIN import_inode_map inode ON inode.source_id=entry.source_inode; + +INSERT INTO _vexfs_acl_states( + workspace_id,inode_id,principal_id,effect,commit_id,permissions,inherit_flags,deleted) +SELECT (SELECT id FROM _vexfs_workspaces WHERE state='importing'),inode.local_id, + state.principal_id,state.effect,commit_row.local_id,state.permissions, + state.inherit_flags,state.deleted +FROM package.acl_states state +JOIN import_inode_map inode ON inode.source_id=state.source_inode +JOIN import_commit_map commit_row ON commit_row.source_id=state.source_commit; + +INSERT INTO _vexfs_snapshots(workspace_id,name,commit_id,created_at) +SELECT (SELECT id FROM _vexfs_workspaces WHERE state='importing'),snapshot.name, + commit_row.local_id,snapshot.created_at +FROM package.snapshots snapshot +JOIN import_commit_map commit_row ON commit_row.source_id=snapshot.source_commit; +)SQL"); + + Statement publish(db.get(), R"SQL( +UPDATE _vexfs_workspaces +SET root_inode=?1,head_commit=?2,history_floor_commit=?3,state='active' +WHERE id=?4 AND state='importing' +)SQL"); + publish.Int64(1, root->second); + publish.Int64(2, head->second); + publish.Int64(3, floor->second); + publish.Int64(4, workspace_id); + publish.Done(); + + { + Statement clean(db.get(), + "SELECT json_extract(vexfs_check(?1,0),'$.ok')"); + clean.Text(1, workspace); + if (!clean.Row() || clean.Int(0) != 1) { + throw Error("imported workspace failed deep integrity check"); + } + } + { + Statement clear(db.get(), + "DELETE FROM _vexfs_dirty_inodes WHERE workspace_id=?1"); + clear.Int64(1, workspace_id); + clear.Done(); + } + { + Statement clear(db.get(), + "DELETE FROM _vexfs_dirty_dentries WHERE workspace_id=?1"); + clear.Int64(1, workspace_id); + clear.Done(); + } + { + Statement clear(db.get(), + "DELETE FROM _vexfs_dirty_xattrs WHERE workspace_id=?1"); + clear.Int64(1, workspace_id); + clear.Done(); + } + { + Statement clear(db.get(), + "DELETE FROM _vexfs_dirty_acl WHERE workspace_id=?1"); + clear.Int64(1, workspace_id); + clear.Done(); + } + Exec(db.get(), R"SQL( +UPDATE _vexfs_meta SET value='1' WHERE key='grep_index_dirty'; +COMMIT; +)SQL"); + + const std::string result = "{\"workspace\":\"" + JsonEscape(workspace) + + "\",\"source_workspace\":\"" + JsonEscape(info.workspace) + + "\",\"source_commit\":" + std::to_string(info.source_commit) + + ",\"versions\":" + std::to_string(info.versions) + + ",\"content_bytes\":" + std::to_string(info.content_bytes) + + ",\"package_checksum\":\"" + info.checksum + "\"}"; + new_database.Release(); + return result; + } catch (...) { + try { Exec(db.get(), "ROLLBACK"); } catch (...) {} + throw; + } +} + +} // namespace vexfs_cli diff --git a/agent_files/cli/vexfs_archive.h b/agent_files/cli/vexfs_archive.h new file mode 100644 index 0000000000..679c4eff2f --- /dev/null +++ b/agent_files/cli/vexfs_archive.h @@ -0,0 +1,18 @@ +#ifndef VEXFS_ARCHIVE_H +#define VEXFS_ARCHIVE_H + +#include + +namespace vexfs_cli { + +// The .vexfs file is a backend-neutral logical package stored in an SQLite +// container. It is not a copy of the source database. +std::string ExportArchive(const std::string &database, const std::string &workspace, + const std::string &snapshot, const std::string &output); +std::string ImportArchive(const std::string &database, const std::string &workspace, + const std::string &input); +std::string VerifyArchive(const std::string &input); + +} // namespace vexfs_cli + +#endif // VEXFS_ARCHIVE_H diff --git a/agent_files/cli/vexfs_main.cpp b/agent_files/cli/vexfs_main.cpp index 6ea60aae55..e9e7222a15 100644 --- a/agent_files/cli/vexfs_main.cpp +++ b/agent_files/cli/vexfs_main.cpp @@ -1,4 +1,5 @@ #include "vexfs_runtime_admin.h" +#include "vexfs_archive.h" #include "vexfs_platform.h" #include @@ -47,6 +48,16 @@ void Usage(std::ostream &output) { " Manage the optional trigram text index\n" " check [--quick] Verify workspace metadata and content\n" " --quick skips SHA-256 BLOB hashing\n" + " quota show Show current usage and hard limits\n" + " quota set --max-bytes N|unlimited --max-files N|unlimited\n" + " --max-file-bytes N|unlimited\n" + " Set workspace quota (sizes are bytes)\n" + " retention show Show history retention and storage usage\n" + " retention set --keep-versions N --keep-days N\n" + " Set explicit history retention policy\n" + " gc [--batch N] Delete one bounded batch of unreferenced history\n" + " Current files, snapshots and handles are protected\n" + " gc pause|resume Pause or resume history deletion\n" " stat PATH Print file metadata as JSON\n" " history PATH [--limit N] [--before N] [--json]\n" " List one page of file versions\n" @@ -63,6 +74,10 @@ void Usage(std::ostream &output) { " snapshot restore NAME [--dry-run] [--force-unmount]\n" " Restore the complete tree as a new commit\n" " snapshot drop NAME Delete a snapshot name, not its history\n" + " export --output FILE [--snapshot NAME]\n" + " Export a checked logical workspace package\n" + " import FILE Verify and atomically publish a new workspace\n" + " archive verify FILE Verify package records and content checksums\n" " mv SOURCE DESTINATION Move a file or directory\n" " ln SOURCE DESTINATION Create a hard link to a regular file\n" " chown UID:GID PATH Store owner IDs (- keeps an ID)\n" @@ -342,6 +357,11 @@ int64_t NonnegativeInteger(const std::string &value, const std::string &name) { return static_cast(result); } +int64_t QuotaValue(const std::string &value, const std::string &name) { + if (value == "unlimited") return -1; + return NonnegativeInteger(value, name); +} + int64_t OwnerId(const std::string &value, const std::string &name) { if (value == "-") return -1; char *end = nullptr; @@ -798,6 +818,31 @@ int Run(const Options &options) { if (options.arguments.size() != 1) throw std::runtime_error("doctor accepts only global options"); return RunDoctor(options); } + if (command == "export") { + const ParsedCommand parsed = ParseCommand( + options.arguments, {"--output", "--snapshot"}, {}); + if (!parsed.positional.empty()) { + throw std::runtime_error("export accepts --output FILE and optional --snapshot NAME"); + } + const std::string output = parsed.Value("--output"); + const std::string snapshot = parsed.Value("--snapshot", false); + std::cout << vexfs_cli::ExportArchive( + options.database, options.workspace, snapshot, output) << '\n'; + return 0; + } + if (command == "import") { + if (options.arguments.size() != 2) throw std::runtime_error("import needs FILE"); + std::cout << vexfs_cli::ImportArchive( + options.database, options.workspace, options.arguments[1]) << '\n'; + return 0; + } + if (command == "archive") { + if (options.arguments.size() != 3 || options.arguments[1] != "verify") { + throw std::runtime_error("archive needs verify FILE"); + } + std::cout << vexfs_cli::VerifyArchive(options.arguments[2]) << '\n'; + return 0; + } // check 必须保持只读边界：不创建数据库、workspace、WAL/SHM，也不发布 staging。 Session session(options, command != "check"); @@ -871,6 +916,86 @@ int Run(const Options &options) { const std::string value = BytesToString(&json); if (options.json) std::cout << value << '\n'; else PrintCheck(value); return JsonBoolean(value, "ok") ? 0 : ExitCode(VEXFS_MOUNT_CORRUPTION); + } else if (command == "quota") { + if (options.arguments.size() < 2) + throw std::runtime_error("quota needs show or set"); + const std::string &action = options.arguments[1]; + vexfs_mount_bytes json{}; + if (action == "show") { + if (options.arguments.size() != 2) + throw std::runtime_error("quota show accepts no arguments"); + Check(vexfs_mount_quota_get(session.get(), &json, &error), error); + } else if (action == "set") { + const ParsedCommand parsed = ParseCommand( + options.arguments, + {"--max-bytes", "--max-files", "--max-file-bytes"}, {}); + if (parsed.positional.size() != 1 || parsed.positional[0] != "set") { + throw std::runtime_error( + "quota set needs --max-bytes, --max-files and --max-file-bytes"); + } + const int64_t max_bytes = QuotaValue( + parsed.Value("--max-bytes"), "max-bytes"); + const int64_t max_files = QuotaValue( + parsed.Value("--max-files"), "max-files"); + const int64_t max_file_bytes = QuotaValue( + parsed.Value("--max-file-bytes"), "max-file-bytes"); + Check(vexfs_mount_quota_set(session.get(), max_bytes, max_files, + max_file_bytes, &json, &error), error); + } else { + throw std::runtime_error("unknown quota command: " + action); + } + std::cout << BytesToString(&json) << '\n'; + } else if (command == "retention") { + if (options.arguments.size() < 2) + throw std::runtime_error("retention needs show or set"); + const std::string &action = options.arguments[1]; + vexfs_mount_bytes json{}; + if (action == "show") { + if (options.arguments.size() != 2) + throw std::runtime_error("retention show accepts no arguments"); + Check(vexfs_mount_retention_get(session.get(), &json, &error), error); + } else if (action == "set") { + const ParsedCommand parsed = ParseCommand( + options.arguments, {"--keep-versions", "--keep-days"}, {}); + if (parsed.positional.size() != 1 || parsed.positional[0] != "set") { + throw std::runtime_error( + "retention set needs --keep-versions N and --keep-days N"); + } + const int64_t keep_versions = NonnegativeInteger( + parsed.Value("--keep-versions"), "keep-versions"); + const int64_t keep_days = NonnegativeInteger( + parsed.Value("--keep-days"), "keep-days"); + if (keep_versions > 1000000 || keep_days > 36500) { + throw CliError(VEXFS_MOUNT_INVALID_ARGUMENT, + "keep-versions must be at most 1000000 and keep-days at most 36500"); + } + Check(vexfs_mount_retention_set( + session.get(), static_cast(keep_versions), + static_cast(keep_days), &json, &error), error); + } else { + throw std::runtime_error("unknown retention command: " + action); + } + std::cout << BytesToString(&json) << '\n'; + } else if (command == "gc") { + if (options.arguments.size() == 2 && + (options.arguments[1] == "pause" || options.arguments[1] == "resume")) { + vexfs_mount_bytes json{}; + Check(vexfs_mount_gc_pause(session.get(), options.arguments[1] == "pause", + &json, &error), error); + std::cout << BytesToString(&json) << '\n'; + return 0; + } + const ParsedCommand parsed = ParseCommand(options.arguments, {"--batch"}, {}); + if (!parsed.positional.empty()) + throw std::runtime_error("gc accepts only --batch N"); + const std::string batch_value = parsed.Value("--batch", false); + const int64_t batch = batch_value.empty() ? 1000 : + PositiveInteger(batch_value, "batch"); + if (batch > 10000) + throw CliError(VEXFS_MOUNT_INVALID_ARGUMENT, "batch must be at most 10000"); + vexfs_mount_bytes json{}; + Check(vexfs_mount_gc(session.get(), static_cast(batch), &json, &error), error); + std::cout << BytesToString(&json) << '\n'; } else if (command == "stat") { if (options.arguments.size() != 2) throw std::runtime_error("stat needs PATH"); vexfs_mount_bytes json{}; diff --git "a/agent_files/linux/package/\344\275\277\347\224\250\350\257\264\346\230\216.md" "b/agent_files/linux/package/\344\275\277\347\224\250\350\257\264\346\230\216.md" index 292d06406c..bf8e9280d2 100644 --- "a/agent_files/linux/package/\344\275\277\347\224\250\350\257\264\346\230\216.md" +++ "b/agent_files/linux/package/\344\275\277\347\224\250\350\257\264\346\230\216.md" @@ -90,6 +90,29 @@ vexdb fs check 检查结构时使用 `vexdb fs check --quick`。发现损坏时命令输出问题和建议，返回退出码 8， 不会自动改写数据库。 +长期工作区可以设置 live 配额、历史保留和显式分批 GC： + +```bash +vexdb fs quota show +vexdb fs quota set --max-bytes 1073741824 --max-files 100000 \ + --max-file-bytes 134217728 +vexdb fs retention set --keep-versions 32 --keep-days 30 +vexdb fs gc --batch 1000 +``` + +`unlimited` 表示不限额。`gc pause/resume` 用于备份验证或维护边界；GC 不会自动执行，也不会 +自动 `VACUUM` 缩小 SQLite 文件。 + +单 workspace 跨机器传递使用校验过的逻辑包，不要复制正在写入的数据库文件： + +```bash +vexdb fs export --snapshot before-agent --output workspace.vexfs +vexdb fs archive verify workspace.vexfs +vexdb fs --db restored.sqlite3 --workspace restored import workspace.vexfs +``` + +导入目标 workspace 必须不存在，任何记录或内容 checksum 失败都会回滚。 + 用 SQLite Backup API 复制整个数据库： ```bash diff --git a/agent_files/macos/package/README.md b/agent_files/macos/package/README.md index 3949679d13..3ab5bab965 100644 --- a/agent_files/macos/package/README.md +++ b/agent_files/macos/package/README.md @@ -74,6 +74,11 @@ vexdb fs snapshot restore before-agent --dry-run vexdb fs snapshot restore before-agent vexdb fs check # 深度检查结构、历史和内容 SHA-256 vexdb fs check --quick # 只检查结构和引用 +vexdb fs quota show # 查看 live 文件数、字节和上限 +vexdb fs retention show # 查看保留策略和可回收历史 +vexdb fs gc --batch 1000 # 显式分批清理无引用历史 +vexdb fs export --snapshot before-agent --output workspace.vexfs +vexdb fs archive verify workspace.vexfs ``` 已挂载 workspace 会在恢复时自动安全卸载并挂回原目录；正常卸载失败时不会开始恢复。 diff --git "a/agent_files/macos/package/\344\275\277\347\224\250\350\257\264\346\230\216.md" "b/agent_files/macos/package/\344\275\277\347\224\250\350\257\264\346\230\216.md" index 8207c5b00e..d0c8966656 100644 --- "a/agent_files/macos/package/\344\275\277\347\224\250\350\257\264\346\230\216.md" +++ "b/agent_files/macos/package/\344\275\277\347\224\250\350\257\264\346\230\216.md" @@ -226,8 +226,9 @@ vexdb fs snapshot restore before-agent --force-unmount vexdb fs snapshot drop before-agent ``` -删除快照名称不会删除 workspace 历史。当前技术预览还没有自动历史清理和 GC。快照用于 -任务检查点，不是独立备份；机器或数据库损坏的恢复仍应使用第 8 节的 SQLite Backup。 +删除快照名称不会立即删除 workspace 历史。VexDB 不会后台自动删历史；需要先设置保留策略， +再显式分批 GC。快照引用的版本、当前版本和打开文件引用的版本不会被删除。快照用于任务检查 +点，不是独立备份；机器或数据库损坏的恢复仍应使用第 8 节的 SQLite Backup。 ### 检查工作区 @@ -241,6 +242,35 @@ vexdb fs --json check 跳过内容哈希，适合频繁检查结构；它不能发现同长度内容被改写。发现损坏时命令仍输出完整 报告并返回退出码 8，不会自动修复或用空内容补文件。先保存数据库副本，再按报告处理。 +### 配额、历史清理和逻辑包 + +```bash +vexdb fs quota show +vexdb fs quota set --max-bytes 1073741824 --max-files 100000 \ + --max-file-bytes 134217728 +vexdb fs quota set --max-bytes unlimited --max-files unlimited \ + --max-file-bytes unlimited + +vexdb fs retention show +vexdb fs retention set --keep-versions 32 --keep-days 30 +vexdb fs gc --batch 1000 +``` + +配额单位是字节；`unlimited` 表示不限额。GC 只回收数据库内不再被引用的历史记录，不会自动 +缩小 `.sqlite3` 文件。备份验证或维护期间可以用 `vexdb fs gc pause`，完成后执行 +`vexdb fs gc resume`。 + +导出一个可以离线校验和跨机器传递的 workspace 逻辑包： + +```bash +vexdb fs export --snapshot before-agent --output workspace.vexfs +vexdb fs archive verify workspace.vexfs +vexdb fs --db restored.sqlite3 --workspace restored import workspace.vexfs +``` + +`.vexfs` 不是整个 SQLite 数据库的物理备份；它只包含一个 workspace 的逻辑内容、历史、 +快照和元数据。导入目标 workspace 必须不存在，校验失败不会留下半个 workspace。 + ## 7. 挂载后用普通 Bash 命令 只有这一步需要启用 FSKit 文件系统扩展。SQLite、向量和上一节的 `vexdb fs` diff --git a/agent_files/macos/package_preview.sh b/agent_files/macos/package_preview.sh index 02d6bad7ed..64d0ff426d 100755 --- a/agent_files/macos/package_preview.sh +++ b/agent_files/macos/package_preview.sh @@ -255,7 +255,7 @@ write_manifest() { echo "product=VexDB-Lite" echo "filesystem=VexFS" echo "preview_version=$VERSION" - echo "contract_version=0.7.0" + echo "contract_version=0.9.0" echo "mount_abi_version=$RUNTIME_ABI" echo "platform=macOS" echo "minimum_macos=26.0" diff --git a/agent_files/mount/common/include/vexfs_runtime_admin.h b/agent_files/mount/common/include/vexfs_runtime_admin.h index a5c09f2c00..0a95063e57 100644 --- a/agent_files/mount/common/include/vexfs_runtime_admin.h +++ b/agent_files/mount/common/include/vexfs_runtime_admin.h @@ -71,6 +71,29 @@ vexfs_mount_status vexfs_mount_snapshot_restore(vexfs_mount_session *session, const char *name, int64_t expected_head, int64_t *new_commit, vexfs_mount_error *error); +// Quota values use -1 for unlimited and non-negative values for a hard limit. +vexfs_mount_status vexfs_mount_quota_get(vexfs_mount_session *session, + vexfs_mount_bytes *json, + vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_quota_set(vexfs_mount_session *session, + int64_t max_bytes, int64_t max_files, + int64_t max_file_bytes, + vexfs_mount_bytes *json, + vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_retention_get(vexfs_mount_session *session, + vexfs_mount_bytes *json, + vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_retention_set(vexfs_mount_session *session, + uint32_t keep_versions, + uint32_t keep_days, + vexfs_mount_bytes *json, + vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_gc_pause(vexfs_mount_session *session, int paused, + vexfs_mount_bytes *json, + vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_gc(vexfs_mount_session *session, uint32_t batch, + vexfs_mount_bytes *json, + vexfs_mount_error *error); #ifdef __cplusplus } diff --git a/agent_files/mount/common/src/vexfs_mount_contract_sqlite.cpp b/agent_files/mount/common/src/vexfs_mount_contract_sqlite.cpp index 5b0922b7ec..0a1d022da9 100644 --- a/agent_files/mount/common/src/vexfs_mount_contract_sqlite.cpp +++ b/agent_files/mount/common/src/vexfs_mount_contract_sqlite.cpp @@ -623,7 +623,7 @@ extern "C" vexfs_mount_status vexfs_mount_diagnostics(vexfs_mount_session *sessi staging.Row(); const char *filename = sqlite3_db_filename(session->db, "main"); const std::string contract_version = contract.ResultText(); - const bool schema_ready = contract_version == "0.7.0"; + const bool schema_ready = contract_version == "0.9.0"; const std::string value = "{\"schema_version\":\"" + JsonEscape(contract_version) + "\",\"schema_ready\":" + (schema_ready ? "true" : "false") + @@ -1342,6 +1342,104 @@ extern "C" vexfs_mount_status vexfs_mount_snapshot_restore(vexfs_mount_session * }); } +extern "C" vexfs_mount_status vexfs_mount_quota_get( + vexfs_mount_session *session, vexfs_mount_bytes *json, + vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + if (json == nullptr) throw CallError(SQLITE_MISUSE, "output is NULL"); + Call call(session->db, "SELECT vexfs_quota_get(?1)"); + call.Text(1, session->workspace.c_str()); + call.Row(); + CopyResult(call, json); + }); +} + +extern "C" vexfs_mount_status vexfs_mount_quota_set( + vexfs_mount_session *session, int64_t max_bytes, + int64_t max_files, int64_t max_file_bytes, + vexfs_mount_bytes *json, vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + UseFullDurability(session); + if (max_bytes < -1 || max_files < -1 || max_file_bytes < -1 || json == nullptr) { + throw CallError(SQLITE_RANGE, + "quota values must be -1 (unlimited) or non-negative"); + } + Call call(session->db, "SELECT vexfs_quota_set(?1,?2,?3,?4)"); + call.Text(1, session->workspace.c_str()); + if (max_bytes < 0) call.Null(2); else call.Int64(2, max_bytes); + if (max_files < 0) call.Null(3); else call.Int64(3, max_files); + if (max_file_bytes < 0) call.Null(4); else call.Int64(4, max_file_bytes); + call.Row(); + CopyResult(call, json); + }); +} + +extern "C" vexfs_mount_status vexfs_mount_retention_get( + vexfs_mount_session *session, vexfs_mount_bytes *json, + vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + if (json == nullptr) throw CallError(SQLITE_MISUSE, "output is NULL"); + Call call(session->db, "SELECT vexfs_retention_get(?1)"); + call.Text(1, session->workspace.c_str()); + call.Row(); + CopyResult(call, json); + }); +} + +extern "C" vexfs_mount_status vexfs_mount_retention_set( + vexfs_mount_session *session, uint32_t keep_versions, + uint32_t keep_days, vexfs_mount_bytes *json, + vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + UseFullDurability(session); + if (json == nullptr) throw CallError(SQLITE_MISUSE, "output is NULL"); + Call call(session->db, "SELECT vexfs_retention_set(?1,?2,?3)"); + call.Text(1, session->workspace.c_str()); + call.Int64(2, keep_versions); + call.Int64(3, keep_days); + call.Row(); + CopyResult(call, json); + }); +} + +extern "C" vexfs_mount_status vexfs_mount_gc( + vexfs_mount_session *session, uint32_t batch, + vexfs_mount_bytes *json, vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + UseFullDurability(session); + if (batch == 0 || batch > 10000 || json == nullptr) { + throw CallError(SQLITE_RANGE, "batch must be between 1 and 10000"); + } + Call call(session->db, "SELECT vexfs_gc(?1,?2)"); + call.Text(1, session->workspace.c_str()); + call.Int64(2, batch); + call.Row(); + CopyResult(call, json); + }); +} + +extern "C" vexfs_mount_status vexfs_mount_gc_pause( + vexfs_mount_session *session, int paused, + vexfs_mount_bytes *json, vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + UseFullDurability(session); + if ((paused != 0 && paused != 1) || json == nullptr) { + throw CallError(SQLITE_RANGE, "paused must be 0 or 1"); + } + Call call(session->db, "SELECT vexfs_gc_pause(?1,?2)"); + call.Text(1, session->workspace.c_str()); + call.Int(2, paused); + call.Row(); + CopyResult(call, json); + }); +} + extern "C" vexfs_mount_status vexfs_mount_handle_open(vexfs_mount_session *session, const char *path, const char *flags, const char *request_id, diff --git a/agent_files/mount/common/test/vexfs_mount_contract_smoke.cpp b/agent_files/mount/common/test/vexfs_mount_contract_smoke.cpp index 157b2f5df4..484499a03e 100644 --- a/agent_files/mount/common/test/vexfs_mount_contract_smoke.cpp +++ b/agent_files/mount/common/test/vexfs_mount_contract_smoke.cpp @@ -216,7 +216,7 @@ int main(int argc, char **argv) { vexfs_mount_bytes diagnostics{}; if (vexfs_mount_diagnostics(session, &diagnostics, &error) != VEXFS_MOUNT_OK) return Fail("diagnostics", error); - if (!Contains(diagnostics, "\"schema_version\":\"0.7.0\"") || + if (!Contains(diagnostics, "\"schema_version\":\"0.9.0\"") || !Contains(diagnostics, "\"workspace_exists\":1")) return Fail("diagnostics content", error); vexfs_mount_free(diagnostics.data); @@ -580,6 +580,71 @@ int main(int argc, char **argv) { if (vexfs_mount_read_file(session, "/agent/sync.txt", &content, &error) != VEXFS_MOUNT_OK || !Equals(content, "ABC")) return Fail("persistent synchronized content", error); vexfs_mount_free(content.data); + + vexfs_mount_bytes quota{}; + if (vexfs_mount_quota_get(session, "a, &error) != VEXFS_MOUNT_OK || + !Contains(quota, "\"max_bytes\":null") || + JsonInteger(quota, "live_files") <= 0) + return Fail("quota get", error); + vexfs_mount_free(quota.data); + quota = {}; + if (vexfs_mount_quota_set(session, -1, -1, 3, "a, &error) != VEXFS_MOUNT_OK || + !Contains(quota, "\"max_file_bytes\":3")) + return Fail("quota set", error); + vexfs_mount_free(quota.data); + if (vexfs_mount_write_file(session, "/agent/versioned.txt", "four", 4, + &version, &error) != VEXFS_MOUNT_NO_SPACE) + return Fail("quota must reject oversized publish", error); + content = {}; + if (vexfs_mount_read_file(session, "/agent/versioned.txt", &content, &error) != + VEXFS_MOUNT_OK || !Equals(content, "old")) + return Fail("quota rejection must be atomic", error); + vexfs_mount_free(content.data); + quota = {}; + if (vexfs_mount_quota_set(session, -1, -1, -1, "a, &error) != VEXFS_MOUNT_OK) + return Fail("quota reset", error); + vexfs_mount_free(quota.data); + + if (vexfs_mount_write_file(session, "/agent/gc.txt", "one", 3, + &version, &error) != VEXFS_MOUNT_OK || + vexfs_mount_write_file(session, "/agent/gc.txt", "two", 3, + &version, &error) != VEXFS_MOUNT_OK || + vexfs_mount_write_file(session, "/agent/gc.txt", "three", 5, + &version, &error) != VEXFS_MOUNT_OK) + return Fail("retention seed", error); + vexfs_mount_bytes retention{}; + if (vexfs_mount_retention_set(session, 1, 0, &retention, &error) != VEXFS_MOUNT_OK || + JsonInteger(retention, "reclaimable_versions") < 2) + return Fail("retention set", error); + vexfs_mount_free(retention.data); + retention = {}; + if (vexfs_mount_gc_pause(session, 1, &retention, &error) != VEXFS_MOUNT_OK || + !Contains(retention, "\"gc_paused\":true")) + return Fail("GC pause", error); + vexfs_mount_free(retention.data); + vexfs_mount_bytes gc{}; + if (vexfs_mount_gc(session, 1, &gc, &error) != VEXFS_MOUNT_BUSY) + return Fail("paused GC must be busy", error); + retention = {}; + if (vexfs_mount_gc_pause(session, 0, &retention, &error) != VEXFS_MOUNT_OK || + !Contains(retention, "\"gc_paused\":false")) + return Fail("GC resume", error); + vexfs_mount_free(retention.data); + gc = {}; + if (vexfs_mount_gc(session, 1, &gc, &error) != VEXFS_MOUNT_OK || + JsonInteger(gc, "deleted_versions") != 1 || + !Contains(gc, "\"has_more\":true")) + return Fail("bounded GC", error); + vexfs_mount_free(gc.data); + gc = {}; + if (vexfs_mount_gc(session, 1000, &gc, &error) != VEXFS_MOUNT_OK || + JsonInteger(gc, "remaining_reclaimable_versions") != 0) + return Fail("complete GC", error); + vexfs_mount_free(gc.data); + retention = {}; + if (vexfs_mount_retention_set(session, 32, 30, &retention, &error) != VEXFS_MOUNT_OK) + return Fail("retention reset", error); + vexfs_mount_free(retention.data); vexfs_mount_session_close(session); // 独占挂载进程崩溃后，新 session 自动认领并发布该 session 已提交的 staging。 @@ -740,9 +805,13 @@ int main(int argc, char **argv) { return Fail("deep integrity check", error); vexfs_mount_free(check.data); if (!Execute(path, - "UPDATE _vexfs_file_versions SET content=X'424144' " - "WHERE inode_id=(SELECT inode_id FROM _vexfs_dentries WHERE name='crash.txt') " - "AND version_no=(SELECT current_version FROM _vexfs_inodes WHERE id=inode_id)")) + "UPDATE _vexfs_chunks SET content=X'424144' WHERE id=(" + "SELECT entry.chunk_id FROM _vexfs_file_versions version " + "JOIN _vexfs_manifest_chunks entry ON entry.manifest_id=version.manifest_id " + "JOIN _vexfs_dentries dentry ON dentry.inode_id=version.inode_id " + "JOIN _vexfs_inodes inode ON inode.id=version.inode_id " + "WHERE dentry.name='crash.txt' AND version.version_no=inode.current_version " + "AND entry.chunk_no=0)")) return Fail("inject checksum corruption", error); check = {}; if (vexfs_mount_check(session, VEXFS_MOUNT_CHECK_QUICK, &check, &error) != @@ -753,7 +822,7 @@ int main(int argc, char **argv) { check = {}; if (vexfs_mount_check(session, 0, &check, &error) != VEXFS_MOUNT_OK || !Contains(check, "\"ok\":false") || - !Contains(check, "VEXFS_CHECKSUM_MISMATCH")) + !Contains(check, "VEXFS_CHUNK_INVALID")) return Fail("checksum corruption report", error); vexfs_mount_free(check.data); content = {}; diff --git a/docs/design/2026-07-16_agent-files-universal-filesystem.md b/docs/design/2026-07-16_agent-files-universal-filesystem.md index b4b2150fee..6f705882b7 100644 --- a/docs/design/2026-07-16_agent-files-universal-filesystem.md +++ b/docs/design/2026-07-16_agent-files-universal-filesystem.md @@ -565,9 +565,10 @@ SELECT vexfs_retention_set('workspace', keep_versions => 20, keep_days => 30); repair 必须先给出计划。缺失文件块时不能填空内容后报告成功。 -SQLite 当前实现只读 check，没有 repair。默认 CLI 使用 deep；每个 canonical 完整 BLOB 按 -64 KiB 流式读取。版本恢复生成的 alias 必须直接引用 canonical 版本，并复制相同的 size 和 -SHA-256。当前仍是 `full-blob-v1`，chunk/manifest 要等后续内容模型落地后再加入检查项。 +SQLite 当前实现只读 check，没有 repair。默认 CLI 使用 deep。当前 `chunked-v1` 中，canonical +版本引用不可变 manifest，manifest 再按顺序引用不超过 64 KiB 的物理 chunk；版本恢复生成的 +alias 必须直接引用同 inode 的 canonical 版本，并复制相同的 size 和 SHA-256。quick 检查引用、 +块数量、顺序和大小，deep 再逐块及整文件计算 SHA-256，全程不组装完整历史文件。 ### 5.7 Worktree changeset @@ -805,25 +806,33 @@ vexfs_dentry_versions vexfs_manifests workspace_id manifest_id + inode_id file_size block_size + chunk_count checksum vexfs_chunks workspace_id - manifest_id - chunk_no + chunk_id + inode_id + size data_blob checksum + +vexfs_manifest_chunks + manifest_id + chunk_no + chunk_id ``` 规则： -- 默认块大小建议 64 KiB； -- 小文件允许内联； +- SQLite `chunked-v1` 固定使用 64 KiB 块，空文件使用零块 manifest； - manifest 发布后不可修改； -- 随机写只产生受影响的新块； -- 第一版不做跨文件去重； +- 物理 chunk 发布后不可修改；同 inode 的版本可以复用未变化的物理块； +- 随机写只产生受影响的新块；版本恢复使用 alias，不复制 manifest 或 chunk； +- 当前不做跨 inode/跨文件去重； - 文件内容全部由数据库 BLOB 管理。 ### 8.5 快照 diff --git a/docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md b/docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md index 9088876283..c023520dba 100644 --- a/docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md +++ b/docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md @@ -90,14 +90,14 @@ vexdb fs --workspace workspace snapshot restore before-refactor - 不为了宣传“更快 grep”而阻塞挂载正确性、备份和跨机器工作区； - 不在核心文件合同中加入 Agent 专用的 prompt、memory 或语义字段。 -## 5. 当前基线：2026-07-22 +## 5. 当前基线：2026-07-23 | 范围 | 当前事实 | 主要缺口 | |---|---|---| -| 统一合同 | SQLite 合同 `0.7.0`，runtime ABI 1；SQL、C ABI、CLI、mount runtime 已连通；已增加跨平台一致性合同和结构化错误状态 | PostgreSQL、DuckDB 尚未实现 VexFS adapter | +| 统一合同 | SQLite 合同 `0.9.0`，runtime ABI 1；SQL、C ABI、CLI、mount runtime 已连通；已增加跨平台一致性合同和结构化错误状态 | PostgreSQL、DuckDB 尚未实现 VexFS adapter | | 文件语义 | 文件、目录、四类时间戳、并发 append、进程锁、mode、symlink、xattr、hardlink、owner/group 元数据、便携 ACL 已进入数据库合同和测试 | ACL 还没有完整授权执行和继承；特殊文件不支持 | -| 版本恢复 | 单文件版本、workspace commit、snapshot、diff、expected-head restore、SQLite Backup 已实现 | retention、GC、配额、审计、逻辑导入导出未完成 | -| 长期校验 | 每个文件版本持久化 SHA-256；只读 `vexfs_check` / `vexdb fs check` 已覆盖结构、历史、快照、staging、BLOB 长度和流式内容校验 | 自动 repair 不在当前范围；chunk/manifest 尚未落地 | +| 版本恢复 | 单文件版本、workspace commit、snapshot、diff、expected-head restore、SQLite Backup、retention、显式分批 GC、live quota 和 format v2 逻辑导入导出已实现 | 审计、自动维护、history/staging/index/total quota、PG/DuckDB 导入端未完成 | +| 长期校验 | `chunked-v1` 使用不可变 manifest、64 KiB 块和逐块/整文件 SHA-256；只读 check 覆盖结构、引用、顺序、大小和流式内容校验 | 自动 repair 和跨文件通用去重不在当前范围 | | macOS | preview.17 已通过 Apple 公证并 staple；本机安装后 Gatekeeper、FSKit 注册、Bash、Git、Python、Node.js、Go、Rust、并发 append、进程锁、打开文件生命周期和强制卸载全部通过 | 尚缺另一台没有源码和 Xcode 构建记录的干净 Mac 验证 | | Linux | libfuse3 真实挂载已通过 root 和普通用户；时间戳、并发 append、进程锁、打开文件生命周期、强制卸载和 helper 崩溃恢复已验证；x86_64/AArch64 manylinux 安装包已在对应架构真机完成无 root 安装回归 | 缓存/TTL、更多干净发行版、长期运行和真实挂载安装回归仍不足 | | Windows | 只有平台边界和规划 | WinFsp adapter、安装签名、路径/ACL/SID 合同均未实现 | @@ -113,7 +113,9 @@ vexdb fs --workspace workspace snapshot restore before-refactor 1. **已完成代码修复**：零用例失败、`PASS_WITH_SKIPS`、`--fail-on-skip`、显式 package/mount CLI 绑定、runtime ABI 单一来源、旧 schema 明确拒绝、数据库符号链接拒绝、备份目标 0600、安装哈希/签名/ABI/合同校验； 2. **已完成本地交付验证**：当前源码 ad-hoc 包通过 64 项文档 smoke 和 2 个 package Gate；这只证明本地包结构与功能，不替代 Developer ID、公证和真实 FSKit； 3. **待完成发行证明**：提交当前实现后，从干净 commit 重新 Developer ID 签名、公证，并在启用 FSKit 的 Mac 上用本包 CLI 运行 `--fail-on-skip` 的真实 mount Gate； -4. **随后进入长期安全**：checksum/check 已完成；继续 retention/GC、配额、逻辑 export/import，再做 chunk + manifest 和 append 合并发布。 +4. **长期安全与分块内容模型已完成**：checksum/check、retention/GC、live quota、SQLite format v2 + 逻辑 export/import、chunk + manifest 和 append 合并发布已进入 spec、runtime/CLI smoke 和 eval； + 下一步回到当前代码签名、公证和新的干净安装包回归。 当前不提前开发 PostgreSQL、DuckDB 或 Windows adapter。 @@ -131,8 +133,47 @@ vexdb fs --workspace workspace snapshot restore before-refactor 观测到的额外 RSS 峰值增长为 0； - 报告：`vexdb_sqlite/build/eval/vexfs/20260722T104155.646993Z-quick-20260718/report.json`。 -这不是 repair，也不是 chunk 校验。当前内容模型明确报告为 `full-blob-v1`；下一项是 -retention/GC，之后才是 quota、export/import 和 chunk/manifest。 +这是合同 `0.7.0` 当时的证据；当前 `0.9.0` 已由下方 `chunked-v1` 证据替代，但自动 repair 仍不在范围内。 + +### 5.3 长期安全最小集合完成证据 + +2026-07-23 SQLite 合同升到 `0.8.0`，旧 schema 明确拒绝且不迁移： + +- quota 覆盖最大 live bytes、live files 和单文件 bytes；数据库触发器维护 O(1) live 计数； +- retention 支持最近版本数和天数；GC 显式分批，保护 current、snapshot、handle 和 alias source， + 活动挂载以及 `gc pause` 会阻止删除； +- `.vexfs` format v1 是独立逻辑记录容器，不是原 SQLite 数据库副本；包含 commit、inode、 + dentry、文件版本、快照、xattr、ACL、principal 和逐记录/内容/整包 SHA-256； +- 导入在一个事务内使用 `importing` 隐藏状态，完成整包校验、ID 映射和深度 check 后才原子发布； +- quick 性能 eval 中，开启配额创建 250 个 1-byte 文件为 0.101915 秒，即 2,453.024 files/s； + 300 个历史版本按 128 条分批 GC 为 0.015086 秒，共 3 批，删除 299 个版本； +- 8 MiB canonical 内容的固定快照导出、独立 verify、导入、hardlink/版本/xattr/ACL 对比以及 + 内容损坏和半成品注入测试通过；export、verify、import 的 BLOB 路径固定使用 1 MiB 缓冲。 + +GC 删除记录后只让 SQLite 页面可复用，不自动 `VACUUM`；format v1 当前只有 SQLite producer/ +consumer，PG/DuckDB adapter 尚未开始。完整证据见 +`docs/reports/2026-07-23_vexfs-retention-quota-logical-archive.md`。 + +### 5.4 chunk + manifest 完成证据 + +2026-07-23 SQLite 合同升到 `0.9.0`，逻辑归档升到 format v2： + +- canonical 文件版本只引用不可变 manifest，不再在版本行保存整文件 BLOB； +- manifest 固定 64 KiB 逻辑块，物理 chunk object 可以被同 inode 的多个版本和多个块位置复用； +- 挂载打开从 chunk 流式写入 staging，publish 两遍扫描 staging 后流式生成 manifest/chunk，固定 + 64 KiB 工作缓冲；不会额外组装 128 MiB 整文件； +- 随机覆盖只新建受影响块；恢复版本继续使用版本别名，不新建 manifest；GC 只删除没有任何 + manifest 引用的物理块； +- quick check 验证 manifest 引用、块数量、顺序、大小和可达性，deep check 再验证逐块和整文件 + SHA-256； +- `.vexfs` format v2 携带 manifest/chunk 记录，verify/import 均逐块处理，导入会恢复块复用； +- 8 MiB、100 次 4 KiB 随机修改样本中，71 个 64 KiB 块受影响，57 个块直接复用，两个版本最终 + 只保存 72 个物理块；发布耗时 0.370 秒； +- 最终 quick eval 为 57 passed、0 failed、18 个平台条件 skipped、2,863 checks，耗时 24.824 秒； + SQLite spec、runtime smoke、VexFS CLI smoke 和 VexDB unified CLI smoke 全部通过。 + +当前仍没有跨文件通用去重、自动 repair、自动 GC 或在线 VACUUM。完整证据见 +`docs/reports/2026-07-23_vexfs-chunk-manifest-v2.md`。 ## 6. 阶段路线图 diff --git a/docs/reports/2026-07-23_vexfs-chunk-manifest-v2.md b/docs/reports/2026-07-23_vexfs-chunk-manifest-v2.md new file mode 100644 index 0000000000..9a89c480d6 --- /dev/null +++ b/docs/reports/2026-07-23_vexfs-chunk-manifest-v2.md @@ -0,0 +1,87 @@ +# VexFS chunk + manifest 与逻辑归档 v2 验证报告 + +- 日期：2026-07-23 +- 分支：`feature/agent_files` +- SQLite 合同：`0.9.0` +- 内容模型：`chunked-v1` +- 逻辑归档格式：`.vexfs` format v2 + +## 1. 结论 + +SQLite 已从“每个 canonical 版本保存一个完整 BLOB”切换为不可变 manifest + 64 KiB 物理 +chunk。挂载打开和发布不再额外组装一份完整文件；随机覆盖只创建受影响的块，未变化的块在 +同一 inode 的版本之间复用。版本恢复继续使用 alias，不重复创建 manifest 或 chunk。 + +这是未正式发版的新 schema，不提供旧实验 schema 迁移；版本不匹配时明确拒绝打开。 + +## 2. 当前数据模型 + +- `_vexfs_file_versions`：canonical 版本保存 `manifest_id`；恢复版本保存 + `source_file_version_id`，两者不能同时存在。 +- `_vexfs_manifests`：保存 inode、文件长度、固定块大小、块数量和整文件 SHA-256。 +- `_vexfs_chunks`：不可变物理块，保存 inode、内容长度、BLOB 和逐块 SHA-256。 +- `_vexfs_manifest_chunks`：保存 manifest 中连续的块序号和物理 chunk 引用。 + +复用范围有意限制在同一 inode。当前不做跨 inode、跨文件通用去重，避免仅凭内容 hash 把 +不同文件的生命周期、安全边界和回收规则绑在一起。 + +## 3. 读写和恢复 + +- 读取 canonical 版本时依次读取 manifest 的 chunk，校验块顺序、长度、逐块 SHA-256 和 + 整文件 SHA-256；alias 会先解析到同 inode 的 canonical 版本。 +- mount 打开文件时，以 64 KiB 为单位把已发布内容写入 staging。 +- mount publish 两遍扫描 staging：第一遍计算文件信息，第二遍建立 manifest 和物理块； + 工作缓冲固定为 64 KiB。 +- 原子 append 在发布锁内刷新到当前 inode 版本，再追加本次后缀，避免两个旧 handle 静默覆盖。 +- staging 当前仍是每个 handle 一个可增长 SQLite BLOB；它是未发布临时状态，不是最终内容模型。 + +## 4. check、GC、配额和归档 + +- quick check 验证 manifest 引用、块数量、连续顺序、长度和可达性。 +- deep check 再验证每个物理块和整文件 SHA-256。 +- GC 先删除可回收版本和 manifest 映射，只删除没有任何 manifest 引用的物理块；共享块不会 + 因删除一个版本而被误删。 +- retention 的 stored/live/reclaimable bytes 按物理 chunk 统计，不把同一块的多个引用重复计费。 +- `.vexfs` format v2 显式保存 manifest/chunk 逻辑记录；verify 逐块校验，import 重建 ID 和 + 块引用，深度 check 通过后才把目标 workspace 原子切换为 active。 + +## 5. 有界性能样本 + +测试文件为 8 MiB，连续执行 100 次 4 KiB 随机覆盖，再发布为新版本： + +| 指标 | 结果 | +|---|---:| +| 随机覆盖次数 | 100 | +| 受影响的 64 KiB 块 | 71 | +| 从上一版本复用的块 | 57 | +| 两个版本最终物理块数 | 72 | +| 100 次 staging 覆盖耗时 | 0.074700 s | +| publish 耗时 | 0.370492 s | +| staging 覆盖吞吐 | 1,338.684 ops/s | + +这组数据证明“未变化块确实被复用”，不是完整设备性能基准。 + +最终全量 quick eval：75 个场景中 57 passed、0 failed、18 skipped，共 2,863 checks，耗时 +24.824 秒，结果为 `PASS_WITH_SKIPS`。18 个 skip 都有明确平台条件：当前环境没有已安装并启用 +的 FSKit extension、当前不是 Linux，或没有显式开启真实 OpenCode 模型调用。 + +- JSON：`vexdb_sqlite/build/eval/vexfs/20260723T085638.564798Z-quick-20260718/report.json` +- Markdown：`vexdb_sqlite/build/eval/vexfs/20260723T085638.564798Z-quick-20260718/report.md` +- SQLite spec：1 passed、0 failed +- runtime smoke：PASS +- VexFS CLI smoke：PASS +- VexDB unified CLI smoke：PASS + +## 6. 仍然存在的边界 + +1. SQL 直接函数 `vexfs_append` 仍会在内存中组装当前完整内容；mount handle append 已走流式发布。 +2. staging 仍是每个 handle 一个可增长 BLOB，极大未发布文件会让 SQLite staging BLOB 很大。 +3. 当前没有跨文件去重、自动 repair、自动 GC 或在线 `VACUUM`。 +4. PostgreSQL 和 DuckDB 还没有 format v2 consumer，也没有 VexFS adapter。 +5. 当前安装的 FSKit 预览包不是基于合同 `0.9.0` 重新签名、公证和安装的发行证据。 + +## 7. 下一步 + +从干净 commit 重新构建 macOS 包，完成 Developer ID 签名、公证、staple 和启用 FSKit 的真实 +mount Gate；随后在另一台干净 Mac 按随包文档验证安装、Bash、Git、卸载和恢复。完成 Phase 1 +发行证明后，再进入 PostgreSQL 远程共享工作区，不提前并行扩展三个数据库。 diff --git a/docs/specs/2026-07-16_vexfs-product-spec.md b/docs/specs/2026-07-16_vexfs-product-spec.md index 5382ba003d..8301a72e6b 100644 --- a/docs/specs/2026-07-16_vexfs-product-spec.md +++ b/docs/specs/2026-07-16_vexfs-product-spec.md @@ -79,16 +79,16 @@ VexFS 的不变最终目标、阶段边界、完成条件和当前下一步统 SQLite MVP 没有数据库用户认证。该阶段的 principal 只能来自受签名 App/extension、当前 macOS 用户、security-scoped 数据库目录和数据库文件权限共同建立的本地身份边界；不能把 PostgreSQL 角色语义直接套到 SQLite。 -### 1.2 当前实现状态（2026-07-21） +### 1.2 当前实现状态（2026-07-23） -SQLite 合同为 `0.7.0`，runtime ABI 为 1。除单文件版本外，当前实现已经提供完整 workspace 时间点恢复： +SQLite 合同为 `0.9.0`，runtime ABI 为 1。除单文件版本外，当前实现已经提供完整 workspace 时间点恢复： - 每个 workspace commit 增量保存 inode、dentry 和 xattr 状态；文件内容继续复用不可变 file version； - snapshot 是“名称 → 固定 workspace commit”的不可变书签，创建不复制目录树或文件内容； - `snapshot create/list/show/diff/drop/restore` 已同时进入 SQL、C ABI 和 CLI； - restore 覆盖目录创建、删除和移动，以及文件内容、mode、symlink target 和 xattr，并生成新的 workspace commit； - restore 必须带 expected-head；旧 head、并发 restore 和打开/retained handle 都会被拒绝，不能静默覆盖； -- 产品尚未正式发版，当前只接受 `0.7.0` schema；其他版本明确拒绝且不改写数据库，不保留历史迁移代码； +- 产品尚未正式发版，当前只接受 `0.9.0` schema；其他版本明确拒绝且不改写数据库，不保留历史迁移代码； - SQLite Online Backup 会同时保存 snapshot、历史树和未发布 staging；数据库重开后仍可 diff 和 restore； - 3,000 文件实测 snapshot create 约 0.16 ms、单文件变化 diff 约 0.65 s、完整 restore 约 1.47 s；10,000 文件压力档分别约 0.17 ms、6.83 s 和 15.42 s。 @@ -124,7 +124,7 @@ SQLite 合同为 `0.7.0`，runtime ABI 为 1。除单文件版本外，当前实 已有的单文件版本能力包括： -- 普通写入保存不可变完整内容并生成递增版本； +- 普通写入生成不可变 manifest 和递增版本；manifest 按 64 KiB 块引用数据库内的物理 chunk； - `vexfs_history` 支持有上限的分页查询，`vexfs_read_version` 读取指定版本； - `vexfs_restore_version` 创建新版本，但复用目标版本的不可变内容，不复制大 BLOB； - 四参数恢复带 `expected_version`，当前版本已变化时返回冲突； @@ -133,8 +133,20 @@ SQLite 合同为 `0.7.0`，runtime ABI 为 1。除单文件版本外，当前实 - mount session 使用独占租约，崩溃遗留 staging 只能显式恢复，不能被新进程静默发布； - `doctor` 不创建、不迁移旧数据库，并能报告合同兼容性和升级需要； - SQLite 主库、WAL 和 SHM 默认限制为当前用户读写，CLI 提供稳定错误码和退出码。 - -当前已经是完整 workspace 时间点恢复，但还不是完整 POSIX 文件系统。SQLite 文件版本 SHA-256 和只读 check 已实现；principal 权限执行、ACL 授权、审计、配额、保留策略、GC、分块和去重仍未实现。四类时间戳和本地最小文件锁已经验证，但跨 gateway 锁、setuid/setgid/sticky bit 和特殊文件没有形成跨平台承诺。hardlink、owner/group 和便携 ACL 当前只能称为“数据库合同已实现并在部分平台接入”，不能称为完整身份权限系统。当前历史只追加、不自动回收；长期运行前必须补保留策略和 GC。snapshot 解决误改和任务检查点，SQLite Backup 才是独立备份，两者不能混为一谈。 +- workspace 配额已覆盖当前文件数、当前内容总字节和单文件字节。计数由数据库触发器维护， + 写入、句柄 publish 和快照恢复在修改前检查，失败不会产生半个版本或 commit；hardlink 不重复计数； +- retention 已支持最近版本数和天数。GC 只能显式分批执行，保护当前版本、快照、打开或保留 + 句柄以及恢复别名引用的 canonical 内容；活动挂载和 `gc pause` 都会阻止删除； +- `vexdb fs export/import` 已提供 format v2 逻辑包。逻辑包使用独立 SQLite 容器保存跨后端 + 记录，不是源数据库副本；它包含 workspace 元数据、commit、inode/dentry 历史、文件版本、 + manifest、chunk、快照、xattr、ACL 和 principal 表。每条记录、每个 chunk、整文件和整包都有 + SHA-256；导入先在 + `importing` 状态完成映射和深度检查，再在同一事务中原子发布； +- export、`archive verify` 和 import 都逐个处理不超过 64 KiB 的 chunk，校验和导入 + 使用同一个事务快照。HEAD 导出遇到未发布句柄时拒绝，指定快照 + 导出不受后续写入影响。当前只实现 SQLite 读写该逻辑格式；PG/DuckDB 消费端尚未实现。 + +当前已经是完整 workspace 时间点恢复，但还不是完整 POSIX 文件系统。SQLite 文件版本 SHA-256、只读 check、live quota、retention、显式 GC、逻辑导入导出和 `chunked-v1` 内容模型已实现；principal 权限执行、ACL 授权、审计、history/staging/index/total quota 和跨文件通用去重仍未实现。四类时间戳和本地最小文件锁已经验证，但跨 gateway 锁、setuid/setgid/sticky bit 和特殊文件没有形成跨平台承诺。hardlink、owner/group 和便携 ACL 当前只能称为“数据库合同已实现并在部分平台接入”，不能称为完整身份权限系统。GC 回收的是数据库可复用页，不会自动执行 `VACUUM` 缩小 SQLite 文件。snapshot 解决误改和任务检查点，SQLite Backup 才是独立备份，两者不能混为一谈。 当前符号链接采用 POSIX 行为：target 是原样字符串，可以指向挂载目录外部。它解决兼容性，但不提供路径隔离；需要隔离时仍由 Agent sandbox 负责。未来可以增加可选的 `internal` 策略，但当前版本没有实现该策略。 @@ -752,7 +764,7 @@ VexFS 使用四层数据保护： | FR-DB-004 | P0 | 适配器不得直接读写数据库物理文件。 | | FR-DB-005 | P0 | 宿主不允许时，不得创建线程、后台任务或外部文件。 | | FR-DB-006 | P0 | PostgreSQL 必须使用 role、MVCC、WAL 和数据库锁能力。 | -| FR-DB-007 | P0 | SQLite 必须正确处理单 writer、busy、savepoint、WAL/reopen 和 Backup API。mount staging 的热点元数据必须与大内容 BLOB 分开；旧内联布局要能原子迁移。 | +| FR-DB-007 | P0 | SQLite 必须正确处理单 writer、busy、savepoint、WAL/reopen 和 Backup API。mount staging 的热点元数据必须与内容块分开；正式发版前的旧实验 schema 明确拒绝，不保留迁移代码。 | | FR-DB-008 | P1 | DuckDB 必须正确处理单进程写入、乐观冲突、checkpoint 和 reopen；默认先支持 SQL/CLI/worktree，可写 mount 只允许 gateway 成为唯一写进程或宿主提供可验证的多进程写入能力。 | | FR-DB-009 | P1 | 同一组合同 spec 必须在所有已支持引擎运行。 | @@ -849,7 +861,7 @@ SELECT * FROM vexfs_stat('workspace', '/reports/final.md'); ### 12.4 版本和快照 -SQLite `0.7.0` 当前已经实现文件版本和 workspace 快照： +SQLite `0.9.0` 当前已经实现文件版本和 workspace 快照： ```sql SELECT vexfs_history('workspace', '/reports/final.md'); @@ -861,7 +873,9 @@ SELECT vexfs_restore_version('workspace', '/reports/final.md', 2, :expected_vers 四参数 `vexfs_history` 返回 `{"entries": [...], "next_before": ...}`，按版本倒序；每页最多 1000 条，CLI 默认 100 条。两参数形式为兼容入口，返回 JSON 数组并固定最多 100 条。每项包含 `version`、`commit`、`parent_commit`、`size`、`checksum`、`created_at`、`message` 和 `current`。`checksum` 是小写 SHA-256；文件和符号链接的 `vexfs_stat` 也返回当前版本的 `checksum`，目录返回 `null`。 -恢复成功返回新版本号。恢复必须使用带 `expected_version` 的四参数接口，不能省略并发检查。恢复版本只引用目标版本的不可变内容，历史中仍保留一条新的 version 和 commit；这不是通用内容去重，普通写入仍保存完整内容。 +恢复成功返回新版本号。恢复必须使用带 `expected_version` 的四参数接口，不能省略并发检查。 +恢复版本只引用同 inode 目标 canonical 版本，历史中仍保留一条新的 version 和 commit；普通 +写入可以复用该 inode 上一版本未变化的物理块，但当前不做跨 inode、跨文件的通用去重。 workspace 快照当前公开合同： @@ -900,8 +914,8 @@ COMMIT; ### 12.7 检查、配额和迁移 ```sql -SELECT vexfs_check('workspace', 1); -- 深度检查，包含所有 canonical BLOB 的 SHA-256 -SELECT vexfs_check('workspace', 0); -- 快速检查，只检查结构、引用和 BLOB 长度 +SELECT vexfs_check('workspace', 1); -- 深度检查，包含逐块和整文件 SHA-256 +SELECT vexfs_check('workspace', 0); -- 快速检查，只检查结构、引用、顺序和块长度 SELECT * FROM vexfs_repair_plan('workspace'); SELECT vexfs_gc('workspace', 1000); SELECT vexfs_quota_set('workspace', :max_bytes, :max_files, :max_file_bytes); @@ -910,15 +924,31 @@ SELECT * FROM vexfs_export_begin('workspace', 'snapshot-name'); SELECT vexfs_import_begin('new-workspace', :format_version); ``` -SQLite `0.7.0` 已实现只读的 `vexfs_check`。它返回 JSON，检查 SQLite 自身、workspace root、 -inode、dentry、可达性、commit 父链、snapshot、当前和历史版本引用、版本别名、BLOB 长度、 +SQLite `0.9.0` 已实现只读的 `vexfs_check`。它返回 JSON，检查 SQLite 自身、workspace root、 +inode、dentry、可达性、commit 父链、snapshot、当前和历史版本引用、版本别名、manifest/chunk、 staging 和 handle。深度模式额外按 64 KiB 流式计算 SHA-256，不把全部历史内容载入内存。 每个问题包含稳定的 `code`、对象、类型、说明和建议动作，最多返回 1000 个问题并报告完整 -问题数。当前内容模型是 `full-blob-v1`，尚未存在的 chunk/manifest 不会被假装成已检查。 +问题数。当前内容模型是 `chunked-v1`；quick 检查 manifest、块数量、顺序、大小和引用，deep +再逐块验证 SHA-256 以及整文件 SHA-256。 对应 CLI 是 `vexdb fs check`，默认深度检查；`--quick` 跳过内容哈希，`--json` 输出原始 JSON。发现损坏时仍输出报告并返回退出码 8。当前没有自动 repair；检查不能修改或填补内容。 +SQLite `0.9.0` 同时公开： + +```sql +SELECT vexfs_quota_get('workspace'); +SELECT vexfs_quota_set('workspace', :max_bytes, :max_files, :max_file_bytes); +SELECT vexfs_retention_get('workspace'); +SELECT vexfs_retention_set('workspace', :keep_versions, :keep_days); +SELECT vexfs_gc_pause('workspace', 1); +SELECT vexfs_gc('workspace', 1000); +``` + +配额参数使用 SQL `NULL` 表示不限额，`0` 表示真实的零配额。降低配额后可以继续做减少用量的 +操作，但不能增加超额用量。GC 不自动运行；每次最多删除调用者给定的批量，并返回是否还有可 +回收版本。CLI 对应 `quota show/set`、`retention show/set`、`gc pause/resume` 和 `gc --batch`。 + ## 13. 权限规则 ### 13.1 两层权限 @@ -1043,7 +1073,17 @@ worktree 清单不保存凭证，也不能被数据库扩展信任为权限依 ### 15.5 逻辑导出格式 -逻辑包至少包含： +SQLite `0.9.0` 已实现 `.vexfs` format v2。它使用独立 SQLite 文件作为逻辑记录容器， +不复制源数据库的业务表、连接信息、密码、WAL 或内部运行句柄。当前命令： + +```bash +vexdb fs export --output workspace.vexfs +vexdb fs export --snapshot before-change --output workspace.vexfs +vexdb fs archive verify workspace.vexfs +vexdb fs --db target.sqlite3 --workspace restored import workspace.vexfs +``` + +逻辑包包含： - format version； - workspace metadata； @@ -1055,6 +1095,17 @@ worktree 清单不保存凭证，也不能被数据库扩展信任为权限依 - 整包 checksum； - 导出完成标记。 +format v2 导出选择 HEAD 或一个固定快照，并保留该 commit 祖先链内仍可恢复的文件版本和 +workspace 历史。canonical 内容保持 manifest + 64 KiB chunk，恢复别名保持对同 inode canonical 版本的引用。 +记录 hash 覆盖记录全部字段，内容 hash 复用文件版本 SHA-256，整包 hash 覆盖 manifest 和 +全部有序记录 hash。校验逐个读取不超过 64 KiB 的 chunk，不把整个工作区载入内存。 + +导入目标 workspace 必须不存在。导入先创建同一事务内不可见的 `importing` workspace，映射 +commit/inode/manifest ID，重建 chunk 引用、当前树和历史，运行深度 `vexfs_check`，成功后才切换成 `active` 并提交。 +任何格式、结构、记录、内容、整包 hash 或深度检查失败都会回滚，目标名称不会留下半成品。 +SQLite 到 SQLite 采用 principal 原值映射；未来 PG/DuckDB 导入必须在 format v2 principal +表之上增加显式目标身份映射，不能自动扩大权限。 + ## 16. 三端能力矩阵 | 能力 | PostgreSQL | SQLite | DuckDB | diff --git a/tests/eval/vexfs/run.py b/tests/eval/vexfs/run.py index 4ff79ff68e..25eab719b2 100644 --- a/tests/eval/vexfs/run.py +++ b/tests/eval/vexfs/run.py @@ -33,7 +33,7 @@ MIB = 1024 * 1024 MAX_FILE_BYTES = 128 * MIB -CONTRACT_VERSION = "0.7.0" +CONTRACT_VERSION = "0.9.0" class EvalFailure(AssertionError): @@ -762,10 +762,12 @@ def version_history(ctx: Context) -> dict[str, Any]: ctx.equal(versions, [1, 2, 3], "文件版本单调增长") inode = db.json("SELECT vexfs_stat('default','/history')")["inode"] history = db.connection.execute( - "SELECT version_no,content,size FROM _vexfs_file_versions " + "SELECT version_no,size,manifest_id FROM _vexfs_file_versions " "WHERE inode_id=? ORDER BY version_no", (inode,)).fetchall() - ctx.equal(history, [(1, b"one", 3), (2, b"two", 3), (3, b"three", 5)], - "历史版本内容") + ctx.equal([(row[0], row[1]) for row in history], [(1, 3), (2, 3), (3, 5)], + "历史版本大小") + ctx.check(all(row[2] is not None for row in history), + "canonical 版本均引用不可变 manifest") public_history = db.json("SELECT vexfs_history('default','/history')") ctx.equal([row["version"] for row in public_history], [3, 2, 1], "公开历史版本倒序") @@ -1013,7 +1015,10 @@ def open_unlink(ctx: Context) -> dict[str, Any]: ctx.expect_error(lambda: db.scalar("SELECT vexfs_stat('default','/open-unlink')"), "path not found") ctx.equal(db.scalar( - "SELECT content FROM _vexfs_file_versions WHERE inode_id=? AND version_no=2", (inode,)), + "SELECT chunk.content FROM _vexfs_file_versions version " + "JOIN _vexfs_manifest_chunks entry ON entry.manifest_id=version.manifest_id " + "JOIN _vexfs_chunks chunk ON chunk.id=entry.chunk_id " + "WHERE version.inode_id=? AND version.version_no=2 AND entry.chunk_no=0", (inode,)), b"NEW", "删除 inode 的已发布历史完整") return {"inode": inode, "published_version": 2} @@ -1317,21 +1322,23 @@ def integrity_checksum_and_aliases(ctx: Context) -> dict[str, Any]: deep = db.json("SELECT vexfs_check('default',1)") quick = db.json("SELECT vexfs_check('default',0)") ctx.check(deep["ok"] and quick["ok"], "干净 workspace 深度和快速检查通过") - ctx.equal(deep["content_model"], "full-blob-v1", "检查明确当前内容模型") + ctx.equal(deep["content_model"], "chunked-v1", "检查明确当前内容模型") ctx.equal(deep["checked"]["snapshots"], 1, "检查覆盖快照引用") db.connection.execute( - "UPDATE _vexfs_file_versions SET content=zeroblob(size) " - "WHERE inode_id=? AND version_no=1", (inode,)) + "UPDATE _vexfs_chunks SET content=zeroblob(size) WHERE id=(" + "SELECT entry.chunk_id FROM _vexfs_file_versions version " + "JOIN _vexfs_manifest_chunks entry ON entry.manifest_id=version.manifest_id " + "WHERE version.inode_id=? AND version.version_no=1 AND entry.chunk_no=0)", (inode,)) quick_after = db.json("SELECT vexfs_check('default',0)") ctx.check(quick_after["ok"], "快速检查明确不读取 BLOB 校验和") deep_after = db.json("SELECT vexfs_check('default',1)") codes = {issue["code"] for issue in deep_after["issues"]} - ctx.check(not deep_after["ok"] and "VEXFS_CHECKSUM_MISMATCH" in codes, + ctx.check(not deep_after["ok"] and "VEXFS_CHUNK_INVALID" in codes, "深度检查发现同长度内容损坏") ctx.expect_error( lambda: db.scalar("SELECT vexfs_read('default','/checked.txt')"), - "checksum does not match") + "chunk is corrupt") cli = run_process( [str(ctx.cli), "--db", str(db.path), "--workspace", "default", @@ -1387,6 +1394,362 @@ def integrity_structural_corruption(ctx: Context) -> dict[str, Any]: return {"issue_count": report["issue_count"], "codes": sorted(codes)} +@case("maintenance.retention-gc", "maintenance", + "保留数、天数、快照、句柄、内容别名、活动挂载和分批 GC 的安全边界") +def retention_gc(ctx: Context) -> dict[str, Any]: + with Database(ctx) as db: + for value in ("one", "two", "three"): + db.scalar("SELECT vexfs_write('default','/history.txt',?)", (value,)) + snapshot_commit = db.scalar( + "SELECT vexfs_snapshot_create('default','protected-version-3')") + db.scalar("SELECT vexfs_write('default','/history.txt','four')") + handle = db.scalar( + "SELECT vexfs_handle_open('default','/history.txt','r','gc-open')") + db.scalar("SELECT vexfs_restore_version('default','/history.txt',1,4)") + policy = db.json("SELECT vexfs_retention_set('default',1,0)") + ctx.equal(policy["keep_versions"], 1, "保留最近版本数") + ctx.equal(policy["keep_days"], 0, "关闭按天保留") + ctx.check(policy["reclaimable_versions"] >= 1, "存在可回收历史") + + db.scalar("SELECT vexfs_mount_session_start('default','gc-mount')") + ctx.expect_error(lambda: db.scalar("SELECT vexfs_gc('default',1)"), + "active mount session") + db.scalar("SELECT vexfs_mount_session_end('default','gc-mount')") + ctx.check(db.json("SELECT vexfs_gc_pause('default',1)")["gc_paused"], + "显式暂停 GC") + ctx.expect_error(lambda: db.scalar("SELECT vexfs_gc('default',1)"), + "GC is paused") + ctx.check(not db.json("SELECT vexfs_gc_pause('default',0)")["gc_paused"], + "显式恢复 GC") + + batches = 0 + deleted = 0 + while True: + result = db.json("SELECT vexfs_gc('default',1)") + batches += 1 + deleted += result["deleted_versions"] + if not result["has_more"]: + break + ctx.check(batches < 100, "GC 必须在有界批次数内完成") + ctx.check(deleted >= 1, "GC 实际删除历史版本") + ctx.equal(db.scalar( + "SELECT CAST(vexfs_read_version('default','/history.txt',3) AS TEXT)"), + "three", "快照引用版本未删除") + ctx.equal(db.scalar( + "SELECT CAST(vexfs_read_version('default','/history.txt',4) AS TEXT)"), + "four", "打开句柄引用版本未删除") + ctx.equal(db.scalar( + "SELECT CAST(vexfs_read('default','/history.txt') AS TEXT)"), + "one", "当前别名版本和 canonical 内容未删除") + check = db.json("SELECT vexfs_check('default',1)") + ctx.check(check["ok"], "GC 后深度检查通过") + db.scalar("SELECT vexfs_handle_close(?,0,'gc-close')", (handle,)) + return {"snapshot_commit": snapshot_commit, "batches": batches, + "deleted_versions": deleted, + "remaining_versions": check["checked"]["versions"]} + + +@case("limits.workspace-quota", "limits", + "文件数、当前字节、单文件配额在发布和快照恢复前原子检查") +def workspace_quota(ctx: Context) -> dict[str, Any]: + with Database(ctx) as db: + db.scalar("SELECT vexfs_write('default','/quota.txt','abc')") + inode = db.json("SELECT vexfs_stat('default','/quota.txt')")["inode"] + before_commit = db.scalar( + "SELECT head_commit FROM _vexfs_workspaces WHERE name='default'") + quota = db.json("SELECT vexfs_quota_set('default',3,1,3)") + ctx.equal((quota["live_bytes"], quota["live_files"]), (3, 1), + "配额统计当前 live usage") + db.scalar("SELECT vexfs_link('default','/quota.txt','/quota-link.txt')") + ctx.equal(db.json("SELECT vexfs_quota_get('default')")["live_files"], 1, + "hardlink 不重复计算文件数") + ctx.expect_error( + lambda: db.scalar("SELECT vexfs_write('default','/second.txt','x')"), + "file quota exceeded") + ctx.expect_error( + lambda: db.scalar("SELECT vexfs_write('default','/quota.txt','abcd')"), + "maximum file size quota exceeded") + ctx.equal(db.scalar("SELECT CAST(vexfs_read('default','/quota.txt') AS TEXT)"), + "abc", "失败写入不改正文") + ctx.equal(db.scalar( + "SELECT count(*) FROM _vexfs_file_versions WHERE inode_id=?", (inode,)), + 1, "失败写入不产生版本") + ctx.equal(db.scalar( + "SELECT head_commit FROM _vexfs_workspaces WHERE name='default'"), + before_commit + 1, "只有 hardlink 成功操作产生 commit") + + db.scalar("SELECT vexfs_quota_set('default',NULL,NULL,NULL)") + snapshot = db.scalar("SELECT vexfs_snapshot_create('default','large')") + db.scalar("SELECT vexfs_write('default','/quota.txt','x')") + current_head = db.scalar( + "SELECT head_commit FROM _vexfs_workspaces WHERE name='default'") + db.scalar("SELECT vexfs_quota_set('default',NULL,NULL,2)") + ctx.expect_error(lambda: db.scalar( + "SELECT vexfs_snapshot_restore('default','large',?)", (current_head,)), + "maximum file size quota") + ctx.equal(db.scalar("SELECT CAST(vexfs_read('default','/quota.txt') AS TEXT)"), + "x", "失败快照恢复不修改工作区") + ctx.equal(db.scalar( + "SELECT head_commit FROM _vexfs_workspaces WHERE name='default'"), + current_head, "失败快照恢复不产生 commit") + return {"snapshot_commit": snapshot, "live_files": 1, + "rejected_operations": 3} + + +@case("backup.logical-export-import", "backup", + "逻辑包记录、整包和 BLOB 校验，指定快照导出，导入原子发布") +def logical_export_import(ctx: Context) -> dict[str, Any]: + with tempfile.TemporaryDirectory(prefix="vexfs-export-eval-") as directory: + base = Path(directory) + source_path = base / "source.sqlite3" + destination_path = base / "destination.sqlite3" + archive = base / "workspace.vexfs" + dirty_archive = base / "dirty-head.vexfs" + corrupt_archive = base / "corrupt.vexfs" + missing_source = base / "missing.sqlite3" + missing_source_archive = base / "missing-source.vexfs" + missing_export = run_process( + [str(ctx.cli), "--db", str(missing_source), "--workspace", "default", + "export", "--output", str(missing_source_archive)], check=False) + ctx.check(missing_export.returncode != 0 and not missing_source.exists() and + not missing_source_archive.exists(), + "不存在的源数据库导出失败且不创建空数据库或归档") + with Database(ctx, source_path) as source: + source.scalar("SELECT vexfs_mkdir('default','/project')") + source.scalar("SELECT vexfs_write('default','/project/data.bin',?)", + (sqlite3.Binary(b"A" * min(ctx.mode.sequential_mib, 16) * MIB),)) + source.scalar("SELECT vexfs_write('default','/project/version.txt','one')") + source.scalar("SELECT vexfs_write('default','/project/version.txt','two')") + inode = source.json( + "SELECT vexfs_stat('default','/project/version.txt')")["inode"] + source.scalar("SELECT vexfs_link('default','/project/version.txt'," + "'/project/version-link.txt')") + source.scalar("SELECT vexfs_xattr_set('default',?,'user.export','value',0)", + (inode,)) + source.scalar("SELECT vexfs_acl_set('default',?," + "'[{\"principal\":\"alice\",\"effect\":\"allow\"," + "\"permissions\":\"read\",\"inherit\":1}]')", (inode,)) + snapshot_commit = source.scalar( + "SELECT vexfs_snapshot_create('default','portable')") + source.scalar("SELECT vexfs_write('default','/project/version.txt','after')") + dirty_handle = source.scalar( + "SELECT vexfs_handle_open('default','/project/version.txt','rw','export-open')") + source.scalar( + "SELECT vexfs_handle_stage_write(?,0,'dirty','export-stage')", + (dirty_handle,)) + + prefix = [str(ctx.cli), "--db", str(source_path), "--workspace", "default"] + dirty_export = run_process( + prefix + ["export", "--output", str(dirty_archive)], check=False) + ctx.check(dirty_export.returncode != 0 and + b"unpublished file handles" in dirty_export.stderr and + not dirty_archive.exists(), + "HEAD 导出拒绝未发布句柄且不留半个包") + export_started = time.perf_counter() + exported = json.loads(run_process( + prefix + ["export", "--snapshot", "portable", "--output", str(archive)] + ).stdout) + export_seconds = time.perf_counter() - export_started + ctx.equal(archive.stat().st_mode & 0o777, 0o600, + "逻辑包默认只允许当前用户读写") + archive.chmod(0o400) + verified = json.loads(run_process( + [str(ctx.cli), "archive", "verify", str(archive)]).stdout) + ctx.check(verified["ok"], "逻辑包独立校验通过") + ctx.equal(exported["source_commit"], snapshot_commit, "导出固定快照 commit") + ctx.equal(exported["package_checksum"], verified["package_checksum"], + "导出和校验整包 hash 一致") + + import_started = time.perf_counter() + imported = json.loads(run_process( + [str(ctx.cli), "--db", str(destination_path), "--workspace", "restored", + "import", str(archive)]).stdout) + import_seconds = time.perf_counter() - import_started + ctx.budget("logical_export_seconds", export_seconds, 15.0) + ctx.budget("logical_import_seconds", import_seconds, 15.0) + with Database(ctx, destination_path, initialize=False) as destination: + ctx.equal(destination_path.stat().st_mode & 0o777, 0o600, + "新目标数据库默认只允许当前用户读写") + ctx.equal(destination.scalar( + "SELECT CAST(vexfs_read('restored','/project/version.txt') AS TEXT)"), + "two", "指定快照内容导入") + left = destination.json( + "SELECT vexfs_stat('restored','/project/version.txt')") + right = destination.json( + "SELECT vexfs_stat('restored','/project/version-link.txt')") + ctx.equal((left["inode"], left["link_count"]), + (right["inode"], 2), "hardlink 关系导入") + ctx.equal(destination.scalar( + "SELECT CAST(vexfs_read_version('restored','/project/version.txt',1) AS TEXT)"), + "one", "文件版本历史导入") + ctx.equal(destination.scalar( + "SELECT CAST(vexfs_xattr_get('restored',?,'user.export') AS TEXT)", + (left["inode"],)), "value", "xattr 导入") + ctx.equal(destination.json( + "SELECT vexfs_acl_get('restored',?)", (left["inode"],))[0]["principal"], + "alice", "ACL 导入") + ctx.check(destination.json("SELECT vexfs_check('restored',1)")["ok"], + "导入后深度检查通过") + + shutil.copyfile(archive, corrupt_archive) + tampered = sqlite3.connect(corrupt_archive) + tampered_row = tampered.execute( + "SELECT id FROM chunks WHERE size>0 ORDER BY id LIMIT 1").fetchone() + ctx.check(tampered_row is not None, "损坏测试找到 canonical 内容") + tampered.execute( + "UPDATE chunks SET content=zeroblob(size) WHERE id=?", + (tampered_row[0],)) + tampered.commit() + tampered.close() + rejected = run_process( + [str(ctx.cli), "archive", "verify", str(corrupt_archive)], check=False) + ctx.check(rejected.returncode != 0 and b"checksum mismatch" in rejected.stderr, + "内容损坏必须被校验拒绝") + rejected_import = run_process( + [str(ctx.cli), "--db", str(destination_path), "--workspace", "partial", + "import", str(corrupt_archive)], check=False) + ctx.check(rejected_import.returncode != 0, "损坏包导入失败") + with Database(ctx, destination_path, initialize=False) as destination: + ctx.equal(destination.scalar( + "SELECT count(*) FROM _vexfs_workspaces WHERE name='partial'"), 0, + "失败导入不发布半个 workspace") + corrupt_destination = base / "corrupt-destination.sqlite3" + rejected_new_database = run_process( + [str(ctx.cli), "--db", str(corrupt_destination), "--workspace", "partial", + "import", str(corrupt_archive)], check=False) + ctx.check(rejected_new_database.returncode != 0 and + not corrupt_destination.exists(), + "损坏包导入新目标失败时删除新建数据库") + return {"source_commit": snapshot_commit, + "archive_bytes": archive.stat().st_size, + "versions": verified["versions"], + "content_bytes": verified["content_bytes"], + "export_seconds": round(export_seconds, 6), + "import_seconds": round(import_seconds, 6), + "package_checksum": imported["package_checksum"]} + + +@case("performance.quota-and-gc-scale", "performance", + "开启配额后的文件创建为常数时间计数，历史 GC 保持固定批次和受控内存") +def quota_and_gc_scale(ctx: Context) -> dict[str, Any]: + with Database(ctx) as db: + file_count = ctx.mode.small_files + db.scalar("SELECT vexfs_mkdir('default','/quota-scale')") + db.scalar("SELECT vexfs_quota_set('default',?,?,?)", + (file_count + 1, file_count + 1, 1024)) + started = time.perf_counter() + db.connection.execute("BEGIN") + try: + for index in range(file_count): + db.scalar("SELECT vexfs_write('default',?,'x')", + (f"/quota-scale/f{index:06d}",)) + db.connection.execute("COMMIT") + except Exception: + db.connection.execute("ROLLBACK") + raise + quota_seconds = time.perf_counter() - started + usage = db.json("SELECT vexfs_quota_get('default')") + ctx.equal(usage["live_files"], file_count, "规模写入文件计数") + ctx.equal(usage["live_bytes"], file_count, "规模写入字节计数") + + versions = ctx.mode.random_ops + db.scalar("SELECT vexfs_quota_set('default',NULL,NULL,NULL)") + for index in range(versions): + db.scalar("SELECT vexfs_write('default','/gc-scale',?)", + (f"value-{index:08d}",)) + db.scalar("SELECT vexfs_retention_set('default',1,0)") + gc_started = time.perf_counter() + batches = 0 + deleted = 0 + while True: + result = db.json("SELECT vexfs_gc('default',128)") + batches += 1 + deleted += result["deleted_versions"] + if not result["has_more"]: + break + ctx.check(batches <= (versions // 128) + 4, "GC 批次数受控") + gc_seconds = time.perf_counter() - gc_started + ctx.equal(deleted, versions - 1, "只保留当前版本") + ctx.check(db.json("SELECT vexfs_check('default',0)")["ok"], + "规模 GC 后结构检查通过") + ctx.budget("quota_scale_seconds", quota_seconds, + max(5.0, file_count / 300.0)) + ctx.budget("gc_scale_seconds", gc_seconds, + max(5.0, versions / 1000.0)) + return {"files": file_count, "versions": versions, + "quota_seconds": round(quota_seconds, 6), + "quota_files_per_second": round(file_count / max(quota_seconds, 1e-9), 3), + "gc_seconds": round(gc_seconds, 6), "gc_batches": batches, + "deleted_versions": deleted} + + +@case("storage.chunk-manifest-reuse", "storage", + "64 KiB manifest/chunk、随机覆盖复用未变化块、恢复别名和 GC 共享块安全") +def chunk_manifest_reuse(ctx: Context) -> dict[str, Any]: + with Database(ctx) as db: + chunk_bytes = 64 * 1024 + original = b"".join(bytes([index + 1]) * chunk_bytes for index in range(4)) + ctx.equal(db.scalar("SELECT vexfs_write('default','/chunked.bin',?)", (original,)), + 1, "创建四块 canonical 版本") + inode = db.json("SELECT vexfs_stat('default','/chunked.bin')")["inode"] + first_manifest = db.scalar( + "SELECT manifest_id FROM _vexfs_file_versions " + "WHERE inode_id=? AND version_no=1", (inode,)) + ctx.equal(db.scalar( + "SELECT count(*) FROM _vexfs_manifest_chunks WHERE manifest_id=?", + (first_manifest,)), 4, "首版本 manifest 有四个块引用") + + handle = db.scalar( + "SELECT vexfs_handle_open('default','/chunked.bin','rw','chunk-open')") + patch_offset = chunk_bytes + 123 + patch = b"changed-block" + generation = db.scalar( + "SELECT vexfs_handle_stage_write(?,?,?,'chunk-patch')", + (handle, patch_offset, patch)) + ctx.equal(db.scalar( + "SELECT vexfs_handle_publish(?,?,'data','chunk-publish')", + (handle, generation)), 2, "随机覆盖发布第二版本") + db.scalar("SELECT vexfs_handle_close(?,0,'chunk-close')", (handle,)) + second_manifest = db.scalar( + "SELECT manifest_id FROM _vexfs_file_versions " + "WHERE inode_id=? AND version_no=2", (inode,)) + reused = db.scalar( + "SELECT count(*) FROM _vexfs_manifest_chunks first " + "JOIN _vexfs_manifest_chunks second " + "ON second.chunk_no=first.chunk_no AND second.chunk_id=first.chunk_id " + "WHERE first.manifest_id=? AND second.manifest_id=?", + (first_manifest, second_manifest)) + ctx.equal(reused, 3, "只重写受影响的一块，其余三块复用") + ctx.equal(db.scalar( + "SELECT count(*) FROM _vexfs_chunks WHERE inode_id=?", (inode,)), 5, + "两个四块版本只保存五个物理块") + expected = bytearray(original) + expected[patch_offset:patch_offset + len(patch)] = patch + ctx.equal(db.scalar("SELECT vexfs_read('default','/chunked.bin')"), bytes(expected), + "分块随机覆盖后的完整内容") + + ctx.equal(db.scalar( + "SELECT vexfs_restore_version('default','/chunked.bin',1,2)"), 3, + "恢复版本仍使用文件版本别名") + ctx.equal(db.scalar( + "SELECT count(*) FROM _vexfs_manifests WHERE workspace_id=(" + "SELECT id FROM _vexfs_workspaces WHERE name='default')"), 2, + "恢复别名不创建重复 manifest") + db.scalar("SELECT vexfs_retention_set('default',1,0)") + gc = db.json("SELECT vexfs_gc('default',100)") + ctx.equal(gc["deleted_versions"], 1, "GC 删除不再需要的覆盖版本") + ctx.equal(db.scalar( + "SELECT count(*) FROM _vexfs_chunks WHERE inode_id=?", (inode,)), 4, + "GC 只删除覆盖版本独有块，保留共享块") + report = db.json("SELECT vexfs_check('default',1)") + ctx.check(report["ok"], "共享块 GC 后深度检查通过") + ctx.equal((report["checked"]["manifests"], report["checked"]["chunks"]), + (1, 4), "检查报告包含真实 manifest 和物理块数量") + return {"logical_bytes": len(original), "chunk_bytes": chunk_bytes, + "reused_chunks": reused, "physical_chunks_before_gc": 5, + "physical_chunks_after_gc": 4} + + @case("recovery.initialization-failure-rollback", "recovery", "新库初始化空间不足时不得留下半张 schema") def initialization_failure_rollback(ctx: Context) -> dict[str, Any]: @@ -2414,6 +2777,10 @@ def performance_random_patches(ctx: Context) -> dict[str, Any]: expected: dict[int, bytes] = {} with Database(ctx) as db: db.scalar("SELECT vexfs_write('default','/random.bin',zeroblob(?))", (logical_size,)) + inode = db.json("SELECT vexfs_stat('default','/random.bin')")["inode"] + base_manifest = db.scalar( + "SELECT manifest_id FROM _vexfs_file_versions " + "WHERE inode_id=? AND version_no=1", (inode,)) handle = db.scalar( "SELECT vexfs_handle_open('default','/random.bin','rw','random-open')") started = time.perf_counter() @@ -2436,14 +2803,45 @@ def performance_random_patches(ctx: Context) -> dict[str, Any]: ctx.equal(row[0], 1, "随机写 staging 行数") ctx.equal(row[1], logical_size, "随机写 logical_size") ctx.check(row[2] <= MAX_FILE_BYTES, "随机写 capacity") + publish_started = time.perf_counter() + ctx.equal(db.scalar( + "SELECT vexfs_handle_publish(?,?,'data','random-publish')", + (handle, generation)), 2, "随机 patch 发布新版本") + publish_seconds = time.perf_counter() - publish_started + db.scalar("SELECT vexfs_handle_close(?,0,'random-close')", (handle,)) + patched_manifest = db.scalar( + "SELECT manifest_id FROM _vexfs_file_versions " + "WHERE inode_id=? AND version_no=2", (inode,)) + affected_chunks = {offset // (64 * 1024) for offset in expected} + total_chunks = logical_size // (64 * 1024) + reused_chunks = db.scalar( + "SELECT count(*) FROM _vexfs_manifest_chunks before " + "JOIN _vexfs_manifest_chunks after " + "ON after.chunk_no=before.chunk_no AND after.chunk_id=before.chunk_id " + "WHERE before.manifest_id=? AND after.manifest_id=?", + (base_manifest, patched_manifest)) + ctx.equal(reused_chunks, total_chunks - len(affected_chunks), + "随机发布只替换受影响的 64 KiB 块") + physical_chunks = db.scalar( + "SELECT count(*) FROM _vexfs_chunks WHERE inode_id=?", (inode,)) + ctx.check(physical_chunks <= 1 + len(affected_chunks), + "重复零块和未变化块不会重复保存") + for offset, data in list(expected.items())[-min(len(expected), 3):]: + ctx.equal(db.scalar( + "SELECT substr(vexfs_read('default','/random.bin'),?,?)", + (offset + 1, patch_size)), data, "发布后随机 patch 内容") ctx.budget("random_patch_seconds", seconds, 120.0) + ctx.budget("random_patch_publish_seconds", publish_seconds, 60.0) storage_before_checkpoint = db_storage(db.path) db.connection.execute("PRAGMA wal_checkpoint(TRUNCATE)").fetchone() storage_after_checkpoint = db_storage(db.path) return {"writes": ctx.mode.random_writes, "unique_slots": len(expected), "logical_bytes": logical_size, "capacity_bytes": row[2], "generation": generation, "seconds": round(seconds, 6), + "publish_seconds": round(publish_seconds, 6), "operations_per_second": round(ctx.mode.random_writes / max(seconds, 1e-9), 3), + "affected_chunks": len(affected_chunks), "reused_chunks": reused_chunks, + "physical_chunks": physical_chunks, "request_rows": db.scalar("SELECT count(*) FROM _vexfs_requests"), "storage_before_checkpoint": storage_before_checkpoint, "storage_after_checkpoint": storage_after_checkpoint} @@ -2522,7 +2920,7 @@ def performance_integrity_check(ctx: Context) -> dict[str, Any]: after_rss = self_max_rss_bytes() ctx.check(quick["ok"] and deep["ok"], "性能数据集检查通过") ctx.equal(deep["checked"]["content_bytes"], logical_bytes, - "深度检查扫描所有 canonical BLOB") + "深度检查扫描所有 canonical manifest/chunk") ctx.equal(deep["checked"]["versions"], file_count, "版本计数") ctx.budget("integrity_check_seconds", deep_seconds, {"quick": 10.0, "full": 45.0, "stress": 90.0}[ctx.mode.name]) @@ -2581,8 +2979,11 @@ def maximum_file(ctx: Context) -> dict[str, Any]: ctx.equal(db.scalar("SELECT length(vexfs_read('default','/max.bin'))"), MAX_FILE_BYTES, "复用内容的恢复版本可读") ctx.equal(db.scalar( - "SELECT sum(length(content)) FROM _vexfs_file_versions WHERE inode_id=" - "(SELECT inode_id FROM _vexfs_dentries WHERE name='max.bin')"), + "SELECT sum(chunk.size) FROM _vexfs_file_versions version " + "JOIN _vexfs_manifest_chunks entry ON entry.manifest_id=version.manifest_id " + "JOIN _vexfs_chunks chunk ON chunk.id=entry.chunk_id " + "WHERE version.inode_id=(SELECT inode_id FROM _vexfs_dentries " + "WHERE name='max.bin')"), 2 * MAX_FILE_BYTES, "restore 不重复保存 128 MiB BLOB") storage_before_checkpoint = db_storage(db.path) db.connection.execute("PRAGMA wal_checkpoint(TRUNCATE)").fetchone() diff --git a/tests/spec/sqlite/agent_files/vexfs_contract.yaml b/tests/spec/sqlite/agent_files/vexfs_contract.yaml index 18c483aa3c..0f729e9a54 100644 --- a/tests/spec/sqlite/agent_files/vexfs_contract.yaml +++ b/tests/spec/sqlite/agent_files/vexfs_contract.yaml @@ -4,7 +4,7 @@ description: VexFS 文件目录、事务、句柄写入、幂等请求和并发 steps: - query: SELECT vexfs_contract_version(); expect: - - - 0.7.0 + - - 0.9.0 - query: SELECT vexfs_init(); expect: - - 1 @@ -219,3 +219,62 @@ steps: - query: SELECT vexfs_snapshot_drop('default', 'spec-baseline'); expect: - - 1 +- query: SELECT json_extract(vexfs_quota_get('default'), '$.max_bytes'), json_extract(vexfs_quota_get('default'), '$.over_quota'); + expect: + - - null + - 0 +- query: SELECT json_extract(vexfs_quota_set('default', NULL, NULL, 5), '$.max_file_bytes'); + expect: + - - 5 +- query: SELECT vexfs_write('default', '/notes/hello.txt', '123456'); + expect_error: maximum file size quota exceeded +- query: SELECT CAST(vexfs_read('default', '/notes/hello.txt') AS TEXT), count(*) FROM _vexfs_file_versions WHERE inode_id=4; + expect: + - - hello + - 6 +- query: SELECT json_extract(vexfs_quota_set('default', NULL, NULL, NULL), '$.over_quota'); + expect: + - - 0 +- query: SELECT vexfs_write('default', '/notes/gc.txt', 'one'); + expect: + - - 1 +- query: SELECT vexfs_write('default', '/notes/gc.txt', 'two'); + expect: + - - 2 +- query: SELECT vexfs_snapshot_create('default', 'gc-protected') > 0; + expect: + - - 1 +- query: SELECT vexfs_write('default', '/notes/gc.txt', 'three'); + expect: + - - 3 +- query: SELECT vexfs_write('default', '/notes/gc.txt', 'four'); + expect: + - - 4 +- query: SELECT json_extract(vexfs_retention_set('default', 1, 0), '$.reclaimable_versions') >= 2; + expect: + - - 1 +- query: SELECT json_extract(vexfs_gc_pause('default', 1), '$.gc_paused'); + expect: + - - 1 +- query: SELECT vexfs_gc('default', 10000); + expect_error: GC is paused +- query: SELECT json_extract(vexfs_gc_pause('default', 0), '$.gc_paused'); + expect: + - - 0 +- query: SELECT json_extract(vexfs_gc('default', 10000), '$.deleted_versions') > 0; + expect: + - - 1 +- query: SELECT json_extract(vexfs_gc('default', 10000), '$.remaining_reclaimable_versions'); + expect: + - - 0 +- query: SELECT CAST(vexfs_read_version('default', '/notes/gc.txt', 2) AS TEXT), CAST(vexfs_read('default', '/notes/gc.txt') AS TEXT); + expect: + - - two + - four +- query: SELECT json_extract(vexfs_check('default', 1), '$.ok'), json_extract(vexfs_check('default', 1), '$.issue_count'); + expect: + - - 1 + - 0 +- query: SELECT vexfs_snapshot_drop('default', 'gc-protected'); + expect: + - - 1 diff --git a/vexdb_sqlite/CMakeLists.txt b/vexdb_sqlite/CMakeLists.txt index 28c31552c5..3d2f838c44 100644 --- a/vexdb_sqlite/CMakeLists.txt +++ b/vexdb_sqlite/CMakeLists.txt @@ -247,6 +247,7 @@ if(VEXDB_SQLITE_DESKTOP) ${REPO_ROOT}/agent_files/cli/vexdb_main.cpp ${REPO_ROOT}/agent_files/cli/vexdb_shell_init.c ${REPO_ROOT}/agent_files/cli/vexfs_main.cpp + ${REPO_ROOT}/agent_files/cli/vexfs_archive.cpp ${REPO_ROOT}/agent_files/cli/vexfs_platform_registry.cpp ${SQLITE_AMALG_DIR}/shell.c) set(VEXDB_CLI_PLATFORM_LIBS) diff --git a/vexdb_sqlite/src/agent_files/vexfs_sqlite.cpp b/vexdb_sqlite/src/agent_files/vexfs_sqlite.cpp index ea4b109def..ae1697445a 100644 --- a/vexdb_sqlite/src/agent_files/vexfs_sqlite.cpp +++ b/vexdb_sqlite/src/agent_files/vexfs_sqlite.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -27,8 +28,9 @@ namespace { -constexpr const char *kContractVersion = "0.7.0"; +constexpr const char *kContractVersion = "0.9.0"; constexpr sqlite3_int64 kMaxStagedBytes = 128LL * 1024LL * 1024LL; +constexpr sqlite3_int64 kContentChunkBytes = 64LL * 1024LL; constexpr sqlite3_int64 kInitialStagingCapacity = 64LL * 1024LL; constexpr sqlite3_int64 kEmptyFileStagingCapacity = 4LL * 1024LL; constexpr sqlite3_int64 kMaxXattrBytes = 64LL * 1024LL; @@ -56,6 +58,12 @@ class SqlError : public std::runtime_error { int code_; }; +struct BlobCloser { + void operator()(sqlite3_blob *blob) const { + if (blob != nullptr) sqlite3_blob_close(blob); + } +}; + class Statement { public: Statement(sqlite3 *db, const char *sql) : db_(db) { @@ -105,6 +113,11 @@ class Statement { if (rc != SQLITE_DONE) Throw("step", rc); } + void Reset() { + Check(sqlite3_reset(stmt_), "reset"); + Check(sqlite3_clear_bindings(stmt_), "clear bindings"); + } + sqlite3_int64 Int64(int column) const { return sqlite3_column_int64(stmt_, column); } int Int(int column) const { return sqlite3_column_int(stmt_, column); } int Type(int column) const { return sqlite3_column_type(stmt_, column); } @@ -136,32 +149,6 @@ class Statement { sqlite3_stmt *stmt_ = nullptr; }; -class ReadOnlyBlob { - public: - ReadOnlyBlob(sqlite3 *db, sqlite3_int64 rowid) { - const int rc = sqlite3_blob_open(db, "main", "_vexfs_file_versions", "content", - rowid, 0, &blob_); - if (rc != SQLITE_OK) throw SqlError("cannot open version BLOB", rc); - } - - ~ReadOnlyBlob() { - if (blob_ != nullptr) sqlite3_blob_close(blob_); - } - - int size() const { return sqlite3_blob_bytes(blob_); } - - void Read(void *output, int bytes, int offset) { - const int rc = sqlite3_blob_read(blob_, output, bytes, offset); - if (rc != SQLITE_OK) throw SqlError("cannot read version BLOB", rc); - } - - ReadOnlyBlob(const ReadOnlyBlob &) = delete; - ReadOnlyBlob &operator=(const ReadOnlyBlob &) = delete; - - private: - sqlite3_blob *blob_ = nullptr; -}; - void Exec(sqlite3 *db, const char *sql) { char *message = nullptr; const int rc = sqlite3_exec(db, sql, nullptr, nullptr, &message); @@ -418,22 +405,44 @@ GrepIndexState GetGrepIndexState(sqlite3 *db) { return found == g_grep_index_connections.end() ? GrepIndexState::kDisabled : found->second; } +std::vector ReadVersionContent(sqlite3 *db, sqlite3_int64 inode_id, + sqlite3_int64 version); + void RebuildGrepIndex(sqlite3 *db) { Savepoint savepoint(db, "vexfs_grep_index_rebuild"); - Exec(db, R"SQL( -DELETE FROM _vexfs_content_fts; + Exec(db, "DELETE FROM _vexfs_content_fts"); + struct IndexedFile { + sqlite3_int64 inode_id; + sqlite3_int64 workspace_id; + sqlite3_int64 version; + }; + std::vector files; + { + Statement rows(db, R"SQL( +SELECT id,workspace_id,current_version FROM _vexfs_inodes +WHERE kind='file' AND deleted_at IS NULL +)SQL"); + while (rows.Row()) { + files.push_back({rows.Int64(0), rows.Int64(1), rows.Int64(2)}); + } + } + Statement insert(db, R"SQL( INSERT INTO _vexfs_content_fts(rowid,workspace_id,inode_id,version_no,content) -SELECT inode.id,inode.workspace_id,inode.id,inode.current_version, - CAST(CASE WHEN version.source_version_no IS NULL - THEN version.content ELSE source.content END AS TEXT) -FROM _vexfs_inodes inode -JOIN _vexfs_file_versions version - ON version.inode_id=inode.id AND version.version_no=inode.current_version -LEFT JOIN _vexfs_file_versions source - ON source.inode_id=version.inode_id AND source.version_no=version.source_version_no -WHERE inode.kind='file' AND inode.deleted_at IS NULL; -INSERT OR REPLACE INTO _vexfs_meta(key,value) VALUES('grep_index_dirty','0'); +VALUES(?1,?2,?1,?3,?4) )SQL"); + for (const IndexedFile &file : files) { + const std::vector content = + ReadVersionContent(db, file.inode_id, file.version); + insert.BindInt64(1, file.inode_id); + insert.BindInt64(2, file.workspace_id); + insert.BindInt64(3, file.version); + insert.BindText(4, content.empty() ? std::string() : + std::string(reinterpret_cast(content.data()), content.size())); + insert.Done(); + insert.Reset(); + } + Exec(db, "INSERT OR REPLACE INTO _vexfs_meta(key,value) " + "VALUES('grep_index_dirty','0')"); savepoint.Release(); } @@ -526,9 +535,21 @@ CREATE TABLE IF NOT EXISTS _vexfs_meta( CREATE TABLE IF NOT EXISTS _vexfs_workspaces( id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE, + state TEXT NOT NULL DEFAULT 'active' CHECK(state IN ('active','importing')), root_inode INTEGER, head_commit INTEGER, history_floor_commit INTEGER, + retention_keep_versions INTEGER NOT NULL DEFAULT 32 + CHECK(retention_keep_versions >= 0), + retention_keep_days INTEGER NOT NULL DEFAULT 30 + CHECK(retention_keep_days >= 0), + gc_paused INTEGER NOT NULL DEFAULT 0 CHECK(gc_paused IN (0,1)), + quota_max_bytes INTEGER CHECK(quota_max_bytes IS NULL OR quota_max_bytes >= 0), + quota_max_files INTEGER CHECK(quota_max_files IS NULL OR quota_max_files >= 0), + quota_max_file_bytes INTEGER + CHECK(quota_max_file_bytes IS NULL OR quota_max_file_bytes >= 0), + live_files INTEGER NOT NULL DEFAULT 0 CHECK(live_files >= 0), + live_bytes INTEGER NOT NULL DEFAULT 0 CHECK(live_bytes >= 0), created_at INTEGER NOT NULL DEFAULT (CAST(strftime('%s','now') AS INTEGER) * 1000) ); CREATE TABLE IF NOT EXISTS _vexfs_inodes( @@ -549,6 +570,35 @@ CREATE TABLE IF NOT EXISTS _vexfs_inodes( ); CREATE INDEX IF NOT EXISTS _vexfs_inodes_workspace_idx ON _vexfs_inodes(workspace_id, id); +CREATE TRIGGER IF NOT EXISTS _vexfs_quota_inode_insert +AFTER INSERT ON _vexfs_inodes +WHEN NEW.kind<>'directory' AND NEW.deleted_at IS NULL BEGIN + UPDATE _vexfs_workspaces + SET live_files=live_files+1,live_bytes=live_bytes+NEW.size + WHERE id=NEW.workspace_id; +END; +CREATE TRIGGER IF NOT EXISTS _vexfs_quota_inode_update +AFTER UPDATE ON _vexfs_inodes BEGIN + UPDATE _vexfs_workspaces + SET live_files=live_files-CASE WHEN OLD.kind<>'directory' AND OLD.deleted_at IS NULL + THEN 1 ELSE 0 END, + live_bytes=live_bytes-CASE WHEN OLD.kind<>'directory' AND OLD.deleted_at IS NULL + THEN OLD.size ELSE 0 END + WHERE id=OLD.workspace_id; + UPDATE _vexfs_workspaces + SET live_files=live_files+CASE WHEN NEW.kind<>'directory' AND NEW.deleted_at IS NULL + THEN 1 ELSE 0 END, + live_bytes=live_bytes+CASE WHEN NEW.kind<>'directory' AND NEW.deleted_at IS NULL + THEN NEW.size ELSE 0 END + WHERE id=NEW.workspace_id; +END; +CREATE TRIGGER IF NOT EXISTS _vexfs_quota_inode_delete +AFTER DELETE ON _vexfs_inodes +WHEN OLD.kind<>'directory' AND OLD.deleted_at IS NULL BEGIN + UPDATE _vexfs_workspaces + SET live_files=live_files-1,live_bytes=live_bytes-OLD.size + WHERE id=OLD.workspace_id; +END; CREATE TABLE IF NOT EXISTS _vexfs_dentries( workspace_id INTEGER NOT NULL, parent_inode INTEGER NOT NULL, @@ -565,16 +615,46 @@ CREATE TABLE IF NOT EXISTS _vexfs_commits( message TEXT NOT NULL, created_at INTEGER NOT NULL DEFAULT (CAST(strftime('%s','now') AS INTEGER) * 1000) ); +CREATE TABLE IF NOT EXISTS _vexfs_manifests( + id INTEGER PRIMARY KEY, + workspace_id INTEGER NOT NULL, + file_size INTEGER NOT NULL CHECK(file_size >= 0), + chunk_size INTEGER NOT NULL CHECK(chunk_size = 65536), + chunk_count INTEGER NOT NULL CHECK(chunk_count >= 0), + checksum TEXT NOT NULL CHECK(length(checksum)=64), + created_at INTEGER NOT NULL DEFAULT (CAST(strftime('%s','now') AS INTEGER) * 1000) +); +CREATE TABLE IF NOT EXISTS _vexfs_chunks( + id INTEGER PRIMARY KEY, + workspace_id INTEGER NOT NULL, + inode_id INTEGER NOT NULL, + content BLOB NOT NULL, + size INTEGER NOT NULL CHECK(size >= 0 AND size <= 65536), + checksum TEXT NOT NULL CHECK(length(checksum)=64), + created_at INTEGER NOT NULL DEFAULT (CAST(strftime('%s','now') AS INTEGER) * 1000) +); +CREATE TABLE IF NOT EXISTS _vexfs_manifest_chunks( + manifest_id INTEGER NOT NULL, + chunk_no INTEGER NOT NULL CHECK(chunk_no >= 0), + chunk_id INTEGER NOT NULL, + PRIMARY KEY(manifest_id, chunk_no) +); +CREATE INDEX IF NOT EXISTS _vexfs_chunks_manifest_idx + ON _vexfs_manifest_chunks(manifest_id, chunk_no); +CREATE INDEX IF NOT EXISTS _vexfs_manifest_chunks_chunk_idx + ON _vexfs_manifest_chunks(chunk_id); CREATE TABLE IF NOT EXISTS _vexfs_file_versions( id INTEGER PRIMARY KEY, inode_id INTEGER NOT NULL, version_no INTEGER NOT NULL, commit_id INTEGER NOT NULL, - content BLOB NOT NULL, + manifest_id INTEGER, size INTEGER NOT NULL, checksum TEXT NOT NULL CHECK(length(checksum)=64), source_version_no INTEGER, created_at INTEGER NOT NULL DEFAULT (CAST(strftime('%s','now') AS INTEGER) * 1000), + CHECK((manifest_id IS NOT NULL AND source_version_no IS NULL) OR + (manifest_id IS NULL AND source_version_no IS NOT NULL)), UNIQUE(inode_id, version_no) ); CREATE TABLE IF NOT EXISTS _vexfs_xattrs( @@ -642,7 +722,8 @@ CREATE TABLE IF NOT EXISTS _vexfs_mount_sessions( created_at INTEGER NOT NULL DEFAULT (CAST(strftime('%s','now') AS INTEGER) * 1000), updated_at INTEGER NOT NULL DEFAULT (CAST(strftime('%s','now') AS INTEGER) * 1000) ); -INSERT OR IGNORE INTO _vexfs_meta(key, value) VALUES('contract_version', '0.7.0'); +INSERT OR IGNORE INTO _vexfs_meta(key, value) VALUES('contract_version', '0.9.0'); +INSERT OR IGNORE INTO _vexfs_meta(key, value) VALUES('content_model', 'chunked-v1'); INSERT OR IGNORE INTO _vexfs_meta(key, value) VALUES('staging_layout', 'split-v1'); )SQL"); CreateHistorySchema(db); @@ -798,7 +879,7 @@ struct Workspace { Workspace FindWorkspace(sqlite3 *db, const std::string &name) { Statement statement(db, "SELECT id, root_inode, COALESCE(head_commit, 0) " - "FROM _vexfs_workspaces WHERE name = ?1"); + "FROM _vexfs_workspaces WHERE name = ?1 AND state='active'"); statement.BindText(1, name); if (!statement.Row()) throw SqlError("workspace not found: " + name, SQLITE_NOTFOUND); return {statement.Int64(0), statement.Int64(1), statement.Int64(2)}; @@ -819,25 +900,102 @@ struct Node { sqlite3_int64 changed_at = 0; }; +struct QuotaPolicy { + sqlite3_int64 max_bytes = -1; + sqlite3_int64 max_files = -1; + sqlite3_int64 max_file_bytes = -1; +}; + +struct QuotaStats { + sqlite3_int64 live_files = 0; + sqlite3_int64 live_bytes = 0; + sqlite3_int64 largest_file_bytes = 0; +}; + +QuotaPolicy ReadQuotaPolicy(sqlite3 *db, sqlite3_int64 workspace_id) { + Statement statement(db, + "SELECT quota_max_bytes,quota_max_files,quota_max_file_bytes " + "FROM _vexfs_workspaces WHERE id=?1"); + statement.BindInt64(1, workspace_id); + if (!statement.Row()) throw SqlError("workspace not found", SQLITE_NOTFOUND); + return { + statement.Type(0) == SQLITE_NULL ? -1 : statement.Int64(0), + statement.Type(1) == SQLITE_NULL ? -1 : statement.Int64(1), + statement.Type(2) == SQLITE_NULL ? -1 : statement.Int64(2), + }; +} + +void CheckQuotaForNewInode(sqlite3 *db, sqlite3_int64 workspace_id, + const char *kind) { + if (std::strcmp(kind, "directory") == 0) return; + Statement usage(db, + "SELECT quota_max_files,live_files FROM _vexfs_workspaces WHERE id=?1"); + usage.BindInt64(1, workspace_id); + if (!usage.Row()) throw SqlError("workspace not found", SQLITE_NOTFOUND); + if (usage.Type(0) != SQLITE_NULL && usage.Int64(1) >= usage.Int64(0)) { + throw SqlError("workspace file quota exceeded", SQLITE_FULL); + } +} + +void CheckQuotaForContentSize(sqlite3 *db, sqlite3_int64 workspace_id, + sqlite3_int64 inode_id, sqlite3_int64 new_size) { + if (new_size < 0) throw SqlError("file size must be non-negative", SQLITE_RANGE); + const QuotaPolicy policy = ReadQuotaPolicy(db, workspace_id); + Statement usage(db, R"SQL( +SELECT live_bytes, + COALESCE((SELECT size FROM _vexfs_inodes + WHERE id=?2 AND workspace_id=?1 AND deleted_at IS NULL + AND kind<>'directory'),0), + EXISTS(SELECT 1 FROM _vexfs_inodes + WHERE id=?2 AND workspace_id=?1 AND deleted_at IS NULL + AND kind<>'directory') +FROM _vexfs_workspaces +WHERE id=?1 +)SQL"); + usage.BindInt64(1, workspace_id); + usage.BindInt64(2, inode_id); + if (!usage.Row()) throw SqlError("workspace not found", SQLITE_NOTFOUND); + const bool is_live = usage.Int(2) != 0; + const sqlite3_int64 old_size = is_live ? usage.Int64(1) : 0; + + // Lowering a quota below current usage must not strand the workspace. Writes that + // shrink a file remain legal until usage is back under the configured limit. + if (policy.max_file_bytes >= 0 && new_size > policy.max_file_bytes && + (!is_live || new_size >= old_size)) { + throw SqlError("maximum file size quota exceeded", SQLITE_FULL); + } + if (policy.max_bytes >= 0 && is_live && new_size >= old_size) { + const sqlite3_int64 unchanged_bytes = usage.Int64(0) - old_size; + if (new_size > policy.max_bytes || + unchanged_bytes > policy.max_bytes - new_size) { + throw SqlError("workspace byte quota exceeded", SQLITE_FULL); + } + } +} + struct VersionStorage { - sqlite3_int64 rowid = 0; + sqlite3_int64 manifest_id = 0; sqlite3_int64 size = 0; sqlite3_int64 canonical_version = 0; + sqlite3_int64 chunk_count = 0; std::string checksum; }; VersionStorage ResolveVersionStorage(sqlite3 *db, sqlite3_int64 inode_id, sqlite3_int64 version) { Statement statement(db, R"SQL( -SELECT CASE WHEN v.source_version_no IS NULL THEN v.rowid ELSE source.rowid END, +SELECT CASE WHEN v.source_version_no IS NULL THEN v.manifest_id ELSE source.manifest_id END, v.size, COALESCE(v.source_version_no,v.version_no), - CASE WHEN v.source_version_no IS NULL THEN v.size ELSE source.size END, + manifest.file_size, v.checksum, - CASE WHEN v.source_version_no IS NULL THEN v.checksum ELSE source.checksum END + CASE WHEN v.source_version_no IS NULL THEN v.checksum ELSE source.checksum END, + manifest.checksum,manifest.chunk_size,manifest.chunk_count FROM _vexfs_file_versions v LEFT JOIN _vexfs_file_versions source ON source.inode_id=v.inode_id AND source.version_no=v.source_version_no +LEFT JOIN _vexfs_manifests manifest + ON manifest.id=CASE WHEN v.source_version_no IS NULL THEN v.manifest_id ELSE source.manifest_id END WHERE v.inode_id=?1 AND v.version_no=?2 )SQL"); statement.BindInt64(1, inode_id); @@ -845,12 +1003,15 @@ WHERE v.inode_id=?1 AND v.version_no=?2 if (!statement.Row()) { throw SqlError("file version not found: " + std::to_string(version), SQLITE_NOTFOUND); } - if (statement.Type(0) == SQLITE_NULL || statement.Int64(1) < 0 || - statement.Int64(1) != statement.Int64(3) || statement.Text(4).size() != 64 || - statement.Text(4) != statement.Text(5)) { + if (statement.Type(0) == SQLITE_NULL || statement.Int64(0) <= 0 || + statement.Int64(1) < 0 || statement.Int64(1) != statement.Int64(3) || + statement.Text(4).size() != 64 || statement.Text(4) != statement.Text(5) || + statement.Text(4) != statement.Text(6) || + statement.Int64(7) != kContentChunkBytes || statement.Int64(8) < 0) { throw SqlError("file version content reference is corrupt", SQLITE_CORRUPT); } - return {statement.Int64(0), statement.Int64(1), statement.Int64(2), statement.Text(4)}; + return {statement.Int64(0), statement.Int64(1), statement.Int64(2), + statement.Int64(8), statement.Text(4)}; } std::vector ReadVersionContent(sqlite3 *db, sqlite3_int64 inode_id, @@ -859,18 +1020,36 @@ std::vector ReadVersionContent(sqlite3 *db, sqlite3_int64 inode_i if (storage.size > kMaxStagedBytes) { throw SqlError("file version is larger than 128 MiB", SQLITE_CORRUPT); } - ReadOnlyBlob blob(db, storage.rowid); - if (blob.size() != storage.size) { - throw SqlError("file version size does not match content", SQLITE_CORRUPT); - } std::vector content(static_cast(storage.size)); - constexpr int kChunkSize = 64 * 1024; - for (sqlite3_int64 offset = 0; offset < storage.size; offset += kChunkSize) { - const int bytes = static_cast( - std::min(kChunkSize, storage.size - offset)); - blob.Read(content.data() + offset, bytes, static_cast(offset)); + Statement chunks(db, R"SQL( +SELECT chunk_no,content,size,checksum +FROM _vexfs_manifest_chunks entry +JOIN _vexfs_chunks chunk ON chunk.id=entry.chunk_id +WHERE entry.manifest_id=?1 ORDER BY entry.chunk_no +)SQL"); + chunks.BindInt64(1, storage.manifest_id); + vexfs::Sha256 file_hash; + sqlite3_int64 offset = 0; + sqlite3_int64 expected_chunk = 0; + while (chunks.Row()) { + const std::vector chunk = chunks.Blob(1); + const sqlite3_int64 expected_size = std::min( + kContentChunkBytes, storage.size - offset); + if (chunks.Int64(0) != expected_chunk || expected_size <= 0 || + chunks.Int64(2) != expected_size || + static_cast(chunk.size()) != expected_size || + chunks.Text(3).size() != 64 || + vexfs::Sha256Hex(chunk.data(), chunk.size()) != chunks.Text(3)) { + throw SqlError("file manifest chunk is corrupt", SQLITE_CORRUPT); + } + std::copy(chunk.begin(), chunk.end(), content.begin() + offset); + file_hash.Update(chunk.data(), chunk.size()); + offset += expected_size; + ++expected_chunk; } - if (vexfs::Sha256Hex(content.data(), content.size()) != storage.checksum) { + if (offset != storage.size || expected_chunk != storage.chunk_count || + expected_chunk != (storage.size + kContentChunkBytes - 1) / kContentChunkBytes || + vexfs::Hex(file_hash.Finish()) != storage.checksum) { throw SqlError("file version checksum does not match content", SQLITE_CORRUPT); } return content; @@ -1078,6 +1257,7 @@ sqlite3_int64 CreateCommit(sqlite3 *db, const Workspace &workspace, const std::s } sqlite3_int64 CreateInode(sqlite3 *db, sqlite3_int64 workspace_id, const char *kind, int mode) { + CheckQuotaForNewInode(db, workspace_id, kind); Statement insert(db, "INSERT INTO _vexfs_inodes(workspace_id, kind, mode, owner_principal, uid, gid, current_version) " "VALUES(?1, ?2, ?3, 'local', ?4, ?5, ?6)"); @@ -1146,22 +1326,21 @@ void UpdateGrepIndexFromVersion(sqlite3 *db, sqlite3_int64 workspace_id, Statement remove(db, "DELETE FROM _vexfs_content_fts WHERE rowid=?1"); remove.BindInt64(1, inode_id); remove.Done(); + Statement live(db, "SELECT 1 FROM _vexfs_inodes WHERE id=?1 AND workspace_id=?2 " + "AND kind='file' AND deleted_at IS NULL"); + live.BindInt64(1, inode_id); + live.BindInt64(2, workspace_id); + if (!live.Row()) return; + const std::vector content = ReadVersionContent(db, inode_id, version); Statement insert(db, R"SQL( INSERT INTO _vexfs_content_fts(rowid,workspace_id,inode_id,version_no,content) -SELECT inode.id,inode.workspace_id,inode.id,version.version_no, - CAST(CASE WHEN version.source_version_no IS NULL - THEN version.content ELSE source.content END AS TEXT) -FROM _vexfs_inodes inode -JOIN _vexfs_file_versions version - ON version.inode_id=inode.id AND version.version_no=?3 -LEFT JOIN _vexfs_file_versions source - ON source.inode_id=version.inode_id AND source.version_no=version.source_version_no -WHERE inode.id=?2 AND inode.workspace_id=?1 AND inode.kind='file' - AND inode.deleted_at IS NULL +VALUES(?1,?2,?1,?3,?4) )SQL"); - insert.BindInt64(1, workspace_id); - insert.BindInt64(2, inode_id); + insert.BindInt64(1, inode_id); + insert.BindInt64(2, workspace_id); insert.BindInt64(3, version); + insert.BindText(4, content.empty() ? std::string() : + std::string(reinterpret_cast(content.data()), content.size())); insert.Done(); } @@ -1189,10 +1368,93 @@ WHERE rowid IN ( remove.Done(); } +sqlite3_int64 StoreManifest(sqlite3 *db, sqlite3_int64 workspace_id, + sqlite3_int64 inode_id, + sqlite3_int64 previous_manifest, + const std::vector &content, + const std::string &checksum) { + const sqlite3_int64 size = static_cast(content.size()); + const sqlite3_int64 chunk_count = + (size + kContentChunkBytes - 1) / kContentChunkBytes; + Statement manifest(db, R"SQL( +INSERT INTO _vexfs_manifests(workspace_id,file_size,chunk_size,chunk_count,checksum) +VALUES(?1,?2,?3,?4,?5) +)SQL"); + manifest.BindInt64(1, workspace_id); + manifest.BindInt64(2, size); + manifest.BindInt64(3, kContentChunkBytes); + manifest.BindInt64(4, chunk_count); + manifest.BindText(5, checksum); + manifest.Done(); + const sqlite3_int64 manifest_id = sqlite3_last_insert_rowid(db); + + Statement previous(db, R"SQL( +SELECT chunk.id,chunk.content,chunk.size,chunk.checksum +FROM _vexfs_manifest_chunks entry +JOIN _vexfs_chunks chunk ON chunk.id=entry.chunk_id +WHERE entry.manifest_id=?1 AND entry.chunk_no=?2 AND chunk.inode_id=?3 +)SQL"); + Statement chunk(db, R"SQL( +INSERT INTO _vexfs_chunks(workspace_id,inode_id,content,size,checksum) +VALUES(?1,?2,?3,?4,?5) +)SQL"); + Statement entry(db, R"SQL( +INSERT INTO _vexfs_manifest_chunks(manifest_id,chunk_no,chunk_id) +VALUES(?1,?2,?3) +)SQL"); + std::unordered_map new_chunks; + for (sqlite3_int64 chunk_no = 0, offset = 0; offset < size; + ++chunk_no, offset += kContentChunkBytes) { + const size_t bytes = static_cast(std::min( + kContentChunkBytes, size - offset)); + std::vector data( + content.begin() + offset, content.begin() + offset + bytes); + const std::string chunk_checksum = vexfs::Sha256Hex(data.data(), data.size()); + sqlite3_int64 chunk_id = 0; + if (previous_manifest > 0) { + previous.BindInt64(1, previous_manifest); + previous.BindInt64(2, chunk_no); + previous.BindInt64(3, inode_id); + if (previous.Row() && previous.Int64(2) == static_cast(bytes) && + previous.Text(3) == chunk_checksum && previous.Blob(1) == data) { + chunk_id = previous.Int64(0); + } + previous.Reset(); + } + const std::string reuse_key = std::to_string(bytes) + ":" + chunk_checksum; + if (chunk_id == 0) { + const auto reused = new_chunks.find(reuse_key); + if (reused != new_chunks.end()) chunk_id = reused->second; + } + if (chunk_id == 0) { + chunk.BindInt64(1, workspace_id); + chunk.BindInt64(2, inode_id); + chunk.BindBlob(3, data); + chunk.BindInt64(4, static_cast(bytes)); + chunk.BindText(5, chunk_checksum); + chunk.Done(); + chunk_id = sqlite3_last_insert_rowid(db); + new_chunks.emplace(reuse_key, chunk_id); + chunk.Reset(); + } + entry.BindInt64(1, manifest_id); + entry.BindInt64(2, chunk_no); + entry.BindInt64(3, chunk_id); + entry.Done(); + entry.Reset(); + } + return manifest_id; +} + sqlite3_int64 StoreVersion(sqlite3 *db, const Workspace &workspace, const Node &node, const std::vector &content, const std::string &message) { Savepoint savepoint(db, "vexfs_store_version"); + AcquireWriteLock(db); + const sqlite3_int64 previous_manifest = node.version > 0 + ? ResolveVersionStorage(db, node.id, node.version).manifest_id : 0; + CheckQuotaForContentSize(db, workspace.id, node.id, + static_cast(content.size())); const sqlite3_int64 version = node.version + 1; Statement advance(db, "UPDATE _vexfs_inodes SET size=?1, current_version=?2, " @@ -1210,14 +1472,16 @@ sqlite3_int64 StoreVersion(sqlite3 *db, const Workspace &workspace, const Node & const sqlite3_int64 commit = CreateCommit(db, workspace, message); const std::string checksum = vexfs::Sha256Hex(content.data(), content.size()); + const sqlite3_int64 manifest_id = StoreManifest( + db, workspace.id, node.id, previous_manifest, content, checksum); Statement insert(db, "INSERT INTO _vexfs_file_versions" - "(inode_id,version_no,commit_id,content,size,checksum) " + "(inode_id,version_no,commit_id,manifest_id,size,checksum) " "VALUES(?1,?2,?3,?4,?5,?6)"); insert.BindInt64(1, node.id); insert.BindInt64(2, version); insert.BindInt64(3, commit); - insert.BindBlob(4, content); + insert.BindInt64(4, manifest_id); insert.BindInt64(5, static_cast(content.size())); insert.BindText(6, checksum); insert.Done(); @@ -1232,6 +1496,7 @@ sqlite3_int64 StoreVersionFromVersion(sqlite3 *db, const Workspace &workspace, c Savepoint savepoint(db, "vexfs_restore_version"); AcquireWriteLock(db); const VersionStorage source = ResolveVersionStorage(db, node.id, source_version); + CheckQuotaForContentSize(db, workspace.id, node.id, source.size); const sqlite3_int64 version = node.version + 1; Statement advance(db, "UPDATE _vexfs_inodes SET size=?1, current_version=?2, " @@ -1249,8 +1514,8 @@ sqlite3_int64 StoreVersionFromVersion(sqlite3 *db, const Workspace &workspace, c const sqlite3_int64 commit = CreateCommit(db, workspace, message); Statement insert(db, "INSERT INTO _vexfs_file_versions" - "(inode_id,version_no,commit_id,content,size,checksum,source_version_no) " - "VALUES(?1,?2,?3,X'',?4,?5,?6)"); + "(inode_id,version_no,commit_id,manifest_id,size,checksum,source_version_no) " + "VALUES(?1,?2,?3,NULL,?4,?5,?6)"); insert.BindInt64(1, node.id); insert.BindInt64(2, version); insert.BindInt64(3, commit); @@ -1677,6 +1942,167 @@ void CreateStaging(sqlite3 *db, const Handle &handle, } } +void CreateStagingFromVersion(sqlite3 *db, const Handle &handle, const Node &node) { + if (node.version <= 0 || node.size == 0) { + CreateStaging(db, handle, {}); + return; + } + const VersionStorage storage = ResolveVersionStorage(db, node.id, node.version); + if (storage.size != node.size) { + throw SqlError("current file size does not match manifest", SQLITE_CORRUPT); + } + sqlite3_int64 capacity = storage.size; + if (handle.writable && capacity < kInitialStagingCapacity) { + capacity = kInitialStagingCapacity; + } + Statement insert(db, + "INSERT INTO _vexfs_staging(handle_id,generation,content,logical_size,capacity) " + "VALUES(?1,?2,X'',?4,?3)"); + insert.BindText(1, handle.id); + insert.BindInt64(2, handle.dirty_generation); + insert.BindInt64(3, capacity); + insert.BindInt64(4, storage.size); + insert.Done(); + Statement data(db, + "INSERT INTO _vexfs_staging_data(handle_id,content) VALUES(?1,zeroblob(?2))"); + data.BindText(1, handle.id); + data.BindInt64(2, capacity); + data.Done(); + const StagingInfo staging = FindStaging(db, handle); + Statement chunks(db, R"SQL( +SELECT entry.chunk_no,chunk.content,chunk.size,chunk.checksum +FROM _vexfs_manifest_chunks entry +JOIN _vexfs_chunks chunk ON chunk.id=entry.chunk_id +WHERE entry.manifest_id=?1 ORDER BY entry.chunk_no +)SQL"); + chunks.BindInt64(1, storage.manifest_id); + vexfs::Sha256 file_hash; + sqlite3_int64 offset = 0; + sqlite3_int64 chunk_no = 0; + while (chunks.Row()) { + const std::vector chunk = chunks.Blob(1); + const sqlite3_int64 expected_size = std::min( + kContentChunkBytes, storage.size - offset); + if (chunks.Int64(0) != chunk_no || expected_size <= 0 || + chunks.Int64(2) != expected_size || + static_cast(chunk.size()) != expected_size || + vexfs::Sha256Hex(chunk.data(), chunk.size()) != chunks.Text(3)) { + throw SqlError("current file manifest chunk is corrupt", SQLITE_CORRUPT); + } + WriteBlob(db, staging.rowid, offset, chunk.data(), expected_size); + file_hash.Update(chunk.data(), chunk.size()); + offset += expected_size; + ++chunk_no; + } + if (offset != storage.size || chunk_no != storage.chunk_count || + vexfs::Hex(file_hash.Finish()) != storage.checksum) { + throw SqlError("current file manifest checksum mismatch", SQLITE_CORRUPT); + } +} + +sqlite3_int64 StoreManifestFromStaging(sqlite3 *db, sqlite3_int64 workspace_id, + sqlite3_int64 inode_id, + sqlite3_int64 previous_manifest, + const StagingInfo &staging, + std::string *checksum_out) { + if (staging.logical_size < 0 || staging.logical_size > staging.capacity || + staging.logical_size > kMaxStagedBytes) { + throw SqlError("staging size is corrupt", SQLITE_CORRUPT); + } + sqlite3_blob *raw_blob = nullptr; + int rc = sqlite3_blob_open(db, "main", "_vexfs_staging_data", "content", + staging.rowid, 0, &raw_blob); + if (rc != SQLITE_OK) throw SqlError("cannot open staging BLOB", rc); + std::unique_ptr blob(raw_blob); + if (sqlite3_blob_bytes(blob.get()) != staging.capacity) { + throw SqlError("staging capacity does not match BLOB", SQLITE_CORRUPT); + } + std::vector buffer(static_cast(kContentChunkBytes)); + std::vector chunk_checksums; + vexfs::Sha256 file_hash; + for (sqlite3_int64 offset = 0; offset < staging.logical_size; + offset += kContentChunkBytes) { + const int bytes = static_cast(std::min( + kContentChunkBytes, staging.logical_size - offset)); + rc = sqlite3_blob_read(blob.get(), buffer.data(), bytes, static_cast(offset)); + if (rc != SQLITE_OK) throw SqlError("cannot read staging BLOB", rc); + file_hash.Update(buffer.data(), static_cast(bytes)); + chunk_checksums.push_back(vexfs::Sha256Hex(buffer.data(), static_cast(bytes))); + } + const std::string checksum = vexfs::Hex(file_hash.Finish()); + const sqlite3_int64 chunk_count = static_cast(chunk_checksums.size()); + Statement manifest(db, R"SQL( +INSERT INTO _vexfs_manifests(workspace_id,file_size,chunk_size,chunk_count,checksum) +VALUES(?1,?2,?3,?4,?5) +)SQL"); + manifest.BindInt64(1, workspace_id); + manifest.BindInt64(2, staging.logical_size); + manifest.BindInt64(3, kContentChunkBytes); + manifest.BindInt64(4, chunk_count); + manifest.BindText(5, checksum); + manifest.Done(); + const sqlite3_int64 manifest_id = sqlite3_last_insert_rowid(db); + Statement previous(db, R"SQL( +SELECT chunk.id,chunk.content,chunk.size,chunk.checksum +FROM _vexfs_manifest_chunks entry +JOIN _vexfs_chunks chunk ON chunk.id=entry.chunk_id +WHERE entry.manifest_id=?1 AND entry.chunk_no=?2 AND chunk.inode_id=?3 +)SQL"); + Statement chunk(db, R"SQL( +INSERT INTO _vexfs_chunks(workspace_id,inode_id,content,size,checksum) +VALUES(?1,?2,?3,?4,?5) +)SQL"); + Statement entry(db, R"SQL( +INSERT INTO _vexfs_manifest_chunks(manifest_id,chunk_no,chunk_id) +VALUES(?1,?2,?3) +)SQL"); + std::unordered_map new_chunks; + for (sqlite3_int64 chunk_no = 0, offset = 0; chunk_no < chunk_count; + ++chunk_no, offset += kContentChunkBytes) { + const int bytes = static_cast(std::min( + kContentChunkBytes, staging.logical_size - offset)); + rc = sqlite3_blob_read(blob.get(), buffer.data(), bytes, static_cast(offset)); + if (rc != SQLITE_OK) throw SqlError("cannot reread staging BLOB", rc); + std::vector content(buffer.begin(), buffer.begin() + bytes); + const std::string &chunk_checksum = + chunk_checksums[static_cast(chunk_no)]; + sqlite3_int64 chunk_id = 0; + if (previous_manifest > 0) { + previous.BindInt64(1, previous_manifest); + previous.BindInt64(2, chunk_no); + previous.BindInt64(3, inode_id); + if (previous.Row() && previous.Int64(2) == bytes && + previous.Text(3) == chunk_checksum && previous.Blob(1) == content) { + chunk_id = previous.Int64(0); + } + previous.Reset(); + } + const std::string reuse_key = std::to_string(bytes) + ":" + chunk_checksum; + if (chunk_id == 0) { + const auto reused = new_chunks.find(reuse_key); + if (reused != new_chunks.end()) chunk_id = reused->second; + } + if (chunk_id == 0) { + chunk.BindInt64(1, workspace_id); + chunk.BindInt64(2, inode_id); + chunk.BindBlob(3, content); + chunk.BindInt64(4, bytes); + chunk.BindText(5, chunk_checksum); + chunk.Done(); + chunk_id = sqlite3_last_insert_rowid(db); + new_chunks.emplace(reuse_key, chunk_id); + chunk.Reset(); + } + entry.BindInt64(1, manifest_id); + entry.BindInt64(2, chunk_no); + entry.BindInt64(3, chunk_id); + entry.Done(); + entry.Reset(); + } + *checksum_out = checksum; + return manifest_id; +} + sqlite3_int64 StageWrite(sqlite3 *db, const Handle &handle, sqlite3_int64 offset, const std::vector &patch) { if (patch.empty()) return handle.dirty_generation; @@ -1730,18 +2156,6 @@ sqlite3_int64 TruncateStaging(sqlite3 *db, const Handle &handle, sqlite3_int64 s return generation; } -std::vector ReadStaging(sqlite3 *db, const Handle &handle, - sqlite3_int64 generation) { - Statement statement(db, - "SELECT substr(d.content,1,s.logical_size) FROM _vexfs_staging s " - "JOIN _vexfs_staging_data d ON d.handle_id=s.handle_id " - "WHERE s.handle_id=?1 AND s.generation=?2"); - statement.BindText(1, handle.id); - statement.BindInt64(2, generation); - if (!statement.Row()) throw SqlError("staged generation is stale or missing", SQLITE_NOTFOUND); - return statement.Blob(0); -} - std::vector ReadStagingRange(sqlite3 *db, const Handle &handle, sqlite3_int64 offset, sqlite3_int64 length) { Statement statement(db, @@ -1775,7 +2189,55 @@ Node NodeById(sqlite3 *db, sqlite3_int64 id, bool include_deleted = false) { return {statement.Int64(0), statement.Text(1), statement.Int64(2), statement.Int64(3), statement.Int(4), statement.Text(5), statement.Int64(6), statement.Int64(7), statement.Int64(8), statement.Int64(9), - statement.Int64(10), statement.Int64(11)}; + statement.Int64(10), statement.Int64(11)}; +} + +sqlite3_int64 StoreVersionFromStaging(sqlite3 *db, const Workspace &workspace, + const Node &node, const Handle &handle, + sqlite3_int64 generation, + const std::string &message) { + Savepoint savepoint(db, "vexfs_store_staged_version"); + AcquireWriteLock(db); + const StagingInfo staging = FindStaging(db, handle); + if (staging.generation != generation) { + throw SqlError("staged generation is stale or missing", SQLITE_NOTFOUND); + } + const sqlite3_int64 previous_manifest = node.version > 0 + ? ResolveVersionStorage(db, node.id, node.version).manifest_id : 0; + CheckQuotaForContentSize(db, workspace.id, node.id, staging.logical_size); + const sqlite3_int64 version = node.version + 1; + Statement advance(db, + "UPDATE _vexfs_inodes SET size=?1,current_version=?2," + "updated_at=CAST(unixepoch('subsec')*1000 AS INTEGER)," + "changed_at=CAST(unixepoch('subsec')*1000 AS INTEGER) " + "WHERE id=?3 AND current_version=?4"); + advance.BindInt64(1, staging.logical_size); + advance.BindInt64(2, version); + advance.BindInt64(3, node.id); + advance.BindInt64(4, node.version); + advance.Done(); + if (sqlite3_changes64(db) != 1) { + throw SqlError("version conflict: file changed before publish", SQLITE_CONSTRAINT); + } + const sqlite3_int64 commit = CreateCommit(db, workspace, message); + std::string checksum; + const sqlite3_int64 manifest_id = StoreManifestFromStaging( + db, workspace.id, node.id, previous_manifest, staging, &checksum); + Statement insert(db, R"SQL( +INSERT INTO _vexfs_file_versions( + inode_id,version_no,commit_id,manifest_id,size,checksum) +VALUES(?1,?2,?3,?4,?5,?6) +)SQL"); + insert.BindInt64(1, node.id); + insert.BindInt64(2, version); + insert.BindInt64(3, commit); + insert.BindInt64(4, manifest_id); + insert.BindInt64(5, staging.logical_size); + insert.BindText(6, checksum); + insert.Done(); + UpdateGrepIndexFromVersion(db, workspace.id, node.id, version); + savepoint.Release(); + return version; } sqlite3_int64 PublishHandle(sqlite3 *db, Handle handle, sqlite3_int64 generation) { @@ -1791,10 +2253,10 @@ sqlite3_int64 PublishHandle(sqlite3 *db, Handle handle, sqlite3_int64 generation if (node.version != handle.expected_version) { throw SqlError("write conflict: file changed after handle_open", SQLITE_CONSTRAINT); } - const std::vector content = ReadStaging(db, handle, generation); Workspace workspace = WorkspaceById(db, handle.workspace_id); Savepoint savepoint(db, "vexfs_publish_handle"); - const sqlite3_int64 version = StoreVersion(db, workspace, node, content, "handle publish"); + const sqlite3_int64 version = StoreVersionFromStaging( + db, workspace, node, handle, generation, "handle publish"); Statement update(db, "UPDATE _vexfs_handles SET expected_version = ?1, published_generation = ?2, " @@ -2110,14 +2572,10 @@ WITH RECURSIVE tree(inode_id,path,kind,current_version,size) AS ( JOIN _vexfs_inodes child ON child.id=d.inode_id AND child.deleted_at IS NULL WHERE tree.kind='directory' ) -SELECT tree.path,tree.inode_id,tree.current_version,tree.size, - CASE WHEN version.source_version_no IS NULL THEN version.content ELSE source.content END, - version.checksum +SELECT tree.path,tree.inode_id,tree.current_version,tree.size,version.checksum FROM tree JOIN _vexfs_file_versions version ON version.inode_id=tree.inode_id AND version.version_no=tree.current_version -LEFT JOIN _vexfs_file_versions source - ON source.inode_id=version.inode_id AND source.version_no=version.source_version_no WHERE tree.kind='file' )SQL"; constexpr const char *indexed_query = R"SQL( @@ -2138,15 +2596,11 @@ ancestors(inode_id,parent_inode,path,depth) AS ( paths(inode_id,path) AS MATERIALIZED ( SELECT inode_id,'/'||path FROM ancestors WHERE parent_inode=?3 ) -SELECT paths.path,inode.id,inode.current_version,inode.size, - CASE WHEN version.source_version_no IS NULL THEN version.content ELSE source.content END, - version.checksum +SELECT paths.path,inode.id,inode.current_version,inode.size,version.checksum FROM paths JOIN _vexfs_inodes inode ON inode.id=paths.inode_id AND inode.deleted_at IS NULL JOIN _vexfs_file_versions version ON version.inode_id=inode.id AND version.version_no=inode.current_version -LEFT JOIN _vexfs_file_versions source - ON source.inode_id=version.inode_id AND source.version_no=version.source_version_no WHERE inode.kind='file' AND (?2='/' OR paths.path=?2 OR substr(paths.path,1,length(?2)+1)=?2||'/') )SQL"; @@ -2174,16 +2628,16 @@ WHERE inode.kind='file' AND while (files.Row()) { ++files_scanned; const sqlite3_int64 expected_size = files.Int64(3); - if (expected_size < 0 || expected_size > kMaxStagedBytes || - files.Type(4) == SQLITE_NULL) { + if (expected_size < 0 || expected_size > kMaxStagedBytes) { throw SqlError("current file content reference is corrupt", SQLITE_CORRUPT); } - const std::vector content = files.Blob(4); + const std::vector content = + ReadVersionContent(db, files.Int64(1), files.Int64(2)); if (static_cast(content.size()) != expected_size) { throw SqlError("current file size does not match content", SQLITE_CORRUPT); } - if (!IsSha256(files.Text(5)) || - vexfs::Sha256Hex(content.data(), content.size()) != files.Text(5)) { + if (!IsSha256(files.Text(4)) || + vexfs::Sha256Hex(content.data(), content.size()) != files.Text(4)) { throw SqlError("current file checksum does not match content", SQLITE_CORRUPT); } bytes_scanned += expected_size; @@ -2348,37 +2802,68 @@ void CompareVersionsFunction(sqlite3_context *context, int, sqlite3_value **valu const sqlite3_int64 to_version = RequiredPositiveInteger(values[3], "to version"); const VersionStorage from = ResolveVersionStorage(db, node.id, from_version); const VersionStorage to = ResolveVersionStorage(db, node.id, to_version); - ReadOnlyBlob from_blob(db, from.rowid); - ReadOnlyBlob to_blob(db, to.rowid); - if (from_blob.size() != from.size || to_blob.size() != to.size) { - throw SqlError("file version size does not match content", SQLITE_CORRUPT); - } - std::array from_chunk{}; - std::array to_chunk{}; + Statement from_rows(db, R"SQL( +SELECT entry.chunk_no,chunk.content,chunk.size,chunk.checksum +FROM _vexfs_manifest_chunks entry +JOIN _vexfs_chunks chunk ON chunk.id=entry.chunk_id +WHERE entry.manifest_id=?1 ORDER BY entry.chunk_no +)SQL"); + Statement to_rows(db, R"SQL( +SELECT entry.chunk_no,chunk.content,chunk.size,chunk.checksum +FROM _vexfs_manifest_chunks entry +JOIN _vexfs_chunks chunk ON chunk.id=entry.chunk_id +WHERE entry.manifest_id=?1 ORDER BY entry.chunk_no +)SQL"); + from_rows.BindInt64(1, from.manifest_id); + to_rows.BindInt64(1, to.manifest_id); vexfs::Sha256 from_hash; vexfs::Sha256 to_hash; bool changed = from.size != to.size; bool binary = false; - const sqlite3_int64 maximum = std::max(from.size, to.size); - for (sqlite3_int64 offset = 0; offset < maximum; offset += from_chunk.size()) { - const int from_bytes = static_cast(std::min( - from_chunk.size(), std::max(0, from.size - offset))); - const int to_bytes = static_cast(std::min( - to_chunk.size(), std::max(0, to.size - offset))); - if (from_bytes > 0) from_blob.Read(from_chunk.data(), from_bytes, static_cast(offset)); - if (to_bytes > 0) to_blob.Read(to_chunk.data(), to_bytes, static_cast(offset)); - from_hash.Update(from_chunk.data(), static_cast(from_bytes)); - to_hash.Update(to_chunk.data(), static_cast(to_bytes)); + sqlite3_int64 chunk_no = 0; + sqlite3_int64 from_chunks = 0; + sqlite3_int64 to_chunks = 0; + sqlite3_int64 from_bytes_total = 0; + sqlite3_int64 to_bytes_total = 0; + while (true) { + const bool has_from = from_rows.Row(); + const bool has_to = to_rows.Row(); + if (!has_from && !has_to) break; + std::vector from_chunk; + std::vector to_chunk; + if (has_from) { + from_chunk = from_rows.Blob(1); + if (from_rows.Int64(0) != chunk_no || + from_rows.Int64(2) != static_cast(from_chunk.size()) || + from_chunk.empty() || from_chunk.size() > kContentChunkBytes || + vexfs::Sha256Hex(from_chunk.data(), from_chunk.size()) != from_rows.Text(3)) { + throw SqlError("file manifest chunk is corrupt", SQLITE_CORRUPT); + } + from_hash.Update(from_chunk.data(), from_chunk.size()); + from_bytes_total += static_cast(from_chunk.size()); + ++from_chunks; + } + if (has_to) { + to_chunk = to_rows.Blob(1); + if (to_rows.Int64(0) != chunk_no || + to_rows.Int64(2) != static_cast(to_chunk.size()) || + to_chunk.empty() || to_chunk.size() > kContentChunkBytes || + vexfs::Sha256Hex(to_chunk.data(), to_chunk.size()) != to_rows.Text(3)) { + throw SqlError("file manifest chunk is corrupt", SQLITE_CORRUPT); + } + to_hash.Update(to_chunk.data(), to_chunk.size()); + to_bytes_total += static_cast(to_chunk.size()); + ++to_chunks; + } binary = binary || - std::find(from_chunk.begin(), from_chunk.begin() + from_bytes, 0) != - from_chunk.begin() + from_bytes || - std::find(to_chunk.begin(), to_chunk.begin() + to_bytes, 0) != - to_chunk.begin() + to_bytes; - if (!changed && (from_bytes != to_bytes || - !std::equal(from_chunk.begin(), from_chunk.begin() + from_bytes, - to_chunk.begin()))) changed = true; - } - if (vexfs::Hex(from_hash.Finish()) != from.checksum || + std::find(from_chunk.begin(), from_chunk.end(), 0) != from_chunk.end() || + std::find(to_chunk.begin(), to_chunk.end(), 0) != to_chunk.end(); + if (!changed && from_chunk != to_chunk) changed = true; + ++chunk_no; + } + if (from_bytes_total != from.size || to_bytes_total != to.size || + from_chunks != from.chunk_count || to_chunks != to.chunk_count || + vexfs::Hex(from_hash.Finish()) != from.checksum || vexfs::Hex(to_hash.Finish()) != to.checksum) { throw SqlError("file version checksum does not match content", SQLITE_CORRUPT); } @@ -2464,6 +2949,8 @@ struct CheckReport { sqlite3_int64 inodes = 0; sqlite3_int64 dentries = 0; sqlite3_int64 versions = 0; + sqlite3_int64 manifests = 0; + sqlite3_int64 chunks = 0; sqlite3_int64 content_bytes = 0; sqlite3_int64 commits = 0; sqlite3_int64 snapshots = 0; @@ -2489,18 +2976,37 @@ bool IsSha256(const std::string &value) { }); } -std::string HashVersionBlob(sqlite3 *db, sqlite3_int64 rowid, sqlite3_int64 expected_size) { - ReadOnlyBlob blob(db, rowid); - if (blob.size() != expected_size) { - throw SqlError("file version size does not match content", SQLITE_CORRUPT); - } +std::string HashManifest(sqlite3 *db, sqlite3_int64 manifest_id, + sqlite3_int64 expected_size, + sqlite3_int64 expected_chunk_count) { + Statement rows(db, R"SQL( +SELECT entry.chunk_no,chunk.content,chunk.size,chunk.checksum +FROM _vexfs_manifest_chunks entry +JOIN _vexfs_chunks chunk ON chunk.id=entry.chunk_id +WHERE entry.manifest_id=?1 ORDER BY entry.chunk_no +)SQL"); + rows.BindInt64(1, manifest_id); vexfs::Sha256 hash; - std::array chunk{}; - for (sqlite3_int64 offset = 0; offset < expected_size; offset += chunk.size()) { - const int bytes = static_cast(std::min( - static_cast(chunk.size()), expected_size - offset)); - blob.Read(chunk.data(), bytes, static_cast(offset)); - hash.Update(chunk.data(), static_cast(bytes)); + sqlite3_int64 chunk_no = 0; + sqlite3_int64 total = 0; + while (rows.Row()) { + const std::vector content = rows.Blob(1); + const sqlite3_int64 expected_chunk_size = std::min( + kContentChunkBytes, expected_size - total); + if (rows.Int64(0) != chunk_no || expected_chunk_size <= 0 || + rows.Int64(2) != expected_chunk_size || + static_cast(content.size()) != expected_chunk_size || + !IsSha256(rows.Text(3)) || + vexfs::Sha256Hex(content.data(), content.size()) != rows.Text(3)) { + throw SqlError("file manifest chunk is corrupt", SQLITE_CORRUPT); + } + hash.Update(content.data(), content.size()); + total += expected_chunk_size; + ++chunk_no; + } + if (total != expected_size || chunk_no != expected_chunk_count || + chunk_no != (expected_size + kContentChunkBytes - 1) / kContentChunkBytes) { + throw SqlError("file manifest chunk count or size is corrupt", SQLITE_CORRUPT); } return vexfs::Hex(hash.Finish()); } @@ -2522,10 +3028,12 @@ std::string CheckReportJson(const CheckReport &report, const std::string &worksp return "{\"ok\":" + std::string(report.total_issues == 0 ? "true" : "false") + ",\"workspace\":\"" + JsonEscape(workspace) + "\",\"mode\":\"" + (deep ? "deep" : "quick") + - "\",\"content_model\":\"full-blob-v1\",\"checked\":{" + + "\",\"content_model\":\"chunked-v1\",\"checked\":{" + "\"inodes\":" + std::to_string(report.inodes) + ",\"dentries\":" + std::to_string(report.dentries) + ",\"versions\":" + std::to_string(report.versions) + + ",\"manifests\":" + std::to_string(report.manifests) + + ",\"chunks\":" + std::to_string(report.chunks) + ",\"content_bytes\":" + std::to_string(report.content_bytes) + ",\"commits\":" + std::to_string(report.commits) + ",\"snapshots\":" + std::to_string(report.snapshots) + @@ -2638,6 +3146,25 @@ void CheckFunction(sqlite3_context *context, int, sqlite3_value **values) { rows.Int(4) != 0}; } } + { + sqlite3_int64 actual_files = 0; + sqlite3_int64 actual_bytes = 0; + for (const auto &entry : inodes) { + if (!entry.second.deleted && entry.second.kind != "directory") { + ++actual_files; + actual_bytes += entry.second.size; + } + } + Statement stored(db, + "SELECT live_files,live_bytes FROM _vexfs_workspaces WHERE id=?1"); + stored.BindInt64(1, workspace.id); + if (!stored.Row() || stored.Int64(0) != actual_files || + stored.Int64(1) != actual_bytes) { + report.Add("VEXFS_QUOTA_USAGE_MISMATCH", "workspace:" + workspace_name, + "quota", "stored live usage counters do not match active inodes", + "restore a verified backup or rebuild quota counters offline"); + } + } const auto root = inodes.find(workspace.root_inode); if (root == inodes.end() || root->second.deleted || root->second.kind != "directory") { report.Add("VEXFS_ROOT_INVALID", "inode:" + std::to_string(workspace.root_inode), @@ -2716,28 +3243,51 @@ void CheckFunction(sqlite3_context *context, int, sqlite3_value **values) { } struct VersionCheck { - sqlite3_int64 rowid = 0; + sqlite3_int64 id = 0; + sqlite3_int64 manifest = 0; sqlite3_int64 commit = 0; sqlite3_int64 size = 0; - sqlite3_int64 content_size = 0; sqlite3_int64 source = 0; + sqlite3_int64 manifest_size = -1; + sqlite3_int64 chunk_size = -1; + sqlite3_int64 chunk_count = -1; + sqlite3_int64 actual_chunk_count = -1; + sqlite3_int64 actual_chunk_bytes = -1; + sqlite3_int64 minimum_chunk = -1; + sqlite3_int64 maximum_chunk = -1; std::string checksum; + std::string manifest_checksum; }; std::map, VersionCheck> versions; { Statement rows(db, R"SQL( -SELECT v.rowid,v.inode_id,v.version_no,v.commit_id,v.size,length(v.content), - COALESCE(v.source_version_no,0),v.checksum +SELECT v.id,v.inode_id,v.version_no,v.commit_id,v.size, + COALESCE(v.source_version_no,0),v.checksum,COALESCE(v.manifest_id,0), + COALESCE(manifest.file_size,-1),COALESCE(manifest.chunk_size,-1), + COALESCE(manifest.chunk_count,-1),COALESCE(chunk_stats.actual_count,-1), + COALESCE(chunk_stats.actual_bytes,-1),COALESCE(chunk_stats.minimum_chunk,-1), + COALESCE(chunk_stats.maximum_chunk,-1),COALESCE(manifest.checksum,'') FROM _vexfs_file_versions v JOIN _vexfs_inodes i ON i.id=v.inode_id +LEFT JOIN _vexfs_manifests manifest ON manifest.id=v.manifest_id +LEFT JOIN ( + SELECT entry.manifest_id,count(*) actual_count,COALESCE(sum(chunk.size),0) actual_bytes, + min(entry.chunk_no) minimum_chunk,max(entry.chunk_no) maximum_chunk + FROM _vexfs_manifest_chunks entry + JOIN _vexfs_chunks chunk ON chunk.id=entry.chunk_id + GROUP BY entry.manifest_id +) chunk_stats ON chunk_stats.manifest_id=manifest.id WHERE i.workspace_id=?1 ORDER BY v.inode_id,v.version_no )SQL"); rows.BindInt64(1, workspace.id); while (rows.Row()) { ++report.versions; - VersionCheck version{rows.Int64(0), rows.Int64(3), rows.Int64(4), - rows.Int64(5), rows.Int64(6), rows.Text(7)}; + VersionCheck version{rows.Int64(0), rows.Int64(7), rows.Int64(3), + rows.Int64(4), rows.Int64(5), rows.Int64(8), + rows.Int64(9), rows.Int64(10), rows.Int64(11), + rows.Int64(12), rows.Int64(13), rows.Int64(14), + rows.Text(6), rows.Text(15)}; versions[{rows.Int64(1), rows.Int64(2)}] = std::move(version); } } @@ -2775,28 +3325,40 @@ ORDER BY v.inode_id,v.version_no } if (version.source == 0) { report.content_bytes += std::max(version.size, 0); - if (version.content_size != version.size) { - report.Add("VEXFS_CONTENT_SIZE_MISMATCH", object, "content", - "stored BLOB length does not match version size", + const sqlite3_int64 expected_chunks = version.size < 0 ? -1 : + (version.size + kContentChunkBytes - 1) / kContentChunkBytes; + if (version.manifest <= 0 || version.manifest_size != version.size || + version.chunk_size != kContentChunkBytes || + version.chunk_count != expected_chunks || + (expected_chunks == 0 && version.actual_chunk_count != -1) || + (expected_chunks > 0 && + (version.actual_chunk_count != expected_chunks || + version.actual_chunk_bytes != version.size || + version.minimum_chunk != 0 || + version.maximum_chunk != expected_chunks - 1)) || + version.manifest_checksum != version.checksum) { + report.Add("VEXFS_MANIFEST_MISMATCH", object, "manifest", + "manifest metadata does not match its canonical version", "restore this file version from a verified snapshot or backup"); } else if (deep && IsSha256(version.checksum)) { try { - const std::string actual = HashVersionBlob(db, version.rowid, version.size); + const std::string actual = HashManifest( + db, version.manifest, version.size, version.chunk_count); if (actual != version.checksum) { report.Add("VEXFS_CHECKSUM_MISMATCH", object, "content", - "stored BLOB does not match its SHA-256", + "manifest chunks do not match the file SHA-256", "restore this file version from a verified snapshot or backup"); } } catch (const SqlError &error) { - report.Add("VEXFS_CONTENT_UNREADABLE", object, "content", error.what(), + report.Add("VEXFS_CHUNK_INVALID", object, "chunk", error.what(), "restore this file version from a verified snapshot or backup"); } } } else { const auto source = versions.find({inode, version.source}); - if (version.content_size != 0) { - report.Add("VEXFS_ALIAS_HAS_CONTENT", object, "content", - "version alias unexpectedly stores its own BLOB", + if (version.manifest != 0) { + report.Add("VEXFS_ALIAS_HAS_MANIFEST", object, "manifest", + "version alias unexpectedly stores its own manifest", "restore version history from a verified backup"); } if (source == versions.end() || source->second.source != 0) { @@ -2811,6 +3373,76 @@ ORDER BY v.inode_id,v.version_no } } } + { + Statement rows(db, + "SELECT id FROM _vexfs_manifests WHERE workspace_id=?1"); + rows.BindInt64(1, workspace.id); + while (rows.Row()) ++report.manifests; + } + { + Statement rows(db, R"SQL( +SELECT id FROM _vexfs_chunks WHERE workspace_id=?1 +)SQL"); + rows.BindInt64(1, workspace.id); + while (rows.Row()) ++report.chunks; + } + { + Statement rows(db, R"SQL( +SELECT manifest.id FROM _vexfs_manifests manifest +WHERE manifest.workspace_id=?1 AND NOT EXISTS( + SELECT 1 FROM _vexfs_file_versions version + WHERE version.manifest_id=manifest.id AND version.source_version_no IS NULL) +)SQL"); + rows.BindInt64(1, workspace.id); + while (rows.Row()) { + report.Add("VEXFS_MANIFEST_UNREFERENCED", + "manifest:" + std::to_string(rows.Int64(0)), "manifest", + "manifest is not referenced by a canonical file version", + "restore from backup or remove the unreachable manifest offline"); + } + } + { + Statement rows(db, R"SQL( +SELECT version.id,manifest.id FROM _vexfs_file_versions version +JOIN _vexfs_inodes inode ON inode.id=version.inode_id +JOIN _vexfs_manifests manifest ON manifest.id=version.manifest_id +WHERE inode.workspace_id=?1 AND manifest.workspace_id<>?1 +)SQL"); + rows.BindInt64(1, workspace.id); + while (rows.Row()) { + report.Add("VEXFS_MANIFEST_WORKSPACE_MISMATCH", + "version-row:" + std::to_string(rows.Int64(0)), "manifest", + "file version references another workspace manifest", + "restore this file version from a verified backup"); + } + } + { + Statement rows(db, R"SQL( +SELECT chunk.id FROM _vexfs_chunks chunk +WHERE NOT EXISTS(SELECT 1 FROM _vexfs_manifest_chunks entry + WHERE entry.chunk_id=chunk.id) +)SQL"); + while (rows.Row()) { + report.Add("VEXFS_CHUNK_UNREFERENCED", + "chunk:" + std::to_string(rows.Int64(0)), "chunk", + "chunk object is not referenced by any manifest", + "restore content tables from a verified backup"); + } + } + { + Statement rows(db, R"SQL( +SELECT entry.rowid FROM _vexfs_manifest_chunks entry +LEFT JOIN _vexfs_manifests manifest ON manifest.id=entry.manifest_id +LEFT JOIN _vexfs_chunks chunk ON chunk.id=entry.chunk_id +WHERE manifest.id IS NULL OR chunk.id IS NULL +)SQL"); + while (rows.Row()) { + report.Add("VEXFS_MANIFEST_CHUNK_REFERENCE_INVALID", + "manifest-chunk:" + std::to_string(rows.Int64(0)), "chunk", + "manifest chunk entry references a missing object", + "restore content tables from a verified backup"); + } + } for (const auto &entry : inodes) { if (entry.second.deleted || entry.second.kind == "directory") continue; const auto version = versions.find({entry.first, entry.second.version}); @@ -3531,6 +4163,43 @@ struct SnapshotTreeNode { using SnapshotTree = std::map; +QuotaStats SnapshotQuotaStats(const SnapshotTree &tree) { + QuotaStats result; + std::unordered_set counted; + for (const auto &[path, node] : tree) { + (void)path; + if (node.kind == "directory" || !counted.insert(node.inode).second) continue; + ++result.live_files; + if (node.size > std::numeric_limits::max() - result.live_bytes) { + throw SqlError("snapshot byte usage is too large", SQLITE_TOOBIG); + } + result.live_bytes += node.size; + result.largest_file_bytes = std::max(result.largest_file_bytes, node.size); + } + return result; +} + +void CheckQuotaForSnapshotRestore(sqlite3 *db, const Workspace &workspace, + const SnapshotTree ¤t, + const SnapshotTree &target) { + const QuotaPolicy policy = ReadQuotaPolicy(db, workspace.id); + const QuotaStats before = SnapshotQuotaStats(current); + const QuotaStats after = SnapshotQuotaStats(target); + if (policy.max_files >= 0 && after.live_files > policy.max_files && + after.live_files >= before.live_files) { + throw SqlError("snapshot exceeds workspace file quota", SQLITE_FULL); + } + if (policy.max_bytes >= 0 && after.live_bytes > policy.max_bytes && + after.live_bytes >= before.live_bytes) { + throw SqlError("snapshot exceeds workspace byte quota", SQLITE_FULL); + } + if (policy.max_file_bytes >= 0 && + after.largest_file_bytes > policy.max_file_bytes && + after.largest_file_bytes >= before.largest_file_bytes) { + throw SqlError("snapshot exceeds maximum file size quota", SQLITE_FULL); + } +} + void ResolveSnapshotCanonicalVersions(sqlite3 *db, SnapshotTree &tree) { std::string requested = "["; bool first = true; @@ -4050,6 +4719,506 @@ void SnapshotDropFunction(sqlite3_context *context, int, sqlite3_value **values) }); } +QuotaStats ReadQuotaStats(sqlite3 *db, sqlite3_int64 workspace_id) { + Statement statement(db, R"SQL( +SELECT workspace.live_files,workspace.live_bytes, + COALESCE((SELECT max(size) FROM _vexfs_inodes + WHERE workspace_id=?1 AND deleted_at IS NULL AND kind<>'directory'),0) +FROM _vexfs_workspaces workspace WHERE workspace.id=?1 +)SQL"); + statement.BindInt64(1, workspace_id); + statement.Row(); + return {statement.Int64(0), statement.Int64(1), statement.Int64(2)}; +} + +std::string NullableQuotaJson(sqlite3_int64 value) { + return value < 0 ? "null" : std::to_string(value); +} + +std::string QuotaJson(const std::string &workspace_name, + const QuotaPolicy &policy, const QuotaStats &stats) { + const bool over_bytes = policy.max_bytes >= 0 && stats.live_bytes > policy.max_bytes; + const bool over_files = policy.max_files >= 0 && stats.live_files > policy.max_files; + const bool over_file_bytes = policy.max_file_bytes >= 0 && + stats.largest_file_bytes > policy.max_file_bytes; + return "{\"workspace\":\"" + JsonEscape(workspace_name) + + "\",\"max_bytes\":" + NullableQuotaJson(policy.max_bytes) + + ",\"max_files\":" + NullableQuotaJson(policy.max_files) + + ",\"max_file_bytes\":" + NullableQuotaJson(policy.max_file_bytes) + + ",\"live_bytes\":" + std::to_string(stats.live_bytes) + + ",\"live_files\":" + std::to_string(stats.live_files) + + ",\"largest_file_bytes\":" + std::to_string(stats.largest_file_bytes) + + ",\"over_quota\":" + + ((over_bytes || over_files || over_file_bytes) ? "true" : "false") + "}"; +} + +sqlite3_int64 OptionalQuotaValue(sqlite3_value *value, const char *name) { + if (sqlite3_value_type(value) == SQLITE_NULL) return -1; + return RequiredNonnegativeInteger(value, name); +} + +void QuotaGetFunction(sqlite3_context *context, int, sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + const std::string workspace_name = RequiredText(values[0], "workspace"); + const Workspace workspace = FindWorkspace(db, workspace_name); + const std::string json = QuotaJson( + workspace_name, ReadQuotaPolicy(db, workspace.id), + ReadQuotaStats(db, workspace.id)); + sqlite3_result_text(context, json.data(), static_cast(json.size()), SQLITE_TRANSIENT); + }); +} + +void QuotaSetFunction(sqlite3_context *context, int, sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + const std::string workspace_name = RequiredText(values[0], "workspace"); + const sqlite3_int64 max_bytes = OptionalQuotaValue(values[1], "max_bytes"); + const sqlite3_int64 max_files = OptionalQuotaValue(values[2], "max_files"); + const sqlite3_int64 max_file_bytes = + OptionalQuotaValue(values[3], "max_file_bytes"); + const Workspace workspace = FindWorkspace(db, workspace_name); + + Savepoint savepoint(db, "vexfs_quota_set"); + AcquireWriteLock(db); + Statement update(db, R"SQL( +UPDATE _vexfs_workspaces +SET quota_max_bytes=?1,quota_max_files=?2,quota_max_file_bytes=?3 +WHERE id=?4 +)SQL"); + if (max_bytes < 0) update.BindNull(1); else update.BindInt64(1, max_bytes); + if (max_files < 0) update.BindNull(2); else update.BindInt64(2, max_files); + if (max_file_bytes < 0) update.BindNull(3); + else update.BindInt64(3, max_file_bytes); + update.BindInt64(4, workspace.id); + update.Done(); + savepoint.Release(); + + const QuotaPolicy policy{max_bytes, max_files, max_file_bytes}; + const std::string json = QuotaJson( + workspace_name, policy, ReadQuotaStats(db, workspace.id)); + sqlite3_result_text(context, json.data(), static_cast(json.size()), SQLITE_TRANSIENT); + }); +} + +struct RetentionPolicy { + sqlite3_int64 keep_versions = 0; + sqlite3_int64 keep_days = 0; + bool gc_paused = false; +}; + +struct RetentionStats { + sqlite3_int64 live_files = 0; + sqlite3_int64 live_bytes = 0; + sqlite3_int64 versions = 0; + sqlite3_int64 stored_version_bytes = 0; + sqlite3_int64 live_storage_bytes = 0; + sqlite3_int64 reclaimable_versions = 0; + sqlite3_int64 reclaimable_bytes = 0; +}; + +RetentionPolicy ReadRetentionPolicy(sqlite3 *db, const Workspace &workspace) { + Statement statement(db, + "SELECT retention_keep_versions,retention_keep_days,gc_paused " + "FROM _vexfs_workspaces WHERE id=?1"); + statement.BindInt64(1, workspace.id); + if (!statement.Row()) throw SqlError("workspace not found", SQLITE_NOTFOUND); + return {statement.Int64(0), statement.Int64(1), statement.Int(2) != 0}; +} + +void PrepareRetentionKeepSet(sqlite3 *db, const Workspace &workspace, + const RetentionPolicy &policy) { + Exec(db, R"SQL( +CREATE TEMP TABLE IF NOT EXISTS _vexfs_gc_keep_versions( + inode_id INTEGER NOT NULL, + version_no INTEGER NOT NULL, + reason TEXT NOT NULL, + PRIMARY KEY(inode_id,version_no) +); +DELETE FROM _vexfs_gc_keep_versions; +)SQL"); + + Statement current(db, R"SQL( +INSERT OR IGNORE INTO _vexfs_gc_keep_versions(inode_id,version_no,reason) +SELECT id,current_version,'current' +FROM _vexfs_inodes +WHERE workspace_id=?1 AND deleted_at IS NULL AND kind<>'directory' AND current_version>0 +)SQL"); + current.BindInt64(1, workspace.id); + current.Done(); + + Statement handles(db, R"SQL( +INSERT OR IGNORE INTO _vexfs_gc_keep_versions(inode_id,version_no,reason) +SELECT inode_id,expected_version,'handle' +FROM _vexfs_handles +WHERE workspace_id=?1 AND state IN ('open','retained') AND expected_version>0 +)SQL"); + handles.BindInt64(1, workspace.id); + handles.Done(); + + if (policy.keep_versions > 0) { + Statement recent(db, R"SQL( +INSERT OR IGNORE INTO _vexfs_gc_keep_versions(inode_id,version_no,reason) +SELECT inode_id,version_no,'recent' +FROM ( + SELECT v.inode_id,v.version_no, + ROW_NUMBER() OVER(PARTITION BY v.inode_id ORDER BY v.version_no DESC) AS rank + FROM _vexfs_file_versions v + JOIN _vexfs_inodes i ON i.id=v.inode_id + WHERE i.workspace_id=?1 +) +WHERE rank<=?2 +)SQL"); + recent.BindInt64(1, workspace.id); + recent.BindInt64(2, policy.keep_versions); + recent.Done(); + } + + if (policy.keep_days > 0) { + Statement recent_days(db, R"SQL( +INSERT OR IGNORE INTO _vexfs_gc_keep_versions(inode_id,version_no,reason) +SELECT v.inode_id,v.version_no,'age' +FROM _vexfs_file_versions v +JOIN _vexfs_inodes i ON i.id=v.inode_id +WHERE i.workspace_id=?1 + AND v.created_at>=CAST(unixepoch('subsec')*1000 AS INTEGER)-(?2*86400000) +)SQL"); + recent_days.BindInt64(1, workspace.id); + recent_days.BindInt64(2, policy.keep_days); + recent_days.Done(); + } + + // 快照恢复按每个 inode 在目标 commit 之前的最后一条完整状态重建。 + // 先排名再过滤 deleted，避免把快照中已经删除的 inode 的更早版本误保留。 + Statement snapshots(db, R"SQL( +INSERT OR IGNORE INTO _vexfs_gc_keep_versions(inode_id,version_no,reason) +WITH ranked AS ( + SELECT snapshot.id AS snapshot_id,state.inode_id,state.current_version,state.deleted_at, + ROW_NUMBER() OVER( + PARTITION BY snapshot.id,state.inode_id ORDER BY state.commit_id DESC) AS rank + FROM _vexfs_snapshots snapshot + JOIN _vexfs_inode_states state + ON state.workspace_id=snapshot.workspace_id AND state.commit_id<=snapshot.commit_id + WHERE snapshot.workspace_id=?1 AND state.kind<>'directory' AND state.current_version>0 +) +SELECT inode_id,current_version,'snapshot' +FROM ranked WHERE rank=1 AND deleted_at IS NULL +)SQL"); + snapshots.BindInt64(1, workspace.id); + snapshots.Done(); + + // restore 生成的版本是不可变内容的别名。任何被保留的别名都必须继续保留 + // 它最终指向的 canonical manifest；循环也兼容未来出现多级别名的情况。 + while (true) { + Statement sources(db, R"SQL( +INSERT OR IGNORE INTO _vexfs_gc_keep_versions(inode_id,version_no,reason) +SELECT version.inode_id,version.source_version_no,'source' +FROM _vexfs_file_versions version +JOIN _vexfs_gc_keep_versions kept + ON kept.inode_id=version.inode_id AND kept.version_no=version.version_no +WHERE version.source_version_no IS NOT NULL +)SQL"); + sources.Done(); + if (sqlite3_changes64(db) == 0) break; + } +} + +RetentionStats ReadRetentionStats(sqlite3 *db, const Workspace &workspace) { + RetentionStats result; + { + Statement current(db, + "SELECT live_files,live_bytes FROM _vexfs_workspaces WHERE id=?1"); + current.BindInt64(1, workspace.id); + current.Row(); + result.live_files = current.Int64(0); + result.live_bytes = current.Int64(1); + } + { + Statement versions(db, R"SQL( +SELECT (SELECT count(*) FROM _vexfs_file_versions v + JOIN _vexfs_inodes i ON i.id=v.inode_id WHERE i.workspace_id=?1), + COALESCE((SELECT sum(size) FROM _vexfs_chunks WHERE workspace_id=?1),0) +)SQL"); + versions.BindInt64(1, workspace.id); + versions.Row(); + result.versions = versions.Int64(0); + result.stored_version_bytes = versions.Int64(1); + } + { + Statement live_storage(db, R"SQL( +WITH live_manifests AS ( + SELECT DISTINCT source.manifest_id + FROM _vexfs_inodes inode + JOIN _vexfs_file_versions current + ON current.inode_id=inode.id AND current.version_no=inode.current_version + JOIN _vexfs_file_versions source + ON source.inode_id=current.inode_id + AND source.version_no=COALESCE(current.source_version_no,current.version_no) + WHERE inode.workspace_id=?1 AND inode.deleted_at IS NULL AND inode.kind<>'directory' +), live_chunks AS ( + SELECT DISTINCT entry.chunk_id FROM live_manifests live + JOIN _vexfs_manifest_chunks entry ON entry.manifest_id=live.manifest_id +) +SELECT COALESCE(sum(chunk.size),0) +FROM live_chunks live JOIN _vexfs_chunks chunk ON chunk.id=live.chunk_id +)SQL"); + live_storage.BindInt64(1, workspace.id); + live_storage.Row(); + result.live_storage_bytes = live_storage.Int64(0); + } + { + Statement reclaimable(db, R"SQL( +WITH reclaimable_versions AS MATERIALIZED ( + SELECT v.id,v.manifest_id + FROM _vexfs_file_versions v + JOIN _vexfs_inodes i ON i.id=v.inode_id + LEFT JOIN _vexfs_gc_keep_versions kept + ON kept.inode_id=v.inode_id AND kept.version_no=v.version_no + WHERE i.workspace_id=?1 AND kept.inode_id IS NULL +), reclaimable_manifests AS MATERIALIZED ( + SELECT manifest_id FROM reclaimable_versions WHERE manifest_id IS NOT NULL +), reclaimable_chunks AS ( + SELECT DISTINCT entry.chunk_id FROM _vexfs_manifest_chunks entry + JOIN reclaimable_manifests manifest ON manifest.manifest_id=entry.manifest_id + WHERE NOT EXISTS( + SELECT 1 FROM _vexfs_manifest_chunks retained + WHERE retained.chunk_id=entry.chunk_id + AND retained.manifest_id NOT IN (SELECT manifest_id FROM reclaimable_manifests)) +) +SELECT (SELECT count(*) FROM reclaimable_versions), + COALESCE((SELECT sum(chunk.size) FROM reclaimable_chunks item + JOIN _vexfs_chunks chunk ON chunk.id=item.chunk_id),0) +)SQL"); + reclaimable.BindInt64(1, workspace.id); + reclaimable.Row(); + result.reclaimable_versions = reclaimable.Int64(0); + result.reclaimable_bytes = reclaimable.Int64(1); + } + return result; +} + +std::string RetentionJson(const std::string &workspace_name, + const RetentionPolicy &policy, + const RetentionStats &stats) { + const sqlite3_int64 retained_history = std::max( + 0, stats.stored_version_bytes - stats.live_storage_bytes); + return "{\"workspace\":\"" + JsonEscape(workspace_name) + + "\",\"keep_versions\":" + std::to_string(policy.keep_versions) + + ",\"keep_days\":" + std::to_string(policy.keep_days) + + ",\"gc_paused\":" + (policy.gc_paused ? "true" : "false") + + ",\"live_files\":" + std::to_string(stats.live_files) + + ",\"live_bytes\":" + std::to_string(stats.live_bytes) + + ",\"stored_versions\":" + std::to_string(stats.versions) + + ",\"stored_version_bytes\":" + std::to_string(stats.stored_version_bytes) + + ",\"retained_history_bytes\":" + std::to_string(retained_history) + + ",\"reclaimable_versions\":" + std::to_string(stats.reclaimable_versions) + + ",\"reclaimable_bytes\":" + std::to_string(stats.reclaimable_bytes) + "}"; +} + +void RetentionGetFunction(sqlite3_context *context, int, sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + const std::string workspace_name = RequiredText(values[0], "workspace"); + const Workspace workspace = FindWorkspace(db, workspace_name); + const RetentionPolicy policy = ReadRetentionPolicy(db, workspace); + PrepareRetentionKeepSet(db, workspace, policy); + const std::string json = RetentionJson( + workspace_name, policy, ReadRetentionStats(db, workspace)); + sqlite3_result_text(context, json.data(), static_cast(json.size()), SQLITE_TRANSIENT); + }); +} + +void RetentionSetFunction(sqlite3_context *context, int, sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + const std::string workspace_name = RequiredText(values[0], "workspace"); + const sqlite3_int64 keep_versions = + RequiredNonnegativeInteger(values[1], "keep_versions"); + const sqlite3_int64 keep_days = RequiredNonnegativeInteger(values[2], "keep_days"); + if (keep_versions > 1000000) { + throw SqlError("keep_versions must be at most 1000000", SQLITE_RANGE); + } + if (keep_days > 36500) { + throw SqlError("keep_days must be at most 36500", SQLITE_RANGE); + } + const Workspace workspace = FindWorkspace(db, workspace_name); + Statement update(db, + "UPDATE _vexfs_workspaces SET retention_keep_versions=?1,retention_keep_days=?2 " + "WHERE id=?3"); + update.BindInt64(1, keep_versions); + update.BindInt64(2, keep_days); + update.BindInt64(3, workspace.id); + update.Done(); + const RetentionPolicy policy = ReadRetentionPolicy(db, workspace); + PrepareRetentionKeepSet(db, workspace, policy); + const std::string json = RetentionJson( + workspace_name, policy, ReadRetentionStats(db, workspace)); + sqlite3_result_text(context, json.data(), static_cast(json.size()), SQLITE_TRANSIENT); + }); +} + +void GcPauseFunction(sqlite3_context *context, int, sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + const std::string workspace_name = RequiredText(values[0], "workspace"); + if (sqlite3_value_type(values[1]) != SQLITE_INTEGER || + (sqlite3_value_int(values[1]) != 0 && sqlite3_value_int(values[1]) != 1)) { + throw SqlError("paused must be 0 or 1", SQLITE_MISMATCH); + } + const Workspace workspace = FindWorkspace(db, workspace_name); + Savepoint savepoint(db, "vexfs_gc_pause"); + AcquireWriteLock(db); + Statement update(db, + "UPDATE _vexfs_workspaces SET gc_paused=?1 WHERE id=?2"); + update.BindInt(1, sqlite3_value_int(values[1])); + update.BindInt64(2, workspace.id); + update.Done(); + savepoint.Release(); + const RetentionPolicy policy = ReadRetentionPolicy(db, workspace); + PrepareRetentionKeepSet(db, workspace, policy); + const std::string json = RetentionJson( + workspace_name, policy, ReadRetentionStats(db, workspace)); + sqlite3_result_text(context, json.data(), static_cast(json.size()), SQLITE_TRANSIENT); + }); +} + +void GcFunction(sqlite3_context *context, int, sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + const std::string workspace_name = RequiredText(values[0], "workspace"); + const sqlite3_int64 batch = RequiredPositiveInteger(values[1], "batch"); + if (batch > 10000) throw SqlError("batch must be at most 10000", SQLITE_RANGE); + const Workspace workspace = FindWorkspace(db, workspace_name); + + Savepoint savepoint(db, "vexfs_gc"); + AcquireWriteLock(db); + { + Statement mounted(db, + "SELECT 1 FROM _vexfs_mount_sessions WHERE workspace_id=?1 " + "AND lease_until>CAST(unixepoch('subsec')*1000 AS INTEGER) LIMIT 1"); + mounted.BindInt64(1, workspace.id); + if (mounted.Row()) { + throw SqlError("workspace has an active mount session; unmount before GC", + SQLITE_BUSY); + } + } + const RetentionPolicy policy = ReadRetentionPolicy(db, workspace); + if (policy.gc_paused) throw SqlError("workspace GC is paused", SQLITE_BUSY); + PrepareRetentionKeepSet(db, workspace, policy); + + Exec(db, R"SQL( +CREATE TEMP TABLE IF NOT EXISTS _vexfs_gc_delete_versions( + id INTEGER PRIMARY KEY, + inode_id INTEGER NOT NULL, + version_no INTEGER NOT NULL, + manifest_id INTEGER NOT NULL, + stored_bytes INTEGER NOT NULL +); +DELETE FROM _vexfs_gc_delete_versions; +)SQL"); + Statement candidates(db, R"SQL( +INSERT INTO _vexfs_gc_delete_versions(id,inode_id,version_no,manifest_id,stored_bytes) +SELECT version.id,version.inode_id,version.version_no, + COALESCE(version.manifest_id,0),COALESCE(manifest.file_size,0) +FROM _vexfs_file_versions version +JOIN _vexfs_inodes inode ON inode.id=version.inode_id +LEFT JOIN _vexfs_manifests manifest ON manifest.id=version.manifest_id +LEFT JOIN _vexfs_gc_keep_versions kept + ON kept.inode_id=version.inode_id AND kept.version_no=version.version_no +WHERE inode.workspace_id=?1 AND kept.inode_id IS NULL + AND (version.source_version_no IS NOT NULL OR NOT EXISTS ( + SELECT 1 FROM _vexfs_file_versions alias + WHERE alias.inode_id=version.inode_id + AND alias.source_version_no=version.version_no + )) +ORDER BY version.source_version_no IS NULL,version.created_at,version.id +LIMIT ?2 +)SQL"); + candidates.BindInt64(1, workspace.id); + candidates.BindInt64(2, batch); + candidates.Done(); + + sqlite3_int64 deleted_versions = 0; + sqlite3_int64 reclaimed_bytes = 0; + { + Statement count(db, + "SELECT count(*) FROM _vexfs_gc_delete_versions"); + count.Row(); + deleted_versions = count.Int64(0); + } + { + Statement bytes(db, R"SQL( +SELECT COALESCE(sum(chunk.size),0) +FROM _vexfs_chunks chunk +WHERE EXISTS( + SELECT 1 FROM _vexfs_manifest_chunks removed + JOIN _vexfs_gc_delete_versions candidate + ON candidate.manifest_id=removed.manifest_id + WHERE removed.chunk_id=chunk.id AND candidate.manifest_id<>0) +AND NOT EXISTS( + SELECT 1 FROM _vexfs_manifest_chunks retained + WHERE retained.chunk_id=chunk.id AND retained.manifest_id NOT IN( + SELECT manifest_id FROM _vexfs_gc_delete_versions WHERE manifest_id<>0)) +)SQL"); + bytes.Row(); + reclaimed_bytes = bytes.Int64(0); + } + if (deleted_versions > 0) { + // 每条 inode state 都是完整状态。若它引用本批删除的版本，它既不是当前 + // 状态，也不是任何快照的边界状态；对应版本已在 keep set 中排除证明。 + Statement states(db, R"SQL( +DELETE FROM _vexfs_inode_states +WHERE workspace_id=?1 AND EXISTS ( + SELECT 1 FROM _vexfs_gc_delete_versions removed + WHERE removed.inode_id=_vexfs_inode_states.inode_id + AND removed.version_no=_vexfs_inode_states.current_version +) +)SQL"); + states.BindInt64(1, workspace.id); + states.Done(); + Exec(db, R"SQL( +DELETE FROM _vexfs_file_versions +WHERE id IN (SELECT id FROM _vexfs_gc_delete_versions); +DELETE FROM _vexfs_manifest_chunks +WHERE manifest_id IN ( + SELECT manifest_id FROM _vexfs_gc_delete_versions WHERE manifest_id<>0 +); +DELETE FROM _vexfs_manifests +WHERE id IN ( + SELECT manifest_id FROM _vexfs_gc_delete_versions WHERE manifest_id<>0 +); +DELETE FROM _vexfs_chunks +WHERE workspace_id=(SELECT workspace_id FROM _vexfs_inodes + WHERE id=(SELECT inode_id FROM _vexfs_gc_delete_versions LIMIT 1)) + AND NOT EXISTS(SELECT 1 FROM _vexfs_manifest_chunks entry + WHERE entry.chunk_id=_vexfs_chunks.id); +)SQL"); + } + + PrepareRetentionKeepSet(db, workspace, policy); + const RetentionStats remaining = ReadRetentionStats(db, workspace); + const std::string json = "{\"workspace\":\"" + JsonEscape(workspace_name) + + "\",\"batch\":" + std::to_string(batch) + + ",\"deleted_versions\":" + std::to_string(deleted_versions) + + ",\"reclaimed_bytes\":" + std::to_string(reclaimed_bytes) + + ",\"remaining_versions\":" + std::to_string(remaining.versions) + + ",\"remaining_reclaimable_versions\":" + + std::to_string(remaining.reclaimable_versions) + + ",\"remaining_reclaimable_bytes\":" + + std::to_string(remaining.reclaimable_bytes) + + ",\"has_more\":" + + (remaining.reclaimable_versions == 0 ? "false" : "true") + "}"; + savepoint.Release(); + sqlite3_result_text(context, json.data(), static_cast(json.size()), SQLITE_TRANSIENT); + }); +} + void RestoreWorkspaceTree(sqlite3 *db, const Workspace &workspace, sqlite3_int64 commit) { Statement clear_acl(db, "DELETE FROM _vexfs_acl_entries WHERE workspace_id=?1"); @@ -4197,8 +5366,8 @@ WITH ranked AS MATERIALIZED ( FROM _vexfs_inode_states s WHERE s.workspace_id=?1 AND s.commit_id<=?2), latest AS MATERIALIZED (SELECT * FROM ranked WHERE vexfs_rank=1) INSERT INTO _vexfs_file_versions( - inode_id,version_no,commit_id,content,size,checksum,source_version_no) -SELECT inode.id,inode.current_version,?3,X'',target.size,source.checksum, + inode_id,version_no,commit_id,manifest_id,size,checksum,source_version_no) +SELECT inode.id,inode.current_version,?3,NULL,target.size,source.checksum, COALESCE(source.source_version_no,source.version_no) FROM _vexfs_inodes inode JOIN latest target ON target.inode_id=inode.id AND target.deleted_at IS NULL @@ -4251,6 +5420,7 @@ void SnapshotRestoreFunction(sqlite3_context *context, int, sqlite3_value **valu if (SnapshotDiffJson(current, target) == "[]") { throw SqlError("snapshot already matches current workspace", SQLITE_MISMATCH); } + CheckQuotaForSnapshotRestore(db, workspace, current, target); RestoreWorkspaceTree(db, workspace, snapshot.commit); PrepareRestoredVersionAliases(db, workspace, snapshot.commit); const sqlite3_int64 commit = CreateCommit(db, workspace, "restore snapshot " + name); @@ -4518,7 +5688,8 @@ void HandleOpenFunction(sqlite3_context *context, int argument_count, sqlite3_va Handle handle{handle_id, workspace.id, node.id, writable, node.version, dirty_generation, 0, "open", owner_session}; - CreateStaging(db, handle, truncate ? std::vector() : ReadNode(db, node)); + if (truncate) CreateStaging(db, handle, {}); + else CreateStagingFromVersion(db, handle, node); StoreRequestText(db, request_id, "handle_open", fingerprint, handle_id); sqlite3_result_text(context, handle_id.data(), static_cast(handle_id.size()), SQLITE_TRANSIENT); }); @@ -4550,6 +5721,10 @@ void HandleStageWriteFunction(sqlite3_context *context, int, sqlite3_value **val if (patch.size() > static_cast(kMaxStagedBytes - offset)) { throw SqlError("staged file is larger than 128 MiB", SQLITE_TOOBIG); } + const StagingInfo staging = FindStaging(db, handle); + const sqlite3_int64 staged_size = std::max( + staging.logical_size, offset + static_cast(patch.size())); + CheckQuotaForContentSize(db, handle.workspace_id, handle.inode_id, staged_size); const sqlite3_int64 generation = StageWrite(db, handle, offset, patch); StoreRequestInteger(db, request_id, "handle_stage_write", fingerprint, generation); sqlite3_result_int64(context, generation); @@ -4587,29 +5762,28 @@ void HandleAppendFunction(sqlite3_context *context, int, sqlite3_value **values) suffix.size() > static_cast(kMaxStagedBytes - node.size)) { throw SqlError("file is larger than the Phase 0 limit (128 MiB)", SQLITE_TOOBIG); } - std::vector content = ReadNode(db, node); - content.insert(content.end(), suffix.begin(), suffix.end()); - const sqlite3_int64 version = StoreVersion( - db, workspace, node, content, "append handle"); - - StagingInfo staging = EnsureStagingCapacity( - db, handle, FindStaging(db, handle), static_cast(content.size())); - if (!content.empty()) { - WriteBlob(db, staging.rowid, 0, content.data(), - static_cast(content.size())); - } - Statement stage(db, - "UPDATE _vexfs_staging SET logical_size=?1," - "updated_at=CAST(unixepoch('subsec')*1000 AS INTEGER) WHERE handle_id=?2"); - stage.BindInt64(1, static_cast(content.size())); - stage.BindText(2, handle.id); - stage.Done(); - Statement update(db, - "UPDATE _vexfs_handles SET expected_version=?1," - "updated_at=CAST(unixepoch('subsec')*1000 AS INTEGER) WHERE id=?2"); - update.BindInt64(1, version); - update.BindText(2, handle.id); - update.Done(); + CheckQuotaForContentSize( + db, workspace.id, node.id, + node.size + static_cast(suffix.size())); + // O_APPEND is serialized against the current inode version, not the version + // that happened to be current when this handle was opened. Refresh this + // handle's clean staging view so concurrent appenders merge instead of + // reporting a normal write conflict. + { + Statement remove_data(db, + "DELETE FROM _vexfs_staging_data WHERE handle_id=?1"); + remove_data.BindText(1, handle.id); + remove_data.Done(); + Statement remove_meta(db, + "DELETE FROM _vexfs_staging WHERE handle_id=?1"); + remove_meta.BindText(1, handle.id); + remove_meta.Done(); + } + handle.expected_version = node.version; + CreateStagingFromVersion(db, handle, node); + const sqlite3_int64 generation = StageWrite(db, handle, node.size, suffix); + handle.dirty_generation = generation; + const sqlite3_int64 version = PublishHandle(db, handle, generation); StoreRequestInteger(db, request_id, "handle_append", fingerprint, version); savepoint.Release(); sqlite3_result_int64(context, version); @@ -4635,6 +5809,10 @@ void HandleTruncateFunction(sqlite3_context *context, int, sqlite3_value **value const Handle handle = FindHandle(db, handle_id); if (handle.state != "open") throw SqlError("handle is not open", SQLITE_NOTFOUND); if (!handle.writable) throw SqlError("handle is read-only", SQLITE_READONLY); + if (size < 0 || size > kMaxStagedBytes) { + throw SqlError("truncate size is out of range", SQLITE_RANGE); + } + CheckQuotaForContentSize(db, handle.workspace_id, handle.inode_id, size); const sqlite3_int64 generation = TruncateStaging(db, handle, size); StoreRequestInteger(db, request_id, "handle_truncate", fingerprint, generation); sqlite3_result_int64(context, generation); @@ -5007,6 +6185,12 @@ extern "C" int vexfs_sqlite_register(sqlite3 *db) { {"vexfs_snapshot_diff", 3, SnapshotDiffFunction, SQLITE_UTF8}, {"vexfs_snapshot_drop", 2, SnapshotDropFunction, SQLITE_UTF8}, {"vexfs_snapshot_restore", 3, SnapshotRestoreFunction, SQLITE_UTF8}, + {"vexfs_quota_get", 1, QuotaGetFunction, SQLITE_UTF8}, + {"vexfs_quota_set", 4, QuotaSetFunction, SQLITE_UTF8}, + {"vexfs_retention_get", 1, RetentionGetFunction, SQLITE_UTF8}, + {"vexfs_retention_set", 3, RetentionSetFunction, SQLITE_UTF8}, + {"vexfs_gc_pause", 2, GcPauseFunction, SQLITE_UTF8}, + {"vexfs_gc", 2, GcFunction, SQLITE_UTF8}, {"vexfs_move", 3, RenameFunction, SQLITE_UTF8}, {"vexfs_rename", 4, RenameFunction, SQLITE_UTF8}, {"vexfs_remove", 3, RemoveFunction, SQLITE_UTF8}, From 8eb5a6c1b4a1c8f2143fc406ef475217447a3b68 Mon Sep 17 00:00:00 2001 From: Four <8122332+HungryFour@users.noreply.github.com> Date: Thu, 23 Jul 2026 17:11:42 +0800 Subject: [PATCH 04/28] Document preview 20 release verification --- ...2026-07-21_vexfs-final-goal-and-roadmap.md | 35 ++++-- ...23_vexfs-preview20-notarized-fskit-gate.md | 106 ++++++++++++++++++ 2 files changed, 132 insertions(+), 9 deletions(-) create mode 100644 docs/reports/2026-07-23_vexfs-preview20-notarized-fskit-gate.md diff --git a/docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md b/docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md index c023520dba..12f3dfe31b 100644 --- a/docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md +++ b/docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md @@ -98,11 +98,11 @@ vexdb fs --workspace workspace snapshot restore before-refactor | 文件语义 | 文件、目录、四类时间戳、并发 append、进程锁、mode、symlink、xattr、hardlink、owner/group 元数据、便携 ACL 已进入数据库合同和测试 | ACL 还没有完整授权执行和继承；特殊文件不支持 | | 版本恢复 | 单文件版本、workspace commit、snapshot、diff、expected-head restore、SQLite Backup、retention、显式分批 GC、live quota 和 format v2 逻辑导入导出已实现 | 审计、自动维护、history/staging/index/total quota、PG/DuckDB 导入端未完成 | | 长期校验 | `chunked-v1` 使用不可变 manifest、64 KiB 块和逐块/整文件 SHA-256；只读 check 覆盖结构、引用、顺序、大小和流式内容校验 | 自动 repair 和跨文件通用去重不在当前范围 | -| macOS | preview.17 已通过 Apple 公证并 staple；本机安装后 Gatekeeper、FSKit 注册、Bash、Git、Python、Node.js、Go、Rust、并发 append、进程锁、打开文件生命周期和强制卸载全部通过 | 尚缺另一台没有源码和 Xcode 构建记录的干净 Mac 验证 | +| macOS | preview.20 已从干净 commit `4f53577884` 构建，通过 Apple 公证并 staple；本机安装后 Gatekeeper、FSKit 注册和 13 个真实挂载 Gate 全部通过 | 尚缺另一台没有源码和 Xcode 构建记录的干净 Mac 验证 | | Linux | libfuse3 真实挂载已通过 root 和普通用户；时间戳、并发 append、进程锁、打开文件生命周期、强制卸载和 helper 崩溃恢复已验证；x86_64/AArch64 manylinux 安装包已在对应架构真机完成无 root 安装回归 | 缓存/TTL、更多干净发行版、长期运行和真实挂载安装回归仍不足 | | Windows | 只有平台边界和规划 | WinFsp adapter、安装签名、路径/ACL/SID 合同均未实现 | | 远程共享 | 统一合同为 PostgreSQL 预留 | 还不能让另一台电脑挂载同一个远程工作区 | -| 性能规模 | SQLite 直连已完成 10 万文件；preview.16 综合 quick 为 300.590 files/s，1 千文件普通 `rg` 为 418.559 ms、索引 grep 为 7.842 ms；固定 full 负载为 31.859 MiB/s 顺序写、215.884 次/s 随机覆盖 | 小文件创建仍比同轮 APFS 慢 40.433 倍，普通 `rg` 慢 23.396 倍；后续仍需重跑 1 万/10 万和真实 Coding Agent | +| 性能规模 | SQLite 直连已完成 10 万文件；preview.20 真机为 46.863 MiB/s 顺序写、1,041.044 次/s 随机覆盖、175.253 files/s，1 千文件普通扫描 428.041 ms、索引搜索 7.449 ms | 小文件创建仍比同轮 APFS 慢 72.012 倍，普通扫描慢 23.422 倍；后续仍需重跑 1 万/10 万和真实 Coding Agent | “合同中已保存”不等于“所有平台已经完整执行”。例如 owner/group 和便携 ACL 已经可以保存、读取和恢复，但身份认证、权限判断和各系统的原生 ACL 映射仍属于后续阶段。 @@ -112,10 +112,12 @@ vexdb fs --workspace workspace snapshot restore before-refactor 1. **已完成代码修复**：零用例失败、`PASS_WITH_SKIPS`、`--fail-on-skip`、显式 package/mount CLI 绑定、runtime ABI 单一来源、旧 schema 明确拒绝、数据库符号链接拒绝、备份目标 0600、安装哈希/签名/ABI/合同校验； 2. **已完成本地交付验证**：当前源码 ad-hoc 包通过 64 项文档 smoke 和 2 个 package Gate；这只证明本地包结构与功能，不替代 Developer ID、公证和真实 FSKit； -3. **待完成发行证明**：提交当前实现后，从干净 commit 重新 Developer ID 签名、公证，并在启用 FSKit 的 Mac 上用本包 CLI 运行 `--fail-on-skip` 的真实 mount Gate； +3. **已完成本机发行证明**：preview.20 从干净 commit 重新 Developer ID 签名、公证，并在 + 启用 FSKit 的 Mac 上用本包 CLI 逐项运行 `--fail-on-skip` 的 13 个真实 mount Gate；只剩 + 另一台干净 Mac 分发验证； 4. **长期安全与分块内容模型已完成**：checksum/check、retention/GC、live quota、SQLite format v2 逻辑 export/import、chunk + manifest 和 append 合并发布已进入 spec、runtime/CLI smoke 和 eval； - 下一步回到当前代码签名、公证和新的干净安装包回归。 + 下一步先完成另一台干净 Mac 分发验证，再集中解决小文件写入性能。 当前不提前开发 PostgreSQL、DuckDB 或 Windows adapter。 @@ -406,10 +408,20 @@ owner、文件历史、macOS 快照在 Linux 恢复、Linux 快照在 macOS 恢 macOS 完成 Apple 公证、干净机器安装和 extension 启用流程；Linux 完成普通用户依赖检查、x86_64/AArch64 包和卸载。安装说明必须与压缩包或安装包一起交付。 -状态（2026-07-23）：**进行中**。历史 `preview.17` 已公证；当前 `preview.19-test` 已完成签名和 -M1 真机快照恢复复测，但仍需从本次干净 commit 重新打包、公证，之后再做额外干净 Mac 分发验证。 +状态（2026-07-23）：**进行中**。当前 `preview.20` 已从干净 commit `4f53577884` 重新构建、 +签名、公证、安装并通过本机真实 FSKit Gate；只剩额外干净 Mac 分发验证。 -- macOS 最新 `preview.17` 已使用 `Developer ID Application: Jian ming Wu (BB5VK42K87)` 签名； +- preview.20 使用 `Developer ID Application: Jian ming Wu (BB5VK42K87)` 签名，Apple 公证 + submission ID 为 `e2883581-7e4a-488a-9087-8c64d4ccded3`；staple、Gatekeeper、解压复验、 + 67 项文档 smoke 和 17 项 package Gate 全部通过； +- 本机 13 个 macOS 必跑 mount 用例全部逐项开启 `--fail-on-skip`：13 passed、0 failed、 + 0 skipped、227 checks；覆盖 Bash、Git、Python/Node/Go/Rust、并发、锁、元数据、重挂载、 + 性能和 1,000 文件； +- 另一台 M1 `192.168.130.217:22` 当前连接超时，尚不能完成额外干净 Mac 分发验证； + +历史包证据继续保留： + +- 历史 `preview.17` 已使用 `Developer ID Application: Jian ming Wu (BB5VK42K87)` 签名； App、FSKit extension、CLI 和 SQLite dylib 的签名链、统一安装回归与文档 smoke 均通过； - 打包会在 Xcode export 前撤销临时 App 注册并删除专用 archive/export/DerivedData；真实 mount eval 会拒绝 module URL 指向 `.xcarchive`、DerivedData 或 dist 副本； @@ -435,8 +447,8 @@ M1 真机快照恢复复测，但仍需从本次干净 commit 重新打包、公 交付物和证据： -- macOS：`dist/vexdb-lite/vexdb-lite-0.1.0-preview.17-macos-arm64.zip` - （SHA-256 `08bd4f3dc3c3f896b51e92422a4a33ae941d5eab90d986f360a7919cf2f7c2d6`）； +- macOS：`dist/vexdb-lite/vexdb-lite-0.1.0-preview.20-macos-arm64.zip` + （SHA-256 `3818155cc0aae66b17b9655b9e2df6f49a84fd1ff75d07fabdaabc03bb01a2ac`）； - Linux x86_64：`dist/release/vexdb-lite-sqlite-files-linux-x86_64.tar.gz` （SHA-256 `7a736d020cb523740b6e5763d27d96dcb7a7124442b851f2f1c4d389b31e2085`）； - Linux AArch64：`dist/release/vexdb-lite-sqlite-files-linux-aarch64.tar.gz` @@ -444,6 +456,8 @@ M1 真机快照恢复复测，但仍需从本次干净 commit 重新打包、公 - 详细报告：`docs/reports/2026-07-21_vexfs-preview9-cross-platform-release.md`。 - preview.16 修复报告：`docs/reports/2026-07-22_vexfs-preview16-p0-fixes.md`。 - preview.17 公证报告：`docs/reports/2026-07-22_vexfs-preview17-apple-notarization.md`。 +- preview.20 公证与真实 FSKit Gate： + `docs/reports/2026-07-23_vexfs-preview20-notarized-fskit-gate.md`。 ### N5. 真实项目和规模基线 @@ -474,6 +488,9 @@ M1 真机快照恢复复测，但仍需从本次干净 commit 重新打包、公 不能写成已通过； - preview.17 公证最终包在真实 FSKit 上完成 14 passed、0 failed、3 skipped、229 checks； 1000 文件创建 333.971 files/s，同轮比 APFS 慢 38.954 倍； +- preview.20 的 13 个 macOS 必跑 Gate 为 13 passed、0 failed、0 skipped、227 checks；8 MiB + 顺序写 46.863 MiB/s、随机写 1,041.044 ops/s，1,000 文件创建 175.253 files/s，同轮比 + APFS 慢 72.012 倍； - 尚需完成 100,000 文件完整复测、真实 Coding Agent 和写入优化。 证据位置： diff --git a/docs/reports/2026-07-23_vexfs-preview20-notarized-fskit-gate.md b/docs/reports/2026-07-23_vexfs-preview20-notarized-fskit-gate.md new file mode 100644 index 0000000000..6573d34866 --- /dev/null +++ b/docs/reports/2026-07-23_vexfs-preview20-notarized-fskit-gate.md @@ -0,0 +1,106 @@ +# VexFS preview.20 公证与真实 FSKit Gate 报告 + +- 日期：2026-07-23 +- 分支：`feature/agent_files` +- 源码 commit：`4f53577884` +- 版本：`0.1.0-preview.20` +- SQLite 合同：`0.9.0` +- runtime ABI：1 +- 平台：macOS 26.3.1 / arm64 + +## 1. 结论 + +preview.20 已从干净 commit 构建，使用 `Developer ID Application: Jian ming Wu +(BB5VK42K87)` 签名，并通过 Apple 公证、staple、Gatekeeper、安装和真实 FSKit 挂载 Gate。 + +本机发行 Gate 为 13 passed、0 failed、0 skipped，共 227 checks。另一台 M1 +`192.168.130.217` 当前 SSH 端口 22 超时，因此额外干净 Mac 分发验证尚未完成。 + +## 2. 交付物 + +- 压缩包：`dist/vexdb-lite/vexdb-lite-0.1.0-preview.20-macos-arm64.zip` +- 独立使用说明：`dist/vexdb-lite/vexdb-lite-0.1.0-preview.20-macos-arm64-使用说明.md` +- SHA-256：`3818155cc0aae66b17b9655b9e2df6f49a84fd1ff75d07fabdaabc03bb01a2ac` +- `source_dirty=false` +- certificate SHA-1：`26C311958B22397631A857D0482CD2F0EA0BF2AA` + +## 3. Apple 签名与公证 + +- App、FSKit extension、CLI 和 `vexdb_lite.dylib` 的 TeamIdentifier 均为 `BB5VK42K87`。 +- Apple 公证状态：`Accepted`。 +- submission ID：`e2883581-7e4a-488a-9087-8c64d4ccded3`。 +- `stapler staple` 与 `stapler validate` 通过。 +- `spctl --assess --type execute` 返回 `source=Notarized Developer ID`。 +- ZIP 解压后再次通过 App deep codesign、CLI/dylib codesign 和统一 CLI smoke。 + +## 4. 包和安装验证 + +- VexDB unified CLI smoke：PASS。 +- 文档 smoke：67 checks，PASS。 +- `package.unified-install`：1 passed、0 failed、0 skipped、17 checks。 +- 安装后 CLI：`vexdb-lite 0.1.0-preview.20 (4f53577884), SQLite 3.45.3`。 +- 安装后合同：`0.9.0`。 +- FSKit extension：enabled。 +- FSKit module URL 指向 + `~/Applications/VexDB Lite.app/Contents/Extensions/VexFSAppEx.appex`，不是 archive、DerivedData + 或 dist 副本。 +- 默认旧测试数据库仍为 schema `0.6.0`，被新 CLI 按合同明确拒绝；测试使用全新 `0.9.0` + 数据库，没有删除或迁移旧数据库。 + +## 5. 真实 FSKit Gate + +13 个 macOS 必跑用例逐项启用 `--fail-on-skip`，结果为 13 passed、0 failed、0 skipped、 +227 checks，总用时 59.278 秒： + +1. `mount.cross-platform-conformance` +2. `mount.timestamps` +3. `mount.concurrent-append` +4. `mount.open-rename-unlink` +5. `mount.read-only-open-lifecycle` +6. `mount.process-locks` +7. `mount.force-unmount` +8. `mount.real-bash` +9. `mount.posix-metadata` +10. `mount.performance` +11. `mount.git-workspace` +12. `mount.real-toolchain-projects` +13. `mount.scale-tree` + +测试覆盖 Bash、Git、Python、Node.js、Go、Rust、并发 append、进程锁、hardlink、symlink、 +xattr、Unicode、快照恢复、强制卸载、重挂载和 1,000 文件工作区。结束后无残留 VexFS 挂载。 + +主要报告： + +- 一致性合同：`vexdb_sqlite/build/eval/vexfs/20260723T090649.098074Z-quick-20260718/report.json` +- 性能：`vexdb_sqlite/build/eval/vexfs/20260723T090717.944326Z-quick-20260718/report.json` +- 工具链：`vexdb_sqlite/build/eval/vexfs/20260723T090721.320982Z-quick-20260718/report.json` +- 规模：`vexdb_sqlite/build/eval/vexfs/20260723T090737.859441Z-quick-20260718/report.json` + +## 6. 本机性能样本 + +- 8 MiB 顺序写：46.863 MiB/s。 +- 100 次随机写：1,041.044 ops/s。 +- 1,000 文件创建：5.706 秒，175.253 files/s。 +- 同轮 APFS：12,620.380 files/s；VexFS 小文件创建慢约 72.012 倍。 +- 普通文件扫描：0.428 秒；同轮 APFS 为 0.018 秒，慢约 23.422 倍。 +- 数据库 trigram 索引搜索：0.007449 秒，候选文件 1 个。 +- 快照：0.007131 秒；恢复：0.271043 秒。 +- 数据库占用：2,342,912 bytes，约 2,342.912 bytes/file。 + +这说明 chunked-v1 没有破坏正确性或大块顺序写，但小文件创建仍是当前最明显的性能问题。 + +## 7. 最终完整性 + +使用已安装 preview.20 创建全新数据库后执行 deep check： + +- `ok=true` +- `content_model=chunked-v1` +- `issue_count=0` +- 无 pending handle、retained handle、staging 或残留 mount。 + +## 8. 尚未完成 + +1. 另一台没有源码/Xcode 构建记录的干净 Mac 安装验证；当前测试机 + `192.168.130.217:22` 连接超时。 +2. 真实 OpenCode 调用仍需单独授权模型外发和额度，不属于自动发行 Gate。 +3. 小文件创建性能仍需继续优化。 From 9c684c51c0315055a9bc00b188624b9d120f8c01 Mon Sep 17 00:00:00 2001 From: Four <8122332+HungryFour@users.noreply.github.com> Date: Thu, 23 Jul 2026 17:30:00 +0800 Subject: [PATCH 05/28] Fix unpublished VexFS history isolation --- .../test/vexfs_mount_contract_smoke.cpp | 17 ++- tests/eval/vexfs/run.py | 45 ++++++ tests/eval/vexfs/run_macos_package_smoke.sh | 143 ++++++++++++++++++ vexdb_sqlite/src/agent_files/vexfs_sqlite.cpp | 81 +++++++++- 4 files changed, 280 insertions(+), 6 deletions(-) create mode 100755 tests/eval/vexfs/run_macos_package_smoke.sh diff --git a/agent_files/mount/common/test/vexfs_mount_contract_smoke.cpp b/agent_files/mount/common/test/vexfs_mount_contract_smoke.cpp index 484499a03e..27f301c5e3 100644 --- a/agent_files/mount/common/test/vexfs_mount_contract_smoke.cpp +++ b/agent_files/mount/common/test/vexfs_mount_contract_smoke.cpp @@ -704,7 +704,9 @@ int main(int argc, char **argv) { vexfs_mount_free(state.data); // The mount-only create-and-open path must produce one history commit and - // one content version, even when macOS metadata arrives before content. + // one content version, even when macOS metadata or an unrelated commit + // arrives before content. In particular, no immutable history row may + // point at the private version-0 staging state. const int64_t commits_before = Scalar(path, "SELECT count(*) FROM _vexfs_commits"); const int64_t versions_before = Scalar(path, "SELECT count(*) FROM _vexfs_file_versions"); const int64_t requests_before = Scalar(path, "SELECT count(*) FROM _vexfs_requests"); @@ -720,6 +722,17 @@ int main(int argc, char **argv) { return Fail("coalesced create has no empty version", error); const int64_t coalesced_inode = JsonInteger(coalesced_stat, "inode"); vexfs_mount_free(coalesced_stat.data); + if (vexfs_mount_mkdir(session, "/agent/unrelated-boundary", &error) != VEXFS_MOUNT_OK) + return Fail("unrelated history boundary", error); + if (Scalar(path, + "SELECT count(*) FROM _vexfs_inode_states WHERE kind='file' " + "AND current_version=0") != 0 || + Scalar(path, + "SELECT count(*) FROM _vexfs_dirty_inodes dirty " + "JOIN _vexfs_inodes inode ON inode.id=dirty.inode_id " + "WHERE inode.id=(SELECT inode_id FROM _vexfs_dentries " + "WHERE name='coalesced.txt') AND inode.current_version=0") != 1) + return Fail("unpublished create history isolation", error); if (vexfs_mount_xattr_set(session, coalesced_inode, "com.apple.provenance", "p", 1, VEXFS_MOUNT_XATTR_ALWAYS_SET, &error) != VEXFS_MOUNT_OK) return Fail("coalesced provenance", error); @@ -733,7 +746,7 @@ int main(int argc, char **argv) { &version, &error) != VEXFS_MOUNT_OK || version != 1) return Fail("coalesced publish close", error); - if (Scalar(path, "SELECT count(*) FROM _vexfs_commits") - commits_before != 1 || + if (Scalar(path, "SELECT count(*) FROM _vexfs_commits") - commits_before != 2 || Scalar(path, "SELECT count(*) FROM _vexfs_file_versions") - versions_before != 1 || Scalar(path, "SELECT count(*) FROM _vexfs_requests") - requests_before != 0) return Fail("coalesced write amplification", error); diff --git a/tests/eval/vexfs/run.py b/tests/eval/vexfs/run.py index 25eab719b2..ef0c4407e2 100644 --- a/tests/eval/vexfs/run.py +++ b/tests/eval/vexfs/run.py @@ -1282,6 +1282,51 @@ def cross_session_snapshot_barrier(ctx: Context) -> dict[str, Any]: "changes_after_publish": len(changes)} +@case("contract.unpublished-create-history", "transaction", + "无关提交不能把尚未发布的 handle_create version 0 写入历史") +def unpublished_create_history(ctx: Context) -> dict[str, Any]: + with Database(ctx) as db: + handle = db.scalar( + "SELECT vexfs_handle_create('default','/index.lock',420,'create-lock')") + inode = db.json("SELECT vexfs_stat('default','/index.lock')")["inode"] + ctx.equal(db.json("SELECT vexfs_stat('default','/index.lock')")["version"], 0, + "新建句柄在首次发布前仍是暂存版本") + db.scalar("SELECT vexfs_xattr_set('default',?,'user.vexfs.test','pending',0)", + (inode,)) + + # Git 会在 index.lock 仍打开时创建 refs 等目录；这个独立提交过去会 + # 错把 version 0 刷进不可变历史，最终令快照无法完整恢复。 + db.scalar("SELECT vexfs_mkdir('default','/refs')") + ctx.equal(db.scalar( + "SELECT count(*) FROM _vexfs_inode_states " + "WHERE inode_id=? AND current_version=0", (inode,)), 0, + "无关提交不记录未发布 inode") + ctx.equal(db.scalar( + "SELECT count(*) FROM _vexfs_dentry_states WHERE inode_id=?", (inode,)), 0, + "无关提交不记录未发布 dentry") + ctx.equal(db.scalar( + "SELECT count(*) FROM _vexfs_dirty_inodes WHERE inode_id=?", (inode,)), 1, + "未发布 inode 保留 dirty 状态") + + generation = db.scalar( + "SELECT vexfs_handle_stage_write(?,0,'git-index','write-lock')", (handle,)) + ctx.equal(db.scalar( + "SELECT vexfs_handle_publish(?,?,'data','publish-lock')", + (handle, generation)), 1, "首次发布生成 version 1") + db.scalar("SELECT vexfs_handle_close(?,0,'close-lock')", (handle,)) + ctx.equal(db.scalar( + "SELECT count(*) FROM _vexfs_inode_states " + "WHERE inode_id=? AND current_version=1", (inode,)), 1, + "首次发布原子写入有效历史") + ctx.equal(db.scalar( + "SELECT count(*) FROM _vexfs_dirty_inodes WHERE inode_id=?", (inode,)), 0, + "首次发布清理 dirty inode") + report = db.json("SELECT vexfs_check('default',1)") + ctx.check(report["ok"], "真实 Git 锁文件顺序通过深度检查") + return {"inode": inode, "generation": generation, + "version": 1, "issue_count": report["issue_count"]} + + @case("integrity.checksum-and-aliases", "integrity", "SHA-256 覆盖普通版本、单文件恢复和 workspace 快照恢复，损坏内容不得返回") def integrity_checksum_and_aliases(ctx: Context) -> dict[str, Any]: diff --git a/tests/eval/vexfs/run_macos_package_smoke.sh b/tests/eval/vexfs/run_macos_package_smoke.sh new file mode 100755 index 0000000000..70e6da0aac --- /dev/null +++ b/tests/eval/vexfs/run_macos_package_smoke.sh @@ -0,0 +1,143 @@ +#!/bin/bash +set -euo pipefail + +# 在没有源码和 Xcode 构建目录的 macOS 上验证已安装发行包。 +# usage: bash run_macos_package_smoke.sh [installed-vexdb] + +VEXDB="${1:-$HOME/.local/bin/vexdb}" +[ -x "$VEXDB" ] || { echo "找不到已安装 vexdb：$VEXDB" >&2; exit 1; } + +ROOT="$(mktemp -d -t vexdb-lite-macos-package-smoke)" +DB="$ROOT/workspace.sqlite3" +RESTORED_DB="$ROOT/restored.sqlite3" +ARCHIVE="$ROOT/workspace.vexfs" +MOUNT_POINT="$ROOT/mnt" +WORKSPACE=clean-install +CHECKS=0 +EXPECTED_VERSION="${VEXDB_LITE_EXPECTED_VERSION:-}" +PAYLOAD="vexdb package smoke" + +fs() { + "$VEXDB" fs --db "$DB" --workspace "$WORKSPACE" "$@" +} + +check_equal() { + local expected="$1" + local actual="$2" + local message="$3" + if [ "$actual" != "$expected" ]; then + echo "$message：期望 '$expected'，实际 '$actual'" >&2 + exit 1 + fi + CHECKS=$((CHECKS + 1)) +} + +check_file() { + [ -f "$1" ] || { echo "缺少文件：$1" >&2; exit 1; } + CHECKS=$((CHECKS + 1)) +} + +cleanup() { + fs unmount --force "$MOUNT_POINT" >/dev/null 2>&1 || true + if [ "${VEXDB_LITE_KEEP_SMOKE:-0}" = 1 ]; then + echo "保留测试目录：$ROOT" >&2 + else + rm -rf "$ROOT" + fi +} +trap cleanup EXIT + +check_equal 0.9.0 "$($VEXDB :memory: 'SELECT vexfs_contract_version();')" "文件合同错误" +VERSION_OUTPUT="$("$VEXDB" --version)" +if [ -n "$EXPECTED_VERSION" ]; then + printf '%s\n' "$VERSION_OUTPUT" | grep -Fq "$EXPECTED_VERSION" +else + printf '%s\n' "$VERSION_OUTPUT" | grep -Eq 'vexdb-lite 0\.1\.0-preview\.[0-9]+' +fi +CHECKS=$((CHECKS + 1)) + +mkdir -p "$MOUNT_POINT" +fs setup >/dev/null +DOCTOR="$(fs --json doctor)" +printf '%s' "$DOCTOR" | /usr/bin/python3 -c \ + 'import json,sys; value=json.load(sys.stdin); assert value["mount_ready"] is True; assert value["extension"] == "enabled"; assert value["database"]["schema_version"] == "0.9.0"' +CHECKS=$((CHECKS + 3)) + +fs mount "$MOUNT_POINT" +mkdir -p "$MOUNT_POINT/project/src" +printf '%s\n' "$PAYLOAD" > "$MOUNT_POINT/project/src/message.txt" +check_equal "$PAYLOAD" "$(cat "$MOUNT_POINT/project/src/message.txt")" "cat 内容错误" +grep -q "$PAYLOAD" "$MOUNT_POINT/project/src/message.txt" +CHECKS=$((CHECKS + 1)) + +cp "$MOUNT_POINT/project/src/message.txt" "$MOUNT_POINT/project/src/copy.txt" +mv "$MOUNT_POINT/project/src/copy.txt" "$MOUNT_POINT/project/src/moved.txt" +check_file "$MOUNT_POINT/project/src/moved.txt" +ln "$MOUNT_POINT/project/src/message.txt" "$MOUNT_POINT/project/src/message-hard.txt" +ln -s message.txt "$MOUNT_POINT/project/src/message-link.txt" +check_equal "$PAYLOAD" "$(cat "$MOUNT_POINT/project/src/message-hard.txt")" "hardlink 内容错误" +check_equal "$PAYLOAD" "$(cat "$MOUNT_POINT/project/src/message-link.txt")" "symlink 内容错误" + +printf '#!/bin/sh\nprintf "agent\\n"\n' > "$MOUNT_POINT/project/run.sh" +chmod 0755 "$MOUNT_POINT/project/run.sh" +check_equal agent "$($MOUNT_POINT/project/run.sh)" "可执行权限错误" + +/usr/bin/python3 -c \ + 'from pathlib import Path; import json,sys; Path(sys.argv[1]).write_text(json.dumps({"ok": True}))' \ + "$MOUNT_POINT/project/result.json" +check_equal True "$(/usr/bin/python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["ok"])' "$MOUNT_POINT/project/result.json")" "Python 文件读写错误" + +/usr/bin/xattr -w user.vexdb.package-smoke yes "$MOUNT_POINT/project/src/message.txt" +check_equal yes "$(/usr/bin/xattr -p user.vexdb.package-smoke "$MOUNT_POINT/project/src/message.txt")" "xattr 错误" + +git -C "$MOUNT_POINT/project" init -q +git -C "$MOUNT_POINT/project" config user.name VexDB-Test +git -C "$MOUNT_POINT/project" config user.email test@vexdb.local +git -C "$MOUNT_POINT/project" add . +git -C "$MOUNT_POINT/project" commit -qm baseline +check_equal '' "$(git -C "$MOUNT_POINT/project" status --porcelain)" "Git 初始工作区不干净" + +fs snapshot create remote-baseline >/dev/null +printf 'changed\n' > "$MOUNT_POINT/project/src/message.txt" +printf 'temporary\n' > "$MOUNT_POINT/project/temporary.txt" +fs snapshot restore remote-baseline >/dev/null +check_equal "$PAYLOAD" "$(cat "$MOUNT_POINT/project/src/message.txt")" "快照没有恢复旧内容" +[ ! -e "$MOUNT_POINT/project/temporary.txt" ] || { echo "快照没有删除新增文件" >&2; exit 1; } +CHECKS=$((CHECKS + 1)) +check_equal '' "$(git -C "$MOUNT_POINT/project" status --porcelain)" "快照恢复后 Git 工作区不干净" + +fs index enable >/dev/null +fs grep "$PAYLOAD" /project | grep -q '/project/src/message.txt' +CHECKS=$((CHECKS + 1)) + +fs unmount "$MOUNT_POINT" +set +e +MOUNT_STATUS="$(fs --json mount status "$MOUNT_POINT")" +MOUNT_STATUS_CODE=$? +set -e +check_equal 1 "$MOUNT_STATUS_CODE" "未挂载状态退出码错误" +check_equal '[]' "$MOUNT_STATUS" "卸载后仍有 mount session" + +CHECK="$(fs --json check)" +printf '%s' "$CHECK" | /usr/bin/python3 -c \ + 'import json,sys; value=json.load(sys.stdin); assert value["ok"] is True; assert value["content_model"] == "chunked-v1"; assert value["issue_count"] == 0' +CHECKS=$((CHECKS + 3)) + +fs export --snapshot remote-baseline --output "$ARCHIVE" >/dev/null +"$VEXDB" fs archive verify "$ARCHIVE" | /usr/bin/python3 -c \ + 'import json,sys; value=json.load(sys.stdin); assert value["ok"] is True; assert value["format_version"] == 2' +CHECKS=$((CHECKS + 2)) +"$VEXDB" fs --db "$RESTORED_DB" --workspace restored import "$ARCHIVE" >/dev/null +check_equal "$PAYLOAD" \ + "$("$VEXDB" fs --db "$RESTORED_DB" --workspace restored cat /project/src/message.txt)" \ + "归档恢复内容错误" +"$VEXDB" fs --db "$RESTORED_DB" --workspace restored --json check | /usr/bin/python3 -c \ + 'import json,sys; value=json.load(sys.stdin); assert value["ok"] is True; assert value["content_model"] == "chunked-v1"' +CHECKS=$((CHECKS + 2)) + +COUNTS="$($VEXDB "$DB" 'SELECT (SELECT count(*) FROM _vexfs_manifests) || "|" || (SELECT count(*) FROM _vexfs_chunks);')" +printf '%s' "$COUNTS" | /usr/bin/python3 -c \ + 'import sys; manifests,chunks=map(int,sys.stdin.read().split("|")); assert manifests > 0; assert chunks > 0' +CHECKS=$((CHECKS + 2)) + +echo "VEXDB MACOS PACKAGE SMOKE: PASS ($CHECKS checks)" diff --git a/vexdb_sqlite/src/agent_files/vexfs_sqlite.cpp b/vexdb_sqlite/src/agent_files/vexfs_sqlite.cpp index ae1697445a..c1cd787e44 100644 --- a/vexdb_sqlite/src/agent_files/vexfs_sqlite.cpp +++ b/vexdb_sqlite/src/agent_files/vexfs_sqlite.cpp @@ -1168,6 +1168,11 @@ SELECT i.workspace_id,i.id,?2,i.kind,i.mode,i.owner_principal,i.uid,i.gid,i.size FROM _vexfs_dirty_inodes dirty JOIN _vexfs_inodes i ON i.workspace_id=dirty.workspace_id AND i.id=dirty.inode_id WHERE dirty.workspace_id=?1 + -- handle_create deliberately keeps a new file at version 0 until its first + -- publish, so several create/write/metadata calls can share one commit. A + -- commit for another inode must not make that private staging state part of + -- immutable history: version 0 has no manifest and cannot be restored. + AND NOT (i.kind='file' AND i.current_version=0) )SQL"); inodes.BindInt64(1, workspace.id); inodes.BindInt64(2, commit); @@ -1183,6 +1188,12 @@ LEFT JOIN _vexfs_dentries d ON d.workspace_id=dirty.workspace_id AND d.parent_inode=dirty.parent_inode AND d.name=dirty.name WHERE dirty.workspace_id=?1 + AND NOT EXISTS ( + SELECT 1 FROM _vexfs_inodes pending + WHERE pending.workspace_id=dirty.workspace_id + AND pending.id=COALESCE(d.inode_id,dirty.inode_id) + AND pending.kind='file' AND pending.current_version=0 + ) )SQL"); dentries.BindInt64(1, workspace.id); dentries.BindInt64(2, commit); @@ -1196,6 +1207,11 @@ SELECT dirty.workspace_id,dirty.inode_id,dirty.name,?2, FROM _vexfs_dirty_xattrs dirty LEFT JOIN _vexfs_xattrs x ON x.inode_id=dirty.inode_id AND x.name=dirty.name WHERE dirty.workspace_id=?1 + AND NOT EXISTS ( + SELECT 1 FROM _vexfs_inodes pending + WHERE pending.workspace_id=dirty.workspace_id AND pending.id=dirty.inode_id + AND pending.kind='file' AND pending.current_version=0 + ) )SQL"); xattrs.BindInt64(1, workspace.id); xattrs.BindInt64(2, commit); @@ -1212,21 +1228,78 @@ LEFT JOIN _vexfs_acl_entries a ON a.workspace_id=dirty.workspace_id AND a.inode_id=dirty.inode_id AND a.principal_id=dirty.principal_id AND a.effect=dirty.effect WHERE dirty.workspace_id=?1 + AND NOT EXISTS ( + SELECT 1 FROM _vexfs_inodes pending + WHERE pending.workspace_id=dirty.workspace_id AND pending.id=dirty.inode_id + AND pending.kind='file' AND pending.current_version=0 + ) )SQL"); acl.BindInt64(1, workspace.id); acl.BindInt64(2, commit); acl.Done(); - Statement clear_inodes(db, "DELETE FROM _vexfs_dirty_inodes WHERE workspace_id=?1"); + // Keep unpublished files dirty across unrelated commits. Once publish + // advances current_version, the next CreateCommit flushes and clears all + // of their inode/dentry/xattr/ACL changes atomically. A version-0 inode + // deleted before publish never entered history, so its dirty rows can be + // discarded. + Statement clear_inodes(db, R"SQL( +DELETE FROM _vexfs_dirty_inodes +WHERE workspace_id=?1 AND ( + NOT EXISTS ( + SELECT 1 FROM _vexfs_inodes pending + WHERE pending.workspace_id=_vexfs_dirty_inodes.workspace_id + AND pending.id=_vexfs_dirty_inodes.inode_id + ) OR EXISTS ( + SELECT 1 FROM _vexfs_inodes publishable + WHERE publishable.workspace_id=_vexfs_dirty_inodes.workspace_id + AND publishable.id=_vexfs_dirty_inodes.inode_id + AND (publishable.kind<>'file' OR publishable.current_version>0 + OR publishable.deleted_at IS NOT NULL) + ) +) +)SQL"); clear_inodes.BindInt64(1, workspace.id); clear_inodes.Done(); - Statement clear_dentries(db, "DELETE FROM _vexfs_dirty_dentries WHERE workspace_id=?1"); + Statement clear_dentries(db, R"SQL( +DELETE FROM _vexfs_dirty_dentries +WHERE workspace_id=?1 AND NOT EXISTS ( + SELECT 1 FROM _vexfs_inodes pending + WHERE pending.workspace_id=_vexfs_dirty_dentries.workspace_id + AND pending.id=COALESCE(( + SELECT live.inode_id FROM _vexfs_dentries live + WHERE live.workspace_id=_vexfs_dirty_dentries.workspace_id + AND live.parent_inode=_vexfs_dirty_dentries.parent_inode + AND live.name=_vexfs_dirty_dentries.name + ),_vexfs_dirty_dentries.inode_id) + AND pending.kind='file' AND pending.current_version=0 + AND pending.deleted_at IS NULL +) +)SQL"); clear_dentries.BindInt64(1, workspace.id); clear_dentries.Done(); - Statement clear_xattrs(db, "DELETE FROM _vexfs_dirty_xattrs WHERE workspace_id=?1"); + Statement clear_xattrs(db, R"SQL( +DELETE FROM _vexfs_dirty_xattrs +WHERE workspace_id=?1 AND NOT EXISTS ( + SELECT 1 FROM _vexfs_inodes pending + WHERE pending.workspace_id=_vexfs_dirty_xattrs.workspace_id + AND pending.id=_vexfs_dirty_xattrs.inode_id + AND pending.kind='file' AND pending.current_version=0 + AND pending.deleted_at IS NULL +) +)SQL"); clear_xattrs.BindInt64(1, workspace.id); clear_xattrs.Done(); - Statement clear_acl(db, "DELETE FROM _vexfs_dirty_acl WHERE workspace_id=?1"); + Statement clear_acl(db, R"SQL( +DELETE FROM _vexfs_dirty_acl +WHERE workspace_id=?1 AND NOT EXISTS ( + SELECT 1 FROM _vexfs_inodes pending + WHERE pending.workspace_id=_vexfs_dirty_acl.workspace_id + AND pending.id=_vexfs_dirty_acl.inode_id + AND pending.kind='file' AND pending.current_version=0 + AND pending.deleted_at IS NULL +) +)SQL"); clear_acl.BindInt64(1, workspace.id); clear_acl.Done(); } From 8a8b0c139a055a8ba88475f32288f3ec4d1d38e4 Mon Sep 17 00:00:00 2001 From: Four <8122332+HungryFour@users.noreply.github.com> Date: Thu, 23 Jul 2026 17:40:44 +0800 Subject: [PATCH 06/28] Filter deleted metadata from VexFS archives --- agent_files/cli/vexfs_archive.cpp | 8 ++++++-- tests/eval/vexfs/run.py | 23 +++++++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/agent_files/cli/vexfs_archive.cpp b/agent_files/cli/vexfs_archive.cpp index 4a6455f2cc..ea6ef381e0 100644 --- a/agent_files/cli/vexfs_archive.cpp +++ b/agent_files/cli/vexfs_archive.cpp @@ -701,7 +701,9 @@ SELECT printf('%020lld:%s',inode_id,hex(name)),inode_id,name,value, (SELECT created_at FROM package.manifest), vexfs_archive_sha256(CAST(json_array(inode_id,name,hex(value), (SELECT created_at FROM package.manifest)) AS BLOB)) -FROM ranked WHERE rank=1 AND deleted=0; +FROM ranked +JOIN package.inodes live ON live.source_id=ranked.inode_id AND live.deleted_at IS NULL +WHERE rank=1 AND deleted=0; INSERT INTO package.acl_entries WITH ranked AS ( @@ -716,7 +718,9 @@ SELECT printf('%020lld:%s:%s',inode_id,hex(principal_id),effect),inode_id,princi vexfs_archive_sha256(CAST(json_array(inode_id,principal_id,effect,permissions, inherit_flags,(SELECT created_at FROM package.manifest), (SELECT created_at FROM package.manifest)) AS BLOB)) -FROM ranked WHERE rank=1 AND deleted=0; +FROM ranked +JOIN package.inodes live ON live.source_id=ranked.inode_id AND live.deleted_at IS NULL +WHERE rank=1 AND deleted=0; INSERT INTO package.principals SELECT hex(principal),principal, diff --git a/tests/eval/vexfs/run.py b/tests/eval/vexfs/run.py index ef0c4407e2..dadc7134bc 100644 --- a/tests/eval/vexfs/run.py +++ b/tests/eval/vexfs/run.py @@ -1574,6 +1574,17 @@ def logical_export_import(ctx: Context) -> dict[str, Any]: source.scalar("SELECT vexfs_acl_set('default',?," "'[{\"principal\":\"alice\",\"effect\":\"allow\"," "\"permissions\":\"read\",\"inherit\":1}]')", (inode,)) + source.scalar("SELECT vexfs_write('default','/project/deleted.lock','temp')") + deleted_inode = source.json( + "SELECT vexfs_stat('default','/project/deleted.lock')")["inode"] + source.scalar( + "SELECT vexfs_xattr_set('default',?,'user.deleted','temp',0)", + (deleted_inode,)) + source.scalar( + "SELECT vexfs_acl_set('default',?," + "'[{\"principal\":\"local\",\"effect\":\"allow\"," + "\"permissions\":\"read,write\",\"inherit\":0}]')", (deleted_inode,)) + source.scalar("SELECT vexfs_remove('default','/project/deleted.lock',0)") snapshot_commit = source.scalar( "SELECT vexfs_snapshot_create('default','portable')") source.scalar("SELECT vexfs_write('default','/project/version.txt','after')") @@ -1604,6 +1615,18 @@ def logical_export_import(ctx: Context) -> dict[str, Any]: ctx.equal(exported["source_commit"], snapshot_commit, "导出固定快照 commit") ctx.equal(exported["package_checksum"], verified["package_checksum"], "导出和校验整包 hash 一致") + with sqlite3.connect(f"file:{archive}?mode=ro", uri=True) as package: + stale_metadata = package.execute( + "SELECT (SELECT count(*) FROM xattrs attribute " + "WHERE NOT EXISTS(SELECT 1 FROM inodes inode " + "WHERE inode.source_id=attribute.source_inode " + "AND inode.deleted_at IS NULL)) + " + "(SELECT count(*) FROM acl_entries entry " + "WHERE NOT EXISTS(SELECT 1 FROM inodes inode " + "WHERE inode.source_id=entry.source_inode " + "AND inode.deleted_at IS NULL))").fetchone()[0] + ctx.equal(stale_metadata, 0, + "快照导出不把已删除 inode 的 xattr/ACL 当作当前元数据") import_started = time.perf_counter() imported = json.loads(run_process( From 7520f24b0946e62814263a4022e26d72c457b230 Mon Sep 17 00:00:00 2001 From: Four <8122332+HungryFour@users.noreply.github.com> Date: Thu, 23 Jul 2026 17:59:19 +0800 Subject: [PATCH 07/28] Document preview 22 clean Mac verification --- ...2026-07-21_vexfs-final-goal-and-roadmap.md | 31 ++++-- ...23_vexfs-preview20-notarized-fskit-gate.md | 4 + ...26-07-23_vexfs-preview22-clean-mac-gate.md | 105 ++++++++++++++++++ tests/eval/vexfs/run_macos_package_smoke.sh | 7 +- 4 files changed, 134 insertions(+), 13 deletions(-) create mode 100644 docs/reports/2026-07-23_vexfs-preview22-clean-mac-gate.md diff --git a/docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md b/docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md index 12f3dfe31b..81e0946b13 100644 --- a/docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md +++ b/docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md @@ -98,7 +98,7 @@ vexdb fs --workspace workspace snapshot restore before-refactor | 文件语义 | 文件、目录、四类时间戳、并发 append、进程锁、mode、symlink、xattr、hardlink、owner/group 元数据、便携 ACL 已进入数据库合同和测试 | ACL 还没有完整授权执行和继承；特殊文件不支持 | | 版本恢复 | 单文件版本、workspace commit、snapshot、diff、expected-head restore、SQLite Backup、retention、显式分批 GC、live quota 和 format v2 逻辑导入导出已实现 | 审计、自动维护、history/staging/index/total quota、PG/DuckDB 导入端未完成 | | 长期校验 | `chunked-v1` 使用不可变 manifest、64 KiB 块和逐块/整文件 SHA-256；只读 check 覆盖结构、引用、顺序、大小和流式内容校验 | 自动 repair 和跨文件通用去重不在当前范围 | -| macOS | preview.20 已从干净 commit `4f53577884` 构建，通过 Apple 公证并 staple；本机安装后 Gatekeeper、FSKit 注册和 13 个真实挂载 Gate 全部通过 | 尚缺另一台没有源码和 Xcode 构建记录的干净 Mac 验证 | +| macOS | preview.22 已从干净 commit `8a8b0c139a` 构建并通过 Apple 公证；本机 13 个真实挂载 Gate 通过，另一台无源码 M1 macOS 26.5.2 的 30 项安装后 smoke 也通过 | macOS 26.0–26.4 与 x86_64 尚未独立发行验证；构建机 26.3.1(a) 的 Code Signing 子系统当前异常 | | Linux | libfuse3 真实挂载已通过 root 和普通用户；时间戳、并发 append、进程锁、打开文件生命周期、强制卸载和 helper 崩溃恢复已验证；x86_64/AArch64 manylinux 安装包已在对应架构真机完成无 root 安装回归 | 缓存/TTL、更多干净发行版、长期运行和真实挂载安装回归仍不足 | | Windows | 只有平台边界和规划 | WinFsp adapter、安装签名、路径/ACL/SID 合同均未实现 | | 远程共享 | 统一合同为 PostgreSQL 预留 | 还不能让另一台电脑挂载同一个远程工作区 | @@ -112,12 +112,12 @@ vexdb fs --workspace workspace snapshot restore before-refactor 1. **已完成代码修复**：零用例失败、`PASS_WITH_SKIPS`、`--fail-on-skip`、显式 package/mount CLI 绑定、runtime ABI 单一来源、旧 schema 明确拒绝、数据库符号链接拒绝、备份目标 0600、安装哈希/签名/ABI/合同校验； 2. **已完成本地交付验证**：当前源码 ad-hoc 包通过 64 项文档 smoke 和 2 个 package Gate；这只证明本地包结构与功能，不替代 Developer ID、公证和真实 FSKit； -3. **已完成本机发行证明**：preview.20 从干净 commit 重新 Developer ID 签名、公证，并在 - 启用 FSKit 的 Mac 上用本包 CLI 逐项运行 `--fail-on-skip` 的 13 个真实 mount Gate；只剩 - 另一台干净 Mac 分发验证； +3. **已完成 macOS arm64 发行证明**：preview.22 从干净 commit 重新 Developer ID 签名、公证， + 本机 13 个真实 mount Gate 通过；另一台没有源码和完整 Xcode 的 M1 也完成安装、真实挂载、 + Git、快照、deep check 和 archive v2 的 30 项 smoke； 4. **长期安全与分块内容模型已完成**：checksum/check、retention/GC、live quota、SQLite format v2 逻辑 export/import、chunk + manifest 和 append 合并发布已进入 spec、runtime/CLI smoke 和 eval； - 下一步先完成另一台干净 Mac 分发验证，再集中解决小文件写入性能。 + 下一步集中解决小文件写入性能，并补 macOS x86_64 交付验证。 当前不提前开发 PostgreSQL、DuckDB 或 Windows adapter。 @@ -408,16 +408,19 @@ owner、文件历史、macOS 快照在 Linux 恢复、Linux 快照在 macOS 恢 macOS 完成 Apple 公证、干净机器安装和 extension 启用流程；Linux 完成普通用户依赖检查、x86_64/AArch64 包和卸载。安装说明必须与压缩包或安装包一起交付。 -状态（2026-07-23）：**进行中**。当前 `preview.20` 已从干净 commit `4f53577884` 重新构建、 -签名、公证、安装并通过本机真实 FSKit Gate；只剩额外干净 Mac 分发验证。 +状态（2026-07-23）：**进行中**。macOS arm64 已由 `preview.22` 从干净 commit +`8a8b0c139a` 完成交付闭环；N4 剩余主要缺口是 macOS x86_64 和更多 Linux 发行版验证。 -- preview.20 使用 `Developer ID Application: Jian ming Wu (BB5VK42K87)` 签名，Apple 公证 - submission ID 为 `e2883581-7e4a-488a-9087-8c64d4ccded3`；staple、Gatekeeper、解压复验、 +- preview.22 使用 `Developer ID Application: Jian ming Wu (BB5VK42K87)` 签名，Apple 公证 + submission ID 为 `406b904b-6b01-4ae7-8283-4db6c909c5cc`；staple、Gatekeeper、解压复验、 67 项文档 smoke 和 17 项 package Gate 全部通过； - 本机 13 个 macOS 必跑 mount 用例全部逐项开启 `--fail-on-skip`：13 passed、0 failed、 0 skipped、227 checks；覆盖 Bash、Git、Python/Node/Go/Rust、并发、锁、元数据、重挂载、 性能和 1,000 文件； -- 另一台 M1 `192.168.130.217:22` 当前连接超时，尚不能完成额外干净 Mac 分发验证； +- 另一台没有源码、只有 Command Line Tools 的 M1 已覆盖安装 preview.22；FSKit extension + enabled，30 项无源码 smoke 通过，覆盖 Git、快照恢复、deep check、archive v2 校验与导入； +- 该真机先后发现并验证修复两个真实问题：未发布 `index.lock` 的 version-0 历史污染，以及 + 快照 archive 把已删除 inode 的 ACL 当作当前 ACL；preview.20/21 因此不再作为推荐包； 历史包证据继续保留： @@ -447,8 +450,8 @@ macOS 完成 Apple 公证、干净机器安装和 extension 启用流程；Linux 交付物和证据： -- macOS：`dist/vexdb-lite/vexdb-lite-0.1.0-preview.20-macos-arm64.zip` - （SHA-256 `3818155cc0aae66b17b9655b9e2df6f49a84fd1ff75d07fabdaabc03bb01a2ac`）； +- macOS：`dist/vexdb-lite/vexdb-lite-0.1.0-preview.22-macos-arm64.zip` + （SHA-256 `773b0c05e47908dfe2b8f890be4a16f04f10ff3c1937843e9e2aea1c3aa25d1c`）； - Linux x86_64：`dist/release/vexdb-lite-sqlite-files-linux-x86_64.tar.gz` （SHA-256 `7a736d020cb523740b6e5763d27d96dcb7a7124442b851f2f1c4d389b31e2085`）； - Linux AArch64：`dist/release/vexdb-lite-sqlite-files-linux-aarch64.tar.gz` @@ -458,6 +461,8 @@ macOS 完成 Apple 公证、干净机器安装和 extension 启用流程；Linux - preview.17 公证报告：`docs/reports/2026-07-22_vexfs-preview17-apple-notarization.md`。 - preview.20 公证与真实 FSKit Gate： `docs/reports/2026-07-23_vexfs-preview20-notarized-fskit-gate.md`。 +- preview.22 无源码 M1 真机 Gate： + `docs/reports/2026-07-23_vexfs-preview22-clean-mac-gate.md`。 ### N5. 真实项目和规模基线 @@ -491,6 +496,8 @@ macOS 完成 Apple 公证、干净机器安装和 extension 启用流程；Linux - preview.20 的 13 个 macOS 必跑 Gate 为 13 passed、0 failed、0 skipped、227 checks；8 MiB 顺序写 46.863 MiB/s、随机写 1,041.044 ops/s，1,000 文件创建 175.253 files/s，同轮比 APFS 慢 72.012 倍； +- preview.22 在另一台无源码 M1 上完成 30 项安装后 smoke；Git workspace、快照恢复、索引、 + deep check 和 archive v2 导出/校验/导入均通过； - 尚需完成 100,000 文件完整复测、真实 Coding Agent 和写入优化。 证据位置： diff --git a/docs/reports/2026-07-23_vexfs-preview20-notarized-fskit-gate.md b/docs/reports/2026-07-23_vexfs-preview20-notarized-fskit-gate.md index 6573d34866..4f1ab071dd 100644 --- a/docs/reports/2026-07-23_vexfs-preview20-notarized-fskit-gate.md +++ b/docs/reports/2026-07-23_vexfs-preview20-notarized-fskit-gate.md @@ -1,5 +1,9 @@ # VexFS preview.20 公证与真实 FSKit Gate 报告 +> 后续状态：preview.20 在第二台 M1 的无源码完整测试中发现 Git `index.lock` +> 历史引用 version 0 的真实问题，已由 preview.22 修复并取代。preview.20 不再作为推荐交付包； +> 最新证据见 `docs/reports/2026-07-23_vexfs-preview22-clean-mac-gate.md`。 + - 日期：2026-07-23 - 分支：`feature/agent_files` - 源码 commit：`4f53577884` diff --git a/docs/reports/2026-07-23_vexfs-preview22-clean-mac-gate.md b/docs/reports/2026-07-23_vexfs-preview22-clean-mac-gate.md new file mode 100644 index 0000000000..0f3eb78544 --- /dev/null +++ b/docs/reports/2026-07-23_vexfs-preview22-clean-mac-gate.md @@ -0,0 +1,105 @@ +# VexFS preview.22 无源码 M1 真机 Gate 报告 + +- 日期：2026-07-23 +- 分支：`feature/agent_files` +- 源码 commit：`8a8b0c139a` +- 版本：`0.1.0-preview.22` +- SQLite 合同：`0.9.0` +- runtime ABI：1 +- 测试机：Apple M1，macOS 26.5.2 + +## 1. 结论 + +preview.22 已完成 Developer ID 签名、Apple 公证、staple、Gatekeeper、包内安装检查， +并在另一台没有 VexDB 源码、只有 Command Line Tools 的 M1 上覆盖安装和真实 FSKit 挂载。 + +无源码真机 smoke 最终为 `PASS (30 checks)`。覆盖 Bash、Git、Python、普通文件、目录、 +hardlink、symlink、可执行权限、xattr、快照创建与恢复、索引 grep、卸载、deep check、 +archive v2 导出、独立校验、导入和导入后 deep check。结束后没有残留挂载。 + +preview.20 和 preview.21 只保留为发现问题的中间证据，不再作为推荐交付包。 + +## 2. 最终交付物 + +- ZIP：`dist/vexdb-lite/vexdb-lite-0.1.0-preview.22-macos-arm64.zip` +- 独立说明：`dist/vexdb-lite/vexdb-lite-0.1.0-preview.22-macos-arm64-使用说明.md` +- SHA-256：`773b0c05e47908dfe2b8f890be4a16f04f10ff3c1937843e9e2aea1c3aa25d1c` +- `source_dirty=false` +- certificate SHA-1：`26C311958B22397631A857D0482CD2F0EA0BF2AA` +- Apple submission ID：`406b904b-6b01-4ae7-8283-4db6c909c5cc` +- Apple 状态：`Accepted` +- Gatekeeper：`source=Notarized Developer ID` + +## 3. 真机发现并修复的问题 + +### 3.1 未发布的新文件进入历史 + +Git 创建 `.git/index.lock` 后，另一个目录操作可能先产生 commit。旧实现会把尚未首次发布、 +`current_version=0` 的 inode 和 dentry 写入不可变历史，但 version 0 没有 manifest，最终 deep +check 报 `VEXFS_HISTORY_VERSION_MISSING`。 + +修复：历史刷写跳过未发布的 version-0 文件，并保留其 dirty inode、dentry、xattr 和 ACL； +首次发布推进为 version 1 时再原子写入历史。该修复没有增加空版本或额外 commit。 + +回归用例：`contract.unpublished-create-history`，8 checks；运行时合同还覆盖了 +`handle_create -> unrelated mkdir -> first publish` 的顺序。 + +### 3.2 快照 archive 带入已删除 inode 的当前 ACL + +preview.21 的 deep check 已通过，但导出 `remote-baseline` 快照后，archive verify 发现 13 条 +当前 ACL 指向快照中已经删除的 Git 临时 inode。历史 ACL 合法，问题是导出器把它们错误地 +重建成当前 ACL。 + +修复：archive 的当前 `xattrs` 和 `acl_entries` 只保留所选快照中仍存活的 inode；历史 +`xattr_states` 和 `acl_states` 不删除。`backup.logical-export-import` 新增已删除 inode 的 +xattr/ACL 场景，现为 21 checks。 + +使用 M1 原始失败数据库重新导出同一个 commit 111 后，结果为：49 versions、27,199 content +bytes，archive verify、导入和导入后 deep check 全部通过，`issue_count=0`。 + +## 4. 本地回归 + +- `VEXFS RUNTIME SMOKE: PASS`。 +- 100 文件 mount-contract 基准：467.757 files/s，仍为每文件 1 commit、1 version。 +- 完整 quick eval：58 passed、0 failed、18 skipped、2,499 checks，25.859 秒。 +- 18 个 skip 是当前源码构建未注册 FSKit、Linux 专用、跨系统专用或需要真实模型授权的用例； + 本次无源码 M1 Gate 单独验证了已安装 FSKit。 +- 文档 smoke：67 checks，PASS。 +- `package.unified-install`：17 checks，PASS。 + +## 5. 无源码 M1 证据 + +测试机没有 VexDB 源码目录，也没有完整 Xcode。安装后的关键结果： + +- CLI：`vexdb-lite 0.1.0-preview.22 (8a8b0c139a), SQLite 3.45.3`。 +- 合同：`0.9.0`。 +- FSKit：`mount_ready=true`、`extension=enabled`。 +- module URL：`~/Applications/VexDB Lite.app/Contents/Extensions/VexFSAppEx.appex`。 +- App：Gatekeeper accepted，Notarized Developer ID。 +- 无源码 smoke：30 checks，PASS。 +- deep check：`content_model=chunked-v1`、`issue_count=0`。 +- archive：format v2 verify、导入、导入后 deep check 全部通过。 + +### 5.1 签名复验环境说明 + +最终 ZIP 已在 M1 macOS 26.5.2 上从 Downloads 全新解压，不经过安装目录或系统缓存，再次完成： + +- ZIP SHA-256 校验； +- App deep codesign； +- CLI 和 dylib codesign； +- Gatekeeper `Notarized Developer ID`； +- manifest 的 source commit、clean source、notarization 状态和 submission ID 核对。 + +构建机当前是 macOS 26.3.1(a)，但 Xcode 只有 macOS 26.5 SDK。完成构建后，该机器的 +Code Signing 子系统会对同一 ZIP 返回 `internal error` 或 `invalid signature`，同时 +`xcodebuild` 报系统缓存目录 I/O error；同一 ZIP 在 26.5.2 上持续验证有效。因此本报告把 +26.5.2 真机结果作为有效签名证据，不把 26.3.1(a) 的瞬时缓存结果算作通过。 + +manifest 的 deployment target 仍是 26.0，但目前真实发行验证只证明 26.5.2；26.0–26.4 +兼容性需要在系统状态正常、对应版本的独立 Mac 上补测。 + +## 6. 当前边界 + +preview.22 证明 macOS 26.5.2 arm64 的安装、FSKit、Bash/Git workspace、版本恢复和离线 +archive 已形成完整闭环。它不代表 macOS 26.0–26.4、x86_64 macOS、Linux 安装包或 +PostgreSQL/DuckDB 文件插件已经达到同一交付状态；这些仍按路线图分别推进。 diff --git a/tests/eval/vexfs/run_macos_package_smoke.sh b/tests/eval/vexfs/run_macos_package_smoke.sh index 70e6da0aac..f43af0301e 100755 --- a/tests/eval/vexfs/run_macos_package_smoke.sh +++ b/tests/eval/vexfs/run_macos_package_smoke.sh @@ -16,6 +16,7 @@ WORKSPACE=clean-install CHECKS=0 EXPECTED_VERSION="${VEXDB_LITE_EXPECTED_VERSION:-}" PAYLOAD="vexdb package smoke" +MOUNTED=0 fs() { "$VEXDB" fs --db "$DB" --workspace "$WORKSPACE" "$@" @@ -38,7 +39,9 @@ check_file() { } cleanup() { - fs unmount --force "$MOUNT_POINT" >/dev/null 2>&1 || true + if [ "$MOUNTED" = 1 ]; then + fs unmount --force "$MOUNT_POINT" >/dev/null 2>&1 || true + fi if [ "${VEXDB_LITE_KEEP_SMOKE:-0}" = 1 ]; then echo "保留测试目录：$ROOT" >&2 else @@ -64,6 +67,7 @@ printf '%s' "$DOCTOR" | /usr/bin/python3 -c \ CHECKS=$((CHECKS + 3)) fs mount "$MOUNT_POINT" +MOUNTED=1 mkdir -p "$MOUNT_POINT/project/src" printf '%s\n' "$PAYLOAD" > "$MOUNT_POINT/project/src/message.txt" check_equal "$PAYLOAD" "$(cat "$MOUNT_POINT/project/src/message.txt")" "cat 内容错误" @@ -111,6 +115,7 @@ fs grep "$PAYLOAD" /project | grep -q '/project/src/message.txt' CHECKS=$((CHECKS + 1)) fs unmount "$MOUNT_POINT" +MOUNTED=0 set +e MOUNT_STATUS="$(fs --json mount status "$MOUNT_POINT")" MOUNT_STATUS_CODE=$? From 3e6d72d6ba78c19aff78e1cb15378197949777db Mon Sep 17 00:00:00 2001 From: Four <8122332+HungryFour@users.noreply.github.com> Date: Fri, 24 Jul 2026 14:49:18 +0800 Subject: [PATCH 08/28] Add PostgreSQL VexFS runtime and validation --- .gitignore | 1 + agent_files/cli/test/vexdb_unified_smoke.sh | 8 +- .../test/vexfs_platform_registry_smoke.cpp | 13 +- .../cli/test/vexfs_platform_stub_smoke.cpp | 3 +- agent_files/cli/vexfs_archive.cpp | 554 ++ agent_files/cli/vexfs_archive.h | 4 + agent_files/cli/vexfs_main.cpp | 161 +- agent_files/cli/vexfs_platform.h | 4 +- agent_files/cli/vexfs_platform_linux.cpp | 81 +- agent_files/cli/vexfs_platform_macos.cpp | 498 +- agent_files/cli/vexfs_platform_registry.cpp | 19 +- agent_files/cli/vexfs_platform_stub.cpp | 1 + .../macos/VexFS.xcodeproj/project.pbxproj | 386 +- agent_files/macos/VexFSApp/ContentView.swift | 216 + agent_files/macos/VexFSApp/Info.plist | 12 + .../macos/VexFSApp/VexFSApp.entitlements | 6 +- agent_files/macos/VexFSAppEx/Info.plist | 2 + .../macos/VexFSAppEx/VexFSAppEx.entitlements | 2 + .../macos/VexFSAppEx/VexFSBackend.swift | 42 +- .../macos/VexFSAppEx/VexFSFileSystem.swift | 62 +- .../VexFSAppEx/VexFSVolume+Operations.swift | 17 +- .../macos/VexFSAppEx/VexFSVolume.swift | 53 +- agent_files/macos/bundle_pg_runtime.sh | 123 + agent_files/macos/generate_xcode_project.rb | 23 +- agent_files/macos/package/README.md | 14 +- agent_files/macos/package/install.sh | 340 +- agent_files/macos/package/uninstall.sh | 3 +- ...77\347\224\250\350\257\264\346\230\216.md" | 69 +- agent_files/macos/package_preview.sh | 334 +- .../common/include/vexfs_runtime_admin.h | 3 - .../mount/common/include/vexfs_runtime_io.h | 5 + .../common/include/vexfs_runtime_types.h | 1 + .../src/vexfs_mount_contract_sqlite.cpp | 779 +- .../test/vexfs_mount_contract_smoke.cpp | 29 +- .../common/test/vexfs_pg_runtime_smoke.cpp | 377 + agent_files/mount/linux/vexfs_fuse.cpp | 118 +- ...-07-16_agent-files-universal-filesystem.md | 89 +- ...2026-07-21_vexfs-final-goal-and-roadmap.md | 128 +- docs/specs/2026-07-16_vexfs-product-spec.md | 173 +- tests/eval/vexfs/Dockerfile.linux-fuse | 2 + tests/eval/vexfs/Dockerfile.pg-version-matrix | 25 + tests/eval/vexfs/README.md | 178 +- tests/eval/vexfs/documentation_smoke.sh | 113 +- tests/eval/vexfs/python.sh | 34 + tests/eval/vexfs/run.py | 783 +- .../vexfs/run_cross_platform_portability.sh | 7 +- .../vexfs/run_linux_mount_in_container.sh | 25 +- tests/eval/vexfs/run_pg_adapter_alpha.sh | 144 + .../eval/vexfs/run_pg_adapter_performance.sh | 76 + tests/eval/vexfs/run_pg_archive_roundtrip.sh | 178 + tests/eval/vexfs/run_pg_backup_performance.sh | 250 + tests/eval/vexfs/run_pg_backup_restore.sh | 217 + tests/eval/vexfs/run_pg_cross_gateway.sh | 75 + ...run_pg_cross_gateway_linux_in_container.sh | 26 + tests/eval/vexfs/run_pg_grep_performance.sh | 277 + tests/eval/vexfs/run_pg_linux_mount.sh | 33 + tests/eval/vexfs/run_pg_macos_credentials.sh | 122 + tests/eval/vexfs/run_pg_macos_mount.sh | 71 + .../vexfs/run_pg_physical_backup_restore.sh | 168 + tests/eval/vexfs/run_pg_remote_macos.sh | 112 + .../eval/vexfs/run_pg_remote_macos_faults.sh | 347 + tests/eval/vexfs/run_pg_remote_macos_node.sh | 166 + tests/eval/vexfs/run_pg_restart_recovery.sh | 61 + tests/eval/vexfs/run_pg_runtime.sh | 166 + tests/eval/vexfs/run_pg_two_mac_opencode.sh | 156 + .../vexfs/run_pg_two_mac_opencode_node.sh | 261 + tests/eval/vexfs/run_pg_version_matrix.sh | 173 + tests/spec/_lib/docker/run_pg.sh | 41 +- tests/spec/_lib/render.py | 1 + .../agent_files/vexfs_pg_archive_export.yaml | 101 + .../pg/agent_files/vexfs_pg_chunk_check.yaml | 73 + .../pg/agent_files/vexfs_pg_grep_index.yaml | 99 + .../agent_files/vexfs_pg_handles_gateway.yaml | 189 + .../vexfs_pg_history_snapshot.yaml | 144 + .../pg/agent_files/vexfs_pg_metadata_acl.yaml | 149 + .../pg/agent_files/vexfs_pg_quota_gc.yaml | 130 + .../vexfs_pg_transaction_role.yaml | 99 + vexdb_pg/sql/vexdb_lite--1.0.sql | 8175 +++++++++++++++++ vexdb_sqlite/CMakeLists.txt | 84 + vexdb_sqlite/src/agent_files/vexfs_sqlite.cpp | 121 +- 80 files changed, 17547 insertions(+), 858 deletions(-) create mode 100644 agent_files/macos/VexFSApp/Info.plist create mode 100755 agent_files/macos/bundle_pg_runtime.sh create mode 100644 agent_files/mount/common/test/vexfs_pg_runtime_smoke.cpp create mode 100644 tests/eval/vexfs/Dockerfile.pg-version-matrix create mode 100755 tests/eval/vexfs/python.sh create mode 100755 tests/eval/vexfs/run_pg_adapter_alpha.sh create mode 100755 tests/eval/vexfs/run_pg_adapter_performance.sh create mode 100755 tests/eval/vexfs/run_pg_archive_roundtrip.sh create mode 100755 tests/eval/vexfs/run_pg_backup_performance.sh create mode 100755 tests/eval/vexfs/run_pg_backup_restore.sh create mode 100755 tests/eval/vexfs/run_pg_cross_gateway.sh create mode 100755 tests/eval/vexfs/run_pg_cross_gateway_linux_in_container.sh create mode 100755 tests/eval/vexfs/run_pg_grep_performance.sh create mode 100755 tests/eval/vexfs/run_pg_linux_mount.sh create mode 100755 tests/eval/vexfs/run_pg_macos_credentials.sh create mode 100755 tests/eval/vexfs/run_pg_macos_mount.sh create mode 100755 tests/eval/vexfs/run_pg_physical_backup_restore.sh create mode 100755 tests/eval/vexfs/run_pg_remote_macos.sh create mode 100755 tests/eval/vexfs/run_pg_remote_macos_faults.sh create mode 100755 tests/eval/vexfs/run_pg_remote_macos_node.sh create mode 100755 tests/eval/vexfs/run_pg_restart_recovery.sh create mode 100755 tests/eval/vexfs/run_pg_runtime.sh create mode 100755 tests/eval/vexfs/run_pg_two_mac_opencode.sh create mode 100755 tests/eval/vexfs/run_pg_two_mac_opencode_node.sh create mode 100755 tests/eval/vexfs/run_pg_version_matrix.sh create mode 100644 tests/spec/pg/agent_files/vexfs_pg_archive_export.yaml create mode 100644 tests/spec/pg/agent_files/vexfs_pg_chunk_check.yaml create mode 100644 tests/spec/pg/agent_files/vexfs_pg_grep_index.yaml create mode 100644 tests/spec/pg/agent_files/vexfs_pg_handles_gateway.yaml create mode 100644 tests/spec/pg/agent_files/vexfs_pg_history_snapshot.yaml create mode 100644 tests/spec/pg/agent_files/vexfs_pg_metadata_acl.yaml create mode 100644 tests/spec/pg/agent_files/vexfs_pg_quota_gc.yaml create mode 100644 tests/spec/pg/agent_files/vexfs_pg_transaction_role.yaml diff --git a/.gitignore b/.gitignore index b6db5cefab..335f0a4b5e 100644 --- a/.gitignore +++ b/.gitignore @@ -76,6 +76,7 @@ m4/lt~obsolete.m4 build/ build-release/ build-release-vex/ +/.cache/ duckdb_benchmark_data/ #*.m4 *.o diff --git a/agent_files/cli/test/vexdb_unified_smoke.sh b/agent_files/cli/test/vexdb_unified_smoke.sh index ba7490336c..62b0d693be 100755 --- a/agent_files/cli/test/vexdb_unified_smoke.sh +++ b/agent_files/cli/test/vexdb_unified_smoke.sh @@ -1,7 +1,8 @@ #!/bin/bash set -euo pipefail -VEXDB="${1:?usage: vexdb_unified_smoke.sh /path/to/vexdb}" +VEXDB_INPUT="${1:?usage: vexdb_unified_smoke.sh /path/to/vexdb}" +VEXDB="$(cd "$(dirname "$VEXDB_INPUT")" && pwd)/$(basename "$VEXDB_INPUT")" TMP_DIR="$(mktemp -d -t vexdb-unified-smoke)" DB="$TMP_DIR/agent.sqlite3" COPY="$TMP_DIR/agent-copy.sqlite3" @@ -16,7 +17,10 @@ cleanup() { trap cleanup EXIT "$VEXDB" --version | grep -q 'vexdb-lite' -cp "$VEXDB" "$VEXFS_EXE" +# Keep the executable next to its package-relative runtime libraries. The real +# installation also exposes vexfs as a symlink to vexdb; copying only the Mach-O +# binary would deliberately break @executable_path dependency resolution. +ln -s "$VEXDB" "$VEXFS_EXE" "$VEXFS_EXE" --help | grep -q 'Commands:' RESULT="$("$VEXDB" "$DB" \ diff --git a/agent_files/cli/test/vexfs_platform_registry_smoke.cpp b/agent_files/cli/test/vexfs_platform_registry_smoke.cpp index f61a8db4e4..40b853fb7f 100644 --- a/agent_files/cli/test/vexfs_platform_registry_smoke.cpp +++ b/agent_files/cli/test/vexfs_platform_registry_smoke.cpp @@ -28,13 +28,14 @@ int main() { try { VexFSPlatformRememberMount(registry, - {source.string(), target.string(), "vexfs", database.string(), "agent"}); + {source.string(), target.string(), "vexfs", "sqlite", database.string(), "agent"}); std::string encoded_source = source.string(); const size_t space = encoded_source.find(' '); if (space != std::string::npos) encoded_source.replace(space, 1, "%20"); auto attached = VexFSPlatformAttachMountRegistry( {{"file://" + encoded_source + "/", target.string(), "vexfs"}}, registry); if (attached.size() != 1 || + attached[0].backend != "sqlite" || attached[0].database != std::filesystem::weakly_canonical(database).string() || attached[0].workspace != "agent") { const std::string details = attached.empty() ? "empty" : @@ -55,6 +56,16 @@ int main() { std::filesystem::remove_all(root); return Fail("forgotten mount identity"); } + const std::string dsn = "postgresql://postgres@127.0.0.1:5432/test"; + VexFSPlatformRememberMount(registry, + {source.string(), target.string(), "vexfs", "postgresql", dsn, "remote"}); + attached = VexFSPlatformAttachMountRegistry( + {{source.string(), target.string(), "vexfs"}}, registry); + if (attached[0].backend != "postgresql" || attached[0].database != dsn || + attached[0].workspace != "remote") { + std::filesystem::remove_all(root); + return Fail("PostgreSQL connection identity must not be normalized as a path"); + } } catch (const std::exception &error) { std::filesystem::remove_all(root); return Fail(error.what()); diff --git a/agent_files/cli/test/vexfs_platform_stub_smoke.cpp b/agent_files/cli/test/vexfs_platform_stub_smoke.cpp index 349436cc32..020ef9e5f0 100644 --- a/agent_files/cli/test/vexfs_platform_stub_smoke.cpp +++ b/agent_files/cli/test/vexfs_platform_stub_smoke.cpp @@ -66,7 +66,8 @@ int main() { return Fail("platform state"); } try { - VexFSPlatformMount("workspace.sqlite3", "default", "/tmp/vexfs-mount"); + VexFSPlatformMount("sqlite", "workspace.sqlite3", "default", + "/tmp/vexfs-mount"); return Fail("unsupported mount succeeded"); } catch (const std::runtime_error &error) { if (std::string(error.what()).find(expected_driver) == std::string::npos) diff --git a/agent_files/cli/vexfs_archive.cpp b/agent_files/cli/vexfs_archive.cpp index ea6ef381e0..35fd6d7397 100644 --- a/agent_files/cli/vexfs_archive.cpp +++ b/agent_files/cli/vexfs_archive.cpp @@ -5,6 +5,10 @@ #include "sqlite3.h" +#if defined(VEXFS_HAVE_LIBPQ) +#include +#endif + #include #include #include @@ -71,6 +75,14 @@ class Statement { void Int64(int index, sqlite3_int64 value) { Check("bind integer", sqlite3_bind_int64(statement_, index, value)); } + void Blob(int index, const void *value, size_t size) { + if (size > static_cast(std::numeric_limits::max())) { + throw Error("archive record BLOB is too large"); + } + Check("bind blob", sqlite3_bind_blob( + statement_, index, size == 0 ? "" : value, static_cast(size), + SQLITE_TRANSIENT)); + } void Null(int index) { Check("bind NULL", sqlite3_bind_null(statement_, index)); } bool Row() { const int rc = sqlite3_step(statement_); @@ -92,6 +104,12 @@ class Statement { return value == nullptr ? std::string() : std::string(reinterpret_cast(value), size); } + const void *Blob(int column) const { return sqlite3_column_blob(statement_, column); } + int Bytes(int column) const { return sqlite3_column_bytes(statement_, column); } + void Reset() { + Check("reset", sqlite3_reset(statement_)); + Check("clear bindings", sqlite3_clear_bindings(statement_)); + } private: void Check(const char *action, int rc) { @@ -443,6 +461,348 @@ ORDER BY package_chunk.id } } +#if defined(VEXFS_HAVE_LIBPQ) + +struct PgConnectionCloser { + void operator()(PGconn *connection) const { + if (connection != nullptr) PQfinish(connection); + } +}; +using PgConnection = std::unique_ptr; + +struct PgResultCloser { + void operator()(PGresult *result) const { + if (result != nullptr) PQclear(result); + } +}; +using PgResult = std::unique_ptr; + +PgConnection OpenPostgres(const std::string &dsn) { + PGconn *raw = PQconnectdb(dsn.c_str()); + PgConnection connection(raw); + if (raw == nullptr || PQstatus(raw) != CONNECTION_OK) { + throw Error("cannot connect to PostgreSQL: " + + std::string(raw == nullptr ? "out of memory" : PQerrorMessage(raw))); + } + return connection; +} + +void PgCommand(PGconn *connection, const char *sql) { + PgResult result(PQexec(connection, sql)); + if (!result || PQresultStatus(result.get()) != PGRES_COMMAND_OK) { + throw Error("PostgreSQL command failed: " + std::string(PQerrorMessage(connection))); + } +} + +std::string PgScalar(PGconn *connection, const char *sql, int parameter_count, + const char *const *values, const int *lengths = nullptr, + const int *formats = nullptr) { + PgResult result(PQexecParams( + connection, sql, parameter_count, nullptr, values, lengths, formats, 0)); + if (!result || PQresultStatus(result.get()) != PGRES_TUPLES_OK || + PQntuples(result.get()) != 1 || PQnfields(result.get()) != 1 || + PQgetisnull(result.get(), 0, 0)) { + throw Error("PostgreSQL query failed: " + std::string(PQerrorMessage(connection))); + } + return std::string(PQgetvalue(result.get(), 0, 0), + static_cast(PQgetlength(result.get(), 0, 0))); +} + +struct PgArchiveExportResult { + sqlite3_int64 source_commit = 0; + std::string checksum; +}; + +PgArchiveExportResult ExportPostgresRecords( + const std::string &dsn, const std::string &workspace, + const std::string &snapshot, const std::string &output) { + if (dsn.empty()) throw Error("PostgreSQL export needs a connection DSN"); + if (workspace.empty()) throw Error("PostgreSQL export needs a workspace name"); + if (output.empty()) throw Error("export output path is empty"); + if (std::filesystem::exists(output)) { + throw Error("export output already exists: " + output); + } + + bool owns_output = false; + try { + { + Database created = Open(output, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | + SQLITE_OPEN_EXCLUSIVE); + Exec(created.get(), "PRAGMA journal_mode=DELETE; PRAGMA synchronous=FULL;"); + } + owns_output = true; + std::filesystem::permissions( + output, std::filesystem::perms::owner_read | + std::filesystem::perms::owner_write, + std::filesystem::perm_options::replace); + + Database package = Open(":memory:", SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | + SQLITE_OPEN_URI); + RegisterHash(package.get()); + Attach(package.get(), output, "package"); + Exec(package.get(), "BEGIN IMMEDIATE"); + CreateArchiveSchema(package.get()); + Exec(package.get(), R"SQL( +CREATE TEMP TABLE incoming( + record_type TEXT NOT NULL,record_key TEXT NOT NULL,record_json TEXT NOT NULL, + PRIMARY KEY(record_type,record_key)); +)SQL"); + Statement incoming(package.get(), + "INSERT INTO incoming(record_type,record_key,record_json) VALUES(?1,?2,?3)"); + Statement chunk(package.get(), R"SQL( +INSERT INTO package.chunks( + record_key,source_manifest,chunk_no,content,size,checksum,record_checksum) +VALUES(?1,json_extract(?2,'$.source_manifest'),json_extract(?2,'$.chunk_no'),?3, + json_extract(?2,'$.size'),json_extract(?2,'$.checksum'),'') +)SQL"); + + PgConnection connection = OpenPostgres(dsn); + PgCommand(connection.get(), "BEGIN ISOLATION LEVEL REPEATABLE READ"); + PgCommand(connection.get(), "SET LOCAL client_min_messages=warning"); + const char *values[2] = {workspace.c_str(), snapshot.empty() ? nullptr : snapshot.c_str()}; + if (PQsendQueryParams(connection.get(), + "SELECT record_type,record_key,record_json::text,content " + "FROM public.vexfs_archive_export_records($1,$2)", + 2, nullptr, values, nullptr, nullptr, 0) != 1) { + throw Error("cannot start PostgreSQL archive stream: " + + std::string(PQerrorMessage(connection.get()))); + } + if (PQsetSingleRowMode(connection.get()) != 1) { + throw Error("cannot enable PostgreSQL single-row archive streaming"); + } + bool completed = false; + while (PGresult *raw = PQgetResult(connection.get())) { + PgResult row(raw); + const ExecStatusType status = PQresultStatus(raw); + if (status == PGRES_TUPLES_OK) { + completed = true; + continue; + } + if (status != PGRES_SINGLE_TUPLE || PQntuples(raw) != 1 || PQnfields(raw) != 4) { + throw Error("PostgreSQL archive stream failed: " + + std::string(PQresultErrorMessage(raw))); + } + const std::string type(PQgetvalue(raw, 0, 0), + static_cast(PQgetlength(raw, 0, 0))); + const std::string key(PQgetvalue(raw, 0, 1), + static_cast(PQgetlength(raw, 0, 1))); + const std::string json(PQgetvalue(raw, 0, 2), + static_cast(PQgetlength(raw, 0, 2))); + if (type == "chunks") { + if (PQgetisnull(raw, 0, 3)) throw Error("PostgreSQL chunk content is NULL"); + size_t decoded_size = 0; + unsigned char *decoded = PQunescapeBytea( + reinterpret_cast(PQgetvalue(raw, 0, 3)), + &decoded_size); + if (decoded == nullptr) throw Error("cannot decode PostgreSQL chunk content"); + try { + chunk.Text(1, key); + chunk.Text(2, json); + chunk.Blob(3, decoded, decoded_size); + chunk.Done(); + chunk.Reset(); + } catch (...) { + PQfreemem(decoded); + throw; + } + PQfreemem(decoded); + } else { + incoming.Text(1, type); + incoming.Text(2, key); + incoming.Text(3, json); + incoming.Done(); + incoming.Reset(); + } + } + if (!completed) { + throw Error("PostgreSQL archive stream ended without completion"); + } + PgCommand(connection.get(), "COMMIT"); + + Exec(package.get(), R"SQL( +INSERT INTO package.manifest( + format_version,source_engine,source_workspace,source_commit,source_snapshot, + root_source_inode,history_floor_source_commit,retention_keep_versions, + retention_keep_days,quota_max_bytes,quota_max_files,quota_max_file_bytes, + created_at,package_checksum,complete) +SELECT json_extract(record_json,'$.format_version'), + json_extract(record_json,'$.source_engine'), + json_extract(record_json,'$.source_workspace'), + json_extract(record_json,'$.source_commit'), + json_extract(record_json,'$.source_snapshot'), + json_extract(record_json,'$.root_source_inode'), + json_extract(record_json,'$.history_floor_source_commit'), + json_extract(record_json,'$.retention_keep_versions'), + json_extract(record_json,'$.retention_keep_days'), + json_extract(record_json,'$.quota_max_bytes'), + json_extract(record_json,'$.quota_max_files'), + json_extract(record_json,'$.quota_max_file_bytes'), + json_extract(record_json,'$.created_at'),NULL,0 +FROM incoming WHERE record_type='manifest'; + +INSERT INTO package.commits( + record_key,source_id,parent_source_id,message,created_at,record_checksum) +SELECT record_key,json_extract(record_json,'$.source_id'), + json_extract(record_json,'$.parent_source_id'),json_extract(record_json,'$.message'), + json_extract(record_json,'$.created_at'),'' +FROM incoming WHERE record_type='commits'; + +INSERT INTO package.inodes( + record_key,source_id,kind,mode,owner_principal,uid,gid,size,current_version, + created_at,accessed_at,updated_at,changed_at,deleted_at,record_checksum) +SELECT record_key,json_extract(record_json,'$.source_id'),json_extract(record_json,'$.kind'), + json_extract(record_json,'$.mode'),json_extract(record_json,'$.owner_principal'), + json_extract(record_json,'$.uid'),json_extract(record_json,'$.gid'), + json_extract(record_json,'$.size'),json_extract(record_json,'$.current_version'), + json_extract(record_json,'$.created_at'),json_extract(record_json,'$.accessed_at'), + json_extract(record_json,'$.updated_at'),json_extract(record_json,'$.changed_at'), + json_extract(record_json,'$.deleted_at'),'' +FROM incoming WHERE record_type='inodes'; + +INSERT INTO package.dentries( + record_key,parent_source_inode,name,inode_source_id,record_checksum) +SELECT record_key,json_extract(record_json,'$.parent_source_inode'), + json_extract(record_json,'$.name'),json_extract(record_json,'$.inode_source_id'),'' +FROM incoming WHERE record_type='dentries'; + +INSERT INTO package.file_versions( + record_key,source_inode,version_no,source_commit,source_manifest,size,checksum, + source_version_no,created_at,record_checksum) +SELECT record_key,json_extract(record_json,'$.source_inode'), + json_extract(record_json,'$.version_no'),json_extract(record_json,'$.source_commit'), + json_extract(record_json,'$.source_manifest'),json_extract(record_json,'$.size'), + json_extract(record_json,'$.checksum'),json_extract(record_json,'$.source_version_no'), + json_extract(record_json,'$.created_at'),'' +FROM incoming WHERE record_type='file_versions'; + +INSERT INTO package.manifests( + record_key,source_id,file_size,chunk_size,chunk_count,checksum,created_at,record_checksum) +SELECT record_key,json_extract(record_json,'$.source_id'), + json_extract(record_json,'$.file_size'),json_extract(record_json,'$.chunk_size'), + json_extract(record_json,'$.chunk_count'),json_extract(record_json,'$.checksum'), + json_extract(record_json,'$.created_at'),'' +FROM incoming WHERE record_type='manifests'; + +INSERT INTO package.inode_states( + record_key,source_inode,source_commit,kind,mode,owner_principal,uid,gid,size, + current_version,created_at,accessed_at,updated_at,changed_at,deleted_at,record_checksum) +SELECT record_key,json_extract(record_json,'$.source_inode'), + json_extract(record_json,'$.source_commit'),json_extract(record_json,'$.kind'), + json_extract(record_json,'$.mode'),json_extract(record_json,'$.owner_principal'), + json_extract(record_json,'$.uid'),json_extract(record_json,'$.gid'), + json_extract(record_json,'$.size'),json_extract(record_json,'$.current_version'), + json_extract(record_json,'$.created_at'),json_extract(record_json,'$.accessed_at'), + json_extract(record_json,'$.updated_at'),json_extract(record_json,'$.changed_at'), + json_extract(record_json,'$.deleted_at'),'' +FROM incoming WHERE record_type='inode_states'; + +INSERT INTO package.dentry_states( + record_key,parent_source_inode,name,source_commit,inode_source_id,deleted,record_checksum) +SELECT record_key,json_extract(record_json,'$.parent_source_inode'), + json_extract(record_json,'$.name'),json_extract(record_json,'$.source_commit'), + json_extract(record_json,'$.inode_source_id'),json_extract(record_json,'$.deleted'),'' +FROM incoming WHERE record_type='dentry_states'; + +INSERT INTO package.xattr_states( + record_key,source_inode,name,source_commit,value,deleted,record_checksum) +SELECT record_key,json_extract(record_json,'$.source_inode'),json_extract(record_json,'$.name'), + json_extract(record_json,'$.source_commit'), + unhex(json_extract(record_json,'$.value_hex')), + json_extract(record_json,'$.deleted'),'' +FROM incoming WHERE record_type='xattr_states'; + +INSERT INTO package.acl_states( + record_key,source_inode,principal_id,effect,source_commit,permissions, + inherit_flags,deleted,record_checksum) +SELECT record_key,json_extract(record_json,'$.source_inode'), + json_extract(record_json,'$.principal_id'),json_extract(record_json,'$.effect'), + json_extract(record_json,'$.source_commit'),json_extract(record_json,'$.permissions'), + json_extract(record_json,'$.inherit_flags'),json_extract(record_json,'$.deleted'),'' +FROM incoming WHERE record_type='acl_states'; + +INSERT INTO package.snapshots(record_key,name,source_commit,created_at,record_checksum) +SELECT record_key,json_extract(record_json,'$.name'), + json_extract(record_json,'$.source_commit'),json_extract(record_json,'$.created_at'),'' +FROM incoming WHERE record_type='snapshots'; + +INSERT INTO package.xattrs(record_key,source_inode,name,value,updated_at,record_checksum) +SELECT record_key,json_extract(record_json,'$.source_inode'),json_extract(record_json,'$.name'), + unhex(json_extract(record_json,'$.value_hex')), + json_extract(record_json,'$.updated_at'),'' +FROM incoming WHERE record_type='xattrs'; + +INSERT INTO package.acl_entries( + record_key,source_inode,principal_id,effect,permissions,inherit_flags, + created_at,updated_at,record_checksum) +SELECT record_key,json_extract(record_json,'$.source_inode'), + json_extract(record_json,'$.principal_id'),json_extract(record_json,'$.effect'), + json_extract(record_json,'$.permissions'),json_extract(record_json,'$.inherit_flags'), + json_extract(record_json,'$.created_at'),json_extract(record_json,'$.updated_at'),'' +FROM incoming WHERE record_type='acl_entries'; + +INSERT INTO package.principals(record_key,source_principal,record_checksum) +SELECT record_key,json_extract(record_json,'$.source_principal'),'' +FROM incoming WHERE record_type='principals'; + +UPDATE package.commits SET record_checksum= + vexfs_archive_sha256(CAST(json_array(source_id,parent_source_id,message,created_at) AS BLOB)); +UPDATE package.inodes SET record_checksum= + vexfs_archive_sha256(CAST(json_array(source_id,kind,mode,owner_principal,uid,gid,size, + current_version,created_at,accessed_at,updated_at,changed_at,deleted_at) AS BLOB)); +UPDATE package.dentries SET record_checksum= + vexfs_archive_sha256(CAST(json_array(parent_source_inode,name,inode_source_id) AS BLOB)); +UPDATE package.file_versions SET record_checksum= + vexfs_archive_sha256(CAST(json_array(source_inode,version_no,source_commit,source_manifest, + size,checksum,source_version_no,created_at) AS BLOB)); +UPDATE package.manifests SET record_checksum= + vexfs_archive_sha256(CAST(json_array(source_id,file_size,chunk_size,chunk_count,checksum, + created_at) AS BLOB)); +UPDATE package.chunks SET record_checksum= + vexfs_archive_sha256(CAST(json_array(source_manifest,chunk_no,size,checksum) AS BLOB)); +UPDATE package.inode_states SET record_checksum= + vexfs_archive_sha256(CAST(json_array(source_inode,source_commit,kind,mode,owner_principal, + uid,gid,size,current_version,created_at,accessed_at,updated_at,changed_at,deleted_at) AS BLOB)); +UPDATE package.dentry_states SET record_checksum= + vexfs_archive_sha256(CAST(json_array(parent_source_inode,name,source_commit,inode_source_id, + deleted) AS BLOB)); +UPDATE package.xattr_states SET record_checksum= + vexfs_archive_sha256(CAST(json_array(source_inode,name,source_commit,hex(value),deleted) AS BLOB)); +UPDATE package.acl_states SET record_checksum= + vexfs_archive_sha256(CAST(json_array(source_inode,principal_id,effect,source_commit, + permissions,inherit_flags,deleted) AS BLOB)); +UPDATE package.snapshots SET record_checksum= + vexfs_archive_sha256(CAST(json_array(name,source_commit,created_at) AS BLOB)); +UPDATE package.xattrs SET record_checksum= + vexfs_archive_sha256(CAST(json_array(source_inode,name,hex(value),updated_at) AS BLOB)); +UPDATE package.acl_entries SET record_checksum= + vexfs_archive_sha256(CAST(json_array(source_inode,principal_id,effect,permissions, + inherit_flags,created_at,updated_at) AS BLOB)); +UPDATE package.principals SET record_checksum= + vexfs_archive_sha256(CAST(json_array(source_principal) AS BLOB)); +)SQL"); + Exec(package.get(), "UPDATE package.manifest SET complete=1"); + const std::string checksum = ComputePackageChecksum(package.get()); + { + Statement finish(package.get(), + "UPDATE package.manifest SET package_checksum=?1"); + finish.Text(1, checksum); + finish.Done(); + } + const sqlite3_int64 source_commit = ScalarInt64( + package.get(), "SELECT source_commit FROM package.manifest"); + Exec(package.get(), "COMMIT"); + return {source_commit, checksum}; + } catch (...) { + if (owns_output) { + std::error_code ignored; + std::filesystem::remove(output, ignored); + } + throw; + } +} + +#endif // VEXFS_HAVE_LIBPQ + } // namespace std::string ExportArchive(const std::string &database, const std::string &workspace, @@ -762,6 +1122,26 @@ WHERE principal<>''; } } +std::string ExportPostgresArchive(const std::string &dsn, const std::string &workspace, + const std::string &snapshot, const std::string &output) { +#if defined(VEXFS_HAVE_LIBPQ) + const PgArchiveExportResult result = + ExportPostgresRecords(dsn, workspace, snapshot, output); + const auto bytes = std::filesystem::file_size(output); + return "{\"format_version\":" + std::to_string(kArchiveFormatVersion) + + ",\"workspace\":\"" + JsonEscape(workspace) + + "\",\"source_commit\":" + std::to_string(result.source_commit) + + ",\"package_checksum\":\"" + result.checksum + + "\",\"bytes\":" + std::to_string(bytes) + "}"; +#else + (void)dsn; + (void)workspace; + (void)snapshot; + (void)output; + throw Error("this vexdb build does not include PostgreSQL client support"); +#endif +} + namespace { struct ArchiveInfo { @@ -1052,6 +1432,151 @@ WHERE source_manifest=?1 ORDER BY chunk_no return info; } +#if defined(VEXFS_HAVE_LIBPQ) + +std::string ImportPostgresRecords(sqlite3 *db, const std::string &dsn, + const std::string &workspace) { + Statement manifest(db, R"SQL( +SELECT json_object( + 'format_version',format_version,'source_engine',source_engine, + 'source_workspace',source_workspace,'source_commit',source_commit, + 'source_snapshot',source_snapshot,'root_source_inode',root_source_inode, + 'history_floor_source_commit',history_floor_source_commit, + 'retention_keep_versions',retention_keep_versions, + 'retention_keep_days',retention_keep_days,'quota_max_bytes',quota_max_bytes, + 'quota_max_files',quota_max_files,'quota_max_file_bytes',quota_max_file_bytes, + 'created_at',created_at) +FROM package.manifest +)SQL"); + if (!manifest.Row()) throw Error("archive manifest is missing"); + const std::string manifest_json = manifest.Text(0); + + PgConnection connection = OpenPostgres(dsn); + PgCommand(connection.get(), "BEGIN"); + try { + PgCommand(connection.get(), "SET LOCAL client_min_messages=warning"); + const char *begin_values[2] = {workspace.c_str(), manifest_json.c_str()}; + const std::string job = PgScalar( + connection.get(), + "SELECT public.vexfs_archive_import_begin($1,$2::jsonb)::text", + 2, begin_values); + + struct RecordQuery { + const char *type; + const char *sql; + bool has_content; + }; + static constexpr RecordQuery queries[] = { + {"commits", R"SQL( +SELECT record_key,json_object( + 'source_id',source_id,'parent_source_id',parent_source_id, + 'message',message,'created_at',created_at),NULL +FROM package.commits ORDER BY record_key)SQL", false}, + {"inodes", R"SQL( +SELECT record_key,json_object( + 'source_id',source_id,'kind',kind,'mode',mode,'owner_principal',owner_principal, + 'uid',uid,'gid',gid,'size',size,'current_version',current_version, + 'created_at',created_at,'accessed_at',accessed_at,'updated_at',updated_at, + 'changed_at',changed_at,'deleted_at',deleted_at),NULL +FROM package.inodes ORDER BY record_key)SQL", false}, + {"dentries", R"SQL( +SELECT record_key,json_object( + 'parent_source_inode',parent_source_inode,'name',name, + 'inode_source_id',inode_source_id),NULL +FROM package.dentries ORDER BY record_key)SQL", false}, + {"file_versions", R"SQL( +SELECT record_key,json_object( + 'source_inode',source_inode,'version_no',version_no,'source_commit',source_commit, + 'source_manifest',source_manifest,'size',size,'checksum',checksum, + 'source_version_no',source_version_no,'created_at',created_at),NULL +FROM package.file_versions ORDER BY record_key)SQL", false}, + {"manifests", R"SQL( +SELECT record_key,json_object( + 'source_id',source_id,'file_size',file_size,'chunk_size',chunk_size, + 'chunk_count',chunk_count,'checksum',checksum,'created_at',created_at),NULL +FROM package.manifests ORDER BY record_key)SQL", false}, + {"chunks", R"SQL( +SELECT record_key,json_object( + 'source_manifest',source_manifest,'chunk_no',chunk_no,'size',size, + 'checksum',checksum),content +FROM package.chunks ORDER BY record_key)SQL", true}, + {"inode_states", R"SQL( +SELECT record_key,json_object( + 'source_inode',source_inode,'source_commit',source_commit,'kind',kind,'mode',mode, + 'owner_principal',owner_principal,'uid',uid,'gid',gid,'size',size, + 'current_version',current_version,'created_at',created_at,'accessed_at',accessed_at, + 'updated_at',updated_at,'changed_at',changed_at,'deleted_at',deleted_at),NULL +FROM package.inode_states ORDER BY record_key)SQL", false}, + {"dentry_states", R"SQL( +SELECT record_key,json_object( + 'parent_source_inode',parent_source_inode,'name',name,'source_commit',source_commit, + 'inode_source_id',inode_source_id,'deleted',deleted),NULL +FROM package.dentry_states ORDER BY record_key)SQL", false}, + {"xattr_states", R"SQL( +SELECT record_key,json_object( + 'source_inode',source_inode,'name',name,'source_commit',source_commit, + 'value_hex',lower(hex(value)),'deleted',deleted),NULL +FROM package.xattr_states ORDER BY record_key)SQL", false}, + {"acl_states", R"SQL( +SELECT record_key,json_object( + 'source_inode',source_inode,'principal_id',principal_id,'effect',effect, + 'source_commit',source_commit,'permissions',permissions, + 'inherit_flags',inherit_flags,'deleted',deleted),NULL +FROM package.acl_states ORDER BY record_key)SQL", false}, + {"snapshots", R"SQL( +SELECT record_key,json_object( + 'name',name,'source_commit',source_commit,'created_at',created_at),NULL +FROM package.snapshots ORDER BY record_key)SQL", false}, + {"xattrs", R"SQL( +SELECT record_key,json_object( + 'source_inode',source_inode,'name',name,'value_hex',lower(hex(value)), + 'updated_at',updated_at),NULL +FROM package.xattrs ORDER BY record_key)SQL", false}, + {"acl_entries", R"SQL( +SELECT record_key,json_object( + 'source_inode',source_inode,'principal_id',principal_id,'effect',effect, + 'permissions',permissions,'inherit_flags',inherit_flags, + 'created_at',created_at,'updated_at',updated_at),NULL +FROM package.acl_entries ORDER BY record_key)SQL", false}, + {"principals", R"SQL( +SELECT record_key,json_object('source_principal',source_principal),NULL +FROM package.principals ORDER BY record_key)SQL", false} + }; + + for (const RecordQuery &query : queries) { + Statement records(db, query.sql); + while (records.Row()) { + const std::string key = records.Text(0); + const std::string json = records.Text(1); + const int content_bytes = query.has_content ? records.Bytes(2) : 0; + const void *content = query.has_content ? records.Blob(2) : nullptr; + const char *values[5] = { + job.c_str(), query.type, key.c_str(), json.c_str(), + query.has_content ? static_cast(content) : nullptr}; + const int lengths[5] = {0, 0, 0, 0, content_bytes}; + const int formats[5] = {0, 0, 0, 0, query.has_content ? 1 : 0}; + (void)PgScalar( + connection.get(), + "SELECT public.vexfs_archive_import_record(" + "$1::bigint,$2,$3,$4::jsonb,$5)::text", + 5, values, lengths, formats); + } + } + const char *finish_values[1] = {job.c_str()}; + const std::string result = PgScalar( + connection.get(), + "SELECT public.vexfs_archive_import_finish($1::bigint)::text", + 1, finish_values); + PgCommand(connection.get(), "COMMIT"); + return result; + } catch (...) { + try { PgCommand(connection.get(), "ROLLBACK"); } catch (...) {} + throw; + } +} + +#endif // VEXFS_HAVE_LIBPQ + } // namespace std::string VerifyArchive(const std::string &input) { @@ -1072,6 +1597,35 @@ std::string VerifyArchive(const std::string &input) { info.checksum + "\"}"; } +std::string ImportPostgresArchive(const std::string &dsn, const std::string &workspace, + const std::string &input) { +#if defined(VEXFS_HAVE_LIBPQ) + if (dsn.empty()) throw Error("PostgreSQL import needs a connection DSN"); + if (workspace.empty() || workspace.size() > 128) { + throw Error("workspace name must be 1..128 bytes"); + } + if (!std::filesystem::is_regular_file(input)) { + throw Error("archive file not found: " + input); + } + Database db = Open(":memory:", SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_URI); + RegisterHash(db.get()); + AttachReadOnly(db.get(), input, "package"); + Exec(db.get(), "PRAGMA query_only=ON"); + const ArchiveInfo info = VerifyAttachedArchive(db.get()); + std::string result = ImportPostgresRecords(db.get(), dsn, workspace); + if (!result.empty() && result.back() == '}') { + result.pop_back(); + result += ",\"package_checksum\":\"" + info.checksum + "\"}"; + } + return result; +#else + (void)dsn; + (void)workspace; + (void)input; + throw Error("this vexdb build does not include PostgreSQL client support"); +#endif +} + std::string ImportArchive(const std::string &database, const std::string &workspace, const std::string &input) { if (database.empty() || database == ":memory:") { diff --git a/agent_files/cli/vexfs_archive.h b/agent_files/cli/vexfs_archive.h index 679c4eff2f..abb78a4eeb 100644 --- a/agent_files/cli/vexfs_archive.h +++ b/agent_files/cli/vexfs_archive.h @@ -9,8 +9,12 @@ namespace vexfs_cli { // container. It is not a copy of the source database. std::string ExportArchive(const std::string &database, const std::string &workspace, const std::string &snapshot, const std::string &output); +std::string ExportPostgresArchive(const std::string &dsn, const std::string &workspace, + const std::string &snapshot, const std::string &output); std::string ImportArchive(const std::string &database, const std::string &workspace, const std::string &input); +std::string ImportPostgresArchive(const std::string &dsn, const std::string &workspace, + const std::string &input); std::string VerifyArchive(const std::string &input); } // namespace vexfs_cli diff --git a/agent_files/cli/vexfs_main.cpp b/agent_files/cli/vexfs_main.cpp index e9e7222a15..835e9a0b22 100644 --- a/agent_files/cli/vexfs_main.cpp +++ b/agent_files/cli/vexfs_main.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include #include @@ -22,7 +23,9 @@ namespace { struct Options { + std::string backend = VEXFS_RUNTIME_BACKEND_SQLITE; std::string database; + std::string dsn; std::string workspace = "default"; std::string mount_point; bool json = false; @@ -33,7 +36,8 @@ std::string g_program_name = "vexfs"; void Usage(std::ostream &output) { output << - "Usage: " << g_program_name << " [--db PATH] [--workspace NAME] COMMAND [ARGS]\n" + "Usage: " << g_program_name << " [--db PATH | --backend pg --dsn DSN] " + "[--workspace NAME] COMMAND [ARGS]\n" "\n" "Commands:\n" " setup [--mount PATH] Initialize and optionally mount a workspace\n" @@ -43,7 +47,7 @@ void Usage(std::ostream &output) { " cat PATH Print a file\n" " ls [PATH] [--json] List a directory\n" " grep [-i] [-l] [-n] [--max-results N] PATTERN [PATH]\n" - " Search current text files inside SQLite\n" + " Search current text files in the workspace\n" " index status|enable|rebuild|disable\n" " Manage the optional trigram text index\n" " check [--quick] Verify workspace metadata and content\n" @@ -89,7 +93,7 @@ void Usage(std::ostream &output) { " mount status [MOUNT_POINT] Show active VexFS mounts\n" " unmount [--force] MOUNT_POINT\n" " Unmount; --force detaches a stale mount\n" - " doctor [--json] Check platform adapter and SQLite state\n"; + " doctor [--json] Check platform adapter and database state\n"; } Options ParseOptions(int argc, char **argv) { @@ -99,6 +103,13 @@ Options ParseOptions(int argc, char **argv) { std::string argument = argv[index]; if (argument == "--db" && index + 1 < argc) { options.database = argv[++index]; + } else if (argument == "--backend" && index + 1 < argc) { + options.backend = argv[++index]; + if (options.backend == "pg" || options.backend == "postgres") { + options.backend = "postgresql"; + } + } else if (argument == "--dsn" && index + 1 < argc) { + options.dsn = argv[++index]; } else if (argument == "--workspace" && index + 1 < argc) { options.workspace = argv[++index]; } else if (argument == "--mount" && index + 1 < argc) { @@ -112,6 +123,13 @@ Options ParseOptions(int argc, char **argv) { options.arguments.push_back(std::move(argument)); } } + if (options.backend != VEXFS_RUNTIME_BACKEND_SQLITE && + options.backend != "postgresql") { + throw std::runtime_error("backend must be sqlite or pg"); + } + if (options.backend == "postgresql" && options.dsn.empty()) { + throw std::runtime_error("PostgreSQL backend needs --dsn DSN"); + } return options; } @@ -168,18 +186,19 @@ class CliError : public std::runtime_error { class Session { public: explicit Session(const Options &options, bool initialize = true) { + const bool postgresql = options.backend == VEXFS_RUNTIME_BACKEND_POSTGRESQL; const std::filesystem::path path(options.database); - if (initialize && path.has_parent_path()) { + if (!postgresql && initialize && path.has_parent_path()) { const bool existed = std::filesystem::exists(path.parent_path()); std::filesystem::create_directories(path.parent_path()); if (!existed) VexFSPlatformProtectDirectory(path.parent_path()); } vexfs_mount_config config{}; config.abi_version = VEXFS_RUNTIME_ABI_VERSION; - config.backend = VEXFS_RUNTIME_BACKEND_SQLITE; - config.connection = options.database.c_str(); + config.backend = options.backend.c_str(); + config.connection = postgresql ? options.dsn.c_str() : options.database.c_str(); config.workspace = options.workspace.c_str(); - config.principal = "local"; + config.principal = postgresql ? "" : "local"; config.operation_timeout_ms = 5000; config.flags = initialize ? 0 : VEXFS_RUNTIME_OPEN_NO_CREATE; vexfs_mount_error error{}; @@ -252,33 +271,33 @@ void PrintNames(const std::string &json) { } } +int64_t JsonInteger(const std::string &json, const std::string &name); +std::string JsonString(const std::string &json, const std::string &name); + void PrintGrep(const std::string &json, bool files_only, bool show_line) { - const std::string path_marker = "\"path\":\""; - const std::string line_marker = "\"line\":"; - const std::string text_marker = "\"text\":\""; - size_t position = 0; - while ((position = json.find(path_marker, position)) != std::string::npos) { - position += path_marker.size(); - const std::string path = JsonUnescape(json, &position); - const size_t line_position = json.find(line_marker, position); - const size_t text_position = json.find(text_marker, position); - if (line_position == std::string::npos || text_position == std::string::npos) { + const std::string matches_marker = "\"matches\":["; + size_t position = json.find(matches_marker); + if (position == std::string::npos) throw std::runtime_error("invalid grep JSON"); + position += matches_marker.size(); + while (position < json.size() && json[position] != ']') { + if (json[position] == ',') ++position; + if (position >= json.size() || json[position] != '{') { throw std::runtime_error("invalid grep JSON"); } - char *end = nullptr; - const long long line = std::strtoll( - json.c_str() + line_position + line_marker.size(), &end, 10); - if (end == json.c_str() + line_position + line_marker.size()) { - throw std::runtime_error("invalid grep line number"); - } - position = text_position + text_marker.size(); - const std::string text = JsonUnescape(json, &position); + const size_t row_start = position; + const size_t row_end = json.find('}', row_start); + if (row_end == std::string::npos) throw std::runtime_error("invalid grep JSON"); + const std::string row = json.substr(row_start, row_end - row_start + 1); + const std::string path = JsonString(row, "path"); + const long long line = JsonInteger(row, "line"); + const std::string text = JsonString(row, "text"); std::cout << path; if (!files_only) { if (show_line) std::cout << ':' << line; std::cout << ':' << text; } std::cout << '\n'; + position = row_end + 1; } } @@ -560,12 +579,41 @@ std::string JsonEscape(const std::string &value) { } void WriteDescriptor(const Options &options, const std::filesystem::path &path) { + auto has_inline_password = [](const std::string &connection) { + std::string compact; + compact.reserve(connection.size()); + for (const unsigned char value : connection) { + if (value != ' ' && value != '\t' && value != '\r' && value != '\n') + compact.push_back(static_cast(std::tolower(value))); + } + if (compact.find("password=") != std::string::npos) return true; + const size_t scheme = compact.find("://"); + if (scheme == std::string::npos) return false; + const size_t authority = scheme + 3; + const size_t at = compact.find('@', authority); + return at != std::string::npos && compact.find(':', authority) < at; + }; + if (options.backend == VEXFS_RUNTIME_BACKEND_POSTGRESQL && + has_inline_password(options.dsn)) { + throw std::runtime_error( + "descriptor refuses an inline PostgreSQL password; use a client certificate " + "or a libpq service without inline secrets"); + } if (path.has_parent_path()) std::filesystem::create_directories(path.parent_path()); std::ofstream output(path, std::ios::binary | std::ios::trunc); if (!output) throw std::runtime_error("cannot write descriptor: " + path.string()); - output << "{\n \"version\": 1,\n \"database_path\": \"" - << JsonEscape(std::filesystem::absolute(options.database).string()) + const std::string connection = options.backend == VEXFS_RUNTIME_BACKEND_POSTGRESQL + ? options.dsn : std::filesystem::absolute(options.database).string(); + output << "{\n \"version\": 2,\n \"backend\": \"" + << JsonEscape(options.backend) + << "\",\n \"connection\": \"" << JsonEscape(connection) << "\",\n \"workspace\": \"" << JsonEscape(options.workspace) << "\"\n}\n"; + output.close(); + std::error_code error; + std::filesystem::permissions( + path, std::filesystem::perms::owner_read | std::filesystem::perms::owner_write, + std::filesystem::perm_options::replace, error); + if (error) throw std::runtime_error("cannot protect descriptor: " + error.message()); } std::string DatabaseDiagnostics(Session &session) { @@ -643,10 +691,15 @@ std::string NormalizedPath(const std::string &path) { } std::vector WorkspaceMounts(const Options &options) { - const std::string database = NormalizedPath(options.database); + const bool postgresql = options.backend == VEXFS_RUNTIME_BACKEND_POSTGRESQL; + const std::string database = postgresql ? options.dsn : NormalizedPath(options.database); std::vector matches; for (const auto &mount : VexFSPlatformInspect().mounts) { - if (!mount.database.empty() && NormalizedPath(mount.database) == database && + if (mount.backend != options.backend) continue; + const bool same_connection = postgresql + ? mount.database == database + : !mount.database.empty() && NormalizedPath(mount.database) == database; + if (same_connection && mount.workspace == options.workspace) { matches.push_back(mount); } @@ -667,6 +720,7 @@ int PrintMountStatus(const Options &options, const std::string &requested_path) std::cout << "{\"source\":\"" << JsonEscape(mounts[index].source) << "\",\"target\":\"" << JsonEscape(mounts[index].target) << "\",\"type\":\"" << JsonEscape(mounts[index].type) + << "\",\"backend\":\"" << JsonEscape(mounts[index].backend) << "\",\"database\":\"" << JsonEscape(mounts[index].database) << "\",\"workspace\":\"" << JsonEscape(mounts[index].workspace) << "\"}"; @@ -682,7 +736,11 @@ int PrintMountStatus(const Options &options, const std::string &requested_path) } int MountWorkspace(const Options &options, const std::string &mount_point) { - return VexFSPlatformMount(options.database, options.workspace, mount_point); + return VexFSPlatformMount( + options.backend, + options.backend == VEXFS_RUNTIME_BACKEND_POSTGRESQL + ? options.dsn : options.database, + options.workspace, mount_point); } int UnmountWorkspace(const std::string &mount_point, bool force) { @@ -759,10 +817,12 @@ int RunDoctor(const Options &options) { << ",\"fskit\":" << (platform.mount_driver_available ? "true" : "false"); } - std::cout - << ",\"database\":"; + std::cout << ",\"backend\":\"" << JsonEscape(options.backend) + << "\",\"database\":"; if (database_readable) std::cout << database_details; - else std::cout << "{\"path\":\"" << JsonEscape(options.database) + else std::cout << "{\"connection\":\"" + << (options.backend == VEXFS_RUNTIME_BACKEND_POSTGRESQL + ? "postgresql" : JsonEscape(options.database)) << "\",\"error\":\"" << JsonEscape(database_error) << "\"}"; std::cout << ",\"mount_count\":" << platform.mounts.size() << "}\n"; } else { @@ -775,9 +835,12 @@ int RunDoctor(const Options &options) { "extension path: " + platform.extension_path + "\n") << (platform.extension_path_matches ? std::string() : "extension path: does not match installed App\n") - << "database: " << options.database << '\n' + << "backend: " << options.backend << '\n' + << "database: " + << (options.backend == VEXFS_RUNTIME_BACKEND_POSTGRESQL + ? "PostgreSQL (DSN hidden)" : options.database) << '\n' << "workspace: " << options.workspace << '\n' - << "SQLite schema: " << (database_ok ? "ok" : + << "database schema: " << (database_ok ? "ok" : (database_readable ? "version mismatch" : database_error)) << '\n' << "active mounts: " << platform.mounts.size() << '\n'; if (database_readable) std::cout << "database details: " << database_details << '\n'; @@ -826,14 +889,24 @@ int Run(const Options &options) { } const std::string output = parsed.Value("--output"); const std::string snapshot = parsed.Value("--snapshot", false); - std::cout << vexfs_cli::ExportArchive( - options.database, options.workspace, snapshot, output) << '\n'; + if (options.backend == "postgresql") { + std::cout << vexfs_cli::ExportPostgresArchive( + options.dsn, options.workspace, snapshot, output) << '\n'; + } else { + std::cout << vexfs_cli::ExportArchive( + options.database, options.workspace, snapshot, output) << '\n'; + } return 0; } if (command == "import") { if (options.arguments.size() != 2) throw std::runtime_error("import needs FILE"); - std::cout << vexfs_cli::ImportArchive( - options.database, options.workspace, options.arguments[1]) << '\n'; + if (options.backend == "postgresql") { + std::cout << vexfs_cli::ImportPostgresArchive( + options.dsn, options.workspace, options.arguments[1]) << '\n'; + } else { + std::cout << vexfs_cli::ImportArchive( + options.database, options.workspace, options.arguments[1]) << '\n'; + } return 0; } if (command == "archive") { @@ -849,7 +922,8 @@ int Run(const Options &options) { vexfs_mount_error error{}; if (command == "init" || command == "setup") { if (options.arguments.size() != 1) throw std::runtime_error(command + " accepts only global options"); - std::cout << options.database << " [" << options.workspace << "]\n"; + std::cout << (options.backend == VEXFS_RUNTIME_BACKEND_POSTGRESQL + ? "postgresql" : options.database) << " [" << options.workspace << "]\n"; if (!options.mount_point.empty()) return MountWorkspace(options, options.mount_point); } else if (command == "mkdir") { if (options.arguments.size() < 2) throw std::runtime_error("mkdir needs PATH"); @@ -1109,8 +1183,6 @@ int Run(const Options &options) { if (parsed.Flag("--dry-run") && parsed.Flag("--force-unmount")) throw std::runtime_error("--force-unmount cannot be combined with --dry-run"); const std::string &name = parsed.positional[1]; - int64_t head = 0; - Check(vexfs_mount_workspace_head(session.get(), &head, &error), error); if (parsed.Flag("--dry-run")) { vexfs_mount_bytes json{}; Check(vexfs_mount_snapshot_diff(session.get(), "HEAD", name.c_str(), @@ -1132,6 +1204,11 @@ int Run(const Options &options) { } } + // Unmount can publish dirty FSKit handles and legitimately advance + // the workspace head. Read expected-head only after the last mount + // is gone, otherwise our own close path causes a false conflict. + int64_t head = 0; + Check(vexfs_mount_workspace_head(session.get(), &head, &error), error); int64_t commit = 0; try { commit = RestoreSnapshotAfterUnmount( diff --git a/agent_files/cli/vexfs_platform.h b/agent_files/cli/vexfs_platform.h index 7b99665aee..bbc72f6d80 100644 --- a/agent_files/cli/vexfs_platform.h +++ b/agent_files/cli/vexfs_platform.h @@ -10,6 +10,7 @@ struct VexFSPlatformMountEntry { std::string type; // Filled only when the mount was created by VexDB-Lite and its live mount // table entry still matches the saved identity record. + std::string backend; std::string database; std::string workspace; }; @@ -32,6 +33,7 @@ struct VexFSPlatformState { std::string VexFSPlatformDefaultDatabasePath(); void VexFSPlatformProtectDirectory(const std::filesystem::path &path); VexFSPlatformState VexFSPlatformInspect(); -int VexFSPlatformMount(const std::string &database, const std::string &workspace, +int VexFSPlatformMount(const std::string &backend, const std::string &connection, + const std::string &workspace, const std::string &mount_point); int VexFSPlatformUnmount(const std::string &mount_point, bool force); diff --git a/agent_files/cli/vexfs_platform_linux.cpp b/agent_files/cli/vexfs_platform_linux.cpp index fc3d6710ba..2009ad5913 100644 --- a/agent_files/cli/vexfs_platform_linux.cpp +++ b/agent_files/cli/vexfs_platform_linux.cpp @@ -1,6 +1,8 @@ #include "vexfs_platform.h" #include "vexfs_platform_registry.h" +#include "vexfs_runtime_types.h" +#include #include #include #include @@ -157,6 +159,41 @@ void PrepareMountPoint(const std::string &path) { throw std::runtime_error("mount point must be empty: " + path); } +// A crashed FUSE helper exposes the directory below the mount again. Make that +// directory read-only so tools cannot mistake local writes for durable VexFS +// writes. Explicit unmount restores the ordinary private-directory mode. +class UnderlyingMountPointGuard { + public: + explicit UnderlyingMountPointGuard(const std::string &path) + : descriptor_(open(path.c_str(), O_RDONLY | O_DIRECTORY | O_CLOEXEC)) { + if (descriptor_ < 0) + throw std::runtime_error("cannot open mount point guard: " + + std::string(std::strerror(errno))); + } + ~UnderlyingMountPointGuard() { + if (descriptor_ >= 0) close(descriptor_); + } + UnderlyingMountPointGuard(const UnderlyingMountPointGuard &) = delete; + UnderlyingMountPointGuard &operator=(const UnderlyingMountPointGuard &) = delete; + + void Arm() const { + if (fchmod(descriptor_, 0500) != 0) + throw std::runtime_error("cannot protect mount point: " + + std::string(std::strerror(errno))); + } + + private: + int descriptor_ = -1; +}; + +void DisarmMountPointGuard(const std::string &path) { + std::error_code error; + if (!std::filesystem::is_directory(path, error)) return; + if (chmod(path.c_str(), 0700) != 0) + throw std::runtime_error("cannot restore mount point permissions: " + + std::string(std::strerror(errno))); +} + std::string KernelVersion() { struct utsname information{}; return uname(&information) == 0 ? information.release : "unknown"; @@ -195,15 +232,18 @@ VexFSPlatformState VexFSPlatformInspect() { return state; } -int VexFSPlatformMount(const std::string &database, const std::string &workspace, +int VexFSPlatformMount(const std::string &backend, const std::string &connection, + const std::string &workspace, const std::string &mount_point) { const std::string mounted_type = MountedFileSystemAt(mount_point); if (mounted_type == "fuse.vexfs") { - const std::string requested_database = NormalizedPath(database); + const std::string requested_database = backend == VEXFS_RUNTIME_BACKEND_SQLITE + ? NormalizedPath(connection) : connection; const std::string requested_target = NormalizedPath(mount_point); for (const auto &mount : MountedVolumes()) { if (NormalizedPath(mount.target) != requested_target) continue; - if (mount.database == requested_database && mount.workspace == workspace) return 0; + if (mount.backend == backend && mount.database == requested_database && + mount.workspace == workspace) return 0; throw std::runtime_error( "mount point already contains a VexFS workspace with different or unknown identity"); } @@ -215,30 +255,45 @@ int VexFSPlatformMount(const std::string &database, const std::string &workspace if (!state.mount_ready) throw std::runtime_error("VexFS libfuse3 mount is not ready: " + state.extension_state); PrepareMountPoint(mount_point); - const std::filesystem::path absolute_database = - std::filesystem::absolute(database).lexically_normal(); - if (absolute_database.has_parent_path()) { - std::filesystem::create_directories(absolute_database.parent_path()); - VexFSPlatformProtectDirectory(absolute_database.parent_path()); + std::string mount_connection = connection; + if (backend == VEXFS_RUNTIME_BACKEND_SQLITE) { + const std::filesystem::path absolute_database = + std::filesystem::absolute(connection).lexically_normal(); + if (absolute_database.has_parent_path()) { + std::filesystem::create_directories(absolute_database.parent_path()); + VexFSPlatformProtectDirectory(absolute_database.parent_path()); + } + mount_connection = absolute_database.string(); } const std::string helper = FuseHelper(); - const int result = RunProcess(helper, - {"--db", absolute_database.string(), "--workspace", workspace, mount_point}); + std::vector arguments = { + "--backend", backend, + backend == VEXFS_RUNTIME_BACKEND_POSTGRESQL ? "--dsn" : "--db", + mount_connection, + "--workspace", workspace, + mount_point}; + UnderlyingMountPointGuard mount_point_guard(mount_point); + const int result = RunProcess(helper, arguments); for (int attempt = 0; attempt < 50; ++attempt) { if (!MountedFileSystemAt(mount_point).empty()) { try { + mount_point_guard.Arm(); VexFSPlatformRememberMount(MountRegistryDirectory(), - {"vexfs", NormalizedPath(mount_point), "fuse.vexfs", - NormalizedPath(database), workspace}); + {"vexfs", NormalizedPath(mount_point), "fuse.vexfs", backend, + backend == VEXFS_RUNTIME_BACKEND_SQLITE + ? NormalizedPath(connection) : connection, + workspace}); } catch (...) { const std::string fusermount = FindExecutable("fusermount3"); if (!fusermount.empty()) RunProcess(fusermount, {"-u", mount_point}); + DisarmMountPointGuard(mount_point); throw; } return 0; } usleep(100 * 1000); } + DisarmMountPointGuard(mount_point); if (result == 0) throw std::runtime_error("vexfs-fuse returned success but the mount is not active"); return result; @@ -248,6 +303,7 @@ int VexFSPlatformUnmount(const std::string &mount_point, bool force) { const std::string mounted_type = MountedFileSystemAt(mount_point); if (mounted_type.empty()) { VexFSPlatformForgetMount(MountRegistryDirectory(), mount_point); + DisarmMountPointGuard(mount_point); return 0; } if (mounted_type != "fuse.vexfs") @@ -261,6 +317,7 @@ int VexFSPlatformUnmount(const std::string &mount_point, bool force) { : std::vector{"-u", mount_point}); if (MountedFileSystemAt(mount_point).empty()) { VexFSPlatformForgetMount(MountRegistryDirectory(), mount_point); + DisarmMountPointGuard(mount_point); return 0; } usleep(200 * 1000); diff --git a/agent_files/cli/vexfs_platform_macos.cpp b/agent_files/cli/vexfs_platform_macos.cpp index 622b0368ca..6680131b8d 100644 --- a/agent_files/cli/vexfs_platform_macos.cpp +++ b/agent_files/cli/vexfs_platform_macos.cpp @@ -1,8 +1,10 @@ #include "vexfs_platform.h" #include "vexfs_platform_registry.h" #include "vexfs_fskit_state.h" +#include "vexfs_runtime_types.h" #include +#include #include #include #include @@ -10,18 +12,35 @@ #include #include +#include +#include +#include +#include #include #include #include #include #include +#include +#include +#include #include #include #include +#if defined(VEXFS_HAVE_LIBPQ) +#include +#endif + namespace { -enum class ExtensionState { kMissing, kEnabled, kDisabled, kRegistered }; +enum class ExtensionState { + kMissing, + kEnabled, + kDisabled, + kRegistered, + kServiceUnavailable, +}; std::string HomeDirectory() { const char *home = std::getenv("HOME"); @@ -58,7 +77,8 @@ struct ProcessOutput { std::string output; }; -int RunProcess(const char *program, const std::vector &arguments) { +int RunProcess(const char *program, const std::vector &arguments, + int timeout_ms = 30'000) { const pid_t child = fork(); if (child < 0) throw std::runtime_error(std::strerror(errno)); if (child == 0) { @@ -71,8 +91,26 @@ int RunProcess(const char *program, const std::vector &arguments) { _exit(127); } int status = 0; - if (waitpid(child, &status, 0) < 0) throw std::runtime_error(std::strerror(errno)); - return WIFEXITED(status) ? WEXITSTATUS(status) : 1; + const auto deadline = std::chrono::steady_clock::now() + + std::chrono::milliseconds(timeout_ms); + while (true) { + const pid_t waited = waitpid(child, &status, WNOHANG); + if (waited == child) + return WIFEXITED(status) ? WEXITSTATUS(status) : 1; + if (waited < 0 && errno != EINTR) + throw std::runtime_error(std::strerror(errno)); + if (std::chrono::steady_clock::now() >= deadline) { + kill(child, SIGTERM); + for (int attempt = 0; attempt < 10; ++attempt) { + if (waitpid(child, &status, WNOHANG) == child) return 124; + usleep(50 * 1000); + } + kill(child, SIGKILL); + while (waitpid(child, &status, 0) < 0 && errno == EINTR) {} + return 124; + } + usleep(50 * 1000); + } } ProcessOutput CaptureProcess(const char *program, const std::vector &arguments) { @@ -123,6 +161,10 @@ ExtensionState GetExtensionState(std::string *module_path) { if (module_path != nullptr) *module_path = module_url; if (fskit_state == 2) return ExtensionState::kEnabled; if (fskit_state == 1) return ExtensionState::kDisabled; + // Negative results mean FSKit itself timed out or rejected the XPC request. + // Falling through to pluginkit used to turn that into the misleading + // "missing" state even when the signed App and extension were on disk. + if (fskit_state < 0) return ExtensionState::kServiceUnavailable; const ProcessOutput result = CaptureProcess("/usr/bin/pluginkit", {"-m", "-A", "-D", "-i", "io.vexdb.vexfs.extension"}); @@ -147,6 +189,7 @@ const char *ExtensionStateName(ExtensionState state) { case ExtensionState::kEnabled: return "enabled"; case ExtensionState::kDisabled: return "disabled"; case ExtensionState::kRegistered: return "registered"; + case ExtensionState::kServiceUnavailable: return "service-unavailable"; case ExtensionState::kMissing: return "missing"; } return "missing"; @@ -212,21 +255,389 @@ void PrepareMountPoint(const std::string &path) { } } -std::filesystem::path WriteMountResourceDescriptor(const std::string &database_path, +// Keep the directory below a live mount non-writable. If FSKit crashes and +// macOS removes the mount, Bash must fail instead of silently writing files to +// the local directory. A normal VexFS unmount restores the directory to 0700. +class UnderlyingMountPointGuard { + public: + explicit UnderlyingMountPointGuard(const std::string &path) + : descriptor_(open(path.c_str(), O_RDONLY | O_DIRECTORY | O_CLOEXEC)) { + if (descriptor_ < 0) + throw std::runtime_error("cannot open mount point guard: " + + std::string(std::strerror(errno))); + } + ~UnderlyingMountPointGuard() { + if (descriptor_ >= 0) close(descriptor_); + } + UnderlyingMountPointGuard(const UnderlyingMountPointGuard &) = delete; + UnderlyingMountPointGuard &operator=(const UnderlyingMountPointGuard &) = delete; + + void Arm() const { + if (fchmod(descriptor_, 0500) != 0) + throw std::runtime_error("cannot protect mount point: " + + std::string(std::strerror(errno))); + } + + private: + int descriptor_ = -1; +}; + +void DisarmMountPointGuard(const std::string &path) { + std::error_code error; + if (!std::filesystem::is_directory(path, error)) return; + if (chmod(path.c_str(), 0700) != 0) + throw std::runtime_error("cannot restore mount point permissions: " + + std::string(std::strerror(errno))); +} + +uint64_t StableHash(const std::string &value) { + uint64_t hash = 1469598103934665603ULL; + for (const unsigned char character : value) { + hash ^= character; + hash *= 1099511628211ULL; + } + return hash; +} + +bool HasInlinePostgreSQLPassword(const std::string &connection) { + std::string lower = connection; + for (char &character : lower) + character = static_cast(std::tolower(static_cast(character))); + if (lower.find("password=") != std::string::npos) return true; + const size_t scheme = lower.find("://"); + if (scheme == std::string::npos) return false; + const size_t authority = scheme + 3; + const size_t at = lower.find('@', authority); + if (at == std::string::npos) return false; + return lower.find(':', authority) < at; +} + +std::string EscapeLibpqValue(const std::string &value) { + std::string escaped; + escaped.reserve(value.size() + 2); + escaped.push_back('\''); + for (const char character : value) { + if (character == '\'' || character == '\\') escaped.push_back('\\'); + escaped.push_back(character); + } + escaped.push_back('\''); + return escaped; +} + +struct PostgreSQLNetworkEndpoint { + std::string host; + int port = 5432; +}; + +bool ParseRemotePostgreSQLEndpoint(const std::string &connection, + PostgreSQLNetworkEndpoint *endpoint) { +#if !defined(VEXFS_HAVE_LIBPQ) + (void)connection; + (void)endpoint; + return false; +#else + char *parse_error = nullptr; + PQconninfoOption *options = PQconninfoParse(connection.c_str(), &parse_error); + if (options == nullptr) { + if (parse_error != nullptr) PQfreemem(parse_error); + return false; + } + struct OptionsGuard { + PQconninfoOption *value; + ~OptionsGuard() { PQconninfoFree(value); } + } guard{options}; + std::string host; + std::string host_address; + std::string port = "5432"; + for (PQconninfoOption *option = options; option->keyword != nullptr; ++option) { + if (option->val == nullptr || option->val[0] == '\0') continue; + if (std::strcmp(option->keyword, "host") == 0) host = option->val; + else if (std::strcmp(option->keyword, "hostaddr") == 0) host_address = option->val; + else if (std::strcmp(option->keyword, "port") == 0) port = option->val; + } + if (!host_address.empty()) host = host_address; + const size_t host_separator = host.find(','); + if (host_separator != std::string::npos) host.resize(host_separator); + const size_t port_separator = port.find(','); + if (port_separator != std::string::npos) port.resize(port_separator); + if (host.size() >= 2 && host.front() == '[' && host.back() == ']') + host = host.substr(1, host.size() - 2); + std::string lower = host; + for (char &character : lower) + character = static_cast(std::tolower(static_cast(character))); + if (host.empty() || host.front() == '/' || lower == "localhost" || lower == "::1" || + lower.rfind("127.", 0) == 0) return false; + char *end = nullptr; + const long parsed_port = std::strtol(port.c_str(), &end, 10); + if (end == port.c_str() || *end != '\0' || parsed_port < 1 || parsed_port > 65535) + return false; + endpoint->host = std::move(host); + endpoint->port = static_cast(parsed_port); + return true; +#endif +} + +bool RequestLocalNetworkAccess(const std::string &connection) { + PostgreSQLNetworkEndpoint endpoint; + if (!ParseRemotePostgreSQLEndpoint(connection, &endpoint)) return false; + const std::filesystem::path app = std::filesystem::path(HomeDirectory()) / + "Applications/VexDB Lite.app"; + if (!std::filesystem::exists(app)) return false; + // The signed host App is sandboxed. Its home directory is the Data folder + // below this container, while the standalone CLI sees the normal user home. + // This request contains only host/port/nonce, never the DSN or credentials. + const std::filesystem::path support = std::filesystem::path(HomeDirectory()) / + "Library/Containers/io.vexdb.vexfs/Data/Library/Application Support/VexDB-Lite"; + std::filesystem::create_directories(support); + VexFSPlatformProtectDirectory(support); + const std::filesystem::path request = support / "local-network-probe.json"; + const std::filesystem::path response = support / "local-network-probe-response.json"; + std::error_code error; + std::filesystem::remove(request, error); + std::filesystem::remove(response, error); + const uint64_t nonce_value = StableHash(connection + std::to_string(getpid()) + + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); + std::ostringstream nonce_stream; + nonce_stream << std::hex << std::setw(16) << std::setfill('0') << nonce_value; + const std::string nonce = nonce_stream.str(); + const std::filesystem::path temporary = + support / ("local-network-probe.new." + std::to_string(getpid())); + std::filesystem::remove(temporary, error); + const int descriptor = open(temporary.c_str(), + O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC | O_NOFOLLOW, 0600); + if (descriptor < 0) return false; + std::ostringstream payload; + payload << "{\"version\":1,\"nonce\":\"" << JsonEscape(nonce) + << "\",\"host\":\"" << JsonEscape(endpoint.host) + << "\",\"port\":" << endpoint.port << "}\n"; + const std::string data = payload.str(); + size_t written = 0; + while (written < data.size()) { + const ssize_t count = write(descriptor, data.data() + written, data.size() - written); + if (count < 0 && errno == EINTR) continue; + if (count <= 0) break; + written += static_cast(count); + } + const bool write_ok = written == data.size() && fsync(descriptor) == 0; + close(descriptor); + if (!write_ok || rename(temporary.c_str(), request.c_str()) != 0) { + std::filesystem::remove(temporary, error); + std::filesystem::remove(request, error); + return false; + } + std::fprintf(stderr, + "VexDB Lite is requesting local network access for PostgreSQL; " + "approve the macOS prompt if it appears.\n"); + RunProcess("/usr/bin/open", {"-a", app.string()}); + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(32); + while (std::chrono::steady_clock::now() < deadline) { + std::ifstream input(response, std::ios::binary); + if (input) { + const std::string result((std::istreambuf_iterator(input)), + std::istreambuf_iterator()); + if (result.find("\"nonce\":\"" + nonce + "\"") != std::string::npos) { + const bool allowed = result.find("\"allowed\":true") != std::string::npos; + std::filesystem::remove(request, error); + std::filesystem::remove(response, error); + if (!allowed) { + std::fprintf(stderr, + "VexDB Lite could not use the local network. Enable it in " + "System Settings > Privacy & Security > Local Network.\n"); + } + return allowed; + } + } + usleep(250 * 1000); + } + std::filesystem::remove(request, error); + std::filesystem::remove(response, error); + std::fprintf(stderr, + "Timed out waiting for VexDB Lite local network approval.\n"); + return false; +} + +std::string StagePostgreSQLPassfile(const std::string &connection, + const std::filesystem::path &directory) { +#if !defined(VEXFS_HAVE_LIBPQ) + (void)directory; + return connection; +#else + char *parse_error = nullptr; + PQconninfoOption *options = PQconninfoParse(connection.c_str(), &parse_error); + if (options == nullptr) { + const std::string message = parse_error == nullptr + ? "invalid PostgreSQL DSN" : std::string(parse_error); + if (parse_error != nullptr) PQfreemem(parse_error); + throw std::runtime_error(message); + } + struct OptionsGuard { + PQconninfoOption *value; + ~OptionsGuard() { PQconninfoFree(value); } + } guard{options}; + + std::filesystem::path source; + for (PQconninfoOption *option = options; option->keyword != nullptr; ++option) { + if (option->val == nullptr || option->val[0] == '\0') continue; + if (std::strcmp(option->keyword, "password") == 0) { + throw std::runtime_error( + "PostgreSQL mount DSN must not resolve an inline password; use passfile"); + } + if (std::strcmp(option->keyword, "passfile") == 0) source = option->val; + } + if (source.empty()) return connection; + source = std::filesystem::absolute(source).lexically_normal(); + + struct stat metadata {}; + const int source_fd = open(source.c_str(), O_RDONLY | O_CLOEXEC | O_NOFOLLOW); + if (source_fd < 0) + throw std::runtime_error("cannot read PostgreSQL passfile: " + + std::string(std::strerror(errno))); + struct FileGuard { + int descriptor; + ~FileGuard() { if (descriptor >= 0) close(descriptor); } + } source_guard{source_fd}; + if (fstat(source_fd, &metadata) != 0) + throw std::runtime_error("cannot inspect PostgreSQL passfile: " + + std::string(std::strerror(errno))); + if (!S_ISREG(metadata.st_mode)) + throw std::runtime_error("PostgreSQL passfile must be a regular file"); + if (metadata.st_uid != getuid()) + throw std::runtime_error("PostgreSQL passfile must be owned by the current user"); + if ((metadata.st_mode & 0077) != 0) + throw std::runtime_error("PostgreSQL passfile permissions must be 0600 or stricter"); + if (metadata.st_size < 0 || metadata.st_size > 64 * 1024) + throw std::runtime_error("PostgreSQL passfile is larger than 64 KiB"); + + const std::filesystem::path staged = directory / ".vexfs-pgpass"; + const std::filesystem::path temporary = + directory / (".vexfs-pgpass.new." + std::to_string(getpid())); + std::error_code error; + std::filesystem::remove(temporary, error); + try { + const int target_fd = open(temporary.c_str(), + O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC | O_NOFOLLOW, 0600); + if (target_fd < 0) + throw std::runtime_error("cannot stage PostgreSQL passfile: " + + std::string(std::strerror(errno))); + FileGuard target_guard{target_fd}; + char buffer[8192]; + for (;;) { + const ssize_t count = read(source_fd, buffer, sizeof(buffer)); + if (count == 0) break; + if (count < 0) { + if (errno == EINTR) continue; + throw std::runtime_error("cannot read PostgreSQL passfile: " + + std::string(std::strerror(errno))); + } + ssize_t written = 0; + while (written < count) { + const ssize_t result = write(target_fd, buffer + written, + static_cast(count - written)); + if (result < 0 && errno == EINTR) continue; + if (result <= 0) + throw std::runtime_error("cannot stage PostgreSQL passfile: " + + std::string(std::strerror(errno))); + written += result; + } + } + if (fsync(target_fd) != 0) + throw std::runtime_error("cannot sync PostgreSQL passfile: " + + std::string(std::strerror(errno))); + if (rename(temporary.c_str(), staged.c_str()) != 0) + throw std::runtime_error(std::strerror(errno)); + } catch (...) { + std::filesystem::remove(temporary, error); + throw; + } + + std::ostringstream rewritten; + bool first = true; + for (PQconninfoOption *option = options; option->keyword != nullptr; ++option) { + if (option->val == nullptr) continue; + if (!first) rewritten << ' '; + first = false; + rewritten << option->keyword << '=' << EscapeLibpqValue( + std::strcmp(option->keyword, "passfile") == 0 + ? staged.string() : std::string(option->val)); + } + return rewritten.str(); +#endif +} + +void RemoveStagedPostgreSQLPassfile(const std::filesystem::path &directory) { + if (directory.empty()) return; + std::error_code error; + std::filesystem::remove(directory / ".vexfs-pgpass", error); + for (std::filesystem::directory_iterator iterator(directory, error), end; + !error && iterator != end; iterator.increment(error)) { + if (iterator->path().filename().string().rfind(".vexfs-pgpass.new.", 0) == 0) + std::filesystem::remove(iterator->path(), error); + } +} + +void RemoveStalePostgreSQLPassfiles() { + const std::filesystem::path root = std::filesystem::path(HomeDirectory()) / + "Library/Application Support/VexDB-Lite/mount-resources"; + std::error_code error; + if (!std::filesystem::is_directory(root, error)) return; + const auto mounts = MountedVolumes(); + for (std::filesystem::directory_iterator iterator(root, error), end; + !error && iterator != end; iterator.increment(error)) { + if (!iterator->is_directory(error)) continue; + const std::string resource = NormalizedPath(iterator->path().string()); + bool active = false; + for (const auto &mount : mounts) { + if (NormalizedPath(mount.source) == resource) { + active = true; + break; + } + } + if (!active) RemoveStagedPostgreSQLPassfile(iterator->path()); + } +} + +std::filesystem::path WriteMountResourceDescriptor(const std::string &backend, + const std::string &connection, const std::string &workspace) { - const std::filesystem::path database = - std::filesystem::absolute(database_path).lexically_normal(); - const std::filesystem::path directory = database.parent_path(); - if (directory.empty() || database.filename().empty()) { - throw std::runtime_error("database path must include a file name"); + std::filesystem::path directory; + std::string descriptor_connection; + if (backend == VEXFS_RUNTIME_BACKEND_SQLITE) { + const std::filesystem::path database = + std::filesystem::absolute(connection).lexically_normal(); + directory = database.parent_path(); + if (directory.empty() || database.filename().empty()) + throw std::runtime_error("database path must include a file name"); + descriptor_connection = database.filename().string(); + } else if (backend == VEXFS_RUNTIME_BACKEND_POSTGRESQL) { + if (connection.empty()) throw std::runtime_error("PostgreSQL DSN is required"); + if (HasInlinePostgreSQLPassword(connection)) { + throw std::runtime_error( + "PostgreSQL mount DSN must not contain a password; use a 0600 passfile, " + "a client certificate, or a libpq service without inline secrets"); + } + std::ostringstream name; + name << std::hex << std::setw(16) << std::setfill('0') + << StableHash(connection + std::string(1, '\0') + workspace); + directory = std::filesystem::path(HomeDirectory()) / + "Library/Application Support/VexDB-Lite/mount-resources" / name.str(); + descriptor_connection = connection; + } else { + throw std::runtime_error("unsupported VexFS backend: " + backend); } std::filesystem::create_directories(directory); + VexFSPlatformProtectDirectory(directory); + if (backend == VEXFS_RUNTIME_BACKEND_POSTGRESQL) + descriptor_connection = StagePostgreSQLPassfile(connection, directory); const std::filesystem::path descriptor = directory / ".vexfs-volume.json"; std::ofstream output(descriptor, std::ios::binary | std::ios::trunc); if (!output) throw std::runtime_error("cannot write mount resource: " + descriptor.string()); - output << "{\n \"version\": 2,\n \"database_file\": \"" - << JsonEscape(database.filename().string()) + output << "{\n \"version\": 3,\n \"backend\": \"" + << JsonEscape(backend) + << "\",\n \"connection\": \"" << JsonEscape(descriptor_connection) << "\",\n \"workspace\": \"" << JsonEscape(workspace) << "\"\n}\n"; + output.close(); + if (chmod(descriptor.c_str(), 0600) != 0) + throw std::runtime_error(std::strerror(errno)); return directory; } @@ -246,6 +657,7 @@ void VexFSPlatformProtectDirectory(const std::filesystem::path &path) { } VexFSPlatformState VexFSPlatformInspect() { + RemoveStalePostgreSQLPassfiles(); VexFSPlatformState state; state.platform = "macos"; state.version = ProductVersion(); @@ -266,15 +678,18 @@ VexFSPlatformState VexFSPlatformInspect() { return state; } -int VexFSPlatformMount(const std::string &database, const std::string &workspace, +int VexFSPlatformMount(const std::string &backend, const std::string &connection, + const std::string &workspace, const std::string &mount_point) { const std::string mounted_type = MountedFileSystemAt(mount_point); if (mounted_type == "vexfs") { - const std::string requested_database = NormalizedPath(database); + const std::string requested_database = backend == VEXFS_RUNTIME_BACKEND_SQLITE + ? NormalizedPath(connection) : connection; const std::string requested_target = NormalizedPath(mount_point); for (const auto &mount : MountedVolumes()) { if (NormalizedPath(mount.target) != requested_target) continue; - if (mount.database == requested_database && mount.workspace == workspace) return 0; + if (mount.backend == backend && mount.database == requested_database && + mount.workspace == workspace) return 0; throw std::runtime_error( "mount point already contains a VexFS workspace with different or unknown identity"); } @@ -285,33 +700,53 @@ int VexFSPlatformMount(const std::string &database, const std::string &workspace } const VexFSPlatformState state = VexFSPlatformInspect(); if (!state.mount_ready) { + if (state.extension_state == "service-unavailable") { + throw std::runtime_error( + "macOS FSKit service is unavailable; reopen System Settings or restart " + "macOS, then run `vexdb fs doctor` again"); + } throw std::runtime_error("VexFS FSKit extension is " + state.extension_state + "; install VexDB Lite.app and enable the VexFS file system extension in System Settings"); } PrepareMountPoint(mount_point); const std::filesystem::path resource_directory = - WriteMountResourceDescriptor(database, workspace); + WriteMountResourceDescriptor(backend, connection, workspace); + UnderlyingMountPointGuard mount_point_guard(mount_point); int result = 1; for (int attempt = 0; attempt < 3; ++attempt) { result = RunProcess("/sbin/mount", - {"-F", "-t", "vexfs", resource_directory.string(), mount_point}); + {"-F", "-t", "vexfs", resource_directory.string(), mount_point}, 15'000); if (MountedFileSystemAt(mount_point) == "vexfs") { + // Keep the protected passfile while this resource is mounted. FSKit can + // reload the resource after its process restarts, and libpq must still + // be able to authenticate. The last unmount removes the staged copy. try { + mount_point_guard.Arm(); VexFSPlatformRememberMount(MountRegistryDirectory(), - {resource_directory.string(), NormalizedPath(mount_point), "vexfs", - NormalizedPath(database), workspace}); + {resource_directory.string(), NormalizedPath(mount_point), "vexfs", backend, + backend == VEXFS_RUNTIME_BACKEND_SQLITE + ? NormalizedPath(connection) : connection, + workspace}); } catch (...) { - RunProcess("/sbin/umount", {mount_point}); + RunProcess("/sbin/umount", {mount_point}, 5'000); + RemoveStagedPostgreSQLPassfile(resource_directory); + DisarmMountPointGuard(mount_point); throw; } return 0; } if (result == 0) break; + if (attempt == 0 && backend == VEXFS_RUNTIME_BACKEND_POSTGRESQL) + RequestLocalNetworkAccess(connection); if (attempt != 2) usleep(500 * 1000); } if (result == 0) { + RemoveStagedPostgreSQLPassfile(resource_directory); + DisarmMountPointGuard(mount_point); throw std::runtime_error("mount command returned success but VexFS is not mounted"); } + RemoveStagedPostgreSQLPassfile(resource_directory); + DisarmMountPointGuard(mount_point); return result; } @@ -319,10 +754,20 @@ int VexFSPlatformUnmount(const std::string &mount_point, bool force) { const std::string mounted_type = MountedFileSystemAt(mount_point); if (mounted_type.empty()) { VexFSPlatformForgetMount(MountRegistryDirectory(), mount_point); + RemoveStalePostgreSQLPassfiles(); + DisarmMountPointGuard(mount_point); return 0; } if (mounted_type != "vexfs") throw std::runtime_error("mount point is not VexFS: " + mounted_type); + std::filesystem::path resource_directory; + const std::string normalized_target = NormalizedPath(mount_point); + for (const auto &mount : MountedVolumes()) { + if (NormalizedPath(mount.target) == normalized_target) { + resource_directory = mount.source; + break; + } + } int result = 1; // fseventsd/Spotlight can briefly hold a newly mounted FSKit volume even // when no user process has an open file. Keep this a normal (non-force) @@ -331,9 +776,20 @@ int VexFSPlatformUnmount(const std::string &mount_point, bool force) { for (int attempt = 0; attempt < kUnmountAttempts; ++attempt) { result = RunProcess("/sbin/umount", force ? std::vector{"-f", mount_point} - : std::vector{mount_point}); + : std::vector{mount_point}, 5'000); if (MountedFileSystemAt(mount_point).empty()) { VexFSPlatformForgetMount(MountRegistryDirectory(), mount_point); + bool resource_still_mounted = false; + for (const auto &mount : MountedVolumes()) { + if (!resource_directory.empty() && + NormalizedPath(mount.source) == NormalizedPath(resource_directory)) { + resource_still_mounted = true; + break; + } + } + if (!resource_still_mounted) + RemoveStagedPostgreSQLPassfile(resource_directory); + DisarmMountPointGuard(mount_point); return 0; } if (attempt + 1 != kUnmountAttempts) usleep(500 * 1000); diff --git a/agent_files/cli/vexfs_platform_registry.cpp b/agent_files/cli/vexfs_platform_registry.cpp index b28df7316d..bcc0f1a1ed 100644 --- a/agent_files/cli/vexfs_platform_registry.cpp +++ b/agent_files/cli/vexfs_platform_registry.cpp @@ -15,7 +15,7 @@ namespace { -constexpr const char *kRegistryVersion = "VEXFS_MOUNT_REGISTRY_V1"; +constexpr const char *kRegistryVersion = "VEXFS_MOUNT_REGISTRY_V2"; std::string PercentDecode(std::string value) { const auto hex = [](char character) -> int { @@ -86,7 +86,13 @@ bool ReadRecord(const std::filesystem::path &path, VexFSPlatformMountEntry *entr if (!std::getline(input, version) || version != kRegistryVersion) return false; return static_cast(input >> std::quoted(entry->source) >> std::quoted(entry->target) >> std::quoted(entry->type) - >> std::quoted(entry->database) >> std::quoted(entry->workspace)); + >> std::quoted(entry->backend) >> std::quoted(entry->database) + >> std::quoted(entry->workspace)); +} + +std::string NormalizeConnection(const std::string &backend, + const std::string &connection) { + return backend == "sqlite" ? NormalizePath(connection) : connection; } } // namespace @@ -104,7 +110,8 @@ std::vector VexFSPlatformAttachMountRegistry( for (auto &live : mounts) { if (NormalizePath(live.target) != NormalizePath(saved.target) || NormalizeSource(live.source) != NormalizeSource(saved.source)) continue; - live.database = NormalizePath(saved.database); + live.backend = saved.backend; + live.database = NormalizeConnection(saved.backend, saved.database); live.workspace = saved.workspace; break; } @@ -114,7 +121,8 @@ std::vector VexFSPlatformAttachMountRegistry( void VexFSPlatformRememberMount(const std::filesystem::path ®istry_directory, const VexFSPlatformMountEntry &mount) { - if (mount.target.empty() || mount.source.empty() || mount.database.empty() || + if (mount.target.empty() || mount.source.empty() || mount.backend.empty() || + mount.database.empty() || mount.workspace.empty()) { throw std::runtime_error("incomplete VexFS mount identity"); } @@ -131,7 +139,8 @@ void VexFSPlatformRememberMount(const std::filesystem::path ®istry_directory, << std::quoted(mount.source) << '\n' << std::quoted(NormalizePath(mount.target)) << '\n' << std::quoted(mount.type) << '\n' - << std::quoted(NormalizePath(mount.database)) << '\n' + << std::quoted(mount.backend) << '\n' + << std::quoted(NormalizeConnection(mount.backend, mount.database)) << '\n' << std::quoted(mount.workspace) << '\n'; if (!output) throw std::runtime_error("cannot finish VexFS mount registry"); } diff --git a/agent_files/cli/vexfs_platform_stub.cpp b/agent_files/cli/vexfs_platform_stub.cpp index 46ce63c104..857661f6d8 100644 --- a/agent_files/cli/vexfs_platform_stub.cpp +++ b/agent_files/cli/vexfs_platform_stub.cpp @@ -90,6 +90,7 @@ VexFSPlatformState VexFSPlatformInspect() { } int VexFSPlatformMount(const std::string &, const std::string &, + const std::string &, const std::string &) { throw std::runtime_error(std::string("VexFS ") + MountDriverName() + " mount adapter is not implemented yet"); diff --git a/agent_files/macos/VexFS.xcodeproj/project.pbxproj b/agent_files/macos/VexFS.xcodeproj/project.pbxproj index 8b06dcbe54..f86a8ae5b6 100644 --- a/agent_files/macos/VexFS.xcodeproj/project.pbxproj +++ b/agent_files/macos/VexFS.xcodeproj/project.pbxproj @@ -7,38 +7,38 @@ objects = { /* Begin PBXBuildFile section */ - 1C608A9018A4422EA1952C9C /* VexFSFileSystem.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2F39B3A14283D310941712A9 /* VexFSFileSystem.swift */; }; - 1EA761C45020F997EB84C888 /* VexFSVolume.swift in Sources */ = {isa = PBXBuildFile; fileRef = 44C93C9A924C2FE4F6908B8B /* VexFSVolume.swift */; }; - 3780F8276301D1A2E0BFA83A /* VexFSItem.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7E0E0BBFF61E43527CEFB2D3 /* VexFSItem.swift */; }; - 3C4A509CFA9740CED0FA6288 /* VexFSAppEx.appex in Embed ExtensionKit Extensions */ = {isa = PBXBuildFile; fileRef = B9426785DC8CB247B8D17175 /* VexFSAppEx.appex */; settings = {ATTRIBUTES = (RemoveHeadersOnCopy, ); }; }; - 42FDB23ACE4332A35F1EC106 /* ContentView.swift in Sources */ = {isa = PBXBuildFile; fileRef = ACD925958EF883345CE46C7D /* ContentView.swift */; }; - 45F16BD430300F6F029AE478 /* Cocoa.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 7462B11A9FB54192939A31FB /* Cocoa.framework */; }; - 5E2138B682A54BE885D561E1 /* FSKit.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 8B6A0DDE2868EC68E6A66B4B /* FSKit.framework */; }; - 6C5EE8691215FE2119F96681 /* Cocoa.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 7462B11A9FB54192939A31FB /* Cocoa.framework */; }; - 94CD0A2563F9AFB2FD7F6D7F /* VexFSApp.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6D28655136F6C3AE749EC51A /* VexFSApp.swift */; }; - B5409F3CA8A4250DE5EC350D /* VexFSAppEx.swift in Sources */ = {isa = PBXBuildFile; fileRef = AE67026F8EE22D6D7FDDF80B /* VexFSAppEx.swift */; }; - EFE51E86624B1AAEE193E04A /* VexFSVolume+Operations.swift in Sources */ = {isa = PBXBuildFile; fileRef = 61EE820F7CF3E4F8A29B0443 /* VexFSVolume+Operations.swift */; }; - FC4015CAE58C58A33AB7D39B /* VexFSBackend.swift in Sources */ = {isa = PBXBuildFile; fileRef = ADC6AEF286D1A4ABBB5D64FC /* VexFSBackend.swift */; }; + 2EE6D8298C04D3E5E5C2460F /* VexFSAppEx.appex in Embed ExtensionKit Extensions */ = {isa = PBXBuildFile; fileRef = 1C486E29BAF93B35AFBC796E /* VexFSAppEx.appex */; settings = {ATTRIBUTES = (RemoveHeadersOnCopy, ); }; }; + 5A5203C7F7F89B325F9FA2CC /* VexFSApp.swift in Sources */ = {isa = PBXBuildFile; fileRef = 73EBA70FC3B95B3888C01E07 /* VexFSApp.swift */; }; + 664CB1C505108619FDE55161 /* VexFSVolume+Operations.swift in Sources */ = {isa = PBXBuildFile; fileRef = A80D16872765D81610B9A653 /* VexFSVolume+Operations.swift */; }; + 7560600D623C3A447E6D6A5D /* Cocoa.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 0CFD2B5341CD367A7E5B577F /* Cocoa.framework */; }; + 77D958E2D11B81318906E1A1 /* VexFSBackend.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7755F957A1B1CD15BE4197E9 /* VexFSBackend.swift */; }; + 95F593B1D1F656649DCEF44F /* VexFSAppEx.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9F0F47620625C8620DE57B91 /* VexFSAppEx.swift */; }; + 98E0DDA029A922158CF93EA6 /* FSKit.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = A5EEC8DE603F197F2CBE0EE9 /* FSKit.framework */; }; + 9FCCFB7E03374E29D1C0836B /* VexFSItem.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1A6D849C6EEFD13A7B598252 /* VexFSItem.swift */; }; + B2893F6D46A0FF824F080C86 /* VexFSFileSystem.swift in Sources */ = {isa = PBXBuildFile; fileRef = DDE6BF41596A81730DBACBEB /* VexFSFileSystem.swift */; }; + B4D586F9CE9676533000CCB8 /* VexFSVolume.swift in Sources */ = {isa = PBXBuildFile; fileRef = B69833294E066A1972FBE013 /* VexFSVolume.swift */; }; + C6035101BDE088E75C65B779 /* Cocoa.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 0CFD2B5341CD367A7E5B577F /* Cocoa.framework */; }; + E3C298958970DDFFA78BF249 /* ContentView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9CE2FF23A36C7BE458FDD3CF /* ContentView.swift */; }; /* End PBXBuildFile section */ /* Begin PBXContainerItemProxy section */ - 27172AE9F45A0ED82BF173F0 /* PBXContainerItemProxy */ = { + D13F97A15279A8E11DE79A2E /* PBXContainerItemProxy */ = { isa = PBXContainerItemProxy; - containerPortal = ED84B2349ED06A80AABDC7EC /* Project object */; + containerPortal = 325223F52457A8234432047D /* Project object */; proxyType = 1; - remoteGlobalIDString = 755E955E75926D9A199A34AE; + remoteGlobalIDString = 45F96B69A7CA797EE1599774; remoteInfo = VexFSAppEx; }; /* End PBXContainerItemProxy section */ /* Begin PBXCopyFilesBuildPhase section */ - AE3E23C090B6C56902DD5920 /* Embed ExtensionKit Extensions */ = { + FFEE7C243A47A3B0D530D694 /* Embed ExtensionKit Extensions */ = { isa = PBXCopyFilesBuildPhase; buildActionMask = 2147483647; dstPath = "$(EXTENSIONS_FOLDER_PATH)"; dstSubfolderSpec = 16; files = ( - 3C4A509CFA9740CED0FA6288 /* VexFSAppEx.appex in Embed ExtensionKit Extensions */, + 2EE6D8298C04D3E5E5C2460F /* VexFSAppEx.appex in Embed ExtensionKit Extensions */, ); name = "Embed ExtensionKit Extensions"; runOnlyForDeploymentPostprocessing = 0; @@ -46,120 +46,120 @@ /* End PBXCopyFilesBuildPhase section */ /* Begin PBXFileReference section */ - 2B548BA0C8FC4FE83BDADFAA /* VexFSApp.entitlements */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.plist.entitlements; path = VexFSApp.entitlements; sourceTree = ""; }; - 2F39B3A14283D310941712A9 /* VexFSFileSystem.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = VexFSFileSystem.swift; sourceTree = ""; }; - 44C93C9A924C2FE4F6908B8B /* VexFSVolume.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = VexFSVolume.swift; sourceTree = ""; }; - 50A67298EECEEE0C501FD3D7 /* VexFS-Bridging-Header.h */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.c.h; path = "VexFS-Bridging-Header.h"; sourceTree = ""; }; - 61EE820F7CF3E4F8A29B0443 /* VexFSVolume+Operations.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = "VexFSVolume+Operations.swift"; sourceTree = ""; }; - 6D28655136F6C3AE749EC51A /* VexFSApp.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = VexFSApp.swift; sourceTree = ""; }; - 7462B11A9FB54192939A31FB /* Cocoa.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = Cocoa.framework; path = Platforms/MacOSX.platform/Developer/SDKs/MacOSX15.0.sdk/System/Library/Frameworks/Cocoa.framework; sourceTree = DEVELOPER_DIR; }; - 7E0E0BBFF61E43527CEFB2D3 /* VexFSItem.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = VexFSItem.swift; sourceTree = ""; }; - 8B6A0DDE2868EC68E6A66B4B /* FSKit.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = FSKit.framework; path = System/Library/Frameworks/FSKit.framework; sourceTree = SDKROOT; }; - 92DD930797D5751B88DD0B4C /* VexFSAppEx.entitlements */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.plist.entitlements; path = VexFSAppEx.entitlements; sourceTree = ""; }; - 955CCBFC4ADDE7F3B2AB6171 /* VexFSApp.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = VexFSApp.app; sourceTree = BUILT_PRODUCTS_DIR; }; - ACD925958EF883345CE46C7D /* ContentView.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = ContentView.swift; sourceTree = ""; }; - ADC6AEF286D1A4ABBB5D64FC /* VexFSBackend.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = VexFSBackend.swift; sourceTree = ""; }; - AE67026F8EE22D6D7FDDF80B /* VexFSAppEx.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = VexFSAppEx.swift; sourceTree = ""; }; - B9426785DC8CB247B8D17175 /* VexFSAppEx.appex */ = {isa = PBXFileReference; explicitFileType = "wrapper.extensionkit-extension"; includeInIndex = 0; path = VexFSAppEx.appex; sourceTree = BUILT_PRODUCTS_DIR; }; - C0B85E0FADEEE8DCCF88E2F2 /* Info.plist */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; + 0CFD2B5341CD367A7E5B577F /* Cocoa.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = Cocoa.framework; path = Platforms/MacOSX.platform/Developer/SDKs/MacOSX15.0.sdk/System/Library/Frameworks/Cocoa.framework; sourceTree = DEVELOPER_DIR; }; + 1A6D849C6EEFD13A7B598252 /* VexFSItem.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = VexFSItem.swift; sourceTree = ""; }; + 1C486E29BAF93B35AFBC796E /* VexFSAppEx.appex */ = {isa = PBXFileReference; explicitFileType = "wrapper.extensionkit-extension"; includeInIndex = 0; path = VexFSAppEx.appex; sourceTree = BUILT_PRODUCTS_DIR; }; + 35C04E17B9594FA63B86D931 /* Info.plist */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; + 6578283192BB1DFFAC0CA85B /* VexFSAppEx.entitlements */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.plist.entitlements; path = VexFSAppEx.entitlements; sourceTree = ""; }; + 66071D46A272BAB8B40E6628 /* VexFSApp.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = VexFSApp.app; sourceTree = BUILT_PRODUCTS_DIR; }; + 73EBA70FC3B95B3888C01E07 /* VexFSApp.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = VexFSApp.swift; sourceTree = ""; }; + 7755F957A1B1CD15BE4197E9 /* VexFSBackend.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = VexFSBackend.swift; sourceTree = ""; }; + 998984630D8E7B97F6BDEFD7 /* VexFS-Bridging-Header.h */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.c.h; path = "VexFS-Bridging-Header.h"; sourceTree = ""; }; + 9CE2FF23A36C7BE458FDD3CF /* ContentView.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = ContentView.swift; sourceTree = ""; }; + 9F0F47620625C8620DE57B91 /* VexFSAppEx.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = VexFSAppEx.swift; sourceTree = ""; }; + A5EEC8DE603F197F2CBE0EE9 /* FSKit.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = FSKit.framework; path = System/Library/Frameworks/FSKit.framework; sourceTree = SDKROOT; }; + A80D16872765D81610B9A653 /* VexFSVolume+Operations.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = "VexFSVolume+Operations.swift"; sourceTree = ""; }; + B69833294E066A1972FBE013 /* VexFSVolume.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = VexFSVolume.swift; sourceTree = ""; }; + DD194AA3764B966088BE5AFB /* VexFSApp.entitlements */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.plist.entitlements; path = VexFSApp.entitlements; sourceTree = ""; }; + DDE6BF41596A81730DBACBEB /* VexFSFileSystem.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = VexFSFileSystem.swift; sourceTree = ""; }; /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ - A44DD1638B2FEDF54E7E6D9A /* Frameworks */ = { + 7541F3BE2913449D1BB71FB3 /* Frameworks */ = { isa = PBXFrameworksBuildPhase; buildActionMask = 2147483647; files = ( - 6C5EE8691215FE2119F96681 /* Cocoa.framework in Frameworks */, + C6035101BDE088E75C65B779 /* Cocoa.framework in Frameworks */, + 98E0DDA029A922158CF93EA6 /* FSKit.framework in Frameworks */, ); runOnlyForDeploymentPostprocessing = 0; }; - FB3B1CE2358B294D0A3F2843 /* Frameworks */ = { + E25AFA851529AA58E6F44A22 /* Frameworks */ = { isa = PBXFrameworksBuildPhase; buildActionMask = 2147483647; files = ( - 45F16BD430300F6F029AE478 /* Cocoa.framework in Frameworks */, - 5E2138B682A54BE885D561E1 /* FSKit.framework in Frameworks */, + 7560600D623C3A447E6D6A5D /* Cocoa.framework in Frameworks */, ); runOnlyForDeploymentPostprocessing = 0; }; /* End PBXFrameworksBuildPhase section */ /* Begin PBXGroup section */ - 5249ACF4B47C0DD1B5FCBCFE /* VexFSApp */ = { + 00D65477AC4B02495692787B /* VexFSApp */ = { isa = PBXGroup; children = ( - 6D28655136F6C3AE749EC51A /* VexFSApp.swift */, - ACD925958EF883345CE46C7D /* ContentView.swift */, - 2B548BA0C8FC4FE83BDADFAA /* VexFSApp.entitlements */, + 73EBA70FC3B95B3888C01E07 /* VexFSApp.swift */, + 9CE2FF23A36C7BE458FDD3CF /* ContentView.swift */, + DD194AA3764B966088BE5AFB /* VexFSApp.entitlements */, ); name = VexFSApp; path = VexFSApp; sourceTree = ""; }; - 9940EB58D403ACAE6F0D63D1 /* VexFSAppEx */ = { + 06B7F04ADB0A548BA2B8A435 /* Products */ = { isa = PBXGroup; children = ( - AE67026F8EE22D6D7FDDF80B /* VexFSAppEx.swift */, - ADC6AEF286D1A4ABBB5D64FC /* VexFSBackend.swift */, - 2F39B3A14283D310941712A9 /* VexFSFileSystem.swift */, - 7E0E0BBFF61E43527CEFB2D3 /* VexFSItem.swift */, - 44C93C9A924C2FE4F6908B8B /* VexFSVolume.swift */, - 61EE820F7CF3E4F8A29B0443 /* VexFSVolume+Operations.swift */, - 50A67298EECEEE0C501FD3D7 /* VexFS-Bridging-Header.h */, - 92DD930797D5751B88DD0B4C /* VexFSAppEx.entitlements */, - C0B85E0FADEEE8DCCF88E2F2 /* Info.plist */, + 66071D46A272BAB8B40E6628 /* VexFSApp.app */, + 1C486E29BAF93B35AFBC796E /* VexFSAppEx.appex */, ); - name = VexFSAppEx; - path = VexFSAppEx; + name = Products; sourceTree = ""; }; - AFB77C8A39FAFE054703C88A /* Products */ = { + 713C285A804373351E655D0C /* VexFSAppEx */ = { isa = PBXGroup; children = ( - 955CCBFC4ADDE7F3B2AB6171 /* VexFSApp.app */, - B9426785DC8CB247B8D17175 /* VexFSAppEx.appex */, + 9F0F47620625C8620DE57B91 /* VexFSAppEx.swift */, + 7755F957A1B1CD15BE4197E9 /* VexFSBackend.swift */, + DDE6BF41596A81730DBACBEB /* VexFSFileSystem.swift */, + 1A6D849C6EEFD13A7B598252 /* VexFSItem.swift */, + B69833294E066A1972FBE013 /* VexFSVolume.swift */, + A80D16872765D81610B9A653 /* VexFSVolume+Operations.swift */, + 998984630D8E7B97F6BDEFD7 /* VexFS-Bridging-Header.h */, + 6578283192BB1DFFAC0CA85B /* VexFSAppEx.entitlements */, + 35C04E17B9594FA63B86D931 /* Info.plist */, ); - name = Products; + name = VexFSAppEx; + path = VexFSAppEx; sourceTree = ""; }; - C33214477C261251221288EE = { + 81CB49E432BA868E3C322A21 = { isa = PBXGroup; children = ( - AFB77C8A39FAFE054703C88A /* Products */, - D07AD92FC8225E0F250C0546 /* Frameworks */, - 5249ACF4B47C0DD1B5FCBCFE /* VexFSApp */, - 9940EB58D403ACAE6F0D63D1 /* VexFSAppEx */, + 06B7F04ADB0A548BA2B8A435 /* Products */, + FAE897193AB397B6F7A4FAA0 /* Frameworks */, + 00D65477AC4B02495692787B /* VexFSApp */, + 713C285A804373351E655D0C /* VexFSAppEx */, ); sourceTree = ""; }; - D07AD92FC8225E0F250C0546 /* Frameworks */ = { + C69A57EBCA9F4671CA6C85D7 /* OS X */ = { isa = PBXGroup; children = ( - FAA5BD3C573B5E15140317D1 /* OS X */, - 8B6A0DDE2868EC68E6A66B4B /* FSKit.framework */, + 0CFD2B5341CD367A7E5B577F /* Cocoa.framework */, ); - name = Frameworks; + name = "OS X"; sourceTree = ""; }; - FAA5BD3C573B5E15140317D1 /* OS X */ = { + FAE897193AB397B6F7A4FAA0 /* Frameworks */ = { isa = PBXGroup; children = ( - 7462B11A9FB54192939A31FB /* Cocoa.framework */, + C69A57EBCA9F4671CA6C85D7 /* OS X */, + A5EEC8DE603F197F2CBE0EE9 /* FSKit.framework */, ); - name = "OS X"; + name = Frameworks; sourceTree = ""; }; /* End PBXGroup section */ /* Begin PBXNativeTarget section */ - 755E955E75926D9A199A34AE /* VexFSAppEx */ = { + 45F96B69A7CA797EE1599774 /* VexFSAppEx */ = { isa = PBXNativeTarget; - buildConfigurationList = 016567E2D1ACC4AB4EFB697A /* Build configuration list for PBXNativeTarget "VexFSAppEx" */; + buildConfigurationList = EF605239B146BE8FFCCED3C8 /* Build configuration list for PBXNativeTarget "VexFSAppEx" */; buildPhases = ( - 5A328AF2F8ACEA4AC8F29B3C /* Build VexFS SQLite Core */, - 8FB42330433A1B49BFEDDE6F /* Sources */, - FB3B1CE2358B294D0A3F2843 /* Frameworks */, - 3028435B675F4CF744C89BC5 /* Resources */, + 01576350A66CB78C2D9A7B63 /* Build VexFS Runtime Core */, + F31DC771BC439880C9127985 /* Sources */, + 7541F3BE2913449D1BB71FB3 /* Frameworks */, + 2BB7DC0912B92CA9E26700AB /* Resources */, ); buildRules = ( ); @@ -167,38 +167,38 @@ ); name = VexFSAppEx; productName = VexFSAppEx; - productReference = B9426785DC8CB247B8D17175 /* VexFSAppEx.appex */; + productReference = 1C486E29BAF93B35AFBC796E /* VexFSAppEx.appex */; productType = "com.apple.product-type.extensionkit-extension"; }; - BAD729588513F880EAA2F59F /* VexFSApp */ = { + F396799EA2CA285284523B58 /* VexFSApp */ = { isa = PBXNativeTarget; - buildConfigurationList = 6581806B5669A7CF0E93B968 /* Build configuration list for PBXNativeTarget "VexFSApp" */; + buildConfigurationList = 5996080E60AD3B987B22BA1B /* Build configuration list for PBXNativeTarget "VexFSApp" */; buildPhases = ( - E1E929D517CB3B3CFC12756C /* Sources */, - A44DD1638B2FEDF54E7E6D9A /* Frameworks */, - 456B33A1846ED212B12332AD /* Resources */, - AE3E23C090B6C56902DD5920 /* Embed ExtensionKit Extensions */, + 254241D5DE8DCA49534AD681 /* Sources */, + E25AFA851529AA58E6F44A22 /* Frameworks */, + F48CEE901D9DC9686C3A637B /* Resources */, + FFEE7C243A47A3B0D530D694 /* Embed ExtensionKit Extensions */, ); buildRules = ( ); dependencies = ( - 8A050C6C471C700BA18BBEF1 /* PBXTargetDependency */, + DB5430C221112C9F1E0E2DE7 /* PBXTargetDependency */, ); name = VexFSApp; productName = VexFSApp; - productReference = 955CCBFC4ADDE7F3B2AB6171 /* VexFSApp.app */; + productReference = 66071D46A272BAB8B40E6628 /* VexFSApp.app */; productType = "com.apple.product-type.application"; }; /* End PBXNativeTarget section */ /* Begin PBXProject section */ - ED84B2349ED06A80AABDC7EC /* Project object */ = { + 325223F52457A8234432047D /* Project object */ = { isa = PBXProject; attributes = { LastSwiftUpdateCheck = 2630; LastUpgradeCheck = 2630; }; - buildConfigurationList = 759F70EA6018D5E6B193E205 /* Build configuration list for PBXProject "VexFS" */; + buildConfigurationList = 0A1337F6EA75BD3565BD75E4 /* Build configuration list for PBXProject "VexFS" */; compatibilityVersion = "Xcode 3.2"; developmentRegion = en; hasScannedForEncodings = 0; @@ -206,28 +206,28 @@ en, Base, ); - mainGroup = C33214477C261251221288EE; + mainGroup = 81CB49E432BA868E3C322A21; minimizedProjectReferenceProxies = 0; preferredProjectObjectVersion = 77; - productRefGroup = AFB77C8A39FAFE054703C88A /* Products */; + productRefGroup = 06B7F04ADB0A548BA2B8A435 /* Products */; projectDirPath = ""; projectRoot = ""; targets = ( - BAD729588513F880EAA2F59F /* VexFSApp */, - 755E955E75926D9A199A34AE /* VexFSAppEx */, + F396799EA2CA285284523B58 /* VexFSApp */, + 45F96B69A7CA797EE1599774 /* VexFSAppEx */, ); }; /* End PBXProject section */ /* Begin PBXResourcesBuildPhase section */ - 3028435B675F4CF744C89BC5 /* Resources */ = { + 2BB7DC0912B92CA9E26700AB /* Resources */ = { isa = PBXResourcesBuildPhase; buildActionMask = 2147483647; files = ( ); runOnlyForDeploymentPostprocessing = 0; }; - 456B33A1846ED212B12332AD /* Resources */ = { + F48CEE901D9DC9686C3A637B /* Resources */ = { isa = PBXResourcesBuildPhase; buildActionMask = 2147483647; files = ( @@ -237,7 +237,7 @@ /* End PBXResourcesBuildPhase section */ /* Begin PBXShellScriptBuildPhase section */ - 5A328AF2F8ACEA4AC8F29B3C /* Build VexFS SQLite Core */ = { + 01576350A66CB78C2D9A7B63 /* Build VexFS Runtime Core */ = { isa = PBXShellScriptBuildPhase; alwaysOutOfDate = 1; buildActionMask = 2147483647; @@ -247,53 +247,120 @@ ); inputPaths = ( ); - name = "Build VexFS SQLite Core"; + name = "Build VexFS Runtime Core"; outputFileListPaths = ( ); outputPaths = ( ); runOnlyForDeploymentPostprocessing = 0; shellPath = /bin/sh; - shellScript = "set -e\nCMAKE_BIN=\"$(command -v cmake || true)\"\n[ -x \"$CMAKE_BIN\" ] || CMAKE_BIN=/opt/homebrew/bin/cmake\n[ -x \"$CMAKE_BIN\" ] || CMAKE_BIN=/usr/local/bin/cmake\n[ -x \"$CMAKE_BIN\" ] || { echo \"cmake is required\" >&2; exit 1; }\nBUILD_DIR=\"$SRCROOT/../../vexdb_sqlite/build-fskit\"\nBUILD_JOBS=\"${VEXDB_LITE_BUILD_JOBS:-4}\"\ncase \"$BUILD_JOBS\" in *[!0-9]*|0) echo \"VEXDB_LITE_BUILD_JOBS must be a positive integer\" >&2; exit 2;; esac\n[ \"$BUILD_JOBS\" -le 4 ] || BUILD_JOBS=4\n\"$CMAKE_BIN\" -S \"$SRCROOT/../../vexdb_sqlite\" -B \"$BUILD_DIR\" \\\n -DCMAKE_BUILD_TYPE=\"$CONFIGURATION\" -DVEXDB_SQLITE_BUILD_TESTS=OFF\n\"$CMAKE_BIN\" --build \"$BUILD_DIR\" --target vexfs_runtime -j \"$BUILD_JOBS\"\n"; + shellScript = "set -e\nCMAKE_BIN=\"$(command -v cmake || true)\"\n[ -x \"$CMAKE_BIN\" ] || CMAKE_BIN=/opt/homebrew/bin/cmake\n[ -x \"$CMAKE_BIN\" ] || CMAKE_BIN=/usr/local/bin/cmake\n[ -x \"$CMAKE_BIN\" ] || { echo \"cmake is required\" >&2; exit 1; }\n# Run-script phases are not per-architecture, so CURRENT_ARCH is often\n# \"undefined_arch\". The packaging flow intentionally builds one architecture\n# at a time and passes ARCHS explicitly.\nBUILD_ARCH=\"${ARCHS:-}\"\ncase \"$BUILD_ARCH\" in\n arm64|x86_64) ;;\n *) echo \"unsupported FSKit build architecture: $BUILD_ARCH\" >&2; exit 2 ;;\nesac\nBUILD_DIR=\"$SRCROOT/../../vexdb_sqlite/build-fskit-$BUILD_ARCH\"\nBUILD_JOBS=\"${VEXDB_LITE_BUILD_JOBS:-4}\"\ncase \"$BUILD_JOBS\" in *[!0-9]*|0) echo \"VEXDB_LITE_BUILD_JOBS must be a positive integer\" >&2; exit 2;; esac\n[ \"$BUILD_JOBS\" -le 4 ] || BUILD_JOBS=4\n\"$CMAKE_BIN\" -S \"$SRCROOT/../../vexdb_sqlite\" -B \"$BUILD_DIR\" \\\n -DCMAKE_BUILD_TYPE=\"$CONFIGURATION\" \\\n -DCMAKE_OSX_ARCHITECTURES=\"$BUILD_ARCH\" \\\n -DVEXFS_REQUIRE_POSTGRESQL=ON \\\n -DVEXDB_SQLITE_BUILD_TESTS=OFF\n\"$CMAKE_BIN\" --build \"$BUILD_DIR\" --target vexfs_runtime -j \"$BUILD_JOBS\"\n"; }; /* End PBXShellScriptBuildPhase section */ /* Begin PBXSourcesBuildPhase section */ - 8FB42330433A1B49BFEDDE6F /* Sources */ = { + 254241D5DE8DCA49534AD681 /* Sources */ = { isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; files = ( - B5409F3CA8A4250DE5EC350D /* VexFSAppEx.swift in Sources */, - FC4015CAE58C58A33AB7D39B /* VexFSBackend.swift in Sources */, - 1C608A9018A4422EA1952C9C /* VexFSFileSystem.swift in Sources */, - 3780F8276301D1A2E0BFA83A /* VexFSItem.swift in Sources */, - 1EA761C45020F997EB84C888 /* VexFSVolume.swift in Sources */, - EFE51E86624B1AAEE193E04A /* VexFSVolume+Operations.swift in Sources */, + 5A5203C7F7F89B325F9FA2CC /* VexFSApp.swift in Sources */, + E3C298958970DDFFA78BF249 /* ContentView.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; - E1E929D517CB3B3CFC12756C /* Sources */ = { + F31DC771BC439880C9127985 /* Sources */ = { isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; files = ( - 94CD0A2563F9AFB2FD7F6D7F /* VexFSApp.swift in Sources */, - 42FDB23ACE4332A35F1EC106 /* ContentView.swift in Sources */, + 95F593B1D1F656649DCEF44F /* VexFSAppEx.swift in Sources */, + 77D958E2D11B81318906E1A1 /* VexFSBackend.swift in Sources */, + B2893F6D46A0FF824F080C86 /* VexFSFileSystem.swift in Sources */, + 9FCCFB7E03374E29D1C0836B /* VexFSItem.swift in Sources */, + B4D586F9CE9676533000CCB8 /* VexFSVolume.swift in Sources */, + 664CB1C505108619FDE55161 /* VexFSVolume+Operations.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; /* End PBXSourcesBuildPhase section */ /* Begin PBXTargetDependency section */ - 8A050C6C471C700BA18BBEF1 /* PBXTargetDependency */ = { + DB5430C221112C9F1E0E2DE7 /* PBXTargetDependency */ = { isa = PBXTargetDependency; name = VexFSAppEx; - target = 755E955E75926D9A199A34AE /* VexFSAppEx */; - targetProxy = 27172AE9F45A0ED82BF173F0 /* PBXContainerItemProxy */; + target = 45F96B69A7CA797EE1599774 /* VexFSAppEx */; + targetProxy = D13F97A15279A8E11DE79A2E /* PBXContainerItemProxy */; }; /* End PBXTargetDependency section */ /* Begin XCBuildConfiguration section */ - 00F1949ED702892DE84D34BC /* Debug */ = { + 45907C82AAF8C8918ECCF570 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + APPLICATION_EXTENSION_API_ONLY = YES; + CODE_SIGN_ENTITLEMENTS = VexFSAppEx/VexFSAppEx.entitlements; + CODE_SIGN_STYLE = Automatic; + CURRENT_PROJECT_VERSION = 1; + DEVELOPMENT_TEAM = BB5VK42K87; + ENABLE_APP_SANDBOX = YES; + ENABLE_HARDENED_RUNTIME = YES; + ENABLE_USER_SCRIPT_SANDBOXING = NO; + GENERATE_INFOPLIST_FILE = YES; + HEADER_SEARCH_PATHS = ( + "$(inherited)", + "\"$(SRCROOT)/../mount/common/include\"", + ); + INFOPLIST_FILE = VexFSAppEx/Info.plist; + INFOPLIST_KEY_CFBundleDisplayName = "VexFS file system"; + INFOPLIST_KEY_NSLocalNetworkUsageDescription = "VexDB Lite needs to connect to PostgreSQL on the local network for shared VexFS workspaces."; + LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/../Frameworks @executable_path/../../../../Frameworks"; + LIBRARY_SEARCH_PATHS = ( + "$(inherited)", + "\"$(SRCROOT)/../../vexdb_sqlite/build-fskit-$(ARCHS)/pg-client/lib\"", + ); + MACOSX_DEPLOYMENT_TARGET = 26.0; + MARKETING_VERSION = 0.1.0; + OTHER_LDFLAGS = ( + "$(inherited)", + "\"$(SRCROOT)/../../vexdb_sqlite/build-fskit-$(ARCHS)/libvexfs_runtime.a\"", + "\"$(SRCROOT)/../../vexdb_sqlite/build-fskit-$(ARCHS)/libvexdb_lite_static.a\"", + "-lsqlite3", + "-lpq", + "-lc++", + ); + PRODUCT_BUNDLE_IDENTIFIER = io.vexdb.vexfs.extension; + PRODUCT_NAME = VexFSAppEx; + SDKROOT = macosx; + SKIP_INSTALL = YES; + SWIFT_OBJC_BRIDGING_HEADER = "VexFSAppEx/VexFS-Bridging-Header.h"; + }; + name = Release; + }; + 92A51B341EF5DE5B723560A1 /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; + CODE_SIGN_ENTITLEMENTS = VexFSApp/VexFSApp.entitlements; + CODE_SIGN_STYLE = Automatic; + COMBINE_HIDPI_IMAGES = YES; + CURRENT_PROJECT_VERSION = 1; + DEVELOPMENT_TEAM = BB5VK42K87; + ENABLE_APP_SANDBOX = YES; + ENABLE_HARDENED_RUNTIME = YES; + GENERATE_INFOPLIST_FILE = YES; + INFOPLIST_FILE = VexFSApp/Info.plist; + INFOPLIST_KEY_CFBundleDisplayName = "VexDB Lite"; + INFOPLIST_KEY_NSLocalNetworkUsageDescription = "VexDB Lite needs to connect to PostgreSQL on the local network for shared VexFS workspaces."; + LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/../Frameworks"; + MACOSX_DEPLOYMENT_TARGET = 26.0; + MARKETING_VERSION = 0.1.0; + PRODUCT_BUNDLE_IDENTIFIER = io.vexdb.vexfs; + PRODUCT_NAME = "VexDB Lite"; + SDKROOT = macosx; + }; + name = Debug; + }; + 9AE13EB7D79BCCA8CA2557ED /* Debug */ = { isa = XCBuildConfiguration; buildSettings = { ALWAYS_SEARCH_USER_PATHS = NO; @@ -356,7 +423,7 @@ }; name = Debug; }; - 2B05C13542B6725C575D64A9 /* Release */ = { + B68EEDB5CA77CBE42EC60919 /* Release */ = { isa = XCBuildConfiguration; buildSettings = { ALWAYS_SEARCH_USER_PATHS = NO; @@ -412,7 +479,7 @@ }; name = Release; }; - 3D1A46E7C3D9E7486EA7515F /* Release */ = { + EDF353F74962E980BAD41E82 /* Debug */ = { isa = XCBuildConfiguration; buildSettings = { APPLICATION_EXTENSION_API_ONLY = YES; @@ -430,14 +497,20 @@ ); INFOPLIST_FILE = VexFSAppEx/Info.plist; INFOPLIST_KEY_CFBundleDisplayName = "VexFS file system"; + INFOPLIST_KEY_NSLocalNetworkUsageDescription = "VexDB Lite needs to connect to PostgreSQL on the local network for shared VexFS workspaces."; LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/../Frameworks @executable_path/../../../../Frameworks"; + LIBRARY_SEARCH_PATHS = ( + "$(inherited)", + "\"$(SRCROOT)/../../vexdb_sqlite/build-fskit-$(ARCHS)/pg-client/lib\"", + ); MACOSX_DEPLOYMENT_TARGET = 26.0; MARKETING_VERSION = 0.1.0; OTHER_LDFLAGS = ( "$(inherited)", - "\"$(SRCROOT)/../../vexdb_sqlite/build-fskit/libvexfs_runtime.a\"", - "\"$(SRCROOT)/../../vexdb_sqlite/build-fskit/libvexdb_lite_static.a\"", + "\"$(SRCROOT)/../../vexdb_sqlite/build-fskit-$(ARCHS)/libvexfs_runtime.a\"", + "\"$(SRCROOT)/../../vexdb_sqlite/build-fskit-$(ARCHS)/libvexdb_lite_static.a\"", "-lsqlite3", + "-lpq", "-lc++", ); PRODUCT_BUNDLE_IDENTIFIER = io.vexdb.vexfs.extension; @@ -446,9 +519,9 @@ SKIP_INSTALL = YES; SWIFT_OBJC_BRIDGING_HEADER = "VexFSAppEx/VexFS-Bridging-Header.h"; }; - name = Release; + name = Debug; }; - 4C1189A9B0E0C89A11F8F389 /* Release */ = { + F1EBB61E4B874C6AC79814CE /* Release */ = { isa = XCBuildConfiguration; buildSettings = { ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; @@ -461,7 +534,9 @@ ENABLE_APP_SANDBOX = YES; ENABLE_HARDENED_RUNTIME = YES; GENERATE_INFOPLIST_FILE = YES; + INFOPLIST_FILE = VexFSApp/Info.plist; INFOPLIST_KEY_CFBundleDisplayName = "VexDB Lite"; + INFOPLIST_KEY_NSLocalNetworkUsageDescription = "VexDB Lite needs to connect to PostgreSQL on the local network for shared VexFS workspaces."; LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/../Frameworks"; MACOSX_DEPLOYMENT_TARGET = 26.0; MARKETING_VERSION = 0.1.0; @@ -471,96 +546,37 @@ }; name = Release; }; - 5B6CE2F7E21FBD659FA20DBF /* Debug */ = { - isa = XCBuildConfiguration; - buildSettings = { - ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; - ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; - CODE_SIGN_ENTITLEMENTS = VexFSApp/VexFSApp.entitlements; - CODE_SIGN_STYLE = Automatic; - COMBINE_HIDPI_IMAGES = YES; - CURRENT_PROJECT_VERSION = 1; - DEVELOPMENT_TEAM = BB5VK42K87; - ENABLE_APP_SANDBOX = YES; - ENABLE_HARDENED_RUNTIME = YES; - GENERATE_INFOPLIST_FILE = YES; - INFOPLIST_KEY_CFBundleDisplayName = "VexDB Lite"; - LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/../Frameworks"; - MACOSX_DEPLOYMENT_TARGET = 26.0; - MARKETING_VERSION = 0.1.0; - PRODUCT_BUNDLE_IDENTIFIER = io.vexdb.vexfs; - PRODUCT_NAME = "VexDB Lite"; - SDKROOT = macosx; - }; - name = Debug; - }; - D4767CC85F241F3F0711CBC2 /* Debug */ = { - isa = XCBuildConfiguration; - buildSettings = { - APPLICATION_EXTENSION_API_ONLY = YES; - CODE_SIGN_ENTITLEMENTS = VexFSAppEx/VexFSAppEx.entitlements; - CODE_SIGN_STYLE = Automatic; - CURRENT_PROJECT_VERSION = 1; - DEVELOPMENT_TEAM = BB5VK42K87; - ENABLE_APP_SANDBOX = YES; - ENABLE_HARDENED_RUNTIME = YES; - ENABLE_USER_SCRIPT_SANDBOXING = NO; - GENERATE_INFOPLIST_FILE = YES; - HEADER_SEARCH_PATHS = ( - "$(inherited)", - "\"$(SRCROOT)/../mount/common/include\"", - ); - INFOPLIST_FILE = VexFSAppEx/Info.plist; - INFOPLIST_KEY_CFBundleDisplayName = "VexFS file system"; - LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/../Frameworks @executable_path/../../../../Frameworks"; - MACOSX_DEPLOYMENT_TARGET = 26.0; - MARKETING_VERSION = 0.1.0; - OTHER_LDFLAGS = ( - "$(inherited)", - "\"$(SRCROOT)/../../vexdb_sqlite/build-fskit/libvexfs_runtime.a\"", - "\"$(SRCROOT)/../../vexdb_sqlite/build-fskit/libvexdb_lite_static.a\"", - "-lsqlite3", - "-lc++", - ); - PRODUCT_BUNDLE_IDENTIFIER = io.vexdb.vexfs.extension; - PRODUCT_NAME = VexFSAppEx; - SDKROOT = macosx; - SKIP_INSTALL = YES; - SWIFT_OBJC_BRIDGING_HEADER = "VexFSAppEx/VexFS-Bridging-Header.h"; - }; - name = Debug; - }; /* End XCBuildConfiguration section */ /* Begin XCConfigurationList section */ - 016567E2D1ACC4AB4EFB697A /* Build configuration list for PBXNativeTarget "VexFSAppEx" */ = { + 0A1337F6EA75BD3565BD75E4 /* Build configuration list for PBXProject "VexFS" */ = { isa = XCConfigurationList; buildConfigurations = ( - 3D1A46E7C3D9E7486EA7515F /* Release */, - D4767CC85F241F3F0711CBC2 /* Debug */, + 9AE13EB7D79BCCA8CA2557ED /* Debug */, + B68EEDB5CA77CBE42EC60919 /* Release */, ); defaultConfigurationIsVisible = 0; defaultConfigurationName = Release; }; - 6581806B5669A7CF0E93B968 /* Build configuration list for PBXNativeTarget "VexFSApp" */ = { + 5996080E60AD3B987B22BA1B /* Build configuration list for PBXNativeTarget "VexFSApp" */ = { isa = XCConfigurationList; buildConfigurations = ( - 4C1189A9B0E0C89A11F8F389 /* Release */, - 5B6CE2F7E21FBD659FA20DBF /* Debug */, + F1EBB61E4B874C6AC79814CE /* Release */, + 92A51B341EF5DE5B723560A1 /* Debug */, ); defaultConfigurationIsVisible = 0; defaultConfigurationName = Release; }; - 759F70EA6018D5E6B193E205 /* Build configuration list for PBXProject "VexFS" */ = { + EF605239B146BE8FFCCED3C8 /* Build configuration list for PBXNativeTarget "VexFSAppEx" */ = { isa = XCConfigurationList; buildConfigurations = ( - 00F1949ED702892DE84D34BC /* Debug */, - 2B05C13542B6725C575D64A9 /* Release */, + 45907C82AAF8C8918ECCF570 /* Release */, + EDF353F74962E980BAD41E82 /* Debug */, ); defaultConfigurationIsVisible = 0; defaultConfigurationName = Release; }; /* End XCConfigurationList section */ }; - rootObject = ED84B2349ED06A80AABDC7EC /* Project object */; + rootObject = 325223F52457A8234432047D /* Project object */; } diff --git a/agent_files/macos/VexFSApp/ContentView.swift b/agent_files/macos/VexFSApp/ContentView.swift index f3323c731b..09a054bf99 100644 --- a/agent_files/macos/VexFSApp/ContentView.swift +++ b/agent_files/macos/VexFSApp/ContentView.swift @@ -1,6 +1,193 @@ import SwiftUI +import Network +import OSLog +import AppKit +import Darwin + +private struct LocalNetworkProbeRequest: Codable { + let version: Int + let nonce: String + let host: String + let port: Int +} + +private struct LocalNetworkProbeResponse: Codable { + let version: Int + let nonce: String + let allowed: Bool + let detail: String +} + +final class LocalNetworkAccess: ObservableObject { + @Published private(set) var status = "尚未检查局域网权限" + private var browser: NWBrowser? + private var probeConnection: NWConnection? + private var probeNonce: String? + private var probeDirectorySource: DispatchSourceFileSystemObject? + private let logger = Logger(subsystem: "io.vexdb.vexfs", category: "LocalNetwork") + + private var supportDirectory: URL { + FileManager.default.homeDirectoryForCurrentUser + .appendingPathComponent("Library/Application Support/VexDB-Lite", isDirectory: true) + } + + private var probeRequestURL: URL { + supportDirectory.appendingPathComponent("local-network-probe.json") + } + + private var probeResponseURL: URL { + supportDirectory.appendingPathComponent("local-network-probe-response.json") + } + + init() { + do { + try FileManager.default.createDirectory( + at: supportDirectory, withIntermediateDirectories: true, + attributes: [.posixPermissions: 0o700]) + let descriptor = open(supportDirectory.path, O_EVTONLY | O_CLOEXEC) + if descriptor >= 0 { + let source = DispatchSource.makeFileSystemObjectSource( + fileDescriptor: descriptor, + eventMask: [.write, .extend, .attrib, .rename], + queue: .main) + source.setEventHandler { [weak self] in + self?.processPendingProbe() + } + source.setCancelHandler { + close(descriptor) + } + probeDirectorySource = source + source.resume() + } + } catch { + logger.error("cannot watch local network requests: \(error.localizedDescription, privacy: .public)") + } + DispatchQueue.main.async { [weak self] in + self?.processPendingProbe() + } + } + + private func updateStatus(_ value: String) { + status = value + UserDefaults.standard.set(value, forKey: "localNetworkStatus") + logger.notice("local network status: \(value, privacy: .public)") + } + + func request() { + browser?.cancel() + updateStatus("正在请求局域网权限…") + let parameters = NWParameters() + parameters.includePeerToPeer = true + let browser = NWBrowser( + for: .bonjour(type: "_vexdb._tcp", domain: nil), using: parameters) + self.browser = browser + browser.stateUpdateHandler = { [weak self] state in + DispatchQueue.main.async { + guard let self else { return } + switch state { + case .ready: + self.updateStatus("局域网检查已就绪，请确认系统设置中的 VexDB Lite 已打开") + case .failed(let error): + self.updateStatus("局域网权限不可用：\(error.localizedDescription)") + case .waiting(let error): + self.updateStatus("等待局域网权限：\(error.localizedDescription)") + case .cancelled: + break + default: + self.updateStatus("正在请求局域网权限…") + } + } + } + browser.start(queue: DispatchQueue(label: "io.vexdb.local-network-access")) + } + + private func finishProbe(_ request: LocalNetworkProbeRequest, allowed: Bool, + detail: String) { + guard probeNonce == request.nonce else { return } + probeConnection?.cancel() + probeConnection = nil + probeNonce = nil + let response = LocalNetworkProbeResponse( + version: 1, nonce: request.nonce, allowed: allowed, detail: detail) + do { + try FileManager.default.createDirectory( + at: supportDirectory, withIntermediateDirectories: true, + attributes: [.posixPermissions: 0o700]) + let data = try JSONEncoder().encode(response) + try data.write(to: probeResponseURL, options: .atomic) + try FileManager.default.setAttributes( + [.posixPermissions: 0o600], ofItemAtPath: probeResponseURL.path) + } catch { + logger.error("cannot write local network response: \(error.localizedDescription, privacy: .public)") + } + updateStatus(allowed + ? "局域网连接已允许：\(request.host):\(request.port)" + : "局域网连接不可用：\(detail)") + } + + func processPendingProbe() { + guard probeNonce == nil, + FileManager.default.fileExists(atPath: probeRequestURL.path) else { return } + do { + let attributes = try FileManager.default.attributesOfItem(atPath: probeRequestURL.path) + guard let permissions = attributes[.posixPermissions] as? NSNumber, + permissions.intValue & 0o077 == 0, + let owner = attributes[.ownerAccountID] as? NSNumber, + owner.uint32Value == getuid() else { + try? FileManager.default.removeItem(at: probeRequestURL) + updateStatus("局域网检查请求的文件权限不安全") + return + } + let data = try Data(contentsOf: probeRequestURL) + let request = try JSONDecoder().decode(LocalNetworkProbeRequest.self, from: data) + try? FileManager.default.removeItem(at: probeRequestURL) + guard request.version == 1, !request.nonce.isEmpty, + !request.host.isEmpty, request.host.count <= 253, + (1...65535).contains(request.port), + let port = NWEndpoint.Port(rawValue: UInt16(request.port)) else { + updateStatus("局域网检查请求无效") + return + } + + probeNonce = request.nonce + updateStatus("正在连接 \(request.host):\(request.port)，请在系统提示中点“允许”…") + let connection = NWConnection(host: NWEndpoint.Host(request.host), port: port, using: .tcp) + probeConnection = connection + connection.stateUpdateHandler = { [weak self] state in + DispatchQueue.main.async { + guard let self, self.probeNonce == request.nonce else { return } + switch state { + case .ready: + self.finishProbe(request, allowed: true, detail: "connected") + case .failed(let error): + self.finishProbe(request, allowed: false, + detail: error.localizedDescription) + case .waiting(let error): + self.updateStatus("等待局域网权限：\(error.localizedDescription)") + case .cancelled: + break + default: + break + } + } + } + connection.start(queue: DispatchQueue(label: "io.vexdb.local-network-probe")) + DispatchQueue.main.asyncAfter(deadline: .now() + 30) { [weak self] in + guard let self, self.probeNonce == request.nonce else { return } + self.finishProbe(request, allowed: false, + detail: "等待系统授权超时，请在本地网络设置中允许 VexDB Lite") + } + } catch { + try? FileManager.default.removeItem(at: probeRequestURL) + updateStatus("局域网检查请求失败：\(error.localizedDescription)") + } + } +} struct ContentView: View { + @StateObject private var localNetwork = LocalNetworkAccess() + private let probeTimer = Timer.publish(every: 0.5, on: .main, in: .common).autoconnect() + private var databasePath: String { FileManager.default.homeDirectoryForCurrentUser .appendingPathComponent("Library/Application Support/VexDB-Lite/default.sqlite3").path @@ -21,6 +208,28 @@ struct ContentView: View { .frame(maxWidth: .infinity, alignment: .leading) .padding(4) } + GroupBox("跨电脑 PostgreSQL 工作区") { + HStack(spacing: 12) { + VStack(alignment: .leading, spacing: 4) { + Text(localNetwork.status) + Text("只有连接局域网中的 PostgreSQL 时需要此权限；本机 SQLite 不需要。") + .font(.footnote) + .foregroundStyle(.secondary) + } + Spacer() + Button("检查权限") { + localNetwork.request() + } + Button("打开系统设置") { + if let url = URL(string: + "x-apple.systempreferences:com.apple.preference.security?Privacy_LocalNetwork") { + NSWorkspace.shared.open(url) + } + } + } + .frame(maxWidth: .infinity, alignment: .leading) + .padding(4) + } Text("默认数据库：\(databasePath)") .font(.system(.footnote, design: .monospaced)) .textSelection(.enabled) @@ -31,5 +240,12 @@ struct ContentView: View { Spacer() } .padding(28) + .onAppear { + localNetwork.request() + localNetwork.processPendingProbe() + } + .onReceive(probeTimer) { _ in + localNetwork.processPendingProbe() + } } } diff --git a/agent_files/macos/VexFSApp/Info.plist b/agent_files/macos/VexFSApp/Info.plist new file mode 100644 index 0000000000..c880c3569e --- /dev/null +++ b/agent_files/macos/VexFSApp/Info.plist @@ -0,0 +1,12 @@ + + + + + NSLocalNetworkUsageDescription + VexDB Lite 需要连接局域网中的 PostgreSQL，挂载共享的 VexFS 工作区。 + NSBonjourServices + + _vexdb._tcp + + + diff --git a/agent_files/macos/VexFSApp/VexFSApp.entitlements b/agent_files/macos/VexFSApp/VexFSApp.entitlements index 13cb114cf8..ee95ab7e58 100644 --- a/agent_files/macos/VexFSApp/VexFSApp.entitlements +++ b/agent_files/macos/VexFSApp/VexFSApp.entitlements @@ -2,7 +2,9 @@ - com.apple.security.app-sandbox - + com.apple.security.app-sandbox + + com.apple.security.network.client + diff --git a/agent_files/macos/VexFSAppEx/Info.plist b/agent_files/macos/VexFSAppEx/Info.plist index df80a37d5c..ee185d69d9 100644 --- a/agent_files/macos/VexFSAppEx/Info.plist +++ b/agent_files/macos/VexFSAppEx/Info.plist @@ -2,6 +2,8 @@ + NSLocalNetworkUsageDescription + VexDB Lite 需要连接局域网中的 PostgreSQL，挂载共享的 VexFS 工作区。 EXAppExtensionAttributes EXExtensionPointIdentifier diff --git a/agent_files/macos/VexFSAppEx/VexFSAppEx.entitlements b/agent_files/macos/VexFSAppEx/VexFSAppEx.entitlements index 40a67e20d7..911b22d949 100644 --- a/agent_files/macos/VexFSAppEx/VexFSAppEx.entitlements +++ b/agent_files/macos/VexFSAppEx/VexFSAppEx.entitlements @@ -6,5 +6,7 @@ com.apple.developer.fskit.fsmodule + com.apple.security.network.client + diff --git a/agent_files/macos/VexFSAppEx/VexFSBackend.swift b/agent_files/macos/VexFSAppEx/VexFSBackend.swift index 537890ae4f..6049070a87 100644 --- a/agent_files/macos/VexFSAppEx/VexFSBackend.swift +++ b/agent_files/macos/VexFSAppEx/VexFSBackend.swift @@ -1,8 +1,10 @@ import Foundation +import OSLog struct VexFSDescriptor: Codable { let version: Int - let database_file: String + let backend: String + let connection: String let workspace: String } @@ -83,11 +85,13 @@ struct VexFSDirectoryListing: Codable { final class VexFSBackend { private var session: OpaquePointer? - private let databasePath: String + private let backend: String + private let connection: String private let workspace: String - init(databasePath: String, workspace: String) throws { - self.databasePath = databasePath + init(backend: String, connection: String, workspace: String) throws { + self.backend = backend + self.connection = connection self.workspace = workspace try reopen() } @@ -100,14 +104,15 @@ final class VexFSBackend { config.flags = UInt32(VEXFS_RUNTIME_EXCLUSIVE_GATEWAY) var opened: OpaquePointer? var error = vexfs_mount_error() - let status = "sqlite".withCString { backendPointer in - databasePath.withCString { databasePointer in + let principal = backend == "sqlite" ? "local" : "" + let status = backend.withCString { backendPointer in + connection.withCString { connectionPointer in workspace.withCString { workspacePointer in - "local".withCString { principalPointer in + principal.withCString { principalPointer in config.backend = backendPointer - config.connection = databasePointer + config.connection = connectionPointer config.workspace = workspacePointer - config.principal = principalPointer + config.principal = backend == "sqlite" ? principalPointer : nil return vexfs_mount_session_open(&config, &opened, &error) } } @@ -371,6 +376,16 @@ final class VexFSBackend { guard let value = String(data: Self.take(&output), encoding: .utf8) else { throw POSIXError(.EIO) } + do { + // handle_create prepares an empty private generation. POSIX create + // must make the directory entry visible before FSKit asks for the + // returned item's attributes, so publish generation 1 now and keep + // the same handle open for a later write. + _ = try publish(handle: value, generation: 1) + } catch { + try? closeHandle(value, retain: true) + throw error + } return value } @@ -501,6 +516,15 @@ final class VexFSBackend { private static func check(_ status: vexfs_mount_status, error: inout vexfs_mount_error) throws { guard status != VEXFS_MOUNT_OK else { return } + let detail = withUnsafePointer(to: &error.message) { pointer in + pointer.withMemoryRebound(to: CChar.self, capacity: 512) { + String(cString: $0) + } + } + if !detail.isEmpty { + Logger.vexfs.error( + "runtime rejected operation: \(detail, privacy: .public)") + } let code: POSIXError.Code switch status { case VEXFS_MOUNT_NOT_FOUND: code = .ENOENT diff --git a/agent_files/macos/VexFSAppEx/VexFSFileSystem.swift b/agent_files/macos/VexFSAppEx/VexFSFileSystem.swift index f44f5e9d60..968726cd2e 100644 --- a/agent_files/macos/VexFSAppEx/VexFSFileSystem.swift +++ b/agent_files/macos/VexFSAppEx/VexFSFileSystem.swift @@ -8,9 +8,9 @@ extension Logger { } enum VexFSIdentity { - static func uuid(databasePath: String, workspace: String) -> UUID { - let normalized = URL(fileURLWithPath: databasePath).standardizedFileURL.path - let digest = SHA256.hash(data: Data("\(normalized)\u{0}\(workspace)".utf8)) + static func uuid(backend: String, connection: String, workspace: String) -> UUID { + let digest = SHA256.hash( + data: Data("\(backend)\u{0}\(connection)\u{0}\(workspace)".utf8)) var bytes = Array(digest.prefix(16)) bytes[6] = (bytes[6] & 0x0f) | 0x50 bytes[8] = (bytes[8] & 0x3f) | 0x80 @@ -28,8 +28,13 @@ final class VexFSFileSystem: FSUnaryFileSystem & FSUnaryFileSystemOperations { private var resource: FSPathURLResource? private var volume: VexFSVolume? + private struct Configuration { + let descriptor: VexFSDescriptor + let connection: String + } + private func configuration(for resource: FSPathURLResource) throws - -> (descriptor: VexFSDescriptor, databaseURL: URL) { + -> Configuration { let root = resource.url.standardizedFileURL var isDirectory: ObjCBool = false guard FileManager.default.fileExists(atPath: root.path, isDirectory: &isDirectory), @@ -39,20 +44,26 @@ final class VexFSFileSystem: FSUnaryFileSystem & FSUnaryFileSystemOperations { let descriptorURL = root.appendingPathComponent(Self.descriptorName, isDirectory: false) let descriptor = try JSONDecoder().decode( VexFSDescriptor.self, from: Data(contentsOf: descriptorURL)) - guard descriptor.version == 2, - !descriptor.database_file.isEmpty, - descriptor.database_file != ".", - descriptor.database_file != "..", - URL(fileURLWithPath: descriptor.database_file).lastPathComponent == - descriptor.database_file else { + guard descriptor.version == 3, + descriptor.backend == "sqlite" || descriptor.backend == "postgresql", + !descriptor.connection.isEmpty, + !descriptor.workspace.isEmpty else { throw POSIXError(.EINVAL) } - let databaseURL = root.appendingPathComponent( - descriptor.database_file, isDirectory: false).standardizedFileURL - guard databaseURL.deletingLastPathComponent() == root else { - throw POSIXError(.EACCES) + if descriptor.backend == "sqlite" { + guard descriptor.connection != ".", descriptor.connection != "..", + URL(fileURLWithPath: descriptor.connection).lastPathComponent == + descriptor.connection else { + throw POSIXError(.EINVAL) + } + let databaseURL = root.appendingPathComponent( + descriptor.connection, isDirectory: false).standardizedFileURL + guard databaseURL.deletingLastPathComponent() == root else { + throw POSIXError(.EACCES) + } + return Configuration(descriptor: descriptor, connection: databaseURL.path) } - return (descriptor, databaseURL) + return Configuration(descriptor: descriptor, connection: descriptor.connection) } func probeResource(resource: FSResource, @@ -61,9 +72,11 @@ final class VexFSFileSystem: FSUnaryFileSystem & FSUnaryFileSystemOperations { return replyHandler(nil, POSIXError(.ENODEV)) } do { - let (descriptor, databaseURL) = try configuration(for: pathResource) + let configuration = try configuration(for: pathResource) + let descriptor = configuration.descriptor let identifier = FSContainerIdentifier(uuid: - VexFSIdentity.uuid(databasePath: databaseURL.path, + VexFSIdentity.uuid(backend: descriptor.backend, + connection: configuration.connection, workspace: descriptor.workspace)) replyHandler(.usable(name: "VexFS \(descriptor.workspace)", containerID: identifier), nil) } catch { @@ -84,10 +97,13 @@ final class VexFSFileSystem: FSUnaryFileSystem & FSUnaryFileSystemOperations { return replyHandler(nil, POSIXError(.EACCES)) } do { - let (descriptor, databaseURL) = try configuration(for: pathResource) - let backend = try VexFSBackend(databasePath: databaseURL.path, + let configuration = try configuration(for: pathResource) + let descriptor = configuration.descriptor + let backend = try VexFSBackend(backend: descriptor.backend, + connection: configuration.connection, workspace: descriptor.workspace) - let identity = VexFSIdentity.uuid(databasePath: databaseURL.path, + let identity = VexFSIdentity.uuid(backend: descriptor.backend, + connection: configuration.connection, workspace: descriptor.workspace) let loadedVolume = try VexFSVolume(backend: backend, workspace: descriptor.workspace, volumeID: identity) @@ -96,7 +112,11 @@ final class VexFSFileSystem: FSUnaryFileSystem & FSUnaryFileSystemOperations { self.volume = loadedVolume replyHandler(loadedVolume, nil) } catch { - Logger.vexfs.error("load rejected resource: \(error.localizedDescription)") + // This contains the runtime error text, never the connection string or + // passfile contents. Keep it public so headless install/eval failures + // can be diagnosed from unified logs without enabling private logging. + Logger.vexfs.error( + "load rejected resource: \(error.localizedDescription, privacy: .public)") pathResource.url.stopAccessingSecurityScopedResource() replyHandler(nil, error) } diff --git a/agent_files/macos/VexFSAppEx/VexFSVolume+Operations.swift b/agent_files/macos/VexFSAppEx/VexFSVolume+Operations.swift index ff95190187..edb5f215af 100644 --- a/agent_files/macos/VexFSAppEx/VexFSVolume+Operations.swift +++ b/agent_files/macos/VexFSAppEx/VexFSVolume+Operations.swift @@ -198,7 +198,14 @@ extension VexFSVolume: FSVolume.Operations { } if nameString == "." { return replyHandler(directory, name, nil) } do { - let path = try childPath(parent: directory, name: nameString) + let generation = try currentCacheGeneration() + let path = try childPath(parent: directory, name: nameString, + generation: generation) + if let (record, parentID) = cachedLookupRecord(path: path, + generation: generation) { + return replyHandler(cachedItem(path: path, record: record, + parentID: parentID), name, nil) + } let record = try backend.stat(path: path) replyHandler(cachedItem(path: path, record: record, parentID: directory.itemID), name, nil) } catch { replyHandler(nil, nil, error) } @@ -246,12 +253,12 @@ extension VexFSVolume: FSVolume.Operations { let record = try backend.stat(path: path) let item = cachedItem(path: path, record: record, parentID: directory.itemID) if let createdHandle { - // The atomic create leaves an empty generation staged. Reuse - // that handle for the first write and publish one version on - // close/synchronize, instead of an empty version plus content. + // createFileHandle has already published the empty file so the + // directory entry is visible. Reuse its handle for later writes; + // the next stage operation supplies the next dirty generation. item.handle = createdHandle item.handleWritable = true - item.dirtyGeneration = 1 + item.dirtyGeneration = nil } recordDirectoryMutation(directory) replyHandler(item, name, nil) diff --git a/agent_files/macos/VexFSAppEx/VexFSVolume.swift b/agent_files/macos/VexFSAppEx/VexFSVolume.swift index 17d33a186a..55e19845d1 100644 --- a/agent_files/macos/VexFSAppEx/VexFSVolume.swift +++ b/agent_files/macos/VexFSAppEx/VexFSVolume.swift @@ -5,11 +5,19 @@ final class VexFSVolume: FSVolume, FSVolume.ReadWriteOperations, FSVolume.OpenCloseOperations, FSVolume.XattrOperations { + private struct CachedLookupRecord { + let record: VexFSStatRecord + let parentID: FSItem.Identifier + let generation: UInt64 + } + let backend: VexFSBackend let workspace: String let rootItem: VexFSItem private let cacheLock = NSLock() private var itemCache: [UInt64: VexFSItem] = [:] + private var lookupRecordCache: [String: CachedLookupRecord] = [:] + private let maximumLookupRecordCount = 100_000 private let maximumReadCacheBytes: UInt64 = 1024 * 1024 private let maximumTotalReadCacheBytes: UInt64 = 64 * 1024 * 1024 private var readCacheBytes: UInt64 = 0 @@ -49,6 +57,7 @@ final class VexFSVolume: FSVolume, cacheLock.lock() for item in itemCache.values { item.readCache = nil } itemCache.removeAll(keepingCapacity: true) + lookupRecordCache.removeAll(keepingCapacity: true) itemCache[rootRecord.inode] = rootItem readCacheBytes = 0 observedCacheGeneration = visibility.generation @@ -60,6 +69,12 @@ final class VexFSVolume: FSVolume, func currentCacheGeneration() throws -> UInt64 { let visibility = try backend.refreshVisibility() cacheLock.lock() + if observedCacheGeneration != visibility.generation { + // 目录枚举带回的元数据只对当时的 workspace generation 有效。 + // 其他 gateway 提交、rename 或 restore 后必须整体丢弃，不能让 + // lookup 为了省一次 stat 而返回旧路径或旧版本。 + lookupRecordCache.removeAll(keepingCapacity: true) + } observedCacheGeneration = visibility.generation let generation = observedCacheGeneration cacheLock.unlock() @@ -96,15 +111,51 @@ final class VexFSVolume: FSVolume, return path } + // 调用方已经刷新过 visibility 时，generation 相同就说明本地记录里的 + // 路径仍然有效。正常的只读遍历不需要为每个文件再跑一次 vexfs_path。 + func currentPath(for item: VexFSItem, generation: UInt64) throws -> String { + cacheLock.lock() + let cacheGeneration = observedCacheGeneration + cacheLock.unlock() + if !item.isUnlinked && generation == cacheGeneration && + item.cacheGeneration == generation { + return item.path + } + return try currentPath(for: item) + } + func childPath(parent: VexFSItem, name: String) throws -> String { let parentPath = try currentPath(for: parent) return parentPath == "/" ? "/\(name)" : "\(parentPath)/\(name)" } + func childPath(parent: VexFSItem, name: String, generation: UInt64) throws -> String { + let parentPath = try currentPath(for: parent, generation: generation) + return parentPath == "/" ? "/\(name)" : "\(parentPath)/\(name)" + } + + func cachedLookupRecord(path: String, + generation: UInt64) -> (VexFSStatRecord, FSItem.Identifier)? { + cacheLock.lock() + defer { cacheLock.unlock() } + guard let cached = lookupRecordCache[path], cached.generation == generation else { + return nil + } + return (cached.record, cached.parentID) + } + func cachedItem(path: String, record: VexFSStatRecord, parentID: FSItem.Identifier) -> VexFSItem { cacheLock.lock() defer { cacheLock.unlock() } + if lookupRecordCache[path] == nil && + lookupRecordCache.count >= maximumLookupRecordCount { + // 防止长时间遍历大量不同目录时缓存无上限增长。清空只会退回 + // 一次权威 stat，不影响内容或正确性。 + lookupRecordCache.removeAll(keepingCapacity: true) + } + lookupRecordCache[path] = CachedLookupRecord( + record: record, parentID: parentID, generation: observedCacheGeneration) if let item = itemCache[record.inode] { item.path = path item.record = record @@ -233,7 +284,7 @@ final class VexFSVolume: FSVolume, item.dirtyGeneration = nil } if item.handle == nil { - let path = try currentPath(for: item) + let path = try currentPath(for: item, generation: generation) if !writable && item.record.size <= maximumReadCacheBytes { let data = try backend.readFile(path: path) if installReadCache(data, for: item) { diff --git a/agent_files/macos/bundle_pg_runtime.sh b/agent_files/macos/bundle_pg_runtime.sh new file mode 100755 index 0000000000..e14105562e --- /dev/null +++ b/agent_files/macos/bundle_pg_runtime.sh @@ -0,0 +1,123 @@ +#!/bin/bash +set -euo pipefail + +# Copy the non-system dynamic-library closure needed by libpq into a portable +# @rpath directory. Usage: +# bundle_pg_runtime.sh ... + +[ "$#" -ge 4 ] || { + echo "usage: $0 binary destination architecture search-dir..." >&2 + exit 2 +} + +BINARY="$1" +DESTINATION="$2" +ARCHITECTURE="$3" +shift 3 + +[ -f "$BINARY" ] || { echo "binary not found: $BINARY" >&2; exit 1; } +case "$ARCHITECTURE" in arm64|x86_64) ;; *) echo "unsupported architecture: $ARCHITECTURE" >&2; exit 2;; esac +OTOOL=/usr/bin/otool +INSTALL_NAME_TOOL=/usr/bin/install_name_tool +LIPO=/usr/bin/lipo +[ -x "$OTOOL" ] || { echo "Apple otool is required" >&2; exit 1; } +[ -x "$INSTALL_NAME_TOOL" ] || { echo "Apple install_name_tool is required" >&2; exit 1; } +[ -x "$LIPO" ] || { echo "Apple lipo is required" >&2; exit 1; } + +mkdir -p "$DESTINATION" +QUEUE="$(mktemp "${TMPDIR:-/tmp}/vexfs-pg-runtime.queue.XXXXXX")" +SEARCH_DIRS="$(mktemp "${TMPDIR:-/tmp}/vexfs-pg-runtime.search.XXXXXX")" +trap 'rm -f "$QUEUE" "$SEARCH_DIRS"' EXIT +printf '%s\n' "$BINARY" > "$QUEUE" +for directory in "$@"; do + [ -d "$directory" ] && printf '%s\n' "$directory" >> "$SEARCH_DIRS" +done + +find_dependency() { + local name="$1" directory + while IFS= read -r directory; do + [ -f "$directory/$name" ] && { printf '%s\n' "$directory/$name"; return 0; } + done < "$SEARCH_DIRS" + return 1 +} + +queue_index=1 +while :; do + item="$(sed -n "${queue_index}p" "$QUEUE")" + [ -n "$item" ] || break + queue_index=$((queue_index + 1)) + + "$OTOOL" -L "$item" | tail -n +2 | awk '{print $1}' | while IFS= read -r dependency; do + case "$dependency" in + /System/Library/*|/usr/lib/*) continue ;; + @rpath/*) + name="${dependency#@rpath/}" + case "$name" in */*) echo "nested @rpath dependency is unsupported: $dependency" >&2; exit 1;; esac + source="$(find_dependency "$name" || true)" + ;; + /*) + source="$dependency" + name="$(basename "$dependency")" + printf '%s\n' "$(dirname "$dependency")" >> "$SEARCH_DIRS" + ;; + @loader_path/*) + relative="${dependency#@loader_path/}" + source="$(cd "$(dirname "$item")" && pwd)/$relative" + name="$(basename "$source")" + ;; + *) + echo "unsupported dynamic-library reference: $dependency in $item" >&2 + exit 1 + ;; + esac + + destination="$DESTINATION/$name" + # A dylib lists its own install name first. If it is already bundled, + # there is nothing to copy or queue again. + if [ ! -f "$destination" ]; then + [ -n "${source:-}" ] && [ -f "$source" ] || { + echo "cannot resolve $dependency required by $item" >&2 + exit 1 + } + cp -fL "$source" "$destination" + chmod 0755 "$destination" + "$LIPO" "$destination" -verify_arch "$ARCHITECTURE" >/dev/null || { + echo "$destination does not contain $ARCHITECTURE" >&2 + exit 1 + } + "$INSTALL_NAME_TOOL" -id "@rpath/$name" "$destination" + printf '%s\n' "$destination" >> "$QUEUE" + fi + if [ "$dependency" != "@rpath/$name" ]; then + "$INSTALL_NAME_TOOL" -change "$dependency" "@rpath/$name" "$item" + fi + done +done + +# Refuse a bundle that still reaches into Homebrew, Conda, MacPorts, or another +# developer-machine path. Every non-system dependency must now resolve locally. +while IFS= read -r item; do + "$OTOOL" -l "$item" | awk ' + $1 == "cmd" && $2 == "LC_RPATH" { getline; getline; print $2 } + ' | while IFS= read -r runpath; do + case "$runpath" in + @*) ;; + *) "$INSTALL_NAME_TOOL" -delete_rpath "$runpath" "$item" ;; + esac + done + "$OTOOL" -L "$item" | tail -n +2 | awk '{print $1}' | while IFS= read -r dependency; do + case "$dependency" in + /System/Library/*|/usr/lib/*) ;; + @rpath/*) + name="${dependency#@rpath/}" + [ -f "$DESTINATION/$name" ] || { + echo "bundled dependency is missing: $dependency for $item" >&2 + exit 1 + } + ;; + *) echo "non-portable dependency remains: $dependency in $item" >&2; exit 1;; + esac + done +done < "$QUEUE" + +echo "Bundled PostgreSQL runtime: $(find "$DESTINATION" -type f -name '*.dylib' | wc -l | tr -d ' ') dylibs" diff --git a/agent_files/macos/generate_xcode_project.rb b/agent_files/macos/generate_xcode_project.rb index 86feb1c9bc..6d3dcc1c54 100644 --- a/agent_files/macos/generate_xcode_project.rb +++ b/agent_files/macos/generate_xcode_project.rb @@ -41,7 +41,7 @@ fskit.source_tree = 'SDKROOT' extension.frameworks_build_phase.add_file_reference(fskit) -build_core = extension.new_shell_script_build_phase('Build VexFS SQLite Core') +build_core = extension.new_shell_script_build_phase('Build VexFS Runtime Core') build_core.always_out_of_date = '1' build_core.shell_script = <<~'SH' set -e @@ -49,12 +49,23 @@ [ -x "$CMAKE_BIN" ] || CMAKE_BIN=/opt/homebrew/bin/cmake [ -x "$CMAKE_BIN" ] || CMAKE_BIN=/usr/local/bin/cmake [ -x "$CMAKE_BIN" ] || { echo "cmake is required" >&2; exit 1; } - BUILD_DIR="$SRCROOT/../../vexdb_sqlite/build-fskit" + # Run-script phases are not per-architecture, so CURRENT_ARCH is often + # "undefined_arch". The packaging flow intentionally builds one architecture + # at a time and passes ARCHS explicitly. + BUILD_ARCH="${ARCHS:-}" + case "$BUILD_ARCH" in + arm64|x86_64) ;; + *) echo "unsupported FSKit build architecture: $BUILD_ARCH" >&2; exit 2 ;; + esac + BUILD_DIR="$SRCROOT/../../vexdb_sqlite/build-fskit-$BUILD_ARCH" BUILD_JOBS="${VEXDB_LITE_BUILD_JOBS:-4}" case "$BUILD_JOBS" in *[!0-9]*|0) echo "VEXDB_LITE_BUILD_JOBS must be a positive integer" >&2; exit 2;; esac [ "$BUILD_JOBS" -le 4 ] || BUILD_JOBS=4 "$CMAKE_BIN" -S "$SRCROOT/../../vexdb_sqlite" -B "$BUILD_DIR" \ - -DCMAKE_BUILD_TYPE="$CONFIGURATION" -DVEXDB_SQLITE_BUILD_TESTS=OFF + -DCMAKE_BUILD_TYPE="$CONFIGURATION" \ + -DCMAKE_OSX_ARCHITECTURES="$BUILD_ARCH" \ + -DVEXFS_REQUIRE_POSTGRESQL=ON \ + -DVEXDB_SQLITE_BUILD_TESTS=OFF "$CMAKE_BIN" --build "$BUILD_DIR" --target vexfs_runtime -j "$BUILD_JOBS" SH extension.build_phases.move(build_core, 0) @@ -107,12 +118,14 @@ 'INFOPLIST_FILE' => 'VexFSAppEx/Info.plist', 'INFOPLIST_KEY_CFBundleDisplayName' => 'VexFS file system', 'LD_RUNPATH_SEARCH_PATHS' => '$(inherited) @executable_path/../Frameworks @executable_path/../../../../Frameworks', + 'LIBRARY_SEARCH_PATHS' => '$(inherited) "$(SRCROOT)/../../vexdb_sqlite/build-fskit-$(ARCHS)/pg-client/lib"', 'MARKETING_VERSION' => '0.1.0', 'OTHER_LDFLAGS' => [ '$(inherited)', - '"$(SRCROOT)/../../vexdb_sqlite/build-fskit/libvexfs_runtime.a"', - '"$(SRCROOT)/../../vexdb_sqlite/build-fskit/libvexdb_lite_static.a"', + '"$(SRCROOT)/../../vexdb_sqlite/build-fskit-$(ARCHS)/libvexfs_runtime.a"', + '"$(SRCROOT)/../../vexdb_sqlite/build-fskit-$(ARCHS)/libvexdb_lite_static.a"', '-lsqlite3', + '-lpq', '-lc++' ], 'PRODUCT_BUNDLE_IDENTIFIER' => 'io.vexdb.vexfs.extension', diff --git a/agent_files/macos/package/README.md b/agent_files/macos/package/README.md index 3ab5bab965..17928b4730 100644 --- a/agent_files/macos/package/README.md +++ b/agent_files/macos/package/README.md @@ -5,7 +5,7 @@ VexDB-Lite 在一个安装包中提供： -- `VexDB Lite.app`：包含 VexFS FSKit 文件系统扩展； +- `.payload/VexDB Lite.app`：安装程序使用的隐藏 App payload，包含 VexFS FSKit 文件系统扩展； - `bin/vexdb`：内置 SQLite、向量检索和文件 SQL 能力的统一命令； - `bin/vexfs`：`vexdb fs` 的兼容快捷入口； - `lib/vexdb_lite.dylib`：供现有 SQLite 程序加载的扩展； @@ -38,6 +38,14 @@ VexDB-Lite 在一个安装包中提供： 安装程序会在后台启动一次 VexDB Lite App，让 macOS 立即发现新安装或刚替换的 FSKit 模块。 它不会替用户打开扩展开关；首次安装仍需在系统设置中允许。 +App payload 放在隐藏目录中，避免解压目录和安装目录同时被 macOS 注册为两个同名 +FSKit 模块。扩展已经获准时，安装程序还会执行一次临时真实挂载；挂载失败不会假报安装成功。 +升级时，旧 App、CLI、SQLite 扩展和 PostgreSQL runtime 只放在隐藏事务目录中；任何一步失败 +都会恢复旧版本，成功后立即删除临时备份，不会在 Applications 中留下可被系统再次识别的 +`.app.disabled` 副本。如果 macOS 要求新签名版本重新授权，安装程序会明确提示，不会静默跳过。 +安装或升级前请先卸载正在使用的 VexFS、exFAT 等 FSKit 卷。安装程序会拒绝在这些卷 +仍挂载时替换扩展，然后只重启当前用户的 `fskit_agent` 来刷新模块；它不会重启 `pkd`、 +清空 LaunchServices 注册库或影响其他用户。 如果 `~/.local/bin` 不在 PATH： @@ -84,6 +92,10 @@ vexdb fs archive verify workspace.vexfs 已挂载 workspace 会在恢复时自动安全卸载并挂回原目录；正常卸载失败时不会开始恢复。 `--force-unmount` 只用于用户明确接受中断打开文件的场景。 +FSKit 异常退出后，底层 mountpoint 会保持不可写，避免 Bash 把文件误写进普通本机目录。 +此时先运行 `vexdb fs unmount --force MOUNTPOINT` 清理，再重新挂载。PostgreSQL 网络中断时， +失败的写命令不会被隐式重放；连接恢复后先读取或 quick check，再重新发出写命令。 + 旧命令继续可用： ```bash diff --git a/agent_files/macos/package/install.sh b/agent_files/macos/package/install.sh index aadee6251c..9c41047a5e 100755 --- a/agent_files/macos/package/install.sh +++ b/agent_files/macos/package/install.sh @@ -8,11 +8,13 @@ BIN_DIR="${VEXDB_LITE_BIN_DIR:-${VEXFS_BIN_DIR:-$HOME/.local/bin}}" LIB_DIR="${VEXDB_LITE_LIB_DIR:-${VEXFS_LIB_DIR:-$HOME/.local/lib/vexdb-lite}}" LSREGISTER="/System/Library/Frameworks/CoreServices.framework/Frameworks/LaunchServices.framework/Support/lsregister" OFFICIAL_TEAM_ID="BB5VK42K87" +PACKAGE_APP="$ROOT/.payload/VexDB Lite.app" -[ -d "$ROOT/VexDB Lite.app" ] || { echo "安装包缺少 VexDB Lite.app" >&2; exit 1; } +[ -d "$PACKAGE_APP" ] || { echo "安装包缺少 .payload/VexDB Lite.app" >&2; exit 1; } [ -x "$ROOT/bin/vexdb" ] || { echo "安装包缺少 bin/vexdb" >&2; exit 1; } [ -x "$ROOT/bin/vexfs" ] || { echo "安装包缺少 bin/vexfs" >&2; exit 1; } [ -f "$ROOT/lib/vexdb_lite.dylib" ] || { echo "安装包缺少 SQLite 扩展" >&2; exit 1; } +[ -f "$ROOT/lib/runtime/libpq.5.dylib" ] || { echo "安装包缺少 PostgreSQL runtime" >&2; exit 1; } [ -f "$ROOT/MANIFEST.txt" ] || { echo "安装包缺少 MANIFEST.txt" >&2; exit 1; } [ -f "$ROOT/SHA256SUMS.txt" ] || { echo "安装包缺少 SHA256SUMS.txt" >&2; exit 1; } @@ -49,10 +51,18 @@ case "$PACKAGE_SIGNATURE" in *) echo "安装包签名类型无效：${PACKAGE_SIGNATURE:-missing}" >&2; exit 1 ;; esac -ACTUAL_RUNTIME_ABI="$("$ROOT/bin/vexdb" fs --json doctor 2>/dev/null \ - | sed -n 's/.*"runtime_abi":\([0-9][0-9]*\).*/\1/p' || true)" +DOCTOR_CONTRACT_OUTPUT="" +ACTUAL_RUNTIME_ABI="" +for ATTEMPT in 1 2 3; do + DOCTOR_CONTRACT_OUTPUT="$("$ROOT/bin/vexdb" fs --json doctor 2>&1 || true)" + ACTUAL_RUNTIME_ABI="$(printf '%s\n' "$DOCTOR_CONTRACT_OUTPUT" \ + | sed -n 's/.*"runtime_abi":\([0-9][0-9]*\).*/\1/p')" + [ -z "$ACTUAL_RUNTIME_ABI" ] || break + sleep 1 +done [ -n "$ACTUAL_RUNTIME_ABI" ] && [ "$ACTUAL_RUNTIME_ABI" = "$EXPECTED_RUNTIME_ABI" ] || { echo "安装包 runtime ABI 不一致：实际 ${ACTUAL_RUNTIME_ABI:-missing}，清单 ${EXPECTED_RUNTIME_ABI:-missing}" >&2 + [ -z "$DOCTOR_CONTRACT_OUTPUT" ] || printf '%s\n' "$DOCTOR_CONTRACT_OUTPUT" >&2 exit 1 } ACTUAL_CONTRACT="$("$ROOT/bin/vexdb" :memory: "SELECT vexfs_contract_version();" 2>/dev/null || true)" @@ -78,16 +88,187 @@ verify_component() { fi } -verify_component "$ROOT/VexDB Lite.app" --deep -verify_component "$ROOT/VexDB Lite.app/Contents/Extensions/VexFSAppEx.appex" +read_code_entitlement() { + local item="$1" key="$2" entitlement_file value + entitlement_file="$(mktemp "${TMPDIR:-/tmp}/vexdb-lite-entitlements.XXXXXX")" || return 1 + if ! /usr/bin/codesign -d --entitlements - --xml "$item" \ + >"$entitlement_file" 2>/dev/null; then + rm -f "$entitlement_file" + return 1 + fi + value="$(/usr/libexec/PlistBuddy -c "Print :$key" \ + "$entitlement_file" 2>/dev/null || true)" + rm -f "$entitlement_file" + [ -n "$value" ] || return 1 + printf '%s\n' "$value" +} + +verify_component "$PACKAGE_APP" --deep +PACKAGE_EXTENSION="$PACKAGE_APP/Contents/Extensions/VexFSAppEx.appex" +verify_component "$PACKAGE_EXTENSION" verify_component "$ROOT/bin/vexdb" verify_component "$ROOT/lib/vexdb_lite.dylib" +find "$ROOT/lib/runtime" \ + "$PACKAGE_APP/Contents/Extensions/VexFSAppEx.appex/Contents/Frameworks" \ + -type f -name '*.dylib' -print0 | while IFS= read -r -d '' library; do + verify_component "$library" + done + +FSKIT_ENTITLEMENT="$(read_code_entitlement "$PACKAGE_EXTENSION" \ + com.apple.developer.fskit.fsmodule || true)" +[ "$FSKIT_ENTITLEMENT" = true ] || { + echo "安装包 FSKit 扩展缺少 fsmodule entitlement" >&2 + exit 1 +} +if [ "$PACKAGE_SIGNATURE" = developer-id ]; then + EXTENSION_TEAM="$(read_code_entitlement "$PACKAGE_EXTENSION" \ + com.apple.developer.team-identifier || true)" + EXTENSION_APPLICATION="$(read_code_entitlement "$PACKAGE_EXTENSION" \ + com.apple.application-identifier || true)" + [ "$EXTENSION_TEAM" = "$EXPECTED_TEAM" ] || { + echo "安装包 FSKit 扩展 team entitlement 错误：${EXTENSION_TEAM:-missing}" >&2 + exit 1 + } + [ "$EXTENSION_APPLICATION" = "$EXPECTED_TEAM.io.vexdb.vexfs.extension" ] || { + echo "安装包 FSKit 扩展 application identifier 错误：${EXTENSION_APPLICATION:-missing}" >&2 + exit 1 + } +fi + +verify_installed_mount() { + local probe_root probe_database probe_mount probe_error attempt actual + probe_root="$(mktemp -d "${TMPDIR:-/tmp}/vexdb-lite-install-mount.XXXXXX")" + probe_database="$probe_root/resource/probe.sqlite3" + probe_mount="$probe_root/mount" + probe_error="$probe_root/mount-error.log" + mkdir -p "$probe_root/resource" "$probe_mount" + + cleanup_probe_mount() { + if "$BIN_DIR/vexdb" fs --db "$probe_database" --workspace install-probe \ + unmount --force "$probe_mount" >/dev/null 2>&1; then + rm -rf "$probe_root" + return 0 + fi + echo "临时 FSKit 验证目录未能卸载，已保留以避免删除挂载内容：$probe_root" >&2 + return 1 + } + + if ! "$BIN_DIR/vexdb" fs --db "$probe_database" \ + --workspace install-probe setup >/dev/null 2>"$probe_error"; then + cat "$probe_error" >&2 + rm -rf "$probe_root" + return 1 + fi + + # FSKit can briefly report the newly registered path before ExtensionKit has + # finished replacing the old helper. A real disposable mount is the only + # reliable readiness check; doctor intentionally reports registry state only. + for attempt in 1 2; do + if "$BIN_DIR/vexdb" fs --db "$probe_database" \ + --workspace install-probe mount "$probe_mount" \ + >/dev/null 2>"$probe_error"; then + if printf '%s\n' ready-from-fskit >"$probe_mount/ready.txt" 2>>"$probe_error"; then + actual="$(cat "$probe_mount/ready.txt" 2>>"$probe_error" || true)" + if [ "$actual" = ready-from-fskit ]; then + if "$BIN_DIR/vexdb" fs --db "$probe_database" \ + --workspace install-probe unmount "$probe_mount" \ + >/dev/null 2>>"$probe_error"; then + rm -rf "$probe_root" + return 0 + fi + cat "$probe_error" >&2 + echo "临时 FSKit 验证目录未能正常卸载，已保留：$probe_root" >&2 + return 1 + fi + fi + fi + if [ "$attempt" -eq 2 ]; then + cat "$probe_error" >&2 + cleanup_probe_mount || true + return 1 + fi + cleanup_probe_mount || return 1 + # The cleanup removed the disposable database. Recreate it for the next + # attempt so no state from a failed helper launch can affect the result. + probe_root="$(mktemp -d "${TMPDIR:-/tmp}/vexdb-lite-install-mount.XXXXXX")" + probe_database="$probe_root/resource/probe.sqlite3" + probe_mount="$probe_root/mount" + probe_error="$probe_root/mount-error.log" + mkdir -p "$probe_root/resource" "$probe_mount" + "$BIN_DIR/vexdb" fs --db "$probe_database" --workspace install-probe \ + setup >/dev/null 2>"$probe_error" || { + cat "$probe_error" >&2 + rm -rf "$probe_root" + return 1 + } + sleep $((attempt * 2)) + done + + return 1 +} + +fskit_mounts_active() { + local active_mounts + active_mounts="$(/sbin/mount 2>/dev/null | tr '[:upper:]' '[:lower:]')" + case "$active_mounts" in + *"(vexfs,"*|*"(exfat,"*|*"(msdos,"*|*"(ftp,"*|\ + *" type vexfs"*|*" type exfat"*|*" type msdos"*|*" type ftp"*) + return 0 + ;; + esac + return 1 +} + +restart_current_user_fskit_agent() { + local current_uid old_pids pid attempt still_running + if fskit_mounts_active; then + echo "检测到正在使用的 FSKit 文件系统；拒绝刷新 fskit_agent。" >&2 + return 1 + fi + current_uid="$(id -u 2>/dev/null || true)" + [ -n "$current_uid" ] || { + echo "无法读取当前用户 ID，不能安全刷新 fskit_agent。" >&2 + return 1 + } + old_pids="$(/usr/bin/pgrep -u "$current_uid" -x fskit_agent 2>/dev/null || true)" + [ -n "$old_pids" ] || return 0 + + # fskit_agent 会忽略 TERM。没有活动 FSKit 卷时用 KILL 结束当前用户的 + # 旧代理，让 launchd 按需拉起一个不含旧 extension UUID 的新进程。 + /usr/bin/pkill -KILL -u "$current_uid" -x fskit_agent >/dev/null 2>&1 || { + echo "无法刷新当前用户的 fskit_agent。" >&2 + return 1 + } + for attempt in 1 2 3 4 5; do + still_running=false + for pid in $old_pids; do + if /bin/kill -0 "$pid" >/dev/null 2>&1; then + still_running=true + break + fi + done + [ "$still_running" = false ] && return 0 + sleep 0.2 + done + echo "旧 fskit_agent 没有退出，无法保证扩展身份已经刷新。" >&2 + return 1 +} mkdir -p "$APP_DIR" "$BIN_DIR" "$LIB_DIR" +# Replacing an FSKit module requires restarting the current user's discovery +# agents. Never do that while any known FSKit volume is mounted, because the +# volume may belong to VexFS or to another FSKit implementation such as exFAT. +if [ -z "$APP_DIR_OVERRIDE" ]; then + if fskit_mounts_active; then + echo "检测到正在使用的 FSKit 文件系统；请先卸载相关卷，再安装或升级 VexDB-Lite。" >&2 + exit 1 + fi +fi + # 旧 VexFS.app 与新 App 沿用同一个 extension ID。两者同时注册时 macOS 可能继续 # 调用旧扩展，表现为 mount "Loading resource: Input/output error"。正常安装时先 -# 撤销旧注册，并在权限允许时把旧 App 改成不会被 LaunchServices 扫描的备份。 +# 撤销旧注册，并在权限允许时把旧 App 放入本次安装事务目录。 # 自定义 APP_DIR 用于测试或集成，不触碰系统中已有的 App 注册。 LEGACY_FOUND=false LEGACY_INDEX=0 @@ -99,7 +280,7 @@ if [ -z "$APP_DIR_OVERRIDE" ]; then if [ -x "$LSREGISTER" ]; then "$LSREGISTER" -u "$LEGACY_APP" >/dev/null 2>&1 || true fi - LEGACY_BACKUP="$APP_DIR/VexFS Legacy $(date +%Y%m%d-%H%M%S)-$LEGACY_INDEX.app.disabled" + LEGACY_BACKUP="$APP_DIR/.VexFS Legacy $(date +%Y%m%d-%H%M%S)-$LEGACY_INDEX.bundle.disabled" if mv "$LEGACY_APP" "$LEGACY_BACKUP" 2>/dev/null; then echo "旧 VexFS App 已停用并保留：$LEGACY_BACKUP" else @@ -110,24 +291,92 @@ fi INSTALL_ID="$(date +%Y%m%d-%H%M%S)-$$" INSTALLED_APP="$APP_DIR/VexDB Lite.app" -STAGED_APP="$APP_DIR/.VexDB Lite $INSTALL_ID.app" +TRANSACTION_DIR="$APP_DIR/.vexdb-lite-install-$INSTALL_ID" +STAGED_APP="$TRANSACTION_DIR/new.bundle" PREVIOUS_APP="" +APP_REPLACED=false +CLI_BACKED_UP=false +LIBRARY_BACKED_UP=false +RUNTIME_BACKED_UP=false +INSTALL_COMMITTED=false +mkdir -p "$TRANSACTION_DIR" + +rollback_install() { + local status="$1" + set +e + if [ "$INSTALL_COMMITTED" != true ] && [ "$APP_REPLACED" = true ]; then + if [ -x "$LSREGISTER" ] && [ -z "$APP_DIR_OVERRIDE" ]; then + "$LSREGISTER" -u "$INSTALLED_APP" >/dev/null 2>&1 || true + fi + rm -rf "$INSTALLED_APP" + if [ -n "$PREVIOUS_APP" ] && [ -d "$PREVIOUS_APP" ]; then + mv "$PREVIOUS_APP" "$INSTALLED_APP" >/dev/null 2>&1 || true + if [ -x "$LSREGISTER" ] && [ -z "$APP_DIR_OVERRIDE" ]; then + "$LSREGISTER" -f "$INSTALLED_APP" >/dev/null 2>&1 || true + pluginkit -a "$INSTALLED_APP/Contents/Extensions/VexFSAppEx.appex" \ + >/dev/null 2>&1 || true + /usr/bin/open -gj "$INSTALLED_APP" >/dev/null 2>&1 || true + fi + fi + if [ "$CLI_BACKED_UP" = true ]; then + cp -p "$TRANSACTION_DIR/previous-vexdb" "$BIN_DIR/vexdb" >/dev/null 2>&1 || true + else + rm -f "$BIN_DIR/vexdb" + fi + if [ "$LIBRARY_BACKED_UP" = true ]; then + cp -p "$TRANSACTION_DIR/previous-vexdb_lite.dylib" \ + "$LIB_DIR/vexdb_lite.dylib" >/dev/null 2>&1 || true + else + rm -f "$LIB_DIR/vexdb_lite.dylib" + fi + rm -rf "$LIB_DIR/runtime" + if [ "$RUNTIME_BACKED_UP" = true ]; then + ditto "$TRANSACTION_DIR/previous-runtime" "$LIB_DIR/runtime" \ + >/dev/null 2>&1 || true + fi + echo "安装失败，已经恢复此前的 VexDB Lite App、CLI 和 runtime。" >&2 + fi + rm -rf "$STAGED_APP" "$TRANSACTION_DIR" + return "$status" +} +trap 'rollback_install $?' EXIT # ditto 会合并已有目录，不能直接覆盖已安装 App。旧 bundle 中多出的文件（例如旧 # embedded.provisionprofile）会留在新签名封装内，导致 codesign 立即失效。先复制到 # 新目录并验签，再整体替换；失败时保留旧 App。 -ditto "$ROOT/VexDB Lite.app" "$STAGED_APP" +if [ -e "$BIN_DIR/vexdb" ]; then + cp -p "$BIN_DIR/vexdb" "$TRANSACTION_DIR/previous-vexdb" + CLI_BACKED_UP=true +fi +if [ -e "$LIB_DIR/vexdb_lite.dylib" ]; then + cp -p "$LIB_DIR/vexdb_lite.dylib" "$TRANSACTION_DIR/previous-vexdb_lite.dylib" + LIBRARY_BACKED_UP=true +fi +if [ -d "$LIB_DIR/runtime" ]; then + ditto "$LIB_DIR/runtime" "$TRANSACTION_DIR/previous-runtime" + RUNTIME_BACKED_UP=true +fi +ditto "$PACKAGE_APP" "$STAGED_APP" if ! /usr/bin/codesign --verify --deep --strict "$STAGED_APP"; then - FAILED_APP="$APP_DIR/VexDB Lite Failed $INSTALL_ID.app.disabled" - mv "$STAGED_APP" "$FAILED_APP" 2>/dev/null || true - echo "安装包 App 签名校验失败，未替换当前版本：$FAILED_APP" >&2 + echo "安装包 App 签名校验失败，未替换当前版本。" >&2 exit 1 fi if [ -d "$INSTALLED_APP" ]; then + # 同 bundle ID 的旧进程仍存活时，open 会只激活旧实例。先结束宿主 App， + # 确保替换后启动的是新二进制和新的权限说明；FSKit 扩展由系统单独管理。 + if /usr/bin/pgrep -x "VexDB Lite" >/dev/null 2>&1; then + /usr/bin/pkill -TERM -x "VexDB Lite" >/dev/null 2>&1 || true + for ATTEMPT in 1 2 3 4 5 6 7 8 9 10; do + /usr/bin/pgrep -x "VexDB Lite" >/dev/null 2>&1 || break + sleep 0.1 + done + /usr/bin/pgrep -x "VexDB Lite" >/dev/null 2>&1 && + /usr/bin/pkill -KILL -x "VexDB Lite" >/dev/null 2>&1 || true + fi if [ -x "$LSREGISTER" ] && [ -z "$APP_DIR_OVERRIDE" ]; then "$LSREGISTER" -u "$INSTALLED_APP" >/dev/null 2>&1 || true fi - PREVIOUS_APP="$APP_DIR/VexDB Lite Previous $INSTALL_ID.app.disabled" + PREVIOUS_APP="$TRANSACTION_DIR/previous.bundle" mv "$INSTALLED_APP" "$PREVIOUS_APP" fi if ! mv "$STAGED_APP" "$INSTALLED_APP"; then @@ -136,9 +385,23 @@ if ! mv "$STAGED_APP" "$INSTALLED_APP"; then echo "无法替换 VexDB Lite.app，已经尝试恢复旧版本" >&2 exit 1 fi +APP_REPLACED=true install -m 0755 "$ROOT/bin/vexdb" "$BIN_DIR/vexdb" ln -sfn vexdb "$BIN_DIR/vexfs" install -m 0644 "$ROOT/lib/vexdb_lite.dylib" "$LIB_DIR/vexdb_lite.dylib" +rm -rf "$LIB_DIR/runtime" +mkdir -p "$LIB_DIR/runtime" +find "$ROOT/lib/runtime" -type f -name '*.dylib' -print0 \ + | while IFS= read -r -d '' library; do + install -m 0755 "$library" "$LIB_DIR/runtime/$(basename "$library")" + done +verify_component "$INSTALLED_APP" --deep +verify_component "$BIN_DIR/vexdb" +verify_component "$LIB_DIR/vexdb_lite.dylib" +find "$LIB_DIR/runtime" -type f -name '*.dylib' -print0 \ + | while IFS= read -r -d '' library; do + verify_component "$library" + done if [ -z "$APP_DIR_OVERRIDE" ]; then INSTALLED_EXTENSION="$INSTALLED_APP/Contents/Extensions/VexFSAppEx.appex" @@ -152,11 +415,13 @@ if [ -z "$APP_DIR_OVERRIDE" ]; then # App has been launched once. Launch it hidden so doctor does not keep # reporting `extension: missing` after a valid upgrade. /usr/bin/open -gj "$INSTALLED_APP" >/dev/null 2>&1 || true - MOUNTS="$(/sbin/mount 2>/dev/null | tr '[:upper:]' '[:lower:]')" - case "$MOUNTS" in - *"(vexfs,"*|*"(exfat,"*|*"(msdos,"*|*" type vexfs"*|*" type exfat"*|*" type msdos"*) ;; - *) pkill -KILL -x fskit_agent >/dev/null 2>&1 || true ;; - esac + # Registration is now stable. Refresh only fskit_agent; rotating pkd here + # can assign another extension UUID after the agent has already cached the + # previous identity and leads to ExtensionKit error 2 during the first mount. + restart_current_user_fskit_agent + sleep 1 + /usr/bin/open -gj "$INSTALLED_APP" >/dev/null 2>&1 || true + sleep 2 # enabled 只表示同 bundle ID 的扩展已经获准使用，不保证 FSKit 当前解析到 # 新安装的 App。用签名 CLI 读取 FSKit 的真实 module URL，避免安装成功但 @@ -173,7 +438,26 @@ if [ -z "$APP_DIR_OVERRIDE" ]; then sleep 1 done case "$EXTENSION_CHECK" in - *'"extension_path_matches":true'*) ;; + *'"extension_path_matches":true'*) + if [[ "$EXTENSION_CHECK" == *'"mount_ready":true'* ]]; then + if ! verify_installed_mount; then + echo "FSKit 已注册，但真实临时挂载仍未就绪；请稍后重试安装或运行 vexdb fs doctor。" >&2 + exit 1 + fi + echo "FSKit 真实挂载检查通过。" + else + case "$EXTENSION_CHECK" in + *'"extension":"disabled"'*) + echo "FSKit 扩展已安装，但新版本尚未获得系统授权。" + echo "请在系统设置 → 通用 → 登录项与扩展 → 文件系统扩展中启用 VexDB Lite，然后重新运行：" + echo " $BIN_DIR/vexdb fs doctor" + ;; + *) + echo "FSKit 扩展路径正确，但系统服务尚未准备好；请稍后运行 $BIN_DIR/vexdb fs doctor。" + ;; + esac + fi + ;; *'"extension_path":""'*) echo "警告：系统尚未返回 FSKit module 路径；启用扩展后请运行 vexdb fs doctor。" >&2 ;; @@ -186,11 +470,29 @@ if [ -z "$APP_DIR_OVERRIDE" ]; then esac fi +# 只有所有复制、签名和注册检查都完成后才提交事务。旧版本不作为长期备份 +# 留在 Applications 中，否则 LaunchServices 仍会把 `.app.disabled` 当成候选 App。 +INSTALL_COMMITTED=true +rm -rf "$TRANSACTION_DIR" +if [ -z "$APP_DIR_OVERRIDE" ]; then + for STALE_BACKUP in \ + "$APP_DIR"/VexDB\ Lite\ Previous\ *.app.disabled \ + "$APP_DIR"/VexDB\ Lite\ Failed\ *.app.disabled; do + [ -d "$STALE_BACKUP" ] || continue + if [ -x "$LSREGISTER" ]; then + "$LSREGISTER" -u "$STALE_BACKUP" >/dev/null 2>&1 || true + fi + rm -rf "$STALE_BACKUP" + done +fi +trap - EXIT + echo "VexDB-Lite 已安装：" echo " App: $APP_DIR/VexDB Lite.app" echo " CLI: $BIN_DIR/vexdb" echo " 文件快捷命令: $BIN_DIR/vexfs" echo " SQLite: $LIB_DIR/vexdb_lite.dylib" +echo " PostgreSQL runtime: $LIB_DIR/runtime/" case ":$PATH:" in *":$BIN_DIR:"*) ;; *) diff --git a/agent_files/macos/package/uninstall.sh b/agent_files/macos/package/uninstall.sh index 2e0e03361b..eb9a2db847 100755 --- a/agent_files/macos/package/uninstall.sh +++ b/agent_files/macos/package/uninstall.sh @@ -7,8 +7,9 @@ LIB_DIR="${VEXDB_LITE_LIB_DIR:-${VEXFS_LIB_DIR:-$HOME/.local/lib/vexdb-lite}}" rm -rf "$APP_DIR/VexDB Lite.app" rm -f "$BIN_DIR/vexdb" "$BIN_DIR/vexfs" "$LIB_DIR/vexdb_lite.dylib" +rm -rf "$LIB_DIR/runtime" rmdir "$LIB_DIR" 2>/dev/null || true -echo "VexDB-Lite App、CLI 和 SQLite 扩展已卸载。" +echo "VexDB-Lite App、CLI、SQLite 扩展和 PostgreSQL runtime 已卸载。" echo "数据库未删除：" echo " $HOME/Library/Application Support/VexDB-Lite/" diff --git "a/agent_files/macos/package/\344\275\277\347\224\250\350\257\264\346\230\216.md" "b/agent_files/macos/package/\344\275\277\347\224\250\350\257\264\346\230\216.md" index d0c8966656..8ddde2494e 100644 --- "a/agent_files/macos/package/\344\275\277\347\224\250\350\257\264\346\230\216.md" +++ "b/agent_files/macos/package/\344\275\277\347\224\250\350\257\264\346\230\216.md" @@ -36,6 +36,18 @@ VEXDB_LITE_ALLOW_ADHOC_INSTALL=1 ./install.sh 安装程序会在后台启动一次 VexDB Lite App，让 macOS 立即发现新安装或刚替换的 FSKit 模块。 首次安装仍需由用户在系统设置中打开文件系统扩展开关。 +安装包内的 App 位于隐藏的 `.payload` 目录，防止解压副本和正式安装副本同时被 +macOS 注册。扩展已经获准时，安装程序会执行一次临时真实挂载；只有写入、读回和 +卸载都成功，安装才会报告完成。 + +升级时，旧 App、CLI、SQLite 扩展和 PostgreSQL runtime 只保存在隐藏事务目录。任何一步 +失败都会恢复旧版本；成功后立即删除临时备份，不会在 Applications 中留下仍会被 macOS +识别的 `.app.disabled` 副本。如果系统要求新签名版本重新授权，安装程序会明确提示并保留 +已安装的新版本，用户启用后运行 `vexdb fs doctor` 即可继续。 + +安装或升级前，请先卸载正在使用的 VexFS、exFAT 等 FSKit 文件系统。安装程序检测到 +这类卷仍挂载时会停止，不会强行中断工作区。替换扩展后，安装程序只重启当前用户的 +`fskit_agent` 来刷新模块，不会重启 `pkd`、清空 LaunchServices 注册库或影响其他用户。 如果终端提示 `command not found: vexdb`，运行： @@ -171,6 +183,55 @@ vexdb fs index disable 索引后，文件写入、单文件版本恢复和完整工作区恢复会同步更新索引。宿主 SQLite 没有 FTS5 时仍能扫描搜索，不会影响基本文件能力。 +### 4.2 PostgreSQL 共享工作区 + +远程 PostgreSQL 必须先安装当前版本的 `vexdb_lite` extension。每个挂载使用真实数据库 +用户和 role 权限，不要在 DSN 中写明文密码。先创建 libpq passfile： + +```bash +printf '%s\n' 'db.example.com:5432:app:agent:你的密码' > ~/.pgpass +chmod 0600 ~/.pgpass +``` + +passfile 必须是当前用户拥有的普通文件，权限为 `0600` 或更严格，不能是符号链接。然后初始化 +并挂载共享工作区： + +```bash +PG_DSN='host=db.example.com port=5432 dbname=app user=agent passfile=/Users/me/.pgpass connect_timeout=8' + +vexdb fs --backend pg --dsn "$PG_DSN" --workspace agent-workspace setup +vexdb fs --backend pg --dsn "$PG_DSN" --workspace agent-workspace \ + mount ~/Work/agent-workspace + +cd ~/Work/agent-workspace +ls +git status +opencode +``` + +连接局域网中的 PostgreSQL 时，首次 mount 失败后 VexDB Lite 会自动打开并发起真实 host/port +检查。看到 macOS 的“本地网络”提示时点“允许”，CLI 会自动重试。若以前拒绝过，请到 +“系统设置 → 隐私与安全性 → 本地网络”打开 VexDB Lite。这个授权只用于局域网连接；SQLite +和回环地址不需要。授权请求只包含 host、port 和随机编号，不包含 DSN、用户名或密码。 + +另一台已安装 VexDB Lite 的 Mac 使用相同 DSN、数据库 role 和 workspace 名称，就会看到同一 +目录树、文件历史和快照。常用管理命令仍使用同一组连接参数： + +```bash +vexdb fs --backend pg --dsn "$PG_DSN" --workspace agent-workspace check +vexdb fs --backend pg --dsn "$PG_DSN" --workspace agent-workspace snapshot list +vexdb fs --backend pg --dsn "$PG_DSN" --workspace agent-workspace snapshot create before-agent +vexdb fs --backend pg --dsn "$PG_DSN" --workspace agent-workspace snapshot restore before-agent +vexdb fs --backend pg --dsn "$PG_DSN" --workspace agent-workspace unmount ~/Work/agent-workspace +``` + +快照用于误操作恢复，不代替数据库备份。管理员仍应使用 `pg_dump/pg_restore` 或 +`pg_basebackup` 备份整个 PostgreSQL；目标机器必须安装匹配版本的 VexDB-Lite PG extension。 + +网络或 PostgreSQL 中断时，当前失败的写命令不会被 VexDB Lite 自动重放。客户端无法可靠 +判断服务端事务是否已经提交，自动重放可能造成重复 append。网络恢复后先运行 `check --quick` +或读取一个已知文件，确认连接和内容状态，再重新发出需要的写命令。 + ## 5. 文件版本 每次写入和恢复都会生成新版本： @@ -340,6 +401,11 @@ vexdb fs unmount --force ~/VexDB 强制卸载会让现有文件句柄失效，适合进程崩溃或无法正常退出时使用。正常情况下仍应先关闭 正在访问挂载目录的程序，再执行普通卸载。 +挂载成功后，VexDB Lite 会把被挂载目录的底层目录设为 `0500`。如果 FSKit extension 意外 +退出、挂载被系统撤销，普通用户不能把文件误写到这个本机目录；写入会明确失败。执行 +`vexdb fs unmount --force MOUNTPOINT` 完成清理后，目录权限恢复为 `0700`，随后可以重新挂载。 +不要手工修改该目录权限或在挂载消失时继续写入。 + macOS 要求用户授权文件系统扩展，安装程序不能静默跳过这一步。 授权开关在“按类别”视图的“文件系统扩展”详情中，不在 `VexDB Lite` 的“按 App”只读详情中。 @@ -364,7 +430,8 @@ vexdb fs --db "$HOME/Desktop/vexdb-backup.sqlite3" ls / ``` 如果检测到旧版 `VexFS.app`，安装程序会先撤销旧文件系统扩展的注册，再把旧 App -改名为 `.disabled` 备份，避免 macOS 继续加载旧扩展。这个过程不会删除任何数据库。 +改成不带 `.app` 后缀的隐藏 `.bundle.disabled` 备份，避免 macOS 继续加载旧扩展。 +这个过程不会删除任何数据库。 ## 9. 在其他 SQLite 程序中加载 diff --git a/agent_files/macos/package_preview.sh b/agent_files/macos/package_preview.sh index 64d0ff426d..9c119dcb83 100755 --- a/agent_files/macos/package_preview.sh +++ b/agent_files/macos/package_preview.sh @@ -29,6 +29,22 @@ fi case "$VERSION" in *[!A-Za-z0-9._-]*) echo "版本只能包含字母、数字、点、下划线和连字符" >&2; exit 2 ;; esac +BUNDLE_MARKETING_VERSION="${VEXDB_LITE_BUNDLE_MARKETING_VERSION:-${VERSION%%-*}}" +BUNDLE_BUILD_VERSION="${VEXDB_LITE_BUNDLE_BUILD_VERSION:-}" +if [ -z "$BUNDLE_BUILD_VERSION" ]; then + preview_number="${VERSION##*.}" + case "$VERSION:$preview_number" in + *-preview.*:[0-9]*) BUNDLE_BUILD_VERSION="$preview_number" ;; + *) BUNDLE_BUILD_VERSION="$(git -C "$ROOT" rev-list --count HEAD 2>/dev/null || echo 1)" ;; + esac +fi +if [[ ! "$BUNDLE_MARKETING_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "Bundle marketing version 必须是数字三段式：$BUNDLE_MARKETING_VERSION" >&2 + exit 2 +fi +case "$BUNDLE_BUILD_VERSION" in + ''|*[!0-9]*|0) echo "Bundle build version 必须是正整数：$BUNDLE_BUILD_VERSION" >&2; exit 2 ;; +esac case "$ARCH" in arm64|x86_64) ;; *) echo "VEXDB_LITE_ARCH 只能是 arm64 或 x86_64" >&2; exit 2 ;; @@ -61,12 +77,39 @@ CMAKE_BIN="$(command -v cmake || true)" command -v xcodebuild >/dev/null || { echo "找不到 xcodebuild" >&2; exit 1; } command -v codesign >/dev/null || { echo "找不到 codesign" >&2; exit 1; } +PYTHON_BIN="${VEXDB_LITE_PYTHON:-}" +if [ -n "$PYTHON_BIN" ]; then + [ -x "$PYTHON_BIN" ] || { echo "VEXDB_LITE_PYTHON 不可执行：$PYTHON_BIN" >&2; exit 1; } + "$PYTHON_BIN" -c 'import sys; raise SystemExit(sys.version_info < (3, 8))' || { + echo "VEXDB_LITE_PYTHON 需要 Python 3.8 或更高版本：$PYTHON_BIN" >&2 + exit 1 + } +else + for candidate in \ + "$(command -v python3 2>/dev/null || true)" \ + /opt/anaconda3/bin/python3 \ + /opt/homebrew/bin/python3 \ + /usr/bin/python3; do + [ -n "$candidate" ] && [ -x "$candidate" ] || continue + if "$candidate" -c 'import sys; raise SystemExit(sys.version_info < (3, 8))' \ + >/dev/null 2>&1; then + PYTHON_BIN="$candidate" + break + fi + done + [ -n "$PYTHON_BIN" ] || { echo "打包测试需要 Python 3.8 或更高版本" >&2; exit 1; } +fi + NCPU="$(sysctl -n hw.ncpu 2>/dev/null || echo 4)" BUILD_JOBS="${VEXDB_LITE_BUILD_JOBS:-$NCPU}" XCODE_JOBS="${VEXDB_LITE_XCODE_JOBS:-2}" +FSKIT_SETTLE_SECONDS="${VEXDB_LITE_FSKIT_SETTLE_SECONDS:-20}" case "$BUILD_JOBS:$XCODE_JOBS" in *[!0-9:]*|0:*|*:0) echo "构建并行度必须是正整数" >&2; exit 2 ;; esac +case "$FSKIT_SETTLE_SECONDS" in + ''|*[!0-9]*) echo "FSKit 收尾等待秒数必须是非负整数" >&2; exit 2 ;; +esac # 打包是交付步骤，不需要占满开发机。默认最多四个 C/C++ 编译任务，Xcode 最多两个。 [ "$BUILD_JOBS" -le 4 ] || BUILD_JOBS=4 BUILD_DIR="$SQLITE_DIR/build-vexdb-lite-package-$ARCH" @@ -79,20 +122,63 @@ STAGE="$DIST_DIR/$PACKAGE_NAME" ZIP="$DIST_DIR/$PACKAGE_NAME.zip" GUIDE="$DIST_DIR/$PACKAGE_NAME-使用说明.md" APP_NAME="VexDB Lite.app" +PAYLOAD_RELATIVE=".payload/$APP_NAME" +PAYLOAD_APP="$STAGE/$PAYLOAD_RELATIVE" LSREGISTER="/System/Library/Frameworks/CoreServices.framework/Frameworks/LaunchServices.framework/Support/lsregister" +FSKIT_RESTORE_REQUIRED=false + +fskit_mounts_active() { + local active_mounts + active_mounts="$(/sbin/mount 2>/dev/null | tr '[:upper:]' '[:lower:]')" + case "$active_mounts" in + *"(vexfs,"*|*"(exfat,"*|*"(msdos,"*|*"(ftp,"*|\ + *" type vexfs"*|*" type exfat"*|*" type msdos"*|*" type ftp"*) + return 0 + ;; + esac + return 1 +} + +restart_current_user_fskit_agent() { + local current_uid old_pids pid attempt still_running + if fskit_mounts_active; then + echo "检测到正在使用的 FSKit 文件系统；拒绝刷新 fskit_agent。" >&2 + return 1 + fi + current_uid="$(id -u 2>/dev/null || true)" + [ -n "$current_uid" ] || return 1 + old_pids="$(/usr/bin/pgrep -u "$current_uid" -x fskit_agent 2>/dev/null || true)" + [ -n "$old_pids" ] || return 0 + /usr/bin/pkill -KILL -u "$current_uid" -x fskit_agent >/dev/null 2>&1 || return 1 + for attempt in 1 2 3 4 5; do + still_running=false + for pid in $old_pids; do + if /bin/kill -0 "$pid" >/dev/null 2>&1; then + still_running=true + break + fi + done + [ "$still_running" = false ] && return 0 + sleep 0.2 + done + return 1 +} unregister_temporary_fskit_apps() { [ -x "$LSREGISTER" ] || return 0 local app for app in \ + "$DERIVED_DATA/Build/Products/Release/$APP_NAME" \ "$ARCHIVE_PATH/Products/Applications/$APP_NAME" \ "$EXPORT_DIR/$APP_NAME" \ "$DERIVED_DATA/Build/Intermediates.noindex/ArchiveIntermediates/VexFSApp/InstallationBuildProductsLocation/Applications/$APP_NAME" \ "$DERIVED_DATA/Build/Intermediates.noindex/ArchiveIntermediates/VexFSApp/BuildProductsPath/Release/$APP_NAME"; do [ -d "$app" ] || continue - # xcodebuild 的登记可能延迟落盘。重复撤销只作用于这两个构建路径， + # Xcode 会自动登记这些构建 App；必须在删除目录前撤销。重复撤销只 + # 作用于当前仍真实存在的确定构建路径， # 不会清除用户已经批准的 extension 启用状态。 "$LSREGISTER" -u "$app" >/dev/null 2>&1 || true + FSKIT_RESTORE_REQUIRED=true done } @@ -113,17 +199,68 @@ restore_installed_fskit_app() { if command -v pluginkit >/dev/null; then pluginkit -a "$installed/Contents/Extensions/VexFSAppEx.appex" >/dev/null 2>&1 || true fi - local mounts - mounts="$(/sbin/mount 2>/dev/null | tr '[:upper:]' '[:lower:]')" - case "$mounts" in - *"(vexfs,"*|*"(exfat,"*|*"(msdos,"*|*" type vexfs"*|*" type exfat"*|*" type msdos"*) ;; - *) pkill -KILL -x fskit_agent >/dev/null 2>&1 || true ;; - esac + # Xcode may rotate the registered extension UUID while temporary products + # are removed. Refresh the idle per-user FSKit agent only after registration + # is stable; do not rotate pkd again and create another identity. + restart_current_user_fskit_agent || { + echo "无法刷新当前用户的 fskit_agent" >&2 + return 1 + } + sleep 1 + /usr/bin/open -gj "$installed" >/dev/null 2>&1 || true + sleep 2 + + local state plugin_state doctor_cli candidate attempt healthy + doctor_cli="" + for candidate in \ + "$HOME/.local/bin/vexdb" \ + "$STAGE/bin/vexdb"; do + if [ -x "$candidate" ]; then + doctor_cli="$candidate" + break + fi + done + healthy=0 + for attempt in 1 2 3 4 5 6 7 8; do + if [ -n "$doctor_cli" ]; then + state="$("$doctor_cli" fs --json doctor 2>/dev/null || true)" + case "$state" in + *'"extension":"service-unavailable"'*) healthy=0 ;; + *'"extension":'*) healthy=$((healthy + 1)) ;; + *) healthy=0 ;; + esac + elif command -v pluginkit >/dev/null; then + plugin_state="$(pluginkit -mAvv -p com.apple.fskit.fsmodule 2>&1 || true)" + case "$plugin_state" in + *'Connection invalid'*) healthy=0 ;; + *'io.vexdb.vexfs.extension'*) healthy=$((healthy + 1)) ;; + *) healthy=0 ;; + esac + fi + [ "$healthy" -lt 3 ] || return 0 + sleep 1 + done + if [ "$SIGN_MODE" = ad-hoc ]; then + echo "警告：当前用户的 FSKit 服务不可查询；ad-hoc 包未安装到系统，跳过服务恢复 Gate。" >&2 + return 0 + fi + echo "清理 Xcode 临时扩展后，当前用户的 FSKit 服务仍不可连接" >&2 + return 1 } cleanup_fskit_build_registration() { unregister_temporary_fskit_apps - restore_installed_fskit_app + # The zip is the deliverable. Keeping an expanded signed App below dist/ + # lets LaunchServices discover a second copy of the same FSKit module. + rm -rf "$STAGE" + if [ "$FSKIT_RESTORE_REQUIRED" = true ]; then + # Let LaunchServices finish every notification caused by Xcode and by + # deleting the expanded stage. Recover only once, after no more bundle + # work remains, so a delayed notification cannot invalidate FSKit again. + sleep "$FSKIT_SETTLE_SECONDS" + restore_installed_fskit_app + FSKIT_RESTORE_REQUIRED=false + fi } # archive/export 即使在后续步骤失败也不能留在 FSKit 的候选模块列表里。 @@ -131,13 +268,27 @@ trap cleanup_fskit_build_registration EXIT mkdir -p "$BUILD_DIR" "$DIST_DIR" +if fskit_mounts_active; then + echo "检测到正在使用的 FSKit 文件系统；请先卸载相关卷，再构建 macOS 包。" >&2 + exit 1 +fi + echo "=== 构建 VexDB-Lite CLI 与 SQLite 扩展 ===" "$CMAKE_BIN" -S "$SQLITE_DIR" -B "$BUILD_DIR" \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_OSX_ARCHITECTURES="$ARCH" \ -DVEXDB_LITE_VERSION="$VERSION" \ + -DVEXFS_REQUIRE_POSTGRESQL=ON \ + -DVEXFS_BUNDLE_POSTGRESQL_RUNTIME=ON \ -DVEXDB_SQLITE_BUILD_TESTS=OFF "$CMAKE_BIN" --build "$BUILD_DIR" --target vexdb_cli vexdb_lite_loadable -j "$BUILD_JOBS" +PG_CLIENT_LIBRARY="$(sed -n 's/^VEXFS_POSTGRESQL_CLIENT_LIBRARY:FILEPATH=//p' \ + "$BUILD_DIR/CMakeCache.txt")" +[ -f "$PG_CLIENT_LIBRARY" ] || { + echo "构建没有解析到可打包的 libpq：${PG_CLIENT_LIBRARY:-missing}" >&2 + exit 1 +} +PG_RUNTIME_SEARCH_DIR="$(cd "$(dirname "$PG_CLIENT_LIBRARY")" && pwd)" echo "=== 构建 VexDB Lite App 与 VexFS extension ===" if [ "$SIGN_MODE" = developer-id ]; then @@ -151,6 +302,8 @@ if [ "$SIGN_MODE" = developer-id ]; then -archivePath "$ARCHIVE_PATH" \ -destination 'generic/platform=macOS' \ ARCHS="$ARCH" ONLY_ACTIVE_ARCH=YES \ + MARKETING_VERSION="$BUNDLE_MARKETING_VERSION" \ + CURRENT_PROJECT_VERSION="$BUNDLE_BUILD_VERSION" \ DEVELOPMENT_TEAM="$TEAM_ID" CODE_SIGN_STYLE=Automatic \ -allowProvisioningUpdates archive # InstallationBuildProductsLocation 可能在 exportArchive 期间被 Xcode 删除； @@ -172,23 +325,41 @@ else -scheme VexFSApp \ -configuration Release \ -derivedDataPath "$DERIVED_DATA" \ - ARCHS="$ARCH" ONLY_ACTIVE_ARCH=YES CODE_SIGNING_ALLOWED=NO build + ARCHS="$ARCH" ONLY_ACTIVE_ARCH=YES \ + MARKETING_VERSION="$BUNDLE_MARKETING_VERSION" \ + CURRENT_PROJECT_VERSION="$BUNDLE_BUILD_VERSION" \ + CODE_SIGNING_ALLOWED=NO build APP_SOURCE="$DERIVED_DATA/Build/Products/Release/$APP_NAME" fi [ -d "$APP_SOURCE" ] || { echo "没有找到 $APP_NAME" >&2; exit 1; } [ -x "$BUILD_DIR/vexdb" ] || { echo "没有找到 vexdb CLI" >&2; exit 1; } [ -f "$BUILD_DIR/vexdb_lite.dylib" ] || { echo "没有找到 vexdb_lite.dylib" >&2; exit 1; } +for bundle in \ + "$APP_SOURCE" \ + "$APP_SOURCE/Contents/Extensions/VexFSAppEx.appex"; do + actual_marketing="$(plutil -extract CFBundleShortVersionString raw -o - \ + "$bundle/Contents/Info.plist")" + actual_build="$(plutil -extract CFBundleVersion raw -o - \ + "$bundle/Contents/Info.plist")" + [ "$actual_marketing" = "$BUNDLE_MARKETING_VERSION" ] || { + echo "Bundle marketing version 错误：$bundle（$actual_marketing）" >&2 + exit 1 + } + [ "$actual_build" = "$BUNDLE_BUILD_VERSION" ] || { + echo "Bundle build version 错误：$bundle（$actual_build）" >&2 + exit 1 + } +done rm -rf "$STAGE" -mkdir -p "$STAGE/bin" "$STAGE/lib" -ditto "$APP_SOURCE" "$STAGE/$APP_NAME" +mkdir -p "$STAGE/bin" "$STAGE/lib/runtime" "$STAGE/.payload" +ditto "$APP_SOURCE" "$PAYLOAD_APP" if [ "$SIGN_MODE" = developer-id ]; then # 先复制交付 App，再撤销并删除 Xcode 的临时同 ID App。这样 LaunchServices # 后续不会重新发现 .xcarchive 并让 FSKit 加载到已经失效的构建路径。 sleep 2 unregister_temporary_fskit_apps rm -rf "$ARCHIVE_PATH" "$EXPORT_DIR" "$DERIVED_DATA" - restore_installed_fskit_app fi install -m 0755 "$BUILD_DIR/vexdb" "$STAGE/bin/vexdb" ln -s vexdb "$STAGE/bin/vexfs" @@ -201,7 +372,34 @@ install -m 0644 "$PACKAGE_DIR/使用说明.md" "$STAGE/使用说明.md" install -m 0644 "$PACKAGE_DIR/使用说明.md" "$GUIDE" install -m 0644 "$ROOT/LICENSE" "$STAGE/LICENSE" -EXTENSION="$STAGE/$APP_NAME/Contents/Extensions/VexFSAppEx.appex" +EXTENSION="$PAYLOAD_APP/Contents/Extensions/VexFSAppEx.appex" +EXTENSION_EXECUTABLE="$EXTENSION/Contents/MacOS/VexFSAppEx" +EXTENSION_FRAMEWORKS="$EXTENSION/Contents/Frameworks" +[ -x "$EXTENSION_EXECUTABLE" ] || { echo "FSKit extension 主程序缺失" >&2; exit 1; } +EXTENSION_SIGN_ENTITLEMENTS="$SCRIPT_DIR/VexFSAppEx/VexFSAppEx.entitlements" +APP_SIGN_ENTITLEMENTS="$SCRIPT_DIR/VexFSApp/VexFSApp.entitlements" +if [ "$SIGN_MODE" = developer-id ]; then + # Xcode injects the application identifier and team identifier required by + # FSKit when it exports the Developer ID App. Bundling libpq invalidates the + # original signature, so preserve the complete exported entitlements before + # re-signing instead of falling back to the smaller source plist. + EXTENSION_SIGN_ENTITLEMENTS="$BUILD_DIR/exported-extension-entitlements.plist" + APP_SIGN_ENTITLEMENTS="$BUILD_DIR/exported-app-entitlements.plist" + codesign -d --entitlements - --xml "$EXTENSION" 2>/dev/null \ + > "$EXTENSION_SIGN_ENTITLEMENTS" + codesign -d --entitlements - --xml "$PAYLOAD_APP" 2>/dev/null \ + > "$APP_SIGN_ENTITLEMENTS" + plutil -lint "$EXTENSION_SIGN_ENTITLEMENTS" "$APP_SIGN_ENTITLEMENTS" >/dev/null +fi + +# The unpacked package and the installed CLI have different relative library +# layouts, so CMake wrote both safe runpaths when it linked the executable. The +# bundler removes any remaining build-machine paths from transitive libraries. +bash "$SCRIPT_DIR/bundle_pg_runtime.sh" \ + "$STAGE/bin/vexdb" "$STAGE/lib/runtime" "$ARCH" "$PG_RUNTIME_SEARCH_DIR" +bash "$SCRIPT_DIR/bundle_pg_runtime.sh" \ + "$EXTENSION_EXECUTABLE" "$EXTENSION_FRAMEWORKS" "$ARCH" "$PG_RUNTIME_SEARCH_DIR" + SIGNATURE=ad-hoc SIGNING_IDENTITY=- SIGNING_TEAM=none @@ -209,14 +407,24 @@ CERTIFICATE_SHA1=none if [ "$SIGN_MODE" = developer-id ]; then CERT_PREFIX="$BUILD_DIR/vexfs-app-certificate-" rm -f "${CERT_PREFIX}"* - codesign -d --extract-certificates="$CERT_PREFIX" "$STAGE/$APP_NAME" + codesign -d --extract-certificates="$CERT_PREFIX" "$PAYLOAD_APP" CERTIFICATE_SHA1="$(openssl x509 -inform DER -in "${CERT_PREFIX}0" -noout -fingerprint -sha1 | cut -d= -f2 | tr -d ':')" [ -n "$CERTIFICATE_SHA1" ] || { echo "无法读取 App 的签名证书" >&2; exit 1; } SIGNING_IDENTITY="$CERTIFICATE_SHA1" SIGNING_TEAM="$TEAM_ID" SIGNATURE=developer-id + find "$STAGE/lib/runtime" "$EXTENSION_FRAMEWORKS" -type f -name '*.dylib' -print0 \ + | while IFS= read -r -d '' library; do + codesign --force --options runtime --timestamp --sign "$SIGNING_IDENTITY" "$library" + done codesign --force --options runtime --timestamp --sign "$SIGNING_IDENTITY" "$STAGE/bin/vexdb" codesign --force --options runtime --timestamp --sign "$SIGNING_IDENTITY" "$STAGE/lib/vexdb_lite.dylib" + codesign --force --options runtime --timestamp --generate-entitlement-der \ + --sign "$SIGNING_IDENTITY" \ + --entitlements "$EXTENSION_SIGN_ENTITLEMENTS" "$EXTENSION" + codesign --force --options runtime --timestamp --generate-entitlement-der \ + --sign "$SIGNING_IDENTITY" \ + --entitlements "$APP_SIGN_ENTITLEMENTS" "$PAYLOAD_APP" verify_team_identifier() { local item="$1" @@ -227,22 +435,62 @@ if [ "$SIGN_MODE" = developer-id ]; then exit 1 fi } - verify_team_identifier "$STAGE/$APP_NAME" + verify_team_identifier "$PAYLOAD_APP" verify_team_identifier "$EXTENSION" verify_team_identifier "$STAGE/bin/vexdb" verify_team_identifier "$STAGE/lib/vexdb_lite.dylib" + find "$STAGE/lib/runtime" "$EXTENSION_FRAMEWORKS" -type f -name '*.dylib' -print0 \ + | while IFS= read -r -d '' library; do verify_team_identifier "$library"; done + SIGNED_EXTENSION_ENTITLEMENTS="$BUILD_DIR/signed-extension-entitlements.plist" + SIGNED_EXTENSION_DIAGNOSTICS="$BUILD_DIR/signed-extension-entitlements.log" + if ! codesign -d --entitlements - --xml "$EXTENSION" \ + >"$SIGNED_EXTENSION_ENTITLEMENTS" 2>"$SIGNED_EXTENSION_DIAGNOSTICS"; then + cat "$SIGNED_EXTENSION_DIAGNOSTICS" >&2 + echo "无法读取重签后的 FSKit 扩展权限" >&2 + exit 1 + fi + if grep -q 'invalid entitlements blob' "$SIGNED_EXTENSION_DIAGNOSTICS" || + ! plutil -lint "$SIGNED_EXTENSION_ENTITLEMENTS" >/dev/null 2>&1; then + cat "$SIGNED_EXTENSION_DIAGNOSTICS" >&2 + echo "FSKit 扩展的 DER 权限无效，系统会忽略这些权限" >&2 + exit 1 + fi + # plutil treats periods as key-path separators. Entitlement names contain + # literal periods, so use PlistBuddy to read the exact top-level keys. + signed_team="$(/usr/libexec/PlistBuddy \ + -c 'Print :com.apple.developer.team-identifier' \ + "$SIGNED_EXTENSION_ENTITLEMENTS" 2>/dev/null || true)" + signed_application="$(/usr/libexec/PlistBuddy \ + -c 'Print :com.apple.application-identifier' \ + "$SIGNED_EXTENSION_ENTITLEMENTS" 2>/dev/null || true)" + [ "$signed_team" = "$TEAM_ID" ] || { + echo "FSKit 扩展缺少 Developer ID team entitlement：${signed_team:-missing}" >&2 + exit 1 + } + [ "$signed_application" = "$TEAM_ID.io.vexdb.vexfs.extension" ] || { + echo "FSKit 扩展 application identifier 错误：${signed_application:-missing}" >&2 + exit 1 + } else # 没有 Developer ID 时仍做完整的 ad-hoc 签名，避免发送过程中破坏 bundle seal。 - codesign --force --sign - --timestamp=none \ - --entitlements "$SCRIPT_DIR/VexFSAppEx/VexFSAppEx.entitlements" "$EXTENSION" - codesign --force --sign - --timestamp=none \ - --entitlements "$SCRIPT_DIR/VexFSApp/VexFSApp.entitlements" "$STAGE/$APP_NAME" + find "$STAGE/lib/runtime" "$EXTENSION_FRAMEWORKS" -type f -name '*.dylib' -print0 \ + | while IFS= read -r -d '' library; do + codesign --force --sign - --timestamp=none "$library" + done codesign --force --sign - --timestamp=none "$STAGE/bin/vexdb" codesign --force --sign - --timestamp=none "$STAGE/lib/vexdb_lite.dylib" + codesign --force --sign - --timestamp=none --generate-entitlement-der \ + --entitlements "$EXTENSION_SIGN_ENTITLEMENTS" "$EXTENSION" + codesign --force --sign - --timestamp=none --generate-entitlement-der \ + --entitlements "$APP_SIGN_ENTITLEMENTS" "$PAYLOAD_APP" fi -codesign --verify --deep --strict --verbose=2 "$STAGE/$APP_NAME" +codesign --verify --deep --strict --verbose=2 "$PAYLOAD_APP" codesign --verify --strict --verbose=2 "$STAGE/bin/vexdb" codesign --verify --strict --verbose=2 "$STAGE/lib/vexdb_lite.dylib" +find "$STAGE/lib/runtime" "$EXTENSION_FRAMEWORKS" -type f -name '*.dylib' -print0 \ + | while IFS= read -r -d '' library; do + codesign --verify --strict --verbose=2 "$library" + done echo "=== 验证统一 SQLite、向量和文件入口 ===" bash "$ROOT/agent_files/cli/test/vexdb_unified_smoke.sh" "$STAGE/bin/vexdb" @@ -255,6 +503,8 @@ write_manifest() { echo "product=VexDB-Lite" echo "filesystem=VexFS" echo "preview_version=$VERSION" + echo "bundle_marketing_version=$BUNDLE_MARKETING_VERSION" + echo "bundle_build_version=$BUNDLE_BUILD_VERSION" echo "contract_version=0.9.0" echo "mount_abi_version=$RUNTIME_ABI" echo "platform=macOS" @@ -284,9 +534,14 @@ write_hashes() { bin/vexdb \ lib/vexdb_lite.dylib \ 使用说明.md \ - "$APP_NAME/Contents/MacOS/VexDB Lite" \ - "$APP_NAME/Contents/Extensions/VexFSAppEx.appex/Contents/MacOS/VexFSAppEx" \ + "$PAYLOAD_RELATIVE/Contents/MacOS/VexDB Lite" \ + "$PAYLOAD_RELATIVE/Contents/Extensions/VexFSAppEx.appex/Contents/MacOS/VexFSAppEx" \ > SHA256SUMS.txt + find lib/runtime \ + "$PAYLOAD_RELATIVE/Contents/Extensions/VexFSAppEx.appex/Contents/Frameworks" \ + -type f -name '*.dylib' -print | LC_ALL=C sort | while IFS= read -r library; do + shasum -a 256 "$library" + done >> SHA256SUMS.txt ) } @@ -303,33 +558,43 @@ write_zip() { ) } -verify_zip() { - local verify_dir verify_root +verify_zip() ( + local verify_dir verify_root verify_app verify_dir="$(mktemp -d "$BUILD_DIR/zip-verify.XXXXXX")" verify_root="$verify_dir/$PACKAGE_NAME" + verify_app="$verify_root/$PAYLOAD_RELATIVE" + cleanup_zip_verification() { + rm -rf "$verify_dir" + } + trap cleanup_zip_verification EXIT ( cd "$DIST_DIR" shasum -a 256 -c "$(basename "$ZIP").sha256" ) ditto -x -k "$ZIP" "$verify_dir" - [ -d "$verify_root/$APP_NAME" ] || { echo "压缩包缺少 $APP_NAME" >&2; return 1; } + [ -d "$verify_app" ] || { echo "压缩包缺少 $PAYLOAD_RELATIVE" >&2; return 1; } [ -x "$verify_root/bin/vexdb" ] || { echo "压缩包缺少 bin/vexdb" >&2; return 1; } [ -L "$verify_root/bin/vexfs" ] || { echo "压缩包中的 bin/vexfs 不是兼容链接" >&2; return 1; } [ -f "$verify_root/lib/vexdb_lite.dylib" ] || { echo "压缩包缺少 SQLite 扩展" >&2; return 1; } + [ -f "$verify_root/lib/runtime/libpq.5.dylib" ] || { echo "压缩包缺少 PostgreSQL runtime" >&2; return 1; } [ -f "$verify_root/使用说明.md" ] || { echo "压缩包缺少使用说明" >&2; return 1; } - codesign --verify --deep --strict --verbose=2 "$verify_root/$APP_NAME" + codesign --verify --deep --strict --verbose=2 "$verify_app" codesign --verify --strict --verbose=2 "$verify_root/bin/vexdb" codesign --verify --strict --verbose=2 "$verify_root/lib/vexdb_lite.dylib" + find "$verify_root/lib/runtime" \ + "$verify_app/Contents/Extensions/VexFSAppEx.appex/Contents/Frameworks" \ + -type f -name '*.dylib' -print0 | while IFS= read -r -d '' library; do + codesign --verify --strict --verbose=2 "$library" + done bash "$ROOT/agent_files/cli/test/vexdb_unified_smoke.sh" "$verify_root/bin/vexdb" - rm -rf "$verify_dir" -} +) write_manifest write_hashes bash "$ROOT/tests/eval/vexfs/documentation_smoke.sh" "$STAGE" -VEXDB_LITE_PACKAGE_STAGE="$STAGE" python3 "$ROOT/tests/eval/vexfs/run.py" \ +VEXDB_LITE_PACKAGE_STAGE="$STAGE" "$PYTHON_BIN" "$ROOT/tests/eval/vexfs/run.py" \ --mode quick --filter package.unified-install --include-package --fail-on-skip \ --build-dir "$BUILD_DIR" write_zip @@ -350,9 +615,9 @@ if [ "$SIGN_MODE" = developer-id ] && [ -n "$NOTARY_PROFILE" ]; then exit 1 fi NOTARIZATION_STATUS=accepted - xcrun stapler staple "$STAGE/$APP_NAME" - xcrun stapler validate "$STAGE/$APP_NAME" - spctl --assess --type execute --verbose=4 "$STAGE/$APP_NAME" + xcrun stapler staple "$PAYLOAD_APP" + xcrun stapler validate "$PAYLOAD_APP" + spctl --assess --type execute --verbose=4 "$PAYLOAD_APP" write_manifest write_hashes bash "$ROOT/tests/eval/vexfs/documentation_smoke.sh" "$STAGE" @@ -360,6 +625,11 @@ if [ "$SIGN_MODE" = developer-id ] && [ -n "$NOTARY_PROFILE" ]; then verify_zip fi +# 成功返回前先完成临时注册清理并验证用户级 FSKit 服务。失败路径仍由 EXIT trap +# 执行同样的尽力清理。 +cleanup_fskit_build_registration +trap - EXIT + echo "" echo "=== VexDB-Lite 技术预览包 ===" ls -lh "$ZIP" "$ZIP.sha256" "$GUIDE" diff --git a/agent_files/mount/common/include/vexfs_runtime_admin.h b/agent_files/mount/common/include/vexfs_runtime_admin.h index 0a95063e57..47f0fef789 100644 --- a/agent_files/mount/common/include/vexfs_runtime_admin.h +++ b/agent_files/mount/common/include/vexfs_runtime_admin.h @@ -72,9 +72,6 @@ vexfs_mount_status vexfs_mount_snapshot_restore(vexfs_mount_session *session, int64_t *new_commit, vexfs_mount_error *error); // Quota values use -1 for unlimited and non-negative values for a hard limit. -vexfs_mount_status vexfs_mount_quota_get(vexfs_mount_session *session, - vexfs_mount_bytes *json, - vexfs_mount_error *error); vexfs_mount_status vexfs_mount_quota_set(vexfs_mount_session *session, int64_t max_bytes, int64_t max_files, int64_t max_file_bytes, diff --git a/agent_files/mount/common/include/vexfs_runtime_io.h b/agent_files/mount/common/include/vexfs_runtime_io.h index 2e96be51f6..7af7df9725 100644 --- a/agent_files/mount/common/include/vexfs_runtime_io.h +++ b/agent_files/mount/common/include/vexfs_runtime_io.h @@ -83,6 +83,11 @@ vexfs_mount_status vexfs_mount_refresh_visibility(vexfs_mount_session *session, vexfs_mount_visibility *visibility, vexfs_mount_error *error); +// 平台 adapter 用它实现 statfs。限制值为 null 时，后端没有设置硬上限。 +vexfs_mount_status vexfs_mount_quota_get(vexfs_mount_session *session, + vexfs_mount_bytes *json, + vexfs_mount_error *error); + vexfs_mount_status vexfs_mount_handle_open(vexfs_mount_session *session, const char *path, const char *flags, const char *request_id, vexfs_mount_bytes *handle, diff --git a/agent_files/mount/common/include/vexfs_runtime_types.h b/agent_files/mount/common/include/vexfs_runtime_types.h index f1c4d644c4..f8b0eebd2d 100644 --- a/agent_files/mount/common/include/vexfs_runtime_types.h +++ b/agent_files/mount/common/include/vexfs_runtime_types.h @@ -13,6 +13,7 @@ extern "C" { #define VEXFS_RUNTIME_OPEN_NO_CREATE 1u #define VEXFS_RUNTIME_EXCLUSIVE_GATEWAY 2u #define VEXFS_RUNTIME_BACKEND_SQLITE "sqlite" +#define VEXFS_RUNTIME_BACKEND_POSTGRESQL "postgresql" #define VEXFS_MOUNT_SYMLINK_MAX 4096u #define VEXFS_MOUNT_TIME_ACCESS 1u #define VEXFS_MOUNT_TIME_MODIFY 2u diff --git a/agent_files/mount/common/src/vexfs_mount_contract_sqlite.cpp b/agent_files/mount/common/src/vexfs_mount_contract_sqlite.cpp index 0a1d022da9..8aa0cff4db 100644 --- a/agent_files/mount/common/src/vexfs_mount_contract_sqlite.cpp +++ b/agent_files/mount/common/src/vexfs_mount_contract_sqlite.cpp @@ -4,6 +4,10 @@ #include "agent_files/vexfs_sqlite.h" #include "vexdb_sqlite.h" +#if defined(VEXFS_HAVE_LIBPQ) +#include +#endif + #include #include #include @@ -14,6 +18,7 @@ #include #include #include +#include #if defined(_WIN32) #include @@ -27,6 +32,10 @@ struct vexfs_mount_session { sqlite3 *db = nullptr; +#if defined(VEXFS_HAVE_LIBPQ) + PGconn *pg = nullptr; +#endif + bool postgresql = false; std::string workspace; std::string principal; std::string session_id; @@ -34,7 +43,9 @@ struct vexfs_mount_session { bool session_started = false; bool no_create = false; bool synchronous_full = true; + uint32_t operation_timeout_ms = 0; std::chrono::steady_clock::time_point next_heartbeat{}; + std::chrono::steady_clock::time_point next_visibility_poll{}; uint64_t contract_calls = 0; uint64_t ordinary_mutation_calls = 0; uint64_t full_boundary_calls = 0; @@ -52,62 +63,327 @@ struct vexfs_mount_session { namespace { constexpr int kUnsupportedBackend = -1001; +constexpr int kPgDatabaseError = -1100; +constexpr int kPgInvalidArgument = -1101; +constexpr int kPgNotFound = -1102; +constexpr int kPgConflict = -1103; +constexpr int kPgReadOnly = -1104; +constexpr int kPgBusy = -1105; +constexpr int kPgPermission = -1106; +constexpr int kPgNoSpace = -1107; +constexpr int kPgCorruption = -1108; +constexpr int kPgNotEmpty = -1109; + +#if defined(VEXFS_HAVE_LIBPQ) +void EnsurePostgreSQLConnection(vexfs_mount_session *session); +#endif class CallError : public std::runtime_error { public: - CallError(int code, const std::string &message) : std::runtime_error(message), code(code) {} + CallError(int code, const std::string &message, + const std::string &backend = VEXFS_RUNTIME_BACKEND_SQLITE) + : std::runtime_error(message), code(code), backend(backend) {} int code; + std::string backend; }; class Call { public: - Call(sqlite3 *db, const char *sql) : db_(db) { - const int rc = sqlite3_prepare_v2(db, sql, -1, &statement_, nullptr); - if (rc != SQLITE_OK) Throw(rc); + Call(sqlite3 *db, const char *sql) : db_(db) { PrepareSQLite(sql); } + Call(vexfs_mount_session *session, const char *sql) { + if (session == nullptr) throw CallError(SQLITE_MISUSE, "mount session is NULL"); +#if defined(VEXFS_HAVE_LIBPQ) + if (session->postgresql) { + EnsurePostgreSQLConnection(session); + pg_ = session->pg; + if (pg_ == nullptr || PQstatus(pg_) != CONNECTION_OK) { + throw CallError(kPgDatabaseError, "PostgreSQL connection is not available", + VEXFS_RUNTIME_BACKEND_POSTGRESQL); + } + PreparePostgreSQL(sql); + return; + } +#endif + db_ = session->db; + PrepareSQLite(sql); } ~Call() { if (statement_ != nullptr) sqlite3_finalize(statement_); +#if defined(VEXFS_HAVE_LIBPQ) + if (pg_result_ != nullptr) PQclear(pg_result_); +#endif } Call(const Call &) = delete; Call &operator=(const Call &) = delete; void Text(int index, const char *value) { if (value == nullptr) throw CallError(SQLITE_MISUSE, "required text is NULL"); +#if defined(VEXFS_HAVE_LIBPQ) + if (pg_ != nullptr) { + PgBind(index, std::string(value), 0, 0, false); + return; + } +#endif Check(sqlite3_bind_text(statement_, index, value, -1, SQLITE_TRANSIENT)); } void Int64(int index, int64_t value) { +#if defined(VEXFS_HAVE_LIBPQ) + if (pg_ != nullptr) { + PgBind(index, std::to_string(value), 0, 0, false); + return; + } +#endif Check(sqlite3_bind_int64(statement_, index, value)); } - void Int(int index, int value) { Check(sqlite3_bind_int(statement_, index, value)); } - void Null(int index) { Check(sqlite3_bind_null(statement_, index)); } + void Int(int index, int value) { Int64(index, value); } + void Null(int index) { +#if defined(VEXFS_HAVE_LIBPQ) + if (pg_ != nullptr) { + PgBind(index, std::string(), 0, 0, true); + return; + } +#endif + Check(sqlite3_bind_null(statement_, index)); + } void Blob(int index, const void *data, uint64_t size) { if (size > static_cast(std::numeric_limits::max())) { throw CallError(SQLITE_TOOBIG, "buffer is too large"); } if (data == nullptr && size != 0) throw CallError(SQLITE_MISUSE, "buffer is NULL"); +#if defined(VEXFS_HAVE_LIBPQ) + if (pg_ != nullptr) { + if (size > static_cast(std::numeric_limits::max())) { + throw CallError(kPgInvalidArgument, + "PostgreSQL parameter is too large", + VEXFS_RUNTIME_BACKEND_POSTGRESQL); + } + PgBind(index, + std::string(static_cast(size == 0 ? "" : data), + static_cast(size)), + 17, 1, false); + return; + } +#endif Check(sqlite3_bind_blob64(statement_, index, size == 0 ? "" : data, size, SQLITE_TRANSIENT)); } void Row() { +#if defined(VEXFS_HAVE_LIBPQ) + if (pg_ != nullptr) { + ExecutePostgreSQL(); + if (PQntuples(pg_result_) < 1) { + throw CallError(kPgNotFound, "PostgreSQL query returned no row", + VEXFS_RUNTIME_BACKEND_POSTGRESQL); + } + return; + } +#endif const int rc = sqlite3_step(statement_); if (rc != SQLITE_ROW) Throw(rc); } bool MaybeRow() { +#if defined(VEXFS_HAVE_LIBPQ) + if (pg_ != nullptr) { + ExecutePostgreSQL(); + return PQntuples(pg_result_) > 0; + } +#endif const int rc = sqlite3_step(statement_); if (rc == SQLITE_ROW) return true; if (rc == SQLITE_DONE) return false; Throw(rc); return false; } - int64_t ResultInt64() const { return sqlite3_column_int64(statement_, 0); } + int64_t ResultInt64() const { +#if defined(VEXFS_HAVE_LIBPQ) + if (pg_ != nullptr) { + const std::string value = ResultText(); + char *end = nullptr; + const long long result = std::strtoll(value.c_str(), &end, 10); + if (end == nullptr || *end != '\0') { + throw CallError(kPgDatabaseError, + "PostgreSQL result is not an integer", + VEXFS_RUNTIME_BACKEND_POSTGRESQL); + } + return static_cast(result); + } +#endif + return sqlite3_column_int64(statement_, 0); + } std::string ResultText() const { +#if defined(VEXFS_HAVE_LIBPQ) + if (pg_ != nullptr) { + if (pg_result_ == nullptr || PQntuples(pg_result_) < 1 || + PQnfields(pg_result_) < 1 || PQgetisnull(pg_result_, 0, 0)) { + return std::string(); + } + return std::string(PQgetvalue(pg_result_, 0, 0), + static_cast(PQgetlength(pg_result_, 0, 0))); + } +#endif const unsigned char *value = sqlite3_column_text(statement_, 0); return value == nullptr ? std::string() : std::string(reinterpret_cast(value)); } - const void *ResultBlob() const { return sqlite3_column_blob(statement_, 0); } - int ResultBytes() const { return sqlite3_column_bytes(statement_, 0); } + const void *ResultBlob() const { +#if defined(VEXFS_HAVE_LIBPQ) + if (pg_ != nullptr) { + PreparePostgreSQLResultBytes(); + return pg_result_bytes_.data(); + } +#endif + return sqlite3_column_blob(statement_, 0); + } + int ResultBytes() const { +#if defined(VEXFS_HAVE_LIBPQ) + if (pg_ != nullptr) { + PreparePostgreSQLResultBytes(); + if (pg_result_bytes_.size() > static_cast(std::numeric_limits::max())) { + throw CallError(kPgInvalidArgument, "PostgreSQL result is too large", + VEXFS_RUNTIME_BACKEND_POSTGRESQL); + } + return static_cast(pg_result_bytes_.size()); + } +#endif + return sqlite3_column_bytes(statement_, 0); + } private: + void PrepareSQLite(const char *sql) { + if (db_ == nullptr) throw CallError(SQLITE_MISUSE, "SQLite connection is NULL"); + const int rc = sqlite3_prepare_v2(db_, sql, -1, &statement_, nullptr); + if (rc != SQLITE_OK) Throw(rc); + } +#if defined(VEXFS_HAVE_LIBPQ) + struct PgParameter { + std::string value; + Oid type = 0; + int format = 0; + bool is_null = true; + }; + + void PreparePostgreSQL(const char *sql) { + pg_sql_.clear(); + const std::string source(sql == nullptr ? "" : sql); + for (size_t index = 0; index < source.size();) { + if (source[index] == '?' && index + 1 < source.size() && + source[index + 1] >= '0' && source[index + 1] <= '9') { + pg_sql_.push_back('$'); + ++index; + while (index < source.size() && source[index] >= '0' && source[index] <= '9') { + pg_sql_.push_back(source[index++]); + } + } else { + pg_sql_.push_back(source[index++]); + } + } + } + + void PgBind(int index, std::string value, Oid type, int format, bool is_null) { + if (index <= 0 || index > 1024) { + throw CallError(kPgInvalidArgument, "PostgreSQL parameter index is invalid", + VEXFS_RUNTIME_BACKEND_POSTGRESQL); + } + if (pg_parameters_.size() < static_cast(index)) { + pg_parameters_.resize(static_cast(index)); + } + pg_parameters_[static_cast(index - 1)] = { + std::move(value), type, format, is_null}; + } + + static int PostgreSQLCode(const char *state) { + if (state == nullptr) return kPgDatabaseError; + const std::string value(state); + if (value == "P0002") return kPgNotFound; + if (value == "40001" || value == "23505") return kPgConflict; + if (value == "42501") return kPgPermission; + if (value == "25006") return kPgReadOnly; + if (value == "53100") return kPgNoSpace; + if (value == "55006" || value == "55P03") return kPgBusy; + if (value == "XX001") return kPgCorruption; + if (value == "2BP01") return kPgNotEmpty; + if (value.rfind("22", 0) == 0 || value == "42809") return kPgInvalidArgument; + return kPgDatabaseError; + } + + void ExecutePostgreSQL() { + if (pg_result_ != nullptr) return; + std::vector types(pg_parameters_.size()); + std::vector values(pg_parameters_.size()); + std::vector lengths(pg_parameters_.size()); + std::vector formats(pg_parameters_.size()); + for (size_t index = 0; index < pg_parameters_.size(); ++index) { + const PgParameter ¶meter = pg_parameters_[index]; + types[index] = parameter.type; + values[index] = parameter.is_null ? nullptr : parameter.value.data(); + lengths[index] = static_cast(parameter.value.size()); + formats[index] = parameter.format; + } + pg_result_ = PQexecParams( + pg_, pg_sql_.c_str(), static_cast(pg_parameters_.size()), + types.empty() ? nullptr : types.data(), + values.empty() ? nullptr : values.data(), + lengths.empty() ? nullptr : lengths.data(), + formats.empty() ? nullptr : formats.data(), 0); + if (pg_result_ == nullptr) { + throw CallError(kPgDatabaseError, PQerrorMessage(pg_), + VEXFS_RUNTIME_BACKEND_POSTGRESQL); + } + const ExecStatusType status = PQresultStatus(pg_result_); + if (status != PGRES_TUPLES_OK) { + const char *state = PQresultErrorField(pg_result_, PG_DIAG_SQLSTATE); + std::string message = PQresultErrorMessage(pg_result_); + if (state != nullptr && *state != '\0') { + message = std::string("SQLSTATE ") + state + ": " + message; + } + throw CallError(PostgreSQLCode(state), message, + VEXFS_RUNTIME_BACKEND_POSTGRESQL); + } + } + + void PreparePostgreSQLResultBytes() const { + if (pg_result_bytes_ready_) return; + pg_result_bytes_ready_ = true; + pg_result_bytes_.clear(); + if (pg_result_ == nullptr || PQntuples(pg_result_) < 1 || + PQnfields(pg_result_) < 1 || PQgetisnull(pg_result_, 0, 0)) return; + if (PQftype(pg_result_, 0) == 17) { + size_t size = 0; + unsigned char *value = PQunescapeBytea( + reinterpret_cast(PQgetvalue(pg_result_, 0, 0)), + &size); + if (value == nullptr) { + throw CallError(kPgDatabaseError, "cannot decode PostgreSQL bytea result", + VEXFS_RUNTIME_BACKEND_POSTGRESQL); + } + pg_result_bytes_.assign(reinterpret_cast(value), size); + PQfreemem(value); + return; + } + pg_result_bytes_.assign(PQgetvalue(pg_result_, 0, 0), + static_cast(PQgetlength(pg_result_, 0, 0))); + if (PQftype(pg_result_, 0) == 114 || PQftype(pg_result_, 0) == 3802) { + std::string compact; + compact.reserve(pg_result_bytes_.size()); + bool quoted = false; + bool escaped = false; + for (const char value : pg_result_bytes_) { + if (quoted) { + compact.push_back(value); + if (escaped) escaped = false; + else if (value == '\\') escaped = true; + else if (value == '"') quoted = false; + } else if (value == '"') { + quoted = true; + compact.push_back(value); + } else if (value != ' ' && value != '\t' && value != '\r' && value != '\n') { + compact.push_back(value); + } + } + pg_result_bytes_ = std::move(compact); + } + } +#endif + void Check(int rc) { if (rc != SQLITE_OK) Throw(rc); } @@ -117,19 +393,95 @@ class Call { sqlite3 *db_ = nullptr; sqlite3_stmt *statement_ = nullptr; +#if defined(VEXFS_HAVE_LIBPQ) + PGconn *pg_ = nullptr; + PGresult *pg_result_ = nullptr; + std::string pg_sql_; + std::vector pg_parameters_; + mutable bool pg_result_bytes_ready_ = false; + mutable std::string pg_result_bytes_; +#endif }; -void ClearError(vexfs_mount_error *error) { +#if defined(VEXFS_HAVE_LIBPQ) +void RequirePostgreSQLCommand(PGconn *connection, PGresult *result, + const char *operation) { + if (result != nullptr && PQresultStatus(result) == PGRES_COMMAND_OK) { + PQclear(result); + return; + } + const std::string message = result == nullptr + ? PQerrorMessage(connection) : PQresultErrorMessage(result); + if (result != nullptr) PQclear(result); + throw CallError(kPgDatabaseError, + std::string(operation) + ": " + message, + VEXFS_RUNTIME_BACKEND_POSTGRESQL); +} + +void EnsurePostgreSQLConnection(vexfs_mount_session *session) { + if (session == nullptr || !session->postgresql || session->pg == nullptr) { + throw CallError(kPgDatabaseError, "PostgreSQL connection is not available", + VEXFS_RUNTIME_BACKEND_POSTGRESQL); + } + if (PQstatus(session->pg) == CONNECTION_OK) return; + + // Only reconnect before a new operation. A command interrupted in flight + // has an unknown commit state, so this layer never replays it implicitly. + PQreset(session->pg); + if (PQstatus(session->pg) != CONNECTION_OK) { + throw CallError(kPgDatabaseError, PQerrorMessage(session->pg), + VEXFS_RUNTIME_BACKEND_POSTGRESQL); + } + if (session->operation_timeout_ms != 0) { + const std::string command = "SET statement_timeout TO " + + std::to_string(session->operation_timeout_ms); + RequirePostgreSQLCommand(session->pg, PQexec(session->pg, command.c_str()), + "restore statement timeout"); + } + RequirePostgreSQLCommand(session->pg, PQexec(session->pg, "LISTEN vexfs_change"), + "restore change notifications"); + if (session->session_started) { + const char *values[] = {session->workspace.c_str(), session->session_id.c_str()}; + PGresult *start = PQexecParams( + session->pg, "SELECT vexfs_mount_session_start($1,$2)", 2, + nullptr, values, nullptr, nullptr, 0); + if (start == nullptr || PQresultStatus(start) != PGRES_TUPLES_OK) { + const std::string message = start == nullptr + ? PQerrorMessage(session->pg) : PQresultErrorMessage(start); + if (start != nullptr) PQclear(start); + throw CallError(kPgDatabaseError, + "restore mount session: " + message, + VEXFS_RUNTIME_BACKEND_POSTGRESQL); + } + PQclear(start); + session->next_heartbeat = + std::chrono::steady_clock::now() + std::chrono::seconds(10); + } +} +#endif + +void ClearError(vexfs_mount_error *error, + const char *backend = VEXFS_RUNTIME_BACKEND_SQLITE) { if (error == nullptr) return; error->status = VEXFS_MOUNT_OK; error->native_code = SQLITE_OK; std::snprintf(error->backend, sizeof(error->backend), "%s", - VEXFS_RUNTIME_BACKEND_SQLITE); + backend); error->message[0] = '\0'; } vexfs_mount_status MapStatus(int sqlite_code, const std::string &) { if (sqlite_code == kUnsupportedBackend) return VEXFS_MOUNT_UNSUPPORTED; + if (sqlite_code == kPgInvalidArgument) return VEXFS_MOUNT_INVALID_ARGUMENT; + if (sqlite_code == kPgNotFound) return VEXFS_MOUNT_NOT_FOUND; + if (sqlite_code == kPgConflict) return VEXFS_MOUNT_CONFLICT; + if (sqlite_code == kPgReadOnly) return VEXFS_MOUNT_READ_ONLY; + if (sqlite_code == kPgBusy) return VEXFS_MOUNT_BUSY; + if (sqlite_code == kPgPermission) return VEXFS_MOUNT_PERMISSION_DENIED; + if (sqlite_code == kPgNoSpace) return VEXFS_MOUNT_NO_SPACE; + if (sqlite_code == kPgCorruption) return VEXFS_MOUNT_CORRUPTION; + if (sqlite_code == kPgNotEmpty) return VEXFS_MOUNT_NOT_EMPTY; + if (sqlite_code == kPgDatabaseError) return VEXFS_MOUNT_DATABASE_ERROR; if (sqlite_code == VEXFS_SQLITE_CONSTRAINT_NOT_EMPTY) return VEXFS_MOUNT_NOT_EMPTY; const int primary = sqlite_code & 0xff; if (primary == SQLITE_BUSY || primary == SQLITE_LOCKED) return VEXFS_MOUNT_BUSY; @@ -145,13 +497,14 @@ vexfs_mount_status MapStatus(int sqlite_code, const std::string &) { } vexfs_mount_status SetError(vexfs_mount_error *error, int sqlite_code, - const std::string &message) { + const std::string &message, + const std::string &backend = VEXFS_RUNTIME_BACKEND_SQLITE) { const vexfs_mount_status status = MapStatus(sqlite_code, message); if (error != nullptr) { error->status = status; error->native_code = sqlite_code; std::snprintf(error->backend, sizeof(error->backend), "%s", - VEXFS_RUNTIME_BACKEND_SQLITE); + backend.c_str()); std::snprintf(error->message, sizeof(error->message), "%s", message.c_str()); } return status; @@ -164,7 +517,7 @@ vexfs_mount_status Guard(vexfs_mount_error *error, Function function) { function(); return VEXFS_MOUNT_OK; } catch (const CallError &exception) { - return SetError(error, exception.code, exception.what()); + return SetError(error, exception.code, exception.what(), exception.backend); } catch (const std::bad_alloc &) { return SetError(error, SQLITE_NOMEM, "out of memory"); } catch (const std::exception &exception) { @@ -177,22 +530,35 @@ vexfs_mount_status Guard(vexfs_mount_error *error, Function function) { template vexfs_mount_status Guard(vexfs_mount_session *session, vexfs_mount_error *error, Function function) { - return Guard(error, [&] { + if (error != nullptr && session != nullptr && session->postgresql) { + ClearError(error, VEXFS_RUNTIME_BACKEND_POSTGRESQL); + } + const vexfs_mount_status status = Guard(error, [&] { if (session == nullptr) throw CallError(SQLITE_MISUSE, "mount session is NULL"); std::lock_guard lock(session->mutex); ++session->contract_calls; function(); }); + if (status == VEXFS_MOUNT_OK && error != nullptr && session != nullptr && + session->postgresql) { + ClearError(error, VEXFS_RUNTIME_BACKEND_POSTGRESQL); + } + return status; } void RequireSession(vexfs_mount_session *session) { - if (session == nullptr || session->db == nullptr) { + if (session == nullptr || + (session->db == nullptr +#if defined(VEXFS_HAVE_LIBPQ) + && session->pg == nullptr +#endif + )) { throw CallError(SQLITE_MISUSE, "mount session is NULL"); } - if (session->exclusive_gateway && session->session_started) { + if (session->session_started) { const auto now = std::chrono::steady_clock::now(); if (now < session->next_heartbeat) return; - Call heartbeat(session->db, "SELECT vexfs_mount_session_heartbeat(?1,?2)"); + Call heartbeat(session, "SELECT vexfs_mount_session_heartbeat(?1,?2)"); heartbeat.Text(1, session->workspace.c_str()); heartbeat.Text(2, session->session_id.c_str()); heartbeat.Row(); @@ -384,7 +750,7 @@ void Exec(sqlite3 *db, const char *sql) { void UseOrdinaryDurability(vexfs_mount_session *session) { ++session->ordinary_mutation_calls; ++session->mutation_epoch; - if (!session->synchronous_full) return; + if (session->postgresql || !session->synchronous_full) return; Exec(session->db, "PRAGMA synchronous=NORMAL"); session->synchronous_full = false; ++session->synchronous_mode_switches; @@ -392,7 +758,7 @@ void UseOrdinaryDurability(vexfs_mount_session *session) { void UseFullDurability(vexfs_mount_session *session) { ++session->full_boundary_calls; - if (session->synchronous_full) return; + if (session->postgresql || session->synchronous_full) return; // fsync/synchronize/snapshot 前切回 FULL。随后必须执行真实写事务；仅修改 // PRAGMA 不会形成持久化屏障。 Exec(session->db, "PRAGMA synchronous=FULL"); @@ -401,7 +767,11 @@ void UseFullDurability(vexfs_mount_session *session) { } void DurabilityBarrier(vexfs_mount_session *session) { - Call barrier(session->db, + if (session->postgresql) { + ++session->durability_barriers; + return; + } + Call barrier(session, "INSERT INTO _vexfs_meta(key,value) VALUES('durability_epoch','1') " "ON CONFLICT(key) DO UPDATE SET value=CAST(CAST(value AS INTEGER)+1 AS TEXT) " "RETURNING value"); @@ -415,7 +785,8 @@ const char *EffectiveRequestId(vexfs_mount_session *session, const char *request // FSKit/libfuse generate a fresh UUID for every callback, so persisting // stage/publish/sync UUIDs cannot deduplicate an OS retry. These operations // are generation-idempotent. Keep open/close/append requests persisted. - if (session->exclusive_gateway && safe_without_persistent_replay) return ""; + if (!session->postgresql && session->exclusive_gateway && + safe_without_persistent_replay) return ""; return request_id; } @@ -445,29 +816,130 @@ extern "C" vexfs_mount_status vexfs_mount_session_open(const vexfs_mount_config if (output != nullptr) *output = nullptr; return Guard(error, [&] { if (config == nullptr || output == nullptr || config->backend == nullptr || - config->connection == nullptr || config->workspace == nullptr || - config->principal == nullptr) { + config->connection == nullptr || config->workspace == nullptr) { throw CallError(SQLITE_MISUSE, - "config, output, backend, connection, workspace and principal are required"); + "config, output, backend, connection and workspace are required"); } if (config->abi_version != VEXFS_RUNTIME_ABI_VERSION) { throw CallError(SQLITE_MISUSE, "unsupported runtime ABI version"); } - if (std::strcmp(config->backend, VEXFS_RUNTIME_BACKEND_SQLITE) != 0) { + const bool sqlite_backend = + std::strcmp(config->backend, VEXFS_RUNTIME_BACKEND_SQLITE) == 0; + const bool postgresql_backend = + std::strcmp(config->backend, VEXFS_RUNTIME_BACKEND_POSTGRESQL) == 0; + if (!sqlite_backend && !postgresql_backend) { throw CallError(kUnsupportedBackend, - "this adapter only supports the sqlite backend"); + "runtime backend must be sqlite or postgresql"); } - if (std::strcmp(config->principal, "local") != 0) { + if (sqlite_backend && + (config->principal == nullptr || std::strcmp(config->principal, "local") != 0)) { throw CallError(SQLITE_AUTH, "sqlite backend is single-user and only accepts the local principal"); } vexfs_mount_session *session = new vexfs_mount_session(); session->workspace = config->workspace; - session->principal = config->principal; + session->principal = config->principal == nullptr ? "" : config->principal; session->session_id = RandomSessionId(); session->exclusive_gateway = (config->flags & VEXFS_RUNTIME_EXCLUSIVE_GATEWAY) != 0; const bool no_create = (config->flags & VEXFS_RUNTIME_OPEN_NO_CREATE) != 0; session->no_create = no_create; + session->operation_timeout_ms = config->operation_timeout_ms; +#if defined(VEXFS_HAVE_LIBPQ) + if (postgresql_backend) { + session->postgresql = true; + session->pg = PQconnectdb(config->connection); + if (session->pg == nullptr || PQstatus(session->pg) != CONNECTION_OK) { + const std::string message = session->pg == nullptr + ? "cannot allocate PostgreSQL connection" + : PQerrorMessage(session->pg); + if (session->pg != nullptr) PQfinish(session->pg); + delete session; + throw CallError(kPgDatabaseError, message, + VEXFS_RUNTIME_BACKEND_POSTGRESQL); + } + try { + if (config->operation_timeout_ms != 0) { + const std::string command = "SET statement_timeout TO " + + std::to_string(config->operation_timeout_ms); + PGresult *result = PQexec(session->pg, command.c_str()); + if (result == nullptr || PQresultStatus(result) != PGRES_COMMAND_OK) { + const std::string message = result == nullptr + ? PQerrorMessage(session->pg) : PQresultErrorMessage(result); + if (result != nullptr) PQclear(result); + throw CallError(kPgDatabaseError, message, + VEXFS_RUNTIME_BACKEND_POSTGRESQL); + } + PQclear(result); + } + Call version(session, "SELECT vexfs_pg_adapter_version()"); + version.Row(); + Call actor(session, "SELECT session_user"); + actor.Row(); + session->principal = actor.ResultText(); + bool workspace_exists = true; + try { + Call workspace(session, "SELECT vexfs_workspace_stat(?1)"); + workspace.Text(1, config->workspace); + workspace.Row(); + } catch (const CallError &exception) { + if (exception.code != kPgNotFound || no_create) throw; + workspace_exists = false; + } + if (!workspace_exists) { + Call create(session, "SELECT vexfs_workspace_create(?1)"); + create.Text(1, config->workspace); + create.Row(); + } + { + PGresult *listen = PQexec(session->pg, "LISTEN vexfs_change"); + if (listen == nullptr || PQresultStatus(listen) != PGRES_COMMAND_OK) { + const std::string message = listen == nullptr + ? PQerrorMessage(session->pg) : PQresultErrorMessage(listen); + if (listen != nullptr) PQclear(listen); + throw CallError(kPgDatabaseError, message, + VEXFS_RUNTIME_BACKEND_POSTGRESQL); + } + PQclear(listen); + } + if (!no_create) { + Call start(session, "SELECT vexfs_mount_session_start(?1,?2)"); + start.Text(1, config->workspace); + start.Text(2, session->session_id.c_str()); + start.Row(); + session->session_started = true; + Call recover(session, + "SELECT vexfs_mount_synchronize(?1,?2,?3)"); + recover.Text(1, config->workspace); + recover.Text(2, ("mount-recover:" + session->session_id).c_str()); + recover.Text(3, session->session_id.c_str()); + recover.Row(); + session->next_heartbeat = + std::chrono::steady_clock::now() + std::chrono::seconds(10); + } + } catch (...) { + if (session->session_started) { + try { + Call end(session, "SELECT vexfs_mount_session_end(?1,?2)"); + end.Text(1, session->workspace.c_str()); + end.Text(2, session->session_id.c_str()); + end.Row(); + } catch (...) { + } + } + PQfinish(session->pg); + delete session; + throw; + } + *output = session; + return; + } +#else + if (postgresql_backend) { + delete session; + throw CallError(kUnsupportedBackend, + "postgresql backend was built without libpq"); + } +#endif PreparePrivateDatabase(config->connection, no_create); int sqlite_open_flags = (no_create ? SQLITE_OPEN_READWRITE : SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE) | @@ -493,12 +965,20 @@ extern "C" vexfs_mount_status vexfs_mount_session_open(const vexfs_mount_config Exec(session->db, "PRAGMA foreign_keys=ON;"); } else { Exec(session->db, - "PRAGMA journal_mode=WAL; PRAGMA synchronous=FULL; PRAGMA foreign_keys=ON;"); + "PRAGMA journal_mode=WAL; PRAGMA synchronous=FULL; PRAGMA foreign_keys=ON;" + // Move automatic checkpoints out of the hottest small-file loop. + // This is a target threshold, not a hard WAL-size limit when readers + // prevent a checkpoint. + "PRAGMA wal_autocheckpoint=8192;" + // One gateway owns one long-lived connection. Keep its suggested page + // cache near 32 MiB so history lookups do not fall back to a tiny + // default cache as the workspace grows. + "PRAGMA cache_size=-32768;"); ProtectSidecar(std::string(config->connection) + "-wal"); ProtectSidecar(std::string(config->connection) + "-shm"); - Call init(session->db, "SELECT vexfs_init()"); + Call init(session, "SELECT vexfs_init()"); init.Row(); - Call workspace(session->db, "SELECT vexfs_workspace_create(?1)"); + Call workspace(session, "SELECT vexfs_workspace_create(?1)"); workspace.Text(1, config->workspace); workspace.Row(); ProtectSidecar(std::string(config->connection) + "-wal"); @@ -506,7 +986,7 @@ extern "C" vexfs_mount_status vexfs_mount_session_open(const vexfs_mount_config if (session->exclusive_gateway) { CheckExistingLocalSession(session->db, config->workspace, session->session_id); - Call start(session->db, "SELECT vexfs_mount_session_start(?1,?2)"); + Call start(session, "SELECT vexfs_mount_session_start(?1,?2)"); start.Text(1, config->workspace); start.Text(2, session->session_id.c_str()); start.Row(); @@ -514,7 +994,7 @@ extern "C" vexfs_mount_status vexfs_mount_session_open(const vexfs_mount_config // A previous gateway may have died after staging writes but before // close. SessionStart claims those retained handles for this new // session; publish them before exposing the remounted workspace. - Call recover(session->db, + Call recover(session, "SELECT vexfs_mount_synchronize(?1,?2,?3)"); recover.Text(1, config->workspace); recover.Text(2, ""); @@ -532,7 +1012,7 @@ extern "C" vexfs_mount_status vexfs_mount_session_open(const vexfs_mount_config } catch (...) { if (session->session_started) { try { - Call end(session->db, "SELECT vexfs_mount_session_end(?1,?2)"); + Call end(session, "SELECT vexfs_mount_session_end(?1,?2)"); end.Text(1, session->workspace.c_str()); end.Text(2, session->session_id.c_str()); end.Row(); @@ -549,12 +1029,16 @@ extern "C" vexfs_mount_status vexfs_mount_session_open(const vexfs_mount_config extern "C" void vexfs_mount_session_close(vexfs_mount_session *session) { if (session == nullptr) return; - if (session->db != nullptr) { + if (session->db != nullptr +#if defined(VEXFS_HAVE_LIBPQ) + || session->pg != nullptr +#endif + ) { std::lock_guard lock(session->mutex); try { if (!session->no_create) { UseFullDurability(session); - Call synchronize(session->db, session->session_started + Call synchronize(session, session->session_started ? "SELECT vexfs_mount_synchronize(?1,'',?2)" : "SELECT vexfs_mount_synchronize(?1,'')"); synchronize.Text(1, session->workspace.c_str()); @@ -564,7 +1048,7 @@ extern "C" void vexfs_mount_session_close(vexfs_mount_session *session) { synchronize.Row(); DurabilityBarrier(session); if (session->session_started) { - Call end(session->db, "SELECT vexfs_mount_session_end(?1,?2)"); + Call end(session, "SELECT vexfs_mount_session_end(?1,?2)"); end.Text(1, session->workspace.c_str()); end.Text(2, session->session_id.c_str()); end.Row(); @@ -574,7 +1058,10 @@ extern "C" void vexfs_mount_session_close(vexfs_mount_session *session) { } catch (...) { // close 不能抛异常；未发布内容仍留在数据库中，后续可诊断和回收。 } - sqlite3_close(session->db); + if (session->db != nullptr) sqlite3_close(session->db); +#if defined(VEXFS_HAVE_LIBPQ) + if (session->pg != nullptr) PQfinish(session->pg); +#endif } delete session; } @@ -584,27 +1071,34 @@ extern "C" vexfs_mount_status vexfs_mount_diagnostics(vexfs_mount_session *sessi vexfs_mount_error *error) { return Guard(session, error, [&] { RequireSession(session); + if (session->postgresql) { + Call diagnostics(session, "SELECT vexfs_diagnostics(?1)"); + diagnostics.Text(1, session->workspace.c_str()); + diagnostics.Row(); + CopyResult(diagnostics, json); + return; + } // doctor 以只读方式打开数据库，不初始化也不改写 schema。 - Call contract(session->db, + Call contract(session, "SELECT value FROM _vexfs_meta WHERE key='contract_version'"); contract.Row(); - Call journal(session->db, "PRAGMA journal_mode"); + Call journal(session, "PRAGMA journal_mode"); journal.Row(); - Call synchronous(session->db, "PRAGMA synchronous"); + Call synchronous(session, "PRAGMA synchronous"); synchronous.Row(); - Call foreign_keys(session->db, "PRAGMA foreign_keys"); + Call foreign_keys(session, "PRAGMA foreign_keys"); foreign_keys.Row(); - Call workspace_exists(session->db, + Call workspace_exists(session, "SELECT count(*) FROM _vexfs_workspaces WHERE name=?1"); workspace_exists.Text(1, session->workspace.c_str()); workspace_exists.Row(); - Call pending(session->db, + Call pending(session, "SELECT count(*) FROM _vexfs_handles WHERE workspace_id=" "(SELECT id FROM _vexfs_workspaces WHERE name=?1) " "AND state IN ('open','retained') AND dirty_generation>published_generation"); pending.Text(1, session->workspace.c_str()); pending.Row(); - Call retained(session->db, + Call retained(session, "SELECT count(*) FROM _vexfs_handles WHERE workspace_id=" "(SELECT id FROM _vexfs_workspaces WHERE name=?1) AND state='retained'"); retained.Text(1, session->workspace.c_str()); @@ -612,7 +1106,7 @@ extern "C" vexfs_mount_status vexfs_mount_diagnostics(vexfs_mount_session *sessi const bool split_staging = sqlite3_table_column_metadata( session->db, "main", "_vexfs_staging", "logical_size", nullptr, nullptr, nullptr, nullptr, nullptr) == SQLITE_OK; - Call staging(session->db, split_staging + Call staging(session, split_staging ? "SELECT COALESCE(sum(s.logical_size),0) FROM _vexfs_staging s " "JOIN _vexfs_handles h ON h.id=s.handle_id WHERE h.workspace_id=" "(SELECT id FROM _vexfs_workspaces WHERE name=?1)" @@ -660,7 +1154,7 @@ extern "C" vexfs_mount_status vexfs_mount_mkdir(vexfs_mount_session *session, return Guard(session, error, [&] { RequireSession(session); UseOrdinaryDurability(session); - Call call(session->db, "SELECT vexfs_mkdir(?1,?2)"); + Call call(session, "SELECT vexfs_mkdir(?1,?2)"); call.Text(1, session->workspace.c_str()); call.Text(2, path); call.Row(); }); } @@ -675,7 +1169,7 @@ extern "C" vexfs_mount_status vexfs_mount_create(vexfs_mount_session *session, if (path == nullptr || kind == nullptr || mode > 0777) { throw CallError(SQLITE_MISUSE, "path, kind and mode 0..0777 are required"); } - Call call(session->db, "SELECT vexfs_create(?1,?2,?3,?4)"); + Call call(session, "SELECT vexfs_create(?1,?2,?3,?4)"); call.Text(1, session->workspace.c_str()); call.Text(2, path); call.Text(3, kind); @@ -693,7 +1187,7 @@ extern "C" vexfs_mount_status vexfs_mount_set_mode(vexfs_mount_session *session, if (inode <= 0 || mode > 0777) { throw CallError(SQLITE_MISUSE, "positive inode and mode 0..0777 are required"); } - Call call(session->db, "SELECT vexfs_set_mode(?1,?2,?3)"); + Call call(session, "SELECT vexfs_set_mode(?1,?2,?3)"); call.Text(1, session->workspace.c_str()); call.Int64(2, inode); call.Int(3, static_cast(mode)); @@ -715,7 +1209,7 @@ extern "C" vexfs_mount_status vexfs_mount_set_times(vexfs_mount_session *session throw CallError(SQLITE_MISUSE, "positive inode, non-negative times and a valid mask are required"); } - Call call(session->db, "SELECT vexfs_set_times(?1,?2,?3,?4,?5)"); + Call call(session, "SELECT vexfs_set_times(?1,?2,?3,?4,?5)"); call.Text(1, session->workspace.c_str()); call.Int64(2, inode); call.Int64(3, accessed_at_ms); @@ -737,7 +1231,7 @@ extern "C" vexfs_mount_status vexfs_mount_symlink(vexfs_mount_session *session, throw CallError(SQLITE_MISUSE, "path and symlink target of 1..4096 bytes are required"); } - Call call(session->db, "SELECT vexfs_symlink(?1,?2,?3)"); + Call call(session, "SELECT vexfs_symlink(?1,?2,?3)"); call.Text(1, session->workspace.c_str()); call.Text(2, path); call.Blob(3, target, target_size); @@ -754,7 +1248,7 @@ extern "C" vexfs_mount_status vexfs_mount_readlink(vexfs_mount_session *session, if (inode <= 0 || target == nullptr) { throw CallError(SQLITE_MISUSE, "positive inode and target output are required"); } - Call call(session->db, "SELECT vexfs_readlink(?1,?2)"); + Call call(session, "SELECT vexfs_readlink(?1,?2)"); call.Text(1, session->workspace.c_str()); call.Int64(2, inode); call.Row(); @@ -772,7 +1266,7 @@ extern "C" vexfs_mount_status vexfs_mount_link(vexfs_mount_session *session, if (source == nullptr || destination == nullptr) { throw CallError(SQLITE_MISUSE, "source and destination are required"); } - Call call(session->db, "SELECT vexfs_link(?1,?2,?3)"); + Call call(session, "SELECT vexfs_link(?1,?2,?3)"); call.Text(1, session->workspace.c_str()); call.Text(2, source); call.Text(3, destination); @@ -792,7 +1286,7 @@ extern "C" vexfs_mount_status vexfs_mount_chown(vexfs_mount_session *session, throw CallError(SQLITE_MISUSE, "positive inode and uid/gid in -1..4294967295 are required"); } - Call call(session->db, "SELECT vexfs_chown(?1,?2,?3,?4)"); + Call call(session, "SELECT vexfs_chown(?1,?2,?3,?4)"); call.Text(1, session->workspace.c_str()); call.Int64(2, inode); call.Int64(3, uid); @@ -809,7 +1303,7 @@ extern "C" vexfs_mount_status vexfs_mount_write_file(vexfs_mount_session *sessio RequireSession(session); UseOrdinaryDurability(session); if (version == nullptr) throw CallError(SQLITE_MISUSE, "version output is NULL"); - Call call(session->db, "SELECT vexfs_write(?1,?2,?3)"); + Call call(session, "SELECT vexfs_write(?1,?2,?3)"); call.Text(1, session->workspace.c_str()); call.Text(2, path); call.Blob(3, data, size); call.Row(); *version = call.ResultInt64(); }); @@ -826,7 +1320,7 @@ extern "C" vexfs_mount_status vexfs_mount_append_file(vexfs_mount_session *sessi throw CallError(SQLITE_MISUSE, "positive inode, content and version output are required"); } - Call call(session->db, "SELECT vexfs_append(?1,?2,?3)"); + Call call(session, "SELECT vexfs_append(?1,?2,?3)"); call.Text(1, session->workspace.c_str()); call.Int64(2, inode); call.Blob(3, data, size); @@ -841,7 +1335,7 @@ extern "C" vexfs_mount_status vexfs_mount_read_file(vexfs_mount_session *session vexfs_mount_error *error) { return Guard(session, error, [&] { RequireSession(session); - Call call(session->db, "SELECT vexfs_read(?1,?2)"); + Call call(session, "SELECT vexfs_read(?1,?2)"); call.Text(1, session->workspace.c_str()); call.Text(2, path); call.Row(); CopyResult(call, content); }); @@ -860,7 +1354,7 @@ extern "C" vexfs_mount_status vexfs_mount_grep(vexfs_mount_session *session, throw CallError(SQLITE_RANGE, "grep needs path, a non-empty pattern, valid flags and limit 1..10240"); } - Call call(session->db, "SELECT vexfs_grep(?1,?2,?3,?4,?5)"); + Call call(session, "SELECT vexfs_grep(?1,?2,?3,?4,?5)"); call.Text(1, session->workspace.c_str()); call.Text(2, path); call.Text(3, pattern); @@ -880,7 +1374,7 @@ extern "C" vexfs_mount_status vexfs_mount_check(vexfs_mount_session *session, if ((flags & ~VEXFS_MOUNT_CHECK_QUICK) != 0) { throw CallError(SQLITE_RANGE, "check flags are invalid"); } - Call call(session->db, "SELECT vexfs_check(?1,?2)"); + Call call(session, "SELECT vexfs_check(?1,?2)"); call.Text(1, session->workspace.c_str()); call.Int(2, (flags & VEXFS_MOUNT_CHECK_QUICK) == 0 ? 1 : 0); call.Row(); @@ -897,8 +1391,15 @@ extern "C" vexfs_mount_status vexfs_mount_grep_index(vexfs_mount_session *sessio if (action == nullptr || action[0] == '\0') { throw CallError(SQLITE_MISUSE, "index action is required"); } - Call call(session->db, "SELECT vexfs_grep_index(?1)"); - call.Text(1, action); + Call call(session, session->postgresql + ? "SELECT vexfs_grep_index(?1,?2)" + : "SELECT vexfs_grep_index(?1)"); + if (session->postgresql) { + call.Text(1, session->workspace.c_str()); + call.Text(2, action); + } else { + call.Text(1, action); + } call.Row(); CopyResult(call, json); }); @@ -910,7 +1411,9 @@ extern "C" vexfs_mount_status vexfs_mount_history(vexfs_mount_session *session, vexfs_mount_error *error) { return Guard(session, error, [&] { RequireSession(session); - Call call(session->db, "SELECT vexfs_history(?1,?2)"); + Call call(session, session->postgresql + ? "SELECT vexfs_history_json(?1,?2,100,0)" + : "SELECT vexfs_history(?1,?2)"); call.Text(1, session->workspace.c_str()); call.Text(2, path); call.Row(); @@ -926,7 +1429,9 @@ extern "C" vexfs_mount_status vexfs_mount_history_page( if (limit == 0 || limit > 1000 || before_version < 0) { throw CallError(SQLITE_RANGE, "history limit must be 1..1000 and before must be non-negative"); } - Call call(session->db, "SELECT vexfs_history(?1,?2,?3,?4)"); + Call call(session, session->postgresql + ? "SELECT vexfs_history_json(?1,?2,?3,?4)" + : "SELECT vexfs_history(?1,?2,?3,?4)"); call.Text(1, session->workspace.c_str()); call.Text(2, path); call.Int64(3, static_cast(limit)); @@ -943,7 +1448,7 @@ extern "C" vexfs_mount_status vexfs_mount_read_version(vexfs_mount_session *sess return Guard(session, error, [&] { RequireSession(session); if (version <= 0) throw CallError(SQLITE_MISUSE, "version must be a positive integer"); - Call call(session->db, "SELECT vexfs_read_version(?1,?2,?3)"); + Call call(session, "SELECT vexfs_read_version(?1,?2,?3)"); call.Text(1, session->workspace.c_str()); call.Text(2, path); call.Int64(3, version); @@ -960,7 +1465,7 @@ extern "C" vexfs_mount_status vexfs_mount_compare_versions( if (from_version <= 0 || to_version <= 0) { throw CallError(SQLITE_RANGE, "versions must be positive integers"); } - Call call(session->db, "SELECT vexfs_compare_versions(?1,?2,?3,?4)"); + Call call(session, "SELECT vexfs_compare_versions(?1,?2,?3,?4)"); call.Text(1, session->workspace.c_str()); call.Text(2, path); call.Int64(3, from_version); @@ -980,7 +1485,7 @@ extern "C" vexfs_mount_status vexfs_mount_restore_version( throw CallError(SQLITE_MISUSE, "target version, expected version and new version output are required"); } - Call call(session->db, "SELECT vexfs_restore_version(?1,?2,?3,?4)"); + Call call(session, "SELECT vexfs_restore_version(?1,?2,?3,?4)"); call.Text(1, session->workspace.c_str()); call.Text(2, path); call.Int64(3, target_version); @@ -995,7 +1500,7 @@ extern "C" vexfs_mount_status vexfs_mount_stat(vexfs_mount_session *session, vexfs_mount_error *error) { return Guard(session, error, [&] { RequireSession(session); - Call call(session->db, "SELECT vexfs_stat(?1,?2)"); + Call call(session, "SELECT vexfs_stat(?1,?2)"); call.Text(1, session->workspace.c_str()); call.Text(2, path); call.Row(); CopyResult(call, json); }); } @@ -1006,7 +1511,7 @@ extern "C" vexfs_mount_status vexfs_mount_path_for_inode(vexfs_mount_session *se vexfs_mount_error *error) { return Guard(session, error, [&] { RequireSession(session); - Call call(session->db, "SELECT vexfs_path(?1,?2)"); + Call call(session, "SELECT vexfs_path(?1,?2)"); call.Text(1, session->workspace.c_str()); call.Int64(2, inode); call.Row(); @@ -1019,7 +1524,9 @@ extern "C" vexfs_mount_status vexfs_mount_list(vexfs_mount_session *session, vexfs_mount_error *error) { return Guard(session, error, [&] { RequireSession(session); - Call call(session->db, "SELECT vexfs_list(?1,?2)"); + Call call(session, session->postgresql + ? "SELECT vexfs_list_json(?1,?2)" + : "SELECT vexfs_list(?1,?2)"); call.Text(1, session->workspace.c_str()); call.Text(2, path); call.Row(); CopyResult(call, json); }); } @@ -1032,10 +1539,11 @@ extern "C" vexfs_mount_status vexfs_mount_list_versioned( if (path == nullptr || json == nullptr) { throw CallError(SQLITE_MISUSE, "path and output are required"); } - Call call(session->db, - "SELECT json_object('version'," - "json_extract(vexfs_stat(?1,?2),'$.version')," - "'entries',json(vexfs_list(?1,?2)))"); + Call call(session, session->postgresql + ? "SELECT vexfs_list_versioned_json(?1,?2)" + : "SELECT json_object('version'," + "json_extract(vexfs_stat(?1,?2),'$.version')," + "'entries',json(vexfs_list(?1,?2)))"); call.Text(1, session->workspace.c_str()); call.Text(2, path); call.Row(); @@ -1049,7 +1557,7 @@ extern "C" vexfs_mount_status vexfs_mount_move(vexfs_mount_session *session, return Guard(session, error, [&] { RequireSession(session); UseOrdinaryDurability(session); - Call call(session->db, "SELECT vexfs_move(?1,?2,?3)"); + Call call(session, "SELECT vexfs_move(?1,?2,?3)"); call.Text(1, session->workspace.c_str()); call.Text(2, source); call.Text(3, destination); call.Row(); }); } @@ -1061,7 +1569,7 @@ extern "C" vexfs_mount_status vexfs_mount_rename(vexfs_mount_session *session, return Guard(session, error, [&] { RequireSession(session); UseOrdinaryDurability(session); - Call call(session->db, "SELECT vexfs_rename(?1,?2,?3,?4)"); + Call call(session, "SELECT vexfs_rename(?1,?2,?3,?4)"); call.Text(1, session->workspace.c_str()); call.Text(2, source); call.Text(3, destination); @@ -1076,7 +1584,7 @@ extern "C" vexfs_mount_status vexfs_mount_remove(vexfs_mount_session *session, return Guard(session, error, [&] { RequireSession(session); UseOrdinaryDurability(session); - Call call(session->db, "SELECT vexfs_remove(?1,?2,?3)"); + Call call(session, "SELECT vexfs_remove(?1,?2,?3)"); call.Text(1, session->workspace.c_str()); call.Text(2, path); call.Int(3, recursive); call.Row(); }); } @@ -1090,7 +1598,7 @@ extern "C" vexfs_mount_status vexfs_mount_xattr_get(vexfs_mount_session *session if (inode <= 0 || name == nullptr || value == nullptr) { throw CallError(SQLITE_MISUSE, "inode, xattr name and output are required"); } - Call call(session->db, "SELECT vexfs_xattr_get(?1,?2,?3)"); + Call call(session, "SELECT vexfs_xattr_get(?1,?2,?3)"); call.Text(1, session->workspace.c_str()); call.Int64(2, inode); call.Text(3, name); @@ -1108,7 +1616,7 @@ extern "C" vexfs_mount_status vexfs_mount_xattr_list(vexfs_mount_session *sessio if (inode <= 0 || json == nullptr) { throw CallError(SQLITE_MISUSE, "inode and output are required"); } - Call call(session->db, "SELECT vexfs_xattr_list(?1,?2)"); + Call call(session, "SELECT vexfs_xattr_list(?1,?2)"); call.Text(1, session->workspace.c_str()); call.Int64(2, inode); call.Row(); @@ -1128,7 +1636,7 @@ extern "C" vexfs_mount_status vexfs_mount_xattr_set(vexfs_mount_session *session (policy != 3 && value == nullptr && size != 0)) { throw CallError(SQLITE_MISUSE, "invalid inode, xattr name, value or policy"); } - Call call(session->db, "SELECT vexfs_xattr_set(?1,?2,?3,?4,?5)"); + Call call(session, "SELECT vexfs_xattr_set(?1,?2,?3,?4,?5)"); call.Text(1, session->workspace.c_str()); call.Int64(2, inode); call.Text(3, name); @@ -1146,7 +1654,7 @@ extern "C" vexfs_mount_status vexfs_mount_acl_get(vexfs_mount_session *session, if (inode <= 0 || json == nullptr) { throw CallError(SQLITE_MISUSE, "inode and output are required"); } - Call call(session->db, "SELECT vexfs_acl_get(?1,?2)"); + Call call(session, "SELECT vexfs_acl_get(?1,?2)"); call.Text(1, session->workspace.c_str()); call.Int64(2, inode); call.Row(); @@ -1163,10 +1671,16 @@ extern "C" vexfs_mount_status vexfs_mount_acl_set(vexfs_mount_session *session, if (inode <= 0 || json == nullptr || size == 0) { throw CallError(SQLITE_MISUSE, "inode and non-empty ACL JSON are required"); } - Call call(session->db, "SELECT vexfs_acl_set(?1,?2,?3)"); + Call call(session, "SELECT vexfs_acl_set(?1,?2,?3)"); call.Text(1, session->workspace.c_str()); call.Int64(2, inode); - call.Blob(3, json, size); + if (session->postgresql) { + const std::string acl(static_cast(json), + static_cast(size)); + call.Text(3, acl.c_str()); + } else { + call.Blob(3, json, size); + } call.Row(); }); } @@ -1177,8 +1691,9 @@ extern "C" vexfs_mount_status vexfs_mount_workspace_head(vexfs_mount_session *se return Guard(session, error, [&] { RequireSession(session); if (head_commit == nullptr) throw CallError(SQLITE_MISUSE, "head output is NULL"); - Call call(session->db, - "SELECT COALESCE(head_commit,0) FROM _vexfs_workspaces WHERE name=?1"); + Call call(session, session->postgresql + ? "SELECT vexfs_workspace_head(?1)" + : "SELECT COALESCE(head_commit,0) FROM _vexfs_workspaces WHERE name=?1"); call.Text(1, session->workspace.c_str()); call.Row(); *head_commit = call.ResultInt64(); @@ -1193,7 +1708,47 @@ extern "C" vexfs_mount_status vexfs_mount_refresh_visibility( if (visibility == nullptr) { throw CallError(SQLITE_MISUSE, "visibility output is NULL"); } - Call data_version(session->db, "PRAGMA data_version"); + if (session->postgresql) { +#if defined(VEXFS_HAVE_LIBPQ) + if (PQconsumeInput(session->pg) == 0) { + throw CallError(kPgDatabaseError, PQerrorMessage(session->pg), + VEXFS_RUNTIME_BACKEND_POSTGRESQL); + } + bool notification = false; + while (PGnotify *notice = PQnotifies(session->pg)) { + notification = true; + PQfreemem(notice); + } + const auto now = std::chrono::steady_clock::now(); + const bool local_mutation = session->visibility_initialized && + session->mutation_epoch != session->observed_mutation_epoch; + const bool periodic_poll = now >= session->next_visibility_poll; + int external_workspace_commit = 0; + if (!session->visibility_initialized || notification || local_mutation || periodic_poll) { + Call head(session, "SELECT vexfs_workspace_head(?1)"); + head.Text(1, session->workspace.c_str()); + head.Row(); + Call generation(session, "SELECT vexfs_cache_generation(?1)"); + generation.Text(1, session->workspace.c_str()); + generation.Row(); + const int64_t current_head = head.ResultInt64(); + const uint64_t current_generation = static_cast( + std::max(0, generation.ResultInt64())); + external_workspace_commit = session->visibility_initialized && notification && + current_head != session->observed_workspace_head ? 1 : 0; + session->observed_workspace_head = current_head; + session->cache_generation = current_generation; + session->observed_mutation_epoch = session->mutation_epoch; + session->visibility_initialized = true; + session->next_visibility_poll = now + std::chrono::seconds(1); + } + visibility->workspace_head = session->observed_workspace_head; + visibility->cache_generation = session->cache_generation; + visibility->external_commit = external_workspace_commit; + return; +#endif + } + Call data_version(session, "PRAGMA data_version"); data_version.Row(); const int64_t current_data_version = data_version.ResultInt64(); const bool external_database_commit = session->visibility_initialized && @@ -1202,7 +1757,7 @@ extern "C" vexfs_mount_status vexfs_mount_refresh_visibility( session->mutation_epoch != session->observed_mutation_epoch; int external_workspace_commit = 0; if (!session->visibility_initialized || external_database_commit || local_mutation) { - Call head(session->db, + Call head(session, "SELECT COALESCE(head_commit,0) FROM _vexfs_workspaces WHERE name=?1"); head.Text(1, session->workspace.c_str()); head.Row(); @@ -1243,13 +1798,13 @@ extern "C" vexfs_mount_status vexfs_mount_snapshot_create(vexfs_mount_session *s // snapshot barrier rejects any dirty handle owned by another session. // This prevents a successful snapshot from silently missing writes. if (!committed_only) { - Call synchronize(session->db, + Call synchronize(session, "SELECT vexfs_mount_synchronize(?1,'',?2)"); synchronize.Text(1, session->workspace.c_str()); synchronize.Text(2, session->session_id.c_str()); synchronize.Row(); } - Call call(session->db, committed_only + Call call(session, committed_only ? "SELECT vexfs_snapshot_create(?1,?2,NULL,'committed-only')" : "SELECT vexfs_snapshot_create(?1,?2)"); call.Text(1, session->workspace.c_str()); @@ -1265,7 +1820,9 @@ extern "C" vexfs_mount_status vexfs_mount_snapshot_list(vexfs_mount_session *ses return Guard(session, error, [&] { RequireSession(session); if (json == nullptr) throw CallError(SQLITE_MISUSE, "output is NULL"); - Call call(session->db, "SELECT vexfs_snapshot_list(?1)"); + Call call(session, session->postgresql + ? "SELECT vexfs_snapshot_list_json(?1)" + : "SELECT vexfs_snapshot_list(?1)"); call.Text(1, session->workspace.c_str()); call.Row(); CopyResult(call, json); @@ -1281,7 +1838,7 @@ extern "C" vexfs_mount_status vexfs_mount_snapshot_show(vexfs_mount_session *ses if (name == nullptr || json == nullptr) { throw CallError(SQLITE_MISUSE, "snapshot name and output are required"); } - Call call(session->db, "SELECT vexfs_snapshot_show(?1,?2)"); + Call call(session, "SELECT vexfs_snapshot_show(?1,?2)"); call.Text(1, session->workspace.c_str()); call.Text(2, name); call.Row(); @@ -1298,7 +1855,7 @@ extern "C" vexfs_mount_status vexfs_mount_snapshot_diff(vexfs_mount_session *ses if (from == nullptr || to == nullptr || json == nullptr) { throw CallError(SQLITE_MISUSE, "from, to and output are required"); } - Call call(session->db, "SELECT vexfs_snapshot_diff(?1,?2,?3)"); + Call call(session, "SELECT vexfs_snapshot_diff(?1,?2,?3)"); call.Text(1, session->workspace.c_str()); call.Text(2, from); call.Text(3, to); @@ -1314,7 +1871,7 @@ extern "C" vexfs_mount_status vexfs_mount_snapshot_drop(vexfs_mount_session *ses RequireSession(session); UseFullDurability(session); if (name == nullptr) throw CallError(SQLITE_MISUSE, "snapshot name is required"); - Call call(session->db, "SELECT vexfs_snapshot_drop(?1,?2)"); + Call call(session, "SELECT vexfs_snapshot_drop(?1,?2)"); call.Text(1, session->workspace.c_str()); call.Text(2, name); call.Row(); @@ -1333,7 +1890,7 @@ extern "C" vexfs_mount_status vexfs_mount_snapshot_restore(vexfs_mount_session * throw CallError(SQLITE_MISUSE, "snapshot name, positive expected head and commit output are required"); } - Call call(session->db, "SELECT vexfs_snapshot_restore(?1,?2,?3)"); + Call call(session, "SELECT vexfs_snapshot_restore(?1,?2,?3)"); call.Text(1, session->workspace.c_str()); call.Text(2, name); call.Int64(3, expected_head); @@ -1348,7 +1905,7 @@ extern "C" vexfs_mount_status vexfs_mount_quota_get( return Guard(session, error, [&] { RequireSession(session); if (json == nullptr) throw CallError(SQLITE_MISUSE, "output is NULL"); - Call call(session->db, "SELECT vexfs_quota_get(?1)"); + Call call(session, "SELECT vexfs_quota_get(?1)"); call.Text(1, session->workspace.c_str()); call.Row(); CopyResult(call, json); @@ -1366,7 +1923,7 @@ extern "C" vexfs_mount_status vexfs_mount_quota_set( throw CallError(SQLITE_RANGE, "quota values must be -1 (unlimited) or non-negative"); } - Call call(session->db, "SELECT vexfs_quota_set(?1,?2,?3,?4)"); + Call call(session, "SELECT vexfs_quota_set(?1,?2,?3,?4)"); call.Text(1, session->workspace.c_str()); if (max_bytes < 0) call.Null(2); else call.Int64(2, max_bytes); if (max_files < 0) call.Null(3); else call.Int64(3, max_files); @@ -1382,7 +1939,7 @@ extern "C" vexfs_mount_status vexfs_mount_retention_get( return Guard(session, error, [&] { RequireSession(session); if (json == nullptr) throw CallError(SQLITE_MISUSE, "output is NULL"); - Call call(session->db, "SELECT vexfs_retention_get(?1)"); + Call call(session, "SELECT vexfs_retention_get(?1)"); call.Text(1, session->workspace.c_str()); call.Row(); CopyResult(call, json); @@ -1397,7 +1954,7 @@ extern "C" vexfs_mount_status vexfs_mount_retention_set( RequireSession(session); UseFullDurability(session); if (json == nullptr) throw CallError(SQLITE_MISUSE, "output is NULL"); - Call call(session->db, "SELECT vexfs_retention_set(?1,?2,?3)"); + Call call(session, "SELECT vexfs_retention_set(?1,?2,?3)"); call.Text(1, session->workspace.c_str()); call.Int64(2, keep_versions); call.Int64(3, keep_days); @@ -1415,7 +1972,7 @@ extern "C" vexfs_mount_status vexfs_mount_gc( if (batch == 0 || batch > 10000 || json == nullptr) { throw CallError(SQLITE_RANGE, "batch must be between 1 and 10000"); } - Call call(session->db, "SELECT vexfs_gc(?1,?2)"); + Call call(session, "SELECT vexfs_gc(?1,?2)"); call.Text(1, session->workspace.c_str()); call.Int64(2, batch); call.Row(); @@ -1432,7 +1989,7 @@ extern "C" vexfs_mount_status vexfs_mount_gc_pause( if ((paused != 0 && paused != 1) || json == nullptr) { throw CallError(SQLITE_RANGE, "paused must be 0 or 1"); } - Call call(session->db, "SELECT vexfs_gc_pause(?1,?2)"); + Call call(session, "SELECT vexfs_gc_pause(?1,?2)"); call.Text(1, session->workspace.c_str()); call.Int(2, paused); call.Row(); @@ -1448,7 +2005,7 @@ extern "C" vexfs_mount_status vexfs_mount_handle_open(vexfs_mount_session *sessi return Guard(session, error, [&] { RequireSession(session); UseOrdinaryDurability(session); - Call call(session->db, "SELECT vexfs_handle_open(?1,?2,?3,?4,?5)"); + Call call(session, "SELECT vexfs_handle_open(?1,?2,?3,?4,?5)"); call.Text(1, session->workspace.c_str()); call.Text(2, path); call.Text(3, flags); call.Text(4, EffectiveRequestId(session, request_id, true)); call.Text(5, session->session_id.c_str()); @@ -1466,7 +2023,7 @@ extern "C" vexfs_mount_status vexfs_mount_handle_create( throw CallError(SQLITE_MISUSE, "path, mode 0..0777, request_id and handle output are required"); } - Call call(session->db, "SELECT vexfs_handle_create(?1,?2,?3,?4,?5)"); + Call call(session, "SELECT vexfs_handle_create(?1,?2,?3,?4,?5)"); call.Text(1, session->workspace.c_str()); call.Text(2, path); call.Int(3, static_cast(mode)); @@ -1487,7 +2044,7 @@ extern "C" vexfs_mount_status vexfs_mount_handle_truncate( throw CallError(SQLITE_MISUSE, "size is too large or generation output is NULL"); } UseOrdinaryDurability(session); - Call call(session->db, "SELECT vexfs_handle_truncate(?1,?2,?3)"); + Call call(session, "SELECT vexfs_handle_truncate(?1,?2,?3)"); call.Text(1, handle); call.Int64(2, static_cast(size)); call.Text(3, EffectiveRequestId(session, request_id, true)); @@ -1505,7 +2062,7 @@ extern "C" vexfs_mount_status vexfs_mount_handle_stage_write( throw CallError(SQLITE_MISUSE, "offset is too large or generation output is NULL"); } UseOrdinaryDurability(session); - Call call(session->db, "SELECT vexfs_handle_stage_write(?1,?2,?3,?4)"); + Call call(session, "SELECT vexfs_handle_stage_write(?1,?2,?3,?4)"); call.Text(1, handle); call.Int64(2, static_cast(offset)); call.Blob(3, data, size); @@ -1526,7 +2083,7 @@ extern "C" vexfs_mount_status vexfs_mount_handle_append( "handle, content, request_id and version output are required"); } UseOrdinaryDurability(session); - Call call(session->db, "SELECT vexfs_handle_append(?1,?2,?3)"); + Call call(session, "SELECT vexfs_handle_append(?1,?2,?3)"); call.Text(1, handle); call.Blob(2, data, size); call.Text(3, request_id); @@ -1546,7 +2103,7 @@ extern "C" vexfs_mount_status vexfs_mount_handle_read(vexfs_mount_session *sessi length > static_cast(std::numeric_limits::max())) { throw CallError(SQLITE_MISUSE, "read range is too large"); } - Call call(session->db, "SELECT vexfs_handle_read(?1,?2,?3)"); + Call call(session, "SELECT vexfs_handle_read(?1,?2,?3)"); call.Text(1, handle); call.Int64(2, static_cast(offset)); call.Int64(3, static_cast(length)); call.Row(); CopyResult(call, content); }); @@ -1564,7 +2121,7 @@ extern "C" vexfs_mount_status vexfs_mount_handle_publish( if (std::strcmp(durability, "full") == 0) UseFullDurability(session); else UseOrdinaryDurability(session); if (version == nullptr) throw CallError(SQLITE_MISUSE, "version output is NULL"); - Call call(session->db, "SELECT vexfs_handle_publish(?1,?2,?3,?4)"); + Call call(session, "SELECT vexfs_handle_publish(?1,?2,?3,?4)"); call.Text(1, handle); call.Int64(2, generation); call.Text(3, durability); call.Text(4, EffectiveRequestId(session, request_id, true)); call.Row(); *version = call.ResultInt64(); @@ -1582,7 +2139,7 @@ extern "C" vexfs_mount_status vexfs_mount_handle_publish_close( } if (std::strcmp(durability, "full") == 0) UseFullDurability(session); else UseOrdinaryDurability(session); - Call call(session->db, "SELECT vexfs_handle_publish_close(?1,?2,?3)"); + Call call(session, "SELECT vexfs_handle_publish_close(?1,?2,?3)"); call.Text(1, handle); call.Int64(2, generation); call.Text(3, durability); @@ -1597,7 +2154,7 @@ extern "C" vexfs_mount_status vexfs_mount_handle_close( return Guard(session, error, [&] { RequireSession(session); UseOrdinaryDurability(session); - Call call(session->db, "SELECT vexfs_handle_close(?1,?2,?3)"); + Call call(session, "SELECT vexfs_handle_close(?1,?2,?3)"); call.Text(1, handle); call.Int(2, retain_unpublished); call.Text(3, EffectiveRequestId(session, request_id, true)); call.Row(); CopyResult(call, state); @@ -1612,7 +2169,7 @@ extern "C" vexfs_mount_status vexfs_mount_synchronize(vexfs_mount_session *sessi RequireSession(session); UseFullDurability(session); if (published == nullptr) throw CallError(SQLITE_MISUSE, "published output is NULL"); - Call call(session->db, "SELECT vexfs_mount_synchronize(?1,?2,?3)"); + Call call(session, "SELECT vexfs_mount_synchronize(?1,?2,?3)"); call.Text(1, session->workspace.c_str()); call.Text(2, EffectiveRequestId(session, request_id, true)); call.Text(3, session->session_id.c_str()); call.Row(); @@ -1630,7 +2187,7 @@ extern "C" vexfs_mount_status vexfs_mount_reclaim(vexfs_mount_session *session, RequireSession(session); UseOrdinaryDurability(session); if (reclaimed == nullptr) throw CallError(SQLITE_MISUSE, "reclaimed output is NULL"); - Call call(session->db, "SELECT vexfs_item_reclaim(?1,?2)"); + Call call(session, "SELECT vexfs_item_reclaim(?1,?2)"); call.Text(1, session->workspace.c_str()); call.Text(2, EffectiveRequestId(session, request_id, true)); call.Row(); diff --git a/agent_files/mount/common/test/vexfs_mount_contract_smoke.cpp b/agent_files/mount/common/test/vexfs_mount_contract_smoke.cpp index 27f301c5e3..b366ff7e5d 100644 --- a/agent_files/mount/common/test/vexfs_mount_contract_smoke.cpp +++ b/agent_files/mount/common/test/vexfs_mount_contract_smoke.cpp @@ -68,8 +68,8 @@ bool Execute(const char *path, const char *sql) { } int RunBenchmark(int file_count) { - if (file_count <= 0 || file_count > 1000) { - std::fprintf(stderr, "benchmark file count must be 1..1000\n"); + if (file_count <= 0 || file_count > 100000) { + std::fprintf(stderr, "benchmark file count must be 1..100000\n"); return 2; } char path[] = "/tmp/vexfs-contract-benchmark-XXXXXX"; @@ -97,6 +97,11 @@ int RunBenchmark(int file_count) { const int64_t versions_before = Scalar(path, "SELECT count(*) FROM _vexfs_file_versions"); const int64_t requests_before = Scalar(path, "SELECT count(*) FROM _vexfs_requests"); const char payload[] = "vexfs benchmark payload\n"; + double create_call_seconds = 0.0; + double stat_call_seconds = 0.0; + double xattr_call_seconds = 0.0; + double stage_call_seconds = 0.0; + double publish_call_seconds = 0.0; const auto started = std::chrono::steady_clock::now(); for (int index = 0; index < file_count; ++index) { char file_path[64]; @@ -104,31 +109,46 @@ int RunBenchmark(int file_count) { std::snprintf(file_path, sizeof(file_path), "/files/f%06d.txt", index); std::snprintf(request, sizeof(request), "create-%06d", index); vexfs_mount_bytes handle{}; + auto call_started = std::chrono::steady_clock::now(); if (vexfs_mount_handle_create(session, file_path, 0644, request, &handle, &error) != VEXFS_MOUNT_OK) return Fail("benchmark create", error); + create_call_seconds += std::chrono::duration( + std::chrono::steady_clock::now() - call_started).count(); const std::string handle_id(static_cast(handle.data), static_cast(handle.size)); vexfs_mount_free(handle.data); vexfs_mount_bytes item_stat{}; + call_started = std::chrono::steady_clock::now(); if (vexfs_mount_stat(session, file_path, &item_stat, &error) != VEXFS_MOUNT_OK) return Fail("benchmark stat", error); + stat_call_seconds += std::chrono::duration( + std::chrono::steady_clock::now() - call_started).count(); const int64_t inode = JsonInteger(item_stat, "inode"); vexfs_mount_free(item_stat.data); + call_started = std::chrono::steady_clock::now(); if (vexfs_mount_xattr_set(session, inode, "com.apple.provenance", "p", 1, VEXFS_MOUNT_XATTR_ALWAYS_SET, &error) != VEXFS_MOUNT_OK) return Fail("benchmark xattr", error); + xattr_call_seconds += std::chrono::duration( + std::chrono::steady_clock::now() - call_started).count(); int64_t generation = 0; std::snprintf(request, sizeof(request), "write-%06d", index); + call_started = std::chrono::steady_clock::now(); if (vexfs_mount_handle_stage_write( session, handle_id.c_str(), 0, payload, sizeof(payload) - 1, request, &generation, &error) != VEXFS_MOUNT_OK) return Fail("benchmark stage", error); + stage_call_seconds += std::chrono::duration( + std::chrono::steady_clock::now() - call_started).count(); int64_t version = 0; + call_started = std::chrono::steady_clock::now(); if (vexfs_mount_handle_publish_close(session, handle_id.c_str(), generation, "data", &version, &error) != VEXFS_MOUNT_OK || version != 1) return Fail("benchmark publish close", error); + publish_call_seconds += std::chrono::duration( + std::chrono::steady_clock::now() - call_started).count(); } const double create_seconds = std::chrono::duration( std::chrono::steady_clock::now() - started).count(); @@ -159,6 +179,9 @@ int RunBenchmark(int file_count) { "\"sync_seconds\":%.6f,\"commit_rows\":%lld,\"version_rows\":%lld," "\"request_rows\":%lld,\"commits_per_file\":%.3f," "\"versions_per_file\":%.3f,\"requests_per_file\":%.3f," + "\"create_call_seconds\":%.6f,\"stat_call_seconds\":%.6f," + "\"xattr_call_seconds\":%.6f,\"stage_call_seconds\":%.6f," + "\"publish_call_seconds\":%.6f," "\"ordinary_mutation_calls\":%lld,\"full_boundary_calls\":%lld," "\"durability_barriers\":%lld,\"database_bytes\":%lld}\n", file_count, create_seconds, file_count / std::max(create_seconds, 1e-9), @@ -166,6 +189,8 @@ int RunBenchmark(int file_count) { static_cast(requests), static_cast(commits) / file_count, static_cast(versions) / file_count, static_cast(requests) / file_count, + create_call_seconds, stat_call_seconds, xattr_call_seconds, + stage_call_seconds, publish_call_seconds, static_cast(ordinary_calls), static_cast(full_calls), static_cast(barriers), static_cast(database_bytes)); unlink(path); diff --git a/agent_files/mount/common/test/vexfs_pg_runtime_smoke.cpp b/agent_files/mount/common/test/vexfs_pg_runtime_smoke.cpp new file mode 100644 index 0000000000..4ff0c41a9f --- /dev/null +++ b/agent_files/mount/common/test/vexfs_pg_runtime_smoke.cpp @@ -0,0 +1,377 @@ +#include "vexfs_runtime_admin.h" + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace { + +int checks = 0; + +int Fail(const char *operation, const vexfs_mount_error *error = nullptr) { + if (error == nullptr) { + std::fprintf(stderr, "VEXFS PG RUNTIME FAIL: %s\n", operation); + } else { + std::fprintf(stderr, + "VEXFS PG RUNTIME FAIL: %s: status=%d backend=%s native=%d %s\n", + operation, error->status, error->backend, error->native_code, error->message); + } + return 1; +} + +bool CheckStatus(const char *operation, vexfs_mount_status status, + vexfs_mount_error &error) { + ++checks; + if (status == VEXFS_MOUNT_OK) return true; + Fail(operation, &error); + return false; +} + +std::string Take(vexfs_mount_bytes *bytes) { + std::string value; + if (bytes->data != nullptr) { + value.assign(static_cast(bytes->data), + static_cast(bytes->size)); + vexfs_mount_free(bytes->data); + } + bytes->data = nullptr; + bytes->size = 0; + return value; +} + +bool Expect(bool condition, const char *message) { + ++checks; + if (condition) return true; + Fail(message); + return false; +} + +int64_t JsonInteger(const std::string &json, const char *key) { + const std::string marker = std::string("\"") + key + "\":"; + const size_t start = json.find(marker); + if (start == std::string::npos) return -1; + return std::strtoll(json.c_str() + start + marker.size(), nullptr, 10); +} + +bool PgCommand(PGconn *connection, const char *sql) { + PGresult *result = PQexec(connection, sql); + const bool ok = result != nullptr && + (PQresultStatus(result) == PGRES_COMMAND_OK || + PQresultStatus(result) == PGRES_TUPLES_OK); + if (!ok) std::fprintf(stderr, "PostgreSQL control command failed: %s\n", + result == nullptr ? PQerrorMessage(connection) : + PQresultErrorMessage(result)); + if (result != nullptr) PQclear(result); + return ok; +} + +bool ReconnectControl(PGconn *connection) { + bool probe_ok = false; + if (PQstatus(connection) == CONNECTION_OK) { + PGresult *probe = PQexec(connection, "SELECT 1"); + probe_ok = probe != nullptr && PQresultStatus(probe) == PGRES_TUPLES_OK; + if (probe != nullptr) PQclear(probe); + } + if (probe_ok) return true; + + PQreset(connection); + if (PQstatus(connection) != CONNECTION_OK) return false; + PGresult *probe = PQexec(connection, "SELECT 1"); + probe_ok = probe != nullptr && PQresultStatus(probe) == PGRES_TUPLES_OK; + if (probe != nullptr) PQclear(probe); + return probe_ok; +} + +bool DropWorkspace(PGconn *connection, const std::string &workspace) { + const char *values[] = {workspace.c_str()}; + PGresult *result = PQexecParams( + connection, "SELECT vexfs_workspace_drop($1,true)", 1, + nullptr, values, nullptr, nullptr, 0); + const bool ok = result != nullptr && PQresultStatus(result) == PGRES_TUPLES_OK; + if (result != nullptr) PQclear(result); + return ok; +} + +bool TerminateRuntimeConnection(PGconn *connection, int64_t backend_pid) { + const std::string pid = std::to_string(backend_pid); + const char *values[] = {pid.c_str()}; + PGresult *result = PQexecParams( + connection, + "SELECT pg_terminate_backend($1::integer) WHERE $1::integer<>pg_backend_pid()", + 1, nullptr, values, nullptr, nullptr, 0); + bool ok = result != nullptr && PQresultStatus(result) == PGRES_TUPLES_OK && + PQntuples(result) == 1 && std::strcmp(PQgetvalue(result, 0, 0), "t") == 0; + if (!ok) std::fprintf(stderr, "could not terminate runtime connection: %s\n", + result == nullptr ? PQerrorMessage(connection) : + PQresultErrorMessage(result)); + if (result != nullptr) PQclear(result); + return ok; +} + +} // namespace + +int main(int argc, char **argv) { + const char *dsn = argc == 2 ? argv[1] : std::getenv("VEXDB_PG_DSN"); + if (dsn == nullptr || *dsn == '\0') { + std::fprintf(stderr, "usage: vexfs_pg_runtime_smoke DSN\n"); + return 2; + } + const std::string workspace = "pg-runtime-contract"; + PGconn *control = PQconnectdb(dsn); + if (control == nullptr || PQstatus(control) != CONNECTION_OK) { + std::fprintf(stderr, "control connection failed: %s\n", + control == nullptr ? "allocation failed" : PQerrorMessage(control)); + if (control != nullptr) PQfinish(control); + return 1; + } + DropWorkspace(control, workspace); + + vexfs_mount_config config{}; + config.abi_version = VEXFS_RUNTIME_ABI_VERSION; + config.backend = VEXFS_RUNTIME_BACKEND_POSTGRESQL; + config.connection = dsn; + config.workspace = workspace.c_str(); + config.principal = nullptr; + config.operation_timeout_ms = 5000; + config.flags = 0; + vexfs_mount_error error{}; + vexfs_mount_session *session = nullptr; + if (!CheckStatus("open", vexfs_mount_session_open(&config, &session, &error), error)) + return 1; + + vexfs_mount_bytes bytes{}; + if (!CheckStatus("diagnostics", vexfs_mount_diagnostics(session, &bytes, &error), error)) + return 1; + const std::string diagnostics = Take(&bytes); + const int64_t backend_pid = JsonInteger(diagnostics, "backend_pid"); + if (!Expect(diagnostics.find("\"backend\":\"postgresql\"") != std::string::npos, + "diagnostics backend") || + !Expect(diagnostics.find("\"schema_version\":\"0.9.0\"") != std::string::npos, + "diagnostics contract version") || + !Expect(diagnostics.find("\"adapter_version\":\"0.4.0-alpha.1\"") != + std::string::npos, + "diagnostics adapter version") || + !Expect(diagnostics.find("\"security_mode\":\"database-role\"") != std::string::npos, + "diagnostics security") || + !Expect(diagnostics.find("\"principal\":\"postgres\"") != std::string::npos, + "diagnostics principal") || + !Expect(backend_pid > 0, "diagnostics backend pid")) return 1; + + if (!CheckStatus("mkdir", vexfs_mount_mkdir(session, "/project", &error), error)) return 1; + int64_t version = 0; + if (!CheckStatus("write v1", vexfs_mount_write_file( + session, "/project/main.txt", "alpha\n", 6, &version, &error), error) || + !Expect(version == 1, "write version 1")) return 1; + if (!CheckStatus("write v2", vexfs_mount_write_file( + session, "/project/main.txt", "beta alpha\n", 11, &version, &error), error) || + !Expect(version == 2, "write version 2")) return 1; + if (!CheckStatus("read", vexfs_mount_read_file( + session, "/project/main.txt", &bytes, &error), error) || + !Expect(Take(&bytes) == "beta alpha\n", "read content")) return 1; + + if (!CheckStatus("stat", vexfs_mount_stat( + session, "/project/main.txt", &bytes, &error), error)) return 1; + const std::string stat = Take(&bytes); + const int64_t inode = JsonInteger(stat, "inode"); + if (!Expect(inode > 0 && JsonInteger(stat, "version") == 2, "stat metadata")) return 1; + if (!CheckStatus("chmod", vexfs_mount_set_mode(session, inode, 0750, &error), error) || + !CheckStatus("times", vexfs_mount_set_times( + session, inode, 1700000000123LL, 1700000100456LL, + VEXFS_MOUNT_TIME_ACCESS | VEXFS_MOUNT_TIME_MODIFY, &error), error) || + !CheckStatus("chown", vexfs_mount_chown(session, inode, 501, 20, &error), error)) return 1; + + if (!CheckStatus("xattr set", vexfs_mount_xattr_set( + session, inode, "user.runtime", "value", 5, + VEXFS_MOUNT_XATTR_MUST_CREATE, &error), error) || + !CheckStatus("xattr get", vexfs_mount_xattr_get( + session, inode, "user.runtime", &bytes, &error), error) || + !Expect(Take(&bytes) == "value", "xattr value")) return 1; + const char acl[] = + "[{\"principal\":\"postgres\",\"effect\":\"allow\"," + "\"permissions\":\"read,write,execute,metadata\",\"inherit\":1}]"; + if (!CheckStatus("acl set", vexfs_mount_acl_set( + session, inode, acl, sizeof(acl) - 1, &error), error) || + !CheckStatus("acl get", vexfs_mount_acl_get(session, inode, &bytes, &error), error) || + !Expect(Take(&bytes).find("\"principal\":\"postgres\"") != std::string::npos, + "acl content")) return 1; + + if (!CheckStatus("hardlink", vexfs_mount_link( + session, "/project/main.txt", "/project/main-link.txt", &error), error) || + !CheckStatus("symlink", vexfs_mount_symlink( + session, "/project/current", "main.txt", 8, &error), error) || + !CheckStatus("list", vexfs_mount_list(session, "/project", &bytes, &error), error)) + return 1; + const std::string listing = Take(&bytes); + if (!Expect(listing.find("\"name\":\"main.txt\"") != std::string::npos && + listing.find("\"name\":\"current\"") != std::string::npos, + "directory listing")) return 1; + + if (!CheckStatus("grep", vexfs_mount_grep( + session, "/project", "alpha", 0, 100, &bytes, &error), error)) return 1; + const std::string grep = Take(&bytes); + if (!Expect(grep.find("\"match_count\":2") != std::string::npos && + grep.find("\"index_used\":false") != std::string::npos, + "grep hardlink matches")) return 1; + if (!CheckStatus("grep index", vexfs_mount_grep_index( + session, "status", &bytes, &error), error)) return 1; + const std::string index = Take(&bytes); + if (!Expect(index.find("\"available\":true") != std::string::npos && + index.find("\"enabled\":false") != std::string::npos && + index.find("\"backend\":\"pg-trgm\"") != std::string::npos, + "trigram capability status") || + !CheckStatus("grep index enable", vexfs_mount_grep_index( + session, "enable", &bytes, &error), error)) return 1; + const std::string enabled_index = Take(&bytes); + if (!Expect(enabled_index.find("\"enabled\":true") != std::string::npos && + enabled_index.find("\"dirty\":false") != std::string::npos && + enabled_index.find("\"indexed_files\":1") != std::string::npos, + "trigram index enabled") || + !CheckStatus("indexed grep", vexfs_mount_grep( + session, "/project", "alpha", 0, 100, &bytes, &error), error)) return 1; + const std::string indexed_grep = Take(&bytes); + if (!Expect(indexed_grep.find("\"match_count\":2") != std::string::npos && + indexed_grep.find("\"index_used\":true") != std::string::npos, + "trigram indexed grep")) return 1; + + vexfs_mount_bytes handle_bytes{}; + if (!CheckStatus("handle create", vexfs_mount_handle_create( + session, "/project/staged.txt", 0644, "create-staged", + &handle_bytes, &error), error)) return 1; + const std::string handle = Take(&handle_bytes); + int64_t generation = 0; + if (!CheckStatus("stage", vexfs_mount_handle_stage_write( + session, handle.c_str(), 0, "staged", 6, "stage-staged", + &generation, &error), error) || + !Expect(generation > 0, "stage generation")) return 1; + int64_t replay_generation = 0; + if (!CheckStatus("stage replay", vexfs_mount_handle_stage_write( + session, handle.c_str(), 0, "staged", 6, "stage-staged", + &replay_generation, &error), error) || + !Expect(replay_generation == generation, "stage replay generation")) return 1; + ++checks; + if (vexfs_mount_handle_stage_write( + session, handle.c_str(), 0, "other", 5, "stage-staged", + &replay_generation, &error) != VEXFS_MOUNT_CONFLICT) + return Fail("stage replay fingerprint", &error); + + vexfs_mount_session *peer = nullptr; + vexfs_mount_config peer_config = config; + if (!CheckStatus("peer open", vexfs_mount_session_open( + &peer_config, &peer, &error), error)) return 1; + ++checks; + if (vexfs_mount_read_file(peer, "/project/staged.txt", &bytes, &error) != + VEXFS_MOUNT_NOT_FOUND) return Fail("unpublished isolation", &error); + if (!CheckStatus("handle read", vexfs_mount_handle_read( + session, handle.c_str(), 0, 64, &bytes, &error), error) || + !Expect(Take(&bytes) == "staged", "staged read")) return 1; + if (!CheckStatus("publish", vexfs_mount_handle_publish( + session, handle.c_str(), generation, "data", "publish-staged", + &version, &error), error) || + !CheckStatus("close handle", vexfs_mount_handle_close( + session, handle.c_str(), 0, "close-staged", &bytes, &error), error)) return 1; + Take(&bytes); + if (!CheckStatus("peer published read", vexfs_mount_read_file( + peer, "/project/staged.txt", &bytes, &error), error) || + !Expect(Take(&bytes) == "staged", "published visibility")) return 1; + + vexfs_mount_visibility visibility{}; + if (!CheckStatus("visibility initial", vexfs_mount_refresh_visibility( + session, &visibility, &error), error)) return 1; + if (!CheckStatus("peer write", vexfs_mount_write_file( + peer, "/project/peer.txt", "peer", 4, &version, &error), error) || + !CheckStatus("visibility external", vexfs_mount_refresh_visibility( + session, &visibility, &error), error) || + !Expect(visibility.external_commit == 1, "external change notification")) return 1; + + int64_t snapshot_commit = 0; + if (!CheckStatus("snapshot", vexfs_mount_snapshot_create( + session, "baseline", VEXFS_SNAPSHOT_COMMITTED_ONLY, + &snapshot_commit, &error), error) || + !CheckStatus("snapshot list", vexfs_mount_snapshot_list( + session, &bytes, &error), error) || + !Expect(Take(&bytes).find("\"name\":\"baseline\"") != std::string::npos, + "snapshot list content")) return 1; + if (!CheckStatus("mutate after snapshot", vexfs_mount_write_file( + session, "/project/main.txt", "changed\n", 8, &version, &error), error)) return 1; + int64_t head = 0; + if (!CheckStatus("workspace head", vexfs_mount_workspace_head( + session, &head, &error), error)) return 1; + int64_t restore_commit = 0; + if (!CheckStatus("snapshot restore", vexfs_mount_snapshot_restore( + session, "baseline", head, &restore_commit, &error), error) || + !CheckStatus("restored read", vexfs_mount_read_file( + session, "/project/main.txt", &bytes, &error), error) || + !Expect(Take(&bytes) == "beta alpha\n", "snapshot restored content")) return 1; + + if (!CheckStatus("deep check", vexfs_mount_check(session, 0, &bytes, &error), error) || + !Expect(Take(&bytes).find("\"ok\":true") != std::string::npos, + "deep check result")) return 1; + + if (!Expect(TerminateRuntimeConnection(control, backend_pid), + "terminate runtime connection")) return 1; + bytes = {}; + const vexfs_mount_status first_after_disconnect = + vexfs_mount_stat(session, "/project/main.txt", &bytes, &error); + ++checks; + if (first_after_disconnect != VEXFS_MOUNT_OK && + first_after_disconnect != VEXFS_MOUNT_DATABASE_ERROR) + return Fail("disconnect status", &error); + if (first_after_disconnect == VEXFS_MOUNT_OK) Take(&bytes); + if (!CheckStatus("reconnect next operation", vexfs_mount_stat( + session, "/project/main.txt", &bytes, &error), error) || + !Expect(JsonInteger(Take(&bytes), "inode") == inode, + "reconnect preserved workspace")) return 1; + + const char *restart_gate = std::getenv("VEXFS_PG_RESTART_GATE_DIR"); + if (restart_gate != nullptr && *restart_gate != '\0') { + const std::string ready = std::string(restart_gate) + "/ready"; + const std::string resume = std::string(restart_gate) + "/continue"; + { + std::ofstream marker(ready, std::ios::trunc); + if (!marker) return Fail("create restart ready marker"); + marker << "ready\n"; + } + const auto deadline = std::chrono::steady_clock::now() + + std::chrono::seconds(120); + while (!std::ifstream(resume).good()) { + if (std::chrono::steady_clock::now() >= deadline) + return Fail("wait for PostgreSQL restart"); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + + bytes = {}; + const vexfs_mount_status first_after_restart = + vexfs_mount_stat(session, "/project/main.txt", &bytes, &error); + ++checks; + if (first_after_restart != VEXFS_MOUNT_OK && + first_after_restart != VEXFS_MOUNT_DATABASE_ERROR) + return Fail("restart first operation status", &error); + if (first_after_restart == VEXFS_MOUNT_OK) Take(&bytes); + if (!CheckStatus("restart reconnect next operation", vexfs_mount_stat( + session, "/project/main.txt", &bytes, &error), error) || + !Expect(JsonInteger(Take(&bytes), "inode") == inode, + "restart reconnect preserved workspace")) return 1; + if (!CheckStatus("restart diagnostics", vexfs_mount_diagnostics( + session, &bytes, &error), error)) return 1; + const int64_t restarted_backend_pid = JsonInteger(Take(&bytes), "backend_pid"); + if (!Expect(restarted_backend_pid > 0 && restarted_backend_pid != backend_pid, + "restart created a new backend connection")) return 1; + + if (!Expect(ReconnectControl(control), + "control reconnect after server restart")) return 1; + } + + vexfs_mount_session_close(peer); + vexfs_mount_session_close(session); + if (!Expect(DropWorkspace(control, workspace), "cleanup workspace")) return 1; + if (!Expect(PgCommand(control, "SELECT 1"), "control connection")) return 1; + PQfinish(control); + std::printf("VEXFS PG RUNTIME CONTRACT: PASS (%d checks)\n", checks); + return 0; +} diff --git a/agent_files/mount/linux/vexfs_fuse.cpp b/agent_files/mount/linux/vexfs_fuse.cpp index b4aa59fe25..524b2b93c5 100644 --- a/agent_files/mount/linux/vexfs_fuse.cpp +++ b/agent_files/mount/linux/vexfs_fuse.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -37,7 +38,8 @@ namespace { struct FuseState { vexfs_mount_session *session = nullptr; - std::string database; + std::string backend; + std::string connection; std::string request_prefix; std::recursive_mutex mutex; std::atomic request_sequence{1}; @@ -139,6 +141,21 @@ int64_t JsonInteger(const std::string &json, const std::string &name) { return static_cast(value); } +int64_t JsonOptionalInteger(const std::string &json, const std::string &name, + int64_t fallback) { + const std::string marker = "\"" + name + "\":"; + size_t position = json.find(marker); + if (position == std::string::npos) throw std::runtime_error("missing JSON integer: " + name); + position += marker.size(); + while (position < json.size() && std::isspace(static_cast(json[position]))) + ++position; + if (json.compare(position, 4, "null") == 0) return fallback; + char *end = nullptr; + const long long value = std::strtoll(json.c_str() + position, &end, 10); + if (end == json.c_str() + position) throw std::runtime_error("invalid JSON integer: " + name); + return static_cast(value); +} + std::string JsonString(const std::string &json, const std::string &name) { const std::string marker = "\"" + name + "\":\""; size_t position = json.find(marker); @@ -414,6 +431,8 @@ FileHandle *Handle(fuse_file_info *info) { return info == nullptr ? nullptr : reinterpret_cast(info->fh); } +int Publish(FuseState *state, FileHandle *handle, const char *durability); + int CreatePath(const char *path, mode_t mode, fuse_file_info *info) { FuseState *state = State(); if (state == nullptr) return -EIO; @@ -429,6 +448,27 @@ int CreatePath(const char *path, mode_t mode, fuse_file_info *info) { handle->append = (info->flags & O_APPEND) != 0; handle->generation = 1; handle->dirty = true; + // FUSE asks getattr for the new path before open(2) returns. PostgreSQL keeps + // handle_create as private staging until publish, so publish the POSIX-visible + // empty file here. Later writes still use the same handle and atomic staging. + const int publish_result = Publish(state, handle.get(), "data"); + if (publish_result != 0) { + vexfs_mount_bytes close_state{}; + vexfs_mount_handle_close(state->session, handle->id.c_str(), 1, + RequestId(state, "close-create-failure").c_str(), + &close_state, &error); + TakeBytes(&close_state); + return publish_result; + } + const int chown_result = ChownCreatedPath(state, path); + if (chown_result != 0) { + vexfs_mount_bytes close_state{}; + vexfs_mount_handle_close(state->session, handle->id.c_str(), 0, + RequestId(state, "close-chown-failure").c_str(), + &close_state, &error); + TakeBytes(&close_state); + return chown_result; + } info->fh = reinterpret_cast(handle.release()); info->direct_io = 1; return 0; @@ -612,10 +652,51 @@ int ReadDirectory(const char *path, void *buffer, fuse_fill_dir_t filler, off_t, int StatFs(const char *, struct statvfs *output) { FuseState *state = State(); if (state == nullptr) return -EIO; - const std::filesystem::path database(state->database); - const std::filesystem::path directory = database.has_parent_path() ? - database.parent_path() : std::filesystem::current_path(); - return statvfs(directory.c_str(), output) == 0 ? 0 : -errno; + if (state->backend == VEXFS_RUNTIME_BACKEND_SQLITE) { + const std::filesystem::path database(state->connection); + const std::filesystem::path directory = database.has_parent_path() ? + database.parent_path() : std::filesystem::current_path(); + return statvfs(directory.c_str(), output) == 0 ? 0 : -errno; + } + + std::lock_guard guard(state->mutex); + vexfs_mount_bytes quota{}; + vexfs_mount_error error{}; + const auto status = vexfs_mount_quota_get(state->session, "a, &error); + if (status != VEXFS_MOUNT_OK) return -Errno(status); + try { + const std::string json = TakeBytes("a); + constexpr uint64_t block_size = 4096; + constexpr uint64_t unlimited_bytes = 1ULL << 50; // 1 PiB virtual ceiling. + constexpr uint64_t unlimited_files = 1ULL << 30; + const uint64_t live_bytes = static_cast( + std::max(0, JsonInteger(json, "live_bytes"))); + const uint64_t live_files = static_cast( + std::max(0, JsonInteger(json, "live_files"))); + const int64_t configured_bytes = JsonOptionalInteger(json, "max_bytes", -1); + const int64_t configured_files = JsonOptionalInteger(json, "max_files", -1); + const uint64_t capacity_bytes = configured_bytes < 0 + ? std::max(unlimited_bytes, live_bytes) + : std::max(static_cast(configured_bytes), live_bytes); + const uint64_t capacity_files = configured_files < 0 + ? std::max(unlimited_files, live_files) + : std::max(static_cast(configured_files), live_files); + std::memset(output, 0, sizeof(*output)); + output->f_bsize = block_size; + output->f_frsize = block_size; + output->f_blocks = (capacity_bytes + block_size - 1) / block_size; + const uint64_t used_blocks = (live_bytes + block_size - 1) / block_size; + output->f_bfree = output->f_bavail = output->f_blocks > used_blocks + ? output->f_blocks - used_blocks : 0; + output->f_files = capacity_files; + output->f_ffree = output->f_favail = capacity_files > live_files + ? capacity_files - live_files : 0; + output->f_namemax = 255; + return 0; + } catch (...) { + TakeBytes("a); + return -EIO; + } } int PathInode(FuseState *state, const char *path, int64_t *inode) { @@ -790,7 +871,8 @@ fuse_operations Operations() { } struct Options { - std::string database; + std::string backend = VEXFS_RUNTIME_BACKEND_SQLITE; + std::string connection; std::string workspace = "default"; std::string mount_point; bool foreground = false; @@ -801,7 +883,13 @@ Options ParseOptions(int argc, char **argv) { Options options; for (int index = 1; index < argc; ++index) { const std::string argument = argv[index]; - if (argument == "--db" && index + 1 < argc) options.database = argv[++index]; + if (argument == "--backend" && index + 1 < argc) { + options.backend = argv[++index]; + if (options.backend == "pg" || options.backend == "postgres") + options.backend = VEXFS_RUNTIME_BACKEND_POSTGRESQL; + } + else if (argument == "--db" && index + 1 < argc) options.connection = argv[++index]; + else if (argument == "--dsn" && index + 1 < argc) options.connection = argv[++index]; else if (argument == "--workspace" && index + 1 < argc) options.workspace = argv[++index]; else if (argument == "--foreground" || argument == "-f") options.foreground = true; else if (argument == "--self-test") options.self_test = true; @@ -810,7 +898,12 @@ Options ParseOptions(int argc, char **argv) { else if (options.mount_point.empty()) options.mount_point = argument; else throw std::runtime_error("only one mount point is allowed"); } - if (options.database.empty()) throw std::runtime_error("--db DATABASE is required"); + if (options.backend != VEXFS_RUNTIME_BACKEND_SQLITE && + options.backend != VEXFS_RUNTIME_BACKEND_POSTGRESQL) + throw std::runtime_error("--backend must be sqlite or postgresql"); + if (options.connection.empty()) + throw std::runtime_error(options.backend == VEXFS_RUNTIME_BACKEND_POSTGRESQL + ? "--dsn DSN is required" : "--db DATABASE is required"); if (!options.self_test && options.mount_point.empty()) throw std::runtime_error("MOUNT_POINT is required"); return options; @@ -856,14 +949,15 @@ int main(int argc, char **argv) { try { const Options options = ParseOptions(argc, argv); FuseState state; - state.database = options.database; + state.backend = options.backend; + state.connection = options.connection; state.request_prefix = NewRequestPrefix(); vexfs_mount_config config{}; config.abi_version = VEXFS_RUNTIME_ABI_VERSION; - config.backend = VEXFS_RUNTIME_BACKEND_SQLITE; - config.connection = options.database.c_str(); + config.backend = options.backend.c_str(); + config.connection = options.connection.c_str(); config.workspace = options.workspace.c_str(); - config.principal = "local"; + config.principal = options.backend == VEXFS_RUNTIME_BACKEND_SQLITE ? "local" : nullptr; config.operation_timeout_ms = 5000; config.flags = options.self_test ? 0 : VEXFS_RUNTIME_EXCLUSIVE_GATEWAY; vexfs_mount_error error{}; diff --git a/docs/design/2026-07-16_agent-files-universal-filesystem.md b/docs/design/2026-07-16_agent-files-universal-filesystem.md index 6f705882b7..190a15fd3c 100644 --- a/docs/design/2026-07-16_agent-files-universal-filesystem.md +++ b/docs/design/2026-07-16_agent-files-universal-filesystem.md @@ -2,14 +2,19 @@ - 日期：2026-07-16 - 分支：`feature/agent_files` -- 文档版本：1.4 -- 状态：macOS `preview.17` 已完成 Developer ID 签名、Apple 公证、staple、本机安装和真实 FSKit 回归，但它不是当前工作树的发行证明；当前统一 runtime ABI 为 1，已修复包 Gate、ABI 清单、安装校验和脏源码发行拦截，待从干净 commit 重新签名、公证并跑真机挂载；Linux libfuse3 双架构预览包已完成；1 万文件、900 秒稳定性与真实工具链已验证；干净 Mac、10 万完整 mount 复测和真实 Coding Agent 待完成 +- 文档版本：1.6 +- 状态：macOS FSKit 与 Linux libfuse3 已复用同一 runtime；PostgreSQL `0.4.0-alpha.1` + 已完成数据库合同、format v2、role/ACL/审计、libpq HostStore、macOS/Linux 真实 mount、 + 跨机器 OpenCode 和逻辑/物理备份；第二台 Mac 的局域网直连只差一次系统授权确认 - 产品名：暂定 `VexFS` - 产品规格：`docs/specs/2026-07-16_vexfs-product-spec.md` - 最终目标和阶段顺序：`docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md` ## 1. 最终定义 +> 2026-07-24 默认入口变更：macOS 0.1 使用系统 NFS client + 本机用户态 gateway；FSKit +> 保留为后续可选原生 adapter。已有 FSKit 章节记录已完成实现和历史证据，不再表示默认发布路径。 + VexFS 是由 PostgreSQL、DuckDB 或 SQLite 加载和管理的数据库扩展。三个宿主共享逻辑合同，但首个闭环固定为 macOS + SQLite，不要求第一版同时完成其他数据库和操作系统。 它给数据库增加一套分层文件管理能力： @@ -26,7 +31,10 @@ VexFS 是由 PostgreSQL、DuckDB 或 SQLite 加载和管理的数据库扩展。 - 完整性检查； - 工作区逻辑导入导出。 -VexFS 的权威核心是数据库扩展。首个产品提供 macOS FSKit App Extension，把 SQLite 中的工作区显示为真实操作系统目录。FSKit extension 是非权威 mount adapter：它通过 SQLite 连接调用公开 VexFS SQL/handle 合同，不保存最终状态，不绕过数据库事务，也不管理 SQLite 的 WAL、锁或物理文件。 +VexFS 的权威核心是数据库扩展。首个产品默认提供只监听 loopback 的 macOS NFS gateway， +把 SQLite 中的工作区显示为真实操作系统目录。NFS gateway 是非权威 mount adapter：它调用 +公开 VexFS SQL/handle 合同，不保存最终状态，不绕过数据库事务，也不管理 SQLite 的 WAL、锁 +或物理文件。FSKit 后续通过同一个 Workspace Engine 接入，不能复制文件管理逻辑。 用户通过数据库连接和 SQL 使用它： @@ -63,7 +71,9 @@ vexfs --workspace agent-workspace grep -R "Result" /reports > **VexFS 给 PostgreSQL、DuckDB 和 SQLite 增加数据库原生的文件管理系统。** -首个可挂载实现从 macOS 26.0+、Apple Silicon 和 SQLite 的垂直闭环开始，Linux libfuse3 预览现已接入同一合同并通过真实普通用户挂载。FSKit 基础接口从 macOS 15.4 提供，但数据库目录资源所需的 `FSPathURLResource` 属于 V2、从 macOS 26.0 提供，因此 15.4–15.x 只保留数据库与 CLI 能力，不承诺无缝挂载。当前阶段先把 macOS/Linux + SQLite 收口为本地开发者预览；PostgreSQL 用于下一阶段的跨电脑共享和权限验证，Windows 后续使用 WinFsp 接入同一 mount runtime。 +默认 NFS 路径不依赖 FSKit V2 的 `FSPathURLResource`，因此新的 macOS 最低版本和 CPU 支持范围 +由 NFS 包真机验证决定，不再先写死为 macOS 26.0+ Apple Silicon。现有 FSKit 垂直闭环继续作为 +可选 adapter 证据保留。Linux libfuse3 已接入同一合同，Windows 后续使用 WinFsp。 ### 1.1 适配工作量必须按 N+M 拆分 @@ -73,7 +83,8 @@ vexfs --workspace agent-workspace grep -R "Result" /reports SQLite adapter ----+ DuckDB adapter ----+--> VexFS 统一合同和 mount runtime PostgreSQL adapter-+ | - +--> macOS FSKit + +--> macOS NFS（默认） + +--> macOS FSKit（后续可选） +--> Linux libfuse3 +--> Windows WinFsp +--> 跨平台 CLI @@ -472,17 +483,22 @@ MVP 采用 close-to-open 一致性： ### 4.4 身份与沙箱模型 -当前 FSKit 本地预览只服务当前 macOS 用户，并绑定一个 SQLite context、workspace 和数据库 principal： +FSKit 本地入口只服务当前 macOS 用户；数据库 principal 由所选后端决定： -- 默认 SQLite 数据库位于 `~/Library/Application Support/VexFS/`； +- 默认 SQLite 数据库位于 `~/Library/Application Support/VexDB-Lite/`； - CLI 把数据库所在目录作为 security-scoped `FSPathURLResource` 交给 FSKit； - extension 只访问该目录内经过描述文件校验的数据库、WAL 和 SHM； - mount point 只允许当前用户访问； -- SQLite principal 固定为 workspace owner，不接受 CLI 自报用户； -- 文件和目录保存并报告数据库中的 mode 和 owner/group 元数据；便携 ACL 可以保存和恢复，但本阶段尚未实现完整路径 ACL 授权和继承； +- SQLite principal 固定为 workspace owner，不接受 CLI 自报用户；SQLite 保存便携 ACL，但 + 当前不执行完整路径 ACL 授权和继承； +- PostgreSQL principal 来自经过 libpq 认证的真实 role，并在 open、目录修改和 publish 时 + 重新执行路径 ACL；CLI 传入的名称或操作系统 uid 不能提升数据库权限； +- 文件和目录保存并报告数据库中的 mode 和 owner/group 元数据； - App sandbox、entitlement、签名和公证配置必须进入安装回归。 -新版 FSKit `FSContext` 可以提供调用者 uid/gid，但 MVP 不据此实现多用户权限。PostgreSQL 阶段增加数据库 role、路径 ACL，以及每次 read/write/publish 所需的权限复查规则。操作系统身份负责保护本地入口，数据库 principal 负责决定数据库文件权限，两者不能互相替代。 +新版 FSKit `FSContext` 可以提供调用者 uid/gid，但当前不据此实现本机多用户共享。PostgreSQL +已经增加数据库 role、路径 ACL，以及每次 read/write/publish 所需的权限复查规则。操作系统 +身份负责保护本地入口，数据库 principal 负责决定数据库文件权限，两者不能互相替代。 ## 5. 三端统一 SQL 合同 @@ -1041,6 +1057,18 @@ vexfs backup verify workspace - SQL 函数默认 `SECURITY INVOKER`； - 内部 schema 不授予普通用户直接写权限； - PostgreSQL 数据库进程不启动 FSKit/FUSE/WinFsp loop 或网络线程；外部 mount adapter 使用正常 PostgreSQL 连接和 role。 +- 当前 `0.4.0-alpha.1` 已实现 chunk/manifest、版本、snapshot、ACL、审计、配额、retention、 + GC、跨 gateway handle/lock/cache invalidation 和 libpq HostStore； +- 变更审计由数据库在同一事务内写入，使用认证后的 `session_user` 和 role OID；记录 workspace + 名称/ID、commit、操作、路径、inode 和前后版本。workspace 删除前先记录，删除后把 ID + 置空但保留名称和证据。details 只保存安全元数据，不保存文件正文、xattr 值、密码或 DSN； +- macOS FSKit 与 Linux FUSE 复用同一 runtime。macOS 的 PostgreSQL passfile 只在挂载资源存在 + 期间以 `0600` 暂存，最后卸载和 `doctor` 会清理； +- mount 成功后，runtime 通过预先打开的目录 fd 把底层 mountpoint 设为 `0500`。FSKit/FUSE + 异常退出时，普通用户不能把文件写进没有数据库语义的本机目录；显式卸载恢复 `0700`； +- 网络中断时失败的 mutation 不做隐式重放，因为客户端无法确认服务端事务是否已经提交； + 恢复后先用可重试读操作确认状态，再接受新的写请求； +- 局域网 PostgreSQL 需要用户给 VexDB Lite 一次 macOS“本地网络”权限，程序不绕过系统授权。 ### DuckDB @@ -1246,7 +1274,10 @@ vexdb_lite/ └── tests/spec/agent_files/ ``` -仓库不增加远程文件协议、SDK 或独立权威服务。macOS FSKit extension 通过自己的 SQLite 连接加载数据库扩展并调用公开合同；PostgreSQL 阶段使用正常客户端连接。任何 adapter 启动前都必须与扩展交换合同版本和能力位，主版本不兼容时拒绝可写挂载。 +仓库不增加对外远程文件协议、SDK 或独立权威服务。macOS 默认 NFS 只绑定 loopback，作为 +系统 mount client 到 Workspace Engine 的本机传输层；跨电脑仍使用正常 PostgreSQL 连接。 +后续 FSKit extension 通过同一 runtime 调用公开合同。任何 adapter 启动前都必须与扩展交换 +合同版本和能力位，主版本不兼容时拒绝可写挂载。 ## 16. 验收测试 @@ -1353,14 +1384,25 @@ vexdb_lite/ - 完成真实项目、Coding Agent、10 万文件和长时间运行 eval； - 完成开发者预览所需的 retention、GC、quota、check 和 export/import 最小集合。 -### Phase 2：PostgreSQL 共享工作区 Alpha — 未开始 - -- 实现 PG HostStore，不重新定义文件语义； -- 数据库 role 映射 principal、owner、ACL、审计和配额； -- 多 gateway 的锁、缓存失效、冲突、断线重连和数据库重启； -- 文件与普通业务 SQL 原子提交； -- 电脑 A 写入后，电脑 B 通过 macOS/Linux mount 继承同一 workspace、历史和快照； -- pg_dump/pg_restore、物理备份和逻辑导出通过恢复测试。 +### Phase 2：PostgreSQL 共享工作区 Alpha — 功能与验收完成 + +- 已完成数据库内路径、内容、版本、workspace commit、snapshot 和冲突保护； +- 已完成 chunk/manifest、checksum/check、quota、retention、分批 GC 和 format v2； +- 已完成 libpq PG HostStore，没有重新定义文件语义； +- 已完成真实 role、principal、owner、ACL、完整变更审计和配额；普通 role 不能读取审计， + `vexfs_check` 会检查审计对象与前后版本字段； +- 已完成多 gateway 锁、缓存失效、冲突、断线重连和数据库重启； +- 已完成两台 Mac 同时挂载下的 extension 崩溃、网络中断和数据库停机 25 项故障测试； +- 已完成 Linux root/uid 1000 helper 崩溃后的底层目录防误写、租约超时接管和 staging 发布； +- 已验证文件与普通业务 SQL 参加同一数据库事务； +- macOS FSKit、Linux FUSE、跨系统同一 PG、第二台 M1 Mac OpenCode 已通过； +- pg_dump/pg_restore、pg_basebackup 和逻辑导出恢复已通过； +- 16 MiB 有界备份性能与 RSS/OOM Gate 23 项已通过；当前审计实现下 format v2 导出/导入 + 为 72.727/106.667 MiB/s，1 GiB 容器没有 OOM； +- 当前数据库源码的自动测试、Linux 真挂载、本机 PostgreSQL FSKit 13 组 247 项、凭据生命周期 + 9 项、macOS↔Linux 跨 gateway 47 项、两台 Mac 文件/快照往返 50 项和 extension/网络/数据库 + 故障恢复 25 项已完成；最后发行验收是从干净提交构建、公证并 staple `preview.38`，再在 + 本机和第二台 Mac 重跑安装 Gate。局域网直连仍需第二台 Mac 的“本地网络”权限。 ### Phase 3：Windows 与跨系统预览 — 未开始 @@ -1406,7 +1448,10 @@ Linux package macOS 可挂载预览支持 macOS 26.0+ Apple Silicon。Phase 1 必须验证 Developer ID + 公证或 Mac App Store/TestFlight 至少一条真实分发路径；CI 检查 App、appex、CLI 的签名链、FSKit entitlement 和系统信任状态。首次启动必须引导用户在系统设置中启用文件系统扩展，不能尝试绕过系统授权。Linux 包必须检查 libfuse3、`fusermount3` 和 `/dev/fuse`，普通用户不能依赖 root helper 才能工作。 -默认 SQLite 数据库位于 `~/Library/Application Support/VexFS/vexfs.sqlite3`。每个 mount 创建不含密码的 `.vexfs-volume.json`，再把数据库父目录作为 security-scoped FSKit source resource 传给系统；extension 只能打开描述文件指定的同目录数据库。用户选择其他位置时沿用同一目录授权模型。卸载 App 不能自动删除数据库，数据删除必须是独立且明确的动作。 +默认 SQLite 数据库位于 `~/Library/Application Support/VexDB-Lite/default.sqlite3`。每个 mount +创建不含密码的 `.vexfs-volume.json`，再把数据库父目录作为 security-scoped FSKit source +resource 传给系统；extension 只能打开描述文件指定的同目录数据库。用户选择其他位置时沿用 +同一目录授权模型。卸载 App 不能自动删除数据库，数据删除必须是独立且明确的动作。 每次 mount 的启动顺序固定为： @@ -1441,4 +1486,6 @@ VexFS 的产品本体是数据库 extension： 这个设计同时满足两件事：用户获得真实文件路径，数据库继续拥有全部已提交权威状态。VexDB-Lite 的差异点不是单独做一个文件服务，而是让数据库扩展提供受管文件能力，再用 mount 把它交给 Bash。 -实施上不按“三数据库 × 三操作系统”开发九套文件系统。当前门槛是把已经跑通的 macOS/Linux + SQLite 闭环收口为可安装、可恢复、可跑真实项目的本地开发者预览；达到路线图中的一致性、分发和规模条件后，再把主要开发重心转到 PostgreSQL 多用户共享工作区。 +实施上不按“三数据库 × 三操作系统”开发九套文件系统。SQLite 和 PostgreSQL 已经复用 +macOS/Linux 的同一 mount runtime 与一致性测试。PostgreSQL 多用户共享工作区代码完成后， +下一步只选择 Windows WinFsp 或 DuckDB HostStore 其中一条线继续，不能重新复制文件语义。 diff --git a/docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md b/docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md index 81e0946b13..c41e0edb7a 100644 --- a/docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md +++ b/docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md @@ -2,10 +2,44 @@ - 文档类型：唯一有效的目标和开发顺序基线 - 所属产品：VexDB-Lite 的文件管理能力 VexFS -- 日期：2026-07-22 +- 日期：2026-07-24 - 分支：`feature/agent_files` -- 文档版本：2.0 -- 当前阶段：Phase 1，SQLite 本地开发者预览 +- 文档版本：2.9 +- macOS 默认入口：本机 NFS gateway；FSKit 延后为可选原生增强 +- 当前阶段：Phase 2 PostgreSQL 功能与验收已完成；最后一次数据审计已让所有变更记录 + 认证用户、workspace、commit、对象和前后版本，并保留 workspace 删除证据。PG 16–19、 + Linux 真挂载、format v2 和备份性能已经按当前审计实现复跑。当前源码的 + `preview.37` Developer ID 包已通过 92 项文档 smoke、19 项 package Gate 和包内 PG runtime， + 已安装到两台 M1 并在远端通过 libpq 读写。真机复测确认 FSKit 已启用；用户再次唤醒系统后， + 本机 PG FSKit 13/13 场景、凭据生命周期和 macOS↔Linux 跨 gateway 已通过。该包没有 Apple + 公证票据，仍不能作为正式分发结论；两台 Mac 当前又通过文件/快照往返 50 项和 extension、 + 网络、数据库故障恢复 25 项。最初产品场景也已由两台 Mac 各运行一次真实 OpenCode 完成 + 串行闭环：7 + 9 + 21 条检查通过，两个完整工作区快照可比较并一键恢复。安装器的旧 UUID + 刷新逻辑已经修复。下一步必须从干净提交 + 原定 FSKit `preview.38` 不再是 0.1 阻塞项。下一步先完成 macOS NFS 默认 adapter、 + 无 FSKit 授权安装包和干净机器 Gate；已有 FSKit 能力保留为后续可选 backend + +## 0. 2026-07-24 路线调整 + +用户把“安装后无缝 Bash”放在第一位，而 FSKit 首次授权入口难找。因此 macOS 发布顺序调整为： + +1. 默认使用 macOS 内置 NFS client + 当前用户运行的本机 gateway； +2. gateway、Linux FUSE、后续 FSKit 和 Windows WinFsp 共用 Workspace Engine 和数据库合同； +3. NFS 只绑定 loopback，不能代替 PostgreSQL 跨电脑共享；远端 workspace 仍由每台电脑的 + 本地 gateway 连接 PG； +4. 先完成 NFS 协议、缓存、锁、xattr、sleep/wake、stale mount、gateway crash 和性能 Gate； +5. FSKit 已有实现和测试不删除，但其 entitlement、系统设置授权和公证不再阻塞 0.1； +6. 后续只有在 FSKit 能明显提供更好的原生集成、权限或性能，并通过同一 eval 后，才将其作为 + `--backend fskit` 提供。 + +新的最近任务顺序： + +1. 冻结 NFS adapter 与 Workspace Engine 的窄接口，不复制数据库逻辑； +2. 做 localhost-only NFS prototype，完成 mount/unmount/status/doctor； +3. 跑 Bash、Git、OpenCode、npm/cargo、文件锁、xattr、fsync 和 gateway 随机 kill； +4. 跑 1,000 小文件与 APFS/现有 FSKit 的同机对照，确认 NFS 是否改善吞吐； +5. 生成不含 FSKit 授权前置的签名公证包，在干净 Mac 验证安装和系统提示； +6. 通过正确性和安装 Gate 后，才把 NFS 设为代码中的默认 backend。 ## 1. 这份文档解决什么问题 @@ -85,23 +119,23 @@ vexdb fs --workspace workspace snapshot restore before-refactor - 不追求所有 POSIX 特殊文件和内核设备语义； - 不替代 Git 的分支、合并和代码协作能力； - 不把大视频、模型文件和数据湖对象作为第一目标； -- 不新建脱离数据库权限和事务的文件服务器协议； +- 不提供脱离数据库权限和事务的远程文件服务器；macOS localhost NFS 只允许作为 mount adapter； - 不让 DuckDB 承担多用户远程文件服务器职责； - 不为了宣传“更快 grep”而阻塞挂载正确性、备份和跨机器工作区； - 不在核心文件合同中加入 Agent 专用的 prompt、memory 或语义字段。 -## 5. 当前基线：2026-07-23 +## 5. 当前基线：2026-07-24 | 范围 | 当前事实 | 主要缺口 | |---|---|---| -| 统一合同 | SQLite 合同 `0.9.0`，runtime ABI 1；SQL、C ABI、CLI、mount runtime 已连通；已增加跨平台一致性合同和结构化错误状态 | PostgreSQL、DuckDB 尚未实现 VexFS adapter | -| 文件语义 | 文件、目录、四类时间戳、并发 append、进程锁、mode、symlink、xattr、hardlink、owner/group 元数据、便携 ACL 已进入数据库合同和测试 | ACL 还没有完整授权执行和继承；特殊文件不支持 | -| 版本恢复 | 单文件版本、workspace commit、snapshot、diff、expected-head restore、SQLite Backup、retention、显式分批 GC、live quota 和 format v2 逻辑导入导出已实现 | 审计、自动维护、history/staging/index/total quota、PG/DuckDB 导入端未完成 | +| 统一合同 | SQLite 合同 `0.9.0`，runtime ABI 1；SQL、C ABI、CLI、mount runtime 已连通；PostgreSQL `0.4.0-alpha.1` 已实现同一文件合同、format v2、ACL、审计、libpq HostStore 和远程 mount | DuckDB 尚未实现 VexFS adapter | +| 文件语义 | 文件、目录、四类时间戳、并发 append、进程锁、mode、symlink、xattr、hardlink、owner/group 元数据、便携 ACL 已进入数据库合同和测试；PG 已执行 ACL 和继承 | SQLite 尚未执行完整 ACL 授权；特殊文件不支持 | +| 版本恢复 | 单文件版本、workspace commit、snapshot、diff、expected-head restore、原生备份、retention、显式分批 GC、live quota 和 format v2 逻辑导入导出已实现 | 自动维护、history/staging/index/total quota 和 DuckDB 导入端未完成 | | 长期校验 | `chunked-v1` 使用不可变 manifest、64 KiB 块和逐块/整文件 SHA-256；只读 check 覆盖结构、引用、顺序、大小和流式内容校验 | 自动 repair 和跨文件通用去重不在当前范围 | -| macOS | preview.22 已从干净 commit `8a8b0c139a` 构建并通过 Apple 公证；本机 13 个真实挂载 Gate 通过，另一台无源码 M1 macOS 26.5.2 的 30 项安装后 smoke 也通过 | macOS 26.0–26.4 与 x86_64 尚未独立发行验证；构建机 26.3.1(a) 的 Code Signing 子系统当前异常 | -| Linux | libfuse3 真实挂载已通过 root 和普通用户；时间戳、并发 append、进程锁、打开文件生命周期、强制卸载和 helper 崩溃恢复已验证；x86_64/AArch64 manylinux 安装包已在对应架构真机完成无 root 安装回归 | 缓存/TTL、更多干净发行版、长期运行和真实挂载安装回归仍不足 | +| macOS | preview.22 已从干净 commit `8a8b0c139a` 构建并通过 Apple 公证；本机 13 个真实挂载 Gate 通过，另一台无源码 M1 macOS 26.5.2 的 30 项安装后 smoke 也通过。preview.37 的 FSKit 已启用，本机 PG FSKit 13/13、247 项和凭据 9 项通过 | preview.37 没有 stapled 公证票据；要从干净提交公证 preview.38。macOS 26.0–26.4 与 x86_64 尚未独立发行验证 | +| Linux | libfuse3 真实挂载已通过 root 和 uid 1000 各 9 组；时间戳、并发 append、进程锁、打开文件生命周期、强制卸载和 helper 崩溃恢复已验证；异常撤销后底层目录保持 0500，显式卸载恢复 0700；x86_64/AArch64 manylinux 安装包已在对应架构真机完成无 root 安装回归 | 更多干净发行版、长期运行和真实挂载安装回归仍不足 | | Windows | 只有平台边界和规划 | WinFsp adapter、安装签名、路径/ACL/SID 合同均未实现 | -| 远程共享 | 统一合同为 PostgreSQL 预留 | 还不能让另一台电脑挂载同一个远程工作区 | +| 远程共享 | PostgreSQL 已通过多 gateway、macOS FSKit、Linux FUSE、数据库重启、当前源码下的 macOS↔Linux 47 项、两台 Mac 文件/快照 50 项、双 Mac 串行真实 OpenCode 7 + 9 + 21 项，以及双机 extension 崩溃、网络中断和数据库停机 25 项 | 当前审计源码对应的公证包尚未生成，需要重跑双机安装 Gate；局域网直连 FSKit 还需要 macOS 本地网络授权；PG 文本索引尚未实现 | | 性能规模 | SQLite 直连已完成 10 万文件；preview.20 真机为 46.863 MiB/s 顺序写、1,041.044 次/s 随机覆盖、175.253 files/s，1 千文件普通扫描 428.041 ms、索引搜索 7.449 ms | 小文件创建仍比同轮 APFS 慢 72.012 倍，普通扫描慢 23.422 倍；后续仍需重跑 1 万/10 万和真实 Coding Agent | “合同中已保存”不等于“所有平台已经完整执行”。例如 owner/group 和便携 ACL 已经可以保存、读取和恢复，但身份认证、权限判断和各系统的原生 ACL 映射仍属于后续阶段。 @@ -119,7 +153,9 @@ vexdb fs --workspace workspace snapshot restore before-refactor 逻辑 export/import、chunk + manifest 和 append 合并发布已进入 spec、runtime/CLI smoke 和 eval； 下一步集中解决小文件写入性能，并补 macOS x86_64 交付验证。 -当前不提前开发 PostgreSQL、DuckDB 或 Windows adapter。 +当前不把主要开发重心完全移出 Phase 1；但已按用户决定完成 PostgreSQL `0.4.0-alpha.1`： +SQL 合同、长期内容模型、完整性检查、空间管理、format v2、ACL、审计、libpq HostStore、 +macOS/Linux mount 和逻辑/物理备份均有真实测试。DuckDB、Windows adapter 仍不提前开发。 ### 5.2 checksum/check 完成证据 @@ -152,8 +188,8 @@ vexdb fs --workspace workspace snapshot restore before-refactor - 8 MiB canonical 内容的固定快照导出、独立 verify、导入、hardlink/版本/xattr/ACL 对比以及 内容损坏和半成品注入测试通过；export、verify、import 的 BLOB 路径固定使用 1 MiB 缓冲。 -GC 删除记录后只让 SQLite 页面可复用，不自动 `VACUUM`；format v1 当前只有 SQLite producer/ -consumer，PG/DuckDB adapter 尚未开始。完整证据见 +GC 删除记录后只让 SQLite 页面可复用，不自动 `VACUUM`；这里记录的是 format v1 当时的 +SQLite 证据。当前 format v2 已支持 SQLite 和 PostgreSQL，DuckDB adapter 尚未开始。完整证据见 `docs/reports/2026-07-23_vexfs-retention-quota-logical-archive.md`。 ### 5.4 chunk + manifest 完成证据 @@ -228,7 +264,7 @@ consumer，PG/DuckDB adapter 尚未开始。完整证据见 - 数据库备份恢复、进程崩溃、断电等价故障和错误卸载有可重复证据； - 发布包、版本清单、使用文档和已知边界放在一起。 -### Phase 2：PostgreSQL 共享工作区 Alpha（macOS + Linux）— 未开始 +### Phase 2：PostgreSQL 共享工作区 Alpha（macOS + Linux）— 功能与验收完成 目标：让电脑 A 创建的 Agent 工作区可以由电脑 B 通过 PostgreSQL 直接挂载，并由数据库角色管理并发和权限。 @@ -242,6 +278,32 @@ consumer，PG/DuckDB adapter 尚未开始。完整证据见 - pg_dump/pg_restore、物理备份和逻辑导入导出； - 电脑 A 写入、电脑 B 挂载并继承同一版本历史和快照。 +当前完成证据： + +- 适配器版本为 `0.4.0-alpha.1`，7 个 PG spec、151 项公共合同/并发 eval、32 项 CLI runtime、 + 74 项 runtime 合同、 + 数据库重启恢复、16/17/18/19 版本矩阵均通过； +- ACL、真实 role、跨 gateway 锁和缓存失效已经进入 PostgreSQL 合同；审计精确记录 + `session_user`/role OID、workspace、commit、path/inode 和 before/after version，workspace + 删除后仍保留名称和证据；普通 role 无权读取； +- `pg_dump/pg_restore`、`pg_basebackup` 和 format v2 SQLite ↔ PG 恢复通过； +- 当前源码下 macOS PG FSKit 13 组 247 项、凭据生命周期 9 项、Linux PG FUSE root/uid 1000 + 各 9 组和 macOS↔Linux 跨系统 47 项通过； +- 两台 Mac 已各运行一次真实 OpenCode：第一台创建并快照，第二台继承同一 Git workspace、 + 修改并快照，第一台回看后恢复完整工作区；三个阶段 7 + 9 + 21 条检查通过； +- 当前源码下两台 Mac 文件、hardlink、symlink、xattr、可执行权限和三次快照往返 50 项通过； +- 两台 Mac 同时挂载的 extension 崩溃、SSH reverse tunnel 中断和 PostgreSQL 停机恢复 25 项通过； + helper 异常退出后底层目录保持 `0500`，不会出现本地假写，显式卸载恢复 `0700`； +- 当前审计实现已复跑 PG 16–19 共 36 项、Linux FUSE root 142 项和 uid 1000 143 项， + 均无失败、无跳过；1000 文件为 649.882 ms、1538.741 files/s，审计数和字段完整性通过； +- 16 MiB 备份性能 Gate 23 项通过，覆盖逻辑/物理备份、克隆启动、format v2 和 RSS/OOM； + 当前数据为逻辑 dump/restore 173.913/175.824 MiB/s、format v2 导出/导入 + 72.727/106.667 MiB/s、`memory.max=1 GiB`、`oom_kill=0`； +- format v2 与备份性能脚本会从所选容器推导本机端口，不能再把容器切到 `5434` 后误测 + 固定在 `5433` 的另一套 PostgreSQL； +- 两台 Mac 的局域网 TCP、libpq 和 runtime 已通过。直连 FSKit 的最后一步需要用户在第二台 + Mac 的系统设置中打开 VexDB Lite 的“本地网络”权限；Apple 不允许程序替用户打开该开关。 + 完成条件： - 两台电脑可以同时操作一个远程 workspace，权限隔离和冲突行为可重复； @@ -249,6 +311,10 @@ consumer，PG/DuckDB adapter 尚未开始。完整证据见 - PostgreSQL 原生备份恢复后文件、历史、权限和业务表处于同一时间点； - macOS 和 Linux 使用同一 mount 一致性测试，不增加 PostgreSQL 专用文件语义。 +当前完成条件已经由本机、Linux、跨系统和双机故障 Gate 证明，Phase 2 状态改为“功能与验收 +完成”。从当前干净提交生成并公证 `preview.38` 仍是必要的发行任务，但不再被误写成 +PostgreSQL 文件合同缺口；之后再决定先做 Windows 还是 DuckDB。 + ### Phase 3：Windows 与跨系统预览 — 未开始 目标：通过 WinFsp 接入同一 mount runtime，使 Windows 既能使用本地 SQLite 工作区，也能挂载 PostgreSQL 远程工作区。 @@ -297,7 +363,7 @@ consumer，PG/DuckDB adapter 尚未开始。完整证据见 下一轮开发固定按以下顺序推进： -### 当前固定顺序（2026-07-23 M1 快照恢复复测后） +### 当前固定顺序（2026-07-23 PostgreSQL Phase 2 收口） 1. **P0（已完成）：修复 FSKit 元数据缓存失效。** hardlink 后立即刷新源 inode；create、symlink、 hardlink、remove、rename 后刷新或失效相关父目录。250 ms TTL 只做兜底，不能代替 @@ -312,22 +378,28 @@ consumer，PG/DuckDB adapter 尚未开始。完整证据见 restore；CLI 正常卸载、恢复并原位重挂载；FSKit 在 `unmount()` / `deactivate()` 关闭 session，在 `activate()` 重开并清理旧缓存。M1 实测 lease 约 2.832 ms 释放，恢复后内容 正确且无残留挂载。 -5. **P0（下一项）：从已提交的干净源码重新签名并公证当前 preview。** 使用已经验证可用的 - `vexdb-lite-notary` profile，要求 Apple `Accepted`、staple、Gatekeeper、安装后 doctor 和 - 快照恢复真机回归全部通过；旧 preview.17 的公证不能替代当前实现的公证。 -6. **P0（紧随公证）：把本次真实挂载快照恢复脚本变成正式 eval。** 覆盖活动 mount 直接 +5. **P0（历史发行证明已完成，当前源码待重做）：** preview.22 已从干净提交签名、公证并 + 通过真实 Gate；当前 preview.37 没有 stapled 票据，不能继承旧包的公证结论。必须使用 + `vexdb-lite-notary` profile 生成 preview.38，并再次要求 Apple `Accepted`、staple、 + Gatekeeper、安装后 doctor 和快照恢复真机回归全部通过。 +6. **P0（已完成）：把本次真实挂载快照恢复脚本变成正式 eval。** 覆盖活动 mount 直接 restore 被拒、一键卸载/恢复/重挂载、lease 立即释放、旧缓存清理、失败回滚、多 workspace 定位和最终无残留挂载，避免以后只靠 `/tmp` 手工脚本发现回归。 -7. **P1：完成干净 Mac 安装。** 在没有源码、Xcode DerivedData 和历史 VexDB Lite 注册记录的 +7. **P1（已完成 arm64 M1）：完成干净 Mac 安装。** 在没有源码、Xcode DerivedData 和历史 VexDB Lite 注册记录的 Mac 上，仅按压缩包文档完成安装、启用、挂载、Bash/Git、快照恢复和卸载。 -8. **P1：完成长期管理能力。** `check` 已完成，下一项固定为 retention/GC，随后是 quota 和 +8. **P1（最小集合已完成）：完成长期管理能力。** `check` 已完成，下一项固定为 retention/GC，随后是 quota 和 export/import，先保证长期使用不会无限增长且可以迁移、恢复。 -9. **P1：恢复规模和 Agent 验证。** 先 1 万文件，再决定是否重跑 10 万；最后执行真实 - OpenCode 工作区。所有大测试继续保留 RSS 上限和低并发。 - -当前 1–4 已完成。接下来严格按 5（当前代码公证）→ 6（永久回归）→ 7（干净 Mac）推进， -然后再做 retention/GC。以上完成前，不把主要开发重心切到 PostgreSQL、DuckDB 或 Windows； -三种数据库和三端适配仍是最终目标，但必须建立在稳定的文件系统合同和发行 Gate 上。 +9. **P1（已完成 PG Alpha 规模和 Agent 验证）：** PostgreSQL 已完成有界性能、1 千文件、 + 真实工具链和双 Mac 串行 OpenCode 完整闭环：第一台创建、第二台继承修改、第一台回看， + 两个 workspace 快照和一键恢复均通过。所有大测试继续保留 RSS 上限和低并发。 +10. **当前唯一动作：** 把已经通过自动测试、本机真挂载和双机故障的工作树形成干净提交， + 生成并公证 `preview.38`，安装后再跑双机安装 Gate。FSKit 当前已经启用，不需要重复处理 + 旧 module；局域网直连另需第二台 Mac 允许 VexDB Lite 使用“本地网络”。 + +PostgreSQL Phase 2 的 format v2、ACL、审计、libpq HostStore、远程 mount、备份和故障恢复 +已经完成。当前不再继续扩展 PG 代码；先关闭两台 Mac 对最终签名版本的系统授权验收点。之后的产品开发只在 +Windows WinFsp 与 DuckDB HostStore 中二选一，不同时铺开。SQLite 的 10 万文件和小文件性能 +仍是独立的 Phase 1 发布优化，不再阻塞 PostgreSQL Alpha 功能结论。 ### N1. 共用 mount 一致性 eval diff --git a/docs/specs/2026-07-16_vexfs-product-spec.md b/docs/specs/2026-07-16_vexfs-product-spec.md index 8301a72e6b..d3a36c2a1c 100644 --- a/docs/specs/2026-07-16_vexfs-product-spec.md +++ b/docs/specs/2026-07-16_vexfs-product-spec.md @@ -5,13 +5,38 @@ - 所属项目：VexDB-Lite - 日期：2026-07-16 - 分支：`feature/agent_files` -- 文档版本：1.3 -- 状态：SQLite 合同、macOS FSKit 垂直闭环和 Linux libfuse3 双架构预览包已实现；当前继续本地开发者预览收口 +- 文档版本：1.8 +- 状态：SQLite `0.9.0`、macOS FSKit、Linux libfuse3 和 arm64 真机交付已实现； +PostgreSQL `0.4.0-alpha.1` 的数据库合同、format v2、ACL、审计、libpq HostStore、 + macOS/Linux 真实 mount、双 Mac 串行 Agent 工作区和备份恢复已实现 - 适用范围：VexFS 完整产品定义；开发顺序以 `docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md` 为准 ## 0. 阅读结论 -VexFS 是由 PostgreSQL、DuckDB 或 SQLite 加载和管理的数据库文件管理扩展。三个数据库共享文件合同，但不要求同时完成。macOS + SQLite 的首个闭环已经跑通，Linux + SQLite 的 libfuse3 真实挂载预览也已经跑通；当前先把这两个本地平台收口成可安装的开发者预览，再进入 PostgreSQL 远程共享工作区。Windows 使用 WinFsp 接入同一 mount runtime；DuckDB 后续先提供 SQL、CLI 和 worktree。 +### 0.0 2026-07-24 macOS 默认挂载决策 + +macOS 0.1 默认使用系统内置 NFS 客户端连接本机 VexFS NFS gateway。FSKit 已完成的实现和 +真机证据继续保留，但降为后续原生增强，不再是首次安装、默认挂载或 0.1 发布 Gate。 + +这个决定只改变 macOS mount adapter，不改变数据库权威核心：NFS gateway、后续 FSKit、 +Linux FUSE 和 Windows WinFsp 必须共用同一 Workspace Engine、mount runtime、数据库合同和 +一致性 eval，不能形成两套文件、版本、权限或备份逻辑。 + +默认 NFS 路径的产品要求是： + +- 用户不需要在“文件系统扩展”中寻找和启用 VexFS； +- `vexdb fs mount` 自动启动当前用户的本机 gateway，并调用系统 NFS 客户端挂载； +- NFS 服务只绑定 loopback，不作为局域网或公网文件服务器； +- mount/unmount/status/doctor 管理 gateway 生命周期和 stale mount； +- FSKit 的签名、entitlement、启用和公证不再阻塞 NFS 版 0.1，但普通 macOS 发行物仍需 + Developer ID 签名与公证，避免 Gatekeeper 阻止 CLI/gateway； +- NFS 协议版本、mount 权限提示、锁、xattr、缓存一致性、sleep/wake 和 gateway crash 必须 + 通过真机 Gate 后，才能宣传“安装后无缝 Bash”。 + +后文中的 FSKit 完成状态仍是历史事实；凡是把 FSKit 写成“默认、MVP 主入口或当前发布前置”的 +旧表述，均由本节替代。 + +VexFS 是由 PostgreSQL、DuckDB 或 SQLite 加载和管理的数据库文件管理扩展。三个数据库共享文件合同，但不要求同时完成。macOS + SQLite 和 Linux + SQLite 的真实挂载预览已经跑通；PostgreSQL `0.4.0-alpha.1` 已完成数据库内合同、format v2、真实 role、ACL、审计、跨 gateway 锁与缓存失效、libpq HostStore、macOS FSKit、Linux FUSE、数据库重启以及逻辑和物理备份恢复。两台真实 Mac 已通过回环隧道串行挂载同一 PG workspace，各运行一次真实 OpenCode，完成创建、继承修改、双快照和整工作区一键恢复；局域网直连 FSKit 只需要用户完成一次 macOS“本地网络”授权。Windows 使用 WinFsp 接入同一 mount runtime；DuckDB 后续先提供 SQL、CLI 和 worktree。 它在数据库中提供文件、目录、路径、版本、快照、权限、审计、配额、完整性检查和逻辑导入导出。文件修改可以和普通业务 SQL 在同一个事务中提交或回滚。 @@ -33,7 +58,10 @@ VexFS 有两种不同的事务入口，文档和产品界面必须明确区分 1. SQL 调用可以参加调用者当前数据库事务，实现“文件 + 业务表”原子提交； 2. mount 中的普通 Bash 操作由 gateway 建立短事务，不能自动加入应用已经打开的数据库事务，多条 Bash 命令也不会自动合成一个事务。 -macOS 可挂载技术闭环支持 macOS 26.0+、Apple Silicon 和 FSKit。FSKit 基础接口从 macOS 15.4 提供，但本产品用来传递数据库目录资源的 `FSPathURLResource` 从 macOS 26.0 才提供。Linux 已有 libfuse3 预览，x86_64 和 AArch64 都已有 manylinux 安装包和对应架构真机安装证据，真实挂载已覆盖 root 与普通用户；Windows、较旧 macOS 和不能启用文件系统扩展的环境仍先使用数据库函数与 CLI。 +macOS 默认挂载改为系统 NFS client + 本机用户态 gateway，不再受 FSKit V2 +`FSPathURLResource` 的 macOS 26.0 最低版本约束；新的最低 macOS 版本和 Intel/Apple Silicon +支持范围由 NFS 发行包真机验证决定。现有 macOS 26.0+ Apple Silicon FSKit 实现作为后续可选 +原生 adapter 保留。Linux 已有 libfuse3 预览，Windows 后续使用 WinFsp。 ### 0.1 最终目标和开发顺序 @@ -79,7 +107,7 @@ VexFS 的不变最终目标、阶段边界、完成条件和当前下一步统 SQLite MVP 没有数据库用户认证。该阶段的 principal 只能来自受签名 App/extension、当前 macOS 用户、security-scoped 数据库目录和数据库文件权限共同建立的本地身份边界；不能把 PostgreSQL 角色语义直接套到 SQLite。 -### 1.2 当前实现状态（2026-07-23） +### 1.2 当前实现状态（2026-07-24） SQLite 合同为 `0.9.0`，runtime ABI 为 1。除单文件版本外，当前实现已经提供完整 workspace 时间点恢复： @@ -105,9 +133,19 @@ SQLite 合同为 `0.9.0`，runtime ABI 为 1。除单文件版本外，当前实 `O_APPEND`、最小 `flock`/`fcntl`、打开文件 rename/unlink、强制卸载和 helper 崩溃恢复已验证； - 同一 SQLite 数据库已完成 macOS → Linux → macOS 往返，内容、Unicode、mode、链接、 xattr、便携 ACL、数字 UID/GID、历史和跨系统快照恢复均通过； -- macOS `preview.17` 已完成 Developer ID 签名、Apple 公证、staple、本机安装和真实 FSKit - 回归；尚缺另一台没有源码和 Xcode 构建记录的干净 Mac 验证。Linux x86_64/AArch64 - `preview.9` 包已完成原生无 root 安装、扩展加载、自检、文件读写和卸载回归。 +- macOS `preview.22` 已完成 Developer ID 签名、Apple 公证、staple、本机安装和真实 FSKit + 回归，另一台没有源码和完整 Xcode 的 M1 Mac 也已完成安装、挂载、Git、OpenCode 和恢复 + 验证。`preview.36` 又修复了重签后 DER entitlement 与原子升级事务；本机已经允许 + `preview.37` 的 FSKit module，`doctor` 连续确认 `extension=enabled` 和 `mount_ready=true`； + 用户再次唤醒系统后，本机 PostgreSQL FSKit 13/13 场景、247 项检查已全部通过，凭据生命周期 + 9 项、macOS↔Linux 跨 gateway 47 项、两台 Mac 文件/快照往返 50 项、双机故障恢复 25 项， + 以及双 Mac 串行真实 OpenCode 的 7 + 9 + 21 条检查也通过。后者覆盖第一台创建、第二台继承 + 同一 Git workspace、第一台回看、两个完整快照和一键恢复。PG 当前 grep 仍是数据库扫描， + 不宣称索引加速。该包仍没有 Apple 公证票据，不能作为正式分发 + 结论；安装器已经修复旧 extension UUID 刷新问题，必须从干净提交构建、公证并 staple + `preview.38` 后再做最终发行复测。Linux + x86_64/AArch64 `preview.9` + 包已完成原生无 root 安装、扩展加载、自检、文件读写和卸载回归。 - Python、Node.js、Go、Rust 和 Git 已在 FSKit 挂载盘中运行并通过重挂载复测；SQLite 直连已通过 10 万文件，FSKit 已通过 1 万文件；10 万文件已完成创建/遍历/抽样读取， 但普通 `rg` 在 3,600 秒内未完成；900 秒混合重开稳定性完成 594,000 @@ -144,9 +182,17 @@ SQLite 合同为 `0.9.0`，runtime ABI 为 1。除单文件版本外，当前实 `importing` 状态完成映射和深度检查，再在同一事务中原子发布； - export、`archive verify` 和 import 都逐个处理不超过 64 KiB 的 chunk，校验和导入 使用同一个事务快照。HEAD 导出遇到未发布句柄时拒绝，指定快照 - 导出不受后续写入影响。当前只实现 SQLite 读写该逻辑格式；PG/DuckDB 消费端尚未实现。 - -当前已经是完整 workspace 时间点恢复，但还不是完整 POSIX 文件系统。SQLite 文件版本 SHA-256、只读 check、live quota、retention、显式 GC、逻辑导入导出和 `chunked-v1` 内容模型已实现；principal 权限执行、ACL 授权、审计、history/staging/index/total quota 和跨文件通用去重仍未实现。四类时间戳和本地最小文件锁已经验证，但跨 gateway 锁、setuid/setgid/sticky bit 和特殊文件没有形成跨平台承诺。hardlink、owner/group 和便携 ACL 当前只能称为“数据库合同已实现并在部分平台接入”，不能称为完整身份权限系统。GC 回收的是数据库可复用页，不会自动执行 `VACUUM` 缩小 SQLite 文件。snapshot 解决误改和任务检查点，SQLite Backup 才是独立备份，两者不能混为一谈。 + 导出不受后续写入影响。当前 SQLite 和 PostgreSQL 已能读写该逻辑格式；DuckDB 消费端尚未实现。 + +当前已经是完整 workspace 时间点恢复，但还不是完整 POSIX 文件系统。SQLite 文件版本 +SHA-256、只读 check、live quota、retention、显式 GC、逻辑导入导出和 `chunked-v1` 内容模型 +已实现；PostgreSQL 还完成了真实 role、路径 ACL 授权与继承、审计、跨 gateway 锁和权限/内容 +失效。SQLite 仍只保存便携 ACL，不执行完整路径授权。产品整体尚未实现 +history/staging/index/total quota、跨文件通用去重、setuid/setgid/sticky bit 和特殊文件。 +hardlink、owner/group 和便携 ACL 已进入公共合同和 macOS/Linux mount 测试，但不能把这些 +能力扩大描述为所有平台的原生 ACL 完全等价。GC 回收的是数据库可复用页，不会自动执行 +`VACUUM` 缩小 SQLite 文件。snapshot 解决误改和任务检查点，SQLite Backup 或 PostgreSQL +原生备份才解决数据库灾难恢复，两者不能混为一谈。 当前符号链接采用 POSIX 行为：target 是原样字符串，可以指向挂载目录外部。它解决兼容性，但不提供路径隔离；需要隔离时仍由 Agent sandbox 负责。未来可以增加可选的 `internal` 策略，但当前版本没有实现该策略。 @@ -584,6 +630,14 @@ VexFS 禁止： | FR-AU-006 | P1 | 支持可选审计 hash 链，用于发现记录被改写。 | | FR-AU-007 | P1 | 工作区可以开启读取审计，默认关闭以控制数据量。 | +PostgreSQL `0.4.0-alpha.1` 当前对 workspace、目录、文件、link、owner、mode、时间戳、 +xattr、ACL、单文件恢复、workspace 恢复和 format v2 导入等变更写审计。每行保存由数据库 +认证的 `session_user` 与 role OID、workspace 名称和 ID、commit、操作、路径、inode、 +`before_version` 与 `after_version`；不保存文件正文、xattr 值、密码或连接串。删除 workspace +前先写审计，外键使用 `ON DELETE SET NULL`，因此删除后仍保留 workspace 名称和最后操作， +不会产生无法识别的孤立记录。普通 role 不能读取审计，`vexfs_check` 会报告对象或前后版本 +不完整的审计行。 + ### 10.8 配额、保留和垃圾回收 | ID | 优先级 | 需求 | @@ -689,29 +743,29 @@ VexFS 使用四层数据保护： | ID | 优先级 | 需求 | |---|---|---| -| FR-MNT-001 | P0 | macOS 发行物必须包含受系统信任的 VexFS App、FSKit App Extension、SQLite 扩展和 CLI；Phase 1 必须验证 Developer ID 公证或 Mac App Store/TestFlight 中至少一条真实分发路径。 | +| FR-MNT-001 | P0 | macOS 发行物必须包含受系统信任的 VexDB CLI、SQLite 扩展和本机 NFS gateway；必须验证 Developer ID 签名、公证、安装、挂载和卸载。FSKit App Extension 是后续可选产物。 | | FR-MNT-002 | P0 | `vexfs mount <目录>` 必须使用当前 context 和 workspace 完成挂载。 | -| FR-MNT-003 | P0 | 必须提供 `mount`、`unmount`、`mount list`、`mount status` 和 `doctor`，并显示 FSKit extension 的安装、启用和版本状态。 | +| FR-MNT-003 | P0 | 必须提供 `mount`、`unmount`、`mount list`、`mount status` 和 `doctor`，并显示 NFS gateway、系统 NFS client、挂载表、端口、进程和版本状态；安装了 FSKit 时再显示可选 adapter 状态。 | | FR-MNT-004 | P0 | mount point 必须是当前操作系统用户控制的空目录，默认权限为 0700。 | | FR-MNT-005 | P0 | 本地 mount 默认只服务当前操作系统用户，不向其他本地用户共享；Linux `allow_other` 默认关闭。 | -| FR-MNT-006 | P0 | FSKit extension 由 macOS 管理生命周期；CLI 不能自行启动长期 root 进程，数据库扩展初始化和 SQL callback 也不能启动文件系统 loop。FSKit extension 可以在自身进程中打开 SQLite 连接。 | +| FR-MNT-006 | P0 | 默认 NFS gateway 以当前用户身份运行，只绑定 loopback，由 CLI 管理生命周期；不能建立长期 root 服务。数据库扩展初始化和 SQL callback 不能启动文件系统 loop 或 NFS 服务。 | | FR-MNT-007 | P1 | 登录后自动挂载必须显式开启，并使用 macOS 允许的用户级生命周期机制。 | | FR-MNT-008 | P0 | mount gateway 停止后，数据库扩展、SQL 和已提交文件必须继续正常工作。 | -| FR-MNT-009 | P0 | FSKit 不可用、extension 未启用、签名不可信或系统版本过低时，doctor 必须给出明确原因、系统设置入口及 CLI/worktree 回退方法。 | +| FR-MNT-009 | P0 | 系统 NFS client 不可用、mount 被系统拒绝、gateway 启动失败、签名不可信或出现 stale mount 时，doctor 必须给出明确原因和清理方法；不能把 FSKit 未启用当作默认挂载失败。 | #### 10.13.2 文件与目录操作 | ID | 优先级 | 需求 | |---|---|---| -| FR-MNT-010 | P0 | FSKit adapter 支持 lookupItem、getAttributes、enumerateDirectory、openItem、closeItem、read、write 和 createItem，并映射到平台无关核心合同。 | -| FR-MNT-011 | P0 | FSKit adapter 支持 removeItem、renameItem、setAttributes、synchronize、reclaimItem 和 statfs 等 MVP 所需操作。 | +| FR-MNT-010 | P0 | macOS NFS adapter 支持 lookup/getattr/readdir/open/close/read/write/create，并映射到平台无关核心合同；后续 FSKit adapter 复用同一合同。 | +| FR-MNT-011 | P0 | macOS NFS adapter 支持 remove/rename/setattr/fsync/commit/statfs 等 MVP 所需操作；协议能力不足时返回稳定错误，不能假装成功。 | | FR-MNT-012 | P0 | 支持 lseek、statfs 和 utimens；chmod、symlink、readlink、hardlink 和 chown 必须按能力矩阵接入。当前 Linux 尚需补齐 utimens 的正式验收。 | | FR-MNT-013 | P0 | 正确处理 O_CREAT、O_EXCL、O_TRUNC、O_APPEND、只读和读写打开模式。 | | FR-MNT-014 | P0 | 同一目录内及跨目录 rename 必须原子生效，不能短暂出现两个名称都不存在。 | | FR-MNT-015 | P0 | open 后 unlink 必须允许已打开句柄继续使用；最后一个句柄关闭后再回收无引用版本。 | | FR-MNT-016 | P0 | 路径规范化、`.`、`..` 和越界规则必须与 SQL 合同一致；当前符号链接明确采用 POSIX target 原样保存合同，未来的 workspace 内限制策略必须作为独立可选合同。 | | FR-MNT-017 | P0 | 不支持设备文件、socket、FIFO 和 setuid/setgid 文件；创建时返回明确错误。 | -| FR-MNT-018 | P1 | MVP Gate 不承诺 advisory flock；Phase 0 记录 macOS/FSKit 实际锁行为，跨 gateway 锁在 PostgreSQL 阶段映射为数据库租约或锁。 | +| FR-MNT-018 | P1 | MVP Gate 不承诺 advisory flock；Phase 0 必须记录 macOS NFS 的 `flock/fcntl` 实际行为。若默认 mount 使用 `nolocks`，doctor 和能力矩阵必须明确报告，不能声称适合在挂载内运行依赖文件锁的数据库。 | | FR-MNT-019 | P1 | 支持 hardlink，并在支持前对 link 调用返回稳定的不支持错误。 | | FR-MNT-020 | P1 | 已支持扩展属性，并把值持久化到 SQLite；未知或不存在的 xattr 返回稳定错误。 | @@ -719,9 +773,9 @@ VexFS 使用四层数据保护： | ID | 优先级 | 需求 | |---|---|---| -| FR-MNT-021 | P0 | 普通文件写入先进入带 request id 的未发布 staging；平台无关 `publish/sync` 以幂等短事务发布完整新版本。macOS adapter 主要由 `synchronize` 和 `closeItem` 触发。 | +| FR-MNT-021 | P0 | 普通文件写入先进入带 request id 的未发布 staging；平台无关 `publish/sync` 以幂等短事务发布完整新版本。macOS NFS adapter 主要由 COMMIT/fsync、close 和安全卸载触发；具体映射必须由真实 syscall/protocol trace 冻结。 | | FR-MNT-022 | P0 | 不能在文件 open 到 close 的整个期间持有长期数据库事务。 | -| FR-MNT-023 | P0 | `write` 成功表示对应数据库 staging 已保存，但不表示新版本已发布。macOS SQLite mount 的未发布 staging 使用 WAL + NORMAL；`synchronize/publish/close` 前必须切回 FULL。电源故障可以丢失最近的未发布 staging，但不能产生半个已发布文件；`synchronize` 成功后完整版本必须达到 FULL。普通 SQL 连接仍使用调用者自己的 durability 配置。 | +| FR-MNT-023 | P0 | `write` 成功的含义必须由 NFS reply、数据库 staging 和 COMMIT/fsync 边界共同定义，不能在数据只存在 gateway 易失内存时伪造已安全保存。SQLite mount 的未发布 staging 使用 WAL + NORMAL；强 publish/sync 前切回 FULL。普通 SQL 连接仍使用调用者自己的 durability 配置。 | | FR-MNT-024 | P0 | create、mkdir、unlink、rename 和 truncate 等单个文件系统操作各自在数据库事务内原子执行；chmod/chown 在进入支持范围后遵守同一规则。 | | FR-MNT-025 | P0 | 多条 Bash 命令和多个 file handle 默认不是一个数据库事务，文档和错误信息不能暗示它们天然原子。 | | FR-MNT-026 | P0 | 多文件或“文件 + 业务 SQL”原子提交必须使用公开 changeset/数据库事务合同。 | @@ -744,7 +798,7 @@ VexFS 使用四层数据保护： | ID | 优先级 | 需求 | |---|---|---| -| FR-MNT-036 | P0 | 正确性优先：MVP Gate 不启用 FSKit `DataCacheHandler` 的 kernel data cache；如 SDK 要求协商缓存，选择 no-cache 模式。后续启用缓存前必须具备版本校验和权限失效方案。 | +| FR-MNT-036 | P0 | 正确性优先：NFS attribute/data cache 的 TTL、close-to-open、COMMIT/fsync 和主动失效行为必须可测；默认参数不能让刚写、rename、restore 或权限变化后的内容静默保持旧值。FSKit DataCacheHandler 留到后续版本。 | | FR-MNT-037 | P0 | 数据库连接中断时，未确认提交的操作必须返回 I/O 错误，不能报告成功。 | | FR-MNT-038 | P0 | gateway 崩溃后，数据库中只能存在完整旧版本、完整新版本或可识别的未发布暂存对象。 | | FR-MNT-039 | P0 | 重新挂载后必须清理或恢复过期暂存对象，不能把半写文件发布给用户。 | @@ -752,7 +806,7 @@ VexFS 使用四层数据保护： | FR-MNT-041 | P0 | mount gateway 不能成为数据库备份的一部分；数据库原生备份必须独立包含全部已提交状态。 | | FR-MNT-042 | P1 | 多个 gateway 同时访问时支持主动失效通知；在此能力完成前，不承诺跨 gateway 实时可见，只保证 close-to-open、冲突检测和不静默覆盖。 | | FR-MNT-043 | P0 | 同一 handle 必须读到自己的 staged writes；其他已打开 handle 继续读取其打开时版本；成功 publish/synchronize 后的新 open 读取新版本。 | -| FR-MNT-044 | P0 | closeItem/reclaimItem 时如果仍有 dirty staging，可以执行可恢复的兜底发布，但失败不能删除 staging、伪造成功或产生半版本。 | +| FR-MNT-044 | P0 | NFS close/COMMIT/fsync 或 gateway 回收 handle 时如果仍有 dirty staging，可以执行可恢复的兜底发布，但失败不能删除 staging、伪造成功或产生半版本。 | ### 10.14 多数据库适配 @@ -1057,6 +1111,15 @@ worktree 清单不保存凭证，也不能被数据库扩展信任为权限依 - 权限和 ACL 需要验证恢复结果； - VexFS 不调用或调度 pg_dump、base backup 或复制。 +当前 PostgreSQL `0.4.0-alpha.1` 已实现文件版本、workspace commit、snapshot、 +expected-head/version、chunk/manifest、SHA-256、quick/deep check、quota、retention、 +分批 GC、ACL、完整变更审计、format v2 和 libpq 远程 mount。`pg_dump/pg_restore` 与 +`pg_basebackup` 已通过恢复测试，恢复后文件、历史、快照、设置、权限、审计和业务表保持一致。 +16 MiB 有界性能 Gate 已验证克隆可启动、`pg_verifybackup`、format v2 导入导出、CLI +峰值 RSS、容器 `memory.max=1 GiB`，且运行前后 `oom_kill=0`。当前审计实现下，逻辑 +dump/restore 为 173.913/175.824 MiB/s，format v2 导出/导入为 72.727/106.667 MiB/s； +这些是 2026-07-24 本机有界样本，不是所有机器的固定承诺。 + ### 15.3 SQLite - 使用 SQLite Online Backup API 或安全数据库备份方式； @@ -1103,8 +1166,9 @@ workspace 历史。canonical 内容保持 manifest + 64 KiB chunk，恢复别名 导入目标 workspace 必须不存在。导入先创建同一事务内不可见的 `importing` workspace，映射 commit/inode/manifest ID，重建 chunk 引用、当前树和历史，运行深度 `vexfs_check`，成功后才切换成 `active` 并提交。 任何格式、结构、记录、内容、整包 hash 或深度检查失败都会回滚，目标名称不会留下半成品。 -SQLite 到 SQLite 采用 principal 原值映射；未来 PG/DuckDB 导入必须在 format v2 principal -表之上增加显式目标身份映射，不能自动扩大权限。 +SQLite 到 SQLite 采用 principal 原值映射；PostgreSQL 导入把目标 owner 绑定到真实数据库 +role，并保留可移植 ACL 信息，不能自动扩大权限。DuckDB 后续也必须在 format v2 principal +表之上实现明确的目标身份映射。 ## 16. 三端能力矩阵 @@ -1117,10 +1181,10 @@ SQLite 到 SQLite 采用 principal 原值映射；未来 PG/DuckDB 导入必须 | 冲突方式 | 锁、MVCC、约束 | busy、单 writer | 乐观冲突 | | WAL/恢复 | PostgreSQL 管理 | SQLite 管理 | DuckDB 管理 | | 原生备份 | PG 工具 | Online Backup API | DB copy/export/安全文件流程 | -| 路径 ACL | 完整 | 应用内 | 应用内 | -| P0 合同 | Phase 2 实现 | Phase 1 收口 | Phase 4 实现 | -| 可写 macOS mount | Phase 2 | 已有 SQLite 预览 | Phase 4 后按单写者条件决定 | -| 可写 Linux mount | Phase 2 | 已有 libfuse3 预览 | Phase 4 后按单写者条件决定 | +| 路径 ACL | 已实现 | 应用内 | 应用内 | +| P0 合同 | 已实现 | Phase 1 收口 | Phase 4 实现 | +| 可写 macOS mount | 已实现 | 已有 SQLite 预览 | Phase 4 后按单写者条件决定 | +| 可写 Linux mount | 已实现 | 已有 libfuse3 预览 | Phase 4 后按单写者条件决定 | | 可写 Windows mount | Phase 3 | Phase 3 | Phase 4 后按单写者条件决定 | 版本约束： @@ -1347,14 +1411,14 @@ chmod/chown 等已明确不支持的权限修改返回 `EPERM`；跨 workspace r | ID | 验收场景 | |---|---| -| AC-MVP-001 | 在干净的 macOS 26.0+ Apple Silicon 环境安装并启用 VexFS FSKit extension 后，`vexfs doctor` 通过，一条 `vexfs mount` 命令得到可 `cd` 的真实目录。 | +| AC-MVP-001 | 在干净 Mac 安装默认 NFS 发行物后，不启用文件系统扩展，`vexdb fs doctor` 通过，一条 `vexdb fs mount` 命令得到可 `cd` 的真实目录。最低系统和 CPU 范围由发行真机矩阵确定。 | | AC-MVP-002 | `ls/cat/cp/mv/rm/mkdir/find/grep/sed`、Python 读写和基础 Git `init/add/commit/checkout` 直接运行。 | -| AC-MVP-003 | FSKit lookup/get attributes/enumerate/open/close/create/read/write/remove/rename/set attributes/synchronize/reclaim 和 statfs 通过核心合同测试。 | +| AC-MVP-003 | macOS NFS mount 的 lookup/getattr/readdir/open/close/create/read/write/remove/rename/setattr/fsync/COMMIT 和 statfs 通过核心合同与真实 Bash 测试。 | | AC-MVP-004 | 同一 handle 读到自己的未发布写入；其他 handle 看到打开时版本；成功 publish/synchronize 后的新 open 看到新版本。 | | AC-MVP-005 | `synchronize` 重复调用不产生重复 commit；close/reclaim 失败不能丢弃 dirty staging 或伪造提交成功。 | -| AC-MVP-006 | `synchronize` 成功后终止 FSKit extension、重启 SQLite 并重新挂载，内容和 checksum 保持正确。 | +| AC-MVP-006 | 强 sync 成功后终止 NFS gateway、重启 SQLite 并重新挂载，内容和 checksum 保持正确；未 sync 窗口的行为有明确测试证据。 | | AC-MVP-007 | SQLite busy、连接中断、空间不足、版本冲突和 stale handle 返回冻结后的稳定 errno。 | -| AC-MVP-008 | MVP 不启用 FSKit kernel data cache，mount point 为当前用户私有，其他本地用户不能访问。 | +| AC-MVP-008 | NFS gateway 只监听 loopback，mount point 为当前用户私有；其他本地用户和局域网主机不能访问 export。 | | AC-MVP-009 | 1,000 个文件、1 GiB 当前内容、100 MiB 单文件完成写入、重挂载、读取和校验，没有半文件或丢失目录项。 | | AC-MVP-010 | gateway 与扩展合同版本不兼容时拒绝可写挂载；卸载 gateway 后，SQL 仍可读写全部已提交文件。 | @@ -1377,8 +1441,8 @@ chmod/chown 等已明确不支持的权限修改返回 `EPERM`；跨 workspace r | AC-P0-011 | 缺失 chunk 时 check 报告损坏，read 不返回伪造内容。 | | AC-P0-012 | 10 万文件和 10 GiB 数据集通过正确性与初始性能目标。 | | AC-P0-013 | 完成一次 context 和 workspace 配置后，用户可以用一条 vexfs mount 命令得到真实目录并直接 cd 进入。 | -| AC-P0-014 | FSKit extension 被终止、数据库重启或连接中断后，已成功 synchronize 的文件仍完整，未成功发布不被伪装成成功。 | -| AC-P0-015 | vexfs doctor 能准确报告 App/extension 未安装、FSKit extension 未启用、系统版本过低、签名或版本不兼容、未初始化、连接失败和权限不足。 | +| AC-P0-014 | NFS gateway 被终止、数据库重启或连接中断后，已成功强 sync 的文件仍完整，未成功发布不被伪装成成功。 | +| AC-P0-015 | `vexdb fs doctor` 能准确报告 gateway 不存在或版本不兼容、NFS client 不可用、mount 权限被拒、stale mount、端口冲突、未初始化、连接失败和权限不足。 | | AC-P0-016 | mount point 默认仅当前操作系统用户可访问，数据库 ACL 变化会影响后续文件操作。 | | AC-P0-017 | 两个 mount 同时修改同一版本时至少一个得到冲突，不能静默覆盖。 | | AC-P0-018 | 不安装或不运行 gateway 时，SQL 和 CLI 数据能力仍然可用。 | @@ -1427,6 +1491,9 @@ chmod/chown 等已明确不支持的权限修改返回 `EPERM`；跨 workspace r - 多连接并发和 deadlock； - WAL/restart； - pg_dump/pg_restore 和物理备份恢复； +- 两个真实 mount gateway 同时在线时，分别注入 helper 崩溃、网络中断和数据库停机； +- helper 异常撤销后底层 mountpoint 必须不可写，不能让普通用户产生未进入数据库的本地文件； +- 网络恢复不得隐式重放提交状态未知的写请求，必须先用幂等读操作确认连接与状态； - extension 安装和升级。 #### DuckDB @@ -1449,7 +1516,7 @@ chmod/chown 等已明确不支持的权限修改返回 `EPERM`；跨 workspace r |---|---|---| | Phase 0 | 统一合同、macOS + SQLite 垂直闭环、Linux 真实挂载验证 | 已完成 | | Phase 1 | SQLite 本地开发者预览：macOS + Linux 可安装、可恢复、可跑真实项目 | 进行中 | -| Phase 2 | PostgreSQL 共享工作区：跨电脑、多用户、数据库权限和并发 | 未开始 | +| Phase 2 | PostgreSQL 共享工作区：跨电脑、多用户、数据库权限和并发 | 代码与自动测试完成；当前源码 Developer ID 包已验证并安装，待重新授权后补跑真挂载 Gate | | Phase 3 | Windows WinFsp 和 macOS/Linux/Windows 跨系统预览 | 未开始 | | Phase 4 | DuckDB 本地分析工作区 Beta | 未开始 | | Phase 5 | VexDB-Lite Files v1 稳定版 | 未开始 | @@ -1512,9 +1579,10 @@ chmod/chown 等已明确不支持的权限修改返回 `EPERM`；跨 workspace r - “安装后无缝 Bash”是第一产品目标； - 产品 P0 与首个 MVP Gate 分开管理，P0 能力不要求全部进入 SQLite 首个闭环； - mount gateway 是第一等入口，但不是权威核心； -- macOS FSKit mount 是 MVP 和 P0 主入口； -- 可挂载 MVP 支持 macOS 26.0+ Apple Silicon，使用 FSUnaryFileSystem、FSVolume 稳定接口和 FSKit V2 的 FSPathURLResource； -- MVP Gate 不启用 FSKit kernel data cache，先保证权限与正确性； +- macOS 默认 mount 是系统 NFS client + 本机用户态 NFS gateway； +- FSKit 作为后续原生增强保留，不是 0.1 首次安装和发布前置； +- NFS gateway 只绑定 loopback，不作为远程共享协议；跨电脑共享仍由每台机器的本地 mount gateway 连接 PostgreSQL； +- 默认 NFS 的协议版本、锁、xattr、缓存、sleep/wake、mount 权限和 crash 恢复必须先通过真机 Gate； - mount 采用 close-to-open 一致性，不承诺完整跨 gateway POSIX 强一致； - 平台无关 publish/sync 是可报告的幂等发布边界；macOS 由 synchronize 映射，close/reclaim 只做可恢复兜底； - 一个 mount 绑定一个数据库 principal，mount point 默认当前用户私有； @@ -1524,7 +1592,7 @@ chmod/chown 等已明确不支持的权限修改返回 `EPERM`；跨 workspace r - 常用命令直接映射 SQL； - 任意普通终端程序优先通过挂载目录运行； - worktree 是不能挂载时的回退，不是权威存储或备份； -- mount gateway 不建立新网络协议，不提供 SDK，不管理数据库； +- macOS NFS 只作为 loopback mount adapter，不开放远程文件服务、不提供 SDK、不管理数据库； - 通过 SQL 调用的文件修改参加调用者当前数据库事务；mount 操作使用 gateway 短事务； - 权限来自数据库可证明身份； - 使用文件版本、快照、数据库原生备份和逻辑导出四层保护； @@ -1568,18 +1636,29 @@ chmod/chown 等已明确不支持的权限修改返回 `EPERM`；跨 workspace r ### 27.1 用户需要安装什么 -macOS 用户安装一个签名并公证的 VexDB-Lite 发行包，包内包含四个产物： +macOS 用户安装一个签名并公证的 VexDB-Lite 发行包，默认包含三个产物： + +1. SQLite VexFS 数据库扩展； +2. 当前用户运行的本机 NFS gateway； +3. `vexdb` CLI，以及指向 `vexdb fs` 的 `vexfs` 兼容入口。 + +默认安装不要求用户进入“文件系统扩展”页面。`vexdb fs mount` 负责启动 gateway、确认它只监听 +loopback、调用系统 NFS client 并验证真实读写。macOS 仍可能针对挂载操作、目标目录或下载的 +可执行文件给出普通系统授权或 Gatekeeper 提示；安装器必须把实际提示做成真机 Gate，不能把 +“不需要 FSKit 授权”写成“macOS 永远不会出现任何授权”。 -1. `VexDB Lite.app`； -2. App 内的 FSKit File System Extension，它承担首个 mount gateway； -3. SQLite VexFS 数据库扩展； -4. `vexdb` CLI，以及指向 `vexdb fs` 的 `vexfs` 兼容入口。 +后续 FSKit 版本可以在发行包中增加 `VexDB Lite.app` 和 App Extension，作为显式选择的 +`--backend fskit`；它不能拥有另一套数据库 schema、版本、权限或备份逻辑。 -macOS 本地预览首先支持 macOS 26.0+ Apple Silicon。Phase 1 必须证明至少一条真实分发路径：优先 Developer ID 签名与公证，如果 FSKit entitlement 或系统策略不允许，则使用 Mac App Store/TestFlight。首次打开 App 后，用户需要在“系统设置 → 通用 → 登录项与扩展 → 文件系统扩展”中启用 VexFS；这是系统安全步骤，产品不能静默绕过。App 必须显示准确引导，`vexfs doctor` 必须检测是否已经启用。 +发行包、解压验证目录和构建目录不得长期暴露第二份同 bundle ID 的 `.app`。打包完成后必须撤销这些临时路径的 LaunchServices 登记并删除展开 stage；安装包把 App 放在隐藏 `.payload` 中。扩展已经获准时，安装器必须以临时数据库执行真实 mount、写入、读回和 unmount，不能只根据注册状态宣布成功。 -默认 SQLite 数据库放在 `~/Library/Application Support/VexDB-Lite/default.sqlite3`。每个 mount 把数据库所在的独立 0700 资源目录作为 security-scoped `FSPathURLResource` 交给系统；目录内的非权威 `.vexfs-volume.json` 只保存合同版本、数据库文件名和 workspace，不保存密码或绝对路径。extension 必须拒绝 `.`、`..`、子路径和任何逃逸该目录的文件名。一个数据库文件独占一个父目录，目录只放 descriptor、数据库、WAL 和 SHM。 +默认 SQLite 数据库放在 `~/Library/Application Support/VexDB-Lite/default.sqlite3`。NFS gateway +直接以当前用户权限打开数据库，不需要 security-scoped `FSPathURLResource`。非权威 mount +descriptor 只保存合同版本、数据库标识、workspace、gateway 实例和 mountpoint，不保存密码。 -CLI 与 FSKit extension 必须在挂载前读取数据库扩展的合同版本和能力位。主版本不兼容时拒绝可写挂载；卸载时先停止新操作、执行 synchronize、报告仍未发布的 handle，再执行 unmount。安装、升级和卸载都不得删除数据库中的已提交文件。 +CLI 与 NFS gateway 必须在挂载前读取数据库扩展的合同版本和能力位。主版本不兼容时拒绝可写 +挂载；卸载时先停止新操作、执行强 sync、报告仍未发布的 handle，再执行 unmount 并退出 +gateway。安装、升级和卸载都不得删除数据库中的已提交文件。 运行时边界保持不变：数据库扩展保存全部权威状态；mount gateway 只把数据库文件展示为本地路径；CLI 管理连接和挂载。gateway 不保存最终文件，不管理数据库。 diff --git a/tests/eval/vexfs/Dockerfile.linux-fuse b/tests/eval/vexfs/Dockerfile.linux-fuse index 66654ae003..3acb81ead1 100644 --- a/tests/eval/vexfs/Dockerfile.linux-fuse +++ b/tests/eval/vexfs/Dockerfile.linux-fuse @@ -8,6 +8,8 @@ RUN apt-get update \ fuse3 \ git \ libfuse3-dev \ + libpq-dev \ + postgresql-client \ pkg-config \ && rm -rf /var/lib/apt/lists/* diff --git a/tests/eval/vexfs/Dockerfile.pg-version-matrix b/tests/eval/vexfs/Dockerfile.pg-version-matrix new file mode 100644 index 0000000000..f4d195505c --- /dev/null +++ b/tests/eval/vexfs/Dockerfile.pg-version-matrix @@ -0,0 +1,25 @@ +ARG PG_MAJOR=16 +FROM postgres:${PG_MAJOR}-bookworm AS build + +ARG PG_MAJOR +ARG JOBS=1 +RUN apt-get update && apt-get install -y --no-install-recommends \ + cmake g++ make \ + libboost-system-dev libboost-thread-dev \ + postgresql-server-dev-${PG_MAJOR} \ + && rm -rf /var/lib/apt/lists/* + +COPY vexdb_lite_src /src/vexdb_lite +WORKDIR /src/vexdb_lite +RUN PG_CONFIG=/usr/lib/postgresql/${PG_MAJOR}/bin/pg_config \ + cmake -S vexdb_pg -B build/pg \ + -DCMAKE_BUILD_TYPE=Release \ + -DBOOST_FALLBACK_INC=/src/vexdb_lite/thirdparties/boost \ + && cmake --build build/pg -j${JOBS} \ + && DESTDIR=/stage cmake --install build/pg + +FROM postgres:${PG_MAJOR}-bookworm +RUN apt-get update && apt-get install -y --no-install-recommends \ + libboost-system1.74.0 libboost-thread1.74.0 \ + && rm -rf /var/lib/apt/lists/* +COPY --from=build /stage/ / diff --git a/tests/eval/vexfs/README.md b/tests/eval/vexfs/README.md index 6ea0405447..3005972bf6 100644 --- a/tests/eval/vexfs/README.md +++ b/tests/eval/vexfs/README.md @@ -13,25 +13,30 @@ bash build_sqlite.sh eval bash build_sqlite.sh eval quick # 指定已有构建目录（评测器会从这里查找 vexdb、扩展和测试产物） -python3 tests/eval/vexfs/run.py --mode quick --build-dir vexdb_sqlite/build +tests/eval/vexfs/python.sh tests/eval/vexfs/run.py --mode quick --build-dir vexdb_sqlite/build # 最大规模（128 MiB 文件、1 万随机操作、1 万小文件） bash build_sqlite.sh eval stress # 只跑某一类或某个用例 -python3 tests/eval/vexfs/run.py --mode full --filter recovery -python3 tests/eval/vexfs/run.py --mode full --filter performance.staged-overwrite -python3 tests/eval/vexfs/run.py --mode full --filter performance.mount-contract-small-files +tests/eval/vexfs/python.sh tests/eval/vexfs/run.py --mode full --filter recovery +tests/eval/vexfs/python.sh tests/eval/vexfs/run.py --mode full --filter performance.staged-overwrite +tests/eval/vexfs/python.sh tests/eval/vexfs/run.py --mode full --filter performance.mount-contract-small-files + +# 真实挂载规模搜索默认最多等待 quick=60、full=300、stress=600 秒；可显式覆盖 +VEXFS_EVAL_NATIVE_SEARCH_TIMEOUT_SECONDS=900 \ + tests/eval/vexfs/python.sh tests/eval/vexfs/run.py --mode stress --filter mount.scale-tree \ + --mount-cli /absolute/path/to/vexfs --fail-on-skip # 把性能预算也作为硬 Gate；默认只记录指标，避免不同机器误报 -python3 tests/eval/vexfs/run.py --mode full --enforce-performance +tests/eval/vexfs/python.sh tests/eval/vexfs/run.py --mode full --enforce-performance # 发版前重新打包和签名后，再显式运行交付包测试 VEXDB_LITE_PACKAGE_STAGE=/absolute/path/to/stage \ - python3 tests/eval/vexfs/run.py --mode full --include-package --fail-on-skip + tests/eval/vexfs/python.sh tests/eval/vexfs/run.py --mode full --include-package --fail-on-skip # 真实挂载和跨平台 Gate 必须显式指定本次要验证的挂载 CLI；任何 SKIP 都失败 -python3 tests/eval/vexfs/run.py --mode full --filter mount \ +tests/eval/vexfs/python.sh tests/eval/vexfs/run.py --mode full --filter mount \ --mount-cli /absolute/path/to/vexfs --fail-on-skip # Linux AArch64/x86_64：在真实 /dev/fuse 上分别以 root 和 uid 1000 @@ -44,13 +49,149 @@ bash tests/eval/vexfs/run_cross_platform_portability.sh # 显式调用真实 OpenCode 模型，在挂载项目中自主改代码并跑测试 # 默认不运行，避免日常回归自动消耗模型额度 VEXFS_EVAL_OPENCODE=1 \ - python3 tests/eval/vexfs/run.py --mode quick --filter mount.real-opencode-project + tests/eval/vexfs/python.sh tests/eval/vexfs/run.py --mode quick --filter mount.real-opencode-project # 可选择模型，默认 openai/gpt-5.4-mini VEXFS_EVAL_OPENCODE=1 VEXFS_EVAL_OPENCODE_MODEL=openai/gpt-5.4-mini \ - python3 tests/eval/vexfs/run.py --mode quick --filter mount.real-opencode-project + tests/eval/vexfs/python.sh tests/eval/vexfs/run.py --mode quick --filter mount.real-opencode-project +``` + +## PostgreSQL adapter + +PG eval 使用真实 PostgreSQL、真实扩展表、libpq、FSKit 和 FUSE，不使用 mock。先把当前 +`vexdb_lite--1.0.sql` 和 `vexdb_lite.control` 安装进测试容器，再按需要执行： + +```bash +# 渲染并运行全部 VexFS PG spec +tests/eval/vexfs/python.sh tests/spec/_lib/render.py --engine pg --out build/spec +VEXDB_PG_CONTAINER=vexdb_pg19-vexfs-dev \ + bash tests/spec/_lib/docker/run_pg.sh test 'pg__vexfs_*' + +# SQL adapter、并发写和 expected-head 恢复冲突 +VEXDB_PG_CONTAINER=vexdb_pg19-vexfs-dev \ + bash tests/eval/vexfs/run_pg_adapter_alpha.sh + +# 1000 个文件的有界性能基线；参数硬限制为 1..10000 +VEXDB_PG_CONTAINER=vexdb_pg19-vexfs-dev VEXDB_PG_PERF_FILES=1000 \ + bash tests/eval/vexfs/run_pg_adapter_performance.sh + +# libpq HostStore、CLI、handle、ACL、审计、多 gateway 和缓存失效 +VEXDB_PG_CONTAINER=vexdb_pg19-vexfs-dev \ +VEXDB_PG_DSN=postgresql://postgres@127.0.0.1:5434/test \ + bash tests/eval/vexfs/run_pg_runtime.sh + +# 数据库停止和重启后的句柄、锁、缓存和内容恢复 +VEXDB_PG_CONTAINER=vexdb_pg19-vexfs-dev \ +VEXDB_PG_DSN=postgresql://postgres@127.0.0.1:5434/test \ + bash tests/eval/vexfs/run_pg_restart_recovery.sh + +# 完整数据库 pg_dump/pg_restore +VEXDB_PG_CONTAINER=vexdb_pg19-vexfs-dev \ + bash tests/eval/vexfs/run_pg_backup_restore.sh + +# pg_basebackup 物理克隆 +VEXDB_PG_CONTAINER=vexdb_pg19-vexfs-dev \ + bash tests/eval/vexfs/run_pg_physical_backup_restore.sh + +# 16 MiB 有界备份性能 Gate：逻辑备份、物理备份、format v2 和峰值 RSS +# 容器 memory.max 必须不超过 1 GiB，脚本也会检查 oom_kill 没有增加 +# 未设置 VEXDB_PG_HOST_PORT 时会读取所选容器发布的 5432/tcp 端口。 +VEXDB_PG_CONTAINER=vexdb_pg19-vexfs-dev \ + bash tests/eval/vexfs/run_pg_backup_performance.sh + +# format v2 SQLite → PostgreSQL → SQLite；未设置 VEXDB_PG_DSN 时， +# 脚本会从所选容器的 5432/tcp 发布端口推导本机 DSN，避免连到另一套 PG。 +VEXDB_PG_CONTAINER=vexdb_pg19-vexfs-dev \ + bash tests/eval/vexfs/run_pg_archive_roundtrip.sh + +# PostgreSQL 16/17/18/19，按顺序运行且每个容器限制 1 GiB +# PG19 仍是预发布版，没有官方 postgres:19 镜像，因此显式复用已安装当前扩展、 +# memory.max 不超过 1 GiB 的 PG19 测试容器。 +VEXDB_PG19_MATRIX_CONTAINER=vexdb_pg19-vexfs-dev \ + bash tests/eval/vexfs/run_pg_version_matrix.sh + +# macOS：真实 FSKit 的 13 组公共 mount 合同 +VEXDB_PG_CONTAINER=vexdb_pg19-vexfs-dev \ +VEXDB_PG_DSN=postgresql://postgres@127.0.0.1:5434/test \ + bash tests/eval/vexfs/run_pg_macos_mount.sh + +# Linux：真实 libfuse3，root 和 uid 1000 +VEXDB_PG_CONTAINER=vexdb_pg19-vexfs-dev \ + bash tests/eval/vexfs/run_pg_linux_mount.sh + +# 同一 PG 工作区在 macOS FSKit 和 Linux FUSE 间往返 +VEXDB_PG_CONTAINER=vexdb_pg19-vexfs-dev \ +VEXDB_PG_DSN=postgresql://postgres@127.0.0.1:5434/test \ + bash tests/eval/vexfs/run_pg_cross_gateway.sh + +# 两台 Mac：真实 FSKit、extension 崩溃、网络中断和数据库停机恢复。 +# 默认通过 SSH reverse tunnel 连接本机 PG，因此不依赖远端 Mac 的本地网络授权。 +VEXFS_REMOTE_HOST= VEXFS_REMOTE_USER= \ +VEXDB_PG_CONTAINER=vexdb_pg19-vexfs-dev VEXFS_LOCAL_PG_PORT=5434 \ + tests/eval/vexfs/run_pg_remote_macos_faults.sh + +# 最初产品场景的 PostgreSQL 版本：Linux PG + 两台 Mac FSKit + 两次真实 OpenCode， +# 串行修改同一个 workspace，列出两个快照并一键恢复。该用例会调用外部模型，必须显式运行。 +VEXFS_REMOTE_HOST= VEXFS_REMOTE_USER= \ +VEXDB_PG_CONTAINER=vexdb_pg19-vexfs-dev VEXFS_LOCAL_PG_PORT=5434 \ +VEXFS_LOCAL_OPENCODE_MODEL=openai/gpt-5.4-mini \ +VEXFS_REMOTE_OPENCODE_MODEL=opencode/north-mini-code-free VEXFS_KEEP_WORKSPACE=1 \ + tests/eval/vexfs/run_pg_two_mac_opencode.sh ``` +备份恢复 eval 会检查 `vexdb_lite.control` 的默认版本为 `1.0`。这是发布门禁： +`pg_dump` 在目标库安装扩展时使用目标机器的默认版本，所以控制文件与对应版本 SQL 必须 +一起安装。 + +macOS 远程 PG 推荐使用 libpq passfile，文件必须是当前用户拥有的普通文件且权限为 `0600`： + +```bash +chmod 0600 ~/.pgpass +vexdb fs --backend pg \ + --dsn 'host=db.example port=5432 dbname=app user=agent passfile=/Users/me/.pgpass' \ + --workspace agent-workspace mount ~/Work/agent-workspace +``` + +DSN 中的明文密码、符号链接 passfile 和过宽权限会在调用 FSKit 前被拒绝。挂载时的受保护 +passfile 副本只在资源仍被挂载时保留，最后卸载或 `vexdb fs doctor` 会清理。可用 +`run_pg_macos_credentials.sh` 做自动回归。 + +两台 Mac 直连局域网 PostgreSQL 前，要分别启用： + +1. “系统设置 → 通用 → 登录项与扩展 → 文件系统扩展”中的 VexDB Lite； +2. “系统设置 → 隐私与安全性 → 本地网络”中的 VexDB Lite。 + +第二项是 macOS 隐私授权，安装脚本不能替用户打开。首次远程 mount 失败时，CLI 会自动打开 +VexDB Lite，用目标 host/port 发起不含凭据的真实连接检查，触发授权并在允许后重试。若用户 +以前拒绝过，CLI 会明确提示上面的系统设置路径。完成后使用 +`run_pg_remote_macos.sh` 执行本机创建、远端直读、远端挂载修改、本机快照恢复、远端最终验证 +五个阶段。无人值守 CI 可用 SSH 隧道把远端 PG 暴露为回环地址，验证同一 FSKit/runtime 合同， +但隧道结果不能替代局域网权限验收。 + +回环隧道下本机和远端端口通常不同。`run_pg_remote_macos.sh` 分别接受 +`VEXFS_LOCAL_PG_HOST`/`VEXFS_LOCAL_PG_PORT` 与 +`VEXFS_REMOTE_PG_HOST`/`VEXFS_REMOTE_PG_PORT`；本机可以直连数据库发布端口，远端只连接 +SSH reverse tunnel，两个 passfile 也按各自 endpoint 生成。 + +`run_pg_remote_macos_faults.sh` 会让两台 Mac 同时挂载一个 PG workspace，并依次杀死远端 +FSKit extension、切断 SSH 隧道、停止 PostgreSQL，再验证没有半文件或静默覆盖。网络恢复后 +先用可重试的只读探针建立新连接，再继续写入；中断时失败的写请求不会被隐式重放，因为客户端 +无法可靠判断服务端是否已经提交。helper 异常退出后，底层 mountpoint 保持 `0500`，防止 Bash +把内容误写进普通本机目录；显式卸载恢复为 `0700`。 + +`run_pg_two_mac_opencode.sh` 只发送脚本生成的临时 Python 项目给 OpenCode，不发送仓库源码、 +DSN 或凭据。第一台和第二台 Mac 串行工作，暂不测试并发冲突。PG 当前的 `vexfs grep` 是数据库 +扫描，`vexfs index status` 返回 `available=false`；因此该用例验证 grep 正确性，但不会把 +尚未实现的 PostgreSQL substring 索引宣传成性能提升。 +Apple Silicon 默认使用远端 `/opt/homebrew/bin/opencode`；不要误用 Rosetta 下的 +`/usr/local/bin/opencode`，后者依赖的 x86_64 Bun 会警告缺少 AVX 并可能崩溃。Intel Mac 可用 +`VEXFS_REMOTE_OPENCODE` 显式覆盖。 + +真实 OpenCode 用例会自动检测当前版本支持的无人值守参数：优先 +`--dangerously-skip-permissions`，旧版本回退 `--auto`。两台机器可以分别通过 +`VEXFS_LOCAL_OPENCODE_MODEL` 和 `VEXFS_REMOTE_OPENCODE_MODEL` 选择各自已配置的模型；远端默认 +使用 `opencode/north-mini-code-free`。`VEXFS_EVAL_OPENCODE_MODEL` 仍可设置第一台的兼容默认值。 + 报告生成在： ```text @@ -62,6 +203,10 @@ vexdb_sqlite/build/eval/vexfs-linux-mount/{root,uid-1000}//report.json 随机状态机使用固定 seed，可通过 `--seed` 复现或扩充样本。 +宿主 eval 要求 Python 3.8+。`tests/eval/vexfs/python.sh` 会优先使用 +`VEXDB_LITE_PYTHON`，否则从 PATH、Homebrew、Anaconda 和系统 Python 中选择第一个满足版本 +要求的解释器，避免 PATH 中旧 Python 3.6 造成语法错误。容器脚本继续使用镜像内 Python。 + 评测结果有三种：`PASS` 表示所有选中用例都执行并通过；`PASS_WITH_SKIPS` 表示已执行用例通过，但有环境不满足的用例未执行；`FAIL` 表示至少一个用例失败， 或在 `--fail-on-skip` 下出现了 `SKIP`。没有匹配到任何用例会直接以非零状态退出。 @@ -72,18 +217,21 @@ vexdb_sqlite/build/eval/vexfs-linux-mount/{root,uid-1000}//report.json - 文件合同：二进制/Unicode、元数据、路径校验、rename replace、递归删除、公开版本历史和指定版本读取。 - 版本恢复：SQL/C ABI/CLI restore、expected-version 冲突、事务回滚、dry-run 不写库、文本 diff 和末尾换行差异。 - 事务：commit、rollback、savepoint、读者快照、目录 verifier。 -- 句柄：flags、稀疏写、truncate、幂等指纹、同步、乐观写冲突；幂等结果至少 +- 句柄：flags、稀疏写、truncate、幂等指纹、同步、乐观写冲突、发布关闭原子回滚；幂等结果至少 保留最近 65,536 条，并以 4,096 条为一批清理更早的完成记录。 - 时间戳和生命周期：birth/access/modify/change、`utimens`、并发原子 append、 `flock`/`fcntl`、打开文件 rename/unlink、普通/强制卸载。 - 恢复：进程中途退出、WAL 重开、integrity check、retained reclaim； - Linux helper 被 `SIGKILL` 后重挂载会自动发布保留的暂存写入。 + Linux helper 被 `SIGKILL` 后，底层目录保持 `0500`、普通用户不能产生本地假文件；30 秒 + session/handle 租约到期后重挂载会自动发布保留的暂存写入，显式卸载恢复 `0700`。 - 长期校验：版本 SHA-256、单文件恢复和 workspace 恢复别名、commit/snapshot/history/dentry 引用、staging 缺失、同长度内容损坏注入、损坏读取拒绝、CLI 退出码 8；性能用例分别记录 quick 元数据检查和 64 KiB 流式 deep hash 的吞吐与 RSS 增长。 - 数据库版本：当前尚未发版，只支持当前 schema，不运行旧 schema 迁移测试。 - 并发：多个真实进程写入、数据库锁和 busy timeout。 - 备份：运行中的 SQLite online backup、恢复、未发布 staging 可见性。 +- PostgreSQL 备份性能：有界验证 `pg_dump/pg_restore`、`pg_basebackup` + `pg_verifybackup` + + 克隆启动、format v2 PG → SQLite；同时检查容器 1 GiB 内存上限、`oom_kill` 和 CLI 峰值 RSS。 - 随机模型：真实数据库状态与 Python 参考文件树持续比对。 - 性能：大量小文件、大文件顺序读写、分块 staging、随机 4 KiB 覆盖、备份吞吐； `performance.mount-contract-small-files` 额外模拟原子创建、macOS provenance xattr、 @@ -114,7 +262,7 @@ vexdb_sqlite/build/eval/vexfs-linux-mount/{root,uid-1000}//report.json |---|---:|---:|---:| | 随机模型操作 | 300 | 2,000 | 10,000 | | 小文件 | 250 | 3,000 | 10,000 | -| 挂载合同小文件 | 250 | 1,000 | 1,000 | +| 挂载合同小文件 | 250 | 10,000 | 100,000 | | 顺序文件 | 8 MiB | 100 MiB | 128 MiB | | staging 文件 | 8 MiB | 100 MiB | 128 MiB | | 随机 4 KiB 写 | 100 | 1,000 | 5,000 | @@ -126,6 +274,12 @@ vexdb_sqlite/build/eval/vexfs-linux-mount/{root,uid-1000}//report.json quick/full/stress 的峰值 RSS 保护线分别为 1 GiB、1.5 GiB、2 GiB；SQLite 临时表使用 磁盘，页缓存限制为 64 MiB。越过保护线会立即失败，不继续扩大测试规模。 +真实挂载规模测试即使原生 `rg`/`grep` 超时，也会继续完成数据库搜索、索引、快照恢复和 +重挂载，再把超时作为带完整指标的失败报告。失败时临时数据库、挂载描述和分阶段 +`eval-checkpoints.jsonl` 会移到该 case 的 `artifacts/preserved-workspace/`，避免数小时测试 +只留下一个超时字符串；成功时这些大文件自动清理。搜索等待时间由 +`VEXFS_EVAL_NATIVE_SEARCH_TIMEOUT_SECONDS` 控制。 + `mount.real-bash` 是环境 Gate。macOS、FSKit 和已启用的 VexFS 系统扩展都满足时， 它会真实挂载并运行 `mkdir`、`cat`、`grep`、`cp`、`mv`、`find`、`rm` 等命令； 条件不满足时报告会明确写 `SKIP` 和原因，不会把未执行伪装成通过。 diff --git a/tests/eval/vexfs/documentation_smoke.sh b/tests/eval/vexfs/documentation_smoke.sh index 6e6506068f..7354e7216c 100755 --- a/tests/eval/vexfs/documentation_smoke.sh +++ b/tests/eval/vexfs/documentation_smoke.sh @@ -8,9 +8,19 @@ TEST_HOME="$TMP_DIR/home" APP_DIR="$TEST_HOME/Applications" BIN_DIR="$TEST_HOME/.local/bin" LIB_DIR="$TEST_HOME/.local/lib/vexdb-lite" +PAYLOAD_APP="$STAGE/.payload/VexDB Lite.app" CHECKS=0 PACKAGE_VERSION="$(sed -n 's/^preview_version=//p' "$STAGE/MANIFEST.txt")" [ -n "$PACKAGE_VERSION" ] || { echo "DOCUMENTATION SMOKE: FAIL: manifest has no preview_version" >&2; exit 1; } +BUNDLE_MARKETING_VERSION="$(sed -n 's/^bundle_marketing_version=//p' "$STAGE/MANIFEST.txt")" +[ -n "$BUNDLE_MARKETING_VERSION" ] || { echo "DOCUMENTATION SMOKE: FAIL: manifest has no bundle marketing version" >&2; exit 1; } +BUNDLE_BUILD_VERSION="$(sed -n 's/^bundle_build_version=//p' "$STAGE/MANIFEST.txt")" +[ -n "$BUNDLE_BUILD_VERSION" ] || { echo "DOCUMENTATION SMOKE: FAIL: manifest has no bundle build version" >&2; exit 1; } +PACKAGE_ARCHITECTURE="$(sed -n 's/^architecture=//p' "$STAGE/MANIFEST.txt")" +case "$PACKAGE_ARCHITECTURE" in + arm64|x86_64) ;; + *) echo "DOCUMENTATION SMOKE: FAIL: invalid manifest architecture: $PACKAGE_ARCHITECTURE" >&2; exit 1 ;; +esac MANIFEST_RUNTIME_ABI="$(sed -n 's/^mount_abi_version=//p' "$STAGE/MANIFEST.txt")" [ -n "$MANIFEST_RUNTIME_ABI" ] || { echo "DOCUMENTATION SMOKE: FAIL: manifest has no runtime ABI" >&2; exit 1; } PACKAGE_SIGNATURE="$(sed -n 's/^signature=//p' "$STAGE/MANIFEST.txt")" @@ -60,15 +70,75 @@ expect_fail() { mkdir -p "$TEST_HOME" -# 交付物和使用说明。 -check test -d "$STAGE/VexDB Lite.app" +# 交付物和使用说明。App 放在隐藏 payload，防止解压目录被 LaunchServices +# 当成第二个可运行副本，从而让 FSKit 随机选到未安装的扩展。 +check test ! -e "$STAGE/VexDB Lite.app" +check test -d "$PAYLOAD_APP" +for bundle in \ + "$PAYLOAD_APP" \ + "$PAYLOAD_APP/Contents/Extensions/VexFSAppEx.appex"; do + equal "$BUNDLE_MARKETING_VERSION" \ + "$(plutil -extract CFBundleShortVersionString raw -o - "$bundle/Contents/Info.plist")" \ + "bundle marketing version" + equal "$BUNDLE_BUILD_VERSION" \ + "$(plutil -extract CFBundleVersion raw -o - "$bundle/Contents/Info.plist")" \ + "bundle build version" +done +equal "$PACKAGE_ARCHITECTURE" \ + "$(lipo -archs "$PAYLOAD_APP/Contents/MacOS/VexDB Lite")" \ + "app architecture" +equal "$PACKAGE_ARCHITECTURE" \ + "$(lipo -archs "$PAYLOAD_APP/Contents/Extensions/VexFSAppEx.appex/Contents/MacOS/VexFSAppEx")" \ + "FSKit extension architecture" check test -x "$STAGE/bin/vexdb" check test -L "$STAGE/bin/vexfs" check test -f "$STAGE/lib/vexdb_lite.dylib" +check test -f "$STAGE/lib/runtime/libpq.5.dylib" +EXTENSION_ENTITLEMENTS="$TMP_DIR/extension-entitlements.plist" +EXTENSION_ENTITLEMENTS_LOG="$TMP_DIR/extension-entitlements.log" +if ! codesign -d --entitlements - --xml \ + "$PAYLOAD_APP/Contents/Extensions/VexFSAppEx.appex" \ + >"$EXTENSION_ENTITLEMENTS" 2>"$EXTENSION_ENTITLEMENTS_LOG"; then + cat "$EXTENSION_ENTITLEMENTS_LOG" >&2 + echo "DOCUMENTATION SMOKE: FAIL: cannot read FSKit entitlements" >&2 + exit 1 +fi +if grep -q 'invalid entitlements blob' "$EXTENSION_ENTITLEMENTS_LOG" || + ! plutil -lint "$EXTENSION_ENTITLEMENTS" >/dev/null 2>&1; then + cat "$EXTENSION_ENTITLEMENTS_LOG" >&2 + echo "DOCUMENTATION SMOKE: FAIL: invalid FSKit DER entitlements" >&2 + exit 1 +fi +CHECKS=$((CHECKS + 1)) +FSKIT_ENTITLEMENT="$(/usr/libexec/PlistBuddy \ + -c 'Print :com.apple.developer.fskit.fsmodule' \ + "$EXTENSION_ENTITLEMENTS" 2>/dev/null || true)" +equal "true" "$FSKIT_ENTITLEMENT" "FSKit module entitlement" +if [ "$PACKAGE_SIGNATURE" = developer-id ]; then + EXTENSION_TEAM="$(/usr/libexec/PlistBuddy \ + -c 'Print :com.apple.developer.team-identifier' \ + "$EXTENSION_ENTITLEMENTS" 2>/dev/null || true)" + EXTENSION_APPLICATION="$(/usr/libexec/PlistBuddy \ + -c 'Print :com.apple.application-identifier' \ + "$EXTENSION_ENTITLEMENTS" 2>/dev/null || true)" + equal "developer-id" "$PACKAGE_SIGNATURE" "Developer ID entitlement checks" + equal "$(sed -n 's/^signing_team=//p' "$STAGE/MANIFEST.txt")" \ + "$EXTENSION_TEAM" "FSKit team entitlement" + equal "$EXTENSION_TEAM.io.vexdb.vexfs.extension" \ + "$EXTENSION_APPLICATION" "FSKit application identifier" +fi check test -f "$STAGE/使用说明.md" check grep -q 'Developer ID' "$STAGE/使用说明.md" check grep -q '按类别' "$STAGE/使用说明.md" check grep -q '文件系统扩展' "$STAGE/使用说明.md" +check grep -q '底层目录设为' "$STAGE/使用说明.md" +check grep -q '不会被 VexDB Lite 自动重放' "$STAGE/使用说明.md" +check grep -q 'verify_installed_mount' "$STAGE/install.sh" +check grep -q '正在使用的 FSKit 文件系统' "$STAGE/install.sh" +check grep -q 'pkill -KILL -u.*fskit_agent' "$STAGE/install.sh" +expect_fail grep -q 'killall -u.*pkd' "$STAGE/install.sh" +expect_fail grep -q 'lsregister.*-kill' "$STAGE/install.sh" +check grep -Eq '不会.*清空 LaunchServices 注册库' "$STAGE/使用说明.md" expect_fail grep -q '没有提供可用的授权入口' "$STAGE/使用说明.md" check grep -Eq '^signature=(developer-id|ad-hoc)$' "$STAGE/MANIFEST.txt" check grep -Eq '^notarization=(not-submitted|accepted)$' "$STAGE/MANIFEST.txt" @@ -88,6 +158,31 @@ esac ) || fail "package SHA256SUMS verification failed" CHECKS=$((CHECKS + 1)) +# 安装事务必须在 App 已替换、CLI 复制失败时恢复旧版本，并且不留下可被 +# LaunchServices 再次发现的 `.app.disabled` 副本。 +ROLLBACK_ROOT="$TMP_DIR/rollback-install" +ROLLBACK_APP_DIR="$ROLLBACK_ROOT/Applications" +ROLLBACK_BIN_DIR="$ROLLBACK_ROOT/read-only-bin" +ROLLBACK_LIB_DIR="$ROLLBACK_ROOT/lib" +mkdir -p "$ROLLBACK_APP_DIR" "$ROLLBACK_BIN_DIR" "$ROLLBACK_LIB_DIR" +ditto "$PAYLOAD_APP" "$ROLLBACK_APP_DIR/VexDB Lite.app" +touch "$ROLLBACK_APP_DIR/VexDB Lite.app/.rollback-marker" +chmod 0555 "$ROLLBACK_BIN_DIR" +expect_fail env \ + HOME="$ROLLBACK_ROOT" \ + VEXDB_LITE_APP_DIR="$ROLLBACK_APP_DIR" \ + VEXDB_LITE_BIN_DIR="$ROLLBACK_BIN_DIR" \ + VEXDB_LITE_LIB_DIR="$ROLLBACK_LIB_DIR" \ + VEXDB_LITE_ALLOW_ADHOC_INSTALL="$ALLOW_ADHOC" \ + VEXDB_LITE_ALLOW_DIRTY_INSTALL="$ALLOW_DIRTY" \ + "$STAGE/install.sh" +chmod 0755 "$ROLLBACK_BIN_DIR" +check test -f "$ROLLBACK_APP_DIR/VexDB Lite.app/.rollback-marker" +equal "" "$(find "$ROLLBACK_APP_DIR" -maxdepth 1 \ + -name '*.app.disabled' -print -quit)" "no discoverable rollback App" +equal "" "$(find "$ROLLBACK_APP_DIR" -maxdepth 1 \ + -name '.vexdb-lite-install-*' -print -quit)" "no installation transaction residue" + # 按文档执行无污染安装。所有安装位置和 HOME 都在临时目录。 HOME="$TEST_HOME" \ VEXDB_LITE_APP_DIR="$APP_DIR" \ @@ -210,7 +305,8 @@ expect_fail env HOME="$TEST_HOME" "$VEXDB" fs stat /after-snapshot/new.txt HOME="$TEST_HOME" "$VEXDB" fs snapshot diff before-agent >/dev/null # doctor 必须如实报告当前机器的 FSKit 状态。临时 HOME 只隔离数据库， -# 不会隔离系统已经注册的扩展，因此测试机可能是 missing、registered 或 enabled。 +# 不会隔离系统已经注册的扩展，因此测试机可能是 missing、registered、disabled +# 或 enabled。 set +e DOCTOR_JSON="$(HOME="$TEST_HOME" "$VEXDB" fs doctor --json 2>/dev/null)" DOCTOR_RC=$? @@ -220,7 +316,7 @@ case "$DOCTOR_STATE" in enabled) equal "0" "$DOCTOR_RC" "doctor enabled extension exit code" ;; - missing|registered) + missing|registered|disabled|service-unavailable) equal "1" "$DOCTOR_RC" "doctor unavailable extension exit code" ;; *) @@ -251,7 +347,16 @@ if /usr/bin/sqlite3 :memory: ".help" | grep -q '^\.load'; then fi CHECKS=$((CHECKS + 1)) LOADABLE_SQLITE="$(command -v sqlite3 || true)" +LOADABLE_SQLITE_ARCHS="" +LOADABLE_SQLITE_COMPATIBLE=0 +if [ -n "$LOADABLE_SQLITE" ]; then + LOADABLE_SQLITE_ARCHS="$(lipo -archs "$LOADABLE_SQLITE" 2>/dev/null || true)" +fi +case " $LOADABLE_SQLITE_ARCHS " in + *" $PACKAGE_ARCHITECTURE "*) LOADABLE_SQLITE_COMPATIBLE=1 ;; +esac if [ -n "$LOADABLE_SQLITE" ] && [ "$LOADABLE_SQLITE" != "/usr/bin/sqlite3" ] && \ + [ "$LOADABLE_SQLITE_COMPATIBLE" = 1 ] && \ "$LOADABLE_SQLITE" :memory: ".help" | grep -q '^\.load'; then contains "$("$LOADABLE_SQLITE" "$SQL_DB" ".load $DYLIB" "SELECT vexdb_version();")" \ "$EXPECTED_PRODUCT_VERSION" "third-party sqlite loadable extension" diff --git a/tests/eval/vexfs/python.sh b/tests/eval/vexfs/python.sh new file mode 100755 index 0000000000..cb37dd416a --- /dev/null +++ b/tests/eval/vexfs/python.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +set -euo pipefail + +PYTHON_BIN="${VEXDB_LITE_PYTHON:-}" +if [ -n "$PYTHON_BIN" ]; then + [ -x "$PYTHON_BIN" ] || { + echo "VEXDB_LITE_PYTHON 不可执行：$PYTHON_BIN" >&2 + exit 2 + } + "$PYTHON_BIN" -c 'import sys; raise SystemExit(sys.version_info < (3, 8))' || { + echo "VEXDB_LITE_PYTHON 需要 Python 3.8 或更高版本：$PYTHON_BIN" >&2 + exit 2 + } +else + for candidate in \ + "$(command -v python3 2>/dev/null || true)" \ + /opt/homebrew/bin/python3 \ + /opt/anaconda3/bin/python3 \ + /usr/bin/python3; do + [ -n "$candidate" ] || continue + [ -x "$candidate" ] || continue + if "$candidate" -c 'import sys; raise SystemExit(sys.version_info < (3, 8))' \ + >/dev/null 2>&1; then + PYTHON_BIN="$candidate" + break + fi + done + [ -n "$PYTHON_BIN" ] || { + echo "找不到 Python 3.8 或更高版本；可设置 VEXDB_LITE_PYTHON" >&2 + exit 2 + } +fi + +exec "$PYTHON_BIN" "$@" diff --git a/tests/eval/vexfs/run.py b/tests/eval/vexfs/run.py index dadc7134bc..ad738b8e57 100644 --- a/tests/eval/vexfs/run.py +++ b/tests/eval/vexfs/run.py @@ -29,6 +29,7 @@ from datetime import datetime, timezone from pathlib import Path from typing import Any, Callable +from urllib.parse import unquote, urlparse MIB = 1024 * 1024 @@ -50,6 +51,32 @@ class EvalSkip(RuntimeError): pass +class FailureArtifactDirectory: + """临时目录成功时清理，失败时移入当前 case 的报告目录。""" + + def __init__(self, ctx: "Context", prefix: str): + self.ctx = ctx + self.path = Path(tempfile.mkdtemp(prefix=prefix)) + + def __enter__(self) -> str: + return str(self.path) + + def __exit__(self, exception_type: type[BaseException] | None, + _exception: BaseException | None, _traceback: Any) -> bool: + if exception_type is None or issubclass(exception_type, EvalSkip): + shutil.rmtree(self.path, ignore_errors=True) + return False + if self.ctx.current_artifacts is None: + return False + destination = self.ctx.current_artifacts / "preserved-workspace" + try: + shutil.move(str(self.path), str(destination)) + except OSError: + # 保留原始临时目录比为了整理报告而覆盖真正的测试异常更重要。 + pass + return False + + @dataclass(frozen=True) class Mode: name: str @@ -323,10 +350,15 @@ def package_unified_install(ctx: Context) -> dict[str, Any]: raise EvalSkip("统一 macOS 安装包只在 macOS 执行") stage = package_stage(ctx) packaged_cli = stage / "bin/vexdb" + payload_app = stage / ".payload/VexDB Lite.app" ctx.check(packaged_cli.exists(), "统一包缺少 bin/vexdb") ctx.check((stage / "bin/vexfs").is_symlink(), "vexfs 必须是兼容链接") - ctx.check((stage / "VexDB Lite.app").is_dir(), "统一包缺少 VexDB Lite.app") + ctx.check(not (stage / "VexDB Lite.app").exists(), + "统一包不能暴露可被 LaunchServices 重复注册的顶层 App") + ctx.check(payload_app.is_dir(), "统一包缺少隐藏 App payload") ctx.check((stage / "lib/vexdb_lite.dylib").exists(), "统一包缺少 SQLite 扩展") + ctx.check((stage / "lib/runtime/libpq.5.dylib").exists(), + "统一包缺少 PostgreSQL runtime") manifest = dict( line.split("=", 1) for line in (stage / "MANIFEST.txt").read_text().splitlines() if "=" in line) @@ -1141,6 +1173,63 @@ def idempotency(ctx: Context) -> dict[str, Any]: return {"request_rows": db.scalar("SELECT count(*) FROM _vexfs_requests")} +@case("contract.publish-close-atomicity", "handles", + "发布并关闭必须原子提交，失败后句柄和 staging 可继续恢复") +def publish_close_atomicity(ctx: Context) -> dict[str, Any]: + with Database(ctx) as db: + db.scalar("SELECT vexfs_write('default','/publish-close','base')") + handle = db.scalar( + "SELECT vexfs_handle_open('default','/publish-close','rw','pc-open')") + generation = db.scalar( + "SELECT vexfs_handle_stage_write(?,0,'next','pc-stage')", (handle,)) + ctx.expect_error(lambda: db.scalar( + "SELECT vexfs_handle_publish_close(?,?,'data')", + (handle, generation + 1)), "generation is stale") + ctx.equal(db.scalar("SELECT state FROM _vexfs_handles WHERE id=?", (handle,)), + "open", "generation 失败后句柄仍可恢复") + ctx.equal(db.scalar( + "SELECT count(*) FROM _vexfs_staging WHERE handle_id=?", (handle,)), + 1, "generation 失败后 staging 未被清理") + ctx.equal(db.scalar( + "SELECT current_version FROM _vexfs_inodes WHERE id=(" + "SELECT inode_id FROM _vexfs_handles WHERE id=?)", (handle,)), + 1, "generation 失败不推进文件版本") + version = db.scalar( + "SELECT vexfs_handle_publish_close(?,?,'data')", (handle, generation)) + ctx.equal(version, 2, "发布并关闭推进版本") + ctx.equal(db.scalar("SELECT state FROM _vexfs_handles WHERE id=?", (handle,)), + "closed", "成功后句柄关闭") + ctx.equal(db.scalar( + "SELECT count(*) FROM _vexfs_staging WHERE handle_id=?", (handle,)), + 0, "成功后 staging 清理") + ctx.equal(db.scalar("SELECT vexfs_read('default','/publish-close')"), b"next", + "发布并关闭写入内容") + + first = db.scalar( + "SELECT vexfs_handle_open('default','/publish-close','rw','pc-first')") + second = db.scalar( + "SELECT vexfs_handle_open('default','/publish-close','rw','pc-second')") + first_generation = db.scalar( + "SELECT vexfs_handle_stage_write(?,0,'one!','pc-first-stage')", (first,)) + second_generation = db.scalar( + "SELECT vexfs_handle_stage_write(?,0,'two!','pc-second-stage')", (second,)) + ctx.equal(db.scalar( + "SELECT vexfs_handle_publish_close(?,?,'data')", + (first, first_generation)), 3, "第一个并发句柄发布") + ctx.expect_error(lambda: db.scalar( + "SELECT vexfs_handle_publish_close(?,?,'data')", + (second, second_generation)), "write conflict") + ctx.equal(db.scalar("SELECT state FROM _vexfs_handles WHERE id=?", (second,)), + "open", "冲突后句柄没有被误关") + ctx.equal(db.scalar( + "SELECT count(*) FROM _vexfs_staging WHERE handle_id=?", (second,)), + 1, "冲突后 staging 保留") + ctx.equal(db.scalar("SELECT vexfs_read('default','/publish-close')"), b"one!", + "冲突后保留已提交内容") + db.scalar("SELECT vexfs_handle_close(?,0,'pc-second-close')", (second,)) + return {"versions": 3, "rollback_paths": 2} + + @case("maintenance.request-retention", "handles", "幂等请求日志有上限，并保留最近请求的重试结果") def request_retention(ctx: Context) -> dict[str, Any]: @@ -2314,9 +2403,14 @@ def cli_command_surface(ctx: Context) -> dict[str, Any]: ctx.equal(index_disabled["enabled"], False, "CLI 关闭文本索引") run_process(prefix + ["descriptor", str(descriptor)]) descriptor_json = json.loads(descriptor.read_text()) + ctx.equal(descriptor_json["version"], 2, "descriptor version") + ctx.equal(descriptor_json["backend"], "sqlite", "descriptor backend") ctx.equal(descriptor_json["workspace"], "eval", "descriptor workspace") - ctx.equal(Path(descriptor_json["database_path"]).resolve(), database.resolve(), + ctx.equal(Path(descriptor_json["connection"]).resolve(), database.resolve(), "descriptor database") + if os.name != "nt": + ctx.equal(descriptor.stat().st_mode & 0o777, 0o600, + "descriptor 只允许当前用户读写") doctor = run_process(prefix + ["--json", "doctor"], check=False) doctor_json = json.loads(doctor.stdout) expected_platform = {"Darwin": "macos", "Linux": "linux", @@ -2452,9 +2546,11 @@ def performance_mount_contract_small_files(ctx: Context) -> dict[str, Any]: executable = ctx.build_dir / "vexfs_runtime_smoke" if not executable.exists(): raise EvalSkip(f"缺少挂载合同 benchmark: {executable}") - file_count = 250 if ctx.mode.name == "quick" else 1_000 + file_count = {"quick": 250, "full": 10_000, "stress": 100_000}[ + ctx.mode.name] + timeout = {"quick": 60, "full": 300, "stress": 3_600}[ctx.mode.name] rss_before = child_max_rss_bytes() - result = run_process([str(executable), "--benchmark", str(file_count)], timeout=60) + result = run_process([str(executable), "--benchmark", str(file_count)], timeout=timeout) metrics = json.loads(result.stdout) rss_after = child_max_rss_bytes() ctx.equal(metrics["files"], file_count, "挂载合同 benchmark 文件数") @@ -2464,7 +2560,8 @@ def performance_mount_contract_small_files(ctx: Context) -> dict[str, Any]: "本地挂载每个文件最多保留 create/close 两条请求") ctx.check(metrics["durability_barriers"] >= 1, "synchronize 执行真实 FULL 屏障") ctx.check(rss_after < 1024 * MIB, "挂载合同 benchmark 峰值内存低于 1 GiB") - ctx.budget("mount_contract_create_seconds", metrics["create_seconds"], 30.0) + ctx.budget("mount_contract_create_seconds", metrics["create_seconds"], + {"quick": 30.0, "full": 300.0, "stress": 3_000.0}[ctx.mode.name]) ctx.budget("mount_contract_sync_seconds", metrics["sync_seconds"], 5.0) metrics["child_max_rss_before_bytes"] = rss_before metrics["child_max_rss_after_bytes"] = rss_after @@ -2588,6 +2685,98 @@ def performance_scale_tree(ctx: Context) -> dict[str, Any]: } +@case("performance.mvp-dataset", "performance", + "MVP 的 1 千文件、1 GiB 当前内容、100 MiB 单文件、100 commits 和 10 snapshots", + modes=("full", "stress")) +def performance_mvp_dataset(ctx: Context) -> dict[str, Any]: + file_count = 1_000 + logical_bytes = 1024 * MIB + large_file_bytes = 100 * MIB + small_total = logical_bytes - large_file_bytes + small_base, small_extra = divmod(small_total, file_count - 1) + rss_before = self_max_rss_bytes() + + with FailureArtifactDirectory(ctx, "vexfs-mvp-dataset-") as directory: + database_path = Path(directory) / "mvp.sqlite3" + db = Database(ctx, database_path) + db.connection.execute("PRAGMA cache_size=-32768") + db.scalar("SELECT vexfs_mkdir('default','/mvp')") + + checksums: list[tuple[str, str]] = [] + started = time.perf_counter() + large_path = "/mvp/file-0000.bin" + large_payload = b"L" * large_file_bytes + checksums.append((large_path, sha256(large_payload))) + db.scalar("SELECT vexfs_write('default',?,?)", (large_path, large_payload)) + del large_payload + + for index in range(1, file_count): + size = small_base + (1 if index <= small_extra else 0) + payload = (index.to_bytes(8, "little") + + bytes([index % 251]) * (size - 8)) + path = f"/mvp/file-{index:04d}.bin" + checksums.append((path, sha256(payload))) + db.scalar("SELECT vexfs_write('default',?,?)", (path, payload)) + write_seconds = time.perf_counter() - started + + for index in range(10): + db.scalar("SELECT vexfs_snapshot_create('default',?)", (f"mvp-{index}",)) + + workspace = db.connection.execute( + "SELECT live_files,live_bytes FROM _vexfs_workspaces " + "WHERE name='default'").fetchone() + if workspace is None: + raise EvalFailure("MVP workspace 不存在") + live_files, live_bytes = workspace + direct_children = len(db.json("SELECT vexfs_list('default','/mvp')")) + commit_count = db.scalar("SELECT count(*) FROM _vexfs_commits") + snapshot_count = db.scalar("SELECT count(*) FROM _vexfs_snapshots") + ctx.equal(db.scalar("PRAGMA integrity_check"), "ok", + "MVP 写入后数据库完整") + db.close() + + reopened = Database(ctx, database_path) + reopened.connection.execute("PRAGMA cache_size=-32768") + read_started = time.perf_counter() + for path, expected_checksum in checksums: + content = reopened.scalar("SELECT vexfs_read('default',?)", (path,)) + ctx.equal(sha256(content), expected_checksum, + f"MVP 重开后文件 checksum: {path}") + read_seconds = time.perf_counter() - read_started + ctx.equal(reopened.scalar("PRAGMA integrity_check"), "ok", + "MVP 重开读取后数据库完整") + reopened.connection.execute("PRAGMA wal_checkpoint(TRUNCATE)").fetchone() + storage = db_storage(database_path) + reopened.close() + + rss_after = self_max_rss_bytes() + ctx.equal(live_files, file_count, "MVP 当前文件数") + ctx.equal(live_bytes, logical_bytes, "MVP 当前内容总量") + ctx.equal(direct_children, file_count, "MVP 单目录直接子项") + ctx.equal(snapshot_count, 10, "MVP 快照数") + ctx.check(commit_count >= 100, "MVP workspace commits 不少于 100") + ctx.check(rss_after < 1024 * MIB, "MVP 数据集峰值内存低于 1 GiB") + ctx.budget("mvp_dataset_write_seconds", write_seconds, 180.0) + ctx.budget("mvp_dataset_read_seconds", read_seconds, 180.0) + return { + "files": live_files, + "logical_bytes": live_bytes, + "direct_children": direct_children, + "max_file_bytes": large_file_bytes, + "commits": commit_count, + "snapshots": snapshot_count, + "write_seconds": round(write_seconds, 6), + "write_mib_per_second": round( + logical_bytes / MIB / max(write_seconds, 1e-9), 3), + "reopen_read_seconds": round(read_seconds, 6), + "read_mib_per_second": round( + logical_bytes / MIB / max(read_seconds, 1e-9), 3), + "max_rss_before_bytes": rss_before, + "max_rss_after_bytes": rss_after, + "storage": storage, + } + + @case("stability.mixed-reopen-soak", "stability", "持续混合读写、元数据、快照、checkpoint 和数据库重开") def stability_mixed_reopen_soak(ctx: Context) -> dict[str, Any]: @@ -3155,10 +3344,16 @@ def mount_cli(ctx: Context) -> Path: return installed if installed.exists() else ctx.cli -def filesystem_cli_prefix(cli: Path, database: Path, workspace: str) -> list[str]: +def filesystem_cli_prefix(cli: Path, database: Path, workspace: str, + backend: str = "sqlite", dsn: str = "") -> list[str]: """同时支持独立 vexfs 名称和统一 vexdb fs 入口。""" name = cli.stem.lower() if cli.suffix.lower() == ".exe" else cli.name.lower() command = [str(cli)] if name == "vexfs" else [str(cli), "fs"] + if backend == "postgresql": + if not dsn: + raise EvalFailure("PostgreSQL mount eval 缺少 VEXFS_MOUNT_DSN") + return command + ["--backend", "pg", "--dsn", dsn, + "--workspace", workspace] return command + ["--db", str(database), "--workspace", workspace] @@ -3189,6 +3384,8 @@ class RealMountAdapter: mount_point: Path prefix: list[str] doctor_before: dict[str, Any] + backend: str + connection_identity: str def prepare_real_mount_adapter(ctx: Context, base: Path, @@ -3213,9 +3410,16 @@ def prepare_real_mount_adapter(ctx: Context, base: Path, else: raise EvalSkip("共享 mount 一致性 eval 当前支持 macOS 和 Linux") + backend = os.environ.get("VEXFS_MOUNT_BACKEND", "sqlite").lower() + if backend in {"pg", "postgres"}: + backend = "postgresql" + if backend not in {"sqlite", "postgresql"}: + raise EvalFailure(f"不支持的 mount eval backend：{backend}") + dsn = os.environ.get("VEXFS_MOUNT_DSN", "") if backend == "postgresql" else "" database = database or base / "mount.sqlite3" mount_point = base / "mnt" - prefix = filesystem_cli_prefix(cli, database, workspace) + prefix = filesystem_cli_prefix(cli, database, workspace, backend, dsn) + connection_identity = dsn if backend == "postgresql" else os.path.realpath(database) run_process(prefix + ["setup"]) doctor = run_process(prefix + ["--json", "doctor"], check=False) try: @@ -3242,7 +3446,8 @@ def prepare_real_mount_adapter(ctx: Context, base: Path, if system == "Linux" and not details.get("mount_ready"): raise EvalSkip(f"libfuse3 adapter={details.get('extension', 'unknown')}") mount_point.mkdir() - return RealMountAdapter(name, cli, database, mount_point, prefix, details) + return RealMountAdapter(name, cli, database, mount_point, prefix, details, + backend, connection_identity) def expect_mount_errno(ctx: Context, operation: Callable[[], Any], expected: int, @@ -3527,7 +3732,7 @@ def cross_platform_mount_conformance(ctx: Context) -> dict[str, Any]: adapter.prefix + ["--json", "mount", "status", str(adapter.mount_point)]).stdout) ctx.equal(len(status), 1, "快照恢复后挂载仍然存在") - ctx.equal(status[0]["database"], os.path.realpath(adapter.database), + ctx.equal(status[0]["database"], adapter.connection_identity, "挂载身份记录数据库") ctx.equal(status[0]["workspace"], "conformance", "挂载身份记录 workspace") clean_diff = run_process(adapter.prefix + ["snapshot", "diff", "baseline"]) @@ -3562,6 +3767,98 @@ def cross_platform_mount_conformance(ctx: Context) -> dict[str, Any]: PORTABILITY_XATTR = "user.vexfs.portable" PORTABILITY_XATTR_VALUE = b"mac-linux-mac\x00metadata" +PG_CROSS_GATEWAY_WORKSPACE = "pg-cross-gateway" +PG_CROSS_GATEWAY_XATTR = "user.vexfs.pg-cross-gateway" +PG_CROSS_GATEWAY_XATTR_VALUE = b"shared-through-postgresql" + + +@case("portability.pg-cross-gateway", "portability", + "macOS FSKit 与 Linux FUSE 通过同一个 PostgreSQL 工作区双向接力") +def pg_cross_gateway_roundtrip(ctx: Context) -> dict[str, Any]: + phase = os.environ.get("VEXFS_PG_CROSS_GATEWAY_PHASE", "") + phases = { + "mac-create": "Darwin", + "linux-modify": "Linux", + "mac-verify": "Darwin", + "linux-final": "Linux", + } + if phase not in phases: + raise EvalSkip( + "需通过 run_pg_cross_gateway.sh 设置 PostgreSQL 跨网关阶段") + if platform.system() != phases[phase]: + raise EvalFailure( + f"阶段 {phase} 必须在 {phases[phase]} 执行，当前为 {platform.system()}") + if os.environ.get("VEXFS_MOUNT_BACKEND", "").lower() not in { + "pg", "postgres", "postgresql"}: + raise EvalFailure("PostgreSQL 跨网关 eval 必须使用 PG backend") + + with tempfile.TemporaryDirectory(prefix=f"vexfs-pg-gateway-{phase}-") as directory: + adapter = prepare_real_mount_adapter( + ctx, Path(directory), PG_CROSS_GATEWAY_WORKSPACE) + mount_adapter_mount(ctx, adapter) + shared = adapter.mount_point / "shared" + try: + if phase == "mac-create": + shared.mkdir() + (shared / "from-mac.txt").write_text("created on macOS\n", encoding="utf-8") + script = shared / "run.sh" + script.write_text("#!/bin/sh\nprintf 'pg-gateway-ok\\n'\n", encoding="utf-8") + script.chmod(0o755) + os.link(shared / "from-mac.txt", shared / "from-mac-hard.txt") + (shared / "from-mac-link.txt").symlink_to("from-mac.txt") + system_setxattr(shared / "from-mac.txt", PG_CROSS_GATEWAY_XATTR, + PG_CROSS_GATEWAY_XATTR_VALUE) + ctx.equal((shared / "from-mac.txt").read_text(encoding="utf-8"), + "created on macOS\n", "macOS 首次写入") + else: + ctx.equal((shared / "from-mac.txt").read_text(encoding="utf-8"), + "created on macOS\n", f"{phase} 读取 macOS 文件") + ctx.equal((shared / "from-mac-hard.txt").stat().st_ino, + (shared / "from-mac.txt").stat().st_ino, + f"{phase} 保留 hardlink") + ctx.equal(os.readlink(shared / "from-mac-link.txt"), "from-mac.txt", + f"{phase} 保留 symlink") + ctx.equal(system_getxattr( + shared / "from-mac.txt", PG_CROSS_GATEWAY_XATTR), + PG_CROSS_GATEWAY_XATTR_VALUE, f"{phase} 保留 xattr") + ctx.equal(run_process([str(shared / "run.sh")]).stdout, + b"pg-gateway-ok\n", f"{phase} 保留可执行 mode") + + if phase == "linux-modify": + (shared / "from-linux.txt").write_text( + "created on Linux\n", encoding="utf-8") + elif phase in {"mac-verify", "linux-final"}: + ctx.equal((shared / "from-linux.txt").read_text(encoding="utf-8"), + "created on Linux\n", f"{phase} 读取 Linux 文件") + if phase == "mac-verify": + (shared / "from-mac-again.txt").write_text( + "created after Linux\n", encoding="utf-8") + elif phase == "linux-final": + ctx.equal((shared / "from-mac-again.txt").read_text(encoding="utf-8"), + "created after Linux\n", "Linux 最终读取 macOS 回写") + finally: + mount_adapter_unmount(ctx, adapter, phase) + + snapshot_name = { + "mac-create": "mac-baseline", + "linux-modify": "linux-baseline", + "mac-verify": "mac-after-linux", + "linux-final": "linux-final", + }[phase] + snapshot = run_process(adapter.prefix + ["snapshot", "create", snapshot_name]) + ctx.check(int(snapshot.stdout.strip()) > 0, f"{phase} 创建快照") + snapshots = run_process(adapter.prefix + ["snapshot", "list"]).stdout + ctx.check(snapshot_name.encode() in snapshots, f"{phase} 快照可见") + check = json.loads(run_process(adapter.prefix + ["--json", "check"]).stdout) + ctx.equal(check["ok"], True, f"{phase} 深度检查") + return { + "phase": phase, + "adapter": adapter.name, + "backend": adapter.backend, + "workspace": PG_CROSS_GATEWAY_WORKSPACE, + "snapshot": snapshot_name, + } + def verify_portable_tree(ctx: Context, adapter: RealMountAdapter, plan_content: bytes, linux_file: bool) -> None: @@ -4067,11 +4364,14 @@ def mount_helper_crash_recovery(ctx: Context) -> dict[str, Any]: ctx.equal(ready, "staged", "子进程已写入但未关闭 fd") process_table = run_process(["ps", "-eo", "pid=,args="]).stdout.decode() helper_pids = [] + identity_marker = (adapter.connection_identity + if adapter.backend == "postgresql" + else str(adapter.database)) for line in process_table.splitlines(): fields = line.strip().split(maxsplit=1) if len(fields) != 2: continue - if ("vexfs-fuse" in fields[1] and str(adapter.database) in fields[1] + if ("vexfs-fuse" in fields[1] and identity_marker in fields[1] and str(adapter.mount_point) in fields[1]): helper_pids.append(int(fields[0])) ctx.equal(len(helper_pids), 1, "定位唯一 FUSE helper") @@ -4079,24 +4379,87 @@ def mount_helper_crash_recovery(ctx: Context) -> dict[str, Any]: child.terminate() child.wait(timeout=10) + # Remove the dead kernel mount without going through VexFS cleanup. + # The covered directory must already be 0500, otherwise Bash could + # silently write local files after a helper crash. + detached = run_process( + ["fusermount3", "-u", "-z", str(adapter.mount_point)], + check=False, timeout=30) + ctx.equal(detached.returncode, 0, "helper 崩溃后内核挂载可分离") + for _ in range(50): + if not os.path.ismount(adapter.mount_point): + break + time.sleep(0.1) + ctx.equal(os.path.ismount(adapter.mount_point), False, + "helper 崩溃后内核挂载已消失") + ctx.equal(adapter.mount_point.stat().st_mode & 0o777, 0o500, + "helper 崩溃后底层目录禁止误写") + blocked = adapter.mount_point / "must-not-be-local.txt" + if os.geteuid() != 0: + try: + blocked.write_text("unsafe", encoding="utf-8") + except PermissionError: + pass + else: + raise EvalFailure("helper 崩溃后底层目录仍可写") + ctx.check(not blocked.exists(), "失败写不能留下本地文件") + forced = run_process( adapter.prefix + ["unmount", "--force", str(adapter.mount_point)], check=False, timeout=60) ctx.equal(forced.returncode, 0, "helper 崩溃后强制卸载") - with sqlite3.connect(adapter.database) as connection: - dirty = connection.execute( - "SELECT count(*) FROM _vexfs_handles " - "WHERE dirty_generation>published_generation").fetchone()[0] - ctx.equal(dirty, 1, "helper 崩溃后 staging 仍在数据库") - connection.execute("UPDATE _vexfs_mount_sessions SET lease_until=0") + ctx.equal(adapter.mount_point.stat().st_mode & 0o777, 0o700, + "显式卸载后底层目录恢复 0700") + if adapter.backend == "postgresql": + dirty_result = run_process([ + "psql", adapter.connection_identity, "-X", "-q", "-t", "-A", + "-v", "ON_ERROR_STOP=1", "-c", + "SELECT count(*) FROM _vexfs.handles AS handle " + "JOIN _vexfs.workspaces AS workspace USING (workspace_id) " + "WHERE workspace.name='helper-crash' " + "AND handle.dirty_generation>handle.published_generation;", + ]) + dirty = int(dirty_result.stdout.decode().strip()) + ctx.equal(dirty, 1, "helper 崩溃后 staging 仍在 PostgreSQL") + run_process([ + "psql", adapter.connection_identity, "-X", "-q", + "-v", "ON_ERROR_STOP=1", "-c", + "UPDATE _vexfs.mount_sessions " + "SET lease_until=clock_timestamp() - interval '1 second' " + "WHERE workspace_id=(SELECT workspace_id FROM _vexfs.workspaces " + "WHERE name='helper-crash'); " + "UPDATE _vexfs.handles " + "SET lease_until=clock_timestamp() - interval '1 second' " + "WHERE workspace_id=(SELECT workspace_id FROM _vexfs.workspaces " + "WHERE name='helper-crash') " + "AND dirty_generation>published_generation;", + ]) + else: + with sqlite3.connect(adapter.database) as connection: + dirty = connection.execute( + "SELECT count(*) FROM _vexfs_handles " + "WHERE dirty_generation>published_generation").fetchone()[0] + ctx.equal(dirty, 1, "helper 崩溃后 staging 仍在数据库") + connection.execute("UPDATE _vexfs_mount_sessions SET lease_until=0") mount_adapter_mount(ctx, adapter) remounted = True ctx.equal(target.read_bytes(), b"CRASHine", "新 session 自动发布崩溃前 staging") - with sqlite3.connect(adapter.database) as connection: - dirty_after = connection.execute( - "SELECT count(*) FROM _vexfs_handles " - "WHERE dirty_generation>published_generation").fetchone()[0] + if adapter.backend == "postgresql": + dirty_after_result = run_process([ + "psql", adapter.connection_identity, "-X", "-q", "-t", "-A", + "-v", "ON_ERROR_STOP=1", "-c", + "SELECT count(*) FROM _vexfs.handles AS handle " + "JOIN _vexfs.workspaces AS workspace USING (workspace_id) " + "WHERE workspace.name='helper-crash' " + "AND handle.dirty_generation>handle.published_generation;", + ]) + dirty_after = int(dirty_after_result.stdout.decode().strip()) + else: + with sqlite3.connect(adapter.database) as connection: + dirty_after = connection.execute( + "SELECT count(*) FROM _vexfs_handles " + "WHERE dirty_generation>published_generation").fetchone()[0] ctx.equal(dirty_after, 0, "恢复后没有未发布 staging") return {"adapter": adapter.name, "helper_pid": helper_pids[0]} finally: @@ -4105,6 +4468,9 @@ def mount_helper_crash_recovery(ctx: Context) -> dict[str, Any]: child.wait(timeout=10) if remounted: mount_adapter_unmount(ctx, adapter, "helper 崩溃恢复") + else: + run_process(adapter.prefix + ["unmount", "--force", + str(adapter.mount_point)], check=False, timeout=60) @case("mount.real-bash", "mount", "真实 FSKit mount 后用 bash 常用文件命令操作") @@ -4113,30 +4479,37 @@ def real_mount_bash(ctx: Context) -> dict[str, Any]: raise EvalSkip("真实 FSKit mount 只在 macOS 执行") with tempfile.TemporaryDirectory(prefix="vexfs-mount-eval-") as directory: base = Path(directory) - database = base / "mount.sqlite3" - mount_point = base / "mnt" - cli = mount_cli(ctx) - prefix = filesystem_cli_prefix(cli, database, "eval") - run_process(prefix + ["setup"]) - doctor = run_process(prefix + ["--json", "doctor"], check=False) - details = json.loads(doctor.stdout) - if details.get("extension") != "enabled": - raise EvalSkip( - f"FSKit extension={details.get('extension', 'unknown')}，需安装并在系统设置中启用") - mount_point.mkdir() + adapter = prepare_real_mount_adapter(ctx, base, "eval") + mount_point = adapter.mount_point + prefix = adapter.prefix + details = adapter.doctor_before expected_target = os.path.realpath(mount_point) - run_process(prefix + ["mount", str(mount_point)], timeout=60) + mount_adapter_mount(ctx, adapter) descriptor: dict[str, Any] = {} mounts: list[dict[str, Any]] = [] - unmount = None try: - descriptor = json.loads((base / ".vexfs-volume.json").read_text()) - ctx.equal(descriptor, {"version": 2, "database_file": "mount.sqlite3", - "workspace": "eval"}, "FSKit 目录资源描述文件") status = run_process(prefix + ["--json", "mount", "status", str(mount_point)]) mounts = json.loads(status.stdout) ctx.equal(len(mounts), 1, "挂载状态数量") ctx.equal(mounts[0]["target"], expected_target, "挂载状态目标") + source = mounts[0]["source"] + source_path = (Path(unquote(urlparse(source).path)) + if source.startswith("file:") else Path(source)) + descriptor = json.loads( + (source_path / ".vexfs-volume.json").read_text()) + expected_connection = (adapter.database.name + if adapter.backend == "sqlite" + else adapter.connection_identity) + ctx.equal(descriptor, {"version": 3, "backend": adapter.backend, + "connection": expected_connection, + "workspace": "eval"}, "FSKit 目录资源描述文件") + if adapter.backend == "sqlite": + descriptor_connection = Path(descriptor["connection"]) + ctx.check(not descriptor_connection.is_absolute(), + "FSKit SQLite 描述文件不得保存绝对路径") + ctx.equal(os.path.realpath(source_path / descriptor_connection), + os.path.realpath(adapter.database), + "FSKit SQLite 描述文件相对路径解析") shell = r''' set -e mkdir -p "$1/project/sub" @@ -4159,15 +4532,14 @@ def real_mount_bash(ctx: Context) -> dict[str, Any]: reverse = run_process(prefix + ["cat", "/project/sub/result.txt"]) ctx.equal(reverse.stdout, b"agent\n", "数据库 CLI 反向读取挂载写入") finally: - unmount = run_process(prefix + ["unmount", str(mount_point)], - check=False, timeout=60) - ctx.equal(unmount.returncode, 0, "卸载退出码") + mount_adapter_unmount(ctx, adapter, "Bash") after = json.loads(run_process(prefix + ["--json", "doctor"]).stdout) ctx.equal(after["mount_count"], 0, "卸载后挂载数") ctx.equal(after["database"]["pending_handles"], 0, "卸载后未发布句柄") ctx.equal(after["database"]["retained_handles"], 0, "卸载后保留句柄") ctx.equal(after["database"]["staging_bytes"], 0, "卸载后暂存字节") - return {"cli": str(cli), "doctor_before": details, "doctor_after": after, + return {"cli": str(adapter.cli), "backend": adapter.backend, + "doctor_before": details, "doctor_after": after, "mounts": mounts, "descriptor": descriptor, "commands": ["mkdir", "printf", "grep", "cp", "mv", "cmp", "ls", @@ -4191,15 +4563,11 @@ def real_linux_mount_bash_git(ctx: Context) -> dict[str, Any]: with tempfile.TemporaryDirectory(prefix="vexfs-linux-mount-eval-") as directory: base = Path(directory) - database = base / "mount.sqlite3" - mount_point = base / "mnt" - prefix = [str(ctx.cli), "--db", str(database), "--workspace", "linux-eval"] - run_process(prefix + ["setup"]) - details = json.loads(run_process(prefix + ["--json", "doctor"], check=False).stdout) - if not details.get("mount_ready"): - raise EvalSkip(f"libfuse3 adapter={details.get('extension', 'unknown')}") - mount_point.mkdir() - run_process(prefix + ["mount", str(mount_point)], timeout=60) + adapter = prepare_real_mount_adapter(ctx, base, "linux-eval") + mount_point = adapter.mount_point + prefix = adapter.prefix + details = adapter.doctor_before + mount_adapter_mount(ctx, adapter) first_unmount = None try: shell = r''' @@ -4257,7 +4625,8 @@ def real_linux_mount_bash_git(ctx: Context) -> dict[str, Any]: after = json.loads(run_process(prefix + ["--json", "doctor"]).stdout) ctx.equal(after["mount_count"], 0, "Linux 卸载后无挂载") ctx.equal(after["database"]["pending_handles"], 0, "Linux 卸载后无未发布句柄") - return {"doctor_before": details, "doctor_after": after, + return {"backend": adapter.backend, + "doctor_before": details, "doctor_after": after, "commands": ["mkdir", "printf", "chmod", "exec", "grep", "cp", "mv", "hardlink", "symlink", "git init", "git add", "git commit", "git status", "unmount", "remount"]} @@ -4270,20 +4639,10 @@ def real_mount_posix_metadata(ctx: Context) -> dict[str, Any]: raise EvalSkip("真实 FSKit mount 只在 macOS 执行") with tempfile.TemporaryDirectory(prefix="vexfs-mount-posix-") as directory: base = Path(directory) - database = base / "mount.sqlite3" - mount_point = base / "mnt" - cli = mount_cli(ctx) - prefix = filesystem_cli_prefix(cli, database, "posix") - run_process(prefix + ["setup"]) - details = json.loads(run_process( - prefix + ["--json", "doctor"], check=False).stdout) - if details.get("extension") != "enabled": - raise EvalSkip( - f"FSKit extension={details.get('extension', 'unknown')}，需安装并在系统设置中启用") - - mount_point.mkdir() - run_process(prefix + ["mount", str(mount_point)], timeout=60) - first_unmount = None + adapter = prepare_real_mount_adapter(ctx, base, "posix") + mount_point = adapter.mount_point + prefix = adapter.prefix + mount_adapter_mount(ctx, adapter) script = mount_point / "project/run.sh" link = mount_point / "project/run-link" dangling = mount_point / "project/dangling" @@ -4311,15 +4670,12 @@ def real_mount_posix_metadata(ctx: Context) -> dict[str, Any]: renamed.unlink() ctx.check(script.exists(), "删除链接不删除目标") finally: - first_unmount = run_process(prefix + ["unmount", str(mount_point)], - check=False, timeout=60) - ctx.equal(first_unmount.returncode, 0, "POSIX 首次卸载退出码") + mount_adapter_unmount(ctx, adapter, "POSIX 首次") # FSKit 的 unload 在 umount 返回后仍可能异步收尾；等待旧 volume 释放同一 # container identity，避免把系统生命周期竞态误判成持久化失败。 time.sleep(2.0) - run_process(prefix + ["mount", str(mount_point)], timeout=60) - second_unmount = None + mount_adapter_mount(ctx, adapter) try: ctx.equal(script.stat().st_mode & 0o777, 0o755, "重挂载后 mode 持久化") ctx.equal(os.readlink(link), "run.sh", "重挂载后链接 target 持久化") @@ -4329,40 +4685,19 @@ def real_mount_posix_metadata(ctx: Context) -> dict[str, Any]: # mount 返回后 FSKit 仍会完成少量异步激活工作；这个回归用例操作很少， # 需要给 activate/deactivate 留出正常交接时间。 time.sleep(2.0) - second_unmount = run_process(prefix + ["unmount", str(mount_point)], - check=False, timeout=60) - if second_unmount.returncode != 0: - run_process(["/sbin/umount", "-f", str(mount_point)], - check=False, timeout=60) - ctx.equal(second_unmount.returncode, 0, "POSIX 第二次卸载退出码") + mount_adapter_unmount(ctx, adapter, "POSIX 第二次") - connection = sqlite3.connect(database, isolation_level=None) - connection.enable_load_extension(True) - connection.load_extension(str(ctx.extension)) - try: - row = connection.execute( - "SELECT i.id,i.kind,i.mode,i.size FROM _vexfs_dentries d " - "JOIN _vexfs_inodes i ON i.id=d.inode_id " - "JOIN _vexfs_workspaces w ON w.id=d.workspace_id " - "WHERE w.name='posix' AND d.name='run-link' AND i.deleted_at IS NULL" - ).fetchone() - ctx.check(row is not None, "SQLite 中存在符号链接 inode") - assert row is not None - ctx.equal((row[1], row[2], row[3]), ("symlink", 0o777, len("run.sh")), - "SQLite 中的链接元数据") - target = connection.execute( - "SELECT vexfs_readlink('posix',?)", (row[0],)).fetchone()[0] - ctx.equal(target, b"run.sh", "SQLite 中保存链接 target") - stored_mode = connection.execute( - "SELECT i.mode FROM _vexfs_dentries d JOIN _vexfs_inodes i ON i.id=d.inode_id " - "JOIN _vexfs_workspaces w ON w.id=d.workspace_id " - "WHERE w.name='posix' AND d.name='run.sh' AND i.deleted_at IS NULL" - ).fetchone()[0] - ctx.equal(stored_mode, 0o755, "SQLite 中保存可执行权限") - finally: - connection.close() - return {"cli": str(cli), "mode": 0o755, "target": "run.sh", - "remounted": True, "database_verified": True} + link_stat = json.loads(run_process( + prefix + ["stat", "/project/run-link"]).stdout) + script_stat = json.loads(run_process( + prefix + ["stat", "/project/run.sh"]).stdout) + ctx.equal((link_stat["kind"], link_stat["mode"], link_stat["size"]), + ("symlink", 0o777, len("run.sh")), + "数据库 CLI 中的链接元数据") + ctx.equal(script_stat["mode"], 0o755, "数据库 CLI 中保存可执行权限") + return {"cli": str(adapter.cli), "backend": adapter.backend, + "mode": 0o755, "target": "run.sh", "remounted": True, + "database_verified": True} @case("mount.performance", "mount", "真实 FSKit 分块顺序写和随机覆盖性能") @@ -4373,18 +4708,10 @@ def real_mount_performance(ctx: Context) -> dict[str, Any]: random_writes = {"quick": 100, "full": 1_000, "stress": 2_000}[ctx.mode.name] with tempfile.TemporaryDirectory(prefix="vexfs-mount-perf-") as directory: base = Path(directory) - database = base / "mount.sqlite3" - mount_point = base / "mnt" - cli = mount_cli(ctx) - prefix = filesystem_cli_prefix(cli, database, "perf") - run_process(prefix + ["setup"]) - details = json.loads(run_process( - prefix + ["--json", "doctor"], check=False).stdout) - if details.get("extension") != "enabled": - raise EvalSkip( - f"FSKit extension={details.get('extension', 'unknown')}，需安装并在系统设置中启用") - mount_point.mkdir() - run_process(prefix + ["mount", str(mount_point)], timeout=60) + adapter = prepare_real_mount_adapter(ctx, base, "perf") + mount_point = adapter.mount_point + prefix = adapter.prefix + mount_adapter_mount(ctx, adapter) fd = -1 try: path = mount_point / "bench.bin" @@ -4420,13 +4747,12 @@ def real_mount_performance(ctx: Context) -> dict[str, Any]: finally: if fd >= 0: os.close(fd) - unmount = run_process(prefix + ["unmount", str(mount_point)], - check=False, timeout=60) - ctx.equal(unmount.returncode, 0, "FSKit 性能测试卸载退出码") + mount_adapter_unmount(ctx, adapter, "性能测试") after = json.loads(run_process(prefix + ["--json", "doctor"]).stdout) ctx.equal(after["mount_count"], 0, "FSKit 性能测试卸载后挂载数") return { - "cli": str(cli), + "cli": str(adapter.cli), + "backend": adapter.backend, "sequential_mib": sequential_mib, "sequential_seconds": round(sequential_seconds, 6), "sequential_mib_per_second": round( @@ -4451,20 +4777,11 @@ def real_mount_git_workspace(ctx: Context) -> dict[str, Any]: raise EvalSkip("系统没有 xattr") with tempfile.TemporaryDirectory(prefix="vexfs-mount-git-") as directory: base = Path(directory) - database = base / "mount.sqlite3" - mount_point = base / "mnt" - cli = mount_cli(ctx) - prefix = filesystem_cli_prefix(cli, database, "git-eval") - run_process(prefix + ["setup"]) - details = json.loads(run_process( - prefix + ["--json", "doctor"], check=False).stdout) - if details.get("extension") != "enabled": - raise EvalSkip( - f"FSKit extension={details.get('extension', 'unknown')}，需安装并在系统设置中启用") - mount_point.mkdir() - run_process(prefix + ["mount", str(mount_point)], timeout=60) + adapter = prepare_real_mount_adapter(ctx, base, "git-eval") + mount_point = adapter.mount_point + prefix = adapter.prefix + mount_adapter_mount(ctx, adapter) project = mount_point / "project" - unmount = None try: project.mkdir() run_process([xattr, "-w", "com.vexfs.eval", "stored-in-sqlite", str(project)]) @@ -4537,25 +4854,27 @@ def real_mount_git_workspace(ctx: Context) -> dict[str, Any]: ctx.equal(run_process(prefix + ["cat", "/project/README.md"]).stdout, b"main\n", "数据库 CLI 反向读取 Git 工作区") finally: - unmount = run_process(prefix + ["unmount", str(mount_point)], - check=False, timeout=60) - ctx.equal(unmount.returncode, 0, "Git 测试卸载退出码") + mount_adapter_unmount(ctx, adapter, "Git 首次") + + mount_adapter_mount(ctx, adapter) + try: + persisted_xattr = run_process( + [xattr, "-p", "com.vexfs.eval", str(project)], timeout=120) + ctx.equal(persisted_xattr.stdout.strip(), b"stored-in-sqlite", + "重挂载后扩展属性持久化") + ctx.equal(run_process([git, "status", "--porcelain"], cwd=project, + env=environment, timeout=120).stdout, + b"", "重挂载后 Git 工作区状态") + finally: + mount_adapter_unmount(ctx, adapter, "Git 第二次") after = json.loads(run_process(prefix + ["--json", "doctor"]).stdout) ctx.equal(after["mount_count"], 0, "Git 测试卸载后挂载数") ctx.equal(after["database"]["pending_handles"], 0, "Git 测试未发布句柄") ctx.equal(after["database"]["retained_handles"], 0, "Git 测试保留句柄") ctx.equal(after["database"]["staging_bytes"], 0, "Git 测试暂存字节") - connection = sqlite3.connect(database) - try: - stored_xattr = connection.execute( - "SELECT value FROM _vexfs_xattrs WHERE name='com.vexfs.eval'" - ).fetchone() - finally: - connection.close() - ctx.equal(None if stored_xattr is None else stored_xattr[0], b"stored-in-sqlite", - "扩展属性持久化到 SQLite") return { - "cli": str(cli), + "cli": str(adapter.cli), + "backend": adapter.backend, "git": run_process([git, "--version"]).stdout.decode().strip(), "filemode": True, "symlinks": True, @@ -4686,15 +5005,22 @@ def run_toolchains() -> dict[str, float]: mount_adapter_unmount(ctx, adapter, "工具链首次") first_unmounted = True - connection = sqlite3.connect(adapter.database) - try: - ctx.equal(connection.execute("PRAGMA integrity_check").fetchone()[0], "ok", - "工具链项目数据库完整性") - stored_files = connection.execute( - "SELECT count(*) FROM _vexfs_inodes WHERE kind='file' AND deleted_at IS NULL" - ).fetchone()[0] - finally: - connection.close() + if adapter.backend == "sqlite": + connection = sqlite3.connect(adapter.database) + try: + ctx.equal(connection.execute("PRAGMA integrity_check").fetchone()[0], "ok", + "工具链项目数据库完整性") + stored_files = connection.execute( + "SELECT count(*) FROM _vexfs_inodes " + "WHERE kind='file' AND deleted_at IS NULL" + ).fetchone()[0] + finally: + connection.close() + else: + integrity = json.loads(run_process( + adapter.prefix + ["--json", "check"]).stdout) + ctx.equal(integrity["ok"], True, "工具链项目 PG workspace 完整性") + stored_files = int(integrity["versions"]) ctx.check(stored_files >= 15, "工具链项目产物进入数据库") mount_adapter_mount(ctx, adapter) @@ -4737,6 +5063,15 @@ def real_mount_opencode_project(ctx: Context) -> dict[str, Any]: if opencode is None or python is None or git is None: raise EvalSkip("真实 OpenCode 用例需要 opencode、python3 和 git") model = os.environ.get("VEXFS_EVAL_OPENCODE_MODEL", "openai/gpt-5.4-mini") + opencode_help_result = run_process([opencode, "run", "--help"]) + opencode_help = (opencode_help_result.stdout + opencode_help_result.stderr).decode( + errors="replace") + if "--dangerously-skip-permissions" in opencode_help: + approval_arguments = ["--dangerously-skip-permissions"] + elif "--auto" in opencode_help: + approval_arguments = ["--auto"] + else: + raise EvalSkip("当前 OpenCode 没有可用于隔离 eval 的自动授权参数") with tempfile.TemporaryDirectory(prefix="vexfs-opencode-project-") as directory: base = Path(directory) @@ -4785,7 +5120,7 @@ def real_mount_opencode_project(ctx: Context) -> dict[str, Any]: ) started = time.perf_counter() result = run_process( - [opencode, "run", "--auto", "--pure", "--format", "json", + [opencode, "run", *approval_arguments, "--pure", "--format", "json", "--model", model, "--dir", str(project), prompt], cwd=project, env=environment, timeout=900) agent_seconds = time.perf_counter() - started @@ -4804,12 +5139,17 @@ def real_mount_opencode_project(ctx: Context) -> dict[str, Any]: stored = run_process(adapter.prefix + ["cat", "/agent-project/calc.py"]) ctx.check(b"NotImplementedError" not in stored.stdout, "OpenCode 修改已经进入数据库权威内容") - connection = sqlite3.connect(adapter.database) - try: - ctx.equal(connection.execute("PRAGMA integrity_check").fetchone()[0], "ok", - "OpenCode 项目数据库完整性") - finally: - connection.close() + if adapter.backend == "sqlite": + connection = sqlite3.connect(adapter.database) + try: + ctx.equal(connection.execute("PRAGMA integrity_check").fetchone()[0], "ok", + "OpenCode 项目数据库完整性") + finally: + connection.close() + else: + integrity = json.loads(run_process( + adapter.prefix + ["--json", "check"]).stdout) + ctx.equal(integrity["ok"], True, "OpenCode 项目 PG workspace 完整性") mount_adapter_mount(ctx, adapter) try: @@ -4834,7 +5174,14 @@ def real_mount_opencode_project(ctx: Context) -> dict[str, Any]: def real_mount_scale_tree(ctx: Context) -> dict[str, Any]: file_count = {"quick": 1_000, "full": 10_000, "stress": 100_000}[ctx.mode.name] directory_count = min(1_000, max(20, file_count // 100)) - with tempfile.TemporaryDirectory(prefix="vexfs-mount-scale-") as directory: + default_native_search_timeout = {"quick": 60, "full": 300, "stress": 600}[ + ctx.mode.name] + native_search_timeout = float(os.environ.get( + "VEXFS_EVAL_NATIVE_SEARCH_TIMEOUT_SECONDS", + str(default_native_search_timeout))) + if native_search_timeout <= 0: + raise EvalFailure("VEXFS_EVAL_NATIVE_SEARCH_TIMEOUT_SECONDS 必须大于 0") + with FailureArtifactDirectory(ctx, "vexfs-mount-scale-") as directory: base = Path(directory) adapter = prepare_real_mount_adapter(ctx, base, "mount-scale") root = adapter.mount_point / "scale" @@ -4842,6 +5189,11 @@ def real_mount_scale_tree(ctx: Context) -> dict[str, Any]: if search is None: raise EvalSkip("系统没有 rg 或 grep") + def checkpoint(stage: str, **values: Any) -> None: + record = {"stage": stage, "at": time.time(), **values} + with (base / "eval-checkpoints.jsonl").open("a", encoding="utf-8") as stream: + stream.write(json.dumps(record, ensure_ascii=False) + "\n") + def create_tree(target: Path) -> None: target.mkdir() for directory_index in range(directory_count): @@ -4857,10 +5209,12 @@ def search_tree(target: Path) -> tuple[float, int, bool]: try: if Path(search).name == "rg": result = run_process( - [search, "-l", "needle", str(target)], timeout=3_600) + [search, "-l", "needle", str(target)], + timeout=native_search_timeout) else: result = run_process( - [search, "-R", "-l", "needle", str(target)], timeout=3_600) + [search, "-R", "-l", "needle", str(target)], + timeout=native_search_timeout) except subprocess.TimeoutExpired: return time.perf_counter() - started, 0, True return time.perf_counter() - started, len( @@ -4878,6 +5232,10 @@ def search_tree(target: Path) -> tuple[float, int, bool]: ctx.equal(native_files, file_count, "原生目录对照文件数") expected_matches = (file_count + 999) // 1_000 ctx.equal(native_matches, expected_matches, "原生目录对照搜索命中数") + checkpoint("native-complete", files=native_files, + create_seconds=native_create_seconds, + walk_seconds=native_walk_seconds, + search_seconds=native_search_seconds) mount_adapter_mount(ctx, adapter) create_seconds = 0.0 @@ -4904,11 +5262,15 @@ def search_tree(target: Path) -> tuple[float, int, bool]: ctx.equal(matches, expected_matches, "真实规模目录文本搜索命中数") finally: mount_adapter_unmount(ctx, adapter, "规模首次") + checkpoint("mount-tree-complete", files=file_count, + create_seconds=create_seconds, walk_seconds=walk_seconds, + search_seconds=search_seconds, + search_timed_out=search_timed_out) database_search_started = time.perf_counter() database_search = json.loads(run_process( - [str(ctx.cli), "--db", str(adapter.database), "--workspace", "mount-scale", - "--json", "grep", "-l", "needle", "/scale", "--max-results", "10240"], + adapter.prefix + ["--json", "grep", "-l", "needle", "/scale", + "--max-results", "10240"], timeout=3_600).stdout) database_search_seconds = time.perf_counter() - database_search_started ctx.equal(database_search["match_count"], expected_matches, @@ -4917,25 +5279,33 @@ def search_tree(target: Path) -> tuple[float, int, bool]: "数据库快路径搜索扫描完整目录树") index_build_started = time.perf_counter() index_status = json.loads(run_process( - [str(ctx.cli), "--db", str(adapter.database), "--workspace", "mount-scale", - "index", "enable"], timeout=3_600).stdout) + adapter.prefix + ["index", "enable"], timeout=3_600).stdout) index_build_seconds = time.perf_counter() - index_build_started - ctx.check(index_status["indexed_files"] >= file_count, - "数据库 trigram 索引覆盖业务文件和 FSKit 系统文件") + index_available = bool(index_status.get("available", True)) + if index_available: + ctx.check(index_status["indexed_files"] >= file_count, + "数据库 trigram 索引覆盖业务文件和 FSKit 系统文件") + else: + ctx.equal(index_status.get("backend"), "postgresql-scan", + "PG 必须如实报告无可用 substring 索引") indexed_search_started = time.perf_counter() indexed_search = json.loads(run_process( - [str(ctx.cli), "--db", str(adapter.database), "--workspace", "mount-scale", - "--json", "grep", "-l", "needle", "/scale", "--max-results", "10240"], + adapter.prefix + ["--json", "grep", "-l", "needle", "/scale", + "--max-results", "10240"], timeout=3_600).stdout) indexed_search_seconds = time.perf_counter() - indexed_search_started - ctx.equal(indexed_search["index_used"], True, "真实挂载数据使用 trigram 索引") + ctx.equal(indexed_search["index_used"], index_available, + "数据库搜索如实报告索引使用状态") ctx.equal(indexed_search["match_count"], expected_matches, "trigram 索引搜索命中数") + checkpoint("database-search-complete", + database_search_seconds=database_search_seconds, + index_build_seconds=index_build_seconds, + indexed_search_seconds=indexed_search_seconds) # 当前真实 FSKit helper 可能来自已安装的上一预览包。性能数据取完后关闭 # 新索引，避免旧 helper 后续修改临时数据库时无法维护这个新合同。 - run_process( - [str(ctx.cli), "--db", str(adapter.database), "--workspace", "mount-scale", - "index", "disable"], timeout=300) + if index_available: + run_process(adapter.prefix + ["index", "disable"], timeout=300) snapshot_started = time.perf_counter() run_process(adapter.prefix + ["snapshot", "create", "scale-baseline"], @@ -4960,27 +5330,47 @@ def search_tree(target: Path) -> tuple[float, int, bool]: ctx.equal(remounted_files, file_count, "真实规模目录重挂载文件数") finally: mount_adapter_unmount(ctx, adapter, "规模第二次") + checkpoint("snapshot-restore-complete", snapshot_seconds=snapshot_seconds, + restore_seconds=restore_seconds, remounted_files=remounted_files) - connection = sqlite3.connect(adapter.database) storage_rows: dict[str, int] = {} - try: - ctx.equal(connection.execute("PRAGMA integrity_check").fetchone()[0], "ok", - "真实规模目录数据库完整性") + if adapter.backend == "sqlite": + connection = sqlite3.connect(adapter.database) + try: + ctx.equal(connection.execute("PRAGMA integrity_check").fetchone()[0], "ok", + "真实规模目录数据库完整性") + storage_rows = { + label: int(connection.execute(f"SELECT count(*) FROM {table}").fetchone()[0]) + for label, table in ( + ("inode_rows", "_vexfs_inodes"), + ("commit_rows", "_vexfs_commits"), + ("file_version_rows", "_vexfs_file_versions"), + ("request_rows", "_vexfs_requests"), + ("inode_history_rows", "_vexfs_inode_states"), + ("dentry_history_rows", "_vexfs_dentry_states"), + ("acl_rows", "_vexfs_acl_entries"), + ("xattr_rows", "_vexfs_xattrs"), + ) + } + finally: + connection.close() + database_bytes = adapter.database.stat().st_size + else: + integrity = json.loads(run_process( + adapter.prefix + ["--json", "check"], timeout=3_600).stdout) + ctx.equal(integrity["ok"], True, "真实规模目录 PG workspace 完整性") + checked = integrity["checked"] storage_rows = { - label: int(connection.execute(f"SELECT count(*) FROM {table}").fetchone()[0]) - for label, table in ( - ("inode_rows", "_vexfs_inodes"), - ("commit_rows", "_vexfs_commits"), - ("file_version_rows", "_vexfs_file_versions"), - ("request_rows", "_vexfs_requests"), - ("inode_history_rows", "_vexfs_inode_states"), - ("dentry_history_rows", "_vexfs_dentry_states"), - ("acl_rows", "_vexfs_acl_entries"), - ("xattr_rows", "_vexfs_xattrs"), - ) + "inode_rows": int(checked["inodes"]), + "commit_rows": int(checked["commits"]), + "file_version_rows": int(checked["versions"]), + "request_rows": int(checked["requests"]), + "inode_history_rows": 0, + "dentry_history_rows": 0, + "acl_rows": int(checked["acl_entries"]), + "xattr_rows": int(checked["xattrs"]), } - finally: - connection.close() + database_bytes = int(integrity["content_bytes"]) create_budget = {"quick": 600.0, "full": 3_600.0, "stress": 14_400.0}[ ctx.mode.name] ctx.check(create_seconds >= 0, "mount_scale_create_seconds 不能为负数") @@ -4993,7 +5383,9 @@ def search_tree(target: Path) -> tuple[float, int, bool]: f"{create_seconds:.3f} > {create_budget:.3f}") if search_timed_out: ctx.checks += 1 - deferred_failures.append("真实规模目录文本搜索在 3600 秒内未完成") + deferred_failures.append( + "真实规模目录文本搜索在 " + f"{native_search_timeout:g} 秒内未完成") metrics = { "adapter": adapter.name, "files": file_count, @@ -5003,6 +5395,7 @@ def search_tree(target: Path) -> tuple[float, int, bool]: "walk_seconds": round(walk_seconds, 6), "search_seconds": round(search_seconds, 6), "search_timed_out": search_timed_out, + "native_search_timeout_seconds": native_search_timeout, "search_matches": matches, "database_search_seconds": round(database_search_seconds, 6), "database_search_matches": database_search["match_count"], @@ -5023,9 +5416,9 @@ def search_tree(target: Path) -> tuple[float, int, bool]: search_seconds / max(native_search_seconds, 1e-9), 3), "snapshot_seconds": round(snapshot_seconds, 6), "restore_seconds": round(restore_seconds, 6), - "database_bytes": adapter.database.stat().st_size, + "database_bytes": database_bytes, "database_bytes_per_file": round( - adapter.database.stat().st_size / max(file_count, 1), 3), + database_bytes / max(file_count, 1), 3), **storage_rows, "commits_per_file": round(storage_rows["commit_rows"] / file_count, 3), "versions_per_file": round(storage_rows["file_version_rows"] / file_count, 3), diff --git a/tests/eval/vexfs/run_cross_platform_portability.sh b/tests/eval/vexfs/run_cross_platform_portability.sh index b787d8c02c..a25d90c544 100755 --- a/tests/eval/vexfs/run_cross_platform_portability.sh +++ b/tests/eval/vexfs/run_cross_platform_portability.sh @@ -2,6 +2,7 @@ set -euo pipefail ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" +PYTHON="$ROOT/tests/eval/vexfs/python.sh" IMAGE="${VEXFS_LINUX_EVAL_IMAGE:-vexdb-lite-vexfs-linux-eval:debian}" HOST_BUILD_DIR="${VEXFS_MACOS_BUILD_DIR:-$ROOT/vexdb_sqlite/build}" MOUNT_CLI="${VEXFS_EVAL_MOUNT_CLI:?set VEXFS_EVAL_MOUNT_CLI to the signed CLI from the current build}" @@ -14,7 +15,7 @@ mkdir -p "$OUTPUT_ROOT/mac-create" "$OUTPUT_ROOT/linux-roundtrip" "$OUTPUT_ROOT/ require_pass() { local report="$1" - python3 -c \ + "$PYTHON" -c \ 'import json,sys; from pathlib import Path; latest=Path(sys.argv[1]); pointer=json.load(open(latest)); reports=sorted(latest.parent.glob("*/report.json"), key=lambda path: path.stat().st_mtime); assert reports, latest; value=json.load(open(reports[-1])); summary=value["summary"]; assert pointer["status"] == "PASS" and summary["failed"] == 0 and summary["passed"] == 1 and summary["skipped"] == 0, summary' \ "$report" } @@ -22,7 +23,7 @@ require_pass() { echo "=== macOS FSKit 创建共享工作区 ===" VEXFS_PORTABILITY_PHASE=mac-create \ VEXFS_PORTABILITY_DB="$DATABASE" \ -python3 "$ROOT/tests/eval/vexfs/run.py" \ +"$PYTHON" "$ROOT/tests/eval/vexfs/run.py" \ --root "$ROOT" \ --build-dir "$HOST_BUILD_DIR" \ --output-dir "$OUTPUT_ROOT/mac-create" \ @@ -53,7 +54,7 @@ require_pass "$OUTPUT_ROOT/linux-roundtrip/latest.json" echo "=== macOS FSKit 回读并恢复 Linux 快照 ===" VEXFS_PORTABILITY_PHASE=mac-verify \ VEXFS_PORTABILITY_DB="$DATABASE" \ -python3 "$ROOT/tests/eval/vexfs/run.py" \ +"$PYTHON" "$ROOT/tests/eval/vexfs/run.py" \ --root "$ROOT" \ --build-dir "$HOST_BUILD_DIR" \ --output-dir "$OUTPUT_ROOT/mac-verify" \ diff --git a/tests/eval/vexfs/run_linux_mount_in_container.sh b/tests/eval/vexfs/run_linux_mount_in_container.sh index 7f5ace4990..98ec85232e 100755 --- a/tests/eval/vexfs/run_linux_mount_in_container.sh +++ b/tests/eval/vexfs/run_linux_mount_in_container.sh @@ -1,11 +1,13 @@ #!/bin/bash set -euo pipefail -BUILD_DIR=/tmp/vexfs-linux-build +BUILD_DIR="${VEXFS_LINUX_BUILD_DIR:-/workspace/.cache/vexfs-linux-build}" OUTPUT_BASE=/workspace/vexdb_sqlite/build/eval/vexfs-linux-mount ROOT_OUTPUT="$OUTPUT_BASE/root" USER_OUTPUT="$OUTPUT_BASE/uid-1000" -BUILD_LOG=/tmp/vexfs-linux-build.log +BUILD_LOG="$BUILD_DIR/build.log" + +mkdir -p "$BUILD_DIR" if ! cmake -S /workspace/vexdb_sqlite -B "$BUILD_DIR" \ -DCMAKE_BUILD_TYPE=Release \ @@ -13,14 +15,28 @@ if ! cmake -S /workspace/vexdb_sqlite -B "$BUILD_DIR" \ tail -n 200 "$BUILD_LOG" exit 1 fi -if ! cmake --build "$BUILD_DIR" -j "$(nproc)" >>"$BUILD_LOG" 2>&1; then +BUILD_JOBS="${VEXFS_BUILD_JOBS:-1}" +case "$BUILD_JOBS" in ''|*[!0-9]*|0) echo "VEXFS_BUILD_JOBS 必须是正整数" >&2; exit 2;; esac +[ "$BUILD_JOBS" -le 2 ] || BUILD_JOBS=2 +if ! cmake --build "$BUILD_DIR" -j "$BUILD_JOBS" >>"$BUILD_LOG" 2>&1; then tail -n 200 "$BUILD_LOG" exit 1 fi read -r -a MOUNT_CASES <<< "${VEXFS_LINUX_EVAL_CASES:-mount.cross-platform-conformance mount.timestamps mount.concurrent-append mount.open-rename-unlink mount.process-locks mount.force-unmount mount.helper-crash-recovery}" +cleanup_pg_workspaces() { + [ "${VEXFS_MOUNT_BACKEND:-sqlite}" = postgresql ] || return 0 + local workspace + for workspace in conformance timestamps append open-life read-life locks \ + force-unmount helper-crash linux-eval toolchains; do + psql "$VEXFS_MOUNT_DSN" -X -q -v ON_ERROR_STOP=1 \ + -c "SELECT vexfs_workspace_drop('$workspace', true);" >/dev/null 2>&1 || true + done +} + for test_case in "${MOUNT_CASES[@]}"; do + cleanup_pg_workspaces python3 /workspace/tests/eval/vexfs/run.py \ --root /workspace \ --build-dir "$BUILD_DIR" \ @@ -35,6 +51,7 @@ chmod 0666 /dev/fuse mkdir -p "$ROOT_OUTPUT" "$USER_OUTPUT" chown vexeval:vexeval "$USER_OUTPUT" for test_case in "${MOUNT_CASES[@]}"; do + cleanup_pg_workspaces su --shell /bin/bash vexeval --command \ "python3 /workspace/tests/eval/vexfs/run.py \ --root /workspace \ @@ -45,4 +62,6 @@ for test_case in "${MOUNT_CASES[@]}"; do --filter $test_case" done +cleanup_pg_workspaces + echo "VEXFS LINUX MOUNT CONFORMANCE: PASS" diff --git a/tests/eval/vexfs/run_pg_adapter_alpha.sh b/tests/eval/vexfs/run_pg_adapter_alpha.sh new file mode 100755 index 0000000000..67036e191d --- /dev/null +++ b/tests/eval/vexfs/run_pg_adapter_alpha.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" +CONTAINER="${VEXDB_PG_CONTAINER:-vexdb_pg19-test}" +DATABASE="${VEXDB_PG_DATABASE:-test}" +WORKSPACE="pg-concurrent" +CHECKS=0 +TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/vexfs-pg-alpha.XXXXXX")" + +cleanup() { + docker exec "$CONTAINER" psql -d "$DATABASE" -X -q -v ON_ERROR_STOP=1 \ + -c "SELECT vexfs_workspace_drop('$WORKSPACE', true);" >/dev/null 2>&1 || true + rm -rf "$TMP_DIR" +} +trap cleanup EXIT + +docker inspect "$CONTAINER" >/dev/null + +# Public function names shared by SQLite must also exist in PostgreSQL. Keep +# this check source-derived so adding a new common SQLite function cannot leave +# the PG adapter silently behind. +sed -n '/static const FunctionDefinition functions\[\]/,/^ };/p' \ + "$ROOT/vexdb_sqlite/src/agent_files/vexfs_sqlite.cpp" \ + | sed -n 's/^[[:space:]]*{"\(vexfs_[a-z0-9_]*\)".*/\1/p' \ + | sort -u >"$TMP_DIR/sqlite-contract.txt" +sed -n '/static const FunctionDefinition functions\[\]/,/^ };/p' \ + "$ROOT/vexdb_sqlite/src/agent_files/vexfs_sqlite.cpp" \ + | sed -n 's/^[[:space:]]*{"\(vexfs_[a-z0-9_]*\)",[[:space:]]*\([0-9][0-9]*\).*/\1|\2/p' \ + | sort -u >"$TMP_DIR/sqlite-arities.txt" +docker exec "$CONTAINER" psql -d "$DATABASE" -X -q -t -A -v ON_ERROR_STOP=1 \ + -c "SELECT DISTINCT p.proname + FROM pg_catalog.pg_proc AS p + JOIN pg_catalog.pg_namespace AS n ON n.oid=p.pronamespace + WHERE n.nspname='public' AND p.proname LIKE 'vexfs_%' + ORDER BY p.proname;" \ + >"$TMP_DIR/pg-contract.txt" +docker exec "$CONTAINER" psql -d "$DATABASE" -X -q -t -A -F '|' \ + -v ON_ERROR_STOP=1 \ + -c "SELECT p.proname,p.pronargs-p.pronargdefaults,p.pronargs + FROM pg_catalog.pg_proc AS p + JOIN pg_catalog.pg_namespace AS n ON n.oid=p.pronamespace + WHERE n.nspname='public' AND p.proname LIKE 'vexfs_%' + ORDER BY p.proname,p.pronargs-p.pronargdefaults,p.pronargs;" \ + >"$TMP_DIR/pg-arities.txt" +MISSING_CONTRACT="$(comm -23 "$TMP_DIR/sqlite-contract.txt" "$TMP_DIR/pg-contract.txt")" +if [[ -n "$MISSING_CONTRACT" ]]; then + echo "PostgreSQL 缺少公共 VexFS SQL 函数：" >&2 + printf '%s\n' "$MISSING_CONTRACT" >&2 + exit 1 +fi +while IFS='|' read -r function_name arguments; do + if ! awk -F '|' -v name="$function_name" -v argc="$arguments" \ + '$1 == name && argc >= $2 && argc <= $3 { found=1 } + END { exit(found ? 0 : 1) }' "$TMP_DIR/pg-arities.txt"; then + echo "PostgreSQL 公共函数不接受 SQLite 参数数量：$function_name/$arguments" >&2 + exit 1 + fi +done <"$TMP_DIR/sqlite-arities.txt" +CONTRACT_STATE="$(docker exec "$CONTAINER" psql -d "$DATABASE" -X -q -t -A \ + -F '|' -v ON_ERROR_STOP=1 \ + -c "SELECT vexfs_pg_adapter_version(),vexfs_contract_version(),vexfs_init();")" +if [[ "$CONTRACT_STATE" != "0.4.0-alpha.1|0.9.0|1" ]]; then + echo "PostgreSQL VexFS 合同版本错误：$CONTRACT_STATE" >&2 + exit 1 +fi +CHECKS=$((CHECKS + $(wc -l <"$TMP_DIR/sqlite-contract.txt") + \ + $(wc -l <"$TMP_DIR/sqlite-arities.txt") + 3)) + +docker exec -i "$CONTAINER" psql -d "$DATABASE" -X -q -v ON_ERROR_STOP=1 </dev/null +SELECT vexfs_workspace_drop('$WORKSPACE', true); +SELECT vexfs_workspace_create('$WORKSPACE'); +SELECT vexfs_write('$WORKSPACE', '/shared.txt', convert_to('base', 'UTF8')); +SQL +CHECKS=$((CHECKS + 3)) + +docker exec "$CONTAINER" psql -d "$DATABASE" -X -q -t -A -v ON_ERROR_STOP=1 \ + -c "SELECT vexfs_write('$WORKSPACE', '/shared.txt', convert_to('writer-a', 'UTF8'));" \ + >"$TMP_DIR/a.out" & +PID_A=$! +docker exec "$CONTAINER" psql -d "$DATABASE" -X -q -t -A -v ON_ERROR_STOP=1 \ + -c "SELECT vexfs_write('$WORKSPACE', '/shared.txt', convert_to('writer-b', 'UTF8'));" \ + >"$TMP_DIR/b.out" & +PID_B=$! + +wait "$PID_A" +wait "$PID_B" + +VERSIONS="$(sort -n "$TMP_DIR/a.out" "$TMP_DIR/b.out" | tr '\n' ',' | sed 's/,$//')" +if [[ "$VERSIONS" != "2,3" ]]; then + echo "并发写版本错误：$VERSIONS" >&2 + exit 1 +fi +CHECKS=$((CHECKS + 1)) + +STATE="$(docker exec "$CONTAINER" psql -d "$DATABASE" -X -q -t -A -F '|' -v ON_ERROR_STOP=1 \ + -c "SELECT i.current_version, count(f.version_no), convert_from(vexfs_read('$WORKSPACE', '/shared.txt'), 'UTF8') IN ('writer-a', 'writer-b') FROM _vexfs.inodes i JOIN _vexfs.file_versions f USING (workspace_id, inode_id) WHERE i.kind='file' AND i.workspace_id=(SELECT workspace_id FROM _vexfs.workspaces WHERE name='$WORKSPACE') GROUP BY i.current_version;")" +if [[ "$STATE" != "3|3|t" ]]; then + echo "并发写最终状态错误：$STATE" >&2 + exit 1 +fi +CHECKS=$((CHECKS + 3)) + +EXPECTED_HEAD="$(docker exec "$CONTAINER" psql -d "$DATABASE" -X -q -t -A -v ON_ERROR_STOP=1 \ + -c "SELECT vexfs_snapshot_create('$WORKSPACE', 'concurrent-restore'); SELECT vexfs_write('$WORKSPACE', '/shared.txt', convert_to('after-snapshot', 'UTF8')); SELECT (vexfs_workspace_stat('$WORKSPACE')->>'head_commit')::bigint;")" +EXPECTED_HEAD="$(printf '%s\n' "$EXPECTED_HEAD" | tail -n 1)" +if [[ "$EXPECTED_HEAD" != "5" ]]; then + echo "并发恢复准备 head 错误：$EXPECTED_HEAD" >&2 + exit 1 +fi +CHECKS=$((CHECKS + 3)) + +docker exec "$CONTAINER" psql -d "$DATABASE" -X -q -t -A -v ON_ERROR_STOP=1 \ + -c "SELECT vexfs_snapshot_restore('$WORKSPACE', 'concurrent-restore', $EXPECTED_HEAD);" \ + >"$TMP_DIR/restore-a.out" 2>"$TMP_DIR/restore-a.err" & +PID_A=$! +docker exec "$CONTAINER" psql -d "$DATABASE" -X -q -t -A -v ON_ERROR_STOP=1 \ + -c "SELECT vexfs_snapshot_restore('$WORKSPACE', 'concurrent-restore', $EXPECTED_HEAD);" \ + >"$TMP_DIR/restore-b.out" 2>"$TMP_DIR/restore-b.err" & +PID_B=$! + +set +e +wait "$PID_A"; RC_A=$? +wait "$PID_B"; RC_B=$? +set -e +if [[ "$RC_A" -eq 0 && "$RC_B" -eq 0 ]] || [[ "$RC_A" -ne 0 && "$RC_B" -ne 0 ]]; then + echo "并发恢复必须一成一败：rc_a=$RC_A rc_b=$RC_B" >&2 + exit 1 +fi +if ! grep -q 'VEXFS_HEAD_CONFLICT' "$TMP_DIR/restore-a.err" "$TMP_DIR/restore-b.err"; then + echo "并发恢复失败端没有返回 VEXFS_HEAD_CONFLICT" >&2 + exit 1 +fi +CHECKS=$((CHECKS + 2)) + +RESTORE_STATE="$(docker exec "$CONTAINER" psql -d "$DATABASE" -X -q -t -A -F '|' -v ON_ERROR_STOP=1 \ + -c "SELECT (vexfs_workspace_stat('$WORKSPACE')->>'head_commit')::bigint, convert_from(vexfs_read('$WORKSPACE', '/shared.txt'), 'UTF8') IN ('writer-a', 'writer-b');")" +if [[ "$RESTORE_STATE" != "6|t" ]]; then + echo "并发恢复最终状态错误：$RESTORE_STATE" >&2 + exit 1 +fi +CHECKS=$((CHECKS + 2)) + +echo "VEXFS PG ADAPTER ALPHA: PASS ($CHECKS checks)" diff --git a/tests/eval/vexfs/run_pg_adapter_performance.sh b/tests/eval/vexfs/run_pg_adapter_performance.sh new file mode 100755 index 0000000000..215dae6159 --- /dev/null +++ b/tests/eval/vexfs/run_pg_adapter_performance.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +set -euo pipefail + +CONTAINER="${VEXDB_PG_CONTAINER:-vexdb_pg19-test}" +DATABASE="${VEXDB_PG_DATABASE:-test}" +FILE_COUNT="${VEXDB_PG_PERF_FILES:-1000}" +WORKSPACE="pg-performance" + +if ! [[ "$FILE_COUNT" =~ ^[0-9]+$ ]] || (( FILE_COUNT < 1 || FILE_COUNT > 10000 )); then + echo "VEXDB_PG_PERF_FILES 必须在 1..10000" >&2 + exit 2 +fi + +docker inspect "$CONTAINER" >/dev/null + +RESULT="$(docker exec -i "$CONTAINER" psql -d "$DATABASE" -X -q -t -A -F '|' \ + -v ON_ERROR_STOP=1 -v file_count="$FILE_COUNT" <<'SQL' | tail -n 1 +SELECT set_config('vexfs.perf_files', :'file_count', false); +DO $$ +DECLARE + v_count integer := current_setting('vexfs.perf_files')::integer; + v_started timestamptz; + v_elapsed_ms double precision; + v_written bigint; +BEGIN + PERFORM public.vexfs_workspace_drop('pg-performance', true); + PERFORM public.vexfs_workspace_create('pg-performance'); + CREATE TEMP TABLE vexfs_perf_result(elapsed_ms double precision) ON COMMIT PRESERVE ROWS; + v_started := clock_timestamp(); + SELECT count(public.vexfs_write( + 'pg-performance', + '/f-' || i::text || '.txt', + convert_to(i::text, 'UTF8'))) + INTO v_written + FROM generate_series(1, v_count) AS g(i); + v_elapsed_ms := extract(epoch FROM clock_timestamp() - v_started) * 1000.0; + IF v_written <> v_count THEN + RAISE EXCEPTION 'write count mismatch: expected %, actual %', v_count, v_written; + END IF; + INSERT INTO vexfs_perf_result VALUES (v_elapsed_ms); +END; +$$; +SELECT round(r.elapsed_ms::numeric, 3), + (public.vexfs_workspace_stat('pg-performance')->>'head_commit')::bigint, + (SELECT count(*) FROM public.vexfs_list('pg-performance', '/')), + min(c.commit_no), + max(c.commit_no), + count(c.commit_no), + (SELECT count(*) FROM _vexfs.audit_events AS audit + WHERE audit.workspace_id = w.workspace_id), + (SELECT bool_and(audit.path IS NOT NULL + AND audit.inode_id IS NOT NULL + AND audit.details ? 'before_version' + AND audit.details ? 'after_version') + FROM _vexfs.audit_events AS audit + WHERE audit.workspace_id = w.workspace_id) + FROM vexfs_perf_result AS r + JOIN _vexfs.workspaces AS w ON w.name = 'pg-performance' + JOIN _vexfs.commits AS c ON c.workspace_id = w.workspace_id + GROUP BY r.elapsed_ms, w.workspace_id; +DO $$ BEGIN PERFORM public.vexfs_workspace_drop('pg-performance', true); END $$; +SQL +)" + +IFS='|' read -r ELAPSED_MS HEAD FILES MIN_COMMIT MAX_COMMIT COMMITS AUDITS AUDIT_COMPLETE <<<"$RESULT" +EXPECTED_HEAD=$((FILE_COUNT + 1)) +if [[ "$HEAD" != "$EXPECTED_HEAD" || "$FILES" != "$FILE_COUNT" || \ + "$MIN_COMMIT" != 1 || "$MAX_COMMIT" != "$EXPECTED_HEAD" || \ + "$COMMITS" != "$EXPECTED_HEAD" || "$AUDITS" != "$EXPECTED_HEAD" || \ + "$AUDIT_COMPLETE" != t ]]; then + echo "PG 性能基线状态错误：$RESULT" >&2 + exit 1 +fi + +RATE="$(awk -v n="$FILE_COUNT" -v ms="$ELAPSED_MS" 'BEGIN { if (ms <= 0) print 0; else printf "%.3f", n * 1000 / ms }')" +echo "VEXFS PG PERFORMANCE: PASS (files=$FILE_COUNT elapsed_ms=$ELAPSED_MS files_per_second=$RATE checks=7)" diff --git a/tests/eval/vexfs/run_pg_archive_roundtrip.sh b/tests/eval/vexfs/run_pg_archive_roundtrip.sh new file mode 100755 index 0000000000..ca3aeabafd --- /dev/null +++ b/tests/eval/vexfs/run_pg_archive_roundtrip.sh @@ -0,0 +1,178 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" +CONTAINER="${VEXDB_PG_CONTAINER:-vexdb_pg19-test}" +DATABASE="${VEXDB_PG_DATABASE:-test}" +DSN="${VEXDB_PG_DSN:-}" +CLI="${VEXFS_EVAL_MOUNT_CLI:-$ROOT/vexdb_sqlite/build/vexdb}" +TMP="$(mktemp -d "${TMPDIR:-/tmp}/vexfs-pg-archive.XXXXXX")" +SQLITE_SOURCE="$TMP/sqlite-source.sqlite3" +SQLITE_TARGET="$TMP/sqlite-target.sqlite3" +SQLITE_PACKAGE="$TMP/sqlite-to-pg.vexfs" +PG_PACKAGE="$TMP/pg-to-sqlite.vexfs" +CORRUPT_PACKAGE="$TMP/corrupt.vexfs" +CHECKS=0 + +cleanup() { + local status=$? + trap - EXIT + local workspace + for workspace in sqlite-to-pg pg-to-sqlite corrupt-import; do + docker exec "$CONTAINER" psql -U postgres -d "$DATABASE" -X -q \ + -c "SELECT vexfs_workspace_drop('$workspace', true);" \ + >/dev/null 2>&1 || true + done + rm -rf "$TMP" + exit "$status" +} +trap cleanup EXIT + +assert_equal() { + local actual=$1 + local expected=$2 + local message=$3 + if [[ "$actual" != "$expected" ]]; then + echo "${message}：期望 ${expected}，实际 ${actual}" >&2 + exit 1 + fi + CHECKS=$((CHECKS + 1)) +} + +[ -x "$CLI" ] || { echo "找不到 vexdb CLI：$CLI" >&2; exit 2; } +docker inspect "$CONTAINER" >/dev/null +if [ -z "$DSN" ]; then + HOST_BINDING="$(docker port "$CONTAINER" 5432/tcp 2>/dev/null | sed -n '1p')" + HOST_PORT="${HOST_BINDING##*:}" + case "$HOST_PORT" in + ''|*[!0-9]*) + echo "无法从容器 $CONTAINER 推导 PostgreSQL 端口；请设置 VEXDB_PG_DSN" >&2 + exit 2 + ;; + esac + DSN="postgresql://postgres@127.0.0.1:${HOST_PORT}/${DATABASE}" +fi + +for workspace in sqlite-to-pg pg-to-sqlite corrupt-import; do + docker exec "$CONTAINER" psql -U postgres -d "$DATABASE" -X -q \ + -c "SELECT vexfs_workspace_drop('$workspace', true);" \ + >/dev/null 2>&1 || true +done + +SQLITE=("$CLI" fs --db "$SQLITE_SOURCE" --workspace sqlite-source) +PG_IMPORT=("$CLI" fs --backend pg --dsn "$DSN" --workspace sqlite-to-pg) +"${SQLITE[@]}" setup >/dev/null +"${SQLITE[@]}" mkdir /tree >/dev/null +printf 'sqlite-v1' | "${SQLITE[@]}" write /tree/value.txt >/dev/null +"${SQLITE[@]}" ln /tree/value.txt /tree/value-hard.txt >/dev/null +printf '[{"principal":"agent-portable","effect":"allow","permissions":"read","inherit":0}]' | \ + "${SQLITE[@]}" setfacl /tree/value.txt >/dev/null +"$CLI" "$SQLITE_SOURCE" \ + "SELECT vexfs_symlink('sqlite-source','/tree/value-link.txt',CAST('value.txt' AS BLOB)); + SELECT vexfs_xattr_set( + 'sqlite-source',(vexfs_stat('sqlite-source','/tree/value.txt')->>'inode'), + 'user.cross-engine',CAST('sqlite-metadata' AS BLOB),0);" >/dev/null +"${SQLITE[@]}" snapshot create baseline >/dev/null +printf 'sqlite-v2' | "${SQLITE[@]}" write /tree/value.txt >/dev/null +"${SQLITE[@]}" export --output "$SQLITE_PACKAGE" >/dev/null + +VERIFY_SQLITE_PACKAGE="$("${SQLITE[@]}" archive verify "$SQLITE_PACKAGE")" +[[ "$VERIFY_SQLITE_PACKAGE" == *'"ok":true'* ]] || { + echo "SQLite 导出包校验失败：$VERIFY_SQLITE_PACKAGE" >&2 + exit 1 +} +CHECKS=$((CHECKS + 1)) +"${PG_IMPORT[@]}" import "$SQLITE_PACKAGE" >/dev/null + +PG_IMPORTED="$(docker exec "$CONTAINER" psql -U postgres -d "$DATABASE" -X -q -t -A -F '|' \ + -v ON_ERROR_STOP=1 -c \ + "SELECT convert_from(vexfs_read('sqlite-to-pg','/tree/value.txt'),'UTF8'), + convert_from(vexfs_read_version('sqlite-to-pg','/tree/value.txt',1),'UTF8'), + (SELECT count(*) FROM vexfs_snapshot_list('sqlite-to-pg')), + (vexfs_stat('sqlite-to-pg','/tree/value.txt')->>'inode') = + (vexfs_stat('sqlite-to-pg','/tree/value-hard.txt')->>'inode'), + convert_from(vexfs_readlink( + 'sqlite-to-pg',(vexfs_stat('sqlite-to-pg','/tree/value-link.txt')->>'inode')::bigint), + 'UTF8'), + convert_from(vexfs_xattr_get( + 'sqlite-to-pg',(vexfs_stat('sqlite-to-pg','/tree/value.txt')->>'inode')::bigint, + 'user.cross-engine'),'UTF8'), + vexfs_acl_get( + 'sqlite-to-pg',(vexfs_stat('sqlite-to-pg','/tree/value.txt')->>'inode')::bigint) + @> '[{\"principal\":\"agent-portable\",\"effect\":\"allow\",\"permissions\":\"read\"}]'::jsonb, + (vexfs_check('sqlite-to-pg',1)->>'ok')::boolean;")" +assert_equal "$PG_IMPORTED" \ + "sqlite-v2|sqlite-v1|1|t|value.txt|sqlite-metadata|t|t" \ + "SQLite 到 PostgreSQL format v2 往返错误" + +# 目标 workspace 已存在时必须拒绝，且不能改动已导入内容。 +if "${PG_IMPORT[@]}" import "$SQLITE_PACKAGE" >/dev/null 2>&1; then + echo "重复导入现有 PostgreSQL workspace 应该失败" >&2 + exit 1 +fi +assert_equal "$("${PG_IMPORT[@]}" cat /tree/value.txt)" "sqlite-v2" \ + "重复导入失败后内容不应改变" + +docker exec -i "$CONTAINER" psql -U postgres -d "$DATABASE" -X -q \ + -v ON_ERROR_STOP=1 >/dev/null <<'SQL' +SELECT vexfs_workspace_create('pg-to-sqlite'); +SELECT vexfs_mkdir('pg-to-sqlite','/tree',true); +SELECT vexfs_write('pg-to-sqlite','/tree/value.txt',convert_to('pg-v1','UTF8')); +SELECT vexfs_link('pg-to-sqlite','/tree/value.txt','/tree/value-hard.txt'); +SELECT vexfs_symlink('pg-to-sqlite','/tree/value-link.txt',convert_to('value.txt','UTF8')); +SELECT vexfs_xattr_set( + 'pg-to-sqlite',(vexfs_stat('pg-to-sqlite','/tree/value.txt')->>'inode')::bigint, + 'user.cross-engine',convert_to('pg-metadata','UTF8'),1); +SELECT vexfs_acl_grant( + 'pg-to-sqlite',(vexfs_stat('pg-to-sqlite','/tree/value.txt')->>'inode')::bigint, + 'agent-portable','read','allow',0); +SELECT vexfs_snapshot_create('pg-to-sqlite','baseline'); +SELECT vexfs_write('pg-to-sqlite','/tree/value.txt',convert_to('pg-v2','UTF8')); +SQL + +PG_EXPORT=("$CLI" fs --backend pg --dsn "$DSN" --workspace pg-to-sqlite) +"${PG_EXPORT[@]}" export --output "$PG_PACKAGE" >/dev/null +VERIFY_PG_PACKAGE="$("${PG_EXPORT[@]}" archive verify "$PG_PACKAGE")" +[[ "$VERIFY_PG_PACKAGE" == *'"ok":true'* ]] || { + echo "PostgreSQL 导出包校验失败：$VERIFY_PG_PACKAGE" >&2 + exit 1 +} +CHECKS=$((CHECKS + 1)) + +SQLITE_IMPORT=("$CLI" fs --db "$SQLITE_TARGET" --workspace pg-imported) +"${SQLITE_IMPORT[@]}" import "$PG_PACKAGE" >/dev/null +SQLITE_IMPORTED="$("$CLI" "$SQLITE_TARGET" \ + "SELECT CAST(vexfs_read('pg-imported','/tree/value.txt') AS TEXT), + CAST(vexfs_read_version('pg-imported','/tree/value.txt',1) AS TEXT), + (SELECT count(*) FROM json_each(vexfs_snapshot_list('pg-imported'))), + json_extract(vexfs_stat('pg-imported','/tree/value.txt'),'$.inode') = + json_extract(vexfs_stat('pg-imported','/tree/value-hard.txt'),'$.inode'), + CAST(vexfs_readlink( + 'pg-imported',json_extract(vexfs_stat('pg-imported','/tree/value-link.txt'),'$.inode')) AS TEXT), + CAST(vexfs_xattr_get( + 'pg-imported',json_extract(vexfs_stat('pg-imported','/tree/value.txt'),'$.inode'), + 'user.cross-engine') AS TEXT), + EXISTS( + SELECT 1 FROM json_each(vexfs_acl_get( + 'pg-imported',json_extract(vexfs_stat('pg-imported','/tree/value.txt'),'$.inode'))) + WHERE json_extract(value,'$.principal')='agent-portable' + AND json_extract(value,'$.permissions')='read'), + json_extract(vexfs_check('pg-imported',1),'$.ok');")" +assert_equal "$SQLITE_IMPORTED" \ + "pg-v2|pg-v1|1|1|value.txt|pg-metadata|1|1" \ + "PostgreSQL 到 SQLite format v2 往返错误" + +cp "$SQLITE_PACKAGE" "$CORRUPT_PACKAGE" +"$CLI" "$CORRUPT_PACKAGE" \ + "UPDATE chunks SET content=zeroblob(length(content)) + WHERE rowid=(SELECT min(rowid) FROM chunks);" >/dev/null +CORRUPT_IMPORT=("$CLI" fs --backend pg --dsn "$DSN" --workspace corrupt-import) +if "${CORRUPT_IMPORT[@]}" import "$CORRUPT_PACKAGE" >/dev/null 2>&1; then + echo "损坏的 format v2 包导入 PostgreSQL 应该失败" >&2 + exit 1 +fi +CORRUPT_EXISTS="$(docker exec "$CONTAINER" psql -U postgres -d "$DATABASE" -X -q -t -A \ + -c "SELECT EXISTS(SELECT 1 FROM _vexfs.workspaces WHERE name='corrupt-import');")" +assert_equal "$CORRUPT_EXISTS" "f" "损坏包失败后不能留下 workspace" + +echo "VEXFS PG ARCHIVE CROSS-ENGINE ROUNDTRIP: PASS ($CHECKS checks)" diff --git a/tests/eval/vexfs/run_pg_backup_performance.sh b/tests/eval/vexfs/run_pg_backup_performance.sh new file mode 100755 index 0000000000..e386bebc96 --- /dev/null +++ b/tests/eval/vexfs/run_pg_backup_performance.sh @@ -0,0 +1,250 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" +CONTAINER="${VEXDB_PG_CONTAINER:-vexdb_pg19-test}" +SOURCE_DB="${VEXDB_PG_BACKUP_PERF_SOURCE_DB:-vexfs_backup_perf_source}" +TARGET_DB="${VEXDB_PG_BACKUP_PERF_TARGET_DB:-vexfs_backup_perf_target}" +WORKSPACE="pg-backup-performance" +PAYLOAD_BYTES="${VEXDB_PG_BACKUP_PERF_BYTES:-16777216}" +FILE_COUNT="${VEXDB_PG_BACKUP_PERF_FILES:-16}" +PG_BIN="${VEXDB_PG_BIN:-/opt/pg19/bin}" +CLONE_PORT="${VEXDB_PG_BACKUP_PERF_CLONE_PORT:-55433}" +HOST_PORT="${VEXDB_PG_HOST_PORT:-}" +CLI="${VEXFS_EVAL_MOUNT_CLI:-$ROOT/vexdb_sqlite/build/vexdb}" +MAX_CLI_RSS_BYTES="${VEXDB_PG_BACKUP_PERF_MAX_CLI_RSS_BYTES:-536870912}" +MIN_LOGICAL_DUMP_MIB_S="${VEXDB_PG_BACKUP_PERF_MIN_LOGICAL_DUMP_MIB_S:-1.0}" +MIN_LOGICAL_RESTORE_MIB_S="${VEXDB_PG_BACKUP_PERF_MIN_LOGICAL_RESTORE_MIB_S:-0.5}" +MIN_PHYSICAL_MIB_S="${VEXDB_PG_BACKUP_PERF_MIN_PHYSICAL_MIB_S:-1.0}" +MIN_ARCHIVE_EXPORT_MIB_S="${VEXDB_PG_BACKUP_PERF_MIN_ARCHIVE_EXPORT_MIB_S:-0.5}" +MIN_ARCHIVE_IMPORT_MIB_S="${VEXDB_PG_BACKUP_PERF_MIN_ARCHIVE_IMPORT_MIB_S:-0.5}" + +TMP="$(mktemp -d "${TMPDIR:-/tmp}/vexfs-pg-backup-perf.XXXXXX")" +DUMP_PATH="/tmp/vexfs-backup-performance.dump" +BACKUP_DIR="/tmp/vexfs-pg-backup-performance" +ARCHIVE="$TMP/postgresql-format-v2.vexfs" +SQLITE_DB="$TMP/imported.sqlite3" +CHECKS=0 + +cleanup() { + local status=$? + trap - EXIT + docker exec "$CONTAINER" "$PG_BIN/pg_ctl" -D "$BACKUP_DIR" \ + -m immediate -w stop >/dev/null 2>&1 || true + docker exec "$CONTAINER" "$PG_BIN/dropdb" --if-exists --force "$TARGET_DB" \ + >/dev/null 2>&1 || true + docker exec "$CONTAINER" "$PG_BIN/dropdb" --if-exists --force "$SOURCE_DB" \ + >/dev/null 2>&1 || true + docker exec "$CONTAINER" rm -rf "$DUMP_PATH" "$BACKUP_DIR" \ + >/dev/null 2>&1 || true + rm -rf "$TMP" + exit "$status" +} +trap cleanup EXIT + +fail() { + echo "$*" >&2 + exit 1 +} + +assert_number_range() { + local value=$1 + local minimum=$2 + local maximum=$3 + local name=$4 + [[ "$value" =~ ^[0-9]+$ ]] || fail "$name 必须是整数" + (( value >= minimum && value <= maximum )) || \ + fail "$name 必须在 $minimum..$maximum" +} + +now_ms() { + perl -MTime::HiRes=time -e 'printf "%.0f\n", time() * 1000' +} + +elapsed_ms() { + local started=$1 + local ended + ended="$(now_ms)" + echo $((ended - started)) +} + +rate_mib_s() { + local bytes=$1 + local milliseconds=$2 + awk -v b="$bytes" -v ms="$milliseconds" \ + 'BEGIN { if (ms <= 0) print "0.000"; else printf "%.3f", b / 1048576.0 / (ms / 1000.0) }' +} + +assert_rate() { + local actual=$1 + local minimum=$2 + local label=$3 + awk -v a="$actual" -v m="$minimum" 'BEGIN { exit !(a + 0 >= m + 0) }' || \ + fail "$label 吞吐不足：实际 ${actual} MiB/s，最低 ${minimum} MiB/s" + CHECKS=$((CHECKS + 1)) +} + +timed_cli() { + local label=$1 + shift + local time_file="$TMP/${label}.time" + /usr/bin/time -l -o "$time_file" "$@" >/dev/null + local elapsed rss + elapsed="$(awk '/ real / { print int(($1 * 1000) + 0.5); exit }' "$time_file")" + rss="$(awk '/maximum resident set size/ { print $1; exit }' "$time_file")" + [[ "$elapsed" =~ ^[0-9]+$ ]] || fail "$label 没有得到耗时" + [[ "$rss" =~ ^[0-9]+$ ]] || fail "$label 没有得到峰值内存" + (( rss <= MAX_CLI_RSS_BYTES )) || \ + fail "$label 峰值内存过高：$rss bytes，限制 $MAX_CLI_RSS_BYTES bytes" + CHECKS=$((CHECKS + 1)) + printf '%s|%s\n' "$elapsed" "$rss" +} + +assert_number_range "$PAYLOAD_BYTES" 1048576 134217728 VEXDB_PG_BACKUP_PERF_BYTES +assert_number_range "$FILE_COUNT" 1 1024 VEXDB_PG_BACKUP_PERF_FILES +(( PAYLOAD_BYTES % FILE_COUNT == 0 )) || \ + fail "VEXDB_PG_BACKUP_PERF_BYTES 必须能被 VEXDB_PG_BACKUP_PERF_FILES 整除" +CHUNK_BYTES=$((PAYLOAD_BYTES / FILE_COUNT)) +(( CHUNK_BYTES <= 8388608 )) || fail "单文件不能超过 8 MiB，请增加文件数" + +[ -x "$CLI" ] || fail "找不到 vexdb CLI：$CLI" +docker inspect "$CONTAINER" >/dev/null +if [ -z "$HOST_PORT" ]; then + HOST_BINDING="$(docker port "$CONTAINER" 5432/tcp 2>/dev/null | sed -n '1p')" + HOST_PORT="${HOST_BINDING##*:}" + case "$HOST_PORT" in + ''|*[!0-9]*) + fail "无法从容器 $CONTAINER 推导 PostgreSQL 端口；请设置 VEXDB_PG_HOST_PORT" + ;; + esac +fi + +MEMORY_MAX="$(docker exec "$CONTAINER" cat /sys/fs/cgroup/memory.max)" +[[ "$MEMORY_MAX" =~ ^[0-9]+$ ]] || fail "PG 容器没有有限的 memory.max：$MEMORY_MAX" +(( MEMORY_MAX <= 1073741824 )) || \ + fail "PG 性能容器内存上限必须不大于 1 GiB，当前为 $MEMORY_MAX bytes" +OOM_BEFORE="$(docker exec "$CONTAINER" awk '$1=="oom_kill" {print $2}' /sys/fs/cgroup/memory.events)" +CHECKS=$((CHECKS + 2)) + +docker exec "$CONTAINER" "$PG_BIN/pg_ctl" -D "$BACKUP_DIR" \ + -m immediate -w stop >/dev/null 2>&1 || true +docker exec "$CONTAINER" rm -rf "$DUMP_PATH" "$BACKUP_DIR" >/dev/null 2>&1 || true +docker exec "$CONTAINER" "$PG_BIN/dropdb" --if-exists --force "$TARGET_DB" \ + >/dev/null 2>&1 || true +docker exec "$CONTAINER" "$PG_BIN/dropdb" --if-exists --force "$SOURCE_DB" \ + >/dev/null 2>&1 || true +docker exec "$CONTAINER" "$PG_BIN/createdb" "$SOURCE_DB" + +docker exec -i "$CONTAINER" "$PG_BIN/psql" -d "$SOURCE_DB" -X -q \ + -v ON_ERROR_STOP=1 -v file_count="$FILE_COUNT" -v chunk_bytes="$CHUNK_BYTES" \ + >/dev/null <<'SQL' +CREATE EXTENSION vexdb_lite VERSION '1.0'; +SELECT public.vexfs_workspace_create('pg-backup-performance'); +SELECT public.vexfs_mkdir('pg-backup-performance', '/payload', true); +SELECT count(public.vexfs_write( + 'pg-backup-performance', + '/payload/file-' || lpad(i::text, 4, '0') || '.bin', + decode(repeat(lpad(to_hex(i % 256), 2, '0'), :'chunk_bytes'::integer), 'hex'))) + FROM generate_series(1, :'file_count'::integer) AS g(i); +SELECT public.vexfs_snapshot_create('pg-backup-performance', 'performance-baseline'); +SQL + +SOURCE_STATE="$(docker exec "$CONTAINER" "$PG_BIN/psql" -d "$SOURCE_DB" \ + -X -q -t -A -F '|' -v ON_ERROR_STOP=1 -c \ + "SELECT sum((public.vexfs_stat('$WORKSPACE', '/payload/' || name)->>'size')::bigint), + count(*), + (public.vexfs_check('$WORKSPACE',1)->>'ok')::boolean, + (SELECT count(*) FROM public.vexfs_snapshot_list('$WORKSPACE')) + FROM public.vexfs_list('$WORKSPACE','/payload');")" +[[ "$SOURCE_STATE" == "$PAYLOAD_BYTES|$FILE_COUNT|t|1" ]] || \ + fail "PG 性能数据源状态错误：$SOURCE_STATE" +CHECKS=$((CHECKS + 4)) + +STARTED="$(now_ms)" +docker exec "$CONTAINER" "$PG_BIN/pg_dump" --format=custom \ + --file="$DUMP_PATH" "$SOURCE_DB" +LOGICAL_DUMP_MS="$(elapsed_ms "$STARTED")" +LOGICAL_DUMP_RATE="$(rate_mib_s "$PAYLOAD_BYTES" "$LOGICAL_DUMP_MS")" +assert_rate "$LOGICAL_DUMP_RATE" "$MIN_LOGICAL_DUMP_MIB_S" "pg_dump" + +docker exec "$CONTAINER" "$PG_BIN/createdb" "$TARGET_DB" +STARTED="$(now_ms)" +docker exec "$CONTAINER" "$PG_BIN/pg_restore" --exit-on-error \ + --single-transaction --dbname="$TARGET_DB" "$DUMP_PATH" +LOGICAL_RESTORE_MS="$(elapsed_ms "$STARTED")" +LOGICAL_RESTORE_RATE="$(rate_mib_s "$PAYLOAD_BYTES" "$LOGICAL_RESTORE_MS")" +assert_rate "$LOGICAL_RESTORE_RATE" "$MIN_LOGICAL_RESTORE_MIB_S" "pg_restore" + +LOGICAL_STATE="$(docker exec "$CONTAINER" "$PG_BIN/psql" -d "$TARGET_DB" \ + -X -q -t -A -F '|' -v ON_ERROR_STOP=1 -c \ + "SELECT sum((public.vexfs_stat('$WORKSPACE', '/payload/' || name)->>'size')::bigint), + count(*), + (public.vexfs_check('$WORKSPACE',1)->>'ok')::boolean + FROM public.vexfs_list('$WORKSPACE','/payload');")" +[[ "$LOGICAL_STATE" == "$PAYLOAD_BYTES|$FILE_COUNT|t" ]] || \ + fail "逻辑恢复后的文件状态错误：$LOGICAL_STATE" +CHECKS=$((CHECKS + 3)) +docker exec "$CONTAINER" "$PG_BIN/dropdb" --force "$TARGET_DB" >/dev/null + +STARTED="$(now_ms)" +docker exec "$CONTAINER" "$PG_BIN/pg_basebackup" -D "$BACKUP_DIR" \ + -U postgres --checkpoint=fast --wal-method=stream --no-slot +PHYSICAL_MS="$(elapsed_ms "$STARTED")" +PHYSICAL_BYTES="$(( $(docker exec "$CONTAINER" du -sk "$BACKUP_DIR" | awk '{print $1}') * 1024 ))" +PHYSICAL_RATE="$(rate_mib_s "$PHYSICAL_BYTES" "$PHYSICAL_MS")" +assert_rate "$PHYSICAL_RATE" "$MIN_PHYSICAL_MIB_S" "pg_basebackup" +docker exec "$CONTAINER" "$PG_BIN/pg_verifybackup" "$BACKUP_DIR" >/dev/null +CHECKS=$((CHECKS + 1)) + +docker exec "$CONTAINER" "$PG_BIN/pg_ctl" -D "$BACKUP_DIR" \ + -o "-p $CLONE_PORT -k /tmp -c listen_addresses=''" \ + -l "$BACKUP_DIR/clone.log" -w start >/dev/null +PHYSICAL_STATE="$(docker exec "$CONTAINER" "$PG_BIN/psql" -h /tmp -p "$CLONE_PORT" \ + -d "$SOURCE_DB" -X -q -t -A -F '|' -v ON_ERROR_STOP=1 -c \ + "SELECT sum((public.vexfs_stat('$WORKSPACE', '/payload/' || name)->>'size')::bigint), + count(*), + (public.vexfs_check('$WORKSPACE',1)->>'ok')::boolean + FROM public.vexfs_list('$WORKSPACE','/payload');")" +[[ "$PHYSICAL_STATE" == "$PAYLOAD_BYTES|$FILE_COUNT|t" ]] || \ + fail "物理恢复后的文件状态错误：$PHYSICAL_STATE" +CHECKS=$((CHECKS + 3)) +docker exec "$CONTAINER" "$PG_BIN/pg_ctl" -D "$BACKUP_DIR" \ + -m fast -w stop >/dev/null + +PG_DSN="${VEXDB_PG_DSN:-postgresql://postgres@127.0.0.1:${HOST_PORT}/${SOURCE_DB}}" +PG_CLI=("$CLI" fs --backend pg --dsn "$PG_DSN" --workspace "$WORKSPACE") +IFS='|' read -r ARCHIVE_EXPORT_MS ARCHIVE_EXPORT_RSS < <( + timed_cli archive-export "${PG_CLI[@]}" export --output "$ARCHIVE") +ARCHIVE_EXPORT_RATE="$(rate_mib_s "$PAYLOAD_BYTES" "$ARCHIVE_EXPORT_MS")" +assert_rate "$ARCHIVE_EXPORT_RATE" "$MIN_ARCHIVE_EXPORT_MIB_S" "format v2 导出" +ARCHIVE_VERIFY="$("${PG_CLI[@]}" archive verify "$ARCHIVE")" +[[ "$ARCHIVE_VERIFY" == *'"ok":true'* ]] || fail "format v2 包校验失败：$ARCHIVE_VERIFY" +CHECKS=$((CHECKS + 1)) + +SQLITE_CLI=("$CLI" fs --db "$SQLITE_DB" --workspace pg-backup-performance-import) +IFS='|' read -r ARCHIVE_IMPORT_MS ARCHIVE_IMPORT_RSS < <( + timed_cli archive-import "${SQLITE_CLI[@]}" import "$ARCHIVE") +ARCHIVE_IMPORT_RATE="$(rate_mib_s "$PAYLOAD_BYTES" "$ARCHIVE_IMPORT_MS")" +assert_rate "$ARCHIVE_IMPORT_RATE" "$MIN_ARCHIVE_IMPORT_MIB_S" "format v2 导入" + +SQLITE_STATE="$("$CLI" "$SQLITE_DB" \ + "SELECT sum(json_extract(value,'$.size')), + count(*), + json_extract(vexfs_check('pg-backup-performance-import',1),'$.ok') + FROM json_each(vexfs_list('pg-backup-performance-import','/payload'));")" +[[ "$SQLITE_STATE" == "$PAYLOAD_BYTES|$FILE_COUNT|1" ]] || \ + fail "format v2 导入 SQLite 后的文件状态错误：$SQLITE_STATE" +CHECKS=$((CHECKS + 3)) + +OOM_AFTER="$(docker exec "$CONTAINER" awk '$1=="oom_kill" {print $2}' /sys/fs/cgroup/memory.events)" +[[ "$OOM_AFTER" == "$OOM_BEFORE" ]] || \ + fail "性能测试触发了容器 OOM：测试前 $OOM_BEFORE，测试后 $OOM_AFTER" +CHECKS=$((CHECKS + 1)) + +echo "VEXFS PG BACKUP PERFORMANCE: PASS ($CHECKS checks)" +echo " payload_bytes=$PAYLOAD_BYTES files=$FILE_COUNT container_memory_max=$MEMORY_MAX oom_kill=$OOM_AFTER" +echo " logical_dump_ms=$LOGICAL_DUMP_MS logical_dump_mib_s=$LOGICAL_DUMP_RATE" +echo " logical_restore_ms=$LOGICAL_RESTORE_MS logical_restore_mib_s=$LOGICAL_RESTORE_RATE" +echo " physical_backup_bytes=$PHYSICAL_BYTES physical_backup_ms=$PHYSICAL_MS physical_backup_mib_s=$PHYSICAL_RATE" +echo " archive_export_ms=$ARCHIVE_EXPORT_MS archive_export_mib_s=$ARCHIVE_EXPORT_RATE archive_export_rss_bytes=$ARCHIVE_EXPORT_RSS" +echo " archive_import_ms=$ARCHIVE_IMPORT_MS archive_import_mib_s=$ARCHIVE_IMPORT_RATE archive_import_rss_bytes=$ARCHIVE_IMPORT_RSS" diff --git a/tests/eval/vexfs/run_pg_backup_restore.sh b/tests/eval/vexfs/run_pg_backup_restore.sh new file mode 100755 index 0000000000..42ed629705 --- /dev/null +++ b/tests/eval/vexfs/run_pg_backup_restore.sh @@ -0,0 +1,217 @@ +#!/usr/bin/env bash +set -euo pipefail + +CONTAINER="${VEXDB_PG_CONTAINER:-vexdb_pg19-test}" +SOURCE_DB="${VEXDB_PG_BACKUP_SOURCE_DB:-vexfs_backup_source_eval}" +TARGET_DB="${VEXDB_PG_BACKUP_TARGET_DB:-vexfs_backup_target_eval}" +DUMP_PATH="/tmp/vexfs-backup-restore-eval.dump" +CHECKS=0 + +cleanup() { + docker exec "$CONTAINER" dropdb --if-exists --force "$TARGET_DB" >/dev/null 2>&1 || true + docker exec "$CONTAINER" dropdb --if-exists --force "$SOURCE_DB" >/dev/null 2>&1 || true + docker exec "$CONTAINER" rm -f "$DUMP_PATH" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +docker inspect "$CONTAINER" >/dev/null +cleanup + +DEFAULT_VERSION="$(docker exec "$CONTAINER" psql -d postgres -X -q -t -A \ + -v ON_ERROR_STOP=1 -c \ + "SELECT default_version FROM pg_available_extensions WHERE name='vexdb_lite';")" +if [[ "$DEFAULT_VERSION" != "1.0" ]]; then + echo "PG 扩展默认版本错误：期望 1.0，实际 ${DEFAULT_VERSION:-missing}。" >&2 + echo "pg_dump 恢复时按默认版本安装扩展，控制文件与 SQL 版本必须一起发布。" >&2 + exit 1 +fi + +docker exec "$CONTAINER" createdb "$SOURCE_DB" + +docker exec -i "$CONTAINER" psql -d "$SOURCE_DB" -X -q -v ON_ERROR_STOP=1 \ + >/dev/null <<'SQL' +CREATE EXTENSION pg_trgm; +CREATE EXTENSION vexdb_lite VERSION '1.0'; +CREATE TABLE public.vexfs_backup_business( + id integer PRIMARY KEY, + checkpoint text NOT NULL +); +INSERT INTO public.vexfs_backup_business VALUES (1, 'same-transaction-boundary'); +SELECT public.vexfs_workspace_create('backup-source'); +SELECT public.vexfs_write( + 'backup-source', + '/workspace.bin', + convert_to(repeat('a', 70000) || '-before', 'UTF8')); +SELECT public.vexfs_mkdir('backup-source', '/links', true); +SELECT public.vexfs_link( + 'backup-source', '/workspace.bin', '/links/workspace-hard.bin'); +SELECT public.vexfs_symlink( + 'backup-source', '/links/current', convert_to('../workspace.bin', 'UTF8')); +SELECT public.vexfs_set_mode( + 'backup-source', + (public.vexfs_stat('backup-source', '/workspace.bin')->>'inode')::bigint, + 493); +SELECT public.vexfs_chown( + 'backup-source', + (public.vexfs_stat('backup-source', '/workspace.bin')->>'inode')::bigint, + 501, + 20); +SELECT public.vexfs_xattr_set( + 'backup-source', + (public.vexfs_stat('backup-source', '/workspace.bin')->>'inode')::bigint, + 'user.backup', + convert_to('metadata-survives', 'UTF8'), + 1); +SELECT public.vexfs_acl_grant( + 'backup-source', + (public.vexfs_stat('backup-source', '/workspace.bin')->>'inode')::bigint, + 'public', + 'read', + 'allow', + 0); +SELECT public.vexfs_snapshot_create('backup-source', 'baseline'); +SELECT public.vexfs_write( + 'backup-source', + '/workspace.bin', + convert_to(repeat('b', 70000) || '-after', 'UTF8')); +SELECT public.vexfs_quota_set('backup-source', 200000, 10, 100000); +SELECT public.vexfs_retention_set('backup-source', 1, 0); +SELECT public.vexfs_grep_index('backup-source', 'enable'); +SQL + +SOURCE_STATE="$(docker exec "$CONTAINER" psql -d "$SOURCE_DB" -X -q -t -A -F '|' \ + -v ON_ERROR_STOP=1 -c \ + "SELECT (public.vexfs_check('backup-source',1)->>'ok')::boolean, + (SELECT count(*) FROM _vexfs.manifests), + (SELECT count(*) FROM _vexfs.chunks), + (public.vexfs_workspace_stat('backup-source')->>'head_commit')::bigint, + (public.vexfs_grep_index('backup-source','status')->>'enabled')::boolean, + (public.vexfs_grep_index('backup-source','status')->>'dirty')::boolean, + (public.vexfs_grep_index('backup-source','status')->>'indexed_files')::integer, + (public.vexfs_grep('backup-source','/','after',0,10)->>'index_used')::boolean;")" +if [[ "$SOURCE_STATE" != "t|3|5|10|t|f|1|t" ]]; then + echo "PG 备份源状态错误：$SOURCE_STATE" >&2 + exit 1 +fi +CHECKS=$((CHECKS + 8)) + +docker exec "$CONTAINER" pg_dump \ + --format=custom \ + --file="$DUMP_PATH" \ + "$SOURCE_DB" +docker exec "$CONTAINER" createdb "$TARGET_DB" +docker exec "$CONTAINER" pg_restore \ + --exit-on-error \ + --single-transaction \ + --dbname="$TARGET_DB" \ + "$DUMP_PATH" + +TARGET_STATE="$(docker exec "$CONTAINER" psql -d "$TARGET_DB" -X -q -t -A -F '|' \ + -v ON_ERROR_STOP=1 -c \ + "SELECT public.vexfs_pg_adapter_version(), + public.vexfs_read('backup-source','/workspace.bin') = + convert_to(repeat('b',70000) || '-after','UTF8'), + public.vexfs_read_version('backup-source','/workspace.bin',1) = + convert_to(repeat('a',70000) || '-before','UTF8'), + (SELECT count(*) FROM public.vexfs_snapshot_list('backup-source')), + (public.vexfs_quota_get('backup-source')->>'max_bytes')::bigint, + (public.vexfs_retention_get('backup-source')->>'keep_versions')::integer, + (SELECT checkpoint FROM public.vexfs_backup_business WHERE id=1), + (public.vexfs_check('backup-source',1)->>'ok')::boolean, + (public.vexfs_stat('backup-source','/workspace.bin')->>'inode') = + (public.vexfs_stat('backup-source','/links/workspace-hard.bin')->>'inode'), + convert_from(public.vexfs_readlink( + 'backup-source', + (public.vexfs_stat('backup-source','/links/current')->>'inode')::bigint), + 'UTF8'), + convert_from(public.vexfs_xattr_get( + 'backup-source', + (public.vexfs_stat('backup-source','/workspace.bin')->>'inode')::bigint, + 'user.backup'), 'UTF8'), + public.vexfs_stat('backup-source','/workspace.bin')->>'mode', + public.vexfs_stat('backup-source','/workspace.bin')->>'uid', + public.vexfs_stat('backup-source','/workspace.bin')->>'gid', + (public.vexfs_acl_get( + 'backup-source', + (public.vexfs_stat('backup-source','/workspace.bin')->>'inode')::bigint) + @> '[{\"principal\":\"public\",\"effect\":\"allow\",\"permissions\":\"read\"}]'::jsonb), + (SELECT count(*) >= 9 FROM public.vexfs_audit_list( + 'backup-source', 1000, NULL)), + (SELECT bool_and(actor IS NOT NULL AND path IS NOT NULL + AND inode_id IS NOT NULL + AND details ? 'before_version' + AND details ? 'after_version') + FROM public.vexfs_audit_list('backup-source', 1000, NULL)), + (SELECT path FROM public.vexfs_audit_list('backup-source', 1000, NULL) + WHERE operation='write' ORDER BY event_id DESC LIMIT 1), + (SELECT details->>'before_version' FROM public.vexfs_audit_list( + 'backup-source', 1000, NULL) + WHERE operation='write' ORDER BY event_id DESC LIMIT 1), + (SELECT details->>'after_version' FROM public.vexfs_audit_list( + 'backup-source', 1000, NULL) + WHERE operation='write' ORDER BY event_id DESC LIMIT 1), + (SELECT actor::text FROM public.vexfs_audit_list( + 'backup-source', 1000, NULL) + WHERE operation='write' ORDER BY event_id DESC LIMIT 1), + (public.vexfs_grep_index('backup-source','status')->>'enabled')::boolean, + (public.vexfs_grep_index('backup-source','status')->>'dirty')::boolean, + (public.vexfs_grep_index('backup-source','status')->>'indexed_files')::integer, + (public.vexfs_grep('backup-source','/','after',0,10)->>'index_used')::boolean, + (public.vexfs_grep('backup-source','/','after',0,10)->>'match_count')::integer;")" +if [[ "$TARGET_STATE" != \ + "0.4.0-alpha.1|t|t|1|200000|1|same-transaction-boundary|t|t|../workspace.bin|metadata-survives|493|501|20|t|t|t|/workspace.bin|1|2|postgres|t|t|0|f|2" ]]; then + echo "PG 恢复后状态错误：$TARGET_STATE" >&2 + exit 1 +fi +CHECKS=$((CHECKS + 26)) + +INDEX_REBUILD_ACTION="$(docker exec "$CONTAINER" psql -d "$TARGET_DB" -X -q -t -A \ + -v ON_ERROR_STOP=1 -c \ + "SELECT (public.vexfs_grep_index('backup-source','rebuild')->>'dirty')::boolean;")" +if [[ "$INDEX_REBUILD_ACTION" != "f" ]]; then + echo "PG 恢复后索引重建动作错误：$INDEX_REBUILD_ACTION" >&2 + exit 1 +fi +INDEX_REBUILD_STATE="$(docker exec "$CONTAINER" psql -d "$TARGET_DB" -X -q -t -A -F '|' \ + -v ON_ERROR_STOP=1 -c \ + "SELECT (public.vexfs_grep_index('backup-source','status')->>'indexed_files')::integer, + (public.vexfs_grep('backup-source','/','after',0,10)->>'index_used')::boolean;")" +if [[ "$INDEX_REBUILD_STATE" != "1|t" ]]; then + echo "PG 恢复后索引重建状态错误：$INDEX_REBUILD_STATE" >&2 + exit 1 +fi +CHECKS=$((CHECKS + 3)) + +RESTORE_STATE="$(docker exec "$CONTAINER" psql -d "$TARGET_DB" -X -q -t -A -F '|' \ + -v ON_ERROR_STOP=1 -c \ + "SELECT public.vexfs_snapshot_restore('backup-source','baseline',10); + SELECT (public.vexfs_workspace_stat('backup-source')->>'head_commit')::bigint, + public.vexfs_read('backup-source','/workspace.bin') = + convert_to(repeat('a',70000) || '-before','UTF8'), + (public.vexfs_check('backup-source',1)->>'ok')::boolean, + (public.vexfs_stat('backup-source','/workspace.bin')->>'inode') = + (public.vexfs_stat('backup-source','/links/workspace-hard.bin')->>'inode'), + convert_from(public.vexfs_xattr_get( + 'backup-source', + (public.vexfs_stat('backup-source','/workspace.bin')->>'inode')::bigint, + 'user.backup'), 'UTF8'), + (public.vexfs_grep('backup-source','/','before',0,10)->>'index_used')::boolean, + (public.vexfs_grep('backup-source','/','before',0,10)->>'match_count')::integer;")" +RESTORE_STATE="$(printf '%s\n' "$RESTORE_STATE" | tail -n 1)" +if [[ "$RESTORE_STATE" != "11|t|t|t|metadata-survives|t|2" ]]; then + echo "PG 恢复库快照状态错误：$RESTORE_STATE" >&2 + exit 1 +fi +CHECKS=$((CHECKS + 7)) + +IDENTITY_STATE="$(docker exec "$CONTAINER" psql -d "$TARGET_DB" -X -q -t -A \ + -v ON_ERROR_STOP=1 -c \ + "SELECT public.vexfs_workspace_create('after-restore') > + (SELECT workspace_id FROM _vexfs.workspaces WHERE name='backup-source');")" +if [[ "$IDENTITY_STATE" != "t" ]]; then + echo "PG 恢复后 identity sequence 没有继续递增：$IDENTITY_STATE" >&2 + exit 1 +fi +CHECKS=$((CHECKS + 1)) + +echo "VEXFS PG BACKUP RESTORE: PASS ($CHECKS checks)" diff --git a/tests/eval/vexfs/run_pg_cross_gateway.sh b/tests/eval/vexfs/run_pg_cross_gateway.sh new file mode 100755 index 0000000000..4b597e3418 --- /dev/null +++ b/tests/eval/vexfs/run_pg_cross_gateway.sh @@ -0,0 +1,75 @@ +#!/bin/bash +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" +PYTHON="$ROOT/tests/eval/vexfs/python.sh" +IMAGE="${VEXFS_LINUX_EVAL_IMAGE:-vexdb-lite-vexfs-linux-eval:debian}" +PG_CONTAINER="${VEXDB_PG_CONTAINER:-vexdb_pg19-test}" +DSN="${VEXDB_PG_DSN:-postgresql://postgres@127.0.0.1:5433/test}" +MOUNT_CLI="${VEXFS_EVAL_MOUNT_CLI:-$HOME/.local/bin/vexdb}" +MEMORY="${VEXFS_DOCKER_MEMORY:-1g}" +MEMORY_SWAP="${VEXFS_DOCKER_MEMORY_SWAP:-1g}" +BUILD_VOLUME="${VEXFS_PG_CROSS_GATEWAY_BUILD_VOLUME:-vexfs-pg-cross-gateway-build}" + +[ "$(uname -s)" = Darwin ] || { echo "跨网关编排需要从 macOS 运行" >&2; exit 2; } +[ -x "$MOUNT_CLI" ] || { echo "找不到已安装的签名 CLI：$MOUNT_CLI" >&2; exit 2; } +docker inspect "$PG_CONTAINER" >/dev/null + +drop_workspace() { + docker exec "$PG_CONTAINER" psql -U postgres -d test -X -q \ + -c "SELECT vexfs_workspace_drop('pg-cross-gateway', true);" \ + >/dev/null 2>&1 || true +} + +cleanup() { + local status=$? + trap - EXIT INT TERM + drop_workspace + exit "$status" +} +trap cleanup EXIT INT TERM +drop_workspace + +docker build \ + --file "$ROOT/tests/eval/vexfs/Dockerfile.linux-fuse" \ + --tag "$IMAGE" \ + "$ROOT/tests/eval/vexfs" + +run_mac_phase() { + local phase=$1 + VEXFS_MOUNT_BACKEND=postgresql \ + VEXFS_MOUNT_DSN="$DSN" \ + VEXFS_PG_CROSS_GATEWAY_PHASE="$phase" \ + "$PYTHON" "$ROOT/tests/eval/vexfs/run.py" \ + --root "$ROOT" \ + --build-dir "$ROOT/vexdb_sqlite/build" \ + --output-dir "$ROOT/vexdb_sqlite/build/eval/vexfs-pg-cross-gateway/macos" \ + --mode quick \ + --mount-cli "$MOUNT_CLI" \ + --fail-on-skip \ + --filter portability.pg-cross-gateway +} + +run_linux_phase() { + local phase=$1 + docker run --rm \ + --memory "$MEMORY" \ + --memory-swap "$MEMORY_SWAP" \ + --network "container:$PG_CONTAINER" \ + --cap-add SYS_ADMIN \ + --device /dev/fuse \ + --security-opt apparmor=unconfined \ + --env "VEXFS_PG_CROSS_GATEWAY_PHASE=$phase" \ + --volume "$ROOT:/workspace" \ + --volume "$BUILD_VOLUME:/vexfs-build" \ + --workdir /workspace \ + "$IMAGE" \ + /bin/bash /workspace/tests/eval/vexfs/run_pg_cross_gateway_linux_in_container.sh +} + +run_mac_phase mac-create +run_linux_phase linux-modify +run_mac_phase mac-verify +run_linux_phase linux-final + +echo "VEXFS PG CROSS GATEWAY ROUNDTRIP: PASS" diff --git a/tests/eval/vexfs/run_pg_cross_gateway_linux_in_container.sh b/tests/eval/vexfs/run_pg_cross_gateway_linux_in_container.sh new file mode 100755 index 0000000000..122f7a4170 --- /dev/null +++ b/tests/eval/vexfs/run_pg_cross_gateway_linux_in_container.sh @@ -0,0 +1,26 @@ +#!/bin/bash +set -euo pipefail + +BUILD_DIR=/vexfs-build +BUILD_LOG=/tmp/vexfs-pg-cross-gateway-build.log + +if ! cmake -S /workspace/vexdb_sqlite -B "$BUILD_DIR" \ + -DCMAKE_BUILD_TYPE=Release \ + -DVEXDB_SQLITE_BUILD_TESTS=ON >"$BUILD_LOG" 2>&1; then + tail -n 200 "$BUILD_LOG" + exit 1 +fi +if ! cmake --build "$BUILD_DIR" -j 1 >>"$BUILD_LOG" 2>&1; then + tail -n 200 "$BUILD_LOG" + exit 1 +fi + +VEXFS_MOUNT_BACKEND=postgresql \ +VEXFS_MOUNT_DSN=postgresql://postgres@127.0.0.1:5432/test \ + python3 /workspace/tests/eval/vexfs/run.py \ + --root /workspace \ + --build-dir "$BUILD_DIR" \ + --output-dir /workspace/vexdb_sqlite/build/eval/vexfs-pg-cross-gateway/linux \ + --mode quick \ + --fail-on-skip \ + --filter portability.pg-cross-gateway diff --git a/tests/eval/vexfs/run_pg_grep_performance.sh b/tests/eval/vexfs/run_pg_grep_performance.sh new file mode 100755 index 0000000000..74bb765497 --- /dev/null +++ b/tests/eval/vexfs/run_pg_grep_performance.sh @@ -0,0 +1,277 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" +CONTAINER="${VEXDB_PG_CONTAINER:-vexdb_pg19-test}" +DATABASE="${VEXDB_PG_DATABASE:-test}" +DSN="${VEXDB_PG_DSN:-postgresql://postgres@127.0.0.1:5433/test}" +CLI="${VEXDB_PG_CLI:-$ROOT/vexdb_sqlite/build/vexdb}" +RG="${VEXDB_PG_RG:-$(command -v rg || true)}" +WORKSPACE="${VEXDB_PG_GREP_WORKSPACE:-pg-grep-performance}" +FILE_COUNT="${VEXDB_PG_GREP_FILES:-5000}" +CONTENT_BYTES="${VEXDB_PG_GREP_CONTENT_BYTES:-2048}" +ITERATIONS="${VEXDB_PG_GREP_ITERATIONS:-5}" +MATCH_EVERY="${VEXDB_PG_GREP_MATCH_EVERY:-1000}" +PATTERN="${VEXDB_PG_GREP_PATTERN:-rare-vexfs-token-2026}" +KEEP_WORKSPACE="${VEXFS_KEEP_WORKSPACE:-0}" +RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)" +OUTPUT_DIR="${VEXFS_EVAL_OUTPUT_DIR:-$ROOT/vexdb_sqlite/build/eval/vexfs-pg-grep/$RUN_ID}" +TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/vexfs-pg-grep-perf.XXXXXX")" +MOUNT_POINT="$TMP_DIR/mount" +MOUNTED=false + +for numeric in "$FILE_COUNT" "$CONTENT_BYTES" "$ITERATIONS" "$MATCH_EVERY"; do + [[ "$numeric" =~ ^[0-9]+$ ]] || { echo "性能参数必须是正整数" >&2; exit 2; } +done +(( FILE_COUNT >= 100 && FILE_COUNT <= 10000 )) || { + echo "VEXDB_PG_GREP_FILES 必须在 100..10000" >&2; exit 2; +} +(( CONTENT_BYTES >= 128 && CONTENT_BYTES <= 8192 )) || { + echo "VEXDB_PG_GREP_CONTENT_BYTES 必须在 128..8192" >&2; exit 2; +} +(( ITERATIONS >= 3 && ITERATIONS <= 9 )) || { + echo "VEXDB_PG_GREP_ITERATIONS 必须在 3..9" >&2; exit 2; +} +(( MATCH_EVERY >= 1 && MATCH_EVERY <= FILE_COUNT )) || { + echo "VEXDB_PG_GREP_MATCH_EVERY 必须在 1..FILE_COUNT" >&2; exit 2; +} +case "$WORKSPACE" in + ''|*[!A-Za-z0-9_-]*) echo "workspace 只能包含字母、数字、下划线和连字符" >&2; exit 2 ;; +esac +case "$PATTERN" in + ''|*[!A-Za-z0-9_-]*) echo "pattern 只能包含字母、数字、下划线和连字符" >&2; exit 2 ;; +esac +[ -x "$CLI" ] || { echo "找不到 vexdb CLI：$CLI" >&2; exit 2; } +[ -x "$RG" ] || { echo "找不到 rg：$RG" >&2; exit 2; } +mkdir -p "$MOUNT_POINT" "$OUTPUT_DIR" + +vexfs() { + "$CLI" fs --backend pg --dsn "$DSN" --workspace "$WORKSPACE" "$@" +} + +drop_workspace() { + docker exec "$CONTAINER" psql -d "$DATABASE" -X -q -v ON_ERROR_STOP=1 \ + -c "SELECT vexfs_workspace_drop('$WORKSPACE', true);" >/dev/null 2>&1 || true +} + +cleanup() { + local status=$? + trap - EXIT INT TERM + cd / + if [ "$MOUNTED" = true ]; then + vexfs unmount --force "$MOUNT_POINT" >/dev/null 2>&1 || true + fi + if [ "$KEEP_WORKSPACE" != 1 ]; then + drop_workspace + fi + rm -rf "$TMP_DIR" + exit "$status" +} +trap cleanup EXIT INT TERM + +median() { + sort -n "$1" | awk ' + { value[NR]=$1 } + END { + if (NR % 2) result=value[(NR+1)/2]; + else result=(value[NR/2]+value[NR/2+1])/2; + printf "%.3f", result; + }' +} + +measure_cli_grep() { + local phase="$1" expected_index="$2" + local times="$TMP_DIR/$phase-times.txt" + local iteration started ended elapsed output + : >"$times" + for iteration in $(seq 1 "$ITERATIONS"); do + output="$TMP_DIR/$phase-$iteration.json" + started="$(/usr/bin/perl -MTime::HiRes=time -e 'printf "%.9f", time')" + /usr/bin/perl -e 'alarm 300; exec @ARGV' \ + "$CLI" fs --backend pg --dsn "$DSN" --workspace "$WORKSPACE" \ + --json grep "$PATTERN" /corpus >"$output" + ended="$(/usr/bin/perl -MTime::HiRes=time -e 'printf "%.9f", time')" + elapsed="$(awk -v start="$started" -v finish="$ended" \ + 'BEGIN { printf "%.3f", (finish-start)*1000 }')" + printf '%s\n' "$elapsed" >>"$times" + grep -Fq "\"index_used\":$expected_index" "$output" || { + echo "$phase 没有按预期报告 index_used=$expected_index" >&2; exit 1; + } + grep -Fq "\"match_count\":$EXPECTED_MATCHES" "$output" || { + echo "$phase 匹配数错误" >&2; sed -n '1p' "$output" >&2; exit 1; + } + done + median "$times" +} + +measure_native_rg() { + local times="$TMP_DIR/native-times.txt" iteration started ended elapsed output count + : >"$times" + for iteration in $(seq 1 "$ITERATIONS"); do + output="$TMP_DIR/native-$iteration.out" + started="$(/usr/bin/perl -MTime::HiRes=time -e 'printf "%.9f", time')" + /usr/bin/perl -e 'alarm 300; exec @ARGV' \ + "$RG" --threads 1 --no-ignore --fixed-strings --files-with-matches \ + "$PATTERN" "$MOUNT_POINT/corpus" >"$output" + ended="$(/usr/bin/perl -MTime::HiRes=time -e 'printf "%.9f", time')" + elapsed="$(awk -v start="$started" -v finish="$ended" \ + 'BEGIN { printf "%.3f", (finish-start)*1000 }')" + printf '%s\n' "$elapsed" >>"$times" + count="$(wc -l <"$output" | tr -d ' ')" + [ "$count" = "$EXPECTED_MATCHES" ] || { + echo "原生 rg 匹配文件数错误：期望 $EXPECTED_MATCHES，实际 $count" >&2; exit 1; + } + done + median "$times" +} + +docker inspect "$CONTAINER" >/dev/null +MEMORY_LIMIT="$(docker inspect "$CONTAINER" --format '{{.HostConfig.Memory}}')" +if (( MEMORY_LIMIT <= 0 || MEMORY_LIMIT > 1073741824 )); then + echo "PG grep 性能测试要求容器 memory.max 在 1 GiB 以内，实际 $MEMORY_LIMIT" >&2 + exit 1 +fi +OOM_BEFORE="$(docker exec "$CONTAINER" sh -lc \ + "awk '\$1==\"oom_kill\"{print \$2}' /sys/fs/cgroup/memory.events")" +drop_workspace +docker exec "$CONTAINER" psql -d "$DATABASE" -X -q -v ON_ERROR_STOP=1 \ + -c "CREATE EXTENSION IF NOT EXISTS pg_trgm;" >/dev/null + +CREATE_RESULT="$(docker exec -i "$CONTAINER" psql -d "$DATABASE" -X -q -t -A -F '|' \ + -v ON_ERROR_STOP=1 -v file_count="$FILE_COUNT" -v content_bytes="$CONTENT_BYTES" \ + -v match_every="$MATCH_EVERY" -v pattern="$PATTERN" -v workspace="$WORKSPACE" \ + <<'SQL' | tail -n 1 +SELECT set_config('vexfs.perf_files', :'file_count', false); +SELECT set_config('vexfs.perf_content_bytes', :'content_bytes', false); +SELECT set_config('vexfs.perf_match_every', :'match_every', false); +SELECT set_config('vexfs.perf_pattern', :'pattern', false); +SELECT set_config('vexfs.perf_workspace', :'workspace', false); +CREATE TEMP TABLE vexfs_grep_build(elapsed_ms double precision) ON COMMIT PRESERVE ROWS; +DO $$ +DECLARE + v_files integer := current_setting('vexfs.perf_files')::integer; + v_bytes integer := current_setting('vexfs.perf_content_bytes')::integer; + v_every integer := current_setting('vexfs.perf_match_every')::integer; + v_pattern text := current_setting('vexfs.perf_pattern'); + v_workspace text := current_setting('vexfs.perf_workspace'); + v_filler text; + v_content text; + v_started timestamptz; +BEGIN + PERFORM vexfs_workspace_create(v_workspace); + PERFORM vexfs_mkdir(v_workspace, '/corpus'); + v_filler := repeat('alpha beta gamma delta epsilon ', (v_bytes / 31) + 2); + v_started := clock_timestamp(); + FOR i IN 1..v_files LOOP + v_content := left(v_filler, v_bytes - 17) || ' ' || lpad(i::text, 16, '0'); + IF i % v_every = 0 THEN + v_content := overlay(v_content PLACING v_pattern FROM 33); + END IF; + PERFORM vexfs_write( + v_workspace, '/corpus/file-' || lpad(i::text, 6, '0') || '.txt', + convert_to(v_content, 'UTF8')); + END LOOP; + INSERT INTO vexfs_grep_build VALUES ( + extract(epoch FROM clock_timestamp() - v_started) * 1000.0); +END $$; +SELECT round(build.elapsed_ms::numeric,3), + (vexfs_workspace_stat(:'workspace')->>'head_commit')::bigint, + (SELECT count(*) FROM vexfs_list(:'workspace','/corpus')) + FROM vexfs_grep_build AS build; +SQL +)" +IFS='|' read -r CREATE_MS HEAD FILES <<<"$CREATE_RESULT" +EXPECTED_HEAD=$((FILE_COUNT + 2)) +EXPECTED_MATCHES=$((FILE_COUNT / MATCH_EVERY)) +[ "$HEAD" = "$EXPECTED_HEAD" ] && [ "$FILES" = "$FILE_COUNT" ] && \ + [ "$EXPECTED_MATCHES" -gt 0 ] || { + echo "PG grep 数据集状态错误：$CREATE_RESULT" >&2; exit 1; + } + +vexfs index disable >/dev/null +vexfs --json grep "$PATTERN" /corpus >/dev/null +SCAN_MEDIAN_MS="$(measure_cli_grep scan false)" + +BUILD_STARTED="$(/usr/bin/perl -MTime::HiRes=time -e 'printf "%.9f", time')" +vexfs index enable >"$TMP_DIR/index-enable.json" +BUILD_ENDED="$(/usr/bin/perl -MTime::HiRes=time -e 'printf "%.9f", time')" +INDEX_BUILD_MS="$(awk -v start="$BUILD_STARTED" -v finish="$BUILD_ENDED" \ + 'BEGIN { printf "%.3f", (finish-start)*1000 }')" +grep -Fq '"enabled":true' "$TMP_DIR/index-enable.json" +grep -Fq '"dirty":false' "$TMP_DIR/index-enable.json" +grep -Fq "\"indexed_files\":$FILE_COUNT" "$TMP_DIR/index-enable.json" + +PLAN="$(docker exec "$CONTAINER" psql -d "$DATABASE" -X -q -t -A \ + -v ON_ERROR_STOP=1 -c \ + "SET enable_seqscan=off; + EXPLAIN (COSTS OFF) SELECT inode_id FROM _vexfs.grep_documents + WHERE content LIKE '%$PATTERN%'; + RESET enable_seqscan;")" +printf '%s\n' "$PLAN" >"$OUTPUT_DIR/pg-plan.txt" +grep -Fq 'vexfs_grep_documents_content_trgm_idx' "$OUTPUT_DIR/pg-plan.txt" || { + echo "PostgreSQL 计划没有使用 trigram GIN" >&2 + cat "$OUTPUT_DIR/pg-plan.txt" >&2 + exit 1 +} + +vexfs --json grep "$PATTERN" /corpus >/dev/null +INDEX_MEDIAN_MS="$(measure_cli_grep index true)" + +vexfs mount "$MOUNT_POINT" >/dev/null +MOUNTED=true +"$RG" --threads 1 --no-ignore --fixed-strings --files-with-matches \ + "$PATTERN" "$MOUNT_POINT/corpus" >/dev/null +NATIVE_MEDIAN_MS="$(measure_native_rg)" +cd / +vexfs unmount "$MOUNT_POINT" >/dev/null +MOUNTED=false + +SCAN_SPEEDUP="$(awk -v scan="$SCAN_MEDIAN_MS" -v idx="$INDEX_MEDIAN_MS" \ + 'BEGIN { printf "%.3f", scan/idx }')" +NATIVE_SPEEDUP="$(awk -v native="$NATIVE_MEDIAN_MS" -v idx="$INDEX_MEDIAN_MS" \ + 'BEGIN { printf "%.3f", native/idx }')" +awk -v scan="$SCAN_MEDIAN_MS" -v idx="$INDEX_MEDIAN_MS" \ + 'BEGIN { exit !(idx < scan) }' || { + echo "PG trigram 没有快于 PG 扫描：index=$INDEX_MEDIAN_MS scan=$SCAN_MEDIAN_MS" >&2 + exit 1 + } +awk -v native="$NATIVE_MEDIAN_MS" -v idx="$INDEX_MEDIAN_MS" \ + 'BEGIN { exit !(idx < native) }' || { + echo "PG trigram 没有快于挂载目录原生 rg：index=$INDEX_MEDIAN_MS native=$NATIVE_MEDIAN_MS" >&2 + exit 1 + } + +INDEX_BYTES="$(docker exec "$CONTAINER" psql -d "$DATABASE" -X -q -t -A \ + -v ON_ERROR_STOP=1 -c \ + "SELECT pg_total_relation_size('_vexfs.grep_documents'::regclass);")" +MEMORY_CURRENT="$(docker exec "$CONTAINER" sh -lc 'cat /sys/fs/cgroup/memory.current')" +MEMORY_PEAK="$(docker exec "$CONTAINER" sh -lc 'cat /sys/fs/cgroup/memory.peak')" +OOM_AFTER="$(docker exec "$CONTAINER" sh -lc \ + "awk '\$1==\"oom_kill\"{print \$2}' /sys/fs/cgroup/memory.events")" +[ "$OOM_AFTER" = "$OOM_BEFORE" ] || { + echo "PG grep 性能测试触发 OOM kill：before=$OOM_BEFORE after=$OOM_AFTER" >&2; exit 1; +} + +cat >"$OUTPUT_DIR/report.json" <"$OUTPUT_DIR/report.md" </dev/null +docker build \ + --file "$ROOT/tests/eval/vexfs/Dockerfile.linux-fuse" \ + --tag "$IMAGE" \ + "$ROOT/tests/eval/vexfs" + +# 与 PG 容器共享网络命名空间，测试流量不会向局域网公开；VexFS 仍通过 +# TCP/libpq 连接 127.0.0.1:5432，等价于远端数据库的数据面。 +docker run --rm \ + --memory "$MEMORY" \ + --memory-swap "$MEMORY_SWAP" \ + --network "container:$PG_CONTAINER" \ + --cap-add SYS_ADMIN \ + --device /dev/fuse \ + --security-opt apparmor=unconfined \ + --env VEXFS_MOUNT_BACKEND=postgresql \ + --env VEXFS_MOUNT_DSN=postgresql://postgres@127.0.0.1:5432/test \ + --env "VEXFS_LINUX_EVAL_CASES=$CASES" \ + --env VEXFS_BUILD_JOBS=1 \ + --volume "$ROOT:/workspace" \ + --workdir /workspace \ + "$IMAGE" \ + /bin/bash /workspace/tests/eval/vexfs/run_linux_mount_in_container.sh diff --git a/tests/eval/vexfs/run_pg_macos_credentials.sh b/tests/eval/vexfs/run_pg_macos_credentials.sh new file mode 100755 index 0000000000..d44c6f7b43 --- /dev/null +++ b/tests/eval/vexfs/run_pg_macos_credentials.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +set -euo pipefail + +[ "$(uname -s)" = Darwin ] || { echo "该测试只支持 macOS" >&2; exit 2; } + +CLI="${VEXFS_EVAL_MOUNT_CLI:-$HOME/.local/bin/vexdb}" +PG_HOST="${VEXFS_PG_HOST:-127.0.0.1}" +PG_PORT="${VEXFS_PG_PORT:-5432}" +PG_DATABASE="${VEXFS_PG_DATABASE:-test}" +PG_USER="${VEXFS_PG_USER:-postgres}" +PG_PASSWORD="${VEXFS_PG_PASSWORD:?VEXFS_PG_PASSWORD is required}" +PG_CONTAINER="${VEXDB_PG_CONTAINER:-}" +WORKSPACE="${VEXFS_PG_WORKSPACE:-pg-macos-credentials}" +RESOURCE_ROOT="$HOME/Library/Application Support/VexDB-Lite/mount-resources" +PASSFILE="$(mktemp "${TMPDIR:-/tmp}/vexfs-pgpass.XXXXXX")" +MOUNT_POINT="$(mktemp -d "${TMPDIR:-/tmp}/vexfs-pg-credential-mount.XXXXXX")" +STALE_DIR="$RESOURCE_ROOT/stale-credential-eval-$$" +LOG_FILE="$(mktemp "${TMPDIR:-/tmp}/vexfs-pg-credential-log.XXXXXX")" +MOUNTED=false +CHECKS=0 + +[ -x "$CLI" ] || { echo "找不到 vexdb CLI：$CLI" >&2; exit 2; } +case "$PG_HOST$PG_PORT$PG_DATABASE$PG_USER$WORKSPACE" in + *[!A-Za-z0-9_.:-]*) echo "连接字段包含不支持的字符" >&2; exit 2 ;; +esac + +pass() { CHECKS=$((CHECKS + 1)); } +fail() { echo "$1" >&2; exit 1; } + +drop_workspace() { + [ -n "$PG_CONTAINER" ] || return 0 + docker exec "$PG_CONTAINER" /opt/pg19/bin/psql -U postgres -d "$PG_DATABASE" -X -q \ + -c "SELECT vexfs_workspace_drop('$WORKSPACE', true);" >/dev/null 2>&1 || true +} + +cleanup() { + local status=$? + trap - EXIT INT TERM + if [ "$MOUNTED" = true ]; then + "$CLI" fs --backend pg --dsn "$DSN" --workspace "$WORKSPACE" \ + unmount --force "$MOUNT_POINT" >/dev/null 2>&1 || true + fi + rm -f "$PASSFILE" "$PASSFILE.link" "$LOG_FILE" + rm -f "$STALE_DIR/.vexfs-pgpass" + rmdir "$STALE_DIR" "$MOUNT_POINT" >/dev/null 2>&1 || true + drop_workspace + exit "$status" +} +trap cleanup EXIT INT TERM + +umask 077 +printf '%s\n' "$PG_HOST:$PG_PORT:$PG_DATABASE:$PG_USER:$PG_PASSWORD" >"$PASSFILE" +chmod 0600 "$PASSFILE" +DSN="host=$PG_HOST port=$PG_PORT dbname=$PG_DATABASE user=$PG_USER passfile=$PASSFILE connect_timeout=8" + +drop_workspace +"$CLI" fs --backend pg --dsn "$DSN" --workspace "$WORKSPACE" setup >/dev/null + +# 过宽权限、符号链接和 DSN 明文密码都必须在调用 FSKit 前拒绝。 +chmod 0644 "$PASSFILE" +if "$CLI" fs --backend pg --dsn "$DSN" --workspace "$WORKSPACE" \ + mount "$MOUNT_POINT" >"$LOG_FILE" 2>&1; then + fail "0644 passfile 被错误接受" +fi +grep -q "0600" "$LOG_FILE" || fail "0644 passfile 错误信息不明确" +pass +chmod 0600 "$PASSFILE" + +ln -s "$PASSFILE" "$PASSFILE.link" +SYMLINK_DSN="host=$PG_HOST port=$PG_PORT dbname=$PG_DATABASE user=$PG_USER passfile=$PASSFILE.link connect_timeout=8" +if "$CLI" fs --backend pg --dsn "$SYMLINK_DSN" --workspace "$WORKSPACE" \ + mount "$MOUNT_POINT" >"$LOG_FILE" 2>&1; then + fail "符号链接 passfile 被错误接受" +fi +pass +rm -f "$PASSFILE.link" + +INLINE_DSN="postgresql://$PG_USER:$PG_PASSWORD@$PG_HOST:$PG_PORT/$PG_DATABASE" +if "$CLI" fs --backend pg --dsn "$INLINE_DSN" --workspace "$WORKSPACE" \ + mount "$MOUNT_POINT" >"$LOG_FILE" 2>&1; then + fail "DSN 明文密码被错误接受" +fi +grep -q "must not contain a password" "$LOG_FILE" || fail "明文密码错误信息不明确" +pass + +# 凭据副本只在挂载期间存在，权限固定为 0600。 +"$CLI" fs --backend pg --dsn "$DSN" --workspace "$WORKSPACE" mount "$MOUNT_POINT" >/dev/null +MOUNTED=true +printf '%s\n' 'credential-reload-ok' >"$MOUNT_POINT/reload.txt" +STAGED_COUNT="$(find "$RESOURCE_ROOT" -name .vexfs-pgpass -type f -print | wc -l | tr -d ' ')" +STAGED_PATH="$(find "$RESOURCE_ROOT" -name .vexfs-pgpass -type f -print -quit)" +[ "$STAGED_COUNT" -eq 1 ] || fail "挂载期间应恰好存在一个受保护凭据副本" +[ "$(stat -f '%Lp' "$STAGED_PATH")" = 600 ] || fail "凭据副本权限不是 0600" +pass +pass + +# 用新的 CLI 进程检查现有挂载，凭据不能只活在首次 mount 命令的内存里。 +"$CLI" fs --backend pg --dsn "$DSN" --workspace "$WORKSPACE" \ + --json mount status "$MOUNT_POINT" | grep -q "\"workspace\":\"$WORKSPACE\"" || \ + fail "新 CLI 进程无法识别 PostgreSQL 挂载" +[ "$(cat "$MOUNT_POINT/reload.txt")" = credential-reload-ok ] || \ + fail "新 CLI 进程下文件读取失败" +pass +pass + +"$CLI" fs --backend pg --dsn "$DSN" --workspace "$WORKSPACE" unmount "$MOUNT_POINT" >/dev/null +MOUNTED=false +if find "$RESOURCE_ROOT" -name .vexfs-pgpass -type f -print -quit | grep -q .; then + fail "最后一个挂载卸载后仍遗留 PostgreSQL 凭据" +fi +pass + +# 模拟 CLI 异常退出的遗留文件，doctor 必须清理未挂载资源中的凭据。 +mkdir -p "$STALE_DIR" +chmod 0700 "$STALE_DIR" +printf '%s\n' stale >"$STALE_DIR/.vexfs-pgpass" +chmod 0600 "$STALE_DIR/.vexfs-pgpass" +"$CLI" fs --json doctor >/dev/null 2>&1 || true +[ ! -e "$STALE_DIR/.vexfs-pgpass" ] || fail "doctor 没有清理未挂载的凭据" +pass + +echo "VEXFS PG MACOS CREDENTIAL LIFECYCLE: PASS ($CHECKS checks)" diff --git a/tests/eval/vexfs/run_pg_macos_mount.sh b/tests/eval/vexfs/run_pg_macos_mount.sh new file mode 100755 index 0000000000..09bc51bf18 --- /dev/null +++ b/tests/eval/vexfs/run_pg_macos_mount.sh @@ -0,0 +1,71 @@ +#!/bin/bash +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" +PYTHON="$ROOT/tests/eval/vexfs/python.sh" +PG_CONTAINER="${VEXDB_PG_CONTAINER:-vexdb_pg19-test}" +DSN="${VEXDB_PG_DSN:-postgresql://postgres@127.0.0.1:5433/test}" +MOUNT_CLI="${VEXFS_EVAL_MOUNT_CLI:-$HOME/.local/bin/vexdb}" +OUTPUT="${VEXFS_MACOS_PG_OUTPUT:-$ROOT/vexdb_sqlite/build/eval/vexfs-pg-macos-mount}" +CASES="${VEXFS_MACOS_PG_CASES:-mount.cross-platform-conformance mount.timestamps mount.concurrent-append mount.open-rename-unlink mount.read-only-open-lifecycle mount.process-locks mount.force-unmount mount.real-bash mount.posix-metadata mount.performance mount.git-workspace mount.real-toolchain-projects mount.scale-tree}" + +[ "$(uname -s)" = Darwin ] || { echo "该脚本只在 macOS 上运行" >&2; exit 2; } +[ -x "$MOUNT_CLI" ] || { echo "找不到已安装的签名 CLI：$MOUNT_CLI" >&2; exit 2; } +docker inspect "$PG_CONTAINER" >/dev/null + +cleanup_pg_workspaces() { + local workspace + for workspace in conformance timestamps append open-life read-life locks \ + force-unmount eval posix perf git-eval toolchains mount-scale opencode; do + docker exec "$PG_CONTAINER" psql -U postgres -d test -X -q \ + -v ON_ERROR_STOP=1 \ + -c "SELECT vexfs_workspace_drop('$workspace', true);" \ + >/dev/null 2>&1 || true + done +} + +cleanup_mounts() { + local mount_point + while IFS= read -r mount_point; do + [ -n "$mount_point" ] || continue + "$MOUNT_CLI" fs --backend pg --dsn "$DSN" --workspace conformance \ + unmount --force "$mount_point" >/dev/null 2>&1 || true + done < <("$MOUNT_CLI" fs --backend pg --dsn "$DSN" --workspace conformance \ + --json mount status 2>/dev/null | \ + "$PYTHON" -c 'import json,sys; [print(x.get("mount_point", "")) for x in json.load(sys.stdin)]' \ + 2>/dev/null || true) +} + +cleanup() { + local status=$? + trap - EXIT INT TERM + cleanup_mounts + cleanup_pg_workspaces + exit "$status" +} +trap cleanup EXIT INT TERM + +mkdir -p "$OUTPUT" +read -r -a mount_cases <<< "$CASES" +if [ "${VEXFS_EVAL_OPENCODE:-0}" = 1 ]; then + mount_cases+=(mount.real-opencode-project) +fi + +for test_case in "${mount_cases[@]}"; do + cleanup_mounts + cleanup_pg_workspaces + VEXFS_MOUNT_BACKEND=postgresql \ + VEXFS_MOUNT_DSN="$DSN" \ + VEXFS_EVAL_OPENCODE="${VEXFS_EVAL_OPENCODE:-0}" \ + VEXFS_EVAL_OPENCODE_MODEL="${VEXFS_EVAL_OPENCODE_MODEL:-openai/gpt-5.4-mini}" \ + "$PYTHON" "$ROOT/tests/eval/vexfs/run.py" \ + --root "$ROOT" \ + --build-dir "$ROOT/vexdb_sqlite/build" \ + --output-dir "$OUTPUT" \ + --mode quick \ + --mount-cli "$MOUNT_CLI" \ + --fail-on-skip \ + --filter "$test_case" +done + +echo "VEXFS PG MACOS MOUNT CONFORMANCE: PASS (${#mount_cases[@]} cases)" diff --git a/tests/eval/vexfs/run_pg_physical_backup_restore.sh b/tests/eval/vexfs/run_pg_physical_backup_restore.sh new file mode 100755 index 0000000000..5399259a84 --- /dev/null +++ b/tests/eval/vexfs/run_pg_physical_backup_restore.sh @@ -0,0 +1,168 @@ +#!/usr/bin/env bash +set -euo pipefail + +CONTAINER="${VEXDB_PG_CONTAINER:-vexdb_pg19-test}" +SOURCE_DB="${VEXDB_PG_PHYSICAL_SOURCE_DB:-vexfs_physical_source_eval}" +BACKUP_DIR="${VEXDB_PG_PHYSICAL_BACKUP_DIR:-/tmp/vexfs-pg-physical-backup}" +CLONE_PORT="${VEXDB_PG_PHYSICAL_CLONE_PORT:-55432}" +PG_BIN="${VEXDB_PG_BIN:-/opt/pg19/bin}" +CHECKS=0 + +cleanup() { + local status=$? + trap - EXIT + docker exec "$CONTAINER" "$PG_BIN/pg_ctl" -D "$BACKUP_DIR" \ + -m immediate -w stop >/dev/null 2>&1 || true + docker exec "$CONTAINER" "$PG_BIN/dropdb" --if-exists --force "$SOURCE_DB" \ + >/dev/null 2>&1 || true + docker exec "$CONTAINER" rm -rf "$BACKUP_DIR" >/dev/null 2>&1 || true + exit "$status" +} +trap cleanup EXIT + +assert_equal() { + local actual=$1 + local expected=$2 + local message=$3 + if [[ "$actual" != "$expected" ]]; then + echo "${message}：期望 ${expected}，实际 ${actual}" >&2 + exit 1 + fi + CHECKS=$((CHECKS + 1)) +} + +docker inspect "$CONTAINER" >/dev/null +docker exec "$CONTAINER" "$PG_BIN/pg_ctl" -D "$BACKUP_DIR" \ + -m immediate -w stop >/dev/null 2>&1 || true +docker exec "$CONTAINER" rm -rf "$BACKUP_DIR" >/dev/null 2>&1 || true +docker exec "$CONTAINER" "$PG_BIN/dropdb" --if-exists --force "$SOURCE_DB" \ + >/dev/null 2>&1 || true +docker exec "$CONTAINER" "$PG_BIN/createdb" "$SOURCE_DB" + +docker exec -i "$CONTAINER" "$PG_BIN/psql" -d "$SOURCE_DB" -X -q \ + -v ON_ERROR_STOP=1 >/dev/null <<'SQL' +CREATE EXTENSION pg_trgm; +CREATE EXTENSION vexdb_lite VERSION '1.0'; +CREATE TABLE public.physical_backup_business( + id integer PRIMARY KEY, + checkpoint text NOT NULL +); +INSERT INTO public.physical_backup_business VALUES (1, 'before-base-backup'); +SELECT public.vexfs_workspace_create('physical-source'); +SELECT public.vexfs_mkdir('physical-source', '/project', true); +SELECT public.vexfs_write( + 'physical-source', '/project/state.txt', convert_to('physical-v1', 'UTF8')); +SELECT public.vexfs_link( + 'physical-source', '/project/state.txt', '/project/state-hard.txt'); +SELECT public.vexfs_symlink( + 'physical-source', '/project/state-link.txt', convert_to('state.txt', 'UTF8')); +SELECT public.vexfs_set_mode( + 'physical-source', + (public.vexfs_stat('physical-source', '/project/state.txt')->>'inode')::bigint, + 488); +SELECT public.vexfs_xattr_set( + 'physical-source', + (public.vexfs_stat('physical-source', '/project/state.txt')->>'inode')::bigint, + 'user.physical', convert_to('base-backup-metadata', 'UTF8'), 1); +SELECT public.vexfs_acl_grant( + 'physical-source', + (public.vexfs_stat('physical-source', '/project/state.txt')->>'inode')::bigint, + 'public', 'read', 'allow', 0); +SELECT public.vexfs_snapshot_create('physical-source', 'baseline'); +SELECT public.vexfs_write( + 'physical-source', '/project/state.txt', convert_to('physical-v2', 'UTF8')); +SELECT public.vexfs_grep_index('physical-source', 'enable'); +SQL + +SOURCE_STATE="$(docker exec "$CONTAINER" "$PG_BIN/psql" -d "$SOURCE_DB" \ + -X -q -t -A -F '|' -v ON_ERROR_STOP=1 -c \ + "SELECT convert_from(public.vexfs_read('physical-source','/project/state.txt'),'UTF8'), + convert_from(public.vexfs_read_version( + 'physical-source','/project/state.txt',1),'UTF8'), + (SELECT count(*) FROM public.vexfs_snapshot_list('physical-source')), + (public.vexfs_check('physical-source',1)->>'ok')::boolean, + (SELECT checkpoint FROM public.physical_backup_business WHERE id=1), + (public.vexfs_grep('physical-source','/project','physical-v2',0,10)->>'index_used')::boolean, + (public.vexfs_grep_index('physical-source','status')->>'indexed_files')::integer;")" +assert_equal "$SOURCE_STATE" "physical-v2|physical-v1|1|t|before-base-backup|t|1" \ + "物理备份源状态错误" + +docker exec "$CONTAINER" "$PG_BIN/pg_basebackup" \ + -D "$BACKUP_DIR" \ + -U postgres \ + --checkpoint=fast \ + --wal-method=stream \ + --no-slot + +# 备份完成后再写入源库；恢复副本必须停在备份完成时的一致边界。 +docker exec "$CONTAINER" "$PG_BIN/psql" -d "$SOURCE_DB" -X -q \ + -v ON_ERROR_STOP=1 -c \ + "INSERT INTO public.physical_backup_business VALUES (2, 'after-base-backup'); + SELECT public.vexfs_write( + 'physical-source','/project/after-backup.txt',convert_to('too-new','UTF8'));" \ + >/dev/null + +docker exec "$CONTAINER" "$PG_BIN/pg_ctl" -D "$BACKUP_DIR" \ + -o "-p $CLONE_PORT -k /tmp -c listen_addresses=''" \ + -l "$BACKUP_DIR/clone.log" -w start >/dev/null + +CLONE_STATE="$(docker exec "$CONTAINER" "$PG_BIN/psql" \ + -h /tmp -p "$CLONE_PORT" -d "$SOURCE_DB" -X -q -t -A -F '|' \ + -v ON_ERROR_STOP=1 -c \ + "SELECT convert_from(public.vexfs_read('physical-source','/project/state.txt'),'UTF8'), + convert_from(public.vexfs_read_version( + 'physical-source','/project/state.txt',1),'UTF8'), + (SELECT count(*) FROM public.vexfs_snapshot_list('physical-source')), + (public.vexfs_check('physical-source',1)->>'ok')::boolean, + (SELECT checkpoint FROM public.physical_backup_business WHERE id=1), + NOT EXISTS(SELECT 1 FROM public.physical_backup_business WHERE id=2), + public.vexfs_stat('physical-source','/project/state.txt')->>'mode', + (public.vexfs_stat('physical-source','/project/state.txt')->>'inode') = + (public.vexfs_stat('physical-source','/project/state-hard.txt')->>'inode'), + convert_from(public.vexfs_readlink( + 'physical-source', + (public.vexfs_stat('physical-source','/project/state-link.txt')->>'inode')::bigint), + 'UTF8'), + convert_from(public.vexfs_xattr_get( + 'physical-source', + (public.vexfs_stat('physical-source','/project/state.txt')->>'inode')::bigint, + 'user.physical'), 'UTF8'), + (public.vexfs_acl_get( + 'physical-source', + (public.vexfs_stat('physical-source','/project/state.txt')->>'inode')::bigint) + @> '[{\"principal\":\"public\",\"effect\":\"allow\",\"permissions\":\"read\"}]'::jsonb), + NOT EXISTS( + SELECT 1 + FROM _vexfs.dentries AS d + JOIN _vexfs.workspaces AS w USING (workspace_id) + WHERE w.name='physical-source' AND d.name='after-backup.txt'), + (public.vexfs_grep_index('physical-source','status')->>'enabled')::boolean, + (public.vexfs_grep_index('physical-source','status')->>'dirty')::boolean, + (public.vexfs_grep('physical-source','/project','physical-v2',0,10)->>'index_used')::boolean, + (public.vexfs_grep('physical-source','/project','physical-v2',0,10)->>'match_count')::integer;")" +assert_equal "$CLONE_STATE" \ + "physical-v2|physical-v1|1|t|before-base-backup|t|488|t|state.txt|base-backup-metadata|t|t|t|f|t|2" \ + "物理备份恢复状态错误" + +RESTORE_STATE="$(docker exec "$CONTAINER" "$PG_BIN/psql" \ + -h /tmp -p "$CLONE_PORT" -d "$SOURCE_DB" -X -q -t -A -F '|' \ + -v ON_ERROR_STOP=1 -c \ + "SELECT public.vexfs_snapshot_restore( + 'physical-source', 'baseline', + (public.vexfs_workspace_stat('physical-source')->>'head_commit')::bigint); + SELECT convert_from(public.vexfs_read( + 'physical-source','/project/state.txt'),'UTF8'), + (public.vexfs_check('physical-source',1)->>'ok')::boolean, + (public.vexfs_stat('physical-source','/project/state.txt')->>'inode') = + (public.vexfs_stat('physical-source','/project/state-hard.txt')->>'inode'), + convert_from(public.vexfs_xattr_get( + 'physical-source', + (public.vexfs_stat('physical-source','/project/state.txt')->>'inode')::bigint, + 'user.physical'),'UTF8'), + (public.vexfs_grep('physical-source','/project','physical-v1',0,10)->>'index_used')::boolean, + (public.vexfs_grep('physical-source','/project','physical-v1',0,10)->>'match_count')::integer;")" +RESTORE_STATE="$(printf '%s\n' "$RESTORE_STATE" | tail -n 1)" +assert_equal "$RESTORE_STATE" "physical-v1|t|t|base-backup-metadata|t|2" \ + "物理恢复副本的快照还原错误" + +echo "VEXFS PG PHYSICAL BACKUP RESTORE: PASS ($CHECKS groups, 29 fields)" diff --git a/tests/eval/vexfs/run_pg_remote_macos.sh b/tests/eval/vexfs/run_pg_remote_macos.sh new file mode 100755 index 0000000000..f8d2f11b7c --- /dev/null +++ b/tests/eval/vexfs/run_pg_remote_macos.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" +NODE_SCRIPT="$ROOT/tests/eval/vexfs/run_pg_remote_macos_node.sh" +REMOTE_HOST="${VEXFS_REMOTE_HOST:?VEXFS_REMOTE_HOST is required}" +REMOTE_USER="${VEXFS_REMOTE_USER:-$USER}" +REMOTE_PASSWORD="${VEXFS_REMOTE_PASSWORD:-}" +PG_HOST="${VEXFS_REMOTE_PG_HOST:?VEXFS_REMOTE_PG_HOST is required}" +PG_PORT="${VEXFS_REMOTE_PG_PORT:-5432}" +LOCAL_PG_HOST="${VEXFS_LOCAL_PG_HOST:-$PG_HOST}" +LOCAL_PG_PORT="${VEXFS_LOCAL_PG_PORT:-$PG_PORT}" +PG_DATABASE="${VEXFS_REMOTE_PG_DATABASE:-test}" +PG_USER="${VEXFS_REMOTE_PG_USER:-postgres}" +PG_PASSWORD="${VEXFS_REMOTE_PG_PASSWORD:?VEXFS_REMOTE_PG_PASSWORD is required}" +PG_CONTAINER="${VEXDB_PG_CONTAINER:-}" +LOCAL_CLI="${VEXFS_EVAL_MOUNT_CLI:-$HOME/.local/bin/vexdb}" +REMOTE_CLI="${VEXFS_REMOTE_CLI:-/Users/$REMOTE_USER/.local/bin/vexdb}" +WORKSPACE="${VEXFS_REMOTE_WORKSPACE:-pg-remote-macos}" +KNOWN_HOSTS="${VEXFS_REMOTE_KNOWN_HOSTS:-/tmp/vexfs-remote-macos-known-hosts}" +REMOTE_NODE="/tmp/vexfs-pg-remote-node-$$.sh" +PASSFILE="/tmp/vexfs-pg-remote-${WORKSPACE}.pass" +TARGET="$REMOTE_USER@$REMOTE_HOST" + +case "$WORKSPACE" in + ''|*[!A-Za-z0-9_-]*) echo "workspace 只能包含字母、数字、下划线和连字符" >&2; exit 2 ;; +esac +case "$PG_HOST$PG_PORT$LOCAL_PG_HOST$LOCAL_PG_PORT$PG_DATABASE$PG_USER" in + *[!A-Za-z0-9_.:-]*) echo "PG 连接字段包含不支持的字符" >&2; exit 2 ;; +esac +[ "$(uname -s)" = Darwin ] || { echo "跨 Mac 编排必须从 macOS 运行" >&2; exit 2; } +[ -x "$LOCAL_CLI" ] || { echo "找不到本机 vexdb CLI：$LOCAL_CLI" >&2; exit 2; } +[ -f "$NODE_SCRIPT" ] || { echo "找不到节点脚本：$NODE_SCRIPT" >&2; exit 2; } + +SSH_OPTIONS=(-o StrictHostKeyChecking=no -o UserKnownHostsFile="$KNOWN_HOSTS" -o ConnectTimeout=8) +if [ -n "$REMOTE_PASSWORD" ]; then + SSHPASS_BIN="$(command -v sshpass || true)" + [ -n "$SSHPASS_BIN" ] || { echo "密码登录需要 sshpass" >&2; exit 2; } + export SSHPASS="$REMOTE_PASSWORD" + SSH=("$SSHPASS_BIN" -e ssh "${SSH_OPTIONS[@]}") + SCP=("$SSHPASS_BIN" -e scp "${SSH_OPTIONS[@]}") +else + SSH=(ssh "${SSH_OPTIONS[@]}") + SCP=(scp "${SSH_OPTIONS[@]}") +fi + +REMOTE_DSN="host=$PG_HOST port=$PG_PORT dbname=$PG_DATABASE user=$PG_USER passfile=$PASSFILE connect_timeout=8" +LOCAL_DSN="host=$LOCAL_PG_HOST port=$LOCAL_PG_PORT dbname=$PG_DATABASE user=$PG_USER passfile=$PASSFILE connect_timeout=8" +REMOTE_PASSFILE_LINE="$PG_HOST:$PG_PORT:$PG_DATABASE:$PG_USER:$PG_PASSWORD" +LOCAL_PASSFILE_LINE="$LOCAL_PG_HOST:$LOCAL_PG_PORT:$PG_DATABASE:$PG_USER:$PG_PASSWORD" + +remote_command() { + "${SSH[@]}" "$TARGET" "$1" +} + +drop_workspace() { + [ -n "$PG_CONTAINER" ] || return 0 + docker exec "$PG_CONTAINER" /opt/pg19/bin/psql -U postgres -d "$PG_DATABASE" -X -q \ + -c "SELECT vexfs_workspace_drop('$WORKSPACE', true);" >/dev/null 2>&1 || true +} + +cleanup() { + local status=$? command + trap - EXIT INT TERM + "$LOCAL_CLI" fs --backend pg --dsn "$LOCAL_DSN" --workspace "$WORKSPACE" \ + unmount --force >/dev/null 2>&1 || true + printf -v command 'rm -f %q %q' "$REMOTE_NODE" "$PASSFILE" + remote_command "$command" >/dev/null 2>&1 || true + rm -f "$PASSFILE" + drop_workspace + exit "$status" +} +trap cleanup EXIT INT TERM + +write_remote_passfile() { + local command + printf -v command 'umask 077; printf "%%s\\n" %q > %q; chmod 0600 %q' \ + "$REMOTE_PASSFILE_LINE" "$PASSFILE" "$PASSFILE" + remote_command "$command" +} + +remote_phase() { + local phase="$1" command + printf -v command \ + 'env VEXFS_NODE_PHASE=%q VEXFS_NODE_CLI=%q VEXFS_NODE_DSN=%q VEXFS_NODE_WORKSPACE=%q /bin/bash %q' \ + "$phase" "$REMOTE_CLI" "$REMOTE_DSN" "$WORKSPACE" "$REMOTE_NODE" + remote_command "$command" +} + +local_phase() { + VEXFS_NODE_PHASE="$1" \ + VEXFS_NODE_CLI="$LOCAL_CLI" \ + VEXFS_NODE_DSN="$LOCAL_DSN" \ + VEXFS_NODE_WORKSPACE="$WORKSPACE" \ + /bin/bash "$NODE_SCRIPT" +} + +drop_workspace +umask 077 +printf '%s\n' "$LOCAL_PASSFILE_LINE" >"$PASSFILE" +chmod 0600 "$PASSFILE" +write_remote_passfile +"${SCP[@]}" "$NODE_SCRIPT" "$TARGET:$REMOTE_NODE" >/dev/null +remote_command "chmod 0700 $REMOTE_NODE" + +local_phase local-create +remote_phase remote-runtime-check +remote_phase remote-modify +local_phase local-restore +remote_phase remote-final + +echo "VEXFS PG TWO-MAC ROUNDTRIP: PASS" diff --git a/tests/eval/vexfs/run_pg_remote_macos_faults.sh b/tests/eval/vexfs/run_pg_remote_macos_faults.sh new file mode 100755 index 0000000000..eba24395de --- /dev/null +++ b/tests/eval/vexfs/run_pg_remote_macos_faults.sh @@ -0,0 +1,347 @@ +#!/bin/bash +set -euo pipefail + +# Keep this file directly executable: CI and local acceptance use the shebang path. + +ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" +REMOTE_HOST="${VEXFS_REMOTE_HOST:?VEXFS_REMOTE_HOST is required}" +REMOTE_USER="${VEXFS_REMOTE_USER:-$USER}" +REMOTE_PASSWORD="${VEXFS_REMOTE_PASSWORD:-}" +LOCAL_CLI="${VEXFS_EVAL_MOUNT_CLI:-$HOME/.local/bin/vexdb}" +REMOTE_CLI="${VEXFS_REMOTE_CLI:-/Users/$REMOTE_USER/.local/bin/vexdb}" +CONTAINER="${VEXDB_PG_CONTAINER:-vexdb_pg19-test}" +PG_BIN="${VEXDB_PG_BIN:-/opt/pg19/bin}" +PG_DATABASE="${VEXFS_REMOTE_PG_DATABASE:-test}" +PG_USER="${VEXFS_REMOTE_PG_USER:-postgres}" +PG_PASSWORD="${VEXFS_REMOTE_PG_PASSWORD:-}" +LOCAL_PG_PORT="${VEXFS_LOCAL_PG_PORT:-5433}" +REMOTE_TUNNEL_PORT="${VEXFS_REMOTE_TUNNEL_PORT:-6545}" +WORKSPACE="${VEXFS_REMOTE_FAULT_WORKSPACE:-pg-two-mac-fault}" +KNOWN_HOSTS="${VEXFS_REMOTE_KNOWN_HOSTS:-/tmp/vexfs-remote-macos-fault-known-hosts}" +TMP="$(mktemp -d "${TMPDIR:-/tmp}/vexfs-pg-two-mac-fault.XXXXXX")" +CONTROL="/tmp/vexfs-pg-fault-tunnel-$$.sock" +LOCAL_MOUNT="$TMP/mount" +LOCAL_PASSFILE="$TMP/pgpass" +REMOTE_ROOT="/tmp/vexfs-pg-two-mac-fault-$$" +REMOTE_MOUNT="$REMOTE_ROOT/mount" +REMOTE_PASSFILE="$REMOTE_ROOT/pgpass" +TARGET="$REMOTE_USER@$REMOTE_HOST" +CHECKS=0 +TUNNEL_RUNNING=false +DATABASE_STOPPED=false +PHASE=initializing + +case "$WORKSPACE" in + ''|*[!A-Za-z0-9_-]*) echo "workspace 只能包含字母、数字、下划线和连字符" >&2; exit 2 ;; +esac +case "$PG_DATABASE$PG_USER$LOCAL_PG_PORT$REMOTE_TUNNEL_PORT" in + *[!A-Za-z0-9_-]*) echo "PG 连接字段包含不支持的字符" >&2; exit 2 ;; +esac +[ "$(uname -s)" = Darwin ] || { echo "双 Mac 故障测试必须从 macOS 运行" >&2; exit 2; } +[ -x "$LOCAL_CLI" ] || { echo "找不到本机 vexdb CLI：$LOCAL_CLI" >&2; exit 2; } +echo "VEXFS PG TWO-MAC FAULT: phase=prerequisites" +docker inspect "$CONTAINER" >/dev/null + +SSH_OPTIONS=(-o StrictHostKeyChecking=no -o UserKnownHostsFile="$KNOWN_HOSTS" -o ConnectTimeout=8) +if [ -n "$REMOTE_PASSWORD" ]; then + SSHPASS_BIN="$(command -v sshpass || true)" + [ -n "$SSHPASS_BIN" ] || { echo "密码登录需要 sshpass" >&2; exit 2; } + export SSHPASS="$REMOTE_PASSWORD" + SSH=("$SSHPASS_BIN" -e ssh "${SSH_OPTIONS[@]}") +else + SSH=(ssh "${SSH_OPTIONS[@]}") +fi + +LOCAL_DSN="host=127.0.0.1 port=$LOCAL_PG_PORT dbname=$PG_DATABASE user=$PG_USER connect_timeout=3" +REMOTE_DSN="host=127.0.0.1 port=$REMOTE_TUNNEL_PORT dbname=$PG_DATABASE user=$PG_USER connect_timeout=3" +if [ -n "$PG_PASSWORD" ]; then + umask 077 + printf '%s:%s:%s:%s:%s\n' 127.0.0.1 "$LOCAL_PG_PORT" \ + "$PG_DATABASE" "$PG_USER" "$PG_PASSWORD" >"$LOCAL_PASSFILE" + chmod 0600 "$LOCAL_PASSFILE" + LOCAL_DSN="$LOCAL_DSN passfile=$LOCAL_PASSFILE" + REMOTE_DSN="$REMOTE_DSN passfile=$REMOTE_PASSFILE" +fi + +fail() { + echo "$*" >&2 + exit 1 +} + +pass() { + CHECKS=$((CHECKS + 1)) +} + +shell_join() { + local output="" argument quoted + for argument in "$@"; do + printf -v quoted '%q' "$argument" + output+="$quoted " + done + printf '%s' "$output" +} + +remote_exec() { + local command + command="$(shell_join "$@")" + printf 'set -euo pipefail\n%s\n' "$command" | "${SSH[@]}" "$TARGET" /bin/bash -s +} + +remote_shell() { + printf 'set -euo pipefail\n%s\n' "$1" | "${SSH[@]}" "$TARGET" /bin/bash -s +} + +remote_vexfs() { + remote_exec "$REMOTE_CLI" fs --backend pg --dsn "$REMOTE_DSN" \ + --workspace "$WORKSPACE" "$@" +} + +local_vexfs() { + "$LOCAL_CLI" fs --backend pg --dsn "$LOCAL_DSN" \ + --workspace "$WORKSPACE" "$@" +} + +start_tunnel() { + [ "$TUNNEL_RUNNING" = false ] || return 0 + rm -f "$CONTROL" + "${SSH[@]}" -M -S "$CONTROL" -fNT -o ExitOnForwardFailure=yes \ + -R "127.0.0.1:$REMOTE_TUNNEL_PORT:127.0.0.1:$LOCAL_PG_PORT" "$TARGET" + TUNNEL_RUNNING=true +} + +stop_tunnel() { + [ "$TUNNEL_RUNNING" = true ] || return 0 + "${SSH[@]}" -S "$CONTROL" -O exit "$TARGET" >/dev/null 2>&1 || true + TUNNEL_RUNNING=false + rm -f "$CONTROL" +} + +database_start() { + if ! docker exec "$CONTAINER" "$PG_BIN/pg_ctl" \ + -D /var/lib/postgresql/data status >/dev/null 2>&1; then + docker exec "$CONTAINER" "$PG_BIN/pg_ctl" \ + -D /var/lib/postgresql/data -w start >/dev/null + fi + DATABASE_STOPPED=false +} + +database_stop() { + docker exec "$CONTAINER" "$PG_BIN/pg_ctl" \ + -D /var/lib/postgresql/data -m fast -w stop >/dev/null + DATABASE_STOPPED=true +} + +drop_workspace() { + docker exec "$CONTAINER" "$PG_BIN/psql" -U "$PG_USER" -d "$PG_DATABASE" \ + -X -q -v ON_ERROR_STOP=1 \ + -c "SELECT public.vexfs_workspace_drop('$WORKSPACE', true);" \ + >/dev/null 2>&1 || true +} + +cleanup() { + local status=$? + trap - EXIT INT TERM + if [ "$status" -ne 0 ]; then + echo "双 Mac 故障 eval 失败：phase=$PHASE status=$status" >&2 + fi + database_start >/dev/null 2>&1 || true + start_tunnel >/dev/null 2>&1 || true + local_vexfs unmount --force "$LOCAL_MOUNT" >/dev/null 2>&1 || true + remote_vexfs unmount --force "$REMOTE_MOUNT" >/dev/null 2>&1 || true + remote_exec rm -rf "$REMOTE_ROOT" >/dev/null 2>&1 || true + stop_tunnel + drop_workspace + rm -rf "$TMP" + exit "$status" +} +trap cleanup EXIT INT TERM + +expect_remote_write_failure() { + local relative=$1 content=$2 description=$3 command + printf -v command 'printf "%%s\\n" %q > %q' "$content" "$REMOTE_MOUNT/$relative" + if remote_exec /usr/bin/perl -e 'alarm 12; exec @ARGV' /bin/bash -c "$command" \ + >/dev/null 2>&1; then + fail "$description：写入意外成功" + fi + pass +} + +expect_database_path_missing() { + local path=$1 description=$2 + if local_vexfs stat "$path" >/dev/null 2>&1; then + fail "$description：数据库中留下了 $path" + fi + pass +} + +wait_remote_unmounted() { + local output="" + for _ in $(seq 1 40); do + output="$(remote_vexfs --json mount status "$REMOTE_MOUNT" 2>/dev/null || true)" + [ "$output" = "[]" ] && return 0 + sleep 0.25 + done + fail "VexFSAppEx 退出后挂载没有在 10 秒内撤销：$output" +} + +wait_remote_mount_read() { + local relative=$1 expected=$2 description=$3 actual="" + for _ in $(seq 1 20); do + if actual="$(remote_exec cat "$REMOTE_MOUNT/$relative" 2>/dev/null)" && \ + [ "$actual" = "$expected" ]; then + pass + return 0 + fi + sleep 0.25 + done + fail "$description：10 秒内没有恢复，最后内容为 [$actual]" +} + +wait_local_mount_read() { + local relative=$1 expected=$2 description=$3 actual="" + for _ in $(seq 1 20); do + if actual="$(cat "$LOCAL_MOUNT/$relative" 2>/dev/null)" && \ + [ "$actual" = "$expected" ]; then + pass + return 0 + fi + sleep 0.25 + done + fail "$description：10 秒内没有恢复，最后内容为 [$actual]" +} + +# A previous FSKit fault run can still be finishing ExtensionKit teardown even +# after both mount tables are empty. Avoid immediately reusing that process window. +sleep 3 +echo "VEXFS PG TWO-MAC FAULT: phase=database-and-tunnel" +database_start +start_tunnel +PHASE=mounting +echo "VEXFS PG TWO-MAC FAULT: phase=mounting" +remote_exec test -x "$REMOTE_CLI" +remote_exec rm -rf "$REMOTE_ROOT" +remote_exec mkdir -p "$REMOTE_MOUNT" +if [ -n "$PG_PASSWORD" ]; then + PASSFILE_LINE="127.0.0.1:$REMOTE_TUNNEL_PORT:$PG_DATABASE:$PG_USER:$PG_PASSWORD" + printf -v command 'umask 077; printf "%%s\\n" %q > %q; chmod 0600 %q' \ + "$PASSFILE_LINE" "$REMOTE_PASSFILE" "$REMOTE_PASSFILE" + remote_shell "$command" +fi +mkdir -p "$LOCAL_MOUNT" +local_vexfs unmount --force "$LOCAL_MOUNT" >/dev/null 2>&1 || true +drop_workspace +# ExtensionKit tears down an unmounted volume asynchronously. A new volume +# started in that short window can inherit the previous termination signal. +# Let both machines finish the old volume before creating the fault-test pair. +sleep 3 +local_vexfs setup >/dev/null +local_vexfs mount "$LOCAL_MOUNT" >/dev/null +remote_vexfs mount "$REMOTE_MOUNT" >/dev/null + +mkdir -p "$LOCAL_MOUNT/shared" +printf '%s\n' baseline-from-local >"$LOCAL_MOUNT/shared/baseline.txt" +[ "$(remote_exec cat "$REMOTE_MOUNT/shared/baseline.txt")" = baseline-from-local ] || \ + fail "第二台 Mac 没有读到第一台的基线文件" +pass +[[ "$(remote_vexfs --json mount status "$REMOTE_MOUNT")" == *"\"workspace\":\"$WORKSPACE\""* ]] || \ + fail "第二台 Mac 的 FSKit 挂载状态不正确" +pass +sleep 2 + +# ExtensionKit 会在 VexFSAppEx 崩溃后撤销挂载。底层目录必须保持 0500， +# 避免 Bash 把文件写进本机目录；显式 mount 后应继续使用原 workspace。 +PHASE=extension-crash +echo "VEXFS PG TWO-MAC FAULT: phase=extension-crash" +remote_exec pgrep -x VexFSAppEx >/dev/null +remote_exec pkill -KILL -x VexFSAppEx +wait_remote_unmounted +pass +[ "$(remote_exec stat -f '%Lp' "$REMOTE_MOUNT")" = 500 ] || \ + fail "VexFSAppEx 崩溃后挂载点没有进入防误写模式" +pass +expect_remote_write_failure unsafe-after-extension-crash.txt should-not-be-local \ + "VexFSAppEx 崩溃后的底层目录保护" +expect_database_path_missing /unsafe-after-extension-crash.txt \ + "VexFSAppEx 崩溃失败写" +remote_vexfs mount "$REMOTE_MOUNT" >/dev/null +[ "$(remote_exec cat "$REMOTE_MOUNT/shared/baseline.txt")" = baseline-from-local ] || \ + fail "VexFSAppEx 崩溃重挂后基线文件丢失" +pass +remote_shell "printf '%s\n' after-extension-remount > $(shell_join "$REMOTE_MOUNT/shared/after-extension-remount.txt")" +[ "$(local_vexfs cat /shared/after-extension-remount.txt)" = after-extension-remount ] || \ + fail "VexFSAppEx 重挂后的写入没有进入 PostgreSQL" +pass + +# 关闭 SSH 反向隧道等价于第二台机器到 PG 的网络中断。挂载保留， +# 当前写必须明确失败；隧道恢复后同一个挂载应在下一次操作自动重连。 +PHASE=network-outage +echo "VEXFS PG TWO-MAC FAULT: phase=network-outage" +stop_tunnel +expect_remote_write_failure shared/during-network-outage.txt should-fail \ + "网络中断期间写入" +start_tunnel +expect_database_path_missing /shared/during-network-outage.txt \ + "网络中断失败写" +wait_remote_mount_read shared/baseline.txt baseline-from-local \ + "网络恢复后的挂载读" +remote_shell "printf '%s\n' after-network-reconnect > $(shell_join "$REMOTE_MOUNT/shared/after-network-reconnect.txt")" +[ "$(local_vexfs cat /shared/after-network-reconnect.txt)" = after-network-reconnect ] || \ + fail "网络恢复后的写入没有进入 PostgreSQL" +pass + +# PostgreSQL 整体停止时，两个 gateway 都会失去连接。服务器恢复后不重挂， +# libpq HostStore 必须在下一次操作自动建立新连接。 +PHASE=database-outage +echo "VEXFS PG TWO-MAC FAULT: phase=database-outage" +database_stop +expect_remote_write_failure shared/during-database-outage.txt should-fail \ + "数据库停止期间写入" +database_start +expect_database_path_missing /shared/during-database-outage.txt \ + "数据库停止失败写" +wait_remote_mount_read shared/baseline.txt baseline-from-local \ + "数据库恢复后的挂载读" +wait_local_mount_read shared/baseline.txt baseline-from-local \ + "数据库恢复后的本机挂载读" +remote_shell "printf '%s\n' after-database-restart > $(shell_join "$REMOTE_MOUNT/shared/after-database-restart.txt")" +[ "$(local_vexfs cat /shared/after-database-restart.txt)" = after-database-restart ] || \ + fail "数据库恢复后的写入没有进入 PostgreSQL" +pass + +printf '%s\n' final-from-local >"$LOCAL_MOUNT/shared/final-from-local.txt" +PHASE=final-consistency +echo "VEXFS PG TWO-MAC FAULT: phase=final-consistency" +[ "$(remote_exec cat "$REMOTE_MOUNT/shared/final-from-local.txt")" = final-from-local ] || \ + fail "故障恢复后第二台 Mac 没有看到第一台写入" +pass +remote_shell "printf '%s\n' final-from-remote > $(shell_join "$REMOTE_MOUNT/shared/final-from-remote.txt")" +[ "$(cat "$LOCAL_MOUNT/shared/final-from-remote.txt")" = final-from-remote ] || \ + fail "故障恢复后第一台 Mac 没有看到第二台写入" +pass + +CHECK_OUTPUT="$(local_vexfs check)" +[[ "$CHECK_OUTPUT" == *"OK workspace=$WORKSPACE"* ]] || \ + fail "双机故障恢复后的 deep check 失败：$CHECK_OUTPUT" +pass +FINAL_STATE="$(docker exec "$CONTAINER" "$PG_BIN/psql" -U "$PG_USER" \ + -d "$PG_DATABASE" -X -q -t -A -F '|' -v ON_ERROR_STOP=1 -c \ + "SELECT (public.vexfs_check('$WORKSPACE',1)->>'ok')::boolean, + (SELECT count(*) FROM public.vexfs_audit_list('$WORKSPACE',1000,NULL)) >= 6, + (SELECT count(*) >= 2 + FROM _vexfs.mount_sessions AS mounted + JOIN _vexfs.workspaces AS workspace USING (workspace_id) + WHERE workspace.name='$WORKSPACE' + AND mounted.lease_until > clock_timestamp());")" +[[ "$FINAL_STATE" == "t|t|t" ]] || fail "双机最终数据库状态错误：$FINAL_STATE" +CHECKS=$((CHECKS + 3)) + +local_vexfs unmount "$LOCAL_MOUNT" >/dev/null +remote_vexfs unmount "$REMOTE_MOUNT" >/dev/null +[ "$(stat -f '%Lp' "$LOCAL_MOUNT")" = 700 ] || \ + fail "本机正常卸载后挂载点权限没有恢复到 0700" +pass +[ "$(remote_exec stat -f '%Lp' "$REMOTE_MOUNT")" = 700 ] || \ + fail "第二台 Mac 正常卸载后挂载点权限没有恢复到 0700" +pass + +echo "VEXFS PG TWO-MAC FAULT RECOVERY: PASS ($CHECKS checks)" diff --git a/tests/eval/vexfs/run_pg_remote_macos_node.sh b/tests/eval/vexfs/run_pg_remote_macos_node.sh new file mode 100755 index 0000000000..baad5a8918 --- /dev/null +++ b/tests/eval/vexfs/run_pg_remote_macos_node.sh @@ -0,0 +1,166 @@ +#!/usr/bin/env bash +set -euo pipefail + +PHASE="${VEXFS_NODE_PHASE:?VEXFS_NODE_PHASE is required}" +CLI="${VEXFS_NODE_CLI:-$HOME/.local/bin/vexdb}" +DSN="${VEXFS_NODE_DSN:?VEXFS_NODE_DSN is required}" +WORKSPACE="${VEXFS_NODE_WORKSPACE:-pg-remote-macos}" +CHECKS=0 +MOUNT_POINT="$(mktemp -d "${TMPDIR:-/tmp}/vexfs-pg-remote-mount.XXXXXX")" +MOUNTED=false + +[ "$(uname -s)" = Darwin ] || { echo "该节点测试只支持 macOS" >&2; exit 2; } +[ -x "$CLI" ] || { echo "找不到 vexdb CLI：$CLI" >&2; exit 2; } + +vexfs() { + "$CLI" fs --backend pg --dsn "$DSN" --workspace "$WORKSPACE" "$@" +} + +pass() { + CHECKS=$((CHECKS + 1)) +} + +equals() { + local actual="$1" expected="$2" description="$3" + if [ "$actual" != "$expected" ]; then + echo "$description：期望 [$expected]，实际 [$actual]" >&2 + exit 1 + fi + pass +} + +contains() { + local actual="$1" expected="$2" description="$3" + case "$actual" in + *"$expected"*) pass ;; + *) echo "$description：缺少 [$expected]，实际 [$actual]" >&2; exit 1 ;; + esac +} + +assert_file() { + [ -f "$1" ] || { echo "$2：文件不存在 $1" >&2; exit 1; } + pass +} + +assert_missing() { + [ ! -e "$1" ] && [ ! -L "$1" ] || { echo "$2：路径仍存在 $1" >&2; exit 1; } + pass +} + +mount_workspace() { + vexfs mount "$MOUNT_POINT" >/dev/null + MOUNTED=true + contains "$(vexfs --json mount status "$MOUNT_POINT")" \ + "\"workspace\":\"$WORKSPACE\"" "FSKit 挂载状态" +} + +unmount_workspace() { + if [ "$MOUNTED" = true ]; then + vexfs unmount "$MOUNT_POINT" >/dev/null + MOUNTED=false + fi +} + +cleanup() { + local status=$? + trap - EXIT INT TERM + if [ "$MOUNTED" = true ]; then + vexfs unmount --force "$MOUNT_POINT" >/dev/null 2>&1 || true + fi + rmdir "$MOUNT_POINT" >/dev/null 2>&1 || true + exit "$status" +} +trap cleanup EXIT INT TERM + +verify_local_baseline() { + assert_file "$MOUNT_POINT/shared/from-local.txt" "本机文件" + equals "$(cat "$MOUNT_POINT/shared/from-local.txt")" "created-on-first-mac" "本机文件内容" + equals "$(readlink "$MOUNT_POINT/shared/from-local-link")" "from-local.txt" "符号链接" + equals "$(stat -f '%i' "$MOUNT_POINT/shared/from-local.txt")" \ + "$(stat -f '%i' "$MOUNT_POINT/shared/from-local-hard")" "hardlink inode" + equals "$(xattr -p com.vexfs.origin "$MOUNT_POINT/shared/from-local.txt")" \ + "first-mac" "扩展属性" + equals "$("$MOUNT_POINT/bin/from-local.sh")" "script-from-first-mac" "可执行权限" +} + +case "$PHASE" in + local-create) + vexfs setup >/dev/null + mount_workspace + mkdir -p "$MOUNT_POINT/shared" "$MOUNT_POINT/bin" + printf '%s\n' 'created-on-first-mac' >"$MOUNT_POINT/shared/from-local.txt" + ln "$MOUNT_POINT/shared/from-local.txt" "$MOUNT_POINT/shared/from-local-hard" + ln -s from-local.txt "$MOUNT_POINT/shared/from-local-link" + printf '%s\n' '#!/bin/sh' 'printf "%s\\n" script-from-first-mac' \ + >"$MOUNT_POINT/bin/from-local.sh" + chmod 0751 "$MOUNT_POINT/bin/from-local.sh" + xattr -w com.vexfs.origin first-mac "$MOUNT_POINT/shared/from-local.txt" + verify_local_baseline + unmount_workspace + vexfs snapshot create local-baseline >/dev/null + contains "$(vexfs snapshot list)" "local-baseline" "首台机器快照" + ;; + remote-modify) + mount_workspace + verify_local_baseline + printf '%s\n' 'created-on-second-mac' >"$MOUNT_POINT/shared/from-remote.tmp" + mv "$MOUNT_POINT/shared/from-remote.tmp" "$MOUNT_POINT/shared/from-remote.txt" + printf '%s\n' 'updated-on-second-mac' >>"$MOUNT_POINT/shared/from-local.txt" + xattr -w com.vexfs.remote second-mac "$MOUNT_POINT/shared/from-local.txt" + equals "$(tail -n 1 "$MOUNT_POINT/shared/from-local.txt")" \ + "updated-on-second-mac" "第二台机器追加" + equals "$(xattr -p com.vexfs.remote "$MOUNT_POINT/shared/from-local.txt")" \ + "second-mac" "第二台机器扩展属性" + unmount_workspace + vexfs snapshot create remote-update >/dev/null + contains "$(vexfs snapshot list)" "remote-update" "第二台机器快照" + ;; + remote-runtime-check) + equals "$(vexfs cat /shared/from-local.txt)" \ + "created-on-first-mac" "第二台机器直接 PG runtime 读取" + contains "$(vexfs check)" "OK workspace=$WORKSPACE" \ + "第二台机器直接 PG runtime deep check" + ;; + local-restore) + mount_workspace + assert_file "$MOUNT_POINT/shared/from-remote.txt" "远端文件回传" + equals "$(cat "$MOUNT_POINT/shared/from-remote.txt")" \ + "created-on-second-mac" "远端文件内容" + equals "$(tail -n 1 "$MOUNT_POINT/shared/from-local.txt")" \ + "updated-on-second-mac" "远端追加回传" + equals "$(xattr -p com.vexfs.remote "$MOUNT_POINT/shared/from-local.txt")" \ + "second-mac" "远端 xattr 回传" + unmount_workspace + vexfs snapshot restore local-baseline --force-unmount >/dev/null + mount_workspace + verify_local_baseline + assert_missing "$MOUNT_POINT/shared/from-remote.txt" "整树快照恢复" + printf '%s\n' 'created-after-restore' >"$MOUNT_POINT/shared/after-restore.txt" + assert_file "$MOUNT_POINT/shared/after-restore.txt" "恢复后新文件" + unmount_workspace + vexfs snapshot create local-restored >/dev/null + contains "$(vexfs snapshot list)" "local-restored" "恢复后快照" + ;; + remote-final) + mount_workspace + verify_local_baseline + assert_missing "$MOUNT_POINT/shared/from-remote.txt" "远端观察整树恢复" + equals "$(cat "$MOUNT_POINT/shared/after-restore.txt")" \ + "created-after-restore" "远端观察恢复后写入" + contains "$(find "$MOUNT_POINT/shared" -maxdepth 1 -print | sort)" \ + "after-restore.txt" "Bash find" + contains "$(grep -R 'created-on-first-mac' "$MOUNT_POINT/shared")" \ + "created-on-first-mac" "Bash grep" + unmount_workspace + contains "$(vexfs snapshot list)" "local-baseline" "最终快照 local-baseline" + contains "$(vexfs snapshot list)" "remote-update" "最终快照 remote-update" + contains "$(vexfs snapshot list)" "local-restored" "最终快照 local-restored" + contains "$(vexfs check)" "OK workspace=$WORKSPACE" "最终 deep check" + ;; + *) + echo "未知节点阶段：$PHASE" >&2 + exit 2 + ;; +esac + +echo "VEXFS PG REMOTE MACOS NODE: PASS (phase=$PHASE, $CHECKS checks)" diff --git a/tests/eval/vexfs/run_pg_restart_recovery.sh b/tests/eval/vexfs/run_pg_restart_recovery.sh new file mode 100755 index 0000000000..bac56e4354 --- /dev/null +++ b/tests/eval/vexfs/run_pg_restart_recovery.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" +CONTAINER="${VEXDB_PG_CONTAINER:-vexdb_pg19-test}" +DATABASE="${VEXDB_PG_DATABASE:-test}" +DSN="${VEXDB_PG_DSN:-postgresql://postgres@127.0.0.1:5433/test}" +BINARY="${VEXDB_PG_RUNTIME_SMOKE:-$ROOT/vexdb_sqlite/build/vexfs_pg_runtime_smoke}" +GATE="$(mktemp -d "${TMPDIR:-/tmp}/vexfs-pg-restart.XXXXXX")" +OUTPUT="$GATE/runtime.out" +ERRORS="$GATE/runtime.err" +PID="" + +cleanup() { + local status=$? + trap - EXIT + if [ -n "$PID" ] && kill -0 "$PID" >/dev/null 2>&1; then + kill "$PID" >/dev/null 2>&1 || true + wait "$PID" >/dev/null 2>&1 || true + fi + if [ "$status" -ne 0 ]; then + sed -n '1,240p' "$OUTPUT" >&2 2>/dev/null || true + sed -n '1,240p' "$ERRORS" >&2 2>/dev/null || true + fi + rm -rf "$GATE" + exit "$status" +} +trap cleanup EXIT + +[ -x "$BINARY" ] || { echo "找不到 PG runtime smoke：$BINARY" >&2; exit 1; } +docker inspect "$CONTAINER" >/dev/null +docker exec "$CONTAINER" psql -d "$DATABASE" -X -q -v ON_ERROR_STOP=1 \ + -c "CREATE EXTENSION IF NOT EXISTS pg_trgm;" >/dev/null + +VEXFS_PG_RESTART_GATE_DIR="$GATE" "$BINARY" "$DSN" >"$OUTPUT" 2>"$ERRORS" & +PID=$! + +ready=0 +for _ in $(seq 1 240); do + if [ -f "$GATE/ready" ]; then + ready=1 + break + fi + if ! kill -0 "$PID" >/dev/null 2>&1; then + wait "$PID" || true + echo "PG runtime 在数据库重启门禁前已经退出" >&2 + exit 1 + fi + sleep 0.25 +done +[ "$ready" -eq 1 ] || { echo "等待 PG runtime 重启门禁超时" >&2; exit 1; } + +docker exec "$CONTAINER" pg_ctl \ + -D /var/lib/postgresql/data -m fast -w restart >/dev/null +touch "$GATE/continue" + +wait "$PID" +PID="" +grep -q '^VEXFS PG RUNTIME CONTRACT: PASS' "$OUTPUT" +cat "$OUTPUT" +echo "VEXFS PG SERVER RESTART RECOVERY: PASS" diff --git a/tests/eval/vexfs/run_pg_runtime.sh b/tests/eval/vexfs/run_pg_runtime.sh new file mode 100755 index 0000000000..f3d094f770 --- /dev/null +++ b/tests/eval/vexfs/run_pg_runtime.sh @@ -0,0 +1,166 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" +CLI="${VEXDB_PG_CLI:-$ROOT/vexdb_sqlite/build/vexdb}" +CONTAINER="${VEXDB_PG_CONTAINER:-vexdb_pg19-test}" +DATABASE="${VEXDB_PG_DATABASE:-test}" +DSN="${VEXDB_PG_DSN:-postgresql://postgres@127.0.0.1:5433/test}" +WORKSPACE="${VEXDB_PG_RUNTIME_WORKSPACE:-pg-runtime-eval}" +IMPORT_WORKSPACE="${WORKSPACE}-import" +CHECKS=0 +TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/vexfs-pg-runtime.XXXXXX")" + +case "$WORKSPACE" in + ''|*[!A-Za-z0-9_-]*) echo "workspace 只能包含字母、数字、下划线和连字符" >&2; exit 2 ;; +esac +[ -x "$CLI" ] || { echo "找不到 vexdb CLI：$CLI" >&2; exit 1; } + +# 开发构建可能使用非系统 libpq；交付包不应依赖这个变量。 +if [ -n "${VEXDB_PG_LIB_DIR:-}" ]; then + export DYLD_LIBRARY_PATH="$VEXDB_PG_LIB_DIR${DYLD_LIBRARY_PATH:+:$DYLD_LIBRARY_PATH}" + export LD_LIBRARY_PATH="$VEXDB_PG_LIB_DIR${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" +fi + +cleanup_workspace() { + local name="$1" + docker exec "$CONTAINER" psql -d "$DATABASE" -X -q -v ON_ERROR_STOP=1 \ + -c "SELECT vexfs_workspace_drop('$name', true);" >/dev/null 2>&1 || true +} + +cleanup() { + local status=$? + trap - EXIT + cleanup_workspace "$WORKSPACE" + cleanup_workspace "$IMPORT_WORKSPACE" + rm -rf "$TMP_DIR" + exit "$status" +} +trap cleanup EXIT + +vexfs() { + "$CLI" fs --backend pg --dsn "$DSN" --workspace "$WORKSPACE" "$@" +} + +vexfs_import() { + "$CLI" fs --backend pg --dsn "$DSN" --workspace "$IMPORT_WORKSPACE" "$@" +} + +contains() { + local file="$1" value="$2" description="$3" + if ! grep -Fq "$value" "$file"; then + echo "${description}：缺少 $value" >&2 + sed -n '1,80p' "$file" >&2 + exit 1 + fi + CHECKS=$((CHECKS + 1)) +} + +equals() { + local actual="$1" expected="$2" description="$3" + if [ "$actual" != "$expected" ]; then + echo "${description}：期望 [$expected]，实际 [$actual]" >&2 + exit 1 + fi + CHECKS=$((CHECKS + 1)) +} + +cleanup_workspace "$WORKSPACE" +cleanup_workspace "$IMPORT_WORKSPACE" +docker exec "$CONTAINER" psql -d "$DATABASE" -X -q -v ON_ERROR_STOP=1 \ + -c "CREATE EXTENSION IF NOT EXISTS pg_trgm;" >/dev/null + +vexfs setup >"$TMP_DIR/setup.out" +contains "$TMP_DIR/setup.out" "postgresql [$WORKSPACE]" "setup backend" + +vexfs mkdir /project/src +printf 'alpha\nbeta alpha\n' >"$TMP_DIR/v1.txt" +printf 'gamma\nbeta alpha\n' >"$TMP_DIR/v2.txt" +vexfs write /project/src/main.txt "$TMP_DIR/v1.txt" >"$TMP_DIR/write-v1.out" +vexfs write /project/src/main.txt "$TMP_DIR/v2.txt" >"$TMP_DIR/write-v2.out" +equals "$(vexfs cat /project/src/main.txt)" "$(cat "$TMP_DIR/v2.txt")" "cat current" +CHECKS=$((CHECKS + 2)) + +vexfs ls /project/src --json >"$TMP_DIR/ls.json" +contains "$TMP_DIR/ls.json" '"name":"main.txt"' "ls file" +contains "$TMP_DIR/ls.json" '"kind":"file"' "ls metadata" +vexfs stat /project/src/main.txt >"$TMP_DIR/stat.json" +contains "$TMP_DIR/stat.json" '"version":2' "stat version" + +vexfs grep -n alpha /project >"$TMP_DIR/grep.out" +contains "$TMP_DIR/grep.out" '/project/src/main.txt:2:beta alpha' "grep line" +vexfs history /project/src/main.txt --json >"$TMP_DIR/history.json" +contains "$TMP_DIR/history.json" '"version":2' "history v2" +contains "$TMP_DIR/history.json" '"version":1' "history v1" +equals "$(vexfs show /project/src/main.txt --version 1)" "$(cat "$TMP_DIR/v1.txt")" \ + "show historical version" + +vexfs ln /project/src/main.txt /project/src/hardlink.txt +equals "$(vexfs cat /project/src/hardlink.txt)" "$(cat "$TMP_DIR/v2.txt")" "hardlink content" +vexfs chown 501:20 /project/src/main.txt +vexfs stat /project/src/main.txt >"$TMP_DIR/chown.json" +contains "$TMP_DIR/chown.json" '"uid":501' "stored uid" +contains "$TMP_DIR/chown.json" '"gid":20' "stored gid" + +printf '%s\n' '[{"principal":"postgres","effect":"allow","permissions":"read,write,execute,metadata","inherit":1}]' \ + >"$TMP_DIR/acl.json" +vexfs setfacl /project/src/main.txt "$TMP_DIR/acl.json" +vexfs getfacl /project/src/main.txt >"$TMP_DIR/getfacl.json" +contains "$TMP_DIR/getfacl.json" '"principal":"postgres"' "ACL principal" +contains "$TMP_DIR/getfacl.json" '"permissions":"read,write,execute,metadata"' "ACL permissions" + +vexfs index status >"$TMP_DIR/index.json" +contains "$TMP_DIR/index.json" '"backend":"pg-trgm"' "PG trigram capability" +contains "$TMP_DIR/index.json" '"available":true' "PG trigram available" +contains "$TMP_DIR/index.json" '"enabled":false' "PG trigram default disabled" +vexfs index enable >"$TMP_DIR/index-enabled.json" +contains "$TMP_DIR/index-enabled.json" '"enabled":true' "PG trigram enabled" +contains "$TMP_DIR/index-enabled.json" '"dirty":false' "PG trigram clean" +vexfs --json grep alpha /project >"$TMP_DIR/indexed-grep.json" +contains "$TMP_DIR/indexed-grep.json" '"index_used":true' "PG grep uses trigram index" +vexfs quota set --max-bytes 1048576 --max-files 100 --max-file-bytes 262144 \ + >"$TMP_DIR/quota.json" +contains "$TMP_DIR/quota.json" '"max_bytes":1048576' "quota max bytes" +vexfs retention set --keep-versions 10 --keep-days 30 >"$TMP_DIR/retention.json" +contains "$TMP_DIR/retention.json" '"keep_versions":10' "retention versions" + +vexfs snapshot create baseline >"$TMP_DIR/snapshot-create.out" +vexfs snapshot list >"$TMP_DIR/snapshot-list.out" +contains "$TMP_DIR/snapshot-list.out" "baseline" "snapshot list" +vexfs snapshot show baseline >"$TMP_DIR/snapshot-show.json" +contains "$TMP_DIR/snapshot-show.json" '"path":"/project/src/main.txt"' "snapshot tree" +printf 'after snapshot\n' >"$TMP_DIR/after.txt" +vexfs write /project/src/main.txt "$TMP_DIR/after.txt" >/dev/null +set +e +vexfs snapshot diff baseline >"$TMP_DIR/snapshot-diff.json" +diff_status=$? +set -e +equals "$diff_status" "1" "snapshot diff changed status" +contains "$TMP_DIR/snapshot-diff.json" '"path":"/project/src/main.txt"' "snapshot diff" +vexfs snapshot restore baseline >/dev/null +equals "$(vexfs cat /project/src/main.txt)" "$(cat "$TMP_DIR/v2.txt")" "snapshot restore" +vexfs --json grep alpha /project >"$TMP_DIR/restored-indexed-grep.json" +contains "$TMP_DIR/restored-indexed-grep.json" '"index_used":true' \ + "snapshot restore keeps PG trigram index current" + +vexfs check >"$TMP_DIR/check.out" +contains "$TMP_DIR/check.out" "OK workspace=$WORKSPACE" "deep check" +vexfs gc --batch 100 >"$TMP_DIR/gc.json" +contains "$TMP_DIR/gc.json" '"deleted_versions"' "bounded GC" + +vexfs export --output "$TMP_DIR/workspace.vexfs" >/dev/null +vexfs archive verify "$TMP_DIR/workspace.vexfs" >"$TMP_DIR/archive-verify.out" +contains "$TMP_DIR/archive-verify.out" '"ok":true' "archive verify" +vexfs_import import "$TMP_DIR/workspace.vexfs" >/dev/null +equals "$(vexfs_import cat /project/src/main.txt)" "$(cat "$TMP_DIR/v2.txt")" \ + "PG archive import" +vexfs_import check >"$TMP_DIR/import-check.out" +contains "$TMP_DIR/import-check.out" "OK workspace=$IMPORT_WORKSPACE" "import deep check" + +vexfs descriptor "$TMP_DIR/workspace.json" >/dev/null +contains "$TMP_DIR/workspace.json" '"backend": "postgresql"' "descriptor backend" +contains "$TMP_DIR/workspace.json" "\"workspace\": \"$WORKSPACE\"" "descriptor workspace" +mode="$(stat -f '%Lp' "$TMP_DIR/workspace.json" 2>/dev/null || stat -c '%a' "$TMP_DIR/workspace.json")" +equals "$mode" "600" "descriptor permissions" + +echo "VEXFS PG RUNTIME: PASS ($CHECKS checks)" diff --git a/tests/eval/vexfs/run_pg_two_mac_opencode.sh b/tests/eval/vexfs/run_pg_two_mac_opencode.sh new file mode 100755 index 0000000000..1f89411944 --- /dev/null +++ b/tests/eval/vexfs/run_pg_two_mac_opencode.sh @@ -0,0 +1,156 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" +NODE_SCRIPT="$ROOT/tests/eval/vexfs/run_pg_two_mac_opencode_node.sh" +REMOTE_HOST="${VEXFS_REMOTE_HOST:?VEXFS_REMOTE_HOST is required}" +REMOTE_USER="${VEXFS_REMOTE_USER:-$USER}" +LOCAL_CLI="${VEXFS_EVAL_MOUNT_CLI:-$HOME/.local/bin/vexdb}" +REMOTE_CLI="${VEXFS_REMOTE_CLI:-/Users/$REMOTE_USER/.local/bin/vexdb}" +LOCAL_OPENCODE="${VEXFS_LOCAL_OPENCODE:-$(command -v opencode || true)}" +# Apple Silicon 优先使用原生 Homebrew。Rosetta 下的 x86_64 Bun 会在无 AVX 的 M1 上 +# 警告并可能在 agent server 启动后崩溃；Intel Mac 可显式覆盖此路径。 +REMOTE_OPENCODE="${VEXFS_REMOTE_OPENCODE:-/opt/homebrew/bin/opencode}" +CONTAINER="${VEXDB_PG_CONTAINER:-vexdb_pg19-vexfs-dev}" +PG_DATABASE="${VEXFS_PG_DATABASE:-test}" +PG_USER="${VEXFS_PG_USER:-postgres}" +LOCAL_PG_PORT="${VEXFS_LOCAL_PG_PORT:-5434}" +REMOTE_TUNNEL_PORT="${VEXFS_REMOTE_TUNNEL_PORT:-6550}" +WORKSPACE="${VEXFS_PG_WORKSPACE:-pg-two-mac-opencode}" +LOCAL_MODEL="${VEXFS_LOCAL_OPENCODE_MODEL:-${VEXFS_EVAL_OPENCODE_MODEL:-openai/gpt-5.4-mini}}" +REMOTE_MODEL="${VEXFS_REMOTE_OPENCODE_MODEL:-opencode/north-mini-code-free}" +KEEP_WORKSPACE="${VEXFS_KEEP_WORKSPACE:-0}" +KNOWN_HOSTS="${VEXFS_REMOTE_KNOWN_HOSTS:-/tmp/vexfs-pg-opencode-known-hosts}" +CONTROL="/tmp/vexfs-pg-opencode-tunnel-$$.sock" +REMOTE_NODE="/tmp/vexfs-pg-opencode-node-$$.sh" +TARGET="$REMOTE_USER@$REMOTE_HOST" +LOCAL_DSN="host=127.0.0.1 port=$LOCAL_PG_PORT dbname=$PG_DATABASE user=$PG_USER connect_timeout=8" +REMOTE_DSN="host=127.0.0.1 port=$REMOTE_TUNNEL_PORT dbname=$PG_DATABASE user=$PG_USER connect_timeout=8" +RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)" +OUTPUT_DIR="${VEXFS_EVAL_OUTPUT_DIR:-$ROOT/vexdb_sqlite/build/eval/vexfs-pg-two-mac-opencode/$RUN_ID}" +LOG_FILE="$OUTPUT_DIR/run.log" +REPORT_FILE="$OUTPUT_DIR/report.md" +TUNNEL_RUNNING=false + +case "$WORKSPACE" in + ''|*[!A-Za-z0-9_-]*) echo "workspace 只能包含字母、数字、下划线和连字符" >&2; exit 2 ;; +esac +[ "$(uname -s)" = Darwin ] || { echo "双 Mac OpenCode 编排必须从 macOS 运行" >&2; exit 2; } +[ -x "$LOCAL_CLI" ] || { echo "找不到本机 vexdb CLI：$LOCAL_CLI" >&2; exit 2; } +[ -x "$LOCAL_OPENCODE" ] || { echo "找不到本机 OpenCode：$LOCAL_OPENCODE" >&2; exit 2; } +[ -f "$NODE_SCRIPT" ] || { echo "找不到节点脚本：$NODE_SCRIPT" >&2; exit 2; } +mkdir -p "$OUTPUT_DIR" +: >"$LOG_FILE" + +SSH_OPTIONS=(-o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=no -o UserKnownHostsFile="$KNOWN_HOSTS") +SSH=(ssh "${SSH_OPTIONS[@]}") +SCP=(scp "${SSH_OPTIONS[@]}") + +shell_join() { + local output="" argument quoted + for argument in "$@"; do + printf -v quoted '%q' "$argument" + output+="$quoted " + done + printf '%s' "$output" +} + +remote_exec() { + local command + command="$(shell_join "$@")" + printf 'set -euo pipefail\n%s\n' "$command" | "${SSH[@]}" "$TARGET" /bin/bash -s +} + +vexfs_local() { + "$LOCAL_CLI" fs --backend pg --dsn "$LOCAL_DSN" --workspace "$WORKSPACE" "$@" +} + +drop_workspace() { + docker exec "$CONTAINER" /opt/pg19/bin/psql -U postgres -d "$PG_DATABASE" -X -q \ + -c "SELECT public.vexfs_workspace_drop('$WORKSPACE', true);" >/dev/null 2>&1 || true +} + +start_tunnel() { + "${SSH[@]}" -M -S "$CONTROL" -fNT -o ExitOnForwardFailure=yes \ + -R "127.0.0.1:$REMOTE_TUNNEL_PORT:127.0.0.1:$LOCAL_PG_PORT" "$TARGET" + TUNNEL_RUNNING=true +} + +cleanup() { + local status=$? + trap - EXIT INT TERM + "$LOCAL_CLI" fs --backend pg --dsn "$LOCAL_DSN" --workspace "$WORKSPACE" \ + unmount --force >/dev/null 2>&1 || true + remote_exec "$REMOTE_CLI" fs --backend pg --dsn "$REMOTE_DSN" \ + --workspace "$WORKSPACE" unmount --force >/dev/null 2>&1 || true + remote_exec rm -f "$REMOTE_NODE" >/dev/null 2>&1 || true + if [ "$TUNNEL_RUNNING" = true ]; then + "${SSH[@]}" -S "$CONTROL" -O exit "$TARGET" >/dev/null 2>&1 || true + fi + if [ "$KEEP_WORKSPACE" != 1 ]; then + drop_workspace + fi + exit "$status" +} +trap cleanup EXIT INT TERM + +run_node() { + local phase="$1" cli="$2" dsn="$3" opencode="$4" model="$5" script="$6" + env VEXFS_NODE_PHASE="$phase" VEXFS_NODE_CLI="$cli" VEXFS_NODE_DSN="$dsn" \ + VEXFS_NODE_WORKSPACE="$WORKSPACE" VEXFS_NODE_OPENCODE="$opencode" \ + VEXFS_EVAL_OPENCODE_MODEL="$model" /bin/bash "$script" +} + +run_remote_node() { + local phase="$1" command + printf -v command \ + 'env VEXFS_NODE_PHASE=%q VEXFS_NODE_CLI=%q VEXFS_NODE_DSN=%q VEXFS_NODE_WORKSPACE=%q VEXFS_NODE_OPENCODE=%q VEXFS_EVAL_OPENCODE_MODEL=%q /bin/bash %q' \ + "$phase" "$REMOTE_CLI" "$REMOTE_DSN" "$WORKSPACE" "$REMOTE_OPENCODE" \ + "$REMOTE_MODEL" "$REMOTE_NODE" + printf 'set -euo pipefail\n%s\n' "$command" | \ + "${SSH[@]}" "$TARGET" /bin/bash -s +} + +drop_workspace +start_tunnel +remote_exec test -x "$REMOTE_CLI" +remote_exec test -x "$REMOTE_OPENCODE" +"${SCP[@]}" "$NODE_SCRIPT" "$TARGET:$REMOTE_NODE" >/dev/null +remote_exec chmod 0700 "$REMOTE_NODE" + +{ + echo "VEXFS PG TWO-MAC OPENCODE: phase=mac1-create" + run_node mac1-create "$LOCAL_CLI" "$LOCAL_DSN" "$LOCAL_OPENCODE" "$LOCAL_MODEL" "$NODE_SCRIPT" + echo "VEXFS PG TWO-MAC OPENCODE: phase=mac2-modify" + run_remote_node mac2-modify + echo "VEXFS PG TWO-MAC OPENCODE: phase=mac1-verify-restore" + run_node mac1-verify-restore "$LOCAL_CLI" "$LOCAL_DSN" "$LOCAL_OPENCODE" "$LOCAL_MODEL" "$NODE_SCRIPT" +} 2>&1 | tee "$LOG_FILE" + +cat >"$REPORT_FILE" <&2; exit 2; } +[ -x "$CLI" ] || { echo "找不到 vexdb CLI：$CLI" >&2; exit 2; } +[ -x "$OPENCODE" ] || { echo "找不到 OpenCode：$OPENCODE" >&2; exit 2; } +[ -x "$PYTHON" ] || { echo "找不到 Python：$PYTHON" >&2; exit 2; } +[ -x "$GIT" ] || { echo "找不到 Git：$GIT" >&2; exit 2; } + +vexfs() { + "$CLI" fs --backend pg --dsn "$DSN" --workspace "$WORKSPACE" "$@" +} + +pass() { + CHECKS=$((CHECKS + 1)) +} + +fail() { + echo "$1" >&2 + exit 1 +} + +equals() { + local actual="$1" expected="$2" description="$3" + [ "$actual" = "$expected" ] || \ + fail "$description：期望 [$expected]，实际 [$actual]" + pass +} + +contains() { + local actual="$1" expected="$2" description="$3" + case "$actual" in + *"$expected"*) pass ;; + *) fail "$description：缺少 [$expected]，实际 [$actual]" ;; + esac +} + +mount_workspace() { + vexfs mount "$MOUNT_POINT" >/dev/null + MOUNTED=true + contains "$(vexfs --json mount status "$MOUNT_POINT")" \ + "\"workspace\":\"$WORKSPACE\"" "FSKit 挂载状态" +} + +unmount_workspace() { + if [ "$MOUNTED" = true ]; then + cd / + vexfs unmount "$MOUNT_POINT" >/dev/null + MOUNTED=false + fi +} + +cleanup() { + local status=$? + trap - EXIT INT TERM + cd / + if [ "$MOUNTED" = true ]; then + vexfs unmount --force "$MOUNT_POINT" >/dev/null 2>&1 || true + fi + rm -f "$OPENCODE_LOG" + rmdir "$MOUNT_POINT" >/dev/null 2>&1 || true + exit "$status" +} +trap cleanup EXIT INT TERM + +approval_argument() { + local help + help="$("$OPENCODE" run --help 2>&1)" + if [[ "$help" == *"--dangerously-skip-permissions"* ]]; then + printf '%s\n' --dangerously-skip-permissions + elif [[ "$help" == *"--auto"* ]]; then + printf '%s\n' --auto + else + fail "当前 OpenCode 没有无人值守授权参数" + fi +} + +run_opencode() { + local prompt="$1" approval started ended + approval="$(approval_argument)" + started="$(date +%s)" + /usr/bin/perl -e 'alarm 900; exec @ARGV' \ + "$OPENCODE" run "$approval" --pure --format json --model "$MODEL" \ + --dir "$PROJECT" "$prompt" >"$OPENCODE_LOG" 2>&1 || { + tail -80 "$OPENCODE_LOG" >&2 + fail "OpenCode 执行失败" + } + ended="$(date +%s)" + [ -s "$OPENCODE_LOG" ] || fail "OpenCode 没有返回执行事件" + pass + printf 'OPENCODE_METRIC phase=%s version=%s model=%s seconds=%s\n' \ + "$PHASE" "$("$OPENCODE" --version 2>/dev/null | tail -1)" "$MODEL" \ + "$((ended - started))" +} + +assert_mac1_state() { + "$PYTHON" -m unittest -v test_calc.py >/dev/null + pass + ! grep -q NotImplementedError calc.py || fail "multiply 仍未实现" + pass +} + +case "$PHASE" in + mac1-create) + vexfs setup >/dev/null + mount_workspace + mkdir -p "$PROJECT" + cd "$PROJECT" + cat >calc.py <<'PY' +def add(left, right): + return left + right + + +def multiply(left, right): + """Return left multiplied by right.""" + raise NotImplementedError("OpenCode must implement multiply") +PY + cat >test_calc.py <<'PY' +import unittest + +from calc import add, multiply + + +class CalcTest(unittest.TestCase): + def test_add(self): + self.assertEqual(add(2, 5), 7) + + def test_multiply(self): + self.assertEqual(multiply(6, 7), 42) + self.assertEqual(multiply(-3, 4), -12) + self.assertEqual(multiply(10, 0), 0) + + +if __name__ == "__main__": + unittest.main() +PY + cat >TASK_MAC1.md <<'MD' +Implement multiply in calc.py. Keep add unchanged and make all unit tests pass. +MD + "$GIT" init -b main >/dev/null + "$GIT" config user.name "VexFS Two-Mac Eval" + "$GIT" config user.email "eval@example.invalid" + "$GIT" add . + "$GIT" commit -m "shared workspace fixture" >/dev/null + run_opencode \ + "Work only in this repository. Read TASK_MAC1.md, edit only calc.py, and run /usr/bin/python3 -m unittest -v. Do not ask questions and do not commit." + assert_mac1_state + equals "$("$GIT" diff --name-only)" "calc.py" "第一台 OpenCode 修改范围" + "$GIT" add calc.py + "$GIT" commit -m "mac1 opencode" >/dev/null + contains "$("$GIT" log -1 --format=%s)" "mac1 opencode" "第一台 Mac Git 提交" + unmount_workspace + vexfs snapshot create mac1-opencode >/dev/null + contains "$(vexfs snapshot list)" "mac1-opencode" "第一台工作区快照" + ;; + + mac2-modify) + mount_workspace + cd "$PROJECT" + assert_mac1_state + contains "$("$GIT" log --format=%s --all)" "mac1 opencode" "继承第一台 Mac Git 提交" + cat >test_power.py <<'PY' +import unittest + +from calc import power + + +class PowerTest(unittest.TestCase): + def test_power(self): + self.assertEqual(power(2, 10), 1024) + self.assertEqual(power(5, 0), 1) + self.assertEqual(power(-3, 3), -27) + + +if __name__ == "__main__": + unittest.main() +PY + cat >TASK_MAC2.md <<'MD' +Add power(base, exponent) to calc.py for non-negative integer exponents. Preserve existing behavior. +MD + "$GIT" add test_power.py TASK_MAC2.md + "$GIT" commit -m "mac2 task fixture" >/dev/null + run_opencode \ + "This repository was created by another Mac. Read TASK_MAC2.md, edit only calc.py, and run /usr/bin/python3 -m unittest -v. Do not ask questions and do not commit." + "$PYTHON" -m unittest -v >/dev/null + pass + equals "$("$GIT" diff --name-only)" "calc.py" "第二台 OpenCode 修改范围" + contains "$(cat calc.py)" "def power" "第二台 OpenCode 实现" + "$GIT" add calc.py + "$GIT" commit -m "mac2 opencode" >/dev/null + unmount_workspace + vexfs snapshot create mac2-opencode >/dev/null + contains "$(vexfs snapshot list)" "mac2-opencode" "第二台工作区快照" + ;; + + mac1-verify-restore) + mount_workspace + cd "$PROJECT" + "$PYTHON" -m unittest -v >/dev/null + pass + contains "$("$GIT" log --format=%s --all)" "mac1 opencode" "继承第一台提交" + contains "$("$GIT" log --format=%s --all)" "mac2 opencode" "继承第二台提交" + contains "$(cat calc.py)" "def multiply" "继承 multiply" + contains "$(cat calc.py)" "def power" "继承 power" + native_grep="$(grep -n 'def power' calc.py)" + contains "$native_grep" "def power" "挂载目录原生 grep" + unmount_workspace + + snapshots="$(vexfs snapshot list)" + contains "$snapshots" "mac1-opencode" "版本列表包含第一台快照" + contains "$snapshots" "mac2-opencode" "版本列表包含第二台快照" + diff_status=0 + diff_output="$(vexfs snapshot diff mac1-opencode --to mac2-opencode)" || diff_status=$? + equals "$diff_status" "1" "两个 workspace 版本存在差异" + contains "$diff_output" "calc.py" "两个 workspace 版本的差异" + history_json="$(vexfs --json history /agent-project/calc.py --limit 100)" + history_count="$(printf '%s' "$history_json" | "$PYTHON" -c 'import json,sys; print(len(json.load(sys.stdin)))')" + [ "$history_count" -ge 1 ] || fail "calc.py 没有可查看的历史版本" + pass + db_grep="$(vexfs --json grep power /agent-project)" + contains "$db_grep" '"match_count":' "数据库 grep" + contains "$db_grep" '"index_used":false' "PG grep 如实报告未用索引" + index_status="$(vexfs index status)" + contains "$index_status" '"available":false' "PG 索引能力边界" + + vexfs snapshot restore mac1-opencode >/dev/null + mount_workspace + cd "$PROJECT" + "$PYTHON" -m unittest -v test_calc.py >/dev/null + pass + [ ! -e test_power.py ] || fail "一键恢复后仍存在第二台测试文件" + pass + ! grep -q 'def power' calc.py || fail "一键恢复后仍存在第二台实现" + pass + equals "$("$GIT" log -1 --format=%s)" "mac1 opencode" "一键恢复后的 Git workspace" + unmount_workspace + contains "$(vexfs check)" "OK workspace=$WORKSPACE" "恢复后 deep check" + printf 'VERSION_METRIC workspace=%s snapshots=2 calc_history=%s grep_index_available=false\n' \ + "$WORKSPACE" "$history_count" + ;; + + *) + fail "未知节点阶段：$PHASE" + ;; +esac + +echo "VEXFS PG TWO-MAC OPENCODE NODE: PASS (phase=$PHASE, $CHECKS checks)" diff --git a/tests/eval/vexfs/run_pg_version_matrix.sh b/tests/eval/vexfs/run_pg_version_matrix.sh new file mode 100755 index 0000000000..e9a3e16c2f --- /dev/null +++ b/tests/eval/vexfs/run_pg_version_matrix.sh @@ -0,0 +1,173 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" +VERSIONS="${VEXDB_PG_MATRIX_VERSIONS:-16 17 18 19}" +MEMORY="${VEXFS_DOCKER_MEMORY:-1g}" +MEMORY_SWAP="${VEXFS_DOCKER_MEMORY_SWAP:-1g}" +CONTEXT="${TMPDIR:-/tmp}/vexfs-pg-version-matrix-context" +CHECKS=0 +ACTIVE_CONTAINER="" +ACTIVE_CONTAINER_OWNED=0 +PG19_CONTAINER="${VEXDB_PG19_MATRIX_CONTAINER:-${VEXDB_PG_CONTAINER:-}}" +PYTHON_BIN="${VEXDB_LITE_PYTHON:-}" + +if [ -n "$PYTHON_BIN" ]; then + [ -x "$PYTHON_BIN" ] || { + echo "VEXDB_LITE_PYTHON 不可执行：$PYTHON_BIN" >&2 + exit 2 + } + "$PYTHON_BIN" -c 'import sys; raise SystemExit(sys.version_info < (3, 8))' || { + echo "VEXDB_LITE_PYTHON 需要 Python 3.8 或更高版本：$PYTHON_BIN" >&2 + exit 2 + } +else + for candidate in \ + "$(command -v python3 2>/dev/null || true)" \ + /opt/homebrew/bin/python3 \ + /opt/anaconda3/bin/python3 \ + /usr/bin/python3; do + [ -n "$candidate" ] && [ -x "$candidate" ] || continue + if "$candidate" -c \ + 'import sys; raise SystemExit(sys.version_info < (3, 8))' \ + >/dev/null 2>&1; then + PYTHON_BIN="$candidate" + break + fi + done + [ -n "$PYTHON_BIN" ] || { + echo "PG 版本矩阵需要 Python 3.8 或更高版本" >&2 + exit 2 + } +fi + +cleanup_container() { + if [ -n "$ACTIVE_CONTAINER" ] && [ "$ACTIVE_CONTAINER_OWNED" -eq 1 ]; then + docker rm -f "$ACTIVE_CONTAINER" >/dev/null 2>&1 || true + fi + ACTIVE_CONTAINER="" + ACTIVE_CONTAINER_OWNED=0 +} + +cleanup() { + local status=$? + trap - EXIT INT TERM + cleanup_container + rm -rf "$CONTEXT" + exit "$status" +} +trap cleanup EXIT INT TERM + +case "$MEMORY" in ''|*[!0-9gGmMkK.]*) echo "VEXFS_DOCKER_MEMORY 格式无效" >&2; exit 2;; esac +case " $VERSIONS " in + *" 19 "*) + [ -n "$PG19_CONTAINER" ] || { + echo "PG19 尚无官方 postgres:19 镜像；请用 VEXDB_PG19_MATRIX_CONTAINER 指定已安装当前扩展的受控 PG19 测试容器" >&2 + exit 2 + } + ;; +esac +rm -rf "$CONTEXT" +mkdir -p "$CONTEXT/vexdb_lite_src" +cp "$ROOT/tests/eval/vexfs/Dockerfile.pg-version-matrix" "$CONTEXT/Dockerfile" +( + cd "$ROOT" + tar -cf - common vexdb_pg thirdparties +) | ( + cd "$CONTEXT/vexdb_lite_src" + tar -xf - +) + +"$PYTHON_BIN" "$ROOT/tests/spec/_lib/render.py" --engine pg --out "$ROOT/build/spec" + +for version in $VERSIONS; do + case "$version" in 16|17|18|19) ;; *) echo "不支持 PG $version" >&2; exit 2;; esac + image="vexdb-pg-vexfs-matrix:$version" + container="vexdb-pg-vexfs-matrix-$version" + cleanup_container + if [ "$version" = 19 ] && [ -n "$PG19_CONTAINER" ]; then + container="$PG19_CONTAINER" + docker inspect "$container" >/dev/null + ACTIVE_CONTAINER="$container" + ACTIVE_CONTAINER_OWNED=0 + else + docker rm -f "$container" >/dev/null 2>&1 || true + docker build \ + --build-arg "PG_MAJOR=$version" \ + --build-arg JOBS=1 \ + --tag "$image" \ + "$CONTEXT" + + docker run -d \ + --name "$container" \ + --memory "$MEMORY" \ + --memory-swap "$MEMORY_SWAP" \ + --env POSTGRES_HOST_AUTH_METHOD=trust \ + --env POSTGRES_DB=test \ + "$image" \ + -c shared_preload_libraries=vexdb_lite >/dev/null + ACTIVE_CONTAINER="$container" + ACTIVE_CONTAINER_OWNED=1 + + initialized=0 + for _ in $(seq 1 240); do + if docker logs "$container" 2>&1 | \ + grep -q 'PostgreSQL init process complete'; then + initialized=1 + break + fi + sleep 0.25 + done + if [ "$initialized" -ne 1 ]; then + docker logs --tail 200 "$container" >&2 + echo "PG $version 初始化超时" >&2 + exit 1 + fi + fi + + ready=0 + for _ in $(seq 1 120); do + if docker exec "$container" psql -U postgres -d test -X -q -t -A \ + -c 'SELECT 1' 2>/dev/null | grep -qx 1; then + ready=1 + break + fi + sleep 0.25 + done + if [ "$ready" -ne 1 ]; then + docker logs --tail 200 "$container" >&2 + echo "PG $version 启动超时" >&2 + exit 1 + fi + MEMORY_MAX="$(docker exec "$container" cat /sys/fs/cgroup/memory.max)" + if [[ ! "$MEMORY_MAX" =~ ^[0-9]+$ ]] || [ "$MEMORY_MAX" -gt 1073741824 ]; then + echo "PG $version 容器必须设置不超过 1 GiB 的 memory.max，实际 $MEMORY_MAX" >&2 + exit 1 + fi + OOM_BEFORE="$(docker exec "$container" awk '$1=="oom_kill" {print $2}' \ + /sys/fs/cgroup/memory.events)" + docker exec "$container" psql -U postgres -d test -X -q -v ON_ERROR_STOP=1 \ + -c "CREATE EXTENSION IF NOT EXISTS vexdb_lite VERSION '1.0';" >/dev/null + + actual_version="$(docker exec "$container" psql -U postgres -d test -X -q -t -A \ + -c "SELECT current_setting('server_version_num')::integer / 10000, + vexfs_pg_adapter_version();")" + if [[ "$actual_version" != "$version|0.4.0-alpha.1" ]]; then + echo "PG $version 版本/adapter 错误：$actual_version" >&2 + exit 1 + fi + CHECKS=$((CHECKS + 2)) + + VEXDB_PG_CONTAINER="$container" \ + bash "$ROOT/tests/spec/_lib/docker/run_pg.sh" test 'pg__vexfs_*' + CHECKS=$((CHECKS + 7)) + OOM_AFTER="$(docker exec "$container" awk '$1=="oom_kill" {print $2}' \ + /sys/fs/cgroup/memory.events)" + if [ "$OOM_AFTER" != "$OOM_BEFORE" ]; then + echo "PG $version 矩阵触发 OOM kill：$OOM_BEFORE -> $OOM_AFTER" >&2 + exit 1 + fi + cleanup_container +done + +echo "VEXFS PG 16-19 VERSION MATRIX: PASS ($CHECKS checks)" diff --git a/tests/spec/_lib/docker/run_pg.sh b/tests/spec/_lib/docker/run_pg.sh index 41b2eff818..55ca00546d 100755 --- a/tests/spec/_lib/docker/run_pg.sh +++ b/tests/spec/_lib/docker/run_pg.sh @@ -19,8 +19,33 @@ ROOT_DIR="$(cd "$(dirname "$0")/../../../.." && pwd)" PG_VEXDB_SRC="${PG_VEXDB_SRC:-$ROOT_DIR}" SPEC_DIR="${ROOT_DIR}/build/spec/pg" RESULTS_DIR="${SPEC_DIR}/results" -IMAGE="vexdb_pg19:latest" -CONTAINER="vexdb_pg19-test" +IMAGE="${VEXDB_PG_IMAGE:-vexdb_pg19:latest}" +CONTAINER="${VEXDB_PG_CONTAINER:-vexdb_pg19-test}" +DATABASE="${VEXDB_PG_DATABASE:-test}" +PYTHON_BIN="${VEXDB_LITE_PYTHON:-}" + +if [[ -n "$PYTHON_BIN" ]]; then + [[ -x "$PYTHON_BIN" ]] || { echo "VEXDB_LITE_PYTHON 不可执行：$PYTHON_BIN" >&2; exit 2; } + "$PYTHON_BIN" -c 'import sys; raise SystemExit(sys.version_info < (3, 8))' || { + echo "VEXDB_LITE_PYTHON 需要 Python 3.8 或更高版本：$PYTHON_BIN" >&2 + exit 2 + } +else + for candidate in \ + "$(command -v python3 2>/dev/null || true)" \ + /opt/homebrew/bin/python3 \ + /opt/anaconda3/bin/python3 \ + /usr/bin/python3; do + [[ -n "$candidate" && -x "$candidate" ]] || continue + if "$candidate" -c \ + 'import sys; raise SystemExit(sys.version_info < (3, 8))' \ + >/dev/null 2>&1; then + PYTHON_BIN="$candidate" + break + fi + done + [[ -n "$PYTHON_BIN" ]] || { echo "PG spec 需要 Python 3.8 或更高版本" >&2; exit 2; } +fi YEL=$'\033[1;33m'; GRN=$'\033[0;32m'; RED=$'\033[0;31m'; NC=$'\033[0m' info() { printf '%s[pg]%s %s\n' "$YEL" "$NC" "$*"; } @@ -83,8 +108,8 @@ cmd_up() { fi info "等待 PG ready..." for _ in $(seq 1 30); do - if docker exec "$CONTAINER" psql -d test -c 'SELECT 1' >/dev/null 2>&1; then - ok "PG ready on localhost:5433 db=test (容器内 5432)" + if docker exec "$CONTAINER" psql -U postgres -d "$DATABASE" -c 'SELECT 1' >/dev/null 2>&1; then + ok "PG ready on localhost:5433 db=$DATABASE (容器内 5432)" return 0 fi sleep 1 @@ -99,7 +124,7 @@ cmd_down() { } cmd_shell() { - docker exec -it "$CONTAINER" psql -d test + docker exec -it "$CONTAINER" psql -U postgres -d "$DATABASE" } cmd_logs() { @@ -130,7 +155,7 @@ cmd_test() { # -X: 不读 .psqlrc; -q: quiet (不输出 CREATE TABLE 等); -t: 不带 header/footer # -A: unaligned; -F '|': 字段分隔符; --no-psqlrc; ON_ERROR_STOP 失败立即停 # 每个 spec 跑前: 删除 public schema 下所有用户表 (保留 extension) - docker exec -i "$CONTAINER" psql -d test -X -q -t -A -F '|' -P pager=off > /dev/null 2>&1 <<'EOF' + docker exec -i "$CONTAINER" psql -U postgres -d "$DATABASE" -X -q -t -A -F '|' -P pager=off > /dev/null 2>&1 <<'EOF' DO $$ DECLARE r record; BEGIN @@ -139,7 +164,7 @@ BEGIN END LOOP; END $$; EOF - docker exec -i "$CONTAINER" psql -d test -X -q -t -A -F '|' -P pager=off \ + docker exec -i "$CONTAINER" psql -U postgres -d "$DATABASE" -X -q -t -A -F '|' -P pager=off \ > "$actual" 2>&1 < "$sql_file" || true # 过滤 psql 命令完成消息 + 服务端日志级别消息 + 报错位置标记 (^ / LINE). # 单条 ERE 替代之前 23 个 -e 模式. @@ -154,7 +179,7 @@ EOF awk 'NF || prev_nf { print } { prev_nf = NF }' "$actual" > "${actual}.tmp" && mv "${actual}.tmp" "$actual" if [[ -f "$expected" ]]; then - if python3 "${ROOT_DIR}/tests/spec/_lib/docker/compare.py" "$expected" "$actual" 2>"$diff_file"; then + if "$PYTHON_BIN" "${ROOT_DIR}/tests/spec/_lib/docker/compare.py" "$expected" "$actual" 2>"$diff_file"; then pass=$((pass+1)) rm -f "$diff_file" printf ' %s%s%s %s\n' "$GRN" "PASS" "$NC" "$name" diff --git a/tests/spec/_lib/render.py b/tests/spec/_lib/render.py index 8d9e08a0c3..d433c99bcb 100644 --- a/tests/spec/_lib/render.py +++ b/tests/spec/_lib/render.py @@ -311,6 +311,7 @@ def emit_pg(spec: dict, dialect: dict) -> tuple[str, str]: sql_lines = [ f"-- spec: {spec['name']}", "-- 抑制 NOTICE/WARNING (扩展输出噪声), 让 expected 纯净", + r"\set VERBOSITY terse", "SET client_min_messages = ERROR;", "", ] diff --git a/tests/spec/pg/agent_files/vexfs_pg_archive_export.yaml b/tests/spec/pg/agent_files/vexfs_pg_archive_export.yaml new file mode 100644 index 0000000000..63f659fc29 --- /dev/null +++ b/tests/spec/pg/agent_files/vexfs_pg_archive_export.yaml @@ -0,0 +1,101 @@ +name: pg__vexfs_archive_export +tags: [vexfs, pg, archive, format-v2] +description: PostgreSQL 通过公开合同在单一事务快照中生成 format v2 逻辑记录 +steps: +- statement: TRUNCATE _vexfs.workspaces CASCADE; +- query: SELECT vexfs_workspace_create('pg-archive') > 0; + expect: + - - true +- query: SELECT vexfs_mkdir('pg-archive', '/tree', true) > 0; + expect: + - - true +- query: SELECT vexfs_write('pg-archive', '/tree/value.txt', convert_to('before', 'UTF8')); + expect: + - - 1 +- query: SELECT vexfs_link('pg-archive', '/tree/value.txt', '/tree/value-hard.txt') > 0; + expect: + - - true +- query: SELECT vexfs_symlink('pg-archive', '/tree/current', convert_to('value.txt', 'UTF8')) > 0; + expect: + - - true +- query: SELECT vexfs_xattr_set('pg-archive', (vexfs_stat('pg-archive', '/tree/value.txt')->>'inode')::bigint, 'user.archive', convert_to('portable', 'UTF8'), 1); + expect: + - - 1 +- query: SELECT vexfs_acl_grant('pg-archive', (vexfs_stat('pg-archive', '/tree/value.txt')->>'inode')::bigint, 'public', 'read', 'allow', 0); + expect: + - - 1 +- query: SELECT vexfs_snapshot_create('pg-archive', 'baseline'); + expect: + - - 7 +- query: SELECT vexfs_write('pg-archive', '/tree/value.txt', convert_to('after', 'UTF8')); + expect: + - - 2 +- query: SELECT vexfs_remove('pg-archive', '/tree/value-hard.txt', false); + expect: + - - 1 +- query: SELECT vexfs_xattr_set('pg-archive', (vexfs_stat('pg-archive', '/tree/value.txt')->>'inode')::bigint, 'user.archive', NULL, 3); + expect: + - - 1 +- query: SELECT vexfs_acl_revoke('pg-archive', (vexfs_stat('pg-archive', '/tree/value.txt')->>'inode')::bigint, 'public', 'allow'); + expect: + - - 1 +- statement: CREATE TEMP TABLE pg_archive_head AS SELECT * FROM vexfs_archive_export_records('pg-archive', NULL); +- query: SELECT record_json->>'source_engine', (record_json->>'format_version')::integer, (record_json->>'source_commit')::bigint FROM pg_archive_head WHERE record_type='manifest'; + expect: + - - postgresql + - 2 + - 11 +- query: SELECT record_type, count(*) FROM pg_archive_head GROUP BY record_type ORDER BY record_type COLLATE "C"; + expect: + - - acl_states + - 2 + - - chunks + - 3 + - - commits + - 11 + - - dentries + - 3 + - - dentry_states + - 8 + - - file_versions + - 3 + - - inode_states + - 8 + - - inodes + - 4 + - - manifest + - 1 + - - manifests + - 3 + - - principals + - 2 + - - snapshots + - 1 + - - xattr_states + - 2 +- query: SELECT count(*) FROM pg_archive_head WHERE record_type='chunks' AND encode(sha256(content),'hex') = record_json->>'checksum'; + expect: + - - 3 +- query: SELECT count(*) FROM pg_archive_head WHERE record_type='dentry_states' AND (record_json->>'deleted')::integer=1 AND record_json->>'name'='value-hard.txt'; + expect: + - - 1 +- query: SELECT count(*) FROM pg_archive_head WHERE record_type='xattr_states' AND (record_json->>'deleted')::integer=1; + expect: + - - 1 +- query: SELECT count(*) FROM pg_archive_head WHERE record_type='acl_states' AND (record_json->>'deleted')::integer=1; + expect: + - - 1 +- statement: CREATE TEMP TABLE pg_archive_baseline AS SELECT * FROM vexfs_archive_export_records('pg-archive', 'baseline'); +- query: SELECT (record_json->>'source_commit')::bigint, record_json->>'source_snapshot' FROM pg_archive_baseline WHERE record_type='manifest'; + expect: + - - 7 + - baseline +- query: SELECT count(*) FROM pg_archive_baseline WHERE record_type='dentries' AND record_json->>'name'='value-hard.txt'; + expect: + - - 1 +- query: SELECT count(*) FROM pg_archive_baseline WHERE record_type='file_versions'; + expect: + - - 2 +- query: SELECT (vexfs_check('pg-archive', 1)->>'ok')::boolean; + expect: + - - true diff --git a/tests/spec/pg/agent_files/vexfs_pg_chunk_check.yaml b/tests/spec/pg/agent_files/vexfs_pg_chunk_check.yaml new file mode 100644 index 0000000000..9e88c691c3 --- /dev/null +++ b/tests/spec/pg/agent_files/vexfs_pg_chunk_check.yaml @@ -0,0 +1,73 @@ +name: pg__vexfs_chunk_check +tags: [vexfs, pg, chunk, checksum, integrity] +description: PostgreSQL VexFS 64 KiB chunk/manifest、块复用、恢复别名和 quick/deep check +steps: +- statement: TRUNCATE _vexfs.workspaces CASCADE; +- query: SELECT vexfs_pg_adapter_version(); + expect: + - - 0.4.0-alpha.1 +- query: SELECT vexfs_workspace_create('pg-a2') > 0; + expect: + - - true +- query: SELECT vexfs_write('pg-a2', '/large.bin', convert_to(repeat('a', 65536) || repeat('b', 65536) || repeat('c', 7), 'UTF8')); + expect: + - - 1 +- query: SELECT octet_length(vexfs_read('pg-a2', '/large.bin')), vexfs_stat('pg-a2', '/large.bin')->>'checksum'; + expect: + - - 131079 + - 57ad989627081a0270bb309d3f9272ae22f302aedc2b907dfd1e676741bc535d +- query: SELECT m.chunk_size, m.chunk_count, count(entry.chunk_id), count(DISTINCT entry.chunk_id) FROM _vexfs.file_versions AS f JOIN _vexfs.manifests AS m ON m.manifest_id=f.manifest_id LEFT JOIN _vexfs.manifest_chunks AS entry ON entry.manifest_id=m.manifest_id WHERE f.workspace_id=(SELECT workspace_id FROM _vexfs.workspaces WHERE name='pg-a2') AND f.version_no=1 GROUP BY m.chunk_size,m.chunk_count; + expect: + - - 65536 + - 3 + - 3 + - 3 +- query: SELECT vexfs_write('pg-a2', '/large.bin', convert_to(repeat('a', 65536) || repeat('b', 65536) || repeat('d', 7), 'UTF8')); + expect: + - - 2 +- query: SELECT (SELECT count(*) FROM _vexfs.manifests WHERE workspace_id=w.workspace_id), (SELECT count(*) FROM _vexfs.chunks WHERE workspace_id=w.workspace_id), (SELECT count(*) FROM _vexfs.manifest_chunks AS first_entry JOIN _vexfs.manifest_chunks AS second_entry ON second_entry.chunk_no=first_entry.chunk_no AND second_entry.chunk_id=first_entry.chunk_id WHERE first_entry.manifest_id=(SELECT manifest_id FROM _vexfs.file_versions WHERE workspace_id=w.workspace_id AND version_no=1) AND second_entry.manifest_id=(SELECT manifest_id FROM _vexfs.file_versions WHERE workspace_id=w.workspace_id AND version_no=2)) FROM _vexfs.workspaces AS w WHERE w.name='pg-a2'; + expect: + - - 2 + - 4 + - 2 +- query: SELECT vexfs_read('pg-a2', '/large.bin') = convert_to(repeat('a', 65536) || repeat('b', 65536) || repeat('d', 7), 'UTF8'); + expect: + - - true +- query: SELECT length(checksum), current::integer FROM vexfs_history('pg-a2', '/large.bin', 1, NULL); + expect: + - - 64 + - 1 +- query: SELECT vexfs_restore_version('pg-a2', '/large.bin', 1, 2); + expect: + - - 3 +- query: SELECT manifest_id IS NULL, source_version_no, checksum FROM _vexfs.file_versions WHERE workspace_id=(SELECT workspace_id FROM _vexfs.workspaces WHERE name='pg-a2') AND version_no=3; + expect: + - - true + - 1 + - 57ad989627081a0270bb309d3f9272ae22f302aedc2b907dfd1e676741bc535d +- query: SELECT vexfs_read('pg-a2', '/large.bin') = convert_to(repeat('a', 65536) || repeat('b', 65536) || repeat('c', 7), 'UTF8'); + expect: + - - true +- query: SELECT (vexfs_check('pg-a2', 0)->>'ok')::boolean, vexfs_check('pg-a2', 0)->>'mode', vexfs_check('pg-a2', 0)->>'content_model'; + expect: + - - true + - quick + - chunked-v1 +- query: SELECT (vexfs_check('pg-a2', 1)->>'ok')::boolean, (vexfs_check('pg-a2', 1)->'checked'->>'canonical_versions')::bigint, (vexfs_check('pg-a2', 1)->>'issue_count')::integer; + expect: + - - true + - 2 + - 0 +- statement: UPDATE _vexfs.chunks SET content=set_byte(content, 0, (get_byte(content, 0)+1)%256) WHERE chunk_id=(SELECT entry.chunk_id FROM _vexfs.file_versions AS f JOIN _vexfs.manifest_chunks AS entry ON entry.manifest_id=f.manifest_id WHERE f.workspace_id=(SELECT workspace_id FROM _vexfs.workspaces WHERE name='pg-a2') AND f.version_no=1 ORDER BY entry.chunk_no LIMIT 1); +- query: SELECT (vexfs_check('pg-a2', 0)->>'ok')::boolean, (vexfs_check('pg-a2', 1)->>'ok')::boolean, (vexfs_check('pg-a2', 1)->>'issue_count')::integer > 0; + expect: + - - true + - false + - true +- query: SELECT vexfs_read('pg-a2', '/large.bin'); + expect_error: VEXFS_CHECKSUM_MISMATCH +- query: SELECT vexfs_check('pg-a2', 2); + expect_error: VEXFS_INVALID_CHECK_MODE +- query: SELECT vexfs_workspace_drop('pg-a2'); + expect: + - - 1 diff --git a/tests/spec/pg/agent_files/vexfs_pg_grep_index.yaml b/tests/spec/pg/agent_files/vexfs_pg_grep_index.yaml new file mode 100644 index 0000000000..53073c8f36 --- /dev/null +++ b/tests/spec/pg/agent_files/vexfs_pg_grep_index.yaml @@ -0,0 +1,99 @@ +name: pg__vexfs_grep_index +tags: [vexfs, pg, grep, index, pg_trgm] +description: PostgreSQL VexFS 可选 trigram 子串索引、回退扫描和快照恢复维护 +steps: +- statement: CREATE EXTENSION IF NOT EXISTS pg_trgm; +- statement: TRUNCATE _vexfs.workspaces CASCADE; +- query: SELECT vexfs_grep_index('status')->>'backend', (vexfs_grep_index('status')->>'available')::boolean; + expect: + - - pg-trgm + - true +- query: SELECT vexfs_grep_index('enable'); + expect_error: VEXFS_WORKSPACE_REQUIRED +- query: SELECT vexfs_workspace_create('pg-grep') > 0; + expect: + - - true +- query: SELECT vexfs_mkdir('pg-grep', '/src') > 0; + expect: + - - true +- query: SELECT vexfs_write('pg-grep', '/src/a.txt', convert_to('Alpha needle 100% foo_bar C:\tmp' || E'\n' || 'second needle', 'UTF8')); + expect: + - - 1 +- query: SELECT vexfs_write('pg-grep', '/src/b.txt', convert_to('ordinary content', 'UTF8')); + expect: + - - 1 +- query: SELECT vexfs_write('pg-grep', '/src/bin.dat', decode('ff00fe', 'hex')); + expect: + - - 1 +- query: SELECT (vexfs_grep('pg-grep', '/src', 'needle', 0, 100)->>'index_used')::boolean, (vexfs_grep('pg-grep', '/src', 'needle', 0, 100)->>'match_count')::integer, (vexfs_grep('pg-grep', '/src', 'needle', 0, 100)->>'binary_files_skipped')::integer; + expect: + - - false + - 2 + - 1 +- query: SELECT (vexfs_grep_index('pg-grep', 'enable')->>'enabled')::boolean, (vexfs_grep_index('pg-grep', 'status')->>'dirty')::boolean, (vexfs_grep_index('pg-grep', 'status')->>'indexed_files')::integer; + expect: + - - true + - false + - 2 +- query: SELECT (vexfs_grep('pg-grep', '/src', 'needle', 0, 100)->>'index_used')::boolean, (vexfs_grep('pg-grep', '/src', 'needle', 0, 100)->>'match_count')::integer, (vexfs_grep('pg-grep', '/src', 'needle', 0, 100)->>'files_scanned')::integer; + expect: + - - true + - 2 + - 1 +- query: SELECT (vexfs_grep('pg-grep', '/src', 'alpha', 1, 100)->>'match_count')::integer, (vexfs_grep('pg-grep', '/src', '100%', 0, 100)->>'match_count')::integer, (vexfs_grep('pg-grep', '/src', 'foo_bar', 0, 100)->>'match_count')::integer, (vexfs_grep('pg-grep', '/src', 'C:\tmp', 0, 100)->>'match_count')::integer; + expect: + - - 1 + - 1 + - 1 + - 1 +- query: SELECT (vexfs_grep('pg-grep', '/src', 'ne', 0, 100)->>'index_used')::boolean, (vexfs_grep('pg-grep', '/src', 'ne', 0, 100)->>'match_count')::integer; + expect: + - - false + - 2 +- query: SELECT vexfs_snapshot_create('pg-grep', 'baseline'); + expect: + - - 5 +- query: SELECT vexfs_write('pg-grep', '/src/a.txt', convert_to('replacement haystack', 'UTF8')); + expect: + - - 2 +- query: SELECT vexfs_write('pg-grep', '/src/new.txt', convert_to('new needle file', 'UTF8')); + expect: + - - 1 +- query: SELECT (vexfs_grep('pg-grep', '/src', 'needle', 0, 100)->>'match_count')::integer, vexfs_grep('pg-grep', '/src', 'needle', 0, 100)->'matches'->0->>'path'; + expect: + - - 1 + - /src/new.txt +- query: SELECT vexfs_snapshot_restore('pg-grep', 'baseline', 7); + expect: + - - 8 +- query: SELECT (vexfs_grep('pg-grep', '/src', 'needle', 0, 100)->>'index_used')::boolean, (vexfs_grep('pg-grep', '/src', 'needle', 0, 100)->>'match_count')::integer, vexfs_grep('pg-grep', '/src', 'needle', 0, 100)->'matches'->0->>'path'; + expect: + - - true + - 2 + - /src/a.txt +- statement: UPDATE _vexfs.workspaces SET grep_index_dirty=true WHERE name='pg-grep'; +- query: SELECT (vexfs_grep('pg-grep', '/src', 'needle', 0, 100)->>'index_used')::boolean, (vexfs_grep_index('pg-grep', 'status')->>'dirty')::boolean; + expect: + - - false + - true +- query: SELECT (vexfs_grep_index('pg-grep', 'rebuild')->>'dirty')::boolean, (vexfs_grep_index('pg-grep', 'status')->>'indexed_files')::integer; + expect: + - - false + - 2 +- query: SELECT vexfs_remove('pg-grep', '/src/a.txt', false); + expect: + - - 1 +- query: SELECT (vexfs_grep('pg-grep', '/src', 'needle', 0, 100)->>'index_used')::boolean, (vexfs_grep('pg-grep', '/src', 'needle', 0, 100)->>'match_count')::integer, (vexfs_grep_index('pg-grep', 'status')->>'indexed_files')::integer; + expect: + - - true + - 0 + - 1 +- query: SELECT (vexfs_grep_index('pg-grep', 'disable')->>'enabled')::boolean; + expect: + - - false +- query: SELECT (vexfs_grep('pg-grep', '/src', 'ordinary', 0, 100)->>'index_used')::boolean; + expect: + - - false +- query: SELECT vexfs_workspace_drop('pg-grep'); + expect: + - - 1 diff --git a/tests/spec/pg/agent_files/vexfs_pg_handles_gateway.yaml b/tests/spec/pg/agent_files/vexfs_pg_handles_gateway.yaml new file mode 100644 index 0000000000..926ff54ae4 --- /dev/null +++ b/tests/spec/pg/agent_files/vexfs_pg_handles_gateway.yaml @@ -0,0 +1,189 @@ +name: pg__vexfs_handles_gateway +tags: [vexfs, pg, handles, gateway, idempotency, locks] +description: PostgreSQL 持久句柄、未发布隔离、多网关冲突、追加、恢复和文件锁合同 +steps: +- statement: TRUNCATE _vexfs.workspaces CASCADE; +- query: SELECT vexfs_workspace_create('pg-handles') > 0; + expect: + - - true +- query: SELECT vexfs_write('pg-handles', '/value.txt', convert_to('base', 'UTF8')); + expect: + - - 1 +- query: SELECT vexfs_mount_session_start('pg-handles', 'gateway-a'); + expect: + - - 1 +- query: SELECT vexfs_mount_session_start('pg-handles', 'gateway-b'); + expect: + - - 1 +- statement: CREATE TEMP TABLE handle_ids(name text PRIMARY KEY, handle text NOT NULL); +- statement: INSERT INTO handle_ids VALUES ('reader', vexfs_handle_open('pg-handles', '/value.txt', 'r', 'open-reader', 'gateway-a')); +- statement: INSERT INTO handle_ids VALUES ('a', vexfs_handle_open('pg-handles', '/value.txt', 'rw', 'open-a', 'gateway-a')); +- statement: INSERT INTO handle_ids VALUES ('b', vexfs_handle_open('pg-handles', '/value.txt', 'rw', 'open-b', 'gateway-b')); +- query: SELECT vexfs_handle_stage_write((SELECT handle FROM handle_ids WHERE name='a'), 0, convert_to('first', 'UTF8'), 'stage-a'); + expect: + - - 1 +- query: SELECT vexfs_handle_stage_write((SELECT handle FROM handle_ids WHERE name='a'), 0, convert_to('first', 'UTF8'), 'stage-a'); + expect: + - - 1 +- query: SELECT vexfs_handle_stage_write((SELECT handle FROM handle_ids WHERE name='a'), 0, convert_to('other', 'UTF8'), 'stage-a'); + expect_error: VEXFS_REQUEST_CONFLICT +- query: SELECT vexfs_handle_stage_write((SELECT handle FROM handle_ids WHERE name='b'), 0, convert_to('second', 'UTF8'), 'stage-b'); + expect: + - - 1 +- query: SELECT convert_from(vexfs_read('pg-handles', '/value.txt'), 'UTF8'), convert_from(vexfs_handle_read((SELECT handle FROM handle_ids WHERE name='a'), 0, 16), 'UTF8'); + expect: + - - base + - first +- query: SELECT vexfs_handle_publish((SELECT handle FROM handle_ids WHERE name='a'), 1, 'data', 'publish-a'); + expect: + - - 2 +- query: SELECT convert_from(vexfs_handle_read((SELECT handle FROM handle_ids WHERE name='reader'), 0, 16), 'UTF8'); + expect: + - - first +- query: SELECT vexfs_handle_publish((SELECT handle FROM handle_ids WHERE name='a'), 1, 'data', 'publish-a'); + expect: + - - 2 +- query: SELECT vexfs_handle_publish((SELECT handle FROM handle_ids WHERE name='a'), 1, 'full', 'publish-a'); + expect_error: VEXFS_REQUEST_CONFLICT +- query: SELECT vexfs_handle_publish((SELECT handle FROM handle_ids WHERE name='b'), 1, 'data', 'publish-b'); + expect_error: VEXFS_VERSION_CONFLICT +- query: SELECT convert_from(vexfs_read('pg-handles', '/value.txt'), 'UTF8'); + expect: + - - first +- query: SELECT vexfs_handle_close((SELECT handle FROM handle_ids WHERE name='b'), true, 'close-b'); + expect: + - - retained +- statement: INSERT INTO handle_ids VALUES ('create', vexfs_handle_create('pg-handles', '/new.txt', 448, 'create-new', 'gateway-a')); +- query: SELECT count(*) FROM vexfs_list('pg-handles', '/') WHERE name='new.txt'; + expect: + - - 0 +- query: SELECT vexfs_handle_stage_write((SELECT handle FROM handle_ids WHERE name='create'), 0, convert_to('new-data', 'UTF8'), 'stage-new'); + expect: + - - 2 +- query: SELECT count(*) FROM vexfs_list('pg-handles', '/') WHERE name='new.txt'; + expect: + - - 0 +- query: SELECT vexfs_handle_publish((SELECT handle FROM handle_ids WHERE name='create'), 2, 'full', 'publish-new'); + expect: + - - 1 +- query: SELECT convert_from(vexfs_read('pg-handles', '/new.txt'), 'UTF8'), (vexfs_stat('pg-handles', '/new.txt')->>'mode')::integer; + expect: + - - new-data + - 448 +- statement: INSERT INTO handle_ids VALUES ('recover', vexfs_handle_open('pg-handles', '/new.txt', 'rw', 'open-recover', 'gateway-a')); +- query: SELECT vexfs_handle_truncate((SELECT handle FROM handle_ids WHERE name='recover'), 3, 'truncate-recover'); + expect: + - - 1 +- query: SELECT vexfs_mount_session_end('pg-handles', 'gateway-a'); + expect: + - - 1 +- query: SELECT vexfs_mount_session_start('pg-handles', 'gateway-a'); + expect: + - - 1 +- query: SELECT convert_from(vexfs_handle_read((SELECT handle FROM handle_ids WHERE name='recover'), 0, 8), 'UTF8'); + expect: + - - new +- query: SELECT vexfs_handle_publish((SELECT handle FROM handle_ids WHERE name='recover'), 1, 'data', 'publish-recover'); + expect: + - - 2 +- statement: INSERT INTO handle_ids VALUES ('append-a', vexfs_handle_open('pg-handles', '/value.txt', 'rw', 'append-open-a', 'gateway-a')); +- statement: INSERT INTO handle_ids VALUES ('append-b', vexfs_handle_open('pg-handles', '/value.txt', 'rw', 'append-open-b', 'gateway-b')); +- query: SELECT vexfs_handle_append((SELECT handle FROM handle_ids WHERE name='append-a'), convert_to('-a', 'UTF8'), 'append-a'); + expect: + - - 3 +- query: SELECT vexfs_handle_append((SELECT handle FROM handle_ids WHERE name='append-b'), convert_to('-b', 'UTF8'), 'append-b'); + expect: + - - 4 +- query: SELECT convert_from(vexfs_read('pg-handles', '/value.txt'), 'UTF8'); + expect: + - - first-a-b +- statement: INSERT INTO handle_ids VALUES ('sync', vexfs_handle_open('pg-handles', '/value.txt', 'rw', 'sync-open', 'gateway-a')); +- query: SELECT vexfs_handle_stage_write((SELECT handle FROM handle_ids WHERE name='sync'), 0, convert_to('sync!', 'UTF8'), 'sync-stage'); + expect: + - - 1 +- query: SELECT vexfs_mount_synchronize('pg-handles', 'sync-all', 'gateway-a'); + expect: + - - 1 +- query: SELECT convert_from(vexfs_read('pg-handles', '/value.txt'), 'UTF8'); + expect: + - - sync!-a-b +- query: SELECT vexfs_lock_acquire('pg-handles', (vexfs_stat('pg-handles', '/value.txt')->>'inode')::bigint, 'gateway-a', 'shared', 'lock-a-shared'); + expect: + - - 1 +- query: SELECT vexfs_lock_acquire('pg-handles', (vexfs_stat('pg-handles', '/value.txt')->>'inode')::bigint, 'gateway-b', 'shared', 'lock-b-shared'); + expect: + - - 1 +- query: SELECT vexfs_lock_acquire('pg-handles', (vexfs_stat('pg-handles', '/value.txt')->>'inode')::bigint, 'gateway-a', 'exclusive', 'lock-a-exclusive-fail'); + expect_error: VEXFS_LOCK_CONFLICT +- query: SELECT vexfs_lock_release('pg-handles', (vexfs_stat('pg-handles', '/value.txt')->>'inode')::bigint, 'gateway-b', 'unlock-b'); + expect: + - - 1 +- query: SELECT vexfs_lock_acquire('pg-handles', (vexfs_stat('pg-handles', '/value.txt')->>'inode')::bigint, 'gateway-a', 'exclusive', 'lock-a-exclusive'); + expect: + - - 1 +- query: SELECT vexfs_lock_acquire('pg-handles', (vexfs_stat('pg-handles', '/value.txt')->>'inode')::bigint, 'gateway-b', 'shared', 'lock-b-shared-fail'); + expect_error: VEXFS_LOCK_CONFLICT +- query: SELECT (vexfs_visibility('pg-handles')->>'workspace_head')::bigint > 1, (vexfs_visibility('pg-handles')->>'cache_generation')::bigint > 1; + expect: + - - true + - true +- query: SELECT vexfs_write('pg-handles', '/open-unlink.txt', convert_to('abcdef', 'UTF8')); + expect: + - - 1 +- statement: INSERT INTO handle_ids VALUES ('unlink-reader', vexfs_handle_open('pg-handles', '/open-unlink.txt', 'r', 'unlink-open-reader', 'gateway-a')); +- statement: INSERT INTO handle_ids VALUES ('unlink-writer', vexfs_handle_open('pg-handles', '/open-unlink.txt', 'rw', 'unlink-open-writer', 'gateway-a')); +- query: SELECT vexfs_remove('pg-handles', '/open-unlink.txt', false); + expect: + - - 1 +- query: SELECT vexfs_stat('pg-handles', '/open-unlink.txt'); + expect_error: VEXFS_PATH_NOT_FOUND +- query: SELECT inode.live, NOT EXISTS (SELECT 1 FROM _vexfs.dentries AS link WHERE link.workspace_id=inode.workspace_id AND link.inode_id=inode.inode_id) FROM _vexfs.inodes AS inode JOIN _vexfs.handles AS handle ON handle.inode_id=inode.inode_id WHERE handle.handle_id=(SELECT handle FROM handle_ids WHERE name='unlink-writer'); + expect: + - - true + - true +- query: SELECT vexfs_handle_stage_write((SELECT handle FROM handle_ids WHERE name='unlink-writer'), 0, convert_to('OLD', 'UTF8'), 'unlink-stage'); + expect: + - - 1 +- query: SELECT vexfs_handle_publish((SELECT handle FROM handle_ids WHERE name='unlink-writer'), 1, 'full', 'unlink-publish'); + expect: + - - 2 +- query: SELECT convert_from(vexfs_handle_read((SELECT handle FROM handle_ids WHERE name='unlink-reader'), 0, 16), 'UTF8'); + expect: + - - OLDdef +- query: SELECT vexfs_write('pg-handles', '/open-unlink.txt', convert_to('NEW', 'UTF8')); + expect: + - - 1 +- query: SELECT convert_from(vexfs_read('pg-handles', '/open-unlink.txt'), 'UTF8'), (vexfs_stat('pg-handles', '/open-unlink.txt')->>'inode')::bigint <> (SELECT inode_id FROM _vexfs.handles WHERE handle_id=(SELECT handle FROM handle_ids WHERE name='unlink-writer')); + expect: + - - NEW + - true +- query: SELECT vexfs_handle_close((SELECT handle FROM handle_ids WHERE name='unlink-writer'), false, 'unlink-close-writer'); + expect: + - - closed +- query: SELECT inode.live FROM _vexfs.inodes AS inode JOIN _vexfs.handles AS handle ON handle.inode_id=inode.inode_id WHERE handle.handle_id=(SELECT handle FROM handle_ids WHERE name='unlink-reader'); + expect: + - - true +- query: SELECT vexfs_handle_close((SELECT handle FROM handle_ids WHERE name='unlink-reader'), false, 'unlink-close-reader'); + expect: + - - closed +- query: SELECT inode.live FROM _vexfs.inodes AS inode JOIN _vexfs.handles AS handle ON handle.inode_id=inode.inode_id WHERE handle.handle_id=(SELECT handle FROM handle_ids WHERE name='unlink-reader'); + expect: + - - false +- query: SELECT convert_from(vexfs_read('pg-handles', '/open-unlink.txt'), 'UTF8'); + expect: + - - NEW +- query: SELECT vexfs_mount_session_heartbeat('pg-handles', 'gateway-a'); + expect: + - - 1 +- query: SELECT vexfs_handle_close((SELECT handle FROM handle_ids WHERE name='a'), false, 'close-a'); + expect: + - - closed +- query: SELECT vexfs_handle_close((SELECT handle FROM handle_ids WHERE name='reader'), false, 'close-reader'); + expect: + - - closed +- query: SELECT vexfs_item_reclaim('pg-handles', 'reclaim') > 0; + expect: + - - true +- query: SELECT (vexfs_check('pg-handles', 1)->>'ok')::boolean; + expect: + - - true diff --git a/tests/spec/pg/agent_files/vexfs_pg_history_snapshot.yaml b/tests/spec/pg/agent_files/vexfs_pg_history_snapshot.yaml new file mode 100644 index 0000000000..30536d60ba --- /dev/null +++ b/tests/spec/pg/agent_files/vexfs_pg_history_snapshot.yaml @@ -0,0 +1,144 @@ +name: pg__vexfs_history_snapshot +tags: [vexfs, pg, history, snapshot, conflict] +description: PostgreSQL VexFS create/move/remove、文件历史、快照和 expected-head 合同 +steps: +- statement: TRUNCATE _vexfs.workspaces CASCADE; +- query: SELECT vexfs_pg_adapter_version(); + expect: + - - 0.4.0-alpha.1 +- query: SELECT vexfs_workspace_create('pg-a1') > 0; + expect: + - - true +- query: SELECT (vexfs_workspace_stat('pg-a1')->>'head_commit')::bigint; + expect: + - - 1 +- query: SELECT vexfs_create('pg-a1', '/empty.txt', 'file', 384) > 0; + expect: + - - true +- query: SELECT (vexfs_stat('pg-a1', '/empty.txt')->>'mode')::integer, octet_length(vexfs_read('pg-a1', '/empty.txt')); + expect: + - - 384 + - 0 +- query: SELECT vexfs_mkdir('pg-a1', '/tree/sub') > 0; + expect: + - - true +- query: SELECT vexfs_write('pg-a1', '/tree/sub/file.txt', convert_to('one', 'UTF8')); + expect: + - - 1 +- query: SELECT vexfs_write('pg-a1', '/tree/sub/file.txt', convert_to('two', 'UTF8')); + expect: + - - 2 +- query: SELECT version, commit_no, size, coalesce(source_version, 0), current::integer FROM vexfs_history('pg-a1', '/tree/sub/file.txt'); + expect: + - - 2 + - 5 + - 3 + - 0 + - 1 + - - 1 + - 4 + - 3 + - 0 + - 0 +- query: SELECT convert_from(vexfs_read_version('pg-a1', '/tree/sub/file.txt', 1), 'UTF8'); + expect: + - - one +- query: SELECT vexfs_restore_version('pg-a1', '/tree/sub/file.txt', 1, 2); + expect: + - - 3 +- query: SELECT vexfs_restore_version('pg-a1', '/tree/sub/file.txt', 2, 2); + expect_error: VEXFS_VERSION_CONFLICT +- query: SELECT vexfs_move('pg-a1', '/tree/sub/file.txt', '/tree/file.txt'); + expect: + - - 1 +- query: SELECT convert_from(vexfs_read('pg-a1', '/tree/file.txt'), 'UTF8'); + expect: + - - one +- query: SELECT vexfs_read('pg-a1', '/tree/sub/file.txt'); + expect_error: VEXFS_PATH_NOT_FOUND +- query: SELECT vexfs_snapshot_create('pg-a1', 'baseline'); + expect: + - - 7 +- query: SELECT (vexfs_snapshot_show('pg-a1', 'baseline')->>'inode_count')::integer, (vexfs_snapshot_show('pg-a1', 'baseline')->>'dentry_count')::integer; + expect: + - - 5 + - 4 +- query: SELECT vexfs_write('pg-a1', '/tree/file.txt', convert_to('three', 'UTF8')); + expect: + - - 4 +- query: SELECT vexfs_write('pg-a1', '/tree/new.txt', convert_to('new', 'UTF8')); + expect: + - - 1 +- query: SELECT vexfs_remove('pg-a1', '/empty.txt', false); + expect: + - - 1 +- query: SELECT vexfs_snapshot_restore('pg-a1', 'baseline', 9); + expect_error: VEXFS_HEAD_CONFLICT +- query: SELECT (vexfs_workspace_stat('pg-a1')->>'head_commit')::bigint; + expect: + - - 10 +- query: SELECT vexfs_snapshot_restore('pg-a1', 'baseline', 10); + expect: + - - 11 +- query: SELECT convert_from(vexfs_read('pg-a1', '/tree/file.txt'), 'UTF8'), (vexfs_stat('pg-a1', '/tree/file.txt')->>'version')::bigint, octet_length(vexfs_read('pg-a1', '/empty.txt')); + expect: + - - one + - 5 + - 0 +- query: SELECT vexfs_stat('pg-a1', '/tree/new.txt'); + expect_error: VEXFS_PATH_NOT_FOUND +- query: SELECT version, source_version, current::integer FROM vexfs_history('pg-a1', '/tree/file.txt', 1, NULL); + expect: + - - 5 + - 1 + - 1 +- query: SELECT vexfs_snapshot_restore('pg-a1', 'baseline', 11); + expect_error: VEXFS_ALREADY_MATCHES +- query: SELECT name, head_commit FROM vexfs_snapshot_list('pg-a1'); + expect: + - - baseline + - 7 +- query: SELECT vexfs_remove('pg-a1', '/tree', false); + expect_error: VEXFS_NOT_EMPTY +- query: SELECT vexfs_remove('pg-a1', '/tree', true); + expect: + - - 1 +- query: SELECT vexfs_stat('pg-a1', '/tree'); + expect_error: VEXFS_PATH_NOT_FOUND +- query: SELECT vexfs_snapshot_restore('pg-a1', 'baseline', 12); + expect: + - - 13 +- query: SELECT vexfs_move('pg-a1', '/tree', '/tree/sub/loop'); + expect_error: VEXFS_INVALID_MOVE +- query: SELECT convert_from(vexfs_read('pg-a1', '/tree/file.txt'), 'UTF8'); + expect: + - - one +- statement: CREATE TABLE vexfs_a1_business (id integer PRIMARY KEY); +- statement: BEGIN; +- statement: INSERT INTO vexfs_a1_business VALUES (1); +- query: SELECT vexfs_remove('pg-a1', '/tree', true); + expect: + - - 1 +- statement: ROLLBACK; +- query: SELECT count(*), convert_from(vexfs_read('pg-a1', '/tree/file.txt'), 'UTF8'), (vexfs_workspace_stat('pg-a1')->>'head_commit')::bigint FROM vexfs_a1_business; + expect: + - - 0 + - one + - 13 +- query: SELECT min(commit_no), max(commit_no), count(*) FROM vexfs_workspace_history('pg-a1', 1000); + expect: + - - 1 + - 13 + - 13 +- query: SELECT operation FROM vexfs_workspace_history('pg-a1', 3); + expect: + - - snapshot_restore + - - remove + - - snapshot_restore +- query: SELECT vexfs_snapshot_drop('pg-a1', 'baseline'); + expect: + - - 1 +- statement: DROP TABLE vexfs_a1_business; +- query: SELECT vexfs_workspace_drop('pg-a1'); + expect: + - - 1 diff --git a/tests/spec/pg/agent_files/vexfs_pg_metadata_acl.yaml b/tests/spec/pg/agent_files/vexfs_pg_metadata_acl.yaml new file mode 100644 index 0000000000..7425ee02dd --- /dev/null +++ b/tests/spec/pg/agent_files/vexfs_pg_metadata_acl.yaml @@ -0,0 +1,149 @@ +name: pg__vexfs_metadata_acl +tags: [vexfs, pg, metadata, symlink, hardlink, xattr, acl, audit] +description: PostgreSQL VexFS 元数据、链接、ACL 权限执行、审计和快照恢复 +steps: +- statement: TRUNCATE _vexfs.workspaces CASCADE; +- statement: DROP ROLE IF EXISTS vexfs_pg_acl_owner; +- statement: DROP ROLE IF EXISTS vexfs_pg_acl_alice; +- statement: DROP ROLE IF EXISTS vexfs_pg_acl_bob; +- statement: CREATE ROLE vexfs_pg_acl_owner NOLOGIN; +- statement: CREATE ROLE vexfs_pg_acl_alice NOLOGIN; +- statement: CREATE ROLE vexfs_pg_acl_bob NOLOGIN; +- statement: SET SESSION AUTHORIZATION vexfs_pg_acl_owner; +- query: SELECT vexfs_workspace_create('pg-metadata') > 0; + expect: + - - true +- query: SELECT vexfs_acl_set('pg-metadata', (vexfs_workspace_stat('pg-metadata')->>'root_inode')::bigint, '[{"principal":"vexfs_pg_acl_alice","effect":"allow","permissions":"read,write,execute,metadata","inherit":1}]'); + expect: + - - 1 +- query: SELECT vexfs_acl_list('pg-metadata', (vexfs_workspace_stat('pg-metadata')->>'root_inode')::bigint) = vexfs_acl_get('pg-metadata', (vexfs_workspace_stat('pg-metadata')->>'root_inode')::bigint); + expect: + - - true +- query: SELECT vexfs_mkdir('pg-metadata', '/docs/tree', true) > 0; + expect: + - - true +- query: SELECT vexfs_write('pg-metadata', '/docs/file.txt', convert_to('hello', 'UTF8')); + expect: + - - 1 +- query: SELECT vexfs_owner_set('pg-metadata', '/docs/file.txt', 'portable-owner'); + expect: + - - portable-owner +- query: SELECT vexfs_link('pg-metadata', '/docs/file.txt', '/docs/linked.txt') = (vexfs_stat('pg-metadata', '/docs/file.txt')->>'inode')::bigint; + expect: + - - true +- query: SELECT vexfs_symlink('pg-metadata', '/docs/current', convert_to('file.txt', 'UTF8')) > 0; + expect: + - - true +- query: SELECT vexfs_set_mode('pg-metadata', (vexfs_stat('pg-metadata', '/docs/file.txt')->>'inode')::bigint, 493); + expect: + - - 493 +- query: SELECT vexfs_chown('pg-metadata', (vexfs_stat('pg-metadata', '/docs/file.txt')->>'inode')::bigint, 501, 20)->>'uid', vexfs_chown('pg-metadata', (vexfs_stat('pg-metadata', '/docs/file.txt')->>'inode')::bigint, -1, 21)->>'gid'; + expect: + - - 501 + - 21 +- query: SELECT vexfs_xattr_set('pg-metadata', (vexfs_stat('pg-metadata', '/docs/file.txt')->>'inode')::bigint, 'user.spec', convert_to('one', 'UTF8'), 1); + expect: + - - 1 +- query: SELECT vexfs_xattr_set('pg-metadata', (vexfs_stat('pg-metadata', '/docs/file.txt')->>'inode')::bigint, 'user.spec', convert_to('duplicate', 'UTF8'), 1); + expect_error: VEXFS_XATTR_EXISTS +- query: SELECT vexfs_snapshot_create('pg-metadata', 'baseline') > 0; + expect: + - - true +- query: SELECT vexfs_acl_grant('pg-metadata', (vexfs_stat('pg-metadata', '/docs/file.txt')->>'inode')::bigint, 'vexfs_pg_acl_alice', 'write', 'deny', 0); + expect: + - - 1 +- statement: RESET SESSION AUTHORIZATION; +- statement: SET SESSION AUTHORIZATION vexfs_pg_acl_alice; +- query: SELECT convert_from(vexfs_read('pg-metadata', '/docs/file.txt'), 'UTF8'); + expect: + - - hello +- query: SELECT vexfs_write('pg-metadata', '/docs/file.txt', convert_to('blocked', 'UTF8')); + expect_error: VEXFS_PERMISSION_DENIED +- query: SELECT vexfs_owner_set('pg-metadata', '/docs/file.txt', 'forged-owner'); + expect_error: VEXFS_PERMISSION_DENIED +- query: SELECT vexfs_acl_delete('pg-metadata', (vexfs_stat('pg-metadata', '/docs/file.txt')->>'inode')::bigint); + expect_error: VEXFS_PERMISSION_DENIED +- query: SELECT vexfs_write('pg-metadata', '/docs/alice.txt', convert_to('alice', 'UTF8')); + expect: + - - 1 +- statement: RESET SESSION AUTHORIZATION; +- statement: SET SESSION AUTHORIZATION vexfs_pg_acl_bob; +- query: SELECT vexfs_read('pg-metadata', '/docs/file.txt'); + expect_error: VEXFS_PERMISSION_DENIED +- statement: RESET SESSION AUTHORIZATION; +- statement: SET SESSION AUTHORIZATION vexfs_pg_acl_owner; +- query: SELECT vexfs_snapshot_restore('pg-metadata', 'baseline', (vexfs_workspace_stat('pg-metadata')->>'head_commit')::bigint) > 0; + expect: + - - true +- query: SELECT convert_from(vexfs_xattr_get('pg-metadata', (vexfs_stat('pg-metadata', '/docs/file.txt')->>'inode')::bigint, 'user.spec'), 'UTF8'), convert_from(vexfs_readlink('pg-metadata', (vexfs_stat('pg-metadata', '/docs/current')->>'inode')::bigint), 'UTF8'), (vexfs_stat('pg-metadata', '/docs/file.txt')->>'link_count')::bigint, vexfs_stat('pg-metadata', '/docs/file.txt')->>'uid', vexfs_stat('pg-metadata', '/docs/file.txt')->>'gid', vexfs_stat('pg-metadata', '/docs/file.txt')->>'owner_principal'; + expect: + - - one + - file.txt + - 2 + - 501 + - 21 + - portable-owner +- statement: RESET SESSION AUTHORIZATION; +- statement: SET SESSION AUTHORIZATION vexfs_pg_acl_alice; +- query: SELECT vexfs_write('pg-metadata', '/docs/file.txt', convert_to('allowed', 'UTF8')); + expect: + - - 2 +- query: SELECT count(*) FROM vexfs_audit_list('pg-metadata', 1000, NULL); + expect_error: VEXFS_PERMISSION_DENIED +- statement: RESET SESSION AUTHORIZATION; +- statement: SET SESSION AUTHORIZATION vexfs_pg_acl_owner; +- query: SELECT actor::text, operation, path, inode_id = (vexfs_stat('pg-metadata', '/docs/file.txt')->>'inode')::bigint, details->>'before_version', details->>'after_version', position('allowed' in details::text) = 0 FROM vexfs_audit_list('pg-metadata', 1000, NULL) WHERE operation = 'write' AND actor = 'vexfs_pg_acl_alice' ORDER BY event_id DESC LIMIT 1; + expect: + - - vexfs_pg_acl_alice + - write + - /docs/file.txt + - true + - 1 + - 2 + - true +- query: SELECT bool_and(actor IS NOT NULL AND path IS NOT NULL AND inode_id IS NOT NULL AND details ? 'before_version' AND details ? 'after_version'), bool_and(details::text !~ 'hello|allowed|blocked') FROM vexfs_audit_list('pg-metadata', 1000, NULL); + expect: + - - true + - true +- query: SELECT vexfs_acl_delete('pg-metadata', (vexfs_stat('pg-metadata', '/docs/file.txt')->>'inode')::bigint) > 0; + expect: + - - true +- statement: RESET SESSION AUTHORIZATION; +- statement: SET SESSION AUTHORIZATION vexfs_pg_acl_alice; +- query: SELECT vexfs_read('pg-metadata', '/docs/file.txt'); + expect_error: VEXFS_PERMISSION_DENIED +- statement: RESET SESSION AUTHORIZATION; +- statement: SET SESSION AUTHORIZATION vexfs_pg_acl_owner; +- query: SELECT vexfs_write('pg-metadata', '/outside.txt', convert_to('outside', 'UTF8')); + expect: + - - 1 +- query: SELECT vexfs_link('pg-metadata', '/outside.txt', '/docs/tree/inside-link') > 0; + expect: + - - true +- query: SELECT vexfs_remove('pg-metadata', '/docs/tree', true); + expect: + - - 1 +- query: SELECT convert_from(vexfs_read('pg-metadata', '/outside.txt'), 'UTF8'); + expect: + - - outside +- query: SELECT (vexfs_check('pg-metadata', 1)->>'ok')::boolean, (SELECT count(*) >= 15 FROM vexfs_audit_list('pg-metadata', 1000, NULL)); + expect: + - - true + - true +- statement: RESET SESSION AUTHORIZATION; +- query: SELECT vexfs_workspace_drop('pg-metadata'); + expect: + - - 1 +- query: SELECT workspace_id IS NULL, workspace_name, actor_role::text, operation, path, inode_id IS NOT NULL, details ? 'before_version', details->'after_version' = 'null'::jsonb FROM _vexfs.audit_events WHERE workspace_name = 'pg-metadata' AND operation = 'workspace_drop' ORDER BY event_id DESC LIMIT 1; + expect: + - - true + - pg-metadata + - postgres + - workspace_drop + - / + - true + - true + - true +- statement: DROP ROLE vexfs_pg_acl_owner; +- statement: DROP ROLE vexfs_pg_acl_alice; +- statement: DROP ROLE vexfs_pg_acl_bob; diff --git a/tests/spec/pg/agent_files/vexfs_pg_quota_gc.yaml b/tests/spec/pg/agent_files/vexfs_pg_quota_gc.yaml new file mode 100644 index 0000000000..f6a261b274 --- /dev/null +++ b/tests/spec/pg/agent_files/vexfs_pg_quota_gc.yaml @@ -0,0 +1,130 @@ +name: pg__vexfs_quota_gc +tags: [vexfs, pg, quota, retention, gc] +description: PostgreSQL VexFS live quota、快照恢复配额、保留策略、别名保护和分批 GC +steps: +- statement: TRUNCATE _vexfs.workspaces CASCADE; +- query: SELECT vexfs_workspace_create('pg-quota') > 0; + expect: + - - true +- query: SELECT vexfs_write('pg-quota', '/quota.txt', convert_to('abc', 'UTF8')); + expect: + - - 1 +- query: SELECT vexfs_snapshot_create('pg-quota', 'large'); + expect: + - - 2 +- query: SELECT vexfs_write('pg-quota', '/quota.txt', convert_to('x', 'UTF8')); + expect: + - - 2 +- query: SELECT (vexfs_quota_set('pg-quota', 1, 1, 2)->>'live_bytes')::bigint, (vexfs_quota_get('pg-quota')->>'over_quota')::boolean; + expect: + - - 1 + - false +- query: SELECT vexfs_snapshot_restore('pg-quota', 'large', 3); + expect_error: VEXFS_QUOTA_FILE_SIZE +- query: SELECT vexfs_write('pg-quota', '/second.txt', convert_to('y', 'UTF8')); + expect_error: VEXFS_QUOTA_BYTES +- query: SELECT vexfs_write('pg-quota', '/quota.txt', convert_to('xyz', 'UTF8')); + expect_error: VEXFS_QUOTA_FILE_SIZE +- query: SELECT convert_from(vexfs_read('pg-quota', '/quota.txt'), 'UTF8'), (vexfs_workspace_stat('pg-quota')->>'head_commit')::bigint, (vexfs_quota_get('pg-quota')->>'live_files')::bigint, (SELECT count(*) FROM _vexfs.file_versions WHERE workspace_id=(SELECT workspace_id FROM _vexfs.workspaces WHERE name='pg-quota')); + expect: + - - x + - 3 + - 1 + - 2 +- query: SELECT (vexfs_quota_set('pg-quota', 0, 1, 0)->>'over_quota')::boolean; + expect: + - - true +- query: SELECT vexfs_write('pg-quota', '/quota.txt', ''::bytea); + expect: + - - 3 +- query: SELECT (vexfs_quota_get('pg-quota')->>'live_bytes')::bigint, (vexfs_quota_get('pg-quota')->>'over_quota')::boolean; + expect: + - - 0 + - false +- query: SELECT vexfs_remove('pg-quota', '/quota.txt'); + expect: + - - 1 +- query: SELECT (vexfs_quota_get('pg-quota')->>'live_files')::bigint, (vexfs_quota_get('pg-quota')->>'live_bytes')::bigint; + expect: + - - 0 + - 0 +- query: SELECT vexfs_quota_set('pg-quota', -1, NULL, NULL); + expect_error: VEXFS_INVALID_QUOTA +- query: SELECT vexfs_workspace_drop('pg-quota'); + expect: + - - 1 + +- query: SELECT vexfs_workspace_create('pg-gc') > 0; + expect: + - - true +- query: SELECT vexfs_write('pg-gc', '/history.txt', convert_to('one', 'UTF8')); + expect: + - - 1 +- query: SELECT vexfs_write('pg-gc', '/history.txt', convert_to('two', 'UTF8')); + expect: + - - 2 +- query: SELECT vexfs_snapshot_create('pg-gc', 'keep-two'); + expect: + - - 3 +- query: SELECT vexfs_write('pg-gc', '/history.txt', convert_to('three', 'UTF8')); + expect: + - - 3 +- query: SELECT vexfs_write('pg-gc', '/history.txt', convert_to('four', 'UTF8')); + expect: + - - 4 +- query: SELECT (vexfs_retention_set('pg-gc', 1, 0)->>'reclaimable_versions')::bigint; + expect: + - - 2 +- query: SELECT (vexfs_gc_pause('pg-gc', 1)->>'gc_paused')::boolean; + expect: + - - true +- query: SELECT vexfs_gc('pg-gc', 1); + expect_error: VEXFS_GC_PAUSED +- query: SELECT (vexfs_gc_pause('pg-gc', 0)->>'gc_paused')::boolean; + expect: + - - false +- query: SELECT (result->>'deleted_versions')::bigint, (result->>'remaining_reclaimable_versions')::bigint FROM (SELECT vexfs_gc('pg-gc', 1) AS result) AS gc; + expect: + - - 1 + - 1 +- query: SELECT (result->>'deleted_versions')::bigint, (result->>'remaining_reclaimable_versions')::bigint FROM (SELECT vexfs_gc('pg-gc', 1) AS result) AS gc; + expect: + - - 1 + - 0 +- query: SELECT convert_from(vexfs_read_version('pg-gc', '/history.txt', 2), 'UTF8'), convert_from(vexfs_read('pg-gc', '/history.txt'), 'UTF8'); + expect: + - - two + - four +- query: SELECT vexfs_read_version('pg-gc', '/history.txt', 1); + expect_error: VEXFS_VERSION_NOT_FOUND +- query: SELECT vexfs_restore_version('pg-gc', '/history.txt', 2, 4); + expect: + - - 5 +- query: SELECT vexfs_snapshot_drop('pg-gc', 'keep-two'); + expect: + - - 1 +- query: SELECT (vexfs_retention_set('pg-gc', 1, 0)->>'reclaimable_versions')::bigint; + expect: + - - 1 +- query: SELECT (result->>'remaining_reclaimable_versions')::bigint, (result->>'remaining_versions')::bigint FROM (SELECT vexfs_gc('pg-gc', 10000) AS result) AS gc; + expect: + - - 0 + - 2 +- query: SELECT (vexfs_retention_get('pg-gc')->>'stored_versions')::bigint, (vexfs_retention_get('pg-gc')->>'stored_version_bytes')::bigint; + expect: + - - 2 + - 3 +- query: SELECT source_version_no, manifest_id IS NULL FROM _vexfs.file_versions WHERE workspace_id=(SELECT workspace_id FROM _vexfs.workspaces WHERE name='pg-gc') AND version_no=5; + expect: + - - 2 + - true +- query: SELECT convert_from(vexfs_read('pg-gc', '/history.txt'), 'UTF8'), (vexfs_check('pg-gc', 1)->>'ok')::boolean, (vexfs_check('pg-gc', 1)->>'issue_count')::integer; + expect: + - - two + - true + - 0 +- query: SELECT vexfs_gc('pg-gc', 0); + expect_error: VEXFS_INVALID_GC_BATCH +- query: SELECT vexfs_workspace_drop('pg-gc'); + expect: + - - 1 diff --git a/tests/spec/pg/agent_files/vexfs_pg_transaction_role.yaml b/tests/spec/pg/agent_files/vexfs_pg_transaction_role.yaml new file mode 100644 index 0000000000..ae8272e54a --- /dev/null +++ b/tests/spec/pg/agent_files/vexfs_pg_transaction_role.yaml @@ -0,0 +1,99 @@ +name: pg__vexfs_transaction_role +tags: [vexfs, pg, transaction, role] +description: PostgreSQL VexFS 基础目录文件、事务回滚和登录角色隔离 +steps: +- statement: TRUNCATE _vexfs.workspaces CASCADE; +- statement: DROP ROLE IF EXISTS vexfs_pg_other; +- query: SELECT vexfs_pg_adapter_version(); + expect: + - - 0.4.0-alpha.1 +- query: SELECT vexfs_contract_version(), vexfs_init(); + expect: + - - 0.9.0 + - 1 +- query: SELECT vexfs_grep_index('status')->>'backend', (vexfs_grep_index('status')->>'available')::boolean; + expect: + - - pg-trgm + - true +- query: SELECT vexfs_grep_index(NULL); + expect_error: VEXFS_INVALID_INDEX_ACTION +- query: SELECT vexfs_workspace_create('pg-contract') > 0; + expect: + - - true +- query: SELECT vexfs_diagnostics('pg-contract')->>'schema_version', vexfs_diagnostics('pg-contract')->>'adapter_version', (vexfs_diagnostics('pg-contract')->>'schema_ready')::boolean; + expect: + - - 0.9.0 + - 0.4.0-alpha.1 + - true +- query: SELECT vexfs_mkdir('pg-contract', '/notes/2026') > 0; + expect: + - - true +- query: SELECT vexfs_write('pg-contract', '/notes/2026/hello.txt', convert_to('hello', 'UTF8')); + expect: + - - 1 +- query: SELECT convert_from(vexfs_read('pg-contract', '/notes/2026/hello.txt'), 'UTF8'); + expect: + - - hello +- statement: CREATE TABLE vexfs_business_txn (id integer PRIMARY KEY, note text NOT NULL); +- statement: BEGIN; +- statement: INSERT INTO vexfs_business_txn VALUES (1, 'same transaction'); +- query: SELECT vexfs_write('pg-contract', '/notes/2026/hello.txt', convert_to('rolled back', 'UTF8')); + expect: + - - 2 +- statement: ROLLBACK; +- query: SELECT count(*), convert_from(vexfs_read('pg-contract', '/notes/2026/hello.txt'), 'UTF8') FROM vexfs_business_txn; + expect: + - - 0 + - hello +- query: SELECT value FROM jsonb_each_text(vexfs_stat('pg-contract', '/notes/2026/hello.txt')) WHERE key = 'version'; + expect: + - - 1 +- query: SELECT vexfs_stat('pg-contract', '/notes/2026/hello.txt')->>'path', vexfs_stat('pg-contract', '/')->>'path'; + expect: + - - /notes/2026/hello.txt + - / +- query: SELECT name, kind, size, version FROM vexfs_list('pg-contract', '/notes/2026'); + expect: + - - hello.txt + - file + - 5 + - 1 +- statement: CREATE ROLE vexfs_pg_other NOLOGIN; +- statement: SET SESSION AUTHORIZATION vexfs_pg_other; +- query: SELECT convert_from(vexfs_read('pg-contract', '/notes/2026/hello.txt'), 'UTF8'); + expect_error: VEXFS_PERMISSION_DENIED +- query: SELECT count(*) FROM _vexfs.workspaces; + expect_error: permission denied +- query: SELECT vexfs_workspace_create('pg-other') > 0; + expect: + - - true +- query: SELECT value FROM jsonb_each_text(vexfs_stat('pg-other', '/')) WHERE key = 'owner'; + expect: + - - vexfs_pg_other +- query: SELECT vexfs_write('pg-other', '/owned.txt', convert_to('owned', 'UTF8')); + expect: + - - 1 +- statement: RESET SESSION AUTHORIZATION; +- statement: ALTER ROLE vexfs_pg_other RENAME TO vexfs_pg_renamed; +- statement: SET SESSION AUTHORIZATION vexfs_pg_renamed; +- query: SELECT convert_from(vexfs_read('pg-other', '/owned.txt'), 'UTF8'); + expect: + - - owned +- query: SELECT value FROM jsonb_each_text(vexfs_stat('pg-other', '/')) WHERE key = 'owner'; + expect: + - - vexfs_pg_renamed +- statement: RESET SESSION AUTHORIZATION; +- statement: DROP ROLE vexfs_pg_renamed; +- statement: CREATE ROLE vexfs_pg_other NOLOGIN; +- statement: SET SESSION AUTHORIZATION vexfs_pg_other; +- query: SELECT convert_from(vexfs_read('pg-other', '/owned.txt'), 'UTF8'); + expect_error: VEXFS_PERMISSION_DENIED +- statement: RESET SESSION AUTHORIZATION; +- query: SELECT vexfs_workspace_drop('pg-other'); + expect: + - - 1 +- query: SELECT vexfs_workspace_drop('pg-contract'); + expect: + - - 1 +- statement: DROP TABLE vexfs_business_txn; +- statement: DROP ROLE vexfs_pg_other; diff --git a/vexdb_pg/sql/vexdb_lite--1.0.sql b/vexdb_pg/sql/vexdb_lite--1.0.sql index bcc8930d14..e55946eae6 100644 --- a/vexdb_pg/sql/vexdb_lite--1.0.sql +++ b/vexdb_pg/sql/vexdb_lite--1.0.sql @@ -306,3 +306,8178 @@ CREATE FUNCTION vexdb_index_info() COMMENT ON FUNCTION vexdb_index_info() IS 'Lists all vexdb_graph indexes with metadata (mirrors duck-side schema)'; + + +-- VexFS PostgreSQL adapter alpha +-- +-- This is the first PostgreSQL vertical slice. PostgreSQL tables, MVCC, WAL +-- and the authenticated session role are authoritative. Mount processes are +-- deliberately not started from the extension. + +CREATE SCHEMA _vexfs; +REVOKE ALL ON SCHEMA _vexfs FROM PUBLIC; + +CREATE TABLE _vexfs.workspaces ( + workspace_id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + name text NOT NULL UNIQUE CHECK (name <> ''), + state text NOT NULL DEFAULT 'active' CHECK (state IN ('active', 'importing')), + owner_oid oid NOT NULL, + owner_role name NOT NULL, + root_inode bigint, + head_commit bigint NOT NULL DEFAULT 0, + history_floor_commit bigint NOT NULL DEFAULT 0 CHECK (history_floor_commit >= 0), + cache_generation bigint NOT NULL DEFAULT 0 CHECK (cache_generation >= 0), + quota_max_bytes bigint CHECK (quota_max_bytes IS NULL OR quota_max_bytes >= 0), + quota_max_files bigint CHECK (quota_max_files IS NULL OR quota_max_files >= 0), + quota_max_file_bytes bigint + CHECK (quota_max_file_bytes IS NULL OR quota_max_file_bytes >= 0), + live_files bigint NOT NULL DEFAULT 0 CHECK (live_files >= 0), + live_bytes bigint NOT NULL DEFAULT 0 CHECK (live_bytes >= 0), + retention_keep_versions integer NOT NULL DEFAULT 32 + CHECK (retention_keep_versions BETWEEN 0 AND 1000000), + retention_keep_days integer NOT NULL DEFAULT 30 + CHECK (retention_keep_days BETWEEN 0 AND 36500), + gc_paused boolean NOT NULL DEFAULT false, + grep_index_enabled boolean NOT NULL DEFAULT false, + grep_index_dirty boolean NOT NULL DEFAULT false, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE TABLE _vexfs.inodes ( + inode_id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + workspace_id bigint NOT NULL REFERENCES _vexfs.workspaces(workspace_id) ON DELETE CASCADE, + kind text NOT NULL CHECK (kind IN ('file', 'directory', 'symlink')), + mode integer NOT NULL CHECK (mode BETWEEN 0 AND 4095), + owner_oid oid NOT NULL, + owner_role name NOT NULL, + owner_principal text NOT NULL, + uid bigint NOT NULL DEFAULT 0 CHECK (uid BETWEEN 0 AND 4294967295), + gid bigint NOT NULL DEFAULT 0 CHECK (gid BETWEEN 0 AND 4294967295), + current_version bigint NOT NULL DEFAULT 0 CHECK (current_version >= 0), + size_bytes bigint NOT NULL DEFAULT 0 CHECK (size_bytes >= 0), + live boolean NOT NULL DEFAULT true, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + accessed_at timestamptz NOT NULL DEFAULT clock_timestamp(), + modified_at timestamptz NOT NULL DEFAULT clock_timestamp(), + changed_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE TABLE _vexfs.dentries ( + workspace_id bigint NOT NULL REFERENCES _vexfs.workspaces(workspace_id) ON DELETE CASCADE, + parent_inode bigint NOT NULL REFERENCES _vexfs.inodes(inode_id) ON DELETE CASCADE, + name text NOT NULL CHECK (name <> '' AND octet_length(name) <= 255), + inode_id bigint NOT NULL REFERENCES _vexfs.inodes(inode_id) ON DELETE CASCADE, + PRIMARY KEY (workspace_id, parent_inode, name) +); + +CREATE INDEX vexfs_dentries_inode_idx + ON _vexfs.dentries(workspace_id, inode_id); + +CREATE TABLE _vexfs.commits ( + workspace_id bigint NOT NULL REFERENCES _vexfs.workspaces(workspace_id) ON DELETE CASCADE, + commit_no bigint NOT NULL CHECK (commit_no > 0), + parent_commit bigint NOT NULL CHECK (parent_commit >= 0), + operation text NOT NULL, + created_by_oid oid NOT NULL, + created_by name NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (workspace_id, commit_no) +); + +CREATE TABLE _vexfs.manifests ( + manifest_id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + workspace_id bigint NOT NULL REFERENCES _vexfs.workspaces(workspace_id) ON DELETE CASCADE, + inode_id bigint NOT NULL REFERENCES _vexfs.inodes(inode_id) ON DELETE CASCADE, + file_size bigint NOT NULL CHECK (file_size >= 0), + chunk_size integer NOT NULL CHECK (chunk_size = 65536), + chunk_count integer NOT NULL CHECK (chunk_count >= 0), + checksum text NOT NULL CHECK (checksum ~ '^[0-9a-f]{64}$'), + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE TABLE _vexfs.chunks ( + chunk_id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + workspace_id bigint NOT NULL REFERENCES _vexfs.workspaces(workspace_id) ON DELETE CASCADE, + inode_id bigint NOT NULL REFERENCES _vexfs.inodes(inode_id) ON DELETE CASCADE, + content bytea NOT NULL, + size_bytes integer NOT NULL CHECK (size_bytes BETWEEN 1 AND 65536), + checksum text NOT NULL CHECK (checksum ~ '^[0-9a-f]{64}$'), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CHECK (octet_length(content) = size_bytes) +); + +CREATE TABLE _vexfs.manifest_chunks ( + manifest_id bigint NOT NULL REFERENCES _vexfs.manifests(manifest_id) ON DELETE CASCADE, + chunk_no integer NOT NULL CHECK (chunk_no >= 0), + chunk_id bigint NOT NULL REFERENCES _vexfs.chunks(chunk_id), + PRIMARY KEY (manifest_id, chunk_no) +); + +CREATE INDEX vexfs_manifest_chunks_chunk_idx + ON _vexfs.manifest_chunks(chunk_id); + +CREATE TABLE _vexfs.xattrs ( + workspace_id bigint NOT NULL REFERENCES _vexfs.workspaces(workspace_id) ON DELETE CASCADE, + inode_id bigint NOT NULL REFERENCES _vexfs.inodes(inode_id) ON DELETE CASCADE, + name text NOT NULL CHECK (name <> '' AND octet_length(name) <= 255), + value bytea NOT NULL CHECK (octet_length(value) <= 65536), + updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (workspace_id, inode_id, name) +); + +CREATE TABLE _vexfs.acl_entries ( + workspace_id bigint NOT NULL REFERENCES _vexfs.workspaces(workspace_id) ON DELETE CASCADE, + inode_id bigint NOT NULL REFERENCES _vexfs.inodes(inode_id) ON DELETE CASCADE, + principal text NOT NULL CHECK (principal <> '' AND octet_length(principal) <= 255), + effect text NOT NULL CHECK (effect IN ('allow', 'deny')), + permissions text NOT NULL CHECK (permissions <> '' AND octet_length(permissions) <= 1024), + inherit_flags integer NOT NULL DEFAULT 0 CHECK (inherit_flags BETWEEN 0 AND 255), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (workspace_id, inode_id, principal, effect) +); + +CREATE INDEX vexfs_acl_entries_principal_idx + ON _vexfs.acl_entries(workspace_id, principal, inode_id); + +CREATE TABLE _vexfs.file_versions ( + workspace_id bigint NOT NULL REFERENCES _vexfs.workspaces(workspace_id) ON DELETE CASCADE, + inode_id bigint NOT NULL REFERENCES _vexfs.inodes(inode_id) ON DELETE CASCADE, + version_no bigint NOT NULL CHECK (version_no > 0), + commit_no bigint NOT NULL, + manifest_id bigint REFERENCES _vexfs.manifests(manifest_id), + source_version_no bigint, + size_bytes bigint NOT NULL CHECK (size_bytes >= 0), + checksum text NOT NULL CHECK (checksum ~ '^[0-9a-f]{64}$'), + created_by_oid oid NOT NULL, + created_by name NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CHECK ((manifest_id IS NOT NULL AND source_version_no IS NULL) OR + (manifest_id IS NULL AND source_version_no IS NOT NULL)), + PRIMARY KEY (workspace_id, inode_id, version_no), + FOREIGN KEY (workspace_id, commit_no) + REFERENCES _vexfs.commits(workspace_id, commit_no) ON DELETE CASCADE +); + +-- Optional, derived search state. This table is deliberately not registered +-- with pg_extension_config_dump: authoritative file versions are restored +-- first, then an enabled workspace reports dirty until an explicit rebuild. +CREATE TABLE _vexfs.grep_documents ( + inode_id bigint PRIMARY KEY REFERENCES _vexfs.inodes(inode_id) ON DELETE CASCADE, + workspace_id bigint NOT NULL REFERENCES _vexfs.workspaces(workspace_id) ON DELETE CASCADE, + version_no bigint NOT NULL CHECK (version_no > 0), + content text NOT NULL, + indexed_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE INDEX vexfs_grep_documents_workspace_idx + ON _vexfs.grep_documents(workspace_id, inode_id); + +CREATE TABLE _vexfs.snapshots ( + snapshot_id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + workspace_id bigint NOT NULL REFERENCES _vexfs.workspaces(workspace_id) ON DELETE CASCADE, + name text NOT NULL CHECK (name <> ''), + head_commit bigint NOT NULL CHECK (head_commit > 0), + created_by_oid oid NOT NULL, + created_by name NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + UNIQUE (workspace_id, name), + FOREIGN KEY (workspace_id, head_commit) + REFERENCES _vexfs.commits(workspace_id, commit_no) +); + +CREATE TABLE _vexfs.snapshot_inodes ( + snapshot_id bigint NOT NULL REFERENCES _vexfs.snapshots(snapshot_id) ON DELETE CASCADE, + inode_id bigint NOT NULL, + kind text NOT NULL CHECK (kind IN ('file', 'directory', 'symlink')), + mode integer NOT NULL CHECK (mode BETWEEN 0 AND 4095), + owner_oid oid NOT NULL, + owner_role name NOT NULL, + owner_principal text NOT NULL, + uid bigint NOT NULL CHECK (uid BETWEEN 0 AND 4294967295), + gid bigint NOT NULL CHECK (gid BETWEEN 0 AND 4294967295), + current_version bigint NOT NULL CHECK (current_version >= 0), + size_bytes bigint NOT NULL CHECK (size_bytes >= 0), + created_at timestamptz NOT NULL, + accessed_at timestamptz NOT NULL, + modified_at timestamptz NOT NULL, + changed_at timestamptz NOT NULL, + PRIMARY KEY (snapshot_id, inode_id) +); + +CREATE TABLE _vexfs.snapshot_dentries ( + snapshot_id bigint NOT NULL REFERENCES _vexfs.snapshots(snapshot_id) ON DELETE CASCADE, + parent_inode bigint NOT NULL, + name text NOT NULL, + inode_id bigint NOT NULL, + PRIMARY KEY (snapshot_id, parent_inode, name) +); + +CREATE TABLE _vexfs.snapshot_xattrs ( + snapshot_id bigint NOT NULL REFERENCES _vexfs.snapshots(snapshot_id) ON DELETE CASCADE, + inode_id bigint NOT NULL, + name text NOT NULL, + value bytea NOT NULL, + updated_at timestamptz NOT NULL, + PRIMARY KEY (snapshot_id, inode_id, name) +); + +CREATE TABLE _vexfs.snapshot_acl_entries ( + snapshot_id bigint NOT NULL REFERENCES _vexfs.snapshots(snapshot_id) ON DELETE CASCADE, + inode_id bigint NOT NULL, + principal text NOT NULL, + effect text NOT NULL CHECK (effect IN ('allow', 'deny')), + permissions text NOT NULL, + inherit_flags integer NOT NULL CHECK (inherit_flags BETWEEN 0 AND 255), + created_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL, + PRIMARY KEY (snapshot_id, inode_id, principal, effect) +); + +CREATE TABLE _vexfs.audit_events ( + event_id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + workspace_id bigint REFERENCES _vexfs.workspaces(workspace_id) ON DELETE SET NULL, + workspace_name text NOT NULL, + commit_no bigint, + actor_oid oid NOT NULL, + actor_role name NOT NULL, + operation text NOT NULL, + path text, + inode_id bigint, + details jsonb NOT NULL DEFAULT '{}'::jsonb, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE INDEX vexfs_audit_events_workspace_idx + ON _vexfs.audit_events(workspace_id, event_id DESC); + +CREATE INDEX vexfs_audit_events_workspace_name_idx + ON _vexfs.audit_events(workspace_name, event_id DESC); + +-- format v2 import is streamed through these transaction-local job rows. They +-- are deliberately not extension configuration data: a committed unfinished +-- job is disposable staging, never authoritative workspace content. +CREATE TABLE _vexfs.archive_import_jobs ( + job_id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + workspace_name text NOT NULL, + owner_oid oid NOT NULL, + owner_role name NOT NULL, + backend_pid integer NOT NULL, + manifest jsonb NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE TABLE _vexfs.archive_import_records ( + job_id bigint NOT NULL REFERENCES _vexfs.archive_import_jobs(job_id) ON DELETE CASCADE, + record_type text NOT NULL, + record_key text NOT NULL, + record_json jsonb NOT NULL, + content bytea, + PRIMARY KEY (job_id, record_type, record_key) +); + +CREATE TABLE _vexfs.mount_sessions ( + workspace_id bigint NOT NULL REFERENCES _vexfs.workspaces(workspace_id) ON DELETE CASCADE, + session_id text NOT NULL CHECK (session_id <> '' AND octet_length(session_id) <= 255), + owner_oid oid NOT NULL, + owner_role name NOT NULL, + lease_until timestamptz NOT NULL, + started_at timestamptz NOT NULL DEFAULT clock_timestamp(), + heartbeat_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (workspace_id, session_id) +); + +CREATE INDEX vexfs_mount_sessions_lease_idx + ON _vexfs.mount_sessions(workspace_id, lease_until); + +CREATE TABLE _vexfs.handles ( + handle_id text PRIMARY KEY CHECK (handle_id <> '' AND octet_length(handle_id) <= 128), + workspace_id bigint NOT NULL REFERENCES _vexfs.workspaces(workspace_id) ON DELETE CASCADE, + inode_id bigint REFERENCES _vexfs.inodes(inode_id) ON DELETE SET NULL, + path text NOT NULL, + flags text NOT NULL, + create_mode integer CHECK (create_mode IS NULL OR create_mode BETWEEN 0 AND 4095), + owner_oid oid NOT NULL, + owner_role name NOT NULL, + session_id text, + expected_version bigint NOT NULL CHECK (expected_version >= 0), + dirty_generation bigint NOT NULL DEFAULT 0 CHECK (dirty_generation >= 0), + published_generation bigint NOT NULL DEFAULT 0 CHECK (published_generation >= 0), + staging bytea NOT NULL DEFAULT ''::bytea, + state text NOT NULL DEFAULT 'open' CHECK (state IN ('open', 'retained', 'closed')), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), + lease_until timestamptz NOT NULL DEFAULT clock_timestamp() + interval '30 seconds' +); + +CREATE INDEX vexfs_handles_workspace_state_idx + ON _vexfs.handles(workspace_id, state, lease_until); +CREATE INDEX vexfs_handles_inode_idx + ON _vexfs.handles(workspace_id, inode_id); + +CREATE TABLE _vexfs.request_replays ( + workspace_id bigint NOT NULL REFERENCES _vexfs.workspaces(workspace_id) ON DELETE CASCADE, + owner_oid oid NOT NULL, + request_id text NOT NULL CHECK (request_id <> '' AND octet_length(request_id) <= 255), + operation text NOT NULL, + argument_hash text NOT NULL CHECK (argument_hash ~ '^[0-9a-f]{64}$'), + result_text text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (workspace_id, owner_oid, request_id) +); + +CREATE INDEX vexfs_request_replays_created_idx + ON _vexfs.request_replays(workspace_id, created_at); + +CREATE TABLE _vexfs.file_locks ( + workspace_id bigint NOT NULL REFERENCES _vexfs.workspaces(workspace_id) ON DELETE CASCADE, + inode_id bigint NOT NULL REFERENCES _vexfs.inodes(inode_id) ON DELETE CASCADE, + owner_oid oid NOT NULL, + session_id text NOT NULL, + lock_kind text NOT NULL CHECK (lock_kind IN ('shared', 'exclusive')), + lease_until timestamptz NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (workspace_id, inode_id, owner_oid, session_id) +); + +CREATE INDEX vexfs_file_locks_lease_idx + ON _vexfs.file_locks(workspace_id, inode_id, lease_until); + +-- PostgreSQL excludes extension-owned table data from pg_dump unless every +-- authoritative relation is registered as extension configuration data. +-- The workspace root reference is intentionally validated by vexfs_check +-- instead of a circular workspaces -> inodes foreign key because circular +-- configuration-table dependencies cannot be restored safely by pg_dump. +-- Version aliases are validated the same way instead of using a self-referencing +-- file_versions foreign key, which would create another circular restore order. +SELECT pg_catalog.pg_extension_config_dump('_vexfs.workspaces', ''); +SELECT pg_catalog.pg_extension_config_dump('_vexfs.inodes', ''); +SELECT pg_catalog.pg_extension_config_dump('_vexfs.dentries', ''); +SELECT pg_catalog.pg_extension_config_dump('_vexfs.commits', ''); +SELECT pg_catalog.pg_extension_config_dump('_vexfs.manifests', ''); +SELECT pg_catalog.pg_extension_config_dump('_vexfs.chunks', ''); +SELECT pg_catalog.pg_extension_config_dump('_vexfs.manifest_chunks', ''); +SELECT pg_catalog.pg_extension_config_dump('_vexfs.file_versions', ''); +SELECT pg_catalog.pg_extension_config_dump('_vexfs.xattrs', ''); +SELECT pg_catalog.pg_extension_config_dump('_vexfs.acl_entries', ''); +SELECT pg_catalog.pg_extension_config_dump('_vexfs.snapshots', ''); +SELECT pg_catalog.pg_extension_config_dump('_vexfs.snapshot_inodes', ''); +SELECT pg_catalog.pg_extension_config_dump('_vexfs.snapshot_dentries', ''); +SELECT pg_catalog.pg_extension_config_dump('_vexfs.snapshot_xattrs', ''); +SELECT pg_catalog.pg_extension_config_dump('_vexfs.snapshot_acl_entries', ''); +SELECT pg_catalog.pg_extension_config_dump('_vexfs.audit_events', ''); +SELECT pg_catalog.pg_extension_config_dump('_vexfs.workspaces_workspace_id_seq', ''); +SELECT pg_catalog.pg_extension_config_dump('_vexfs.inodes_inode_id_seq', ''); +SELECT pg_catalog.pg_extension_config_dump('_vexfs.manifests_manifest_id_seq', ''); +SELECT pg_catalog.pg_extension_config_dump('_vexfs.chunks_chunk_id_seq', ''); +SELECT pg_catalog.pg_extension_config_dump('_vexfs.snapshots_snapshot_id_seq', ''); +SELECT pg_catalog.pg_extension_config_dump('_vexfs.audit_events_event_id_seq', ''); + +CREATE FUNCTION _vexfs.reset_live_usage_on_workspace_insert() +RETURNS trigger +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = pg_catalog, _vexfs +AS $$ +BEGIN + -- pg_dump restores the saved workspace row before inode rows. Rebuild the + -- derived counters through inode triggers instead of counting them twice. + NEW.live_files := 0; + NEW.live_bytes := 0; + NEW.grep_index_dirty := NEW.grep_index_enabled; + RETURN NEW; +END; +$$; + +CREATE TRIGGER vexfs_workspace_reset_live_usage +BEFORE INSERT ON _vexfs.workspaces +FOR EACH ROW EXECUTE FUNCTION _vexfs.reset_live_usage_on_workspace_insert(); + +CREATE FUNCTION _vexfs.update_live_usage() +RETURNS trigger +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = pg_catalog, _vexfs +AS $$ +BEGIN + IF TG_OP IN ('UPDATE', 'DELETE') + AND OLD.kind <> 'directory' AND OLD.live THEN + UPDATE _vexfs.workspaces + SET live_files = live_files - 1, + live_bytes = live_bytes - OLD.size_bytes + WHERE workspace_id = OLD.workspace_id; + END IF; + IF TG_OP IN ('INSERT', 'UPDATE') + AND NEW.kind <> 'directory' AND NEW.live THEN + UPDATE _vexfs.workspaces + SET live_files = live_files + 1, + live_bytes = live_bytes + NEW.size_bytes + WHERE workspace_id = NEW.workspace_id; + END IF; + IF TG_OP = 'DELETE' THEN + RETURN OLD; + END IF; + RETURN NEW; +END; +$$; + +CREATE TRIGGER vexfs_inode_live_usage_insert +AFTER INSERT ON _vexfs.inodes +FOR EACH ROW EXECUTE FUNCTION _vexfs.update_live_usage(); + +CREATE TRIGGER vexfs_inode_live_usage_update +AFTER UPDATE OF workspace_id, kind, size_bytes, live ON _vexfs.inodes +FOR EACH ROW EXECUTE FUNCTION _vexfs.update_live_usage(); + +CREATE TRIGGER vexfs_inode_live_usage_delete +AFTER DELETE ON _vexfs.inodes +FOR EACH ROW EXECUTE FUNCTION _vexfs.update_live_usage(); + +CREATE FUNCTION _vexfs.path_parts(p_path text) +RETURNS text[] +LANGUAGE plpgsql +IMMUTABLE +STRICT +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_parts text[]; + v_part text; +BEGIN + IF left(p_path, 1) <> '/' THEN + RAISE EXCEPTION 'VEXFS_INVALID_PATH: path must be absolute' + USING ERRCODE = '22023'; + END IF; + IF p_path = '/' THEN + RETURN ARRAY[]::text[]; + END IF; + + v_parts := string_to_array(substr(p_path, 2), '/'); + FOREACH v_part IN ARRAY v_parts LOOP + IF v_part = '' OR v_part = '.' OR v_part = '..' THEN + RAISE EXCEPTION 'VEXFS_INVALID_PATH: invalid path component' + USING ERRCODE = '22023'; + END IF; + IF octet_length(v_part) > 255 THEN + RAISE EXCEPTION 'VEXFS_INVALID_PATH: path component is too long' + USING ERRCODE = '22023'; + END IF; + END LOOP; + RETURN v_parts; +END; +$$; + +CREATE FUNCTION _vexfs.permission_contains( + p_permissions text, + p_required text) +RETURNS boolean +LANGUAGE sql +IMMUTABLE +STRICT +PARALLEL SAFE +SET search_path = pg_catalog, _vexfs +AS $$ + SELECT EXISTS ( + SELECT 1 + FROM regexp_split_to_table(lower(p_permissions), '\s*,\s*') AS token + WHERE token IN (lower(p_required), 'admin')) +$$; + +CREATE FUNCTION _vexfs.has_acl_permission( + p_workspace_id bigint, + p_inode_id bigint, + p_permission text) +RETURNS boolean +LANGUAGE plpgsql +SECURITY DEFINER +STABLE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_actor_oid oid; + v_superuser boolean; + v_owner_oid oid; + v_denied boolean; + v_allowed boolean; +BEGIN + SELECT r.oid, r.rolsuper INTO STRICT v_actor_oid, v_superuser + FROM pg_catalog.pg_roles AS r + WHERE r.rolname = session_user; + SELECT i.owner_oid INTO v_owner_oid + FROM _vexfs.inodes AS i + WHERE i.workspace_id = p_workspace_id + AND i.inode_id = p_inode_id + AND i.live; + IF NOT FOUND THEN + RETURN false; + END IF; + IF v_superuser OR v_owner_oid = v_actor_oid THEN + RETURN true; + END IF; + + SELECT coalesce(bool_or( + acl.effect = 'deny' + AND _vexfs.permission_contains(acl.permissions, p_permission)), false), + coalesce(bool_or( + acl.effect = 'allow' + AND _vexfs.permission_contains(acl.permissions, p_permission)), false) + INTO v_denied, v_allowed + FROM _vexfs.acl_entries AS acl + LEFT JOIN pg_catalog.pg_roles AS acl_role + ON acl_role.rolname = acl.principal + WHERE acl.workspace_id = p_workspace_id + AND acl.inode_id = p_inode_id + AND (acl.principal IN (session_user, 'public', '*') + OR (acl_role.oid IS NOT NULL + AND pg_catalog.pg_has_role(v_actor_oid, acl_role.oid, 'member'))); + RETURN NOT v_denied AND v_allowed; +END; +$$; + +CREATE FUNCTION _vexfs.require_inode_permission( + p_workspace_id bigint, + p_inode_id bigint, + p_permission text) +RETURNS void +LANGUAGE plpgsql +SECURITY DEFINER +STABLE +SET search_path = pg_catalog, _vexfs +AS $$ +BEGIN + IF NOT _vexfs.has_acl_permission( + p_workspace_id, p_inode_id, p_permission) THEN + RAISE EXCEPTION 'VEXFS_PERMISSION_DENIED: % permission denied for inode %', + p_permission, p_inode_id + USING ERRCODE = '42501'; + END IF; +END; +$$; + +CREATE FUNCTION _vexfs.require_workspace( + p_name text, + p_permission text DEFAULT 'read') +RETURNS _vexfs.workspaces +LANGUAGE plpgsql +SECURITY DEFINER +STABLE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; + v_principal_oid oid; + v_superuser boolean; +BEGIN + SELECT * INTO v_workspace + FROM _vexfs.workspaces AS w + WHERE w.name = p_name; + IF NOT FOUND THEN + RAISE EXCEPTION 'VEXFS_WORKSPACE_NOT_FOUND: %', p_name + USING ERRCODE = 'P0002'; + END IF; + + SELECT r.oid, r.rolsuper INTO v_principal_oid, v_superuser + FROM pg_catalog.pg_roles AS r + WHERE r.rolname = session_user; + IF v_workspace.state <> 'active' AND NOT coalesce(v_superuser, false) THEN + RAISE EXCEPTION 'VEXFS_WORKSPACE_BUSY: workspace % is being imported', p_name + USING ERRCODE = '55006'; + END IF; + IF v_workspace.owner_oid <> v_principal_oid AND NOT coalesce(v_superuser, false) THEN + IF v_workspace.root_inode IS NULL OR NOT _vexfs.has_acl_permission( + v_workspace.workspace_id, v_workspace.root_inode, p_permission) THEN + RAISE EXCEPTION 'VEXFS_PERMISSION_DENIED: workspace % belongs to role %', + p_name, v_workspace.owner_role + USING ERRCODE = '42501'; + END IF; + END IF; + RETURN v_workspace; +END; +$$; + +CREATE FUNCTION _vexfs.resolve_path(p_workspace_id bigint, p_path text) +RETURNS bigint +LANGUAGE plpgsql +SECURITY DEFINER +STABLE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_parts text[]; + v_part text; + v_inode bigint; + v_next bigint; + v_kind text; +BEGIN + SELECT w.root_inode INTO v_inode + FROM _vexfs.workspaces AS w + WHERE w.workspace_id = p_workspace_id; + IF v_inode IS NULL THEN + RAISE EXCEPTION 'VEXFS_WORKSPACE_NOT_FOUND: %', p_workspace_id + USING ERRCODE = 'P0002'; + END IF; + + v_parts := _vexfs.path_parts(p_path); + FOREACH v_part IN ARRAY v_parts LOOP + PERFORM _vexfs.require_inode_permission( + p_workspace_id, v_inode, 'execute'); + SELECT d.inode_id INTO v_next + FROM _vexfs.dentries AS d + WHERE d.workspace_id = p_workspace_id + AND d.parent_inode = v_inode + AND d.name = v_part; + IF NOT FOUND THEN + RAISE EXCEPTION 'VEXFS_PATH_NOT_FOUND: %', p_path + USING ERRCODE = 'P0002'; + END IF; + SELECT i.kind INTO v_kind + FROM _vexfs.inodes AS i + WHERE i.inode_id = v_next; + v_inode := v_next; + END LOOP; + RETURN v_inode; +END; +$$; + +CREATE FUNCTION _vexfs.resolve_parent(p_workspace_id bigint, p_path text) +RETURNS TABLE(parent_inode bigint, entry_name text) +LANGUAGE plpgsql +SECURITY DEFINER +STABLE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_parts text[]; + v_count integer; + v_parent_path text; +BEGIN + v_parts := _vexfs.path_parts(p_path); + v_count := coalesce(array_length(v_parts, 1), 0); + IF v_count = 0 THEN + RAISE EXCEPTION 'VEXFS_INVALID_PATH: workspace root has no parent' + USING ERRCODE = '22023'; + END IF; + + entry_name := v_parts[v_count]; + IF v_count = 1 THEN + v_parent_path := '/'; + ELSE + v_parent_path := '/' || array_to_string(v_parts[1:v_count - 1], '/'); + END IF; + parent_inode := _vexfs.resolve_path(p_workspace_id, v_parent_path); + RETURN NEXT; +END; +$$; + +CREATE FUNCTION _vexfs.path_for_inode( + p_workspace_id bigint, + p_inode_id bigint) +RETURNS text +LANGUAGE sql +SECURITY DEFINER +STABLE +SET search_path = pg_catalog, _vexfs +AS $$ + WITH RECURSIVE paths(inode_id, path, visited) AS ( + SELECT w.root_inode, '/'::text, ARRAY[w.root_inode]::bigint[] + FROM _vexfs.workspaces AS w + WHERE w.workspace_id = p_workspace_id + UNION ALL + SELECT d.inode_id, + CASE WHEN paths.path = '/' THEN '/' || d.name + ELSE paths.path || '/' || d.name END, + paths.visited || d.inode_id + FROM paths + JOIN _vexfs.dentries AS d + ON d.workspace_id = p_workspace_id + AND d.parent_inode = paths.inode_id + WHERE NOT d.inode_id = ANY(paths.visited) + AND octet_length(paths.path) + octet_length(d.name) + 1 <= 4096) + SELECT paths.path + FROM paths + WHERE paths.inode_id = p_inode_id + ORDER BY paths.path + LIMIT 1 +$$; + +CREATE FUNCTION _vexfs.record_commit( + p_workspace_id bigint, + p_operation text, + p_path text DEFAULT NULL, + p_inode_id bigint DEFAULT NULL, + p_details jsonb DEFAULT '{}'::jsonb) +RETURNS bigint +LANGUAGE plpgsql +SECURITY DEFINER +VOLATILE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_commit bigint; + v_principal_oid oid; +BEGIN + SELECT r.oid INTO STRICT v_principal_oid + FROM pg_catalog.pg_roles AS r + WHERE r.rolname = session_user; + + UPDATE _vexfs.workspaces + SET head_commit = head_commit + 1, + history_floor_commit = CASE + WHEN history_floor_commit = 0 THEN head_commit + 1 + ELSE history_floor_commit + END, + cache_generation = cache_generation + 1 + WHERE workspace_id = p_workspace_id + RETURNING head_commit INTO v_commit; + IF NOT FOUND THEN + RAISE EXCEPTION 'VEXFS_WORKSPACE_NOT_FOUND: %', p_workspace_id + USING ERRCODE = 'P0002'; + END IF; + + INSERT INTO _vexfs.commits( + workspace_id, commit_no, parent_commit, operation, + created_by_oid, created_by) + VALUES ( + p_workspace_id, v_commit, v_commit - 1, p_operation, + v_principal_oid, session_user); + PERFORM _vexfs.audit( + p_workspace_id, v_commit, p_operation, + p_path, p_inode_id, p_details); + PERFORM pg_catalog.pg_notify( + 'vexfs_change', + jsonb_build_object( + 'workspace_id', p_workspace_id, + 'head_commit', v_commit, + 'operation', p_operation)::text); + RETURN v_commit; +END; +$$; + +CREATE FUNCTION _vexfs.audit( + p_workspace_id bigint, + p_commit_no bigint, + p_operation text, + p_path text DEFAULT NULL, + p_inode_id bigint DEFAULT NULL, + p_details jsonb DEFAULT '{}'::jsonb) +RETURNS bigint +LANGUAGE plpgsql +SECURITY DEFINER +VOLATILE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_event bigint; + v_actor_oid oid; + v_workspace_name text; +BEGIN + SELECT r.oid INTO STRICT v_actor_oid + FROM pg_catalog.pg_roles AS r + WHERE r.rolname = session_user; + SELECT w.name INTO STRICT v_workspace_name + FROM _vexfs.workspaces AS w + WHERE w.workspace_id = p_workspace_id; + INSERT INTO _vexfs.audit_events( + workspace_id, workspace_name, commit_no, actor_oid, actor_role, + operation, path, inode_id, details) + VALUES ( + p_workspace_id, v_workspace_name, p_commit_no, v_actor_oid, session_user, + p_operation, p_path, p_inode_id, coalesce(p_details, '{}'::jsonb)) + RETURNING event_id INTO v_event; + RETURN v_event; +END; +$$; + +CREATE FUNCTION _vexfs.inherit_acl( + p_workspace_id bigint, + p_parent_inode bigint, + p_child_inode bigint) +RETURNS void +LANGUAGE sql +SECURITY DEFINER +VOLATILE +SET search_path = pg_catalog, _vexfs +AS $$ + INSERT INTO _vexfs.acl_entries( + workspace_id, inode_id, principal, effect, + permissions, inherit_flags, created_at, updated_at) + SELECT p_workspace_id, + p_child_inode, + acl.principal, + acl.effect, + acl.permissions, + acl.inherit_flags, + clock_timestamp(), + clock_timestamp() + FROM _vexfs.acl_entries AS acl + WHERE acl.workspace_id = p_workspace_id + AND acl.inode_id = p_parent_inode + AND acl.inherit_flags <> 0 + ON CONFLICT (workspace_id, inode_id, principal, effect) DO NOTHING +$$; + +CREATE FUNCTION _vexfs.random_token(p_prefix text) +RETURNS text +LANGUAGE sql +VOLATILE +SET search_path = pg_catalog, _vexfs +AS $$ + SELECT p_prefix || '-' || encode(pg_catalog.sha256(convert_to( + clock_timestamp()::text || ':' || random()::text || ':' || + pg_catalog.pg_backend_pid()::text, 'UTF8')), 'hex') +$$; + +CREATE FUNCTION _vexfs.request_digest(p_arguments jsonb) +RETURNS text +LANGUAGE sql +IMMUTABLE +STRICT +PARALLEL SAFE +SET search_path = pg_catalog, _vexfs +AS $$ + SELECT encode(pg_catalog.sha256(convert_to(p_arguments::text, 'UTF8')), 'hex') +$$; + +CREATE FUNCTION _vexfs.lock_request( + p_workspace_id bigint, + p_request_id text) +RETURNS void +LANGUAGE plpgsql +SECURITY DEFINER +VOLATILE +SET search_path = pg_catalog, _vexfs +AS $$ +BEGIN + IF p_request_id IS NOT NULL AND p_request_id <> '' THEN + PERFORM pg_catalog.pg_advisory_xact_lock( + pg_catalog.hashtextextended( + p_workspace_id::text || ':' || session_user || ':' || p_request_id, + 1448239925)); + END IF; +END; +$$; + +CREATE FUNCTION _vexfs.replay_get( + p_workspace_id bigint, + p_request_id text, + p_operation text, + p_argument_hash text) +RETURNS text +LANGUAGE plpgsql +SECURITY DEFINER +STABLE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_actor oid; + v_operation text; + v_hash text; + v_result text; +BEGIN + IF p_request_id IS NULL OR p_request_id = '' THEN + RETURN NULL; + END IF; + SELECT r.oid INTO STRICT v_actor + FROM pg_catalog.pg_roles AS r WHERE r.rolname = session_user; + SELECT replay.operation, replay.argument_hash, replay.result_text + INTO v_operation, v_hash, v_result + FROM _vexfs.request_replays AS replay + WHERE replay.workspace_id = p_workspace_id + AND replay.owner_oid = v_actor + AND replay.request_id = p_request_id; + IF NOT FOUND THEN + RETURN NULL; + END IF; + IF v_operation IS DISTINCT FROM p_operation OR v_hash IS DISTINCT FROM p_argument_hash THEN + RAISE EXCEPTION 'VEXFS_REQUEST_CONFLICT: request id was reused with different arguments' + USING ERRCODE = '40001'; + END IF; + RETURN v_result; +END; +$$; + +CREATE FUNCTION _vexfs.replay_put( + p_workspace_id bigint, + p_request_id text, + p_operation text, + p_argument_hash text, + p_result text) +RETURNS void +LANGUAGE plpgsql +SECURITY DEFINER +VOLATILE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_actor oid; +BEGIN + IF p_request_id IS NULL OR p_request_id = '' THEN + RETURN; + END IF; + IF octet_length(p_request_id) > 255 THEN + RAISE EXCEPTION 'VEXFS_INVALID_REQUEST: request id is too long' + USING ERRCODE = '22023'; + END IF; + SELECT r.oid INTO STRICT v_actor + FROM pg_catalog.pg_roles AS r WHERE r.rolname = session_user; + INSERT INTO _vexfs.request_replays( + workspace_id, owner_oid, request_id, operation, argument_hash, result_text) + VALUES ( + p_workspace_id, v_actor, p_request_id, p_operation, p_argument_hash, p_result) + ON CONFLICT (workspace_id, owner_oid, request_id) DO NOTHING; + DELETE FROM _vexfs.request_replays AS replay + WHERE replay.workspace_id = p_workspace_id + AND replay.created_at < clock_timestamp() - interval '7 days'; +END; +$$; + +CREATE FUNCTION _vexfs.require_handle( + p_handle text, + p_permission text DEFAULT 'read') +RETURNS _vexfs.handles +LANGUAGE plpgsql +SECURITY DEFINER +VOLATILE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_handle _vexfs.handles%ROWTYPE; + v_actor oid; +BEGIN + SELECT r.oid INTO STRICT v_actor + FROM pg_catalog.pg_roles AS r WHERE r.rolname = session_user; + SELECT * INTO v_handle + FROM _vexfs.handles AS handle + WHERE handle.handle_id = p_handle + AND handle.owner_oid = v_actor + AND handle.state IN ('open', 'retained') + FOR UPDATE; + IF NOT FOUND THEN + RAISE EXCEPTION 'VEXFS_HANDLE_NOT_FOUND: %', p_handle + USING ERRCODE = 'P0002'; + END IF; + IF v_handle.inode_id IS NOT NULL THEN + PERFORM _vexfs.require_inode_permission( + v_handle.workspace_id, v_handle.inode_id, p_permission); + END IF; + UPDATE _vexfs.handles + SET lease_until = clock_timestamp() + interval '30 seconds', + updated_at = clock_timestamp(), + state = 'open' + WHERE handle_id = p_handle; + v_handle.lease_until := clock_timestamp() + interval '30 seconds'; + v_handle.state := 'open'; + RETURN v_handle; +END; +$$; + +CREATE FUNCTION _vexfs.reap_orphan_inodes(p_workspace_id bigint) +RETURNS bigint +LANGUAGE plpgsql +SECURITY DEFINER +VOLATILE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_count bigint; +BEGIN + WITH reaped AS ( + UPDATE _vexfs.inodes AS inode + SET live = false, + changed_at = clock_timestamp() + WHERE inode.workspace_id = p_workspace_id + AND inode.live + AND inode.kind <> 'directory' + AND NOT EXISTS ( + SELECT 1 + FROM _vexfs.dentries AS link + WHERE link.workspace_id = inode.workspace_id + AND link.inode_id = inode.inode_id) + AND NOT EXISTS ( + SELECT 1 + FROM _vexfs.handles AS handle + WHERE handle.workspace_id = inode.workspace_id + AND handle.inode_id = inode.inode_id + AND handle.state IN ('open', 'retained')) + RETURNING 1) + SELECT count(*) INTO v_count FROM reaped; + RETURN v_count; +END; +$$; + +CREATE FUNCTION _vexfs.overlay_bytes( + p_base bytea, + p_offset bigint, + p_value bytea) +RETURNS bytea +LANGUAGE plpgsql +IMMUTABLE +STRICT +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_size bigint; + v_end bigint; + v_prefix bytea; + v_padding bytea := ''::bytea; + v_suffix bytea := ''::bytea; +BEGIN + IF p_offset < 0 OR p_offset > 134217728 THEN + RAISE EXCEPTION 'VEXFS_INVALID_RANGE: offset is outside 0..134217728' + USING ERRCODE = '22023'; + END IF; + v_size := octet_length(p_base); + v_end := p_offset + octet_length(p_value); + IF v_end > 134217728 THEN + RAISE EXCEPTION 'VEXFS_FILE_TOO_LARGE: staged file exceeds 128 MiB' + USING ERRCODE = '54000'; + END IF; + v_prefix := substring(p_base FROM 1 FOR least(p_offset, v_size)::integer); + IF p_offset > v_size THEN + v_padding := decode(repeat('00', (p_offset - v_size)::integer), 'hex'); + END IF; + IF v_end < v_size THEN + v_suffix := substring(p_base FROM (v_end + 1)::integer); + END IF; + RETURN v_prefix || v_padding || p_value || v_suffix; +END; +$$; + +CREATE FUNCTION _vexfs.resolve_version_storage( + p_workspace_id bigint, + p_inode_id bigint, + p_version bigint) +RETURNS TABLE( + manifest_id bigint, + size_bytes bigint, + checksum text, + canonical_version bigint, + chunk_count integer) +LANGUAGE plpgsql +SECURITY DEFINER +STABLE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_version_manifest bigint; + v_source_version bigint; + v_source_manifest bigint; + v_resolved_manifest bigint; + v_source_size bigint; + v_source_checksum text; + v_manifest_workspace bigint; + v_manifest_inode bigint; + v_manifest_size bigint; + v_manifest_chunk_size integer; + v_manifest_checksum text; +BEGIN + SELECT v.manifest_id, + v.source_version_no, + v.size_bytes, + v.checksum, + source.manifest_id, + source.size_bytes, + source.checksum, + source.version_no + INTO v_version_manifest, + v_source_version, + size_bytes, + checksum, + v_source_manifest, + v_source_size, + v_source_checksum, + canonical_version + FROM _vexfs.file_versions AS v + LEFT JOIN _vexfs.file_versions AS source + ON source.workspace_id = v.workspace_id + AND source.inode_id = v.inode_id + AND source.version_no = v.source_version_no + AND source.source_version_no IS NULL + WHERE v.workspace_id = p_workspace_id + AND v.inode_id = p_inode_id + AND v.version_no = p_version; + IF NOT FOUND THEN + RAISE EXCEPTION 'VEXFS_VERSION_NOT_FOUND: %', p_version + USING ERRCODE = 'P0002'; + END IF; + + IF v_source_version IS NULL THEN + v_resolved_manifest := v_version_manifest; + canonical_version := p_version; + ELSE + v_resolved_manifest := v_source_manifest; + IF canonical_version IS NULL + OR v_source_size IS DISTINCT FROM size_bytes + OR v_source_checksum IS DISTINCT FROM checksum THEN + RAISE EXCEPTION 'VEXFS_CORRUPT: version alias metadata is invalid' + USING ERRCODE = 'XX001'; + END IF; + END IF; + IF v_resolved_manifest IS NULL THEN + RAISE EXCEPTION 'VEXFS_CORRUPT: version manifest is missing' + USING ERRCODE = 'XX001'; + END IF; + + SELECT m.workspace_id, + m.inode_id, + m.file_size, + m.chunk_size, + m.chunk_count, + m.checksum + INTO v_manifest_workspace, + v_manifest_inode, + v_manifest_size, + v_manifest_chunk_size, + chunk_count, + v_manifest_checksum + FROM _vexfs.manifests AS m + WHERE m.manifest_id = v_resolved_manifest; + IF NOT FOUND + OR v_manifest_workspace <> p_workspace_id + OR v_manifest_inode <> p_inode_id + OR v_manifest_size <> size_bytes + OR v_manifest_chunk_size <> 65536 + OR chunk_count <> ((size_bytes + 65535) / 65536)::integer + OR v_manifest_checksum IS DISTINCT FROM checksum THEN + RAISE EXCEPTION 'VEXFS_CORRUPT: version manifest metadata is invalid' + USING ERRCODE = 'XX001'; + END IF; + manifest_id := v_resolved_manifest; + RETURN NEXT; +END; +$$; + +CREATE FUNCTION _vexfs.store_manifest( + p_workspace_id bigint, + p_inode_id bigint, + p_previous_manifest bigint, + p_content bytea) +RETURNS bigint +LANGUAGE plpgsql +SECURITY DEFINER +VOLATILE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_manifest bigint; + v_file_size bigint; + v_file_checksum text; + v_chunk_count integer; + v_chunk_no integer := 0; + v_offset integer := 0; + v_chunk bytea; + v_chunk_size integer; + v_chunk_checksum text; + v_chunk_id bigint; +BEGIN + IF p_content IS NULL THEN + RAISE EXCEPTION 'VEXFS_INVALID_CONTENT: content cannot be null' + USING ERRCODE = '22004'; + END IF; + v_file_size := octet_length(p_content); + IF v_file_size > 134217728 THEN + RAISE EXCEPTION 'VEXFS_FILE_TOO_LARGE: maximum file size is 128 MiB' + USING ERRCODE = '54000'; + END IF; + v_file_checksum := encode(pg_catalog.sha256(p_content), 'hex'); + v_chunk_count := ((v_file_size + 65535) / 65536)::integer; + + INSERT INTO _vexfs.manifests( + workspace_id, inode_id, file_size, chunk_size, chunk_count, checksum) + VALUES ( + p_workspace_id, p_inode_id, v_file_size, 65536, + v_chunk_count, v_file_checksum) + RETURNING manifest_id INTO v_manifest; + + WHILE v_offset < v_file_size LOOP + v_chunk_size := least(65536, v_file_size - v_offset)::integer; + v_chunk := substring(p_content FROM v_offset + 1 FOR v_chunk_size); + v_chunk_checksum := encode(pg_catalog.sha256(v_chunk), 'hex'); + v_chunk_id := NULL; + + IF p_previous_manifest IS NOT NULL THEN + SELECT c.chunk_id INTO v_chunk_id + FROM _vexfs.manifest_chunks AS entry + JOIN _vexfs.manifests AS previous + ON previous.manifest_id = entry.manifest_id + JOIN _vexfs.chunks AS c + ON c.chunk_id = entry.chunk_id + WHERE entry.manifest_id = p_previous_manifest + AND entry.chunk_no = v_chunk_no + AND previous.workspace_id = p_workspace_id + AND previous.inode_id = p_inode_id + AND c.workspace_id = p_workspace_id + AND c.inode_id = p_inode_id + AND c.size_bytes = v_chunk_size + AND c.checksum = v_chunk_checksum + AND c.content = v_chunk; + END IF; + + IF v_chunk_id IS NULL THEN + SELECT c.chunk_id INTO v_chunk_id + FROM _vexfs.manifest_chunks AS entry + JOIN _vexfs.chunks AS c ON c.chunk_id = entry.chunk_id + WHERE entry.manifest_id = v_manifest + AND c.workspace_id = p_workspace_id + AND c.inode_id = p_inode_id + AND c.size_bytes = v_chunk_size + AND c.checksum = v_chunk_checksum + AND c.content = v_chunk + LIMIT 1; + END IF; + + IF v_chunk_id IS NULL THEN + INSERT INTO _vexfs.chunks( + workspace_id, inode_id, content, size_bytes, checksum) + VALUES ( + p_workspace_id, p_inode_id, v_chunk, + v_chunk_size, v_chunk_checksum) + RETURNING chunk_id INTO v_chunk_id; + END IF; + + INSERT INTO _vexfs.manifest_chunks(manifest_id, chunk_no, chunk_id) + VALUES (v_manifest, v_chunk_no, v_chunk_id); + v_chunk_no := v_chunk_no + 1; + v_offset := v_offset + v_chunk_size; + END LOOP; + RETURN v_manifest; +END; +$$; + +CREATE FUNCTION _vexfs.read_version_content( + p_workspace_id bigint, + p_inode_id bigint, + p_version bigint) +RETURNS bytea +LANGUAGE plpgsql +SECURITY DEFINER +STABLE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_manifest bigint; + v_size bigint; + v_checksum text; + v_canonical bigint; + v_chunk_count integer; + v_content bytea := ''::bytea; + v_expected_chunk integer := 0; + v_offset bigint := 0; + v_expected_size integer; + v_row record; +BEGIN + SELECT storage.manifest_id, + storage.size_bytes, + storage.checksum, + storage.canonical_version, + storage.chunk_count + INTO v_manifest, + v_size, + v_checksum, + v_canonical, + v_chunk_count + FROM _vexfs.resolve_version_storage( + p_workspace_id, p_inode_id, p_version) AS storage; + + IF v_size > 134217728 THEN + RAISE EXCEPTION 'VEXFS_CORRUPT: file version exceeds 128 MiB' + USING ERRCODE = 'XX001'; + END IF; + FOR v_row IN + SELECT entry.chunk_no, + c.content, + c.size_bytes, + c.checksum, + c.workspace_id, + c.inode_id + FROM _vexfs.manifest_chunks AS entry + JOIN _vexfs.chunks AS c ON c.chunk_id = entry.chunk_id + WHERE entry.manifest_id = v_manifest + ORDER BY entry.chunk_no + LOOP + v_expected_size := least(65536, v_size - v_offset)::integer; + IF v_row.chunk_no <> v_expected_chunk + OR v_expected_size <= 0 + OR v_row.size_bytes <> v_expected_size + OR octet_length(v_row.content) <> v_expected_size + OR v_row.workspace_id <> p_workspace_id + OR v_row.inode_id <> p_inode_id + OR v_row.checksum <> encode(pg_catalog.sha256(v_row.content), 'hex') THEN + RAISE EXCEPTION 'VEXFS_CHECKSUM_MISMATCH: file manifest chunk is invalid' + USING ERRCODE = 'XX001'; + END IF; + v_content := v_content || v_row.content; + v_offset := v_offset + v_row.size_bytes; + v_expected_chunk := v_expected_chunk + 1; + END LOOP; + + IF v_offset <> v_size + OR v_expected_chunk <> v_chunk_count + OR encode(pg_catalog.sha256(v_content), 'hex') <> v_checksum THEN + RAISE EXCEPTION 'VEXFS_CHECKSUM_MISMATCH: file content does not match manifest' + USING ERRCODE = 'XX001'; + END IF; + RETURN v_content; +END; +$$; + +CREATE FUNCTION _vexfs.enforce_quota( + p_workspace_id bigint, + p_inode_id bigint, + p_new_size bigint) +RETURNS void +LANGUAGE plpgsql +SECURITY DEFINER +STABLE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_max_bytes bigint; + v_max_files bigint; + v_max_file_bytes bigint; + v_live_bytes bigint; + v_live_files bigint; + v_old_size bigint := 0; + v_old_live boolean := false; + v_new_file boolean; + v_projected_bytes bigint; +BEGIN + IF p_new_size < 0 THEN + RAISE EXCEPTION 'VEXFS_INVALID_SIZE: file size must be non-negative' + USING ERRCODE = '22023'; + END IF; + SELECT w.quota_max_bytes, + w.quota_max_files, + w.quota_max_file_bytes, + w.live_bytes, + w.live_files + INTO STRICT v_max_bytes, + v_max_files, + v_max_file_bytes, + v_live_bytes, + v_live_files + FROM _vexfs.workspaces AS w + WHERE w.workspace_id = p_workspace_id; + + IF p_inode_id IS NOT NULL THEN + SELECT i.size_bytes, i.live + INTO v_old_size, v_old_live + FROM _vexfs.inodes AS i + WHERE i.workspace_id = p_workspace_id + AND i.inode_id = p_inode_id + AND i.kind <> 'directory'; + IF NOT FOUND THEN + RAISE EXCEPTION 'VEXFS_CORRUPT: quota target inode is invalid' + USING ERRCODE = 'XX001'; + END IF; + END IF; + v_new_file := p_inode_id IS NULL OR NOT v_old_live; + v_projected_bytes := v_live_bytes + - CASE WHEN v_old_live THEN v_old_size ELSE 0 END + + p_new_size; + + IF v_max_file_bytes IS NOT NULL + AND p_new_size > v_max_file_bytes + AND (v_new_file OR p_new_size >= v_old_size) THEN + RAISE EXCEPTION 'VEXFS_QUOTA_FILE_SIZE: maximum file size quota exceeded' + USING ERRCODE = '53100'; + END IF; + IF v_max_bytes IS NOT NULL + AND v_projected_bytes > v_max_bytes + AND v_projected_bytes >= v_live_bytes THEN + RAISE EXCEPTION 'VEXFS_QUOTA_BYTES: workspace byte quota exceeded' + USING ERRCODE = '53100'; + END IF; + IF v_new_file + AND v_max_files IS NOT NULL + AND v_live_files + 1 > v_max_files THEN + RAISE EXCEPTION 'VEXFS_QUOTA_FILES: workspace file quota exceeded' + USING ERRCODE = '53100'; + END IF; +END; +$$; + +CREATE FUNCTION _vexfs.enforce_snapshot_quota( + p_workspace_id bigint, + p_snapshot_id bigint) +RETURNS void +LANGUAGE plpgsql +SECURITY DEFINER +STABLE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_max_bytes bigint; + v_max_files bigint; + v_max_file_bytes bigint; + v_live_bytes bigint; + v_live_files bigint; + v_live_largest bigint; + v_target_bytes bigint; + v_target_files bigint; + v_target_largest bigint; +BEGIN + SELECT w.quota_max_bytes, + w.quota_max_files, + w.quota_max_file_bytes, + w.live_bytes, + w.live_files, + coalesce(max(i.size_bytes) FILTER (WHERE i.kind <> 'directory' AND i.live), 0) + INTO STRICT v_max_bytes, + v_max_files, + v_max_file_bytes, + v_live_bytes, + v_live_files, + v_live_largest + FROM _vexfs.workspaces AS w + LEFT JOIN _vexfs.inodes AS i ON i.workspace_id = w.workspace_id + WHERE w.workspace_id = p_workspace_id + GROUP BY w.workspace_id; + + SELECT count(*) FILTER (WHERE s.kind <> 'directory'), + coalesce(sum(s.size_bytes) FILTER (WHERE s.kind <> 'directory'), 0), + coalesce(max(s.size_bytes) FILTER (WHERE s.kind <> 'directory'), 0) + INTO v_target_files, v_target_bytes, v_target_largest + FROM _vexfs.snapshot_inodes AS s + WHERE s.snapshot_id = p_snapshot_id; + + IF v_max_file_bytes IS NOT NULL + AND v_target_largest > v_max_file_bytes + AND v_target_largest >= v_live_largest THEN + RAISE EXCEPTION 'VEXFS_QUOTA_FILE_SIZE: maximum file size quota exceeded' + USING ERRCODE = '53100'; + END IF; + IF v_max_bytes IS NOT NULL + AND v_target_bytes > v_max_bytes + AND v_target_bytes >= v_live_bytes THEN + RAISE EXCEPTION 'VEXFS_QUOTA_BYTES: workspace byte quota exceeded' + USING ERRCODE = '53100'; + END IF; + IF v_max_files IS NOT NULL + AND v_target_files > v_max_files + AND v_target_files >= v_live_files THEN + RAISE EXCEPTION 'VEXFS_QUOTA_FILES: workspace file quota exceeded' + USING ERRCODE = '53100'; + END IF; +END; +$$; + +CREATE FUNCTION _vexfs.retention_keep_versions(p_workspace_id bigint) +RETURNS TABLE(inode_id bigint, version_no bigint) +LANGUAGE sql +SECURITY DEFINER +STABLE +SET search_path = pg_catalog, _vexfs +AS $$ +WITH policy AS ( + SELECT w.retention_keep_versions AS keep_versions, + w.retention_keep_days AS keep_days + FROM _vexfs.workspaces AS w + WHERE w.workspace_id = p_workspace_id +), +ranked AS ( + SELECT f.inode_id, + f.version_no, + row_number() OVER ( + PARTITION BY f.inode_id ORDER BY f.version_no DESC) AS rank + FROM _vexfs.file_versions AS f + WHERE f.workspace_id = p_workspace_id +), +direct_keep AS ( + SELECT i.inode_id, i.current_version AS version_no + FROM _vexfs.inodes AS i + WHERE i.workspace_id = p_workspace_id + AND i.kind <> 'directory' + AND i.live + AND i.current_version > 0 + UNION + SELECT r.inode_id, r.version_no + FROM ranked AS r + CROSS JOIN policy AS p + WHERE p.keep_versions > 0 + AND r.rank <= p.keep_versions + UNION + SELECT f.inode_id, f.version_no + FROM _vexfs.file_versions AS f + CROSS JOIN policy AS p + WHERE f.workspace_id = p_workspace_id + AND p.keep_days > 0 + AND f.created_at >= clock_timestamp() - make_interval(days => p.keep_days) + UNION + SELECT s.inode_id, s.current_version + FROM _vexfs.snapshots AS snapshot + JOIN _vexfs.snapshot_inodes AS s + ON s.snapshot_id = snapshot.snapshot_id + WHERE snapshot.workspace_id = p_workspace_id + AND s.kind <> 'directory' + AND s.current_version > 0 +), +with_sources AS ( + SELECT keep.inode_id, keep.version_no + FROM direct_keep AS keep + UNION + SELECT source.inode_id, source.source_version_no + FROM direct_keep AS keep + JOIN _vexfs.file_versions AS source + ON source.workspace_id = p_workspace_id + AND source.inode_id = keep.inode_id + AND source.version_no = keep.version_no + WHERE source.source_version_no IS NOT NULL +) +SELECT DISTINCT keep.inode_id, keep.version_no + FROM with_sources AS keep + WHERE keep.version_no IS NOT NULL +$$; + +CREATE FUNCTION _vexfs.retention_status( + p_workspace_id bigint, + p_workspace_name text) +RETURNS jsonb +LANGUAGE plpgsql +SECURITY DEFINER +STABLE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_keep_versions integer; + v_keep_days integer; + v_gc_paused boolean; + v_live_files bigint; + v_live_bytes bigint; + v_versions bigint; + v_stored_bytes bigint; + v_live_storage_bytes bigint; + v_reclaimable_versions bigint; + v_reclaimable_bytes bigint; +BEGIN + SELECT w.retention_keep_versions, + w.retention_keep_days, + w.gc_paused, + w.live_files, + w.live_bytes + INTO STRICT v_keep_versions, + v_keep_days, + v_gc_paused, + v_live_files, + v_live_bytes + FROM _vexfs.workspaces AS w + WHERE w.workspace_id = p_workspace_id; + + SELECT count(*) INTO v_versions + FROM _vexfs.file_versions AS f + WHERE f.workspace_id = p_workspace_id; + SELECT coalesce(sum(c.size_bytes), 0) INTO v_stored_bytes + FROM _vexfs.chunks AS c + WHERE c.workspace_id = p_workspace_id; + + WITH live_manifests AS ( + SELECT DISTINCT coalesce(current_version.manifest_id, source.manifest_id) AS manifest_id + FROM _vexfs.inodes AS i + JOIN _vexfs.file_versions AS current_version + ON current_version.workspace_id = i.workspace_id + AND current_version.inode_id = i.inode_id + AND current_version.version_no = i.current_version + LEFT JOIN _vexfs.file_versions AS source + ON source.workspace_id = current_version.workspace_id + AND source.inode_id = current_version.inode_id + AND source.version_no = current_version.source_version_no + WHERE i.workspace_id = p_workspace_id + AND i.kind <> 'directory' + AND i.live + ), + live_chunks AS ( + SELECT DISTINCT entry.chunk_id + FROM live_manifests AS live + JOIN _vexfs.manifest_chunks AS entry + ON entry.manifest_id = live.manifest_id + ) + SELECT coalesce(sum(c.size_bytes), 0) INTO v_live_storage_bytes + FROM live_chunks AS live + JOIN _vexfs.chunks AS c ON c.chunk_id = live.chunk_id; + + WITH keep AS MATERIALIZED ( + SELECT * FROM _vexfs.retention_keep_versions(p_workspace_id) + ), + reclaimable AS MATERIALIZED ( + SELECT f.inode_id, f.version_no, f.manifest_id + FROM _vexfs.file_versions AS f + LEFT JOIN keep + ON keep.inode_id = f.inode_id + AND keep.version_no = f.version_no + WHERE f.workspace_id = p_workspace_id + AND keep.inode_id IS NULL + ), + reclaimable_manifests AS MATERIALIZED ( + SELECT r.manifest_id + FROM reclaimable AS r + WHERE r.manifest_id IS NOT NULL + ), + reclaimable_chunks AS ( + SELECT DISTINCT entry.chunk_id + FROM reclaimable_manifests AS candidate + JOIN _vexfs.manifest_chunks AS entry + ON entry.manifest_id = candidate.manifest_id + WHERE NOT EXISTS ( + SELECT 1 + FROM _vexfs.manifest_chunks AS retained + WHERE retained.chunk_id = entry.chunk_id + AND NOT EXISTS ( + SELECT 1 + FROM reclaimable_manifests AS removed + WHERE removed.manifest_id = retained.manifest_id)) + ) + SELECT (SELECT count(*) FROM reclaimable), + coalesce(( + SELECT sum(c.size_bytes) + FROM reclaimable_chunks AS item + JOIN _vexfs.chunks AS c ON c.chunk_id = item.chunk_id), 0) + INTO v_reclaimable_versions, v_reclaimable_bytes; + + RETURN jsonb_build_object( + 'workspace', p_workspace_name, + 'keep_versions', v_keep_versions, + 'keep_days', v_keep_days, + 'gc_paused', v_gc_paused, + 'live_files', v_live_files, + 'live_bytes', v_live_bytes, + 'stored_versions', v_versions, + 'stored_version_bytes', v_stored_bytes, + 'retained_history_bytes', greatest( + v_stored_bytes - v_live_storage_bytes, 0), + 'reclaimable_versions', v_reclaimable_versions, + 'reclaimable_bytes', v_reclaimable_bytes); +END; +$$; + +CREATE FUNCTION _vexfs.quota_status( + p_workspace_id bigint, + p_workspace_name text) +RETURNS jsonb +LANGUAGE plpgsql +SECURITY DEFINER +STABLE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; + v_largest bigint; +BEGIN + SELECT * INTO STRICT v_workspace + FROM _vexfs.workspaces AS w + WHERE w.workspace_id = p_workspace_id; + SELECT coalesce(max(i.size_bytes), 0) INTO v_largest + FROM _vexfs.inodes AS i + WHERE i.workspace_id = p_workspace_id + AND i.kind <> 'directory' + AND i.live; + RETURN jsonb_build_object( + 'workspace', p_workspace_name, + 'max_bytes', v_workspace.quota_max_bytes, + 'max_files', v_workspace.quota_max_files, + 'max_file_bytes', v_workspace.quota_max_file_bytes, + 'live_bytes', v_workspace.live_bytes, + 'live_files', v_workspace.live_files, + 'largest_file_bytes', v_largest, + 'over_quota', + (v_workspace.quota_max_bytes IS NOT NULL + AND v_workspace.live_bytes > v_workspace.quota_max_bytes) + OR (v_workspace.quota_max_files IS NOT NULL + AND v_workspace.live_files > v_workspace.quota_max_files) + OR (v_workspace.quota_max_file_bytes IS NOT NULL + AND v_largest > v_workspace.quota_max_file_bytes)); +END; +$$; + +CREATE FUNCTION public.vexfs_pg_adapter_version() +RETURNS text +LANGUAGE sql +IMMUTABLE +PARALLEL SAFE +AS $$ SELECT '0.4.0-alpha.1'::text $$; + +-- These two functions are part of the database-neutral VexFS SQL contract. +-- PostgreSQL creates the private schema atomically in CREATE EXTENSION, so init +-- is an idempotent readiness probe rather than a schema migration entrypoint. +CREATE FUNCTION public.vexfs_init() +RETURNS integer +LANGUAGE sql +STABLE +PARALLEL SAFE +AS $$ SELECT 1 $$; + +CREATE FUNCTION public.vexfs_contract_version() +RETURNS text +LANGUAGE sql +IMMUTABLE +PARALLEL SAFE +AS $$ SELECT '0.9.0'::text $$; + +CREATE FUNCTION public.vexfs_workspace_create(p_name text) +RETURNS bigint +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace_id bigint; + v_root_inode bigint; + v_principal_oid oid; +BEGIN + IF p_name IS NULL OR btrim(p_name) = '' OR p_name <> btrim(p_name) THEN + RAISE EXCEPTION 'VEXFS_INVALID_WORKSPACE: invalid workspace name' + USING ERRCODE = '22023'; + END IF; + + SELECT r.oid INTO STRICT v_principal_oid + FROM pg_catalog.pg_roles AS r + WHERE r.rolname = session_user; + + INSERT INTO _vexfs.workspaces(name, owner_oid, owner_role) + VALUES (p_name, v_principal_oid, session_user) + RETURNING workspace_id INTO v_workspace_id; + + INSERT INTO _vexfs.inodes( + workspace_id, kind, mode, owner_oid, owner_role, owner_principal) + VALUES ( + v_workspace_id, 'directory', 493, + v_principal_oid, session_user, session_user) + RETURNING inode_id INTO v_root_inode; + + UPDATE _vexfs.workspaces + SET root_inode = v_root_inode + WHERE workspace_id = v_workspace_id; + PERFORM _vexfs.record_commit( + v_workspace_id, 'workspace_create', '/', v_root_inode, + jsonb_build_object('before_version', NULL, 'after_version', 0)); + RETURN v_workspace_id; +END; +$$; + +CREATE FUNCTION public.vexfs_workspace_drop(p_name text, p_missing_ok boolean DEFAULT false) +RETURNS integer +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; +BEGIN + SELECT * INTO v_workspace + FROM _vexfs.workspaces AS w + WHERE w.name = p_name; + IF NOT FOUND THEN + IF p_missing_ok THEN + RETURN 0; + END IF; + RAISE EXCEPTION 'VEXFS_WORKSPACE_NOT_FOUND: %', p_name + USING ERRCODE = 'P0002'; + END IF; + + PERFORM _vexfs.require_workspace(p_name, 'admin'); + PERFORM _vexfs.record_commit( + v_workspace.workspace_id, 'workspace_drop', '/', v_workspace.root_inode, + jsonb_build_object( + 'before_version', v_workspace.head_commit, + 'after_version', NULL)); + DELETE FROM _vexfs.workspaces WHERE workspace_id = v_workspace.workspace_id; + RETURN 1; +END; +$$; + +CREATE FUNCTION public.vexfs_mkdir(p_workspace text, p_path text, p_recursive boolean DEFAULT true) +RETURNS bigint +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; + v_parts text[]; + v_part text; + v_count integer; + v_index integer := 0; + v_parent bigint; + v_next bigint; + v_kind text; + v_created boolean := false; + v_principal_oid oid; +BEGIN + v_workspace := _vexfs.require_workspace(p_workspace, 'write'); + SELECT r.oid INTO STRICT v_principal_oid + FROM pg_catalog.pg_roles AS r + WHERE r.rolname = session_user; + PERFORM 1 FROM _vexfs.workspaces WHERE workspace_id = v_workspace.workspace_id FOR UPDATE; + v_parts := _vexfs.path_parts(p_path); + v_count := coalesce(array_length(v_parts, 1), 0); + v_parent := v_workspace.root_inode; + IF v_count = 0 THEN + RETURN v_parent; + END IF; + + FOREACH v_part IN ARRAY v_parts LOOP + v_index := v_index + 1; + PERFORM _vexfs.require_inode_permission( + v_workspace.workspace_id, v_parent, 'execute'); + SELECT d.inode_id INTO v_next + FROM _vexfs.dentries AS d + WHERE d.workspace_id = v_workspace.workspace_id + AND d.parent_inode = v_parent + AND d.name = v_part; + IF FOUND THEN + SELECT i.kind INTO v_kind FROM _vexfs.inodes AS i WHERE i.inode_id = v_next; + IF v_kind <> 'directory' THEN + RAISE EXCEPTION 'VEXFS_NOT_DIRECTORY: %', p_path + USING ERRCODE = '42809'; + END IF; + ELSE + PERFORM _vexfs.require_inode_permission( + v_workspace.workspace_id, v_parent, 'write'); + IF NOT p_recursive AND v_index < v_count THEN + RAISE EXCEPTION 'VEXFS_PATH_NOT_FOUND: parent directory is missing' + USING ERRCODE = 'P0002'; + END IF; + INSERT INTO _vexfs.inodes( + workspace_id, kind, mode, owner_oid, owner_role, owner_principal) + VALUES ( + v_workspace.workspace_id, 'directory', 493, + v_principal_oid, session_user, session_user) + RETURNING inode_id INTO v_next; + INSERT INTO _vexfs.dentries(workspace_id, parent_inode, name, inode_id) + VALUES (v_workspace.workspace_id, v_parent, v_part, v_next); + PERFORM _vexfs.inherit_acl( + v_workspace.workspace_id, v_parent, v_next); + UPDATE _vexfs.inodes + SET modified_at = clock_timestamp(), + changed_at = clock_timestamp() + WHERE inode_id = v_parent; + v_created := true; + END IF; + v_parent := v_next; + END LOOP; + + IF v_created THEN + PERFORM _vexfs.record_commit( + v_workspace.workspace_id, 'mkdir', p_path, v_parent, + jsonb_build_object('before_version', NULL, 'after_version', 0)); + END IF; + RETURN v_parent; +END; +$$; + +CREATE FUNCTION public.vexfs_write(p_workspace text, p_path text, p_content bytea) +RETURNS bigint +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; + v_parts text[]; + v_count integer; + v_name text; + v_parent_path text; + v_parent bigint; + v_inode bigint; + v_kind text; + v_current_version bigint; + v_version bigint; + v_commit bigint; + v_previous_manifest bigint; + v_manifest bigint; + v_checksum text; + v_principal_oid oid; +BEGIN + IF p_content IS NULL THEN + RAISE EXCEPTION 'VEXFS_INVALID_CONTENT: content cannot be null' + USING ERRCODE = '22004'; + END IF; + v_workspace := _vexfs.require_workspace(p_workspace, 'write'); + SELECT r.oid INTO STRICT v_principal_oid + FROM pg_catalog.pg_roles AS r + WHERE r.rolname = session_user; + PERFORM 1 FROM _vexfs.workspaces WHERE workspace_id = v_workspace.workspace_id FOR UPDATE; + v_parts := _vexfs.path_parts(p_path); + v_count := coalesce(array_length(v_parts, 1), 0); + IF v_count = 0 THEN + RAISE EXCEPTION 'VEXFS_IS_DIRECTORY: cannot write workspace root' + USING ERRCODE = '42809'; + END IF; + + v_name := v_parts[v_count]; + IF v_count = 1 THEN + v_parent_path := '/'; + ELSE + v_parent_path := '/' || array_to_string(v_parts[1:v_count - 1], '/'); + END IF; + v_parent := _vexfs.resolve_path(v_workspace.workspace_id, v_parent_path); + PERFORM _vexfs.require_inode_permission( + v_workspace.workspace_id, v_parent, 'write'); + SELECT i.kind INTO v_kind FROM _vexfs.inodes AS i WHERE i.inode_id = v_parent; + IF v_kind <> 'directory' THEN + RAISE EXCEPTION 'VEXFS_NOT_DIRECTORY: %', v_parent_path + USING ERRCODE = '42809'; + END IF; + + SELECT d.inode_id INTO v_inode + FROM _vexfs.dentries AS d + WHERE d.workspace_id = v_workspace.workspace_id + AND d.parent_inode = v_parent + AND d.name = v_name; + IF FOUND THEN + SELECT i.kind, i.current_version + INTO v_kind, v_current_version + FROM _vexfs.inodes AS i + WHERE i.inode_id = v_inode + FOR UPDATE; + IF v_kind <> 'file' THEN + RAISE EXCEPTION 'VEXFS_NOT_REGULAR_FILE: %', p_path + USING ERRCODE = '42809'; + END IF; + PERFORM _vexfs.require_inode_permission( + v_workspace.workspace_id, v_inode, 'write'); + PERFORM _vexfs.enforce_quota( + v_workspace.workspace_id, v_inode, octet_length(p_content)); + SELECT coalesce(max(f.version_no), 0) + 1 INTO v_version + FROM _vexfs.file_versions AS f + WHERE f.workspace_id = v_workspace.workspace_id + AND f.inode_id = v_inode; + SELECT storage.manifest_id INTO v_previous_manifest + FROM _vexfs.resolve_version_storage( + v_workspace.workspace_id, v_inode, v_current_version) AS storage; + UPDATE _vexfs.inodes + SET current_version = v_version, + size_bytes = octet_length(p_content), + modified_at = clock_timestamp() + WHERE inode_id = v_inode; + ELSE + PERFORM _vexfs.enforce_quota( + v_workspace.workspace_id, NULL, octet_length(p_content)); + v_version := 1; + INSERT INTO _vexfs.inodes( + workspace_id, kind, mode, owner_oid, owner_role, owner_principal, + current_version, size_bytes) + VALUES ( + v_workspace.workspace_id, 'file', 420, + v_principal_oid, session_user, session_user, + v_version, octet_length(p_content)) + RETURNING inode_id INTO v_inode; + INSERT INTO _vexfs.dentries(workspace_id, parent_inode, name, inode_id) + VALUES (v_workspace.workspace_id, v_parent, v_name, v_inode); + PERFORM _vexfs.inherit_acl( + v_workspace.workspace_id, v_parent, v_inode); + END IF; + + v_manifest := _vexfs.store_manifest( + v_workspace.workspace_id, v_inode, v_previous_manifest, p_content); + SELECT m.checksum INTO STRICT v_checksum + FROM _vexfs.manifests AS m + WHERE m.manifest_id = v_manifest; + v_commit := _vexfs.record_commit( + v_workspace.workspace_id, 'write', p_path, v_inode, + jsonb_build_object( + 'before_version', v_current_version, + 'after_version', v_version)); + INSERT INTO _vexfs.file_versions( + workspace_id, inode_id, version_no, commit_no, manifest_id, + size_bytes, checksum, created_by_oid, created_by) + VALUES ( + v_workspace.workspace_id, v_inode, v_version, v_commit, v_manifest, + octet_length(p_content), v_checksum, v_principal_oid, session_user); + UPDATE _vexfs.inodes + SET changed_at = clock_timestamp() + WHERE inode_id = v_inode; + UPDATE _vexfs.inodes + SET modified_at = clock_timestamp(), + changed_at = clock_timestamp() + WHERE inode_id = v_parent; + RETURN v_version; +END; +$$; + +CREATE FUNCTION public.vexfs_read(p_workspace text, p_path text) +RETURNS bytea +LANGUAGE plpgsql +SECURITY DEFINER +STABLE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; + v_inode bigint; + v_kind text; + v_version bigint; +BEGIN + v_workspace := _vexfs.require_workspace(p_workspace, 'read'); + v_inode := _vexfs.resolve_path(v_workspace.workspace_id, p_path); + PERFORM _vexfs.require_inode_permission( + v_workspace.workspace_id, v_inode, 'read'); + SELECT i.kind, i.current_version INTO v_kind, v_version + FROM _vexfs.inodes AS i WHERE i.inode_id = v_inode; + IF v_kind <> 'file' THEN + RAISE EXCEPTION 'VEXFS_NOT_REGULAR_FILE: %', p_path + USING ERRCODE = '42809'; + END IF; + RETURN _vexfs.read_version_content( + v_workspace.workspace_id, v_inode, v_version); +END; +$$; + +CREATE FUNCTION public.vexfs_list(p_workspace text, p_path text) +RETURNS TABLE(name text, inode bigint, kind text, size bigint, version bigint) +LANGUAGE plpgsql +SECURITY DEFINER +STABLE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; + v_parent bigint; + v_kind text; +BEGIN + v_workspace := _vexfs.require_workspace(p_workspace, 'read'); + v_parent := _vexfs.resolve_path(v_workspace.workspace_id, p_path); + PERFORM _vexfs.require_inode_permission( + v_workspace.workspace_id, v_parent, 'read'); + SELECT i.kind INTO v_kind FROM _vexfs.inodes AS i WHERE i.inode_id = v_parent; + IF v_kind <> 'directory' THEN + RAISE EXCEPTION 'VEXFS_NOT_DIRECTORY: %', p_path + USING ERRCODE = '42809'; + END IF; + + RETURN QUERY + SELECT d.name, + i.inode_id, + i.kind, + coalesce(f.size_bytes, 0)::bigint, + i.current_version + FROM _vexfs.dentries AS d + JOIN _vexfs.inodes AS i ON i.inode_id = d.inode_id + LEFT JOIN _vexfs.file_versions AS f + ON f.workspace_id = i.workspace_id + AND f.inode_id = i.inode_id + AND f.version_no = i.current_version + WHERE d.workspace_id = v_workspace.workspace_id + AND d.parent_inode = v_parent + ORDER BY d.name; +END; +$$; + +CREATE FUNCTION public.vexfs_list_json(p_workspace text, p_path text) +RETURNS jsonb +LANGUAGE sql +SECURITY DEFINER +STABLE +SET search_path = pg_catalog, _vexfs +AS $$ + SELECT coalesce(jsonb_agg(jsonb_build_object( + 'name', item.name, + 'inode', item.inode, + 'kind', item.kind, + 'size', item.size, + 'version', item.version, + 'mode', inode.mode, + 'created_at', (extract(epoch FROM inode.created_at) * 1000)::bigint, + 'accessed_at', (extract(epoch FROM inode.accessed_at) * 1000)::bigint, + 'updated_at', (extract(epoch FROM inode.modified_at) * 1000)::bigint, + 'changed_at', (extract(epoch FROM inode.changed_at) * 1000)::bigint, + 'uid', inode.uid, + 'gid', inode.gid, + 'link_count', CASE WHEN inode.kind = 'directory' THEN 2 + ( + SELECT count(*) FROM _vexfs.dentries AS child_entry + JOIN _vexfs.inodes AS child ON child.inode_id = child_entry.inode_id + WHERE child_entry.workspace_id = inode.workspace_id + AND child_entry.parent_inode = inode.inode_id + AND child.kind = 'directory' AND child.live) + ELSE (SELECT count(*) FROM _vexfs.dentries AS link + WHERE link.workspace_id = inode.workspace_id + AND link.inode_id = inode.inode_id) END) + ORDER BY item.name), '[]'::jsonb) + FROM public.vexfs_list(p_workspace, p_path) AS item + JOIN _vexfs.inodes AS inode ON inode.inode_id = item.inode +$$; + +CREATE FUNCTION _vexfs.try_utf8(p_content bytea) +RETURNS text +LANGUAGE plpgsql +IMMUTABLE +STRICT +PARALLEL SAFE +SET search_path = pg_catalog, _vexfs +AS $$ +BEGIN + RETURN convert_from(p_content, 'UTF8'); +EXCEPTION WHEN character_not_in_repertoire OR untranslatable_character THEN + RETURN NULL; +END; +$$; + +CREATE FUNCTION _vexfs.grep_index_available() +RETURNS boolean +LANGUAGE sql +STABLE +PARALLEL SAFE +SET search_path = pg_catalog, _vexfs +AS $$ + SELECT EXISTS ( + SELECT 1 FROM pg_catalog.pg_extension WHERE extname = 'pg_trgm') +$$; + +CREATE FUNCTION _vexfs.literal_like_pattern(p_pattern text) +RETURNS text +LANGUAGE sql +IMMUTABLE +STRICT +PARALLEL SAFE +SET search_path = pg_catalog, _vexfs +AS $$ + SELECT '%' || replace( + replace( + replace(p_pattern, chr(92), chr(92) || chr(92)), + '%', chr(92) || '%'), + '_', chr(92) || '_') || '%' +$$; + +CREATE FUNCTION _vexfs.ensure_grep_trgm_index() +RETURNS void +LANGUAGE plpgsql +SECURITY DEFINER +VOLATILE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_schema name; +BEGIN + SELECT n.nspname INTO v_schema + FROM pg_catalog.pg_extension AS e + JOIN pg_catalog.pg_namespace AS n ON n.oid = e.extnamespace + WHERE e.extname = 'pg_trgm'; + IF v_schema IS NULL THEN + RAISE EXCEPTION 'VEXFS_INDEX_UNAVAILABLE: install pg_trgm before enabling the index' + USING ERRCODE = '55000'; + END IF; + EXECUTE format( + 'CREATE INDEX IF NOT EXISTS vexfs_grep_documents_content_trgm_idx ' + 'ON _vexfs.grep_documents USING gin (content %I.gin_trgm_ops)', + v_schema); +END; +$$; + +CREATE FUNCTION _vexfs.refresh_grep_document( + p_workspace_id bigint, + p_inode_id bigint) +RETURNS void +LANGUAGE plpgsql +SECURITY DEFINER +VOLATILE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_inode _vexfs.inodes%ROWTYPE; + v_content bytea; + v_text text; +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM _vexfs.workspaces AS w + WHERE w.workspace_id = p_workspace_id AND w.grep_index_enabled) THEN + RETURN; + END IF; + DELETE FROM _vexfs.grep_documents AS document + WHERE document.inode_id = p_inode_id; + SELECT * INTO v_inode + FROM _vexfs.inodes AS inode + WHERE inode.workspace_id = p_workspace_id + AND inode.inode_id = p_inode_id; + IF NOT FOUND OR NOT v_inode.live OR v_inode.kind <> 'file' + OR v_inode.current_version < 1 + OR NOT EXISTS ( + SELECT 1 FROM _vexfs.file_versions AS version + WHERE version.workspace_id = p_workspace_id + AND version.inode_id = p_inode_id + AND version.version_no = v_inode.current_version) THEN + RETURN; + END IF; + v_content := _vexfs.read_version_content( + p_workspace_id, p_inode_id, v_inode.current_version); + IF position(decode('00', 'hex') IN v_content) > 0 THEN + RETURN; + END IF; + v_text := _vexfs.try_utf8(v_content); + IF v_text IS NULL THEN + RETURN; + END IF; + INSERT INTO _vexfs.grep_documents( + inode_id, workspace_id, version_no, content) + VALUES (p_inode_id, p_workspace_id, v_inode.current_version, v_text) + ON CONFLICT (inode_id) DO UPDATE + SET workspace_id = EXCLUDED.workspace_id, + version_no = EXCLUDED.version_no, + content = EXCLUDED.content, + indexed_at = clock_timestamp(); +EXCEPTION WHEN OTHERS THEN + UPDATE _vexfs.workspaces + SET grep_index_dirty = true + WHERE workspace_id = p_workspace_id; +END; +$$; + +CREATE FUNCTION _vexfs.refresh_grep_document_from_inode() +RETURNS trigger +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = pg_catalog, _vexfs +AS $$ +BEGIN + PERFORM _vexfs.refresh_grep_document(NEW.workspace_id, NEW.inode_id); + RETURN NEW; +END; +$$; + +CREATE TRIGGER vexfs_grep_document_inode_insert +AFTER INSERT ON _vexfs.inodes +FOR EACH ROW EXECUTE FUNCTION _vexfs.refresh_grep_document_from_inode(); + +CREATE TRIGGER vexfs_grep_document_inode_update +AFTER UPDATE OF current_version, live, kind ON _vexfs.inodes +FOR EACH ROW EXECUTE FUNCTION _vexfs.refresh_grep_document_from_inode(); + +CREATE FUNCTION _vexfs.refresh_grep_document_from_version() +RETURNS trigger +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = pg_catalog, _vexfs +AS $$ +BEGIN + PERFORM _vexfs.refresh_grep_document(NEW.workspace_id, NEW.inode_id); + RETURN NEW; +END; +$$; + +CREATE TRIGGER vexfs_grep_document_version_insert +AFTER INSERT ON _vexfs.file_versions +FOR EACH ROW EXECUTE FUNCTION _vexfs.refresh_grep_document_from_version(); + +CREATE FUNCTION _vexfs.rebuild_grep_index(p_workspace_id bigint) +RETURNS bigint +LANGUAGE plpgsql +SECURITY DEFINER +VOLATILE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_inode record; + v_content bytea; + v_text text; + v_count bigint := 0; +BEGIN + DELETE FROM _vexfs.grep_documents AS document + WHERE document.workspace_id = p_workspace_id; + FOR v_inode IN + SELECT inode.inode_id, inode.current_version + FROM _vexfs.inodes AS inode + WHERE inode.workspace_id = p_workspace_id + AND inode.live AND inode.kind = 'file' + AND inode.current_version > 0 + ORDER BY inode.inode_id + LOOP + v_content := _vexfs.read_version_content( + p_workspace_id, v_inode.inode_id, v_inode.current_version); + IF position(decode('00', 'hex') IN v_content) > 0 THEN + CONTINUE; + END IF; + v_text := _vexfs.try_utf8(v_content); + IF v_text IS NULL THEN + CONTINUE; + END IF; + INSERT INTO _vexfs.grep_documents( + inode_id, workspace_id, version_no, content) + VALUES ( + v_inode.inode_id, p_workspace_id, + v_inode.current_version, v_text); + v_count := v_count + 1; + END LOOP; + UPDATE _vexfs.workspaces + SET grep_index_dirty = false + WHERE workspace_id = p_workspace_id; + -- The first indexed grep happens immediately after enable/rebuild. Refresh + -- planner statistics now so PostgreSQL does not scan the derived table + -- using stale pre-build row counts instead of the trigram GIN index. + EXECUTE 'ANALYZE _vexfs.grep_documents'; + RETURN v_count; +END; +$$; + +CREATE FUNCTION _vexfs.grep_indexed_files( + p_workspace_id bigint, + p_root_inode bigint, + p_root_path text, + p_pattern text, + p_ignore_case boolean) +RETURNS TABLE( + path text, + inode_id bigint, + current_version bigint, + size_bytes bigint) +LANGUAGE plpgsql +SECURITY DEFINER +STABLE +SET search_path = pg_catalog, _vexfs +SET enable_seqscan = off +AS $$ +DECLARE + v_like text := _vexfs.literal_like_pattern(p_pattern); +BEGIN + IF p_ignore_case THEN + RETURN QUERY + WITH RECURSIVE + candidates AS MATERIALIZED ( + SELECT document.inode_id, document.workspace_id, document.version_no + FROM _vexfs.grep_documents AS document + WHERE document.content ILIKE v_like ESCAPE E'\\' + ), + tree(inode_id, path) AS ( + SELECT p_root_inode, p_root_path + UNION ALL + SELECT d.inode_id, + CASE WHEN tree.path = '/' THEN '/' || d.name + ELSE tree.path || '/' || d.name END + FROM tree + JOIN _vexfs.inodes AS parent + ON parent.inode_id = tree.inode_id + AND parent.kind = 'directory' AND parent.live + JOIN _vexfs.dentries AS d + ON d.workspace_id = p_workspace_id + AND d.parent_inode = tree.inode_id + ) + SELECT tree.path, inode.inode_id, + inode.current_version, inode.size_bytes + FROM candidates AS candidate + JOIN _vexfs.inodes AS inode + ON inode.inode_id = candidate.inode_id + AND inode.workspace_id = p_workspace_id + AND inode.live AND inode.kind = 'file' + AND candidate.workspace_id = p_workspace_id + AND candidate.version_no = inode.current_version + JOIN tree ON tree.inode_id = inode.inode_id + ORDER BY tree.path; + ELSE + RETURN QUERY + WITH RECURSIVE + candidates AS MATERIALIZED ( + SELECT document.inode_id, document.workspace_id, document.version_no + FROM _vexfs.grep_documents AS document + WHERE document.content LIKE v_like ESCAPE E'\\' + ), + tree(inode_id, path) AS ( + SELECT p_root_inode, p_root_path + UNION ALL + SELECT d.inode_id, + CASE WHEN tree.path = '/' THEN '/' || d.name + ELSE tree.path || '/' || d.name END + FROM tree + JOIN _vexfs.inodes AS parent + ON parent.inode_id = tree.inode_id + AND parent.kind = 'directory' AND parent.live + JOIN _vexfs.dentries AS d + ON d.workspace_id = p_workspace_id + AND d.parent_inode = tree.inode_id + ) + SELECT tree.path, inode.inode_id, + inode.current_version, inode.size_bytes + FROM candidates AS candidate + JOIN _vexfs.inodes AS inode + ON inode.inode_id = candidate.inode_id + AND inode.workspace_id = p_workspace_id + AND inode.live AND inode.kind = 'file' + AND candidate.workspace_id = p_workspace_id + AND candidate.version_no = inode.current_version + JOIN tree ON tree.inode_id = inode.inode_id + ORDER BY tree.path; + END IF; +END; +$$; + +CREATE FUNCTION _vexfs.grep_files( + p_workspace_id bigint, + p_root_inode bigint, + p_root_path text, + p_pattern text, + p_ignore_case boolean, + p_use_index boolean) +RETURNS TABLE( + path text, + inode_id bigint, + current_version bigint, + size_bytes bigint) +LANGUAGE plpgsql +SECURITY DEFINER +STABLE +SET search_path = pg_catalog, _vexfs +AS $$ +BEGIN + IF NOT p_use_index THEN + RETURN QUERY + WITH RECURSIVE tree(inode_id, path) AS ( + SELECT p_root_inode, p_root_path + UNION ALL + SELECT d.inode_id, + CASE WHEN tree.path = '/' THEN '/' || d.name + ELSE tree.path || '/' || d.name END + FROM tree + JOIN _vexfs.inodes AS parent + ON parent.inode_id = tree.inode_id + AND parent.kind = 'directory' AND parent.live + JOIN _vexfs.dentries AS d + ON d.workspace_id = p_workspace_id + AND d.parent_inode = tree.inode_id + ) + SELECT tree.path, inode.inode_id, + inode.current_version, inode.size_bytes + FROM tree + JOIN _vexfs.inodes AS inode + ON inode.inode_id = tree.inode_id + AND inode.live AND inode.kind = 'file' + ORDER BY tree.path; + RETURN; + END IF; + RETURN QUERY + SELECT indexed.path, indexed.inode_id, + indexed.current_version, indexed.size_bytes + FROM _vexfs.grep_indexed_files( + p_workspace_id, p_root_inode, p_root_path, + p_pattern, p_ignore_case) AS indexed; +END; +$$; + +CREATE FUNCTION public.vexfs_grep( + p_workspace text, + p_path text, + p_pattern text, + p_flags integer, + p_limit integer) +RETURNS jsonb +LANGUAGE plpgsql +SECURITY DEFINER +STABLE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; + v_root bigint; + v_root_path text; + v_file record; + v_line record; + v_content bytea; + v_text text; + v_matches jsonb := '[]'::jsonb; + v_match_count bigint := 0; + v_files_scanned bigint := 0; + v_bytes_scanned bigint := 0; + v_binary_skipped bigint := 0; + v_truncated boolean := false; + v_ignore_case boolean; + v_files_only boolean; + v_index_used boolean; +BEGIN + IF p_pattern IS NULL OR p_pattern = '' OR octet_length(p_pattern) > 4096 THEN + RAISE EXCEPTION 'VEXFS_INVALID_GREP: pattern must be 1..4096 bytes' + USING ERRCODE = '22023'; + END IF; + IF p_flags < 0 OR (p_flags & ~3) <> 0 OR p_limit < 1 OR p_limit > 10240 THEN + RAISE EXCEPTION 'VEXFS_INVALID_GREP: flags or result limit is invalid' + USING ERRCODE = '22023'; + END IF; + v_ignore_case := (p_flags & 1) <> 0; + v_files_only := (p_flags & 2) <> 0; + v_workspace := _vexfs.require_workspace(p_workspace, 'read'); + v_root := _vexfs.resolve_path(v_workspace.workspace_id, p_path); + PERFORM _vexfs.require_inode_permission( + v_workspace.workspace_id, v_root, 'read'); + v_root_path := CASE WHEN p_path = '/' THEN '/' + ELSE rtrim(p_path, '/') END; + v_index_used := v_workspace.grep_index_enabled + AND NOT v_workspace.grep_index_dirty + AND _vexfs.grep_index_available() + AND char_length(p_pattern) >= 3; + FOR v_file IN + SELECT * FROM _vexfs.grep_files( + v_workspace.workspace_id, v_root, v_root_path, + p_pattern, v_ignore_case, v_index_used) + LOOP + PERFORM _vexfs.require_inode_permission( + v_workspace.workspace_id, v_file.inode_id, 'read'); + v_content := _vexfs.read_version_content( + v_workspace.workspace_id, v_file.inode_id, v_file.current_version); + v_files_scanned := v_files_scanned + 1; + v_bytes_scanned := v_bytes_scanned + octet_length(v_content); + IF position(decode('00', 'hex') IN v_content) > 0 THEN + v_binary_skipped := v_binary_skipped + 1; + CONTINUE; + END IF; + v_text := _vexfs.try_utf8(v_content); + IF v_text IS NULL THEN + v_binary_skipped := v_binary_skipped + 1; + CONTINUE; + END IF; + FOR v_line IN + SELECT line.text, line.ordinality + FROM regexp_split_to_table(v_text, E'\\n') + WITH ORDINALITY AS line(text, ordinality) + LOOP + IF (v_ignore_case AND position(lower(p_pattern) IN lower(v_line.text)) > 0) + OR (NOT v_ignore_case AND position(p_pattern IN v_line.text) > 0) THEN + v_matches := v_matches || jsonb_build_array(jsonb_build_object( + 'path', v_file.path, + 'line', v_line.ordinality, + 'text', v_line.text)); + v_match_count := v_match_count + 1; + IF v_match_count >= p_limit THEN + v_truncated := true; + EXIT; + END IF; + IF v_files_only THEN + EXIT; + END IF; + END IF; + END LOOP; + EXIT WHEN v_truncated; + END LOOP; + RETURN jsonb_build_object( + 'matches', v_matches, + 'match_count', v_match_count, + 'files_scanned', v_files_scanned, + 'bytes_scanned', v_bytes_scanned, + 'binary_files_skipped', v_binary_skipped, + 'index_used', v_index_used, + 'truncated', v_truncated); +END; +$$; + +-- SQLite exposes a database-wide one-argument form. PostgreSQL settings are +-- workspace-scoped, so the one-argument form is only a server capability probe. +CREATE FUNCTION public.vexfs_grep_index(p_action text) +RETURNS jsonb +LANGUAGE plpgsql +STABLE +PARALLEL SAFE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_available boolean; +BEGIN + IF p_action IS NULL OR p_action NOT IN ('status', 'enable', 'rebuild', 'disable') THEN + RAISE EXCEPTION 'VEXFS_INVALID_INDEX_ACTION: use status, enable, rebuild, or disable' + USING ERRCODE = '22023'; + END IF; + IF p_action <> 'status' THEN + RAISE EXCEPTION 'VEXFS_WORKSPACE_REQUIRED: use vexfs_grep_index(workspace, action)' + USING ERRCODE = '22023'; + END IF; + v_available := _vexfs.grep_index_available(); + RETURN jsonb_build_object( + 'enabled', false, + 'available', v_available, + 'dirty', false, + 'backend', CASE WHEN v_available THEN 'pg-trgm' ELSE 'postgresql-scan' END, + 'indexed_files', 0, + 'requested_action', p_action); +END; +$$; + +CREATE FUNCTION public.vexfs_grep_index(p_workspace text, p_action text) +RETURNS jsonb +LANGUAGE plpgsql +SECURITY DEFINER +VOLATILE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; + v_available boolean; + v_enabled boolean; + v_dirty boolean; + v_indexed_files bigint; +BEGIN + IF p_action IS NULL OR p_action NOT IN ('status', 'enable', 'rebuild', 'disable') THEN + RAISE EXCEPTION 'VEXFS_INVALID_INDEX_ACTION: use status, enable, rebuild, or disable' + USING ERRCODE = '22023'; + END IF; + v_workspace := _vexfs.require_workspace( + p_workspace, CASE WHEN p_action = 'status' THEN 'read' ELSE 'admin' END); + v_available := _vexfs.grep_index_available(); + IF p_action = 'enable' THEN + IF NOT v_available THEN + RAISE EXCEPTION 'VEXFS_INDEX_UNAVAILABLE: install pg_trgm before enabling the index' + USING ERRCODE = '55000'; + END IF; + PERFORM _vexfs.ensure_grep_trgm_index(); + UPDATE _vexfs.workspaces + SET grep_index_enabled = true, + grep_index_dirty = true + WHERE workspace_id = v_workspace.workspace_id; + PERFORM _vexfs.rebuild_grep_index(v_workspace.workspace_id); + ELSIF p_action = 'rebuild' THEN + IF NOT v_workspace.grep_index_enabled THEN + RAISE EXCEPTION 'VEXFS_INDEX_DISABLED: enable the workspace index first' + USING ERRCODE = '55000'; + END IF; + IF NOT v_available THEN + RAISE EXCEPTION 'VEXFS_INDEX_UNAVAILABLE: install pg_trgm before rebuilding the index' + USING ERRCODE = '55000'; + END IF; + PERFORM _vexfs.ensure_grep_trgm_index(); + UPDATE _vexfs.workspaces + SET grep_index_dirty = true + WHERE workspace_id = v_workspace.workspace_id; + PERFORM _vexfs.rebuild_grep_index(v_workspace.workspace_id); + ELSIF p_action = 'disable' THEN + DELETE FROM _vexfs.grep_documents AS document + WHERE document.workspace_id = v_workspace.workspace_id; + UPDATE _vexfs.workspaces + SET grep_index_enabled = false, + grep_index_dirty = false + WHERE workspace_id = v_workspace.workspace_id; + END IF; + SELECT workspace.grep_index_enabled, workspace.grep_index_dirty + INTO v_enabled, v_dirty + FROM _vexfs.workspaces AS workspace + WHERE workspace.workspace_id = v_workspace.workspace_id; + SELECT count(*) INTO v_indexed_files + FROM _vexfs.grep_documents AS document + WHERE document.workspace_id = v_workspace.workspace_id; + RETURN jsonb_build_object( + 'enabled', v_enabled, + 'available', v_available, + 'dirty', v_dirty, + 'backend', CASE WHEN v_available THEN 'pg-trgm' ELSE 'postgresql-scan' END, + 'indexed_files', v_indexed_files, + 'requested_action', p_action, + 'message', CASE + WHEN NOT v_available THEN 'install pg_trgm to enable the optional substring index' + WHEN NOT v_enabled THEN 'substring index is available but disabled for this workspace' + WHEN v_dirty THEN 'substring index is dirty; grep falls back to PostgreSQL scan' + ELSE 'substring index is ready' + END); +END; +$$; + +CREATE FUNCTION public.vexfs_stat(p_workspace text, p_path text) +RETURNS jsonb +LANGUAGE plpgsql +SECURITY DEFINER +STABLE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; + v_inode bigint; + v_result jsonb; +BEGIN + v_workspace := _vexfs.require_workspace(p_workspace, 'read'); + v_inode := _vexfs.resolve_path(v_workspace.workspace_id, p_path); + PERFORM _vexfs.require_inode_permission( + v_workspace.workspace_id, v_inode, 'read'); + SELECT jsonb_build_object( + 'path', _vexfs.path_for_inode(v_workspace.workspace_id, i.inode_id), + 'inode', i.inode_id, + 'kind', i.kind, + 'mode', i.mode, + 'owner', coalesce(r.rolname, i.owner_role), + 'owner_principal', i.owner_principal, + 'uid', i.uid, + 'gid', i.gid, + 'version', i.current_version, + 'size', i.size_bytes, + 'checksum', f.checksum, + 'created_at', (extract(epoch FROM i.created_at) * 1000)::bigint, + 'accessed_at', (extract(epoch FROM i.accessed_at) * 1000)::bigint, + 'updated_at', (extract(epoch FROM i.modified_at) * 1000)::bigint, + 'modified_at', (extract(epoch FROM i.modified_at) * 1000)::bigint, + 'changed_at', (extract(epoch FROM i.changed_at) * 1000)::bigint, + 'link_count', CASE + WHEN i.kind = 'directory' THEN 2 + ( + SELECT count(*) + FROM _vexfs.dentries AS child_entry + JOIN _vexfs.inodes AS child + ON child.inode_id = child_entry.inode_id + WHERE child_entry.workspace_id = i.workspace_id + AND child_entry.parent_inode = i.inode_id + AND child.kind = 'directory' + AND child.live) + ELSE ( + SELECT count(*) + FROM _vexfs.dentries AS link + WHERE link.workspace_id = i.workspace_id + AND link.inode_id = i.inode_id) + END, + 'workspace_head', v_workspace.head_commit) + INTO v_result + FROM _vexfs.inodes AS i + LEFT JOIN pg_catalog.pg_roles AS r ON r.oid = i.owner_oid + LEFT JOIN _vexfs.file_versions AS f + ON f.workspace_id = i.workspace_id + AND f.inode_id = i.inode_id + AND f.version_no = i.current_version + WHERE i.inode_id = v_inode; + RETURN v_result; +END; +$$; + +CREATE FUNCTION public.vexfs_list_versioned_json(p_workspace text, p_path text) +RETURNS jsonb +LANGUAGE sql +SECURITY DEFINER +STABLE +SET search_path = pg_catalog, _vexfs +AS $$ + SELECT jsonb_build_object( + 'version', (public.vexfs_stat(p_workspace, p_path)->>'workspace_head')::bigint, + 'entries', public.vexfs_list_json(p_workspace, p_path)) +$$; + +CREATE FUNCTION public.vexfs_path(p_workspace text, p_inode bigint) +RETURNS text +LANGUAGE plpgsql +SECURITY DEFINER +STABLE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; + v_path text; +BEGIN + v_workspace := _vexfs.require_workspace(p_workspace, 'read'); + PERFORM _vexfs.require_inode_permission( + v_workspace.workspace_id, p_inode, 'read'); + WITH RECURSIVE paths(inode_id, path, visited) AS ( + SELECT v_workspace.root_inode, '/'::text, ARRAY[v_workspace.root_inode]::bigint[] + UNION ALL + SELECT d.inode_id, + CASE WHEN paths.path = '/' THEN '/' || d.name + ELSE paths.path || '/' || d.name END, + paths.visited || d.inode_id + FROM paths + JOIN _vexfs.dentries AS d + ON d.workspace_id = v_workspace.workspace_id + AND d.parent_inode = paths.inode_id + WHERE NOT d.inode_id = ANY(paths.visited) + AND octet_length(paths.path) + octet_length(d.name) + 1 <= 4096) + SELECT paths.path INTO v_path + FROM paths + WHERE paths.inode_id = p_inode + ORDER BY paths.path + LIMIT 1; + IF NOT FOUND THEN + RAISE EXCEPTION 'VEXFS_INODE_NOT_FOUND: %', p_inode + USING ERRCODE = 'P0002'; + END IF; + RETURN v_path; +END; +$$; + +CREATE FUNCTION public.vexfs_append( + p_workspace text, + p_inode bigint, + p_content bytea) +RETURNS bigint +LANGUAGE plpgsql +SECURITY DEFINER +VOLATILE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; + v_path text; + v_current bytea; +BEGIN + IF p_content IS NULL THEN + RAISE EXCEPTION 'VEXFS_INVALID_CONTENT: content cannot be null' + USING ERRCODE = '22004'; + END IF; + v_workspace := _vexfs.require_workspace(p_workspace, 'write'); + PERFORM 1 FROM _vexfs.workspaces + WHERE workspace_id = v_workspace.workspace_id FOR UPDATE; + PERFORM _vexfs.require_inode_permission( + v_workspace.workspace_id, p_inode, 'write'); + v_path := public.vexfs_path(p_workspace, p_inode); + v_current := public.vexfs_read(p_workspace, v_path); + RETURN public.vexfs_write(p_workspace, v_path, v_current || p_content); +END; +$$; + +CREATE FUNCTION public.vexfs_mount_session_start( + p_workspace text, + p_session_id text) +RETURNS integer +LANGUAGE plpgsql +SECURITY DEFINER +VOLATILE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; + v_actor oid; +BEGIN + IF p_session_id IS NULL OR p_session_id = '' OR octet_length(p_session_id) > 255 THEN + RAISE EXCEPTION 'VEXFS_INVALID_SESSION: session id must be 1..255 bytes' + USING ERRCODE = '22023'; + END IF; + v_workspace := _vexfs.require_workspace(p_workspace, 'write'); + SELECT r.oid INTO STRICT v_actor + FROM pg_catalog.pg_roles AS r WHERE r.rolname = session_user; + + UPDATE _vexfs.handles AS handle + SET state = CASE + WHEN handle.dirty_generation > handle.published_generation + THEN 'retained' ELSE 'closed' END, + lease_until = CASE + WHEN handle.dirty_generation > handle.published_generation + THEN clock_timestamp() + interval '24 hours' + ELSE clock_timestamp() END, + updated_at = clock_timestamp() + WHERE handle.workspace_id = v_workspace.workspace_id + AND handle.state = 'open' + AND handle.lease_until <= clock_timestamp() + AND NOT EXISTS ( + SELECT 1 FROM _vexfs.mount_sessions AS mounted + WHERE mounted.workspace_id = handle.workspace_id + AND mounted.session_id = handle.session_id + AND mounted.owner_oid = handle.owner_oid + AND mounted.lease_until > clock_timestamp()); + PERFORM _vexfs.reap_orphan_inodes(v_workspace.workspace_id); + + INSERT INTO _vexfs.mount_sessions( + workspace_id, session_id, owner_oid, owner_role, lease_until) + VALUES ( + v_workspace.workspace_id, p_session_id, v_actor, session_user, + clock_timestamp() + interval '30 seconds') + ON CONFLICT (workspace_id, session_id) DO UPDATE + SET lease_until = EXCLUDED.lease_until, + heartbeat_at = clock_timestamp(), + owner_role = session_user + WHERE _vexfs.mount_sessions.owner_oid = v_actor; + IF NOT FOUND THEN + RAISE EXCEPTION 'VEXFS_SESSION_CONFLICT: session id belongs to another role' + USING ERRCODE = '55006'; + END IF; + + UPDATE _vexfs.handles + SET state = 'open', + session_id = p_session_id, + lease_until = clock_timestamp() + interval '30 seconds', + updated_at = clock_timestamp() + WHERE workspace_id = v_workspace.workspace_id + AND owner_oid = v_actor + AND state = 'retained' + AND dirty_generation > published_generation + AND NOT EXISTS ( + SELECT 1 FROM _vexfs.mount_sessions AS active + WHERE active.workspace_id = v_workspace.workspace_id + AND active.session_id = _vexfs.handles.session_id + AND active.session_id <> p_session_id + AND active.owner_oid = v_actor + AND active.lease_until > clock_timestamp()); + RETURN 1; +END; +$$; + +CREATE FUNCTION public.vexfs_mount_session_heartbeat( + p_workspace text, + p_session_id text) +RETURNS integer +LANGUAGE plpgsql +SECURITY DEFINER +VOLATILE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; + v_actor oid; +BEGIN + v_workspace := _vexfs.require_workspace(p_workspace, 'write'); + SELECT r.oid INTO STRICT v_actor + FROM pg_catalog.pg_roles AS r WHERE r.rolname = session_user; + UPDATE _vexfs.mount_sessions + SET lease_until = clock_timestamp() + interval '30 seconds', + heartbeat_at = clock_timestamp() + WHERE workspace_id = v_workspace.workspace_id + AND session_id = p_session_id + AND owner_oid = v_actor; + IF NOT FOUND THEN + RAISE EXCEPTION 'VEXFS_SESSION_STALE: mount session is not active' + USING ERRCODE = '55006'; + END IF; + UPDATE _vexfs.handles + SET lease_until = clock_timestamp() + interval '30 seconds', + updated_at = clock_timestamp() + WHERE workspace_id = v_workspace.workspace_id + AND session_id = p_session_id + AND owner_oid = v_actor + AND state = 'open'; + UPDATE _vexfs.file_locks + SET lease_until = clock_timestamp() + interval '30 seconds' + WHERE workspace_id = v_workspace.workspace_id + AND session_id = p_session_id + AND owner_oid = v_actor; + RETURN 1; +END; +$$; + +CREATE FUNCTION public.vexfs_mount_session_end( + p_workspace text, + p_session_id text) +RETURNS integer +LANGUAGE plpgsql +SECURITY DEFINER +VOLATILE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; + v_actor oid; +BEGIN + v_workspace := _vexfs.require_workspace(p_workspace, 'write'); + SELECT r.oid INTO STRICT v_actor + FROM pg_catalog.pg_roles AS r WHERE r.rolname = session_user; + UPDATE _vexfs.handles + SET state = CASE WHEN dirty_generation > published_generation + THEN 'retained' ELSE 'closed' END, + lease_until = CASE WHEN dirty_generation > published_generation + THEN clock_timestamp() + interval '24 hours' + ELSE clock_timestamp() END, + updated_at = clock_timestamp() + WHERE workspace_id = v_workspace.workspace_id + AND session_id = p_session_id + AND owner_oid = v_actor + AND state = 'open'; + PERFORM _vexfs.reap_orphan_inodes(v_workspace.workspace_id); + DELETE FROM _vexfs.file_locks + WHERE workspace_id = v_workspace.workspace_id + AND session_id = p_session_id + AND owner_oid = v_actor; + DELETE FROM _vexfs.mount_sessions + WHERE workspace_id = v_workspace.workspace_id + AND session_id = p_session_id + AND owner_oid = v_actor; + RETURN 1; +END; +$$; + +CREATE FUNCTION public.vexfs_handle_open( + p_workspace text, + p_path text, + p_flags text, + p_request_id text, + p_session_id text DEFAULT NULL) +RETURNS text +LANGUAGE plpgsql +SECURITY DEFINER +VOLATILE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; + v_actor oid; + v_inode bigint; + v_kind text; + v_version bigint; + v_content bytea; + v_handle text; + v_hash text; + v_cached text; + v_create boolean; + v_truncate boolean; + v_writable boolean; + v_parent bigint; + v_name text; +BEGIN + IF p_flags IS NULL OR p_flags !~ '^[rwct]+$' + OR (position('r' IN p_flags) = 0 AND position('w' IN p_flags) = 0) THEN + RAISE EXCEPTION 'VEXFS_INVALID_FLAGS: flags must use r,w,c,t and contain r or w' + USING ERRCODE = '22023'; + END IF; + v_writable := position('w' IN p_flags) > 0; + v_create := position('c' IN p_flags) > 0; + v_truncate := position('t' IN p_flags) > 0; + IF (v_create OR v_truncate) AND NOT v_writable THEN + RAISE EXCEPTION 'VEXFS_INVALID_FLAGS: create and truncate require write access' + USING ERRCODE = '22023'; + END IF; + v_workspace := _vexfs.require_workspace( + p_workspace, CASE WHEN v_writable THEN 'write' ELSE 'read' END); + SELECT r.oid INTO STRICT v_actor + FROM pg_catalog.pg_roles AS r WHERE r.rolname = session_user; + IF p_session_id IS NOT NULL AND NOT EXISTS ( + SELECT 1 FROM _vexfs.mount_sessions AS mounted + WHERE mounted.workspace_id = v_workspace.workspace_id + AND mounted.session_id = p_session_id + AND mounted.owner_oid = v_actor + AND mounted.lease_until > clock_timestamp()) THEN + RAISE EXCEPTION 'VEXFS_SESSION_STALE: mount session is not active' + USING ERRCODE = '55006'; + END IF; + v_hash := _vexfs.request_digest(jsonb_build_object( + 'workspace', p_workspace, 'path', p_path, 'flags', p_flags, + 'session', p_session_id)); + PERFORM _vexfs.lock_request(v_workspace.workspace_id, p_request_id); + v_cached := _vexfs.replay_get( + v_workspace.workspace_id, p_request_id, 'handle_open', v_hash); + IF v_cached IS NOT NULL THEN + RETURN v_cached; + END IF; + + BEGIN + v_inode := _vexfs.resolve_path(v_workspace.workspace_id, p_path); + SELECT i.kind, i.current_version INTO STRICT v_kind, v_version + FROM _vexfs.inodes AS i + WHERE i.workspace_id = v_workspace.workspace_id + AND i.inode_id = v_inode AND i.live; + IF v_kind <> 'file' THEN + RAISE EXCEPTION 'VEXFS_NOT_REGULAR_FILE: %', p_path + USING ERRCODE = '42809'; + END IF; + PERFORM _vexfs.require_inode_permission( + v_workspace.workspace_id, v_inode, + CASE WHEN v_writable THEN 'write' ELSE 'read' END); + v_content := CASE WHEN v_truncate THEN ''::bytea ELSE + _vexfs.read_version_content(v_workspace.workspace_id, v_inode, v_version) END; + EXCEPTION WHEN SQLSTATE 'P0002' THEN + IF NOT v_create THEN + RAISE; + END IF; + SELECT rp.parent_inode, rp.entry_name INTO v_parent, v_name + FROM _vexfs.resolve_parent(v_workspace.workspace_id, p_path) AS rp; + PERFORM _vexfs.require_inode_permission( + v_workspace.workspace_id, v_parent, 'write'); + v_inode := NULL; + v_version := 0; + v_content := ''::bytea; + END; + v_handle := _vexfs.random_token('handle'); + INSERT INTO _vexfs.handles( + handle_id, workspace_id, inode_id, path, flags, create_mode, + owner_oid, owner_role, session_id, expected_version, + dirty_generation, staging) + VALUES ( + v_handle, v_workspace.workspace_id, v_inode, p_path, p_flags, + CASE WHEN v_inode IS NULL THEN 420 ELSE NULL END, + v_actor, session_user, p_session_id, v_version, + CASE WHEN v_truncate OR v_inode IS NULL THEN 1 ELSE 0 END, + v_content); + PERFORM _vexfs.replay_put( + v_workspace.workspace_id, p_request_id, 'handle_open', v_hash, v_handle); + RETURN v_handle; +END; +$$; + +CREATE FUNCTION public.vexfs_handle_create( + p_workspace text, + p_path text, + p_mode integer, + p_request_id text, + p_session_id text DEFAULT NULL) +RETURNS text +LANGUAGE plpgsql +SECURITY DEFINER +VOLATILE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; + v_actor oid; + v_parent bigint; + v_name text; + v_handle text; + v_hash text; + v_cached text; +BEGIN + IF p_mode < 0 OR p_mode > 4095 THEN + RAISE EXCEPTION 'VEXFS_INVALID_MODE: %', p_mode USING ERRCODE = '22023'; + END IF; + v_workspace := _vexfs.require_workspace(p_workspace, 'write'); + SELECT r.oid INTO STRICT v_actor + FROM pg_catalog.pg_roles AS r WHERE r.rolname = session_user; + IF p_session_id IS NOT NULL AND NOT EXISTS ( + SELECT 1 FROM _vexfs.mount_sessions AS mounted + WHERE mounted.workspace_id = v_workspace.workspace_id + AND mounted.session_id = p_session_id + AND mounted.owner_oid = v_actor + AND mounted.lease_until > clock_timestamp()) THEN + RAISE EXCEPTION 'VEXFS_SESSION_STALE: mount session is not active' + USING ERRCODE = '55006'; + END IF; + v_hash := _vexfs.request_digest(jsonb_build_object( + 'workspace', p_workspace, 'path', p_path, 'mode', p_mode, + 'session', p_session_id)); + PERFORM _vexfs.lock_request(v_workspace.workspace_id, p_request_id); + v_cached := _vexfs.replay_get( + v_workspace.workspace_id, p_request_id, 'handle_create', v_hash); + IF v_cached IS NOT NULL THEN + RETURN v_cached; + END IF; + SELECT rp.parent_inode, rp.entry_name INTO v_parent, v_name + FROM _vexfs.resolve_parent(v_workspace.workspace_id, p_path) AS rp; + PERFORM _vexfs.require_inode_permission( + v_workspace.workspace_id, v_parent, 'write'); + IF EXISTS ( + SELECT 1 FROM _vexfs.dentries AS d + WHERE d.workspace_id = v_workspace.workspace_id + AND d.parent_inode = v_parent AND d.name = v_name) THEN + RAISE EXCEPTION 'VEXFS_ALREADY_EXISTS: %', p_path + USING ERRCODE = '23505'; + END IF; + PERFORM _vexfs.enforce_quota(v_workspace.workspace_id, NULL, 0); + v_handle := _vexfs.random_token('handle'); + INSERT INTO _vexfs.handles( + handle_id, workspace_id, path, flags, create_mode, + owner_oid, owner_role, session_id, expected_version, + dirty_generation, staging) + VALUES ( + v_handle, v_workspace.workspace_id, p_path, 'rwct', p_mode, + v_actor, session_user, p_session_id, 0, 1, ''::bytea); + PERFORM _vexfs.replay_put( + v_workspace.workspace_id, p_request_id, 'handle_create', v_hash, v_handle); + RETURN v_handle; +END; +$$; + +CREATE FUNCTION public.vexfs_handle_stage_write( + p_handle text, + p_offset bigint, + p_content bytea, + p_request_id text) +RETURNS bigint +LANGUAGE plpgsql +SECURITY DEFINER +VOLATILE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_handle _vexfs.handles%ROWTYPE; + v_hash text; + v_cached text; + v_generation bigint; + v_staging bytea; +BEGIN + IF p_content IS NULL THEN + RAISE EXCEPTION 'VEXFS_INVALID_CONTENT: content cannot be null' + USING ERRCODE = '22004'; + END IF; + SELECT * INTO v_handle FROM _vexfs.require_handle(p_handle, 'write'); + IF position('w' IN v_handle.flags) = 0 THEN + RAISE EXCEPTION 'VEXFS_READ_ONLY_HANDLE: %', p_handle + USING ERRCODE = '25006'; + END IF; + v_hash := _vexfs.request_digest(jsonb_build_object( + 'handle', p_handle, 'offset', p_offset, + 'content_size', octet_length(p_content), + 'content_checksum', encode(pg_catalog.sha256(p_content), 'hex'))); + PERFORM _vexfs.lock_request(v_handle.workspace_id, p_request_id); + v_cached := _vexfs.replay_get( + v_handle.workspace_id, p_request_id, 'handle_stage_write', v_hash); + IF v_cached IS NOT NULL THEN + RETURN v_cached::bigint; + END IF; + v_staging := _vexfs.overlay_bytes(v_handle.staging, p_offset, p_content); + PERFORM _vexfs.enforce_quota( + v_handle.workspace_id, v_handle.inode_id, octet_length(v_staging)); + v_generation := v_handle.dirty_generation + 1; + UPDATE _vexfs.handles + SET staging = v_staging, + dirty_generation = v_generation, + lease_until = clock_timestamp() + interval '30 seconds', + updated_at = clock_timestamp() + WHERE handle_id = p_handle; + PERFORM _vexfs.replay_put( + v_handle.workspace_id, p_request_id, 'handle_stage_write', + v_hash, v_generation::text); + RETURN v_generation; +END; +$$; + +CREATE FUNCTION public.vexfs_handle_truncate( + p_handle text, + p_size bigint, + p_request_id text) +RETURNS bigint +LANGUAGE plpgsql +SECURITY DEFINER +VOLATILE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_handle _vexfs.handles%ROWTYPE; + v_hash text; + v_cached text; + v_generation bigint; + v_staging bytea; + v_current_size bigint; +BEGIN + IF p_size < 0 OR p_size > 134217728 THEN + RAISE EXCEPTION 'VEXFS_INVALID_SIZE: staged file must be 0..128 MiB' + USING ERRCODE = '22023'; + END IF; + SELECT * INTO v_handle FROM _vexfs.require_handle(p_handle, 'write'); + IF position('w' IN v_handle.flags) = 0 THEN + RAISE EXCEPTION 'VEXFS_READ_ONLY_HANDLE: %', p_handle + USING ERRCODE = '25006'; + END IF; + v_hash := _vexfs.request_digest(jsonb_build_object( + 'handle', p_handle, 'size', p_size)); + PERFORM _vexfs.lock_request(v_handle.workspace_id, p_request_id); + v_cached := _vexfs.replay_get( + v_handle.workspace_id, p_request_id, 'handle_truncate', v_hash); + IF v_cached IS NOT NULL THEN + RETURN v_cached::bigint; + END IF; + v_current_size := octet_length(v_handle.staging); + IF p_size <= v_current_size THEN + v_staging := substring(v_handle.staging FROM 1 FOR p_size::integer); + ELSIF p_size = 0 THEN + v_staging := ''::bytea; + ELSE + v_staging := _vexfs.overlay_bytes( + v_handle.staging, p_size - 1, decode('00', 'hex')); + END IF; + PERFORM _vexfs.enforce_quota( + v_handle.workspace_id, v_handle.inode_id, p_size); + v_generation := v_handle.dirty_generation + 1; + UPDATE _vexfs.handles + SET staging = v_staging, + dirty_generation = v_generation, + lease_until = clock_timestamp() + interval '30 seconds', + updated_at = clock_timestamp() + WHERE handle_id = p_handle; + PERFORM _vexfs.replay_put( + v_handle.workspace_id, p_request_id, 'handle_truncate', + v_hash, v_generation::text); + RETURN v_generation; +END; +$$; + +CREATE FUNCTION public.vexfs_handle_read( + p_handle text, + p_offset bigint, + p_length bigint) +RETURNS bytea +LANGUAGE plpgsql +SECURITY DEFINER +VOLATILE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_handle _vexfs.handles%ROWTYPE; + v_content bytea; + v_version bigint; +BEGIN + IF p_offset < 0 OR p_length < 0 + OR p_offset > 134217728 OR p_length > 134217728 THEN + RAISE EXCEPTION 'VEXFS_INVALID_RANGE: offset and length must be within 128 MiB' + USING ERRCODE = '22023'; + END IF; + SELECT * INTO v_handle FROM _vexfs.require_handle(p_handle, 'read'); + IF position('r' IN v_handle.flags) = 0 THEN + RAISE EXCEPTION 'VEXFS_WRITE_ONLY_HANDLE: %', p_handle + USING ERRCODE = '42501'; + END IF; + IF position('w' IN v_handle.flags) = 0 AND v_handle.inode_id IS NOT NULL THEN + SELECT i.current_version INTO STRICT v_version + FROM _vexfs.inodes AS i + WHERE i.workspace_id = v_handle.workspace_id + AND i.inode_id = v_handle.inode_id; + v_content := CASE WHEN v_version = 0 THEN ''::bytea ELSE + _vexfs.read_version_content( + v_handle.workspace_id, v_handle.inode_id, v_version) END; + ELSE + v_content := v_handle.staging; + END IF; + RETURN substring(v_content FROM (p_offset + 1)::integer FOR p_length::integer); +END; +$$; + +CREATE FUNCTION public.vexfs_handle_publish( + p_handle text, + p_generation bigint, + p_durability text, + p_request_id text) +RETURNS bigint +LANGUAGE plpgsql +SECURITY DEFINER +VOLATILE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_handle _vexfs.handles%ROWTYPE; + v_hash text; + v_cached text; + v_current_version bigint; + v_path text; + v_version bigint; + v_inode bigint; + v_parent bigint; + v_name text; + v_previous_manifest bigint; + v_manifest bigint; + v_checksum text; + v_commit bigint; + v_principal_oid oid; +BEGIN + IF p_durability NOT IN ('none', 'data', 'full') THEN + RAISE EXCEPTION 'VEXFS_INVALID_DURABILITY: use none, data, or full' + USING ERRCODE = '22023'; + END IF; + SELECT * INTO v_handle FROM _vexfs.require_handle(p_handle, 'write'); + IF position('w' IN v_handle.flags) = 0 THEN + RAISE EXCEPTION 'VEXFS_READ_ONLY_HANDLE: %', p_handle + USING ERRCODE = '25006'; + END IF; + v_hash := _vexfs.request_digest(jsonb_build_object( + 'handle', p_handle, 'generation', p_generation, + 'durability', p_durability)); + PERFORM _vexfs.lock_request(v_handle.workspace_id, p_request_id); + v_cached := _vexfs.replay_get( + v_handle.workspace_id, p_request_id, 'handle_publish', v_hash); + IF v_cached IS NOT NULL THEN + RETURN v_cached::bigint; + END IF; + IF p_generation <= v_handle.published_generation THEN + v_version := v_handle.expected_version; + PERFORM _vexfs.replay_put( + v_handle.workspace_id, p_request_id, 'handle_publish', + v_hash, v_version::text); + RETURN v_version; + END IF; + IF p_generation <> v_handle.dirty_generation THEN + RAISE EXCEPTION 'VEXFS_STALE_GENERATION: expected %, received %', + v_handle.dirty_generation, p_generation + USING ERRCODE = '40001'; + END IF; + PERFORM pg_catalog.set_config( + 'synchronous_commit', + CASE p_durability WHEN 'none' THEN 'off' + WHEN 'data' THEN 'local' + ELSE 'on' END, + true); + PERFORM 1 FROM _vexfs.workspaces + WHERE workspace_id = v_handle.workspace_id FOR UPDATE; + IF v_handle.inode_id IS NULL THEN + SELECT rp.parent_inode, rp.entry_name INTO v_parent, v_name + FROM _vexfs.resolve_parent( + v_handle.workspace_id, v_handle.path) AS rp; + IF EXISTS ( + SELECT 1 + FROM _vexfs.dentries AS d + WHERE d.workspace_id = v_handle.workspace_id + AND d.parent_inode = v_parent + AND d.name = v_name) THEN + RAISE EXCEPTION 'VEXFS_VERSION_CONFLICT: create destination now exists' + USING ERRCODE = '40001'; + END IF; + v_version := public.vexfs_write( + (SELECT w.name FROM _vexfs.workspaces AS w + WHERE w.workspace_id = v_handle.workspace_id), + v_handle.path, v_handle.staging); + v_inode := _vexfs.resolve_path(v_handle.workspace_id, v_handle.path); + UPDATE _vexfs.inodes + SET mode = coalesce(v_handle.create_mode, 420), + changed_at = clock_timestamp() + WHERE inode_id = v_inode; + ELSE + SELECT i.current_version INTO v_current_version + FROM _vexfs.inodes AS i + WHERE i.workspace_id = v_handle.workspace_id + AND i.inode_id = v_handle.inode_id + AND i.live AND i.kind = 'file' + FOR UPDATE; + IF NOT FOUND OR v_current_version <> v_handle.expected_version THEN + RAISE EXCEPTION 'VEXFS_VERSION_CONFLICT: file changed after handle open' + USING ERRCODE = '40001'; + END IF; + v_path := _vexfs.path_for_inode(v_handle.workspace_id, v_handle.inode_id); + IF v_path IS NOT NULL THEN + v_version := public.vexfs_write( + (SELECT w.name FROM _vexfs.workspaces AS w + WHERE w.workspace_id = v_handle.workspace_id), + v_path, v_handle.staging); + ELSE + -- POSIX keeps an unlinked file alive while an open descriptor still + -- references its inode. Publish the new version without recreating + -- a directory entry; the inode is reaped after the final handle closes. + PERFORM _vexfs.enforce_quota( + v_handle.workspace_id, v_handle.inode_id, + octet_length(v_handle.staging)); + SELECT coalesce(max(version.version_no), 0) + 1 + INTO v_version + FROM _vexfs.file_versions AS version + WHERE version.workspace_id = v_handle.workspace_id + AND version.inode_id = v_handle.inode_id; + SELECT storage.manifest_id INTO v_previous_manifest + FROM _vexfs.resolve_version_storage( + v_handle.workspace_id, v_handle.inode_id, + v_current_version) AS storage; + UPDATE _vexfs.inodes + SET current_version = v_version, + size_bytes = octet_length(v_handle.staging), + modified_at = clock_timestamp(), + changed_at = clock_timestamp() + WHERE workspace_id = v_handle.workspace_id + AND inode_id = v_handle.inode_id; + v_manifest := _vexfs.store_manifest( + v_handle.workspace_id, v_handle.inode_id, + v_previous_manifest, v_handle.staging); + SELECT manifest.checksum INTO STRICT v_checksum + FROM _vexfs.manifests AS manifest + WHERE manifest.manifest_id = v_manifest; + SELECT role.oid INTO STRICT v_principal_oid + FROM pg_catalog.pg_roles AS role + WHERE role.rolname = session_user; + v_commit := _vexfs.record_commit( + v_handle.workspace_id, 'handle_write_orphan', + v_handle.path, v_handle.inode_id, + jsonb_build_object( + 'before_version', v_current_version, + 'after_version', v_version, + 'unlinked', true)); + INSERT INTO _vexfs.file_versions( + workspace_id, inode_id, version_no, commit_no, manifest_id, + size_bytes, checksum, created_by_oid, created_by) + VALUES ( + v_handle.workspace_id, v_handle.inode_id, v_version, + v_commit, v_manifest, octet_length(v_handle.staging), + v_checksum, v_principal_oid, session_user); + END IF; + v_inode := v_handle.inode_id; + END IF; + UPDATE _vexfs.handles + SET inode_id = v_inode, + path = coalesce(v_path, _vexfs.handles.path), + expected_version = v_version, + published_generation = p_generation, + lease_until = clock_timestamp() + interval '30 seconds', + updated_at = clock_timestamp() + WHERE handle_id = p_handle; + PERFORM _vexfs.replay_put( + v_handle.workspace_id, p_request_id, 'handle_publish', + v_hash, v_version::text); + RETURN v_version; +END; +$$; + +CREATE FUNCTION public.vexfs_handle_append( + p_handle text, + p_content bytea, + p_request_id text) +RETURNS bigint +LANGUAGE plpgsql +SECURITY DEFINER +VOLATILE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_handle _vexfs.handles%ROWTYPE; + v_hash text; + v_cached text; + v_current_version bigint; + v_current bytea; + v_generation bigint; + v_version bigint; + v_publish_request text; +BEGIN + IF p_content IS NULL THEN + RAISE EXCEPTION 'VEXFS_INVALID_CONTENT: content cannot be null' + USING ERRCODE = '22004'; + END IF; + SELECT * INTO v_handle FROM _vexfs.require_handle(p_handle, 'write'); + IF position('w' IN v_handle.flags) = 0 THEN + RAISE EXCEPTION 'VEXFS_READ_ONLY_HANDLE: %', p_handle + USING ERRCODE = '25006'; + END IF; + v_hash := _vexfs.request_digest(jsonb_build_object( + 'handle', p_handle, + 'content_size', octet_length(p_content), + 'content_checksum', encode(pg_catalog.sha256(p_content), 'hex'))); + PERFORM _vexfs.lock_request(v_handle.workspace_id, p_request_id); + v_cached := _vexfs.replay_get( + v_handle.workspace_id, p_request_id, 'handle_append', v_hash); + IF v_cached IS NOT NULL THEN + RETURN v_cached::bigint; + END IF; + PERFORM 1 FROM _vexfs.workspaces + WHERE workspace_id = v_handle.workspace_id FOR UPDATE; + IF v_handle.inode_id IS NULL THEN + v_current := v_handle.staging; + ELSE + IF v_handle.dirty_generation <> v_handle.published_generation THEN + RAISE EXCEPTION 'VEXFS_DIRTY_HANDLE: append requires a clean handle' + USING ERRCODE = '55006'; + END IF; + SELECT i.current_version INTO v_current_version + FROM _vexfs.inodes AS i + WHERE i.workspace_id = v_handle.workspace_id + AND i.inode_id = v_handle.inode_id + AND i.live AND i.kind = 'file' + FOR UPDATE; + IF NOT FOUND THEN + RAISE EXCEPTION 'VEXFS_VERSION_CONFLICT: open inode is unavailable' + USING ERRCODE = '40001'; + END IF; + v_current := _vexfs.read_version_content( + v_handle.workspace_id, v_handle.inode_id, v_current_version); + UPDATE _vexfs.handles + SET expected_version = v_current_version, + staging = v_current, + updated_at = clock_timestamp() + WHERE handle_id = p_handle; + END IF; + IF octet_length(v_current) + octet_length(p_content) > 134217728 THEN + RAISE EXCEPTION 'VEXFS_FILE_TOO_LARGE: appended file exceeds 128 MiB' + USING ERRCODE = '54000'; + END IF; + PERFORM _vexfs.enforce_quota( + v_handle.workspace_id, v_handle.inode_id, + octet_length(v_current) + octet_length(p_content)); + v_generation := greatest( + v_handle.dirty_generation, v_handle.published_generation) + 1; + UPDATE _vexfs.handles + SET staging = v_current || p_content, + dirty_generation = v_generation, + lease_until = clock_timestamp() + interval '30 seconds', + updated_at = clock_timestamp() + WHERE handle_id = p_handle; + v_publish_request := 'append-publish:' || coalesce( + nullif(p_request_id, ''), _vexfs.random_token('request')); + v_version := public.vexfs_handle_publish( + p_handle, v_generation, 'data', v_publish_request); + PERFORM _vexfs.replay_put( + v_handle.workspace_id, p_request_id, 'handle_append', + v_hash, v_version::text); + RETURN v_version; +END; +$$; + +CREATE FUNCTION public.vexfs_handle_publish_close( + p_handle text, + p_generation bigint, + p_durability text) +RETURNS bigint +LANGUAGE plpgsql +SECURITY DEFINER +VOLATILE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_handle _vexfs.handles%ROWTYPE; + v_actor oid; + v_version bigint; +BEGIN + SELECT r.oid INTO STRICT v_actor + FROM pg_catalog.pg_roles AS r WHERE r.rolname = session_user; + SELECT * INTO v_handle + FROM _vexfs.handles AS handle + WHERE handle.handle_id = p_handle + AND handle.owner_oid = v_actor + FOR UPDATE; + IF NOT FOUND THEN + RAISE EXCEPTION 'VEXFS_HANDLE_NOT_FOUND: %', p_handle + USING ERRCODE = 'P0002'; + END IF; + IF v_handle.state = 'closed' THEN + IF p_generation <= v_handle.published_generation THEN + RETURN v_handle.expected_version; + END IF; + RAISE EXCEPTION 'VEXFS_HANDLE_CLOSED: %', p_handle + USING ERRCODE = 'P0002'; + END IF; + v_version := public.vexfs_handle_publish( + p_handle, p_generation, p_durability, + 'publish-close:' || p_handle || ':' || p_generation::text || ':' || p_durability); + UPDATE _vexfs.handles + SET state = 'closed', + lease_until = clock_timestamp(), + staging = ''::bytea, + updated_at = clock_timestamp() + WHERE handle_id = p_handle; + PERFORM _vexfs.reap_orphan_inodes(v_handle.workspace_id); + RETURN v_version; +END; +$$; + +CREATE FUNCTION public.vexfs_handle_close( + p_handle text, + p_retain_unpublished boolean, + p_request_id text) +RETURNS text +LANGUAGE plpgsql +SECURITY DEFINER +VOLATILE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_handle _vexfs.handles%ROWTYPE; + v_actor oid; + v_hash text; + v_cached text; + v_state text; +BEGIN + SELECT r.oid INTO STRICT v_actor + FROM pg_catalog.pg_roles AS r WHERE r.rolname = session_user; + SELECT * INTO v_handle + FROM _vexfs.handles AS handle + WHERE handle.handle_id = p_handle + AND handle.owner_oid = v_actor + FOR UPDATE; + IF NOT FOUND THEN + RAISE EXCEPTION 'VEXFS_HANDLE_NOT_FOUND: %', p_handle + USING ERRCODE = 'P0002'; + END IF; + v_hash := _vexfs.request_digest(jsonb_build_object( + 'handle', p_handle, 'retain', p_retain_unpublished)); + PERFORM _vexfs.lock_request(v_handle.workspace_id, p_request_id); + v_cached := _vexfs.replay_get( + v_handle.workspace_id, p_request_id, 'handle_close', v_hash); + IF v_cached IS NOT NULL THEN + RETURN v_cached; + END IF; + v_state := CASE + WHEN p_retain_unpublished + AND v_handle.dirty_generation > v_handle.published_generation + THEN 'retained' ELSE 'closed' END; + UPDATE _vexfs.handles + SET state = v_state, + lease_until = CASE WHEN v_state = 'retained' + THEN clock_timestamp() + interval '24 hours' + ELSE clock_timestamp() END, + staging = CASE WHEN v_state = 'retained' + THEN staging ELSE ''::bytea END, + updated_at = clock_timestamp() + WHERE handle_id = p_handle; + PERFORM _vexfs.reap_orphan_inodes(v_handle.workspace_id); + PERFORM _vexfs.replay_put( + v_handle.workspace_id, p_request_id, 'handle_close', v_hash, v_state); + RETURN v_state; +END; +$$; + +CREATE FUNCTION public.vexfs_mount_synchronize( + p_workspace text, + p_request_id text, + p_session_id text DEFAULT NULL) +RETURNS bigint +LANGUAGE plpgsql +SECURITY DEFINER +VOLATILE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; + v_actor oid; + v_hash text; + v_cached text; + v_handle record; + v_count bigint := 0; +BEGIN + v_workspace := _vexfs.require_workspace(p_workspace, 'write'); + SELECT r.oid INTO STRICT v_actor + FROM pg_catalog.pg_roles AS r WHERE r.rolname = session_user; + v_hash := _vexfs.request_digest(jsonb_build_object( + 'workspace', p_workspace, 'session', p_session_id)); + PERFORM _vexfs.lock_request(v_workspace.workspace_id, p_request_id); + v_cached := _vexfs.replay_get( + v_workspace.workspace_id, p_request_id, 'mount_synchronize', v_hash); + IF v_cached IS NOT NULL THEN + RETURN v_cached::bigint; + END IF; + FOR v_handle IN + SELECT handle.handle_id, handle.dirty_generation + FROM _vexfs.handles AS handle + WHERE handle.workspace_id = v_workspace.workspace_id + AND handle.owner_oid = v_actor + AND handle.state IN ('open', 'retained') + AND handle.dirty_generation > handle.published_generation + AND ((p_session_id IS NULL AND handle.session_id IS NULL) + OR handle.session_id = p_session_id) + ORDER BY handle.handle_id + LOOP + PERFORM public.vexfs_handle_publish( + v_handle.handle_id, v_handle.dirty_generation, 'full', + 'synchronize:' || coalesce(nullif(p_request_id, ''), 'anonymous') || + ':' || v_handle.handle_id || ':' || v_handle.dirty_generation::text); + v_count := v_count + 1; + END LOOP; + PERFORM _vexfs.replay_put( + v_workspace.workspace_id, p_request_id, 'mount_synchronize', + v_hash, v_count::text); + RETURN v_count; +END; +$$; + +CREATE FUNCTION public.vexfs_item_reclaim( + p_workspace text, + p_request_id text) +RETURNS bigint +LANGUAGE plpgsql +SECURITY DEFINER +VOLATILE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; + v_hash text; + v_cached text; + v_count bigint; +BEGIN + v_workspace := _vexfs.require_workspace(p_workspace, 'write'); + v_hash := _vexfs.request_digest(jsonb_build_object('workspace', p_workspace)); + PERFORM _vexfs.lock_request(v_workspace.workspace_id, p_request_id); + v_cached := _vexfs.replay_get( + v_workspace.workspace_id, p_request_id, 'item_reclaim', v_hash); + IF v_cached IS NOT NULL THEN + RETURN v_cached::bigint; + END IF; + DELETE FROM _vexfs.file_locks + WHERE workspace_id = v_workspace.workspace_id + AND lease_until <= clock_timestamp(); + DELETE FROM _vexfs.mount_sessions + WHERE workspace_id = v_workspace.workspace_id + AND lease_until <= clock_timestamp(); + WITH removed AS ( + DELETE FROM _vexfs.handles + WHERE workspace_id = v_workspace.workspace_id + AND (state = 'closed' + OR (state = 'retained' AND lease_until <= clock_timestamp())) + RETURNING 1) + SELECT count(*) INTO v_count FROM removed; + PERFORM _vexfs.reap_orphan_inodes(v_workspace.workspace_id); + DELETE FROM _vexfs.request_replays AS replay + WHERE replay.workspace_id = v_workspace.workspace_id + AND replay.created_at < clock_timestamp() - interval '7 days'; + PERFORM _vexfs.replay_put( + v_workspace.workspace_id, p_request_id, 'item_reclaim', + v_hash, v_count::text); + RETURN v_count; +END; +$$; + +CREATE FUNCTION public.vexfs_lock_acquire( + p_workspace text, + p_inode bigint, + p_session_id text, + p_lock_kind text, + p_request_id text) +RETURNS integer +LANGUAGE plpgsql +SECURITY DEFINER +VOLATILE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; + v_actor oid; + v_hash text; + v_cached text; +BEGIN + IF p_lock_kind NOT IN ('shared', 'exclusive') THEN + RAISE EXCEPTION 'VEXFS_INVALID_LOCK: lock kind must be shared or exclusive' + USING ERRCODE = '22023'; + END IF; + v_workspace := _vexfs.require_workspace( + p_workspace, CASE WHEN p_lock_kind = 'exclusive' THEN 'write' ELSE 'read' END); + SELECT r.oid INTO STRICT v_actor + FROM pg_catalog.pg_roles AS r WHERE r.rolname = session_user; + IF NOT EXISTS ( + SELECT 1 FROM _vexfs.mount_sessions AS mounted + WHERE mounted.workspace_id = v_workspace.workspace_id + AND mounted.session_id = p_session_id + AND mounted.owner_oid = v_actor + AND mounted.lease_until > clock_timestamp()) THEN + RAISE EXCEPTION 'VEXFS_SESSION_STALE: mount session is not active' + USING ERRCODE = '55006'; + END IF; + PERFORM _vexfs.require_inode_permission( + v_workspace.workspace_id, p_inode, + CASE WHEN p_lock_kind = 'exclusive' THEN 'write' ELSE 'read' END); + v_hash := _vexfs.request_digest(jsonb_build_object( + 'workspace', p_workspace, 'inode', p_inode, + 'session', p_session_id, 'kind', p_lock_kind)); + PERFORM _vexfs.lock_request(v_workspace.workspace_id, p_request_id); + v_cached := _vexfs.replay_get( + v_workspace.workspace_id, p_request_id, 'lock_acquire', v_hash); + IF v_cached IS NOT NULL THEN + RETURN v_cached::integer; + END IF; + PERFORM 1 FROM _vexfs.workspaces + WHERE workspace_id = v_workspace.workspace_id FOR UPDATE; + DELETE FROM _vexfs.file_locks + WHERE workspace_id = v_workspace.workspace_id + AND inode_id = p_inode + AND lease_until <= clock_timestamp(); + IF EXISTS ( + SELECT 1 FROM _vexfs.file_locks AS held + WHERE held.workspace_id = v_workspace.workspace_id + AND held.inode_id = p_inode + AND NOT (held.owner_oid = v_actor AND held.session_id = p_session_id) + AND (p_lock_kind = 'exclusive' OR held.lock_kind = 'exclusive')) THEN + RAISE EXCEPTION 'VEXFS_LOCK_CONFLICT: inode % is already locked', p_inode + USING ERRCODE = '55P03'; + END IF; + INSERT INTO _vexfs.file_locks( + workspace_id, inode_id, owner_oid, session_id, lock_kind, lease_until) + VALUES ( + v_workspace.workspace_id, p_inode, v_actor, p_session_id, + p_lock_kind, clock_timestamp() + interval '30 seconds') + ON CONFLICT (workspace_id, inode_id, owner_oid, session_id) DO UPDATE + SET lock_kind = EXCLUDED.lock_kind, + lease_until = EXCLUDED.lease_until; + PERFORM _vexfs.replay_put( + v_workspace.workspace_id, p_request_id, 'lock_acquire', v_hash, '1'); + RETURN 1; +END; +$$; + +CREATE FUNCTION public.vexfs_lock_release( + p_workspace text, + p_inode bigint, + p_session_id text, + p_request_id text) +RETURNS integer +LANGUAGE plpgsql +SECURITY DEFINER +VOLATILE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; + v_actor oid; + v_hash text; + v_cached text; + v_count integer; +BEGIN + v_workspace := _vexfs.require_workspace(p_workspace, 'read'); + SELECT r.oid INTO STRICT v_actor + FROM pg_catalog.pg_roles AS r WHERE r.rolname = session_user; + v_hash := _vexfs.request_digest(jsonb_build_object( + 'workspace', p_workspace, 'inode', p_inode, 'session', p_session_id)); + PERFORM _vexfs.lock_request(v_workspace.workspace_id, p_request_id); + v_cached := _vexfs.replay_get( + v_workspace.workspace_id, p_request_id, 'lock_release', v_hash); + IF v_cached IS NOT NULL THEN + RETURN v_cached::integer; + END IF; + DELETE FROM _vexfs.file_locks + WHERE workspace_id = v_workspace.workspace_id + AND inode_id = p_inode + AND owner_oid = v_actor + AND session_id = p_session_id; + GET DIAGNOSTICS v_count = ROW_COUNT; + PERFORM _vexfs.replay_put( + v_workspace.workspace_id, p_request_id, 'lock_release', + v_hash, v_count::text); + RETURN v_count; +END; +$$; + +CREATE FUNCTION public.vexfs_visibility(p_workspace text) +RETURNS jsonb +LANGUAGE plpgsql +SECURITY DEFINER +STABLE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; +BEGIN + v_workspace := _vexfs.require_workspace(p_workspace, 'read'); + RETURN jsonb_build_object( + 'workspace_head', v_workspace.head_commit, + 'cache_generation', v_workspace.cache_generation); +END; +$$; + +CREATE FUNCTION public.vexfs_workspace_head(p_workspace text) +RETURNS bigint +LANGUAGE sql +SECURITY DEFINER +STABLE +SET search_path = pg_catalog, _vexfs +AS $$ + SELECT (public.vexfs_visibility(p_workspace)->>'workspace_head')::bigint +$$; + +CREATE FUNCTION public.vexfs_cache_generation(p_workspace text) +RETURNS bigint +LANGUAGE sql +SECURITY DEFINER +STABLE +SET search_path = pg_catalog, _vexfs +AS $$ + SELECT (public.vexfs_visibility(p_workspace)->>'cache_generation')::bigint +$$; + +CREATE FUNCTION public.vexfs_diagnostics(p_workspace text) +RETURNS jsonb +LANGUAGE plpgsql +SECURITY DEFINER +STABLE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; + v_pending bigint; + v_retained bigint; + v_staging_bytes bigint; +BEGIN + v_workspace := _vexfs.require_workspace(p_workspace, 'read'); + SELECT count(*) FILTER ( + WHERE handle.dirty_generation > handle.published_generation), + count(*) FILTER (WHERE handle.state = 'retained'), + coalesce(sum(octet_length(handle.staging)) FILTER ( + WHERE handle.state IN ('open', 'retained')), 0) + INTO v_pending, v_retained, v_staging_bytes + FROM _vexfs.handles AS handle + WHERE handle.workspace_id = v_workspace.workspace_id; + RETURN jsonb_build_object( + 'schema_version', public.vexfs_contract_version(), + 'adapter_version', public.vexfs_pg_adapter_version(), + 'schema_ready', public.vexfs_contract_version() = '0.9.0', + 'backend', 'postgresql', + 'workspace', p_workspace, + 'workspace_exists', 1, + 'security_mode', 'database-role', + 'principal', session_user, + 'backend_pid', pg_backend_pid(), + 'server_version', current_setting('server_version'), + 'synchronous_commit', current_setting('synchronous_commit'), + 'workspace_head', v_workspace.head_commit, + 'cache_generation', v_workspace.cache_generation, + 'pending_handles', v_pending, + 'retained_handles', v_retained, + 'staging_bytes', v_staging_bytes); +END; +$$; + +CREATE FUNCTION public.vexfs_compare_versions( + p_workspace text, + p_path text, + p_from_version bigint, + p_to_version bigint) +RETURNS jsonb +LANGUAGE plpgsql +SECURITY DEFINER +STABLE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_from bytea; + v_to bytea; +BEGIN + IF p_from_version <= 0 OR p_to_version <= 0 THEN + RAISE EXCEPTION 'VEXFS_INVALID_VERSION: versions must be positive' + USING ERRCODE = '22023'; + END IF; + v_from := public.vexfs_read_version(p_workspace, p_path, p_from_version); + v_to := public.vexfs_read_version(p_workspace, p_path, p_to_version); + RETURN jsonb_build_object( + 'path', p_path, + 'from', p_from_version, + 'to', p_to_version, + 'changed', v_from IS DISTINCT FROM v_to, + 'binary', position(decode('00', 'hex') IN v_from) > 0 + OR position(decode('00', 'hex') IN v_to) > 0, + 'from_size', octet_length(v_from), + 'to_size', octet_length(v_to)); +END; +$$; + +CREATE FUNCTION public.vexfs_symlink( + p_workspace text, + p_path text, + p_target bytea) +RETURNS bigint +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; + v_parent bigint; + v_name text; + v_inode bigint; + v_manifest bigint; + v_checksum text; + v_commit bigint; + v_principal_oid oid; +BEGIN + IF p_target IS NULL OR octet_length(p_target) = 0 + OR octet_length(p_target) > 4096 + OR position(decode('00', 'hex') IN p_target) > 0 THEN + RAISE EXCEPTION 'VEXFS_INVALID_SYMLINK: target must be 1..4096 bytes without NUL' + USING ERRCODE = '22023'; + END IF; + v_workspace := _vexfs.require_workspace(p_workspace, 'write'); + PERFORM 1 FROM _vexfs.workspaces + WHERE workspace_id = v_workspace.workspace_id FOR UPDATE; + SELECT rp.parent_inode, rp.entry_name INTO v_parent, v_name + FROM _vexfs.resolve_parent(v_workspace.workspace_id, p_path) AS rp; + PERFORM _vexfs.require_inode_permission( + v_workspace.workspace_id, v_parent, 'write'); + IF EXISTS ( + SELECT 1 FROM _vexfs.dentries AS d + WHERE d.workspace_id = v_workspace.workspace_id + AND d.parent_inode = v_parent AND d.name = v_name) THEN + RAISE EXCEPTION 'VEXFS_ALREADY_EXISTS: %', p_path + USING ERRCODE = '23505'; + END IF; + PERFORM _vexfs.enforce_quota( + v_workspace.workspace_id, NULL, octet_length(p_target)); + SELECT r.oid INTO STRICT v_principal_oid + FROM pg_catalog.pg_roles AS r WHERE r.rolname = session_user; + INSERT INTO _vexfs.inodes( + workspace_id, kind, mode, owner_oid, owner_role, owner_principal, + current_version, size_bytes) + VALUES ( + v_workspace.workspace_id, 'symlink', 511, + v_principal_oid, session_user, session_user, 1, octet_length(p_target)) + RETURNING inode_id INTO v_inode; + INSERT INTO _vexfs.dentries(workspace_id, parent_inode, name, inode_id) + VALUES (v_workspace.workspace_id, v_parent, v_name, v_inode); + PERFORM _vexfs.inherit_acl( + v_workspace.workspace_id, v_parent, v_inode); + v_manifest := _vexfs.store_manifest( + v_workspace.workspace_id, v_inode, NULL, p_target); + SELECT m.checksum INTO STRICT v_checksum + FROM _vexfs.manifests AS m WHERE m.manifest_id = v_manifest; + v_commit := _vexfs.record_commit( + v_workspace.workspace_id, 'symlink', p_path, v_inode, + jsonb_build_object('before_version', NULL, 'after_version', 1)); + INSERT INTO _vexfs.file_versions( + workspace_id, inode_id, version_no, commit_no, manifest_id, + size_bytes, checksum, created_by_oid, created_by) + VALUES ( + v_workspace.workspace_id, v_inode, 1, v_commit, v_manifest, + octet_length(p_target), v_checksum, v_principal_oid, session_user); + UPDATE _vexfs.inodes + SET modified_at = clock_timestamp(), changed_at = clock_timestamp() + WHERE inode_id = v_parent; + RETURN v_inode; +END; +$$; + +CREATE FUNCTION public.vexfs_link( + p_workspace text, + p_source text, + p_target text) +RETURNS bigint +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; + v_source_inode bigint; + v_source_kind text; + v_source_version bigint; + v_parent bigint; + v_name text; +BEGIN + v_workspace := _vexfs.require_workspace(p_workspace, 'write'); + PERFORM 1 FROM _vexfs.workspaces + WHERE workspace_id = v_workspace.workspace_id FOR UPDATE; + v_source_inode := _vexfs.resolve_path(v_workspace.workspace_id, p_source); + PERFORM _vexfs.require_inode_permission( + v_workspace.workspace_id, v_source_inode, 'read'); + SELECT i.kind, i.current_version INTO STRICT v_source_kind, v_source_version + FROM _vexfs.inodes AS i WHERE i.inode_id = v_source_inode; + IF v_source_kind <> 'file' THEN + RAISE EXCEPTION 'VEXFS_INVALID_HARDLINK: source must be a regular file' + USING ERRCODE = '22023'; + END IF; + SELECT rp.parent_inode, rp.entry_name INTO v_parent, v_name + FROM _vexfs.resolve_parent(v_workspace.workspace_id, p_target) AS rp; + PERFORM _vexfs.require_inode_permission( + v_workspace.workspace_id, v_parent, 'write'); + IF EXISTS ( + SELECT 1 FROM _vexfs.dentries AS d + WHERE d.workspace_id = v_workspace.workspace_id + AND d.parent_inode = v_parent AND d.name = v_name) THEN + RAISE EXCEPTION 'VEXFS_ALREADY_EXISTS: %', p_target + USING ERRCODE = '23505'; + END IF; + INSERT INTO _vexfs.dentries(workspace_id, parent_inode, name, inode_id) + VALUES (v_workspace.workspace_id, v_parent, v_name, v_source_inode); + UPDATE _vexfs.inodes + SET modified_at = clock_timestamp(), changed_at = clock_timestamp() + WHERE inode_id = v_parent; + PERFORM _vexfs.record_commit( + v_workspace.workspace_id, 'link', p_target, v_source_inode, + jsonb_build_object( + 'before_version', v_source_version, + 'after_version', v_source_version, + 'source_path', p_source, + 'target_path', p_target)); + RETURN v_source_inode; +END; +$$; + +CREATE FUNCTION public.vexfs_readlink( + p_workspace text, + p_inode bigint) +RETURNS bytea +LANGUAGE plpgsql +SECURITY DEFINER +STABLE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; + v_version bigint; +BEGIN + v_workspace := _vexfs.require_workspace(p_workspace, 'read'); + PERFORM _vexfs.require_inode_permission( + v_workspace.workspace_id, p_inode, 'read'); + SELECT i.current_version INTO v_version + FROM _vexfs.inodes AS i + WHERE i.workspace_id = v_workspace.workspace_id + AND i.inode_id = p_inode + AND i.kind = 'symlink' + AND i.live; + IF NOT FOUND THEN + RAISE EXCEPTION 'VEXFS_NOT_SYMLINK: %', p_inode + USING ERRCODE = '42809'; + END IF; + RETURN _vexfs.read_version_content( + v_workspace.workspace_id, p_inode, v_version); +END; +$$; + +CREATE FUNCTION public.vexfs_set_mode( + p_workspace text, + p_inode bigint, + p_mode integer) +RETURNS integer +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; + v_before_mode integer; + v_version bigint; + v_path text; +BEGIN + IF p_mode < 0 OR p_mode > 4095 THEN + RAISE EXCEPTION 'VEXFS_INVALID_MODE: %', p_mode USING ERRCODE = '22023'; + END IF; + v_workspace := _vexfs.require_workspace(p_workspace, 'metadata'); + PERFORM _vexfs.require_inode_permission( + v_workspace.workspace_id, p_inode, 'metadata'); + SELECT i.mode, i.current_version + INTO v_before_mode, v_version + FROM _vexfs.inodes AS i + WHERE i.workspace_id = v_workspace.workspace_id + AND i.inode_id = p_inode AND i.live AND i.kind <> 'symlink' + FOR UPDATE; + IF NOT FOUND THEN + RAISE EXCEPTION 'VEXFS_INODE_NOT_FOUND_OR_SYMLINK: %', p_inode + USING ERRCODE = 'P0002'; + END IF; + v_path := _vexfs.path_for_inode(v_workspace.workspace_id, p_inode); + UPDATE _vexfs.inodes + SET mode = p_mode, changed_at = clock_timestamp() + WHERE workspace_id = v_workspace.workspace_id + AND inode_id = p_inode AND live AND kind <> 'symlink'; + PERFORM _vexfs.record_commit( + v_workspace.workspace_id, 'chmod', v_path, p_inode, + jsonb_build_object( + 'before_version', v_version, + 'after_version', v_version, + 'before_mode', v_before_mode, + 'after_mode', p_mode)); + RETURN p_mode; +END; +$$; + +CREATE FUNCTION public.vexfs_set_times( + p_workspace text, + p_inode bigint, + p_accessed_at_ms bigint, + p_modified_at_ms bigint, + p_mask integer) +RETURNS integer +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; + v_version bigint; + v_path text; +BEGIN + IF p_accessed_at_ms < 0 OR p_modified_at_ms < 0 OR p_mask NOT IN (1, 2, 3) THEN + RAISE EXCEPTION 'VEXFS_INVALID_TIME: invalid millisecond timestamp or mask' + USING ERRCODE = '22023'; + END IF; + v_workspace := _vexfs.require_workspace(p_workspace, 'metadata'); + PERFORM _vexfs.require_inode_permission( + v_workspace.workspace_id, p_inode, 'metadata'); + SELECT i.current_version INTO v_version + FROM _vexfs.inodes AS i + WHERE i.workspace_id = v_workspace.workspace_id + AND i.inode_id = p_inode AND i.live + FOR UPDATE; + IF NOT FOUND THEN + RAISE EXCEPTION 'VEXFS_INODE_NOT_FOUND: %', p_inode USING ERRCODE = 'P0002'; + END IF; + v_path := _vexfs.path_for_inode(v_workspace.workspace_id, p_inode); + UPDATE _vexfs.inodes + SET accessed_at = CASE WHEN (p_mask & 1) <> 0 + THEN to_timestamp(p_accessed_at_ms / 1000.0) + ELSE accessed_at END, + modified_at = CASE WHEN (p_mask & 2) <> 0 + THEN to_timestamp(p_modified_at_ms / 1000.0) + ELSE modified_at END, + changed_at = clock_timestamp() + WHERE workspace_id = v_workspace.workspace_id + AND inode_id = p_inode AND live; + PERFORM _vexfs.record_commit( + v_workspace.workspace_id, 'set_times', v_path, p_inode, + jsonb_build_object( + 'before_version', v_version, + 'after_version', v_version, + 'mask', p_mask)); + RETURN p_mask; +END; +$$; + +CREATE FUNCTION public.vexfs_chown( + p_workspace text, + p_inode bigint, + p_uid bigint, + p_gid bigint) +RETURNS jsonb +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; + v_result jsonb; + v_before_uid bigint; + v_before_gid bigint; + v_version bigint; + v_path text; +BEGIN + IF p_uid < -1 OR p_uid > 4294967295 OR p_gid < -1 OR p_gid > 4294967295 THEN + RAISE EXCEPTION 'VEXFS_INVALID_OWNER: uid/gid must be -1 or 0..4294967295' + USING ERRCODE = '22023'; + END IF; + v_workspace := _vexfs.require_workspace(p_workspace, 'metadata'); + PERFORM _vexfs.require_inode_permission( + v_workspace.workspace_id, p_inode, 'metadata'); + SELECT i.uid, i.gid, i.current_version + INTO v_before_uid, v_before_gid, v_version + FROM _vexfs.inodes AS i + WHERE i.workspace_id = v_workspace.workspace_id + AND i.inode_id = p_inode AND i.live + FOR UPDATE; + IF NOT FOUND THEN + RAISE EXCEPTION 'VEXFS_INODE_NOT_FOUND: %', p_inode USING ERRCODE = 'P0002'; + END IF; + v_path := _vexfs.path_for_inode(v_workspace.workspace_id, p_inode); + UPDATE _vexfs.inodes + SET uid = CASE WHEN p_uid = -1 THEN uid ELSE p_uid END, + gid = CASE WHEN p_gid = -1 THEN gid ELSE p_gid END, + changed_at = clock_timestamp() + WHERE workspace_id = v_workspace.workspace_id + AND inode_id = p_inode AND live + RETURNING jsonb_build_object('uid', uid, 'gid', gid) INTO v_result; + PERFORM _vexfs.record_commit( + v_workspace.workspace_id, 'chown', v_path, p_inode, + jsonb_build_object( + 'before_version', v_version, + 'after_version', v_version, + 'before_uid', v_before_uid, + 'before_gid', v_before_gid, + 'after_uid', v_result->'uid', + 'after_gid', v_result->'gid')); + RETURN v_result; +END; +$$; + +-- owner_principal is portable archive metadata. PostgreSQL authorization keeps +-- using owner_oid/session_user, so changing this value cannot grant a database +-- role access or impersonate another authenticated principal. +CREATE FUNCTION public.vexfs_owner_set( + p_workspace text, + p_path text, + p_owner_principal text) +RETURNS text +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; + v_inode bigint; + v_before_owner text; + v_version bigint; +BEGIN + IF p_owner_principal IS NULL OR p_owner_principal = '' + OR octet_length(p_owner_principal) > 255 THEN + RAISE EXCEPTION 'VEXFS_INVALID_OWNER: owner principal must be 1..255 bytes' + USING ERRCODE = '22023'; + END IF; + v_workspace := _vexfs.require_workspace(p_workspace, 'admin'); + v_inode := _vexfs.resolve_path(v_workspace.workspace_id, p_path); + PERFORM _vexfs.require_inode_permission( + v_workspace.workspace_id, v_inode, 'admin'); + SELECT i.owner_principal, i.current_version + INTO v_before_owner, v_version + FROM _vexfs.inodes AS i + WHERE i.workspace_id = v_workspace.workspace_id + AND i.inode_id = v_inode AND i.live + FOR UPDATE; + IF NOT FOUND THEN + RAISE EXCEPTION 'VEXFS_INODE_NOT_FOUND: %', v_inode USING ERRCODE = 'P0002'; + END IF; + UPDATE _vexfs.inodes + SET owner_principal = p_owner_principal, + changed_at = clock_timestamp() + WHERE workspace_id = v_workspace.workspace_id + AND inode_id = v_inode AND live; + PERFORM _vexfs.record_commit( + v_workspace.workspace_id, 'set_owner_principal', p_path, v_inode, + jsonb_build_object( + 'before_version', v_version, + 'after_version', v_version, + 'before_owner', v_before_owner, + 'after_owner', p_owner_principal)); + RETURN p_owner_principal; +END; +$$; + +CREATE FUNCTION public.vexfs_xattr_get( + p_workspace text, + p_inode bigint, + p_name text) +RETURNS bytea +LANGUAGE plpgsql +SECURITY DEFINER +STABLE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; + v_value bytea; +BEGIN + v_workspace := _vexfs.require_workspace(p_workspace, 'read'); + PERFORM _vexfs.require_inode_permission( + v_workspace.workspace_id, p_inode, 'read'); + SELECT x.value INTO v_value + FROM _vexfs.xattrs AS x + WHERE x.workspace_id = v_workspace.workspace_id + AND x.inode_id = p_inode AND x.name = p_name; + IF NOT FOUND THEN + RAISE EXCEPTION 'VEXFS_XATTR_NOT_FOUND: %', p_name USING ERRCODE = 'P0002'; + END IF; + RETURN v_value; +END; +$$; + +CREATE FUNCTION public.vexfs_xattr_list( + p_workspace text, + p_inode bigint) +RETURNS jsonb +LANGUAGE plpgsql +SECURITY DEFINER +STABLE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; + v_result jsonb; +BEGIN + v_workspace := _vexfs.require_workspace(p_workspace, 'read'); + PERFORM _vexfs.require_inode_permission( + v_workspace.workspace_id, p_inode, 'read'); + SELECT coalesce(jsonb_agg(x.name ORDER BY x.name), '[]'::jsonb) + INTO v_result + FROM _vexfs.xattrs AS x + WHERE x.workspace_id = v_workspace.workspace_id + AND x.inode_id = p_inode; + RETURN v_result; +END; +$$; + +CREATE FUNCTION public.vexfs_xattr_set( + p_workspace text, + p_inode bigint, + p_name text, + p_value bytea, + p_policy integer DEFAULT 0) +RETURNS integer +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; + v_exists boolean; + v_version bigint; + v_path text; +BEGIN + IF p_name IS NULL OR p_name = '' OR octet_length(p_name) > 255 + OR p_policy NOT BETWEEN 0 AND 3 + OR (p_policy <> 3 AND (p_value IS NULL OR octet_length(p_value) > 65536)) THEN + RAISE EXCEPTION 'VEXFS_INVALID_XATTR: invalid name, value, or policy' + USING ERRCODE = '22023'; + END IF; + v_workspace := _vexfs.require_workspace(p_workspace, 'metadata'); + PERFORM _vexfs.require_inode_permission( + v_workspace.workspace_id, p_inode, 'metadata'); + SELECT i.current_version INTO v_version + FROM _vexfs.inodes AS i + WHERE i.workspace_id = v_workspace.workspace_id + AND i.inode_id = p_inode AND i.live + FOR UPDATE; + IF NOT FOUND THEN + RAISE EXCEPTION 'VEXFS_INODE_NOT_FOUND: %', p_inode USING ERRCODE = 'P0002'; + END IF; + v_path := _vexfs.path_for_inode(v_workspace.workspace_id, p_inode); + SELECT EXISTS ( + SELECT 1 FROM _vexfs.xattrs AS x + WHERE x.workspace_id = v_workspace.workspace_id + AND x.inode_id = p_inode AND x.name = p_name) INTO v_exists; + IF p_policy = 1 AND v_exists THEN + RAISE EXCEPTION 'VEXFS_XATTR_EXISTS: %', p_name USING ERRCODE = '23505'; + END IF; + IF p_policy IN (2, 3) AND NOT v_exists THEN + RAISE EXCEPTION 'VEXFS_XATTR_NOT_FOUND: %', p_name USING ERRCODE = 'P0002'; + END IF; + IF p_policy = 3 THEN + DELETE FROM _vexfs.xattrs + WHERE workspace_id = v_workspace.workspace_id + AND inode_id = p_inode AND name = p_name; + ELSE + INSERT INTO _vexfs.xattrs(workspace_id, inode_id, name, value) + VALUES (v_workspace.workspace_id, p_inode, p_name, p_value) + ON CONFLICT (workspace_id, inode_id, name) DO UPDATE + SET value = EXCLUDED.value, updated_at = clock_timestamp(); + END IF; + UPDATE _vexfs.inodes SET changed_at = clock_timestamp() + WHERE inode_id = p_inode; + PERFORM _vexfs.record_commit( + v_workspace.workspace_id, + CASE WHEN p_policy = 3 THEN 'remove_xattr' ELSE 'set_xattr' END, + v_path, p_inode, + jsonb_build_object( + 'before_version', v_version, + 'after_version', v_version, + 'xattr_name', p_name, + 'existed_before', v_exists, + 'exists_after', p_policy <> 3)); + RETURN 1; +END; +$$; + +CREATE FUNCTION public.vexfs_acl_get( + p_workspace text, + p_inode bigint) +RETURNS jsonb +LANGUAGE plpgsql +SECURITY DEFINER +STABLE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; + v_result jsonb; +BEGIN + v_workspace := _vexfs.require_workspace(p_workspace, 'read'); + PERFORM _vexfs.require_inode_permission( + v_workspace.workspace_id, p_inode, 'read'); + SELECT coalesce(jsonb_agg(jsonb_build_object( + 'principal', acl.principal, + 'effect', acl.effect, + 'permissions', acl.permissions, + 'inherit', acl.inherit_flags) + ORDER BY acl.principal, acl.effect), '[]'::jsonb) + INTO v_result + FROM _vexfs.acl_entries AS acl + WHERE acl.workspace_id = v_workspace.workspace_id + AND acl.inode_id = p_inode; + RETURN v_result; +END; +$$; + +CREATE FUNCTION public.vexfs_acl_list( + p_workspace text, + p_inode bigint) +RETURNS jsonb +LANGUAGE sql +SECURITY DEFINER +STABLE +SET search_path = pg_catalog, _vexfs +AS $$ SELECT public.vexfs_acl_get(p_workspace, p_inode) $$; + +CREATE FUNCTION public.vexfs_acl_set( + p_workspace text, + p_inode bigint, + p_acl jsonb) +RETURNS integer +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; + v_entry jsonb; + v_principal text; + v_effect text; + v_permissions text; + v_inherit integer; + v_count integer := 0; + v_before_count integer; + v_version bigint; + v_path text; +BEGIN + IF jsonb_typeof(p_acl) <> 'array' OR jsonb_array_length(p_acl) > 1024 THEN + RAISE EXCEPTION 'VEXFS_INVALID_ACL: ACL must be an array with at most 1024 entries' + USING ERRCODE = '22023'; + END IF; + v_workspace := _vexfs.require_workspace(p_workspace, 'admin'); + PERFORM _vexfs.require_inode_permission( + v_workspace.workspace_id, p_inode, 'admin'); + SELECT i.current_version INTO v_version + FROM _vexfs.inodes AS i + WHERE i.workspace_id = v_workspace.workspace_id + AND i.inode_id = p_inode AND i.live + FOR UPDATE; + IF NOT FOUND THEN + RAISE EXCEPTION 'VEXFS_INODE_NOT_FOUND: %', p_inode USING ERRCODE = 'P0002'; + END IF; + v_path := _vexfs.path_for_inode(v_workspace.workspace_id, p_inode); + SELECT count(*) INTO v_before_count + FROM _vexfs.acl_entries AS acl + WHERE acl.workspace_id = v_workspace.workspace_id + AND acl.inode_id = p_inode; + DELETE FROM _vexfs.acl_entries + WHERE workspace_id = v_workspace.workspace_id AND inode_id = p_inode; + FOR v_entry IN SELECT value FROM jsonb_array_elements(p_acl) + LOOP + v_principal := coalesce(v_entry->>'principal', v_entry->>'principal_id'); + v_effect := coalesce(v_entry->>'effect', 'allow'); + v_permissions := v_entry->>'permissions'; + v_inherit := coalesce((v_entry->>'inherit')::integer, + (v_entry->>'inherit_flags')::integer, 0); + IF v_principal IS NULL OR v_principal = '' OR octet_length(v_principal) > 255 + OR v_effect NOT IN ('allow', 'deny') + OR v_permissions IS NULL OR v_permissions = '' + OR octet_length(v_permissions) > 1024 + OR v_inherit NOT BETWEEN 0 AND 255 THEN + RAISE EXCEPTION 'VEXFS_INVALID_ACL: invalid ACL entry' + USING ERRCODE = '22023'; + END IF; + INSERT INTO _vexfs.acl_entries( + workspace_id, inode_id, principal, effect, + permissions, inherit_flags) + VALUES ( + v_workspace.workspace_id, p_inode, v_principal, v_effect, + v_permissions, v_inherit); + v_count := v_count + 1; + END LOOP; + UPDATE _vexfs.inodes SET changed_at = clock_timestamp() + WHERE inode_id = p_inode; + PERFORM _vexfs.record_commit( + v_workspace.workspace_id, 'set_acl', v_path, p_inode, + jsonb_build_object( + 'before_version', v_version, + 'after_version', v_version, + 'before_entries', v_before_count, + 'after_entries', v_count)); + RETURN v_count; +END; +$$; + +CREATE FUNCTION public.vexfs_acl_grant( + p_workspace text, + p_inode bigint, + p_principal text, + p_permissions text, + p_effect text DEFAULT 'allow', + p_inherit integer DEFAULT 0) +RETURNS integer +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; + v_version bigint; + v_path text; +BEGIN + IF p_principal IS NULL OR p_principal = '' OR octet_length(p_principal) > 255 + OR p_permissions IS NULL OR p_permissions = '' OR octet_length(p_permissions) > 1024 + OR p_effect NOT IN ('allow', 'deny') OR p_inherit NOT BETWEEN 0 AND 255 THEN + RAISE EXCEPTION 'VEXFS_INVALID_ACL: invalid grant' USING ERRCODE = '22023'; + END IF; + v_workspace := _vexfs.require_workspace(p_workspace, 'admin'); + PERFORM _vexfs.require_inode_permission( + v_workspace.workspace_id, p_inode, 'admin'); + SELECT i.current_version INTO v_version + FROM _vexfs.inodes AS i + WHERE i.workspace_id = v_workspace.workspace_id + AND i.inode_id = p_inode AND i.live + FOR UPDATE; + IF NOT FOUND THEN + RAISE EXCEPTION 'VEXFS_INODE_NOT_FOUND: %', p_inode USING ERRCODE = 'P0002'; + END IF; + v_path := _vexfs.path_for_inode(v_workspace.workspace_id, p_inode); + INSERT INTO _vexfs.acl_entries( + workspace_id, inode_id, principal, effect, permissions, inherit_flags) + VALUES ( + v_workspace.workspace_id, p_inode, p_principal, + p_effect, p_permissions, p_inherit) + ON CONFLICT (workspace_id, inode_id, principal, effect) DO UPDATE + SET permissions = EXCLUDED.permissions, + inherit_flags = EXCLUDED.inherit_flags, + updated_at = clock_timestamp(); + UPDATE _vexfs.inodes SET changed_at = clock_timestamp() + WHERE inode_id = p_inode; + PERFORM _vexfs.record_commit( + v_workspace.workspace_id, 'grant_acl', v_path, p_inode, + jsonb_build_object( + 'before_version', v_version, + 'after_version', v_version, + 'principal', p_principal, + 'effect', p_effect, + 'permissions', p_permissions, + 'inherit', p_inherit)); + RETURN 1; +END; +$$; + +CREATE FUNCTION public.vexfs_acl_revoke( + p_workspace text, + p_inode bigint, + p_principal text, + p_effect text DEFAULT NULL) +RETURNS integer +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; + v_removed integer; + v_version bigint; + v_path text; +BEGIN + IF p_effect IS NOT NULL AND p_effect NOT IN ('allow', 'deny') THEN + RAISE EXCEPTION 'VEXFS_INVALID_ACL: invalid effect' USING ERRCODE = '22023'; + END IF; + v_workspace := _vexfs.require_workspace(p_workspace, 'admin'); + PERFORM _vexfs.require_inode_permission( + v_workspace.workspace_id, p_inode, 'admin'); + SELECT i.current_version INTO v_version + FROM _vexfs.inodes AS i + WHERE i.workspace_id = v_workspace.workspace_id + AND i.inode_id = p_inode AND i.live + FOR UPDATE; + IF NOT FOUND THEN + RAISE EXCEPTION 'VEXFS_INODE_NOT_FOUND: %', p_inode USING ERRCODE = 'P0002'; + END IF; + v_path := _vexfs.path_for_inode(v_workspace.workspace_id, p_inode); + DELETE FROM _vexfs.acl_entries + WHERE workspace_id = v_workspace.workspace_id + AND inode_id = p_inode + AND principal = p_principal + AND (p_effect IS NULL OR effect = p_effect); + GET DIAGNOSTICS v_removed = ROW_COUNT; + IF v_removed > 0 THEN + UPDATE _vexfs.inodes SET changed_at = clock_timestamp() + WHERE inode_id = p_inode; + PERFORM _vexfs.record_commit( + v_workspace.workspace_id, 'revoke_acl', v_path, p_inode, + jsonb_build_object( + 'before_version', v_version, + 'after_version', v_version, + 'principal', p_principal, + 'effect', p_effect, + 'removed_entries', v_removed)); + END IF; + RETURN v_removed; +END; +$$; + +CREATE FUNCTION public.vexfs_acl_delete( + p_workspace text, + p_inode bigint) +RETURNS integer +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; + v_removed integer; + v_version bigint; + v_path text; +BEGIN + v_workspace := _vexfs.require_workspace(p_workspace, 'admin'); + PERFORM _vexfs.require_inode_permission( + v_workspace.workspace_id, p_inode, 'admin'); + SELECT i.current_version INTO v_version + FROM _vexfs.inodes AS i + WHERE i.workspace_id = v_workspace.workspace_id + AND i.inode_id = p_inode AND i.live + FOR UPDATE; + IF NOT FOUND THEN + RAISE EXCEPTION 'VEXFS_INODE_NOT_FOUND: %', p_inode USING ERRCODE = 'P0002'; + END IF; + v_path := _vexfs.path_for_inode(v_workspace.workspace_id, p_inode); + DELETE FROM _vexfs.acl_entries + WHERE workspace_id = v_workspace.workspace_id AND inode_id = p_inode; + GET DIAGNOSTICS v_removed = ROW_COUNT; + IF v_removed > 0 THEN + UPDATE _vexfs.inodes SET changed_at = clock_timestamp() + WHERE workspace_id = v_workspace.workspace_id AND inode_id = p_inode; + PERFORM _vexfs.record_commit( + v_workspace.workspace_id, 'delete_acl', v_path, p_inode, + jsonb_build_object( + 'before_version', v_version, + 'after_version', v_version, + 'removed_entries', v_removed)); + END IF; + RETURN v_removed; +END; +$$; + +CREATE FUNCTION public.vexfs_audit_list( + p_workspace text, + p_limit integer DEFAULT 100, + p_before_event bigint DEFAULT NULL) +RETURNS TABLE( + event_id bigint, + commit_no bigint, + actor name, + operation text, + path text, + inode_id bigint, + details jsonb, + created_at timestamptz) +LANGUAGE plpgsql +SECURITY DEFINER +STABLE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; +BEGIN + IF p_limit NOT BETWEEN 1 AND 1000 THEN + RAISE EXCEPTION 'VEXFS_INVALID_LIMIT: limit must be between 1 and 1000' + USING ERRCODE = '22023'; + END IF; + v_workspace := _vexfs.require_workspace(p_workspace, 'admin'); + RETURN QUERY + SELECT event.event_id, + event.commit_no, + coalesce(role.rolname, event.actor_role)::name, + event.operation, + event.path, + event.inode_id, + event.details, + event.created_at + FROM _vexfs.audit_events AS event + LEFT JOIN pg_catalog.pg_roles AS role ON role.oid = event.actor_oid + WHERE event.workspace_id = v_workspace.workspace_id + AND (p_before_event IS NULL OR event.event_id < p_before_event) + ORDER BY event.event_id DESC + LIMIT p_limit; +END; +$$; + +CREATE FUNCTION public.vexfs_workspace_stat(p_workspace text) +RETURNS jsonb +LANGUAGE plpgsql +SECURITY DEFINER +STABLE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; + v_owner name; +BEGIN + v_workspace := _vexfs.require_workspace(p_workspace, 'read'); + SELECT r.rolname INTO v_owner + FROM pg_catalog.pg_roles AS r + WHERE r.oid = v_workspace.owner_oid; + RETURN jsonb_build_object( + 'workspace_id', v_workspace.workspace_id, + 'name', v_workspace.name, + 'owner', coalesce(v_owner, v_workspace.owner_role), + 'root_inode', v_workspace.root_inode, + 'head_commit', v_workspace.head_commit, + 'history_floor_commit', v_workspace.history_floor_commit, + 'cache_generation', v_workspace.cache_generation, + 'live_files', v_workspace.live_files, + 'live_bytes', v_workspace.live_bytes, + 'created_at', v_workspace.created_at); +END; +$$; + +CREATE FUNCTION public.vexfs_workspace_history( + p_workspace text, p_limit integer DEFAULT 100) +RETURNS TABLE( + commit_no bigint, + parent_commit bigint, + operation text, + created_by name, + created_at timestamptz) +LANGUAGE plpgsql +SECURITY DEFINER +STABLE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; +BEGIN + IF p_limit < 1 OR p_limit > 1000 THEN + RAISE EXCEPTION 'VEXFS_INVALID_LIMIT: limit must be between 1 and 1000' + USING ERRCODE = '22023'; + END IF; + v_workspace := _vexfs.require_workspace(p_workspace, 'read'); + RETURN QUERY + SELECT c.commit_no, + c.parent_commit, + c.operation, + coalesce(r.rolname, c.created_by)::name, + c.created_at + FROM _vexfs.commits AS c + LEFT JOIN pg_catalog.pg_roles AS r ON r.oid = c.created_by_oid + WHERE c.workspace_id = v_workspace.workspace_id + ORDER BY c.commit_no DESC + LIMIT p_limit; +END; +$$; + +CREATE FUNCTION public.vexfs_create( + p_workspace text, p_path text, p_kind text, p_mode integer) +RETURNS bigint +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; + v_parent bigint; + v_name text; + v_parent_kind text; + v_inode bigint; + v_version bigint; + v_commit bigint; + v_manifest bigint; + v_checksum text; + v_principal_oid oid; +BEGIN + IF p_kind NOT IN ('file', 'directory') THEN + RAISE EXCEPTION 'VEXFS_INVALID_KIND: %', p_kind + USING ERRCODE = '22023'; + END IF; + IF p_mode < 0 OR p_mode > 4095 THEN + RAISE EXCEPTION 'VEXFS_INVALID_MODE: %', p_mode + USING ERRCODE = '22023'; + END IF; + + v_workspace := _vexfs.require_workspace(p_workspace, 'write'); + PERFORM 1 FROM _vexfs.workspaces + WHERE workspace_id = v_workspace.workspace_id FOR UPDATE; + SELECT rp.parent_inode, rp.entry_name INTO v_parent, v_name + FROM _vexfs.resolve_parent(v_workspace.workspace_id, p_path) AS rp; + PERFORM _vexfs.require_inode_permission( + v_workspace.workspace_id, v_parent, 'write'); + SELECT i.kind INTO v_parent_kind + FROM _vexfs.inodes AS i WHERE i.inode_id = v_parent; + IF v_parent_kind <> 'directory' THEN + RAISE EXCEPTION 'VEXFS_NOT_DIRECTORY: parent of %', p_path + USING ERRCODE = '42809'; + END IF; + IF EXISTS ( + SELECT 1 FROM _vexfs.dentries AS d + WHERE d.workspace_id = v_workspace.workspace_id + AND d.parent_inode = v_parent + AND d.name = v_name) THEN + RAISE EXCEPTION 'VEXFS_ALREADY_EXISTS: %', p_path + USING ERRCODE = '23505'; + END IF; + + SELECT r.oid INTO STRICT v_principal_oid + FROM pg_catalog.pg_roles AS r WHERE r.rolname = session_user; + IF p_kind = 'file' THEN + PERFORM _vexfs.enforce_quota( + v_workspace.workspace_id, NULL, 0); + END IF; + v_version := CASE WHEN p_kind = 'file' THEN 1 ELSE 0 END; + INSERT INTO _vexfs.inodes( + workspace_id, kind, mode, owner_oid, owner_role, + owner_principal, current_version) + VALUES ( + v_workspace.workspace_id, p_kind, p_mode, + v_principal_oid, session_user, session_user, v_version) + RETURNING inode_id INTO v_inode; + INSERT INTO _vexfs.dentries(workspace_id, parent_inode, name, inode_id) + VALUES (v_workspace.workspace_id, v_parent, v_name, v_inode); + PERFORM _vexfs.inherit_acl( + v_workspace.workspace_id, v_parent, v_inode); + v_commit := _vexfs.record_commit( + v_workspace.workspace_id, 'create', p_path, v_inode, + jsonb_build_object( + 'before_version', NULL, + 'after_version', v_version, + 'kind', p_kind, + 'mode', p_mode)); + UPDATE _vexfs.inodes + SET modified_at = clock_timestamp(), + changed_at = clock_timestamp() + WHERE inode_id = v_parent; + + IF p_kind = 'file' THEN + v_manifest := _vexfs.store_manifest( + v_workspace.workspace_id, v_inode, NULL, ''::bytea); + SELECT m.checksum INTO STRICT v_checksum + FROM _vexfs.manifests AS m + WHERE m.manifest_id = v_manifest; + INSERT INTO _vexfs.file_versions( + workspace_id, inode_id, version_no, commit_no, manifest_id, + size_bytes, checksum, created_by_oid, created_by) + VALUES ( + v_workspace.workspace_id, v_inode, 1, v_commit, + v_manifest, 0, v_checksum, v_principal_oid, session_user); + END IF; + RETURN v_inode; +END; +$$; + +CREATE FUNCTION public.vexfs_move( + p_workspace text, p_source text, p_target text) +RETURNS integer +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; + v_source_parent bigint; + v_source_name text; + v_target_parent bigint; + v_target_name text; + v_source_inode bigint; + v_source_kind text; + v_source_version bigint; + v_target_parent_kind text; + v_cycle boolean; +BEGIN + v_workspace := _vexfs.require_workspace(p_workspace, 'write'); + PERFORM 1 FROM _vexfs.workspaces + WHERE workspace_id = v_workspace.workspace_id FOR UPDATE; + SELECT rp.parent_inode, rp.entry_name INTO v_source_parent, v_source_name + FROM _vexfs.resolve_parent(v_workspace.workspace_id, p_source) AS rp; + SELECT rp.parent_inode, rp.entry_name INTO v_target_parent, v_target_name + FROM _vexfs.resolve_parent(v_workspace.workspace_id, p_target) AS rp; + PERFORM _vexfs.require_inode_permission( + v_workspace.workspace_id, v_source_parent, 'write'); + PERFORM _vexfs.require_inode_permission( + v_workspace.workspace_id, v_target_parent, 'write'); + + SELECT d.inode_id, i.kind, i.current_version + INTO v_source_inode, v_source_kind, v_source_version + FROM _vexfs.dentries AS d + JOIN _vexfs.inodes AS i ON i.inode_id = d.inode_id + WHERE d.workspace_id = v_workspace.workspace_id + AND d.parent_inode = v_source_parent + AND d.name = v_source_name; + IF NOT FOUND THEN + RAISE EXCEPTION 'VEXFS_PATH_NOT_FOUND: %', p_source + USING ERRCODE = 'P0002'; + END IF; + PERFORM _vexfs.require_inode_permission( + v_workspace.workspace_id, v_source_inode, 'write'); + IF v_source_parent = v_target_parent AND v_source_name = v_target_name THEN + RETURN 1; + END IF; + SELECT i.kind INTO v_target_parent_kind + FROM _vexfs.inodes AS i WHERE i.inode_id = v_target_parent; + IF v_target_parent_kind <> 'directory' THEN + RAISE EXCEPTION 'VEXFS_NOT_DIRECTORY: parent of %', p_target + USING ERRCODE = '42809'; + END IF; + IF EXISTS ( + SELECT 1 FROM _vexfs.dentries AS d + WHERE d.workspace_id = v_workspace.workspace_id + AND d.parent_inode = v_target_parent + AND d.name = v_target_name) THEN + RAISE EXCEPTION 'VEXFS_ALREADY_EXISTS: %', p_target + USING ERRCODE = '23505'; + END IF; + + IF v_source_kind = 'directory' THEN + WITH RECURSIVE descendants(inode_id) AS ( + SELECT v_source_inode + UNION ALL + SELECT d.inode_id + FROM _vexfs.dentries AS d + JOIN descendants AS x ON d.parent_inode = x.inode_id + WHERE d.workspace_id = v_workspace.workspace_id + ) + SELECT EXISTS( + SELECT 1 FROM descendants WHERE inode_id = v_target_parent) + INTO v_cycle; + IF v_cycle THEN + RAISE EXCEPTION 'VEXFS_INVALID_MOVE: cannot move a directory into itself' + USING ERRCODE = '22023'; + END IF; + END IF; + + UPDATE _vexfs.dentries + SET parent_inode = v_target_parent, + name = v_target_name + WHERE workspace_id = v_workspace.workspace_id + AND parent_inode = v_source_parent + AND name = v_source_name; + UPDATE _vexfs.inodes + SET modified_at = clock_timestamp(), + changed_at = clock_timestamp() + WHERE inode_id IN (v_source_parent, v_target_parent); + PERFORM _vexfs.record_commit( + v_workspace.workspace_id, 'move', p_target, v_source_inode, + jsonb_build_object( + 'before_version', v_source_version, + 'after_version', v_source_version, + 'before_path', p_source, + 'after_path', p_target)); + RETURN 1; +END; +$$; + +CREATE FUNCTION public.vexfs_rename( + p_workspace text, + p_source text, + p_target text, + p_replace integer DEFAULT 0) +RETURNS integer +LANGUAGE plpgsql +SECURITY DEFINER +VOLATILE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; + v_source_parent bigint; + v_source_name text; + v_target_parent bigint; + v_target_name text; + v_source_inode bigint; + v_source_kind text; + v_source_version bigint; + v_target_inode bigint; + v_target_kind text; + v_cycle boolean; +BEGIN + IF p_replace NOT IN (0, 1) THEN + RAISE EXCEPTION 'VEXFS_INVALID_RENAME: replace must be 0 or 1' + USING ERRCODE = '22023'; + END IF; + v_workspace := _vexfs.require_workspace(p_workspace, 'write'); + PERFORM 1 FROM _vexfs.workspaces + WHERE workspace_id = v_workspace.workspace_id FOR UPDATE; + SELECT rp.parent_inode, rp.entry_name INTO v_source_parent, v_source_name + FROM _vexfs.resolve_parent(v_workspace.workspace_id, p_source) AS rp; + SELECT rp.parent_inode, rp.entry_name INTO v_target_parent, v_target_name + FROM _vexfs.resolve_parent(v_workspace.workspace_id, p_target) AS rp; + PERFORM _vexfs.require_inode_permission( + v_workspace.workspace_id, v_source_parent, 'write'); + PERFORM _vexfs.require_inode_permission( + v_workspace.workspace_id, v_target_parent, 'write'); + SELECT d.inode_id, i.kind, i.current_version + INTO v_source_inode, v_source_kind, v_source_version + FROM _vexfs.dentries AS d + JOIN _vexfs.inodes AS i ON i.inode_id = d.inode_id + WHERE d.workspace_id = v_workspace.workspace_id + AND d.parent_inode = v_source_parent AND d.name = v_source_name; + IF NOT FOUND THEN + RAISE EXCEPTION 'VEXFS_PATH_NOT_FOUND: %', p_source + USING ERRCODE = 'P0002'; + END IF; + IF v_source_parent = v_target_parent AND v_source_name = v_target_name THEN + RETURN 1; + END IF; + SELECT d.inode_id, i.kind INTO v_target_inode, v_target_kind + FROM _vexfs.dentries AS d + JOIN _vexfs.inodes AS i ON i.inode_id = d.inode_id + WHERE d.workspace_id = v_workspace.workspace_id + AND d.parent_inode = v_target_parent AND d.name = v_target_name; + IF FOUND THEN + IF p_replace = 0 THEN + RAISE EXCEPTION 'VEXFS_ALREADY_EXISTS: %', p_target + USING ERRCODE = '23505'; + END IF; + IF (v_source_kind = 'directory') <> (v_target_kind = 'directory') THEN + RAISE EXCEPTION 'VEXFS_RENAME_TYPE_MISMATCH: %', p_target + USING ERRCODE = '42809'; + END IF; + IF v_target_kind = 'directory' AND EXISTS ( + SELECT 1 FROM _vexfs.dentries AS d + WHERE d.workspace_id = v_workspace.workspace_id + AND d.parent_inode = v_target_inode) THEN + RAISE EXCEPTION 'VEXFS_NOT_EMPTY: %', p_target + USING ERRCODE = '2BP01'; + END IF; + IF v_target_inode = v_source_inode THEN + DELETE FROM _vexfs.dentries + WHERE workspace_id = v_workspace.workspace_id + AND parent_inode = v_source_parent AND name = v_source_name; + UPDATE _vexfs.inodes + SET modified_at = clock_timestamp(), changed_at = clock_timestamp() + WHERE inode_id = v_source_parent; + PERFORM _vexfs.record_commit( + v_workspace.workspace_id, 'rename', p_target, v_source_inode, + jsonb_build_object( + 'before_version', v_source_version, + 'after_version', v_source_version, + 'before_path', p_source, + 'after_path', p_target, + 'replaced_inode', v_target_inode)); + RETURN 1; + END IF; + END IF; + IF v_source_kind = 'directory' THEN + WITH RECURSIVE descendants(inode_id) AS ( + SELECT v_source_inode + UNION ALL + SELECT d.inode_id + FROM _vexfs.dentries AS d + JOIN descendants AS tree ON tree.inode_id = d.parent_inode + WHERE d.workspace_id = v_workspace.workspace_id) + SELECT EXISTS(SELECT 1 FROM descendants WHERE inode_id = v_target_parent) + INTO v_cycle; + IF v_cycle THEN + RAISE EXCEPTION 'VEXFS_INVALID_RENAME: cannot move a directory into itself' + USING ERRCODE = '22023'; + END IF; + END IF; + IF v_target_inode IS NOT NULL THEN + DELETE FROM _vexfs.dentries + WHERE workspace_id = v_workspace.workspace_id + AND parent_inode = v_target_parent AND name = v_target_name; + IF v_target_kind = 'directory' OR NOT EXISTS ( + SELECT 1 FROM _vexfs.dentries AS link + WHERE link.workspace_id = v_workspace.workspace_id + AND link.inode_id = v_target_inode) + AND NOT EXISTS ( + SELECT 1 FROM _vexfs.handles AS handle + WHERE handle.workspace_id = v_workspace.workspace_id + AND handle.inode_id = v_target_inode + AND handle.state IN ('open', 'retained')) THEN + UPDATE _vexfs.inodes + SET live = false, changed_at = clock_timestamp() + WHERE inode_id = v_target_inode; + END IF; + END IF; + UPDATE _vexfs.dentries + SET parent_inode = v_target_parent, name = v_target_name + WHERE workspace_id = v_workspace.workspace_id + AND parent_inode = v_source_parent AND name = v_source_name; + UPDATE _vexfs.inodes + SET modified_at = clock_timestamp(), changed_at = clock_timestamp() + WHERE inode_id IN (v_source_parent, v_target_parent); + PERFORM _vexfs.record_commit( + v_workspace.workspace_id, 'rename', p_target, v_source_inode, + jsonb_build_object( + 'before_version', v_source_version, + 'after_version', v_source_version, + 'before_path', p_source, + 'after_path', p_target, + 'replaced_inode', v_target_inode)); + RETURN 1; +END; +$$; + +CREATE FUNCTION public.vexfs_remove( + p_workspace text, p_path text, p_recursive boolean DEFAULT false) +RETURNS integer +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; + v_parent bigint; + v_name text; + v_inode bigint; + v_kind text; + v_current_version bigint; +BEGIN + v_workspace := _vexfs.require_workspace(p_workspace, 'write'); + PERFORM 1 FROM _vexfs.workspaces + WHERE workspace_id = v_workspace.workspace_id FOR UPDATE; + SELECT rp.parent_inode, rp.entry_name INTO v_parent, v_name + FROM _vexfs.resolve_parent(v_workspace.workspace_id, p_path) AS rp; + PERFORM _vexfs.require_inode_permission( + v_workspace.workspace_id, v_parent, 'write'); + SELECT d.inode_id, i.kind, i.current_version + INTO v_inode, v_kind, v_current_version + FROM _vexfs.dentries AS d + JOIN _vexfs.inodes AS i ON i.inode_id = d.inode_id + WHERE d.workspace_id = v_workspace.workspace_id + AND d.parent_inode = v_parent + AND d.name = v_name; + IF NOT FOUND THEN + RAISE EXCEPTION 'VEXFS_PATH_NOT_FOUND: %', p_path + USING ERRCODE = 'P0002'; + END IF; + PERFORM _vexfs.require_inode_permission( + v_workspace.workspace_id, v_inode, 'write'); + IF v_kind = 'directory' AND NOT p_recursive AND EXISTS ( + SELECT 1 FROM _vexfs.dentries AS d + WHERE d.workspace_id = v_workspace.workspace_id + AND d.parent_inode = v_inode) THEN + RAISE EXCEPTION 'VEXFS_NOT_EMPTY: %', p_path + USING ERRCODE = '2BP01'; + END IF; + + IF v_kind = 'directory' AND p_recursive THEN + WITH RECURSIVE subtree(inode_id) AS ( + SELECT v_inode + UNION + SELECT d.inode_id + FROM _vexfs.dentries AS d + JOIN subtree AS s ON d.parent_inode = s.inode_id + WHERE d.workspace_id = v_workspace.workspace_id + ), + deleted AS ( + DELETE FROM _vexfs.dentries AS d + WHERE d.workspace_id = v_workspace.workspace_id + AND ((d.parent_inode = v_parent AND d.name = v_name) + OR d.parent_inode IN ( + SELECT tree.inode_id FROM subtree AS tree)) + RETURNING d.inode_id + ), + affected AS ( + SELECT DISTINCT deleted.inode_id FROM deleted + ) + UPDATE _vexfs.inodes AS i + SET live = false, + changed_at = clock_timestamp() + WHERE i.workspace_id = v_workspace.workspace_id + AND i.live + AND ((i.kind = 'directory' AND i.inode_id IN ( + SELECT tree.inode_id FROM subtree AS tree)) + OR (i.kind <> 'directory' + AND i.inode_id IN ( + SELECT item.inode_id FROM affected AS item) + AND NOT EXISTS ( + SELECT 1 FROM _vexfs.dentries AS remaining + WHERE remaining.workspace_id = v_workspace.workspace_id + AND remaining.inode_id = i.inode_id) + AND NOT EXISTS ( + SELECT 1 FROM _vexfs.handles AS handle + WHERE handle.workspace_id = v_workspace.workspace_id + AND handle.inode_id = i.inode_id + AND handle.state IN ('open', 'retained')))); + ELSE + DELETE FROM _vexfs.dentries + WHERE workspace_id = v_workspace.workspace_id + AND parent_inode = v_parent + AND name = v_name; + IF v_kind = 'directory' OR NOT EXISTS ( + SELECT 1 + FROM _vexfs.dentries AS remaining + WHERE remaining.workspace_id = v_workspace.workspace_id + AND remaining.inode_id = v_inode) + AND NOT EXISTS ( + SELECT 1 FROM _vexfs.handles AS handle + WHERE handle.workspace_id = v_workspace.workspace_id + AND handle.inode_id = v_inode + AND handle.state IN ('open', 'retained')) THEN + UPDATE _vexfs.inodes + SET live = false, + changed_at = clock_timestamp() + WHERE workspace_id = v_workspace.workspace_id + AND inode_id = v_inode + AND live; + END IF; + END IF; + UPDATE _vexfs.inodes + SET modified_at = clock_timestamp(), + changed_at = clock_timestamp() + WHERE inode_id = v_parent; + PERFORM _vexfs.record_commit( + v_workspace.workspace_id, 'remove', p_path, v_inode, + jsonb_build_object( + 'before_version', v_current_version, + 'after_version', NULL, + 'recursive', p_recursive, + 'kind', v_kind)); + RETURN 1; +END; +$$; + +CREATE FUNCTION public.vexfs_history( + p_workspace text, + p_path text, + p_limit integer DEFAULT 100, + p_before_version bigint DEFAULT NULL) +RETURNS TABLE( + version bigint, + commit_no bigint, + size bigint, + checksum text, + source_version bigint, + created_by name, + created_at timestamptz, + current boolean) +LANGUAGE plpgsql +SECURITY DEFINER +STABLE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; + v_inode bigint; + v_current bigint; + v_kind text; +BEGIN + IF p_limit < 1 OR p_limit > 1000 THEN + RAISE EXCEPTION 'VEXFS_INVALID_LIMIT: limit must be between 1 and 1000' + USING ERRCODE = '22023'; + END IF; + v_workspace := _vexfs.require_workspace(p_workspace, 'read'); + v_inode := _vexfs.resolve_path(v_workspace.workspace_id, p_path); + PERFORM _vexfs.require_inode_permission( + v_workspace.workspace_id, v_inode, 'read'); + SELECT i.kind, i.current_version INTO v_kind, v_current + FROM _vexfs.inodes AS i WHERE i.inode_id = v_inode; + IF v_kind = 'directory' THEN + RAISE EXCEPTION 'VEXFS_IS_DIRECTORY: %', p_path + USING ERRCODE = '42809'; + END IF; + + RETURN QUERY + SELECT f.version_no, + f.commit_no, + f.size_bytes, + f.checksum, + f.source_version_no, + coalesce(r.rolname, f.created_by)::name, + f.created_at, + f.version_no = v_current + FROM _vexfs.file_versions AS f + LEFT JOIN pg_catalog.pg_roles AS r ON r.oid = f.created_by_oid + WHERE f.workspace_id = v_workspace.workspace_id + AND f.inode_id = v_inode + AND (p_before_version IS NULL OR f.version_no < p_before_version) + ORDER BY f.version_no DESC + LIMIT p_limit; +END; +$$; + +CREATE FUNCTION public.vexfs_history_json( + p_workspace text, + p_path text, + p_limit integer DEFAULT 100, + p_before_version bigint DEFAULT 0) +RETURNS jsonb +LANGUAGE sql +SECURITY DEFINER +STABLE +SET search_path = pg_catalog, _vexfs +AS $$ + WITH rows AS MATERIALIZED ( + SELECT history.*, + row_number() OVER (ORDER BY history.version DESC) AS ordinal + FROM public.vexfs_history( + p_workspace, p_path, p_limit + 1, + nullif(p_before_version, 0)) AS history + ), page AS ( + SELECT * FROM rows WHERE ordinal <= p_limit + ) + SELECT jsonb_build_object( + 'entries', coalesce(( + SELECT jsonb_agg(jsonb_build_object( + 'version', item.version, + 'commit', item.commit_no, + 'parent_commit', commit.parent_commit, + 'size', item.size, + 'created_at', (extract(epoch FROM item.created_at) * 1000)::bigint, + 'message', commit.operation, + 'checksum', item.checksum, + 'current', item.current) + ORDER BY item.version DESC) + FROM page AS item + JOIN _vexfs.workspaces AS workspace ON workspace.name = p_workspace + JOIN _vexfs.commits AS commit + ON commit.workspace_id = workspace.workspace_id + AND commit.commit_no = item.commit_no), '[]'::jsonb), + 'next_before', CASE WHEN EXISTS ( + SELECT 1 FROM rows WHERE ordinal > p_limit) + THEN (SELECT min(version) FROM page) ELSE NULL END) +$$; + +CREATE FUNCTION public.vexfs_read_version( + p_workspace text, p_path text, p_version bigint) +RETURNS bytea +LANGUAGE plpgsql +SECURITY DEFINER +STABLE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; + v_inode bigint; + v_kind text; +BEGIN + v_workspace := _vexfs.require_workspace(p_workspace, 'read'); + v_inode := _vexfs.resolve_path(v_workspace.workspace_id, p_path); + PERFORM _vexfs.require_inode_permission( + v_workspace.workspace_id, v_inode, 'read'); + SELECT i.kind INTO v_kind FROM _vexfs.inodes AS i WHERE i.inode_id = v_inode; + IF v_kind = 'directory' THEN + RAISE EXCEPTION 'VEXFS_IS_DIRECTORY: %', p_path + USING ERRCODE = '42809'; + END IF; + RETURN _vexfs.read_version_content( + v_workspace.workspace_id, v_inode, p_version); +END; +$$; + +CREATE FUNCTION public.vexfs_restore_version( + p_workspace text, + p_path text, + p_version bigint, + p_expected_version bigint) +RETURNS bigint +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; + v_inode bigint; + v_kind text; + v_current bigint; + v_new_version bigint; + v_commit bigint; + v_size bigint; + v_checksum text; + v_canonical bigint; + v_principal_oid oid; +BEGIN + v_workspace := _vexfs.require_workspace(p_workspace, 'write'); + PERFORM 1 FROM _vexfs.workspaces + WHERE workspace_id = v_workspace.workspace_id FOR UPDATE; + v_inode := _vexfs.resolve_path(v_workspace.workspace_id, p_path); + PERFORM _vexfs.require_inode_permission( + v_workspace.workspace_id, v_inode, 'write'); + SELECT i.kind, i.current_version INTO v_kind, v_current + FROM _vexfs.inodes AS i WHERE i.inode_id = v_inode FOR UPDATE; + IF v_kind = 'directory' THEN + RAISE EXCEPTION 'VEXFS_IS_DIRECTORY: %', p_path + USING ERRCODE = '42809'; + END IF; + IF v_current <> p_expected_version THEN + RAISE EXCEPTION 'VEXFS_VERSION_CONFLICT: expected %, actual %', + p_expected_version, v_current + USING ERRCODE = '40001'; + END IF; + SELECT storage.size_bytes, + storage.checksum, + storage.canonical_version + INTO v_size, v_checksum, v_canonical + FROM _vexfs.resolve_version_storage( + v_workspace.workspace_id, v_inode, p_version) AS storage; + + SELECT r.oid INTO STRICT v_principal_oid + FROM pg_catalog.pg_roles AS r WHERE r.rolname = session_user; + SELECT coalesce(max(f.version_no), 0) + 1 INTO v_new_version + FROM _vexfs.file_versions AS f + WHERE f.workspace_id = v_workspace.workspace_id + AND f.inode_id = v_inode; + v_commit := _vexfs.record_commit( + v_workspace.workspace_id, 'restore_version', p_path, v_inode, + jsonb_build_object( + 'before_version', v_current, + 'after_version', v_new_version, + 'source_version', p_version, + 'canonical_source_version', v_canonical)); + INSERT INTO _vexfs.file_versions( + workspace_id, inode_id, version_no, commit_no, source_version_no, + size_bytes, checksum, created_by_oid, created_by) + VALUES ( + v_workspace.workspace_id, v_inode, v_new_version, v_commit, v_canonical, + v_size, v_checksum, v_principal_oid, session_user); + UPDATE _vexfs.inodes + SET current_version = v_new_version, + size_bytes = v_size, + modified_at = clock_timestamp(), + changed_at = clock_timestamp() + WHERE inode_id = v_inode; + RETURN v_new_version; +END; +$$; + +CREATE FUNCTION public.vexfs_snapshot_create( + p_workspace text, + p_name text, + p_expected_head bigint DEFAULT NULL, + p_mode text DEFAULT 'consistent') +RETURNS bigint +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; + v_head bigint; + v_snapshot_id bigint; + v_principal_oid oid; +BEGIN + IF p_name IS NULL OR btrim(p_name) = '' OR p_name <> btrim(p_name) THEN + RAISE EXCEPTION 'VEXFS_INVALID_SNAPSHOT: invalid snapshot name' + USING ERRCODE = '22023'; + END IF; + IF p_mode NOT IN ('consistent', 'committed-only') THEN + RAISE EXCEPTION 'VEXFS_INVALID_SNAPSHOT_MODE: use consistent or committed-only' + USING ERRCODE = '22023'; + END IF; + v_workspace := _vexfs.require_workspace(p_workspace, 'write'); + SELECT w.head_commit INTO v_head + FROM _vexfs.workspaces AS w + WHERE w.workspace_id = v_workspace.workspace_id + FOR UPDATE; + IF p_expected_head IS NOT NULL AND p_expected_head <> v_head THEN + RAISE EXCEPTION 'VEXFS_HEAD_CONFLICT: expected %, actual %', + p_expected_head, v_head + USING ERRCODE = '40001'; + END IF; + IF p_mode = 'consistent' AND EXISTS ( + SELECT 1 FROM _vexfs.handles AS handle + WHERE handle.workspace_id = v_workspace.workspace_id + AND handle.state IN ('open', 'retained') + AND handle.dirty_generation > handle.published_generation) THEN + RAISE EXCEPTION 'VEXFS_UNPUBLISHED_HANDLES: synchronize the owning mount or use committed-only' + USING ERRCODE = '55006'; + END IF; + SELECT r.oid INTO STRICT v_principal_oid + FROM pg_catalog.pg_roles AS r WHERE r.rolname = session_user; + INSERT INTO _vexfs.snapshots( + workspace_id, name, head_commit, created_by_oid, created_by) + VALUES ( + v_workspace.workspace_id, p_name, v_head, + v_principal_oid, session_user) + RETURNING snapshot_id INTO v_snapshot_id; + + WITH RECURSIVE reachable(inode_id) AS ( + SELECT v_workspace.root_inode + UNION + SELECT d.inode_id + FROM _vexfs.dentries AS d + JOIN reachable AS r ON d.parent_inode = r.inode_id + WHERE d.workspace_id = v_workspace.workspace_id + ) + INSERT INTO _vexfs.snapshot_inodes( + snapshot_id, inode_id, kind, mode, owner_oid, owner_role, owner_principal, + uid, gid, current_version, size_bytes, created_at, accessed_at, + modified_at, changed_at) + SELECT v_snapshot_id, + i.inode_id, + i.kind, + i.mode, + i.owner_oid, + i.owner_role, + i.owner_principal, + i.uid, + i.gid, + i.current_version, + i.size_bytes, + i.created_at, + i.accessed_at, + i.modified_at, + i.changed_at + FROM reachable AS x + JOIN _vexfs.inodes AS i ON i.inode_id = x.inode_id; + + INSERT INTO _vexfs.snapshot_dentries( + snapshot_id, parent_inode, name, inode_id) + SELECT v_snapshot_id, d.parent_inode, d.name, d.inode_id + FROM _vexfs.dentries AS d + JOIN _vexfs.snapshot_inodes AS p + ON p.snapshot_id = v_snapshot_id AND p.inode_id = d.parent_inode + JOIN _vexfs.snapshot_inodes AS c + ON c.snapshot_id = v_snapshot_id AND c.inode_id = d.inode_id + WHERE d.workspace_id = v_workspace.workspace_id; + + INSERT INTO _vexfs.snapshot_xattrs( + snapshot_id, inode_id, name, value, updated_at) + SELECT v_snapshot_id, x.inode_id, x.name, x.value, x.updated_at + FROM _vexfs.xattrs AS x + JOIN _vexfs.snapshot_inodes AS i + ON i.snapshot_id = v_snapshot_id AND i.inode_id = x.inode_id + WHERE x.workspace_id = v_workspace.workspace_id; + + INSERT INTO _vexfs.snapshot_acl_entries( + snapshot_id, inode_id, principal, effect, permissions, + inherit_flags, created_at, updated_at) + SELECT v_snapshot_id, acl.inode_id, acl.principal, acl.effect, + acl.permissions, acl.inherit_flags, acl.created_at, acl.updated_at + FROM _vexfs.acl_entries AS acl + JOIN _vexfs.snapshot_inodes AS i + ON i.snapshot_id = v_snapshot_id AND i.inode_id = acl.inode_id + WHERE acl.workspace_id = v_workspace.workspace_id; + PERFORM _vexfs.audit( + v_workspace.workspace_id, v_head, 'snapshot_create', + '/', v_workspace.root_inode, + jsonb_build_object( + 'before_version', v_head, + 'after_version', v_head, + 'snapshot', p_name, + 'mode', p_mode)); + RETURN v_head; +END; +$$; + +CREATE FUNCTION public.vexfs_snapshot_list(p_workspace text) +RETURNS TABLE( + name text, + head_commit bigint, + created_by name, + created_at timestamptz) +LANGUAGE plpgsql +SECURITY DEFINER +STABLE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; +BEGIN + v_workspace := _vexfs.require_workspace(p_workspace, 'read'); + RETURN QUERY + SELECT s.name, + s.head_commit, + coalesce(r.rolname, s.created_by)::name, + s.created_at + FROM _vexfs.snapshots AS s + LEFT JOIN pg_catalog.pg_roles AS r ON r.oid = s.created_by_oid + WHERE s.workspace_id = v_workspace.workspace_id + ORDER BY s.created_at DESC, s.snapshot_id DESC; +END; +$$; + +CREATE FUNCTION public.vexfs_snapshot_list_json(p_workspace text) +RETURNS jsonb +LANGUAGE sql +SECURITY DEFINER +STABLE +SET search_path = pg_catalog, _vexfs +AS $$ + SELECT coalesce(jsonb_agg(jsonb_build_object( + 'name', snapshot.name, + 'commit', snapshot.head_commit, + 'created_by', snapshot.created_by, + 'created_at', (extract(epoch FROM snapshot.created_at) * 1000)::bigint) + ORDER BY snapshot.created_at DESC), '[]'::jsonb) + FROM public.vexfs_snapshot_list(p_workspace) AS snapshot +$$; + +CREATE FUNCTION public.vexfs_snapshot_show(p_workspace text, p_name text) +RETURNS jsonb +LANGUAGE plpgsql +SECURITY DEFINER +STABLE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; + v_result jsonb; +BEGIN + v_workspace := _vexfs.require_workspace(p_workspace, 'read'); + SELECT jsonb_build_object( + 'name', s.name, + 'commit', s.head_commit, + 'head_commit', s.head_commit, + 'inode_count', (SELECT count(*) FROM _vexfs.snapshot_inodes AS i + WHERE i.snapshot_id = s.snapshot_id), + 'dentry_count', (SELECT count(*) FROM _vexfs.snapshot_dentries AS d + WHERE d.snapshot_id = s.snapshot_id), + 'xattr_count', (SELECT count(*) FROM _vexfs.snapshot_xattrs AS x + WHERE x.snapshot_id = s.snapshot_id), + 'acl_count', (SELECT count(*) FROM _vexfs.snapshot_acl_entries AS acl + WHERE acl.snapshot_id = s.snapshot_id), + 'created_by', coalesce(r.rolname, s.created_by), + 'created_at', (extract(epoch FROM s.created_at) * 1000)::bigint, + 'entries', (SELECT coalesce(jsonb_agg(jsonb_build_object( + 'path', entry.path, + 'inode', entry.inode_id, + 'kind', entry.kind, + 'mode', entry.mode, + 'owner_principal', entry.owner_principal, + 'uid', entry.uid, + 'gid', entry.gid, + 'size', entry.size_bytes, + 'version', entry.current_version, + 'checksum', entry.content_checksum) + ORDER BY entry.path), '[]'::jsonb) + FROM _vexfs.tree_at( + v_workspace.workspace_id, s.snapshot_id) AS entry)) + INTO v_result + FROM _vexfs.snapshots AS s + LEFT JOIN pg_catalog.pg_roles AS r ON r.oid = s.created_by_oid + WHERE s.workspace_id = v_workspace.workspace_id + AND s.name = p_name; + IF NOT FOUND THEN + RAISE EXCEPTION 'VEXFS_SNAPSHOT_NOT_FOUND: %', p_name + USING ERRCODE = 'P0002'; + END IF; + RETURN v_result; +END; +$$; + +CREATE FUNCTION _vexfs.tree_at( + p_workspace_id bigint, + p_snapshot_id bigint DEFAULT NULL) +RETURNS TABLE( + path text, + inode_id bigint, + kind text, + mode integer, + owner_principal text, + uid bigint, + gid bigint, + size_bytes bigint, + current_version bigint, + content_checksum text, + metadata_checksum text) +LANGUAGE plpgsql +SECURITY DEFINER +STABLE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_root bigint; +BEGIN + SELECT w.root_inode INTO STRICT v_root + FROM _vexfs.workspaces AS w WHERE w.workspace_id = p_workspace_id; + IF p_snapshot_id IS NULL THEN + RETURN QUERY + WITH RECURSIVE tree(path, inode_id) AS ( + SELECT '/'::text, v_root + UNION ALL + SELECT CASE WHEN tree.path = '/' THEN '/' || d.name + ELSE tree.path || '/' || d.name END, + d.inode_id + FROM tree + JOIN _vexfs.dentries AS d + ON d.workspace_id = p_workspace_id + AND d.parent_inode = tree.inode_id), + metadata AS ( + SELECT i.inode_id, + encode(pg_catalog.sha256(convert_to(jsonb_build_object( + 'mode', i.mode, + 'owner', i.owner_principal, + 'uid', i.uid, + 'gid', i.gid, + 'created_at', i.created_at, + 'accessed_at', i.accessed_at, + 'modified_at', i.modified_at, + 'changed_at', i.changed_at, + 'xattrs', coalesce(( + SELECT jsonb_agg(jsonb_build_array(x.name, encode(x.value, 'hex')) + ORDER BY x.name) + FROM _vexfs.xattrs AS x + WHERE x.workspace_id = p_workspace_id + AND x.inode_id = i.inode_id), '[]'::jsonb), + 'acl', coalesce(( + SELECT jsonb_agg(jsonb_build_array( + acl.principal, acl.effect, acl.permissions, + acl.inherit_flags) + ORDER BY acl.principal, acl.effect) + FROM _vexfs.acl_entries AS acl + WHERE acl.workspace_id = p_workspace_id + AND acl.inode_id = i.inode_id), '[]'::jsonb))::text, 'UTF8')), 'hex') + AS metadata_checksum + FROM _vexfs.inodes AS i + WHERE i.workspace_id = p_workspace_id AND i.live) + SELECT tree.path, + i.inode_id, + i.kind, + i.mode, + i.owner_principal, + i.uid, + i.gid, + i.size_bytes, + i.current_version, + f.checksum, + metadata.metadata_checksum + FROM tree + JOIN _vexfs.inodes AS i ON i.inode_id = tree.inode_id + JOIN metadata ON metadata.inode_id = i.inode_id + LEFT JOIN _vexfs.file_versions AS f + ON f.workspace_id = p_workspace_id + AND f.inode_id = i.inode_id + AND f.version_no = i.current_version; + ELSE + RETURN QUERY + WITH RECURSIVE tree(path, inode_id) AS ( + SELECT '/'::text, v_root + UNION ALL + SELECT CASE WHEN tree.path = '/' THEN '/' || d.name + ELSE tree.path || '/' || d.name END, + d.inode_id + FROM tree + JOIN _vexfs.snapshot_dentries AS d + ON d.snapshot_id = p_snapshot_id + AND d.parent_inode = tree.inode_id), + metadata AS ( + SELECT i.inode_id, + encode(pg_catalog.sha256(convert_to(jsonb_build_object( + 'mode', i.mode, + 'owner', i.owner_principal, + 'uid', i.uid, + 'gid', i.gid, + 'created_at', i.created_at, + 'accessed_at', i.accessed_at, + 'modified_at', i.modified_at, + 'changed_at', i.changed_at, + 'xattrs', coalesce(( + SELECT jsonb_agg(jsonb_build_array(x.name, encode(x.value, 'hex')) + ORDER BY x.name) + FROM _vexfs.snapshot_xattrs AS x + WHERE x.snapshot_id = p_snapshot_id + AND x.inode_id = i.inode_id), '[]'::jsonb), + 'acl', coalesce(( + SELECT jsonb_agg(jsonb_build_array( + acl.principal, acl.effect, acl.permissions, + acl.inherit_flags) + ORDER BY acl.principal, acl.effect) + FROM _vexfs.snapshot_acl_entries AS acl + WHERE acl.snapshot_id = p_snapshot_id + AND acl.inode_id = i.inode_id), '[]'::jsonb))::text, 'UTF8')), 'hex') + AS metadata_checksum + FROM _vexfs.snapshot_inodes AS i + WHERE i.snapshot_id = p_snapshot_id) + SELECT tree.path, + i.inode_id, + i.kind, + i.mode, + i.owner_principal, + i.uid, + i.gid, + i.size_bytes, + i.current_version, + f.checksum, + metadata.metadata_checksum + FROM tree + JOIN _vexfs.snapshot_inodes AS i + ON i.snapshot_id = p_snapshot_id AND i.inode_id = tree.inode_id + JOIN metadata ON metadata.inode_id = i.inode_id + LEFT JOIN _vexfs.file_versions AS f + ON f.workspace_id = p_workspace_id + AND f.inode_id = i.inode_id + AND f.version_no = i.current_version; + END IF; +END; +$$; + +CREATE FUNCTION public.vexfs_snapshot_diff( + p_workspace text, + p_from text, + p_to text DEFAULT 'HEAD') +RETURNS jsonb +LANGUAGE plpgsql +SECURITY DEFINER +STABLE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; + v_from_snapshot bigint; + v_to_snapshot bigint; + v_changes jsonb; +BEGIN + v_workspace := _vexfs.require_workspace(p_workspace, 'admin'); + IF p_from <> 'HEAD' THEN + SELECT s.snapshot_id INTO v_from_snapshot + FROM _vexfs.snapshots AS s + WHERE s.workspace_id = v_workspace.workspace_id AND s.name = p_from; + IF NOT FOUND THEN + RAISE EXCEPTION 'VEXFS_SNAPSHOT_NOT_FOUND: %', p_from + USING ERRCODE = 'P0002'; + END IF; + END IF; + IF p_to <> 'HEAD' THEN + SELECT s.snapshot_id INTO v_to_snapshot + FROM _vexfs.snapshots AS s + WHERE s.workspace_id = v_workspace.workspace_id AND s.name = p_to; + IF NOT FOUND THEN + RAISE EXCEPTION 'VEXFS_SNAPSHOT_NOT_FOUND: %', p_to + USING ERRCODE = 'P0002'; + END IF; + END IF; + WITH old_tree AS ( + SELECT * FROM _vexfs.tree_at(v_workspace.workspace_id, v_from_snapshot)), + new_tree AS ( + SELECT * FROM _vexfs.tree_at(v_workspace.workspace_id, v_to_snapshot)), + changes AS ( + SELECT coalesce(old_tree.path, new_tree.path) AS path, + CASE WHEN old_tree.path IS NULL THEN 'added' + WHEN new_tree.path IS NULL THEN 'removed' + ELSE 'modified' END AS change, + old_tree.kind AS old_kind, + new_tree.kind AS new_kind, + old_tree.size_bytes AS old_size, + new_tree.size_bytes AS new_size, + old_tree.current_version AS old_version, + new_tree.current_version AS new_version + FROM old_tree + FULL JOIN new_tree USING (path) + WHERE old_tree.path IS NULL + OR new_tree.path IS NULL + OR old_tree.kind IS DISTINCT FROM new_tree.kind + OR old_tree.size_bytes IS DISTINCT FROM new_tree.size_bytes + OR old_tree.content_checksum IS DISTINCT FROM new_tree.content_checksum + OR old_tree.metadata_checksum IS DISTINCT FROM new_tree.metadata_checksum) + SELECT coalesce(jsonb_agg(jsonb_build_object( + 'path', changes.path, + 'change', changes.change, + 'old_kind', changes.old_kind, + 'new_kind', changes.new_kind, + 'old_size', changes.old_size, + 'new_size', changes.new_size, + 'old_version', changes.old_version, + 'new_version', changes.new_version) + ORDER BY changes.path), '[]'::jsonb) + INTO v_changes + FROM changes; + RETURN jsonb_build_object( + 'workspace', p_workspace, + 'from', p_from, + 'to', p_to, + 'changes', v_changes); +END; +$$; + +CREATE FUNCTION public.vexfs_snapshot_drop(p_workspace text, p_name text) +RETURNS integer +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; + v_snapshot_head bigint; +BEGIN + v_workspace := _vexfs.require_workspace(p_workspace, 'admin'); + DELETE FROM _vexfs.snapshots + WHERE workspace_id = v_workspace.workspace_id + AND name = p_name + RETURNING head_commit INTO v_snapshot_head; + IF NOT FOUND THEN + RAISE EXCEPTION 'VEXFS_SNAPSHOT_NOT_FOUND: %', p_name + USING ERRCODE = 'P0002'; + END IF; + PERFORM _vexfs.audit( + v_workspace.workspace_id, v_workspace.head_commit, 'snapshot_drop', + '/', v_workspace.root_inode, + jsonb_build_object( + 'before_version', v_workspace.head_commit, + 'after_version', v_workspace.head_commit, + 'snapshot', p_name, + 'snapshot_version', v_snapshot_head)); + RETURN 1; +END; +$$; + +CREATE FUNCTION public.vexfs_snapshot_restore( + p_workspace text, p_name text, p_expected_head bigint) +RETURNS bigint +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; + v_current_head bigint; + v_snapshot_id bigint; + v_snapshot_head bigint; + v_commit bigint; + v_same boolean; +BEGIN + v_workspace := _vexfs.require_workspace(p_workspace, 'write'); + SELECT w.head_commit INTO v_current_head + FROM _vexfs.workspaces AS w + WHERE w.workspace_id = v_workspace.workspace_id + FOR UPDATE; + IF v_current_head <> p_expected_head THEN + RAISE EXCEPTION 'VEXFS_HEAD_CONFLICT: expected %, actual %', + p_expected_head, v_current_head + USING ERRCODE = '40001'; + END IF; + SELECT s.snapshot_id, s.head_commit INTO v_snapshot_id, v_snapshot_head + FROM _vexfs.snapshots AS s + WHERE s.workspace_id = v_workspace.workspace_id + AND s.name = p_name + FOR SHARE; + IF NOT FOUND THEN + RAISE EXCEPTION 'VEXFS_SNAPSHOT_NOT_FOUND: %', p_name + USING ERRCODE = 'P0002'; + END IF; + IF v_snapshot_head = v_current_head THEN + RAISE EXCEPTION 'VEXFS_ALREADY_MATCHES: snapshot already matches workspace head' + USING ERRCODE = '22000'; + END IF; + + IF EXISTS ( + SELECT 1 + FROM _vexfs.snapshot_inodes AS s + LEFT JOIN _vexfs.file_versions AS source + ON source.workspace_id = v_workspace.workspace_id + AND source.inode_id = s.inode_id + AND source.version_no = s.current_version + WHERE s.snapshot_id = v_snapshot_id + AND s.kind <> 'directory' + AND source.version_no IS NULL) THEN + RAISE EXCEPTION 'VEXFS_CORRUPT: snapshot references a missing file version' + USING ERRCODE = 'XX001'; + END IF; + + SELECT + NOT EXISTS ( + (SELECT d.parent_inode, d.name, d.inode_id + FROM _vexfs.dentries AS d + WHERE d.workspace_id = v_workspace.workspace_id) + EXCEPT + (SELECT d.parent_inode, d.name, d.inode_id + FROM _vexfs.snapshot_dentries AS d + WHERE d.snapshot_id = v_snapshot_id)) + AND NOT EXISTS ( + (SELECT d.parent_inode, d.name, d.inode_id + FROM _vexfs.snapshot_dentries AS d + WHERE d.snapshot_id = v_snapshot_id) + EXCEPT + (SELECT d.parent_inode, d.name, d.inode_id + FROM _vexfs.dentries AS d + WHERE d.workspace_id = v_workspace.workspace_id)) + AND NOT EXISTS ( + SELECT 1 + FROM _vexfs.snapshot_inodes AS s + LEFT JOIN _vexfs.inodes AS i + ON i.workspace_id = v_workspace.workspace_id + AND i.inode_id = s.inode_id + LEFT JOIN _vexfs.file_versions AS current_version + ON current_version.workspace_id = v_workspace.workspace_id + AND current_version.inode_id = i.inode_id + AND current_version.version_no = i.current_version + LEFT JOIN _vexfs.file_versions AS source_version + ON source_version.workspace_id = v_workspace.workspace_id + AND source_version.inode_id = s.inode_id + AND source_version.version_no = s.current_version + WHERE s.snapshot_id = v_snapshot_id + AND (i.inode_id IS NULL + OR i.kind IS DISTINCT FROM s.kind + OR i.mode IS DISTINCT FROM s.mode + OR i.owner_oid IS DISTINCT FROM s.owner_oid + OR i.owner_principal IS DISTINCT FROM s.owner_principal + OR i.uid IS DISTINCT FROM s.uid + OR i.gid IS DISTINCT FROM s.gid + OR i.size_bytes IS DISTINCT FROM s.size_bytes + OR i.created_at IS DISTINCT FROM s.created_at + OR i.accessed_at IS DISTINCT FROM s.accessed_at + OR i.modified_at IS DISTINCT FROM s.modified_at + OR i.changed_at IS DISTINCT FROM s.changed_at + OR (s.kind <> 'directory' + AND (current_version.size_bytes IS DISTINCT FROM source_version.size_bytes + OR current_version.checksum IS DISTINCT FROM source_version.checksum)))) + AND NOT EXISTS ( + (SELECT x.inode_id, x.name, x.value + FROM _vexfs.xattrs AS x + WHERE x.workspace_id = v_workspace.workspace_id) + EXCEPT + (SELECT x.inode_id, x.name, x.value + FROM _vexfs.snapshot_xattrs AS x + WHERE x.snapshot_id = v_snapshot_id)) + AND NOT EXISTS ( + (SELECT x.inode_id, x.name, x.value + FROM _vexfs.snapshot_xattrs AS x + WHERE x.snapshot_id = v_snapshot_id) + EXCEPT + (SELECT x.inode_id, x.name, x.value + FROM _vexfs.xattrs AS x + WHERE x.workspace_id = v_workspace.workspace_id)) + AND NOT EXISTS ( + (SELECT acl.inode_id, acl.principal, acl.effect, + acl.permissions, acl.inherit_flags + FROM _vexfs.acl_entries AS acl + WHERE acl.workspace_id = v_workspace.workspace_id) + EXCEPT + (SELECT acl.inode_id, acl.principal, acl.effect, + acl.permissions, acl.inherit_flags + FROM _vexfs.snapshot_acl_entries AS acl + WHERE acl.snapshot_id = v_snapshot_id)) + AND NOT EXISTS ( + (SELECT acl.inode_id, acl.principal, acl.effect, + acl.permissions, acl.inherit_flags + FROM _vexfs.snapshot_acl_entries AS acl + WHERE acl.snapshot_id = v_snapshot_id) + EXCEPT + (SELECT acl.inode_id, acl.principal, acl.effect, + acl.permissions, acl.inherit_flags + FROM _vexfs.acl_entries AS acl + WHERE acl.workspace_id = v_workspace.workspace_id)) + INTO v_same; + IF v_same THEN + RAISE EXCEPTION 'VEXFS_ALREADY_MATCHES: snapshot already matches workspace state' + USING ERRCODE = '22000'; + END IF; + + PERFORM _vexfs.enforce_snapshot_quota( + v_workspace.workspace_id, v_snapshot_id); + v_commit := _vexfs.record_commit( + v_workspace.workspace_id, 'snapshot_restore', '/', v_workspace.root_inode, + jsonb_build_object( + 'before_version', v_current_head, + 'after_version', v_current_head + 1, + 'snapshot', p_name, + 'snapshot_version', v_snapshot_head)); + DELETE FROM _vexfs.dentries + WHERE workspace_id = v_workspace.workspace_id; + INSERT INTO _vexfs.dentries(workspace_id, parent_inode, name, inode_id) + SELECT v_workspace.workspace_id, d.parent_inode, d.name, d.inode_id + FROM _vexfs.snapshot_dentries AS d + WHERE d.snapshot_id = v_snapshot_id; + UPDATE _vexfs.inodes + SET live = false + WHERE workspace_id = v_workspace.workspace_id + AND live; + UPDATE _vexfs.inodes AS i + SET kind = s.kind, + mode = s.mode, + owner_oid = s.owner_oid, + owner_role = s.owner_role, + owner_principal = s.owner_principal, + uid = s.uid, + gid = s.gid, + size_bytes = s.size_bytes, + live = true, + created_at = s.created_at, + accessed_at = s.accessed_at, + modified_at = s.modified_at, + changed_at = s.changed_at + FROM _vexfs.snapshot_inodes AS s + WHERE s.snapshot_id = v_snapshot_id + AND i.workspace_id = v_workspace.workspace_id + AND i.inode_id = s.inode_id; + WITH restored AS ( + INSERT INTO _vexfs.file_versions( + workspace_id, inode_id, version_no, commit_no, source_version_no, + size_bytes, checksum, created_by_oid, created_by) + SELECT v_workspace.workspace_id, + s.inode_id, + (SELECT coalesce(max(all_versions.version_no), 0) + 1 + FROM _vexfs.file_versions AS all_versions + WHERE all_versions.workspace_id = v_workspace.workspace_id + AND all_versions.inode_id = s.inode_id), + v_commit, + coalesce(source.source_version_no, source.version_no), + source.size_bytes, + source.checksum, + (SELECT r.oid FROM pg_catalog.pg_roles AS r + WHERE r.rolname = session_user), + session_user + FROM _vexfs.snapshot_inodes AS s + JOIN _vexfs.inodes AS current_inode + ON current_inode.workspace_id = v_workspace.workspace_id + AND current_inode.inode_id = s.inode_id + JOIN _vexfs.file_versions AS source + ON source.workspace_id = v_workspace.workspace_id + AND source.inode_id = s.inode_id + AND source.version_no = s.current_version + JOIN _vexfs.file_versions AS current_content + ON current_content.workspace_id = v_workspace.workspace_id + AND current_content.inode_id = current_inode.inode_id + AND current_content.version_no = current_inode.current_version + WHERE s.snapshot_id = v_snapshot_id + AND s.kind <> 'directory' + AND (current_content.size_bytes IS DISTINCT FROM source.size_bytes + OR current_content.checksum IS DISTINCT FROM source.checksum) + RETURNING inode_id, version_no + ) + UPDATE _vexfs.inodes AS i + SET current_version = r.version_no + FROM restored AS r + WHERE i.workspace_id = v_workspace.workspace_id + AND i.inode_id = r.inode_id; + + DELETE FROM _vexfs.xattrs + WHERE workspace_id = v_workspace.workspace_id; + INSERT INTO _vexfs.xattrs( + workspace_id, inode_id, name, value, updated_at) + SELECT v_workspace.workspace_id, x.inode_id, x.name, x.value, x.updated_at + FROM _vexfs.snapshot_xattrs AS x + WHERE x.snapshot_id = v_snapshot_id; + + DELETE FROM _vexfs.acl_entries + WHERE workspace_id = v_workspace.workspace_id; + INSERT INTO _vexfs.acl_entries( + workspace_id, inode_id, principal, effect, permissions, + inherit_flags, created_at, updated_at) + SELECT v_workspace.workspace_id, acl.inode_id, acl.principal, acl.effect, + acl.permissions, acl.inherit_flags, acl.created_at, acl.updated_at + FROM _vexfs.snapshot_acl_entries AS acl + WHERE acl.snapshot_id = v_snapshot_id; + RETURN v_commit; +END; +$$; + +CREATE FUNCTION public.vexfs_quota_get(p_workspace text) +RETURNS jsonb +LANGUAGE plpgsql +SECURITY DEFINER +STABLE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; +BEGIN + v_workspace := _vexfs.require_workspace(p_workspace, 'read'); + RETURN _vexfs.quota_status(v_workspace.workspace_id, p_workspace); +END; +$$; + +CREATE FUNCTION public.vexfs_quota_set( + p_workspace text, + p_max_bytes bigint, + p_max_files bigint, + p_max_file_bytes bigint) +RETURNS jsonb +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; +BEGIN + IF p_max_bytes < 0 OR p_max_files < 0 OR p_max_file_bytes < 0 THEN + RAISE EXCEPTION 'VEXFS_INVALID_QUOTA: quota values must be non-negative or null' + USING ERRCODE = '22023'; + END IF; + v_workspace := _vexfs.require_workspace(p_workspace, 'admin'); + UPDATE _vexfs.workspaces + SET quota_max_bytes = p_max_bytes, + quota_max_files = p_max_files, + quota_max_file_bytes = p_max_file_bytes + WHERE workspace_id = v_workspace.workspace_id; + RETURN _vexfs.quota_status(v_workspace.workspace_id, p_workspace); +END; +$$; + +CREATE FUNCTION public.vexfs_retention_get(p_workspace text) +RETURNS jsonb +LANGUAGE plpgsql +SECURITY DEFINER +STABLE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; +BEGIN + v_workspace := _vexfs.require_workspace(p_workspace, 'read'); + RETURN _vexfs.retention_status(v_workspace.workspace_id, p_workspace); +END; +$$; + +CREATE FUNCTION public.vexfs_retention_set( + p_workspace text, + p_keep_versions integer, + p_keep_days integer) +RETURNS jsonb +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; +BEGIN + IF p_keep_versions IS NULL OR p_keep_versions < 0 OR p_keep_versions > 1000000 THEN + RAISE EXCEPTION 'VEXFS_INVALID_RETENTION: keep_versions must be between 0 and 1000000' + USING ERRCODE = '22023'; + END IF; + IF p_keep_days IS NULL OR p_keep_days < 0 OR p_keep_days > 36500 THEN + RAISE EXCEPTION 'VEXFS_INVALID_RETENTION: keep_days must be between 0 and 36500' + USING ERRCODE = '22023'; + END IF; + v_workspace := _vexfs.require_workspace(p_workspace, 'admin'); + UPDATE _vexfs.workspaces + SET retention_keep_versions = p_keep_versions, + retention_keep_days = p_keep_days + WHERE workspace_id = v_workspace.workspace_id; + RETURN _vexfs.retention_status(v_workspace.workspace_id, p_workspace); +END; +$$; + +CREATE FUNCTION public.vexfs_gc_pause( + p_workspace text, + p_paused integer) +RETURNS jsonb +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; +BEGIN + IF p_paused NOT IN (0, 1) THEN + RAISE EXCEPTION 'VEXFS_INVALID_GC_STATE: paused must be 0 or 1' + USING ERRCODE = '22023'; + END IF; + v_workspace := _vexfs.require_workspace(p_workspace, 'admin'); + UPDATE _vexfs.workspaces + SET gc_paused = p_paused = 1 + WHERE workspace_id = v_workspace.workspace_id; + RETURN _vexfs.retention_status(v_workspace.workspace_id, p_workspace); +END; +$$; + +CREATE FUNCTION public.vexfs_gc( + p_workspace text, + p_batch integer) +RETURNS jsonb +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; + v_gc_paused boolean; + v_inodes bigint[]; + v_versions bigint[]; + v_manifests bigint[]; + v_deleted_versions bigint := 0; + v_reclaimed_bytes bigint := 0; + v_remaining jsonb; +BEGIN + IF p_batch IS NULL OR p_batch < 1 OR p_batch > 10000 THEN + RAISE EXCEPTION 'VEXFS_INVALID_GC_BATCH: batch must be between 1 and 10000' + USING ERRCODE = '22023'; + END IF; + v_workspace := _vexfs.require_workspace(p_workspace, 'admin'); + SELECT w.gc_paused INTO v_gc_paused + FROM _vexfs.workspaces AS w + WHERE w.workspace_id = v_workspace.workspace_id + FOR UPDATE; + IF v_gc_paused THEN + RAISE EXCEPTION 'VEXFS_GC_PAUSED: workspace GC is paused' + USING ERRCODE = '55006'; + END IF; + + SELECT array_agg(candidate.inode_id ORDER BY candidate.ordinal), + array_agg(candidate.version_no ORDER BY candidate.ordinal), + array_agg(candidate.manifest_id ORDER BY candidate.ordinal) + INTO v_inodes, v_versions, v_manifests + FROM ( + SELECT f.inode_id, + f.version_no, + f.manifest_id, + row_number() OVER ( + ORDER BY f.source_version_no IS NULL, + f.created_at, + f.inode_id, + f.version_no) AS ordinal + FROM _vexfs.file_versions AS f + LEFT JOIN _vexfs.retention_keep_versions( + v_workspace.workspace_id) AS keep + ON keep.inode_id = f.inode_id + AND keep.version_no = f.version_no + WHERE f.workspace_id = v_workspace.workspace_id + AND keep.inode_id IS NULL + AND (f.source_version_no IS NOT NULL OR NOT EXISTS ( + SELECT 1 + FROM _vexfs.file_versions AS alias + WHERE alias.workspace_id = f.workspace_id + AND alias.inode_id = f.inode_id + AND alias.source_version_no = f.version_no)) + ORDER BY f.source_version_no IS NULL, + f.created_at, + f.inode_id, + f.version_no + LIMIT p_batch + ) AS candidate; + + v_deleted_versions := coalesce(array_length(v_versions, 1), 0); + IF v_deleted_versions > 0 THEN + DELETE FROM _vexfs.file_versions AS f + USING unnest(v_inodes, v_versions) AS candidate(inode_id, version_no) + WHERE f.workspace_id = v_workspace.workspace_id + AND f.inode_id = candidate.inode_id + AND f.version_no = candidate.version_no; + + DELETE FROM _vexfs.manifests AS m + WHERE m.workspace_id = v_workspace.workspace_id + AND m.manifest_id = ANY(array_remove(v_manifests, NULL)) + AND NOT EXISTS ( + SELECT 1 FROM _vexfs.file_versions AS f + WHERE f.manifest_id = m.manifest_id); + + WITH removed AS ( + DELETE FROM _vexfs.chunks AS c + WHERE c.workspace_id = v_workspace.workspace_id + AND NOT EXISTS ( + SELECT 1 FROM _vexfs.manifest_chunks AS entry + WHERE entry.chunk_id = c.chunk_id) + RETURNING c.size_bytes + ) + SELECT coalesce(sum(removed.size_bytes), 0) + INTO v_reclaimed_bytes + FROM removed; + END IF; + + v_remaining := _vexfs.retention_status( + v_workspace.workspace_id, p_workspace); + RETURN jsonb_build_object( + 'workspace', p_workspace, + 'batch', p_batch, + 'deleted_versions', v_deleted_versions, + 'reclaimed_bytes', v_reclaimed_bytes, + 'remaining_versions', (v_remaining->>'stored_versions')::bigint, + 'remaining_reclaimable_versions', + (v_remaining->>'reclaimable_versions')::bigint, + 'remaining_reclaimable_bytes', + (v_remaining->>'reclaimable_bytes')::bigint, + 'has_more', (v_remaining->>'reclaimable_versions')::bigint > 0); +END; +$$; + +CREATE FUNCTION public.vexfs_check( + p_workspace text, + p_deep integer DEFAULT 1) +RETURNS jsonb +LANGUAGE plpgsql +SECURITY DEFINER +STABLE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; + v_issues jsonb := '[]'::jsonb; + v_bad bigint; + v_inode_count bigint; + v_dentry_count bigint; + v_version_count bigint; + v_manifest_count bigint; + v_chunk_count bigint; + v_commit_count bigint; + v_request_count bigint; + v_acl_count bigint; + v_xattr_count bigint; + v_audit_count bigint; + v_handle_count bigint; + v_checked_versions bigint := 0; + v_content_bytes bigint := 0; + v_started timestamptz := clock_timestamp(); + v_row record; + v_error text; +BEGIN + IF p_deep NOT IN (0, 1) THEN + RAISE EXCEPTION 'VEXFS_INVALID_CHECK_MODE: deep must be 0 or 1' + USING ERRCODE = '22023'; + END IF; + v_workspace := _vexfs.require_workspace(p_workspace, 'read'); + + SELECT count(*) INTO v_inode_count + FROM _vexfs.inodes AS i + WHERE i.workspace_id = v_workspace.workspace_id; + SELECT count(*) INTO v_dentry_count + FROM _vexfs.dentries AS d + WHERE d.workspace_id = v_workspace.workspace_id; + SELECT count(*) INTO v_version_count + FROM _vexfs.file_versions AS f + WHERE f.workspace_id = v_workspace.workspace_id; + SELECT count(*) INTO v_manifest_count + FROM _vexfs.manifests AS m + WHERE m.workspace_id = v_workspace.workspace_id; + SELECT count(*) INTO v_chunk_count + FROM _vexfs.chunks AS c + WHERE c.workspace_id = v_workspace.workspace_id; + SELECT count(*) INTO v_commit_count + FROM _vexfs.commits AS c + WHERE c.workspace_id = v_workspace.workspace_id; + SELECT count(*) INTO v_request_count + FROM _vexfs.request_replays AS request + WHERE request.workspace_id = v_workspace.workspace_id; + SELECT count(*) INTO v_acl_count + FROM _vexfs.acl_entries AS acl + WHERE acl.workspace_id = v_workspace.workspace_id; + SELECT count(*) INTO v_xattr_count + FROM _vexfs.xattrs AS xattr + WHERE xattr.workspace_id = v_workspace.workspace_id; + SELECT count(*) INTO v_audit_count + FROM _vexfs.audit_events AS audit + WHERE audit.workspace_id = v_workspace.workspace_id; + SELECT count(*) INTO v_bad + FROM _vexfs.audit_events AS audit + WHERE audit.workspace_id = v_workspace.workspace_id + AND (audit.workspace_name <> p_workspace + OR audit.actor_role IS NULL + OR audit.operation IS NULL + OR audit.path IS NULL + OR audit.inode_id IS NULL + OR NOT audit.details ? 'before_version' + OR NOT audit.details ? 'after_version'); + IF v_bad <> 0 THEN + v_issues := v_issues || jsonb_build_array(jsonb_build_object( + 'code', 'audit_contract', + 'message', format('%s audit events are missing actor, object, or version metadata', v_bad))); + END IF; + SELECT count(*) INTO v_handle_count + FROM _vexfs.handles AS handle + WHERE handle.workspace_id = v_workspace.workspace_id; + SELECT coalesce(sum(c.size_bytes), 0) INTO v_content_bytes + FROM _vexfs.chunks AS c + WHERE c.workspace_id = v_workspace.workspace_id; + + SELECT count(*) INTO v_bad + FROM _vexfs.inodes AS i + WHERE i.inode_id = v_workspace.root_inode + AND i.workspace_id = v_workspace.workspace_id + AND i.kind = 'directory' + AND i.live; + IF v_bad <> 1 THEN + v_issues := v_issues || jsonb_build_array(jsonb_build_object( + 'code', 'root', 'message', 'workspace root inode is invalid')); + END IF; + + SELECT count(*) INTO v_bad + FROM _vexfs.dentries AS d + LEFT JOIN _vexfs.inodes AS parent ON parent.inode_id = d.parent_inode + LEFT JOIN _vexfs.inodes AS child ON child.inode_id = d.inode_id + WHERE d.workspace_id = v_workspace.workspace_id + AND (parent.inode_id IS NULL + OR child.inode_id IS NULL + OR parent.workspace_id <> d.workspace_id + OR child.workspace_id <> d.workspace_id + OR parent.kind <> 'directory' + OR NOT parent.live + OR NOT child.live); + IF v_bad <> 0 THEN + v_issues := v_issues || jsonb_build_array(jsonb_build_object( + 'code', 'dentry', 'count', v_bad, + 'message', 'dentry parent or child is invalid')); + END IF; + + WITH RECURSIVE reachable(inode_id) AS ( + SELECT v_workspace.root_inode + UNION + SELECT d.inode_id + FROM _vexfs.dentries AS d + JOIN reachable AS r ON r.inode_id = d.parent_inode + WHERE d.workspace_id = v_workspace.workspace_id + ) + SELECT count(*) INTO v_bad + FROM _vexfs.dentries AS d + WHERE d.workspace_id = v_workspace.workspace_id + AND NOT EXISTS ( + SELECT 1 FROM reachable AS r WHERE r.inode_id = d.parent_inode); + IF v_bad <> 0 THEN + v_issues := v_issues || jsonb_build_array(jsonb_build_object( + 'code', 'reachability', 'count', v_bad, + 'message', 'current dentry tree is not reachable from root')); + END IF; + + WITH RECURSIVE reachable(inode_id) AS ( + SELECT v_workspace.root_inode + UNION + SELECT d.inode_id + FROM _vexfs.dentries AS d + JOIN reachable AS r ON r.inode_id = d.parent_inode + WHERE d.workspace_id = v_workspace.workspace_id + ) + SELECT count(*) INTO v_bad + FROM _vexfs.inodes AS i + WHERE i.workspace_id = v_workspace.workspace_id + AND i.live + AND NOT EXISTS ( + SELECT 1 FROM reachable AS r WHERE r.inode_id = i.inode_id); + IF v_bad <> 0 THEN + v_issues := v_issues || jsonb_build_array(jsonb_build_object( + 'code', 'unreachable_inode', 'count', v_bad, + 'message', 'live inode is not reachable from root')); + END IF; + + SELECT count(*) INTO v_bad + FROM _vexfs.inodes AS i + WHERE i.workspace_id = v_workspace.workspace_id + AND i.kind = 'directory' + AND i.inode_id <> v_workspace.root_inode + AND (SELECT count(*) FROM _vexfs.dentries AS d + WHERE d.workspace_id = v_workspace.workspace_id + AND d.inode_id = i.inode_id) <> CASE WHEN i.live THEN 1 ELSE 0 END; + IF v_bad <> 0 THEN + v_issues := v_issues || jsonb_build_array(jsonb_build_object( + 'code', 'directory_link', 'count', v_bad, + 'message', 'directory has an invalid parent link count')); + END IF; + + SELECT count(*) INTO v_bad + FROM _vexfs.commits AS c + WHERE c.workspace_id = v_workspace.workspace_id + AND c.commit_no BETWEEN 1 AND v_workspace.head_commit + AND c.parent_commit = c.commit_no - 1; + IF v_workspace.head_commit < 1 OR v_bad <> v_workspace.head_commit + OR EXISTS ( + SELECT 1 FROM _vexfs.commits AS c + WHERE c.workspace_id = v_workspace.workspace_id + AND c.commit_no > v_workspace.head_commit) THEN + v_issues := v_issues || jsonb_build_array(jsonb_build_object( + 'code', 'commit_chain', + 'message', 'workspace commit chain is not continuous')); + END IF; + + SELECT count(*) INTO v_bad + FROM _vexfs.inodes AS i + LEFT JOIN _vexfs.file_versions AS f + ON f.workspace_id = i.workspace_id + AND f.inode_id = i.inode_id + AND f.version_no = i.current_version + WHERE i.workspace_id = v_workspace.workspace_id + AND ((i.kind <> 'directory' AND ( + i.current_version < 1 + OR f.version_no IS NULL + OR i.size_bytes IS DISTINCT FROM f.size_bytes)) + OR (i.kind = 'directory' AND i.current_version <> 0)); + IF v_bad <> 0 THEN + v_issues := v_issues || jsonb_build_array(jsonb_build_object( + 'code', 'current_version', 'count', v_bad, + 'message', 'inode current version is invalid')); + END IF; + + SELECT count(*) INTO v_bad + FROM _vexfs.workspaces AS w + WHERE w.workspace_id = v_workspace.workspace_id + AND (w.live_files IS DISTINCT FROM ( + SELECT count(*) + FROM _vexfs.inodes AS i + WHERE i.workspace_id = v_workspace.workspace_id + AND i.kind <> 'directory' + AND i.live) + OR w.live_bytes IS DISTINCT FROM ( + SELECT coalesce(sum(i.size_bytes), 0) + FROM _vexfs.inodes AS i + WHERE i.workspace_id = v_workspace.workspace_id + AND i.kind <> 'directory' + AND i.live)); + IF v_bad <> 0 THEN + v_issues := v_issues || jsonb_build_array(jsonb_build_object( + 'code', 'live_usage', + 'message', 'workspace live usage counters are invalid')); + END IF; + + SELECT count(*) INTO v_bad + FROM _vexfs.file_versions AS alias + LEFT JOIN _vexfs.file_versions AS source + ON source.workspace_id = alias.workspace_id + AND source.inode_id = alias.inode_id + AND source.version_no = alias.source_version_no + WHERE alias.workspace_id = v_workspace.workspace_id + AND alias.source_version_no IS NOT NULL + AND (source.version_no IS NULL + OR source.source_version_no IS NOT NULL + OR source.manifest_id IS NULL + OR source.size_bytes IS DISTINCT FROM alias.size_bytes + OR source.checksum IS DISTINCT FROM alias.checksum); + IF v_bad <> 0 THEN + v_issues := v_issues || jsonb_build_array(jsonb_build_object( + 'code', 'version_alias', 'count', v_bad, + 'message', 'file version alias is invalid')); + END IF; + + SELECT count(*) INTO v_bad + FROM _vexfs.file_versions AS f + LEFT JOIN _vexfs.manifests AS m ON m.manifest_id = f.manifest_id + WHERE f.workspace_id = v_workspace.workspace_id + AND f.manifest_id IS NOT NULL + AND (m.manifest_id IS NULL + OR m.workspace_id <> f.workspace_id + OR m.inode_id <> f.inode_id + OR m.file_size <> f.size_bytes + OR m.checksum IS DISTINCT FROM f.checksum + OR m.chunk_size <> 65536 + OR m.chunk_count <> ((m.file_size + 65535) / 65536)::integer); + IF v_bad <> 0 THEN + v_issues := v_issues || jsonb_build_array(jsonb_build_object( + 'code', 'manifest_metadata', 'count', v_bad, + 'message', 'manifest metadata does not match file version')); + END IF; + + SELECT count(*) INTO v_bad + FROM _vexfs.manifests AS m + WHERE m.workspace_id = v_workspace.workspace_id + AND ( + (SELECT count(*) FROM _vexfs.manifest_chunks AS entry + WHERE entry.manifest_id = m.manifest_id) <> m.chunk_count + OR EXISTS ( + SELECT 1 + FROM _vexfs.manifest_chunks AS entry + JOIN _vexfs.chunks AS c ON c.chunk_id = entry.chunk_id + WHERE entry.manifest_id = m.manifest_id + AND (entry.chunk_no >= m.chunk_count + OR c.workspace_id <> m.workspace_id + OR c.inode_id <> m.inode_id + OR c.size_bytes <> CASE + WHEN entry.chunk_no < m.chunk_count - 1 THEN 65536 + ELSE (m.file_size - entry.chunk_no::bigint * 65536)::integer + END)) + OR coalesce(( + SELECT sum(c.size_bytes) + FROM _vexfs.manifest_chunks AS entry + JOIN _vexfs.chunks AS c ON c.chunk_id = entry.chunk_id + WHERE entry.manifest_id = m.manifest_id), 0) <> m.file_size); + IF v_bad <> 0 THEN + v_issues := v_issues || jsonb_build_array(jsonb_build_object( + 'code', 'manifest_chunks', 'count', v_bad, + 'message', 'manifest chunk sequence or size is invalid')); + END IF; + + SELECT count(*) INTO v_bad + FROM _vexfs.manifests AS m + WHERE m.workspace_id = v_workspace.workspace_id + AND NOT EXISTS ( + SELECT 1 FROM _vexfs.file_versions AS f + WHERE f.manifest_id = m.manifest_id); + IF v_bad <> 0 THEN + v_issues := v_issues || jsonb_build_array(jsonb_build_object( + 'code', 'orphan_manifest', 'count', v_bad, + 'message', 'manifest is not referenced by a canonical version')); + END IF; + + SELECT count(*) INTO v_bad + FROM _vexfs.chunks AS c + WHERE c.workspace_id = v_workspace.workspace_id + AND NOT EXISTS ( + SELECT 1 FROM _vexfs.manifest_chunks AS entry + WHERE entry.chunk_id = c.chunk_id); + IF v_bad <> 0 THEN + v_issues := v_issues || jsonb_build_array(jsonb_build_object( + 'code', 'orphan_chunk', 'count', v_bad, + 'message', 'chunk is not referenced by a manifest')); + END IF; + + SELECT count(*) INTO v_bad + FROM _vexfs.snapshots AS s + WHERE s.workspace_id = v_workspace.workspace_id + AND ( + NOT EXISTS ( + SELECT 1 FROM _vexfs.snapshot_inodes AS i + WHERE i.snapshot_id = s.snapshot_id + AND i.inode_id = v_workspace.root_inode + AND i.kind = 'directory') + OR EXISTS ( + SELECT 1 + FROM _vexfs.snapshot_dentries AS d + LEFT JOIN _vexfs.snapshot_inodes AS parent + ON parent.snapshot_id = d.snapshot_id + AND parent.inode_id = d.parent_inode + LEFT JOIN _vexfs.snapshot_inodes AS child + ON child.snapshot_id = d.snapshot_id + AND child.inode_id = d.inode_id + WHERE d.snapshot_id = s.snapshot_id + AND (parent.inode_id IS NULL + OR child.inode_id IS NULL + OR parent.kind <> 'directory')) + OR EXISTS ( + SELECT 1 + FROM _vexfs.snapshot_xattrs AS x + LEFT JOIN _vexfs.snapshot_inodes AS i + ON i.snapshot_id = x.snapshot_id + AND i.inode_id = x.inode_id + WHERE x.snapshot_id = s.snapshot_id + AND i.inode_id IS NULL) + OR EXISTS ( + SELECT 1 + FROM _vexfs.snapshot_acl_entries AS acl + LEFT JOIN _vexfs.snapshot_inodes AS i + ON i.snapshot_id = acl.snapshot_id + AND i.inode_id = acl.inode_id + WHERE acl.snapshot_id = s.snapshot_id + AND i.inode_id IS NULL) + OR EXISTS ( + SELECT 1 + FROM _vexfs.snapshot_inodes AS i + LEFT JOIN _vexfs.file_versions AS f + ON f.workspace_id = s.workspace_id + AND f.inode_id = i.inode_id + AND f.version_no = i.current_version + WHERE i.snapshot_id = s.snapshot_id + AND i.kind <> 'directory' + AND f.version_no IS NULL)); + IF v_bad <> 0 THEN + v_issues := v_issues || jsonb_build_array(jsonb_build_object( + 'code', 'snapshot', 'count', v_bad, + 'message', 'snapshot tree or file reference is invalid')); + END IF; + + IF p_deep = 1 THEN + FOR v_row IN + SELECT f.inode_id, f.version_no + FROM _vexfs.file_versions AS f + WHERE f.workspace_id = v_workspace.workspace_id + AND f.manifest_id IS NOT NULL + ORDER BY f.inode_id, f.version_no + LOOP + v_checked_versions := v_checked_versions + 1; + BEGIN + PERFORM _vexfs.read_version_content( + v_workspace.workspace_id, v_row.inode_id, v_row.version_no); + EXCEPTION WHEN OTHERS THEN + GET STACKED DIAGNOSTICS v_error = MESSAGE_TEXT; + IF jsonb_array_length(v_issues) < 100 THEN + v_issues := v_issues || jsonb_build_array(jsonb_build_object( + 'code', 'content_checksum', + 'inode', v_row.inode_id, + 'version', v_row.version_no, + 'message', v_error)); + END IF; + END; + END LOOP; + END IF; + + SELECT coalesce(jsonb_agg(issue.value || jsonb_build_object( + 'object', coalesce( + issue.value->>'object', issue.value->>'inode', p_workspace), + 'suggestion', coalesce( + issue.value->>'suggestion', 'run deep check and restore from a verified snapshot'))), + '[]'::jsonb) + INTO v_issues + FROM jsonb_array_elements(v_issues) AS issue(value); + RETURN jsonb_build_object( + 'ok', jsonb_array_length(v_issues) = 0, + 'workspace', p_workspace, + 'mode', CASE WHEN p_deep = 1 THEN 'deep' ELSE 'quick' END, + 'content_model', 'chunked-v1', + 'issue_count', jsonb_array_length(v_issues), + 'versions', v_version_count, + 'content_bytes', v_content_bytes, + 'elapsed_ms', greatest(0, ( + extract(epoch FROM clock_timestamp() - v_started) * 1000)::bigint), + 'issues', v_issues, + 'checked', jsonb_build_object( + 'inodes', v_inode_count, + 'dentries', v_dentry_count, + 'versions', v_version_count, + 'manifests', v_manifest_count, + 'chunks', v_chunk_count, + 'commits', v_commit_count, + 'requests', v_request_count, + 'acl_entries', v_acl_count, + 'xattrs', v_xattr_count, + 'audit_events', v_audit_count, + 'handles', v_handle_count, + 'canonical_versions', v_checked_versions)); +END; +$$; + +-- format v2 的 PostgreSQL 生产端。CLI 只读取这个公开、带权限检查的记录流， +-- 不需要也不允许直接读取 _vexfs 私有表。临时表把一次导出固定在当前数据库 +-- 事务快照中，并把 PG 的完整 snapshot 表转换成 format v2 的状态变更记录。 +CREATE FUNCTION public.vexfs_archive_export_records( + p_workspace text, + p_snapshot text DEFAULT NULL) +RETURNS TABLE( + record_type text, + record_key text, + record_json jsonb, + content bytea) +LANGUAGE plpgsql +SECURITY DEFINER +VOLATILE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; + v_source_commit bigint; + v_selected_snapshot bigint; + v_selected_name text; + v_created_at bigint := floor(extract(epoch FROM clock_timestamp()) * 1000)::bigint; +BEGIN + v_workspace := _vexfs.require_workspace(p_workspace, 'admin'); + IF p_snapshot IS NULL OR p_snapshot = '' OR p_snapshot = 'HEAD' THEN + v_source_commit := v_workspace.head_commit; + v_selected_snapshot := NULL; + v_selected_name := NULL; + ELSE + SELECT s.snapshot_id, s.head_commit, s.name + INTO v_selected_snapshot, v_source_commit, v_selected_name + FROM _vexfs.snapshots AS s + WHERE s.workspace_id = v_workspace.workspace_id + AND s.name = p_snapshot; + IF NOT FOUND THEN + RAISE EXCEPTION 'VEXFS_SNAPSHOT_NOT_FOUND: %', p_snapshot + USING ERRCODE = 'P0002'; + END IF; + END IF; + IF v_source_commit <= 0 THEN + RAISE EXCEPTION 'VEXFS_ARCHIVE_EMPTY: workspace has no committed state' + USING ERRCODE = '55000'; + END IF; + + DROP TABLE IF EXISTS pg_temp.vexfs_export_checkpoints; + DROP TABLE IF EXISTS pg_temp.vexfs_export_inodes; + DROP TABLE IF EXISTS pg_temp.vexfs_export_dentries; + DROP TABLE IF EXISTS pg_temp.vexfs_export_xattrs; + DROP TABLE IF EXISTS pg_temp.vexfs_export_acls; + + CREATE TEMP TABLE vexfs_export_checkpoints( + commit_no bigint PRIMARY KEY, + snapshot_id bigint, + created_at_ms bigint NOT NULL, + selected boolean NOT NULL DEFAULT false) + ON COMMIT DROP; + INSERT INTO pg_temp.vexfs_export_checkpoints( + commit_no, snapshot_id, created_at_ms) + SELECT s.head_commit, + min(s.snapshot_id), + floor(extract(epoch FROM min(s.created_at)) * 1000)::bigint + FROM _vexfs.snapshots AS s + WHERE s.workspace_id = v_workspace.workspace_id + AND s.head_commit <= v_source_commit + GROUP BY s.head_commit; + INSERT INTO pg_temp.vexfs_export_checkpoints( + commit_no, snapshot_id, created_at_ms, selected) + VALUES ( + v_source_commit, + v_selected_snapshot, + coalesce((SELECT floor(extract(epoch FROM s.created_at) * 1000)::bigint + FROM _vexfs.snapshots AS s + WHERE s.snapshot_id = v_selected_snapshot), v_created_at), + true) + ON CONFLICT (commit_no) DO UPDATE + SET snapshot_id = EXCLUDED.snapshot_id, + created_at_ms = EXCLUDED.created_at_ms, + selected = true; + + CREATE TEMP TABLE vexfs_export_inodes( + commit_no bigint NOT NULL, + inode_id bigint NOT NULL, + kind text NOT NULL, + mode integer NOT NULL, + owner_principal text NOT NULL, + uid bigint NOT NULL, + gid bigint NOT NULL, + size_bytes bigint NOT NULL, + current_version bigint NOT NULL, + created_at_ms bigint NOT NULL, + accessed_at_ms bigint NOT NULL, + updated_at_ms bigint NOT NULL, + changed_at_ms bigint NOT NULL, + PRIMARY KEY (commit_no, inode_id)) ON COMMIT DROP; + INSERT INTO pg_temp.vexfs_export_inodes + SELECT cp.commit_no, + i.inode_id, + i.kind, + i.mode, + i.owner_principal, + i.uid, + i.gid, + i.size_bytes, + i.current_version, + floor(extract(epoch FROM i.created_at) * 1000)::bigint, + floor(extract(epoch FROM i.accessed_at) * 1000)::bigint, + floor(extract(epoch FROM i.modified_at) * 1000)::bigint, + floor(extract(epoch FROM i.changed_at) * 1000)::bigint + FROM pg_temp.vexfs_export_checkpoints AS cp + JOIN _vexfs.snapshot_inodes AS i ON i.snapshot_id = cp.snapshot_id + WHERE cp.snapshot_id IS NOT NULL; + INSERT INTO pg_temp.vexfs_export_inodes + SELECT cp.commit_no, + i.inode_id, + i.kind, + i.mode, + i.owner_principal, + i.uid, + i.gid, + i.size_bytes, + i.current_version, + floor(extract(epoch FROM i.created_at) * 1000)::bigint, + floor(extract(epoch FROM i.accessed_at) * 1000)::bigint, + floor(extract(epoch FROM i.modified_at) * 1000)::bigint, + floor(extract(epoch FROM i.changed_at) * 1000)::bigint + FROM pg_temp.vexfs_export_checkpoints AS cp + JOIN _vexfs.inodes AS i + ON i.workspace_id = v_workspace.workspace_id AND i.live + WHERE cp.selected AND cp.snapshot_id IS NULL + ON CONFLICT (commit_no, inode_id) DO UPDATE SET + kind = EXCLUDED.kind, + mode = EXCLUDED.mode, + owner_principal = EXCLUDED.owner_principal, + uid = EXCLUDED.uid, + gid = EXCLUDED.gid, + size_bytes = EXCLUDED.size_bytes, + current_version = EXCLUDED.current_version, + created_at_ms = EXCLUDED.created_at_ms, + accessed_at_ms = EXCLUDED.accessed_at_ms, + updated_at_ms = EXCLUDED.updated_at_ms, + changed_at_ms = EXCLUDED.changed_at_ms; + + CREATE TEMP TABLE vexfs_export_dentries( + commit_no bigint NOT NULL, + parent_inode bigint NOT NULL, + name text NOT NULL, + inode_id bigint NOT NULL, + PRIMARY KEY (commit_no, parent_inode, name)) ON COMMIT DROP; + INSERT INTO pg_temp.vexfs_export_dentries + SELECT cp.commit_no, d.parent_inode, d.name, d.inode_id + FROM pg_temp.vexfs_export_checkpoints AS cp + JOIN _vexfs.snapshot_dentries AS d ON d.snapshot_id = cp.snapshot_id + WHERE cp.snapshot_id IS NOT NULL; + INSERT INTO pg_temp.vexfs_export_dentries + SELECT cp.commit_no, d.parent_inode, d.name, d.inode_id + FROM pg_temp.vexfs_export_checkpoints AS cp + JOIN _vexfs.dentries AS d ON d.workspace_id = v_workspace.workspace_id + WHERE cp.selected AND cp.snapshot_id IS NULL + ON CONFLICT (commit_no, parent_inode, name) DO UPDATE + SET inode_id = EXCLUDED.inode_id; + + CREATE TEMP TABLE vexfs_export_xattrs( + commit_no bigint NOT NULL, + inode_id bigint NOT NULL, + name text NOT NULL, + value bytea NOT NULL, + updated_at_ms bigint NOT NULL, + PRIMARY KEY (commit_no, inode_id, name)) ON COMMIT DROP; + INSERT INTO pg_temp.vexfs_export_xattrs + SELECT cp.commit_no, + x.inode_id, + x.name, + x.value, + floor(extract(epoch FROM x.updated_at) * 1000)::bigint + FROM pg_temp.vexfs_export_checkpoints AS cp + JOIN _vexfs.snapshot_xattrs AS x ON x.snapshot_id = cp.snapshot_id + WHERE cp.snapshot_id IS NOT NULL; + INSERT INTO pg_temp.vexfs_export_xattrs + SELECT cp.commit_no, + x.inode_id, + x.name, + x.value, + floor(extract(epoch FROM x.updated_at) * 1000)::bigint + FROM pg_temp.vexfs_export_checkpoints AS cp + JOIN _vexfs.xattrs AS x ON x.workspace_id = v_workspace.workspace_id + WHERE cp.selected AND cp.snapshot_id IS NULL + ON CONFLICT (commit_no, inode_id, name) DO UPDATE + SET value = EXCLUDED.value, updated_at_ms = EXCLUDED.updated_at_ms; + + CREATE TEMP TABLE vexfs_export_acls( + commit_no bigint NOT NULL, + inode_id bigint NOT NULL, + principal text NOT NULL, + effect text NOT NULL, + permissions text NOT NULL, + inherit_flags integer NOT NULL, + created_at_ms bigint NOT NULL, + updated_at_ms bigint NOT NULL, + PRIMARY KEY (commit_no, inode_id, principal, effect)) ON COMMIT DROP; + INSERT INTO pg_temp.vexfs_export_acls + SELECT cp.commit_no, + acl.inode_id, + acl.principal, + acl.effect, + acl.permissions, + acl.inherit_flags, + floor(extract(epoch FROM acl.created_at) * 1000)::bigint, + floor(extract(epoch FROM acl.updated_at) * 1000)::bigint + FROM pg_temp.vexfs_export_checkpoints AS cp + JOIN _vexfs.snapshot_acl_entries AS acl ON acl.snapshot_id = cp.snapshot_id + WHERE cp.snapshot_id IS NOT NULL; + INSERT INTO pg_temp.vexfs_export_acls + SELECT cp.commit_no, + acl.inode_id, + acl.principal, + acl.effect, + acl.permissions, + acl.inherit_flags, + floor(extract(epoch FROM acl.created_at) * 1000)::bigint, + floor(extract(epoch FROM acl.updated_at) * 1000)::bigint + FROM pg_temp.vexfs_export_checkpoints AS cp + JOIN _vexfs.acl_entries AS acl ON acl.workspace_id = v_workspace.workspace_id + WHERE cp.selected AND cp.snapshot_id IS NULL + ON CONFLICT (commit_no, inode_id, principal, effect) DO UPDATE SET + permissions = EXCLUDED.permissions, + inherit_flags = EXCLUDED.inherit_flags, + created_at_ms = EXCLUDED.created_at_ms, + updated_at_ms = EXCLUDED.updated_at_ms; + + record_type := 'manifest'; + record_key := 'manifest'; + record_json := jsonb_build_object( + 'format_version', 2, + 'source_engine', 'postgresql', + 'source_workspace', p_workspace, + 'source_commit', v_source_commit, + 'source_snapshot', v_selected_name, + 'root_source_inode', v_workspace.root_inode, + 'history_floor_source_commit', v_workspace.history_floor_commit, + 'retention_keep_versions', v_workspace.retention_keep_versions, + 'retention_keep_days', v_workspace.retention_keep_days, + 'quota_max_bytes', v_workspace.quota_max_bytes, + 'quota_max_files', v_workspace.quota_max_files, + 'quota_max_file_bytes', v_workspace.quota_max_file_bytes, + 'created_at', v_created_at); + content := NULL; + RETURN NEXT; + + RETURN QUERY + SELECT 'commits'::text, + lpad(c.commit_no::text, 20, '0'), + jsonb_build_object( + 'source_id', c.commit_no, + 'parent_source_id', nullif(c.parent_commit, 0), + 'message', c.operation, + 'created_at', floor(extract(epoch FROM c.created_at) * 1000)::bigint), + NULL::bytea + FROM _vexfs.commits AS c + WHERE c.workspace_id = v_workspace.workspace_id + AND c.commit_no <= v_source_commit + ORDER BY c.commit_no; + + RETURN QUERY + WITH latest AS ( + SELECT DISTINCT ON (state.inode_id) + state.* + FROM pg_temp.vexfs_export_inodes AS state + WHERE state.commit_no <= v_source_commit + ORDER BY state.inode_id, state.commit_no DESC), + selected AS ( + SELECT state.inode_id + FROM pg_temp.vexfs_export_inodes AS state + WHERE state.commit_no = v_source_commit) + SELECT 'inodes'::text, + lpad(latest.inode_id::text, 20, '0'), + jsonb_build_object( + 'source_id', latest.inode_id, + 'kind', latest.kind, + 'mode', latest.mode, + 'owner_principal', latest.owner_principal, + 'uid', latest.uid, + 'gid', latest.gid, + 'size', latest.size_bytes, + 'current_version', latest.current_version, + 'created_at', latest.created_at_ms, + 'accessed_at', latest.accessed_at_ms, + 'updated_at', latest.updated_at_ms, + 'changed_at', latest.changed_at_ms, + 'deleted_at', CASE WHEN selected.inode_id IS NULL + THEN v_created_at ELSE NULL END), + NULL::bytea + FROM latest + LEFT JOIN selected USING (inode_id) + ORDER BY latest.inode_id; + + RETURN QUERY + SELECT 'dentries'::text, + lpad(d.parent_inode::text, 20, '0') || ':' || encode(convert_to(d.name, 'UTF8'), 'hex'), + jsonb_build_object( + 'parent_source_inode', d.parent_inode, + 'name', d.name, + 'inode_source_id', d.inode_id), + NULL::bytea + FROM pg_temp.vexfs_export_dentries AS d + WHERE d.commit_no = v_source_commit + ORDER BY d.parent_inode, d.name; + + RETURN QUERY + SELECT 'file_versions'::text, + lpad(f.inode_id::text, 20, '0') || ':' || lpad(f.version_no::text, 20, '0'), + jsonb_build_object( + 'source_inode', f.inode_id, + 'version_no', f.version_no, + 'source_commit', f.commit_no, + 'source_manifest', f.manifest_id, + 'size', f.size_bytes, + 'checksum', f.checksum, + 'source_version_no', f.source_version_no, + 'created_at', floor(extract(epoch FROM f.created_at) * 1000)::bigint), + NULL::bytea + FROM _vexfs.file_versions AS f + WHERE f.workspace_id = v_workspace.workspace_id + AND f.commit_no <= v_source_commit + AND EXISTS ( + SELECT 1 FROM pg_temp.vexfs_export_inodes AS state + WHERE state.inode_id = f.inode_id) + ORDER BY f.inode_id, f.version_no; + + RETURN QUERY + SELECT 'manifests'::text, + lpad(m.manifest_id::text, 20, '0'), + jsonb_build_object( + 'source_id', m.manifest_id, + 'file_size', m.file_size, + 'chunk_size', m.chunk_size, + 'chunk_count', m.chunk_count, + 'checksum', m.checksum, + 'created_at', floor(extract(epoch FROM m.created_at) * 1000)::bigint), + NULL::bytea + FROM _vexfs.manifests AS m + WHERE m.workspace_id = v_workspace.workspace_id + AND EXISTS ( + SELECT 1 FROM _vexfs.file_versions AS f + WHERE f.workspace_id = v_workspace.workspace_id + AND f.commit_no <= v_source_commit + AND f.manifest_id = m.manifest_id + AND EXISTS ( + SELECT 1 FROM pg_temp.vexfs_export_inodes AS state + WHERE state.inode_id = f.inode_id)) + ORDER BY m.manifest_id; + + RETURN QUERY + SELECT 'chunks'::text, + lpad(entry.manifest_id::text, 20, '0') || ':' || + lpad(entry.chunk_no::text, 20, '0'), + jsonb_build_object( + 'source_manifest', entry.manifest_id, + 'chunk_no', entry.chunk_no, + 'size', c.size_bytes, + 'checksum', c.checksum), + c.content + FROM _vexfs.manifest_chunks AS entry + JOIN _vexfs.chunks AS c ON c.chunk_id = entry.chunk_id + WHERE EXISTS ( + SELECT 1 FROM _vexfs.manifests AS m + WHERE m.manifest_id = entry.manifest_id + AND m.workspace_id = v_workspace.workspace_id + AND EXISTS ( + SELECT 1 FROM _vexfs.file_versions AS f + WHERE f.workspace_id = v_workspace.workspace_id + AND f.commit_no <= v_source_commit + AND f.manifest_id = m.manifest_id + AND EXISTS ( + SELECT 1 FROM pg_temp.vexfs_export_inodes AS state + WHERE state.inode_id = f.inode_id))) + ORDER BY entry.manifest_id, entry.chunk_no; + + RETURN QUERY + WITH inode_keys AS ( + SELECT DISTINCT state.inode_id FROM pg_temp.vexfs_export_inodes AS state), + grid AS ( + SELECT cp.commit_no, + cp.created_at_ms AS checkpoint_created_at, + key.inode_id, + (state.inode_id IS NOT NULL) AS present, + lag(state.inode_id IS NOT NULL) OVER ( + PARTITION BY key.inode_id ORDER BY cp.commit_no) AS was_present + FROM pg_temp.vexfs_export_checkpoints AS cp + CROSS JOIN inode_keys AS key + LEFT JOIN pg_temp.vexfs_export_inodes AS state + ON state.commit_no = cp.commit_no AND state.inode_id = key.inode_id), + emitted AS ( + SELECT grid.commit_no, + grid.checkpoint_created_at, + coalesce(current_state.inode_id, previous_state.inode_id) AS inode_id, + coalesce(current_state.kind, previous_state.kind) AS kind, + coalesce(current_state.mode, previous_state.mode) AS mode, + coalesce(current_state.owner_principal, previous_state.owner_principal) AS owner_principal, + coalesce(current_state.uid, previous_state.uid) AS uid, + coalesce(current_state.gid, previous_state.gid) AS gid, + coalesce(current_state.size_bytes, previous_state.size_bytes) AS size_bytes, + coalesce(current_state.current_version, previous_state.current_version) AS current_version, + coalesce(current_state.created_at_ms, previous_state.created_at_ms) AS created_at_ms, + coalesce(current_state.accessed_at_ms, previous_state.accessed_at_ms) AS accessed_at_ms, + coalesce(current_state.updated_at_ms, previous_state.updated_at_ms) AS updated_at_ms, + coalesce(current_state.changed_at_ms, previous_state.changed_at_ms) AS changed_at_ms, + grid.present + FROM grid + LEFT JOIN pg_temp.vexfs_export_inodes AS current_state + ON current_state.commit_no = grid.commit_no + AND current_state.inode_id = grid.inode_id + LEFT JOIN LATERAL ( + SELECT state.* + FROM pg_temp.vexfs_export_inodes AS state + WHERE state.inode_id = grid.inode_id + AND state.commit_no < grid.commit_no + ORDER BY state.commit_no DESC LIMIT 1) AS previous_state ON true + WHERE grid.present OR (grid.was_present AND NOT grid.present)) + SELECT 'inode_states'::text, + lpad(emitted.inode_id::text, 20, '0') || ':' || + lpad(emitted.commit_no::text, 20, '0'), + jsonb_build_object( + 'source_inode', emitted.inode_id, + 'source_commit', emitted.commit_no, + 'kind', emitted.kind, + 'mode', emitted.mode, + 'owner_principal', emitted.owner_principal, + 'uid', emitted.uid, + 'gid', emitted.gid, + 'size', emitted.size_bytes, + 'current_version', emitted.current_version, + 'created_at', emitted.created_at_ms, + 'accessed_at', emitted.accessed_at_ms, + 'updated_at', emitted.updated_at_ms, + 'changed_at', emitted.changed_at_ms, + 'deleted_at', CASE WHEN emitted.present THEN NULL + ELSE emitted.checkpoint_created_at END), + NULL::bytea + FROM emitted + ORDER BY emitted.inode_id, emitted.commit_no; + + RETURN QUERY + WITH keys AS ( + SELECT DISTINCT state.parent_inode, state.name + FROM pg_temp.vexfs_export_dentries AS state), + grid AS ( + SELECT cp.commit_no, + key.parent_inode, + key.name, + state.inode_id, + (state.inode_id IS NOT NULL) AS present, + lag(state.inode_id IS NOT NULL) OVER ( + PARTITION BY key.parent_inode, key.name ORDER BY cp.commit_no) AS was_present + FROM pg_temp.vexfs_export_checkpoints AS cp + CROSS JOIN keys AS key + LEFT JOIN pg_temp.vexfs_export_dentries AS state + ON state.commit_no = cp.commit_no + AND state.parent_inode = key.parent_inode + AND state.name = key.name) + SELECT 'dentry_states'::text, + lpad(grid.parent_inode::text, 20, '0') || ':' || + encode(convert_to(grid.name, 'UTF8'), 'hex') || ':' || + lpad(grid.commit_no::text, 20, '0'), + jsonb_build_object( + 'parent_source_inode', grid.parent_inode, + 'name', grid.name, + 'source_commit', grid.commit_no, + 'inode_source_id', coalesce(grid.inode_id, previous.inode_id), + 'deleted', CASE WHEN grid.present THEN 0 ELSE 1 END), + NULL::bytea + FROM grid + LEFT JOIN LATERAL ( + SELECT state.inode_id + FROM pg_temp.vexfs_export_dentries AS state + WHERE state.parent_inode = grid.parent_inode + AND state.name = grid.name + AND state.commit_no < grid.commit_no + ORDER BY state.commit_no DESC LIMIT 1) AS previous ON true + WHERE grid.present OR (grid.was_present AND NOT grid.present) + ORDER BY grid.parent_inode, grid.name, grid.commit_no; + + RETURN QUERY + WITH keys AS ( + SELECT DISTINCT state.inode_id, state.name + FROM pg_temp.vexfs_export_xattrs AS state), + grid AS ( + SELECT cp.commit_no, + key.inode_id, + key.name, + state.value, + (state.inode_id IS NOT NULL) AS present, + lag(state.inode_id IS NOT NULL) OVER ( + PARTITION BY key.inode_id, key.name ORDER BY cp.commit_no) AS was_present + FROM pg_temp.vexfs_export_checkpoints AS cp + CROSS JOIN keys AS key + LEFT JOIN pg_temp.vexfs_export_xattrs AS state + ON state.commit_no = cp.commit_no + AND state.inode_id = key.inode_id + AND state.name = key.name) + SELECT 'xattr_states'::text, + lpad(grid.inode_id::text, 20, '0') || ':' || + encode(convert_to(grid.name, 'UTF8'), 'hex') || ':' || + lpad(grid.commit_no::text, 20, '0'), + jsonb_build_object( + 'source_inode', grid.inode_id, + 'name', grid.name, + 'source_commit', grid.commit_no, + 'value_hex', encode(coalesce(grid.value, previous.value), 'hex'), + 'deleted', CASE WHEN grid.present THEN 0 ELSE 1 END), + NULL::bytea + FROM grid + LEFT JOIN LATERAL ( + SELECT state.value + FROM pg_temp.vexfs_export_xattrs AS state + WHERE state.inode_id = grid.inode_id + AND state.name = grid.name + AND state.commit_no < grid.commit_no + ORDER BY state.commit_no DESC LIMIT 1) AS previous ON true + WHERE grid.present OR (grid.was_present AND NOT grid.present) + ORDER BY grid.inode_id, grid.name, grid.commit_no; + + RETURN QUERY + WITH keys AS ( + SELECT DISTINCT state.inode_id, state.principal, state.effect + FROM pg_temp.vexfs_export_acls AS state), + grid AS ( + SELECT cp.commit_no, + key.inode_id, + key.principal, + key.effect, + state.permissions, + state.inherit_flags, + (state.inode_id IS NOT NULL) AS present, + lag(state.inode_id IS NOT NULL) OVER ( + PARTITION BY key.inode_id, key.principal, key.effect + ORDER BY cp.commit_no) AS was_present + FROM pg_temp.vexfs_export_checkpoints AS cp + CROSS JOIN keys AS key + LEFT JOIN pg_temp.vexfs_export_acls AS state + ON state.commit_no = cp.commit_no + AND state.inode_id = key.inode_id + AND state.principal = key.principal + AND state.effect = key.effect) + SELECT 'acl_states'::text, + lpad(grid.inode_id::text, 20, '0') || ':' || + encode(convert_to(grid.principal, 'UTF8'), 'hex') || ':' || + grid.effect || ':' || lpad(grid.commit_no::text, 20, '0'), + jsonb_build_object( + 'source_inode', grid.inode_id, + 'principal_id', grid.principal, + 'effect', grid.effect, + 'source_commit', grid.commit_no, + 'permissions', coalesce(grid.permissions, previous.permissions), + 'inherit_flags', coalesce(grid.inherit_flags, previous.inherit_flags), + 'deleted', CASE WHEN grid.present THEN 0 ELSE 1 END), + NULL::bytea + FROM grid + LEFT JOIN LATERAL ( + SELECT state.permissions, state.inherit_flags + FROM pg_temp.vexfs_export_acls AS state + WHERE state.inode_id = grid.inode_id + AND state.principal = grid.principal + AND state.effect = grid.effect + AND state.commit_no < grid.commit_no + ORDER BY state.commit_no DESC LIMIT 1) AS previous ON true + WHERE grid.present OR (grid.was_present AND NOT grid.present) + ORDER BY grid.inode_id, grid.principal, grid.effect, grid.commit_no; + + RETURN QUERY + SELECT 'snapshots'::text, + encode(convert_to(s.name, 'UTF8'), 'hex'), + jsonb_build_object( + 'name', s.name, + 'source_commit', s.head_commit, + 'created_at', floor(extract(epoch FROM s.created_at) * 1000)::bigint), + NULL::bytea + FROM _vexfs.snapshots AS s + WHERE s.workspace_id = v_workspace.workspace_id + AND s.head_commit <= v_source_commit + ORDER BY s.head_commit, s.snapshot_id; + + RETURN QUERY + SELECT 'xattrs'::text, + lpad(x.inode_id::text, 20, '0') || ':' || encode(convert_to(x.name, 'UTF8'), 'hex'), + jsonb_build_object( + 'source_inode', x.inode_id, + 'name', x.name, + 'value_hex', encode(x.value, 'hex'), + 'updated_at', x.updated_at_ms), + NULL::bytea + FROM pg_temp.vexfs_export_xattrs AS x + WHERE x.commit_no = v_source_commit + ORDER BY x.inode_id, x.name; + + RETURN QUERY + SELECT 'acl_entries'::text, + lpad(acl.inode_id::text, 20, '0') || ':' || + encode(convert_to(acl.principal, 'UTF8'), 'hex') || ':' || acl.effect, + jsonb_build_object( + 'source_inode', acl.inode_id, + 'principal_id', acl.principal, + 'effect', acl.effect, + 'permissions', acl.permissions, + 'inherit_flags', acl.inherit_flags, + 'created_at', acl.created_at_ms, + 'updated_at', acl.updated_at_ms), + NULL::bytea + FROM pg_temp.vexfs_export_acls AS acl + WHERE acl.commit_no = v_source_commit + ORDER BY acl.inode_id, acl.principal, acl.effect; + + RETURN QUERY + WITH principals AS ( + SELECT state.owner_principal AS principal + FROM pg_temp.vexfs_export_inodes AS state + UNION + SELECT state.principal FROM pg_temp.vexfs_export_acls AS state) + SELECT 'principals'::text, + encode(convert_to(principals.principal, 'UTF8'), 'hex'), + jsonb_build_object('source_principal', principals.principal), + NULL::bytea + FROM principals + WHERE principals.principal <> '' + ORDER BY principals.principal; +END; +$$; + +CREATE FUNCTION public.vexfs_archive_import_begin( + p_workspace text, + p_manifest jsonb) +RETURNS bigint +LANGUAGE plpgsql +SECURITY DEFINER +VOLATILE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_job bigint; + v_actor oid; +BEGIN + IF p_workspace IS NULL OR btrim(p_workspace) = '' OR p_workspace <> btrim(p_workspace) + OR octet_length(p_workspace) > 128 THEN + RAISE EXCEPTION 'VEXFS_INVALID_WORKSPACE: invalid workspace name' + USING ERRCODE = '22023'; + END IF; + IF p_manifest IS NULL OR jsonb_typeof(p_manifest) <> 'object' + OR coalesce((p_manifest->>'format_version')::integer, 0) <> 2 + OR coalesce((p_manifest->>'source_commit')::bigint, 0) <= 0 + OR coalesce((p_manifest->>'root_source_inode')::bigint, 0) <= 0 + OR coalesce((p_manifest->>'history_floor_source_commit')::bigint, 0) <= 0 + OR (p_manifest->>'history_floor_source_commit')::bigint > + (p_manifest->>'source_commit')::bigint + OR coalesce((p_manifest->>'retention_keep_versions')::integer, -1) + NOT BETWEEN 0 AND 1000000 + OR coalesce((p_manifest->>'retention_keep_days')::integer, -1) + NOT BETWEEN 0 AND 36500 THEN + RAISE EXCEPTION 'VEXFS_INVALID_ARCHIVE: invalid format v2 manifest' + USING ERRCODE = '22023'; + END IF; + IF (p_manifest->>'quota_max_bytes') IS NOT NULL + AND (p_manifest->>'quota_max_bytes')::bigint < 0 THEN + RAISE EXCEPTION 'VEXFS_INVALID_ARCHIVE: invalid max bytes' + USING ERRCODE = '22023'; + END IF; + IF (p_manifest->>'quota_max_files') IS NOT NULL + AND (p_manifest->>'quota_max_files')::bigint < 0 THEN + RAISE EXCEPTION 'VEXFS_INVALID_ARCHIVE: invalid max files' + USING ERRCODE = '22023'; + END IF; + IF (p_manifest->>'quota_max_file_bytes') IS NOT NULL + AND (p_manifest->>'quota_max_file_bytes')::bigint < 0 THEN + RAISE EXCEPTION 'VEXFS_INVALID_ARCHIVE: invalid max file bytes' + USING ERRCODE = '22023'; + END IF; + IF EXISTS (SELECT 1 FROM _vexfs.workspaces AS w WHERE w.name = p_workspace) THEN + RAISE EXCEPTION 'VEXFS_WORKSPACE_EXISTS: %', p_workspace + USING ERRCODE = '23505'; + END IF; + SELECT r.oid INTO STRICT v_actor + FROM pg_catalog.pg_roles AS r WHERE r.rolname = session_user; + DELETE FROM _vexfs.archive_import_jobs AS job + WHERE job.owner_oid = v_actor + AND job.created_at < clock_timestamp() - interval '1 day'; + INSERT INTO _vexfs.archive_import_jobs( + workspace_name, owner_oid, owner_role, backend_pid, manifest) + VALUES (p_workspace, v_actor, session_user, pg_catalog.pg_backend_pid(), p_manifest) + RETURNING job_id INTO v_job; + RETURN v_job; +END; +$$; + +CREATE FUNCTION public.vexfs_archive_import_record( + p_job bigint, + p_record_type text, + p_record_key text, + p_record_json jsonb, + p_content bytea DEFAULT NULL) +RETURNS bigint +LANGUAGE plpgsql +SECURITY DEFINER +VOLATILE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_actor oid; + v_count bigint; +BEGIN + SELECT r.oid INTO STRICT v_actor + FROM pg_catalog.pg_roles AS r WHERE r.rolname = session_user; + PERFORM 1 FROM _vexfs.archive_import_jobs AS job + WHERE job.job_id = p_job + AND job.owner_oid = v_actor + AND job.backend_pid = pg_catalog.pg_backend_pid(); + IF NOT FOUND THEN + RAISE EXCEPTION 'VEXFS_IMPORT_JOB_NOT_FOUND: %', p_job + USING ERRCODE = 'P0002'; + END IF; + IF p_record_type NOT IN ( + 'commits', 'inodes', 'dentries', 'file_versions', 'manifests', 'chunks', + 'inode_states', 'dentry_states', 'xattr_states', 'acl_states', 'snapshots', + 'xattrs', 'acl_entries', 'principals') + OR p_record_key IS NULL OR p_record_key = '' OR octet_length(p_record_key) > 1024 + OR p_record_json IS NULL OR jsonb_typeof(p_record_json) <> 'object' + OR octet_length(p_record_json::text) > 1048576 THEN + RAISE EXCEPTION 'VEXFS_INVALID_ARCHIVE_RECORD: invalid type, key, or JSON' + USING ERRCODE = '22023'; + END IF; + IF p_record_type = 'chunks' THEN + IF p_content IS NULL OR octet_length(p_content) < 1 + OR octet_length(p_content) > 65536 + OR coalesce((p_record_json->>'size')::integer, -1) <> octet_length(p_content) + OR p_record_json->>'checksum' <> encode(pg_catalog.sha256(p_content), 'hex') THEN + RAISE EXCEPTION 'VEXFS_INVALID_ARCHIVE_CHUNK: size or checksum mismatch' + USING ERRCODE = '22023'; + END IF; + ELSIF p_content IS NOT NULL THEN + RAISE EXCEPTION 'VEXFS_INVALID_ARCHIVE_RECORD: only chunks carry content' + USING ERRCODE = '22023'; + END IF; + INSERT INTO _vexfs.archive_import_records( + job_id, record_type, record_key, record_json, content) + VALUES (p_job, p_record_type, p_record_key, p_record_json, p_content); + SELECT count(*) INTO v_count + FROM _vexfs.archive_import_records AS record + WHERE record.job_id = p_job; + IF v_count > 10000000 THEN + RAISE EXCEPTION 'VEXFS_ARCHIVE_TOO_LARGE: record limit exceeded' + USING ERRCODE = '54000'; + END IF; + RETURN v_count; +END; +$$; + +CREATE FUNCTION public.vexfs_archive_import_abort(p_job bigint) +RETURNS integer +LANGUAGE plpgsql +SECURITY DEFINER +VOLATILE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_actor oid; + v_removed integer; +BEGIN + SELECT r.oid INTO STRICT v_actor + FROM pg_catalog.pg_roles AS r WHERE r.rolname = session_user; + DELETE FROM _vexfs.archive_import_jobs AS job + WHERE job.job_id = p_job + AND job.owner_oid = v_actor + AND job.backend_pid = pg_catalog.pg_backend_pid(); + GET DIAGNOSTICS v_removed = ROW_COUNT; + RETURN v_removed; +END; +$$; + +CREATE FUNCTION public.vexfs_archive_import_finish(p_job bigint) +RETURNS jsonb +LANGUAGE plpgsql +SECURITY DEFINER +VOLATILE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_job _vexfs.archive_import_jobs%ROWTYPE; + v_actor oid; + v_workspace_id bigint; + v_source_head bigint; + v_source_floor bigint; + v_source_root bigint; + v_head bigint; + v_floor bigint; + v_root bigint; + v_bad bigint; + v_check jsonb; + v_versions bigint; + v_content_bytes bigint; +BEGIN + SELECT r.oid INTO STRICT v_actor + FROM pg_catalog.pg_roles AS r WHERE r.rolname = session_user; + SELECT * INTO v_job + FROM _vexfs.archive_import_jobs AS job + WHERE job.job_id = p_job + AND job.owner_oid = v_actor + AND job.backend_pid = pg_catalog.pg_backend_pid() + FOR UPDATE; + IF NOT FOUND THEN + RAISE EXCEPTION 'VEXFS_IMPORT_JOB_NOT_FOUND: %', p_job + USING ERRCODE = 'P0002'; + END IF; + IF EXISTS (SELECT 1 FROM _vexfs.workspaces AS w WHERE w.name = v_job.workspace_name) THEN + RAISE EXCEPTION 'VEXFS_WORKSPACE_EXISTS: %', v_job.workspace_name + USING ERRCODE = '23505'; + END IF; + v_source_head := (v_job.manifest->>'source_commit')::bigint; + v_source_floor := (v_job.manifest->>'history_floor_source_commit')::bigint; + v_source_root := (v_job.manifest->>'root_source_inode')::bigint; + + SELECT count(*) INTO v_bad + FROM _vexfs.archive_import_records AS record + WHERE record.job_id = p_job + AND record.record_type = 'commits'; + IF v_bad < 1 THEN + RAISE EXCEPTION 'VEXFS_INVALID_ARCHIVE: commits are missing' + USING ERRCODE = '22023'; + END IF; + WITH commits AS ( + SELECT (record.record_json->>'source_id')::bigint AS source_id, + (record.record_json->>'parent_source_id')::bigint AS parent_source_id, + lag((record.record_json->>'source_id')::bigint) OVER ( + ORDER BY (record.record_json->>'source_id')::bigint) AS expected_parent + FROM _vexfs.archive_import_records AS record + WHERE record.job_id = p_job AND record.record_type = 'commits') + SELECT count(*) INTO v_bad + FROM commits + WHERE parent_source_id IS DISTINCT FROM expected_parent; + IF v_bad <> 0 THEN + RAISE EXCEPTION 'VEXFS_INVALID_ARCHIVE: commit chain is not linear' + USING ERRCODE = '22023'; + END IF; + + DROP TABLE IF EXISTS pg_temp.vexfs_import_commit_map; + DROP TABLE IF EXISTS pg_temp.vexfs_import_inode_map; + DROP TABLE IF EXISTS pg_temp.vexfs_import_manifest_map; + DROP TABLE IF EXISTS pg_temp.vexfs_import_chunk_map; + DROP TABLE IF EXISTS pg_temp.vexfs_import_snapshot_map; + CREATE TEMP TABLE vexfs_import_commit_map( + source_id bigint PRIMARY KEY, + local_id bigint UNIQUE NOT NULL) ON COMMIT DROP; + CREATE TEMP TABLE vexfs_import_inode_map( + source_id bigint PRIMARY KEY, + local_id bigint UNIQUE NOT NULL) ON COMMIT DROP; + CREATE TEMP TABLE vexfs_import_manifest_map( + source_id bigint PRIMARY KEY, + local_id bigint UNIQUE NOT NULL) ON COMMIT DROP; + CREATE TEMP TABLE vexfs_import_chunk_map( + source_manifest bigint NOT NULL, + chunk_no integer NOT NULL, + local_id bigint UNIQUE NOT NULL, + PRIMARY KEY(source_manifest, chunk_no)) ON COMMIT DROP; + CREATE TEMP TABLE vexfs_import_snapshot_map( + name text PRIMARY KEY, + source_commit bigint NOT NULL, + local_id bigint UNIQUE NOT NULL) ON COMMIT DROP; + + INSERT INTO pg_temp.vexfs_import_commit_map(source_id, local_id) + SELECT (record.record_json->>'source_id')::bigint, + row_number() OVER (ORDER BY (record.record_json->>'source_id')::bigint)::bigint + FROM _vexfs.archive_import_records AS record + WHERE record.job_id = p_job AND record.record_type = 'commits'; + SELECT map.local_id INTO v_head + FROM pg_temp.vexfs_import_commit_map AS map WHERE map.source_id = v_source_head; + IF NOT FOUND THEN + RAISE EXCEPTION 'VEXFS_INVALID_ARCHIVE: source head is missing' + USING ERRCODE = '22023'; + END IF; + SELECT map.local_id INTO v_floor + FROM pg_temp.vexfs_import_commit_map AS map WHERE map.source_id = v_source_floor; + IF NOT FOUND THEN + RAISE EXCEPTION 'VEXFS_INVALID_ARCHIVE: history floor is missing' + USING ERRCODE = '22023'; + END IF; + + INSERT INTO _vexfs.workspaces( + name, state, owner_oid, owner_role, root_inode, head_commit, + history_floor_commit, cache_generation, quota_max_bytes, quota_max_files, + quota_max_file_bytes, retention_keep_versions, retention_keep_days, created_at) + VALUES ( + v_job.workspace_name, 'importing', v_actor, session_user, NULL, 0, 0, 0, + (v_job.manifest->>'quota_max_bytes')::bigint, + (v_job.manifest->>'quota_max_files')::bigint, + (v_job.manifest->>'quota_max_file_bytes')::bigint, + (v_job.manifest->>'retention_keep_versions')::integer, + (v_job.manifest->>'retention_keep_days')::integer, + to_timestamp((v_job.manifest->>'created_at')::bigint / 1000.0)) + RETURNING workspace_id INTO v_workspace_id; + + INSERT INTO _vexfs.commits( + workspace_id, commit_no, parent_commit, operation, + created_by_oid, created_by, created_at) + SELECT v_workspace_id, + map.local_id, + map.local_id - 1, + record.record_json->>'message', + v_actor, + session_user, + to_timestamp((record.record_json->>'created_at')::bigint / 1000.0) + FROM _vexfs.archive_import_records AS record + JOIN pg_temp.vexfs_import_commit_map AS map + ON map.source_id = (record.record_json->>'source_id')::bigint + WHERE record.job_id = p_job AND record.record_type = 'commits' + ORDER BY map.local_id; + + INSERT INTO pg_temp.vexfs_import_inode_map(source_id, local_id) + SELECT (record.record_json->>'source_id')::bigint, + nextval(pg_get_serial_sequence('_vexfs.inodes', 'inode_id')) + FROM _vexfs.archive_import_records AS record + WHERE record.job_id = p_job AND record.record_type = 'inodes' + ORDER BY (record.record_json->>'source_id')::bigint; + INSERT INTO _vexfs.inodes( + inode_id, workspace_id, kind, mode, owner_oid, owner_role, owner_principal, + uid, gid, current_version, size_bytes, live, created_at, accessed_at, + modified_at, changed_at) + OVERRIDING SYSTEM VALUE + SELECT map.local_id, + v_workspace_id, + record.record_json->>'kind', + (record.record_json->>'mode')::integer, + v_actor, + session_user, + record.record_json->>'owner_principal', + (record.record_json->>'uid')::bigint, + (record.record_json->>'gid')::bigint, + CASE WHEN record.record_json->>'kind' = 'directory' THEN 0 + ELSE (record.record_json->>'current_version')::bigint END, + (record.record_json->>'size')::bigint, + (record.record_json->>'deleted_at') IS NULL, + to_timestamp((record.record_json->>'created_at')::bigint / 1000.0), + to_timestamp((record.record_json->>'accessed_at')::bigint / 1000.0), + to_timestamp((record.record_json->>'updated_at')::bigint / 1000.0), + to_timestamp((record.record_json->>'changed_at')::bigint / 1000.0) + FROM _vexfs.archive_import_records AS record + JOIN pg_temp.vexfs_import_inode_map AS map + ON map.source_id = (record.record_json->>'source_id')::bigint + WHERE record.job_id = p_job AND record.record_type = 'inodes' + ORDER BY map.source_id; + SELECT map.local_id INTO v_root + FROM pg_temp.vexfs_import_inode_map AS map WHERE map.source_id = v_source_root; + IF NOT FOUND OR NOT EXISTS ( + SELECT 1 FROM _vexfs.inodes AS i + WHERE i.inode_id = v_root AND i.workspace_id = v_workspace_id + AND i.kind = 'directory' AND i.live) THEN + RAISE EXCEPTION 'VEXFS_INVALID_ARCHIVE: root inode is missing or invalid' + USING ERRCODE = '22023'; + END IF; + + INSERT INTO pg_temp.vexfs_import_manifest_map(source_id, local_id) + SELECT (record.record_json->>'source_id')::bigint, + nextval(pg_get_serial_sequence('_vexfs.manifests', 'manifest_id')) + FROM _vexfs.archive_import_records AS record + WHERE record.job_id = p_job AND record.record_type = 'manifests' + ORDER BY (record.record_json->>'source_id')::bigint; + INSERT INTO _vexfs.manifests( + manifest_id, workspace_id, inode_id, file_size, chunk_size, + chunk_count, checksum, created_at) + OVERRIDING SYSTEM VALUE + SELECT manifest_map.local_id, + v_workspace_id, + inode_map.local_id, + (manifest.record_json->>'file_size')::bigint, + (manifest.record_json->>'chunk_size')::integer, + (manifest.record_json->>'chunk_count')::integer, + manifest.record_json->>'checksum', + to_timestamp((manifest.record_json->>'created_at')::bigint / 1000.0) + FROM _vexfs.archive_import_records AS manifest + JOIN pg_temp.vexfs_import_manifest_map AS manifest_map + ON manifest_map.source_id = (manifest.record_json->>'source_id')::bigint + JOIN _vexfs.archive_import_records AS version + ON version.job_id = p_job AND version.record_type = 'file_versions' + AND (version.record_json->>'source_manifest')::bigint = manifest_map.source_id + AND (version.record_json->>'source_version_no') IS NULL + JOIN pg_temp.vexfs_import_inode_map AS inode_map + ON inode_map.source_id = (version.record_json->>'source_inode')::bigint + WHERE manifest.job_id = p_job AND manifest.record_type = 'manifests' + ORDER BY manifest_map.source_id; + + INSERT INTO pg_temp.vexfs_import_chunk_map( + source_manifest, chunk_no, local_id) + SELECT (record.record_json->>'source_manifest')::bigint, + (record.record_json->>'chunk_no')::integer, + nextval(pg_get_serial_sequence('_vexfs.chunks', 'chunk_id')) + FROM _vexfs.archive_import_records AS record + WHERE record.job_id = p_job AND record.record_type = 'chunks' + ORDER BY (record.record_json->>'source_manifest')::bigint, + (record.record_json->>'chunk_no')::integer; + INSERT INTO _vexfs.chunks( + chunk_id, workspace_id, inode_id, content, size_bytes, checksum) + OVERRIDING SYSTEM VALUE + SELECT chunk_map.local_id, + v_workspace_id, + inode_map.local_id, + record.content, + (record.record_json->>'size')::integer, + record.record_json->>'checksum' + FROM _vexfs.archive_import_records AS record + JOIN pg_temp.vexfs_import_chunk_map AS chunk_map + ON chunk_map.source_manifest = (record.record_json->>'source_manifest')::bigint + AND chunk_map.chunk_no = (record.record_json->>'chunk_no')::integer + JOIN _vexfs.archive_import_records AS version + ON version.job_id = p_job AND version.record_type = 'file_versions' + AND (version.record_json->>'source_manifest')::bigint = chunk_map.source_manifest + AND (version.record_json->>'source_version_no') IS NULL + JOIN pg_temp.vexfs_import_inode_map AS inode_map + ON inode_map.source_id = (version.record_json->>'source_inode')::bigint + WHERE record.job_id = p_job AND record.record_type = 'chunks' + ORDER BY chunk_map.source_manifest, chunk_map.chunk_no; + INSERT INTO _vexfs.manifest_chunks(manifest_id, chunk_no, chunk_id) + SELECT manifest_map.local_id, chunk_map.chunk_no, chunk_map.local_id + FROM pg_temp.vexfs_import_chunk_map AS chunk_map + JOIN pg_temp.vexfs_import_manifest_map AS manifest_map + ON manifest_map.source_id = chunk_map.source_manifest; + + INSERT INTO _vexfs.file_versions( + workspace_id, inode_id, version_no, commit_no, manifest_id, + source_version_no, size_bytes, checksum, created_by_oid, created_by, created_at) + SELECT v_workspace_id, + inode_map.local_id, + (record.record_json->>'version_no')::bigint, + commit_map.local_id, + manifest_map.local_id, + (record.record_json->>'source_version_no')::bigint, + (record.record_json->>'size')::bigint, + record.record_json->>'checksum', + v_actor, + session_user, + to_timestamp((record.record_json->>'created_at')::bigint / 1000.0) + FROM _vexfs.archive_import_records AS record + JOIN pg_temp.vexfs_import_inode_map AS inode_map + ON inode_map.source_id = (record.record_json->>'source_inode')::bigint + JOIN pg_temp.vexfs_import_commit_map AS commit_map + ON commit_map.source_id = (record.record_json->>'source_commit')::bigint + LEFT JOIN pg_temp.vexfs_import_manifest_map AS manifest_map + ON manifest_map.source_id = (record.record_json->>'source_manifest')::bigint + WHERE record.job_id = p_job AND record.record_type = 'file_versions' + ORDER BY inode_map.source_id, (record.record_json->>'version_no')::bigint; + + INSERT INTO _vexfs.dentries(workspace_id, parent_inode, name, inode_id) + SELECT v_workspace_id, parent_map.local_id, + record.record_json->>'name', child_map.local_id + FROM _vexfs.archive_import_records AS record + JOIN pg_temp.vexfs_import_inode_map AS parent_map + ON parent_map.source_id = (record.record_json->>'parent_source_inode')::bigint + JOIN pg_temp.vexfs_import_inode_map AS child_map + ON child_map.source_id = (record.record_json->>'inode_source_id')::bigint + WHERE record.job_id = p_job AND record.record_type = 'dentries'; + INSERT INTO _vexfs.xattrs(workspace_id, inode_id, name, value, updated_at) + SELECT v_workspace_id, inode_map.local_id, + record.record_json->>'name', + decode(record.record_json->>'value_hex', 'hex'), + to_timestamp((record.record_json->>'updated_at')::bigint / 1000.0) + FROM _vexfs.archive_import_records AS record + JOIN pg_temp.vexfs_import_inode_map AS inode_map + ON inode_map.source_id = (record.record_json->>'source_inode')::bigint + WHERE record.job_id = p_job AND record.record_type = 'xattrs'; + INSERT INTO _vexfs.acl_entries( + workspace_id, inode_id, principal, effect, permissions, + inherit_flags, created_at, updated_at) + SELECT v_workspace_id, inode_map.local_id, + record.record_json->>'principal_id', record.record_json->>'effect', + record.record_json->>'permissions', + (record.record_json->>'inherit_flags')::integer, + to_timestamp((record.record_json->>'created_at')::bigint / 1000.0), + to_timestamp((record.record_json->>'updated_at')::bigint / 1000.0) + FROM _vexfs.archive_import_records AS record + JOIN pg_temp.vexfs_import_inode_map AS inode_map + ON inode_map.source_id = (record.record_json->>'source_inode')::bigint + WHERE record.job_id = p_job AND record.record_type = 'acl_entries'; + + INSERT INTO pg_temp.vexfs_import_snapshot_map(name, source_commit, local_id) + SELECT record.record_json->>'name', + (record.record_json->>'source_commit')::bigint, + nextval(pg_get_serial_sequence('_vexfs.snapshots', 'snapshot_id')) + FROM _vexfs.archive_import_records AS record + WHERE record.job_id = p_job AND record.record_type = 'snapshots' + ORDER BY (record.record_json->>'source_commit')::bigint, + record.record_json->>'name'; + INSERT INTO _vexfs.snapshots( + snapshot_id, workspace_id, name, head_commit, + created_by_oid, created_by, created_at) + OVERRIDING SYSTEM VALUE + SELECT snapshot_map.local_id, + v_workspace_id, + snapshot_map.name, + commit_map.local_id, + v_actor, + session_user, + to_timestamp((record.record_json->>'created_at')::bigint / 1000.0) + FROM pg_temp.vexfs_import_snapshot_map AS snapshot_map + JOIN pg_temp.vexfs_import_commit_map AS commit_map + ON commit_map.source_id = snapshot_map.source_commit + JOIN _vexfs.archive_import_records AS record + ON record.job_id = p_job AND record.record_type = 'snapshots' + AND record.record_json->>'name' = snapshot_map.name; + + INSERT INTO _vexfs.snapshot_inodes( + snapshot_id, inode_id, kind, mode, owner_oid, owner_role, owner_principal, + uid, gid, current_version, size_bytes, created_at, accessed_at, + modified_at, changed_at) + SELECT snapshot_map.local_id, + inode_map.local_id, + state.kind, + state.mode, + v_actor, + session_user, + state.owner_principal, + state.uid, + state.gid, + CASE WHEN state.kind = 'directory' THEN 0 + ELSE state.current_version END, + state.size_bytes, + to_timestamp(state.created_at_ms / 1000.0), + to_timestamp(state.accessed_at_ms / 1000.0), + to_timestamp(state.updated_at_ms / 1000.0), + to_timestamp(state.changed_at_ms / 1000.0) + FROM pg_temp.vexfs_import_snapshot_map AS snapshot_map + JOIN LATERAL ( + SELECT DISTINCT ON ((record.record_json->>'source_inode')::bigint) + (record.record_json->>'source_inode')::bigint AS source_inode, + record.record_json->>'kind' AS kind, + (record.record_json->>'mode')::integer AS mode, + record.record_json->>'owner_principal' AS owner_principal, + (record.record_json->>'uid')::bigint AS uid, + (record.record_json->>'gid')::bigint AS gid, + (record.record_json->>'size')::bigint AS size_bytes, + (record.record_json->>'current_version')::bigint AS current_version, + (record.record_json->>'created_at')::bigint AS created_at_ms, + (record.record_json->>'accessed_at')::bigint AS accessed_at_ms, + (record.record_json->>'updated_at')::bigint AS updated_at_ms, + (record.record_json->>'changed_at')::bigint AS changed_at_ms, + record.record_json->>'deleted_at' AS deleted_at + FROM _vexfs.archive_import_records AS record + WHERE record.job_id = p_job AND record.record_type = 'inode_states' + AND (record.record_json->>'source_commit')::bigint <= snapshot_map.source_commit + ORDER BY (record.record_json->>'source_inode')::bigint, + (record.record_json->>'source_commit')::bigint DESC) AS state + ON state.deleted_at IS NULL + JOIN pg_temp.vexfs_import_inode_map AS inode_map + ON inode_map.source_id = state.source_inode; + + INSERT INTO _vexfs.snapshot_dentries(snapshot_id, parent_inode, name, inode_id) + SELECT snapshot_map.local_id, parent_map.local_id, state.name, child_map.local_id + FROM pg_temp.vexfs_import_snapshot_map AS snapshot_map + JOIN LATERAL ( + SELECT DISTINCT ON ( + (record.record_json->>'parent_source_inode')::bigint, + record.record_json->>'name') + (record.record_json->>'parent_source_inode')::bigint AS parent_source, + record.record_json->>'name' AS name, + (record.record_json->>'inode_source_id')::bigint AS child_source, + (record.record_json->>'deleted')::integer AS deleted + FROM _vexfs.archive_import_records AS record + WHERE record.job_id = p_job AND record.record_type = 'dentry_states' + AND (record.record_json->>'source_commit')::bigint <= snapshot_map.source_commit + ORDER BY (record.record_json->>'parent_source_inode')::bigint, + record.record_json->>'name', + (record.record_json->>'source_commit')::bigint DESC) AS state + ON state.deleted = 0 + JOIN pg_temp.vexfs_import_inode_map AS parent_map + ON parent_map.source_id = state.parent_source + JOIN pg_temp.vexfs_import_inode_map AS child_map + ON child_map.source_id = state.child_source; + + INSERT INTO _vexfs.snapshot_xattrs(snapshot_id, inode_id, name, value, updated_at) + SELECT snapshot_map.local_id, inode_map.local_id, state.name, + decode(state.value_hex, 'hex'), + to_timestamp((snapshot_record.record_json->>'created_at')::bigint / 1000.0) + FROM pg_temp.vexfs_import_snapshot_map AS snapshot_map + JOIN _vexfs.archive_import_records AS snapshot_record + ON snapshot_record.job_id = p_job + AND snapshot_record.record_type = 'snapshots' + AND snapshot_record.record_json->>'name' = snapshot_map.name + JOIN LATERAL ( + SELECT DISTINCT ON ( + (record.record_json->>'source_inode')::bigint, + record.record_json->>'name') + (record.record_json->>'source_inode')::bigint AS source_inode, + record.record_json->>'name' AS name, + record.record_json->>'value_hex' AS value_hex, + (record.record_json->>'deleted')::integer AS deleted + FROM _vexfs.archive_import_records AS record + WHERE record.job_id = p_job AND record.record_type = 'xattr_states' + AND (record.record_json->>'source_commit')::bigint <= snapshot_map.source_commit + ORDER BY (record.record_json->>'source_inode')::bigint, + record.record_json->>'name', + (record.record_json->>'source_commit')::bigint DESC) AS state + ON state.deleted = 0 + JOIN pg_temp.vexfs_import_inode_map AS inode_map + ON inode_map.source_id = state.source_inode; + + INSERT INTO _vexfs.snapshot_acl_entries( + snapshot_id, inode_id, principal, effect, permissions, + inherit_flags, created_at, updated_at) + SELECT snapshot_map.local_id, inode_map.local_id, + state.principal, state.effect, state.permissions, state.inherit_flags, + to_timestamp((snapshot_record.record_json->>'created_at')::bigint / 1000.0), + to_timestamp((snapshot_record.record_json->>'created_at')::bigint / 1000.0) + FROM pg_temp.vexfs_import_snapshot_map AS snapshot_map + JOIN _vexfs.archive_import_records AS snapshot_record + ON snapshot_record.job_id = p_job + AND snapshot_record.record_type = 'snapshots' + AND snapshot_record.record_json->>'name' = snapshot_map.name + JOIN LATERAL ( + SELECT DISTINCT ON ( + (record.record_json->>'source_inode')::bigint, + record.record_json->>'principal_id', + record.record_json->>'effect') + (record.record_json->>'source_inode')::bigint AS source_inode, + record.record_json->>'principal_id' AS principal, + record.record_json->>'effect' AS effect, + record.record_json->>'permissions' AS permissions, + (record.record_json->>'inherit_flags')::integer AS inherit_flags, + (record.record_json->>'deleted')::integer AS deleted + FROM _vexfs.archive_import_records AS record + WHERE record.job_id = p_job AND record.record_type = 'acl_states' + AND (record.record_json->>'source_commit')::bigint <= snapshot_map.source_commit + ORDER BY (record.record_json->>'source_inode')::bigint, + record.record_json->>'principal_id', + record.record_json->>'effect', + (record.record_json->>'source_commit')::bigint DESC) AS state + ON state.deleted = 0 + JOIN pg_temp.vexfs_import_inode_map AS inode_map + ON inode_map.source_id = state.source_inode; + + UPDATE _vexfs.workspaces + SET root_inode = v_root, + head_commit = v_head, + history_floor_commit = v_floor, + cache_generation = 1, + state = 'active' + WHERE workspace_id = v_workspace_id; + SELECT count(*) INTO v_bad + FROM _vexfs.workspaces AS w + WHERE w.workspace_id = v_workspace_id + AND ((w.quota_max_bytes IS NOT NULL AND w.live_bytes > w.quota_max_bytes) + OR (w.quota_max_files IS NOT NULL AND w.live_files > w.quota_max_files) + OR (w.quota_max_file_bytes IS NOT NULL AND EXISTS ( + SELECT 1 FROM _vexfs.inodes AS i + WHERE i.workspace_id = w.workspace_id + AND i.live AND i.kind <> 'directory' + AND i.size_bytes > w.quota_max_file_bytes))); + IF v_bad <> 0 THEN + RAISE EXCEPTION 'VEXFS_INVALID_ARCHIVE: imported live data exceeds its quota' + USING ERRCODE = '22023'; + END IF; + v_check := public.vexfs_check(v_job.workspace_name, 1); + IF NOT coalesce((v_check->>'ok')::boolean, false) THEN + RAISE EXCEPTION 'VEXFS_INVALID_ARCHIVE: imported workspace failed deep check: %', + v_check::text USING ERRCODE = 'XX001'; + END IF; + SELECT count(*), coalesce(sum(f.size_bytes), 0) + INTO v_versions, v_content_bytes + FROM _vexfs.file_versions AS f + WHERE f.workspace_id = v_workspace_id; + PERFORM _vexfs.audit( + v_workspace_id, v_head, 'archive_import', '/', v_root, + jsonb_build_object( + 'before_version', NULL, + 'after_version', v_head, + 'source_engine', v_job.manifest->>'source_engine', + 'source_workspace', v_job.manifest->>'source_workspace', + 'source_commit', v_source_head)); + DELETE FROM _vexfs.archive_import_jobs WHERE job_id = p_job; + RETURN jsonb_build_object( + 'workspace', v_job.workspace_name, + 'source_workspace', v_job.manifest->>'source_workspace', + 'source_commit', v_source_head, + 'versions', v_versions, + 'content_bytes', v_content_bytes, + 'head_commit', v_head, + 'root_inode', v_root); +END; +$$; + +REVOKE ALL ON ALL TABLES IN SCHEMA _vexfs FROM PUBLIC; +REVOKE ALL ON ALL SEQUENCES IN SCHEMA _vexfs FROM PUBLIC; +REVOKE ALL ON ALL FUNCTIONS IN SCHEMA _vexfs FROM PUBLIC; + +COMMENT ON SCHEMA _vexfs IS + 'Private PostgreSQL storage for the VexFS adapter; use public vexfs_* functions'; +COMMENT ON FUNCTION public.vexfs_pg_adapter_version() IS + 'Returns the PostgreSQL VexFS adapter alpha implementation version'; +COMMENT ON FUNCTION public.vexfs_contract_version() IS + 'Returns the database-neutral VexFS SQL contract version implemented by this adapter'; +COMMENT ON FUNCTION public.vexfs_init() IS + 'Idempotent PostgreSQL VexFS readiness probe; CREATE EXTENSION owns schema initialization'; +COMMENT ON FUNCTION public.vexfs_workspace_create(text) IS + 'Creates a PostgreSQL-managed VexFS workspace owned by the authenticated session role'; +COMMENT ON FUNCTION public.vexfs_write(text, text, bytea) IS + 'Writes a complete file version in the caller transaction'; +COMMENT ON FUNCTION public.vexfs_read(text, text) IS + 'Reads the current file version from PostgreSQL-managed VexFS storage'; diff --git a/vexdb_sqlite/CMakeLists.txt b/vexdb_sqlite/CMakeLists.txt index 3d2f838c44..b6169d0741 100644 --- a/vexdb_sqlite/CMakeLists.txt +++ b/vexdb_sqlite/CMakeLists.txt @@ -24,6 +24,65 @@ endif() find_package(Threads REQUIRED) +# Desktop CLI/mount runtime can connect to the PostgreSQL VexFS adapter through +# normal libpq. Embedded/mobile SQLite builds stay independent of libpq. +option(VEXFS_ENABLE_POSTGRESQL "Build PostgreSQL VexFS client support when libpq is available" ON) +option(VEXFS_REQUIRE_POSTGRESQL "Fail configuration when PostgreSQL client support is unavailable" OFF) +option(VEXFS_BUNDLE_POSTGRESQL_RUNTIME "Use package-relative libpq runpaths" OFF) +if(VEXFS_ENABLE_POSTGRESQL AND NOT CMAKE_SYSTEM_NAME STREQUAL "iOS" AND NOT ANDROID) + find_package(PostgreSQL QUIET) + if(NOT PostgreSQL_FOUND) + find_program(VEXFS_PG_CONFIG pg_config + HINTS /opt/homebrew/opt/libpq/bin /usr/local/opt/libpq/bin /opt/anaconda3/bin) + if(VEXFS_PG_CONFIG) + execute_process(COMMAND "${VEXFS_PG_CONFIG}" --includedir + OUTPUT_VARIABLE _VEXFS_PG_INCLUDEDIR OUTPUT_STRIP_TRAILING_WHITESPACE) + execute_process(COMMAND "${VEXFS_PG_CONFIG}" --libdir + OUTPUT_VARIABLE _VEXFS_PG_LIBDIR OUTPUT_STRIP_TRAILING_WHITESPACE) + endif() + find_path(VEXFS_LIBPQ_INCLUDE_DIR libpq-fe.h + HINTS "${_VEXFS_PG_INCLUDEDIR}" /opt/homebrew/opt/libpq/include + /usr/local/opt/libpq/include /opt/anaconda3/include) + find_library(VEXFS_LIBPQ_LIBRARY NAMES pq libpq + HINTS "${_VEXFS_PG_LIBDIR}" /opt/homebrew/opt/libpq/lib + /usr/local/opt/libpq/lib /opt/anaconda3/lib) + if(VEXFS_LIBPQ_INCLUDE_DIR AND VEXFS_LIBPQ_LIBRARY) + set(PostgreSQL_FOUND TRUE) + set(PostgreSQL_INCLUDE_DIRS "${VEXFS_LIBPQ_INCLUDE_DIR}") + set(PostgreSQL_LIBRARIES "${VEXFS_LIBPQ_LIBRARY}") + execute_process(COMMAND "${VEXFS_PG_CONFIG}" --version + OUTPUT_VARIABLE PostgreSQL_VERSION_STRING OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_QUIET) + endif() + endif() + if(PostgreSQL_FOUND) + message(STATUS "PostgreSQL VexFS client: ${PostgreSQL_VERSION_STRING}") + set(_VEXFS_POSTGRESQL_CLIENT_LIBRARY "") + foreach(_VEXFS_POSTGRESQL_LIBRARY IN LISTS PostgreSQL_LIBRARIES) + if(EXISTS "${_VEXFS_POSTGRESQL_LIBRARY}") + set(_VEXFS_POSTGRESQL_CLIENT_LIBRARY "${_VEXFS_POSTGRESQL_LIBRARY}") + break() + endif() + endforeach() + if(NOT _VEXFS_POSTGRESQL_CLIENT_LIBRARY AND + DEFINED VEXFS_LIBPQ_LIBRARY AND EXISTS "${VEXFS_LIBPQ_LIBRARY}") + set(_VEXFS_POSTGRESQL_CLIENT_LIBRARY "${VEXFS_LIBPQ_LIBRARY}") + endif() + set(VEXFS_POSTGRESQL_CLIENT_LIBRARY "${_VEXFS_POSTGRESQL_CLIENT_LIBRARY}" + CACHE FILEPATH "Resolved libpq runtime used by VexFS" FORCE) + if(APPLE AND VEXFS_POSTGRESQL_CLIENT_LIBRARY) + file(MAKE_DIRECTORY "${CMAKE_BINARY_DIR}/pg-client/lib") + configure_file("${VEXFS_POSTGRESQL_CLIENT_LIBRARY}" + "${CMAKE_BINARY_DIR}/pg-client/lib/libpq.dylib" COPYONLY) + endif() + else() + message(STATUS "PostgreSQL VexFS client disabled: libpq development files not found") + endif() +endif() +if(VEXFS_REQUIRE_POSTGRESQL AND NOT PostgreSQL_FOUND) + message(FATAL_ERROR "PostgreSQL VexFS client support requires libpq development files") +endif() + # macOS：默认对齐真实硬件架构。两个坑：① anaconda clang 默认 target 是 x86_64； # ② 在 Rosetta(x86_64) 进程里 `uname -m` 会谎报 x86_64。都会编出与系统 arm64 # sqlite3 不匹配的库。用 sysctl hw.optional.arm64 探真实硬件（不被 Rosetta 欺骗）。 @@ -226,6 +285,11 @@ target_include_directories(vexfs_runtime PUBLIC ${SQLITE_AMALG_DIR}) target_compile_definitions(vexfs_runtime PRIVATE VEXDB_SQLITE_CORE=1) target_link_libraries(vexfs_runtime PUBLIC vexdb_lite_static) +if(PostgreSQL_FOUND) + target_compile_definitions(vexfs_runtime PRIVATE VEXFS_HAVE_LIBPQ=1) + target_include_directories(vexfs_runtime PRIVATE ${PostgreSQL_INCLUDE_DIRS}) + target_link_libraries(vexfs_runtime PUBLIC ${PostgreSQL_LIBRARIES}) +endif() set_target_properties(vexfs_runtime PROPERTIES OUTPUT_NAME "vexfs_runtime") if(NOT CMAKE_SYSTEM_NAME STREQUAL "iOS" AND NOT ANDROID) @@ -277,7 +341,17 @@ if(VEXDB_SQLITE_DESKTOP) target_compile_definitions(vexdb_cli PRIVATE VEXFS_EMBEDDED_MAIN=1 VEXDB_SQLITE_CORE=1) target_link_libraries(vexdb_cli PRIVATE vexfs_runtime sqlite3_amalg ${VEXDB_CLI_PLATFORM_LIBS} Threads::Threads ${CMAKE_DL_LIBS}) + if(PostgreSQL_FOUND) + target_compile_definitions(vexdb_cli PRIVATE VEXFS_HAVE_LIBPQ=1) + target_include_directories(vexdb_cli PRIVATE ${PostgreSQL_INCLUDE_DIRS}) + target_link_libraries(vexdb_cli PRIVATE ${PostgreSQL_LIBRARIES}) + endif() set_target_properties(vexdb_cli PROPERTIES OUTPUT_NAME "vexdb" LINKER_LANGUAGE CXX) + if(APPLE AND VEXFS_BUNDLE_POSTGRESQL_RUNTIME) + set_target_properties(vexdb_cli PROPERTIES + BUILD_WITH_INSTALL_RPATH ON + INSTALL_RPATH "@executable_path/../lib/runtime;@executable_path/../lib/vexdb-lite/runtime") + endif() if(WIN32) add_custom_command(TARGET vexdb_cli POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different @@ -338,6 +412,16 @@ if(VEXDB_SQLITE_BUILD_TESTS) target_link_libraries(vexfs_runtime_smoke PRIVATE vexfs_runtime sqlite3_amalg Threads::Threads ${CMAKE_DL_LIBS}) + if(PostgreSQL_FOUND) + add_executable(vexfs_pg_runtime_smoke + ${REPO_ROOT}/agent_files/mount/common/test/vexfs_pg_runtime_smoke.cpp) + target_include_directories(vexfs_pg_runtime_smoke PRIVATE + ${REPO_ROOT}/agent_files/mount/common/include ${PostgreSQL_INCLUDE_DIRS}) + target_link_libraries(vexfs_pg_runtime_smoke PRIVATE + vexfs_runtime sqlite3_amalg ${PostgreSQL_LIBRARIES} + Threads::Threads ${CMAKE_DL_LIBS}) + endif() + add_executable(vexfs_platform_registry_smoke ${REPO_ROOT}/agent_files/cli/test/vexfs_platform_registry_smoke.cpp ${REPO_ROOT}/agent_files/cli/vexfs_platform_registry.cpp) diff --git a/vexdb_sqlite/src/agent_files/vexfs_sqlite.cpp b/vexdb_sqlite/src/agent_files/vexfs_sqlite.cpp index c1cd787e44..cfd505fb97 100644 --- a/vexdb_sqlite/src/agent_files/vexfs_sqlite.cpp +++ b/vexdb_sqlite/src/agent_files/vexfs_sqlite.cpp @@ -93,8 +93,12 @@ class Statement { } void BindBlob(int index, const std::vector &value) { - const void *data = value.empty() ? static_cast("") : value.data(); - Check(sqlite3_bind_blob64(stmt_, index, data, value.size(), SQLITE_STATIC), + BindBlobView(index, value.empty() ? nullptr : value.data(), value.size()); + } + + void BindBlobView(int index, const void *data, size_t size) { + const void *bound = size == 0 ? static_cast("") : data; + Check(sqlite3_bind_blob64(stmt_, index, bound, size, SQLITE_STATIC), "bind blob"); } @@ -135,6 +139,21 @@ class Statement { return std::vector(data, data + bytes); } + const unsigned char *BlobData(int column) const { + return static_cast(sqlite3_column_blob(stmt_, column)); + } + + int BlobBytes(int column) const { return sqlite3_column_bytes(stmt_, column); } + + bool BlobEquals(int column, const void *data, size_t size) const { + const int bytes = BlobBytes(column); + if (bytes < 0 || static_cast(bytes) != size) return false; + if (size == 0) return true; + const unsigned char *stored = BlobData(column); + return stored != nullptr && data != nullptr && + std::memcmp(stored, data, size) == 0; + } + private: void Check(int rc, const char *action) { if (rc != SQLITE_OK) Throw(action, rc); @@ -1032,18 +1051,19 @@ WHERE entry.manifest_id=?1 ORDER BY entry.chunk_no sqlite3_int64 offset = 0; sqlite3_int64 expected_chunk = 0; while (chunks.Row()) { - const std::vector chunk = chunks.Blob(1); const sqlite3_int64 expected_size = std::min( kContentChunkBytes, storage.size - offset); + const int chunk_bytes = chunks.BlobBytes(1); + const unsigned char *chunk = chunks.BlobData(1); if (chunks.Int64(0) != expected_chunk || expected_size <= 0 || chunks.Int64(2) != expected_size || - static_cast(chunk.size()) != expected_size || + chunk_bytes != expected_size || chunk == nullptr || chunks.Text(3).size() != 64 || - vexfs::Sha256Hex(chunk.data(), chunk.size()) != chunks.Text(3)) { + vexfs::Sha256Hex(chunk, static_cast(chunk_bytes)) != chunks.Text(3)) { throw SqlError("file manifest chunk is corrupt", SQLITE_CORRUPT); } - std::copy(chunk.begin(), chunk.end(), content.begin() + offset); - file_hash.Update(chunk.data(), chunk.size()); + std::memcpy(content.data() + offset, chunk, static_cast(chunk_bytes)); + file_hash.Update(chunk, static_cast(chunk_bytes)); offset += expected_size; ++expected_chunk; } @@ -1445,10 +1465,11 @@ sqlite3_int64 StoreManifest(sqlite3 *db, sqlite3_int64 workspace_id, sqlite3_int64 inode_id, sqlite3_int64 previous_manifest, const std::vector &content, - const std::string &checksum) { + std::string *checksum_out) { const sqlite3_int64 size = static_cast(content.size()); const sqlite3_int64 chunk_count = (size + kContentChunkBytes - 1) / kContentChunkBytes; + const std::string pending_checksum(64, '0'); Statement manifest(db, R"SQL( INSERT INTO _vexfs_manifests(workspace_id,file_size,chunk_size,chunk_count,checksum) VALUES(?1,?2,?3,?4,?5) @@ -1457,7 +1478,7 @@ VALUES(?1,?2,?3,?4,?5) manifest.BindInt64(2, size); manifest.BindInt64(3, kContentChunkBytes); manifest.BindInt64(4, chunk_count); - manifest.BindText(5, checksum); + manifest.BindText(5, pending_checksum); manifest.Done(); const sqlite3_int64 manifest_id = sqlite3_last_insert_rowid(db); @@ -1476,20 +1497,21 @@ INSERT INTO _vexfs_manifest_chunks(manifest_id,chunk_no,chunk_id) VALUES(?1,?2,?3) )SQL"); std::unordered_map new_chunks; + vexfs::Sha256 file_hash; for (sqlite3_int64 chunk_no = 0, offset = 0; offset < size; ++chunk_no, offset += kContentChunkBytes) { const size_t bytes = static_cast(std::min( kContentChunkBytes, size - offset)); - std::vector data( - content.begin() + offset, content.begin() + offset + bytes); - const std::string chunk_checksum = vexfs::Sha256Hex(data.data(), data.size()); + const unsigned char *data = content.data() + offset; + file_hash.Update(data, bytes); + const std::string chunk_checksum = vexfs::Sha256Hex(data, bytes); sqlite3_int64 chunk_id = 0; if (previous_manifest > 0) { previous.BindInt64(1, previous_manifest); previous.BindInt64(2, chunk_no); previous.BindInt64(3, inode_id); if (previous.Row() && previous.Int64(2) == static_cast(bytes) && - previous.Text(3) == chunk_checksum && previous.Blob(1) == data) { + previous.Text(3) == chunk_checksum && previous.BlobEquals(1, data, bytes)) { chunk_id = previous.Int64(0); } previous.Reset(); @@ -1502,7 +1524,7 @@ VALUES(?1,?2,?3) if (chunk_id == 0) { chunk.BindInt64(1, workspace_id); chunk.BindInt64(2, inode_id); - chunk.BindBlob(3, data); + chunk.BindBlobView(3, data, bytes); chunk.BindInt64(4, static_cast(bytes)); chunk.BindText(5, chunk_checksum); chunk.Done(); @@ -1516,6 +1538,13 @@ VALUES(?1,?2,?3) entry.Done(); entry.Reset(); } + const std::string checksum = vexfs::Hex(file_hash.Finish()); + Statement finalize(db, + "UPDATE _vexfs_manifests SET checksum=?1 WHERE id=?2"); + finalize.BindText(1, checksum); + finalize.BindInt64(2, manifest_id); + finalize.Done(); + *checksum_out = checksum; return manifest_id; } @@ -1544,9 +1573,9 @@ sqlite3_int64 StoreVersion(sqlite3 *db, const Workspace &workspace, const Node & } const sqlite3_int64 commit = CreateCommit(db, workspace, message); - const std::string checksum = vexfs::Sha256Hex(content.data(), content.size()); + std::string checksum; const sqlite3_int64 manifest_id = StoreManifest( - db, workspace.id, node.id, previous_manifest, content, checksum); + db, workspace.id, node.id, previous_manifest, content, &checksum); Statement insert(db, "INSERT INTO _vexfs_file_versions" "(inode_id,version_no,commit_id,manifest_id,size,checksum) " @@ -2091,19 +2120,9 @@ sqlite3_int64 StoreManifestFromStaging(sqlite3 *db, sqlite3_int64 workspace_id, throw SqlError("staging capacity does not match BLOB", SQLITE_CORRUPT); } std::vector buffer(static_cast(kContentChunkBytes)); - std::vector chunk_checksums; - vexfs::Sha256 file_hash; - for (sqlite3_int64 offset = 0; offset < staging.logical_size; - offset += kContentChunkBytes) { - const int bytes = static_cast(std::min( - kContentChunkBytes, staging.logical_size - offset)); - rc = sqlite3_blob_read(blob.get(), buffer.data(), bytes, static_cast(offset)); - if (rc != SQLITE_OK) throw SqlError("cannot read staging BLOB", rc); - file_hash.Update(buffer.data(), static_cast(bytes)); - chunk_checksums.push_back(vexfs::Sha256Hex(buffer.data(), static_cast(bytes))); - } - const std::string checksum = vexfs::Hex(file_hash.Finish()); - const sqlite3_int64 chunk_count = static_cast(chunk_checksums.size()); + const sqlite3_int64 chunk_count = + (staging.logical_size + kContentChunkBytes - 1) / kContentChunkBytes; + const std::string pending_checksum(64, '0'); Statement manifest(db, R"SQL( INSERT INTO _vexfs_manifests(workspace_id,file_size,chunk_size,chunk_count,checksum) VALUES(?1,?2,?3,?4,?5) @@ -2112,7 +2131,7 @@ VALUES(?1,?2,?3,?4,?5) manifest.BindInt64(2, staging.logical_size); manifest.BindInt64(3, kContentChunkBytes); manifest.BindInt64(4, chunk_count); - manifest.BindText(5, checksum); + manifest.BindText(5, pending_checksum); manifest.Done(); const sqlite3_int64 manifest_id = sqlite3_last_insert_rowid(db); Statement previous(db, R"SQL( @@ -2130,22 +2149,24 @@ INSERT INTO _vexfs_manifest_chunks(manifest_id,chunk_no,chunk_id) VALUES(?1,?2,?3) )SQL"); std::unordered_map new_chunks; + vexfs::Sha256 file_hash; for (sqlite3_int64 chunk_no = 0, offset = 0; chunk_no < chunk_count; ++chunk_no, offset += kContentChunkBytes) { const int bytes = static_cast(std::min( kContentChunkBytes, staging.logical_size - offset)); rc = sqlite3_blob_read(blob.get(), buffer.data(), bytes, static_cast(offset)); - if (rc != SQLITE_OK) throw SqlError("cannot reread staging BLOB", rc); - std::vector content(buffer.begin(), buffer.begin() + bytes); - const std::string &chunk_checksum = - chunk_checksums[static_cast(chunk_no)]; + if (rc != SQLITE_OK) throw SqlError("cannot read staging BLOB", rc); + file_hash.Update(buffer.data(), static_cast(bytes)); + const std::string chunk_checksum = + vexfs::Sha256Hex(buffer.data(), static_cast(bytes)); sqlite3_int64 chunk_id = 0; if (previous_manifest > 0) { previous.BindInt64(1, previous_manifest); previous.BindInt64(2, chunk_no); previous.BindInt64(3, inode_id); if (previous.Row() && previous.Int64(2) == bytes && - previous.Text(3) == chunk_checksum && previous.Blob(1) == content) { + previous.Text(3) == chunk_checksum && + previous.BlobEquals(1, buffer.data(), static_cast(bytes))) { chunk_id = previous.Int64(0); } previous.Reset(); @@ -2158,7 +2179,7 @@ VALUES(?1,?2,?3) if (chunk_id == 0) { chunk.BindInt64(1, workspace_id); chunk.BindInt64(2, inode_id); - chunk.BindBlob(3, content); + chunk.BindBlobView(3, buffer.data(), static_cast(bytes)); chunk.BindInt64(4, bytes); chunk.BindText(5, chunk_checksum); chunk.Done(); @@ -2172,6 +2193,12 @@ VALUES(?1,?2,?3) entry.Done(); entry.Reset(); } + const std::string checksum = vexfs::Hex(file_hash.Finish()); + Statement finalize(db, + "UPDATE _vexfs_manifests SET checksum=?1 WHERE id=?2"); + finalize.BindText(1, checksum); + finalize.BindInt64(2, manifest_id); + finalize.Done(); *checksum_out = checksum; return manifest_id; } @@ -2269,7 +2296,9 @@ sqlite3_int64 StoreVersionFromStaging(sqlite3 *db, const Workspace &workspace, const Node &node, const Handle &handle, sqlite3_int64 generation, const std::string &message) { - Savepoint savepoint(db, "vexfs_store_staged_version"); + // PublishHandleInTransaction always owns the surrounding savepoint. Keeping + // another savepoint here added two SQL statements to every small-file close + // without adding another rollback boundary. AcquireWriteLock(db); const StagingInfo staging = FindStaging(db, handle); if (staging.generation != generation) { @@ -2309,11 +2338,11 @@ VALUES(?1,?2,?3,?4,?5,?6) insert.BindText(6, checksum); insert.Done(); UpdateGrepIndexFromVersion(db, workspace.id, node.id, version); - savepoint.Release(); return version; } -sqlite3_int64 PublishHandle(sqlite3 *db, Handle handle, sqlite3_int64 generation) { +sqlite3_int64 PublishHandleInTransaction(sqlite3 *db, Handle handle, + sqlite3_int64 generation) { if (handle.state == "closed") throw SqlError("handle is closed", SQLITE_NOTFOUND); if (generation <= handle.published_generation) { return NodeById(db, handle.inode_id, true).version; @@ -2327,7 +2356,6 @@ sqlite3_int64 PublishHandle(sqlite3 *db, Handle handle, sqlite3_int64 generation throw SqlError("write conflict: file changed after handle_open", SQLITE_CONSTRAINT); } Workspace workspace = WorkspaceById(db, handle.workspace_id); - Savepoint savepoint(db, "vexfs_publish_handle"); const sqlite3_int64 version = StoreVersionFromStaging( db, workspace, node, handle, generation, "handle publish"); @@ -2338,6 +2366,13 @@ sqlite3_int64 PublishHandle(sqlite3 *db, Handle handle, sqlite3_int64 generation update.BindInt64(2, generation); update.BindText(3, handle.id); update.Done(); + return version; +} + +sqlite3_int64 PublishHandle(sqlite3 *db, Handle handle, sqlite3_int64 generation) { + Savepoint savepoint(db, "vexfs_publish_handle"); + const sqlite3_int64 version = PublishHandleInTransaction( + db, std::move(handle), generation); savepoint.Release(); return version; } @@ -5856,7 +5891,8 @@ void HandleAppendFunction(sqlite3_context *context, int, sqlite3_value **values) CreateStagingFromVersion(db, handle, node); const sqlite3_int64 generation = StageWrite(db, handle, node.size, suffix); handle.dirty_generation = generation; - const sqlite3_int64 version = PublishHandle(db, handle, generation); + const sqlite3_int64 version = PublishHandleInTransaction( + db, std::move(handle), generation); StoreRequestInteger(db, request_id, "handle_append", fingerprint, version); savepoint.Release(); sqlite3_result_int64(context, version); @@ -5945,7 +5981,8 @@ void HandlePublishCloseFunction(sqlite3_context *context, int, sqlite3_value **v throw SqlError("durability must be none, data, or full", SQLITE_MISMATCH); } Savepoint savepoint(db, "vexfs_handle_publish_close"); - const sqlite3_int64 version = PublishHandle(db, FindHandle(db, handle_id), generation); + const sqlite3_int64 version = PublishHandleInTransaction( + db, FindHandle(db, handle_id), generation); Statement update(db, "UPDATE _vexfs_handles SET state='closed',lease_until=NULL," "updated_at=CAST(strftime('%s','now') AS INTEGER)*1000 WHERE id=?1"); From d90a8fc654fed6985d00ab18a92e2acf5d532cce Mon Sep 17 00:00:00 2001 From: Four <8122332+HungryFour@users.noreply.github.com> Date: Fri, 24 Jul 2026 15:16:30 +0800 Subject: [PATCH 09/28] Document macOS NFS prototype findings --- ...-07-16_agent-files-universal-filesystem.md | 9 +++- ...2026-07-21_vexfs-final-goal-and-roadmap.md | 35 ++++++++++++- docs/specs/2026-07-16_vexfs-product-spec.md | 49 +++++++++++++------ 3 files changed, 75 insertions(+), 18 deletions(-) diff --git a/docs/design/2026-07-16_agent-files-universal-filesystem.md b/docs/design/2026-07-16_agent-files-universal-filesystem.md index 190a15fd3c..d9e6257e39 100644 --- a/docs/design/2026-07-16_agent-files-universal-filesystem.md +++ b/docs/design/2026-07-16_agent-files-universal-filesystem.md @@ -2,7 +2,7 @@ - 日期：2026-07-16 - 分支：`feature/agent_files` -- 文档版本：1.6 +- 文档版本：1.7 - 状态：macOS FSKit 与 Linux libfuse3 已复用同一 runtime；PostgreSQL `0.4.0-alpha.1` 已完成数据库合同、format v2、role/ACL/审计、libpq HostStore、macOS/Linux 真实 mount、 跨机器 OpenCode 和逻辑/物理备份；第二台 Mac 的局域网直连只差一次系统授权确认 @@ -15,6 +15,13 @@ > 2026-07-24 默认入口变更：macOS 0.1 使用系统 NFS client + 本机用户态 gateway；FSKit > 保留为后续可选原生 adapter。已有 FSKit 章节记录已完成实现和历史证据，不再表示默认发布路径。 +同日真机传输原型已确认 macOS 26.3.1 arm64 可以由当前用户挂载只监听 localhost 的 +NFSv3 服务，无需 FSKit extension；基础 Bash、mode、symlink 和 Git commit 均通过。原型同时 +确认 NFS adapter 不能直接照搬开源 demo：macOS `com.apple.provenance` 会产生 `._*` +AppleDouble sidecar，优化镜像原型也只有约 75 个用户小文件/秒。正式 adapter 必须在 +Workspace Engine 边界内吸收 AppleDouble/xattr，并用批量写回和 metadata cache 达到性能 Gate。 +这两个问题属于 mount adapter，不允许修改数据库文件、版本、权限或备份的权威合同。 + VexFS 是由 PostgreSQL、DuckDB 或 SQLite 加载和管理的数据库扩展。三个宿主共享逻辑合同，但首个闭环固定为 macOS + SQLite，不要求第一版同时完成其他数据库和操作系统。 它给数据库增加一套分层文件管理能力： diff --git a/docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md b/docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md index c41e0edb7a..9c8dd4e546 100644 --- a/docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md +++ b/docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md @@ -4,7 +4,7 @@ - 所属产品：VexDB-Lite 的文件管理能力 VexFS - 日期：2026-07-24 - 分支：`feature/agent_files` -- 文档版本：2.9 +- 文档版本：3.0 - macOS 默认入口：本机 NFS gateway；FSKit 延后为可选原生增强 - 当前阶段：Phase 2 PostgreSQL 功能与验收已完成；最后一次数据审计已让所有变更记录 认证用户、workspace、commit、对象和前后版本，并保留 workspace 删除证据。PG 16–19、 @@ -41,6 +41,39 @@ 5. 生成不含 FSKit 授权前置的签名公证包，在干净 Mac 验证安装和系统提示； 6. 通过正确性和安装 Gate 后，才把 NFS 设为代码中的默认 backend。 +### 0.1 2026-07-24 本机 NFS 原型验证 + +已在 macOS 26.3.1 arm64 真机完成第一轮传输层验证。测试使用只监听 +`127.0.0.1:11111` 的用户态 NFSv3 服务和系统 `/sbin/mount_nfs`，没有启用或调用 +VexFS FSKit extension。 + +已确认： + +- 当前用户可以直接挂载到 `/tmp` 目录，mount 表显示 `mounted by Four`； +- `ls`、`cat`、`grep`、创建、`mkdir`、`cp`、`mv`、`rm`、`chmod` 和 symlink 通过； +- 挂载目录中完成了 `git init`、`git add` 和一次 commit； +- `mount_nfs` 支持固定 `port`/`mountport`，因此 gateway 不需要占用系统 portmapper； +- 功能原型证明“NFS 默认、FSKit 可选”可行，但尚未连接 VexFS SQLite/PG HostStore，不能写成 + VexDB NFS adapter 已完成。 + +同时发现两个发布阻塞项： + +1. NFSv3 没有 macOS named attributes。macOS 26.3.1 为新文件写入 + `com.apple.provenance` 时，会额外创建 `._文件名` AppleDouble 文件。200 个用户文件实际产生 + 200 个 sidecar；必须决定由 gateway 内部吸收、隐藏并映射为 VexFS xattr，不能让它们污染 + workspace 历史、配额、grep 和快照。 +2. 优化构建、关闭 debug 日志并使用 AgentFS 同类挂载参数后，200 个 1 字节文件耗时 + 2.66 秒，约 75 个用户文件/秒；同轮 `/tmp` APFS 为 0.03 秒。这个镜像原型不是数据库 + adapter，不能代表最终吞吐，但已经证明不能只靠更换 mount 协议自动解决小文件性能。 + +因此最近任务增加两个硬 Gate： + +- NFS eval 必须断言用户视图、数据库路径、历史、快照和配额中都没有可见 `._*` 垃圾对象， + 同时保留需要的 macOS xattr； +- 先实现批量提交、写回合并和有界 metadata cache，再做同机 1,000/10,000 文件对照。 + NFS 默认发布至少不能明显低于当前 FSKit 基线；若达不到，继续作为实验 backend，不能仅因 + 免授权就替换默认入口。 + ## 1. 这份文档解决什么问题 VexFS 已经从 SQLite SQL/CLI 发展到 macOS FSKit 和 Linux libfuse3 真实挂载。如果继续按单个功能推进，很容易出现三个问题： diff --git a/docs/specs/2026-07-16_vexfs-product-spec.md b/docs/specs/2026-07-16_vexfs-product-spec.md index d3a36c2a1c..65b69c9555 100644 --- a/docs/specs/2026-07-16_vexfs-product-spec.md +++ b/docs/specs/2026-07-16_vexfs-product-spec.md @@ -5,7 +5,7 @@ - 所属项目：VexDB-Lite - 日期：2026-07-16 - 分支：`feature/agent_files` -- 文档版本：1.8 +- 文档版本：1.9 - 状态：SQLite `0.9.0`、macOS FSKit、Linux libfuse3 和 arm64 真机交付已实现； PostgreSQL `0.4.0-alpha.1` 的数据库合同、format v2、ACL、审计、libpq HostStore、 macOS/Linux 真实 mount、双 Mac 串行 Agent 工作区和备份恢复已实现 @@ -33,6 +33,13 @@ Linux FUSE 和 Windows WinFsp 必须共用同一 Workspace Engine、mount runtim - NFS 协议版本、mount 权限提示、锁、xattr、缓存一致性、sleep/wake 和 gateway crash 必须 通过真机 Gate 后，才能宣传“安装后无缝 Bash”。 +2026-07-24 已完成不连接数据库的 localhost NFSv3 传输原型。macOS 26.3.1 arm64 当前用户可 +直接 mount/unmount，普通 Bash 文件操作、mode、symlink 和 `git init/add/commit` 已通过； +这只证明系统入口可行，不表示 VexDB NFS adapter 已实现。原型还发现 macOS +`com.apple.provenance` 会在 NFSv3 上落成每文件一个 `._*` AppleDouble sidecar；优化后的镜像 +原型创建 200 个用户文件耗时 2.66 秒，同轮 APFS 为 0.03 秒。因此 AppleDouble 内部映射和 +小文件写回合并是默认 NFS 发布 Gate，不能把“免 FSKit 授权”当作唯一选型标准。 + 后文中的 FSKit 完成状态仍是历史事实；凡是把 FSKit 写成“默认、MVP 主入口或当前发布前置”的 旧表述，均由本节替代。 @@ -231,7 +238,9 @@ VexFS 的目标是把普通文件管理放进数据库事务、权限、版本 VexFS 由三部分组成： 1. **数据库扩展**：权威状态、SQL 合同、事务、权限、版本、快照、审计和备份兼容。 -2. **mount gateway**：通过数据库公开合同把 workspace 显示为真实操作系统目录；macOS 使用 FSKit App Extension，Linux 使用 libfuse3，Windows 后续使用 WinFsp。 +2. **mount gateway**：通过数据库公开合同把 workspace 显示为真实操作系统目录；macOS + 默认使用 localhost NFS gateway，FSKit 是后续可选 adapter；Linux 使用 libfuse3，Windows + 后续使用 WinFsp。 3. **配套 CLI**：安装、连接、mount/unmount、直接文件命令、worktree、shell、exec、导入导出和运维入口。 数据库扩展是权威核心。没有 mount gateway 或 CLI 时，全部数据能力仍可通过 SQL 使用；mount gateway 崩溃或卸载不能损坏数据库中已经提交的文件。 @@ -248,16 +257,19 @@ VexFS 由三部分组成： ### 4.1 MVP Gate 目标 -- 只支持 macOS 26.0+、Apple Silicon、SQLite、单 gateway、单 mount 和单数据库 principal； -- 采用 FSKit `FSUnaryFileSystem` 和 `FSVolume`，把一个 workspace 挂载为真实目录； +- 首个默认发行只支持经真机矩阵确认的 macOS 版本和 CPU、SQLite、单 gateway、单 mount 和 + 单数据库 principal； +- 采用系统 NFS client + 只监听 loopback 的当前用户 gateway，把一个 workspace 挂载为真实目录； - 支持普通文件和目录的最小调用集，以及 O_CREAT、O_EXCL、O_TRUNC 和 O_APPEND； - 明确采用 close-to-open 一致性，不宣传完整 POSIX 强一致； -- MVP 不启用 FSKit kernel data cache，权限与正确性优先； -- 允许在 adapter 内对小型只读文件做有边界的打开期缓存，但必须覆盖并通过写打开、 +- NFS attribute/data cache 使用可测试的有界 TTL、COMMIT/fsync 和主动失效规则，权限与正确性优先； +- 允许在 gateway 内对 metadata 和小型只读文件做有边界的缓存，但必须覆盖并通过写打开、 truncate、rename、unlink、同名重建和大文件回退测试； -- FSKit `write`、`synchronize`、`closeItem` 和 `reclaimItem` 的成功含义及故障恢复规则可测试； -- `synchronize` 成功后的内容在 FSKit extension 被终止和数据库 reopen 后仍完整； -- 完成签名并公证的 macOS App、FSKit extension、SQLite 扩展和 CLI 组合包，以及版本握手、doctor 和卸载流程； +- NFS `WRITE`、`COMMIT`、close、fsync 和 unmount 的成功含义及故障恢复规则可测试； +- sync 成功后的内容在 gateway 被终止和数据库 reopen 后仍完整； +- `._*` AppleDouble 不能作为普通用户文件进入路径、历史、快照、配额和 grep；需要保留的 + macOS xattr 必须映射到统一 VexFS xattr 合同； +- 完成签名并公证的 macOS CLI、gateway 和 SQLite 扩展组合包，以及版本握手、doctor 和卸载流程； - Bash、Python 和基础 Git 流程不需要 VexFS 专用命令； - 1,000 个文件、1 GiB 当前内容和 100 MiB 单文件通过正确性测试。 @@ -1687,7 +1699,10 @@ vexdb fs --workspace agent-workspace setup \ --mount ~/VexDB/agent-workspace ``` -未传 `--db` 时使用 `~/Library/Application Support/VexDB-Lite/default.sqlite3`。`setup` 初始化 SQLite 和 workspace；传入 `--mount` 时再检查 macOS 和 FSKit、App/extension、签名和合同版本，创建私有 mount point 并请求系统挂载。用户没有启用 extension 时必须给出准确的系统设置位置，不能假装挂载成功。 +未传 `--db` 时使用 `~/Library/Application Support/VexDB-Lite/default.sqlite3`。`setup` 初始化 +SQLite 和 workspace；传入 `--mount` 时检查 gateway 签名、数据库合同、loopback 端口和系统 +NFS client，创建私有 mount point 并请求系统挂载。默认流程不检查或要求启用 FSKit extension。 +显式使用 `--backend fskit` 时，才检查 App Extension 和系统启用状态。 context 配置不能保存明文密码。认证使用数据库支持的密码文件、环境变量、系统凭证或交互提示。需要登录后自动挂载时，用户显式增加 `--persist`。 @@ -1701,11 +1716,11 @@ python scripts/generate.py git status ``` -这是真实挂载目录，不是 checkout 副本。程序通过普通文件 API 读写；FSKit -`synchronize` 映射到平台无关 sync。macOS mount 的专用 SQLite 连接使用 +这是真实挂载目录，不是 checkout 副本。程序通过普通文件 API 读写；NFS `COMMIT`、fsync、 +close 和安全 unmount 映射到平台无关 sync。macOS mount 的专用 SQLite 连接使用 `WAL + NORMAL staging + FULL publish`：普通 write 返回时内容仍未发布，sync 成功时 完整版本已经发布并达到 FULL。这个策略不修改其他业务 SQLite 连接，`doctor` 必须显示 -实际配置和风险。`closeItem/reclaimItem` 只负责句柄结束和可恢复兜底。另一个 handle +实际配置和风险。NFS handle close 和 gateway 故障恢复只负责句柄结束和可恢复兜底。另一个 handle 在重新 open 后看到已发布版本；MVP 不承诺多个 gateway 实时同步。 常用生命周期命令： @@ -1713,7 +1728,8 @@ git status - `vexfs mount ~/VexFS/agent-workspace`：挂载当前 workspace； - `vexfs mount status`：查看连接、principal、workspace 和待处理 handle； - `vexfs unmount ~/VexFS/agent-workspace`：安全卸载； -- `vexfs doctor`：检查 App、FSKit extension、签名、SQLite 扩展、连接、权限和版本。 +- `vexfs doctor`：检查 gateway、系统 NFS client、loopback 端口、挂载表、签名、SQLite + 扩展、连接、权限和版本；显式选择 FSKit 时再检查 extension。 ### 27.5 Agent 使用 @@ -1740,7 +1756,8 @@ vexfs txn run --workspace agent-workspace -- \ VexFS 承诺： -- macOS 完成一次 FSKit extension 启用和 setup 后，workspace 是可以直接 `cd` 进入的真实路径； +- macOS 完成 setup 后，一条 mount 命令启动本机 gateway 并得到可以直接 `cd` 的真实路径； +- 默认入口不要求用户进入“文件系统扩展”页面； - Bash、ls、cat、rg、sed、python、git、编译器和 apply_patch 直接可用； - 不需要为每个系统命令开发 VexFS 版本； - 每个文件操作仍由数据库统一处理事务、权限、版本、配额和审计； @@ -1748,7 +1765,7 @@ VexFS 承诺： VexFS 不承诺： -- MVP 覆盖 Linux、Windows、macOS 26.0 以下版本、普通容器和不能启用 FSKit extension 环境的真实挂载； +- 首个默认发行覆盖 Linux、Windows、普通容器或未进入 NFS 真机矩阵的 macOS 版本； - 把多条 Bash 命令自动变成一个数据库事务； - 支持设备文件、socket、FIFO、setuid 或全部 POSIX 特殊行为； - 通过 mount point 获得高于数据库 principal 的权限。 From 3a97841f937ec5dfbd9f07754b67849145c5d142 Mon Sep 17 00:00:00 2001 From: Four <8122332+HungryFour@users.noreply.github.com> Date: Fri, 24 Jul 2026 17:37:06 +0800 Subject: [PATCH 10/28] Add macOS NFS mount backend for VexFS --- .gitignore | 3 + agent_files/cli/vexfs_main.cpp | 20 +- agent_files/cli/vexfs_platform.h | 5 +- agent_files/cli/vexfs_platform_linux.cpp | 9 +- agent_files/cli/vexfs_platform_macos.cpp | 387 ++- agent_files/cli/vexfs_platform_stub.cpp | 4 +- agent_files/macos/package/README.md | 44 +- agent_files/macos/package/install.sh | 53 +- agent_files/macos/package/uninstall.sh | 3 +- ...77\347\224\250\350\257\264\346\230\216.md" | 58 +- agent_files/macos/package_preview.sh | 23 +- .../mount/macos/nfs_gateway/Cargo.lock | 345 +++ .../mount/macos/nfs_gateway/Cargo.toml | 16 + .../mount/macos/nfs_gateway/THIRD_PARTY.md | 8 + .../nfs_gateway/include/vexfs_nfs_gateway.h | 31 + .../mount/macos/nfs_gateway/src/lib.rs | 1430 ++++++++++ .../nfs_gateway/vendor/nfsserve/Cargo.toml | 47 + .../macos/nfs_gateway/vendor/nfsserve/LICENSE | 29 + .../nfs_gateway/vendor/nfsserve/README.md | 194 ++ .../nfs_gateway/vendor/nfsserve/src/config.rs | 1 + .../vendor/nfsserve/src/context.rs | 28 + .../vendor/nfsserve/src/fs_util.rs | 178 ++ .../nfs_gateway/vendor/nfsserve/src/lib.rs | 23 + .../nfs_gateway/vendor/nfsserve/src/mount.rs | 41 + .../vendor/nfsserve/src/mount_handlers.rs | 207 ++ .../nfs_gateway/vendor/nfsserve/src/nfs.rs | 560 ++++ .../vendor/nfsserve/src/nfs_handlers.rs | 2302 +++++++++++++++++ .../vendor/nfsserve/src/portmap.rs | 25 + .../vendor/nfsserve/src/portmap_handlers.rs | 90 + .../nfs_gateway/vendor/nfsserve/src/rpc.rs | 513 ++++ .../vendor/nfsserve/src/rpcwire.rs | 178 ++ .../nfs_gateway/vendor/nfsserve/src/tcp.rs | 215 ++ .../nfsserve/src/transaction_tracker.rs | 57 + .../nfs_gateway/vendor/nfsserve/src/vfs.rs | 285 ++ .../vendor/nfsserve/src/write_counter.rs | 43 + .../nfs_gateway/vendor/nfsserve/src/xdr.rs | 234 ++ .../nfs_gateway/vexfs_nfs_gateway_main.cpp | 83 + ...2026-07-21_vexfs-final-goal-and-roadmap.md | 59 +- docs/specs/2026-07-16_vexfs-product-spec.md | 37 +- tests/eval/vexfs/README.md | 24 +- tests/eval/vexfs/documentation_smoke.sh | 42 +- tests/eval/vexfs/run.py | 72 +- tests/eval/vexfs/run_macos_nfs_mount.sh | 221 ++ tests/eval/vexfs/run_macos_package_smoke.sh | 2 +- vexdb_sqlite/CMakeLists.txt | 63 + 45 files changed, 8080 insertions(+), 212 deletions(-) create mode 100644 agent_files/mount/macos/nfs_gateway/Cargo.lock create mode 100644 agent_files/mount/macos/nfs_gateway/Cargo.toml create mode 100644 agent_files/mount/macos/nfs_gateway/THIRD_PARTY.md create mode 100644 agent_files/mount/macos/nfs_gateway/include/vexfs_nfs_gateway.h create mode 100644 agent_files/mount/macos/nfs_gateway/src/lib.rs create mode 100644 agent_files/mount/macos/nfs_gateway/vendor/nfsserve/Cargo.toml create mode 100644 agent_files/mount/macos/nfs_gateway/vendor/nfsserve/LICENSE create mode 100644 agent_files/mount/macos/nfs_gateway/vendor/nfsserve/README.md create mode 100644 agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/config.rs create mode 100644 agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/context.rs create mode 100644 agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/fs_util.rs create mode 100644 agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/lib.rs create mode 100644 agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/mount.rs create mode 100644 agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/mount_handlers.rs create mode 100644 agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/nfs.rs create mode 100644 agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/nfs_handlers.rs create mode 100644 agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/portmap.rs create mode 100644 agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/portmap_handlers.rs create mode 100644 agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/rpc.rs create mode 100644 agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/rpcwire.rs create mode 100644 agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/tcp.rs create mode 100644 agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/transaction_tracker.rs create mode 100644 agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/vfs.rs create mode 100644 agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/write_counter.rs create mode 100644 agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/xdr.rs create mode 100644 agent_files/mount/macos/nfs_gateway/vexfs_nfs_gateway_main.cpp create mode 100755 tests/eval/vexfs/run_macos_nfs_mount.sh diff --git a/.gitignore b/.gitignore index 335f0a4b5e..30f5f8dec7 100644 --- a/.gitignore +++ b/.gitignore @@ -381,6 +381,9 @@ __pycache__/ *.egg-info/ *.whl +# Rust build artifacts +agent_files/mount/macos/nfs_gateway/target/ + # DuckDB runtime / built extensions *.duckdb *.duckdb.wal diff --git a/agent_files/cli/vexfs_main.cpp b/agent_files/cli/vexfs_main.cpp index 835e9a0b22..f552ed3077 100644 --- a/agent_files/cli/vexfs_main.cpp +++ b/agent_files/cli/vexfs_main.cpp @@ -28,6 +28,7 @@ struct Options { std::string dsn; std::string workspace = "default"; std::string mount_point; + std::string mount_driver; bool json = false; std::vector arguments; }; @@ -37,7 +38,7 @@ std::string g_program_name = "vexfs"; void Usage(std::ostream &output) { output << "Usage: " << g_program_name << " [--db PATH | --backend pg --dsn DSN] " - "[--workspace NAME] COMMAND [ARGS]\n" + "[--workspace NAME] [--mount-driver DRIVER] COMMAND [ARGS]\n" "\n" "Commands:\n" " setup [--mount PATH] Initialize and optionally mount a workspace\n" @@ -90,6 +91,7 @@ void Usage(std::ostream &output) { " rm [-r] PATH Remove a file or directory\n" " descriptor OUTPUT Write a portable workspace descriptor\n" " mount MOUNT_POINT Mount through the platform adapter\n" + " macOS defaults to nfs; fskit is optional\n" " mount status [MOUNT_POINT] Show active VexFS mounts\n" " unmount [--force] MOUNT_POINT\n" " Unmount; --force detaches a stale mount\n" @@ -114,6 +116,8 @@ Options ParseOptions(int argc, char **argv) { options.workspace = argv[++index]; } else if (argument == "--mount" && index + 1 < argc) { options.mount_point = argv[++index]; + } else if (argument == "--mount-driver" && index + 1 < argc) { + options.mount_driver = argv[++index]; } else if (argument == "--json") { options.json = true; } else if (argument == "--help" || argument == "-h") { @@ -694,7 +698,7 @@ std::vector WorkspaceMounts(const Options &options) { const bool postgresql = options.backend == VEXFS_RUNTIME_BACKEND_POSTGRESQL; const std::string database = postgresql ? options.dsn : NormalizedPath(options.database); std::vector matches; - for (const auto &mount : VexFSPlatformInspect().mounts) { + for (const auto &mount : VexFSPlatformInspect(options.mount_driver).mounts) { if (mount.backend != options.backend) continue; const bool same_connection = postgresql ? mount.database == database @@ -710,7 +714,7 @@ std::vector WorkspaceMounts(const Options &options) { int PrintMountStatus(const Options &options, const std::string &requested_path) { const std::string requested = requested_path.empty() ? "" : NormalizedPath(requested_path); std::vector mounts; - for (const auto &mount : VexFSPlatformInspect().mounts) { + for (const auto &mount : VexFSPlatformInspect(options.mount_driver).mounts) { if (requested.empty() || NormalizedPath(mount.target) == requested) mounts.push_back(mount); } if (options.json) { @@ -740,7 +744,7 @@ int MountWorkspace(const Options &options, const std::string &mount_point) { options.backend, options.backend == VEXFS_RUNTIME_BACKEND_POSTGRESQL ? options.dsn : options.database, - options.workspace, mount_point); + options.workspace, mount_point, options.mount_driver); } int UnmountWorkspace(const std::string &mount_point, bool force) { @@ -785,7 +789,7 @@ int64_t RestoreSnapshotAfterUnmount(Session &session, const std::string &name, } int RunDoctor(const Options &options) { - const VexFSPlatformState platform = VexFSPlatformInspect(); + const VexFSPlatformState platform = VexFSPlatformInspect(options.mount_driver); std::string database_details; std::string database_error; try { @@ -815,7 +819,7 @@ int RunDoctor(const Options &options) { << "\",\"macos_supported\":" << (platform.platform_supported ? "true" : "false") << ",\"fskit\":" - << (platform.mount_driver_available ? "true" : "false"); + << (platform.extension_state == "enabled" ? "true" : "false"); } std::cout << ",\"backend\":\"" << JsonEscape(options.backend) << "\",\"database\":"; @@ -830,7 +834,9 @@ int RunDoctor(const Options &options) { << (platform.platform_supported ? " (supported)" : " (unsupported)") << '\n' << "mount driver: " << platform.mount_driver << ' ' << (platform.mount_driver_available ? "(available)" : "(unavailable)") << '\n' - << "mount state: " << platform.extension_state << '\n' + << (platform.platform == "macos" && platform.mount_driver == "NFSv3" + ? "optional FSKit: " : "mount state: ") + << platform.extension_state << '\n' << (platform.extension_path.empty() ? std::string() : "extension path: " + platform.extension_path + "\n") << (platform.extension_path_matches ? std::string() : diff --git a/agent_files/cli/vexfs_platform.h b/agent_files/cli/vexfs_platform.h index bbc72f6d80..914f83e156 100644 --- a/agent_files/cli/vexfs_platform.h +++ b/agent_files/cli/vexfs_platform.h @@ -32,8 +32,9 @@ struct VexFSPlatformState { // Database operations use vexfs_runtime_admin.h; platform adapters stay independent. std::string VexFSPlatformDefaultDatabasePath(); void VexFSPlatformProtectDirectory(const std::filesystem::path &path); -VexFSPlatformState VexFSPlatformInspect(); +VexFSPlatformState VexFSPlatformInspect(const std::string &mount_driver = {}); int VexFSPlatformMount(const std::string &backend, const std::string &connection, const std::string &workspace, - const std::string &mount_point); + const std::string &mount_point, + const std::string &mount_driver = {}); int VexFSPlatformUnmount(const std::string &mount_point, bool force); diff --git a/agent_files/cli/vexfs_platform_linux.cpp b/agent_files/cli/vexfs_platform_linux.cpp index 2009ad5913..110ceb6652 100644 --- a/agent_files/cli/vexfs_platform_linux.cpp +++ b/agent_files/cli/vexfs_platform_linux.cpp @@ -211,7 +211,9 @@ void VexFSPlatformProtectDirectory(const std::filesystem::path &path) { if (chmod(path.c_str(), 0700) != 0) throw std::runtime_error(std::strerror(errno)); } -VexFSPlatformState VexFSPlatformInspect() { +VexFSPlatformState VexFSPlatformInspect(const std::string &mount_driver) { + if (!mount_driver.empty() && mount_driver != "fuse" && mount_driver != "libfuse3") + throw std::runtime_error("Linux mount driver must be fuse"); VexFSPlatformState state; state.platform = "linux"; state.version = KernelVersion(); @@ -234,7 +236,8 @@ VexFSPlatformState VexFSPlatformInspect() { int VexFSPlatformMount(const std::string &backend, const std::string &connection, const std::string &workspace, - const std::string &mount_point) { + const std::string &mount_point, + const std::string &mount_driver) { const std::string mounted_type = MountedFileSystemAt(mount_point); if (mounted_type == "fuse.vexfs") { const std::string requested_database = backend == VEXFS_RUNTIME_BACKEND_SQLITE @@ -251,7 +254,7 @@ int VexFSPlatformMount(const std::string &backend, const std::string &connection } if (!mounted_type.empty()) throw std::runtime_error("mount point is already used by " + mounted_type); - const VexFSPlatformState state = VexFSPlatformInspect(); + const VexFSPlatformState state = VexFSPlatformInspect(mount_driver); if (!state.mount_ready) throw std::runtime_error("VexFS libfuse3 mount is not ready: " + state.extension_state); PrepareMountPoint(mount_point); diff --git a/agent_files/cli/vexfs_platform_macos.cpp b/agent_files/cli/vexfs_platform_macos.cpp index 6680131b8d..6591d1961b 100644 --- a/agent_files/cli/vexfs_platform_macos.cpp +++ b/agent_files/cli/vexfs_platform_macos.cpp @@ -5,13 +5,18 @@ #include #include +#include +#include +#include #include +#include #include #include #include #include #include +#include #include #include #include @@ -207,16 +212,28 @@ std::filesystem::path MountRegistryDirectory() { "Library/Application Support/VexDB-Lite/mounts"; } +bool IsLoopbackNfsSource(const std::string &source) { + return source == "127.0.0.1:/" || source.rfind("127.0.0.1:", 0) == 0; +} + std::vector MountedVolumes() { struct statfs *entries = nullptr; const int count = getmntinfo(&entries, MNT_NOWAIT); std::vector mounts; for (int index = 0; index < count; ++index) { - if (std::strcmp(entries[index].f_fstypename, "vexfs") != 0) continue; + const std::string type = entries[index].f_fstypename; + const std::string source = entries[index].f_mntfromname; + if (type != "vexfs" && !(type == "nfs" && IsLoopbackNfsSource(source))) continue; mounts.push_back({entries[index].f_mntfromname, entries[index].f_mntonname, entries[index].f_fstypename}); } - return VexFSPlatformAttachMountRegistry(std::move(mounts), MountRegistryDirectory()); + mounts = VexFSPlatformAttachMountRegistry(std::move(mounts), MountRegistryDirectory()); + mounts.erase(std::remove_if(mounts.begin(), mounts.end(), [](const auto &mount) { + // A loopback NFS mount is VexFS only when the protected registry proves + // the target/source identity. Do not claim unrelated localhost exports. + return mount.type == "nfs" && mount.backend.empty(); + }), mounts.end()); + return mounts; } std::string ProductVersion() { @@ -299,6 +316,228 @@ uint64_t StableHash(const std::string &value) { return hash; } +std::string RequestedMountDriver(std::string driver) { + if (driver.empty()) return "nfs"; + for (char &character : driver) + character = static_cast(std::tolower(static_cast(character))); + if (driver == "nfs" || driver == "nfsv3") return "nfs"; + if (driver == "fskit") return "fskit"; + throw std::runtime_error("macOS mount driver must be nfs or fskit"); +} + +std::filesystem::path NfsGatewayExecutable() { + if (const char *configured = std::getenv("VEXFS_NFS_GATEWAY")) + return std::filesystem::absolute(configured).lexically_normal(); + uint32_t size = 4096; + std::vector buffer(size); + if (_NSGetExecutablePath(buffer.data(), &size) != 0) { + buffer.resize(size); + if (_NSGetExecutablePath(buffer.data(), &size) != 0) return {}; + } + return std::filesystem::path(buffer.data()).parent_path() / "vexfs-nfs-gateway"; +} + +std::filesystem::path NfsGatewayStateRoot() { + return std::filesystem::path(HomeDirectory()) / + "Library/Application Support/VexDB-Lite/nfs-gateways"; +} + +std::filesystem::path NfsGatewayStateDirectory(const std::string &mount_point) { + std::ostringstream name; + name << "mount-" << std::hex << std::setw(16) << std::setfill('0') + << StableHash(NormalizedPath(mount_point)); + return NfsGatewayStateRoot() / name.str(); +} + +struct NfsGatewayRecord { + pid_t pid = -1; + uint16_t port = 0; + std::filesystem::path executable; + std::filesystem::path resource_directory; +}; + +std::filesystem::path NfsGatewayRecordPath(const std::string &mount_point) { + return NfsGatewayStateDirectory(mount_point) / "gateway.record"; +} + +void WriteNfsGatewayRecord(const std::string &mount_point, + const NfsGatewayRecord &record) { + const std::filesystem::path directory = NfsGatewayStateDirectory(mount_point); + std::filesystem::create_directories(directory); + VexFSPlatformProtectDirectory(directory); + const std::filesystem::path destination = NfsGatewayRecordPath(mount_point); + const std::filesystem::path temporary = destination.string() + ".tmp-" + + std::to_string(getpid()); + { + std::ofstream output(temporary, std::ios::binary | std::ios::trunc); + if (!output) throw std::runtime_error("cannot write NFS gateway state"); + output << "VEXFS_NFS_GATEWAY_V1\n" << record.pid << '\n' << record.port << '\n' + << std::quoted(record.executable.string()) << '\n' + << std::quoted(record.resource_directory.string()) << '\n'; + if (!output) throw std::runtime_error("cannot finish NFS gateway state"); + } + if (chmod(temporary.c_str(), 0600) != 0) { + std::error_code ignored; + std::filesystem::remove(temporary, ignored); + throw std::runtime_error("cannot protect NFS gateway state"); + } + std::error_code error; + std::filesystem::rename(temporary, destination, error); + if (error) { + std::filesystem::remove(temporary, error); + throw std::runtime_error("cannot publish NFS gateway state"); + } +} + +bool ReadNfsGatewayRecord(const std::string &mount_point, NfsGatewayRecord *record) { + std::ifstream input(NfsGatewayRecordPath(mount_point), std::ios::binary); + std::string version; + long long pid = -1; + unsigned int port = 0; + std::string executable; + std::string resource; + if (!std::getline(input, version) || version != "VEXFS_NFS_GATEWAY_V1" || + !(input >> pid >> port >> std::quoted(executable) >> std::quoted(resource)) || + pid <= 0 || port == 0 || port > 65535) return false; + record->pid = static_cast(pid); + record->port = static_cast(port); + record->executable = executable; + record->resource_directory = resource; + return true; +} + +bool ProcessMatchesGateway(const NfsGatewayRecord &record) { + char path[PROC_PIDPATHINFO_MAXSIZE]{}; + if (proc_pidpath(record.pid, path, sizeof(path)) <= 0) return false; + return NormalizedPath(path) == NormalizedPath(record.executable.string()); +} + +uint16_t AvailableLoopbackPort() { + const int descriptor = socket(AF_INET, SOCK_STREAM, 0); + if (descriptor < 0) throw std::runtime_error("cannot allocate NFS loopback socket"); + struct SocketGuard { + int value; + ~SocketGuard() { if (value >= 0) close(value); } + } guard{descriptor}; + sockaddr_in address{}; + address.sin_family = AF_INET; + address.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + address.sin_port = 0; + if (bind(descriptor, reinterpret_cast(&address), sizeof(address)) != 0) + throw std::runtime_error("cannot reserve NFS loopback port: " + + std::string(std::strerror(errno))); + socklen_t size = sizeof(address); + if (getsockname(descriptor, reinterpret_cast(&address), &size) != 0) + throw std::runtime_error("cannot inspect NFS loopback port"); + return ntohs(address.sin_port); +} + +bool LoopbackPortReady(uint16_t port) { + const int descriptor = socket(AF_INET, SOCK_STREAM, 0); + if (descriptor < 0) return false; + sockaddr_in address{}; + address.sin_family = AF_INET; + address.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + address.sin_port = htons(port); + const bool ready = connect(descriptor, reinterpret_cast(&address), + sizeof(address)) == 0; + close(descriptor); + return ready; +} + +std::string ReadGatewayLog(const std::filesystem::path &path) { + std::ifstream input(path, std::ios::binary); + if (!input) return {}; + std::string content((std::istreambuf_iterator(input)), + std::istreambuf_iterator()); + constexpr size_t kLimit = 4096; + if (content.size() > kLimit) content.erase(0, content.size() - kLimit); + while (!content.empty() && (content.back() == '\n' || content.back() == '\r')) + content.pop_back(); + return content; +} + +NfsGatewayRecord StartNfsGateway(const std::string &backend, + const std::string &connection, + const std::string &workspace, + const std::string &mount_point, + const std::filesystem::path &resource_directory) { + const std::filesystem::path executable = NfsGatewayExecutable(); + if (executable.empty() || access(executable.c_str(), X_OK) != 0) + throw std::runtime_error("vexfs-nfs-gateway is missing next to the vexdb command"); + const uint16_t port = AvailableLoopbackPort(); + const std::filesystem::path log = resource_directory / "gateway.log"; + const pid_t child = fork(); + if (child < 0) throw std::runtime_error(std::strerror(errno)); + if (child == 0) { + setsid(); + signal(SIGHUP, SIG_IGN); + umask(0077); + const int input = open("/dev/null", O_RDONLY); + const int output = open(log.c_str(), O_WRONLY | O_CREAT | O_APPEND, 0600); + if (input >= 0) dup2(input, STDIN_FILENO); + if (output >= 0) { + dup2(output, STDOUT_FILENO); + dup2(output, STDERR_FILENO); + } + if (input > STDERR_FILENO) close(input); + if (output > STDERR_FILENO) close(output); + const std::string port_text = std::to_string(port); + const char *principal = backend == VEXFS_RUNTIME_BACKEND_SQLITE ? "local" : ""; + std::vector arguments = { + executable.string(), "--backend", backend, "--connection", connection, + "--workspace", workspace, "--principal", principal, + "--listen", "127.0.0.1", "--port", port_text, + "--timeout-ms", "30000"}; + std::vector values; + for (std::string &argument : arguments) values.push_back(argument.data()); + values.push_back(nullptr); + execv(executable.c_str(), values.data()); + _exit(127); + } + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(8); + int status = 0; + while (std::chrono::steady_clock::now() < deadline) { + if (LoopbackPortReady(port)) { + NfsGatewayRecord record{child, port, executable, resource_directory}; + try { + WriteNfsGatewayRecord(mount_point, record); + } catch (...) { + kill(child, SIGTERM); + while (waitpid(child, &status, 0) < 0 && errno == EINTR) {} + throw; + } + return record; + } + const pid_t waited = waitpid(child, &status, WNOHANG); + if (waited == child) { + const std::string details = ReadGatewayLog(log); + throw std::runtime_error(details.empty() + ? "VexFS NFS gateway exited before it became ready" : details); + } + usleep(50 * 1000); + } + kill(child, SIGTERM); + while (waitpid(child, &status, 0) < 0 && errno == EINTR) {} + throw std::runtime_error("timed out waiting for VexFS NFS gateway"); +} + +void StopNfsGateway(const std::string &mount_point) { + NfsGatewayRecord record; + if (!ReadNfsGatewayRecord(mount_point, &record) || !ProcessMatchesGateway(record)) return; + kill(record.pid, SIGTERM); + for (int attempt = 0; attempt < 40; ++attempt) { + if (kill(record.pid, 0) != 0 && errno == ESRCH) return; + usleep(50 * 1000); + } + if (ProcessMatchesGateway(record)) kill(record.pid, SIGKILL); +} + +void RemoveNfsGatewayState(const std::string &mount_point) { + std::error_code ignored; + std::filesystem::remove_all(NfsGatewayStateDirectory(mount_point), ignored); +} + bool HasInlinePostgreSQLPassword(const std::string &connection) { std::string lower = connection; for (char &character : lower) @@ -564,6 +803,50 @@ std::string StagePostgreSQLPassfile(const std::string &connection, #endif } +struct NfsGatewayResource { + std::filesystem::path directory; + std::string connection; +}; + +NfsGatewayResource PrepareNfsGatewayResource(const std::string &backend, + const std::string &connection, + const std::string &mount_point) { + StopNfsGateway(mount_point); + RemoveNfsGatewayState(mount_point); + NfsGatewayResource resource; + resource.directory = NfsGatewayStateDirectory(mount_point); + std::filesystem::create_directories(resource.directory); + VexFSPlatformProtectDirectory(resource.directory); + try { + if (backend == VEXFS_RUNTIME_BACKEND_SQLITE) { + const std::filesystem::path database = + std::filesystem::absolute(connection).lexically_normal(); + if (database.filename().empty()) + throw std::runtime_error("database path must include a file name"); + if (database.has_parent_path()) { + const bool existed = std::filesystem::exists(database.parent_path()); + std::filesystem::create_directories(database.parent_path()); + if (!existed) VexFSPlatformProtectDirectory(database.parent_path()); + } + resource.connection = database.string(); + } else if (backend == VEXFS_RUNTIME_BACKEND_POSTGRESQL) { + if (connection.empty()) throw std::runtime_error("PostgreSQL DSN is required"); + if (HasInlinePostgreSQLPassword(connection)) { + throw std::runtime_error( + "PostgreSQL mount DSN must not contain a password; use a 0600 passfile, " + "a client certificate, or a libpq service without inline secrets"); + } + resource.connection = StagePostgreSQLPassfile(connection, resource.directory); + } else { + throw std::runtime_error("unsupported VexFS backend: " + backend); + } + } catch (...) { + RemoveNfsGatewayState(mount_point); + throw; + } + return resource; +} + void RemoveStagedPostgreSQLPassfile(const std::filesystem::path &directory) { if (directory.empty()) return; std::error_code error; @@ -656,15 +939,15 @@ void VexFSPlatformProtectDirectory(const std::filesystem::path &path) { if (chmod(path.c_str(), 0700) != 0) throw std::runtime_error(std::strerror(errno)); } -VexFSPlatformState VexFSPlatformInspect() { +VexFSPlatformState VexFSPlatformInspect(const std::string &mount_driver) { RemoveStalePostgreSQLPassfiles(); + const std::string driver = RequestedMountDriver(mount_driver); VexFSPlatformState state; state.platform = "macos"; state.version = ProductVersion(); state.platform_supported = ProductVersionSupported(state.version); - state.mount_driver = "FSKit"; void *framework = dlopen("/System/Library/Frameworks/FSKit.framework/FSKit", RTLD_LAZY); - state.mount_driver_available = framework != nullptr; + const bool fskit_available = framework != nullptr; if (framework != nullptr) dlclose(framework); const ExtensionState extension = GetExtensionState(&state.extension_path); state.extension_state = ExtensionStateName(extension); @@ -672,17 +955,32 @@ VexFSPlatformState VexFSPlatformInspect() { state.extension_path_matches = !state.extension_path.empty() && NormalizedPath(state.extension_path) == NormalizedPath(expected); } - state.mount_ready = state.platform_supported && state.mount_driver_available && - extension == ExtensionState::kEnabled && state.extension_path_matches; + if (driver == "nfs") { + state.mount_driver = "NFSv3"; + const std::filesystem::path gateway = NfsGatewayExecutable(); + state.mount_driver_available = access("/sbin/mount_nfs", X_OK) == 0 && + !gateway.empty() && access(gateway.c_str(), X_OK) == 0; + // FSKit is still reported as an optional adapter, but its registration + // never blocks the default NFS path. + state.mount_ready = state.platform_supported && state.mount_driver_available; + } else { + state.mount_driver = "FSKit"; + state.mount_driver_available = fskit_available; + state.mount_ready = state.platform_supported && state.mount_driver_available && + extension == ExtensionState::kEnabled && + state.extension_path_matches; + } state.mounts = MountedVolumes(); return state; } int VexFSPlatformMount(const std::string &backend, const std::string &connection, const std::string &workspace, - const std::string &mount_point) { + const std::string &mount_point, + const std::string &mount_driver) { + const std::string driver = RequestedMountDriver(mount_driver); const std::string mounted_type = MountedFileSystemAt(mount_point); - if (mounted_type == "vexfs") { + if (mounted_type == "vexfs" || mounted_type == "nfs") { const std::string requested_database = backend == VEXFS_RUNTIME_BACKEND_SQLITE ? NormalizedPath(connection) : connection; const std::string requested_target = NormalizedPath(mount_point); @@ -698,8 +996,13 @@ int VexFSPlatformMount(const std::string &backend, const std::string &connection if (!mounted_type.empty()) { throw std::runtime_error("mount point is already used by " + mounted_type); } - const VexFSPlatformState state = VexFSPlatformInspect(); + const VexFSPlatformState state = VexFSPlatformInspect(driver); if (!state.mount_ready) { + if (driver == "nfs") { + throw std::runtime_error( + "VexFS NFS mount is not ready; verify that mount_nfs and " + "vexfs-nfs-gateway are installed next to vexdb"); + } if (state.extension_state == "service-unavailable") { throw std::runtime_error( "macOS FSKit service is unavailable; reopen System Settings or restart " @@ -709,6 +1012,56 @@ int VexFSPlatformMount(const std::string &backend, const std::string &connection "; install VexDB Lite.app and enable the VexFS file system extension in System Settings"); } PrepareMountPoint(mount_point); + if (driver == "nfs") { + const NfsGatewayResource resource = + PrepareNfsGatewayResource(backend, connection, mount_point); + UnderlyingMountPointGuard mount_point_guard(mount_point); + NfsGatewayRecord gateway; + try { + try { + gateway = StartNfsGateway(backend, resource.connection, workspace, + mount_point, resource.directory); + } catch (...) { + if (backend != VEXFS_RUNTIME_BACKEND_POSTGRESQL || + !RequestLocalNetworkAccess(connection)) throw; + gateway = StartNfsGateway(backend, resource.connection, workspace, + mount_point, resource.directory); + } + // Keep I/O bounded if the per-mount gateway exits, but allow one + // request to wait longer than the runtime's 30-second database + // busy timeout. dumbtimer makes the 10-second interval explicit. + const std::string options = + "vers=3,tcp,locallocks,soft,dumbtimer,timeo=100,retrans=4," + "deadtimeout=60,actimeo=1,port=" + + std::to_string(gateway.port) + ",mountport=" + + std::to_string(gateway.port); + const int result = RunProcess("/sbin/mount_nfs", + {"-o", options, "127.0.0.1:/", mount_point}, 30'000); + if (result != 0 || MountedFileSystemAt(mount_point) != "nfs") { + if (result == 0) + throw std::runtime_error( + "mount_nfs returned success but VexFS is not mounted"); + StopNfsGateway(mount_point); + RemoveNfsGatewayState(mount_point); + DisarmMountPointGuard(mount_point); + return result; + } + mount_point_guard.Arm(); + VexFSPlatformRememberMount(MountRegistryDirectory(), + {"127.0.0.1:/", NormalizedPath(mount_point), "nfs", backend, + backend == VEXFS_RUNTIME_BACKEND_SQLITE + ? NormalizedPath(connection) : connection, + workspace}); + return 0; + } catch (...) { + if (MountedFileSystemAt(mount_point) == "nfs") + RunProcess("/sbin/umount", {"-f", mount_point}, 5'000); + StopNfsGateway(mount_point); + RemoveNfsGatewayState(mount_point); + DisarmMountPointGuard(mount_point); + throw; + } + } const std::filesystem::path resource_directory = WriteMountResourceDescriptor(backend, connection, workspace); UnderlyingMountPointGuard mount_point_guard(mount_point); @@ -753,21 +1106,27 @@ int VexFSPlatformMount(const std::string &backend, const std::string &connection int VexFSPlatformUnmount(const std::string &mount_point, bool force) { const std::string mounted_type = MountedFileSystemAt(mount_point); if (mounted_type.empty()) { + StopNfsGateway(mount_point); + RemoveNfsGatewayState(mount_point); VexFSPlatformForgetMount(MountRegistryDirectory(), mount_point); RemoveStalePostgreSQLPassfiles(); DisarmMountPointGuard(mount_point); return 0; } - if (mounted_type != "vexfs") + if (mounted_type != "vexfs" && mounted_type != "nfs") throw std::runtime_error("mount point is not VexFS: " + mounted_type); std::filesystem::path resource_directory; const std::string normalized_target = NormalizedPath(mount_point); + bool verified_mount = false; for (const auto &mount : MountedVolumes()) { if (NormalizedPath(mount.target) == normalized_target) { resource_directory = mount.source; + verified_mount = true; break; } } + if (!verified_mount) + throw std::runtime_error("mount point identity cannot be verified as VexFS"); int result = 1; // fseventsd/Spotlight can briefly hold a newly mounted FSKit volume even // when no user process has an open file. Keep this a normal (non-force) @@ -778,6 +1137,10 @@ int VexFSPlatformUnmount(const std::string &mount_point, bool force) { force ? std::vector{"-f", mount_point} : std::vector{mount_point}, 5'000); if (MountedFileSystemAt(mount_point).empty()) { + if (mounted_type == "nfs") { + StopNfsGateway(mount_point); + RemoveNfsGatewayState(mount_point); + } VexFSPlatformForgetMount(MountRegistryDirectory(), mount_point); bool resource_still_mounted = false; for (const auto &mount : MountedVolumes()) { @@ -787,7 +1150,7 @@ int VexFSPlatformUnmount(const std::string &mount_point, bool force) { break; } } - if (!resource_still_mounted) + if (mounted_type == "vexfs" && !resource_still_mounted) RemoveStagedPostgreSQLPassfile(resource_directory); DisarmMountPointGuard(mount_point); return 0; diff --git a/agent_files/cli/vexfs_platform_stub.cpp b/agent_files/cli/vexfs_platform_stub.cpp index 857661f6d8..8dec2b81f2 100644 --- a/agent_files/cli/vexfs_platform_stub.cpp +++ b/agent_files/cli/vexfs_platform_stub.cpp @@ -77,7 +77,7 @@ void VexFSPlatformProtectDirectory(const std::filesystem::path &path) { #endif } -VexFSPlatformState VexFSPlatformInspect() { +VexFSPlatformState VexFSPlatformInspect(const std::string &) { VexFSPlatformState state; state.platform = PlatformName(); state.version = "unknown"; @@ -91,7 +91,7 @@ VexFSPlatformState VexFSPlatformInspect() { int VexFSPlatformMount(const std::string &, const std::string &, const std::string &, - const std::string &) { + const std::string &, const std::string &) { throw std::runtime_error(std::string("VexFS ") + MountDriverName() + " mount adapter is not implemented yet"); } diff --git a/agent_files/macos/package/README.md b/agent_files/macos/package/README.md index 17928b4730..a0724419e2 100644 --- a/agent_files/macos/package/README.md +++ b/agent_files/macos/package/README.md @@ -5,9 +5,10 @@ VexDB-Lite 在一个安装包中提供： -- `.payload/VexDB Lite.app`：安装程序使用的隐藏 App payload，包含 VexFS FSKit 文件系统扩展； +- `.payload/VexDB Lite.app`：可选的 VexFS FSKit 文件系统扩展宿主； - `bin/vexdb`：内置 SQLite、向量检索和文件 SQL 能力的统一命令； - `bin/vexfs`：`vexdb fs` 的兼容快捷入口； +- `bin/vexfs-nfs-gateway`：macOS 默认真实挂载使用的本机 NFS gateway； - `lib/vexdb_lite.dylib`：供现有 SQLite 程序加载的扩展； - 安装和卸载脚本。 @@ -15,7 +16,9 @@ VexDB-Lite 在一个安装包中提供： - macOS 26.0 或更高版本； - Apple Silicon 包要求 arm64 Mac； -- 只有真实挂载需要 macOS 允许 `VexDB Lite` 文件系统扩展；系统显示的是宿主 App 名，内部文件系统名才是 VexFS。 +- 默认真实挂载使用 macOS 自带的 NFS client，不要求安装驱动或打开系统扩展开关； +- FSKit 是可选 driver。只有显式使用 `--mount-driver fskit` 时才需要 macOS 允许 + `VexDB Lite` 文件系统扩展。 ## 安装 @@ -36,16 +39,14 @@ VexDB-Lite 在一个安装包中提供： - SQLite 扩展：`~/.local/lib/vexdb-lite/vexdb_lite.dylib` - 默认数据库：`~/Library/Application Support/VexDB-Lite/default.sqlite3` -安装程序会在后台启动一次 VexDB Lite App，让 macOS 立即发现新安装或刚替换的 FSKit 模块。 -它不会替用户打开扩展开关；首次安装仍需在系统设置中允许。 -App payload 放在隐藏目录中，避免解压目录和安装目录同时被 macOS 注册为两个同名 -FSKit 模块。扩展已经获准时，安装程序还会执行一次临时真实挂载；挂载失败不会假报安装成功。 +安装程序会用默认 NFS driver 执行一次临时真实挂载；只有 gateway 启动、写入、读回和 +卸载都成功才报告完成。App payload 放在隐藏目录中，避免解压目录和安装目录同时被 +macOS 注册为两个同名 FSKit 模块。FSKit 未授权不影响默认 NFS。 升级时，旧 App、CLI、SQLite 扩展和 PostgreSQL runtime 只放在隐藏事务目录中；任何一步失败 都会恢复旧版本，成功后立即删除临时备份，不会在 Applications 中留下可被系统再次识别的 `.app.disabled` 副本。如果 macOS 要求新签名版本重新授权，安装程序会明确提示，不会静默跳过。 -安装或升级前请先卸载正在使用的 VexFS、exFAT 等 FSKit 卷。安装程序会拒绝在这些卷 -仍挂载时替换扩展，然后只重启当前用户的 `fskit_agent` 来刷新模块；它不会重启 `pkd`、 -清空 LaunchServices 注册库或影响其他用户。 +安装或升级前请先卸载正在使用的 VexFS。安装程序不会重启 `pkd`、清空 LaunchServices +注册库或修改其他 App 的扩展状态。 如果 `~/.local/bin` 不在 PATH： @@ -92,8 +93,8 @@ vexdb fs archive verify workspace.vexfs 已挂载 workspace 会在恢复时自动安全卸载并挂回原目录；正常卸载失败时不会开始恢复。 `--force-unmount` 只用于用户明确接受中断打开文件的场景。 -FSKit 异常退出后，底层 mountpoint 会保持不可写，避免 Bash 把文件误写进普通本机目录。 -此时先运行 `vexdb fs unmount --force MOUNTPOINT` 清理，再重新挂载。PostgreSQL 网络中断时， +NFS gateway 异常退出后，底层 mountpoint 会保持不可写，避免 Bash 把文件误写进普通本机 +目录。此时先运行 `vexdb fs unmount --force MOUNTPOINT` 清理，再重新挂载。PostgreSQL 网络中断时， 失败的写命令不会被隐式重放；连接恢复后先读取或 quick check，再重新发出写命令。 旧命令继续可用： @@ -104,13 +105,6 @@ vexfs ls / ## 真实 Bash 挂载 -1. 打开 `~/Applications/VexDB Lite.app`； -2. 进入“系统设置 → 通用 → 登录项与扩展”，滚动到扩展区域； -3. 切换到“按类别”视图； -4. 在“文件系统扩展”这一行点右侧 ⓘ，找到 `VexDB Lite` 并打开它的开关； -5. 如果看到的是 `VexDB Lite / FSKit Modules` 只读详情，说明仍在“按 App”视图，返回上一层切换视图； -6. 运行： - ```bash vexdb fs doctor vexdb fs mount ~/VexDB @@ -119,6 +113,16 @@ ls cat hello.txt ``` +gateway 只监听 `127.0.0.1`，mount status 的来源应是 `127.0.0.1:/`。状态文件和日志位于 +`~/Library/Application Support/VexDB-Lite/nfs-gateways/`，正常卸载会清理对应目录。 + +要单独测试可选 FSKit： + +```bash +vexdb fs --mount-driver fskit doctor +vexdb fs --mount-driver fskit mount ~/VexDB-FSKit +``` + ## 签名状态 打开 `MANIFEST.txt` 查看： @@ -130,8 +134,8 @@ cat hello.txt 默认拒绝安装任何脏源码 Developer ID 包。只有真机开发验证时才可显式运行 `VEXDB_LITE_ALLOW_DIRTY_INSTALL=1 ./install.sh`；正式发布和普通安装禁止使用这个开关。 -FSKit 首次使用需要 macOS 允许该文件系统扩展。安装程序不能静默启用； -请在“按类别”视图的“文件系统扩展”中完成这一步。 +FSKit 首次使用需要 macOS 允许该文件系统扩展，安装程序不能静默启用；这只影响显式 +选择 FSKit 的用户，不影响默认 NFS 挂载。 ## 卸载 diff --git a/agent_files/macos/package/install.sh b/agent_files/macos/package/install.sh index 9c41047a5e..3bad514ebb 100755 --- a/agent_files/macos/package/install.sh +++ b/agent_files/macos/package/install.sh @@ -13,6 +13,7 @@ PACKAGE_APP="$ROOT/.payload/VexDB Lite.app" [ -d "$PACKAGE_APP" ] || { echo "安装包缺少 .payload/VexDB Lite.app" >&2; exit 1; } [ -x "$ROOT/bin/vexdb" ] || { echo "安装包缺少 bin/vexdb" >&2; exit 1; } [ -x "$ROOT/bin/vexfs" ] || { echo "安装包缺少 bin/vexfs" >&2; exit 1; } +[ -x "$ROOT/bin/vexfs-nfs-gateway" ] || { echo "安装包缺少 NFS gateway" >&2; exit 1; } [ -f "$ROOT/lib/vexdb_lite.dylib" ] || { echo "安装包缺少 SQLite 扩展" >&2; exit 1; } [ -f "$ROOT/lib/runtime/libpq.5.dylib" ] || { echo "安装包缺少 PostgreSQL runtime" >&2; exit 1; } [ -f "$ROOT/MANIFEST.txt" ] || { echo "安装包缺少 MANIFEST.txt" >&2; exit 1; } @@ -107,6 +108,7 @@ verify_component "$PACKAGE_APP" --deep PACKAGE_EXTENSION="$PACKAGE_APP/Contents/Extensions/VexFSAppEx.appex" verify_component "$PACKAGE_EXTENSION" verify_component "$ROOT/bin/vexdb" +verify_component "$ROOT/bin/vexfs-nfs-gateway" verify_component "$ROOT/lib/vexdb_lite.dylib" find "$ROOT/lib/runtime" \ "$PACKAGE_APP/Contents/Extensions/VexFSAppEx.appex/Contents/Frameworks" \ @@ -149,7 +151,7 @@ verify_installed_mount() { rm -rf "$probe_root" return 0 fi - echo "临时 FSKit 验证目录未能卸载，已保留以避免删除挂载内容：$probe_root" >&2 + echo "临时 NFS 验证目录未能卸载，已保留以避免删除挂载内容：$probe_root" >&2 return 1 } @@ -160,16 +162,14 @@ verify_installed_mount() { return 1 fi - # FSKit can briefly report the newly registered path before ExtensionKit has - # finished replacing the old helper. A real disposable mount is the only - # reliable readiness check; doctor intentionally reports registry state only. + # 真实临时挂载是验证系统 NFS client、gateway 和数据库合同的最终检查。 for attempt in 1 2; do if "$BIN_DIR/vexdb" fs --db "$probe_database" \ --workspace install-probe mount "$probe_mount" \ >/dev/null 2>"$probe_error"; then - if printf '%s\n' ready-from-fskit >"$probe_mount/ready.txt" 2>>"$probe_error"; then + if printf '%s\n' ready-from-nfs >"$probe_mount/ready.txt" 2>>"$probe_error"; then actual="$(cat "$probe_mount/ready.txt" 2>>"$probe_error" || true)" - if [ "$actual" = ready-from-fskit ]; then + if [ "$actual" = ready-from-nfs ]; then if "$BIN_DIR/vexdb" fs --db "$probe_database" \ --workspace install-probe unmount "$probe_mount" \ >/dev/null 2>>"$probe_error"; then @@ -177,7 +177,7 @@ verify_installed_mount() { return 0 fi cat "$probe_error" >&2 - echo "临时 FSKit 验证目录未能正常卸载，已保留：$probe_root" >&2 + echo "临时 NFS 验证目录未能正常卸载，已保留：$probe_root" >&2 return 1 fi fi @@ -296,6 +296,7 @@ STAGED_APP="$TRANSACTION_DIR/new.bundle" PREVIOUS_APP="" APP_REPLACED=false CLI_BACKED_UP=false +GATEWAY_BACKED_UP=false LIBRARY_BACKED_UP=false RUNTIME_BACKED_UP=false INSTALL_COMMITTED=false @@ -323,6 +324,12 @@ rollback_install() { else rm -f "$BIN_DIR/vexdb" fi + if [ "$GATEWAY_BACKED_UP" = true ]; then + cp -p "$TRANSACTION_DIR/previous-vexfs-nfs-gateway" \ + "$BIN_DIR/vexfs-nfs-gateway" >/dev/null 2>&1 || true + else + rm -f "$BIN_DIR/vexfs-nfs-gateway" + fi if [ "$LIBRARY_BACKED_UP" = true ]; then cp -p "$TRANSACTION_DIR/previous-vexdb_lite.dylib" \ "$LIB_DIR/vexdb_lite.dylib" >/dev/null 2>&1 || true @@ -348,6 +355,11 @@ if [ -e "$BIN_DIR/vexdb" ]; then cp -p "$BIN_DIR/vexdb" "$TRANSACTION_DIR/previous-vexdb" CLI_BACKED_UP=true fi +if [ -e "$BIN_DIR/vexfs-nfs-gateway" ]; then + cp -p "$BIN_DIR/vexfs-nfs-gateway" \ + "$TRANSACTION_DIR/previous-vexfs-nfs-gateway" + GATEWAY_BACKED_UP=true +fi if [ -e "$LIB_DIR/vexdb_lite.dylib" ]; then cp -p "$LIB_DIR/vexdb_lite.dylib" "$TRANSACTION_DIR/previous-vexdb_lite.dylib" LIBRARY_BACKED_UP=true @@ -387,6 +399,7 @@ if ! mv "$STAGED_APP" "$INSTALLED_APP"; then fi APP_REPLACED=true install -m 0755 "$ROOT/bin/vexdb" "$BIN_DIR/vexdb" +install -m 0755 "$ROOT/bin/vexfs-nfs-gateway" "$BIN_DIR/vexfs-nfs-gateway" ln -sfn vexdb "$BIN_DIR/vexfs" install -m 0644 "$ROOT/lib/vexdb_lite.dylib" "$LIB_DIR/vexdb_lite.dylib" rm -rf "$LIB_DIR/runtime" @@ -397,12 +410,19 @@ find "$ROOT/lib/runtime" -type f -name '*.dylib' -print0 \ done verify_component "$INSTALLED_APP" --deep verify_component "$BIN_DIR/vexdb" +verify_component "$BIN_DIR/vexfs-nfs-gateway" verify_component "$LIB_DIR/vexdb_lite.dylib" find "$LIB_DIR/runtime" -type f -name '*.dylib' -print0 \ | while IFS= read -r -d '' library; do verify_component "$library" done +if ! verify_installed_mount; then + echo "默认 NFS 挂载检查失败；安装没有通过真实文件读写验证。" >&2 + exit 1 +fi +echo "默认 NFS 真实挂载检查通过。" + if [ -z "$APP_DIR_OVERRIDE" ]; then INSTALLED_EXTENSION="$INSTALLED_APP/Contents/Extensions/VexFSAppEx.appex" if [ -x "$LSREGISTER" ]; then @@ -415,13 +435,8 @@ if [ -z "$APP_DIR_OVERRIDE" ]; then # App has been launched once. Launch it hidden so doctor does not keep # reporting `extension: missing` after a valid upgrade. /usr/bin/open -gj "$INSTALLED_APP" >/dev/null 2>&1 || true - # Registration is now stable. Refresh only fskit_agent; rotating pkd here - # can assign another extension UUID after the agent has already cached the - # previous identity and leads to ExtensionKit error 2 during the first mount. - restart_current_user_fskit_agent - sleep 1 /usr/bin/open -gj "$INSTALLED_APP" >/dev/null 2>&1 || true - sleep 2 + sleep 1 # enabled 只表示同 bundle ID 的扩展已经获准使用，不保证 FSKit 当前解析到 # 新安装的 App。用签名 CLI 读取 FSKit 的真实 module URL，避免安装成功但 @@ -429,7 +444,7 @@ if [ -z "$APP_DIR_OVERRIDE" ]; then EXTENSION_CHECK="" for ATTEMPT in 1 2 3 4 5; do EXTENSION_CHECK="$(VEXFS_EXPECTED_EXTENSION_PATH="$INSTALLED_EXTENSION" \ - "$BIN_DIR/vexdb" fs --json doctor 2>/dev/null || true)" + "$BIN_DIR/vexdb" fs --mount-driver fskit --json doctor 2>/dev/null || true)" case "$EXTENSION_CHECK" in *'"extension_path_matches":true'*) break ;; esac @@ -440,11 +455,7 @@ if [ -z "$APP_DIR_OVERRIDE" ]; then case "$EXTENSION_CHECK" in *'"extension_path_matches":true'*) if [[ "$EXTENSION_CHECK" == *'"mount_ready":true'* ]]; then - if ! verify_installed_mount; then - echo "FSKit 已注册，但真实临时挂载仍未就绪；请稍后重试安装或运行 vexdb fs doctor。" >&2 - exit 1 - fi - echo "FSKit 真实挂载检查通过。" + echo "可选 FSKit 扩展已启用。" else case "$EXTENSION_CHECK" in *'"extension":"disabled"'*) @@ -462,10 +473,9 @@ if [ -z "$APP_DIR_OVERRIDE" ]; then echo "警告：系统尚未返回 FSKit module 路径；启用扩展后请运行 vexdb fs doctor。" >&2 ;; *) - echo "FSKit 当前没有加载新安装的扩展，安装未完成：" >&2 + echo "警告：可选 FSKit 当前没有加载新安装的扩展；默认 NFS 不受影响。" >&2 echo "$EXTENSION_CHECK" >&2 echo "期望路径：$INSTALLED_EXTENSION" >&2 - exit 1 ;; esac fi @@ -490,6 +500,7 @@ trap - EXIT echo "VexDB-Lite 已安装：" echo " App: $APP_DIR/VexDB Lite.app" echo " CLI: $BIN_DIR/vexdb" +echo " NFS gateway: $BIN_DIR/vexfs-nfs-gateway" echo " 文件快捷命令: $BIN_DIR/vexfs" echo " SQLite: $LIB_DIR/vexdb_lite.dylib" echo " PostgreSQL runtime: $LIB_DIR/runtime/" diff --git a/agent_files/macos/package/uninstall.sh b/agent_files/macos/package/uninstall.sh index eb9a2db847..985d9f0f71 100755 --- a/agent_files/macos/package/uninstall.sh +++ b/agent_files/macos/package/uninstall.sh @@ -6,7 +6,8 @@ BIN_DIR="${VEXDB_LITE_BIN_DIR:-${VEXFS_BIN_DIR:-$HOME/.local/bin}}" LIB_DIR="${VEXDB_LITE_LIB_DIR:-${VEXFS_LIB_DIR:-$HOME/.local/lib/vexdb-lite}}" rm -rf "$APP_DIR/VexDB Lite.app" -rm -f "$BIN_DIR/vexdb" "$BIN_DIR/vexfs" "$LIB_DIR/vexdb_lite.dylib" +rm -f "$BIN_DIR/vexdb" "$BIN_DIR/vexfs" "$BIN_DIR/vexfs-nfs-gateway" \ + "$LIB_DIR/vexdb_lite.dylib" rm -rf "$LIB_DIR/runtime" rmdir "$LIB_DIR" 2>/dev/null || true diff --git "a/agent_files/macos/package/\344\275\277\347\224\250\350\257\264\346\230\216.md" "b/agent_files/macos/package/\344\275\277\347\224\250\350\257\264\346\230\216.md" index 8ddde2494e..595c7da245 100644 --- "a/agent_files/macos/package/\344\275\277\347\224\250\350\257\264\346\230\216.md" +++ "b/agent_files/macos/package/\344\275\277\347\224\250\350\257\264\346\230\216.md" @@ -34,20 +34,17 @@ VEXDB_LITE_ALLOW_ADHOC_INSTALL=1 ./install.sh - SQLite 扩展：`~/.local/lib/vexdb-lite/vexdb_lite.dylib` - 默认数据库：`~/Library/Application Support/VexDB-Lite/default.sqlite3` -安装程序会在后台启动一次 VexDB Lite App，让 macOS 立即发现新安装或刚替换的 FSKit 模块。 -首次安装仍需由用户在系统设置中打开文件系统扩展开关。 -安装包内的 App 位于隐藏的 `.payload` 目录，防止解压副本和正式安装副本同时被 -macOS 注册。扩展已经获准时，安装程序会执行一次临时真实挂载；只有写入、读回和 -卸载都成功，安装才会报告完成。 +安装程序会用默认 NFS driver 执行一次临时真实挂载；只有本机 gateway 启动、写入、读回 +和卸载都成功才会报告完成。默认 NFS 使用 macOS 自带 client，不要求用户安装驱动或打开 +系统扩展开关。安装包内的 App 位于隐藏的 `.payload` 目录，只用于可选 FSKit driver。 升级时，旧 App、CLI、SQLite 扩展和 PostgreSQL runtime 只保存在隐藏事务目录。任何一步 失败都会恢复旧版本；成功后立即删除临时备份，不会在 Applications 中留下仍会被 macOS 识别的 `.app.disabled` 副本。如果系统要求新签名版本重新授权，安装程序会明确提示并保留 已安装的新版本，用户启用后运行 `vexdb fs doctor` 即可继续。 -安装或升级前，请先卸载正在使用的 VexFS、exFAT 等 FSKit 文件系统。安装程序检测到 -这类卷仍挂载时会停止，不会强行中断工作区。替换扩展后，安装程序只重启当前用户的 -`fskit_agent` 来刷新模块，不会重启 `pkd`、清空 LaunchServices 注册库或影响其他用户。 +安装或升级前，请先卸载正在使用的 VexFS。安装程序不会强行中断工作区，也不会重启 +`pkd`、清空 LaunchServices 注册库或修改其他 App 的扩展状态。 如果终端提示 `command not found: vexdb`，运行： @@ -334,17 +331,8 @@ vexdb fs --db restored.sqlite3 --workspace restored import workspace.vexfs ## 7. 挂载后用普通 Bash 命令 -只有这一步需要启用 FSKit 文件系统扩展。SQLite、向量和上一节的 `vexdb fs` -命令不依赖挂载。 - -第一次使用： - -1. 打开 `~/Applications/VexDB Lite.app`； -2. 打开“系统设置 → 通用 → 登录项与扩展”，滚动到扩展区域； -3. 切换到“按类别”视图； -4. 在“文件系统扩展”这一行点右侧 ⓘ，找到 `VexDB Lite` 并打开它的开关； -5. 如果看到的是 `VexDB Lite / FSKit Modules` 只读详情，说明仍在“按 App”视图，返回上一层切换视图； -6. 回到终端检查并挂载。 +macOS 默认使用系统自带的 NFS client 和 VexDB-Lite 本机 gateway，不要求启用 FSKit。 +gateway 只监听 `127.0.0.1`，不会把 workspace 暴露到局域网。 ```bash vexdb fs doctor @@ -352,6 +340,12 @@ vexdb fs mount ~/VexDB vexdb fs mount status ``` +`doctor` 应显示 `mount driver: NFSv3 (available)`；`mount status` 的来源应是 +`127.0.0.1:/`、类型应是 `nfs`。每个挂载的 PID、端口和日志放在 +`~/Library/Application Support/VexDB-Lite/nfs-gateways/`，正常卸载会清理。 +NFS 请求最长等待约 40 秒；gateway 或数据库长时间不可用时，Bash 会收到 I/O 错误而不是 +永久卡住。恢复时先运行 `vexdb fs unmount --force 挂载目录`，再重新挂载。 + 挂载成功后，可以直接使用模型和开发者熟悉的 Bash 命令： ```bash @@ -382,9 +376,8 @@ Git 可以正常使用 `core.filemode=true` 和 `core.symlinks=true`；普通文 字符/块设备和 socket 等特殊文件仍明确不支持，不会在数据库里伪造。符号链接 target 按 POSIX 规则原样 保存，可以指向挂载目录外；VexFS 本身不是安全沙箱。 -`chown` 设置的数字 UID/GID 会原样保存在数据库和快照中。Linux 挂载会按数字 ID 显示；macOS -普通用户的 FSKit 挂载由系统强制使用 `noowners`，所以 Bash `stat` 会显示当前挂载用户。此时可用 -`vexdb fs stat PATH` 查看数据库保存的原始 UID/GID。这是显示映射，不会改写数据库里的 owner。 +`chown` 设置的数字 UID/GID 会原样保存在数据库和快照中。NFS 挂载使用当前调用者的 +UID/GID；可用 `vexdb fs stat PATH` 查看数据库保存的原始值。 卸载目录： @@ -401,14 +394,20 @@ vexdb fs unmount --force ~/VexDB 强制卸载会让现有文件句柄失效，适合进程崩溃或无法正常退出时使用。正常情况下仍应先关闭 正在访问挂载目录的程序，再执行普通卸载。 -挂载成功后，VexDB Lite 会把被挂载目录的底层目录设为 `0500`。如果 FSKit extension 意外 +挂载成功后，VexDB Lite 会把被挂载目录的底层目录设为 `0500`。如果 NFS gateway 意外 退出、挂载被系统撤销，普通用户不能把文件误写到这个本机目录；写入会明确失败。执行 `vexdb fs unmount --force MOUNTPOINT` 完成清理后，目录权限恢复为 `0700`，随后可以重新挂载。 不要手工修改该目录权限或在挂载消失时继续写入。 -macOS 要求用户授权文件系统扩展，安装程序不能静默跳过这一步。 -授权开关在“按类别”视图的“文件系统扩展”详情中，不在 `VexDB Lite` -的“按 App”只读详情中。 +FSKit 保留为可选 driver。需要测试时显式运行： + +```bash +vexdb fs --mount-driver fskit doctor +vexdb fs --mount-driver fskit mount ~/VexDB-FSKit +``` + +只有这个可选路径要求在“系统设置 → 通用 → 登录项与扩展”的“按类别”视图中允许 +`VexDB Lite` 文件系统扩展。 ## 8. 备份和迁移 @@ -476,9 +475,10 @@ SELECT vexdb_version(); ### `vexdb fs` 能用，但 `mount` 失败 -先运行 `vexdb fs doctor`。如果输出是 `extension=disabled`，进入系统设置的 -“登录项与扩展”，切换到“按类别”，然后在“文件系统扩展”详情中打开 -`VexDB Lite`。“按 App”视图里的 `FSKit Modules` 详情是只读的。 +先运行 `vexdb fs doctor`，确认默认 driver 是 `NFSv3`。再检查 +`~/Library/Application Support/VexDB-Lite/nfs-gateways/` 下对应挂载的 `gateway.log`。 +如果只有显式 FSKit 路径输出 `extension=disabled`，再进入系统设置启用扩展；它不影响 +默认 NFS。 ### 首次打开被 macOS 阻止 diff --git a/agent_files/macos/package_preview.sh b/agent_files/macos/package_preview.sh index 9c119dcb83..008f2340e9 100755 --- a/agent_files/macos/package_preview.sh +++ b/agent_files/macos/package_preview.sh @@ -281,7 +281,7 @@ echo "=== 构建 VexDB-Lite CLI 与 SQLite 扩展 ===" -DVEXFS_REQUIRE_POSTGRESQL=ON \ -DVEXFS_BUNDLE_POSTGRESQL_RUNTIME=ON \ -DVEXDB_SQLITE_BUILD_TESTS=OFF -"$CMAKE_BIN" --build "$BUILD_DIR" --target vexdb_cli vexdb_lite_loadable -j "$BUILD_JOBS" +"$CMAKE_BIN" --build "$BUILD_DIR" --target vexfs_cli vexdb_lite_loadable -j "$BUILD_JOBS" PG_CLIENT_LIBRARY="$(sed -n 's/^VEXFS_POSTGRESQL_CLIENT_LIBRARY:FILEPATH=//p' \ "$BUILD_DIR/CMakeCache.txt")" [ -f "$PG_CLIENT_LIBRARY" ] || { @@ -333,6 +333,7 @@ else fi [ -d "$APP_SOURCE" ] || { echo "没有找到 $APP_NAME" >&2; exit 1; } [ -x "$BUILD_DIR/vexdb" ] || { echo "没有找到 vexdb CLI" >&2; exit 1; } +[ -x "$BUILD_DIR/vexfs-nfs-gateway" ] || { echo "没有找到 VexFS NFS gateway" >&2; exit 1; } [ -f "$BUILD_DIR/vexdb_lite.dylib" ] || { echo "没有找到 vexdb_lite.dylib" >&2; exit 1; } for bundle in \ "$APP_SOURCE" \ @@ -352,7 +353,7 @@ for bundle in \ done rm -rf "$STAGE" -mkdir -p "$STAGE/bin" "$STAGE/lib/runtime" "$STAGE/.payload" +mkdir -p "$STAGE/bin" "$STAGE/lib/runtime" "$STAGE/licenses" "$STAGE/.payload" ditto "$APP_SOURCE" "$PAYLOAD_APP" if [ "$SIGN_MODE" = developer-id ]; then # 先复制交付 App，再撤销并删除 Xcode 的临时同 ID App。这样 LaunchServices @@ -362,6 +363,7 @@ if [ "$SIGN_MODE" = developer-id ]; then rm -rf "$ARCHIVE_PATH" "$EXPORT_DIR" "$DERIVED_DATA" fi install -m 0755 "$BUILD_DIR/vexdb" "$STAGE/bin/vexdb" +install -m 0755 "$BUILD_DIR/vexfs-nfs-gateway" "$STAGE/bin/vexfs-nfs-gateway" ln -s vexdb "$STAGE/bin/vexfs" install -m 0644 "$BUILD_DIR/vexdb_lite.dylib" "$STAGE/lib/vexdb_lite.dylib" install -m 0755 "$PACKAGE_DIR/install.sh" "$STAGE/install.sh" @@ -371,6 +373,10 @@ install -m 0644 "$PACKAGE_DIR/README.md" "$STAGE/README.md" install -m 0644 "$PACKAGE_DIR/使用说明.md" "$STAGE/使用说明.md" install -m 0644 "$PACKAGE_DIR/使用说明.md" "$GUIDE" install -m 0644 "$ROOT/LICENSE" "$STAGE/LICENSE" +install -m 0644 "$ROOT/agent_files/mount/macos/nfs_gateway/THIRD_PARTY.md" \ + "$STAGE/THIRD_PARTY.md" +install -m 0644 "$ROOT/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/LICENSE" \ + "$STAGE/licenses/nfsserve-LICENSE" EXTENSION="$PAYLOAD_APP/Contents/Extensions/VexFSAppEx.appex" EXTENSION_EXECUTABLE="$EXTENSION/Contents/MacOS/VexFSAppEx" @@ -397,6 +403,8 @@ fi # bundler removes any remaining build-machine paths from transitive libraries. bash "$SCRIPT_DIR/bundle_pg_runtime.sh" \ "$STAGE/bin/vexdb" "$STAGE/lib/runtime" "$ARCH" "$PG_RUNTIME_SEARCH_DIR" +bash "$SCRIPT_DIR/bundle_pg_runtime.sh" \ + "$STAGE/bin/vexfs-nfs-gateway" "$STAGE/lib/runtime" "$ARCH" "$PG_RUNTIME_SEARCH_DIR" bash "$SCRIPT_DIR/bundle_pg_runtime.sh" \ "$EXTENSION_EXECUTABLE" "$EXTENSION_FRAMEWORKS" "$ARCH" "$PG_RUNTIME_SEARCH_DIR" @@ -418,6 +426,7 @@ if [ "$SIGN_MODE" = developer-id ]; then codesign --force --options runtime --timestamp --sign "$SIGNING_IDENTITY" "$library" done codesign --force --options runtime --timestamp --sign "$SIGNING_IDENTITY" "$STAGE/bin/vexdb" + codesign --force --options runtime --timestamp --sign "$SIGNING_IDENTITY" "$STAGE/bin/vexfs-nfs-gateway" codesign --force --options runtime --timestamp --sign "$SIGNING_IDENTITY" "$STAGE/lib/vexdb_lite.dylib" codesign --force --options runtime --timestamp --generate-entitlement-der \ --sign "$SIGNING_IDENTITY" \ @@ -438,6 +447,7 @@ if [ "$SIGN_MODE" = developer-id ]; then verify_team_identifier "$PAYLOAD_APP" verify_team_identifier "$EXTENSION" verify_team_identifier "$STAGE/bin/vexdb" + verify_team_identifier "$STAGE/bin/vexfs-nfs-gateway" verify_team_identifier "$STAGE/lib/vexdb_lite.dylib" find "$STAGE/lib/runtime" "$EXTENSION_FRAMEWORKS" -type f -name '*.dylib' -print0 \ | while IFS= read -r -d '' library; do verify_team_identifier "$library"; done @@ -478,6 +488,7 @@ else codesign --force --sign - --timestamp=none "$library" done codesign --force --sign - --timestamp=none "$STAGE/bin/vexdb" + codesign --force --sign - --timestamp=none "$STAGE/bin/vexfs-nfs-gateway" codesign --force --sign - --timestamp=none "$STAGE/lib/vexdb_lite.dylib" codesign --force --sign - --timestamp=none --generate-entitlement-der \ --entitlements "$EXTENSION_SIGN_ENTITLEMENTS" "$EXTENSION" @@ -486,6 +497,7 @@ else fi codesign --verify --deep --strict --verbose=2 "$PAYLOAD_APP" codesign --verify --strict --verbose=2 "$STAGE/bin/vexdb" +codesign --verify --strict --verbose=2 "$STAGE/bin/vexfs-nfs-gateway" codesign --verify --strict --verbose=2 "$STAGE/lib/vexdb_lite.dylib" find "$STAGE/lib/runtime" "$EXTENSION_FRAMEWORKS" -type f -name '*.dylib' -print0 \ | while IFS= read -r -d '' library; do @@ -531,7 +543,10 @@ write_hashes() { uninstall.sh \ README.md \ LICENSE \ + THIRD_PARTY.md \ + licenses/nfsserve-LICENSE \ bin/vexdb \ + bin/vexfs-nfs-gateway \ lib/vexdb_lite.dylib \ 使用说明.md \ "$PAYLOAD_RELATIVE/Contents/MacOS/VexDB Lite" \ @@ -575,13 +590,17 @@ verify_zip() ( ditto -x -k "$ZIP" "$verify_dir" [ -d "$verify_app" ] || { echo "压缩包缺少 $PAYLOAD_RELATIVE" >&2; return 1; } [ -x "$verify_root/bin/vexdb" ] || { echo "压缩包缺少 bin/vexdb" >&2; return 1; } + [ -x "$verify_root/bin/vexfs-nfs-gateway" ] || { echo "压缩包缺少 NFS gateway" >&2; return 1; } [ -L "$verify_root/bin/vexfs" ] || { echo "压缩包中的 bin/vexfs 不是兼容链接" >&2; return 1; } [ -f "$verify_root/lib/vexdb_lite.dylib" ] || { echo "压缩包缺少 SQLite 扩展" >&2; return 1; } [ -f "$verify_root/lib/runtime/libpq.5.dylib" ] || { echo "压缩包缺少 PostgreSQL runtime" >&2; return 1; } [ -f "$verify_root/使用说明.md" ] || { echo "压缩包缺少使用说明" >&2; return 1; } + [ -f "$verify_root/THIRD_PARTY.md" ] || { echo "压缩包缺少第三方软件说明" >&2; return 1; } + [ -f "$verify_root/licenses/nfsserve-LICENSE" ] || { echo "压缩包缺少 nfsserve 许可证" >&2; return 1; } codesign --verify --deep --strict --verbose=2 "$verify_app" codesign --verify --strict --verbose=2 "$verify_root/bin/vexdb" + codesign --verify --strict --verbose=2 "$verify_root/bin/vexfs-nfs-gateway" codesign --verify --strict --verbose=2 "$verify_root/lib/vexdb_lite.dylib" find "$verify_root/lib/runtime" \ "$verify_app/Contents/Extensions/VexFSAppEx.appex/Contents/Frameworks" \ diff --git a/agent_files/mount/macos/nfs_gateway/Cargo.lock b/agent_files/mount/macos/nfs_gateway/Cargo.lock new file mode 100644 index 0000000000..c1b6b6903b --- /dev/null +++ b/agent_files/mount/macos/nfs_gateway/Cargo.lock @@ -0,0 +1,345 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "async-trait" +version = "0.1.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "filetime" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" +dependencies = [ + "cfg-if", + "libc", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "mio" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" +dependencies = [ + "libc", + "wasi", + "windows-sys", +] + +[[package]] +name = "nfsserve" +version = "0.11.0" +dependencies = [ + "anyhow", + "async-trait", + "byteorder", + "filetime", + "num-derive", + "num-traits", + "tokio", + "tracing", +] + +[[package]] +name = "num-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tokio" +version = "1.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys", +] + +[[package]] +name = "tokio-macros" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6328af13490e73a9b4694030fafd93f8c8c6a9dede33e821c3fc63eddf8042ba" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "vexfs-nfs-gateway" +version = "0.1.0" +dependencies = [ + "async-trait", + "libc", + "nfsserve", + "serde", + "serde_json", + "tokio", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/agent_files/mount/macos/nfs_gateway/Cargo.toml b/agent_files/mount/macos/nfs_gateway/Cargo.toml new file mode 100644 index 0000000000..6afaa4d7b9 --- /dev/null +++ b/agent_files/mount/macos/nfs_gateway/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "vexfs-nfs-gateway" +version = "0.1.0" +edition = "2021" +publish = false + +[lib] +crate-type = ["staticlib"] + +[dependencies] +async-trait = "0.1.91" +libc = "0.2.189" +nfsserve = { path = "vendor/nfsserve" } +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +tokio = { version = "1.53", features = ["rt-multi-thread", "macros", "net", "signal", "sync", "time"] } diff --git a/agent_files/mount/macos/nfs_gateway/THIRD_PARTY.md b/agent_files/mount/macos/nfs_gateway/THIRD_PARTY.md new file mode 100644 index 0000000000..0fe01d1a10 --- /dev/null +++ b/agent_files/mount/macos/nfs_gateway/THIRD_PARTY.md @@ -0,0 +1,8 @@ +# Third-party dependency + +The macOS NFS gateway vendors `nfsserve` 0.11.0 from + under its BSD-3-Clause license. +The local fork adds the NFSv3 LINK and COMMIT procedures required by the +VexFS mount contract. Its license is kept at `vendor/nfsserve/LICENSE`. + +Cargo.lock records the exact transitive dependency versions used by the build. diff --git a/agent_files/mount/macos/nfs_gateway/include/vexfs_nfs_gateway.h b/agent_files/mount/macos/nfs_gateway/include/vexfs_nfs_gateway.h new file mode 100644 index 0000000000..1299deba43 --- /dev/null +++ b/agent_files/mount/macos/nfs_gateway/include/vexfs_nfs_gateway.h @@ -0,0 +1,31 @@ +#ifndef VEXFS_NFS_GATEWAY_H +#define VEXFS_NFS_GATEWAY_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct vexfs_nfs_gateway_config { + uint32_t abi_version; + const char *backend; + const char *connection; + const char *workspace; + const char *principal; + const char *listen_address; + uint16_t port; + uint32_t operation_timeout_ms; +} vexfs_nfs_gateway_config; + +// Blocks until SIGINT/SIGTERM or a fatal listener error. The caller owns +// process creation and lifecycle; this function owns one runtime session. +int vexfs_nfs_gateway_run(const vexfs_nfs_gateway_config *config, + char *error_message, size_t error_message_size); + +#ifdef __cplusplus +} +#endif + +#endif // VEXFS_NFS_GATEWAY_H diff --git a/agent_files/mount/macos/nfs_gateway/src/lib.rs b/agent_files/mount/macos/nfs_gateway/src/lib.rs new file mode 100644 index 0000000000..2bd255bb5e --- /dev/null +++ b/agent_files/mount/macos/nfs_gateway/src/lib.rs @@ -0,0 +1,1430 @@ +use async_trait::async_trait; +use nfsserve::nfs::{ + fattr3, fileid3, filename3, ftype3, nfspath3, nfsstat3, nfstime3, sattr3, set_atime, set_gid3, + set_mode3, set_mtime, set_size3, set_uid3, specdata3, +}; +use nfsserve::tcp::{NFSTcp, NFSTcpListener}; +use nfsserve::vfs::{DirEntry, NFSFileSystem, ReadDirResult, VFSCapabilities}; +use serde::Deserialize; +use std::collections::HashMap; +use std::ffi::{CStr, CString}; +use std::os::raw::{c_char, c_int, c_void}; +use std::ptr; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Mutex; +use std::time::{SystemTime, UNIX_EPOCH}; + +const RUNTIME_ABI_VERSION: u32 = 1; +const RUNTIME_EXCLUSIVE_GATEWAY: u32 = 2; +const XATTR_ALWAYS_SET: c_int = 0; +const XATTR_DELETE: c_int = 3; +const TIME_ACCESS: u32 = 1; +const TIME_MODIFY: u32 = 2; +const APPLEDOUBLE_XATTR: &str = "io.vexdb.macos.appledouble"; +const SIDECAR_INODE_START: u64 = u64::MAX - 1024; + +#[repr(C)] +pub struct VexfsNfsGatewayConfig { + abi_version: u32, + backend: *const c_char, + connection: *const c_char, + workspace: *const c_char, + principal: *const c_char, + listen_address: *const c_char, + port: u16, + operation_timeout_ms: u32, +} + +#[repr(C)] +struct RuntimeConfig { + abi_version: u32, + backend: *const c_char, + connection: *const c_char, + workspace: *const c_char, + principal: *const c_char, + operation_timeout_ms: u32, + flags: u32, +} + +#[repr(C)] +struct RuntimeBytes { + data: *mut c_void, + size: u64, +} + +#[repr(C)] +struct RuntimeError { + status: c_int, + native_code: c_int, + backend: [c_char; 16], + message: [c_char; 512], +} + +enum RuntimeSessionOpaque {} +type RuntimeStatus = c_int; + +extern "C" { + fn vexfs_mount_session_open( + config: *const RuntimeConfig, + session: *mut *mut RuntimeSessionOpaque, + error: *mut RuntimeError, + ) -> RuntimeStatus; + fn vexfs_mount_session_close(session: *mut RuntimeSessionOpaque); + fn vexfs_mount_stat( + session: *mut RuntimeSessionOpaque, + path: *const c_char, + json: *mut RuntimeBytes, + error: *mut RuntimeError, + ) -> RuntimeStatus; + fn vexfs_mount_path_for_inode( + session: *mut RuntimeSessionOpaque, + inode: i64, + path: *mut RuntimeBytes, + error: *mut RuntimeError, + ) -> RuntimeStatus; + fn vexfs_mount_list( + session: *mut RuntimeSessionOpaque, + path: *const c_char, + json: *mut RuntimeBytes, + error: *mut RuntimeError, + ) -> RuntimeStatus; + fn vexfs_mount_read_file( + session: *mut RuntimeSessionOpaque, + path: *const c_char, + content: *mut RuntimeBytes, + error: *mut RuntimeError, + ) -> RuntimeStatus; + fn vexfs_mount_mkdir( + session: *mut RuntimeSessionOpaque, + path: *const c_char, + error: *mut RuntimeError, + ) -> RuntimeStatus; + fn vexfs_mount_set_mode( + session: *mut RuntimeSessionOpaque, + inode: i64, + mode: u32, + error: *mut RuntimeError, + ) -> RuntimeStatus; + fn vexfs_mount_set_times( + session: *mut RuntimeSessionOpaque, + inode: i64, + accessed_at_ms: i64, + modified_at_ms: i64, + mask: u32, + error: *mut RuntimeError, + ) -> RuntimeStatus; + fn vexfs_mount_chown( + session: *mut RuntimeSessionOpaque, + inode: i64, + uid: i64, + gid: i64, + error: *mut RuntimeError, + ) -> RuntimeStatus; + fn vexfs_mount_symlink( + session: *mut RuntimeSessionOpaque, + path: *const c_char, + target: *const c_void, + target_size: u64, + error: *mut RuntimeError, + ) -> RuntimeStatus; + fn vexfs_mount_readlink( + session: *mut RuntimeSessionOpaque, + inode: i64, + target: *mut RuntimeBytes, + error: *mut RuntimeError, + ) -> RuntimeStatus; + fn vexfs_mount_rename( + session: *mut RuntimeSessionOpaque, + source: *const c_char, + destination: *const c_char, + replace: c_int, + error: *mut RuntimeError, + ) -> RuntimeStatus; + fn vexfs_mount_link( + session: *mut RuntimeSessionOpaque, + source: *const c_char, + destination: *const c_char, + error: *mut RuntimeError, + ) -> RuntimeStatus; + fn vexfs_mount_remove( + session: *mut RuntimeSessionOpaque, + path: *const c_char, + recursive: c_int, + error: *mut RuntimeError, + ) -> RuntimeStatus; + fn vexfs_mount_xattr_get( + session: *mut RuntimeSessionOpaque, + inode: i64, + name: *const c_char, + value: *mut RuntimeBytes, + error: *mut RuntimeError, + ) -> RuntimeStatus; + fn vexfs_mount_xattr_set( + session: *mut RuntimeSessionOpaque, + inode: i64, + name: *const c_char, + value: *const c_void, + size: u64, + policy: c_int, + error: *mut RuntimeError, + ) -> RuntimeStatus; + fn vexfs_mount_handle_open( + session: *mut RuntimeSessionOpaque, + path: *const c_char, + flags: *const c_char, + request_id: *const c_char, + handle: *mut RuntimeBytes, + error: *mut RuntimeError, + ) -> RuntimeStatus; + fn vexfs_mount_handle_create( + session: *mut RuntimeSessionOpaque, + path: *const c_char, + mode: u32, + request_id: *const c_char, + handle: *mut RuntimeBytes, + error: *mut RuntimeError, + ) -> RuntimeStatus; + fn vexfs_mount_handle_stage_write( + session: *mut RuntimeSessionOpaque, + handle: *const c_char, + offset: u64, + data: *const c_void, + size: u64, + request_id: *const c_char, + generation: *mut i64, + error: *mut RuntimeError, + ) -> RuntimeStatus; + fn vexfs_mount_handle_truncate( + session: *mut RuntimeSessionOpaque, + handle: *const c_char, + size: u64, + request_id: *const c_char, + generation: *mut i64, + error: *mut RuntimeError, + ) -> RuntimeStatus; + fn vexfs_mount_handle_publish_close( + session: *mut RuntimeSessionOpaque, + handle: *const c_char, + generation: i64, + durability: *const c_char, + version: *mut i64, + error: *mut RuntimeError, + ) -> RuntimeStatus; + fn vexfs_mount_handle_close( + session: *mut RuntimeSessionOpaque, + handle: *const c_char, + retain_unpublished: c_int, + request_id: *const c_char, + state: *mut RuntimeBytes, + error: *mut RuntimeError, + ) -> RuntimeStatus; + fn vexfs_mount_synchronize( + session: *mut RuntimeSessionOpaque, + request_id: *const c_char, + published: *mut i64, + error: *mut RuntimeError, + ) -> RuntimeStatus; + fn vexfs_mount_free(memory: *mut c_void); +} + +#[derive(Clone, Deserialize)] +struct FileStat { + inode: i64, + kind: String, + mode: u32, + uid: i64, + gid: i64, + size: i64, + link_count: i64, + accessed_at: i64, + updated_at: i64, + changed_at: i64, +} + +#[derive(Clone, Deserialize)] +struct ListEntry { + name: String, + inode: i64, + kind: String, + mode: u32, + uid: i64, + gid: i64, + size: i64, + link_count: i64, + accessed_at: i64, + updated_at: i64, + changed_at: i64, +} + +impl From for FileStat { + fn from(entry: ListEntry) -> Self { + Self { + inode: entry.inode, + kind: entry.kind, + mode: entry.mode, + uid: entry.uid, + gid: entry.gid, + size: entry.size, + link_count: entry.link_count, + accessed_at: entry.accessed_at, + updated_at: entry.updated_at, + changed_at: entry.changed_at, + } + } +} + +#[derive(Clone, Hash, PartialEq, Eq)] +struct SidecarKey { + parent: fileid3, + name: Vec, +} + +struct Sidecar { + key: SidecarKey, + target_inode: Option, + data: Vec, + mode: u32, + updated_at_ms: i64, +} + +struct RuntimeState { + session: *mut RuntimeSessionOpaque, + root_inode: fileid3, + uid: u32, + gid: u32, + next_sidecar_inode: fileid3, + sidecars_by_key: HashMap, + sidecars: HashMap, +} + +unsafe impl Send for RuntimeState {} + +impl Drop for RuntimeState { + fn drop(&mut self) { + if !self.session.is_null() { + unsafe { vexfs_mount_session_close(self.session) }; + self.session = ptr::null_mut(); + } + } +} + +pub struct VexfsNfs { + state: Mutex, + request_prefix: String, + request_sequence: AtomicU64, +} + +fn empty_error() -> RuntimeError { + RuntimeError { + status: 0, + native_code: 0, + backend: [0; 16], + message: [0; 512], + } +} + +fn runtime_to_nfs(status: RuntimeStatus) -> nfsstat3 { + match status { + 1 => nfsstat3::NFS3ERR_INVAL, + 2 => nfsstat3::NFS3ERR_NOENT, + 3 => nfsstat3::NFS3ERR_EXIST, + 4 => nfsstat3::NFS3ERR_ROFS, + 5 => nfsstat3::NFS3ERR_JUKEBOX, + 8 => nfsstat3::NFS3ERR_ACCES, + 9 => nfsstat3::NFS3ERR_NOSPC, + 11 => nfsstat3::NFS3ERR_NOTSUPP, + 12 => nfsstat3::NFS3ERR_NOTEMPTY, + _ => nfsstat3::NFS3ERR_IO, + } +} + +fn checked_cstring(value: &str) -> Result { + CString::new(value).map_err(|_| nfsstat3::NFS3ERR_INVAL) +} + +fn bytes_to_string(bytes: RuntimeBytes) -> Result { + let value = unsafe { + if bytes.data.is_null() { + Vec::new() + } else { + let value = + std::slice::from_raw_parts(bytes.data.cast::(), bytes.size as usize).to_vec(); + vexfs_mount_free(bytes.data); + value + } + }; + String::from_utf8(value).map_err(|_| nfsstat3::NFS3ERR_IO) +} + +fn take_bytes(bytes: RuntimeBytes) -> Vec { + unsafe { + if bytes.data.is_null() { + Vec::new() + } else { + let value = + std::slice::from_raw_parts(bytes.data.cast::(), bytes.size as usize).to_vec(); + vexfs_mount_free(bytes.data); + value + } + } +} + +fn runtime_result(status: RuntimeStatus) -> Result<(), nfsstat3> { + if status == 0 { + Ok(()) + } else { + Err(runtime_to_nfs(status)) + } +} + +fn now_ms() -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis() + .min(i64::MAX as u128) as i64 +} + +fn nfs_time(milliseconds: i64) -> nfstime3 { + let value = milliseconds.max(0) as u64; + nfstime3 { + seconds: (value / 1000).min(u32::MAX as u64) as u32, + nseconds: ((value % 1000) * 1_000_000) as u32, + } +} + +fn stat_attr(stat: &FileStat) -> fattr3 { + let ftype = match stat.kind.as_str() { + "directory" => ftype3::NF3DIR, + "symlink" => ftype3::NF3LNK, + _ => ftype3::NF3REG, + }; + fattr3 { + ftype, + mode: stat.mode & 0o7777, + nlink: stat.link_count.max(1).min(u32::MAX as i64) as u32, + uid: stat.uid.max(0).min(u32::MAX as i64) as u32, + gid: stat.gid.max(0).min(u32::MAX as i64) as u32, + size: stat.size.max(0) as u64, + used: stat.size.max(0) as u64, + rdev: specdata3::default(), + fsid: 1, + fileid: stat.inode as u64, + atime: nfs_time(stat.accessed_at), + mtime: nfs_time(stat.updated_at), + ctime: nfs_time(stat.changed_at), + } +} + +impl RuntimeState { + fn stat_path(&mut self, path: &str) -> Result { + let path = checked_cstring(path)?; + let mut bytes = RuntimeBytes { + data: ptr::null_mut(), + size: 0, + }; + let mut error = empty_error(); + let status = + unsafe { vexfs_mount_stat(self.session, path.as_ptr(), &mut bytes, &mut error) }; + runtime_result(status)?; + serde_json::from_str(&bytes_to_string(bytes)?).map_err(|_| nfsstat3::NFS3ERR_IO) + } + + fn path_for_inode(&mut self, inode: fileid3) -> Result { + if self.sidecars.contains_key(&inode) { + return Err(nfsstat3::NFS3ERR_INVAL); + } + let mut bytes = RuntimeBytes { + data: ptr::null_mut(), + size: 0, + }; + let mut error = empty_error(); + let status = unsafe { + vexfs_mount_path_for_inode(self.session, inode as i64, &mut bytes, &mut error) + }; + runtime_result(status)?; + bytes_to_string(bytes) + } + + fn child_path(&mut self, parent: fileid3, name: &[u8]) -> Result { + if name.is_empty() || name.contains(&b'/') || name.contains(&0) { + return Err(nfsstat3::NFS3ERR_INVAL); + } + let name = std::str::from_utf8(name).map_err(|_| nfsstat3::NFS3ERR_INVAL)?; + let parent = self.path_for_inode(parent)?; + Ok(if parent == "/" { + format!("/{name}") + } else { + format!("{parent}/{name}") + }) + } + + fn list(&mut self, path: &str) -> Result, nfsstat3> { + let path = checked_cstring(path)?; + let mut bytes = RuntimeBytes { + data: ptr::null_mut(), + size: 0, + }; + let mut error = empty_error(); + let status = + unsafe { vexfs_mount_list(self.session, path.as_ptr(), &mut bytes, &mut error) }; + runtime_result(status)?; + serde_json::from_str(&bytes_to_string(bytes)?).map_err(|_| nfsstat3::NFS3ERR_IO) + } + + fn read_file(&mut self, path: &str) -> Result, nfsstat3> { + let path = checked_cstring(path)?; + let mut bytes = RuntimeBytes { + data: ptr::null_mut(), + size: 0, + }; + let mut error = empty_error(); + let status = + unsafe { vexfs_mount_read_file(self.session, path.as_ptr(), &mut bytes, &mut error) }; + runtime_result(status)?; + Ok(take_bytes(bytes)) + } + + fn request_id(prefix: &str, sequence: &AtomicU64, operation: &str) -> CString { + CString::new(format!( + "{prefix}-{operation}-{}", + sequence.fetch_add(1, Ordering::Relaxed) + )) + .expect("request id has no NUL") + } + + fn create_file( + &mut self, + path: &str, + mode: u32, + prefix: &str, + sequence: &AtomicU64, + ) -> Result<(), nfsstat3> { + let path = checked_cstring(path)?; + let request = Self::request_id(prefix, sequence, "create"); + let mut handle = RuntimeBytes { + data: ptr::null_mut(), + size: 0, + }; + let mut error = empty_error(); + let status = unsafe { + vexfs_mount_handle_create( + self.session, + path.as_ptr(), + mode & 0o7777, + request.as_ptr(), + &mut handle, + &mut error, + ) + }; + runtime_result(status)?; + let handle = checked_cstring(&bytes_to_string(handle)?)?; + let durability = CString::new("full").unwrap(); + let mut version = 0; + let status = unsafe { + vexfs_mount_handle_publish_close( + self.session, + handle.as_ptr(), + 1, + durability.as_ptr(), + &mut version, + &mut error, + ) + }; + if status != 0 { + self.close_failed_handle(&handle, prefix, sequence); + } + runtime_result(status) + } + + fn write_file( + &mut self, + path: &str, + offset: u64, + data: &[u8], + prefix: &str, + sequence: &AtomicU64, + ) -> Result<(), nfsstat3> { + let path = checked_cstring(path)?; + let mode = CString::new("rw").unwrap(); + let open_request = Self::request_id(prefix, sequence, "open-write"); + let mut handle = RuntimeBytes { + data: ptr::null_mut(), + size: 0, + }; + let mut error = empty_error(); + let status = unsafe { + vexfs_mount_handle_open( + self.session, + path.as_ptr(), + mode.as_ptr(), + open_request.as_ptr(), + &mut handle, + &mut error, + ) + }; + runtime_result(status)?; + let handle = checked_cstring(&bytes_to_string(handle)?)?; + let write_request = Self::request_id(prefix, sequence, "write"); + let mut generation = 0; + let status = unsafe { + vexfs_mount_handle_stage_write( + self.session, + handle.as_ptr(), + offset, + data.as_ptr().cast(), + data.len() as u64, + write_request.as_ptr(), + &mut generation, + &mut error, + ) + }; + if status != 0 { + self.close_failed_handle(&handle, prefix, sequence); + return Err(runtime_to_nfs(status)); + } + let durability = CString::new("full").unwrap(); + let mut version = 0; + let status = unsafe { + vexfs_mount_handle_publish_close( + self.session, + handle.as_ptr(), + generation, + durability.as_ptr(), + &mut version, + &mut error, + ) + }; + if status != 0 { + self.close_failed_handle(&handle, prefix, sequence); + } + runtime_result(status) + } + + fn truncate_file( + &mut self, + path: &str, + size: u64, + prefix: &str, + sequence: &AtomicU64, + ) -> Result<(), nfsstat3> { + let path = checked_cstring(path)?; + let mode = CString::new("rw").unwrap(); + let open_request = Self::request_id(prefix, sequence, "open-truncate"); + let mut handle = RuntimeBytes { + data: ptr::null_mut(), + size: 0, + }; + let mut error = empty_error(); + let status = unsafe { + vexfs_mount_handle_open( + self.session, + path.as_ptr(), + mode.as_ptr(), + open_request.as_ptr(), + &mut handle, + &mut error, + ) + }; + runtime_result(status)?; + let handle = checked_cstring(&bytes_to_string(handle)?)?; + let request = Self::request_id(prefix, sequence, "truncate"); + let mut generation = 0; + let status = unsafe { + vexfs_mount_handle_truncate( + self.session, + handle.as_ptr(), + size, + request.as_ptr(), + &mut generation, + &mut error, + ) + }; + if status != 0 { + self.close_failed_handle(&handle, prefix, sequence); + return Err(runtime_to_nfs(status)); + } + let durability = CString::new("full").unwrap(); + let mut version = 0; + let status = unsafe { + vexfs_mount_handle_publish_close( + self.session, + handle.as_ptr(), + generation, + durability.as_ptr(), + &mut version, + &mut error, + ) + }; + if status != 0 { + self.close_failed_handle(&handle, prefix, sequence); + } + runtime_result(status) + } + + fn close_failed_handle(&mut self, handle: &CString, prefix: &str, sequence: &AtomicU64) { + let request = Self::request_id(prefix, sequence, "close-failed"); + let mut state = RuntimeBytes { + data: ptr::null_mut(), + size: 0, + }; + let mut error = empty_error(); + unsafe { + vexfs_mount_handle_close( + self.session, + handle.as_ptr(), + 1, + request.as_ptr(), + &mut state, + &mut error, + ); + } + let _ = take_bytes(state); + } + + fn target_name(sidecar_name: &[u8]) -> Option<&[u8]> { + sidecar_name + .strip_prefix(b"._") + .filter(|name| !name.is_empty() && *name != b"." && *name != b"..") + } + + fn sidecar_target( + &mut self, + parent: fileid3, + sidecar_name: &[u8], + ) -> Result<(String, fileid3), nfsstat3> { + let target_name = Self::target_name(sidecar_name).ok_or(nfsstat3::NFS3ERR_NOENT)?; + let path = self.child_path(parent, target_name)?; + let stat = self.stat_path(&path)?; + Ok((path, stat.inode as u64)) + } + + fn get_xattr(&mut self, inode: fileid3, name: &str) -> Result, nfsstat3> { + let name = checked_cstring(name)?; + let mut bytes = RuntimeBytes { + data: ptr::null_mut(), + size: 0, + }; + let mut error = empty_error(); + let status = unsafe { + vexfs_mount_xattr_get( + self.session, + inode as i64, + name.as_ptr(), + &mut bytes, + &mut error, + ) + }; + runtime_result(status)?; + Ok(take_bytes(bytes)) + } + + fn set_xattr( + &mut self, + inode: fileid3, + name: &str, + data: &[u8], + policy: c_int, + ) -> Result<(), nfsstat3> { + let name = checked_cstring(name)?; + let mut error = empty_error(); + let status = unsafe { + vexfs_mount_xattr_set( + self.session, + inode as i64, + name.as_ptr(), + data.as_ptr().cast(), + data.len() as u64, + policy, + &mut error, + ) + }; + runtime_result(status) + } + + fn allocate_sidecar( + &mut self, + key: SidecarKey, + target_inode: Option, + data: Vec, + mode: u32, + ) -> fileid3 { + if let Some(inode) = self.sidecars_by_key.get(&key) { + return *inode; + } + let inode = self.next_sidecar_inode; + self.next_sidecar_inode = self.next_sidecar_inode.saturating_sub(1); + self.sidecars_by_key.insert(key.clone(), inode); + self.sidecars.insert( + inode, + Sidecar { + key, + target_inode, + data, + mode, + updated_at_ms: now_ms(), + }, + ); + inode + } + + fn lookup_sidecar(&mut self, parent: fileid3, name: &[u8]) -> Result { + let key = SidecarKey { + parent, + name: name.to_vec(), + }; + if let Some(inode) = self.sidecars_by_key.get(&key) { + return Ok(*inode); + } + let (_, target_inode) = self.sidecar_target(parent, name)?; + let data = self.get_xattr(target_inode, APPLEDOUBLE_XATTR)?; + Ok(self.allocate_sidecar(key, Some(target_inode), data, 0o600)) + } + + fn sidecar_attr(&self, inode: fileid3) -> Result { + let sidecar = self.sidecars.get(&inode).ok_or(nfsstat3::NFS3ERR_STALE)?; + Ok(fattr3 { + ftype: ftype3::NF3REG, + mode: sidecar.mode, + nlink: 1, + uid: self.uid, + gid: self.gid, + size: sidecar.data.len() as u64, + used: sidecar.data.len() as u64, + rdev: specdata3::default(), + fsid: 1, + fileid: inode, + atime: nfs_time(sidecar.updated_at_ms), + mtime: nfs_time(sidecar.updated_at_ms), + ctime: nfs_time(sidecar.updated_at_ms), + }) + } + + fn persist_sidecar(&mut self, inode: fileid3) -> Result<(), nfsstat3> { + let (key, target_inode, data) = { + let sidecar = self.sidecars.get(&inode).ok_or(nfsstat3::NFS3ERR_STALE)?; + ( + sidecar.key.clone(), + sidecar.target_inode, + sidecar.data.clone(), + ) + }; + let target_inode = match target_inode { + Some(value) => value, + None => self.sidecar_target(key.parent, &key.name)?.1, + }; + self.set_xattr(target_inode, APPLEDOUBLE_XATTR, &data, XATTR_ALWAYS_SET)?; + if let Some(sidecar) = self.sidecars.get_mut(&inode) { + sidecar.target_inode = Some(target_inode); + } + Ok(()) + } + + fn remove_sidecar(&mut self, parent: fileid3, name: &[u8]) -> Result<(), nfsstat3> { + let key = SidecarKey { + parent, + name: name.to_vec(), + }; + let inode = self.sidecars_by_key.remove(&key); + let target_inode = inode + .and_then(|value| self.sidecars.remove(&value)) + .and_then(|value| value.target_inode) + .or_else(|| self.sidecar_target(parent, name).ok().map(|value| value.1)); + let Some(target_inode) = target_inode else { + return Err(nfsstat3::NFS3ERR_NOENT); + }; + match self.set_xattr(target_inode, APPLEDOUBLE_XATTR, &[], XATTR_DELETE) { + Ok(()) | Err(nfsstat3::NFS3ERR_NOENT) => Ok(()), + Err(status) => Err(status), + } + } +} + +impl VexfsNfs { + fn request_id(&self, operation: &str) -> CString { + RuntimeState::request_id(&self.request_prefix, &self.request_sequence, operation) + } +} + +impl Drop for VexfsNfs { + fn drop(&mut self) { + let request = self.request_id("shutdown-sync"); + let Ok(state) = self.state.lock() else { return }; + let mut published = 0; + let mut error = empty_error(); + unsafe { + vexfs_mount_synchronize(state.session, request.as_ptr(), &mut published, &mut error); + } + } +} + +#[async_trait] +impl NFSFileSystem for VexfsNfs { + fn capabilities(&self) -> VFSCapabilities { + VFSCapabilities::ReadWrite + } + + fn root_dir(&self) -> fileid3 { + self.state.lock().expect("runtime mutex").root_inode + } + + async fn lookup(&self, dirid: fileid3, filename: &filename3) -> Result { + let mut state = self.state.lock().map_err(|_| nfsstat3::NFS3ERR_IO)?; + if filename.as_ref() == b"." { + return Ok(dirid); + } + if filename.as_ref() == b".." { + let path = state.path_for_inode(dirid)?; + if path == "/" { + return Ok(dirid); + } + let parent = path.rsplit_once('/').map(|value| value.0).unwrap_or(""); + let parent = if parent.is_empty() { "/" } else { parent }; + return Ok(state.stat_path(parent)?.inode as u64); + } + if RuntimeState::target_name(filename.as_ref()).is_some() { + return state.lookup_sidecar(dirid, filename.as_ref()); + } + let path = state.child_path(dirid, filename.as_ref())?; + Ok(state.stat_path(&path)?.inode as u64) + } + + async fn getattr(&self, id: fileid3) -> Result { + let mut state = self.state.lock().map_err(|_| nfsstat3::NFS3ERR_IO)?; + if state.sidecars.contains_key(&id) { + return state.sidecar_attr(id); + } + let path = state.path_for_inode(id)?; + Ok(stat_attr(&state.stat_path(&path)?)) + } + + async fn setattr(&self, id: fileid3, attrs: sattr3) -> Result { + let mut state = self.state.lock().map_err(|_| nfsstat3::NFS3ERR_IO)?; + if state.sidecars.contains_key(&id) { + if let set_mode3::mode(mode) = attrs.mode { + state.sidecars.get_mut(&id).unwrap().mode = mode & 0o7777; + } + if let set_size3::size(size) = attrs.size { + state + .sidecars + .get_mut(&id) + .unwrap() + .data + .resize(size as usize, 0); + state.persist_sidecar(id)?; + } + return state.sidecar_attr(id); + } + let path = state.path_for_inode(id)?; + let mut error = empty_error(); + if let set_mode3::mode(mode) = attrs.mode { + runtime_result(unsafe { + vexfs_mount_set_mode(state.session, id as i64, mode & 0o7777, &mut error) + })?; + } + let uid = match attrs.uid { + set_uid3::uid(value) => value as i64, + set_uid3::Void => -1, + }; + let gid = match attrs.gid { + set_gid3::gid(value) => value as i64, + set_gid3::Void => -1, + }; + if uid >= 0 || gid >= 0 { + runtime_result(unsafe { + vexfs_mount_chown(state.session, id as i64, uid, gid, &mut error) + })?; + } + if let set_size3::size(size) = attrs.size { + state.truncate_file(&path, size, &self.request_prefix, &self.request_sequence)?; + } + let now = now_ms(); + let (accessed, access_mask) = match attrs.atime { + set_atime::DONT_CHANGE => (0, 0), + set_atime::SET_TO_SERVER_TIME => (now, TIME_ACCESS), + set_atime::SET_TO_CLIENT_TIME(value) => ( + (value.seconds as i64) * 1000 + (value.nseconds as i64) / 1_000_000, + TIME_ACCESS, + ), + }; + let (modified, modify_mask) = match attrs.mtime { + set_mtime::DONT_CHANGE => (0, 0), + set_mtime::SET_TO_SERVER_TIME => (now, TIME_MODIFY), + set_mtime::SET_TO_CLIENT_TIME(value) => ( + (value.seconds as i64) * 1000 + (value.nseconds as i64) / 1_000_000, + TIME_MODIFY, + ), + }; + if access_mask | modify_mask != 0 { + runtime_result(unsafe { + vexfs_mount_set_times( + state.session, + id as i64, + accessed, + modified, + access_mask | modify_mask, + &mut error, + ) + })?; + } + Ok(stat_attr(&state.stat_path(&path)?)) + } + + async fn read( + &self, + id: fileid3, + offset: u64, + count: u32, + ) -> Result<(Vec, bool), nfsstat3> { + let mut state = self.state.lock().map_err(|_| nfsstat3::NFS3ERR_IO)?; + let data = if let Some(sidecar) = state.sidecars.get(&id) { + sidecar.data.clone() + } else { + let path = state.path_for_inode(id)?; + state.read_file(&path)? + }; + let start = (offset as usize).min(data.len()); + let end = start.saturating_add(count as usize).min(data.len()); + Ok((data[start..end].to_vec(), end == data.len())) + } + + async fn write(&self, id: fileid3, offset: u64, data: &[u8]) -> Result { + let mut state = self.state.lock().map_err(|_| nfsstat3::NFS3ERR_IO)?; + if state.sidecars.contains_key(&id) { + let sidecar = state.sidecars.get_mut(&id).unwrap(); + let offset = offset as usize; + if offset.saturating_add(data.len()) > sidecar.data.len() { + sidecar.data.resize(offset.saturating_add(data.len()), 0); + } + sidecar.data[offset..offset + data.len()].copy_from_slice(data); + sidecar.updated_at_ms = now_ms(); + state.persist_sidecar(id)?; + return state.sidecar_attr(id); + } + let path = state.path_for_inode(id)?; + state.write_file( + &path, + offset, + data, + &self.request_prefix, + &self.request_sequence, + )?; + Ok(stat_attr(&state.stat_path(&path)?)) + } + + async fn create( + &self, + dirid: fileid3, + filename: &filename3, + attrs: sattr3, + ) -> Result<(fileid3, fattr3), nfsstat3> { + let mut state = self.state.lock().map_err(|_| nfsstat3::NFS3ERR_IO)?; + let mode = match attrs.mode { + set_mode3::mode(value) => value, + set_mode3::Void => 0o644, + }; + if RuntimeState::target_name(filename.as_ref()).is_some() { + let key = SidecarKey { + parent: dirid, + name: filename.as_ref().to_vec(), + }; + let target_inode = state + .sidecar_target(dirid, filename.as_ref()) + .ok() + .map(|value| value.1); + let inode = state.allocate_sidecar(key, target_inode, Vec::new(), mode); + return Ok((inode, state.sidecar_attr(inode)?)); + } + let path = state.child_path(dirid, filename.as_ref())?; + state.create_file(&path, mode, &self.request_prefix, &self.request_sequence)?; + let stat = state.stat_path(&path)?; + let mut error = empty_error(); + runtime_result(unsafe { + vexfs_mount_chown( + state.session, + stat.inode, + state.uid as i64, + state.gid as i64, + &mut error, + ) + })?; + let stat = state.stat_path(&path)?; + Ok((stat.inode as u64, stat_attr(&stat))) + } + + async fn create_exclusive( + &self, + dirid: fileid3, + filename: &filename3, + ) -> Result { + Ok(self.create(dirid, filename, sattr3::default()).await?.0) + } + + async fn mkdir( + &self, + dirid: fileid3, + dirname: &filename3, + ) -> Result<(fileid3, fattr3), nfsstat3> { + let mut state = self.state.lock().map_err(|_| nfsstat3::NFS3ERR_IO)?; + let path = state.child_path(dirid, dirname.as_ref())?; + let path_c = checked_cstring(&path)?; + let mut error = empty_error(); + runtime_result(unsafe { vexfs_mount_mkdir(state.session, path_c.as_ptr(), &mut error) })?; + let stat = state.stat_path(&path)?; + let mode = 0o755; + runtime_result(unsafe { + vexfs_mount_set_mode(state.session, stat.inode, mode & 0o7777, &mut error) + })?; + runtime_result(unsafe { + vexfs_mount_chown( + state.session, + stat.inode, + state.uid as i64, + state.gid as i64, + &mut error, + ) + })?; + let stat = state.stat_path(&path)?; + Ok((stat.inode as u64, stat_attr(&stat))) + } + + async fn remove(&self, dirid: fileid3, filename: &filename3) -> Result<(), nfsstat3> { + let mut state = self.state.lock().map_err(|_| nfsstat3::NFS3ERR_IO)?; + if RuntimeState::target_name(filename.as_ref()).is_some() { + return state.remove_sidecar(dirid, filename.as_ref()); + } + let path = state.child_path(dirid, filename.as_ref())?; + let path = checked_cstring(&path)?; + let mut error = empty_error(); + runtime_result(unsafe { vexfs_mount_remove(state.session, path.as_ptr(), 0, &mut error) }) + } + + async fn rename( + &self, + from_dirid: fileid3, + from_filename: &filename3, + to_dirid: fileid3, + to_filename: &filename3, + ) -> Result<(), nfsstat3> { + let mut state = self.state.lock().map_err(|_| nfsstat3::NFS3ERR_IO)?; + let from_sidecar = RuntimeState::target_name(from_filename.as_ref()).is_some(); + let to_sidecar = RuntimeState::target_name(to_filename.as_ref()).is_some(); + if from_sidecar || to_sidecar { + if !from_sidecar || !to_sidecar { + return Err(nfsstat3::NFS3ERR_INVAL); + } + let inode = state.lookup_sidecar(from_dirid, from_filename.as_ref())?; + let data = state.sidecars.get(&inode).unwrap().data.clone(); + state.remove_sidecar(from_dirid, from_filename.as_ref())?; + let key = SidecarKey { + parent: to_dirid, + name: to_filename.as_ref().to_vec(), + }; + let target = state + .sidecar_target(to_dirid, to_filename.as_ref()) + .ok() + .map(|value| value.1); + let new_inode = state.allocate_sidecar(key, target, data, 0o600); + return state.persist_sidecar(new_inode); + } + let source = checked_cstring(&state.child_path(from_dirid, from_filename.as_ref())?)?; + let destination = checked_cstring(&state.child_path(to_dirid, to_filename.as_ref())?)?; + let mut error = empty_error(); + runtime_result(unsafe { + vexfs_mount_rename( + state.session, + source.as_ptr(), + destination.as_ptr(), + 1, + &mut error, + ) + }) + } + + async fn link( + &self, + id: fileid3, + dirid: fileid3, + filename: &filename3, + ) -> Result { + let mut state = self.state.lock().map_err(|_| nfsstat3::NFS3ERR_IO)?; + if state.sidecars.contains_key(&id) + || RuntimeState::target_name(filename.as_ref()).is_some() + { + return Err(nfsstat3::NFS3ERR_NOTSUPP); + } + let source_path = state.path_for_inode(id)?; + let destination_path = state.child_path(dirid, filename.as_ref())?; + let source = checked_cstring(&source_path)?; + let destination = checked_cstring(&destination_path)?; + let mut error = empty_error(); + runtime_result(unsafe { + vexfs_mount_link( + state.session, + source.as_ptr(), + destination.as_ptr(), + &mut error, + ) + })?; + Ok(stat_attr(&state.stat_path(&source_path)?)) + } + + async fn commit(&self, id: fileid3, _offset: u64, _count: u32) -> Result { + let request = self.request_id("nfs-commit"); + let mut state = self.state.lock().map_err(|_| nfsstat3::NFS3ERR_IO)?; + if state.sidecars.contains_key(&id) { + state.persist_sidecar(id)?; + return state.sidecar_attr(id); + } + let mut published = 0; + let mut error = empty_error(); + runtime_result(unsafe { + vexfs_mount_synchronize(state.session, request.as_ptr(), &mut published, &mut error) + })?; + let path = state.path_for_inode(id)?; + Ok(stat_attr(&state.stat_path(&path)?)) + } + + async fn readdir( + &self, + dirid: fileid3, + start_after: fileid3, + max_entries: usize, + ) -> Result { + let mut state = self.state.lock().map_err(|_| nfsstat3::NFS3ERR_IO)?; + let path = state.path_for_inode(dirid)?; + let mut entries = state.list(&path)?; + entries.sort_by_key(|entry| entry.inode); + let start = if start_after == 0 { + 0 + } else { + entries + .iter() + .position(|entry| entry.inode as u64 == start_after) + .map(|value| value + 1) + .unwrap_or(0) + }; + let end = start.saturating_add(max_entries).min(entries.len()); + let complete = end == entries.len(); + let entries = entries[start..end] + .iter() + .map(|entry| { + let stat: FileStat = entry.clone().into(); + DirEntry { + fileid: entry.inode as u64, + name: entry.name.as_bytes().into(), + attr: stat_attr(&stat), + } + }) + .collect(); + Ok(ReadDirResult { + entries, + end: complete, + }) + } + + async fn symlink( + &self, + dirid: fileid3, + linkname: &filename3, + target: &nfspath3, + _attrs: &sattr3, + ) -> Result<(fileid3, fattr3), nfsstat3> { + let mut state = self.state.lock().map_err(|_| nfsstat3::NFS3ERR_IO)?; + let path = checked_cstring(&state.child_path(dirid, linkname.as_ref())?)?; + let mut error = empty_error(); + runtime_result(unsafe { + vexfs_mount_symlink( + state.session, + path.as_ptr(), + target.as_ref().as_ptr().cast(), + target.as_ref().len() as u64, + &mut error, + ) + })?; + let path = path.to_str().map_err(|_| nfsstat3::NFS3ERR_IO)?; + let stat = state.stat_path(path)?; + Ok((stat.inode as u64, stat_attr(&stat))) + } + + async fn readlink(&self, id: fileid3) -> Result { + let state = self.state.lock().map_err(|_| nfsstat3::NFS3ERR_IO)?; + let mut bytes = RuntimeBytes { + data: ptr::null_mut(), + size: 0, + }; + let mut error = empty_error(); + let status = + unsafe { vexfs_mount_readlink(state.session, id as i64, &mut bytes, &mut error) }; + runtime_result(status)?; + Ok(take_bytes(bytes).into()) + } +} + +fn config_string(pointer: *const c_char, name: &str) -> Result { + if pointer.is_null() { + return Err(format!("{name} is required")); + } + unsafe { CStr::from_ptr(pointer) } + .to_str() + .map(str::to_owned) + .map_err(|_| format!("{name} is not UTF-8")) +} + +fn write_error(buffer: *mut c_char, size: usize, message: &str) { + if buffer.is_null() || size == 0 { + return; + } + let bytes = message.as_bytes(); + let count = bytes.len().min(size - 1); + unsafe { + ptr::copy_nonoverlapping(bytes.as_ptr(), buffer.cast::(), count); + *buffer.add(count) = 0; + } +} + +fn runtime_error_message(error: &RuntimeError) -> String { + unsafe { CStr::from_ptr(error.message.as_ptr()) } + .to_string_lossy() + .into_owned() +} + +fn run_gateway(config: &VexfsNfsGatewayConfig) -> Result<(), String> { + if config.abi_version != RUNTIME_ABI_VERSION { + return Err("NFS gateway ABI mismatch".to_string()); + } + let backend = config_string(config.backend, "backend")?; + let connection = config_string(config.connection, "connection")?; + let workspace = config_string(config.workspace, "workspace")?; + let principal = config_string(config.principal, "principal")?; + let listen = config_string(config.listen_address, "listen address")?; + if listen != "127.0.0.1" { + return Err("NFS gateway may only bind 127.0.0.1".to_string()); + } + if config.port == 0 { + return Err("NFS gateway port is required".to_string()); + } + let backend_c = CString::new(backend).map_err(|_| "backend contains NUL")?; + let connection_c = CString::new(connection).map_err(|_| "connection contains NUL")?; + let workspace_c = CString::new(workspace).map_err(|_| "workspace contains NUL")?; + let principal_c = CString::new(principal).map_err(|_| "principal contains NUL")?; + let runtime_config = RuntimeConfig { + abi_version: RUNTIME_ABI_VERSION, + backend: backend_c.as_ptr(), + connection: connection_c.as_ptr(), + workspace: workspace_c.as_ptr(), + principal: principal_c.as_ptr(), + operation_timeout_ms: config.operation_timeout_ms.max(1), + flags: RUNTIME_EXCLUSIVE_GATEWAY, + }; + let mut error = empty_error(); + let mut session = ptr::null_mut(); + let status = unsafe { vexfs_mount_session_open(&runtime_config, &mut session, &mut error) }; + if status != 0 { + let message = runtime_error_message(&error); + return Err(if message.is_empty() { + format!("runtime session open failed with status {status}") + } else { + message + }); + } + let mut state = RuntimeState { + session, + root_inode: 1, + uid: unsafe { libc::getuid() }, + gid: unsafe { libc::getgid() }, + next_sidecar_inode: SIDECAR_INODE_START, + sidecars_by_key: HashMap::new(), + sidecars: HashMap::new(), + }; + let root = state + .stat_path("/") + .map_err(|status| format!("root stat failed: {status:?}"))?; + state.root_inode = root.inode as u64; + let prefix = format!("nfs-{}-{}", unsafe { libc::getpid() }, now_ms()); + let filesystem = VexfsNfs { + state: Mutex::new(state), + request_prefix: prefix, + request_sequence: AtomicU64::new(1), + }; + let bind_address = format!("{listen}:{}", config.port); + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .build() + .map_err(|error| error.to_string())?; + runtime.block_on(async move { + let listener = NFSTcpListener::bind(&bind_address, filesystem) + .await + .map_err(|error| error.to_string())?; + #[cfg(unix)] + { + use tokio::signal::unix::{signal, SignalKind}; + let mut terminate = + signal(SignalKind::terminate()).map_err(|error| error.to_string())?; + let mut interrupt = + signal(SignalKind::interrupt()).map_err(|error| error.to_string())?; + loop { + tokio::select! { + result = listener.handle_forever() => { + match result { + Ok(()) => return Err("NFS listener stopped unexpectedly".to_string()), + Err(error) => { + // nfsserve 0.11 returns from its whole accept loop for one + // transient accept(2) error. A mounted filesystem must stay + // alive, so retain the listener and retry instead of turning a + // short macOS network interruption into a stale mount. + eprintln!("vexfs-nfs-gateway: transient accept error: {error}"); + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + } + }, + _ = terminate.recv() => return Ok(()), + _ = interrupt.recv() => return Ok(()), + } + } + } + #[cfg(not(unix))] + listener + .handle_forever() + .await + .map_err(|error| error.to_string()) + })?; + // Dropping the listener drops the filesystem and closes the runtime session. + Ok(()) +} + +/// Runs the NFS gateway until it receives a termination signal. +/// +/// # Safety +/// +/// `config` must point to a valid `VexfsNfsGatewayConfig` for the duration of +/// this call. When non-null, `error_message` must reference a writable buffer +/// containing at least `error_message_size` bytes. +#[no_mangle] +pub unsafe extern "C" fn vexfs_nfs_gateway_run( + config: *const VexfsNfsGatewayConfig, + error_message: *mut c_char, + error_message_size: usize, +) -> c_int { + if config.is_null() { + write_error( + error_message, + error_message_size, + "gateway config is required", + ); + return 2; + } + let result = std::panic::catch_unwind(|| run_gateway(unsafe { &*config })); + match result { + Ok(Ok(())) => 0, + Ok(Err(message)) => { + write_error(error_message, error_message_size, &message); + 1 + } + Err(_) => { + write_error(error_message, error_message_size, "NFS gateway panicked"); + 1 + } + } +} diff --git a/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/Cargo.toml b/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/Cargo.toml new file mode 100644 index 0000000000..64cc2773e9 --- /dev/null +++ b/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/Cargo.toml @@ -0,0 +1,47 @@ +[package] +name = "nfsserve" +version = "0.11.0" +edition = "2021" +authors = ["Rajat Arya ", "Assaf Vayner "] +description = "A Rust NFS Server implementation" +homepage = "https://github.com/huggingface/nfsserve" +repository = "https://github.com/huggingface/nfsserve" +readme = "README.md" +keywords = ["nfs"] +license = "BSD-3-Clause" +categories = ["network-programming", "filesystem"] +publish = false +include = ["src/**/*", "src/*", "Cargo.toml", "LICENSE", "README.md"] + + +[lib] +doctest = false + +[dependencies] +byteorder = "1.4" +num-traits = "0.2" +num-derive = "0.4" +tokio = { version = "1", features = ["net", "io-util", "sync", "fs", "rt", "macros"], default-features = false } +tracing = "0.1.31" +anyhow = "1" +async-trait = "0.1.9" +filetime = "0.2" + +# demo +tracing-subscriber = { version = "0.3", features = ["tracing-log"], optional = true } +intaglio = { version = "1.6", optional = true } + +[features] +strict = [] +demo = ["tracing-subscriber", "tokio/rt-multi-thread", "intaglio"] + + +[[example]] +name = "demo" +required-features = ["demo"] +path = "examples/demo.rs" + +[[example]] +name = "mirrorfs" +required-features = ["demo"] +path = "examples/mirrorfs.rs" diff --git a/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/LICENSE b/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/LICENSE new file mode 100644 index 0000000000..d8ced14353 --- /dev/null +++ b/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/LICENSE @@ -0,0 +1,29 @@ +BSD 3-Clause License + +Copyright (c) 2023, XetData +Copyright (c) 2025, Hugging Face + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/README.md b/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/README.md new file mode 100644 index 0000000000..2d056863d4 --- /dev/null +++ b/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/README.md @@ -0,0 +1,194 @@ +Rust NFSv3 Server +================= +This is an incomplete but very functional implementation of an NFSv3 server +in Rust. + +Why? You may ask. + +I wanted to implement a user-mode file-system mount that is truly cross-platform. +What is a protocol that pretty much every OS supports? NFS. + +Why not FUSE you may ask: +1. FUSE is annoying to users on Mac and Windows (drivers necessary). +2. It takes a lot of care to build a FUSE driver for remote filesystems. +NFS clients however have a lot of historical robustification for +slow-responding, or perhaps, never-responding servers. +3. The OS is pretty good at caching NFS. There are established principles for +cache eviction, for metadata, or for data. With a FUSE driver I have to do +a lot of the work myself. + +So, this is a FUSE-like user-mode filesystem API that basically works by +creating a localhost NFSv3 server you can mount. + +This is used in [hf-mount](https://github.com/huggingface/hf-mount) to provide the `hf-mount` +functionality. + +Run the Demo +============ +To run the demofs, this will host an NFS server on localhost:11111 +``` +cargo build --example demo --features demo +./target/debug/examples/demo +``` + +To mount. On Linux (sudo may be required): +``` +mkdir demo +mount.nfs -o user,noacl,nolock,vers=3,tcp,wsize=1048576,rsize=131072,actimeo=120,port=11111,mountport=11111 localhost:/ demo +``` + +On Mac: +``` +mkdir demo +mount_nfs -o nolocks,vers=3,tcp,rsize=131072,actimeo=120,port=11111,mountport=11111 localhost:/ demo +``` + +On Windows (Pro required as Home does not have NFS client): +``` +mount.exe -o anon,nolock,mtype=soft,fileaccess=6,casesensitive,lang=ansi,rsize=128,wsize=128,timeout=60,retry=2 \\127.0.0.1\\ X: +``` + +Note that the demo filesystem is *writable*. + +Usage +===== + +You simply need to implement the vfs::NFSFileSystem +trait. See demofs.rs for an example and bin/main.rs for how to actually start +a service. The interface generally not difficult to implement; demanding mainly +the ability to associate every file system object (directory/file) with a 64-bit +ID. Directory listing can be a bit complicated due to the pagination requirements. + +TODO and Seeking Contributors +============================= + - Improve documentation + - More things in Mount Protocol and NFS Protocol has to be implemented. + There are a bunch of messages that reply as "Unavailable". For instance, + we implement `READDIR_PLUS` but not `READDIR` which is usually fine, except + that Windows insists on always trying READDIR first. + Link creation is also not supported. + - The RPC message handling in `nfs_handlers.rs` leaves a lot to be desired. + The response serialization is very manual. Some cleanup will be good. + - Windows mount "kinda" works (only on Windows 11 Pro with the NFS server), + but prints a lot of garbage due to various unimplemented APIs. Windows 11 + somehow tries to poll with very old NFS protocols constantly. + - Many many perf optimizations. + - Maybe pull in the mount command from [xet-core](https://github.com/huggingface/xet-core/blob/main/rust/gitxetcore/src/xetmnt/mod.rs) + so the user does not need to remember the `-o` incantations above. + - Maybe make an SMB3 implementation so we can work on Windows Home edition + - NFSv4 has some write performance optimizations that would be quite nice. + The protocol is a bit more involving to implement though as it is somewhat + stateful. + +Relevant RFCs +============= + - XDR is the message format: RFC 1014. https://datatracker.ietf.org/doc/html/rfc1014 + - SUN RPC is the RPC wire format: RFC 1057 https://datatracker.ietf.org/doc/html/rfc1057 + - NFS is at RFC 1813 https://datatracker.ietf.org/doc/html/rfc1813 + - NFS Mount Protocol is at RFC 1813 Appendix I. https://datatracker.ietf.org/doc/html/rfc1813#appendix-I + - PortMapper is at RFC 1057 Appendix A https://datatracker.ietf.org/doc/html/rfc1057#appendix-A + +Basic Source Layout +=================== + - context.rs: A connection context object that is passed around containing + connection information, VFS information, etc. + - xdr.rs: Serialization / Deserialization routines for XDR structures + - tcp.rs: Main TCP handling entry point + - rpcwire.rs: Reads and write RPC messages from a TCP socket and performs outer + most RPC message decoding, redirecting to NFS/Mount/Portmapper + implementations as needed. + - rpc.rs: The structure of a RPC call and reply. All XDR encoded. + - portmap.rs/portmap\_handlers.rs: The XDR structures required by the Portmapper protocol and the Portmapper RPC handlers. + - mount.rs/mount\_handlers.rs: The XDR structures required by the Mount protocol and the Mount RPC handlers. + - nfs.rs/nfs\_handlers.rs: The XDR structures required by the NFS protocol and the NFS RPC handlers. + + +More More Details Than Necessary +================================ +The basic way a message works is: +1. We read a collection of fragments off a TCP stream +(a 4 byte length header followed by a bunch of bytes) +2. We assemble the fragments into a record +3. The Record is of a SUN RPC message type. +4. A message tells us 3 pieces of information, + - The RPC Program (just an integer denoting + a protocol "class". For instance NFS protocol is 100003, the Portmapper protocol is 100000). + - The version of the RPC program (ex: 3 = NFSv3, 4 = NFSv4, etc) + - The method invoked (Which NFS method to call) (See for instance nfs.rs top comment for the list) +5. Continuing to decode the message will give us the arguments of the method +6. And we take the method response, wrap it around a record and return it. + +Portmapper +---------- +First, lets get portmapper out of the way. This is a *very* old mechanism which +is rarely used anymore. The portmapper is a daemon which runs on a machine running +on port 111. When NFS, or other RPC services start, they register with the +portmapper service with the port they are listening on (Say NFS on 2049). +Then when another machine wants to connect to NFS, they first ask the port mapper +on 111 to ask about which port NFS is listening on, then connects to the returned +port. + +We do not strictly need to implement this protocol as this is pretty much +unused these days (NFSv4 does not use the portmapper for instance). If `-o port` and `-o mountport` +are specified, Linux and Mac's builtin NFS client do not need it either. +But this was useful for debugging and testing as libnfs seems to require a +portmapper, but it annoyingly hardcodes it to 111. I modified the source to +change it to 12000 for testing and implemented the one `PMAPPROC_GETPORT` +method so I can test with libnfs. + + +NFS Basics +========== +The way NFS works is that every file system object (dir/file/symlink) has 2 +ways in which it can be addressed: + +1. `fileid3: u64` . A 64-bit integer. Equivalent to an inode number. +2. `nfs_fh3`: A variable opaque object up to 64 bytes long. + +Basically anytime the client tries to access any information about an object, +it needs an `nfs_fh3`. The purpose of the `nfs_fh3` serves 2 purposes: + + - Allow server to cache additional query information in the handle that may exceed + 64-bit. For instance if the server has multiple exports on different disk volumes, + I may need a few more bits to identify the disk volume. + - Allow client to identify when server has "restarted" and thus client has to + clear all caches. the `nfs_fh3` handle should contain a token that is unique + to when the NFS server first started up which allows the server to check that + the handle is still valid. If the server has restarted, all previous handles + will therefore be "expired" and any usage of them should trigger a handle expiry + error informing the clients to expunge all caches. + + +However, the only way to obtain an `nfs_fh3` for a file is via directory traversal. +i.e. There is a lookup method +`LOOKUP(directory's handle, filename of file/dir in directory)` +which returns the handle for the filename. + +For instance to get the handle of a file "dir/a.txt", I first need the handle +for the directory "dir/", then query `LOOKUP(handle, "a.txt")`. + +The question is then, how do I get my first handle? That is what the MOUNT +protocol addresses. + +Mount +----- +The MOUNT protocol provides a list of "exports", (in the simplest case. Just "/") +and the client will request to MNT("/") which will return the handle of this +root directory. + +Normally the server can and do maintain a list of mounts which can be queried, +and really the client can UMNT (unmount) as well. But in our case we +only implement MNT and EXPORT which suffices. NFS clients generally +ignore the return message of UMNT as there is really nothing the +client can do on a UMNT failure. As such our Mount protocol implementation +is entirely stateless. + +NFS +--- +The NFS protocol itself is pretty straightforward with most annoyances +due to handling of the XDR messaging format (in paticular with optional, +lists, etc). + +What is nice is that the design of NFS is completely stateless. It is mostly +sit down and implement all the methods that are hit and test them against a +client. diff --git a/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/config.rs b/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/config.rs new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/config.rs @@ -0,0 +1 @@ + diff --git a/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/context.rs b/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/context.rs new file mode 100644 index 0000000000..b86c8d261b --- /dev/null +++ b/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/context.rs @@ -0,0 +1,28 @@ +use std::fmt; +use std::sync::Arc; + +use tokio::sync::mpsc; + +use crate::transaction_tracker::TransactionTracker; +use crate::vfs::NFSFileSystem; + +#[derive(Clone)] +pub struct RPCContext { + pub local_port: u16, + pub client_addr: String, + pub auth: crate::rpc::auth_unix, + pub vfs: Arc, + pub mount_signal: Option>, + pub export_name: Arc, + pub transaction_tracker: Arc, +} + +impl fmt::Debug for RPCContext { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.debug_struct("RPCContext") + .field("local_port", &self.local_port) + .field("client_addr", &self.client_addr) + .field("auth", &self.auth) + .finish() + } +} diff --git a/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/fs_util.rs b/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/fs_util.rs new file mode 100644 index 0000000000..54e590295e --- /dev/null +++ b/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/fs_util.rs @@ -0,0 +1,178 @@ +use std::fs::{Metadata, Permissions}; +#[cfg(unix)] +use std::os::unix::fs::{MetadataExt, PermissionsExt}; +use std::path::Path; + +use tokio::fs::OpenOptions; +use tracing::debug; + +use crate::nfs::*; + +/// Compares if file metadata has changed in a significant way +#[cfg(any(target_os = "linux", target_os = "macos"))] +pub fn metadata_differ(lhs: &Metadata, rhs: &Metadata) -> bool { + lhs.ino() != rhs.ino() || lhs.mtime() != rhs.mtime() || lhs.len() != rhs.len() || lhs.file_type() != rhs.file_type() +} +pub fn fattr3_differ(lhs: &fattr3, rhs: &fattr3) -> bool { + lhs.fileid != rhs.fileid + || lhs.mtime.seconds != rhs.mtime.seconds + || lhs.mtime.nseconds != rhs.mtime.nseconds + || lhs.size != rhs.size + || lhs.ftype as u32 != rhs.ftype as u32 +} + +/// path.exists() is terrifyingly unsafe as that +/// traverses symlinks. This can cause deadlocks if we have a +/// recursive symlink. +pub fn exists_no_traverse(path: &Path) -> bool { + path.symlink_metadata().is_ok() +} + +fn mode_unmask(mode: u32) -> u32 { + // it is possible to create a file we cannot write to. + // we force writable always. + let mode = mode | 0x80; + let mode = Permissions::from_mode(mode); + mode.mode() & 0x1FF +} + +/// Converts fs Metadata to NFS fattr3 +pub fn metadata_to_fattr3(fid: fileid3, meta: &Metadata) -> fattr3 { + let size = meta.size(); + let file_mode = mode_unmask(meta.mode()); + if meta.is_file() { + fattr3 { + ftype: ftype3::NF3REG, + mode: file_mode, + nlink: 1, + uid: meta.uid(), + gid: meta.gid(), + size, + used: size, + rdev: specdata3::default(), + fsid: 0, + fileid: fid, + atime: nfstime3 { + seconds: meta.atime() as u32, + nseconds: meta.atime_nsec() as u32, + }, + mtime: nfstime3 { + seconds: meta.mtime() as u32, + nseconds: meta.mtime_nsec() as u32, + }, + ctime: nfstime3 { + seconds: meta.ctime() as u32, + nseconds: meta.ctime_nsec() as u32, + }, + } + } else if meta.is_symlink() { + fattr3 { + ftype: ftype3::NF3LNK, + mode: file_mode, + nlink: 1, + uid: meta.uid(), + gid: meta.gid(), + size, + used: size, + rdev: specdata3::default(), + fsid: 0, + fileid: fid, + atime: nfstime3 { + seconds: meta.atime() as u32, + nseconds: meta.atime_nsec() as u32, + }, + mtime: nfstime3 { + seconds: meta.mtime() as u32, + nseconds: meta.mtime_nsec() as u32, + }, + ctime: nfstime3 { + seconds: meta.ctime() as u32, + nseconds: meta.ctime_nsec() as u32, + }, + } + } else { + fattr3 { + ftype: ftype3::NF3DIR, + mode: file_mode, + nlink: 2, + uid: meta.uid(), + gid: meta.gid(), + size, + used: size, + rdev: specdata3::default(), + fsid: 0, + fileid: fid, + atime: nfstime3 { + seconds: meta.atime() as u32, + nseconds: meta.atime_nsec() as u32, + }, + mtime: nfstime3 { + seconds: meta.mtime() as u32, + nseconds: meta.mtime_nsec() as u32, + }, + ctime: nfstime3 { + seconds: meta.ctime() as u32, + nseconds: meta.ctime_nsec() as u32, + }, + } + } +} + +/// Set attributes of a path +pub async fn path_setattr(path: &Path, setattr: &sattr3) -> Result<(), nfsstat3> { + match setattr.atime { + set_atime::SET_TO_SERVER_TIME => { + let _ = filetime::set_file_atime(path, filetime::FileTime::now()); + }, + set_atime::SET_TO_CLIENT_TIME(time) => { + let _ = filetime::set_file_atime(path, time.into()); + }, + _ => {}, + }; + match setattr.mtime { + set_mtime::SET_TO_SERVER_TIME => { + let _ = filetime::set_file_mtime(path, filetime::FileTime::now()); + }, + set_mtime::SET_TO_CLIENT_TIME(time) => { + let _ = filetime::set_file_mtime(path, time.into()); + }, + _ => {}, + }; + if let set_mode3::mode(mode) = setattr.mode { + debug!(" -- set permissions {:?} {:?}", path, mode); + let mode = mode_unmask(mode); + let _ = std::fs::set_permissions(path, Permissions::from_mode(mode)); + }; + if let set_uid3::uid(_) = setattr.uid { + debug!("Set uid not implemented"); + } + if let set_gid3::gid(_) = setattr.gid { + debug!("Set gid not implemented"); + } + if let set_size3::size(size3) = setattr.size { + let file = OpenOptions::new() + .read(true) + .write(true) + .truncate(false) + .open(path) + .await + .or(Err(nfsstat3::NFS3ERR_IO))?; + debug!(" -- set size {:?} {:?}", path, size3); + file.set_len(size3).await.or(Err(nfsstat3::NFS3ERR_IO))?; + } + Ok(()) +} + +/// Set attributes of a file +pub async fn file_setattr(file: &std::fs::File, setattr: &sattr3) -> Result<(), nfsstat3> { + if let set_mode3::mode(mode) = setattr.mode { + debug!(" -- set permissions {:?}", mode); + let mode = mode_unmask(mode); + let _ = file.set_permissions(Permissions::from_mode(mode)); + } + if let set_size3::size(size3) = setattr.size { + debug!(" -- set size {:?}", size3); + file.set_len(size3).or(Err(nfsstat3::NFS3ERR_IO))?; + } + Ok(()) +} diff --git a/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/lib.rs b/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/lib.rs new file mode 100644 index 0000000000..a5e1dcb868 --- /dev/null +++ b/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/lib.rs @@ -0,0 +1,23 @@ +#![cfg_attr(feature = "strict", deny(warnings))] + +mod context; +mod rpc; +mod rpcwire; +mod write_counter; +pub mod xdr; + +mod mount; +mod mount_handlers; + +mod portmap; +mod portmap_handlers; + +pub mod nfs; +mod nfs_handlers; + +#[cfg(not(target_os = "windows"))] +pub mod fs_util; + +pub mod tcp; +mod transaction_tracker; +pub mod vfs; diff --git a/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/mount.rs b/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/mount.rs new file mode 100644 index 0000000000..fb77d2073a --- /dev/null +++ b/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/mount.rs @@ -0,0 +1,41 @@ +// this is just a complete enumeration of everything in the RFC +#![allow(dead_code)] +// And its nice to keep the original RFC names and case +#![allow(non_camel_case_types)] + +use std::io::{Read, Write}; + +use byteorder::{ReadBytesExt, WriteBytesExt}; +use num_derive::{FromPrimitive, ToPrimitive}; +use num_traits::cast::FromPrimitive; + +use crate::xdr::*; +// Transcribed from RFC 1057 Appendix A + +pub const PROGRAM: u32 = 100005; +pub const VERSION: u32 = 3; + +pub const MNTPATHLEN: u32 = 1024; /* Maximum bytes in a path name */ +pub const MNTNAMLEN: u32 = 255; /* Maximum bytes in a name */ +pub const FHSIZE3: u32 = 64; /* Maximum bytes in a V3 file handle */ + +pub type fhandle3 = Vec; +pub type dirpath = Vec; +pub type name = Vec; + +#[allow(non_camel_case_types)] +#[derive(Copy, Clone, Debug, FromPrimitive, ToPrimitive)] +#[repr(u32)] +pub enum mountstat3 { + MNT3_OK = 0, /* no error */ + MNT3ERR_PERM = 1, /* Not owner */ + MNT3ERR_NOENT = 2, /* No such file or directory */ + MNT3ERR_IO = 5, /* I/O error */ + MNT3ERR_ACCES = 13, /* Permission denied */ + MNT3ERR_NOTDIR = 20, /* Not a directory */ + MNT3ERR_INVAL = 22, /* Invalid argument */ + MNT3ERR_NAMETOOLONG = 63, /* Filename too long */ + MNT3ERR_NOTSUPP = 10004, /* Operation not supported */ + MNT3ERR_SERVERFAULT = 10006, /* A failure on the server */ +} +xdr_enum_serde!(mountstat3); diff --git a/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/mount_handlers.rs b/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/mount_handlers.rs new file mode 100644 index 0000000000..6b192911f2 --- /dev/null +++ b/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/mount_handlers.rs @@ -0,0 +1,207 @@ +use std::io::{Read, Write}; + +use num_derive::{FromPrimitive, ToPrimitive}; +use num_traits::cast::{FromPrimitive, ToPrimitive}; +use tracing::debug; + +use crate::context::RPCContext; +use crate::mount::*; +use crate::rpc::*; +use crate::xdr::*; + +/* +From RFC 1813 Appendix I +program MOUNT_PROGRAM { + version MOUNT_V3 { + void MOUNTPROC3_NULL(void) = 0; + mountres3 MOUNTPROC3_MNT(dirpath) = 1; + mountlist MOUNTPROC3_DUMP(void) = 2; + void MOUNTPROC3_UMNT(dirpath) = 3; + void MOUNTPROC3_UMNTALL(void) = 4; + exports MOUNTPROC3_EXPORT(void) = 5; + } = 3; +} = 100005; +*/ + +#[allow(non_camel_case_types)] +#[allow(clippy::upper_case_acronyms)] +#[derive(Copy, Clone, Debug, FromPrimitive, ToPrimitive)] +enum MountProgram { + MOUNTPROC3_NULL = 0, + MOUNTPROC3_MNT = 1, + MOUNTPROC3_DUMP = 2, + MOUNTPROC3_UMNT = 3, + MOUNTPROC3_UMNTALL = 4, + MOUNTPROC3_EXPORT = 5, + INVALID, +} + +pub async fn handle_mount( + xid: u32, + call: call_body, + input: &mut impl Read, + output: &mut impl Write, + context: &RPCContext, +) -> Result<(), anyhow::Error> { + let prog = MountProgram::from_u32(call.proc).unwrap_or(MountProgram::INVALID); + + match prog { + MountProgram::MOUNTPROC3_NULL => mountproc3_null(xid, input, output)?, + MountProgram::MOUNTPROC3_MNT => mountproc3_mnt(xid, input, output, context).await?, + MountProgram::MOUNTPROC3_UMNT => mountproc3_umnt(xid, input, output, context).await?, + MountProgram::MOUNTPROC3_UMNTALL => mountproc3_umnt_all(xid, input, output, context).await?, + MountProgram::MOUNTPROC3_EXPORT => mountproc3_export(xid, input, output, context)?, + _ => { + proc_unavail_reply_message(xid).serialize(output)?; + }, + } + Ok(()) +} + +pub fn mountproc3_null(xid: u32, _: &mut impl Read, output: &mut impl Write) -> Result<(), anyhow::Error> { + debug!("mountproc3_null({:?}) ", xid); + // build an RPC reply + let msg = make_success_reply(xid); + debug!("\t{:?} --> {:?}", xid, msg); + msg.serialize(output)?; + Ok(()) +} + +#[allow(non_camel_case_types)] +#[derive(Clone, Debug)] +struct mountres3_ok { + fhandle: fhandle3, // really same thing as nfs::nfs_fh3 + auth_flavors: Vec, +} +xdr_struct!(mountres3_ok, fhandle, auth_flavors); + +pub async fn mountproc3_mnt( + xid: u32, + input: &mut impl Read, + output: &mut impl Write, + context: &RPCContext, +) -> Result<(), anyhow::Error> { + let mut path = dirpath::new(); + path.deserialize(input)?; + let utf8path = std::str::from_utf8(&path).unwrap_or_default(); + debug!("mountproc3_mnt({:?},{:?}) ", xid, utf8path); + let path = if let Some(path) = utf8path.strip_prefix(context.export_name.as_str()) { + let path = path.trim_start_matches('/').trim_end_matches('/').trim().as_bytes(); + let mut new_path = Vec::with_capacity(path.len() + 1); + new_path.push(b'/'); + new_path.extend_from_slice(path); + new_path + } else { + // invalid export + debug!("{:?} --> no matching export", xid); + make_success_reply(xid).serialize(output)?; + mountstat3::MNT3ERR_NOENT.serialize(output)?; + return Ok(()); + }; + if let Ok(fileid) = context.vfs.path_to_id(&path).await { + let response = mountres3_ok { + fhandle: context.vfs.id_to_fh(fileid).data, + auth_flavors: vec![ + auth_flavor::AUTH_NULL.to_u32().unwrap(), + auth_flavor::AUTH_UNIX.to_u32().unwrap(), + ], + }; + debug!("{:?} --> {:?}", xid, response); + if let Some(ref chan) = context.mount_signal { + let _ = chan.send(true).await; + } + make_success_reply(xid).serialize(output)?; + mountstat3::MNT3_OK.serialize(output)?; + response.serialize(output)?; + } else { + debug!("{:?} --> MNT3ERR_NOENT", xid); + make_success_reply(xid).serialize(output)?; + mountstat3::MNT3ERR_NOENT.serialize(output)?; + } + Ok(()) +} + +/* + exports MOUNTPROC3_EXPORT(void) = 5; + + typedef struct groupnode *groups; + + struct groupnode { + name gr_name; + groups gr_next; + }; + + typedef struct exportnode *exports; + + struct exportnode { + dirpath ex_dir; + groups ex_groups; + exports ex_next; + }; + +DESCRIPTION + + Procedure EXPORT returns a list of all the exported file + systems and which clients are allowed to mount each one. + The names in the group list are implementation-specific + and cannot be directly interpreted by clients. These names + can represent hosts or groups of hosts. + +IMPLEMENTATION + + This procedure generally returns the contents of a list of + shared or exported file systems. These are the file + systems which are made available to NFS version 3 protocol + clients. + */ + +pub fn mountproc3_export( + xid: u32, + _: &mut impl Read, + output: &mut impl Write, + context: &RPCContext, +) -> Result<(), anyhow::Error> { + debug!("mountproc3_export({:?}) ", xid); + make_success_reply(xid).serialize(output)?; + true.serialize(output)?; + // dirpath + context.export_name.as_bytes().to_vec().serialize(output)?; + // groups + false.serialize(output)?; + // next exports + false.serialize(output)?; + Ok(()) +} + +pub async fn mountproc3_umnt( + xid: u32, + input: &mut impl Read, + output: &mut impl Write, + context: &RPCContext, +) -> Result<(), anyhow::Error> { + let mut path = dirpath::new(); + path.deserialize(input)?; + let utf8path = std::str::from_utf8(&path).unwrap_or_default(); + debug!("mountproc3_umnt({:?},{:?}) ", xid, utf8path); + if let Some(ref chan) = context.mount_signal { + let _ = chan.send(false).await; + } + make_success_reply(xid).serialize(output)?; + mountstat3::MNT3_OK.serialize(output)?; + Ok(()) +} + +pub async fn mountproc3_umnt_all( + xid: u32, + _input: &mut impl Read, + output: &mut impl Write, + context: &RPCContext, +) -> Result<(), anyhow::Error> { + debug!("mountproc3_umnt_all({:?}) ", xid); + if let Some(ref chan) = context.mount_signal { + let _ = chan.send(false).await; + } + make_success_reply(xid).serialize(output)?; + mountstat3::MNT3_OK.serialize(output)?; + Ok(()) +} diff --git a/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/nfs.rs b/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/nfs.rs new file mode 100644 index 0000000000..3412f4fc9f --- /dev/null +++ b/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/nfs.rs @@ -0,0 +1,560 @@ +// this is just a complete enumeration of everything in the RFC +#![allow(dead_code)] +// And its nice to keep the original RFC names and case +#![allow(non_camel_case_types)] + +use std::fmt; +use std::io::{Read, Write}; + +use byteorder::{ReadBytesExt, WriteBytesExt}; +use filetime; +use num_derive::{FromPrimitive, ToPrimitive}; +use num_traits::cast::FromPrimitive; + +use crate::xdr::*; + +// Transcribed from RFC 1813. + +// Section 2.2 Constants +/// These are the RPC constants needed to call the NFS Version 3 +/// service. They are given in decimal. +pub const PROGRAM: u32 = 100003; +pub const VERSION: u32 = 3; + +// Section 2.4 Sizes +// +/// The maximum size in bytes of the opaque file handle. +pub const NFS3_FHSIZE: u32 = 64; + +/// The size in bytes of the opaque cookie verifier passed by +/// READDIR and READDIRPLUS. +pub const NFS3_COOKIEVERFSIZE: u32 = 8; + +/// The size in bytes of the opaque verifier used for +/// exclusive CREATE. +pub const NFS3_CREATEVERFSIZE: u32 = 8; + +/// The size in bytes of the opaque verifier used for +/// asynchronous WRITE. +pub const NFS3_WRITEVERFSIZE: u32 = 8; + +// Section 2.5 Basic Data Types +#[allow(non_camel_case_types)] +#[derive(Default, Clone)] +pub struct nfsstring(pub Vec); +impl nfsstring { + pub fn len(&self) -> usize { + self.0.len() + } + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } +} +impl From> for nfsstring { + fn from(value: Vec) -> Self { + Self(value) + } +} +impl From<&[u8]> for nfsstring { + fn from(value: &[u8]) -> Self { + Self(value.into()) + } +} +impl AsRef<[u8]> for nfsstring { + fn as_ref(&self) -> &[u8] { + &self.0 + } +} + +impl std::ops::Deref for nfsstring { + type Target = Vec; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} +impl fmt::Debug for nfsstring { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", String::from_utf8_lossy(&self.0)) + } +} +impl fmt::Display for nfsstring { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", String::from_utf8_lossy(&self.0)) + } +} +pub type opaque = u8; +pub type filename3 = nfsstring; +pub type nfspath3 = nfsstring; +pub type fileid3 = u64; +pub type cookie3 = u64; +pub type cookieverf3 = [opaque; NFS3_COOKIEVERFSIZE as usize]; +pub type createverf3 = [opaque; NFS3_CREATEVERFSIZE as usize]; +pub type writeverf3 = [opaque; NFS3_WRITEVERFSIZE as usize]; +pub type uid3 = u32; +pub type gid3 = u32; +pub type size3 = u64; +pub type offset3 = u64; +pub type mode3 = u32; +pub type count3 = u32; + +#[allow(non_camel_case_types)] +#[derive(Copy, Clone, Debug, FromPrimitive, ToPrimitive)] +#[repr(u32)] +pub enum nfsstat3 { + /// Indicates the call completed successfully. + NFS3_OK = 0, + /// Not owner. The operation was not allowed because the + /// caller is either not a privileged user (root) or not the + /// owner of the target of the operation. + NFS3ERR_PERM = 1, + /// No such file or directory. The file or directory name + /// specified does not exist. + NFS3ERR_NOENT = 2, + /// I/O error. A hard error (for example, a disk error) + /// occurred while processing the requested operation. + NFS3ERR_IO = 5, + /// I/O error. No such device or address. + NFS3ERR_NXIO = 6, + /// Permission denied. The caller does not have the correct + /// permission to perform the requested operation. Contrast + /// this with NFS3ERR_PERM, which restricts itself to owner + /// or privileged user permission failures. + NFS3ERR_ACCES = 13, + /// File exists. The file specified already exists. + NFS3ERR_EXIST = 17, + /// Attempt to do a cross-device hard link. + NFS3ERR_XDEV = 18, + /// No such device. + NFS3ERR_NODEV = 19, + /// Not a directory. The caller specified a non-directory in + /// a directory operation. + NFS3ERR_NOTDIR = 20, + /// Is a directory. The caller specified a directory in a + /// non-directory operation. + NFS3ERR_ISDIR = 21, + /// Invalid argument or unsupported argument for an + /// operation. Two examples are attempting a READLINK on an + /// object other than a symbolic link or attempting to + /// SETATTR a time field on a server that does not support + /// this operation. + NFS3ERR_INVAL = 22, + /// File too large. The operation would have caused a file to + /// grow beyond the server's limit. + NFS3ERR_FBIG = 27, + /// No space left on device. The operation would have caused + /// the server's file system to exceed its limit. + NFS3ERR_NOSPC = 28, + /// Read-only file system. A modifying operation was + /// attempted on a read-only file system. + NFS3ERR_ROFS = 30, + /// Too many hard links. + NFS3ERR_MLINK = 31, + /// The filename in an operation was too long. + NFS3ERR_NAMETOOLONG = 63, + /// An attempt was made to remove a directory that was not empty. + NFS3ERR_NOTEMPTY = 66, + /// Resource (quota) hard limit exceeded. The user's resource + /// limit on the server has been exceeded. + NFS3ERR_DQUOT = 69, + /// Invalid file handle. The file handle given in the + /// arguments was invalid. The file referred to by that file + /// handle no longer exists or access to it has been + /// revoked. + NFS3ERR_STALE = 70, + /// Too many levels of remote in path. The file handle given + /// in the arguments referred to a file on a non-local file + /// system on the server. + NFS3ERR_REMOTE = 71, + /// Illegal NFS file handle. The file handle failed internal + /// consistency checks. + NFS3ERR_BADHANDLE = 10001, + /// Update synchronization mismatch was detected during a + /// SETATTR operation. + NFS3ERR_NOT_SYNC = 10002, + /// READDIR or READDIRPLUS cookie is stale + NFS3ERR_BAD_COOKIE = 10003, + /// Operation is not supported. + NFS3ERR_NOTSUPP = 10004, + /// Buffer or request is too small. + NFS3ERR_TOOSMALL = 10005, + /// An error occurred on the server which does not map to any + /// of the legal NFS version 3 protocol error values. The + /// client should translate this into an appropriate error. + /// UNIX clients may choose to translate this to EIO. + NFS3ERR_SERVERFAULT = 10006, + /// An attempt was made to create an object of a type not + /// supported by the server. + NFS3ERR_BADTYPE = 10007, + /// The server initiated the request, but was not able to + /// complete it in a timely fashion. The client should wait + /// and then try the request with a new RPC transaction ID. + /// For example, this error should be returned from a server + /// that supports hierarchical storage and receives a request + /// to process a file that has been migrated. In this case, + /// the server should start the immigration process and + /// respond to client with this error. + NFS3ERR_JUKEBOX = 10008, +} + +xdr_enum_serde!(nfsstat3); + +/// File Type +#[allow(non_camel_case_types)] +#[derive(Copy, Clone, Debug, Default, FromPrimitive, ToPrimitive)] +#[repr(u32)] +pub enum ftype3 { + /// Regular File + #[default] + NF3REG = 1, + /// Directory + NF3DIR = 2, + /// Block Special Device + NF3BLK = 3, + /// Character Special Device + NF3CHR = 4, + /// Symbolic Link + NF3LNK = 5, + /// Socket + NF3SOCK = 6, + /// Named Pipe + NF3FIFO = 7, +} +xdr_enum_serde!(ftype3); +/// Device Number information. Ex: Major / Minor device +#[allow(non_camel_case_types)] +#[derive(Copy, Clone, Debug, Default)] +#[repr(C)] +pub struct specdata3 { + pub specdata1: u32, + pub specdata2: u32, +} +xdr_struct!(specdata3, specdata1, specdata2); + +/// File Handle information +#[allow(non_camel_case_types)] +#[derive(Clone, Debug)] +pub struct nfs_fh3 { + pub data: Vec, +} +xdr_struct!(nfs_fh3, data); +#[allow(clippy::derivable_impls)] +impl Default for nfs_fh3 { + fn default() -> nfs_fh3 { + nfs_fh3 { data: Vec::new() } + } +} + +#[allow(non_camel_case_types)] +#[derive(Copy, Clone, Debug, Default)] +#[repr(C)] +pub struct nfstime3 { + pub seconds: u32, + pub nseconds: u32, +} +xdr_struct!(nfstime3, seconds, nseconds); + +impl From for filetime::FileTime { + fn from(time: nfstime3) -> Self { + filetime::FileTime::from_unix_time(time.seconds as i64, time.nseconds) + } +} + +#[allow(non_camel_case_types)] +#[derive(Copy, Clone, Debug, Default)] +pub struct fattr3 { + pub ftype: ftype3, + pub mode: mode3, + pub nlink: u32, + pub uid: uid3, + pub gid: gid3, + pub size: size3, + pub used: size3, + pub rdev: specdata3, + pub fsid: u64, + pub fileid: fileid3, + pub atime: nfstime3, + pub mtime: nfstime3, + pub ctime: nfstime3, +} +xdr_struct!(fattr3, ftype, mode, nlink, uid, gid, size, used, rdev, fsid, fileid, atime, mtime, ctime); + +// Section 3.3.19. Procedure 19: FSINFO - Get static file system Information +// The following constants are used in fsinfo to construct the bitmask 'properties', +// which represents the file system properties. + +/// If this bit is 1 (TRUE), the file system supports hard links. +pub const FSF_LINK: u32 = 0x0001; + +/// If this bit is 1 (TRUE), the file system supports symbolic links. +pub const FSF_SYMLINK: u32 = 0x0002; + +/// If this bit is 1 (TRUE), the information returned by +/// PATHCONF is identical for every file and directory +/// in the file system. If it is 0 (FALSE), the client +/// should retrieve PATHCONF information for each file +/// and directory as required. +pub const FSF_HOMOGENEOUS: u32 = 0x0008; + +/// If this bit is 1 (TRUE), the server will set the +/// times for a file via SETATTR if requested (to the +/// accuracy indicated by time_delta). If it is 0 +/// (FALSE), the server cannot set times as requested. +pub const FSF_CANSETTIME: u32 = 0x0010; + +#[allow(non_camel_case_types)] +#[derive(Debug, Default)] +pub struct fsinfo3 { + pub obj_attributes: post_op_attr, + pub rtmax: u32, + pub rtpref: u32, + pub rtmult: u32, + pub wtmax: u32, + pub wtpref: u32, + pub wtmult: u32, + pub dtpref: u32, + pub maxfilesize: size3, + pub time_delta: nfstime3, + pub properties: u32, +} +xdr_struct!( + fsinfo3, + obj_attributes, + rtmax, + rtpref, + rtmult, + wtmax, + wtpref, + wtmult, + dtpref, + maxfilesize, + time_delta, + properties +); + +#[allow(non_camel_case_types)] +#[derive(Copy, Clone, Debug, Default)] +pub struct wcc_attr { + pub size: size3, + pub mtime: nfstime3, + pub ctime: nfstime3, +} +xdr_struct!(wcc_attr, size, mtime, ctime); + +#[allow(non_camel_case_types)] +#[derive(Copy, Clone, Debug, Default)] +#[repr(u32)] +pub enum pre_op_attr { + #[default] + Void, + attributes(wcc_attr), +} +xdr_bool_union!(pre_op_attr, attributes, wcc_attr); + +#[allow(non_camel_case_types)] +#[derive(Copy, Clone, Debug, Default)] +#[repr(u32)] +pub enum post_op_attr { + #[default] + Void, + attributes(fattr3), +} +xdr_bool_union!(post_op_attr, attributes, fattr3); + +#[allow(non_camel_case_types)] +#[derive(Copy, Clone, Debug, Default)] +pub struct wcc_data { + pub before: pre_op_attr, + pub after: post_op_attr, +} +xdr_struct!(wcc_data, before, after); + +#[allow(non_camel_case_types)] +#[derive(Clone, Debug, Default)] +#[repr(u32)] +pub enum post_op_fh3 { + #[default] + Void, + handle(nfs_fh3), +} +xdr_bool_union!(post_op_fh3, handle, nfs_fh3); + +#[allow(non_camel_case_types)] +#[derive(Copy, Clone, Debug, FromPrimitive, ToPrimitive)] +#[repr(u32)] +/// This enum is only used as a discriminant for set_atime / set_mtime +/// and should not be used directly. +pub enum _time_how { + DONT_CHANGE = 0, + SET_TO_SERVER_TIME = 1, + SET_TO_CLIENT_TIME = 2, +} +xdr_enum_serde!(_time_how); + +#[allow(non_camel_case_types)] +#[derive(Copy, Clone, Debug)] +#[repr(u32)] +pub enum set_mode3 { + Void, + mode(mode3), +} +xdr_bool_union!(set_mode3, mode, mode3); + +#[allow(non_camel_case_types)] +#[derive(Copy, Clone, Debug)] +#[repr(u32)] +pub enum set_uid3 { + Void, + uid(uid3), +} +xdr_bool_union!(set_uid3, uid, uid3); + +#[allow(non_camel_case_types)] +#[derive(Copy, Clone, Debug)] +#[repr(u32)] +pub enum set_gid3 { + Void, + gid(gid3), +} +xdr_bool_union!(set_gid3, gid, gid3); + +#[allow(non_camel_case_types)] +#[derive(Copy, Clone, Debug)] +#[repr(u32)] +pub enum set_size3 { + Void, + size(size3), +} +xdr_bool_union!(set_size3, size, size3); + +#[allow(non_camel_case_types)] +#[derive(Copy, Clone, Debug)] +#[repr(u32)] +/// discriminant is time_how +pub enum set_atime { + DONT_CHANGE, + SET_TO_SERVER_TIME, + SET_TO_CLIENT_TIME(nfstime3), +} +impl XDR for set_atime { + fn serialize(&self, dest: &mut R) -> std::io::Result<()> { + match self { + set_atime::DONT_CHANGE => { + 0_u32.serialize(dest)?; + }, + set_atime::SET_TO_SERVER_TIME => { + 1_u32.serialize(dest)?; + }, + set_atime::SET_TO_CLIENT_TIME(v) => { + 2_u32.serialize(dest)?; + v.serialize(dest)?; + }, + } + Ok(()) + } + fn deserialize(&mut self, src: &mut R) -> std::io::Result<()> { + let mut c: u32 = 0; + c.deserialize(src)?; + if c == 0 { + *self = set_atime::DONT_CHANGE; + } else if c == 1 { + *self = set_atime::SET_TO_SERVER_TIME; + } else if c == 2 { + let mut r = nfstime3::default(); + r.deserialize(src)?; + *self = set_atime::SET_TO_CLIENT_TIME(r); + } else { + return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "Invalid value for set_atime")); + } + Ok(()) + } +} + +#[allow(non_camel_case_types)] +#[derive(Copy, Clone, Debug)] +#[repr(u32)] +/// discriminant is time_how +pub enum set_mtime { + DONT_CHANGE, + SET_TO_SERVER_TIME, + SET_TO_CLIENT_TIME(nfstime3), +} + +impl XDR for set_mtime { + fn serialize(&self, dest: &mut R) -> std::io::Result<()> { + match self { + set_mtime::DONT_CHANGE => { + 0_u32.serialize(dest)?; + }, + set_mtime::SET_TO_SERVER_TIME => { + 1_u32.serialize(dest)?; + }, + set_mtime::SET_TO_CLIENT_TIME(v) => { + 2_u32.serialize(dest)?; + v.serialize(dest)?; + }, + } + Ok(()) + } + fn deserialize(&mut self, src: &mut R) -> std::io::Result<()> { + let mut c: u32 = 0; + c.deserialize(src)?; + if c == 0 { + *self = set_mtime::DONT_CHANGE; + } else if c == 1 { + *self = set_mtime::SET_TO_SERVER_TIME; + } else if c == 2 { + let mut r = nfstime3::default(); + r.deserialize(src)?; + *self = set_mtime::SET_TO_CLIENT_TIME(r); + } else { + return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "Invalid value for set_mtime")); + } + Ok(()) + } +} + +#[allow(non_camel_case_types)] +#[derive(Copy, Clone, Debug)] +pub struct sattr3 { + pub mode: set_mode3, + pub uid: set_uid3, + pub gid: set_gid3, + pub size: set_size3, + pub atime: set_atime, + pub mtime: set_mtime, +} +xdr_struct!(sattr3, mode, uid, gid, size, atime, mtime); +impl Default for sattr3 { + fn default() -> sattr3 { + sattr3 { + mode: set_mode3::Void, + uid: set_uid3::Void, + gid: set_gid3::Void, + size: set_size3::Void, + atime: set_atime::DONT_CHANGE, + mtime: set_mtime::DONT_CHANGE, + } + } +} + +#[allow(non_camel_case_types)] +#[derive(Clone, Debug, Default)] +pub struct diropargs3 { + pub dir: nfs_fh3, + pub name: filename3, +} +xdr_struct!(diropargs3, dir, name); + +#[allow(non_camel_case_types)] +#[derive(Debug, Default)] +pub struct symlinkdata3 { + pub symlink_attributes: sattr3, + pub symlink_data: nfspath3, +} +xdr_struct!(symlinkdata3, symlink_attributes, symlink_data); + +/// We define the root handle here +pub fn get_root_mount_handle() -> Vec { + vec![0] +} diff --git a/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/nfs_handlers.rs b/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/nfs_handlers.rs new file mode 100644 index 0000000000..eebb594a94 --- /dev/null +++ b/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/nfs_handlers.rs @@ -0,0 +1,2302 @@ +#![allow(clippy::upper_case_acronyms)] +#![allow(dead_code)] +use std::io::{Read, Write}; + +use byteorder::{ReadBytesExt, WriteBytesExt}; +use num_derive::{FromPrimitive, ToPrimitive}; +use num_traits::cast::FromPrimitive; +use tracing::{debug, error, trace, warn}; + +use crate::context::RPCContext; +use crate::nfs; +use crate::rpc::*; +use crate::vfs::VFSCapabilities; +use crate::xdr::*; +/* +program NFS_PROGRAM { + version NFS_V3 { + + void + NFSPROC3_NULL(void) = 0; + + GETATTR3res + NFSPROC3_GETATTR(GETATTR3args) = 1; + + SETATTR3res + NFSPROC3_SETATTR(SETATTR3args) = 2; + + LOOKUP3res + NFSPROC3_LOOKUP(LOOKUP3args) = 3; + + ACCESS3res + NFSPROC3_ACCESS(ACCESS3args) = 4; + + READLINK3res + NFSPROC3_READLINK(READLINK3args) = 5; + + READ3res + NFSPROC3_READ(READ3args) = 6; + + WRITE3res + NFSPROC3_WRITE(WRITE3args) = 7; + + CREATE3res + NFSPROC3_CREATE(CREATE3args) = 8; + + MKDIR3res + NFSPROC3_MKDIR(MKDIR3args) = 9; + + SYMLINK3res + NFSPROC3_SYMLINK(SYMLINK3args) = 10; + + MKNOD3res + NFSPROC3_MKNOD(MKNOD3args) = 11; + + REMOVE3res + NFSPROC3_REMOVE(REMOVE3args) = 12; + + RMDIR3res + NFSPROC3_RMDIR(RMDIR3args) = 13; + + RENAME3res + NFSPROC3_RENAME(RENAME3args) = 14; + + LINK3res + NFSPROC3_LINK(LINK3args) = 15; + + READDIR3res + NFSPROC3_READDIR(READDIR3args) = 16; + + READDIRPLUS3res + NFSPROC3_READDIRPLUS(READDIRPLUS3args) = 17; + + FSSTAT3res + NFSPROC3_FSSTAT(FSSTAT3args) = 18; + + FSINFO3res + NFSPROC3_FSINFO(FSINFO3args) = 19; + + PATHCONF3res + NFSPROC3_PATHCONF(PATHCONF3args) = 20; + + COMMIT3res + NFSPROC3_COMMIT(COMMIT3args) = 21; + + } = 3; +} = 100003; +*/ + +#[allow(non_camel_case_types)] +#[allow(clippy::upper_case_acronyms)] +#[derive(Copy, Clone, Debug, FromPrimitive, ToPrimitive)] +enum NFSProgram { + NFSPROC3_NULL = 0, + NFSPROC3_GETATTR = 1, + NFSPROC3_SETATTR = 2, + NFSPROC3_LOOKUP = 3, + NFSPROC3_ACCESS = 4, + NFSPROC3_READLINK = 5, + NFSPROC3_READ = 6, + NFSPROC3_WRITE = 7, + NFSPROC3_CREATE = 8, + NFSPROC3_MKDIR = 9, + NFSPROC3_SYMLINK = 10, + NFSPROC3_MKNOD = 11, + NFSPROC3_REMOVE = 12, + NFSPROC3_RMDIR = 13, + NFSPROC3_RENAME = 14, + NFSPROC3_LINK = 15, + NFSPROC3_READDIR = 16, + NFSPROC3_READDIRPLUS = 17, + NFSPROC3_FSSTAT = 18, + NFSPROC3_FSINFO = 19, + NFSPROC3_PATHCONF = 20, + NFSPROC3_COMMIT = 21, + INVALID = 22, +} + +pub async fn handle_nfs( + xid: u32, + call: call_body, + input: &mut impl Read, + output: &mut impl Write, + context: &RPCContext, +) -> Result<(), anyhow::Error> { + if call.vers != nfs::VERSION { + warn!("Invalid NFS Version number {} != {}", call.vers, nfs::VERSION); + prog_mismatch_reply_message(xid, nfs::VERSION).serialize(output)?; + return Ok(()); + } + let prog = NFSProgram::from_u32(call.proc).unwrap_or(NFSProgram::INVALID); + + match prog { + NFSProgram::NFSPROC3_NULL => nfsproc3_null(xid, input, output)?, + NFSProgram::NFSPROC3_GETATTR => nfsproc3_getattr(xid, input, output, context).await?, + NFSProgram::NFSPROC3_LOOKUP => nfsproc3_lookup(xid, input, output, context).await?, + NFSProgram::NFSPROC3_READ => nfsproc3_read(xid, input, output, context).await?, + NFSProgram::NFSPROC3_FSINFO => nfsproc3_fsinfo(xid, input, output, context).await?, + NFSProgram::NFSPROC3_ACCESS => nfsproc3_access(xid, input, output, context).await?, + NFSProgram::NFSPROC3_PATHCONF => nfsproc3_pathconf(xid, input, output, context).await?, + NFSProgram::NFSPROC3_FSSTAT => nfsproc3_fsstat(xid, input, output, context).await?, + NFSProgram::NFSPROC3_READDIR => nfsproc3_readdir(xid, input, output, context).await?, + NFSProgram::NFSPROC3_READDIRPLUS => nfsproc3_readdirplus(xid, input, output, context).await?, + NFSProgram::NFSPROC3_WRITE => nfsproc3_write(xid, input, output, context).await?, + NFSProgram::NFSPROC3_CREATE => nfsproc3_create(xid, input, output, context).await?, + NFSProgram::NFSPROC3_SETATTR => nfsproc3_setattr(xid, input, output, context).await?, + NFSProgram::NFSPROC3_REMOVE => nfsproc3_remove(xid, input, output, context).await?, + NFSProgram::NFSPROC3_RMDIR => nfsproc3_remove(xid, input, output, context).await?, + NFSProgram::NFSPROC3_RENAME => nfsproc3_rename(xid, input, output, context).await?, + NFSProgram::NFSPROC3_LINK => nfsproc3_link(xid, input, output, context).await?, + NFSProgram::NFSPROC3_MKDIR => nfsproc3_mkdir(xid, input, output, context).await?, + NFSProgram::NFSPROC3_SYMLINK => nfsproc3_symlink(xid, input, output, context).await?, + NFSProgram::NFSPROC3_READLINK => nfsproc3_readlink(xid, input, output, context).await?, + NFSProgram::NFSPROC3_COMMIT => nfsproc3_commit(xid, input, output, context).await?, + _ => { + warn!("Unimplemented message {:?}", prog); + proc_unavail_reply_message(xid).serialize(output)?; + }, /* + NFSPROC3_MKNOD, + INVALID*/ + } + Ok(()) +} + +pub fn nfsproc3_null(xid: u32, _: &mut impl Read, output: &mut impl Write) -> Result<(), anyhow::Error> { + debug!("nfsproc3_null({:?}) ", xid); + let msg = make_success_reply(xid); + debug!("\t{:?} --> {:?}", xid, msg); + msg.serialize(output)?; + Ok(()) +} +/* +GETATTR3res NFSPROC3_GETATTR(GETATTR3args) = 1; +struct GETATTR3args { + nfs_fh3 object; +}; + +struct GETATTR3resok { + fattr3 obj_attributes; +}; + +union GETATTR3res switch (nfsstat3 status) { + case NFS3_OK: + GETATTR3resok resok; + default: + void; +}; + */ +pub async fn nfsproc3_getattr( + xid: u32, + input: &mut impl Read, + output: &mut impl Write, + context: &RPCContext, +) -> Result<(), anyhow::Error> { + let mut handle = nfs::nfs_fh3::default(); + handle.deserialize(input)?; + debug!("nfsproc3_getattr({:?},{:?}) ", xid, handle); + + let id = context.vfs.fh_to_id(&handle); + // fail if unable to convert file handle + if let Err(stat) = id { + make_success_reply(xid).serialize(output)?; + stat.serialize(output)?; + return Ok(()); + } + let id = id.unwrap(); + match context.vfs.getattr(id).await { + Ok(fh) => { + debug!(" {:?} --> {:?}", xid, fh); + make_success_reply(xid).serialize(output)?; + nfs::nfsstat3::NFS3_OK.serialize(output)?; + fh.serialize(output)?; + }, + Err(stat) => { + error!("getattr error {:?} --> {:?}", xid, stat); + make_success_reply(xid).serialize(output)?; + stat.serialize(output)?; + }, + } + Ok(()) +} + +/* + LOOKUP3res NFSPROC3_LOOKUP(LOOKUP3args) = 3; + + struct LOOKUP3args { + diropargs3 what; + }; + + struct LOOKUP3resok { + nfs_fh3 object; + post_op_attr obj_attributes; + post_op_attr dir_attributes; + }; + + struct LOOKUP3resfail { + post_op_attr dir_attributes; + }; + + union LOOKUP3res switch (nfsstat3 status) { + case NFS3_OK: + LOOKUP3resok resok; + default: + LOOKUP3resfail resfail; + }; +* +*/ +pub async fn nfsproc3_lookup( + xid: u32, + input: &mut impl Read, + output: &mut impl Write, + context: &RPCContext, +) -> Result<(), anyhow::Error> { + let mut dirops = nfs::diropargs3::default(); + dirops.deserialize(input)?; + debug!("nfsproc3_lookup({:?},{:?}) ", xid, dirops); + + let dirid = context.vfs.fh_to_id(&dirops.dir); + // fail if unable to convert file handle + if let Err(stat) = dirid { + make_success_reply(xid).serialize(output)?; + stat.serialize(output)?; + nfs::post_op_attr::Void.serialize(output)?; + return Ok(()); + } + let dirid = dirid.unwrap(); + + let dir_attr = match context.vfs.getattr(dirid).await { + Ok(v) => nfs::post_op_attr::attributes(v), + Err(_) => nfs::post_op_attr::Void, + }; + match context.vfs.lookup(dirid, &dirops.name).await { + Ok(fid) => { + let obj_attr = match context.vfs.getattr(fid).await { + Ok(v) => nfs::post_op_attr::attributes(v), + Err(_) => nfs::post_op_attr::Void, + }; + + debug!("lookup success {:?} --> {:?}", xid, obj_attr); + make_success_reply(xid).serialize(output)?; + nfs::nfsstat3::NFS3_OK.serialize(output)?; + context.vfs.id_to_fh(fid).serialize(output)?; + obj_attr.serialize(output)?; + dir_attr.serialize(output)?; + }, + Err(stat) => { + debug!("lookup error {:?}({:?}) --> {:?}", xid, dirops.name, stat); + make_success_reply(xid).serialize(output)?; + stat.serialize(output)?; + dir_attr.serialize(output)?; + }, + } + Ok(()) +} + +#[allow(non_camel_case_types)] +#[derive(Debug, Default)] +struct READ3args { + file: nfs::nfs_fh3, + offset: nfs::offset3, + count: nfs::count3, +} +xdr_struct!(READ3args, file, offset, count); + +#[allow(non_camel_case_types)] +#[derive(Debug, Default)] +struct READ3resok { + file_attributes: nfs::post_op_attr, + count: nfs::count3, + eof: bool, + data: Vec, +} +xdr_struct!(READ3resok, file_attributes, count, eof, data); +/* +READ3res NFSPROC3_READ(READ3args) = 6; + +struct READ3args { + nfs_fh3 file; + offset3 offset; + count3 count; +}; + +struct READ3resok { + post_op_attr file_attributes; + count3 count; + bool eof; + opaque data<>; +}; + +struct READ3resfail { + post_op_attr file_attributes; +}; + +union READ3res switch (nfsstat3 status) { +case NFS3_OK: + READ3resok resok; +default: + READ3resfail resfail; +}; + */ +pub async fn nfsproc3_read( + xid: u32, + input: &mut impl Read, + output: &mut impl Write, + context: &RPCContext, +) -> Result<(), anyhow::Error> { + let mut args = READ3args::default(); + args.deserialize(input)?; + debug!("nfsproc3_read({:?},{:?}) ", xid, args); + + let id = context.vfs.fh_to_id(&args.file); + if let Err(stat) = id { + make_success_reply(xid).serialize(output)?; + stat.serialize(output)?; + nfs::post_op_attr::Void.serialize(output)?; + return Ok(()); + } + let id = id.unwrap(); + + let obj_attr = match context.vfs.getattr(id).await { + Ok(v) => nfs::post_op_attr::attributes(v), + Err(_) => nfs::post_op_attr::Void, + }; + match context.vfs.read(id, args.offset, args.count).await { + Ok((bytes, eof)) => { + let res = READ3resok { + file_attributes: obj_attr, + count: bytes.len() as u32, + eof, + data: bytes, + }; + make_success_reply(xid).serialize(output)?; + nfs::nfsstat3::NFS3_OK.serialize(output)?; + res.serialize(output)?; + }, + Err(stat) => { + error!("read error {:?} --> {:?}", xid, stat); + make_success_reply(xid).serialize(output)?; + stat.serialize(output)?; + obj_attr.serialize(output)?; + }, + } + Ok(()) +} + +/* + + FSINFO3res NFSPROC3_FSINFO(FSINFO3args) = 19; + + const FSF3_LINK = 0x0001; + const FSF3_SYMLINK = 0x0002; + const FSF3_HOMOGENEOUS = 0x0008; + const FSF3_CANSETTIME = 0x0010; + + struct FSINFOargs { + nfs_fh3 fsroot; + }; + + struct FSINFO3resok { + post_op_attr obj_attributes; + uint32 rtmax; + uint32 rtpref; + uint32 rtmult; + uint32 wtmax; + uint32 wtpref; + uint32 wtmult; + uint32 dtpref; + size3 maxfilesize; + nfstime3 time_delta; + uint32 properties; + }; + + struct FSINFO3resfail { + post_op_attr obj_attributes; + }; + + union FSINFO3res switch (nfsstat3 status) { + case NFS3_OK: + FSINFO3resok resok; + default: + FSINFO3resfail resfail; + }; +*/ + +pub async fn nfsproc3_fsinfo( + xid: u32, + input: &mut impl Read, + output: &mut impl Write, + context: &RPCContext, +) -> Result<(), anyhow::Error> { + let mut handle = nfs::nfs_fh3::default(); + handle.deserialize(input)?; + debug!("nfsproc3_fsinfo({:?},{:?}) ", xid, handle); + + let id = context.vfs.fh_to_id(&handle); + // fail if unable to convert file handle + if let Err(stat) = id { + make_success_reply(xid).serialize(output)?; + stat.serialize(output)?; + nfs::post_op_attr::Void.serialize(output)?; + return Ok(()); + } + let id = id.unwrap(); + + match context.vfs.fsinfo(id).await { + Ok(fsinfo) => { + debug!(" {:?} --> {:?}", xid, fsinfo); + make_success_reply(xid).serialize(output)?; + nfs::nfsstat3::NFS3_OK.serialize(output)?; + fsinfo.serialize(output)?; + }, + Err(stat) => { + error!("fsinfo error {:?} --> {:?}", xid, stat); + make_success_reply(xid).serialize(output)?; + stat.serialize(output)?; + }, + } + Ok(()) +} + +const ACCESS3_READ: u32 = 0x0001; +const ACCESS3_LOOKUP: u32 = 0x0002; +const ACCESS3_MODIFY: u32 = 0x0004; +const ACCESS3_EXTEND: u32 = 0x0008; +const ACCESS3_DELETE: u32 = 0x0010; +const ACCESS3_EXECUTE: u32 = 0x0020; +/* + + ACCESS3res NFSPROC3_ACCESS(ACCESS3args) = 4; + + + struct ACCESS3args { + nfs_fh3 object; + uint32 access; + }; + + struct ACCESS3resok { + post_op_attr obj_attributes; + uint32 access; + }; + + struct ACCESS3resfail { + post_op_attr obj_attributes; + }; + + union ACCESS3res switch (nfsstat3 status) { + case NFS3_OK: + ACCESS3resok resok; + default: + ACCESS3resfail resfail; + }; +*/ + +pub async fn nfsproc3_access( + xid: u32, + input: &mut impl Read, + output: &mut impl Write, + context: &RPCContext, +) -> Result<(), anyhow::Error> { + let mut handle = nfs::nfs_fh3::default(); + handle.deserialize(input)?; + let mut access: u32 = 0; + access.deserialize(input)?; + debug!("nfsproc3_access({:?},{:?},{:?})", xid, handle, access); + + let id = context.vfs.fh_to_id(&handle); + // fail if unable to convert file handle + if let Err(stat) = id { + make_success_reply(xid).serialize(output)?; + stat.serialize(output)?; + nfs::post_op_attr::Void.serialize(output)?; + return Ok(()); + } + let id = id.unwrap(); + + let obj_attr = match context.vfs.getattr(id).await { + Ok(v) => nfs::post_op_attr::attributes(v), + Err(_) => nfs::post_op_attr::Void, + }; + // TODO better checks here + if !matches!(context.vfs.capabilities(), VFSCapabilities::ReadWrite) { + access &= ACCESS3_READ | ACCESS3_LOOKUP; + } + debug!(" {:?} ---> {:?}", xid, access); + make_success_reply(xid).serialize(output)?; + nfs::nfsstat3::NFS3_OK.serialize(output)?; + obj_attr.serialize(output)?; + access.serialize(output)?; + Ok(()) +} + +#[allow(non_camel_case_types)] +#[derive(Debug, Default)] +struct PATHCONF3resok { + obj_attributes: nfs::post_op_attr, + linkmax: u32, + name_max: u32, + no_trunc: bool, + chown_restricted: bool, + case_insensitive: bool, + case_preserving: bool, +} +xdr_struct!( + PATHCONF3resok, + obj_attributes, + linkmax, + name_max, + no_trunc, + chown_restricted, + case_insensitive, + case_preserving +); +/* + + PATHCONF3res NFSPROC3_PATHCONF(PATHCONF3args) = 20; + + struct PATHCONF3args { + nfs_fh3 object; + }; + + struct PATHCONF3resok { + post_op_attr obj_attributes; + uint32 linkmax; + uint32 name_max; + bool no_trunc; + bool chown_restricted; + bool case_insensitive; + bool case_preserving; + }; + + struct PATHCONF3resfail { + post_op_attr obj_attributes; + }; + + union PATHCONF3res switch (nfsstat3 status) { + case NFS3_OK: + PATHCONF3resok resok; + default: + PATHCONF3resfail resfail; + }; +*/ +pub async fn nfsproc3_pathconf( + xid: u32, + input: &mut impl Read, + output: &mut impl Write, + context: &RPCContext, +) -> Result<(), anyhow::Error> { + let mut handle = nfs::nfs_fh3::default(); + handle.deserialize(input)?; + debug!("nfsproc3_pathconf({:?},{:?})", xid, handle); + + let id = context.vfs.fh_to_id(&handle); + // fail if unable to convert file handle + if let Err(stat) = id { + make_success_reply(xid).serialize(output)?; + stat.serialize(output)?; + nfs::post_op_attr::Void.serialize(output)?; + return Ok(()); + } + let id = id.unwrap(); + + let obj_attr = match context.vfs.getattr(id).await { + Ok(v) => nfs::post_op_attr::attributes(v), + Err(_) => nfs::post_op_attr::Void, + }; + let res = PATHCONF3resok { + obj_attributes: obj_attr, + linkmax: 0, + name_max: 32768, + no_trunc: true, + chown_restricted: true, + case_insensitive: false, + case_preserving: true, + }; + debug!(" {:?} ---> {:?}", xid, res); + make_success_reply(xid).serialize(output)?; + nfs::nfsstat3::NFS3_OK.serialize(output)?; + res.serialize(output)?; + Ok(()) +} + +#[allow(non_camel_case_types)] +#[derive(Debug, Default)] +struct FSSTAT3resok { + obj_attributes: nfs::post_op_attr, + tbytes: nfs::size3, + fbytes: nfs::size3, + abytes: nfs::size3, + tfiles: nfs::size3, + ffiles: nfs::size3, + afiles: nfs::size3, + invarsec: u32, +} +xdr_struct!(FSSTAT3resok, obj_attributes, tbytes, fbytes, abytes, tfiles, ffiles, afiles, invarsec); + +/* + FSSTAT3res NFSPROC3_FSSTAT(FSSTAT3args) = 18; + + struct FSSTAT3args { + nfs_fh3 fsroot; + }; + + struct FSSTAT3resok { + post_op_attr obj_attributes; + size3 tbytes; + size3 fbytes; + size3 abytes; + size3 tfiles; + size3 ffiles; + size3 afiles; + uint32 invarsec; + }; + + struct FSSTAT3resfail { + post_op_attr obj_attributes; + }; + + union FSSTAT3res switch (nfsstat3 status) { + case NFS3_OK: + FSSTAT3resok resok; + default: + FSSTAT3resfail resfail; + }; + +*/ + +pub async fn nfsproc3_fsstat( + xid: u32, + input: &mut impl Read, + output: &mut impl Write, + context: &RPCContext, +) -> Result<(), anyhow::Error> { + let mut handle = nfs::nfs_fh3::default(); + handle.deserialize(input)?; + debug!("nfsproc3_fsstat({:?},{:?}) ", xid, handle); + let id = context.vfs.fh_to_id(&handle); + // fail if unable to convert file handle + if let Err(stat) = id { + make_success_reply(xid).serialize(output)?; + stat.serialize(output)?; + nfs::post_op_attr::Void.serialize(output)?; + return Ok(()); + } + let id = id.unwrap(); + + let obj_attr = match context.vfs.getattr(id).await { + Ok(v) => nfs::post_op_attr::attributes(v), + Err(_) => nfs::post_op_attr::Void, + }; + let res = FSSTAT3resok { + obj_attributes: obj_attr, + tbytes: 1024 * 1024 * 1024 * 1024, + fbytes: 1024 * 1024 * 1024 * 1024, + abytes: 1024 * 1024 * 1024 * 1024, + tfiles: 1024 * 1024 * 1024, + ffiles: 1024 * 1024 * 1024, + afiles: 1024 * 1024 * 1024, + invarsec: u32::MAX, + }; + make_success_reply(xid).serialize(output)?; + nfs::nfsstat3::NFS3_OK.serialize(output)?; + debug!(" {:?} ---> {:?}", xid, res); + res.serialize(output)?; + Ok(()) +} + +#[allow(non_camel_case_types)] +#[derive(Debug, Default)] +struct READDIRPLUS3args { + dir: nfs::nfs_fh3, + cookie: nfs::cookie3, + cookieverf: nfs::cookieverf3, + dircount: nfs::count3, + maxcount: nfs::count3, +} +xdr_struct!(READDIRPLUS3args, dir, cookie, cookieverf, dircount, maxcount); + +#[allow(non_camel_case_types)] +#[derive(Debug, Default)] +struct entry3 { + fileid: nfs::fileid3, + name: nfs::filename3, + cookie: nfs::cookie3, +} +xdr_struct!(entry3, fileid, name, cookie); + +#[allow(non_camel_case_types)] +#[derive(Debug, Default)] +struct READDIR3args { + dir: nfs::nfs_fh3, + cookie: nfs::cookie3, + cookieverf: nfs::cookieverf3, + dircount: nfs::count3, +} +xdr_struct!(READDIR3args, dir, cookie, cookieverf, dircount); + +#[allow(non_camel_case_types)] +#[derive(Debug, Default)] +struct entryplus3 { + fileid: nfs::fileid3, + name: nfs::filename3, + cookie: nfs::cookie3, + name_attributes: nfs::post_op_attr, + name_handle: nfs::post_op_fh3, +} +xdr_struct!(entryplus3, fileid, name, cookie, name_attributes, name_handle); +/* + + READDIRPLUS3res NFSPROC3_READDIRPLUS(READDIRPLUS3args) = 17; + + struct READDIRPLUS3args { + nfs_fh3 dir; + cookie3 cookie; + cookieverf3 cookieverf; + count3 dircount; + count3 maxcount; + }; + + + struct dirlistplus3 { + entryplus3 *entries; + bool eof; + }; + + struct READDIRPLUS3resok { + post_op_attr dir_attributes; + cookieverf3 cookieverf; + dirlistplus3 reply; + }; + struct READDIRPLUS3resfail { + post_op_attr dir_attributes; + }; +*/ +pub async fn nfsproc3_readdirplus( + xid: u32, + input: &mut impl Read, + output: &mut impl Write, + context: &RPCContext, +) -> Result<(), anyhow::Error> { + let mut args = READDIRPLUS3args::default(); + args.deserialize(input)?; + debug!("nfsproc3_readdirplus({:?},{:?}) ", xid, args); + + let dirid = context.vfs.fh_to_id(&args.dir); + // fail if unable to convert file handle + if let Err(stat) = dirid { + make_success_reply(xid).serialize(output)?; + stat.serialize(output)?; + nfs::post_op_attr::Void.serialize(output)?; + return Ok(()); + } + let dirid = dirid.unwrap(); + let dir_attr_maybe = context.vfs.getattr(dirid).await; + + let dir_attr = match dir_attr_maybe { + Ok(v) => nfs::post_op_attr::attributes(v), + Err(_) => nfs::post_op_attr::Void, + }; + + let dirversion = if let Ok(ref dir_attr) = dir_attr_maybe { + let cvf_version = (dir_attr.mtime.seconds as u64) << 32 | (dir_attr.mtime.nseconds as u64); + cvf_version.to_be_bytes() + } else { + nfs::cookieverf3::default() + }; + debug!(" -- Dir attr {:?}", dir_attr); + debug!(" -- Dir version {:?}", dirversion); + let has_version = args.cookieverf != nfs::cookieverf3::default(); + // initial call should hve empty cookie verf + // subsequent calls should have cvf_version as defined above + // which is based off the mtime. + // + // TODO: This is *far* too aggressive. and unnecessary. + // The client should maintain this correctly typically. + // + // The way cookieverf is handled is quite interesting... + // + // There are 2 notes in the RFC of interest: + // 1. If the + // server detects that the cookie is no longer valid, the + // server will reject the READDIR request with the status, + // NFS3ERR_BAD_COOKIE. The client should be careful to + // avoid holding directory entry cookies across operations + // that modify the directory contents, such as REMOVE and + // CREATE. + // + // 2. One implementation of the cookie-verifier mechanism might + // be for the server to use the modification time of the + // directory. This might be overly restrictive, however. A + // better approach would be to record the time of the last + // directory modification that changed the directory + // organization in a way that would make it impossible to + // reliably interpret a cookie. Servers in which directory + // cookies are always valid are free to use zero as the + // verifier always. + // + // Basically, as long as the cookie is "kinda" intepretable, + // we should keep accepting it. + // On testing, the Mac NFS client pretty much expects that + // especially on highly concurrent modifications to the directory. + // + // 1. If part way through a directory enumeration we fail with BAD_COOKIE + // if the directory contents change, the client listing may fail resulting + // in a "no such file or directory" error. + // 2. if we cache readdir results. i.e. we think of a readdir as two parts a. enumerating everything first b. the + // cookie is then used to paginate the enumeration we can run into file time synchronization issues. i.e. while + // one listing occurs and another file is touched, the listing may report an outdated file status. + // + // This cache also appears to have to be *quite* long lasting + // as the client may hold on to a directory enumerator + // with unbounded time. + // + // Basically, if we think about how linux directory listing works + // is that you just get an enumerator. There is no mechanic available for + // "restarting" a pagination and this enumerator is assumed to be valid + // even across directory modifications and should reflect changes + // immediately. + // + // The best solution is simply to really completely avoid sending + // BAD_COOKIE all together and to ignore the cookie mechanism. + // + /*if args.cookieverf != nfs::cookieverf3::default() && args.cookieverf != dirversion { + info!(" -- Dir version mismatch. Received {:?}", args.cookieverf); + make_success_reply(xid).serialize(output)?; + nfs::nfsstat3::NFS3ERR_BAD_COOKIE.serialize(output)?; + dir_attr.serialize(output)?; + return Ok(()); + }*/ + // subtract off the final entryplus* field (which must be false) and the eof + let max_bytes_allowed = args.maxcount as usize - 128; + // args.dircount is bytes of just fileid, name, cookie. + // This is hard to ballpark, so we just divide it by 16 + let estimated_max_results = args.dircount / 16; + let max_dircount_bytes = args.dircount as usize; + let mut ctr = 0; + match context.vfs.readdir(dirid, args.cookie, estimated_max_results as usize).await { + Ok(result) => { + // we count dir_count seperately as it is just a subset of fields + let mut accumulated_dircount: usize = 0; + let mut all_entries_written = true; + + // this is a wrapper around a writer that also just counts the number of bytes + // written + let mut counting_output = crate::write_counter::WriteCounter::new(output); + + make_success_reply(xid).serialize(&mut counting_output)?; + nfs::nfsstat3::NFS3_OK.serialize(&mut counting_output)?; + dir_attr.serialize(&mut counting_output)?; + dirversion.serialize(&mut counting_output)?; + for entry in result.entries { + let obj_attr = entry.attr; + let handle = nfs::post_op_fh3::handle(context.vfs.id_to_fh(entry.fileid)); + + let entry = entryplus3 { + fileid: entry.fileid, + name: entry.name, + cookie: entry.fileid, + name_attributes: nfs::post_op_attr::attributes(obj_attr), + name_handle: handle, + }; + // write the entry into a buffer first + let mut write_buf: Vec = Vec::new(); + let mut write_cursor = std::io::Cursor::new(&mut write_buf); + // true flag for the entryplus3* to mark that this contains an entry + true.serialize(&mut write_cursor)?; + entry.serialize(&mut write_cursor)?; + write_cursor.flush()?; + let added_dircount = std::mem::size_of::() // fileid + + std::mem::size_of::() + entry.name.len() // name + + std::mem::size_of::(); // cookie + let added_output_bytes = write_buf.len(); + // check if we can write without hitting the limits + if added_output_bytes + counting_output.bytes_written() < max_bytes_allowed + && added_dircount + accumulated_dircount < max_dircount_bytes + { + trace!(" -- dirent {:?}", entry); + // commit the entry + ctr += 1; + counting_output.write_all(&write_buf)?; + accumulated_dircount += added_dircount; + trace!( + " -- lengths: {:?} / {:?} {:?} / {:?}", + accumulated_dircount, + max_dircount_bytes, + counting_output.bytes_written(), + max_bytes_allowed + ); + } else { + trace!(" -- insufficient space. truncating"); + all_entries_written = false; + break; + } + } + // false flag for the final entryplus* linked list + false.serialize(&mut counting_output)?; + // eof flag is only valid here if we wrote everything + if all_entries_written { + debug!(" -- readdir eof {:?}", result.end); + result.end.serialize(&mut counting_output)?; + } else { + debug!(" -- readdir eof {:?}", false); + false.serialize(&mut counting_output)?; + } + debug!( + "readir {}, has_version {}, start at {}, flushing {} entries, complete {}", + dirid, has_version, args.cookie, ctr, all_entries_written + ); + }, + Err(stat) => { + error!("readdir error {:?} --> {:?} ", xid, stat); + make_success_reply(xid).serialize(output)?; + stat.serialize(output)?; + dir_attr.serialize(output)?; + }, + }; + Ok(()) +} + +pub async fn nfsproc3_readdir( + xid: u32, + input: &mut impl Read, + output: &mut impl Write, + context: &RPCContext, +) -> Result<(), anyhow::Error> { + let mut args = READDIR3args::default(); + args.deserialize(input)?; + debug!("nfsproc3_readdirplus({:?},{:?}) ", xid, args); + + let dirid = context.vfs.fh_to_id(&args.dir); + // fail if unable to convert file handle + if let Err(stat) = dirid { + make_success_reply(xid).serialize(output)?; + stat.serialize(output)?; + nfs::post_op_attr::Void.serialize(output)?; + return Ok(()); + } + let dirid = dirid.unwrap(); + let dir_attr_maybe = context.vfs.getattr(dirid).await; + + let dir_attr = match dir_attr_maybe { + Ok(v) => nfs::post_op_attr::attributes(v), + Err(_) => nfs::post_op_attr::Void, + }; + + let dirversion = if let Ok(ref dir_attr) = dir_attr_maybe { + let cvf_version = (dir_attr.mtime.seconds as u64) << 32 | (dir_attr.mtime.nseconds as u64); + cvf_version.to_be_bytes() + } else { + nfs::cookieverf3::default() + }; + debug!(" -- Dir attr {:?}", dir_attr); + debug!(" -- Dir version {:?}", dirversion); + let has_version = args.cookieverf != nfs::cookieverf3::default(); + // subtract off the final entryplus* field (which must be false) and the eof + let max_bytes_allowed = args.dircount as usize - 128; + // args.dircount is bytes of just fileid, name, cookie. + // This is hard to ballpark, so we just divide it by 16 + let estimated_max_results = args.dircount / 16; + let mut ctr = 0; + match context.vfs.readdir_simple(dirid, estimated_max_results as usize).await { + Ok(result) => { + // we count dir_count seperately as it is just a subset of fields + let mut accumulated_dircount: usize = 0; + let mut all_entries_written = true; + + // this is a wrapper around a writer that also just counts the number of bytes + // written + let mut counting_output = crate::write_counter::WriteCounter::new(output); + + make_success_reply(xid).serialize(&mut counting_output)?; + nfs::nfsstat3::NFS3_OK.serialize(&mut counting_output)?; + dir_attr.serialize(&mut counting_output)?; + dirversion.serialize(&mut counting_output)?; + for entry in result.entries { + let entry = entry3 { + fileid: entry.fileid, + name: entry.name, + cookie: entry.fileid, + }; + // write the entry into a buffer first + let mut write_buf: Vec = Vec::new(); + let mut write_cursor = std::io::Cursor::new(&mut write_buf); + // true flag for the entryplus3* to mark that this contains an entry + true.serialize(&mut write_cursor)?; + entry.serialize(&mut write_cursor)?; + write_cursor.flush()?; + let added_dircount = std::mem::size_of::() // fileid + + std::mem::size_of::() + entry.name.len() // name + + std::mem::size_of::(); // cookie + let added_output_bytes = write_buf.len(); + // check if we can write without hitting the limits + if added_output_bytes + counting_output.bytes_written() < max_bytes_allowed { + trace!(" -- dirent {:?}", entry); + // commit the entry + ctr += 1; + counting_output.write_all(&write_buf)?; + accumulated_dircount += added_dircount; + trace!( + " -- lengths: {:?} / {:?} / {:?}", + accumulated_dircount, + counting_output.bytes_written(), + max_bytes_allowed + ); + } else { + trace!(" -- insufficient space. truncating"); + all_entries_written = false; + break; + } + } + // false flag for the final entryplus* linked list + false.serialize(&mut counting_output)?; + // eof flag is only valid here if we wrote everything + if all_entries_written { + debug!(" -- readdir eof {:?}", result.end); + result.end.serialize(&mut counting_output)?; + } else { + debug!(" -- readdir eof {:?}", false); + false.serialize(&mut counting_output)?; + } + debug!( + "readir {}, has_version {}, start at {}, flushing {} entries, complete {}", + dirid, has_version, args.cookie, ctr, all_entries_written + ); + }, + Err(stat) => { + error!("readdir error {:?} --> {:?} ", xid, stat); + make_success_reply(xid).serialize(output)?; + stat.serialize(output)?; + dir_attr.serialize(output)?; + }, + }; + Ok(()) +} + +#[allow(non_camel_case_types)] +#[derive(Copy, Clone, Debug, Default, FromPrimitive, ToPrimitive)] +#[repr(u32)] +pub enum stable_how { + #[default] + UNSTABLE = 0, + DATA_SYNC = 1, + FILE_SYNC = 2, +} +xdr_enum_serde!(stable_how); + +#[allow(non_camel_case_types)] +#[derive(Debug, Default)] +struct WRITE3args { + file: nfs::nfs_fh3, + offset: nfs::offset3, + count: nfs::count3, + stable: u32, + data: Vec, +} +xdr_struct!(WRITE3args, file, offset, count, stable, data); + +#[allow(non_camel_case_types)] +#[derive(Debug, Default)] +struct WRITE3resok { + file_wcc: nfs::wcc_data, + count: nfs::count3, + committed: stable_how, + verf: nfs::writeverf3, +} +xdr_struct!(WRITE3resok, file_wcc, count, committed, verf); +/* +enum stable_how { + UNSTABLE = 0, + DATA_SYNC = 1, + FILE_SYNC = 2 +}; + + +struct WRITE3args { + nfs_fh3 file; + offset3 offset; + count3 count; + stable_how stable; + opaque data<>; +}; + +struct WRITE3resok { + wcc_data file_wcc; + count3 count; + stable_how committed; + writeverf3 verf; +}; + + +struct WRITE3resfail { + wcc_data file_wcc; +}; + + +union WRITE3res switch (nfsstat3 status) { + case NFS3_OK: + WRITE3resok resok; + default: + WRITE3resfail resfail; +}; + + */ +pub async fn nfsproc3_write( + xid: u32, + input: &mut impl Read, + output: &mut impl Write, + context: &RPCContext, +) -> Result<(), anyhow::Error> { + // if we do not have write capabilities + if !matches!(context.vfs.capabilities(), VFSCapabilities::ReadWrite) { + warn!("No write capabilities."); + make_success_reply(xid).serialize(output)?; + nfs::nfsstat3::NFS3ERR_ROFS.serialize(output)?; + nfs::wcc_data::default().serialize(output)?; + return Ok(()); + } + + let mut args = WRITE3args::default(); + args.deserialize(input)?; + debug!("nfsproc3_write({:?},...) ", xid); + // sanity check the length + if args.data.len() != args.count as usize { + garbage_args_reply_message(xid).serialize(output)?; + return Ok(()); + } + + let id = context.vfs.fh_to_id(&args.file); + if let Err(stat) = id { + make_success_reply(xid).serialize(output)?; + stat.serialize(output)?; + nfs::wcc_data::default().serialize(output)?; + return Ok(()); + } + let id = id.unwrap(); + + // get the object attributes before the write + let pre_obj_attr = match context.vfs.getattr(id).await { + Ok(v) => { + let wccattr = nfs::wcc_attr { + size: v.size, + mtime: v.mtime, + ctime: v.ctime, + }; + nfs::pre_op_attr::attributes(wccattr) + }, + Err(_) => nfs::pre_op_attr::Void, + }; + + match context.vfs.write(id, args.offset, &args.data).await { + Ok(fattr) => { + debug!("write success {:?} --> {:?}", xid, fattr); + let res = WRITE3resok { + file_wcc: nfs::wcc_data { + before: pre_obj_attr, + after: nfs::post_op_attr::attributes(fattr), + }, + count: args.count, + committed: stable_how::FILE_SYNC, + verf: context.vfs.serverid(), + }; + make_success_reply(xid).serialize(output)?; + nfs::nfsstat3::NFS3_OK.serialize(output)?; + res.serialize(output)?; + }, + Err(stat) => { + error!("write error {:?} --> {:?}", xid, stat); + make_success_reply(xid).serialize(output)?; + stat.serialize(output)?; + nfs::wcc_data::default().serialize(output)?; + }, + } + Ok(()) +} + +#[allow(non_camel_case_types)] +#[derive(Debug, Default)] +struct COMMIT3args { + file: nfs::nfs_fh3, + offset: nfs::offset3, + count: nfs::count3, +} +xdr_struct!(COMMIT3args, file, offset, count); + +#[allow(non_camel_case_types)] +#[derive(Debug, Default)] +struct COMMIT3resok { + file_wcc: nfs::wcc_data, + verf: nfs::writeverf3, +} +xdr_struct!(COMMIT3resok, file_wcc, verf); + +/// NFSv3 COMMIT. The VFS implementation decides how to make the requested +/// range durable. The write verifier must match WRITE responses from the same +/// server lifetime. +pub async fn nfsproc3_commit( + xid: u32, + input: &mut impl Read, + output: &mut impl Write, + context: &RPCContext, +) -> Result<(), anyhow::Error> { + let mut args = COMMIT3args::default(); + args.deserialize(input)?; + debug!("nfsproc3_commit({:?}, {:?}, {:?})", xid, args.offset, args.count); + + let id = match context.vfs.fh_to_id(&args.file) { + Ok(value) => value, + Err(status) => { + make_success_reply(xid).serialize(output)?; + status.serialize(output)?; + nfs::wcc_data::default().serialize(output)?; + return Ok(()); + }, + }; + let pre_attr = match context.vfs.getattr(id).await { + Ok(value) => nfs::pre_op_attr::attributes(nfs::wcc_attr { + size: value.size, + mtime: value.mtime, + ctime: value.ctime, + }), + Err(_) => nfs::pre_op_attr::Void, + }; + + match context.vfs.commit(id, args.offset, args.count).await { + Ok(value) => { + make_success_reply(xid).serialize(output)?; + nfs::nfsstat3::NFS3_OK.serialize(output)?; + COMMIT3resok { + file_wcc: nfs::wcc_data { + before: pre_attr, + after: nfs::post_op_attr::attributes(value), + }, + verf: context.vfs.serverid(), + } + .serialize(output)?; + }, + Err(status) => { + let post_attr = match context.vfs.getattr(id).await { + Ok(value) => nfs::post_op_attr::attributes(value), + Err(_) => nfs::post_op_attr::Void, + }; + make_success_reply(xid).serialize(output)?; + status.serialize(output)?; + nfs::wcc_data { + before: pre_attr, + after: post_attr, + } + .serialize(output)?; + }, + } + Ok(()) +} + +#[allow(non_camel_case_types)] +#[derive(Copy, Clone, Debug, Default, FromPrimitive, ToPrimitive)] +#[repr(u32)] +pub enum createmode3 { + #[default] + UNCHECKED = 0, + GUARDED = 1, + EXCLUSIVE = 2, +} +xdr_enum_serde!(createmode3); +/* +CREATE3res NFSPROC3_CREATE(CREATE3args) = 8; + + enum createmode3 { + UNCHECKED = 0, + GUARDED = 1, + EXCLUSIVE = 2 + }; + + union createhow3 switch (createmode3 mode) { + case UNCHECKED: + case GUARDED: + sattr3 obj_attributes; + case EXCLUSIVE: + createverf3 verf; + }; + + struct CREATE3args { + diropargs3 where; + createhow3 how; + }; + + struct CREATE3resok { + post_op_fh3 obj; + post_op_attr obj_attributes; + wcc_data dir_wcc; + }; + + struct CREATE3resfail { + wcc_data dir_wcc; + }; + + union CREATE3res switch (nfsstat3 status) { + case NFS3_OK: + CREATE3resok resok; + default: + CREATE3resfail resfail; + }; +*/ + +pub async fn nfsproc3_create( + xid: u32, + input: &mut impl Read, + output: &mut impl Write, + context: &RPCContext, +) -> Result<(), anyhow::Error> { + // if we do not have write capabilities + if !matches!(context.vfs.capabilities(), VFSCapabilities::ReadWrite) { + warn!("No write capabilities."); + make_success_reply(xid).serialize(output)?; + nfs::nfsstat3::NFS3ERR_ROFS.serialize(output)?; + nfs::wcc_data::default().serialize(output)?; + return Ok(()); + } + + let mut dirops = nfs::diropargs3::default(); + dirops.deserialize(input)?; + let mut createhow = createmode3::default(); + createhow.deserialize(input)?; + + debug!("nfsproc3_create({:?}, {:?}, {:?}) ", xid, dirops, createhow); + + // find the directory we are supposed to create the + // new file in + let dirid = context.vfs.fh_to_id(&dirops.dir); + if let Err(stat) = dirid { + // directory does not exist + make_success_reply(xid).serialize(output)?; + stat.serialize(output)?; + nfs::wcc_data::default().serialize(output)?; + error!("Directory does not exist"); + return Ok(()); + } + // found the directory, get the attributes + let dirid = dirid.unwrap(); + + // get the object attributes before the write + let pre_dir_attr = match context.vfs.getattr(dirid).await { + Ok(v) => { + let wccattr = nfs::wcc_attr { + size: v.size, + mtime: v.mtime, + ctime: v.ctime, + }; + nfs::pre_op_attr::attributes(wccattr) + }, + Err(stat) => { + error!("Cannot stat directory"); + make_success_reply(xid).serialize(output)?; + stat.serialize(output)?; + nfs::wcc_data::default().serialize(output)?; + return Ok(()); + }, + }; + let mut target_attributes = nfs::sattr3::default(); + + match createhow { + createmode3::UNCHECKED => { + target_attributes.deserialize(input)?; + debug!("create unchecked {:?}", target_attributes); + }, + createmode3::GUARDED => { + target_attributes.deserialize(input)?; + debug!("create guarded {:?}", target_attributes); + if context.vfs.lookup(dirid, &dirops.name).await.is_ok() { + // file exists. Fail with NFS3ERR_EXIST. + // Re-read dir attributes + // for post op attr + let post_dir_attr = match context.vfs.getattr(dirid).await { + Ok(v) => nfs::post_op_attr::attributes(v), + Err(_) => nfs::post_op_attr::Void, + }; + + make_success_reply(xid).serialize(output)?; + nfs::nfsstat3::NFS3ERR_EXIST.serialize(output)?; + nfs::wcc_data { + before: pre_dir_attr, + after: post_dir_attr, + } + .serialize(output)?; + return Ok(()); + } + }, + createmode3::EXCLUSIVE => { + debug!("create exclusive"); + }, + } + + let fid: Result; + let postopattr: nfs::post_op_attr; + // fill in the fid and post op attr here + if matches!(createhow, createmode3::EXCLUSIVE) { + // the API for exclusive is very slightly different + // We are not returning a post op attribute + fid = context.vfs.create_exclusive(dirid, &dirops.name).await; + postopattr = nfs::post_op_attr::Void; + } else { + // create! + let res = context.vfs.create(dirid, &dirops.name, target_attributes).await; + fid = res.map(|x| x.0); + postopattr = if let Ok((_, fattr)) = res { + nfs::post_op_attr::attributes(fattr) + } else { + nfs::post_op_attr::Void + }; + } + + // Re-read dir attributes for post op attr + let post_dir_attr = match context.vfs.getattr(dirid).await { + Ok(v) => nfs::post_op_attr::attributes(v), + Err(_) => nfs::post_op_attr::Void, + }; + let wcc_res = nfs::wcc_data { + before: pre_dir_attr, + after: post_dir_attr, + }; + + match fid { + Ok(fid) => { + debug!("create success --> {:?}, {:?}", fid, postopattr); + make_success_reply(xid).serialize(output)?; + nfs::nfsstat3::NFS3_OK.serialize(output)?; + // serialize CREATE3resok + let fh = context.vfs.id_to_fh(fid); + nfs::post_op_fh3::handle(fh).serialize(output)?; + postopattr.serialize(output)?; + wcc_res.serialize(output)?; + }, + Err(e) => { + error!("create error --> {:?}", e); + // serialize CREATE3resfail + make_success_reply(xid).serialize(output)?; + e.serialize(output)?; + wcc_res.serialize(output)?; + }, + } + + Ok(()) +} + +#[allow(non_camel_case_types)] +#[derive(Copy, Clone, Debug, Default)] +#[repr(u32)] +pub enum sattrguard3 { + #[default] + Void, + obj_ctime(nfs::nfstime3), +} +xdr_bool_union!(sattrguard3, obj_ctime, nfs::nfstime3); + +#[allow(non_camel_case_types)] +#[derive(Clone, Debug, Default)] +struct SETATTR3args { + object: nfs::nfs_fh3, + new_attribute: nfs::sattr3, + guard: sattrguard3, +} +xdr_struct!(SETATTR3args, object, new_attribute, guard); + +/* + SETATTR3res NFSPROC3_SETATTR(SETATTR3args) = 2; + + union sattrguard3 switch (bool check) { + case TRUE: + nfstime3 obj_ctime; + case FALSE: + void; + }; + + struct SETATTR3args { + nfs_fh3 object; + sattr3 new_attributes; + sattrguard3 guard; + }; + + struct SETATTR3resok { + wcc_data obj_wcc; + }; + + struct SETATTR3resfail { + wcc_data obj_wcc; + }; + union SETATTR3res switch (nfsstat3 status) { + case NFS3_OK: + SETATTR3resok resok; + default: + SETATTR3resfail resfail; + }; +*/ + +pub async fn nfsproc3_setattr( + xid: u32, + input: &mut impl Read, + output: &mut impl Write, + context: &RPCContext, +) -> Result<(), anyhow::Error> { + if !matches!(context.vfs.capabilities(), VFSCapabilities::ReadWrite) { + warn!("No write capabilities."); + make_success_reply(xid).serialize(output)?; + nfs::nfsstat3::NFS3ERR_ROFS.serialize(output)?; + nfs::wcc_data::default().serialize(output)?; + return Ok(()); + } + let mut args = SETATTR3args::default(); + args.deserialize(input)?; + debug!("nfsproc3_setattr({:?},{:?}) ", xid, args); + + let id = context.vfs.fh_to_id(&args.object); + // fail if unable to convert file handle + if let Err(stat) = id { + make_success_reply(xid).serialize(output)?; + stat.serialize(output)?; + return Ok(()); + } + let id = id.unwrap(); + + let ctime; + + let pre_op_attr = match context.vfs.getattr(id).await { + Ok(v) => { + let wccattr = nfs::wcc_attr { + size: v.size, + mtime: v.mtime, + ctime: v.ctime, + }; + ctime = v.ctime; + nfs::pre_op_attr::attributes(wccattr) + }, + Err(stat) => { + make_success_reply(xid).serialize(output)?; + stat.serialize(output)?; + nfs::wcc_data::default().serialize(output)?; + return Ok(()); + }, + }; + // handle the guard + match args.guard { + sattrguard3::Void => {}, + sattrguard3::obj_ctime(c) => { + if c.seconds != ctime.seconds || c.nseconds != ctime.nseconds { + make_success_reply(xid).serialize(output)?; + nfs::nfsstat3::NFS3ERR_NOT_SYNC.serialize(output)?; + nfs::wcc_data::default().serialize(output)?; + } + }, + } + + match context.vfs.setattr(id, args.new_attribute).await { + Ok(post_op_attr) => { + debug!(" setattr success {:?} --> {:?}", xid, post_op_attr); + let wcc_res = nfs::wcc_data { + before: pre_op_attr, + after: nfs::post_op_attr::attributes(post_op_attr), + }; + make_success_reply(xid).serialize(output)?; + nfs::nfsstat3::NFS3_OK.serialize(output)?; + wcc_res.serialize(output)?; + }, + Err(stat) => { + error!("setattr error {:?} --> {:?}", xid, stat); + make_success_reply(xid).serialize(output)?; + stat.serialize(output)?; + nfs::wcc_data::default().serialize(output)?; + }, + } + Ok(()) +} + +/* + REMOVE3res NFSPROC3_REMOVE(REMOVE3args) = 12; + + struct REMOVE3args { + diropargs3 object; + }; + + struct REMOVE3resok { + wcc_data dir_wcc; + }; + + struct REMOVE3resfail { + wcc_data dir_wcc; + }; + + union REMOVE3res switch (nfsstat3 status) { + case NFS3_OK: + REMOVE3resok resok; + default: + REMOVE3resfail resfail; + }; + + RMDIR is basically identically structured +*/ + +pub async fn nfsproc3_remove( + xid: u32, + input: &mut impl Read, + output: &mut impl Write, + context: &RPCContext, +) -> Result<(), anyhow::Error> { + // if we do not have write capabilities + if !matches!(context.vfs.capabilities(), VFSCapabilities::ReadWrite) { + warn!("No write capabilities."); + make_success_reply(xid).serialize(output)?; + nfs::nfsstat3::NFS3ERR_ROFS.serialize(output)?; + nfs::wcc_data::default().serialize(output)?; + return Ok(()); + } + + let mut dirops = nfs::diropargs3::default(); + dirops.deserialize(input)?; + + debug!("nfsproc3_remove({:?}, {:?}) ", xid, dirops); + + // find the directory with the file + let dirid = context.vfs.fh_to_id(&dirops.dir); + if let Err(stat) = dirid { + // directory does not exist + make_success_reply(xid).serialize(output)?; + stat.serialize(output)?; + nfs::wcc_data::default().serialize(output)?; + error!("Directory does not exist"); + return Ok(()); + } + let dirid = dirid.unwrap(); + + // get the object attributes before the write + let pre_dir_attr = match context.vfs.getattr(dirid).await { + Ok(v) => { + let wccattr = nfs::wcc_attr { + size: v.size, + mtime: v.mtime, + ctime: v.ctime, + }; + nfs::pre_op_attr::attributes(wccattr) + }, + Err(stat) => { + error!("Cannot stat directory"); + make_success_reply(xid).serialize(output)?; + stat.serialize(output)?; + nfs::wcc_data::default().serialize(output)?; + return Ok(()); + }, + }; + + // delete! + let res = context.vfs.remove(dirid, &dirops.name).await; + + // Re-read dir attributes for post op attr + let post_dir_attr = match context.vfs.getattr(dirid).await { + Ok(v) => nfs::post_op_attr::attributes(v), + Err(_) => nfs::post_op_attr::Void, + }; + let wcc_res = nfs::wcc_data { + before: pre_dir_attr, + after: post_dir_attr, + }; + + match res { + Ok(()) => { + debug!("remove success"); + make_success_reply(xid).serialize(output)?; + nfs::nfsstat3::NFS3_OK.serialize(output)?; + wcc_res.serialize(output)?; + }, + Err(e) => { + error!("remove error {:?} --> {:?}", xid, e); + // serialize CREATE3resfail + make_success_reply(xid).serialize(output)?; + e.serialize(output)?; + wcc_res.serialize(output)?; + }, + } + + Ok(()) +} + +/* + RENAME3res NFSPROC3_RENAME(RENAME3args) = 14; + + struct RENAME3args { + diropargs3 from; + diropargs3 to; + }; + + struct RENAME3resok { + wcc_data fromdir_wcc; + wcc_data todir_wcc; + }; + + struct RENAME3resfail { + wcc_data fromdir_wcc; + wcc_data todir_wcc; + }; + + union RENAME3res switch (nfsstat3 status) { + case NFS3_OK: + RENAME3resok resok; + default: + RENAME3resfail resfail; + }; +*/ + +pub async fn nfsproc3_rename( + xid: u32, + input: &mut impl Read, + output: &mut impl Write, + context: &RPCContext, +) -> Result<(), anyhow::Error> { + // if we do not have write capabilities + if !matches!(context.vfs.capabilities(), VFSCapabilities::ReadWrite) { + warn!("No write capabilities."); + make_success_reply(xid).serialize(output)?; + nfs::nfsstat3::NFS3ERR_ROFS.serialize(output)?; + nfs::wcc_data::default().serialize(output)?; + return Ok(()); + } + + let mut fromdirops = nfs::diropargs3::default(); + let mut todirops = nfs::diropargs3::default(); + fromdirops.deserialize(input)?; + todirops.deserialize(input)?; + + debug!("nfsproc3_rename({:?}, {:?}, {:?}) ", xid, fromdirops, todirops); + + // find the from directory + let from_dirid = context.vfs.fh_to_id(&fromdirops.dir); + if let Err(stat) = from_dirid { + // directory does not exist + make_success_reply(xid).serialize(output)?; + stat.serialize(output)?; + nfs::wcc_data::default().serialize(output)?; + error!("Directory does not exist"); + return Ok(()); + } + + // find the to directory + let to_dirid = context.vfs.fh_to_id(&todirops.dir); + if let Err(stat) = to_dirid { + // directory does not exist + make_success_reply(xid).serialize(output)?; + stat.serialize(output)?; + nfs::wcc_data::default().serialize(output)?; + error!("Directory does not exist"); + return Ok(()); + } + + // found the directory, get the attributes + let from_dirid = from_dirid.unwrap(); + let to_dirid = to_dirid.unwrap(); + + // get the object attributes before the write + let pre_from_dir_attr = match context.vfs.getattr(from_dirid).await { + Ok(v) => { + let wccattr = nfs::wcc_attr { + size: v.size, + mtime: v.mtime, + ctime: v.ctime, + }; + nfs::pre_op_attr::attributes(wccattr) + }, + Err(stat) => { + error!("Cannot stat directory"); + make_success_reply(xid).serialize(output)?; + stat.serialize(output)?; + nfs::wcc_data::default().serialize(output)?; + return Ok(()); + }, + }; + + // get the object attributes before the write + let pre_to_dir_attr = match context.vfs.getattr(to_dirid).await { + Ok(v) => { + let wccattr = nfs::wcc_attr { + size: v.size, + mtime: v.mtime, + ctime: v.ctime, + }; + nfs::pre_op_attr::attributes(wccattr) + }, + Err(stat) => { + error!("Cannot stat directory"); + make_success_reply(xid).serialize(output)?; + stat.serialize(output)?; + nfs::wcc_data::default().serialize(output)?; + return Ok(()); + }, + }; + + // rename! + let res = context.vfs.rename(from_dirid, &fromdirops.name, to_dirid, &todirops.name).await; + + // Re-read dir attributes for post op attr + let post_from_dir_attr = match context.vfs.getattr(from_dirid).await { + Ok(v) => nfs::post_op_attr::attributes(v), + Err(_) => nfs::post_op_attr::Void, + }; + let post_to_dir_attr = match context.vfs.getattr(to_dirid).await { + Ok(v) => nfs::post_op_attr::attributes(v), + Err(_) => nfs::post_op_attr::Void, + }; + let from_wcc_res = nfs::wcc_data { + before: pre_from_dir_attr, + after: post_from_dir_attr, + }; + + let to_wcc_res = nfs::wcc_data { + before: pre_to_dir_attr, + after: post_to_dir_attr, + }; + + match res { + Ok(()) => { + debug!("rename success"); + make_success_reply(xid).serialize(output)?; + nfs::nfsstat3::NFS3_OK.serialize(output)?; + from_wcc_res.serialize(output)?; + to_wcc_res.serialize(output)?; + }, + Err(e) => { + error!("rename error {:?} --> {:?}", xid, e); + // serialize CREATE3resfail + make_success_reply(xid).serialize(output)?; + e.serialize(output)?; + from_wcc_res.serialize(output)?; + to_wcc_res.serialize(output)?; + }, + } + + Ok(()) +} + +#[allow(non_camel_case_types)] +#[derive(Debug, Default)] +struct LINK3args { + file: nfs::nfs_fh3, + link: nfs::diropargs3, +} +xdr_struct!(LINK3args, file, link); + +pub async fn nfsproc3_link( + xid: u32, + input: &mut impl Read, + output: &mut impl Write, + context: &RPCContext, +) -> Result<(), anyhow::Error> { + if !matches!(context.vfs.capabilities(), VFSCapabilities::ReadWrite) { + make_success_reply(xid).serialize(output)?; + nfs::nfsstat3::NFS3ERR_ROFS.serialize(output)?; + nfs::post_op_attr::Void.serialize(output)?; + nfs::wcc_data::default().serialize(output)?; + return Ok(()); + } + + let mut args = LINK3args::default(); + args.deserialize(input)?; + debug!("nfsproc3_link({:?}, {:?})", xid, args); + + let fileid = match context.vfs.fh_to_id(&args.file) { + Ok(value) => value, + Err(status) => { + make_success_reply(xid).serialize(output)?; + status.serialize(output)?; + nfs::post_op_attr::Void.serialize(output)?; + nfs::wcc_data::default().serialize(output)?; + return Ok(()); + }, + }; + let file_attr = match context.vfs.getattr(fileid).await { + Ok(value) => value, + Err(status) => { + make_success_reply(xid).serialize(output)?; + status.serialize(output)?; + nfs::post_op_attr::Void.serialize(output)?; + nfs::wcc_data::default().serialize(output)?; + return Ok(()); + }, + }; + if matches!(file_attr.ftype, nfs::ftype3::NF3DIR) { + make_success_reply(xid).serialize(output)?; + nfs::nfsstat3::NFS3ERR_ISDIR.serialize(output)?; + nfs::post_op_attr::attributes(file_attr).serialize(output)?; + nfs::wcc_data::default().serialize(output)?; + return Ok(()); + } + + let dirid = match context.vfs.fh_to_id(&args.link.dir) { + Ok(value) => value, + Err(status) => { + make_success_reply(xid).serialize(output)?; + status.serialize(output)?; + nfs::post_op_attr::attributes(file_attr).serialize(output)?; + nfs::wcc_data::default().serialize(output)?; + return Ok(()); + }, + }; + let pre_dir_attr = match context.vfs.getattr(dirid).await { + Ok(value) => nfs::pre_op_attr::attributes(nfs::wcc_attr { + size: value.size, + mtime: value.mtime, + ctime: value.ctime, + }), + Err(status) => { + make_success_reply(xid).serialize(output)?; + status.serialize(output)?; + nfs::post_op_attr::attributes(file_attr).serialize(output)?; + nfs::wcc_data::default().serialize(output)?; + return Ok(()); + }, + }; + + let result = context.vfs.link(fileid, dirid, &args.link.name).await; + let post_dir_attr = match context.vfs.getattr(dirid).await { + Ok(value) => nfs::post_op_attr::attributes(value), + Err(_) => nfs::post_op_attr::Void, + }; + let dir_wcc = nfs::wcc_data { + before: pre_dir_attr, + after: post_dir_attr, + }; + match result { + Ok(value) => { + make_success_reply(xid).serialize(output)?; + nfs::nfsstat3::NFS3_OK.serialize(output)?; + nfs::post_op_attr::attributes(value).serialize(output)?; + dir_wcc.serialize(output)?; + }, + Err(status) => { + let post_file_attr = match context.vfs.getattr(fileid).await { + Ok(value) => nfs::post_op_attr::attributes(value), + Err(_) => nfs::post_op_attr::Void, + }; + make_success_reply(xid).serialize(output)?; + status.serialize(output)?; + post_file_attr.serialize(output)?; + dir_wcc.serialize(output)?; + }, + } + Ok(()) +} + +/* + MKDIR3res NFSPROC3_MKDIR(MKDIR3args) = 9; + + struct MKDIR3args { + diropargs3 where; + sattr3 attributes; + }; + + struct MKDIR3resok { + post_op_fh3 obj; + post_op_attr obj_attributes; + wcc_data dir_wcc; + }; + + struct MKDIR3resfail { + wcc_data dir_wcc; + }; + + union MKDIR3res switch (nfsstat3 status) { + case NFS3_OK: + MKDIR3resok resok; + default: + MKDIR3resfail resfail; + }; + +*/ + +#[allow(non_camel_case_types)] +#[derive(Debug, Default)] +struct MKDIR3args { + dirops: nfs::diropargs3, + attributes: nfs::sattr3, +} +xdr_struct!(MKDIR3args, dirops, attributes); + +pub async fn nfsproc3_mkdir( + xid: u32, + input: &mut impl Read, + output: &mut impl Write, + context: &RPCContext, +) -> Result<(), anyhow::Error> { + // if we do not have write capabilities + if !matches!(context.vfs.capabilities(), VFSCapabilities::ReadWrite) { + warn!("No write capabilities."); + make_success_reply(xid).serialize(output)?; + nfs::nfsstat3::NFS3ERR_ROFS.serialize(output)?; + nfs::wcc_data::default().serialize(output)?; + return Ok(()); + } + let mut args = MKDIR3args::default(); + args.deserialize(input)?; + + debug!("nfsproc3_mkdir({:?}, {:?}) ", xid, args); + + // find the directory we are supposed to create the + // new file in + let dirid = context.vfs.fh_to_id(&args.dirops.dir); + if let Err(stat) = dirid { + // directory does not exist + make_success_reply(xid).serialize(output)?; + stat.serialize(output)?; + nfs::wcc_data::default().serialize(output)?; + error!("Directory does not exist"); + return Ok(()); + } + // found the directory, get the attributes + let dirid = dirid.unwrap(); + + // get the object attributes before the write + let pre_dir_attr = match context.vfs.getattr(dirid).await { + Ok(v) => { + let wccattr = nfs::wcc_attr { + size: v.size, + mtime: v.mtime, + ctime: v.ctime, + }; + nfs::pre_op_attr::attributes(wccattr) + }, + Err(stat) => { + error!("Cannot stat directory"); + make_success_reply(xid).serialize(output)?; + stat.serialize(output)?; + nfs::wcc_data::default().serialize(output)?; + return Ok(()); + }, + }; + + let res = context.vfs.mkdir(dirid, &args.dirops.name).await; + + // Re-read dir attributes for post op attr + let post_dir_attr = match context.vfs.getattr(dirid).await { + Ok(v) => nfs::post_op_attr::attributes(v), + Err(_) => nfs::post_op_attr::Void, + }; + let wcc_res = nfs::wcc_data { + before: pre_dir_attr, + after: post_dir_attr, + }; + + match res { + Ok((fid, fattr)) => { + debug!("mkdir success --> {:?}, {:?}", fid, fattr); + make_success_reply(xid).serialize(output)?; + nfs::nfsstat3::NFS3_OK.serialize(output)?; + // serialize CREATE3resok + let fh = context.vfs.id_to_fh(fid); + nfs::post_op_fh3::handle(fh).serialize(output)?; + nfs::post_op_attr::attributes(fattr).serialize(output)?; + wcc_res.serialize(output)?; + }, + Err(e) => { + debug!("mkdir error {:?} --> {:?}", xid, e); + // serialize CREATE3resfail + make_success_reply(xid).serialize(output)?; + e.serialize(output)?; + wcc_res.serialize(output)?; + }, + } + + Ok(()) +} + +/* + SYMLINK3res NFSPROC3_SYMLINK(SYMLINK3args) = 10; + + struct symlinkdata3 { + sattr3 symlink_attributes; + nfspath3 symlink_data; + }; + + struct SYMLINK3args { + diropargs3 where; + symlinkdata3 symlink; + }; + + struct SYMLINK3resok { + post_op_fh3 obj; + post_op_attr obj_attributes; + wcc_data dir_wcc; + }; + + struct SYMLINK3resfail { + wcc_data dir_wcc; + }; + + union SYMLINK3res switch (nfsstat3 status) { + case NFS3_OK: + SYMLINK3resok resok; + default: + SYMLINK3resfail resfail; + }; +*/ + +#[allow(non_camel_case_types)] +#[derive(Debug, Default)] +struct SYMLINK3args { + dirops: nfs::diropargs3, + symlink: nfs::symlinkdata3, +} +xdr_struct!(SYMLINK3args, dirops, symlink); + +pub async fn nfsproc3_symlink( + xid: u32, + input: &mut impl Read, + output: &mut impl Write, + context: &RPCContext, +) -> Result<(), anyhow::Error> { + // if we do not have write capabilities + if !matches!(context.vfs.capabilities(), VFSCapabilities::ReadWrite) { + warn!("No write capabilities."); + make_success_reply(xid).serialize(output)?; + nfs::nfsstat3::NFS3ERR_ROFS.serialize(output)?; + nfs::wcc_data::default().serialize(output)?; + return Ok(()); + } + let mut args = SYMLINK3args::default(); + args.deserialize(input)?; + + debug!("nfsproc3_symlink({:?}, {:?}) ", xid, args); + + // find the directory we are supposed to create the + // new file in + let dirid = context.vfs.fh_to_id(&args.dirops.dir); + if let Err(stat) = dirid { + // directory does not exist + make_success_reply(xid).serialize(output)?; + stat.serialize(output)?; + nfs::wcc_data::default().serialize(output)?; + error!("Directory does not exist"); + return Ok(()); + } + // found the directory, get the attributes + let dirid = dirid.unwrap(); + + // get the object attributes before the write + let pre_dir_attr = match context.vfs.getattr(dirid).await { + Ok(v) => { + let wccattr = nfs::wcc_attr { + size: v.size, + mtime: v.mtime, + ctime: v.ctime, + }; + nfs::pre_op_attr::attributes(wccattr) + }, + Err(stat) => { + error!("Cannot stat directory"); + make_success_reply(xid).serialize(output)?; + stat.serialize(output)?; + nfs::wcc_data::default().serialize(output)?; + return Ok(()); + }, + }; + + let res = context + .vfs + .symlink(dirid, &args.dirops.name, &args.symlink.symlink_data, &args.symlink.symlink_attributes) + .await; + + // Re-read dir attributes for post op attr + let post_dir_attr = match context.vfs.getattr(dirid).await { + Ok(v) => nfs::post_op_attr::attributes(v), + Err(_) => nfs::post_op_attr::Void, + }; + let wcc_res = nfs::wcc_data { + before: pre_dir_attr, + after: post_dir_attr, + }; + + match res { + Ok((fid, fattr)) => { + debug!("symlink success --> {:?}, {:?}", fid, fattr); + make_success_reply(xid).serialize(output)?; + nfs::nfsstat3::NFS3_OK.serialize(output)?; + // serialize CREATE3resok + let fh = context.vfs.id_to_fh(fid); + nfs::post_op_fh3::handle(fh).serialize(output)?; + nfs::post_op_attr::attributes(fattr).serialize(output)?; + wcc_res.serialize(output)?; + }, + Err(e) => { + debug!("symlink error --> {:?}", e); + // serialize CREATE3resfail + make_success_reply(xid).serialize(output)?; + e.serialize(output)?; + wcc_res.serialize(output)?; + }, + } + + Ok(()) +} + +/* + + READLINK3res NFSPROC3_READLINK(READLINK3args) = 5; + + struct READLINK3args { + nfs_fh3 symlink; + }; + + struct READLINK3resok { + post_op_attr symlink_attributes; + nfspath3 data; + }; + + struct READLINK3resfail { + post_op_attr symlink_attributes; + }; + + union READLINK3res switch (nfsstat3 status) { + case NFS3_OK: + READLINK3resok resok; + default: + READLINK3resfail resfail; + }; +*/ +pub async fn nfsproc3_readlink( + xid: u32, + input: &mut impl Read, + output: &mut impl Write, + context: &RPCContext, +) -> Result<(), anyhow::Error> { + let mut handle = nfs::nfs_fh3::default(); + handle.deserialize(input)?; + debug!("nfsproc3_readlink({:?},{:?}) ", xid, handle); + + let id = context.vfs.fh_to_id(&handle); + // fail if unable to convert file handle + if let Err(stat) = id { + make_success_reply(xid).serialize(output)?; + stat.serialize(output)?; + return Ok(()); + } + let id = id.unwrap(); + // if the id does not exist, we fail + let symlink_attr = match context.vfs.getattr(id).await { + Ok(v) => nfs::post_op_attr::attributes(v), + Err(stat) => { + make_success_reply(xid).serialize(output)?; + stat.serialize(output)?; + nfs::post_op_attr::Void.serialize(output)?; + return Ok(()); + }, + }; + match context.vfs.readlink(id).await { + Ok(path) => { + debug!(" {:?} --> {:?}", xid, path); + make_success_reply(xid).serialize(output)?; + nfs::nfsstat3::NFS3_OK.serialize(output)?; + symlink_attr.serialize(output)?; + path.serialize(output)?; + }, + Err(stat) => { + // failed to read link + // retry with failure and the post_op_attr + make_success_reply(xid).serialize(output)?; + stat.serialize(output)?; + symlink_attr.serialize(output)?; + }, + } + Ok(()) +} diff --git a/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/portmap.rs b/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/portmap.rs new file mode 100644 index 0000000000..805348cf91 --- /dev/null +++ b/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/portmap.rs @@ -0,0 +1,25 @@ +// this is just a complete enumeration of everything in the RFC +#![allow(dead_code)] +// And its nice to keep the original RFC names and case +#![allow(non_camel_case_types)] + +use std::io::{Read, Write}; + +use crate::xdr::*; +// Transcribed from RFC 1057 Appendix A + +/// Device Number information. Ex: Major / Minor device +#[allow(non_camel_case_types)] +#[derive(Copy, Clone, Debug, Default)] +#[repr(C)] +pub struct mapping { + pub prog: u32, + pub vers: u32, + pub prot: u32, + pub port: u32, +} +xdr_struct!(mapping, prog, vers, prot, port); +pub const IPPROTO_TCP: u32 = 6; /* protocol number for TCP/IP */ +pub const IPPROTO_UDP: u32 = 17; /* protocol number for UDP/IP */ +pub const PROGRAM: u32 = 100000; +pub const VERSION: u32 = 2; diff --git a/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/portmap_handlers.rs b/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/portmap_handlers.rs new file mode 100644 index 0000000000..ce76ecaad5 --- /dev/null +++ b/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/portmap_handlers.rs @@ -0,0 +1,90 @@ +use std::io::{Read, Write}; + +use num_derive::{FromPrimitive, ToPrimitive}; +use num_traits::cast::FromPrimitive; +use tracing::{debug, error}; + +use crate::context::RPCContext; +use crate::portmap; +use crate::rpc::*; +use crate::xdr::*; + +/* + From RFC 1057 Appendix A + + program PMAP_PROG { + version PMAP_VERS { + void PMAPPROC_NULL(void) = 0; + bool PMAPPROC_SET(mapping) = 1; + bool PMAPPROC_UNSET(mapping) = 2; + unsigned int PMAPPROC_GETPORT(mapping) = 3; + pmaplist PMAPPROC_DUMP(void) = 4; + call_result PMAPPROC_CALLIT(call_args) = 5; + } = 2; + } = 100000; +*/ + +#[allow(non_camel_case_types)] +#[allow(clippy::upper_case_acronyms)] +#[derive(Copy, Clone, Debug, FromPrimitive, ToPrimitive)] +enum PortmapProgram { + PMAPPROC_NULL = 0, + PMAPPROC_SET = 1, + PMAPPROC_UNSET = 2, + PMAPPROC_GETPORT = 3, + PMAPPROC_DUMP = 4, + PMAPPROC_CALLIT = 5, + INVALID, +} + +pub fn handle_portmap( + xid: u32, + call: call_body, + input: &mut impl Read, + output: &mut impl Write, + context: &RPCContext, +) -> Result<(), anyhow::Error> { + if call.vers != portmap::VERSION { + error!("Invalid Portmap Version number {} != {}", call.vers, portmap::VERSION); + prog_mismatch_reply_message(xid, portmap::VERSION).serialize(output)?; + return Ok(()); + } + let prog = PortmapProgram::from_u32(call.proc).unwrap_or(PortmapProgram::INVALID); + + match prog { + PortmapProgram::PMAPPROC_NULL => pmapproc_null(xid, input, output)?, + PortmapProgram::PMAPPROC_GETPORT => pmapproc_getport(xid, input, output, context)?, + _ => { + proc_unavail_reply_message(xid).serialize(output)?; + }, + } + Ok(()) +} + +pub fn pmapproc_null(xid: u32, _: &mut impl Read, output: &mut impl Write) -> Result<(), anyhow::Error> { + debug!("pmapproc_null({:?}) ", xid); + // build an RPC reply + let msg = make_success_reply(xid); + debug!("\t{:?} --> {:?}", xid, msg); + msg.serialize(output)?; + Ok(()) +} + +/* + * We fake a portmapper here. And always direct back to the same host port + */ +pub fn pmapproc_getport( + xid: u32, + read: &mut impl Read, + output: &mut impl Write, + context: &RPCContext, +) -> Result<(), anyhow::Error> { + let mut mapping = portmap::mapping::default(); + mapping.deserialize(read)?; + debug!("pmapproc_getport({:?}, {:?}) ", xid, mapping); + make_success_reply(xid).serialize(output)?; + let port = context.local_port as u32; + debug!("\t{:?} --> {:?}", xid, port); + port.serialize(output)?; + Ok(()) +} diff --git a/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/rpc.rs b/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/rpc.rs new file mode 100644 index 0000000000..e38d2a390e --- /dev/null +++ b/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/rpc.rs @@ -0,0 +1,513 @@ +// this is just a complete enumeration of everything in the RFC +#![allow(dead_code)] +// And its nice to keep the original RFC names and case +#![allow(non_camel_case_types)] + +use std::io::{Read, Write}; + +use byteorder::{ReadBytesExt, WriteBytesExt}; +use num_derive::{FromPrimitive, ToPrimitive}; +use num_traits::cast::FromPrimitive; + +use crate::xdr::*; +// Transcribed from RFC 1057 + +#[allow(non_camel_case_types)] +#[allow(clippy::upper_case_acronyms)] +#[derive(Copy, Clone, Debug, FromPrimitive, ToPrimitive)] +#[repr(u32)] +/// This is only defined as the discriminant for rpc_body and should not +/// be used directly +pub enum _msg_type { + CALL = 0, + REPLY = 1, +} +xdr_enum_serde!(_msg_type); + +#[allow(non_camel_case_types)] +#[derive(Copy, Clone, Debug, FromPrimitive, ToPrimitive)] +#[repr(u32)] +/// This is only defined as the discriminant for reply_body and should not +/// be used directly +pub enum _reply_stat { + MSG_ACCEPTED = 0, + MSG_DENIED = 1, +} +xdr_enum_serde!(_reply_stat); + +#[allow(non_camel_case_types)] +#[allow(clippy::upper_case_acronyms)] +#[derive(Copy, Clone, Debug, FromPrimitive, ToPrimitive)] +#[repr(u32)] +/// This is only defined as the discriminant for accept_body and should not +/// be used directly +pub enum _accept_stat { + /// RPC executed successfully + SUCCESS = 0, + /// remote hasn't exported program + PROG_UNAVAIL = 1, + /// remote can't support version # + PROG_MISMATCH = 2, + /// program can't support procedure + PROC_UNAVAIL = 3, + /// procedure can't decode params + GARBAGE_ARGS = 4, +} +xdr_enum_serde!(_accept_stat); + +#[allow(non_camel_case_types)] +#[derive(Copy, Clone, Debug, FromPrimitive, ToPrimitive)] +#[repr(u32)] +/// This is only defined as the discriminant for reject_body and should not +/// be used directly +pub enum _reject_stat { + /// RPC version number != 2 + RPC_MISMATCH = 0, + /// remote can't authenticate caller + AUTH_ERROR = 1, +} +xdr_enum_serde!(_reject_stat); + +#[allow(non_camel_case_types)] +#[derive(Copy, Clone, Debug, Default, FromPrimitive, ToPrimitive)] +#[repr(u32)] +/// Why authentication failed +pub enum auth_stat { + /// bad credentials (seal broken) + #[default] + AUTH_BADCRED = 1, + /// client must begin new session + AUTH_REJECTEDCRED = 2, + /// bad verifier (seal broken) + AUTH_BADVERF = 3, + /// verifier expired or replayed + AUTH_REJECTEDVERF = 4, + /// rejected for security reasons + AUTH_TOOWEAK = 5, +} +xdr_enum_serde!(auth_stat); + +#[allow(non_camel_case_types)] +#[derive(Copy, Clone, Debug, FromPrimitive, ToPrimitive)] +#[repr(u32)] +#[non_exhaustive] +pub enum auth_flavor { + AUTH_NULL = 0, + AUTH_UNIX = 1, + AUTH_SHORT = 2, + AUTH_DES = 3, /* and more to be defined */ +} +xdr_enum_serde!(auth_flavor); + +#[allow(non_camel_case_types)] +#[derive(Clone, Debug, Default)] +pub struct auth_unix { + stamp: u32, + machinename: Vec, + uid: u32, + gid: u32, + gids: Vec, +} +xdr_struct!(auth_unix, stamp, machinename, uid, gid, gids); + +///Provisions for authentication of caller to service and vice-versa are +///provided as a part of the RPC protocol. The call message has two +///authentication fields, the credentials and verifier. The reply +///message has one authentication field, the response verifier. The RPC +///protocol specification defines all three fields to be the following +///opaque type (in the eXternal Data Representation (XDR) language [9]): +/// +/// In other words, any "opaque_auth" structure is an "auth_flavor" +///enumeration followed by bytes which are opaque to (uninterpreted by) +///the RPC protocol implementation. +/// +///The interpretation and semantics of the data contained within the +///authentication fields is specified by individual, independent +///authentication protocol specifications. (Section 9 defines the +///various authentication protocols.) +/// +///If authentication parameters were rejected, the reply message +///contains information stating why they were rejected. +#[allow(non_camel_case_types)] +#[derive(Clone, Debug)] +pub struct opaque_auth { + pub flavor: auth_flavor, + pub body: Vec, +} +xdr_struct!(opaque_auth, flavor, body); +impl Default for opaque_auth { + fn default() -> opaque_auth { + opaque_auth { + flavor: auth_flavor::AUTH_NULL, + body: Vec::new(), + } + } +} + +#[allow(non_camel_case_types)] +#[derive(Clone, Debug, Default)] +///All messages start with a transaction identifier, xid, followed by a +///two-armed discriminated union. The union's discriminant is a +///msg_type which switches to one of the two types of the message. The +///xid of a REPLY message always matches that of the initiating CALL +///message. NB: The xid field is only used for clients matching reply +///messages with call messages or for servers detecting retransmissions; +///the service side cannot treat this id as any type of sequence number. +pub struct rpc_msg { + pub xid: u32, + pub body: rpc_body, +} +xdr_struct!(rpc_msg, xid, body); + +#[allow(non_camel_case_types)] +#[allow(clippy::upper_case_acronyms)] +#[derive(Clone, Debug)] +#[repr(u32)] +/// Discriminant is msg_type +pub enum rpc_body { + CALL(call_body), + REPLY(reply_body), +} + +impl Default for rpc_body { + fn default() -> rpc_body { + rpc_body::CALL(call_body::default()) + } +} +impl XDR for rpc_body { + fn serialize(&self, dest: &mut R) -> std::io::Result<()> { + match self { + rpc_body::CALL(v) => { + 0_u32.serialize(dest)?; + v.serialize(dest)?; + }, + rpc_body::REPLY(v) => { + 1_u32.serialize(dest)?; + v.serialize(dest)?; + }, + } + Ok(()) + } + fn deserialize(&mut self, src: &mut R) -> std::io::Result<()> { + let mut c: u32 = 0; + c.deserialize(src)?; + if c == 0 { + let mut r = call_body::default(); + r.deserialize(src)?; + *self = rpc_body::CALL(r); + } else if c == 1 { + let mut r = reply_body::default(); + r.deserialize(src)?; + *self = rpc_body::REPLY(r); + } + Ok(()) + } +} + +///The RPC call message has three unsigned integer fields -- remote +///program number, remote program version number, and remote procedure +///number -- which uniquely identify the procedure to be called. +///Program numbers are administered by some central authority (like +///Sun). Once implementors have a program number, they can implement +///their remote program; the first implementation would most likely have +///the version number 1. Because most new protocols evolve, a version +///field of the call message identifies which version of the protocol +///the caller is using. Version numbers make speaking old and new +///protocols through the same server process possible. +/// +///The procedure number identifies the procedure to be called. These +///numbers are documented in the specific program's protocol +///specification. For example, a file service's protocol specification +///may state that its procedure number 5 is "read" and procedure number +///12 is "write". +/// +///Just as remote program protocols may change over several versions, +///the actual RPC message protocol could also change. Therefore, the +///call message also has in it the RPC version number, which is always +///equal to two for the version of RPC described here. +/// +///The reply message to a request message has enough information to +///distinguish the following error conditions: +/// +///(1) The remote implementation of RPC does not speak protocol version +/// 2. The lowest and highest supported RPC version numbers are returned. +/// +///(2) The remote program is not available on the remote system. +/// +///(3) The remote program does not support the requested version number. +///The lowest and highest supported remote program version numbers are +///returned. +/// +///(4) The requested procedure number does not exist. (This is usually +///a client side protocol or programming error.) +/// +///(5) The parameters to the remote procedure appear to be garbage from +///the server's point of view. (Again, this is usually caused by a +///disagreement about the protocol between client and service.) +/// +/// In version 2 of the RPC protocol specification, rpcvers must be equal +///to 2. The fields prog, vers, and proc specify the remote program, +///its version number, and the procedure within the remote program to be +///called. After these fields are two authentication parameters: cred +///(authentication credentials) and verf (authentication verifier). The +///two authentication parameters are followed by the parameters to the +///remote procedure, which are specified by the specific program +///protocol. +#[allow(non_camel_case_types)] +#[derive(Clone, Debug, Default)] +pub struct call_body { + /// Must be = 2 + pub rpcvers: u32, + pub prog: u32, + pub vers: u32, + pub proc: u32, + pub cred: opaque_auth, + pub verf: opaque_auth, + /* procedure specific parameters start here */ +} + +xdr_struct!(call_body, rpcvers, prog, vers, proc, cred, verf); + +#[allow(non_camel_case_types)] +#[derive(Clone, Debug)] +#[repr(u32)] +pub enum reply_body { + MSG_ACCEPTED(accepted_reply), + MSG_DENIED(rejected_reply), +} + +impl Default for reply_body { + fn default() -> reply_body { + reply_body::MSG_ACCEPTED(accepted_reply::default()) + } +} + +impl XDR for reply_body { + fn serialize(&self, dest: &mut R) -> std::io::Result<()> { + match self { + reply_body::MSG_ACCEPTED(v) => { + 0_u32.serialize(dest)?; + v.serialize(dest)?; + }, + reply_body::MSG_DENIED(v) => { + 1_u32.serialize(dest)?; + v.serialize(dest)?; + }, + } + Ok(()) + } + fn deserialize(&mut self, src: &mut R) -> std::io::Result<()> { + let mut c: u32 = 0; + c.deserialize(src)?; + if c == 0 { + let mut r = accepted_reply::default(); + r.deserialize(src)?; + *self = reply_body::MSG_ACCEPTED(r); + } else if c == 1 { + let mut r = rejected_reply::default(); + r.deserialize(src)?; + *self = reply_body::MSG_DENIED(r); + } + Ok(()) + } +} + +#[allow(non_camel_case_types)] +#[derive(Copy, Clone, Debug, Default)] +#[repr(C)] +pub struct mismatch_info { + pub low: u32, + pub high: u32, +} +xdr_struct!(mismatch_info, low, high); + +///Reply to an RPC call that was accepted by the server: +///There could be an error even though the call was accepted. The first +///field is an authentication verifier that the server generates in +///order to validate itself to the client. It is followed by a union +///whose discriminant is an enum accept_stat. The SUCCESS arm of the +///union is protocol specific. The PROG_UNAVAIL, PROC_UNAVAIL, and +///GARBAGE_ARGS arms of the union are void. The PROG_MISMATCH arm +///specifies the lowest and highest version numbers of the remote +///program supported by the server. +/// Discriminant is reply_stat +#[allow(non_camel_case_types)] +#[derive(Clone, Debug, Default)] +pub struct accepted_reply { + pub verf: opaque_auth, + pub reply_data: accept_body, +} +xdr_struct!(accepted_reply, verf, reply_data); + +#[allow(non_camel_case_types)] +#[allow(clippy::upper_case_acronyms)] +#[derive(Copy, Clone, Debug, Default)] +#[repr(u32)] +/// Discriminant is accept_stat +pub enum accept_body { + #[default] + SUCCESS, + PROG_UNAVAIL, + /// remote can't support version # + PROG_MISMATCH(mismatch_info), + PROC_UNAVAIL, + /// procedure can't decode params + GARBAGE_ARGS, +} +impl XDR for accept_body { + fn serialize(&self, dest: &mut R) -> std::io::Result<()> { + match self { + accept_body::SUCCESS => { + 0_u32.serialize(dest)?; + }, + accept_body::PROG_UNAVAIL => { + 1_u32.serialize(dest)?; + }, + accept_body::PROG_MISMATCH(v) => { + 2_u32.serialize(dest)?; + v.serialize(dest)?; + }, + accept_body::PROC_UNAVAIL => { + 3_u32.serialize(dest)?; + }, + accept_body::GARBAGE_ARGS => { + 4_u32.serialize(dest)?; + }, + } + Ok(()) + } + fn deserialize(&mut self, src: &mut R) -> std::io::Result<()> { + let mut c: u32 = 0; + c.deserialize(src)?; + if c == 0 { + *self = accept_body::SUCCESS; + } else if c == 1 { + *self = accept_body::PROG_UNAVAIL; + } else if c == 2 { + let mut r = mismatch_info::default(); + r.deserialize(src)?; + *self = accept_body::PROG_MISMATCH(r); + } else if c == 3 { + *self = accept_body::PROC_UNAVAIL; + } else { + *self = accept_body::GARBAGE_ARGS; + } + Ok(()) + } +} + +#[allow(non_camel_case_types)] +#[derive(Clone, Debug)] +#[repr(u32)] +///Reply to an RPC call that was rejected by the server: +/// +///The call can be rejected for two reasons: either the server is not +///running a compatible version of the RPC protocol (RPC_MISMATCH), or +///the server refuses to authenticate the caller (AUTH_ERROR). In case +///of an RPC version mismatch, the server returns the lowest and highest +///supported RPC version numbers. In case of refused authentication, +///failure status is returned. +/// Discriminant is reject_stat +pub enum rejected_reply { + RPC_MISMATCH(mismatch_info), + AUTH_ERROR(auth_stat), +} + +impl Default for rejected_reply { + fn default() -> rejected_reply { + rejected_reply::RPC_MISMATCH(mismatch_info::default()) + } +} + +impl XDR for rejected_reply { + fn serialize(&self, dest: &mut R) -> std::io::Result<()> { + match self { + rejected_reply::RPC_MISMATCH(v) => { + 0_u32.serialize(dest)?; + v.serialize(dest)?; + }, + rejected_reply::AUTH_ERROR(v) => { + 1_u32.serialize(dest)?; + v.serialize(dest)?; + }, + } + Ok(()) + } + fn deserialize(&mut self, src: &mut R) -> std::io::Result<()> { + let mut c: u32 = 0; + c.deserialize(src)?; + if c == 0 { + let mut r = mismatch_info::default(); + r.deserialize(src)?; + *self = rejected_reply::RPC_MISMATCH(r); + } else if c == 1 { + let mut r = auth_stat::default(); + r.deserialize(src)?; + *self = rejected_reply::AUTH_ERROR(r); + } + Ok(()) + } +} + +pub fn proc_unavail_reply_message(xid: u32) -> rpc_msg { + let reply = reply_body::MSG_ACCEPTED(accepted_reply { + verf: opaque_auth::default(), + reply_data: accept_body::PROC_UNAVAIL, + }); + rpc_msg { + xid, + body: rpc_body::REPLY(reply), + } +} +pub fn prog_unavail_reply_message(xid: u32) -> rpc_msg { + let reply = reply_body::MSG_ACCEPTED(accepted_reply { + verf: opaque_auth::default(), + reply_data: accept_body::PROG_UNAVAIL, + }); + rpc_msg { + xid, + body: rpc_body::REPLY(reply), + } +} +pub fn prog_mismatch_reply_message(xid: u32, accepted_ver: u32) -> rpc_msg { + let reply = reply_body::MSG_ACCEPTED(accepted_reply { + verf: opaque_auth::default(), + reply_data: accept_body::PROG_MISMATCH(mismatch_info { + low: accepted_ver, + high: accepted_ver, + }), + }); + rpc_msg { + xid, + body: rpc_body::REPLY(reply), + } +} +pub fn garbage_args_reply_message(xid: u32) -> rpc_msg { + let reply = reply_body::MSG_ACCEPTED(accepted_reply { + verf: opaque_auth::default(), + reply_data: accept_body::GARBAGE_ARGS, + }); + rpc_msg { + xid, + body: rpc_body::REPLY(reply), + } +} + +pub fn rpc_vers_mismatch(xid: u32) -> rpc_msg { + let reply = reply_body::MSG_DENIED(rejected_reply::RPC_MISMATCH(mismatch_info::default())); + rpc_msg { + xid, + body: rpc_body::REPLY(reply), + } +} + +pub fn make_success_reply(xid: u32) -> rpc_msg { + let reply = reply_body::MSG_ACCEPTED(accepted_reply { + verf: opaque_auth::default(), + reply_data: accept_body::SUCCESS, + }); + rpc_msg { + xid, + body: rpc_body::REPLY(reply), + } +} diff --git a/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/rpcwire.rs b/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/rpcwire.rs new file mode 100644 index 0000000000..33200c6e46 --- /dev/null +++ b/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/rpcwire.rs @@ -0,0 +1,178 @@ +use std::io::{Cursor, Read, Write}; + +use anyhow::anyhow; +use tokio::io::{AsyncReadExt, AsyncWriteExt, DuplexStream}; +use tokio::sync::mpsc; +use tracing::{debug, error, trace, warn}; + +use crate::context::RPCContext; +use crate::rpc::*; +use crate::xdr::*; +use crate::{mount, mount_handlers, nfs, nfs_handlers, portmap, portmap_handlers}; + +// Information from RFC 5531 +// https://datatracker.ietf.org/doc/html/rfc5531 + +const NFS_ACL_PROGRAM: u32 = 100227; +const NFS_ID_MAP_PROGRAM: u32 = 100270; +const NFS_METADATA_PROGRAM: u32 = 200024; + +async fn handle_rpc( + input: &mut impl Read, + output: &mut impl Write, + mut context: RPCContext, +) -> Result { + let mut recv = rpc_msg::default(); + recv.deserialize(input)?; + let xid = recv.xid; + if let rpc_body::CALL(call) = recv.body { + if let auth_flavor::AUTH_UNIX = call.cred.flavor { + let mut auth = auth_unix::default(); + auth.deserialize(&mut Cursor::new(&call.cred.body))?; + context.auth = auth; + } + if call.rpcvers != 2 { + warn!("Invalid RPC version {} != 2", call.rpcvers); + rpc_vers_mismatch(xid).serialize(output)?; + return Ok(true); + } + + if context.transaction_tracker.is_retransmission(xid, &context.client_addr) { + // This is a retransmission + // Drop the message and return + debug!("Retransmission detected, xid: {}, client_addr: {}, call: {:?}", xid, context.client_addr, call); + return Ok(false); + } + + let res = { + if call.prog == nfs::PROGRAM { + nfs_handlers::handle_nfs(xid, call, input, output, &context).await + } else if call.prog == portmap::PROGRAM { + portmap_handlers::handle_portmap(xid, call, input, output, &context) + } else if call.prog == mount::PROGRAM { + mount_handlers::handle_mount(xid, call, input, output, &context).await + } else if call.prog == NFS_ACL_PROGRAM + || call.prog == NFS_ID_MAP_PROGRAM + || call.prog == NFS_METADATA_PROGRAM + { + trace!("ignoring NFS_ACL packet"); + prog_unavail_reply_message(xid).serialize(output)?; + Ok(()) + } else { + warn!("Unknown RPC Program number {} != {}", call.prog, nfs::PROGRAM); + prog_unavail_reply_message(xid).serialize(output)?; + Ok(()) + } + } + .map(|_| true); + context.transaction_tracker.mark_processed(xid, &context.client_addr); + res + } else { + error!("Unexpectedly received a Reply instead of a Call"); + Err(anyhow!("Bad RPC Call format")) + } +} + +/// RFC 1057 Section 10 +/// When RPC messages are passed on top of a byte stream transport +/// protocol (like TCP), it is necessary to delimit one message from +/// another in order to detect and possibly recover from protocol errors. +/// This is called record marking (RM). Sun uses this RM/TCP/IP +/// transport for passing RPC messages on TCP streams. One RPC message +/// fits into one RM record. +/// +/// A record is composed of one or more record fragments. A record +/// fragment is a four-byte header followed by 0 to (2**31) - 1 bytes of +/// fragment data. The bytes encode an unsigned binary number; as with +/// XDR integers, the byte order is from highest to lowest. The number +/// encodes two values -- a boolean which indicates whether the fragment +/// is the last fragment of the record (bit value 1 implies the fragment +/// is the last fragment) and a 31-bit unsigned binary value which is the +/// length in bytes of the fragment's data. The boolean value is the +/// highest-order bit of the header; the length is the 31 low-order bits. +/// (Note that this record specification is NOT in XDR standard form!) +async fn read_fragment(socket: &mut DuplexStream, append_to: &mut Vec) -> Result { + let mut header_buf = [0_u8; 4]; + socket.read_exact(&mut header_buf).await?; + let fragment_header = u32::from_be_bytes(header_buf); + let is_last = (fragment_header & (1 << 31)) > 0; + let length = (fragment_header & ((1 << 31) - 1)) as usize; + trace!("Reading fragment length:{}, last:{}", length, is_last); + let start_offset = append_to.len(); + append_to.resize(append_to.len() + length, 0); + socket.read_exact(&mut append_to[start_offset..]).await?; + trace!("Finishing Reading fragment length:{}, last:{}", length, is_last); + Ok(is_last) +} + +pub async fn write_fragment(socket: &mut tokio::net::TcpStream, buf: &[u8]) -> Result<(), anyhow::Error> { + // TODO: split into many fragments + assert!(buf.len() < (1 << 31)); + // set the last flag + let fragment_header = buf.len() as u32 + (1 << 31); + let header_buf = u32::to_be_bytes(fragment_header); + socket.write_all(&header_buf).await?; + trace!("Writing fragment length:{}", buf.len()); + socket.write_all(buf).await?; + Ok(()) +} + +pub type SocketMessageType = Result, anyhow::Error>; + +/// The Socket Message Handler reads from a TcpStream and spawns off +/// subtasks to handle each message. replies are queued into the +/// reply_send_channel. +#[derive(Debug)] +pub struct SocketMessageHandler { + cur_fragment: Vec, + socket_receive_channel: DuplexStream, + reply_send_channel: mpsc::UnboundedSender, + context: RPCContext, +} + +impl SocketMessageHandler { + /// Creates a new SocketMessageHandler with the receiver for queued message replies + pub fn new(context: &RPCContext) -> (Self, DuplexStream, mpsc::UnboundedReceiver) { + let (socksend, sockrecv) = tokio::io::duplex(256000); + let (msgsend, msgrecv) = mpsc::unbounded_channel(); + ( + Self { + cur_fragment: Vec::new(), + socket_receive_channel: sockrecv, + reply_send_channel: msgsend, + context: context.clone(), + }, + socksend, + msgrecv, + ) + } + + /// Reads a fragment from the socket. This should be looped. + pub async fn read(&mut self) -> Result<(), anyhow::Error> { + let is_last = read_fragment(&mut self.socket_receive_channel, &mut self.cur_fragment).await?; + if is_last { + let fragment = std::mem::take(&mut self.cur_fragment); + let context = self.context.clone(); + let send = self.reply_send_channel.clone(); + tokio::spawn(async move { + let mut write_buf: Vec = Vec::new(); + let mut write_cursor = Cursor::new(&mut write_buf); + let maybe_reply = handle_rpc(&mut Cursor::new(fragment), &mut write_cursor, context).await; + match maybe_reply { + Err(e) => { + error!("RPC Error: {:?}", e); + let _ = send.send(Err(e)); + }, + Ok(true) => { + let _ = std::io::Write::flush(&mut write_cursor); + let _ = send.send(Ok(write_buf)); + }, + Ok(false) => { + // do not reply + }, + } + }); + } + Ok(()) + } +} diff --git a/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/tcp.rs b/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/tcp.rs new file mode 100644 index 0000000000..08f1c8eadb --- /dev/null +++ b/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/tcp.rs @@ -0,0 +1,215 @@ +use std::io; +use std::net::{IpAddr, SocketAddr}; +use std::sync::Arc; +use std::time::Duration; + +use anyhow; +use async_trait::async_trait; +use tokio::io::AsyncWriteExt; +use tokio::net::TcpListener; +use tokio::sync::mpsc; +use tracing::{debug, error, info}; + +use crate::context::RPCContext; +use crate::rpcwire::*; +use crate::transaction_tracker::TransactionTracker; +use crate::vfs::NFSFileSystem; + +/// A NFS Tcp Connection Handler +pub struct NFSTcpListener { + listener: TcpListener, + port: u16, + arcfs: Arc, + mount_signal: Option>, + export_name: Arc, + transaction_tracker: Arc, +} + +pub fn generate_host_ip(hostnum: u16) -> String { + format!("127.88.{}.{}", ((hostnum >> 8) & 0xFF) as u8, (hostnum & 0xFF) as u8) +} + +/// processes an established socket +async fn process_socket(mut socket: tokio::net::TcpStream, context: RPCContext) -> Result<(), anyhow::Error> { + let (mut message_handler, mut socksend, mut msgrecvchan) = SocketMessageHandler::new(&context); + let _ = socket.set_nodelay(true); + + tokio::spawn(async move { + loop { + if let Err(e) = message_handler.read().await { + debug!("Message loop broken due to {:?}", e); + break; + } + } + }); + loop { + tokio::select! { + _ = socket.readable() => { + let mut buf = [0; 128000]; + + match socket.try_read(&mut buf) { + Ok(0) => { + return Ok(()); + } + Ok(n) => { + let _ = socksend.write_all(&buf[..n]).await; + } + Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => { + continue; + } + Err(e) => { + debug!("Message handling closed : {:?}", e); + return Err(e.into()); + } + } + + }, + reply = msgrecvchan.recv() => { + match reply { + Some(Err(e)) => { + debug!("Message handling closed : {:?}", e); + return Err(e); + } + Some(Ok(msg)) => { + if let Err(e) = write_fragment(&mut socket, &msg).await { + error!("Write error {:?}", e); + } + } + None => { + return Err(anyhow::anyhow!("Unexpected socket context termination")); + } + } + } + } + } +} + +#[async_trait] +pub trait NFSTcp: Send + Sync { + /// Gets the true listening port. Useful if the bound port number is 0 + fn get_listen_port(&self) -> u16; + + /// Gets the true listening IP. Useful on windows when the IP may be random + fn get_listen_ip(&self) -> IpAddr; + + /// Sets a mount listener. A "true" signal will be sent on a mount + /// and a "false" will be sent on an unmount + fn set_mount_listener(&mut self, signal: mpsc::Sender); + + /// Loops forever and never returns handling all incoming connections. + async fn handle_forever(&self) -> io::Result<()>; +} + +impl NFSTcpListener { + /// Binds to a ipstr of the form [ip address]:port. For instance + /// "127.0.0.1:12000". fs is an instance of an implementation + /// of NFSFileSystem. + pub async fn bind(ipstr: &str, fs: T) -> io::Result> { + let (ip, port) = ipstr + .split_once(':') + .ok_or_else(|| io::Error::new(io::ErrorKind::AddrNotAvailable, "IP Address must be of form ip:port"))?; + let port = port + .parse::() + .map_err(|_| io::Error::new(io::ErrorKind::AddrNotAvailable, "Port not in range 0..=65535"))?; + + let arcfs: Arc = Arc::new(fs); + + if ip == "auto" { + let mut num_tries_left = 32; + + for try_ip in 1u16.. { + let ip = generate_host_ip(try_ip); + + let result = NFSTcpListener::bind_internal(&ip, port, arcfs.clone()).await; + + match &result { + Err(_) => { + if num_tries_left == 0 { + return result; + } else { + num_tries_left -= 1; + continue; + } + }, + Ok(_) => { + return result; + }, + } + } + unreachable!(); // Does not detect automatically that loop above never terminates. + } else { + // Otherwise, try this. + NFSTcpListener::bind_internal(ip, port, arcfs).await + } + } + + async fn bind_internal(ip: &str, port: u16, arcfs: Arc) -> io::Result> { + let ipstr = format!("{ip}:{port}"); + let listener = TcpListener::bind(&ipstr).await?; + info!("Listening on {:?}", &ipstr); + + let port = match listener.local_addr().unwrap() { + SocketAddr::V4(s) => s.port(), + SocketAddr::V6(s) => s.port(), + }; + Ok(NFSTcpListener { + listener, + port, + arcfs, + mount_signal: None, + export_name: Arc::from("/".to_string()), + transaction_tracker: Arc::new(TransactionTracker::new(Duration::from_secs(60))), + }) + } + + /// Sets an optional NFS export name. + /// + /// - `export_name`: The desired export name without slashes. + /// + /// Example: Name `foo` results in the export path `/foo`. + /// Default path is `/` if not set. + pub fn with_export_name>(&mut self, export_name: S) { + self.export_name = Arc::new(format!("/{}", export_name.as_ref().trim_end_matches('/').trim_start_matches('/'))) + } +} + +#[async_trait] +impl NFSTcp for NFSTcpListener { + /// Gets the true listening port. Useful if the bound port number is 0 + fn get_listen_port(&self) -> u16 { + let addr = self.listener.local_addr().unwrap(); + addr.port() + } + + fn get_listen_ip(&self) -> IpAddr { + let addr = self.listener.local_addr().unwrap(); + addr.ip() + } + + /// Sets a mount listener. A "true" signal will be sent on a mount + /// and a "false" will be sent on an unmount + fn set_mount_listener(&mut self, signal: mpsc::Sender) { + self.mount_signal = Some(signal); + } + + /// Loops forever and never returns handling all incoming connections. + async fn handle_forever(&self) -> io::Result<()> { + loop { + let (socket, _) = self.listener.accept().await?; + let context = RPCContext { + local_port: self.port, + client_addr: socket.peer_addr().unwrap().to_string(), + auth: crate::rpc::auth_unix::default(), + vfs: self.arcfs.clone(), + mount_signal: self.mount_signal.clone(), + export_name: self.export_name.clone(), + transaction_tracker: self.transaction_tracker.clone(), + }; + info!("Accepting connection from {}", context.client_addr); + debug!("Accepting socket {:?} {:?}", socket, context); + tokio::spawn(async move { + let _ = process_socket(socket, context).await; + }); + } + } +} diff --git a/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/transaction_tracker.rs b/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/transaction_tracker.rs new file mode 100644 index 0000000000..ed44af073e --- /dev/null +++ b/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/transaction_tracker.rs @@ -0,0 +1,57 @@ +use std::collections::{hash_map, HashMap}; +use std::sync::Mutex; +use std::time::{Duration, SystemTime}; + +/// `TransactionTracker` tracks the state of transactions to detect retransmissions. +pub struct TransactionTracker { + retention_period: Duration, + transactions: Mutex>, +} + +impl TransactionTracker { + pub fn new(retention_period: Duration) -> Self { + Self { + retention_period, + transactions: Mutex::new(HashMap::new()), + } + } + + /// Checks if the transaction is a retransmission. + /// If it's a new transaction, it is marked as `InProgress`. + /// + /// Returns `true` if the transaction is a retransmission, `false` otherwise. + pub fn is_retransmission(&self, xid: u32, client_addr: &str) -> bool { + let key = (xid, client_addr.to_string()); + let mut transactions = self.transactions.lock().expect("unable to unlock transactions mutex"); + housekeeping(&mut transactions, self.retention_period); + if let hash_map::Entry::Vacant(e) = transactions.entry(key) { + e.insert(TransactionState::InProgress); + false + } else { + true + } + } + + /// Marks the transaction as processed. + pub fn mark_processed(&self, xid: u32, client_addr: &str) { + let key = (xid, client_addr.to_string()); + let completion_time = SystemTime::now(); + let mut transactions = self.transactions.lock().expect("unable to unlock transactions mutex"); + if let Some(tx) = transactions.get_mut(&key) { + *tx = TransactionState::Completed(completion_time); + } + } +} + +fn housekeeping(transactions: &mut HashMap<(u32, String), TransactionState>, max_age: Duration) { + let mut cutoff = SystemTime::now() - max_age; + transactions.retain(|_, v| match v { + TransactionState::InProgress => true, + TransactionState::Completed(completion_time) => completion_time >= &mut cutoff, + }); +} + +pub enum TransactionState { + InProgress, + Completed(SystemTime), +} diff --git a/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/vfs.rs b/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/vfs.rs new file mode 100644 index 0000000000..89673d0742 --- /dev/null +++ b/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/vfs.rs @@ -0,0 +1,285 @@ +use std::cmp::Ordering; +use std::sync::Once; +use std::time::SystemTime; + +use async_trait::async_trait; + +use crate::nfs; +use crate::nfs::*; +#[derive(Default, Debug)] +pub struct DirEntrySimple { + pub fileid: fileid3, + pub name: filename3, +} +#[derive(Default, Debug)] +pub struct ReadDirSimpleResult { + pub entries: Vec, + pub end: bool, +} + +#[derive(Default, Debug)] +pub struct DirEntry { + pub fileid: fileid3, + pub name: filename3, + pub attr: fattr3, +} +#[derive(Default, Debug)] +pub struct ReadDirResult { + pub entries: Vec, + pub end: bool, +} + +impl ReadDirSimpleResult { + fn from_readdir_result(result: &ReadDirResult) -> ReadDirSimpleResult { + let entries: Vec = result + .entries + .iter() + .map(|e| DirEntrySimple { + fileid: e.fileid, + name: e.name.clone(), + }) + .collect(); + ReadDirSimpleResult { + entries, + end: result.end, + } + } +} + +static mut GENERATION_NUMBER: u64 = 0; +static GENERATION_NUMBER_INIT: Once = Once::new(); + +fn get_generation_number() -> u64 { + unsafe { + GENERATION_NUMBER_INIT.call_once(|| { + GENERATION_NUMBER = SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_millis() as u64; + }); + GENERATION_NUMBER + } +} + +/// What capabilities are supported +pub enum VFSCapabilities { + ReadOnly, + ReadWrite, +} + +/// The basic API to implement to provide an NFS file system +/// +/// Opaque FH +/// --------- +/// Files are only uniquely identified by a 64-bit file id. (basically an inode number) +/// We automatically produce internally the opaque filehandle which is comprised of +/// - A 64-bit generation number derived from the server startup time +/// (i.e. so the opaque file handle expires when the NFS server restarts) +/// - The 64-bit file id +// +/// readdir pagination +/// ------------------ +/// We do not use cookie verifier. We just use the start_after. The +/// implementation should allow startat to start at any position. That is, +/// the next query to readdir may be the last entry in the previous readdir +/// response. +// +/// There is a wierd annoying thing about readdir that limits the number +/// of bytes in the response (instead of the number of entries). The caller +/// will have to truncate the readdir response / issue more calls to readdir +/// accordingly to fill up the expected number of bytes without exceeding it. +// +/// Other requirements +/// ------------------ +/// getattr needs to be fast. NFS uses that a lot +// +/// The 0 fileid is reserved and should not be used +#[async_trait] +pub trait NFSFileSystem: Sync { + /// Returns the set of capabilities supported + fn capabilities(&self) -> VFSCapabilities; + /// Returns the ID the of the root directory "/" + fn root_dir(&self) -> fileid3; + /// Look up the id of a path in a directory + /// + /// i.e. given a directory dir/ containing a file a.txt + /// this may call lookup(id_of("dir/"), "a.txt") + /// and this should return the id of the file "dir/a.txt" + /// + /// This method should be fast as it is used very frequently. + async fn lookup(&self, dirid: fileid3, filename: &filename3) -> Result; + + /// Returns the attributes of an id. + /// This method should be fast as it is used very frequently. + async fn getattr(&self, id: fileid3) -> Result; + + /// Sets the attributes of an id + /// this should return Err(nfsstat3::NFS3ERR_ROFS) if readonly + async fn setattr(&self, id: fileid3, setattr: sattr3) -> Result; + + /// Reads the contents of a file returning (bytes, EOF) + /// Note that offset/count may go past the end of the file and that + /// in that case, all bytes till the end of file are returned. + /// EOF must be flagged if the end of the file is reached by the read. + async fn read(&self, id: fileid3, offset: u64, count: u32) -> Result<(Vec, bool), nfsstat3>; + + /// Writes the contents of a file returning (bytes, EOF) + /// Note that offset/count may go past the end of the file and that + /// in that case, the file is extended. + /// If not supported due to readonly file system + /// this should return Err(nfsstat3::NFS3ERR_ROFS) + async fn write(&self, id: fileid3, offset: u64, data: &[u8]) -> Result; + + /// Creates a file with the following attributes. + /// If not supported due to readonly file system + /// this should return Err(nfsstat3::NFS3ERR_ROFS) + async fn create(&self, dirid: fileid3, filename: &filename3, attr: sattr3) -> Result<(fileid3, fattr3), nfsstat3>; + + /// Creates a file if it does not already exist + /// this should return Err(nfsstat3::NFS3ERR_ROFS) + async fn create_exclusive(&self, dirid: fileid3, filename: &filename3) -> Result; + + /// Makes a directory with the following attributes. + /// If not supported dur to readonly file system + /// this should return Err(nfsstat3::NFS3ERR_ROFS) + async fn mkdir(&self, dirid: fileid3, dirname: &filename3) -> Result<(fileid3, fattr3), nfsstat3>; + + /// Removes a file. + /// If not supported due to readonly file system + /// this should return Err(nfsstat3::NFS3ERR_ROFS) + async fn remove(&self, dirid: fileid3, filename: &filename3) -> Result<(), nfsstat3>; + + /// Removes a file. + /// If not supported due to readonly file system + /// this should return Err(nfsstat3::NFS3ERR_ROFS) + async fn rename( + &self, + from_dirid: fileid3, + from_filename: &filename3, + to_dirid: fileid3, + to_filename: &filename3, + ) -> Result<(), nfsstat3>; + + /// Creates another directory entry for an existing non-directory inode. + /// If not supported due to a read-only file system this should return + /// Err(nfsstat3::NFS3ERR_ROFS). + async fn link( + &self, + id: fileid3, + dirid: fileid3, + filename: &filename3, + ) -> Result; + + /// Makes writes in the requested byte range durable. Implementations that + /// already make every write FILE_SYNC may keep the default implementation. + async fn commit( + &self, + id: fileid3, + _offset: offset3, + _count: count3, + ) -> Result { + self.getattr(id).await + } + + /// Returns the contents of a directory with pagination. + /// Directory listing should be deterministic. + /// Up to max_entries may be returned, and start_after is used + /// to determine where to start returning entries from. + /// + /// For instance if the directory has entry with ids [1,6,2,11,8,9] + /// and start_after=6, readdir should returning 2,11,8,... + // + async fn readdir( + &self, + dirid: fileid3, + start_after: fileid3, + max_entries: usize, + ) -> Result; + + /// Simple version of readdir. + /// Only need to return filename and id + async fn readdir_simple(&self, dirid: fileid3, count: usize) -> Result { + Ok(ReadDirSimpleResult::from_readdir_result(&self.readdir(dirid, 0, count).await?)) + } + + /// Makes a symlink with the following attributes. + /// If not supported due to readonly file system + /// this should return Err(nfsstat3::NFS3ERR_ROFS) + async fn symlink( + &self, + dirid: fileid3, + linkname: &filename3, + symlink: &nfspath3, + attr: &sattr3, + ) -> Result<(fileid3, fattr3), nfsstat3>; + + /// Reads a symlink + async fn readlink(&self, id: fileid3) -> Result; + + /// Get static file system Information + async fn fsinfo(&self, root_fileid: fileid3) -> Result { + let dir_attr: nfs::post_op_attr = match self.getattr(root_fileid).await { + Ok(v) => nfs::post_op_attr::attributes(v), + Err(_) => nfs::post_op_attr::Void, + }; + + let res = fsinfo3 { + obj_attributes: dir_attr, + rtmax: 1024 * 1024, + rtpref: 1024 * 124, + rtmult: 1024 * 1024, + wtmax: 1024 * 1024, + wtpref: 1024 * 1024, + wtmult: 1024 * 1024, + dtpref: 1024 * 1024, + maxfilesize: 128 * 1024 * 1024 * 1024, + time_delta: nfs::nfstime3 { + seconds: 0, + nseconds: 1000000, + }, + properties: nfs::FSF_LINK + | nfs::FSF_SYMLINK + | nfs::FSF_HOMOGENEOUS + | nfs::FSF_CANSETTIME, + }; + Ok(res) + } + + /// Converts the fileid to an opaque NFS file handle. Optional. + fn id_to_fh(&self, id: fileid3) -> nfs_fh3 { + let gennum = get_generation_number(); + let mut ret: Vec = Vec::new(); + ret.extend_from_slice(&gennum.to_le_bytes()); + ret.extend_from_slice(&id.to_le_bytes()); + nfs_fh3 { data: ret } + } + /// Converts an opaque NFS file handle to a fileid. Optional. + fn fh_to_id(&self, id: &nfs_fh3) -> Result { + if id.data.len() != 16 { + return Err(nfsstat3::NFS3ERR_BADHANDLE); + } + let gen = u64::from_le_bytes(id.data[0..8].try_into().unwrap()); + let id = u64::from_le_bytes(id.data[8..16].try_into().unwrap()); + let gennum = get_generation_number(); + match gen.cmp(&gennum) { + Ordering::Less => Err(nfsstat3::NFS3ERR_STALE), + Ordering::Greater => Err(nfsstat3::NFS3ERR_BADHANDLE), + Ordering::Equal => Ok(id), + } + } + /// Converts a complete path to a fileid. Optional. + /// The default implementation walks the directory structure with lookup() + async fn path_to_id(&self, path: &[u8]) -> Result { + let splits = path.split(|&r| r == b'/'); + let mut fid = self.root_dir(); + for component in splits { + if component.is_empty() { + continue; + } + fid = self.lookup(fid, &component.into()).await?; + } + Ok(fid) + } + + fn serverid(&self) -> cookieverf3 { + let gennum = get_generation_number(); + gennum.to_le_bytes() + } +} diff --git a/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/write_counter.rs b/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/write_counter.rs new file mode 100644 index 0000000000..dc9ce4b444 --- /dev/null +++ b/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/write_counter.rs @@ -0,0 +1,43 @@ +#![allow(dead_code)] +use std::io::Write; + +// from https://stackoverflow.com/questions/42187591/how-to-keep-track-of-how-many-bytes-written-when-using-stdiowrite + +pub struct WriteCounter { + inner: W, + count: usize, +} + +impl WriteCounter +where + W: Write, +{ + pub fn new(inner: W) -> Self { + WriteCounter { inner, count: 0 } + } + + pub fn into_inner(self) -> W { + self.inner + } + + pub fn bytes_written(&self) -> usize { + self.count + } +} + +impl Write for WriteCounter +where + W: Write, +{ + fn write(&mut self, buf: &[u8]) -> std::io::Result { + let res = self.inner.write(buf); + if let Ok(size) = res { + self.count += size + } + res + } + + fn flush(&mut self) -> std::io::Result<()> { + self.inner.flush() + } +} diff --git a/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/xdr.rs b/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/xdr.rs new file mode 100644 index 0000000000..8f4f93ccfc --- /dev/null +++ b/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/xdr.rs @@ -0,0 +1,234 @@ +use std::io::{Read, Write}; + +use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt}; +pub type XDREndian = BigEndian; +use crate::nfs::nfsstring; + +/// See https://datatracker.ietf.org/doc/html/rfc1014 +#[allow(clippy::upper_case_acronyms)] +pub trait XDR { + fn serialize(&self, dest: &mut R) -> std::io::Result<()>; + fn deserialize(&mut self, src: &mut R) -> std::io::Result<()>; +} + +/// Serializes a basic enumeration. +/// Casts everything as u32 BigEndian +#[allow(non_camel_case_types)] +#[macro_export] +macro_rules! xdr_enum_serde { + ($t:ident) => { + impl XDR for $t { + fn serialize(&self, dest: &mut R) -> std::io::Result<()> { + dest.write_u32::(*self as u32) + } + fn deserialize(&mut self, src: &mut R) -> std::io::Result<()> { + let r: u32 = src.read_u32::()?; + if let Some(p) = FromPrimitive::from_u32(r) { + *self = p; + } else { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("Invalid value for {}", stringify!($t)), + )); + } + Ok(()) + } + } + }; +} + +/// Serializes a bool as a 4 byte big endian integer. +impl XDR for bool { + fn serialize(&self, dest: &mut R) -> std::io::Result<()> { + let val: u32 = *self as u32; + dest.write_u32::(val) + } + fn deserialize(&mut self, src: &mut R) -> std::io::Result<()> { + let val: u32 = src.read_u32::()?; + *self = val > 0; + Ok(()) + } +} + +/// Serializes a i32 as a 4 byte big endian integer. +impl XDR for i32 { + fn serialize(&self, dest: &mut R) -> std::io::Result<()> { + dest.write_i32::(*self) + } + fn deserialize(&mut self, src: &mut R) -> std::io::Result<()> { + *self = src.read_i32::()?; + Ok(()) + } +} + +/// Serializes a i64 as a 8 byte big endian integer. +impl XDR for i64 { + fn serialize(&self, dest: &mut R) -> std::io::Result<()> { + dest.write_i64::(*self) + } + fn deserialize(&mut self, src: &mut R) -> std::io::Result<()> { + *self = src.read_i64::()?; + Ok(()) + } +} + +/// Serializes a u32 as a 4 byte big endian integer. +impl XDR for u32 { + fn serialize(&self, dest: &mut R) -> std::io::Result<()> { + dest.write_u32::(*self) + } + fn deserialize(&mut self, src: &mut R) -> std::io::Result<()> { + *self = src.read_u32::()?; + Ok(()) + } +} + +/// Serializes a u64 as a 8 byte big endian integer. +impl XDR for u64 { + fn serialize(&self, dest: &mut R) -> std::io::Result<()> { + dest.write_u64::(*self) + } + fn deserialize(&mut self, src: &mut R) -> std::io::Result<()> { + *self = src.read_u64::()?; + Ok(()) + } +} + +impl XDR for [u8; N] { + fn serialize(&self, dest: &mut R) -> std::io::Result<()> { + dest.write_all(self) + } + fn deserialize(&mut self, src: &mut R) -> std::io::Result<()> { + src.read_exact(self) + } +} + +impl XDR for Vec { + fn serialize(&self, dest: &mut R) -> std::io::Result<()> { + assert!(self.len() < u32::MAX as usize); + let length = self.len() as u32; + length.serialize(dest)?; + dest.write_all(self)?; + // write padding + let pad = ((4 - length % 4) % 4) as usize; + let zeros: [u8; 4] = [0, 0, 0, 0]; + if pad > 0 { + dest.write_all(&zeros[..pad])?; + } + Ok(()) + } + fn deserialize(&mut self, src: &mut R) -> std::io::Result<()> { + let mut length: u32 = 0; + length.deserialize(src)?; + self.resize(length as usize, 0); + src.read_exact(self)?; + // read padding + let pad = ((4 - length % 4) % 4) as usize; + let mut zeros: [u8; 4] = [0, 0, 0, 0]; + src.read_exact(&mut zeros[..pad])?; + Ok(()) + } +} + +impl XDR for nfsstring { + fn serialize(&self, dest: &mut R) -> std::io::Result<()> { + self.0.serialize(dest) + } + fn deserialize(&mut self, src: &mut R) -> std::io::Result<()> { + self.0.deserialize(src) + } +} + +impl XDR for Vec { + fn serialize(&self, dest: &mut R) -> std::io::Result<()> { + assert!(self.len() < u32::MAX as usize); + let length = self.len() as u32; + length.serialize(dest)?; + for i in self { + i.serialize(dest)?; + } + Ok(()) + } + fn deserialize(&mut self, src: &mut R) -> std::io::Result<()> { + let mut length: u32 = 0; + length.deserialize(src)?; + self.resize(length as usize, 0); + for i in self { + i.deserialize(src)?; + } + Ok(()) + } +} + +#[allow(non_camel_case_types)] +#[macro_export] +macro_rules! xdr_struct { + ( + $t:ident, + $($element:ident),* + ) => { + impl XDR for $t { + fn serialize(&self, dest: &mut R) -> std::io::Result<()> { + $(self.$element.serialize(dest)?;)* + Ok(()) + } + fn deserialize(&mut self, src: &mut R) -> std::io::Result<()> { + $(self.$element.deserialize(src)?;)* + Ok(()) + } + } + }; +} + +/// This macro only handles XDR Unions of the form +/// union pre_op_attr switch (bool attributes_follow) { +/// case TRUE: +/// wcc_attr attributes; +/// case FALSE: +/// void; +/// }; +/// This is translated to +/// enum pre_op_attr { +/// Void, +/// attributes(wcc_attr) +/// } +/// The serde methods can be generated with XDRBoolUnion(pre_op_attr, attributes, wcc_attr) +/// The "true" type must have the Default trait +#[allow(non_camel_case_types)] +#[macro_export] +macro_rules! xdr_bool_union { + ( + $t:ident, $enumcase:ident, $enumtype:ty + ) => { + impl XDR for $t { + fn serialize(&self, dest: &mut R) -> std::io::Result<()> { + match self { + $t::Void => { + false.serialize(dest)?; + }, + $t::$enumcase(v) => { + true.serialize(dest)?; + v.serialize(dest)?; + }, + } + Ok(()) + } + fn deserialize(&mut self, src: &mut R) -> std::io::Result<()> { + let mut c: bool = false; + c.deserialize(src)?; + if c == false { + *self = $t::Void; + } else { + let mut r = <$enumtype>::default(); + r.deserialize(src)?; + *self = $t::$enumcase(r); + } + Ok(()) + } + } + }; +} + +pub(crate) use xdr_bool_union; +pub(crate) use xdr_enum_serde; +pub(crate) use xdr_struct; diff --git a/agent_files/mount/macos/nfs_gateway/vexfs_nfs_gateway_main.cpp b/agent_files/mount/macos/nfs_gateway/vexfs_nfs_gateway_main.cpp new file mode 100644 index 0000000000..98ac65a0a6 --- /dev/null +++ b/agent_files/mount/macos/nfs_gateway/vexfs_nfs_gateway_main.cpp @@ -0,0 +1,83 @@ +#include "vexfs_nfs_gateway.h" +#include "vexfs_runtime_types.h" + +#include +#include +#include +#include +#include + +namespace { + +struct Options { + std::string backend; + std::string connection; + std::string workspace; + std::string principal; + std::string listen_address = "127.0.0.1"; + uint16_t port = 0; + uint32_t timeout_ms = 30'000; +}; + +uint32_t ParseUnsigned(const char *value, uint32_t maximum, const char *name) { + char *end = nullptr; + const unsigned long parsed = std::strtoul(value, &end, 10); + if (end == value || *end != '\0' || parsed == 0 || parsed > maximum) + throw std::runtime_error(std::string("invalid ") + name); + return static_cast(parsed); +} + +Options Parse(int argc, char **argv) { + Options options; + for (int index = 1; index < argc; ++index) { + const std::string argument = argv[index]; + if (index + 1 >= argc) throw std::runtime_error("missing value for " + argument); + const char *value = argv[++index]; + if (argument == "--backend") options.backend = value; + else if (argument == "--connection") options.connection = value; + else if (argument == "--workspace") options.workspace = value; + else if (argument == "--principal") options.principal = value; + else if (argument == "--listen") options.listen_address = value; + else if (argument == "--port") + options.port = static_cast(ParseUnsigned(value, 65535, "port")); + else if (argument == "--timeout-ms") + options.timeout_ms = ParseUnsigned(value, 3'600'000, "timeout"); + else throw std::runtime_error("unknown argument: " + argument); + } + if (options.backend == VEXFS_RUNTIME_BACKEND_SQLITE && options.principal.empty()) + options.principal = "local"; + if (options.backend.empty() || options.connection.empty() || + options.workspace.empty() || options.port == 0) + throw std::runtime_error( + "backend, connection, workspace and port are required"); + if (options.listen_address != "127.0.0.1") + throw std::runtime_error("NFS gateway must listen on 127.0.0.1"); + return options; +} + +} // namespace + +int main(int argc, char **argv) { + try { + const Options options = Parse(argc, argv); + const vexfs_nfs_gateway_config config{ + VEXFS_RUNTIME_ABI_VERSION, + options.backend.c_str(), + options.connection.c_str(), + options.workspace.c_str(), + options.principal.c_str(), + options.listen_address.c_str(), + options.port, + options.timeout_ms, + }; + char error[1024]{}; + const int result = vexfs_nfs_gateway_run(&config, error, sizeof(error)); + if (result != 0) + std::cerr << "vexfs-nfs-gateway: " + << (error[0] == '\0' ? "gateway failed" : error) << '\n'; + return result; + } catch (const std::exception &error) { + std::cerr << "vexfs-nfs-gateway: " << error.what() << '\n'; + return 2; + } +} diff --git a/docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md b/docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md index 9c8dd4e546..a791a9eed5 100644 --- a/docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md +++ b/docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md @@ -4,7 +4,7 @@ - 所属产品：VexDB-Lite 的文件管理能力 VexFS - 日期：2026-07-24 - 分支：`feature/agent_files` -- 文档版本：3.0 +- 文档版本：3.1 - macOS 默认入口：本机 NFS gateway；FSKit 延后为可选原生增强 - 当前阶段：Phase 2 PostgreSQL 功能与验收已完成；最后一次数据审计已让所有变更记录 认证用户、workspace、commit、对象和前后版本，并保留 workspace 删除证据。PG 16–19、 @@ -15,9 +15,10 @@ 公证票据，仍不能作为正式分发结论；两台 Mac 当前又通过文件/快照往返 50 项和 extension、 网络、数据库故障恢复 25 项。最初产品场景也已由两台 Mac 各运行一次真实 OpenCode 完成 串行闭环：7 + 9 + 21 条检查通过，两个完整工作区快照可比较并一键恢复。安装器的旧 UUID - 刷新逻辑已经修复。下一步必须从干净提交 - 原定 FSKit `preview.38` 不再是 0.1 阻塞项。下一步先完成 macOS NFS 默认 adapter、 - 无 FSKit 授权安装包和干净机器 Gate；已有 FSKit 能力保留为后续可选 backend + 刷新逻辑已经修复。macOS 默认 NFS adapter、AppleDouble 隔离、NFSv3 LINK/COMMIT、 + 安装包 gateway 和 35 项真机 eval 已完成。原定 FSKit `preview.38` 不再是 0.1 阻塞项。 + 下一步是从干净提交生成签名公证包，在未启用 FSKit 的干净 Mac 跑安装 Gate，并补 + gateway crash、sleep/wake、文件锁和长期 Coding Agent 负载。 ## 0. 2026-07-24 路线调整 @@ -34,17 +35,18 @@ 新的最近任务顺序： -1. 冻结 NFS adapter 与 Workspace Engine 的窄接口，不复制数据库逻辑； -2. 做 localhost-only NFS prototype，完成 mount/unmount/status/doctor； -3. 跑 Bash、Git、OpenCode、npm/cargo、文件锁、xattr、fsync 和 gateway 随机 kill； -4. 跑 1,000 小文件与 APFS/现有 FSKit 的同机对照，确认 NFS 是否改善吞吐； -5. 生成不含 FSKit 授权前置的签名公证包，在干净 Mac 验证安装和系统提示； -6. 通过正确性和安装 Gate 后，才把 NFS 设为代码中的默认 backend。 +1. **已完成**：冻结 NFS adapter 与 Workspace Engine 的窄接口，不复制数据库逻辑； +2. **已完成**：实现 localhost-only NFS gateway、mount/unmount/status/doctor； +3. **部分完成**：Bash、Git、hardlink、xattr、fsync 已进入真机 eval；OpenCode、npm/cargo、 + 文件锁和 sleep/wake 尚待补齐；gateway 单次 `SIGKILL` 后的强制卸载、状态清理和重挂载已覆盖； +4. **已完成首轮**：1,000 小文件为 181.689 files/s，同轮 APFS 为 14304.029 files/s； +5. **待完成**：生成不含 FSKit 授权前置的签名公证包，在干净 Mac 验证安装和系统提示； +6. **代码已完成，发行待 Gate**：默认 backend 已改为 NFS，FSKit 只在显式选择时使用。 -### 0.1 2026-07-24 本机 NFS 原型验证 +### 0.1 2026-07-24 本机 NFS 实现与验证 -已在 macOS 26.3.1 arm64 真机完成第一轮传输层验证。测试使用只监听 -`127.0.0.1:11111` 的用户态 NFSv3 服务和系统 `/sbin/mount_nfs`，没有启用或调用 +已在 macOS 26.3.1 arm64 真机完成连接 SQLite mount runtime 的默认 NFS adapter。测试使用 +动态 loopback 端口的用户态 NFSv3 gateway 和系统 `/sbin/mount_nfs`，没有启用或调用 VexFS FSKit extension。 已确认： @@ -53,26 +55,25 @@ VexFS FSKit extension。 - `ls`、`cat`、`grep`、创建、`mkdir`、`cp`、`mv`、`rm`、`chmod` 和 symlink 通过； - 挂载目录中完成了 `git init`、`git add` 和一次 commit； - `mount_nfs` 支持固定 `port`/`mountport`，因此 gateway 不需要占用系统 portmapper； -- 功能原型证明“NFS 默认、FSKit 可选”可行，但尚未连接 VexFS SQLite/PG HostStore，不能写成 - VexDB NFS adapter 已完成。 +- NFS gateway 已连接平台无关 `vexfs_mount_*` C ABI；SQLite 已完成，PG 复用同一 HostStore； +- 公开版 `nfsserve` 缺少 LINK/COMMIT，仓库内固定 fork 已补协议 handler，真实 hardlink 的 + inode、link count、共享内容、重挂载和快照恢复均通过； +- 41 项真实 NFS eval、30 项 package smoke 通过。 -同时发现两个发布阻塞项： +同时确认两个性能和实现边界： 1. NFSv3 没有 macOS named attributes。macOS 26.3.1 为新文件写入 - `com.apple.provenance` 时，会额外创建 `._文件名` AppleDouble 文件。200 个用户文件实际产生 - 200 个 sidecar；必须决定由 gateway 内部吸收、隐藏并映射为 VexFS xattr，不能让它们污染 - workspace 历史、配额、grep 和快照。 -2. 优化构建、关闭 debug 日志并使用 AgentFS 同类挂载参数后，200 个 1 字节文件耗时 - 2.66 秒，约 75 个用户文件/秒；同轮 `/tmp` APFS 为 0.03 秒。这个镜像原型不是数据库 - adapter，不能代表最终吞吐，但已经证明不能只靠更换 mount 协议自动解决小文件性能。 + `com.apple.provenance` 时会尝试写 `._文件名`。当前 gateway 已将 raw AppleDouble 数据吸收为 + 目标 inode xattr，用户树和数据库 dentry 都为 0 个 `._*`。 +2. 当前真实 SQLite adapter 创建 1,000 个 2-byte 文件为 5.503902 秒、181.689 files/s；同轮 + APFS 为 0.069910 秒、14304.029 files/s，约慢 78.7 倍。它与此前 FSKit 约 175 files/s + 同级，0.1 可用但不是性能达标终点。 因此最近任务增加两个硬 Gate： -- NFS eval 必须断言用户视图、数据库路径、历史、快照和配额中都没有可见 `._*` 垃圾对象， - 同时保留需要的 macOS xattr； -- 先实现批量提交、写回合并和有界 metadata cache，再做同机 1,000/10,000 文件对照。 - NFS 默认发布至少不能明显低于当前 FSKit 基线；若达不到，继续作为实验 backend，不能仅因 - 免授权就替换默认入口。 +- AppleDouble 隔离已进入 eval；后续每次 NFS 发布必须保留该断言； +- 0.2 实现批量提交、写回合并和有界 metadata cache，再做同机 1,000/10,000 文件对照； +- 0.1 发布前仍需完成干净 Mac 安装、公证、重复 crash/sleep-wake、文件锁与长期运行 Gate。 ## 1. 这份文档解决什么问题 @@ -165,11 +166,11 @@ vexdb fs --workspace workspace snapshot restore before-refactor | 文件语义 | 文件、目录、四类时间戳、并发 append、进程锁、mode、symlink、xattr、hardlink、owner/group 元数据、便携 ACL 已进入数据库合同和测试；PG 已执行 ACL 和继承 | SQLite 尚未执行完整 ACL 授权；特殊文件不支持 | | 版本恢复 | 单文件版本、workspace commit、snapshot、diff、expected-head restore、原生备份、retention、显式分批 GC、live quota 和 format v2 逻辑导入导出已实现 | 自动维护、history/staging/index/total quota 和 DuckDB 导入端未完成 | | 长期校验 | `chunked-v1` 使用不可变 manifest、64 KiB 块和逐块/整文件 SHA-256；只读 check 覆盖结构、引用、顺序、大小和流式内容校验 | 自动 repair 和跨文件通用去重不在当前范围 | -| macOS | preview.22 已从干净 commit `8a8b0c139a` 构建并通过 Apple 公证；本机 13 个真实挂载 Gate 通过，另一台无源码 M1 macOS 26.5.2 的 30 项安装后 smoke 也通过。preview.37 的 FSKit 已启用，本机 PG FSKit 13/13、247 项和凭据 9 项通过 | preview.37 没有 stapled 公证票据；要从干净提交公证 preview.38。macOS 26.0–26.4 与 x86_64 尚未独立发行验证 | +| macOS | 默认 NFS 已连接 SQLite runtime，35 项真机 eval 和 30 项 package smoke 通过；hardlink、COMMIT/fsync、AppleDouble 隔离、Git、重挂载和快照恢复均已验证。既有 FSKit 仍保留并有 preview.22/preview.37 真机证据 | 默认 NFS 尚未从干净提交生成公证包，也未在未启用 FSKit 的干净 Mac 验证；gateway crash/sleep-wake、锁、长期 Agent、macOS 26.0–26.4 和 x86_64 待补 | | Linux | libfuse3 真实挂载已通过 root 和 uid 1000 各 9 组；时间戳、并发 append、进程锁、打开文件生命周期、强制卸载和 helper 崩溃恢复已验证；异常撤销后底层目录保持 0500，显式卸载恢复 0700；x86_64/AArch64 manylinux 安装包已在对应架构真机完成无 root 安装回归 | 更多干净发行版、长期运行和真实挂载安装回归仍不足 | | Windows | 只有平台边界和规划 | WinFsp adapter、安装签名、路径/ACL/SID 合同均未实现 | | 远程共享 | PostgreSQL 已通过多 gateway、macOS FSKit、Linux FUSE、数据库重启、当前源码下的 macOS↔Linux 47 项、两台 Mac 文件/快照 50 项、双 Mac 串行真实 OpenCode 7 + 9 + 21 项，以及双机 extension 崩溃、网络中断和数据库停机 25 项 | 当前审计源码对应的公证包尚未生成，需要重跑双机安装 Gate；局域网直连 FSKit 还需要 macOS 本地网络授权；PG 文本索引尚未实现 | -| 性能规模 | SQLite 直连已完成 10 万文件；preview.20 真机为 46.863 MiB/s 顺序写、1,041.044 次/s 随机覆盖、175.253 files/s，1 千文件普通扫描 428.041 ms、索引搜索 7.449 ms | 小文件创建仍比同轮 APFS 慢 72.012 倍，普通扫描慢 23.422 倍；后续仍需重跑 1 万/10 万和真实 Coding Agent | +| 性能规模 | SQLite 直连已完成 10 万文件；默认 NFS 1,000 小文件为 5.503902 秒、181.689 files/s，同轮 APFS 0.069910 秒、14304.029 files/s；此前 FSKit 为约 175 files/s | NFS 小文件仍比 APFS 慢约 78.7 倍；0.2 需批量提交、写回合并、metadata cache，并重跑 1 万/10 万和真实 Coding Agent | “合同中已保存”不等于“所有平台已经完整执行”。例如 owner/group 和便携 ACL 已经可以保存、读取和恢复，但身份认证、权限判断和各系统的原生 ACL 映射仍属于后续阶段。 diff --git a/docs/specs/2026-07-16_vexfs-product-spec.md b/docs/specs/2026-07-16_vexfs-product-spec.md index 65b69c9555..c2f89730ca 100644 --- a/docs/specs/2026-07-16_vexfs-product-spec.md +++ b/docs/specs/2026-07-16_vexfs-product-spec.md @@ -5,8 +5,8 @@ - 所属项目：VexDB-Lite - 日期：2026-07-16 - 分支：`feature/agent_files` -- 文档版本：1.9 -- 状态：SQLite `0.9.0`、macOS FSKit、Linux libfuse3 和 arm64 真机交付已实现； +- 文档版本：2.0 +- 状态：SQLite `0.9.0`、macOS 默认 NFS、可选 FSKit、Linux libfuse3 和 arm64 真机交付已实现； PostgreSQL `0.4.0-alpha.1` 的数据库合同、format v2、ACL、审计、libpq HostStore、 macOS/Linux 真实 mount、双 Mac 串行 Agent 工作区和备份恢复已实现 - 适用范围：VexFS 完整产品定义；开发顺序以 `docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md` 为准 @@ -33,12 +33,22 @@ Linux FUSE 和 Windows WinFsp 必须共用同一 Workspace Engine、mount runtim - NFS 协议版本、mount 权限提示、锁、xattr、缓存一致性、sleep/wake 和 gateway crash 必须 通过真机 Gate 后，才能宣传“安装后无缝 Bash”。 -2026-07-24 已完成不连接数据库的 localhost NFSv3 传输原型。macOS 26.3.1 arm64 当前用户可 -直接 mount/unmount，普通 Bash 文件操作、mode、symlink 和 `git init/add/commit` 已通过； -这只证明系统入口可行，不表示 VexDB NFS adapter 已实现。原型还发现 macOS -`com.apple.provenance` 会在 NFSv3 上落成每文件一个 `._*` AppleDouble sidecar；优化后的镜像 -原型创建 200 个用户文件耗时 2.66 秒，同轮 APFS 为 0.03 秒。因此 AppleDouble 内部映射和 -小文件写回合并是默认 NFS 发布 Gate，不能把“免 FSKit 授权”当作唯一选型标准。 +2026-07-24 已完成连接 SQLite mount runtime 的 localhost NFSv3 adapter，并设为 macOS CLI +默认 driver。gateway 只监听 loopback，由 `mount/unmount/status/doctor` 管理独立 PID、端口、 +日志和受保护状态目录。真实 macOS 26.3.1 arm64 eval 共 41 项，覆盖 Bash、hardlink、symlink、 +mode、fsync、xattr、原子替换、open-then-unlink、Git、gateway `SIGKILL` 恢复、卸载/重挂载和完整 workspace 快照恢复； +package smoke 30 项也通过。公开版 `nfsserve` 缺少 NFSv3 LINK/COMMIT，当前仓库固定维护一个 +小型 fork，LINK 调用数据库 hardlink 合同，COMMIT 调用强同步并返回同一 server verifier。 + +默认挂载使用 bounded soft NFS：固定 10 秒重传间隔、最多 4 次，并设 60 秒 dead timeout。 +它让 gateway 崩溃时的文件调用最终返回错误而不是永久卡死，同时覆盖 runtime 最长 30 秒的 +数据库 busy timeout。应用必须把返回的 I/O 错误当成失败，不能假定写入成功。 + +macOS 写入的 AppleDouble 由 gateway 吸收为目标 inode 的 +`io.vexdb.macos.appledouble` xattr，不进入用户目录、dentry、grep 或快照树。1000 个 2-byte +小文件创建为 5.503902 秒、181.689 files/s；同轮 APFS 为 0.069910 秒、14304.029 files/s， +约慢 78.7 倍。这个吞吐与此前 FSKit 约 175 files/s 同级，满足“不比现有默认入口明显退化”的 +0.1 入口条件，但远未接近原生盘，列为 0.2 批量提交、写回合并和 metadata cache 优化项。 后文中的 FSKit 完成状态仍是历史事实；凡是把 FSKit 写成“默认、MVP 主入口或当前发布前置”的 旧表述，均由本节替代。 @@ -449,7 +459,11 @@ git diff vexfs mount status ``` -用户不需要给 `ls/cat/rg/sed/python` 添加 `vexfs` 前缀。文件在平台 adapter 成功 publish/sync 后提交为新版本；macOS MVP 由 FSKit `synchronize` 和可报告结果的 close 路径触发。需要把多个文件和业务 SQL 一起原子提交时，必须使用显式 changeset/事务入口，不能假设多条 Bash 命令天然属于同一个数据库事务。 +用户不需要给 `ls/cat/rg/sed/python` 添加 `vexfs` 前缀。文件在平台 adapter 成功 +publish/sync 后提交为新版本；macOS 默认 NFS 的 WRITE 返回 FILE_SYNC，显式 COMMIT/fsync、 +gateway 同步和安全卸载都映射到数据库强同步。可选 FSKit 继续使用 `synchronize` 和可报告 +结果的 close 路径。需要把多个文件和业务 SQL 一起原子提交时，必须使用显式 +changeset/事务入口，不能假设多条 Bash 命令天然属于同一个数据库事务。 ## 8. 系统边界和所有权 @@ -778,7 +792,7 @@ VexFS 使用四层数据保护： | FR-MNT-016 | P0 | 路径规范化、`.`、`..` 和越界规则必须与 SQL 合同一致；当前符号链接明确采用 POSIX target 原样保存合同，未来的 workspace 内限制策略必须作为独立可选合同。 | | FR-MNT-017 | P0 | 不支持设备文件、socket、FIFO 和 setuid/setgid 文件；创建时返回明确错误。 | | FR-MNT-018 | P1 | MVP Gate 不承诺 advisory flock；Phase 0 必须记录 macOS NFS 的 `flock/fcntl` 实际行为。若默认 mount 使用 `nolocks`，doctor 和能力矩阵必须明确报告，不能声称适合在挂载内运行依赖文件锁的数据库。 | -| FR-MNT-019 | P1 | 支持 hardlink，并在支持前对 link 调用返回稳定的不支持错误。 | +| FR-MNT-019 | P1 | 支持 hardlink。SQLite、Linux FUSE、macOS FSKit 和默认 NFS 已接入同一合同；NFSv3 LINK 由仓库内固定 fork 实现并进入真实 eval。 | | FR-MNT-020 | P1 | 已支持扩展属性，并把值持久化到 SQLite；未知或不存在的 xattr 返回稳定错误。 | #### 10.13.3 写入、版本与事务 @@ -1436,6 +1450,9 @@ chmod/chown 等已明确不支持的权限修改返回 `EPERM`；跨 workspace r 当前验收已经额外覆盖 `chmod` 的 `0000..0777` mode、可执行脚本、symlink、readlink、xattr、hardlink、owner/group 元数据和便携 ACL；Linux 已覆盖 chown 和 hardlink 的 mount 映射。仍不验收多 gateway、完整 ACL 授权、跨 gateway flock、共享可写 mmap、10 万文件和 DuckDB mount。 +2026-07-24 默认 NFS 已通过 35 项真机回归和 30 项 package smoke，包含 hardlink 与 fsync； +正式 0.1 仍需补干净 Mac 安装、公证包、gateway crash/sleep-wake、锁行为和长期负载 Gate。 + ### 20.2 P0 产品验收 | ID | 验收场景 | diff --git a/tests/eval/vexfs/README.md b/tests/eval/vexfs/README.md index 3005972bf6..696017f46d 100644 --- a/tests/eval/vexfs/README.md +++ b/tests/eval/vexfs/README.md @@ -6,7 +6,7 @@ ## 运行 ```bash -# 默认 full：构建后覆盖功能、恢复、并发、备份、性能和 FSKit 编译 +# 默认 full：构建后覆盖功能、恢复、并发、备份、性能和平台 adapter 编译 bash build_sqlite.sh eval # 日常快速回归 @@ -39,11 +39,17 @@ VEXDB_LITE_PACKAGE_STAGE=/absolute/path/to/stage \ tests/eval/vexfs/python.sh tests/eval/vexfs/run.py --mode full --filter mount \ --mount-cli /absolute/path/to/vexfs --fail-on-skip +# macOS 默认 NFS：真实 Bash、hardlink/symlink、fsync、xattr、Git、 +# 1000 个小文件、卸载/重挂载、完整快照恢复；文件数硬限制为 1..10000 +VEXDB_NFS_FILE_COUNT=1000 \ + bash tests/eval/vexfs/run_macos_nfs_mount.sh \ + /absolute/path/to/vexdb + # Linux AArch64/x86_64：在真实 /dev/fuse 上分别以 root 和 uid 1000 # 运行与 macOS 完全相同的 mount 合同（需要 Docker Desktop 或 Docker Engine） bash tests/eval/vexfs/run_linux_mount.sh -# 用同一个 SQLite 文件完成 macOS FSKit → Linux FUSE → macOS FSKit 往返 +# 用同一个 SQLite 文件完成 macOS adapter → Linux FUSE → macOS adapter 往返 bash tests/eval/vexfs/run_cross_platform_portability.sh # 显式调用真实 OpenCode 模型，在挂载项目中自主改代码并跑测试 @@ -280,15 +286,17 @@ quick/full/stress 的峰值 RSS 保护线分别为 1 GiB、1.5 GiB、2 GiB；SQL 只留下一个超时字符串；成功时这些大文件自动清理。搜索等待时间由 `VEXFS_EVAL_NATIVE_SEARCH_TIMEOUT_SECONDS` 控制。 -`mount.real-bash` 是环境 Gate。macOS、FSKit 和已启用的 VexFS 系统扩展都满足时， -它会真实挂载并运行 `mkdir`、`cat`、`grep`、`cp`、`mv`、`find`、`rm` 等命令； -条件不满足时报告会明确写 `SKIP` 和原因，不会把未执行伪装成通过。 +`mount.real-bash` 是 macOS 默认 NFS 环境 Gate。它只使用系统自带 NFS client 和本机 +loopback gateway，不要求用户启用 FSKit；会真实运行 `mkdir`、`cat`、`grep`、`cp`、 +`mv`、`find`、`rm` 等命令。`run_macos_nfs_mount.sh` 是更完整的发版前 Gate，还覆盖 +hardlink、fsync、Git、AppleDouble 隔离、重挂载和快照恢复。条件不满足时必须失败或明确 +报告 `SKIP`，不能用 C ABI smoke 冒充真实挂载通过。 `mount.real-linux-bash-git` 是 Linux 环境 Gate。构建中存在 `vexfs-fuse`，且 `/dev/fuse`、`fusermount3` 和 Git 可用时，它会以真实挂载目录验证完整 Git workspace，并在卸载和重挂载后再次运行 `git status`。 `mount.cross-platform-conformance` 是平台一致性 Gate。Linux 可通过 -`run_linux_mount.sh` 自动准备 libfuse3 环境并验证 root/普通用户；macOS 必须先在 -“系统设置 → 通用 → 登录项与扩展 → 文件系统扩展”中启用 VexDB Lite，再运行该 -用例。缺少真实挂载条件时结果是 `SKIP`，不会用 C ABI smoke 冒充平台通过。 +`run_linux_mount.sh` 自动准备 libfuse3 环境并验证 root/普通用户；macOS 默认使用 NFS， +无需系统扩展授权。只有显式传 `--mount-driver fskit` 时才要求在系统设置中启用 VexDB Lite。 +缺少真实挂载条件时结果是 `SKIP`，不会用 C ABI smoke 冒充平台通过。 diff --git a/tests/eval/vexfs/documentation_smoke.sh b/tests/eval/vexfs/documentation_smoke.sh index 7354e7216c..47b89dda69 100755 --- a/tests/eval/vexfs/documentation_smoke.sh +++ b/tests/eval/vexfs/documentation_smoke.sh @@ -92,8 +92,12 @@ equal "$PACKAGE_ARCHITECTURE" \ "FSKit extension architecture" check test -x "$STAGE/bin/vexdb" check test -L "$STAGE/bin/vexfs" +check test -x "$STAGE/bin/vexfs-nfs-gateway" check test -f "$STAGE/lib/vexdb_lite.dylib" check test -f "$STAGE/lib/runtime/libpq.5.dylib" +equal "$PACKAGE_ARCHITECTURE" \ + "$(lipo -archs "$STAGE/bin/vexfs-nfs-gateway")" \ + "NFS gateway architecture" EXTENSION_ENTITLEMENTS="$TMP_DIR/extension-entitlements.plist" EXTENSION_ENTITLEMENTS_LOG="$TMP_DIR/extension-entitlements.log" if ! codesign -d --entitlements - --xml \ @@ -128,14 +132,17 @@ if [ "$PACKAGE_SIGNATURE" = developer-id ]; then "$EXTENSION_APPLICATION" "FSKit application identifier" fi check test -f "$STAGE/使用说明.md" +check test -f "$STAGE/THIRD_PARTY.md" +check test -f "$STAGE/licenses/nfsserve-LICENSE" +check grep -q 'nfsserve 0.11.0' "$STAGE/THIRD_PARTY.md" +check grep -q 'BSD 3-Clause License' "$STAGE/licenses/nfsserve-LICENSE" check grep -q 'Developer ID' "$STAGE/使用说明.md" -check grep -q '按类别' "$STAGE/使用说明.md" -check grep -q '文件系统扩展' "$STAGE/使用说明.md" +check grep -q '默认使用系统自带的 NFS client' "$STAGE/使用说明.md" +check grep -q '127.0.0.1:/' "$STAGE/使用说明.md" check grep -q '底层目录设为' "$STAGE/使用说明.md" check grep -q '不会被 VexDB Lite 自动重放' "$STAGE/使用说明.md" check grep -q 'verify_installed_mount' "$STAGE/install.sh" -check grep -q '正在使用的 FSKit 文件系统' "$STAGE/install.sh" -check grep -q 'pkill -KILL -u.*fskit_agent' "$STAGE/install.sh" +check grep -q '默认 NFS 真实挂载检查通过' "$STAGE/install.sh" expect_fail grep -q 'killall -u.*pkd' "$STAGE/install.sh" expect_fail grep -q 'lsregister.*-kill' "$STAGE/install.sh" check grep -Eq '不会.*清空 LaunchServices 注册库' "$STAGE/使用说明.md" @@ -194,10 +201,12 @@ VEXDB_LITE_ALLOW_DIRTY_INSTALL="$ALLOW_DIRTY" \ VEXDB="$BIN_DIR/vexdb" VEXFS="$BIN_DIR/vexfs" +NFS_GATEWAY="$BIN_DIR/vexfs-nfs-gateway" DYLIB="$LIB_DIR/vexdb_lite.dylib" check test -d "$APP_DIR/VexDB Lite.app" check test -x "$VEXDB" check test -L "$VEXFS" +check test -x "$NFS_GATEWAY" check test -f "$DYLIB" contains "$("$VEXDB" --version)" "$EXPECTED_PRODUCT_VERSION" "version" contains "$(env HOME="$TEST_HOME" PATH="$BIN_DIR:/usr/bin:/bin:/usr/sbin:/sbin" vexdb --version)" \ @@ -304,26 +313,10 @@ equal "first" "$(HOME="$TEST_HOME" "$VEXDB" fs cat /docs/plan.md)" \ expect_fail env HOME="$TEST_HOME" "$VEXDB" fs stat /after-snapshot/new.txt HOME="$TEST_HOME" "$VEXDB" fs snapshot diff before-agent >/dev/null -# doctor 必须如实报告当前机器的 FSKit 状态。临时 HOME 只隔离数据库， -# 不会隔离系统已经注册的扩展，因此测试机可能是 missing、registered、disabled -# 或 enabled。 -set +e -DOCTOR_JSON="$(HOME="$TEST_HOME" "$VEXDB" fs doctor --json 2>/dev/null)" -DOCTOR_RC=$? -set -e -DOCTOR_STATE="$(printf '%s\n' "$DOCTOR_JSON" | sed -n 's/.*"extension":"\([^"]*\)".*/\1/p')" -case "$DOCTOR_STATE" in - enabled) - equal "0" "$DOCTOR_RC" "doctor enabled extension exit code" - ;; - missing|registered|disabled|service-unavailable) - equal "1" "$DOCTOR_RC" "doctor unavailable extension exit code" - ;; - *) - fail "unexpected doctor extension state: ${DOCTOR_STATE:-empty}" - ;; -esac -contains "$DOCTOR_JSON" "\"extension\":\"$DOCTOR_STATE\"" "doctor extension state" +# 默认 doctor 只要求系统 NFS client 和同包 gateway；FSKit 状态只作为可选信息。 +DOCTOR_JSON="$(HOME="$TEST_HOME" "$VEXDB" fs --json doctor)" +contains "$DOCTOR_JSON" '"mount_driver":"NFSv3"' "default NFS driver" +contains "$DOCTOR_JSON" '"mount_ready":true' "default NFS readiness" contains "$DOCTOR_JSON" '"schema_ready":true' "doctor database state" # 数据库备份、拒绝覆盖和备份验证。 @@ -371,6 +364,7 @@ VEXDB_LITE_LIB_DIR="$LIB_DIR" \ check test ! -e "$APP_DIR/VexDB Lite.app" check test ! -e "$VEXDB" check test ! -e "$VEXFS" +check test ! -e "$NFS_GATEWAY" check test ! -e "$DYLIB" check test -f "$DEFAULT_DB" check test -f "$BACKUP_DB" diff --git a/tests/eval/vexfs/run.py b/tests/eval/vexfs/run.py index ad738b8e57..1e71855762 100644 --- a/tests/eval/vexfs/run.py +++ b/tests/eval/vexfs/run.py @@ -2415,7 +2415,7 @@ def cli_command_surface(ctx: Context) -> dict[str, Any]: doctor_json = json.loads(doctor.stdout) expected_platform = {"Darwin": "macos", "Linux": "linux", "Windows": "windows"}.get(platform.system(), "unsupported") - expected_driver = {"Darwin": "FSKit", "Linux": "libfuse3", + expected_driver = {"Darwin": "NFSv3", "Linux": "libfuse3", "Windows": "WinFsp"}.get(platform.system(), "none") ctx.equal(doctor_json["platform"], expected_platform, "doctor platform") ctx.equal(doctor_json["mount_driver"], expected_driver, "doctor mount driver") @@ -3337,7 +3337,7 @@ def fskit_unsigned_build(ctx: Context) -> dict[str, Any]: def mount_cli(ctx: Context) -> Path: - """FSKit 只向受信任调用方公开扩展；发行 Gate 必须显式绑定被测 CLI。""" + """发行 Gate 必须显式绑定本次构建的 mount CLI。""" if ctx.mount_cli_override is not None: return ctx.mount_cli_override installed = Path.home() / ".local/bin/vexfs" @@ -3394,7 +3394,7 @@ def prepare_real_mount_adapter(ctx: Context, base: Path, """为当前系统准备真实 mount；测试主体不包含平台分支。""" system = platform.system() if system == "Darwin": - name = "fskit" + name = "nfs" cli = mount_cli(ctx) elif system == "Linux": name = "libfuse3" @@ -3427,22 +3427,10 @@ def prepare_real_mount_adapter(ctx: Context, base: Path, except json.JSONDecodeError as error: raise EvalFailure( f"{name} doctor 没有返回 JSON: {doctor.stdout.decode(errors='replace')}") from error - if system == "Darwin" and details.get("extension") != "enabled": - raise EvalSkip( - f"FSKit extension={details.get('extension', 'unknown')}，需安装并在系统设置中启用") if system == "Darwin": - module_path = details.get("extension_path", "") - installed_extensions = { - str(path.resolve()) - for path in ( - Path.home() / "Applications/VexDB Lite.app/Contents/Extensions/VexFSAppEx.appex", - Path("/Applications/VexDB Lite.app/Contents/Extensions/VexFSAppEx.appex"), - ) - if path.exists() - } - ctx.check(bool(module_path), "doctor 必须返回 FSKit 真实 module URL") - ctx.check(str(Path(module_path).resolve()) in installed_extensions, - "FSKit module URL 必须指向正式安装 App，不能指向 archive/DerivedData") + ctx.equal(details.get("mount_driver"), "NFSv3", "macOS 默认 mount driver") + if not details.get("mount_ready"): + raise EvalSkip("macOS NFS gateway 或系统 mount_nfs 不可用") if system == "Linux" and not details.get("mount_ready"): raise EvalSkip(f"libfuse3 adapter={details.get('extension', 'unknown')}") mount_point.mkdir() @@ -4473,10 +4461,10 @@ def mount_helper_crash_recovery(ctx: Context) -> dict[str, Any]: str(adapter.mount_point)], check=False, timeout=60) -@case("mount.real-bash", "mount", "真实 FSKit mount 后用 bash 常用文件命令操作") +@case("mount.real-bash", "mount", "真实 macOS NFS mount 后用 bash 常用文件命令操作") def real_mount_bash(ctx: Context) -> dict[str, Any]: if platform.system() != "Darwin": - raise EvalSkip("真实 FSKit mount 只在 macOS 执行") + raise EvalSkip("真实 macOS NFS mount 只在 macOS 执行") with tempfile.TemporaryDirectory(prefix="vexfs-mount-eval-") as directory: base = Path(directory) adapter = prepare_real_mount_adapter(ctx, base, "eval") @@ -4493,23 +4481,9 @@ def real_mount_bash(ctx: Context) -> dict[str, Any]: ctx.equal(len(mounts), 1, "挂载状态数量") ctx.equal(mounts[0]["target"], expected_target, "挂载状态目标") source = mounts[0]["source"] - source_path = (Path(unquote(urlparse(source).path)) - if source.startswith("file:") else Path(source)) - descriptor = json.loads( - (source_path / ".vexfs-volume.json").read_text()) - expected_connection = (adapter.database.name - if adapter.backend == "sqlite" - else adapter.connection_identity) - ctx.equal(descriptor, {"version": 3, "backend": adapter.backend, - "connection": expected_connection, - "workspace": "eval"}, "FSKit 目录资源描述文件") - if adapter.backend == "sqlite": - descriptor_connection = Path(descriptor["connection"]) - ctx.check(not descriptor_connection.is_absolute(), - "FSKit SQLite 描述文件不得保存绝对路径") - ctx.equal(os.path.realpath(source_path / descriptor_connection), - os.path.realpath(adapter.database), - "FSKit SQLite 描述文件相对路径解析") + ctx.equal(source, "127.0.0.1:/", "NFS gateway 只从 loopback export 挂载") + ctx.equal(mounts[0]["type"], "nfs", "macOS 默认真实挂载类型") + descriptor = {"driver": "nfs", "source": source} shell = r''' set -e mkdir -p "$1/project/sub" @@ -4518,6 +4492,8 @@ def real_mount_bash(ctx: Context) -> dict[str, Any]: cp "$1/project/input.txt" "$1/project/copy.txt" mv "$1/project/copy.txt" "$1/project/moved.txt" cmp "$1/project/input.txt" "$1/project/moved.txt" +ln "$1/project/input.txt" "$1/project/input-hard.txt" +ln -s input.txt "$1/project/input-link.txt" /bin/ls -la "$1/project" find "$1/project" -type f | sort rm "$1/project/moved.txt" @@ -4529,10 +4505,24 @@ def real_mount_bash(ctx: Context) -> dict[str, Any]: "bash find 输出") ctx.equal((mount_point / "project/sub/result.txt").read_text(), "agent\n", "挂载路径结果") - reverse = run_process(prefix + ["cat", "/project/sub/result.txt"]) - ctx.equal(reverse.stdout, b"agent\n", "数据库 CLI 反向读取挂载写入") + source_file = mount_point / "project/input.txt" + hardlink = mount_point / "project/input-hard.txt" + symlink = mount_point / "project/input-link.txt" + ctx.equal(source_file.stat().st_ino, hardlink.stat().st_ino, + "NFS hardlink inode") + ctx.equal(source_file.stat().st_nlink, 2, "NFS hardlink 数量") + ctx.equal(os.readlink(symlink), "input.txt", "NFS symlink target") + with source_file.open("ab", buffering=0) as stream: + stream.write(b"fsync\n") + os.fsync(stream.fileno()) + ctx.check(hardlink.read_bytes().endswith(b"fsync\n"), + "NFS fsync 后 hardlink 共享内容") + sidecar_names = [path.name for path in mount_point.rglob("._*")] + ctx.equal(sidecar_names, [], "AppleDouble sidecar 不得出现在 workspace") finally: mount_adapter_unmount(ctx, adapter, "Bash") + reverse = run_process(prefix + ["cat", "/project/sub/result.txt"]) + ctx.equal(reverse.stdout, b"agent\n", "数据库 CLI 反向读取挂载写入") after = json.loads(run_process(prefix + ["--json", "doctor"]).stdout) ctx.equal(after["mount_count"], 0, "卸载后挂载数") ctx.equal(after["database"]["pending_handles"], 0, "卸载后未发布句柄") @@ -4542,8 +4532,8 @@ def real_mount_bash(ctx: Context) -> dict[str, Any]: "doctor_before": details, "doctor_after": after, "mounts": mounts, "descriptor": descriptor, - "commands": ["mkdir", "printf", "grep", "cp", "mv", "cmp", "ls", - "find", "rm", "cat", "vexfs cat"]} + "commands": ["mkdir", "printf", "grep", "cp", "mv", "cmp", "hardlink", + "symlink", "fsync", "ls", "find", "rm", "cat", "vexfs cat"]} @case("mount.real-linux-bash-git", "mount", diff --git a/tests/eval/vexfs/run_macos_nfs_mount.sh b/tests/eval/vexfs/run_macos_nfs_mount.sh new file mode 100755 index 0000000000..9c011930ff --- /dev/null +++ b/tests/eval/vexfs/run_macos_nfs_mount.sh @@ -0,0 +1,221 @@ +#!/bin/bash +set -euo pipefail + +# 真实 macOS NFS 挂载回归。默认只创建 1000 个小文件，避免压满本机内存。 +# usage: bash run_macos_nfs_mount.sh [vexdb] + +VEXDB="${1:-${VEXDB_BIN:-}}" +if [ -z "$VEXDB" ]; then + REPO_ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" + VEXDB="$REPO_ROOT/vexdb_sqlite/build-nfs-dev/vexdb" +fi +[ -x "$VEXDB" ] || { echo "找不到 vexdb：$VEXDB" >&2; exit 1; } + +FILE_COUNT="${VEXDB_NFS_FILE_COUNT:-1000}" +case "$FILE_COUNT" in + ''|*[!0-9]*) echo "VEXDB_NFS_FILE_COUNT 必须是正整数" >&2; exit 2 ;; +esac +[ "$FILE_COUNT" -gt 0 ] && [ "$FILE_COUNT" -le 10000 ] || { + echo "VEXDB_NFS_FILE_COUNT 必须在 1 到 10000 之间" >&2 + exit 2 +} + +ROOT="$(mktemp -d -t vexdb-nfs-real-eval)" +TEST_HOME="$ROOT/home" +DB="$ROOT/workspace.sqlite3" +MOUNT_POINT="$ROOT/mnt" +WORKSPACE=nfs-real +MOUNTED=0 +CHECKS=0 + +mkdir -p "$TEST_HOME" "$MOUNT_POINT" + +fs() { + HOME="$TEST_HOME" "$VEXDB" fs --db "$DB" --workspace "$WORKSPACE" "$@" +} + +check() { + CHECKS=$((CHECKS + 1)) + "$@" +} + +equal() { + local expected="$1" actual="$2" message="$3" + [ "$expected" = "$actual" ] || { + echo "${message}：期望 '$expected'，实际 '$actual'" >&2 + exit 1 + } + CHECKS=$((CHECKS + 1)) +} + +unmount_now() { + fs unmount "$MOUNT_POINT" + MOUNTED=0 +} + +cleanup() { + if [ "$MOUNTED" = 1 ]; then + fs unmount --force "$MOUNT_POINT" >/dev/null 2>&1 || true + fi + if [ "${VEXDB_KEEP_NFS_EVAL:-0}" = 1 ]; then + echo "保留测试目录：$ROOT" >&2 + else + rm -rf "$ROOT" + fi +} +trap cleanup EXIT + +fs setup >/dev/null +DOCTOR="$(fs --json doctor)" +printf '%s' "$DOCTOR" | /usr/bin/python3 -c \ + 'import json,sys; v=json.load(sys.stdin); assert v["mount_driver"]=="NFSv3"; assert v["mount_ready"] is True; assert v["database"]["schema_ready"] is True' +CHECKS=$((CHECKS + 3)) + +fs mount "$MOUNT_POINT" +MOUNTED=1 +STATUS="$(fs --json mount status "$MOUNT_POINT")" +printf '%s' "$STATUS" | /usr/bin/python3 -c \ + 'import json,sys; v=json.load(sys.stdin); assert len(v)==1; assert v[0]["source"]=="127.0.0.1:/"; assert v[0]["type"]=="nfs"' +CHECKS=$((CHECKS + 3)) + +mkdir -p "$MOUNT_POINT/project/src" +printf 'base\n' > "$MOUNT_POINT/project/src/data.txt" +ln "$MOUNT_POINT/project/src/data.txt" "$MOUNT_POINT/project/src/data-hard.txt" +ln -s data.txt "$MOUNT_POINT/project/src/data-link.txt" +printf '%s\n' '-hard' >> "$MOUNT_POINT/project/src/data-hard.txt" +equal "$(stat -f %i "$MOUNT_POINT/project/src/data.txt")" \ + "$(stat -f %i "$MOUNT_POINT/project/src/data-hard.txt")" "hardlink inode" +equal 2 "$(stat -f %l "$MOUNT_POINT/project/src/data.txt")" "hardlink 数量" +equal data.txt "$(readlink "$MOUNT_POINT/project/src/data-link.txt")" "symlink target" +equal 'base +-hard' "$(cat "$MOUNT_POINT/project/src/data.txt")" "hardlink 共享内容" + +printf '#!/bin/sh\nprintf "nfs-ok\\n"\n' > "$MOUNT_POINT/project/run.sh" +chmod 0755 "$MOUNT_POINT/project/run.sh" +equal nfs-ok "$($MOUNT_POINT/project/run.sh)" "可执行权限" + +/usr/bin/xattr -w user.vexdb.nfs-eval yes "$MOUNT_POINT/project/src/data.txt" +equal yes "$(/usr/bin/xattr -p user.vexdb.nfs-eval "$MOUNT_POINT/project/src/data.txt")" \ + "扩展属性" + +printf 'old\n' > "$MOUNT_POINT/project/atomic.txt" +printf 'new\n' > "$MOUNT_POINT/project/atomic.next" +mv -f "$MOUNT_POINT/project/atomic.next" "$MOUNT_POINT/project/atomic.txt" +equal new "$(cat "$MOUNT_POINT/project/atomic.txt")" "原子替换" + +/usr/bin/python3 -c \ + 'import os,sys; p=sys.argv[1]; f=open(p,"ab",buffering=0); f.write(b"fsync\n"); os.fsync(f.fileno()); f.close()' \ + "$MOUNT_POINT/project/src/data.txt" +printf 'open-unlink\n' > "$MOUNT_POINT/project/open-unlink.txt" +/usr/bin/python3 -c \ + 'import os,sys; p=sys.argv[1]; f=open(p,"rb"); os.unlink(p); assert f.read()==b"open-unlink\n"; f.close()' \ + "$MOUNT_POINT/project/open-unlink.txt" +CHECKS=$((CHECKS + 2)) + +git -C "$MOUNT_POINT/project" init -q +git -C "$MOUNT_POINT/project" config user.name VexFS-NFS-Eval +git -C "$MOUNT_POINT/project" config user.email vexfs-nfs@example.invalid +git -C "$MOUNT_POINT/project" add . +git -C "$MOUNT_POINT/project" commit -qm baseline +equal '' "$(git -C "$MOUNT_POINT/project" status --porcelain)" "Git 工作区" + +PERF_JSON="$(/usr/bin/python3 - "$MOUNT_POINT/perf" "$FILE_COUNT" <<'PY' +import json +import pathlib +import sys +import time + +root = pathlib.Path(sys.argv[1]) +count = int(sys.argv[2]) +root.mkdir() +started = time.perf_counter() +for index in range(count): + (root / f"f-{index:05d}.txt").write_text("x\n", encoding="utf-8") +seconds = time.perf_counter() - started +print(json.dumps({"files": count, "seconds": round(seconds, 6), + "files_per_second": round(count / max(seconds, 1e-9), 3)})) +PY +)" +printf '%s' "$PERF_JSON" | /usr/bin/python3 -c \ + 'import json,sys; v=json.load(sys.stdin); assert v["files"]>0; assert v["files_per_second"]>0' +CHECKS=$((CHECKS + 2)) + +equal '' "$(find "$MOUNT_POINT" -name '._*' -print -quit)" "挂载树不能出现 AppleDouble" +unmount_now + +APPLEDOUBLE_DENTRIES="$($VEXDB "$DB" \ + "SELECT count(*) FROM _vexfs_dentry_states WHERE substr(name,1,2)='._';")" +equal 0 "$APPLEDOUBLE_DENTRIES" "数据库不能保存 AppleDouble dentry" +CHECK_JSON="$(fs --json check)" +printf '%s' "$CHECK_JSON" | /usr/bin/python3 -c \ + 'import json,sys; v=json.load(sys.stdin); assert v["ok"] is True; assert v["issue_count"]==0' +CHECKS=$((CHECKS + 2)) +fs snapshot create baseline >/dev/null +check fs history /project/src/data.txt --limit 5 + +fs mount "$MOUNT_POINT" +MOUNTED=1 +equal "$(stat -f %i "$MOUNT_POINT/project/src/data.txt")" \ + "$(stat -f %i "$MOUNT_POINT/project/src/data-hard.txt")" "重挂载 hardlink inode" +equal 2 "$(stat -f %l "$MOUNT_POINT/project/src/data.txt")" "重挂载 hardlink 数量" +equal yes "$(/usr/bin/xattr -p user.vexdb.nfs-eval "$MOUNT_POINT/project/src/data.txt")" \ + "重挂载扩展属性" +equal '' "$(git -C "$MOUNT_POINT/project" status --porcelain)" "重挂载 Git 工作区" +printf 'changed\n' > "$MOUNT_POINT/project/src/data.txt" +printf 'temporary\n' > "$MOUNT_POINT/project/temporary.txt" +unmount_now + +fs snapshot restore baseline >/dev/null +fs mount "$MOUNT_POINT" +MOUNTED=1 +equal "$(stat -f %i "$MOUNT_POINT/project/src/data.txt")" \ + "$(stat -f %i "$MOUNT_POINT/project/src/data-hard.txt")" "恢复后 hardlink inode" +equal 2 "$(stat -f %l "$MOUNT_POINT/project/src/data.txt")" "恢复后 hardlink 数量" +check test ! -e "$MOUNT_POINT/project/temporary.txt" +equal '' "$(git -C "$MOUNT_POINT/project" status --porcelain)" "快照恢复后 Git 工作区" +unmount_now + +# 模拟 gateway 异常退出。挂载记录必须仍能识别这块盘，强制卸载要能清理 +# 进程状态，随后同一个 workspace 可以重新挂载并继续读取。 +fs mount "$MOUNT_POINT" +MOUNTED=1 +GATEWAY_RECORD="$(find "$TEST_HOME/Library/Application Support/VexDB-Lite/nfs-gateways" \ + -name gateway.record -type f -print -quit)" +check test -f "$GATEWAY_RECORD" +GATEWAY_PID="$(sed -n '2p' "$GATEWAY_RECORD")" +case "$GATEWAY_PID" in + ''|*[!0-9]*) echo "无效的 NFS gateway PID：$GATEWAY_PID" >&2; exit 1 ;; +esac +check kill -9 "$GATEWAY_PID" +for _ in 1 2 3 4 5 6 7 8 9 10; do + if ! kill -0 "$GATEWAY_PID" 2>/dev/null; then break; fi + sleep 0.1 +done +if kill -0 "$GATEWAY_PID" 2>/dev/null; then + echo "NFS gateway 被杀后仍然存活：$GATEWAY_PID" >&2 + exit 1 +fi +CHECKS=$((CHECKS + 1)) +CRASH_STATUS="$(fs --json mount status "$MOUNT_POINT")" +printf '%s' "$CRASH_STATUS" | /usr/bin/python3 -c \ + 'import json,sys; v=json.load(sys.stdin); assert len(v)==1; assert v[0]["type"]=="nfs"' +CHECKS=$((CHECKS + 1)) +fs unmount --force "$MOUNT_POINT" +MOUNTED=0 +equal '' "$(find "$TEST_HOME/Library/Application Support/VexDB-Lite/nfs-gateways" \ + -name gateway.record -type f -print -quit)" "gateway 崩溃后的状态清理" +fs mount "$MOUNT_POINT" +MOUNTED=1 +equal 'base +-hard +fsync' "$(cat "$MOUNT_POINT/project/src/data.txt")" "gateway 崩溃后的重新挂载" +unmount_now + +equal '[]' "$(fs --json mount status "$MOUNT_POINT")" "卸载后挂载状态" +FINAL_DOCTOR="$(fs --json doctor)" +printf '%s' "$FINAL_DOCTOR" | /usr/bin/python3 -c \ + 'import json,sys; v=json.load(sys.stdin); assert v["mount_count"]==0; assert v["database"]["pending_handles"]==0; assert v["database"]["staging_bytes"]==0' +CHECKS=$((CHECKS + 3)) + +echo "VEXDB MACOS NFS REAL MOUNT: PASS ($CHECKS checks)" +echo "small_file_metrics=$PERF_JSON" diff --git a/tests/eval/vexfs/run_macos_package_smoke.sh b/tests/eval/vexfs/run_macos_package_smoke.sh index f43af0301e..98faeb7344 100755 --- a/tests/eval/vexfs/run_macos_package_smoke.sh +++ b/tests/eval/vexfs/run_macos_package_smoke.sh @@ -63,7 +63,7 @@ mkdir -p "$MOUNT_POINT" fs setup >/dev/null DOCTOR="$(fs --json doctor)" printf '%s' "$DOCTOR" | /usr/bin/python3 -c \ - 'import json,sys; value=json.load(sys.stdin); assert value["mount_ready"] is True; assert value["extension"] == "enabled"; assert value["database"]["schema_version"] == "0.9.0"' + 'import json,sys; value=json.load(sys.stdin); assert value["mount_ready"] is True; assert value["mount_driver"] == "NFSv3"; assert value["database"]["schema_version"] == "0.9.0"' CHECKS=$((CHECKS + 3)) fs mount "$MOUNT_POINT" diff --git a/vexdb_sqlite/CMakeLists.txt b/vexdb_sqlite/CMakeLists.txt index b6169d0741..d078010c76 100644 --- a/vexdb_sqlite/CMakeLists.txt +++ b/vexdb_sqlite/CMakeLists.txt @@ -305,6 +305,66 @@ if(VEXDB_SQLITE_BUILD_TESTS OR VEXDB_SQLITE_DESKTOP) endif() if(VEXDB_SQLITE_DESKTOP) + if(APPLE) + find_program(VEXFS_CARGO cargo) + if(NOT VEXFS_CARGO) + message(FATAL_ERROR + "macOS 默认 NFS gateway 需要 Rust cargo；请安装 Rust toolchain") + endif() + set(VEXFS_NFS_GATEWAY_DIR + "${REPO_ROOT}/agent_files/mount/macos/nfs_gateway") + set(VEXFS_NFS_CARGO_TARGET_DIR + "${CMAKE_BINARY_DIR}/nfs-gateway-cargo") + if(CMAKE_OSX_ARCHITECTURES STREQUAL "arm64") + set(VEXFS_NFS_RUST_TARGET "aarch64-apple-darwin") + elseif(CMAKE_OSX_ARCHITECTURES STREQUAL "x86_64") + set(VEXFS_NFS_RUST_TARGET "x86_64-apple-darwin") + else() + message(FATAL_ERROR + "macOS NFS gateway 当前要求单架构构建（arm64 或 x86_64）；" + "通用包由发布脚本合并两个独立构建") + endif() + set(VEXFS_NFS_RUST_LIBRARY + "${VEXFS_NFS_CARGO_TARGET_DIR}/${VEXFS_NFS_RUST_TARGET}/release/libvexfs_nfs_gateway.a") + file(GLOB_RECURSE VEXFS_NFS_RUST_SOURCES CONFIGURE_DEPENDS + "${VEXFS_NFS_GATEWAY_DIR}/src/*.rs" + "${VEXFS_NFS_GATEWAY_DIR}/vendor/nfsserve/src/*.rs") + add_custom_command( + OUTPUT "${VEXFS_NFS_RUST_LIBRARY}" + COMMAND "${CMAKE_COMMAND}" -E env + "CARGO_TARGET_DIR=${VEXFS_NFS_CARGO_TARGET_DIR}" + "${VEXFS_CARGO}" build --release --locked + --target "${VEXFS_NFS_RUST_TARGET}" + --manifest-path "${VEXFS_NFS_GATEWAY_DIR}/Cargo.toml" + DEPENDS + "${VEXFS_NFS_GATEWAY_DIR}/Cargo.toml" + "${VEXFS_NFS_GATEWAY_DIR}/Cargo.lock" + "${VEXFS_NFS_GATEWAY_DIR}/vendor/nfsserve/Cargo.toml" + ${VEXFS_NFS_RUST_SOURCES} + COMMENT "Build macOS VexFS NFS protocol adapter" + VERBATIM) + add_custom_target(vexfs_nfs_gateway_rust_build + DEPENDS "${VEXFS_NFS_RUST_LIBRARY}") + add_library(vexfs_nfs_gateway_rust STATIC IMPORTED GLOBAL) + set_target_properties(vexfs_nfs_gateway_rust PROPERTIES + IMPORTED_LOCATION "${VEXFS_NFS_RUST_LIBRARY}") + add_dependencies(vexfs_nfs_gateway_rust vexfs_nfs_gateway_rust_build) + + add_executable(vexfs_nfs_gateway + ${VEXFS_NFS_GATEWAY_DIR}/vexfs_nfs_gateway_main.cpp) + target_include_directories(vexfs_nfs_gateway PRIVATE + ${VEXFS_NFS_GATEWAY_DIR}/include + ${REPO_ROOT}/agent_files/mount/common/include) + target_link_libraries(vexfs_nfs_gateway PRIVATE + vexfs_nfs_gateway_rust vexfs_runtime sqlite3_amalg + Threads::Threads ${CMAKE_DL_LIBS}) + if(PostgreSQL_FOUND) + target_link_libraries(vexfs_nfs_gateway PRIVATE ${PostgreSQL_LIBRARIES}) + endif() + set_target_properties(vexfs_nfs_gateway PROPERTIES + OUTPUT_NAME "vexfs-nfs-gateway" LINKER_LANGUAGE CXX) + endif() + set_source_files_properties(${SQLITE_AMALG_DIR}/shell.c PROPERTIES COMPILE_DEFINITIONS "main=vexdb_sqlite_shell_main;SQLITE_SHELL_INIT_PROC=vexdb_shell_init") set(VEXDB_CLI_SOURCES @@ -366,6 +426,9 @@ if(VEXDB_SQLITE_DESKTOP) COMMENT "Create vexfs compatibility entry") endif() add_custom_target(vexfs_cli DEPENDS vexdb_cli) + if(APPLE) + add_dependencies(vexfs_cli vexfs_nfs_gateway) + endif() if(CMAKE_SYSTEM_NAME STREQUAL "Linux") find_package(PkgConfig QUIET) From f9d305bd15705effc91a4b4c78a99f723a9aab01 Mon Sep 17 00:00:00 2001 From: Four <8122332+HungryFour@users.noreply.github.com> Date: Fri, 24 Jul 2026 17:47:38 +0800 Subject: [PATCH 11/28] Expand macOS NFS workspace eval coverage --- ...2026-07-21_vexfs-final-goal-and-roadmap.md | 16 ++--- docs/specs/2026-07-16_vexfs-product-spec.md | 11 ++-- tests/eval/vexfs/run_macos_nfs_mount.sh | 62 +++++++++++++++++++ 3 files changed, 77 insertions(+), 12 deletions(-) diff --git a/docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md b/docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md index a791a9eed5..d872a318d7 100644 --- a/docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md +++ b/docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md @@ -37,9 +37,9 @@ 1. **已完成**：冻结 NFS adapter 与 Workspace Engine 的窄接口，不复制数据库逻辑； 2. **已完成**：实现 localhost-only NFS gateway、mount/unmount/status/doctor； -3. **部分完成**：Bash、Git、hardlink、xattr、fsync 已进入真机 eval；OpenCode、npm/cargo、 - 文件锁和 sleep/wake 尚待补齐；gateway 单次 `SIGKILL` 后的强制卸载、状态清理和重挂载已覆盖； -4. **已完成首轮**：1,000 小文件为 181.689 files/s，同轮 APFS 为 14304.029 files/s； +3. **部分完成**：Bash、Git、hardlink、xattr、fsync、单机跨进程 `flock`/`fcntl`、轻量 npm/Cargo 和单次真实 OpenCode 已进入真机 eval； + 跨机器锁、长任务和 sleep/wake 尚待补齐；gateway 单次 `SIGKILL` 后的强制卸载、状态清理和重挂载已覆盖； +4. **已完成首轮**：1,000 小文件为 205.643 files/s，同轮 APFS 为 16940.431 files/s； 5. **待完成**：生成不含 FSKit 授权前置的签名公证包，在干净 Mac 验证安装和系统提示； 6. **代码已完成，发行待 Gate**：默认 backend 已改为 NFS，FSKit 只在显式选择时使用。 @@ -58,22 +58,22 @@ VexFS FSKit extension。 - NFS gateway 已连接平台无关 `vexfs_mount_*` C ABI；SQLite 已完成，PG 复用同一 HostStore； - 公开版 `nfsserve` 缺少 LINK/COMMIT，仓库内固定 fork 已补协议 handler，真实 hardlink 的 inode、link count、共享内容、重挂载和快照恢复均通过； -- 41 项真实 NFS eval、30 项 package smoke 通过。 +- 45 项完整 NFS eval、13 项真实 OpenCode eval、30 项 package smoke 通过。 同时确认两个性能和实现边界： 1. NFSv3 没有 macOS named attributes。macOS 26.3.1 为新文件写入 `com.apple.provenance` 时会尝试写 `._文件名`。当前 gateway 已将 raw AppleDouble 数据吸收为 目标 inode xattr，用户树和数据库 dentry 都为 0 个 `._*`。 -2. 当前真实 SQLite adapter 创建 1,000 个 2-byte 文件为 5.503902 秒、181.689 files/s；同轮 - APFS 为 0.069910 秒、14304.029 files/s，约慢 78.7 倍。它与此前 FSKit 约 175 files/s +2. 当前真实 SQLite adapter 创建 1,000 个 2-byte 文件为 4.862788 秒、205.643 files/s；同轮 + APFS 为 0.059030 秒、16940.431 files/s，约慢 82.4 倍。它与此前 FSKit 约 175 files/s 同级，0.1 可用但不是性能达标终点。 因此最近任务增加两个硬 Gate： - AppleDouble 隔离已进入 eval；后续每次 NFS 发布必须保留该断言； - 0.2 实现批量提交、写回合并和有界 metadata cache，再做同机 1,000/10,000 文件对照； -- 0.1 发布前仍需完成干净 Mac 安装、公证、重复 crash/sleep-wake、文件锁与长期运行 Gate。 +- 0.1 发布前仍需完成干净 Mac 安装、公证、重复 crash/sleep-wake、跨机器锁边界与长期运行 Gate。 ## 1. 这份文档解决什么问题 @@ -170,7 +170,7 @@ vexdb fs --workspace workspace snapshot restore before-refactor | Linux | libfuse3 真实挂载已通过 root 和 uid 1000 各 9 组；时间戳、并发 append、进程锁、打开文件生命周期、强制卸载和 helper 崩溃恢复已验证；异常撤销后底层目录保持 0500，显式卸载恢复 0700；x86_64/AArch64 manylinux 安装包已在对应架构真机完成无 root 安装回归 | 更多干净发行版、长期运行和真实挂载安装回归仍不足 | | Windows | 只有平台边界和规划 | WinFsp adapter、安装签名、路径/ACL/SID 合同均未实现 | | 远程共享 | PostgreSQL 已通过多 gateway、macOS FSKit、Linux FUSE、数据库重启、当前源码下的 macOS↔Linux 47 项、两台 Mac 文件/快照 50 项、双 Mac 串行真实 OpenCode 7 + 9 + 21 项，以及双机 extension 崩溃、网络中断和数据库停机 25 项 | 当前审计源码对应的公证包尚未生成，需要重跑双机安装 Gate；局域网直连 FSKit 还需要 macOS 本地网络授权；PG 文本索引尚未实现 | -| 性能规模 | SQLite 直连已完成 10 万文件；默认 NFS 1,000 小文件为 5.503902 秒、181.689 files/s，同轮 APFS 0.069910 秒、14304.029 files/s；此前 FSKit 为约 175 files/s | NFS 小文件仍比 APFS 慢约 78.7 倍；0.2 需批量提交、写回合并、metadata cache，并重跑 1 万/10 万和真实 Coding Agent | +| 性能规模 | SQLite 直连已完成 10 万文件；默认 NFS 1,000 小文件为 4.862788 秒、205.643 files/s，同轮 APFS 0.059030 秒、16940.431 files/s；此前 FSKit 为约 175 files/s | NFS 小文件仍比 APFS 慢约 82.4 倍；0.2 需批量提交、写回合并、metadata cache，并重跑 1 万/10 万和真实 Coding Agent | “合同中已保存”不等于“所有平台已经完整执行”。例如 owner/group 和便携 ACL 已经可以保存、读取和恢复，但身份认证、权限判断和各系统的原生 ACL 映射仍属于后续阶段。 diff --git a/docs/specs/2026-07-16_vexfs-product-spec.md b/docs/specs/2026-07-16_vexfs-product-spec.md index c2f89730ca..568a9609c1 100644 --- a/docs/specs/2026-07-16_vexfs-product-spec.md +++ b/docs/specs/2026-07-16_vexfs-product-spec.md @@ -35,10 +35,13 @@ Linux FUSE 和 Windows WinFsp 必须共用同一 Workspace Engine、mount runtim 2026-07-24 已完成连接 SQLite mount runtime 的 localhost NFSv3 adapter，并设为 macOS CLI 默认 driver。gateway 只监听 loopback，由 `mount/unmount/status/doctor` 管理独立 PID、端口、 -日志和受保护状态目录。真实 macOS 26.3.1 arm64 eval 共 41 项，覆盖 Bash、hardlink、symlink、 -mode、fsync、xattr、原子替换、open-then-unlink、Git、gateway `SIGKILL` 恢复、卸载/重挂载和完整 workspace 快照恢复； +日志和受保护状态目录。真实 macOS 26.3.1 arm64 eval 共 45 项，覆盖 Bash、hardlink、symlink、 +mode、fsync、xattr、原子替换、open-then-unlink、单机跨进程 `flock`/`fcntl`、轻量 npm/Cargo、Git、gateway `SIGKILL` 恢复、卸载/重挂载和完整 workspace 快照恢复； package smoke 30 项也通过。公开版 `nfsserve` 缺少 NFSv3 LINK/COMMIT，当前仓库固定维护一个 小型 fork，LINK 调用数据库 hardlink 合同，COMMIT 调用强同步并返回同一 server verifier。 +另有独立的真实 OpenCode Gate：OpenCode 1.18.3 使用 `openai/gpt-5.4-mini` 在默认 NFS +挂载项目中修复代码并运行测试，13 项检查在 28.349 秒内通过；卸载和重挂载后的数据库内容、 +Git diff 与单测结果保持一致。 默认挂载使用 bounded soft NFS：固定 10 秒重传间隔、最多 4 次，并设 60 秒 dead timeout。 它让 gateway 崩溃时的文件调用最终返回错误而不是永久卡死，同时覆盖 runtime 最长 30 秒的 @@ -46,8 +49,8 @@ package smoke 30 项也通过。公开版 `nfsserve` 缺少 NFSv3 LINK/COMMIT， macOS 写入的 AppleDouble 由 gateway 吸收为目标 inode 的 `io.vexdb.macos.appledouble` xattr，不进入用户目录、dentry、grep 或快照树。1000 个 2-byte -小文件创建为 5.503902 秒、181.689 files/s；同轮 APFS 为 0.069910 秒、14304.029 files/s， -约慢 78.7 倍。这个吞吐与此前 FSKit 约 175 files/s 同级，满足“不比现有默认入口明显退化”的 +小文件创建为 4.862788 秒、205.643 files/s；同轮 APFS 为 0.059030 秒、16940.431 files/s， +约慢 82.4 倍。这个吞吐与此前 FSKit 约 175 files/s 同级，满足“不比现有默认入口明显退化”的 0.1 入口条件，但远未接近原生盘，列为 0.2 批量提交、写回合并和 metadata cache 优化项。 后文中的 FSKit 完成状态仍是历史事实；凡是把 FSKit 写成“默认、MVP 主入口或当前发布前置”的 diff --git a/tests/eval/vexfs/run_macos_nfs_mount.sh b/tests/eval/vexfs/run_macos_nfs_mount.sh index 9c011930ff..265c72b7ef 100755 --- a/tests/eval/vexfs/run_macos_nfs_mount.sh +++ b/tests/eval/vexfs/run_macos_nfs_mount.sh @@ -98,6 +98,68 @@ equal nfs-ok "$($MOUNT_POINT/project/run.sh)" "可执行权限" equal yes "$(/usr/bin/xattr -p user.vexdb.nfs-eval "$MOUNT_POINT/project/src/data.txt")" \ "扩展属性" +/usr/bin/python3 - "$MOUNT_POINT/project/process.lock" <<'PY' +import fcntl +import multiprocessing +import os +import sys + +path = sys.argv[1] +open(path, "wb").write(b"lock\n") +multiprocessing.set_start_method("fork") + + +def acquire(handle, kind, nonblocking=False): + flag = fcntl.LOCK_EX | (fcntl.LOCK_NB if nonblocking else 0) + if kind == "flock": + fcntl.flock(handle, flag) + else: + fcntl.lockf(handle, flag, 1, 0, os.SEEK_SET) + + +def hold(kind, ready, release): + with open(path, "r+b", buffering=0) as handle: + acquire(handle, kind) + ready.set() + release.wait(5) + + +for kind in ("flock", "fcntl"): + ready = multiprocessing.Event() + release = multiprocessing.Event() + process = multiprocessing.Process(target=hold, args=(kind, ready, release)) + process.start() + assert ready.wait(5), f"{kind} holder did not acquire lock" + blocked = False + with open(path, "r+b", buffering=0) as handle: + try: + acquire(handle, kind, nonblocking=True) + except BlockingIOError: + blocked = True + release.set() + process.join(5) + assert process.exitcode == 0, f"{kind} holder failed: {process.exitcode}" + assert blocked, f"{kind} allowed two exclusive holders" +PY +CHECKS=$((CHECKS + 2)) + +mkdir -p "$MOUNT_POINT/project/node" +printf '%s\n' \ + '{"name":"vexfs-nfs-eval","version":"1.0.0","private":true,"scripts":{"test":"node index.js"}}' \ + > "$MOUNT_POINT/project/node/package.json" +printf '%s\n' 'console.log("npm-ok")' > "$MOUNT_POINT/project/node/index.js" +HOME="$TEST_HOME" npm --prefix "$MOUNT_POINT/project/node" install \ + --package-lock-only --ignore-scripts --offline --no-audit --no-fund >/dev/null +equal npm-ok "$(HOME="$TEST_HOME" npm --prefix "$MOUNT_POINT/project/node" test --silent)" \ + "npm 项目" + +cargo init --quiet --bin --vcs none "$MOUNT_POINT/project/rust" +printf '%s\n' 'fn main() { println!("cargo-ok"); }' \ + > "$MOUNT_POINT/project/rust/src/main.rs" +equal cargo-ok "$(cargo run --quiet --offline \ + --manifest-path "$MOUNT_POINT/project/rust/Cargo.toml")" "Cargo 项目" +printf '%s\n' '/rust/target/' > "$MOUNT_POINT/project/.gitignore" + printf 'old\n' > "$MOUNT_POINT/project/atomic.txt" printf 'new\n' > "$MOUNT_POINT/project/atomic.next" mv -f "$MOUNT_POINT/project/atomic.next" "$MOUNT_POINT/project/atomic.txt" From fd2419bcea502b13eebaf7ea4d81f3e09c19790b Mon Sep 17 00:00:00 2001 From: Four <8122332+HungryFour@users.noreply.github.com> Date: Fri, 24 Jul 2026 18:16:11 +0800 Subject: [PATCH 12/28] Support macOS 13 for the NFS mount path --- agent_files/cli/vexfs_platform_macos.cpp | 7 ++++--- agent_files/macos/package/README.md | 4 ++-- agent_files/macos/package/install.sh | 14 ++++++++++++++ ...275\277\347\224\250\350\257\264\346\230\216.md" | 6 +++--- agent_files/macos/package_preview.sh | 10 +++++++++- agent_files/mount/macos/nfs_gateway/THIRD_PARTY.md | 2 +- .../2026-07-21_vexfs-final-goal-and-roadmap.md | 2 +- docs/specs/2026-07-16_vexfs-product-spec.md | 8 +++++--- tests/eval/vexfs/documentation_smoke.sh | 10 ++++++++++ vexdb_sqlite/CMakeLists.txt | 13 +++++++++++++ 10 files changed, 62 insertions(+), 14 deletions(-) diff --git a/agent_files/cli/vexfs_platform_macos.cpp b/agent_files/cli/vexfs_platform_macos.cpp index 6591d1961b..0227bd4c32 100644 --- a/agent_files/cli/vexfs_platform_macos.cpp +++ b/agent_files/cli/vexfs_platform_macos.cpp @@ -243,10 +243,11 @@ std::string ProductVersion() { return buffer; } -bool ProductVersionSupported(const std::string &version) { +bool ProductVersionSupported(const std::string &version, const std::string &driver) { char *end = nullptr; const long major = std::strtol(version.c_str(), &end, 10); - return end != version.c_str() && major >= 26; + if (end == version.c_str()) return false; + return major >= (driver == "nfs" ? 13 : 26); } std::string MountedFileSystemAt(const std::string &path) { @@ -945,7 +946,7 @@ VexFSPlatformState VexFSPlatformInspect(const std::string &mount_driver) { VexFSPlatformState state; state.platform = "macos"; state.version = ProductVersion(); - state.platform_supported = ProductVersionSupported(state.version); + state.platform_supported = ProductVersionSupported(state.version, driver); void *framework = dlopen("/System/Library/Frameworks/FSKit.framework/FSKit", RTLD_LAZY); const bool fskit_available = framework != nullptr; if (framework != nullptr) dlclose(framework); diff --git a/agent_files/macos/package/README.md b/agent_files/macos/package/README.md index a0724419e2..235075541b 100644 --- a/agent_files/macos/package/README.md +++ b/agent_files/macos/package/README.md @@ -14,10 +14,10 @@ VexDB-Lite 在一个安装包中提供： ## 系统要求 -- macOS 26.0 或更高版本； +- 默认 NFS、CLI 和 SQLite 扩展支持 macOS 13.0 或更高版本； - Apple Silicon 包要求 arm64 Mac； - 默认真实挂载使用 macOS 自带的 NFS client，不要求安装驱动或打开系统扩展开关； -- FSKit 是可选 driver。只有显式使用 `--mount-driver fskit` 时才需要 macOS 允许 +- FSKit 是 macOS 26.0+ 的可选 driver。只有显式使用 `--mount-driver fskit` 时才需要 macOS 允许 `VexDB Lite` 文件系统扩展。 ## 安装 diff --git a/agent_files/macos/package/install.sh b/agent_files/macos/package/install.sh index 3bad514ebb..99c46dfb42 100755 --- a/agent_files/macos/package/install.sh +++ b/agent_files/macos/package/install.sh @@ -19,6 +19,20 @@ PACKAGE_APP="$ROOT/.payload/VexDB Lite.app" [ -f "$ROOT/MANIFEST.txt" ] || { echo "安装包缺少 MANIFEST.txt" >&2; exit 1; } [ -f "$ROOT/SHA256SUMS.txt" ] || { echo "安装包缺少 SHA256SUMS.txt" >&2; exit 1; } +MINIMUM_MACOS="$(sed -n 's/^minimum_macos=//p' "$ROOT/MANIFEST.txt")" +CURRENT_MACOS="$(/usr/bin/sw_vers -productVersion 2>/dev/null || true)" +[ -n "$MINIMUM_MACOS" ] || { echo "安装包清单缺少 minimum_macos" >&2; exit 1; } +[ -n "$CURRENT_MACOS" ] || { echo "无法读取当前 macOS 版本" >&2; exit 1; } +if ! /usr/bin/awk -v current="$CURRENT_MACOS" -v required="$MINIMUM_MACOS" 'BEGIN { + split(current, c, "."); split(required, r, "."); + if ((c[1] + 0) > (r[1] + 0)) exit 0; + if ((c[1] + 0) < (r[1] + 0)) exit 1; + exit !((c[2] + 0) >= (r[2] + 0)); + }'; then + echo "VexDB-Lite 默认 NFS 需要 macOS $MINIMUM_MACOS 或更高版本；当前为 $CURRENT_MACOS" >&2 + exit 1 +fi + (cd "$ROOT" && /usr/bin/shasum -a 256 -c SHA256SUMS.txt >/dev/null) || { echo "安装包文件哈希校验失败" >&2 exit 1 diff --git "a/agent_files/macos/package/\344\275\277\347\224\250\350\257\264\346\230\216.md" "b/agent_files/macos/package/\344\275\277\347\224\250\350\257\264\346\230\216.md" index 595c7da245..937895aaf8 100644 --- "a/agent_files/macos/package/\344\275\277\347\224\250\350\257\264\346\230\216.md" +++ "b/agent_files/macos/package/\344\275\277\347\224\250\350\257\264\346\230\216.md" @@ -1,7 +1,7 @@ # VexDB-Lite macOS 使用说明 适用版本：`0.1.0-preview` 系列；准确版本见同目录 `MANIFEST.txt`。 -适用平台：Apple Silicon Mac、macOS 26.0 或更高版本 +适用平台：Apple Silicon Mac、macOS 13.0 或更高版本；可选 FSKit 需要 macOS 26.0+ VexDB-Lite 把三种能力放进同一个 SQLite 数据库： @@ -43,8 +43,8 @@ VEXDB_LITE_ALLOW_ADHOC_INSTALL=1 ./install.sh 识别的 `.app.disabled` 副本。如果系统要求新签名版本重新授权，安装程序会明确提示并保留 已安装的新版本，用户启用后运行 `vexdb fs doctor` 即可继续。 -安装或升级前，请先卸载正在使用的 VexFS。安装程序不会强行中断工作区，也不会重启 -`pkd`、清空 LaunchServices 注册库或修改其他 App 的扩展状态。 +安装或升级前，请先卸载正在使用的 VexFS。安装程序不会强行中断工作区，不会重启 `pkd`， +不会清空 LaunchServices 注册库或修改其他 App 的扩展状态。 如果终端提示 `command not found: vexdb`，运行： diff --git a/agent_files/macos/package_preview.sh b/agent_files/macos/package_preview.sh index 008f2340e9..96f7054d4b 100755 --- a/agent_files/macos/package_preview.sh +++ b/agent_files/macos/package_preview.sh @@ -15,6 +15,8 @@ RUNTIME_ABI="$(sed -n 's/^#define VEXFS_RUNTIME_ABI_VERSION \([0-9][0-9]*\)u$/\1 VERSION="${VEXDB_LITE_PREVIEW_VERSION:-${VEXFS_PREVIEW_VERSION:-0.1.0-preview.1}}" ARCH="${VEXDB_LITE_ARCH:-${VEXFS_ARCH:-arm64}}" +MINIMUM_MACOS="${VEXDB_LITE_MINIMUM_MACOS:-13.0}" +FSKIT_MINIMUM_MACOS=26.0 SIGN_MODE="${VEXDB_LITE_SIGN_MODE:-${VEXFS_SIGN_MODE:-ad-hoc}}" TEAM_ID="${VEXDB_LITE_TEAM_ID:-${VEXFS_TEAM_ID:-BB5VK42K87}}" EXPORT_OPTIONS="${VEXDB_LITE_EXPORT_OPTIONS:-${VEXFS_EXPORT_OPTIONS:-$SCRIPT_DIR/ExportOptions-DeveloperID.plist}}" @@ -29,6 +31,10 @@ fi case "$VERSION" in *[!A-Za-z0-9._-]*) echo "版本只能包含字母、数字、点、下划线和连字符" >&2; exit 2 ;; esac +if [[ ! "$MINIMUM_MACOS" =~ ^[0-9]+\.[0-9]+$ ]]; then + echo "VEXDB_LITE_MINIMUM_MACOS 必须是 major.minor：$MINIMUM_MACOS" >&2 + exit 2 +fi BUNDLE_MARKETING_VERSION="${VEXDB_LITE_BUNDLE_MARKETING_VERSION:-${VERSION%%-*}}" BUNDLE_BUILD_VERSION="${VEXDB_LITE_BUNDLE_BUILD_VERSION:-}" if [ -z "$BUNDLE_BUILD_VERSION" ]; then @@ -277,6 +283,7 @@ echo "=== 构建 VexDB-Lite CLI 与 SQLite 扩展 ===" "$CMAKE_BIN" -S "$SQLITE_DIR" -B "$BUILD_DIR" \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_OSX_ARCHITECTURES="$ARCH" \ + -DCMAKE_OSX_DEPLOYMENT_TARGET="$MINIMUM_MACOS" \ -DVEXDB_LITE_VERSION="$VERSION" \ -DVEXFS_REQUIRE_POSTGRESQL=ON \ -DVEXFS_BUNDLE_POSTGRESQL_RUNTIME=ON \ @@ -520,7 +527,8 @@ write_manifest() { echo "contract_version=0.9.0" echo "mount_abi_version=$RUNTIME_ABI" echo "platform=macOS" - echo "minimum_macos=26.0" + echo "minimum_macos=$MINIMUM_MACOS" + echo "fskit_minimum_macos=$FSKIT_MINIMUM_MACOS" echo "architecture=$ARCH" echo "source_branch=$GIT_BRANCH" echo "source_commit=$GIT_COMMIT" diff --git a/agent_files/mount/macos/nfs_gateway/THIRD_PARTY.md b/agent_files/mount/macos/nfs_gateway/THIRD_PARTY.md index 0fe01d1a10..9450e3e158 100644 --- a/agent_files/mount/macos/nfs_gateway/THIRD_PARTY.md +++ b/agent_files/mount/macos/nfs_gateway/THIRD_PARTY.md @@ -1,6 +1,6 @@ # Third-party dependency -The macOS NFS gateway vendors `nfsserve` 0.11.0 from +The macOS NFS gateway vendors nfsserve 0.11.0 from under its BSD-3-Clause license. The local fork adds the NFSv3 LINK and COMMIT procedures required by the VexFS mount contract. Its license is kept at `vendor/nfsserve/LICENSE`. diff --git a/docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md b/docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md index d872a318d7..5af801df2a 100644 --- a/docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md +++ b/docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md @@ -166,7 +166,7 @@ vexdb fs --workspace workspace snapshot restore before-refactor | 文件语义 | 文件、目录、四类时间戳、并发 append、进程锁、mode、symlink、xattr、hardlink、owner/group 元数据、便携 ACL 已进入数据库合同和测试；PG 已执行 ACL 和继承 | SQLite 尚未执行完整 ACL 授权；特殊文件不支持 | | 版本恢复 | 单文件版本、workspace commit、snapshot、diff、expected-head restore、原生备份、retention、显式分批 GC、live quota 和 format v2 逻辑导入导出已实现 | 自动维护、history/staging/index/total quota 和 DuckDB 导入端未完成 | | 长期校验 | `chunked-v1` 使用不可变 manifest、64 KiB 块和逐块/整文件 SHA-256；只读 check 覆盖结构、引用、顺序、大小和流式内容校验 | 自动 repair 和跨文件通用去重不在当前范围 | -| macOS | 默认 NFS 已连接 SQLite runtime，35 项真机 eval 和 30 项 package smoke 通过；hardlink、COMMIT/fsync、AppleDouble 隔离、Git、重挂载和快照恢复均已验证。既有 FSKit 仍保留并有 preview.22/preview.37 真机证据 | 默认 NFS 尚未从干净提交生成公证包，也未在未启用 FSKit 的干净 Mac 验证；gateway crash/sleep-wake、锁、长期 Agent、macOS 26.0–26.4 和 x86_64 待补 | +| macOS | 默认 NFS 已连接 SQLite runtime，45 项完整真机 eval、13 项 OpenCode eval 和 30 项 package smoke 通过；hardlink、COMMIT/fsync、AppleDouble 隔离、Git、单机锁、npm/Cargo、重挂载和快照恢复均已验证；macOS 13.0 deployment target 构建通过。preview.40 ad-hoc 完整打包通过 99 项文档/安装检查、18 项干净包安装 eval、哈希和签名校验；既有 FSKit 仍保留并有 preview.22/preview.37 真机证据 | 默认 NFS 尚未从干净提交生成公证包，也未在 macOS 13–15 或未启用 FSKit 的干净 Mac 验证；重复 crash/sleep-wake、跨机器锁、长期 Agent 和 x86_64 待补 | | Linux | libfuse3 真实挂载已通过 root 和 uid 1000 各 9 组；时间戳、并发 append、进程锁、打开文件生命周期、强制卸载和 helper 崩溃恢复已验证；异常撤销后底层目录保持 0500，显式卸载恢复 0700；x86_64/AArch64 manylinux 安装包已在对应架构真机完成无 root 安装回归 | 更多干净发行版、长期运行和真实挂载安装回归仍不足 | | Windows | 只有平台边界和规划 | WinFsp adapter、安装签名、路径/ACL/SID 合同均未实现 | | 远程共享 | PostgreSQL 已通过多 gateway、macOS FSKit、Linux FUSE、数据库重启、当前源码下的 macOS↔Linux 47 项、两台 Mac 文件/快照 50 项、双 Mac 串行真实 OpenCode 7 + 9 + 21 项，以及双机 extension 崩溃、网络中断和数据库停机 25 项 | 当前审计源码对应的公证包尚未生成，需要重跑双机安装 Gate；局域网直连 FSKit 还需要 macOS 本地网络授权；PG 文本索引尚未实现 | diff --git a/docs/specs/2026-07-16_vexfs-product-spec.md b/docs/specs/2026-07-16_vexfs-product-spec.md index 568a9609c1..d97298425c 100644 --- a/docs/specs/2026-07-16_vexfs-product-spec.md +++ b/docs/specs/2026-07-16_vexfs-product-spec.md @@ -79,9 +79,11 @@ VexFS 有两种不同的事务入口，文档和产品界面必须明确区分 2. mount 中的普通 Bash 操作由 gateway 建立短事务，不能自动加入应用已经打开的数据库事务，多条 Bash 命令也不会自动合成一个事务。 macOS 默认挂载改为系统 NFS client + 本机用户态 gateway，不再受 FSKit V2 -`FSPathURLResource` 的 macOS 26.0 最低版本约束；新的最低 macOS 版本和 Intel/Apple Silicon -支持范围由 NFS 发行包真机验证决定。现有 macOS 26.0+ Apple Silicon FSKit 实现作为后续可选 +`FSPathURLResource` 的 macOS 26.0 最低版本约束；默认 NFS core 已完成 macOS 13.0 deployment +target 构建，Intel/Apple Silicon 的完整支持范围仍由发行包真机验证决定。现有 macOS 26.0+ Apple Silicon FSKit 实现作为后续可选 原生 adapter 保留。Linux 已有 libfuse3 预览，Windows 后续使用 WinFsp。 +打包时必须同时写入 `minimum_macos=13.0` 和 `fskit_minimum_macos=26.0`，并保证 +NFS gateway 同时能在压缩包目录和 `~/.local` 安装目录找到随包的 PostgreSQL runtime。 ### 0.1 最终目标和开发顺序 @@ -318,7 +320,7 @@ MVP Gate 不等于完整产品。它只回答三个问题：数据库能否保 ### 4.4 非目标 -- MVP Gate 不承诺 Linux、Windows、macOS 26.0 以下版本和所有容器都能真实挂载； +- MVP Gate 不承诺 Linux、Windows、macOS 13.0 以下版本和所有容器都能真实挂载； - 不把 mount gateway 做成另一套权威文件服务器； - 不建立另一套网络协议； - 不提供必须接入的 SDK； diff --git a/tests/eval/vexfs/documentation_smoke.sh b/tests/eval/vexfs/documentation_smoke.sh index 47b89dda69..5b5ca60765 100755 --- a/tests/eval/vexfs/documentation_smoke.sh +++ b/tests/eval/vexfs/documentation_smoke.sh @@ -70,6 +70,11 @@ expect_fail() { mkdir -p "$TEST_HOME" +equal "13.0" "$(sed -n 's/^minimum_macos=//p' "$STAGE/MANIFEST.txt")" \ + "default NFS minimum macOS" +equal "26.0" "$(sed -n 's/^fskit_minimum_macos=//p' "$STAGE/MANIFEST.txt")" \ + "optional FSKit minimum macOS" + # 交付物和使用说明。App 放在隐藏 payload，防止解压目录被 LaunchServices # 当成第二个可运行副本，从而让 FSKit 随机选到未安装的扩展。 check test ! -e "$STAGE/VexDB Lite.app" @@ -98,6 +103,11 @@ check test -f "$STAGE/lib/runtime/libpq.5.dylib" equal "$PACKAGE_ARCHITECTURE" \ "$(lipo -archs "$STAGE/bin/vexfs-nfs-gateway")" \ "NFS gateway architecture" +GATEWAY_LOAD_COMMANDS="$(otool -l "$STAGE/bin/vexfs-nfs-gateway")" +contains "$GATEWAY_LOAD_COMMANDS" "@executable_path/../lib/runtime" \ + "NFS gateway package runtime path" +contains "$GATEWAY_LOAD_COMMANDS" "@executable_path/../lib/vexdb-lite/runtime" \ + "NFS gateway installed runtime path" EXTENSION_ENTITLEMENTS="$TMP_DIR/extension-entitlements.plist" EXTENSION_ENTITLEMENTS_LOG="$TMP_DIR/extension-entitlements.log" if ! codesign -d --entitlements - --xml \ diff --git a/vexdb_sqlite/CMakeLists.txt b/vexdb_sqlite/CMakeLists.txt index d078010c76..ceb389834a 100644 --- a/vexdb_sqlite/CMakeLists.txt +++ b/vexdb_sqlite/CMakeLists.txt @@ -99,6 +99,13 @@ if(APPLE AND NOT CMAKE_OSX_ARCHITECTURES AND NOT CMAKE_TOOLCHAIN_FILE endif() message(STATUS "macOS arch 对齐真实硬件: ${CMAKE_OSX_ARCHITECTURES}") endif() +if(APPLE AND NOT CMAKE_OSX_DEPLOYMENT_TARGET AND NOT CMAKE_TOOLCHAIN_FILE + AND NOT CMAKE_SYSTEM_NAME STREQUAL "iOS") + # 默认 NFS 路径不依赖 FSKit。FSKit App 保持自己的 macOS 26 target， + # CLI、SQLite extension 和 NFS gateway 保持对更低版本 macOS 的兼容。 + set(CMAKE_OSX_DEPLOYMENT_TARGET "13.0" CACHE STRING "" FORCE) + message(STATUS "macOS NFS core deployment target: ${CMAKE_OSX_DEPLOYMENT_TARGET}") +endif() get_filename_component(REPO_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/.." ABSOLUTE) @@ -333,6 +340,7 @@ if(VEXDB_SQLITE_DESKTOP) OUTPUT "${VEXFS_NFS_RUST_LIBRARY}" COMMAND "${CMAKE_COMMAND}" -E env "CARGO_TARGET_DIR=${VEXFS_NFS_CARGO_TARGET_DIR}" + "MACOSX_DEPLOYMENT_TARGET=${CMAKE_OSX_DEPLOYMENT_TARGET}" "${VEXFS_CARGO}" build --release --locked --target "${VEXFS_NFS_RUST_TARGET}" --manifest-path "${VEXFS_NFS_GATEWAY_DIR}/Cargo.toml" @@ -363,6 +371,11 @@ if(VEXDB_SQLITE_DESKTOP) endif() set_target_properties(vexfs_nfs_gateway PROPERTIES OUTPUT_NAME "vexfs-nfs-gateway" LINKER_LANGUAGE CXX) + if(VEXFS_BUNDLE_POSTGRESQL_RUNTIME) + set_target_properties(vexfs_nfs_gateway PROPERTIES + BUILD_WITH_INSTALL_RPATH ON + INSTALL_RPATH "@executable_path/../lib/runtime;@executable_path/../lib/vexdb-lite/runtime") + endif() endif() set_source_files_properties(${SQLITE_AMALG_DIR}/shell.c PROPERTIES From 50329a9cc17ad1127bae30d31ae52b5fc5353d67 Mon Sep 17 00:00:00 2001 From: Four <8122332+HungryFour@users.noreply.github.com> Date: Sun, 26 Jul 2026 13:02:46 +0800 Subject: [PATCH 13/28] Optimize SQLite VexFS staging overlays --- .../src/vexfs_mount_contract_sqlite.cpp | 8 +- tests/eval/vexfs/run.py | 117 ++- vexdb_sqlite/src/agent_files/vexfs_sqlite.cpp | 741 +++++++++++++----- vexdb_sqlite/test/vexfs_static_smoke.c | 4 +- 4 files changed, 646 insertions(+), 224 deletions(-) diff --git a/agent_files/mount/common/src/vexfs_mount_contract_sqlite.cpp b/agent_files/mount/common/src/vexfs_mount_contract_sqlite.cpp index 8aa0cff4db..dacbc6f8d3 100644 --- a/agent_files/mount/common/src/vexfs_mount_contract_sqlite.cpp +++ b/agent_files/mount/common/src/vexfs_mount_contract_sqlite.cpp @@ -1082,6 +1082,9 @@ extern "C" vexfs_mount_status vexfs_mount_diagnostics(vexfs_mount_session *sessi Call contract(session, "SELECT value FROM _vexfs_meta WHERE key='contract_version'"); contract.Row(); + Call layout(session, + "SELECT value FROM _vexfs_meta WHERE key='staging_layout'"); + layout.Row(); Call journal(session, "PRAGMA journal_mode"); journal.Row(); Call synchronous(session, "PRAGMA synchronous"); @@ -1117,9 +1120,12 @@ extern "C" vexfs_mount_status vexfs_mount_diagnostics(vexfs_mount_session *sessi staging.Row(); const char *filename = sqlite3_db_filename(session->db, "main"); const std::string contract_version = contract.ResultText(); - const bool schema_ready = contract_version == "0.9.0"; + const std::string staging_layout = layout.ResultText(); + const bool schema_ready = contract_version == "0.9.0" && + staging_layout == "overlay-v1"; const std::string value = "{\"schema_version\":\"" + JsonEscape(contract_version) + + "\",\"staging_layout\":\"" + JsonEscape(staging_layout) + "\",\"schema_ready\":" + (schema_ready ? "true" : "false") + ",\"backend\":\"sqlite\",\"connection\":\"" + JsonEscape(filename == nullptr ? "" : filename) + diff --git a/tests/eval/vexfs/run.py b/tests/eval/vexfs/run.py index 1e71855762..e9ba048778 100644 --- a/tests/eval/vexfs/run.py +++ b/tests/eval/vexfs/run.py @@ -462,6 +462,9 @@ def schema_workspaces(ctx: Context) -> dict[str, Any]: with Database(ctx) as db: ctx.equal(db.scalar("SELECT vexfs_contract_version()"), CONTRACT_VERSION, "合同版本") ctx.equal(db.scalar("SELECT vexfs_init()"), 1, "重复初始化") + ctx.equal(db.scalar( + "SELECT value FROM _vexfs_meta WHERE key='staging_layout'"), + "overlay-v1", "staging 使用基础版本加脏块覆盖") default_id = db.scalar("SELECT vexfs_workspace_create('default')") ctx.equal(default_id, 1, "默认 workspace id") other_id = db.scalar("SELECT vexfs_workspace_create('other')") @@ -1326,6 +1329,44 @@ def session_reclaim(ctx: Context) -> dict[str, Any]: return {"published": 2, "reclaimed": 1} +@case("contract.active-mount-read-view", "contract", + "活动挂载的已接受 staging 对普通读取立即可见，普通 SDK staging 仍保持私有") +def active_mount_read_view(ctx: Context) -> dict[str, Any]: + with Database(ctx) as db: + db.scalar("SELECT vexfs_write('default','/live-view','base')") + + private = db.scalar( + "SELECT vexfs_handle_open('default','/live-view','rw','private-open')") + db.scalar("SELECT vexfs_handle_stage_write(?,0,'hidden','private-write')", + (private,)) + ctx.equal(db.scalar( + "SELECT CAST(vexfs_read('default','/live-view') AS TEXT)"), + "base", "普通 SDK staging 不能泄漏到当前文件视图") + db.scalar("SELECT vexfs_handle_close(?,0,'private-close')", (private,)) + + db.scalar("SELECT vexfs_mount_session_start('default','live-mount')") + mounted = db.scalar( + "SELECT vexfs_handle_open('default','/live-view','rw','mount-open','live-mount')") + db.scalar("SELECT vexfs_handle_stage_write(?,0,'mounted','mount-write')", + (mounted,)) + ctx.equal(db.scalar( + "SELECT CAST(vexfs_read('default','/live-view') AS TEXT)"), + "mounted", "活动挂载 staging 立即成为当前读取视图") + ctx.equal(db.scalar( + "SELECT CAST(vexfs_read_range('default','/live-view',1,4) AS TEXT)"), + "ount", "活动挂载范围读取使用相同实时视图") + ctx.equal(db.scalar( + "SELECT vexfs_mount_synchronize('default','live-sync','live-mount')"), + 1, "同步仍会把实时视图封成版本") + db.scalar("SELECT vexfs_mount_session_end('default','live-mount')") + ctx.equal(db.scalar( + "SELECT CAST(vexfs_read('default','/live-view') AS TEXT)"), + "mounted", "挂载结束后从已发布版本读取") + return {"private_staging_visible": False, + "active_mount_staging_visible": True, + "published_versions": 2} + + @case("contract.cross-session-snapshot-barrier", "concurrency", "管理会话不能静默快照另一个挂载会话尚未发布的写入") def cross_session_snapshot_barrier(ctx: Context) -> dict[str, Any]: @@ -1511,7 +1552,7 @@ def integrity_structural_corruption(ctx: Context) -> dict[str, Any]: db.connection.execute( "UPDATE _vexfs_file_versions SET source_version_no=999996 " "WHERE source_version_no IS NOT NULL") - db.connection.execute("DELETE FROM _vexfs_staging_data WHERE handle_id=?", (handle,)) + db.connection.execute("DELETE FROM _vexfs_staging_chunks WHERE handle_id=?", (handle,)) report = db.json("SELECT vexfs_check('default',0)") codes = {issue["code"] for issue in report["issues"]} @@ -1858,18 +1899,34 @@ def chunk_manifest_reuse(ctx: Context) -> dict[str, Any]: handle = db.scalar( "SELECT vexfs_handle_open('default','/chunked.bin','rw','chunk-open')") + staging = db.connection.execute( + "SELECT base_manifest_id,base_size,base_visible_size,logical_size,capacity " + "FROM _vexfs_staging WHERE handle_id=?", (handle,)).fetchone() + ctx.equal(staging, (first_manifest, len(original), len(original), len(original), 0), + "打开已有文件只记录基础 manifest，不复制正文") patch_offset = chunk_bytes + 123 patch = b"changed-block" generation = db.scalar( "SELECT vexfs_handle_stage_write(?,?,?,'chunk-patch')", (handle, patch_offset, patch)) + ctx.equal(db.scalar( + "SELECT capacity FROM _vexfs_staging WHERE handle_id=?", (handle,)), + chunk_bytes, "小范围覆盖只保存一个脏块") ctx.equal(db.scalar( "SELECT vexfs_handle_publish(?,?,'data','chunk-publish')", (handle, generation)), 2, "随机覆盖发布第二版本") - db.scalar("SELECT vexfs_handle_close(?,0,'chunk-close')", (handle,)) second_manifest = db.scalar( "SELECT manifest_id FROM _vexfs_file_versions " "WHERE inode_id=? AND version_no=2", (inode,)) + ctx.equal(db.connection.execute( + "SELECT base_manifest_id,base_size,base_visible_size,logical_size,capacity " + "FROM _vexfs_staging WHERE handle_id=?", (handle,)).fetchone(), + (second_manifest, len(original), len(original), len(original), 0), + "发布后 staging 重基于新 manifest 并清空脏块") + ctx.equal(db.scalar( + "SELECT count(*) FROM _vexfs_staging_chunks WHERE handle_id=?", (handle,)), + 0, "发布后不长期累积脏块") + db.scalar("SELECT vexfs_handle_close(?,0,'chunk-close')", (handle,)) reused = db.scalar( "SELECT count(*) FROM _vexfs_manifest_chunks first " "JOIN _vexfs_manifest_chunks second " @@ -1907,6 +1964,42 @@ def chunk_manifest_reuse(ctx: Context) -> dict[str, Any]: "physical_chunks_after_gc": 4} +@case("storage.overlay-truncate-regrow", "storage", + "已有文件截短再扩展时不恢复旧尾部，只保存边界脏块") +def overlay_truncate_regrow(ctx: Context) -> dict[str, Any]: + chunk_bytes = 64 * 1024 + original = b"A" * chunk_bytes + b"B" * chunk_bytes + with Database(ctx) as db: + db.scalar("SELECT vexfs_write('default','/truncate.bin',?)", (original,)) + handle = db.scalar( + "SELECT vexfs_handle_open('default','/truncate.bin','rw','truncate-open')") + generation = db.scalar( + "SELECT vexfs_handle_truncate(?,100,'truncate-small')", (handle,)) + generation = db.scalar( + "SELECT vexfs_handle_truncate(?,?,'truncate-regrow')", + (handle, len(original))) + ctx.equal(db.scalar("SELECT vexfs_handle_read(?,0,100)", (handle,)), + original[:100], "截短前缀保持不变") + ctx.equal(db.scalar( + "SELECT vexfs_handle_read(?,100,?)", (handle, len(original) - 100)), + bytes(len(original) - 100), "重新扩展部分全部为零") + ctx.equal(db.scalar( + "SELECT capacity FROM _vexfs_staging WHERE handle_id=?", (handle,)), + 100, "截短只物化 EOF 前仍可见的边界内容") + ctx.equal(db.scalar( + "SELECT vexfs_handle_publish(?,?,'data','truncate-publish')", + (handle, generation)), 2, "截短再扩展发布新版本") + db.scalar("SELECT vexfs_handle_close(?,0,'truncate-close')", (handle,)) + content = db.scalar("SELECT vexfs_read('default','/truncate.bin')") + ctx.equal(content[:100], original[:100], "发布后前缀保持不变") + ctx.equal(content[100:], bytes(len(original) - 100), + "发布后旧尾部不会复活") + ctx.check(db.json("SELECT vexfs_check('default',1)")["ok"], + "截短再扩展后深度检查通过") + return {"logical_bytes": len(original), "dirty_bytes": 100, + "generation": generation, "version": 2} + + @case("recovery.initialization-failure-rollback", "recovery", "新库初始化空间不足时不得留下半张 schema") def initialization_failure_rollback(ctx: Context) -> dict[str, Any]: @@ -2986,14 +3079,15 @@ def performance_staged_overwrite(ctx: Context) -> dict[str, Any]: (handle, offset, data, f"staged-{offset}")) stage_seconds = time.perf_counter() - started row = db.connection.execute( - "SELECT count(*),s.logical_size,s.capacity,length(d.content) " - "FROM _vexfs_staging s JOIN _vexfs_staging_data d USING(handle_id) " + "SELECT count(DISTINCT s.handle_id),max(s.logical_size),max(s.capacity)," + "count(d.chunk_no),COALESCE(sum(length(d.content)),0) " + "FROM _vexfs_staging s LEFT JOIN _vexfs_staging_chunks d USING(handle_id) " "WHERE s.handle_id=?", (handle,)).fetchone() ctx.equal(row[0], 1, "每个 handle 只能有一行 staging") ctx.equal(row[1], total, "staging logical_size") - ctx.equal(row[2], row[3], "capacity 与 BLOB 长度") - ctx.check(total <= row[2] <= MAX_FILE_BYTES, "staging capacity 边界") - ctx.check(row[2] < max(total * 2, 64 * 1024 + 1), "几何扩容不能超过 2x") + ctx.equal(row[2], row[4], "capacity 与脏块总长度") + ctx.equal(row[2], row[3] * 64 * 1024, "每个脏块固定为 64 KiB") + ctx.check(total <= row[2] < total + 64 * 1024, "顺序写物理暂存边界") started = time.perf_counter() version = db.scalar( "SELECT vexfs_handle_publish(?,?,'full','staged-publish')", @@ -3016,7 +3110,8 @@ def performance_staged_overwrite(ctx: Context) -> dict[str, Any]: db.connection.execute("PRAGMA wal_checkpoint(TRUNCATE)").fetchone() storage_after_vacuum = db_storage(db.path) return {"logical_bytes": total, "capacity_bytes": row[2], - "staging_rows": row[0], "generation": generation, "version": version, + "staging_rows": row[0], "dirty_chunks": row[3], + "generation": generation, "version": version, "stage_seconds": round(stage_seconds, 6), "stage_mib_per_second": round(total / MIB / max(stage_seconds, 1e-9), 3), "publish_seconds": round(publish_seconds, 6), @@ -3054,12 +3149,15 @@ def performance_random_patches(ctx: Context) -> dict[str, Any]: for offset, data in list(expected.items())[-min(len(expected), 50):]: ctx.equal(db.scalar("SELECT vexfs_handle_read(?,?,?)", (handle, offset, patch_size)), data, "随机 patch 内容") + affected_chunks = {offset // (64 * 1024) for offset in expected} row = db.connection.execute( "SELECT count(*),logical_size,capacity FROM _vexfs_staging WHERE handle_id=?", (handle,)).fetchone() ctx.equal(row[0], 1, "随机写 staging 行数") ctx.equal(row[1], logical_size, "随机写 logical_size") - ctx.check(row[2] <= MAX_FILE_BYTES, "随机写 capacity") + ctx.equal(row[2], len(affected_chunks) * 64 * 1024, + "随机写只暂存受影响的 64 KiB 块") + ctx.check(row[2] < logical_size, "随机写不复制整份基础文件") publish_started = time.perf_counter() ctx.equal(db.scalar( "SELECT vexfs_handle_publish(?,?,'data','random-publish')", @@ -3069,7 +3167,6 @@ def performance_random_patches(ctx: Context) -> dict[str, Any]: patched_manifest = db.scalar( "SELECT manifest_id FROM _vexfs_file_versions " "WHERE inode_id=? AND version_no=2", (inode,)) - affected_chunks = {offset // (64 * 1024) for offset in expected} total_chunks = logical_size // (64 * 1024) reused_chunks = db.scalar( "SELECT count(*) FROM _vexfs_manifest_chunks before " diff --git a/vexdb_sqlite/src/agent_files/vexfs_sqlite.cpp b/vexdb_sqlite/src/agent_files/vexfs_sqlite.cpp index cfd505fb97..dd4bc22d3e 100644 --- a/vexdb_sqlite/src/agent_files/vexfs_sqlite.cpp +++ b/vexdb_sqlite/src/agent_files/vexfs_sqlite.cpp @@ -31,8 +31,6 @@ namespace { constexpr const char *kContractVersion = "0.9.0"; constexpr sqlite3_int64 kMaxStagedBytes = 128LL * 1024LL * 1024LL; constexpr sqlite3_int64 kContentChunkBytes = 64LL * 1024LL; -constexpr sqlite3_int64 kInitialStagingCapacity = 64LL * 1024LL; -constexpr sqlite3_int64 kEmptyFileStagingCapacity = 4LL * 1024LL; constexpr sqlite3_int64 kMaxXattrBytes = 64LL * 1024LL; constexpr sqlite3_int64 kMaxSymlinkBytes = 4096; constexpr sqlite3_int64 kRequestRetentionRows = 64LL * 1024LL; @@ -58,12 +56,6 @@ class SqlError : public std::runtime_error { int code_; }; -struct BlobCloser { - void operator()(sqlite3_blob *blob) const { - if (blob != nullptr) sqlite3_blob_close(blob); - } -}; - class Statement { public: Statement(sqlite3 *db, const char *sql) : db_(db) { @@ -542,6 +534,12 @@ void EnsureSchemaSlow(sqlite3 *db) { if (schema_exists) { if (version != kContractVersion) throw SqlError("unsupported VexFS schema version: " + version, SQLITE_MISMATCH); + Statement layout(db, + "SELECT value FROM _vexfs_meta WHERE key='staging_layout' LIMIT 1"); + if (!layout.Row() || layout.Text(0) != "overlay-v1") { + throw SqlError("unsupported VexFS staging layout; create a new database", + SQLITE_MISMATCH); + } DetectGrepIndex(db); return; } @@ -716,15 +714,20 @@ CREATE INDEX IF NOT EXISTS _vexfs_handles_workspace_idx CREATE TABLE IF NOT EXISTS _vexfs_staging( handle_id TEXT PRIMARY KEY, generation INTEGER NOT NULL, - content BLOB NOT NULL, - logical_size INTEGER NOT NULL, - capacity INTEGER NOT NULL, + base_manifest_id INTEGER, + base_size INTEGER NOT NULL CHECK(base_size >= 0), + base_visible_size INTEGER NOT NULL CHECK(base_visible_size >= 0), + logical_size INTEGER NOT NULL CHECK(logical_size >= 0), + capacity INTEGER NOT NULL CHECK(capacity >= 0), created_at INTEGER NOT NULL DEFAULT (CAST(strftime('%s','now') AS INTEGER) * 1000), - updated_at INTEGER NOT NULL DEFAULT (CAST(strftime('%s','now') AS INTEGER) * 1000) + updated_at INTEGER NOT NULL DEFAULT (CAST(strftime('%s','now') AS INTEGER) * 1000), + CHECK(base_visible_size <= base_size) ); -CREATE TABLE IF NOT EXISTS _vexfs_staging_data( - handle_id TEXT PRIMARY KEY, - content BLOB NOT NULL +CREATE TABLE IF NOT EXISTS _vexfs_staging_chunks( + handle_id TEXT NOT NULL, + chunk_no INTEGER NOT NULL CHECK(chunk_no >= 0), + content BLOB NOT NULL CHECK(length(content) > 0 AND length(content) <= 65536), + PRIMARY KEY(handle_id, chunk_no) ); CREATE TABLE IF NOT EXISTS _vexfs_requests( request_id TEXT PRIMARY KEY, @@ -743,7 +746,7 @@ CREATE TABLE IF NOT EXISTS _vexfs_mount_sessions( ); INSERT OR IGNORE INTO _vexfs_meta(key, value) VALUES('contract_version', '0.9.0'); INSERT OR IGNORE INTO _vexfs_meta(key, value) VALUES('content_model', 'chunked-v1'); -INSERT OR IGNORE INTO _vexfs_meta(key, value) VALUES('staging_layout', 'split-v1'); +INSERT OR IGNORE INTO _vexfs_meta(key, value) VALUES('staging_layout', 'overlay-v1'); )SQL"); CreateHistorySchema(db); EnsureAclHistoryTriggers(db); @@ -1075,6 +1078,64 @@ WHERE entry.manifest_id=?1 ORDER BY entry.chunk_no return content; } +std::vector ReadVersionRange(sqlite3 *db, sqlite3_int64 inode_id, + sqlite3_int64 version, + sqlite3_int64 offset, + sqlite3_int64 length) { + const VersionStorage storage = ResolveVersionStorage(db, inode_id, version); + if (storage.size > kMaxStagedBytes) { + throw SqlError("file version is larger than 128 MiB", SQLITE_CORRUPT); + } + if (offset < 0 || length < 0 || offset > kMaxStagedBytes || + length > kMaxStagedBytes) { + throw SqlError("read range must be within 0..128 MiB", SQLITE_RANGE); + } + if (length == 0 || offset >= storage.size) return {}; + + const sqlite3_int64 end = std::min(storage.size, offset + length); + const sqlite3_int64 first_chunk = offset / kContentChunkBytes; + const sqlite3_int64 last_chunk = (end - 1) / kContentChunkBytes; + std::vector content; + content.reserve(static_cast(end - offset)); + + Statement chunks(db, R"SQL( +SELECT entry.chunk_no,chunk.content,chunk.size,chunk.checksum +FROM _vexfs_manifest_chunks entry +JOIN _vexfs_chunks chunk ON chunk.id=entry.chunk_id +WHERE entry.manifest_id=?1 AND entry.chunk_no BETWEEN ?2 AND ?3 +ORDER BY entry.chunk_no +)SQL"); + chunks.BindInt64(1, storage.manifest_id); + chunks.BindInt64(2, first_chunk); + chunks.BindInt64(3, last_chunk); + sqlite3_int64 expected_chunk = first_chunk; + while (chunks.Row()) { + const sqlite3_int64 chunk_no = chunks.Int64(0); + const sqlite3_int64 chunk_offset = chunk_no * kContentChunkBytes; + const sqlite3_int64 expected_size = std::min( + kContentChunkBytes, storage.size - chunk_offset); + const int chunk_bytes = chunks.BlobBytes(1); + const unsigned char *chunk = chunks.BlobData(1); + if (chunk_no != expected_chunk || expected_size <= 0 || + chunks.Int64(2) != expected_size || chunk_bytes != expected_size || + chunk == nullptr || chunks.Text(3).size() != 64 || + vexfs::Sha256Hex(chunk, static_cast(chunk_bytes)) != chunks.Text(3)) { + throw SqlError("file manifest range chunk is corrupt", SQLITE_CORRUPT); + } + const sqlite3_int64 copy_start = std::max(offset, chunk_offset) - + chunk_offset; + const sqlite3_int64 copy_end = std::min( + end, chunk_offset + expected_size) - chunk_offset; + content.insert(content.end(), chunk + copy_start, chunk + copy_end); + ++expected_chunk; + } + if (expected_chunk != last_chunk + 1 || + content.size() != static_cast(end - offset)) { + throw SqlError("file manifest range is incomplete", SQLITE_CORRUPT); + } + return content; +} + bool FindChild(sqlite3 *db, sqlite3_int64 workspace_id, sqlite3_int64 parent, const std::string &name, Node *node) { Statement statement(db, @@ -1324,8 +1385,8 @@ WHERE workspace_id=?1 AND NOT EXISTS ( clear_acl.Done(); } -sqlite3_int64 CreateCommit(sqlite3 *db, const Workspace &workspace, const std::string &message) { - Savepoint savepoint(db, "vexfs_create_commit"); +sqlite3_int64 CreateCommitInTransaction(sqlite3 *db, const Workspace &workspace, + const std::string &message) { Statement insert(db, "INSERT INTO _vexfs_commits(workspace_id, parent_commit, message) " "SELECT id, NULLIF(head_commit, 0), ?2 FROM _vexfs_workspaces WHERE id=?1"); @@ -1345,6 +1406,13 @@ sqlite3_int64 CreateCommit(sqlite3 *db, const Workspace &workspace, const std::s update.BindInt64(1, commit); update.BindInt64(2, workspace.id); update.Done(); + return commit; +} + +sqlite3_int64 CreateCommit(sqlite3 *db, const Workspace &workspace, + const std::string &message) { + Savepoint savepoint(db, "vexfs_create_commit"); + const sqlite3_int64 commit = CreateCommitInTransaction(db, workspace, message); savepoint.Release(); return commit; } @@ -1926,121 +1994,125 @@ Handle FindHandle(sqlite3 *db, const std::string &id) { } struct StagingInfo { - sqlite3_int64 rowid = 0; sqlite3_int64 generation = 0; + sqlite3_int64 base_manifest_id = 0; + sqlite3_int64 base_size = 0; + sqlite3_int64 base_visible_size = 0; sqlite3_int64 logical_size = 0; sqlite3_int64 capacity = 0; }; StagingInfo FindStaging(sqlite3 *db, const Handle &handle) { Statement statement(db, - "SELECT d.rowid, s.generation, s.logical_size, s.capacity " - "FROM _vexfs_staging s JOIN _vexfs_staging_data d ON d.handle_id=s.handle_id " - "WHERE s.handle_id = ?1"); + "SELECT generation,COALESCE(base_manifest_id,0),base_size,base_visible_size," + "logical_size,capacity FROM _vexfs_staging WHERE handle_id=?1"); statement.BindText(1, handle.id); if (!statement.Row()) throw SqlError("staging area not found", SQLITE_NOTFOUND); - return {statement.Int64(0), statement.Int64(1), statement.Int64(2), statement.Int64(3)}; + return {statement.Int64(0), statement.Int64(1), statement.Int64(2), + statement.Int64(3), statement.Int64(4), statement.Int64(5)}; } -void WriteBlob(sqlite3 *db, sqlite3_int64 rowid, sqlite3_int64 offset, - const void *data, sqlite3_int64 size) { - if (size == 0) return; - if (offset < 0 || size < 0 || offset > std::numeric_limits::max() || - size > std::numeric_limits::max() || offset + size > std::numeric_limits::max()) { - throw SqlError("staging BLOB range is out of range", SQLITE_RANGE); - } - sqlite3_blob *blob = nullptr; - int rc = sqlite3_blob_open(db, "main", "_vexfs_staging_data", "content", rowid, 1, &blob); - if (rc != SQLITE_OK) { - throw SqlError("cannot open staging BLOB: " + std::string(sqlite3_errmsg(db))); - } - rc = sqlite3_blob_write(blob, data, static_cast(size), static_cast(offset)); - const int close_rc = sqlite3_blob_close(blob); - if (rc != SQLITE_OK) { - throw SqlError("cannot write staging BLOB: " + std::string(sqlite3_errmsg(db))); - } - if (close_rc != SQLITE_OK) { - throw SqlError("cannot close staging BLOB: " + std::string(sqlite3_errmsg(db))); - } +sqlite3_int64 DirtyChunkSize(sqlite3 *db, const std::string &handle_id, + sqlite3_int64 chunk_no) { + Statement statement(db, + "SELECT length(content) FROM _vexfs_staging_chunks " + "WHERE handle_id=?1 AND chunk_no=?2"); + statement.BindText(1, handle_id); + statement.BindInt64(2, chunk_no); + return statement.Row() ? statement.Int64(0) : 0; } -void ZeroBlobRange(sqlite3 *db, sqlite3_int64 rowid, sqlite3_int64 offset, - sqlite3_int64 size) { - static const std::array zeros{}; - if (size == 0) return; - if (offset < 0 || size < 0 || offset > std::numeric_limits::max() || - size > std::numeric_limits::max() || offset + size > std::numeric_limits::max()) { - throw SqlError("staging zero range is out of range", SQLITE_RANGE); - } - sqlite3_blob *blob = nullptr; - int rc = sqlite3_blob_open(db, "main", "_vexfs_staging_data", "content", rowid, 1, &blob); - if (rc != SQLITE_OK) { - throw SqlError("cannot open staging BLOB: " + std::string(sqlite3_errmsg(db))); - } - while (size > 0) { - const sqlite3_int64 chunk = std::min(size, zeros.size()); - rc = sqlite3_blob_write(blob, zeros.data(), static_cast(chunk), - static_cast(offset)); - if (rc != SQLITE_OK) break; - offset += chunk; - size -= chunk; - } - const int close_rc = sqlite3_blob_close(blob); - if (rc != SQLITE_OK) { - throw SqlError("cannot zero staging BLOB: " + std::string(sqlite3_errmsg(db))); - } - if (close_rc != SQLITE_OK) { - throw SqlError("cannot close staging BLOB: " + std::string(sqlite3_errmsg(db))); +void StoreDirtyChunk(sqlite3 *db, const std::string &handle_id, + sqlite3_int64 chunk_no, + const std::vector &content) { + if (content.empty() || content.size() > static_cast(kContentChunkBytes)) { + throw SqlError("staging chunk must be within 1..65536 bytes", SQLITE_CORRUPT); } + Statement statement(db, R"SQL( +INSERT INTO _vexfs_staging_chunks(handle_id,chunk_no,content) +VALUES(?1,?2,?3) +ON CONFLICT(handle_id,chunk_no) DO UPDATE SET content=excluded.content +)SQL"); + statement.BindText(1, handle_id); + statement.BindInt64(2, chunk_no); + statement.BindBlob(3, content); + statement.Done(); } -StagingInfo EnsureStagingCapacity(sqlite3 *db, const Handle &handle, StagingInfo staging, - sqlite3_int64 required) { - if (required <= staging.capacity) return staging; - if (required > kMaxStagedBytes) - throw SqlError("staged file is larger than 128 MiB", SQLITE_TOOBIG); - sqlite3_int64 capacity = staging.capacity == 0 - ? kEmptyFileStagingCapacity - : std::max(staging.capacity, kInitialStagingCapacity); - while (capacity < required) capacity = std::min(kMaxStagedBytes, capacity * 2); - Statement grow(db, - "UPDATE _vexfs_staging_data SET content=CAST(content||zeroblob(?1) AS BLOB) " - "WHERE handle_id=?2"); - grow.BindInt64(1, capacity - staging.capacity); - grow.BindText(2, handle.id); - grow.Done(); - Statement update(db, - "UPDATE _vexfs_staging SET capacity=?1, " - "updated_at=CAST(strftime('%s','now') AS INTEGER)*1000 WHERE handle_id=?2"); - update.BindInt64(1, capacity); - update.BindText(2, handle.id); - update.Done(); - staging.capacity = capacity; - return staging; +std::vector ReadEffectiveChunk(sqlite3 *db, const Handle &handle, + const StagingInfo &staging, + sqlite3_int64 chunk_no, + sqlite3_int64 *dirty_bytes_out = nullptr) { + if (chunk_no < 0) throw SqlError("staging chunk number is out of range", SQLITE_RANGE); + std::vector content(static_cast(kContentChunkBytes), 0); + Statement dirty(db, + "SELECT content FROM _vexfs_staging_chunks WHERE handle_id=?1 AND chunk_no=?2"); + dirty.BindText(1, handle.id); + dirty.BindInt64(2, chunk_no); + if (dirty.Row()) { + const int dirty_bytes = dirty.BlobBytes(0); + if (dirty_bytes <= 0 || dirty_bytes > kContentChunkBytes || + dirty.BlobData(0) == nullptr) { + throw SqlError("staging chunk is corrupt", SQLITE_CORRUPT); + } + std::memcpy(content.data(), dirty.BlobData(0), static_cast(dirty_bytes)); + if (dirty_bytes_out != nullptr) *dirty_bytes_out = dirty_bytes; + return content; + } + if (dirty_bytes_out != nullptr) *dirty_bytes_out = 0; + + const sqlite3_int64 chunk_offset = chunk_no * kContentChunkBytes; + if (chunk_offset >= staging.base_visible_size) return content; + if (staging.base_manifest_id <= 0 || staging.base_size <= chunk_offset || + staging.base_visible_size > staging.base_size) { + throw SqlError("staging base reference is corrupt", SQLITE_CORRUPT); + } + Statement base(db, R"SQL( +SELECT chunk.content,chunk.size,chunk.checksum +FROM _vexfs_manifest_chunks entry +JOIN _vexfs_chunks chunk ON chunk.id=entry.chunk_id +WHERE entry.manifest_id=?1 AND entry.chunk_no=?2 +)SQL"); + base.BindInt64(1, staging.base_manifest_id); + base.BindInt64(2, chunk_no); + if (!base.Row()) throw SqlError("staging base chunk is missing", SQLITE_CORRUPT); + const sqlite3_int64 expected_size = std::min( + kContentChunkBytes, staging.base_size - chunk_offset); + if (expected_size <= 0 || base.Int64(1) != expected_size || + base.BlobBytes(0) != expected_size || base.BlobData(0) == nullptr || + base.Text(2).size() != 64 || + vexfs::Sha256Hex(base.BlobData(0), static_cast(expected_size)) != base.Text(2)) { + throw SqlError("staging base chunk is corrupt", SQLITE_CORRUPT); + } + const sqlite3_int64 visible = std::min( + expected_size, staging.base_visible_size - chunk_offset); + std::memcpy(content.data(), base.BlobData(0), static_cast(visible)); + return content; } void CreateStaging(sqlite3 *db, const Handle &handle, const std::vector &content) { - sqlite3_int64 capacity = static_cast(content.size()); - if (handle.writable && capacity > 0 && capacity < kInitialStagingCapacity) { - capacity = kInitialStagingCapacity; + if (content.size() > static_cast(kMaxStagedBytes)) { + throw SqlError("staged file is larger than 128 MiB", SQLITE_TOOBIG); } + const sqlite3_int64 chunk_count = + (static_cast(content.size()) + kContentChunkBytes - 1) / + kContentChunkBytes; Statement insert(db, - "INSERT INTO _vexfs_staging(handle_id,generation,content,logical_size,capacity) " - "VALUES(?1,?2,X'',?4,?3)"); + "INSERT INTO _vexfs_staging(handle_id,generation,base_manifest_id,base_size," + "base_visible_size,logical_size,capacity) VALUES(?1,?2,NULL,0,0,?3,?4)"); insert.BindText(1, handle.id); insert.BindInt64(2, handle.dirty_generation); - insert.BindInt64(3, capacity); + insert.BindInt64(3, static_cast(content.size())); insert.BindInt64(4, static_cast(content.size())); insert.Done(); - Statement data(db, - "INSERT INTO _vexfs_staging_data(handle_id,content) VALUES(?1,zeroblob(?2))"); - data.BindText(1, handle.id); - data.BindInt64(2, capacity); - data.Done(); - if (!content.empty()) { - const StagingInfo staging = FindStaging(db, handle); - WriteBlob(db, staging.rowid, 0, content.data(), static_cast(content.size())); + for (sqlite3_int64 chunk_no = 0; chunk_no < chunk_count; ++chunk_no) { + const sqlite3_int64 offset = chunk_no * kContentChunkBytes; + const size_t bytes = static_cast(std::min( + kContentChunkBytes, static_cast(content.size()) - offset)); + std::vector chunk(content.begin() + offset, + content.begin() + offset + bytes); + StoreDirtyChunk(db, handle.id, chunk_no, chunk); } } @@ -2053,75 +2125,41 @@ void CreateStagingFromVersion(sqlite3 *db, const Handle &handle, const Node &nod if (storage.size != node.size) { throw SqlError("current file size does not match manifest", SQLITE_CORRUPT); } - sqlite3_int64 capacity = storage.size; - if (handle.writable && capacity < kInitialStagingCapacity) { - capacity = kInitialStagingCapacity; - } Statement insert(db, - "INSERT INTO _vexfs_staging(handle_id,generation,content,logical_size,capacity) " - "VALUES(?1,?2,X'',?4,?3)"); + "INSERT INTO _vexfs_staging(handle_id,generation,base_manifest_id,base_size," + "base_visible_size,logical_size,capacity) VALUES(?1,?2,?3,?4,?4,?4,0)"); insert.BindText(1, handle.id); insert.BindInt64(2, handle.dirty_generation); - insert.BindInt64(3, capacity); + insert.BindInt64(3, storage.manifest_id); insert.BindInt64(4, storage.size); insert.Done(); - Statement data(db, - "INSERT INTO _vexfs_staging_data(handle_id,content) VALUES(?1,zeroblob(?2))"); - data.BindText(1, handle.id); - data.BindInt64(2, capacity); - data.Done(); - const StagingInfo staging = FindStaging(db, handle); - Statement chunks(db, R"SQL( -SELECT entry.chunk_no,chunk.content,chunk.size,chunk.checksum -FROM _vexfs_manifest_chunks entry -JOIN _vexfs_chunks chunk ON chunk.id=entry.chunk_id -WHERE entry.manifest_id=?1 ORDER BY entry.chunk_no -)SQL"); - chunks.BindInt64(1, storage.manifest_id); - vexfs::Sha256 file_hash; - sqlite3_int64 offset = 0; - sqlite3_int64 chunk_no = 0; - while (chunks.Row()) { - const std::vector chunk = chunks.Blob(1); - const sqlite3_int64 expected_size = std::min( - kContentChunkBytes, storage.size - offset); - if (chunks.Int64(0) != chunk_no || expected_size <= 0 || - chunks.Int64(2) != expected_size || - static_cast(chunk.size()) != expected_size || - vexfs::Sha256Hex(chunk.data(), chunk.size()) != chunks.Text(3)) { - throw SqlError("current file manifest chunk is corrupt", SQLITE_CORRUPT); - } - WriteBlob(db, staging.rowid, offset, chunk.data(), expected_size); - file_hash.Update(chunk.data(), chunk.size()); - offset += expected_size; - ++chunk_no; - } - if (offset != storage.size || chunk_no != storage.chunk_count || - vexfs::Hex(file_hash.Finish()) != storage.checksum) { - throw SqlError("current file manifest checksum mismatch", SQLITE_CORRUPT); - } } sqlite3_int64 StoreManifestFromStaging(sqlite3 *db, sqlite3_int64 workspace_id, sqlite3_int64 inode_id, sqlite3_int64 previous_manifest, + const Handle &handle, const StagingInfo &staging, std::string *checksum_out) { - if (staging.logical_size < 0 || staging.logical_size > staging.capacity || - staging.logical_size > kMaxStagedBytes) { + if (staging.logical_size < 0 || staging.logical_size > kMaxStagedBytes || + staging.base_size < 0 || staging.base_size > kMaxStagedBytes || + staging.base_visible_size < 0 || + staging.base_visible_size > staging.base_size || staging.capacity < 0 || + staging.capacity > kMaxStagedBytes) { throw SqlError("staging size is corrupt", SQLITE_CORRUPT); } - sqlite3_blob *raw_blob = nullptr; - int rc = sqlite3_blob_open(db, "main", "_vexfs_staging_data", "content", - staging.rowid, 0, &raw_blob); - if (rc != SQLITE_OK) throw SqlError("cannot open staging BLOB", rc); - std::unique_ptr blob(raw_blob); - if (sqlite3_blob_bytes(blob.get()) != staging.capacity) { - throw SqlError("staging capacity does not match BLOB", SQLITE_CORRUPT); - } - std::vector buffer(static_cast(kContentChunkBytes)); + Statement dirty_stats(db, + "SELECT count(*),COALESCE(sum(length(content)),0),COALESCE(max(chunk_no),-1) " + "FROM _vexfs_staging_chunks WHERE handle_id=?1"); + dirty_stats.BindText(1, handle.id); + dirty_stats.Row(); const sqlite3_int64 chunk_count = (staging.logical_size + kContentChunkBytes - 1) / kContentChunkBytes; + if (dirty_stats.Int64(1) != staging.capacity || + (dirty_stats.Int64(0) > 0 && dirty_stats.Int64(2) >= chunk_count)) { + throw SqlError("staging chunk set is corrupt", SQLITE_CORRUPT); + } + std::vector buffer(static_cast(kContentChunkBytes)); const std::string pending_checksum(64, '0'); Statement manifest(db, R"SQL( INSERT INTO _vexfs_manifests(workspace_id,file_size,chunk_size,chunk_count,checksum) @@ -2147,6 +2185,15 @@ VALUES(?1,?2,?3,?4,?5) Statement entry(db, R"SQL( INSERT INTO _vexfs_manifest_chunks(manifest_id,chunk_no,chunk_id) VALUES(?1,?2,?3) +)SQL"); + Statement staged_chunk(db, R"SQL( +SELECT dirty.content,base.id,base.content,base.size,base.checksum,base.inode_id +FROM (SELECT 1) seed +LEFT JOIN _vexfs_staging_chunks dirty + ON dirty.handle_id=?1 AND dirty.chunk_no=?2 +LEFT JOIN _vexfs_manifest_chunks base_entry + ON base_entry.manifest_id=?3 AND base_entry.chunk_no=?2 +LEFT JOIN _vexfs_chunks base ON base.id=base_entry.chunk_id )SQL"); std::unordered_map new_chunks; vexfs::Sha256 file_hash; @@ -2154,13 +2201,54 @@ VALUES(?1,?2,?3) ++chunk_no, offset += kContentChunkBytes) { const int bytes = static_cast(std::min( kContentChunkBytes, staging.logical_size - offset)); - rc = sqlite3_blob_read(blob.get(), buffer.data(), bytes, static_cast(offset)); - if (rc != SQLITE_OK) throw SqlError("cannot read staging BLOB", rc); + std::fill(buffer.begin(), buffer.end(), 0); + sqlite3_int64 chunk_id = 0; + std::string base_checksum; + staged_chunk.BindText(1, handle.id); + staged_chunk.BindInt64(2, chunk_no); + staged_chunk.BindInt64(3, staging.base_manifest_id); + if (!staged_chunk.Row()) { + throw SqlError("cannot resolve staging chunk", SQLITE_CORRUPT); + } + if (staged_chunk.Type(0) != SQLITE_NULL) { + const int dirty_bytes = staged_chunk.BlobBytes(0); + if (dirty_bytes <= 0 || dirty_bytes > kContentChunkBytes || + staged_chunk.BlobData(0) == nullptr) { + throw SqlError("staging chunk is corrupt", SQLITE_CORRUPT); + } + std::memcpy(buffer.data(), staged_chunk.BlobData(0), + static_cast(dirty_bytes)); + } else if (offset < staging.base_visible_size) { + const sqlite3_int64 expected_base_size = std::min( + kContentChunkBytes, staging.base_size - offset); + if (staging.base_manifest_id <= 0 || expected_base_size <= 0 || + staged_chunk.Type(1) == SQLITE_NULL || staged_chunk.Int64(3) != expected_base_size || + staged_chunk.BlobBytes(2) != expected_base_size || + staged_chunk.BlobData(2) == nullptr || staged_chunk.Text(4).size() != 64 || + staged_chunk.Int64(5) != inode_id) { + throw SqlError("staging base chunk is corrupt", SQLITE_CORRUPT); + } + const sqlite3_int64 visible = std::min( + expected_base_size, staging.base_visible_size - offset); + std::memcpy(buffer.data(), staged_chunk.BlobData(2), + static_cast(visible)); + base_checksum = staged_chunk.Text(4); + if (visible == expected_base_size && expected_base_size == bytes) { + chunk_id = staged_chunk.Int64(1); + } else if (vexfs::Sha256Hex( + staged_chunk.BlobData(2), + static_cast(expected_base_size)) != base_checksum) { + throw SqlError("staging base chunk checksum mismatch", SQLITE_CORRUPT); + } + } + staged_chunk.Reset(); file_hash.Update(buffer.data(), static_cast(bytes)); const std::string chunk_checksum = vexfs::Sha256Hex(buffer.data(), static_cast(bytes)); - sqlite3_int64 chunk_id = 0; - if (previous_manifest > 0) { + if (chunk_id > 0 && chunk_checksum != base_checksum) { + throw SqlError("staging base chunk checksum mismatch", SQLITE_CORRUPT); + } + if (chunk_id == 0 && previous_manifest > 0) { previous.BindInt64(1, previous_manifest); previous.BindInt64(2, chunk_no); previous.BindInt64(3, inode_id); @@ -2206,20 +2294,50 @@ VALUES(?1,?2,?3) sqlite3_int64 StageWrite(sqlite3 *db, const Handle &handle, sqlite3_int64 offset, const std::vector &patch) { if (patch.empty()) return handle.dirty_generation; - const sqlite3_int64 required = offset + static_cast(patch.size()); - StagingInfo staging = EnsureStagingCapacity(db, handle, FindStaging(db, handle), required); - if (offset > staging.logical_size) { - ZeroBlobRange(db, staging.rowid, staging.logical_size, offset - staging.logical_size); + if (offset < 0 || offset > kMaxStagedBytes || + patch.size() > static_cast(kMaxStagedBytes - offset)) { + throw SqlError("staged file is larger than 128 MiB", SQLITE_TOOBIG); } - WriteBlob(db, staging.rowid, offset, patch.data(), static_cast(patch.size())); + const sqlite3_int64 required = offset + static_cast(patch.size()); + const StagingInfo staging = FindStaging(db, handle); + sqlite3_int64 capacity_delta = 0; const sqlite3_int64 logical_size = std::max(staging.logical_size, required); + size_t patch_offset = 0; + while (patch_offset < patch.size()) { + const sqlite3_int64 absolute = offset + static_cast(patch_offset); + const sqlite3_int64 chunk_no = absolute / kContentChunkBytes; + const sqlite3_int64 within = absolute % kContentChunkBytes; + const size_t bytes = std::min( + patch.size() - patch_offset, + static_cast(kContentChunkBytes - within)); + const bool full_chunk = within == 0 && bytes == static_cast(kContentChunkBytes); + sqlite3_int64 previous_dirty_bytes = 0; + std::vector chunk; + if (full_chunk) { + previous_dirty_bytes = DirtyChunkSize(db, handle.id, chunk_no); + chunk.assign(patch.begin() + static_cast(patch_offset), + patch.begin() + static_cast(patch_offset + bytes)); + } else { + chunk = ReadEffectiveChunk( + db, handle, staging, chunk_no, &previous_dirty_bytes); + std::memcpy(chunk.data() + within, patch.data() + patch_offset, bytes); + const sqlite3_int64 chunk_offset = chunk_no * kContentChunkBytes; + const sqlite3_int64 stored_bytes = std::min( + kContentChunkBytes, logical_size - chunk_offset); + chunk.resize(static_cast(stored_bytes)); + } + StoreDirtyChunk(db, handle.id, chunk_no, chunk); + capacity_delta += static_cast(chunk.size()) - previous_dirty_bytes; + patch_offset += bytes; + } const sqlite3_int64 generation = handle.dirty_generation + 1; Statement stage(db, - "UPDATE _vexfs_staging SET generation=?1, logical_size=?2, " - "updated_at=CAST(strftime('%s','now') AS INTEGER)*1000 WHERE handle_id=?3"); + "UPDATE _vexfs_staging SET generation=?1,logical_size=?2,capacity=capacity+?3," + "updated_at=CAST(strftime('%s','now') AS INTEGER)*1000 WHERE handle_id=?4"); stage.BindInt64(1, generation); stage.BindInt64(2, logical_size); - stage.BindText(3, handle.id); + stage.BindInt64(3, capacity_delta); + stage.BindText(4, handle.id); stage.Done(); Statement update(db, "UPDATE _vexfs_handles SET dirty_generation=?1, " @@ -2235,17 +2353,41 @@ sqlite3_int64 TruncateStaging(sqlite3 *db, const Handle &handle, sqlite3_int64 s throw SqlError("file size is out of range", SQLITE_RANGE); StagingInfo staging = FindStaging(db, handle); if (size == staging.logical_size) return handle.dirty_generation; - staging = EnsureStagingCapacity(db, handle, staging, size); - if (size > staging.logical_size) { - ZeroBlobRange(db, staging.rowid, staging.logical_size, size - staging.logical_size); + sqlite3_int64 base_visible_size = staging.base_visible_size; + if (size < staging.logical_size) { + const sqlite3_int64 remainder = size % kContentChunkBytes; + std::vector boundary; + if (remainder != 0) { + boundary = ReadEffectiveChunk(db, handle, staging, size / kContentChunkBytes); + boundary.resize(static_cast(remainder)); + } + const sqlite3_int64 first_removed = + (size + kContentChunkBytes - 1) / kContentChunkBytes; + Statement remove(db, + "DELETE FROM _vexfs_staging_chunks WHERE handle_id=?1 AND chunk_no>=?2"); + remove.BindText(1, handle.id); + remove.BindInt64(2, first_removed); + remove.Done(); + if (remainder != 0) { + StoreDirtyChunk(db, handle.id, size / kContentChunkBytes, boundary); + } + base_visible_size = std::min(base_visible_size, size); } + Statement dirty_bytes(db, + "SELECT COALESCE(sum(length(content)),0) FROM _vexfs_staging_chunks " + "WHERE handle_id=?1"); + dirty_bytes.BindText(1, handle.id); + dirty_bytes.Row(); const sqlite3_int64 generation = handle.dirty_generation + 1; Statement stage(db, - "UPDATE _vexfs_staging SET generation=?1, logical_size=?2, " - "updated_at=CAST(strftime('%s','now') AS INTEGER)*1000 WHERE handle_id=?3"); + "UPDATE _vexfs_staging SET generation=?1,logical_size=?2,base_visible_size=?3," + "capacity=?4,updated_at=CAST(strftime('%s','now') AS INTEGER)*1000 " + "WHERE handle_id=?5"); stage.BindInt64(1, generation); stage.BindInt64(2, size); - stage.BindText(3, handle.id); + stage.BindInt64(3, base_visible_size); + stage.BindInt64(4, dirty_bytes.Int64(0)); + stage.BindText(5, handle.id); stage.Done(); Statement update(db, "UPDATE _vexfs_handles SET dirty_generation=?1, " @@ -2258,16 +2400,53 @@ sqlite3_int64 TruncateStaging(sqlite3 *db, const Handle &handle, sqlite3_int64 s std::vector ReadStagingRange(sqlite3 *db, const Handle &handle, sqlite3_int64 offset, sqlite3_int64 length) { - Statement statement(db, - "SELECT CASE WHEN ?2>=logical_size THEN X'' " - "ELSE substr(d.content,?2+1,min(?3,logical_size-?2)) END " - "FROM _vexfs_staging s JOIN _vexfs_staging_data d ON d.handle_id=s.handle_id " - "WHERE s.handle_id=?1"); - statement.BindText(1, handle.id); - statement.BindInt64(2, offset); - statement.BindInt64(3, length); - if (!statement.Row()) throw SqlError("staging area not found", SQLITE_NOTFOUND); - return statement.Blob(0); + if (offset < 0 || length < 0 || offset > kMaxStagedBytes || + length > kMaxStagedBytes) { + throw SqlError("read range must be within 0..128 MiB", SQLITE_RANGE); + } + const StagingInfo staging = FindStaging(db, handle); + if (length == 0 || offset >= staging.logical_size) return {}; + const sqlite3_int64 bytes = std::min( + length, staging.logical_size - offset); + std::vector result(static_cast(bytes)); + sqlite3_int64 copied = 0; + while (copied < bytes) { + const sqlite3_int64 absolute = offset + copied; + const sqlite3_int64 chunk_no = absolute / kContentChunkBytes; + const sqlite3_int64 within = absolute % kContentChunkBytes; + const sqlite3_int64 take = std::min( + bytes - copied, kContentChunkBytes - within); + const std::vector chunk = + ReadEffectiveChunk(db, handle, staging, chunk_no); + std::memcpy(result.data() + copied, chunk.data() + within, + static_cast(take)); + copied += take; + } + return result; +} + +bool FindActiveMountStaging(sqlite3 *db, sqlite3_int64 workspace_id, + sqlite3_int64 inode_id, Handle *result) { + Statement statement(db, R"SQL( +SELECT h.id,h.workspace_id,h.inode_id,h.writable,h.expected_version, + h.dirty_generation,h.published_generation,h.state, + COALESCE(h.owner_session,'') +FROM _vexfs_handles h +JOIN _vexfs_mount_sessions session + ON session.workspace_id=h.workspace_id AND session.session_id=h.owner_session +WHERE h.workspace_id=?1 AND h.inode_id=?2 AND h.writable=1 + AND h.state='open' AND h.dirty_generation>h.published_generation + AND session.lease_until>CAST(unixepoch('subsec')*1000 AS INTEGER) +ORDER BY h.updated_at DESC,h.id DESC +LIMIT 1 +)SQL"); + statement.BindInt64(1, workspace_id); + statement.BindInt64(2, inode_id); + if (!statement.Row()) return false; + *result = {statement.Text(0), statement.Int64(1), statement.Int64(2), + statement.Int(3) != 0, statement.Int64(4), statement.Int64(5), + statement.Int64(6), statement.Text(7), statement.Text(8)}; + return true; } Workspace WorkspaceById(sqlite3 *db, sqlite3_int64 id) { @@ -2295,11 +2474,12 @@ Node NodeById(sqlite3 *db, sqlite3_int64 id, bool include_deleted = false) { sqlite3_int64 StoreVersionFromStaging(sqlite3 *db, const Workspace &workspace, const Node &node, const Handle &handle, sqlite3_int64 generation, - const std::string &message) { + const std::string &message, + bool transaction_owned = false) { // PublishHandleInTransaction always owns the surrounding savepoint. Keeping // another savepoint here added two SQL statements to every small-file close // without adding another rollback boundary. - AcquireWriteLock(db); + if (!transaction_owned) AcquireWriteLock(db); const StagingInfo staging = FindStaging(db, handle); if (staging.generation != generation) { throw SqlError("staged generation is stale or missing", SQLITE_NOTFOUND); @@ -2321,10 +2501,12 @@ sqlite3_int64 StoreVersionFromStaging(sqlite3 *db, const Workspace &workspace, if (sqlite3_changes64(db) != 1) { throw SqlError("version conflict: file changed before publish", SQLITE_CONSTRAINT); } - const sqlite3_int64 commit = CreateCommit(db, workspace, message); + const sqlite3_int64 commit = transaction_owned + ? CreateCommitInTransaction(db, workspace, message) + : CreateCommit(db, workspace, message); std::string checksum; const sqlite3_int64 manifest_id = StoreManifestFromStaging( - db, workspace.id, node.id, previous_manifest, staging, &checksum); + db, workspace.id, node.id, previous_manifest, handle, staging, &checksum); Statement insert(db, R"SQL( INSERT INTO _vexfs_file_versions( inode_id,version_no,commit_id,manifest_id,size,checksum) @@ -2338,11 +2520,23 @@ VALUES(?1,?2,?3,?4,?5,?6) insert.BindText(6, checksum); insert.Done(); UpdateGrepIndexFromVersion(db, workspace.id, node.id, version); + Statement rebase(db, + "UPDATE _vexfs_staging SET base_manifest_id=?1,base_size=logical_size," + "base_visible_size=logical_size,capacity=0," + "updated_at=CAST(strftime('%s','now') AS INTEGER)*1000 WHERE handle_id=?2"); + rebase.BindInt64(1, manifest_id); + rebase.BindText(2, handle.id); + rebase.Done(); + Statement clear_overlay(db, + "DELETE FROM _vexfs_staging_chunks WHERE handle_id=?1"); + clear_overlay.BindText(1, handle.id); + clear_overlay.Done(); return version; } sqlite3_int64 PublishHandleInTransaction(sqlite3 *db, Handle handle, - sqlite3_int64 generation) { + sqlite3_int64 generation, + bool transaction_owned = false) { if (handle.state == "closed") throw SqlError("handle is closed", SQLITE_NOTFOUND); if (generation <= handle.published_generation) { return NodeById(db, handle.inode_id, true).version; @@ -2357,7 +2551,7 @@ sqlite3_int64 PublishHandleInTransaction(sqlite3 *db, Handle handle, } Workspace workspace = WorkspaceById(db, handle.workspace_id); const sqlite3_int64 version = StoreVersionFromStaging( - db, workspace, node, handle, generation, "handle publish"); + db, workspace, node, handle, generation, "handle publish", transaction_owned); Statement update(db, "UPDATE _vexfs_handles SET expected_version = ?1, published_generation = ?2, " @@ -2635,10 +2829,43 @@ void ReadFunction(sqlite3_context *context, int, sqlite3_value **values) { EnsureSchema(db); const Workspace workspace = FindWorkspace(db, RequiredText(values[0], "workspace")); const Node node = Resolve(db, workspace, PathParts(RequiredText(values[1], "path"))); + // A mounted workspace stores its mutable head in staging and seals version + // history in batches. Other readers must still observe NFS FILE_SYNC + // writes immediately. Ordinary SDK handles remain private because they + // do not own the active mount session. + Handle mounted; + if (node.kind == "file" && + FindActiveMountStaging(db, workspace.id, node.id, &mounted)) { + ResultBlob(context, ReadStagingRange(db, mounted, 0, kMaxStagedBytes)); + return; + } ResultBlob(context, ReadNode(db, node)); }); } +void ReadRangeFunction(sqlite3_context *context, int, sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + const Workspace workspace = FindWorkspace(db, RequiredText(values[0], "workspace")); + const Node node = Resolve(db, workspace, PathParts(RequiredText(values[1], "path"))); + const sqlite3_int64 offset = RequiredNonnegativeInteger(values[2], "offset"); + const sqlite3_int64 length = RequiredNonnegativeInteger(values[3], "length"); + if (node.kind != "file") throw SqlError("path is not a file", SQLITE_MISMATCH); + Handle mounted; + if (length != 0 && + FindActiveMountStaging(db, workspace.id, node.id, &mounted)) { + ResultBlob(context, ReadStagingRange(db, mounted, offset, length)); + return; + } + if (node.version == 0 || length == 0 || offset >= node.size) { + ResultBlob(context, std::vector{}); + return; + } + ResultBlob(context, ReadVersionRange(db, node.id, node.version, offset, length)); + }); +} + void GrepFunction(sqlite3_context *context, int, sqlite3_value **values) { Guard(context, [&] { sqlite3 *db = sqlite3_context_db_handle(context); @@ -3670,10 +3897,15 @@ WHERE manifest.id IS NULL OR chunk.id IS NULL { Statement rows(db, R"SQL( SELECT h.id,h.inode_id,h.state,h.dirty_generation,h.published_generation, - s.generation,s.logical_size,s.capacity,length(d.content) + s.generation,s.base_manifest_id,s.base_size,s.base_visible_size, + s.logical_size,s.capacity,dirty.chunk_count,dirty.chunk_bytes,dirty.max_chunk FROM _vexfs_handles h LEFT JOIN _vexfs_staging s ON s.handle_id=h.id -LEFT JOIN _vexfs_staging_data d ON d.handle_id=h.id +LEFT JOIN ( + SELECT handle_id,count(*) chunk_count,COALESCE(sum(length(content)),0) chunk_bytes, + max(chunk_no) max_chunk + FROM _vexfs_staging_chunks GROUP BY handle_id +) dirty ON dirty.handle_id=h.id WHERE h.workspace_id=?1 )SQL"); rows.BindInt64(1, workspace.id); @@ -3694,18 +3926,31 @@ WHERE h.workspace_id=?1 } if (active) { ++report.staging_objects; - if (rows.Type(5) == SQLITE_NULL || rows.Type(8) == SQLITE_NULL) { + const sqlite3_int64 dirty_chunks = + rows.Type(11) == SQLITE_NULL ? 0 : rows.Int64(11); + const sqlite3_int64 dirty_bytes = + rows.Type(12) == SQLITE_NULL ? 0 : rows.Int64(12); + const sqlite3_int64 expected_chunks = rows.Type(9) == SQLITE_NULL ? 0 : + (rows.Int64(9) + kContentChunkBytes - 1) / kContentChunkBytes; + if (rows.Type(5) == SQLITE_NULL || + (rows.Int64(10) > 0 && rows.Type(11) == SQLITE_NULL)) { report.Add("VEXFS_STAGING_MISSING", object, "staging", "open or retained handle has no complete staging object", "discard the handle; recover unpublished work from backup if available"); - } else if (rows.Int64(5) != rows.Int64(3) || rows.Int64(6) < 0 || - rows.Int64(7) < rows.Int64(6) || - rows.Int64(7) > kMaxStagedBytes || rows.Int64(8) != rows.Int64(7)) { + } else if (rows.Int64(5) != rows.Int64(3) || rows.Int64(7) < 0 || + rows.Int64(8) < 0 || rows.Int64(8) > rows.Int64(7) || + rows.Int64(8) > rows.Int64(9) || rows.Int64(9) < 0 || + rows.Int64(9) > kMaxStagedBytes || rows.Int64(10) < 0 || + rows.Int64(10) > kMaxStagedBytes || + (rows.Int64(7) > 0 && + (rows.Type(6) == SQLITE_NULL || rows.Int64(6) <= 0)) || + dirty_bytes != rows.Int64(10) || + (dirty_chunks > 0 && rows.Int64(13) >= expected_chunks)) { report.Add("VEXFS_STAGING_INVALID", object, "staging", "staging generation, size, or capacity is inconsistent", "discard the handle; recover unpublished work from backup if available"); } - } else if (rows.Type(5) != SQLITE_NULL || rows.Type(8) != SQLITE_NULL) { + } else if (rows.Type(5) != SQLITE_NULL || rows.Type(11) != SQLITE_NULL) { report.Add("VEXFS_CLOSED_HANDLE_STAGING", object, "staging", "closed handle still owns staging data", "run a future repair or garbage-collection command after backing up"); @@ -3722,7 +3967,7 @@ WHERE h.workspace_id=?1 "run a future repair or garbage-collection command after backing up"); } Statement orphan_data(db, - "SELECT d.handle_id FROM _vexfs_staging_data d LEFT JOIN _vexfs_staging s " + "SELECT d.handle_id FROM _vexfs_staging_chunks d LEFT JOIN _vexfs_staging s " "ON s.handle_id=d.handle_id WHERE s.handle_id IS NULL"); while (orphan_data.Row()) { report.Add("VEXFS_ORPHAN_STAGING_DATA", "handle:" + orphan_data.Text(0), @@ -5684,13 +5929,22 @@ void HandleCreateFunction(sqlite3_context *context, int argument_count, const std::string workspace_name = RequiredText(values[0], "workspace"); const std::string path = RequiredText(values[1], "path"); const int mode = RequiredMode(values[2]); - const std::string request_id = RequiredText(values[3], "request_id"); - const std::string owner_session = argument_count == 5 - ? RequiredText(values[4], "owner_session") : std::string(); + const bool owned_create = argument_count == 7; + const sqlite3_int64 uid = owned_create + ? RequiredOwnerId(values[3], "uid") : 0; + const sqlite3_int64 gid = owned_create + ? RequiredOwnerId(values[4], "gid") : 0; + const int request_index = owned_create ? 5 : 3; + const int owner_session_index = owned_create ? 6 : 4; + const std::string request_id = RequiredText(values[request_index], "request_id"); + const std::string owner_session = argument_count == 5 || owned_create + ? RequiredText(values[owner_session_index], "owner_session") : std::string(); RequestFingerprint fingerprint_builder("handle_create"); fingerprint_builder.AddText(workspace_name); fingerprint_builder.AddText(path); fingerprint_builder.AddInt64(mode); + fingerprint_builder.AddInt64(uid); + fingerprint_builder.AddInt64(gid); fingerprint_builder.AddText(owner_session); const auto fingerprint = fingerprint_builder.Finish(); const CachedRequest cached = FindRequest(db, request_id, "handle_create", fingerprint); @@ -5711,6 +5965,15 @@ void HandleCreateFunction(sqlite3_context *context, int argument_count, throw SqlError("destination already exists", SQLITE_CONSTRAINT); } const sqlite3_int64 inode = CreateInode(db, workspace.id, "file", mode); + if (owned_create) { + Statement owner(db, + "UPDATE _vexfs_inodes SET uid=?1,gid=?2 WHERE workspace_id=?3 AND id=?4"); + owner.BindInt64(1, uid); + owner.BindInt64(2, gid); + owner.BindInt64(3, workspace.id); + owner.BindInt64(4, inode); + owner.Done(); + } AddDentry(db, workspace.id, parent.id, name, inode); const std::string handle_id = RandomHandleId(); @@ -5721,7 +5984,7 @@ void HandleCreateFunction(sqlite3_context *context, int argument_count, insert.BindText(1, handle_id); insert.BindInt64(2, workspace.id); insert.BindInt64(3, inode); - if (argument_count == 5) insert.BindText(4, owner_session); + if (argument_count == 5 || owned_create) insert.BindText(4, owner_session); else insert.BindNull(4); insert.Done(); @@ -5879,7 +6142,7 @@ void HandleAppendFunction(sqlite3_context *context, int, sqlite3_value **values) // reporting a normal write conflict. { Statement remove_data(db, - "DELETE FROM _vexfs_staging_data WHERE handle_id=?1"); + "DELETE FROM _vexfs_staging_chunks WHERE handle_id=?1"); remove_data.BindText(1, handle.id); remove_data.Done(); Statement remove_meta(db, @@ -5988,7 +6251,7 @@ void HandlePublishCloseFunction(sqlite3_context *context, int, sqlite3_value **v "updated_at=CAST(strftime('%s','now') AS INTEGER)*1000 WHERE id=?1"); update.BindText(1, handle_id); update.Done(); - Statement clear_data(db, "DELETE FROM _vexfs_staging_data WHERE handle_id=?1"); + Statement clear_data(db, "DELETE FROM _vexfs_staging_chunks WHERE handle_id=?1"); clear_data.BindText(1, handle_id); clear_data.Done(); Statement clear(db, "DELETE FROM _vexfs_staging WHERE handle_id=?1"); @@ -6028,7 +6291,7 @@ void HandleCloseFunction(sqlite3_context *context, int, sqlite3_value **values) update.BindText(3, handle.id); update.Done(); if (!retain) { - Statement clear_data(db, "DELETE FROM _vexfs_staging_data WHERE handle_id=?1"); + Statement clear_data(db, "DELETE FROM _vexfs_staging_chunks WHERE handle_id=?1"); clear_data.BindText(1, handle.id); clear_data.Done(); Statement clear(db, "DELETE FROM _vexfs_staging WHERE handle_id=?1"); @@ -6057,7 +6320,7 @@ void RetainOpenHandles(sqlite3 *db, const Workspace &workspace, update.BindText(2, session_id); update.Done(); Statement clear_data(db, - "DELETE FROM _vexfs_staging_data WHERE handle_id IN " + "DELETE FROM _vexfs_staging_chunks WHERE handle_id IN " "(SELECT id FROM _vexfs_handles WHERE workspace_id=?1 AND state='closed')"); clear_data.BindInt64(1, workspace.id); clear_data.Done(); @@ -6197,6 +6460,58 @@ void SynchronizeFunction(sqlite3_context *context, int argument_count, sqlite3_v }); } +void PublishCloseAllFunction(sqlite3_context *context, int argument_count, + sqlite3_value **values) { + Guard(context, [&] { + sqlite3 *db = sqlite3_context_db_handle(context); + EnsureSchema(db); + const Workspace workspace = FindWorkspace( + db, RequiredText(values[0], "workspace")); + const std::string owner_session = RequiredText(values[1], "owner_session"); + if (owner_session.empty()) { + throw SqlError("owner_session must not be empty", SQLITE_MISMATCH); + } + const sqlite3_int64 limit = argument_count == 3 + ? sqlite3_value_int64(values[2]) : 0; + if (limit < 0) { + throw SqlError("limit must be zero or positive", SQLITE_RANGE); + } + Savepoint savepoint(db, "vexfs_publish_close_all"); + AcquireWriteLock(db); + Statement query(db, + "SELECT id,dirty_generation FROM _vexfs_handles " + "WHERE workspace_id=?1 AND owner_session=?2 AND state='open' " + "AND writable=1 AND dirty_generation>published_generation ORDER BY id " + "LIMIT ?3"); + query.BindInt64(1, workspace.id); + query.BindText(2, owner_session); + query.BindInt64(3, limit == 0 ? -1 : limit); + std::vector> pending; + while (query.Row()) pending.emplace_back(query.Text(0), query.Int64(1)); + for (const auto &item : pending) { + // The batch already owns one rollback boundary. Avoid two extra + // SAVEPOINT statements per file while keeping the whole batch + // atomic if any publish fails. + PublishHandleInTransaction( + db, FindHandle(db, item.first), item.second, true); + Statement close(db, + "UPDATE _vexfs_handles SET state='closed',lease_until=NULL," + "updated_at=CAST(strftime('%s','now') AS INTEGER)*1000 WHERE id=?1"); + close.BindText(1, item.first); + close.Done(); + Statement clear_data(db, + "DELETE FROM _vexfs_staging_chunks WHERE handle_id=?1"); + clear_data.BindText(1, item.first); + clear_data.Done(); + Statement clear(db, "DELETE FROM _vexfs_staging WHERE handle_id=?1"); + clear.BindText(1, item.first); + clear.Done(); + } + savepoint.Release(); + sqlite3_result_int64(context, static_cast(pending.size())); + }); +} + void ReclaimFunction(sqlite3_context *context, int, sqlite3_value **values) { Guard(context, [&] { sqlite3 *db = sqlite3_context_db_handle(context); @@ -6218,7 +6533,7 @@ void ReclaimFunction(sqlite3_context *context, int, sqlite3_value **values) { std::vector handles; while (query.Row()) handles.push_back(query.Text(0)); for (const std::string &handle : handles) { - Statement clear_data(db, "DELETE FROM _vexfs_staging_data WHERE handle_id=?1"); + Statement clear_data(db, "DELETE FROM _vexfs_staging_chunks WHERE handle_id=?1"); clear_data.BindText(1, handle); clear_data.Done(); Statement clear(db, "DELETE FROM _vexfs_staging WHERE handle_id=?1"); @@ -6260,6 +6575,7 @@ extern "C" int vexfs_sqlite_register(sqlite3 *db) { {"vexfs_write", 3, WriteFunction, SQLITE_UTF8}, {"vexfs_append", 3, AppendFunction, SQLITE_UTF8}, {"vexfs_read", 2, ReadFunction, SQLITE_UTF8}, + {"vexfs_read_range", 4, ReadRangeFunction, SQLITE_UTF8}, {"vexfs_grep", 5, GrepFunction, SQLITE_UTF8}, {"vexfs_grep_index", 1, GrepIndexFunction, SQLITE_UTF8}, {"vexfs_history", 2, HistoryFunction, SQLITE_UTF8}, @@ -6306,6 +6622,7 @@ extern "C" int vexfs_sqlite_register(sqlite3 *db) { {"vexfs_remove", 3, RemoveFunction, SQLITE_UTF8}, {"vexfs_handle_create", 4, HandleCreateFunction, SQLITE_UTF8}, {"vexfs_handle_create", 5, HandleCreateFunction, SQLITE_UTF8}, + {"vexfs_handle_create", 7, HandleCreateFunction, SQLITE_UTF8}, {"vexfs_handle_open", 4, HandleOpenFunction, SQLITE_UTF8}, {"vexfs_handle_open", 5, HandleOpenFunction, SQLITE_UTF8}, {"vexfs_handle_stage_write", 4, HandleStageWriteFunction, SQLITE_UTF8}, @@ -6320,6 +6637,8 @@ extern "C" int vexfs_sqlite_register(sqlite3 *db) { {"vexfs_mount_session_end", 2, SessionEndFunction, SQLITE_UTF8}, {"vexfs_mount_synchronize", 2, SynchronizeFunction, SQLITE_UTF8}, {"vexfs_mount_synchronize", 3, SynchronizeFunction, SQLITE_UTF8}, + {"vexfs_mount_publish_close_all", 2, PublishCloseAllFunction, SQLITE_UTF8}, + {"vexfs_mount_publish_close_all", 3, PublishCloseAllFunction, SQLITE_UTF8}, {"vexfs_item_reclaim", 2, ReclaimFunction, SQLITE_UTF8}, }; // sqlite3_create_function_v2 exists throughout the supported SQLite range. Its diff --git a/vexdb_sqlite/test/vexfs_static_smoke.c b/vexdb_sqlite/test/vexfs_static_smoke.c index 7fd524e48a..f625668c68 100644 --- a/vexdb_sqlite/test/vexfs_static_smoke.c +++ b/vexdb_sqlite/test/vexfs_static_smoke.c @@ -155,7 +155,7 @@ int main(void) { free(text); free(handle); - // 暂存区每个 handle 只有一行；跨 64 KiB 写入按容量扩展，并补零空洞。 + // 暂存元数据每个 handle 只有一行；稀疏写只保存碰到的 64 KiB 块，并补零空洞。 if (!exec_ok(db, "SELECT vexfs_write('default','/notes/grow.bin',X'');")) return fail(db, "grow seed"); handle = scalar_text(db, @@ -170,7 +170,7 @@ int main(void) { if (scalar_int(db, sql, &ok) != 1 || !ok) return fail(db, "bounded staging rows"); snprintf(sql, sizeof(sql), "SELECT capacity FROM _vexfs_staging WHERE handle_id='%s'", handle); - if (scalar_int(db, sql, &ok) != 131072 || !ok) return fail(db, "geometric staging capacity"); + if (scalar_int(db, sql, &ok) != 4465 || !ok) return fail(db, "dirty chunk capacity"); snprintf(sql, sizeof(sql), "SELECT hex(vexfs_handle_read('%s',69999,2))", handle); text = scalar_text(db, sql); From 9bfe48a883c3d6d745be1af4a74d9d6fe7479d2a Mon Sep 17 00:00:00 2001 From: Four <8122332+HungryFour@users.noreply.github.com> Date: Mon, 27 Jul 2026 13:56:15 +0800 Subject: [PATCH 14/28] Optimize PostgreSQL VexFS staged publishing --- agent_files/cli/vexfs_platform_macos.cpp | 14 +- .../mount/common/include/vexfs_runtime_io.h | 41 + .../src/vexfs_mount_contract_sqlite.cpp | 481 +++++- .../test/vexfs_mount_contract_smoke.cpp | 63 +- .../common/test/vexfs_pg_runtime_smoke.cpp | 149 +- .../mount/macos/nfs_gateway/src/lib.rs | 1095 +++++++++++-- .../vendor/nfsserve/src/nfs_handlers.rs | 14 +- .../nfs_gateway/vendor/nfsserve/src/vfs.rs | 12 +- ...2026-07-21_vexfs-final-goal-and-roadmap.md | 57 +- tests/eval/vexfs/README.md | 31 +- tests/eval/vexfs/benchmark_mount.py | 95 ++ tests/eval/vexfs/run.py | 430 ++++- tests/eval/vexfs/run_pg_macos_mount.sh | 10 +- tests/spec/_lib/docker/run_sqlite.sh | 37 +- .../pg/agent_files/vexfs_pg_chunk_check.yaml | 36 +- .../agent_files/vexfs_pg_handles_gateway.yaml | 48 + .../agent_files/vexfs_pg_staging_overlay.yaml | 108 ++ .../sqlite/agent_files/vexfs_contract.yaml | 4 + vexdb_pg/sql/vexdb_lite--1.0.sql | 1434 ++++++++++++++++- 19 files changed, 3832 insertions(+), 327 deletions(-) create mode 100644 tests/eval/vexfs/benchmark_mount.py create mode 100644 tests/spec/pg/agent_files/vexfs_pg_staging_overlay.yaml diff --git a/agent_files/cli/vexfs_platform_macos.cpp b/agent_files/cli/vexfs_platform_macos.cpp index 0227bd4c32..ccb66ae7c3 100644 --- a/agent_files/cli/vexfs_platform_macos.cpp +++ b/agent_files/cli/vexfs_platform_macos.cpp @@ -527,7 +527,10 @@ void StopNfsGateway(const std::string &mount_point) { NfsGatewayRecord record; if (!ReadNfsGatewayRecord(mount_point, &record) || !ProcessMatchesGateway(record)) return; kill(record.pid, SIGTERM); - for (int attempt = 0; attempt < 40; ++attempt) { + // SIGTERM starts a graceful database-backed staging flush. Large small-file + // bursts are committed in bounded batches, so allow them to finish before + // falling back to SIGKILL. The filesystem is already detached at this point. + for (int attempt = 0; attempt < 600; ++attempt) { if (kill(record.pid, 0) != 0 && errno == ESRCH) return; usleep(50 * 1000); } @@ -1028,11 +1031,12 @@ int VexFSPlatformMount(const std::string &backend, const std::string &connection gateway = StartNfsGateway(backend, resource.connection, workspace, mount_point, resource.directory); } - // Keep I/O bounded if the per-mount gateway exits, but allow one - // request to wait longer than the runtime's 30-second database - // busy timeout. dumbtimer makes the 10-second interval explicit. + // Keep I/O bounded if the per-mount gateway exits. Keep the normal + // adaptive RTT estimator: dumbtimer makes sparse fsync workloads + // wait one fixed timeout for every dirty NFS block on macOS. const std::string options = - "vers=3,tcp,locallocks,soft,dumbtimer,timeo=100,retrans=4," + "vers=3,tcp,locallocks,soft,timeo=10,retrans=4," + "rsize=1048576,wsize=1048576," "deadtimeout=60,actimeo=1,port=" + std::to_string(gateway.port) + ",mountport=" + std::to_string(gateway.port); diff --git a/agent_files/mount/common/include/vexfs_runtime_io.h b/agent_files/mount/common/include/vexfs_runtime_io.h index 7af7df9725..fa3d15052e 100644 --- a/agent_files/mount/common/include/vexfs_runtime_io.h +++ b/agent_files/mount/common/include/vexfs_runtime_io.h @@ -36,12 +36,24 @@ vexfs_mount_status vexfs_mount_readlink(vexfs_mount_session *session, int64_t in vexfs_mount_status vexfs_mount_write_file(vexfs_mount_session *session, const char *path, const void *data, uint64_t size, int64_t *version, vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_write_file_range(vexfs_mount_session *session, + const char *path, uint64_t offset, + const void *data, uint64_t size, + const char *request_id, + const char *durability, + int64_t *version, + vexfs_mount_error *error); vexfs_mount_status vexfs_mount_append_file(vexfs_mount_session *session, int64_t inode, const void *data, uint64_t size, int64_t *version, vexfs_mount_error *error); vexfs_mount_status vexfs_mount_read_file(vexfs_mount_session *session, const char *path, vexfs_mount_bytes *content, vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_read_file_range(vexfs_mount_session *session, + const char *path, uint64_t offset, + uint64_t length, + vexfs_mount_bytes *content, + vexfs_mount_error *error); vexfs_mount_status vexfs_mount_stat(vexfs_mount_session *session, const char *path, vexfs_mount_bytes *json, vexfs_mount_error *error); vexfs_mount_status vexfs_mount_path_for_inode(vexfs_mount_session *session, int64_t inode, @@ -97,12 +109,28 @@ vexfs_mount_status vexfs_mount_handle_create(vexfs_mount_session *session, const char *request_id, vexfs_mount_bytes *handle, vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_handle_create_owned_durable( + vexfs_mount_session *session, const char *path, uint32_t mode, + int64_t uid, int64_t gid, const char *request_id, + const char *durability, vexfs_mount_bytes *handle, + vexfs_mount_error *error); +// 创建文件时同时返回 stat。PostgreSQL adapter 会在同一条 SQL 内完成， +// 避免远程挂载在 CREATE 后再发一次 vexfs_stat 往返。 +vexfs_mount_status vexfs_mount_handle_create_owned_stat_durable( + vexfs_mount_session *session, const char *path, uint32_t mode, + int64_t uid, int64_t gid, const char *request_id, + const char *durability, vexfs_mount_bytes *handle, + vexfs_mount_bytes *stat_json, vexfs_mount_error *error); vexfs_mount_status vexfs_mount_handle_stage_write(vexfs_mount_session *session, const char *handle, uint64_t offset, const void *data, uint64_t size, const char *request_id, int64_t *generation, vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_handle_stage_write_durable( + vexfs_mount_session *session, const char *handle, uint64_t offset, + const void *data, uint64_t size, const char *request_id, + const char *durability, int64_t *generation, vexfs_mount_error *error); vexfs_mount_status vexfs_mount_handle_append(vexfs_mount_session *session, const char *handle, const void *data, uint64_t size, const char *request_id, @@ -136,6 +164,19 @@ vexfs_mount_status vexfs_mount_handle_close(vexfs_mount_session *session, vexfs_mount_status vexfs_mount_synchronize(vexfs_mount_session *session, const char *request_id, int64_t *published, vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_publish_close_all( + vexfs_mount_session *session, const char *durability, + int64_t *published, vexfs_mount_error *error); +vexfs_mount_status vexfs_mount_publish_close_batch( + vexfs_mount_session *session, const char *durability, int64_t max_count, + int64_t *published, vexfs_mount_error *error); +// PostgreSQL-only background path. It uses a dedicated libpq connection and +// publishes exactly the handle generations encoded in claims_json, so callers +// do not need to hold the gateway-wide RuntimeState mutex during database I/O. +vexfs_mount_status vexfs_mount_publish_close_claimed( + vexfs_mount_session *session, const char *durability, + const char *claims_json, vexfs_mount_bytes *result_json, + vexfs_mount_error *error); vexfs_mount_status vexfs_mount_reclaim(vexfs_mount_session *session, const char *request_id, int64_t *reclaimed, vexfs_mount_error *error); diff --git a/agent_files/mount/common/src/vexfs_mount_contract_sqlite.cpp b/agent_files/mount/common/src/vexfs_mount_contract_sqlite.cpp index dacbc6f8d3..befaabeeee 100644 --- a/agent_files/mount/common/src/vexfs_mount_contract_sqlite.cpp +++ b/agent_files/mount/common/src/vexfs_mount_contract_sqlite.cpp @@ -34,6 +34,7 @@ struct vexfs_mount_session { sqlite3 *db = nullptr; #if defined(VEXFS_HAVE_LIBPQ) PGconn *pg = nullptr; + PGconn *publisher_pg = nullptr; #endif bool postgresql = false; std::string workspace; @@ -42,7 +43,8 @@ struct vexfs_mount_session { bool exclusive_gateway = false; bool session_started = false; bool no_create = false; - bool synchronous_full = true; + // SQLite PRAGMA synchronous: 0=OFF, 1=NORMAL, 2=FULL. + int synchronous_mode = 2; uint32_t operation_timeout_ms = 0; std::chrono::steady_clock::time_point next_heartbeat{}; std::chrono::steady_clock::time_point next_visibility_poll{}; @@ -58,6 +60,7 @@ struct vexfs_mount_session { uint64_t observed_mutation_epoch = 0; bool visibility_initialized = false; std::recursive_mutex mutex; + std::mutex publisher_mutex; }; namespace { @@ -76,6 +79,7 @@ constexpr int kPgNotEmpty = -1109; #if defined(VEXFS_HAVE_LIBPQ) void EnsurePostgreSQLConnection(vexfs_mount_session *session); +void EnsurePostgreSQLPublisherConnection(vexfs_mount_session *session); #endif class CallError : public std::runtime_error { @@ -107,6 +111,16 @@ class Call { db_ = session->db; PrepareSQLite(sql); } +#if defined(VEXFS_HAVE_LIBPQ) + Call(PGconn *connection, const char *sql) { + if (connection == nullptr || PQstatus(connection) != CONNECTION_OK) { + throw CallError(kPgDatabaseError, "PostgreSQL connection is not available", + VEXFS_RUNTIME_BACKEND_POSTGRESQL); + } + pg_ = connection; + PreparePostgreSQL(sql); + } +#endif ~Call() { if (statement_ != nullptr) sqlite3_finalize(statement_); #if defined(VEXFS_HAVE_LIBPQ) @@ -209,18 +223,20 @@ class Call { #endif return sqlite3_column_int64(statement_, 0); } - std::string ResultText() const { + std::string ResultText(int column = 0) const { #if defined(VEXFS_HAVE_LIBPQ) if (pg_ != nullptr) { if (pg_result_ == nullptr || PQntuples(pg_result_) < 1 || - PQnfields(pg_result_) < 1 || PQgetisnull(pg_result_, 0, 0)) { + column < 0 || column >= PQnfields(pg_result_) || + PQgetisnull(pg_result_, 0, column)) { return std::string(); } - return std::string(PQgetvalue(pg_result_, 0, 0), - static_cast(PQgetlength(pg_result_, 0, 0))); + return std::string(PQgetvalue(pg_result_, 0, column), + static_cast(PQgetlength(pg_result_, 0, column))); } #endif - const unsigned char *value = sqlite3_column_text(statement_, 0); + if (column < 0 || column >= sqlite3_column_count(statement_)) return std::string(); + const unsigned char *value = sqlite3_column_text(statement_, column); return value == nullptr ? std::string() : std::string(reinterpret_cast(value)); } @@ -458,6 +474,32 @@ void EnsurePostgreSQLConnection(vexfs_mount_session *session) { std::chrono::steady_clock::now() + std::chrono::seconds(10); } } + +void EnsurePostgreSQLPublisherConnection(vexfs_mount_session *session) { + if (session == nullptr || !session->postgresql || session->publisher_pg == nullptr) { + throw CallError(kPgDatabaseError, + "PostgreSQL publisher connection is not available", + VEXFS_RUNTIME_BACKEND_POSTGRESQL); + } + if (PQstatus(session->publisher_pg) == CONNECTION_OK) return; + + // As with the foreground connection, never replay a command whose commit + // result is unknown. The gateway keeps the claimed generations and retries + // them only after the database reports their current state. + PQreset(session->publisher_pg); + if (PQstatus(session->publisher_pg) != CONNECTION_OK) { + throw CallError(kPgDatabaseError, PQerrorMessage(session->publisher_pg), + VEXFS_RUNTIME_BACKEND_POSTGRESQL); + } + if (session->operation_timeout_ms != 0) { + const std::string command = "SET statement_timeout TO " + + std::to_string(session->operation_timeout_ms); + RequirePostgreSQLCommand( + session->publisher_pg, + PQexec(session->publisher_pg, command.c_str()), + "restore publisher statement timeout"); + } +} #endif void ClearError(vexfs_mount_error *error, @@ -546,6 +588,31 @@ vexfs_mount_status Guard(vexfs_mount_session *session, vexfs_mount_error *error, return status; } +#if defined(VEXFS_HAVE_LIBPQ) +template +vexfs_mount_status GuardPublisher(vexfs_mount_session *session, + vexfs_mount_error *error, + Function function) { + if (error != nullptr) { + ClearError(error, VEXFS_RUNTIME_BACKEND_POSTGRESQL); + } + const vexfs_mount_status status = Guard(error, [&] { + if (session == nullptr || !session->postgresql) { + throw CallError(kUnsupportedBackend, + "background publisher requires PostgreSQL", + VEXFS_RUNTIME_BACKEND_POSTGRESQL); + } + std::lock_guard lock(session->publisher_mutex); + EnsurePostgreSQLPublisherConnection(session); + function(); + }); + if (status == VEXFS_MOUNT_OK && error != nullptr) { + ClearError(error, VEXFS_RUNTIME_BACKEND_POSTGRESQL); + } + return status; +} +#endif + void RequireSession(vexfs_mount_session *session) { if (session == nullptr || (session->db == nullptr @@ -632,18 +699,33 @@ void PreparePrivateDatabase(const char *path, bool no_create) { } void ProtectSidecar(const std::string &path) { - const int descriptor = OpenPrivateFile(path.c_str(), true); - if (descriptor < 0) { + // Never open and close a WAL/SHM sidecar while SQLite owns the database. + // POSIX drops every fcntl lock that this process holds on a file when any + // descriptor for that file is closed. Opening -shm here used to release + // SQLite's WAL locks, allowing a peer process to truncate the still-mapped + // wal-index and crash the gateway with SIGBUS. + struct stat info {}; + if (lstat(path.c_str(), &info) != 0) { if (errno == ENOENT) return; - throw CallError(SQLITE_AUTH, "SQLite sidecar is not a private regular file: " + path); + throw CallError(SQLITE_AUTH, + "cannot inspect SQLite sidecar: " + path + ": " + + std::strerror(errno)); } - try { - ValidatePrivateFileDescriptor(descriptor, path, true); - } catch (...) { - close(descriptor); - throw; + if (S_ISLNK(info.st_mode) || !S_ISREG(info.st_mode)) { + throw CallError(SQLITE_AUTH, + "SQLite sidecar is not a private regular file: " + path); + } + if (info.st_uid != geteuid()) { + throw CallError(SQLITE_AUTH, + "SQLite sidecar is not owned by the current user: " + path); + } + // The database lives in a user-private directory, so a path-based chmod is + // safe here and, unlike open/fchmod/close, cannot disturb SQLite WAL locks. + if ((info.st_mode & 0777) != 0600 && chmod(path.c_str(), 0600) != 0) { + throw CallError(SQLITE_AUTH, + "cannot protect SQLite sidecar: " + path + ": " + + std::strerror(errno)); } - close(descriptor); } #else void PreparePrivateDatabase(const char *, bool) {} @@ -750,22 +832,37 @@ void Exec(sqlite3 *db, const char *sql) { void UseOrdinaryDurability(vexfs_mount_session *session) { ++session->ordinary_mutation_calls; ++session->mutation_epoch; - if (session->postgresql || !session->synchronous_full) return; + if (session->postgresql || session->synchronous_mode == 1) return; Exec(session->db, "PRAGMA synchronous=NORMAL"); - session->synchronous_full = false; + session->synchronous_mode = 1; + ++session->synchronous_mode_switches; +} + +void UseRelaxedDurability(vexfs_mount_session *session) { + ++session->ordinary_mutation_calls; + ++session->mutation_epoch; + if (session->postgresql || session->synchronous_mode == 0) return; + Exec(session->db, "PRAGMA synchronous=OFF"); + session->synchronous_mode = 0; ++session->synchronous_mode_switches; } void UseFullDurability(vexfs_mount_session *session) { ++session->full_boundary_calls; - if (session->postgresql || session->synchronous_full) return; + if (session->postgresql || session->synchronous_mode == 2) return; // fsync/synchronize/snapshot 前切回 FULL。随后必须执行真实写事务；仅修改 // PRAGMA 不会形成持久化屏障。 Exec(session->db, "PRAGMA synchronous=FULL"); - session->synchronous_full = true; + session->synchronous_mode = 2; ++session->synchronous_mode_switches; } +void UseRequestedDurability(vexfs_mount_session *session, const char *durability) { + if (std::strcmp(durability, "full") == 0) UseFullDurability(session); + else if (std::strcmp(durability, "none") == 0) UseRelaxedDurability(session); + else UseOrdinaryDurability(session); +} + void DurabilityBarrier(vexfs_mount_session *session) { if (session->postgresql) { ++session->durability_barriers; @@ -857,6 +954,20 @@ extern "C" vexfs_mount_status vexfs_mount_session_open(const vexfs_mount_config throw CallError(kPgDatabaseError, message, VEXFS_RUNTIME_BACKEND_POSTGRESQL); } + session->publisher_pg = PQconnectdb(config->connection); + if (session->publisher_pg == nullptr + || PQstatus(session->publisher_pg) != CONNECTION_OK) { + const std::string message = session->publisher_pg == nullptr + ? "cannot allocate PostgreSQL publisher connection" + : PQerrorMessage(session->publisher_pg); + if (session->publisher_pg != nullptr) { + PQfinish(session->publisher_pg); + } + PQfinish(session->pg); + delete session; + throw CallError(kPgDatabaseError, message, + VEXFS_RUNTIME_BACKEND_POSTGRESQL); + } try { if (config->operation_timeout_ms != 0) { const std::string command = "SET statement_timeout TO " + @@ -870,6 +981,16 @@ extern "C" vexfs_mount_status vexfs_mount_session_open(const vexfs_mount_config VEXFS_RUNTIME_BACKEND_POSTGRESQL); } PQclear(result); + result = PQexec(session->publisher_pg, command.c_str()); + if (result == nullptr || PQresultStatus(result) != PGRES_COMMAND_OK) { + const std::string message = result == nullptr + ? PQerrorMessage(session->publisher_pg) + : PQresultErrorMessage(result); + if (result != nullptr) PQclear(result); + throw CallError(kPgDatabaseError, message, + VEXFS_RUNTIME_BACKEND_POSTGRESQL); + } + PQclear(result); } Call version(session, "SELECT vexfs_pg_adapter_version()"); version.Row(); @@ -926,6 +1047,7 @@ extern "C" vexfs_mount_status vexfs_mount_session_open(const vexfs_mount_config } catch (...) { } } + PQfinish(session->publisher_pg); PQfinish(session->pg); delete session; throw; @@ -1006,7 +1128,7 @@ extern "C" vexfs_mount_status vexfs_mount_session_open(const vexfs_mount_config // Ordinary filesystem callbacks use WAL/NORMAL. Explicit // synchronize, fsync, snapshot and safe close switch to FULL. Exec(session->db, "PRAGMA synchronous=NORMAL"); - session->synchronous_full = false; + session->synchronous_mode = 1; ++session->synchronous_mode_switches; } } catch (...) { @@ -1038,14 +1160,30 @@ extern "C" void vexfs_mount_session_close(vexfs_mount_session *session) { try { if (!session->no_create) { UseFullDurability(session); - Call synchronize(session, session->session_started - ? "SELECT vexfs_mount_synchronize(?1,'',?2)" - : "SELECT vexfs_mount_synchronize(?1,'')"); - synchronize.Text(1, session->workspace.c_str()); - if (session->session_started) { - synchronize.Text(2, session->session_id.c_str()); + if (session->postgresql && session->session_started) { + // A gateway can own thousands of small staged files. One + // all-in-one synchronize transaction may exceed the normal + // statement timeout and roll every publish back during + // unmount. Commit bounded shutdown batches instead; no NFS + // request can arrive after the volume has been detached. + for (;;) { + Call publish(session, + "SELECT vexfs_mount_publish_close_all(?1,?2,'full',64)"); + publish.Text(1, session->workspace.c_str()); + publish.Text(2, session->session_id.c_str()); + publish.Row(); + if (publish.ResultInt64() == 0) break; + } + } else { + Call synchronize(session, session->session_started + ? "SELECT vexfs_mount_synchronize(?1,'',?2)" + : "SELECT vexfs_mount_synchronize(?1,'')"); + synchronize.Text(1, session->workspace.c_str()); + if (session->session_started) { + synchronize.Text(2, session->session_id.c_str()); + } + synchronize.Row(); } - synchronize.Row(); DurabilityBarrier(session); if (session->session_started) { Call end(session, "SELECT vexfs_mount_session_end(?1,?2)"); @@ -1060,6 +1198,11 @@ extern "C" void vexfs_mount_session_close(vexfs_mount_session *session) { } if (session->db != nullptr) sqlite3_close(session->db); #if defined(VEXFS_HAVE_LIBPQ) + if (session->publisher_pg != nullptr) { + std::lock_guard publisher_lock(session->publisher_mutex); + PQfinish(session->publisher_pg); + session->publisher_pg = nullptr; + } if (session->pg != nullptr) PQfinish(session->pg); #endif } @@ -1315,6 +1458,71 @@ extern "C" vexfs_mount_status vexfs_mount_write_file(vexfs_mount_session *sessio }); } +extern "C" vexfs_mount_status vexfs_mount_write_file_range( + vexfs_mount_session *session, const char *path, uint64_t offset, + const void *data, uint64_t size, const char *request_id, + const char *durability, int64_t *version, vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + if (path == nullptr || request_id == nullptr || durability == nullptr || + version == nullptr || (size > 0 && data == nullptr) || + offset > static_cast(std::numeric_limits::max())) { + throw CallError(SQLITE_MISUSE, + "path, content, request, durability and version are required"); + } + UseRequestedDurability(session, durability); +#if defined(VEXFS_HAVE_LIBPQ) + if (session->postgresql) { + Call call(session, "SELECT vexfs_write_range(?1,?2,?3,?4,?5,?6,?7)"); + call.Text(1, session->workspace.c_str()); + call.Text(2, path); + call.Int64(3, static_cast(offset)); + call.Blob(4, data, size); + call.Text(5, request_id); + call.Text(6, session->session_id.c_str()); + call.Text(7, durability); + call.Row(); + *version = call.ResultInt64(); + return; + } +#endif + const std::string open_request = std::string(request_id) + ":open"; + Call open(session, "SELECT vexfs_handle_open(?1,?2,'rw',?3,?4)"); + open.Text(1, session->workspace.c_str()); + open.Text(2, path); + open.Text(3, EffectiveRequestId(session, open_request.c_str(), true)); + open.Text(4, session->session_id.c_str()); + open.Row(); + const std::string handle = open.ResultText(); + try { + const std::string stage_request = std::string(request_id) + ":stage"; + Call stage(session, "SELECT vexfs_handle_stage_write(?1,?2,?3,?4)"); + stage.Text(1, handle.c_str()); + stage.Int64(2, static_cast(offset)); + stage.Blob(3, data, size); + stage.Text(4, EffectiveRequestId(session, stage_request.c_str(), true)); + stage.Row(); + const int64_t generation = stage.ResultInt64(); + Call publish(session, "SELECT vexfs_handle_publish_close(?1,?2,?3)"); + publish.Text(1, handle.c_str()); + publish.Int64(2, generation); + publish.Text(3, durability); + publish.Row(); + *version = publish.ResultInt64(); + } catch (...) { + try { + const std::string close_request = std::string(request_id) + ":close-failed"; + Call close(session, "SELECT vexfs_handle_close(?1,0,?2)"); + close.Text(1, handle.c_str()); + close.Text(2, close_request.c_str()); + close.Row(); + } catch (...) { + } + throw; + } + }); +} + extern "C" vexfs_mount_status vexfs_mount_append_file(vexfs_mount_session *session, int64_t inode, const void *data, uint64_t size, int64_t *version, @@ -1347,6 +1555,27 @@ extern "C" vexfs_mount_status vexfs_mount_read_file(vexfs_mount_session *session }); } +extern "C" vexfs_mount_status vexfs_mount_read_file_range( + vexfs_mount_session *session, const char *path, uint64_t offset, + uint64_t length, vexfs_mount_bytes *content, vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + if (path == nullptr || content == nullptr || + offset > static_cast(std::numeric_limits::max()) || + length > static_cast(std::numeric_limits::max())) { + throw CallError(SQLITE_MISUSE, + "path, content and a valid read range are required"); + } + Call call(session, "SELECT vexfs_read_range(?1,?2,?3,?4)"); + call.Text(1, session->workspace.c_str()); + call.Text(2, path); + call.Int64(3, static_cast(offset)); + call.Int64(4, static_cast(length)); + call.Row(); + CopyResult(call, content); + }); +} + extern "C" vexfs_mount_status vexfs_mount_grep(vexfs_mount_session *session, const char *path, const char *pattern, uint32_t flags, uint32_t limit, @@ -2040,6 +2269,95 @@ extern "C" vexfs_mount_status vexfs_mount_handle_create( }); } +extern "C" vexfs_mount_status vexfs_mount_handle_create_owned_durable( + vexfs_mount_session *session, const char *path, uint32_t mode, + int64_t uid, int64_t gid, const char *request_id, + const char *durability, vexfs_mount_bytes *handle, + vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + if (path == nullptr || request_id == nullptr || durability == nullptr || + handle == nullptr || mode > 07777 || uid < 0 || gid < 0) { + throw CallError(SQLITE_MISUSE, + "path, mode 0..07777, owner, request, durability and handle are required"); + } + UseRequestedDurability(session, durability); + Call call(session, "SELECT vexfs_handle_create(?1,?2,?3,?4,?5,?6,?7)"); + call.Text(1, session->workspace.c_str()); + call.Text(2, path); + call.Int(3, static_cast(mode)); + call.Int64(4, uid); + call.Int64(5, gid); + call.Text(6, EffectiveRequestId(session, request_id, true)); + call.Text(7, session->session_id.c_str()); + call.Row(); + CopyResult(call, handle); + }); +} + +extern "C" vexfs_mount_status vexfs_mount_handle_create_owned_stat_durable( + vexfs_mount_session *session, const char *path, uint32_t mode, + int64_t uid, int64_t gid, const char *request_id, + const char *durability, vexfs_mount_bytes *handle, + vexfs_mount_bytes *stat_json, vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + if (path == nullptr || request_id == nullptr || durability == nullptr || + handle == nullptr || stat_json == nullptr || mode > 07777 || uid < 0 || gid < 0) { + throw CallError( + SQLITE_MISUSE, + "path, mode 0..07777, owner, request, durability, handle and stat are required"); + } + UseRequestedDurability(session, durability); +#if defined(VEXFS_HAVE_LIBPQ) + if (session->postgresql) { + // The server wrapper runs create and stat as separate PL/pgSQL + // statements so the STABLE stat function sees the new inode. Both + // results still travel in one libpq request, removing one WAN RTT. + Call call(session, + "SELECT created.handle, created.stat::text " + "FROM vexfs_handle_create_stat(?1,?2,?3,?4,?5,?6,?7) AS created"); + call.Text(1, session->workspace.c_str()); + call.Text(2, path); + call.Int(3, static_cast(mode)); + call.Int64(4, uid); + call.Int64(5, gid); + call.Text(6, EffectiveRequestId(session, request_id, true)); + call.Text(7, session->session_id.c_str()); + call.Row(); + CopyString(call.ResultText(0), handle); + CopyString(call.ResultText(1), stat_json); + return; + } +#endif + // SQLite is local, so keep the existing functions as the source of truth. + // The adapter still exposes the same combined contract to platform code. + std::string handle_value; + { + Call create(session, "SELECT vexfs_handle_create(?1,?2,?3,?4,?5,?6,?7)"); + create.Text(1, session->workspace.c_str()); + create.Text(2, path); + create.Int(3, static_cast(mode)); + create.Int64(4, uid); + create.Int64(5, gid); + create.Text(6, EffectiveRequestId(session, request_id, true)); + create.Text(7, session->session_id.c_str()); + create.Row(); + handle_value = create.ResultText(); + } + std::string stat_value; + { + Call stat(session, "SELECT vexfs_stat(?1,?2)"); + stat.Text(1, session->workspace.c_str()); + stat.Text(2, path); + stat.Row(); + stat_value = stat.ResultText(); + } + CopyString(handle_value, handle); + CopyString(stat_value, stat_json); + }); +} + extern "C" vexfs_mount_status vexfs_mount_handle_truncate( vexfs_mount_session *session, const char *handle, uint64_t size, const char *request_id, int64_t *generation, vexfs_mount_error *error) { @@ -2078,6 +2396,29 @@ extern "C" vexfs_mount_status vexfs_mount_handle_stage_write( }); } +extern "C" vexfs_mount_status vexfs_mount_handle_stage_write_durable( + vexfs_mount_session *session, const char *handle, uint64_t offset, const void *data, + uint64_t size, const char *request_id, const char *durability, + int64_t *generation, vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + if (handle == nullptr || request_id == nullptr || durability == nullptr || + generation == nullptr || (size > 0 && data == nullptr) || + offset > static_cast(std::numeric_limits::max())) { + throw CallError(SQLITE_MISUSE, + "handle, content, request, durability and generation are required"); + } + UseRequestedDurability(session, durability); + Call call(session, "SELECT vexfs_handle_stage_write(?1,?2,?3,?4)"); + call.Text(1, handle); + call.Int64(2, static_cast(offset)); + call.Blob(3, data, size); + call.Text(4, EffectiveRequestId(session, request_id, true)); + call.Row(); + *generation = call.ResultInt64(); + }); +} + extern "C" vexfs_mount_status vexfs_mount_handle_append( vexfs_mount_session *session, const char *handle, const void *data, uint64_t size, const char *request_id, int64_t *version, vexfs_mount_error *error) { @@ -2124,8 +2465,7 @@ extern "C" vexfs_mount_status vexfs_mount_handle_publish( if (durability == nullptr) { throw CallError(SQLITE_MISUSE, "durability is NULL"); } - if (std::strcmp(durability, "full") == 0) UseFullDurability(session); - else UseOrdinaryDurability(session); + UseRequestedDurability(session, durability); if (version == nullptr) throw CallError(SQLITE_MISUSE, "version output is NULL"); Call call(session, "SELECT vexfs_handle_publish(?1,?2,?3,?4)"); call.Text(1, handle); call.Int64(2, generation); call.Text(3, durability); @@ -2143,8 +2483,7 @@ extern "C" vexfs_mount_status vexfs_mount_handle_publish_close( throw CallError(SQLITE_MISUSE, "handle, durability and version output are required"); } - if (std::strcmp(durability, "full") == 0) UseFullDurability(session); - else UseOrdinaryDurability(session); + UseRequestedDurability(session, durability); Call call(session, "SELECT vexfs_handle_publish_close(?1,?2,?3)"); call.Text(1, handle); call.Int64(2, generation); @@ -2185,6 +2524,86 @@ extern "C" vexfs_mount_status vexfs_mount_synchronize(vexfs_mount_session *sessi }); } +extern "C" vexfs_mount_status vexfs_mount_publish_close_all( + vexfs_mount_session *session, const char *durability, + int64_t *published, vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + if (durability == nullptr || published == nullptr) { + throw CallError(SQLITE_MISUSE, + "durability and published output are required"); + } + UseRequestedDurability(session, durability); + Call call(session, session->postgresql + ? "SELECT vexfs_mount_publish_close_all(?1,?2,?3)" + : "SELECT vexfs_mount_publish_close_all(?1,?2)"); + call.Text(1, session->workspace.c_str()); + call.Text(2, session->session_id.c_str()); + if (session->postgresql) call.Text(3, durability); + call.Row(); + *published = call.ResultInt64(); + if (*published > 0) ++session->mutation_epoch; + }); +} + +extern "C" vexfs_mount_status vexfs_mount_publish_close_batch( + vexfs_mount_session *session, const char *durability, int64_t max_count, + int64_t *published, vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + if (durability == nullptr || published == nullptr || max_count <= 0) { + throw CallError(SQLITE_MISUSE, + "durability, a positive max_count and published output are required"); + } + UseRequestedDurability(session, durability); + Call call(session, session->postgresql + ? "SELECT vexfs_mount_publish_close_all(?1,?2,?3,?4)" + : "SELECT vexfs_mount_publish_close_all(?1,?2,?3)"); + call.Text(1, session->workspace.c_str()); + call.Text(2, session->session_id.c_str()); + if (session->postgresql) { + call.Text(3, durability); + call.Int64(4, max_count); + } else { + call.Int64(3, max_count); + } + call.Row(); + *published = call.ResultInt64(); + if (*published > 0) ++session->mutation_epoch; + }); +} + +extern "C" vexfs_mount_status vexfs_mount_publish_close_claimed( + vexfs_mount_session *session, const char *durability, + const char *claims_json, vexfs_mount_bytes *result_json, + vexfs_mount_error *error) { +#if defined(VEXFS_HAVE_LIBPQ) + return GuardPublisher(session, error, [&] { + if (durability == nullptr || claims_json == nullptr || result_json == nullptr) { + throw CallError(kPgInvalidArgument, + "durability, claims and result output are required", + VEXFS_RUNTIME_BACKEND_POSTGRESQL); + } + Call call(session->publisher_pg, + "SELECT vexfs_mount_publish_close_claimed(?1,?2,?3,?4::jsonb)"); + call.Text(1, session->workspace.c_str()); + call.Text(2, session->session_id.c_str()); + call.Text(3, durability); + call.Text(4, claims_json); + call.Row(); + CopyResult(call, result_json); + }); +#else + (void)session; + (void)durability; + (void)claims_json; + (void)result_json; + return SetError(error, kUnsupportedBackend, + "background publisher requires PostgreSQL", + VEXFS_RUNTIME_BACKEND_POSTGRESQL); +#endif +} + extern "C" vexfs_mount_status vexfs_mount_reclaim(vexfs_mount_session *session, const char *request_id, int64_t *reclaimed, diff --git a/agent_files/mount/common/test/vexfs_mount_contract_smoke.cpp b/agent_files/mount/common/test/vexfs_mount_contract_smoke.cpp index b366ff7e5d..2ec9d8bfa5 100644 --- a/agent_files/mount/common/test/vexfs_mount_contract_smoke.cpp +++ b/agent_files/mount/common/test/vexfs_mount_contract_smoke.cpp @@ -109,21 +109,17 @@ int RunBenchmark(int file_count) { std::snprintf(file_path, sizeof(file_path), "/files/f%06d.txt", index); std::snprintf(request, sizeof(request), "create-%06d", index); vexfs_mount_bytes handle{}; + vexfs_mount_bytes item_stat{}; auto call_started = std::chrono::steady_clock::now(); - if (vexfs_mount_handle_create(session, file_path, 0644, request, - &handle, &error) != VEXFS_MOUNT_OK) + if (vexfs_mount_handle_create_owned_stat_durable( + session, file_path, 0644, getuid(), getgid(), request, "none", + &handle, &item_stat, &error) != VEXFS_MOUNT_OK) return Fail("benchmark create", error); create_call_seconds += std::chrono::duration( std::chrono::steady_clock::now() - call_started).count(); const std::string handle_id(static_cast(handle.data), static_cast(handle.size)); vexfs_mount_free(handle.data); - vexfs_mount_bytes item_stat{}; - call_started = std::chrono::steady_clock::now(); - if (vexfs_mount_stat(session, file_path, &item_stat, &error) != VEXFS_MOUNT_OK) - return Fail("benchmark stat", error); - stat_call_seconds += std::chrono::duration( - std::chrono::steady_clock::now() - call_started).count(); const int64_t inode = JsonInteger(item_stat, "inode"); vexfs_mount_free(item_stat.data); call_started = std::chrono::steady_clock::now(); @@ -135,21 +131,31 @@ int RunBenchmark(int file_count) { int64_t generation = 0; std::snprintf(request, sizeof(request), "write-%06d", index); call_started = std::chrono::steady_clock::now(); - if (vexfs_mount_handle_stage_write( + if (vexfs_mount_handle_stage_write_durable( session, handle_id.c_str(), 0, payload, sizeof(payload) - 1, - request, &generation, &error) != VEXFS_MOUNT_OK) + request, "none", &generation, &error) != VEXFS_MOUNT_OK) return Fail("benchmark stage", error); stage_call_seconds += std::chrono::duration( std::chrono::steady_clock::now() - call_started).count(); - int64_t version = 0; - call_started = std::chrono::steady_clock::now(); - if (vexfs_mount_handle_publish_close(session, handle_id.c_str(), generation, - "data", &version, &error) != - VEXFS_MOUNT_OK || version != 1) - return Fail("benchmark publish close", error); - publish_call_seconds += std::chrono::duration( - std::chrono::steady_clock::now() - call_started).count(); } + const auto publish_started = std::chrono::steady_clock::now(); + int64_t batch_published = 0; + int64_t total_published = 0; + while (total_published < file_count) { + if (vexfs_mount_publish_close_batch(session, "full", 64, + &batch_published, &error) != + VEXFS_MOUNT_OK || batch_published <= 0 || batch_published > 64) { + return Fail("benchmark bounded publish close", error); + } + total_published += batch_published; + } + if (total_published != file_count) { + std::fprintf(stderr, "benchmark batch published=%lld expected=%d\n", + static_cast(total_published), file_count); + return Fail("benchmark batch publish close", error); + } + publish_call_seconds = std::chrono::duration( + std::chrono::steady_clock::now() - publish_started).count(); const double create_seconds = std::chrono::duration( std::chrono::steady_clock::now() - started).count(); const int64_t commits = Scalar(path, "SELECT count(*) FROM _vexfs_commits") - commits_before; @@ -489,6 +495,27 @@ int main(int argc, char **argv) { if (vexfs_mount_write_file(session, "/agent/versioned.txt", "new", 3, &version, &error) != VEXFS_MOUNT_OK || version != 2) return Fail("version update", error); + std::string range_source(70032, 'x'); + range_source.replace(65530, 16, "0123456789abcdef"); + if (vexfs_mount_write_file(session, "/agent/range.bin", range_source.data(), + range_source.size(), &version, &error) != VEXFS_MOUNT_OK) + return Fail("range seed", error); + vexfs_mount_bytes range_content{}; + if (vexfs_mount_read_file_range(session, "/agent/range.bin", 65530, 16, + &range_content, &error) != VEXFS_MOUNT_OK || + !Equals(range_content, "0123456789abcdef")) + return Fail("range read across chunks", error); + vexfs_mount_free(range_content.data); + if (vexfs_mount_write_file_range(session, "/agent/range.bin", 65534, + "RANGE", 5, "range-write-1", "full", + &version, &error) != VEXFS_MOUNT_OK) + return Fail("range write", error); + range_content = {}; + if (vexfs_mount_read_file_range(session, "/agent/range.bin", 65530, 16, + &range_content, &error) != VEXFS_MOUNT_OK || + !Equals(range_content, "0123RANGE9abcdef")) + return Fail("range read after write", error); + vexfs_mount_free(range_content.data); vexfs_mount_bytes history{}; if (vexfs_mount_history(session, "/agent/versioned.txt", &history, &error) != VEXFS_MOUNT_OK || !Contains(history, "\"version\":2") || !Contains(history, "\"version\":1")) diff --git a/agent_files/mount/common/test/vexfs_pg_runtime_smoke.cpp b/agent_files/mount/common/test/vexfs_pg_runtime_smoke.cpp index 4ff0c41a9f..95b506e7fe 100644 --- a/agent_files/mount/common/test/vexfs_pg_runtime_smoke.cpp +++ b/agent_files/mount/common/test/vexfs_pg_runtime_smoke.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -130,6 +131,9 @@ int main(int argc, char **argv) { if (control != nullptr) PQfinish(control); return 1; } + const std::string database_principal = PQuser(control); + const std::string principal_json = + "\"principal\":\"" + database_principal + "\""; DropWorkspace(control, workspace); vexfs_mount_config config{}; @@ -159,12 +163,125 @@ int main(int argc, char **argv) { "diagnostics adapter version") || !Expect(diagnostics.find("\"security_mode\":\"database-role\"") != std::string::npos, "diagnostics security") || - !Expect(diagnostics.find("\"principal\":\"postgres\"") != std::string::npos, + !Expect(diagnostics.find(principal_json) != std::string::npos, "diagnostics principal") || !Expect(backend_pid > 0, "diagnostics backend pid")) return 1; if (!CheckStatus("mkdir", vexfs_mount_mkdir(session, "/project", &error), error)) return 1; int64_t version = 0; + vexfs_mount_bytes owned_handle_bytes{}; + vexfs_mount_bytes owned_stat_bytes{}; + if (!CheckStatus("owned handle create with stat", vexfs_mount_handle_create_owned_stat_durable( + session, "/project/owned.txt", 04755, 501, 20, + "create-owned", "full", &owned_handle_bytes, &owned_stat_bytes, &error), error)) + return 1; + const std::string owned_handle = Take(&owned_handle_bytes); + const std::string owned_stat = Take(&owned_stat_bytes); + if (!Expect(JsonInteger(owned_stat, "version") == 0, + "owned create visible before first publish") || + !Expect(JsonInteger(owned_stat, "mode") == 04755, + "owned create special mode") || + !Expect(JsonInteger(owned_stat, "uid") == 501, "owned create uid") || + !Expect(JsonInteger(owned_stat, "gid") == 20, "owned create gid")) return 1; + int64_t owned_generation = 0; + if (!CheckStatus("owned create stage", vexfs_mount_handle_stage_write( + session, owned_handle.c_str(), 0, "owned", 5, "stage-owned", + &owned_generation, &error), error) || + !Expect(owned_generation == 2, "owned create generation") || + !CheckStatus("owned create publish close", vexfs_mount_handle_publish_close( + session, owned_handle.c_str(), owned_generation, "full", &version, &error), + error) || + !Expect(version == 1, "owned create first version") || + !CheckStatus("owned create read", vexfs_mount_read_file( + session, "/project/owned.txt", &bytes, &error), error) || + !Expect(Take(&bytes) == "owned", "owned create content")) return 1; + vexfs_mount_bytes batch_a_bytes{}; + vexfs_mount_bytes batch_b_bytes{}; + if (!CheckStatus("batch create a", vexfs_mount_handle_create_owned_durable( + session, "/project/batch-a.txt", 0644, 501, 20, + "create-batch-a", "none", &batch_a_bytes, &error), error) || + !CheckStatus("batch create b", vexfs_mount_handle_create_owned_durable( + session, "/project/batch-b.txt", 0644, 501, 20, + "create-batch-b", "none", &batch_b_bytes, &error), error)) return 1; + const std::string batch_a = Take(&batch_a_bytes); + const std::string batch_b = Take(&batch_b_bytes); + int64_t batch_a_generation = 0; + int64_t batch_b_generation = 0; + if (!CheckStatus("batch stage a", vexfs_mount_handle_stage_write_durable( + session, batch_a.c_str(), 0, "batch-a", 7, + "stage-batch-a", "none", &batch_a_generation, &error), error) || + !CheckStatus("batch stage b", vexfs_mount_handle_stage_write_durable( + session, batch_b.c_str(), 0, "batch-b", 7, + "stage-batch-b", "none", &batch_b_generation, &error), error) || + !Expect(batch_a_generation == 2 && batch_b_generation == 2, + "batch staged generations")) return 1; + int64_t batch_published = 0; + if (!CheckStatus("bounded batch publish", vexfs_mount_publish_close_batch( + session, "full", 1, &batch_published, &error), error) || + !Expect(batch_published == 1, "bounded batch publish count") || + !CheckStatus("batch publish remainder", vexfs_mount_publish_close_all( + session, "full", &batch_published, &error), error) || + !Expect(batch_published == 1, "batch publish remainder count") || + !CheckStatus("batch publish replay", vexfs_mount_publish_close_all( + session, "full", &batch_published, &error), error) || + !Expect(batch_published == 0, "batch publish is idempotent") || + !CheckStatus("batch read a", vexfs_mount_read_file( + session, "/project/batch-a.txt", &bytes, &error), error) || + !Expect(Take(&bytes) == "batch-a", "batch content a") || + !CheckStatus("batch read b", vexfs_mount_read_file( + session, "/project/batch-b.txt", &bytes, &error), error) || + !Expect(Take(&bytes) == "batch-b", "batch content b")) return 1; + + vexfs_mount_bytes claimed_handle_bytes{}; + if (!CheckStatus("claimed create", vexfs_mount_handle_create_owned_durable( + session, "/project/claimed.txt", 0644, 501, 20, + "create-claimed", "none", &claimed_handle_bytes, &error), error)) return 1; + const std::string claimed_handle = Take(&claimed_handle_bytes); + int64_t claimed_generation = 0; + if (!CheckStatus("claimed stage", vexfs_mount_handle_stage_write_durable( + session, claimed_handle.c_str(), 0, "claimed", 7, + "stage-claimed", "none", &claimed_generation, &error), error)) return 1; + const std::string claims = "[{\"handle\":\"" + claimed_handle + + "\",\"generation\":" + std::to_string(claimed_generation) + "}]"; + if (!Expect(PgCommand(control, + "BEGIN; SELECT 1 FROM _vexfs.workspaces " + "WHERE name='pg-runtime-contract' FOR UPDATE"), + "lock workspace for publisher concurrency")) return 1; + std::atomic publisher_started{false}; + vexfs_mount_status publisher_status = VEXFS_MOUNT_DATABASE_ERROR; + vexfs_mount_error publisher_error{}; + vexfs_mount_bytes publisher_result{}; + std::thread publisher([&] { + publisher_started.store(true); + publisher_status = vexfs_mount_publish_close_claimed( + session, "full", claims.c_str(), &publisher_result, &publisher_error); + }); + while (!publisher_started.load()) std::this_thread::yield(); + std::this_thread::sleep_for(std::chrono::milliseconds(150)); + const auto foreground_started = std::chrono::steady_clock::now(); + const vexfs_mount_status foreground_status = vexfs_mount_stat( + session, "/project/owned.txt", &bytes, &error); + const auto foreground_elapsed = std::chrono::steady_clock::now() - foreground_started; + const bool unlocked = PgCommand(control, "ROLLBACK"); + publisher.join(); + if (!CheckStatus("foreground stat during blocked publisher", foreground_status, error) || + !Expect(Take(&bytes).find("\"inode\":") != std::string::npos, + "foreground stat result during publisher") || + !Expect(foreground_elapsed < std::chrono::seconds(1), + "background publisher does not hold foreground runtime mutex") || + !Expect(unlocked, "unlock workspace after concurrency probe") || + !CheckStatus("claimed publisher", publisher_status, publisher_error)) return 1; + const std::string claimed_result = Take(&publisher_result); + if (!Expect(claimed_result.find(claimed_handle) != std::string::npos, + "claimed publisher result") || + !CheckStatus("claimed publisher replay", vexfs_mount_publish_close_claimed( + session, "full", claims.c_str(), &bytes, &error), error) || + !Expect(Take(&bytes).find(claimed_handle) != std::string::npos, + "claimed publisher replay result") || + !CheckStatus("claimed content", vexfs_mount_read_file( + session, "/project/claimed.txt", &bytes, &error), error) || + !Expect(Take(&bytes) == "claimed", "claimed content value")) return 1; + if (!CheckStatus("write v1", vexfs_mount_write_file( session, "/project/main.txt", "alpha\n", 6, &version, &error), error) || !Expect(version == 1, "write version 1")) return 1; @@ -192,13 +309,13 @@ int main(int argc, char **argv) { !CheckStatus("xattr get", vexfs_mount_xattr_get( session, inode, "user.runtime", &bytes, &error), error) || !Expect(Take(&bytes) == "value", "xattr value")) return 1; - const char acl[] = - "[{\"principal\":\"postgres\",\"effect\":\"allow\"," + const std::string acl = + "[{\"principal\":\"" + database_principal + "\",\"effect\":\"allow\"," "\"permissions\":\"read,write,execute,metadata\",\"inherit\":1}]"; if (!CheckStatus("acl set", vexfs_mount_acl_set( - session, inode, acl, sizeof(acl) - 1, &error), error) || + session, inode, acl.data(), acl.size(), &error), error) || !CheckStatus("acl get", vexfs_mount_acl_get(session, inode, &bytes, &error), error) || - !Expect(Take(&bytes).find("\"principal\":\"postgres\"") != std::string::npos, + !Expect(Take(&bytes).find(principal_json) != std::string::npos, "acl content")) return 1; if (!CheckStatus("hardlink", vexfs_mount_link( @@ -230,7 +347,7 @@ int main(int argc, char **argv) { const std::string enabled_index = Take(&bytes); if (!Expect(enabled_index.find("\"enabled\":true") != std::string::npos && enabled_index.find("\"dirty\":false") != std::string::npos && - enabled_index.find("\"indexed_files\":1") != std::string::npos, + enabled_index.find("\"indexed_files\":5") != std::string::npos, "trigram index enabled") || !CheckStatus("indexed grep", vexfs_mount_grep( session, "/project", "alpha", 0, 100, &bytes, &error), error)) return 1; @@ -239,6 +356,26 @@ int main(int argc, char **argv) { indexed_grep.find("\"index_used\":true") != std::string::npos, "trigram indexed grep")) return 1; + std::string range_source(70032, 'x'); + range_source.replace(65530, 16, "0123456789abcdef"); + if (!CheckStatus("range seed", vexfs_mount_write_file( + session, "/project/range.bin", range_source.data(), range_source.size(), + &version, &error), error) || + !CheckStatus("range read", vexfs_mount_read_file_range( + session, "/project/range.bin", 65530, 16, &bytes, &error), error) || + !Expect(Take(&bytes) == "0123456789abcdef", "range read across chunks")) return 1; + if (!CheckStatus("range write", vexfs_mount_write_file_range( + session, "/project/range.bin", 65534, "RANGE", 5, + "pg-range-write-1", "full", &version, &error), error)) return 1; + const int64_t range_version = version; + if (!CheckStatus("range write replay", vexfs_mount_write_file_range( + session, "/project/range.bin", 65534, "RANGE", 5, + "pg-range-write-1", "full", &version, &error), error) || + !Expect(version == range_version, "range write replay version") || + !CheckStatus("range read after write", vexfs_mount_read_file_range( + session, "/project/range.bin", 65530, 16, &bytes, &error), error) || + !Expect(Take(&bytes) == "0123RANGE9abcdef", "range read after write")) return 1; + vexfs_mount_bytes handle_bytes{}; if (!CheckStatus("handle create", vexfs_mount_handle_create( session, "/project/staged.txt", 0644, "create-staged", diff --git a/agent_files/mount/macos/nfs_gateway/src/lib.rs b/agent_files/mount/macos/nfs_gateway/src/lib.rs index 2bd255bb5e..8db631b1aa 100644 --- a/agent_files/mount/macos/nfs_gateway/src/lib.rs +++ b/agent_files/mount/macos/nfs_gateway/src/lib.rs @@ -11,7 +11,7 @@ use std::ffi::{CStr, CString}; use std::os::raw::{c_char, c_int, c_void}; use std::ptr; use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::Mutex; +use std::sync::{Arc, Mutex}; use std::time::{SystemTime, UNIX_EPOCH}; const RUNTIME_ABI_VERSION: u32 = 1; @@ -21,7 +21,22 @@ const XATTR_DELETE: c_int = 3; const TIME_ACCESS: u32 = 1; const TIME_MODIFY: u32 = 2; const APPLEDOUBLE_XATTR: &str = "io.vexdb.macos.appledouble"; +// NFSv3 can only expose macOS xattrs as visible `._name` files. Keep the +// default Bash workspace clean and report mounted xattrs as unsupported; +// native VexFS xattr APIs remain available through the database/runtime. +const APPLEDOUBLE_SIDECARS_ENABLED: bool = false; const SIDECAR_INODE_START: u64 = u64::MAX - 1024; +const METADATA_CACHE_LIMIT: usize = 100_000; +const METADATA_CACHE_TTL_MS: i64 = 250; +const DEFERRED_PUBLISH_IDLE_MS: i64 = 500; +const DEFERRED_PUBLISH_MAX_AGE_MS: i64 = 30_000; +const DEFERRED_PUBLISH_DIRTY_BYTES: u64 = 4 * 1024 * 1024; +const DEFERRED_PUBLISH_BATCH_BYTES: u64 = 8 * 1024 * 1024; +const DEFERRED_PUBLISH_FILE_THRESHOLD: usize = 1_024; +// Remote PostgreSQL may spend tens of milliseconds publishing one version. +// Keep one background lock hold below the macOS NFS retransmit window; a large +// batch makes unrelated foreground reads look like a dead server. +const DEFERRED_PUBLISH_BATCH_SIZE: i64 = 8; #[repr(C)] pub struct VexfsNfsGatewayConfig { @@ -60,6 +75,13 @@ struct RuntimeError { message: [c_char; 512], } +#[repr(C)] +struct RuntimeVisibility { + workspace_head: i64, + cache_generation: u64, + external_commit: c_int, +} + enum RuntimeSessionOpaque {} type RuntimeStatus = c_int; @@ -88,9 +110,16 @@ extern "C" { json: *mut RuntimeBytes, error: *mut RuntimeError, ) -> RuntimeStatus; - fn vexfs_mount_read_file( + fn vexfs_mount_refresh_visibility( + session: *mut RuntimeSessionOpaque, + visibility: *mut RuntimeVisibility, + error: *mut RuntimeError, + ) -> RuntimeStatus; + fn vexfs_mount_read_file_range( session: *mut RuntimeSessionOpaque, path: *const c_char, + offset: u64, + length: u64, content: *mut RuntimeBytes, error: *mut RuntimeError, ) -> RuntimeStatus; @@ -176,21 +205,26 @@ extern "C" { handle: *mut RuntimeBytes, error: *mut RuntimeError, ) -> RuntimeStatus; - fn vexfs_mount_handle_create( + fn vexfs_mount_handle_create_owned_stat_durable( session: *mut RuntimeSessionOpaque, path: *const c_char, mode: u32, + uid: i64, + gid: i64, request_id: *const c_char, + durability: *const c_char, handle: *mut RuntimeBytes, + stat_json: *mut RuntimeBytes, error: *mut RuntimeError, ) -> RuntimeStatus; - fn vexfs_mount_handle_stage_write( + fn vexfs_mount_handle_stage_write_durable( session: *mut RuntimeSessionOpaque, handle: *const c_char, offset: u64, data: *const c_void, size: u64, request_id: *const c_char, + durability: *const c_char, generation: *mut i64, error: *mut RuntimeError, ) -> RuntimeStatus; @@ -202,6 +236,14 @@ extern "C" { generation: *mut i64, error: *mut RuntimeError, ) -> RuntimeStatus; + fn vexfs_mount_handle_read( + session: *mut RuntimeSessionOpaque, + handle: *const c_char, + offset: u64, + length: u64, + content: *mut RuntimeBytes, + error: *mut RuntimeError, + ) -> RuntimeStatus; fn vexfs_mount_handle_publish_close( session: *mut RuntimeSessionOpaque, handle: *const c_char, @@ -224,6 +266,20 @@ extern "C" { published: *mut i64, error: *mut RuntimeError, ) -> RuntimeStatus; + fn vexfs_mount_publish_close_batch( + session: *mut RuntimeSessionOpaque, + durability: *const c_char, + max_count: i64, + published: *mut i64, + error: *mut RuntimeError, + ) -> RuntimeStatus; + fn vexfs_mount_publish_close_claimed( + session: *mut RuntimeSessionOpaque, + durability: *const c_char, + claims_json: *const c_char, + result_json: *mut RuntimeBytes, + error: *mut RuntimeError, + ) -> RuntimeStatus; fn vexfs_mount_free(memory: *mut c_void); } @@ -287,14 +343,48 @@ struct Sidecar { updated_at_ms: i64, } +struct PendingWrite { + handle: CString, + generation: i64, + logical_size: u64, + updated_at_ms: i64, + first_dirty_at_ms: i64, + dirty_bytes: u64, + publishing_generation: Option, +} + +#[derive(Clone)] +struct PublishCandidate { + inode: fileid3, + handle: CString, + generation: i64, + logical_size: u64, +} + +#[derive(Deserialize)] +struct PublishedClaim { + handle: String, + generation: i64, + #[allow(dead_code)] + version: i64, +} + struct RuntimeState { session: *mut RuntimeSessionOpaque, + postgresql_backend: bool, root_inode: fileid3, uid: u32, gid: u32, + strict_durability: bool, next_sidecar_inode: fileid3, sidecars_by_key: HashMap, sidecars: HashMap, + pending_writes: HashMap, + paths: HashMap, + inodes_by_path: HashMap, + stats: HashMap, + directory_entries: HashMap, fileid3>>, + metadata_cache_checked_at_ms: i64, } unsafe impl Send for RuntimeState {} @@ -309,7 +399,7 @@ impl Drop for RuntimeState { } pub struct VexfsNfs { - state: Mutex, + state: Arc>, request_prefix: String, request_sequence: AtomicU64, } @@ -377,6 +467,45 @@ fn runtime_result(status: RuntimeStatus) -> Result<(), nfsstat3> { } } +fn publish_claimed_batch( + session: *mut RuntimeSessionOpaque, + candidates: &[PublishCandidate], +) -> Result, nfsstat3> { + let claims: Result, nfsstat3> = candidates + .iter() + .map(|candidate| { + Ok(serde_json::json!({ + "handle": candidate + .handle + .to_str() + .map_err(|_| nfsstat3::NFS3ERR_IO)?, + "generation": candidate.generation, + })) + }) + .collect(); + let claims = CString::new( + serde_json::to_string(&claims?).map_err(|_| nfsstat3::NFS3ERR_IO)?, + ) + .map_err(|_| nfsstat3::NFS3ERR_IO)?; + let durability = CString::new("full").unwrap(); + let mut result = RuntimeBytes { + data: ptr::null_mut(), + size: 0, + }; + let mut error = empty_error(); + let status = unsafe { + vexfs_mount_publish_close_claimed( + session, + durability.as_ptr(), + claims.as_ptr(), + &mut result, + &mut error, + ) + }; + runtime_result(status)?; + serde_json::from_str(&bytes_to_string(result)?).map_err(|_| nfsstat3::NFS3ERR_IO) +} + fn now_ms() -> i64 { SystemTime::now() .duration_since(UNIX_EPOCH) @@ -417,7 +546,56 @@ fn stat_attr(stat: &FileStat) -> fattr3 { } impl RuntimeState { + fn refresh_metadata_cache_if_stale(&mut self) -> Result<(), nfsstat3> { + let now = now_ms(); + if now.saturating_sub(self.metadata_cache_checked_at_ms) < METADATA_CACHE_TTL_MS { + return Ok(()); + } + let mut visibility = RuntimeVisibility { + workspace_head: 0, + cache_generation: 0, + external_commit: 0, + }; + let mut error = empty_error(); + runtime_result(unsafe { + vexfs_mount_refresh_visibility(self.session, &mut visibility, &mut error) + })?; + // Local create/write/publish paths update their cached entries directly. + // Only another database connection can make those entries stale. Paths + // stay available so already-issued NFS file handles remain usable. + if visibility.external_commit != 0 { + self.stats.clear(); + self.directory_entries.clear(); + } + self.metadata_cache_checked_at_ms = now; + Ok(()) + } + + fn clear_metadata_cache(&mut self) { + self.paths.clear(); + self.inodes_by_path.clear(); + self.stats.clear(); + self.directory_entries.clear(); + } + + fn cache_stat(&mut self, path: &str, stat: &FileStat) { + if self.stats.len() >= METADATA_CACHE_LIMIT { + self.clear_metadata_cache(); + } + let inode = stat.inode as u64; + self.paths.insert(inode, path.to_string()); + self.inodes_by_path.insert(path.to_string(), inode); + self.stats.insert(inode, stat.clone()); + } + fn stat_path(&mut self, path: &str) -> Result { + self.refresh_metadata_cache_if_stale()?; + if let Some(inode) = self.inodes_by_path.get(path) { + if let Some(stat) = self.stats.get(inode) { + return Ok(stat.clone()); + } + } + let path_text = path.to_string(); let path = checked_cstring(path)?; let mut bytes = RuntimeBytes { data: ptr::null_mut(), @@ -427,13 +605,19 @@ impl RuntimeState { let status = unsafe { vexfs_mount_stat(self.session, path.as_ptr(), &mut bytes, &mut error) }; runtime_result(status)?; - serde_json::from_str(&bytes_to_string(bytes)?).map_err(|_| nfsstat3::NFS3ERR_IO) + let stat: FileStat = + serde_json::from_str(&bytes_to_string(bytes)?).map_err(|_| nfsstat3::NFS3ERR_IO)?; + self.cache_stat(&path_text, &stat); + Ok(stat) } fn path_for_inode(&mut self, inode: fileid3) -> Result { if self.sidecars.contains_key(&inode) { return Err(nfsstat3::NFS3ERR_INVAL); } + if let Some(path) = self.paths.get(&inode) { + return Ok(path.clone()); + } let mut bytes = RuntimeBytes { data: ptr::null_mut(), size: 0, @@ -443,7 +627,10 @@ impl RuntimeState { vexfs_mount_path_for_inode(self.session, inode as i64, &mut bytes, &mut error) }; runtime_result(status)?; - bytes_to_string(bytes) + let path = bytes_to_string(bytes)?; + self.paths.insert(inode, path.clone()); + self.inodes_by_path.insert(path.clone(), inode); + Ok(path) } fn child_path(&mut self, parent: fileid3, name: &[u8]) -> Result { @@ -460,6 +647,7 @@ impl RuntimeState { } fn list(&mut self, path: &str) -> Result, nfsstat3> { + let path_text = path.to_string(); let path = checked_cstring(path)?; let mut bytes = RuntimeBytes { data: ptr::null_mut(), @@ -469,18 +657,82 @@ impl RuntimeState { let status = unsafe { vexfs_mount_list(self.session, path.as_ptr(), &mut bytes, &mut error) }; runtime_result(status)?; - serde_json::from_str(&bytes_to_string(bytes)?).map_err(|_| nfsstat3::NFS3ERR_IO) + let entries: Vec = + serde_json::from_str(&bytes_to_string(bytes)?).map_err(|_| nfsstat3::NFS3ERR_IO)?; + if let Ok(parent) = self.stat_path(&path_text) { + let parent_inode = parent.inode as u64; + let mut cached = HashMap::new(); + for entry in &entries { + let inode = entry.inode as u64; + cached.insert(entry.name.as_bytes().to_vec(), inode); + let child_path = if path_text == "/" { + format!("/{}", entry.name) + } else { + format!("{path_text}/{}", entry.name) + }; + let stat: FileStat = entry.clone().into(); + self.cache_stat(&child_path, &stat); + } + self.directory_entries.insert(parent_inode, cached); + } + Ok(entries) } - fn read_file(&mut self, path: &str) -> Result, nfsstat3> { + fn read_file_range( + &mut self, + path: &str, + offset: u64, + length: u64, + ) -> Result, nfsstat3> { let path = checked_cstring(path)?; let mut bytes = RuntimeBytes { data: ptr::null_mut(), size: 0, }; let mut error = empty_error(); - let status = - unsafe { vexfs_mount_read_file(self.session, path.as_ptr(), &mut bytes, &mut error) }; + let status = unsafe { + vexfs_mount_read_file_range( + self.session, + path.as_ptr(), + offset, + length, + &mut bytes, + &mut error, + ) + }; + runtime_result(status)?; + Ok(take_bytes(bytes)) + } + + fn read_inode_range( + &mut self, + inode: fileid3, + path: &str, + offset: u64, + length: u64, + ) -> Result, nfsstat3> { + let Some(pending) = self.pending_writes.get(&inode) else { + return self.read_file_range(path, offset, length); + }; + if pending.publishing_generation.is_some() { + return Err(nfsstat3::NFS3ERR_JUKEBOX); + } + let handle = pending.handle.clone(); + let mut bytes = RuntimeBytes { + data: ptr::null_mut(), + size: 0, + }; + let mut error = empty_error(); + let status = unsafe { + vexfs_mount_handle_read( + self.session, + handle.as_ptr(), + offset, + length, + &mut bytes, + &mut error, + ) + }; runtime_result(status)?; Ok(take_bytes(bytes)) } @@ -499,92 +751,150 @@ impl RuntimeState { mode: u32, prefix: &str, sequence: &AtomicU64, - ) -> Result<(), nfsstat3> { + ) -> Result<(CString, FileStat), nfsstat3> { let path = checked_cstring(path)?; let request = Self::request_id(prefix, sequence, "create"); + let durability = CString::new(if self.strict_durability { + "full" + } else { + "none" + }) + .unwrap(); let mut handle = RuntimeBytes { data: ptr::null_mut(), size: 0, }; + let mut stat_json = RuntimeBytes { + data: ptr::null_mut(), + size: 0, + }; let mut error = empty_error(); let status = unsafe { - vexfs_mount_handle_create( + vexfs_mount_handle_create_owned_stat_durable( self.session, path.as_ptr(), mode & 0o7777, + self.uid as i64, + self.gid as i64, request.as_ptr(), + durability.as_ptr(), &mut handle, + &mut stat_json, &mut error, ) }; runtime_result(status)?; let handle = checked_cstring(&bytes_to_string(handle)?)?; - let durability = CString::new("full").unwrap(); - let mut version = 0; - let status = unsafe { - vexfs_mount_handle_publish_close( - self.session, - handle.as_ptr(), - 1, - durability.as_ptr(), - &mut version, - &mut error, - ) - }; - if status != 0 { - self.close_failed_handle(&handle, prefix, sequence); - } - runtime_result(status) + let stat = + serde_json::from_str(&bytes_to_string(stat_json)?).map_err(|_| nfsstat3::NFS3ERR_IO)?; + Ok((handle, stat)) } - fn write_file( + fn stage_file_write( &mut self, + inode: fileid3, path: &str, offset: u64, data: &[u8], prefix: &str, sequence: &AtomicU64, ) -> Result<(), nfsstat3> { - let path = checked_cstring(path)?; - let mode = CString::new("rw").unwrap(); - let open_request = Self::request_id(prefix, sequence, "open-write"); - let mut handle = RuntimeBytes { - data: ptr::null_mut(), - size: 0, - }; - let mut error = empty_error(); - let status = unsafe { - vexfs_mount_handle_open( - self.session, - path.as_ptr(), - mode.as_ptr(), - open_request.as_ptr(), - &mut handle, - &mut error, - ) - }; - runtime_result(status)?; - let handle = checked_cstring(&bytes_to_string(handle)?)?; - let write_request = Self::request_id(prefix, sequence, "write"); + if self + .pending_writes + .get(&inode) + .is_some_and(|pending| pending.publishing_generation.is_some()) + { + return Err(nfsstat3::NFS3ERR_JUKEBOX); + } + if !self.pending_writes.contains_key(&inode) { + let path = checked_cstring(path)?; + let mode = CString::new("rw").unwrap(); + let request = Self::request_id(prefix, sequence, "open-staged-write"); + let mut handle = RuntimeBytes { + data: ptr::null_mut(), + size: 0, + }; + let mut error = empty_error(); + let status = unsafe { + vexfs_mount_handle_open( + self.session, + path.as_ptr(), + mode.as_ptr(), + request.as_ptr(), + &mut handle, + &mut error, + ) + }; + runtime_result(status)?; + let logical_size = self + .stat_path(std::str::from_utf8(path.as_bytes()).unwrap())? + .size + .max(0) as u64; + let dirty_at = now_ms(); + self.pending_writes.insert( + inode, + PendingWrite { + handle: checked_cstring(&bytes_to_string(handle)?)?, + generation: 0, + logical_size, + updated_at_ms: dirty_at, + first_dirty_at_ms: dirty_at, + dirty_bytes: 0, + publishing_generation: None, + }, + ); + } + let handle = self.pending_writes.get(&inode).unwrap().handle.clone(); + let request = Self::request_id(prefix, sequence, "stage-write"); + // NFS UNSTABLE writes become durable at COMMIT. Making every incoming + // block FULL turns a sequential file into thousands of fsync calls. + let durability = CString::new("none").unwrap(); let mut generation = 0; + let mut error = empty_error(); let status = unsafe { - vexfs_mount_handle_stage_write( + vexfs_mount_handle_stage_write_durable( self.session, handle.as_ptr(), offset, data.as_ptr().cast(), data.len() as u64, - write_request.as_ptr(), + request.as_ptr(), + durability.as_ptr(), &mut generation, &mut error, ) }; - if status != 0 { - self.close_failed_handle(&handle, prefix, sequence); - return Err(runtime_to_nfs(status)); + runtime_result(status)?; + let pending = self.pending_writes.get_mut(&inode).unwrap(); + pending.generation = generation; + pending.logical_size = pending + .logical_size + .max(offset.saturating_add(data.len() as u64)); + pending.dirty_bytes = pending.dirty_bytes.saturating_add(data.len() as u64); + let updated_at_ms = now_ms(); + pending.updated_at_ms = updated_at_ms; + let logical_size = pending.logical_size; + if let Some(stat) = self.stats.get_mut(&inode) { + stat.size = logical_size.min(i64::MAX as u64) as i64; + stat.updated_at = updated_at_ms; + stat.changed_at = updated_at_ms; } - let durability = CString::new("full").unwrap(); + Ok(()) + } + + fn publish_pending(&mut self, inode: fileid3, durability: &str) -> Result { + let Some(pending) = self.pending_writes.get(&inode) else { + return Ok(false); + }; + if pending.publishing_generation.is_some() { + return Err(nfsstat3::NFS3ERR_JUKEBOX); + } + let handle = pending.handle.clone(); + let generation = pending.generation; + let logical_size = pending.logical_size; + let durability = CString::new(durability).unwrap(); let mut version = 0; + let mut error = empty_error(); let status = unsafe { vexfs_mount_handle_publish_close( self.session, @@ -595,19 +905,244 @@ impl RuntimeState { &mut error, ) }; - if status != 0 { - self.close_failed_handle(&handle, prefix, sequence); + runtime_result(status)?; + self.pending_writes.remove(&inode); + if let Some(stat) = self.stats.get_mut(&inode) { + stat.size = logical_size.min(i64::MAX as u64) as i64; + stat.updated_at = now_ms(); + stat.changed_at = stat.updated_at; + } + Ok(true) + } + + fn publish_pending_batch( + &mut self, + durability: &str, + max_count: i64, + ) -> Result { + if self.pending_writes.is_empty() || max_count <= 0 { + return Ok(0); + } + let mut handles: Vec<(fileid3, Vec, u64)> = self + .pending_writes + .iter() + .map(|(inode, pending)| { + ( + *inode, + pending.handle.as_bytes().to_vec(), + pending.logical_size, + ) + }) + .collect(); + handles.sort_by(|left, right| left.1.cmp(&right.1)); + handles.truncate(max_count as usize); + let durability = CString::new(durability).unwrap(); + let mut published = 0; + let mut error = empty_error(); + let status = unsafe { + vexfs_mount_publish_close_batch( + self.session, + durability.as_ptr(), + max_count, + &mut published, + &mut error, + ) + }; + runtime_result(status)?; + let published = published.max(0) as usize; + if published > handles.len() { + return Err(nfsstat3::NFS3ERR_IO); + } + let now = now_ms(); + for (inode, _, logical_size) in handles.into_iter().take(published) { + self.pending_writes.remove(&inode); + if let Some(stat) = self.stats.get_mut(&inode) { + stat.size = logical_size.min(i64::MAX as u64) as i64; + stat.updated_at = now; + stat.changed_at = now; + } + } + Ok(published) + } + + fn claim_deferred_publish_batch(&mut self, now: i64) -> Vec { + let available: Vec = self + .pending_writes + .iter() + .filter(|(_, pending)| pending.publishing_generation.is_none()) + .map(|(inode, _)| *inode) + .collect(); + if available.is_empty() { + return Vec::new(); + } + let dirty_bytes = available.iter().fold(0_u64, |total, inode| { + total.saturating_add( + self.pending_writes + .get(inode) + .map(|pending| pending.dirty_bytes.max(1)) + .unwrap_or(0), + ) + }); + let oldest_age = available + .iter() + .filter_map(|inode| self.pending_writes.get(inode)) + .map(|pending| now.saturating_sub(pending.first_dirty_at_ms)) + .max() + .unwrap_or(0); + let latest_update = available + .iter() + .filter_map(|inode| self.pending_writes.get(inode)) + .map(|pending| pending.updated_at_ms) + .max() + .unwrap_or(now); + let globally_idle = now.saturating_sub(latest_update) >= DEFERRED_PUBLISH_IDLE_MS; + let force = available.len() >= DEFERRED_PUBLISH_FILE_THRESHOLD + || dirty_bytes >= DEFERRED_PUBLISH_DIRTY_BYTES + || oldest_age >= DEFERRED_PUBLISH_MAX_AGE_MS; + if !globally_idle && !force { + return Vec::new(); + } + + let mut ordered = available; + ordered.sort_by(|left, right| { + let left_pending = self.pending_writes.get(left).unwrap(); + let right_pending = self.pending_writes.get(right).unwrap(); + left_pending + .updated_at_ms + .cmp(&right_pending.updated_at_ms) + .then_with(|| left_pending.handle.as_bytes().cmp(right_pending.handle.as_bytes())) + }); + + let mut claimed = Vec::new(); + let mut claimed_bytes = 0_u64; + for inode in ordered { + if claimed.len() >= DEFERRED_PUBLISH_BATCH_SIZE as usize { + break; + } + let pending = self.pending_writes.get(&inode).unwrap(); + // Thresholds wake the publisher during a long foreground burst, + // but they must never close the handle currently receiving NFS + // WRITE calls. Older idle files may be drained concurrently; the + // active file is published by COMMIT/fsync or after it becomes idle. + if !globally_idle + && now.saturating_sub(pending.updated_at_ms) < DEFERRED_PUBLISH_IDLE_MS + { + continue; + } + let candidate_bytes = pending.dirty_bytes.max(1); + if !claimed.is_empty() + && claimed_bytes.saturating_add(candidate_bytes) + > DEFERRED_PUBLISH_BATCH_BYTES + { + break; + } + claimed_bytes = claimed_bytes.saturating_add(candidate_bytes); + claimed.push(PublishCandidate { + inode, + handle: pending.handle.clone(), + generation: pending.generation, + logical_size: pending.logical_size, + }); + } + for candidate in &claimed { + if let Some(pending) = self.pending_writes.get_mut(&candidate.inode) { + if pending.generation == candidate.generation { + pending.publishing_generation = Some(candidate.generation); + } + } + } + claimed + } + + fn release_publish_claims(&mut self, candidates: &[PublishCandidate]) { + for candidate in candidates { + if let Some(pending) = self.pending_writes.get_mut(&candidate.inode) { + if pending.publishing_generation == Some(candidate.generation) { + pending.publishing_generation = None; + } + } } - runtime_result(status) + } + + fn complete_publish_claims( + &mut self, + candidates: &[PublishCandidate], + published: &[PublishedClaim], + ) -> Result { + if published.len() != candidates.len() { + self.release_publish_claims(candidates); + return Err(nfsstat3::NFS3ERR_IO); + } + for candidate in candidates { + let handle = candidate + .handle + .to_str() + .map_err(|_| nfsstat3::NFS3ERR_IO)?; + if !published + .iter() + .any(|item| item.handle == handle && item.generation == candidate.generation) + { + self.release_publish_claims(candidates); + return Err(nfsstat3::NFS3ERR_IO); + } + } + let updated_at = now_ms(); + let mut completed = 0; + for candidate in candidates { + let matches = self.pending_writes.get(&candidate.inode).is_some_and(|pending| { + pending.handle.as_bytes() == candidate.handle.as_bytes() + && pending.generation == candidate.generation + && pending.publishing_generation == Some(candidate.generation) + }); + if !matches { + continue; + } + self.pending_writes.remove(&candidate.inode); + if let Some(stat) = self.stats.get_mut(&candidate.inode) { + stat.size = candidate.logical_size.min(i64::MAX as u64) as i64; + stat.updated_at = updated_at; + stat.changed_at = updated_at; + } + completed += 1; + } + Ok(completed) } fn truncate_file( &mut self, + inode: fileid3, path: &str, size: u64, prefix: &str, sequence: &AtomicU64, ) -> Result<(), nfsstat3> { + if let Some(pending) = self.pending_writes.get(&inode) { + if pending.publishing_generation.is_some() { + return Err(nfsstat3::NFS3ERR_JUKEBOX); + } + let handle = pending.handle.clone(); + let request = Self::request_id(prefix, sequence, "truncate-pending"); + let mut generation = 0; + let mut error = empty_error(); + let status = unsafe { + vexfs_mount_handle_truncate( + self.session, + handle.as_ptr(), + size, + request.as_ptr(), + &mut generation, + &mut error, + ) + }; + runtime_result(status)?; + let pending = self.pending_writes.get_mut(&inode).unwrap(); + pending.generation = generation; + pending.logical_size = size; + pending.updated_at_ms = now_ms(); + pending.dirty_bytes = pending.dirty_bytes.saturating_add(1); + self.publish_pending(inode, "full")?; + return Ok(()); + } let path = checked_cstring(path)?; let mode = CString::new("rw").unwrap(); let open_request = Self::request_id(prefix, sequence, "open-truncate"); @@ -846,18 +1381,6 @@ impl VexfsNfs { } } -impl Drop for VexfsNfs { - fn drop(&mut self) { - let request = self.request_id("shutdown-sync"); - let Ok(state) = self.state.lock() else { return }; - let mut published = 0; - let mut error = empty_error(); - unsafe { - vexfs_mount_synchronize(state.session, request.as_ptr(), &mut published, &mut error); - } - } -} - #[async_trait] impl NFSFileSystem for VexfsNfs { fn capabilities(&self) -> VFSCapabilities { @@ -870,6 +1393,7 @@ impl NFSFileSystem for VexfsNfs { async fn lookup(&self, dirid: fileid3, filename: &filename3) -> Result { let mut state = self.state.lock().map_err(|_| nfsstat3::NFS3ERR_IO)?; + state.refresh_metadata_cache_if_stale()?; if filename.as_ref() == b"." { return Ok(dirid); } @@ -883,8 +1407,17 @@ impl NFSFileSystem for VexfsNfs { return Ok(state.stat_path(parent)?.inode as u64); } if RuntimeState::target_name(filename.as_ref()).is_some() { + if !APPLEDOUBLE_SIDECARS_ENABLED { + return Err(nfsstat3::NFS3ERR_NOENT); + } return state.lookup_sidecar(dirid, filename.as_ref()); } + if let Some(entries) = state.directory_entries.get(&dirid) { + return entries + .get(filename.as_ref()) + .copied() + .ok_or(nfsstat3::NFS3ERR_NOENT); + } let path = state.child_path(dirid, filename.as_ref())?; Ok(state.stat_path(&path)?.inode as u64) } @@ -895,7 +1428,12 @@ impl NFSFileSystem for VexfsNfs { return state.sidecar_attr(id); } let path = state.path_for_inode(id)?; - Ok(stat_attr(&state.stat_path(&path)?)) + let mut attr = stat_attr(&state.stat_path(&path)?); + if let Some(pending) = state.pending_writes.get(&id) { + attr.size = pending.logical_size; + attr.used = pending.logical_size; + } + Ok(attr) } async fn setattr(&self, id: fileid3, attrs: sattr3) -> Result { @@ -936,7 +1474,13 @@ impl NFSFileSystem for VexfsNfs { })?; } if let set_size3::size(size) = attrs.size { - state.truncate_file(&path, size, &self.request_prefix, &self.request_sequence)?; + state.truncate_file( + id, + &path, + size, + &self.request_prefix, + &self.request_sequence, + )?; } let now = now_ms(); let (accessed, access_mask) = match attrs.atime { @@ -967,6 +1511,7 @@ impl NFSFileSystem for VexfsNfs { ) })?; } + state.stats.remove(&id); Ok(stat_attr(&state.stat_path(&path)?)) } @@ -977,18 +1522,29 @@ impl NFSFileSystem for VexfsNfs { count: u32, ) -> Result<(Vec, bool), nfsstat3> { let mut state = self.state.lock().map_err(|_| nfsstat3::NFS3ERR_IO)?; - let data = if let Some(sidecar) = state.sidecars.get(&id) { - sidecar.data.clone() - } else { - let path = state.path_for_inode(id)?; - state.read_file(&path)? - }; - let start = (offset as usize).min(data.len()); - let end = start.saturating_add(count as usize).min(data.len()); - Ok((data[start..end].to_vec(), end == data.len())) + if let Some(sidecar) = state.sidecars.get(&id) { + let start = (offset as usize).min(sidecar.data.len()); + let end = start.saturating_add(count as usize).min(sidecar.data.len()); + return Ok((sidecar.data[start..end].to_vec(), end == sidecar.data.len())); + } + let path = state.path_for_inode(id)?; + let stat = state.stat_path(&path)?; + let data = state.read_inode_range(id, &path, offset, count as u64)?; + let size = state + .pending_writes + .get(&id) + .map(|pending| pending.logical_size) + .unwrap_or(stat.size.max(0) as u64); + Ok((data, offset.saturating_add(count as u64) >= size)) } - async fn write(&self, id: fileid3, offset: u64, data: &[u8]) -> Result { + async fn write( + &self, + id: fileid3, + offset: u64, + data: &[u8], + stable_requested: bool, + ) -> Result<(fattr3, bool), nfsstat3> { let mut state = self.state.lock().map_err(|_| nfsstat3::NFS3ERR_IO)?; if state.sidecars.contains_key(&id) { let sidecar = state.sidecars.get_mut(&id).unwrap(); @@ -999,17 +1555,53 @@ impl NFSFileSystem for VexfsNfs { sidecar.data[offset..offset + data.len()].copy_from_slice(data); sidecar.updated_at_ms = now_ms(); state.persist_sidecar(id)?; - return state.sidecar_attr(id); + if state.strict_durability && stable_requested { + let request = self.request_id("nfs-sidecar-write"); + let mut published = 0; + let mut error = empty_error(); + runtime_result(unsafe { + vexfs_mount_synchronize( + state.session, + request.as_ptr(), + &mut published, + &mut error, + ) + })?; + } + return Ok(( + state.sidecar_attr(id)?, + !state.strict_durability || stable_requested, + )); } let path = state.path_for_inode(id)?; - state.write_file( + state.stage_file_write( + id, &path, offset, data, &self.request_prefix, &self.request_sequence, )?; - Ok(stat_attr(&state.stat_path(&path)?)) + if state.strict_durability && stable_requested { + state.publish_pending(id, "full")?; + return Ok((stat_attr(&state.stat_path(&path)?), true)); + } + let mut attr = stat_attr(&state.stat_path(&path)?); + if let Some(pending) = state.pending_writes.get(&id) { + attr.size = pending.logical_size; + attr.used = pending.logical_size; + } + // Balanced mode coalesces small files in database-backed staging, but + // once one handle has accumulated enough write traffic it must make + // the NFS client issue COMMIT at fsync/close. Otherwise a large active + // file can sit behind thousands of older idle files in the background + // queue and still be unpublished when the gateway is stopped. + let requires_foreground_commit = state + .pending_writes + .get(&id) + .is_some_and(|pending| pending.dirty_bytes >= DEFERRED_PUBLISH_DIRTY_BYTES); + // Strict mode reports UNSTABLE unless this request was actually FULL. + Ok((attr, !state.strict_durability && !requires_foreground_commit)) } async fn create( @@ -1024,6 +1616,9 @@ impl NFSFileSystem for VexfsNfs { set_mode3::Void => 0o644, }; if RuntimeState::target_name(filename.as_ref()).is_some() { + if !APPLEDOUBLE_SIDECARS_ENABLED { + return Err(nfsstat3::NFS3ERR_NOTSUPP); + } let key = SidecarKey { parent: dirid, name: filename.as_ref().to_vec(), @@ -1036,19 +1631,26 @@ impl NFSFileSystem for VexfsNfs { return Ok((inode, state.sidecar_attr(inode)?)); } let path = state.child_path(dirid, filename.as_ref())?; - state.create_file(&path, mode, &self.request_prefix, &self.request_sequence)?; - let stat = state.stat_path(&path)?; - let mut error = empty_error(); - runtime_result(unsafe { - vexfs_mount_chown( - state.session, - stat.inode, - state.uid as i64, - state.gid as i64, - &mut error, - ) - })?; - let stat = state.stat_path(&path)?; + let (handle, stat) = + state.create_file(&path, mode, &self.request_prefix, &self.request_sequence)?; + state.cache_stat(&path, &stat); + let dirty_at = now_ms(); + state.pending_writes.insert( + stat.inode as u64, + PendingWrite { + handle, + generation: 1, + logical_size: 0, + updated_at_ms: dirty_at, + first_dirty_at_ms: dirty_at, + dirty_bytes: 0, + publishing_generation: None, + }, + ); + if let Some(entries) = state.directory_entries.get_mut(&dirid) { + entries.insert(filename.as_ref().to_vec(), stat.inode as u64); + } + state.stats.remove(&dirid); Ok((stat.inode as u64, stat_attr(&stat))) } @@ -1084,19 +1686,47 @@ impl NFSFileSystem for VexfsNfs { &mut error, ) })?; + state.stats.remove(&(stat.inode as u64)); let stat = state.stat_path(&path)?; + if let Some(entries) = state.directory_entries.get_mut(&dirid) { + entries.insert(dirname.as_ref().to_vec(), stat.inode as u64); + } + state + .directory_entries + .insert(stat.inode as u64, HashMap::new()); + state.stats.remove(&dirid); Ok((stat.inode as u64, stat_attr(&stat))) } async fn remove(&self, dirid: fileid3, filename: &filename3) -> Result<(), nfsstat3> { let mut state = self.state.lock().map_err(|_| nfsstat3::NFS3ERR_IO)?; if RuntimeState::target_name(filename.as_ref()).is_some() { + if !APPLEDOUBLE_SIDECARS_ENABLED { + return Err(nfsstat3::NFS3ERR_NOENT); + } return state.remove_sidecar(dirid, filename.as_ref()); } let path = state.child_path(dirid, filename.as_ref())?; - let path = checked_cstring(&path)?; + let inode = state.stat_path(&path).ok().map(|stat| stat.inode as u64); + if let Some(inode) = inode { + state.publish_pending(inode, "none")?; + } + let path_c = checked_cstring(&path)?; let mut error = empty_error(); - runtime_result(unsafe { vexfs_mount_remove(state.session, path.as_ptr(), 0, &mut error) }) + runtime_result(unsafe { + vexfs_mount_remove(state.session, path_c.as_ptr(), 0, &mut error) + })?; + if let Some(inode) = inode { + state.paths.remove(&inode); + state.stats.remove(&inode); + state.directory_entries.remove(&inode); + } + state.inodes_by_path.remove(&path); + if let Some(entries) = state.directory_entries.get_mut(&dirid) { + entries.remove(filename.as_ref()); + } + state.stats.remove(&dirid); + Ok(()) } async fn rename( @@ -1110,6 +1740,9 @@ impl NFSFileSystem for VexfsNfs { let from_sidecar = RuntimeState::target_name(from_filename.as_ref()).is_some(); let to_sidecar = RuntimeState::target_name(to_filename.as_ref()).is_some(); if from_sidecar || to_sidecar { + if !APPLEDOUBLE_SIDECARS_ENABLED { + return Err(nfsstat3::NFS3ERR_NOTSUPP); + } if !from_sidecar || !to_sidecar { return Err(nfsstat3::NFS3ERR_INVAL); } @@ -1127,8 +1760,16 @@ impl NFSFileSystem for VexfsNfs { let new_inode = state.allocate_sidecar(key, target, data, 0o600); return state.persist_sidecar(new_inode); } - let source = checked_cstring(&state.child_path(from_dirid, from_filename.as_ref())?)?; - let destination = checked_cstring(&state.child_path(to_dirid, to_filename.as_ref())?)?; + let source_path = state.child_path(from_dirid, from_filename.as_ref())?; + let destination_path = state.child_path(to_dirid, to_filename.as_ref())?; + if let Ok(stat) = state.stat_path(&source_path) { + state.publish_pending(stat.inode as u64, "none")?; + } + if let Ok(stat) = state.stat_path(&destination_path) { + state.publish_pending(stat.inode as u64, "none")?; + } + let source = checked_cstring(&source_path)?; + let destination = checked_cstring(&destination_path)?; let mut error = empty_error(); runtime_result(unsafe { vexfs_mount_rename( @@ -1138,7 +1779,9 @@ impl NFSFileSystem for VexfsNfs { 1, &mut error, ) - }) + })?; + state.clear_metadata_cache(); + Ok(()) } async fn link( @@ -1153,6 +1796,7 @@ impl NFSFileSystem for VexfsNfs { { return Err(nfsstat3::NFS3ERR_NOTSUPP); } + state.publish_pending(id, "none")?; let source_path = state.path_for_inode(id)?; let destination_path = state.child_path(dirid, filename.as_ref())?; let source = checked_cstring(&source_path)?; @@ -1166,21 +1810,29 @@ impl NFSFileSystem for VexfsNfs { &mut error, ) })?; + state.inodes_by_path.insert(destination_path.clone(), id); + if let Some(entries) = state.directory_entries.get_mut(&dirid) { + entries.insert(filename.as_ref().to_vec(), id); + } + state.stats.remove(&id); + state.stats.remove(&dirid); Ok(stat_attr(&state.stat_path(&source_path)?)) } async fn commit(&self, id: fileid3, _offset: u64, _count: u32) -> Result { - let request = self.request_id("nfs-commit"); let mut state = self.state.lock().map_err(|_| nfsstat3::NFS3ERR_IO)?; if state.sidecars.contains_key(&id) { state.persist_sidecar(id)?; return state.sidecar_attr(id); } - let mut published = 0; - let mut error = empty_error(); - runtime_result(unsafe { - vexfs_mount_synchronize(state.session, request.as_ptr(), &mut published, &mut error) - })?; + if state.publish_pending(id, "full")? { + let path = state.path_for_inode(id)?; + return Ok(stat_attr(&state.stat_path(&path)?)); + } + // NFS COMMIT applies only to this file/range. macOS may emit a final + // COMMIT for an inode that is already clean while unmounting. Treating + // that as a workspace-wide synchronize publishes unrelated handles + // without closing them and leaves stale staging metadata behind. let path = state.path_for_inode(id)?; Ok(stat_attr(&state.stat_path(&path)?)) } @@ -1206,10 +1858,18 @@ impl NFSFileSystem for VexfsNfs { }; let end = start.saturating_add(max_entries).min(entries.len()); let complete = end == entries.len(); + let pending_sizes: HashMap = state + .pending_writes + .iter() + .map(|(inode, pending)| (*inode, pending.logical_size)) + .collect(); let entries = entries[start..end] .iter() .map(|entry| { - let stat: FileStat = entry.clone().into(); + let mut stat: FileStat = entry.clone().into(); + if let Some(size) = pending_sizes.get(&(entry.inode as u64)) { + stat.size = (*size).min(i64::MAX as u64) as i64; + } DirEntry { fileid: entry.inode as u64, name: entry.name.as_bytes().into(), @@ -1231,7 +1891,8 @@ impl NFSFileSystem for VexfsNfs { _attrs: &sattr3, ) -> Result<(fileid3, fattr3), nfsstat3> { let mut state = self.state.lock().map_err(|_| nfsstat3::NFS3ERR_IO)?; - let path = checked_cstring(&state.child_path(dirid, linkname.as_ref())?)?; + let path_text = state.child_path(dirid, linkname.as_ref())?; + let path = checked_cstring(&path_text)?; let mut error = empty_error(); runtime_result(unsafe { vexfs_mount_symlink( @@ -1242,8 +1903,11 @@ impl NFSFileSystem for VexfsNfs { &mut error, ) })?; - let path = path.to_str().map_err(|_| nfsstat3::NFS3ERR_IO)?; - let stat = state.stat_path(path)?; + let stat = state.stat_path(&path_text)?; + if let Some(entries) = state.directory_entries.get_mut(&dirid) { + entries.insert(linkname.as_ref().to_vec(), stat.inode as u64); + } + state.stats.remove(&dirid); Ok((stat.inode as u64, stat_attr(&stat))) } @@ -1304,6 +1968,7 @@ fn run_gateway(config: &VexfsNfsGatewayConfig) -> Result<(), String> { if config.port == 0 { return Err("NFS gateway port is required".to_string()); } + let postgresql_backend = backend == "postgresql"; let backend_c = CString::new(backend).map_err(|_| "backend contains NUL")?; let connection_c = CString::new(connection).map_err(|_| "connection contains NUL")?; let workspace_c = CString::new(workspace).map_err(|_| "workspace contains NUL")?; @@ -1330,20 +1995,30 @@ fn run_gateway(config: &VexfsNfsGatewayConfig) -> Result<(), String> { } let mut state = RuntimeState { session, + postgresql_backend, root_inode: 1, uid: unsafe { libc::getuid() }, gid: unsafe { libc::getgid() }, + strict_durability: std::env::var_os("VEXFS_NFS_STRICT_DURABILITY") + .is_some_and(|value| value != "0"), next_sidecar_inode: SIDECAR_INODE_START, sidecars_by_key: HashMap::new(), sidecars: HashMap::new(), + pending_writes: HashMap::new(), + paths: HashMap::new(), + inodes_by_path: HashMap::new(), + stats: HashMap::new(), + directory_entries: HashMap::new(), + metadata_cache_checked_at_ms: now_ms(), }; let root = state .stat_path("/") .map_err(|status| format!("root stat failed: {status:?}"))?; state.root_inode = root.inode as u64; let prefix = format!("nfs-{}-{}", unsafe { libc::getpid() }, now_ms()); + let shared_state = Arc::new(Mutex::new(state)); let filesystem = VexfsNfs { - state: Mutex::new(state), + state: Arc::clone(&shared_state), request_prefix: prefix, request_sequence: AtomicU64::new(1), }; @@ -1354,6 +2029,72 @@ fn run_gateway(config: &VexfsNfsGatewayConfig) -> Result<(), String> { .build() .map_err(|error| error.to_string())?; runtime.block_on(async move { + let publish_state = Arc::clone(&shared_state); + tokio::spawn(async move { + let mut interval = tokio::time::interval(std::time::Duration::from_millis(25)); + loop { + interval.tick().await; + let now = now_ms(); + let publish_job = { + let Ok(mut state) = publish_state.lock() else { + return; + }; + if state.postgresql_backend { + let candidates = state.claim_deferred_publish_batch(now); + if candidates.is_empty() { + None + } else { + Some((state.session as usize, candidates)) + } + } else { + let cutoff = now.saturating_sub(DEFERRED_PUBLISH_IDLE_MS); + let idle = state + .pending_writes + .values() + .map(|pending| pending.updated_at_ms) + .max() + .is_some_and(|updated_at_ms| updated_at_ms <= cutoff); + if idle { + if let Err(status) = state + .publish_pending_batch("full", DEFERRED_PUBLISH_BATCH_SIZE) + { + eprintln!( + "vexfs-nfs-gateway: deferred batch publish failed: {status:?}" + ); + } + } + None + } + }; + let Some((session, candidates)) = publish_job else { + continue; + }; + let result = publish_claimed_batch( + session as *mut RuntimeSessionOpaque, + &candidates, + ); + let Ok(mut state) = publish_state.lock() else { + return; + }; + match result { + Ok(published) => { + if let Err(status) = + state.complete_publish_claims(&candidates, &published) + { + eprintln!( + "vexfs-nfs-gateway: deferred publish result mismatch: {status:?}" + ); + } + } + Err(status) => { + state.release_publish_claims(&candidates); + eprintln!( + "vexfs-nfs-gateway: deferred claimed publish failed: {status:?}" + ); + } + } + } + }); let listener = NFSTcpListener::bind(&bind_address, filesystem) .await .map_err(|error| error.to_string())?; @@ -1428,3 +2169,125 @@ pub unsafe extern "C" fn vexfs_nfs_gateway_run( } } } + +#[cfg(test)] +#[export_name = "vexfs_mount_session_close"] +extern "C" fn test_vexfs_mount_session_close(_session: *mut RuntimeSessionOpaque) {} + +#[cfg(test)] +mod tests { + use super::*; + + fn state_with_pending(entries: &[(u64, i64, i64, u64)]) -> RuntimeState { + let mut pending_writes = HashMap::new(); + for (inode, updated_at_ms, first_dirty_at_ms, dirty_bytes) in entries { + pending_writes.insert( + *inode, + PendingWrite { + handle: CString::new(format!("handle-{inode}")).unwrap(), + generation: 1, + logical_size: *dirty_bytes, + updated_at_ms: *updated_at_ms, + first_dirty_at_ms: *first_dirty_at_ms, + dirty_bytes: *dirty_bytes, + publishing_generation: None, + }, + ); + } + RuntimeState { + session: ptr::null_mut(), + postgresql_backend: true, + root_inode: 1, + uid: 0, + gid: 0, + strict_durability: false, + next_sidecar_inode: SIDECAR_INODE_START, + sidecars_by_key: HashMap::new(), + sidecars: HashMap::new(), + pending_writes, + paths: HashMap::new(), + inodes_by_path: HashMap::new(), + stats: HashMap::new(), + directory_entries: HashMap::new(), + metadata_cache_checked_at_ms: 0, + } + } + + #[test] + fn foreground_write_burst_is_coalesced_until_globally_idle() { + let now = 10_000; + let mut state = state_with_pending(&[ + (1, now - DEFERRED_PUBLISH_IDLE_MS, now - DEFERRED_PUBLISH_IDLE_MS, 4_096), + (2, now, now, 4_096), + ]); + let claimed = state.claim_deferred_publish_batch(now); + assert!(claimed.is_empty()); + assert_eq!( + state.pending_writes.get(&1).unwrap().publishing_generation, + None + ); + assert_eq!( + state.pending_writes.get(&2).unwrap().publishing_generation, + None + ); + + let claimed = state.claim_deferred_publish_batch(now + DEFERRED_PUBLISH_IDLE_MS); + assert_eq!(claimed.len(), 2); + } + + #[test] + fn dirty_byte_threshold_flushes_before_idle_timeout() { + let now = 20_000; + let mut state = state_with_pending(&[( + 7, + now, + now, + DEFERRED_PUBLISH_DIRTY_BYTES, + )]); + let claimed = state.claim_deferred_publish_batch(now); + assert!(claimed.is_empty()); + let claimed = + state.claim_deferred_publish_batch(now + DEFERRED_PUBLISH_IDLE_MS); + assert_eq!(claimed.len(), 1); + assert_eq!(claimed[0].inode, 7); + } + + #[test] + fn oldest_dirty_write_has_a_bounded_publish_delay() { + let now = 30_000; + let mut state = state_with_pending(&[( + 9, + now - DEFERRED_PUBLISH_IDLE_MS, + now - DEFERRED_PUBLISH_MAX_AGE_MS, + 1, + )]); + let claimed = state.claim_deferred_publish_batch(now); + assert_eq!(claimed.len(), 1); + state.release_publish_claims(&claimed); + assert_eq!( + state.pending_writes.get(&9).unwrap().publishing_generation, + None + ); + } + + #[test] + fn file_count_threshold_bounds_a_continuous_small_file_burst() { + let now = 40_000; + let entries: Vec<_> = (0..DEFERRED_PUBLISH_FILE_THRESHOLD) + .map(|inode| { + let updated = if inode + 1 == DEFERRED_PUBLISH_FILE_THRESHOLD { + now + } else { + now - DEFERRED_PUBLISH_IDLE_MS + }; + (inode as u64 + 1, updated, updated, 2) + }) + .collect(); + let mut state = state_with_pending(&entries); + let claimed = state.claim_deferred_publish_batch(now); + assert_eq!(claimed.len(), DEFERRED_PUBLISH_BATCH_SIZE as usize); + assert!(claimed.iter().all(|candidate| { + candidate.inode != DEFERRED_PUBLISH_FILE_THRESHOLD as u64 + })); + } +} diff --git a/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/nfs_handlers.rs b/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/nfs_handlers.rs index eebb594a94..b15d433e82 100644 --- a/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/nfs_handlers.rs +++ b/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/nfs_handlers.rs @@ -1185,8 +1185,12 @@ pub async fn nfsproc3_write( Err(_) => nfs::pre_op_attr::Void, }; - match context.vfs.write(id, args.offset, &args.data).await { - Ok(fattr) => { + match context + .vfs + .write(id, args.offset, &args.data, args.stable != 0) + .await + { + Ok((fattr, committed)) => { debug!("write success {:?} --> {:?}", xid, fattr); let res = WRITE3resok { file_wcc: nfs::wcc_data { @@ -1194,7 +1198,11 @@ pub async fn nfsproc3_write( after: nfs::post_op_attr::attributes(fattr), }, count: args.count, - committed: stable_how::FILE_SYNC, + committed: if committed { + stable_how::FILE_SYNC + } else { + stable_how::UNSTABLE + }, verf: context.vfs.serverid(), }; make_success_reply(xid).serialize(output)?; diff --git a/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/vfs.rs b/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/vfs.rs index 89673d0742..1397362773 100644 --- a/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/vfs.rs +++ b/agent_files/mount/macos/nfs_gateway/vendor/nfsserve/src/vfs.rs @@ -120,12 +120,18 @@ pub trait NFSFileSystem: Sync { /// EOF must be flagged if the end of the file is reached by the read. async fn read(&self, id: fileid3, offset: u64, count: u32) -> Result<(Vec, bool), nfsstat3>; - /// Writes the contents of a file returning (bytes, EOF) + /// Writes file contents and returns (attributes, committed_to_stable_storage). /// Note that offset/count may go past the end of the file and that /// in that case, the file is extended. /// If not supported due to readonly file system /// this should return Err(nfsstat3::NFS3ERR_ROFS) - async fn write(&self, id: fileid3, offset: u64, data: &[u8]) -> Result; + async fn write( + &self, + id: fileid3, + offset: u64, + data: &[u8], + stable_requested: bool, + ) -> Result<(fattr3, bool), nfsstat3>; /// Creates a file with the following attributes. /// If not supported due to readonly file system @@ -223,7 +229,7 @@ pub trait NFSFileSystem: Sync { let res = fsinfo3 { obj_attributes: dir_attr, rtmax: 1024 * 1024, - rtpref: 1024 * 124, + rtpref: 1024 * 1024, rtmult: 1024 * 1024, wtmax: 1024 * 1024, wtpref: 1024 * 1024, diff --git a/docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md b/docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md index 5af801df2a..73cd301f68 100644 --- a/docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md +++ b/docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md @@ -4,21 +4,15 @@ - 所属产品：VexDB-Lite 的文件管理能力 VexFS - 日期：2026-07-24 - 分支：`feature/agent_files` -- 文档版本：3.1 +- 文档版本：3.2 - macOS 默认入口：本机 NFS gateway；FSKit 延后为可选原生增强 -- 当前阶段：Phase 2 PostgreSQL 功能与验收已完成；最后一次数据审计已让所有变更记录 - 认证用户、workspace、commit、对象和前后版本，并保留 workspace 删除证据。PG 16–19、 - Linux 真挂载、format v2 和备份性能已经按当前审计实现复跑。当前源码的 - `preview.37` Developer ID 包已通过 92 项文档 smoke、19 项 package Gate 和包内 PG runtime， - 已安装到两台 M1 并在远端通过 libpq 读写。真机复测确认 FSKit 已启用；用户再次唤醒系统后， - 本机 PG FSKit 13/13 场景、凭据生命周期和 macOS↔Linux 跨 gateway 已通过。该包没有 Apple - 公证票据，仍不能作为正式分发结论；两台 Mac 当前又通过文件/快照往返 50 项和 extension、 - 网络、数据库故障恢复 25 项。最初产品场景也已由两台 Mac 各运行一次真实 OpenCode 完成 - 串行闭环：7 + 9 + 21 条检查通过，两个完整工作区快照可比较并一键恢复。安装器的旧 UUID - 刷新逻辑已经修复。macOS 默认 NFS adapter、AppleDouble 隔离、NFSv3 LINK/COMMIT、 - 安装包 gateway 和 35 项真机 eval 已完成。原定 FSKit `preview.38` 不再是 0.1 阻塞项。 - 下一步是从干净提交生成签名公证包，在未启用 FSKit 的干净 Mac 跑安装 Gate，并补 - gateway crash、sleep/wake、文件锁和长期 Coding Agent 负载。 +- 当前阶段：Phase 2 PostgreSQL 数据库合同和本机默认 NFS 验收已完成。PG VexFS spec 8/8、 + libpq runtime 105 项、macOS 默认 NFS 14/14 场景共 252 项检查全部通过，无跳过。两个独立 + gateway 已同时挂载同一 PG workspace 并完成双向 `fsync`、版本、快照和清理。PG 已加入 + 64 KiB range read/COW、同 session 批量发布、正确的 NFS `UNSTABLE → COMMIT` 持久化语义 + 和 250 ms 跨 gateway 元数据刷新。历史 FSKit 两 Mac、macOS↔Linux、PG 16–19、备份、 + format v2 和真实 OpenCode 证据继续有效，但当前默认 NFS 源码对应的新公证包和第二台 Mac + NFS 复验还没有完成。下一步先做发行 Gate；数据库方向随后进入 DuckDB adapter。 ## 0. 2026-07-24 路线调整 @@ -62,9 +56,9 @@ VexFS FSKit extension。 同时确认两个性能和实现边界： -1. NFSv3 没有 macOS named attributes。macOS 26.3.1 为新文件写入 - `com.apple.provenance` 时会尝试写 `._文件名`。当前 gateway 已将 raw AppleDouble 数据吸收为 - 目标 inode xattr，用户树和数据库 dentry 都为 0 个 `._*`。 +1. NFSv3 没有透明的 macOS named attributes。macOS 26.3.1 会尝试用 `._文件名` 写 + AppleDouble。当前产品选择 Bash 目录干净：gateway 明确返回不支持，不创建 `._*`；数据库 + xattr 合同仍保留，需要透明挂载 xattr 时使用 FSKit 或后续 NFSv4。 2. 当前真实 SQLite adapter 创建 1,000 个 2-byte 文件为 4.862788 秒、205.643 files/s；同轮 APFS 为 0.059030 秒、16940.431 files/s，约慢 82.4 倍。它与此前 FSKit 约 175 files/s 同级，0.1 可用但不是性能达标终点。 @@ -158,7 +152,7 @@ vexdb fs --workspace workspace snapshot restore before-refactor - 不为了宣传“更快 grep”而阻塞挂载正确性、备份和跨机器工作区； - 不在核心文件合同中加入 Agent 专用的 prompt、memory 或语义字段。 -## 5. 当前基线：2026-07-24 +## 5. 当前基线：2026-07-26 | 范围 | 当前事实 | 主要缺口 | |---|---|---| @@ -166,11 +160,11 @@ vexdb fs --workspace workspace snapshot restore before-refactor | 文件语义 | 文件、目录、四类时间戳、并发 append、进程锁、mode、symlink、xattr、hardlink、owner/group 元数据、便携 ACL 已进入数据库合同和测试；PG 已执行 ACL 和继承 | SQLite 尚未执行完整 ACL 授权；特殊文件不支持 | | 版本恢复 | 单文件版本、workspace commit、snapshot、diff、expected-head restore、原生备份、retention、显式分批 GC、live quota 和 format v2 逻辑导入导出已实现 | 自动维护、history/staging/index/total quota 和 DuckDB 导入端未完成 | | 长期校验 | `chunked-v1` 使用不可变 manifest、64 KiB 块和逐块/整文件 SHA-256；只读 check 覆盖结构、引用、顺序、大小和流式内容校验 | 自动 repair 和跨文件通用去重不在当前范围 | -| macOS | 默认 NFS 已连接 SQLite runtime，45 项完整真机 eval、13 项 OpenCode eval 和 30 项 package smoke 通过；hardlink、COMMIT/fsync、AppleDouble 隔离、Git、单机锁、npm/Cargo、重挂载和快照恢复均已验证；macOS 13.0 deployment target 构建通过。preview.40 ad-hoc 完整打包通过 99 项文档/安装检查、18 项干净包安装 eval、哈希和签名校验；既有 FSKit 仍保留并有 preview.22/preview.37 真机证据 | 默认 NFS 尚未从干净提交生成公证包，也未在 macOS 13–15 或未启用 FSKit 的干净 Mac 验证；重复 crash/sleep-wake、跨机器锁、长期 Agent 和 x86_64 待补 | +| macOS | 默认 NFS 已连接 SQLite 和 PG runtime；PG 14/14 场景、252 项检查通过，SQLite 完整真机 eval、OpenCode、package smoke、hardlink、COMMIT/fsync、Git、单机锁、npm/Cargo、重挂载和快照恢复均有证据；NFSv3 xattr 明确不支持且不生成 `._*`；macOS 13.0 deployment target 构建通过；既有 FSKit 仍保留历史真机证据 | 当前默认 NFS 源码尚未从干净提交生成公证包，也未在 macOS 13–15 或未启用 FSKit 的干净 Mac 验证；重复 crash/sleep-wake、跨机器锁、长期 Agent 和 x86_64 待补 | | Linux | libfuse3 真实挂载已通过 root 和 uid 1000 各 9 组；时间戳、并发 append、进程锁、打开文件生命周期、强制卸载和 helper 崩溃恢复已验证；异常撤销后底层目录保持 0500，显式卸载恢复 0700；x86_64/AArch64 manylinux 安装包已在对应架构真机完成无 root 安装回归 | 更多干净发行版、长期运行和真实挂载安装回归仍不足 | | Windows | 只有平台边界和规划 | WinFsp adapter、安装签名、路径/ACL/SID 合同均未实现 | -| 远程共享 | PostgreSQL 已通过多 gateway、macOS FSKit、Linux FUSE、数据库重启、当前源码下的 macOS↔Linux 47 项、两台 Mac 文件/快照 50 项、双 Mac 串行真实 OpenCode 7 + 9 + 21 项，以及双机 extension 崩溃、网络中断和数据库停机 25 项 | 当前审计源码对应的公证包尚未生成，需要重跑双机安装 Gate；局域网直连 FSKit 还需要 macOS 本地网络授权；PG 文本索引尚未实现 | -| 性能规模 | SQLite 直连已完成 10 万文件；默认 NFS 1,000 小文件为 4.862788 秒、205.643 files/s，同轮 APFS 0.059030 秒、16940.431 files/s；此前 FSKit 为约 175 files/s | NFS 小文件仍比 APFS 慢约 82.4 倍；0.2 需批量提交、写回合并、metadata cache，并重跑 1 万/10 万和真实 Coding Agent | +| 远程共享 | PostgreSQL 已通过多 gateway、macOS FSKit、Linux FUSE、数据库重启、macOS↔Linux 47 项、两台 Mac 文件/快照 50 项、双 Mac 串行真实 OpenCode 7 + 9 + 21 项和故障恢复 25 项；当前默认 NFS 又完成本机双 gateway 15 项，PG `pg_trgm` 索引已实现 | 当前默认 NFS 源码对应的公证包和第二台 Mac NFS 复验尚未完成；同时覆盖同一文件只报告版本冲突，不自动合并 | +| 性能规模 | SQLite 直连已完成 10 万文件；SQLite staging 已改成 base manifest + 变长脏块覆盖；SQLite NFS 三轮 1,000 小文件中位数 904.400 files/s，16 MiB 写 383.600 MiB/s；quick 挂载 eval 的 8 MiB 写为 310.118 MiB/s、100 次随机覆盖为 3,522.124 ops/s；PG NFS 仍使用独立基线 | 小文件达到 AgentFS 的 97%，顺序写为 2.18 倍，随机写约 80%；仍需补 1 万文件、长 Agent、冷读、严格持久化和完整 SHA-256 发布成本基线 | “合同中已保存”不等于“所有平台已经完整执行”。例如 owner/group 和便携 ACL 已经可以保存、读取和恢复，但身份认证、权限判断和各系统的原生 ACL 映射仍属于后续阶段。 @@ -184,8 +178,8 @@ vexdb fs --workspace workspace snapshot restore before-refactor 本机 13 个真实 mount Gate 通过；另一台没有源码和完整 Xcode 的 M1 也完成安装、真实挂载、 Git、快照、deep check 和 archive v2 的 30 项 smoke； 4. **长期安全与分块内容模型已完成**：checksum/check、retention/GC、live quota、SQLite format v2 - 逻辑 export/import、chunk + manifest 和 append 合并发布已进入 spec、runtime/CLI smoke 和 eval； - 下一步集中解决小文件写入性能，并补 macOS x86_64 交付验证。 + 逻辑 export/import、chunk + manifest、base manifest + 脏块 staging 和 append 合并发布已进入 + runtime/CLI smoke 和 eval；下一步验证 1 万文件、长期工作区和 macOS x86_64 交付。 当前不把主要开发重心完全移出 Phase 1；但已按用户决定完成 PostgreSQL `0.4.0-alpha.1`： SQL 合同、长期内容模型、完整性检查、空间管理、format v2、ACL、审计、libpq HostStore、 @@ -314,8 +308,8 @@ SQLite 证据。当前 format v2 已支持 SQLite 和 PostgreSQL，DuckDB adapte 当前完成证据： -- 适配器版本为 `0.4.0-alpha.1`，7 个 PG spec、151 项公共合同/并发 eval、32 项 CLI runtime、 - 74 项 runtime 合同、 +- 适配器版本为 `0.4.0-alpha.1`，8 个 PG VexFS spec、151 项公共合同/并发 eval、32 项 CLI runtime、 + 105 项当前 libpq runtime 合同、 数据库重启恢复、16/17/18/19 版本矩阵均通过； - ACL、真实 role、跨 gateway 锁和缓存失效已经进入 PostgreSQL 合同；审计精确记录 `session_user`/role OID、workspace、commit、path/inode 和 before/after version，workspace @@ -337,6 +331,12 @@ SQLite 证据。当前 format v2 已支持 SQLite 和 PostgreSQL，DuckDB adapte 固定在 `5433` 的另一套 PostgreSQL； - 两台 Mac 的局域网 TCP、libpq 和 runtime 已通过。直连 FSKit 的最后一步需要用户在第二台 Mac 的系统设置中打开 VexDB Lite 的“本地网络”权限；Apple 不允许程序替用户打开该开关。 +- 当前默认 NFS 在本机连续通过 14/14 场景、252 项检查；两个独立 gateway 同挂一个 PG + workspace，第一端创建并 `fsync`、第二端继承并覆盖、第一端刷新读回，最后验证两个版本、 + 快照、`pending_handles=0` 和 `staging_bytes=0`； +- NFS 普通 WRITE 正确返回 `UNSTABLE`，COMMIT/fsync/500 ms idle batch/卸载用 `full` 发布； + PG 同 session 批量发布在一个事务内完成。千文件不再出现修复前的 60 秒 `rg` 超时，正式 + file version/commit/request 分别为 1.002/1.066/3.013 每文件。 完成条件： @@ -345,9 +345,10 @@ SQLite 证据。当前 format v2 已支持 SQLite 和 PostgreSQL，DuckDB adapte - PostgreSQL 原生备份恢复后文件、历史、权限和业务表处于同一时间点； - macOS 和 Linux 使用同一 mount 一致性测试，不增加 PostgreSQL 专用文件语义。 -当前完成条件已经由本机、Linux、跨系统和双机故障 Gate 证明，Phase 2 状态改为“功能与验收 -完成”。从当前干净提交生成并公证 `preview.38` 仍是必要的发行任务，但不再被误写成 -PostgreSQL 文件合同缺口；之后再决定先做 Windows 还是 DuckDB。 +当前数据库合同和本机默认 NFS 完成条件已经由本机、Linux、跨系统、历史双机和故障 Gate +证明，Phase 2 状态为“功能与本机验收完成”。从当前干净提交生成并公证新包、在第二台 Mac +按默认 NFS 重跑共享 workspace，仍是发行任务，但不再被误写成 PostgreSQL 文件合同缺口； +数据库方向下一步进入 DuckDB adapter，Windows mount adapter 按独立平台路线推进。 ### Phase 3：Windows 与跨系统预览 — 未开始 diff --git a/tests/eval/vexfs/README.md b/tests/eval/vexfs/README.md index 696017f46d..63b1b5969c 100644 --- a/tests/eval/vexfs/README.md +++ b/tests/eval/vexfs/README.md @@ -28,6 +28,11 @@ VEXFS_EVAL_NATIVE_SEARCH_TIMEOUT_SECONDS=900 \ tests/eval/vexfs/python.sh tests/eval/vexfs/run.py --mode stress --filter mount.scale-tree \ --mount-cli /absolute/path/to/vexfs --fail-on-skip +# 固定复现“1000 小文件后写 8 MiB、完整读、重挂载冷读”；失败时保留现场 +tests/eval/vexfs/python.sh tests/eval/vexfs/run.py --mode quick \ + --filter mount.scale-read-after-small-files \ + --mount-cli /absolute/path/to/vexdb --fail-on-skip + # 把性能预算也作为硬 Gate；默认只记录指标，避免不同机器误报 tests/eval/vexfs/python.sh tests/eval/vexfs/run.py --mode full --enforce-performance @@ -116,11 +121,22 @@ VEXDB_PG_CONTAINER=vexdb_pg19-vexfs-dev \ VEXDB_PG19_MATRIX_CONTAINER=vexdb_pg19-vexfs-dev \ bash tests/eval/vexfs/run_pg_version_matrix.sh -# macOS：真实 FSKit 的 13 组公共 mount 合同 +# macOS：默认通过真实 NFSv3 gateway 运行公共 mount 合同 VEXDB_PG_CONTAINER=vexdb_pg19-vexfs-dev \ VEXDB_PG_DSN=postgresql://postgres@127.0.0.1:5434/test \ bash tests/eval/vexfs/run_pg_macos_mount.sh +# PG overlay/publisher 专项：真实挂载性能、1000 小文件后冷读、跨会话缓存失效 +VEXFS_MACOS_PG_CASES='mount.performance mount.scale-read-after-small-files mount.external-cache-invalidation' \ +VEXDB_PG_DSN=postgresql://postgres@127.0.0.1:5434/test \ + bash tests/eval/vexfs/run_pg_macos_mount.sh + +# 强持久化：fsync、强制卸载、重挂载读取；strict 性能数字单独记录 +VEXFS_NFS_STRICT_DURABILITY=1 \ +VEXFS_MACOS_PG_CASES='mount.force-unmount mount.performance' \ +VEXDB_PG_DSN=postgresql://postgres@127.0.0.1:5434/test \ + bash tests/eval/vexfs/run_pg_macos_mount.sh + # Linux：真实 libfuse3，root 和 uid 1000 VEXDB_PG_CONTAINER=vexdb_pg19-vexfs-dev \ bash tests/eval/vexfs/run_pg_linux_mount.sh @@ -238,6 +254,9 @@ vexdb_sqlite/build/eval/vexfs-linux-mount/{root,uid-1000}//report.json - 备份：运行中的 SQLite online backup、恢复、未发布 staging 可见性。 - PostgreSQL 备份性能：有界验证 `pg_dump/pg_restore`、`pg_basebackup` + `pg_verifybackup` + 克隆启动、format v2 PG → SQLite；同时检查容器 1 GiB 内存上限、`oom_kill` 和 CLI 峰值 RSS。 +- PostgreSQL staging：基础 manifest + 64 KiB 脏块、范围读写、truncate 缩短后零填充、 + 未变化 chunk 复用、精确 generation claim、断线重试幂等、后台发布不持有前台 runtime 锁； + 调度器覆盖全局空闲、4 MiB 脏写、1024 文件、30 秒最长等待和活跃 handle 隔离。 - 随机模型：真实数据库状态与 Python 参考文件树持续比对。 - 性能：大量小文件、大文件顺序读写、分块 staging、随机 4 KiB 覆盖、备份吞吐； `performance.mount-contract-small-files` 额外模拟原子创建、macOS provenance xattr、 @@ -292,6 +311,16 @@ loopback gateway，不要求用户启用 FSKit；会真实运行 `mkdir`、`cat` hardlink、fsync、Git、AppleDouble 隔离、重挂载和快照恢复。条件不满足时必须失败或明确 报告 `SKIP`，不能用 C ABI smoke 冒充真实挂载通过。 +`mount.scale-read-after-small-files` 专门覆盖曾经出现过的规模退化：quick 模式先创建 +1000 个小文件，再写入并完整读取 8 MiB 文件，随后卸载、重挂载并校验冷读 SHA-256。 +full/stress 模式分别提高到 3000/5000 个小文件和 16/32 MiB；所有读取都采用 1 MiB +流式缓冲，不会把整个工作区一次性装进内存。 + +`mount.external-cache-invalidation` 覆盖缓存优化的正确性：先通过真实挂载填充文件和目录 +缓存，再由第二个独立数据库会话覆盖一个文件并创建一个文件；挂载端必须在 5 秒内看到 +新内容和新目录项。它防止为了小文件性能把缓存改成永不失效，也会在卸载后通过 CLI 再次 +核对数据库权威内容。 + `mount.real-linux-bash-git` 是 Linux 环境 Gate。构建中存在 `vexfs-fuse`，且 `/dev/fuse`、`fusermount3` 和 Git 可用时，它会以真实挂载目录验证完整 Git workspace，并在卸载和重挂载后再次运行 `git status`。 diff --git a/tests/eval/vexfs/benchmark_mount.py b/tests/eval/vexfs/benchmark_mount.py new file mode 100644 index 0000000000..1bc8fca265 --- /dev/null +++ b/tests/eval/vexfs/benchmark_mount.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +"""Small, bounded benchmark shared by native, VexFS, and competitor mounts.""" + +import argparse +import json +import os +from pathlib import Path +import statistics +import time + + +def timed(fn): + start = time.perf_counter() + value = fn() + return time.perf_counter() - start, value + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("root", type=Path) + parser.add_argument("--files", type=int, default=1000) + parser.add_argument("--rounds", type=int, default=3) + parser.add_argument("--large-mib", type=int, default=16) + args = parser.parse_args() + args.root.mkdir(parents=True, exist_ok=True) + rounds = [] + block = b"v" * (1024 * 1024) + + for run in range(args.rounds): + root = args.root / f"run-{run}" + root.mkdir() + + def create_small(): + for index in range(args.files): + (root / f"f-{index:05d}.txt").write_bytes(b"x\n") + + create_s, _ = timed(create_small) + list_s, entries = timed(lambda: list(os.scandir(root))) + names = [entry for entry in entries if entry.name.startswith("f-")] + appledouble_count = sum(entry.name.startswith("._") for entry in entries) + assert len(names) == args.files + + def stat_all(): + return sum(entry.stat().st_size for entry in names) + + stat_s, total_size = timed(stat_all) + assert total_size == args.files * 2 + large_path = root / "large.bin" + + def write_large(): + with large_path.open("wb", buffering=1024 * 1024) as handle: + for _ in range(args.large_mib): + handle.write(block) + handle.flush() + os.fsync(handle.fileno()) + + write_s, _ = timed(write_large) if args.large_mib else (0.0, None) + + def read_large(): + total = 0 + with large_path.open("rb", buffering=1024 * 1024) as handle: + while chunk := handle.read(1024 * 1024): + total += len(chunk) + return total + + read_s, read_bytes = timed(read_large) if args.large_mib else (0.0, 0) + assert read_bytes == args.large_mib * 1024 * 1024 + rounds.append({ + "create_seconds": create_s, + "create_files_per_second": args.files / create_s, + "list_seconds": list_s, + "listed_entries": len(entries), + "appledouble_entries": appledouble_count, + "stat_seconds": stat_s, + "write_seconds": write_s, + "write_mib_per_second": args.large_mib / write_s if write_s else 0.0, + "read_seconds": read_s, + "read_mib_per_second": args.large_mib / read_s if read_s else 0.0, + }) + + medians = { + key: statistics.median(round_[key] for round_ in rounds) + for key in rounds[0] + } + print(json.dumps({ + "files": args.files, + "round_count": args.rounds, + "large_mib": args.large_mib, + "rounds": rounds, + "median": medians, + }, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/eval/vexfs/run.py b/tests/eval/vexfs/run.py index e9ba048778..3542e537a1 100644 --- a/tests/eval/vexfs/run.py +++ b/tests/eval/vexfs/run.py @@ -3465,7 +3465,7 @@ def filesystem_cli_prefix(cli: Path, database: Path, workspace: str, "chmod-exec", "hardlink", "symlink", - "xattr", + "xattr-when-platform-driver-supports-it", "unicode-name", "stable-errno", "snapshot-restore", @@ -3530,7 +3530,7 @@ def prepare_real_mount_adapter(ctx: Context, base: Path, raise EvalSkip("macOS NFS gateway 或系统 mount_nfs 不可用") if system == "Linux" and not details.get("mount_ready"): raise EvalSkip(f"libfuse3 adapter={details.get('extension', 'unknown')}") - mount_point.mkdir() + mount_point.mkdir(parents=True) return RealMountAdapter(name, cli, database, mount_point, prefix, details, backend, connection_identity) @@ -3548,6 +3548,20 @@ def expect_mount_errno(ctx: Context, operation: Callable[[], Any], expected: int raise EvalFailure(f"{message}: 预期 errno={expected} 但操作成功") +def expect_mount_errno_one_of(ctx: Context, operation: Callable[[], Any], + expected: set[int], message: str) -> None: + ctx.checks += 1 + try: + operation() + except OSError as error: + if error.errno not in expected: + raise EvalFailure( + f"{message}: expected errno in {sorted(expected)}, " + f"actual={error.errno} ({error})") + return + raise EvalFailure(f"{message}: 预期失败但操作成功") + + def system_setxattr(path: Path, name: str, value: bytes) -> None: if hasattr(os, "setxattr"): os.setxattr(path, name, value) @@ -3639,9 +3653,35 @@ def mount_adapter_unmount(ctx: Context, adapter: RealMountAdapter, result = run_process(adapter.prefix + ["unmount", str(adapter.mount_point)], check=False, timeout=60) ctx.equal(result.returncode, 0, f"{adapter.name} {label}卸载退出码") + target_status: list[dict[str, Any]] | None = None if adapter.name == "fskit": # FSKit 在 umount 返回后仍会异步释放 volume identity；重挂载前给系统收尾时间。 time.sleep(2.0) + elif adapter.name == "nfs": + # umount 返回与 getmntinfo/注册表视图收敛不是同一个时刻。 + # 按目标轮询，既不依赖固定机器速度，也不受其他 VexFS 挂载影响。 + deadline = time.monotonic() + 3.0 + while time.monotonic() < deadline: + status = run_process( + adapter.prefix + ["--json", "mount", "status", + str(adapter.mount_point)], + check=False, timeout=10) + try: + target_status = json.loads(status.stdout) + if target_status == []: + break + except json.JSONDecodeError: + pass + time.sleep(0.1) + if target_status is None: + status = run_process( + adapter.prefix + ["--json", "mount", "status", str(adapter.mount_point)], + check=False, timeout=10) + try: + target_status = json.loads(status.stdout) + except json.JSONDecodeError: + target_status = None + ctx.equal(target_status, [], f"{adapter.name} {label}目标卸载状态") def run_shared_mount_operations(ctx: Context, adapter: RealMountAdapter) -> dict[str, Any]: @@ -3711,9 +3751,18 @@ def run_shared_mount_operations(ctx: Context, adapter: RealMountAdapter) -> dict xattr_name = "user.vexfs.conformance" xattr_value = b"stored-in-database\x00binary" - system_setxattr(data, xattr_name, xattr_value) - ctx.equal(system_getxattr(data, xattr_name), xattr_value, "共享合同 xattr 读取") - ctx.check(xattr_name in system_listxattr(data), "共享合同 xattr 列表") + xattr_supported = adapter.name != "nfs" + if xattr_supported: + system_setxattr(data, xattr_name, xattr_value) + ctx.equal(system_getxattr(data, xattr_name), xattr_value, "共享合同 xattr 读取") + ctx.check(xattr_name in system_listxattr(data), "共享合同 xattr 列表") + else: + expect_mount_errno_one_of( + ctx, + lambda: system_setxattr(data, xattr_name, xattr_value), + {errno.EPERM, errno.ENOTSUP}, + "macOS NFSv3 必须明确拒绝无法无污染持久化的 xattr", + ) script = root / "run.sh" script.write_text("#!/bin/sh\nprintf 'vexfs-conformance\\n'\n", encoding="utf-8") @@ -3736,8 +3785,9 @@ def run_shared_mount_operations(ctx: Context, adapter: RealMountAdapter) -> dict "hardlink_inode_equal": data.stat().st_ino == hardlink.stat().st_ino, "hardlink_count": data.stat().st_nlink, "symlink_target": os.readlink(symlink), + "xattr_supported": xattr_supported, "xattr_name": xattr_name, - "xattr_value": system_getxattr(data, xattr_name), + "xattr_value": system_getxattr(data, xattr_name) if xattr_supported else b"", "unicode": unicode_file.read_text(encoding="utf-8"), "script_output": run_process([str(script)], cwd=root).stdout, "entries": sorted(path.name for path in root.iterdir()), @@ -3757,8 +3807,9 @@ def verify_shared_mount_after_restore(ctx: Context, adapter: RealMountAdapter, ctx.equal(data.stat().st_ino, hardlink.stat().st_ino, "恢复后 hardlink inode") ctx.equal(data.stat().st_nlink, expected["hardlink_count"], "恢复后 hardlink count") ctx.equal(os.readlink(symlink), expected["symlink_target"], "恢复后 symlink") - ctx.equal(system_getxattr(data, expected["xattr_name"]), expected["xattr_value"], - "恢复后 xattr") + if expected["xattr_supported"]: + ctx.equal(system_getxattr(data, expected["xattr_name"]), expected["xattr_value"], + "恢复后 xattr") ctx.equal(unicode_file.read_text(encoding="utf-8"), expected["unicode"], "恢复后 Unicode") ctx.equal(run_process([str(script)], cwd=root).stdout, expected["script_output"], @@ -3770,13 +3821,13 @@ def verify_shared_mount_after_restore(ctx: Context, adapter: RealMountAdapter, "mode": expected["mode"], "hardlink_count": data.stat().st_nlink, "symlink_target": os.readlink(symlink), - "xattrs": system_listxattr(data), + "xattrs": system_listxattr(data) if expected["xattr_supported"] else [], "unicode_name": unicode_file.name, } @case("mount.cross-platform-conformance", "mount", - "macOS FSKit 与 Linux FUSE 执行同一份挂载、元数据、快照和重挂载合同") + "macOS NFS 与 Linux FUSE 执行共享挂载合同，并验证 NFSv3 xattr 边界") def cross_platform_mount_conformance(ctx: Context) -> dict[str, Any]: with tempfile.TemporaryDirectory(prefix="vexfs-mount-conformance-") as directory: base = Path(directory) @@ -3828,14 +3879,20 @@ def cross_platform_mount_conformance(ctx: Context) -> dict[str, Any]: mount_adapter_unmount(ctx, adapter, "第二次") after = json.loads(run_process(adapter.prefix + ["--json", "doctor"]).stdout) - ctx.equal(after["mount_count"], 0, "共享合同卸载后无挂载") + target_status = json.loads(run_process( + adapter.prefix + ["--json", "mount", "status", + str(adapter.mount_point)], check=False).stdout) + ctx.equal(target_status, [], "共享合同目标卸载后无挂载") ctx.equal(after["database"]["pending_handles"], 0, "共享合同无未发布句柄") ctx.equal(after["database"]["retained_handles"], 0, "共享合同无保留句柄") ctx.equal(after["database"]["staging_bytes"], 0, "共享合同无暂存字节") return { "contract_version": MOUNT_CONFORMANCE_VERSION, "adapter": adapter.name, - "capabilities": list(MOUNT_CONFORMANCE_CAPABILITIES), + "capabilities": [capability for capability in MOUNT_CONFORMANCE_CAPABILITIES + if adapter.name != "nfs" or not capability.startswith("xattr-")], + "limitations": (["macos-nfsv3-mounted-xattr"] + if adapter.name == "nfs" else []), "doctor_before": adapter.doctor_before, "doctor_after": after, "mounts": mounts, @@ -3844,6 +3901,75 @@ def cross_platform_mount_conformance(ctx: Context) -> dict[str, Any]: } +@case("mount.pg-shared-workspace", "mount", + "两个独立 NFS gateway 同时挂载一个 PG workspace 并双向继承") +def pg_shared_workspace(ctx: Context) -> dict[str, Any]: + backend = os.environ.get("VEXFS_MOUNT_BACKEND", "sqlite").lower() + if backend not in {"pg", "postgres", "postgresql"}: + raise EvalSkip("共享 workspace 用例只适用于 PostgreSQL") + if platform.system() != "Darwin": + raise EvalSkip("当前共享 gateway 用例需要 macOS NFS") + with tempfile.TemporaryDirectory(prefix="vexfs-pg-shared-") as directory: + base = Path(directory) + first = prepare_real_mount_adapter(ctx, base / "first", "pg-shared-local") + second = prepare_real_mount_adapter(ctx, base / "second", "pg-shared-local") + first_mounted = False + second_mounted = False + try: + mount_adapter_mount(ctx, first) + first_mounted = True + mount_adapter_mount(ctx, second) + second_mounted = True + + first_project = first.mount_point / "workspace" + second_project = second.mount_point / "workspace" + first_project.mkdir() + first_file = first_project / "shared.txt" + descriptor = os.open(first_file, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0o644) + try: + os.write(descriptor, b"created-by-first\n") + os.fsync(descriptor) + finally: + os.close(descriptor) + time.sleep(1.2) + ctx.equal(second_project.joinpath("shared.txt").read_bytes(), + b"created-by-first\n", "第二个 gateway 继承第一个写入") + + second_file = second_project / "shared.txt" + descriptor = os.open(second_file, os.O_TRUNC | os.O_WRONLY) + try: + os.write(descriptor, b"modified-by-second\n") + os.fsync(descriptor) + finally: + os.close(descriptor) + time.sleep(1.2) + ctx.equal(first_file.read_bytes(), b"modified-by-second\n", + "第一个 gateway 刷新第二个写入") + finally: + if second_mounted: + mount_adapter_unmount(ctx, second, "PG 共享第二挂载") + if first_mounted: + mount_adapter_unmount(ctx, first, "PG 共享第一挂载") + + snapshot_commit = int(run_process( + first.prefix + ["snapshot", "create", "two-gateway"]).stdout.strip()) + ctx.check(snapshot_commit > 0, "共享 workspace 快照") + history = json.loads(run_process( + first.prefix + ["--json", "history", "/workspace/shared.txt", "--limit", "100"] + ).stdout) + ctx.check(len(history) >= 2, "共享文件保留两个 gateway 的版本") + after = json.loads(run_process(first.prefix + ["--json", "doctor"]).stdout) + ctx.equal(after["database"]["pending_handles"], 0, "共享 workspace 无待发布句柄") + ctx.equal(after["database"]["staging_bytes"], 0, "共享 workspace 无暂存字节") + return { + "backend": first.backend, + "gateways": 2, + "snapshot_commit": snapshot_commit, + "file_versions": len(history), + "doctor_after": after, + } + + PORTABILITY_WORKSPACE = "cross-os" PORTABILITY_ACL = [ {"principal": "agent-portable", "effect": "allow", @@ -4621,7 +4747,6 @@ def real_mount_bash(ctx: Context) -> dict[str, Any]: reverse = run_process(prefix + ["cat", "/project/sub/result.txt"]) ctx.equal(reverse.stdout, b"agent\n", "数据库 CLI 反向读取挂载写入") after = json.loads(run_process(prefix + ["--json", "doctor"]).stdout) - ctx.equal(after["mount_count"], 0, "卸载后挂载数") ctx.equal(after["database"]["pending_handles"], 0, "卸载后未发布句柄") ctx.equal(after["database"]["retained_handles"], 0, "卸载后保留句柄") ctx.equal(after["database"]["staging_bytes"], 0, "卸载后暂存字节") @@ -4787,10 +4912,86 @@ def real_mount_posix_metadata(ctx: Context) -> dict[str, Any]: "database_verified": True} -@case("mount.performance", "mount", "真实 FSKit 分块顺序写和随机覆盖性能") +@case("mount.sqlite-live-cli-wal", "mount", + "SQLite 真实挂载在大 WAL 下允许第二个 CLI 反复读取且网关继续工作") +def real_mount_sqlite_live_cli_wal(ctx: Context) -> dict[str, Any]: + if platform.system() != "Darwin": + raise EvalSkip("SQLite WAL 多进程真机回归当前只在 macOS NFS 执行") + with tempfile.TemporaryDirectory(prefix="vexfs-mount-wal-peer-") as directory: + base = Path(directory) + adapter = prepare_real_mount_adapter(ctx, base, "wal-peer") + if adapter.backend != "sqlite": + raise EvalSkip("SQLite WAL 多进程回归不适用于 PostgreSQL backend") + mount_adapter_mount(ctx, adapter) + shm = Path(f"{adapter.database}-shm") + wal = Path(f"{adapter.database}-wal") + try: + root = adapter.mount_point / "wal-load" + root.mkdir() + for index in range(1_000): + (root / f"f-{index:04d}.txt").write_bytes(b"x\n") + large = root / "large.bin" + with large.open("wb", buffering=MIB) as stream: + for _ in range(8): + stream.write(b"v" * MIB) + stream.flush() + os.fsync(stream.fileno()) + + ctx.check(shm.exists() and wal.exists(), "真实挂载必须建立 WAL sidecar") + shm_before = shm.stat() + wal_before = wal.stat() + cli_reads = 100 + for _ in range(cli_reads): + result = run_process(adapter.prefix + ["cat", "/wal-load/f-0500.txt"]) + ctx.equal(result.stdout, b"x\n", "第二个 CLI 读取挂载中的 workspace") + shm_after_peer = shm.stat() + wal_after_peer = wal.stat() + ctx.equal(shm_after_peer.st_ino, shm_before.st_ino, + "第二个 CLI 不得替换 SQLite SHM") + ctx.check(shm_after_peer.st_size >= shm_before.st_size, + "第二个 CLI 不得截断 SQLite SHM") + + continued_files = 250 + for index in range(continued_files): + (root / f"after-peer-{index:04d}.txt").write_text( + f"after-peer-{index:04d}\n") + ctx.equal((root / "after-peer-0249.txt").read_text(), + "after-peer-0249\n", "第二个 CLI 退出后挂载网关继续读写") + reverse = run_process( + adapter.prefix + ["cat", "/wal-load/after-peer-0249.txt"]) + ctx.equal(reverse.stdout, b"after-peer-0249\n", + "挂载后续写入仍可由 CLI 读取") + mounts = json.loads(run_process( + adapter.prefix + ["--json", "mount", "status", + str(adapter.mount_point)]).stdout) + ctx.equal(len(mounts), 1, "WAL peer 压力后网关仍在运行") + finally: + mount_adapter_unmount(ctx, adapter, "SQLite WAL peer") + after = json.loads(run_process(adapter.prefix + ["--json", "doctor"]).stdout) + ctx.equal(after["mount_count"], adapter.doctor_before["mount_count"], + "WAL peer 回归卸载后恢复原有挂载数") + ctx.equal(after["database"]["pending_handles"], 0, + "WAL peer 回归卸载后未发布句柄") + return { + "cli": str(adapter.cli), + "small_files_before_peer": 1_000, + "large_mib_before_peer": 8, + "cli_peer_reads": cli_reads, + "continued_files": continued_files, + "shm_inode_before": shm_before.st_ino, + "shm_inode_after_peer": shm_after_peer.st_ino, + "shm_bytes_before": shm_before.st_size, + "shm_bytes_after_peer": shm_after_peer.st_size, + "wal_bytes_before": wal_before.st_size, + "wal_bytes_after_peer": wal_after_peer.st_size, + "doctor_after": after, + } + + +@case("mount.performance", "mount", "真实挂载的分块顺序写、范围读和随机覆盖性能") def real_mount_performance(ctx: Context) -> dict[str, Any]: if platform.system() != "Darwin": - raise EvalSkip("真实 FSKit mount 只在 macOS 执行") + raise EvalSkip("真实 macOS mount 性能测试只在 macOS 执行") sequential_mib = {"quick": 8, "full": 32, "stress": 64}[ctx.mode.name] random_writes = {"quick": 100, "full": 1_000, "stress": 2_000}[ctx.mode.name] with tempfile.TemporaryDirectory(prefix="vexfs-mount-perf-") as directory: @@ -4825,10 +5026,10 @@ def real_mount_performance(ctx: Context) -> dict[str, Any]: random_seconds = time.perf_counter() - started for offset in sampled_offsets: ctx.equal(os.pread(fd, len(patch), offset), patch, - "FSKit 随机覆盖读回") + "挂载盘范围读取必须读回随机覆盖") os.close(fd) fd = -1 - ctx.equal(path.stat().st_size, sequential_mib * MIB, "FSKit 性能文件长度") + ctx.equal(path.stat().st_size, sequential_mib * MIB, "挂载盘性能文件长度") ctx.budget("mount_sequential_seconds", sequential_seconds, 60.0) ctx.budget("mount_random_seconds", random_seconds, 120.0) finally: @@ -4836,7 +5037,6 @@ def real_mount_performance(ctx: Context) -> dict[str, Any]: os.close(fd) mount_adapter_unmount(ctx, adapter, "性能测试") after = json.loads(run_process(prefix + ["--json", "doctor"]).stdout) - ctx.equal(after["mount_count"], 0, "FSKit 性能测试卸载后挂载数") return { "cli": str(adapter.cli), "backend": adapter.backend, @@ -4852,6 +5052,171 @@ def real_mount_performance(ctx: Context) -> dict[str, Any]: } +@case("mount.scale-read-after-small-files", "mount", + "创建大量小文件后写入、完整读取并重挂载冷读大文件") +def real_mount_scale_read_after_small_files(ctx: Context) -> dict[str, Any]: + if platform.system() != "Darwin": + raise EvalSkip("小文件后大文件读取回归当前只在 macOS NFS 执行") + file_count = {"quick": 1_000, "full": 3_000, "stress": 5_000}[ + ctx.mode.name] + large_mib = {"quick": 8, "full": 16, "stress": 32}[ctx.mode.name] + with FailureArtifactDirectory(ctx, "vexfs-mount-scale-read-") as directory: + base = Path(directory) + adapter = prepare_real_mount_adapter(ctx, base, "scale-read") + root = adapter.mount_point / "scale-read" + large = root / "large.bin" + block = bytes(range(256)) * 4096 + expected_hash = hashlib.sha256(block * large_mib).hexdigest() + + def read_all() -> tuple[int, str]: + digest = hashlib.sha256() + total = 0 + with large.open("rb", buffering=MIB) as stream: + for chunk in iter(lambda: stream.read(MIB), b""): + total += len(chunk) + digest.update(chunk) + return total, digest.hexdigest() + + mount_adapter_mount(ctx, adapter) + try: + root.mkdir() + started = time.perf_counter() + for index in range(file_count): + (root / f"f-{index:05d}.txt").write_bytes(b"x\n") + create_seconds = time.perf_counter() - started + + started = time.perf_counter() + with large.open("wb", buffering=MIB) as stream: + for _ in range(large_mib): + stream.write(block) + stream.flush() + os.fsync(stream.fileno()) + write_seconds = time.perf_counter() - started + + started = time.perf_counter() + first_bytes, first_hash = read_all() + first_read_seconds = time.perf_counter() - started + ctx.equal(first_bytes, large_mib * MIB, + "大量小文件后的首次完整读取长度") + ctx.equal(first_hash, expected_hash, + "大量小文件后的首次完整读取校验和") + finally: + mount_adapter_unmount(ctx, adapter, "规模读取首次") + + database_stat = json.loads(run_process( + adapter.prefix + ["stat", "/scale-read/large.bin"]).stdout) + ctx.equal(database_stat["size"], large_mib * MIB, + "数据库权威视图中的大文件长度") + + mount_adapter_mount(ctx, adapter) + try: + started = time.perf_counter() + cold_bytes, cold_hash = read_all() + cold_read_seconds = time.perf_counter() - started + ctx.equal(cold_bytes, large_mib * MIB, "重挂载冷读长度") + ctx.equal(cold_hash, expected_hash, "重挂载冷读校验和") + finally: + mount_adapter_unmount(ctx, adapter, "规模读取重挂载") + + integrity = json.loads(run_process( + adapter.prefix + ["--json", "check", "--quick"]).stdout) + ctx.equal(integrity["ok"], True, "规模读取后的数据库检查") + ctx.equal(integrity["versions"], file_count + 1, + "卸载前的全部小文件和大文件必须形成正式版本") + ctx.equal(integrity["checked"]["manifests"], file_count + 1, + "卸载后 manifest 数量必须覆盖全部文件") + doctor_after = json.loads(run_process( + adapter.prefix + ["--json", "doctor"]).stdout) + ctx.equal(doctor_after["database"]["pending_handles"], 0, + "卸载后不能遗留未发布 handle") + ctx.equal(doctor_after["database"]["staging_bytes"], 0, + "卸载后不能遗留脏 staging 字节") + ctx.budget("mount_scale_create_seconds", create_seconds, 1_200.0) + ctx.budget("mount_scale_first_read_seconds", first_read_seconds, 60.0) + ctx.budget("mount_scale_cold_read_seconds", cold_read_seconds, 60.0) + return { + "adapter": adapter.name, + "backend": adapter.backend, + "small_files": file_count, + "create_seconds": round(create_seconds, 6), + "files_per_second": round(file_count / max(create_seconds, 1e-9), 3), + "large_mib": large_mib, + "write_seconds": round(write_seconds, 6), + "write_mib_per_second": round(large_mib / max(write_seconds, 1e-9), 3), + "first_read_seconds": round(first_read_seconds, 6), + "first_read_mib_per_second": round( + large_mib / max(first_read_seconds, 1e-9), 3), + "cold_read_seconds": round(cold_read_seconds, 6), + "cold_read_mib_per_second": round( + large_mib / max(cold_read_seconds, 1e-9), 3), + "sha256": expected_hash, + "database_version": database_stat["version"], + "doctor_after": doctor_after, + "integrity": integrity, + } + + +@case("mount.external-cache-invalidation", "mount", + "第二个数据库会话提交后，真实挂载只失效外部修改的元数据缓存") +def real_mount_external_cache_invalidation(ctx: Context) -> dict[str, Any]: + if platform.system() != "Darwin": + raise EvalSkip("外部提交缓存失效当前只在 macOS 真实挂载执行") + with FailureArtifactDirectory(ctx, "vexfs-mount-external-cache-") as directory: + base = Path(directory) + adapter = prepare_real_mount_adapter(ctx, base, "external-cache") + shared = adapter.mount_point / "shared.txt" + peer = adapter.mount_point / "peer.txt" + mount_adapter_mount(ctx, adapter) + try: + with shared.open("wb") as stream: + stream.write(b"old\n") + stream.flush() + os.fsync(stream.fileno()) + ctx.equal(shared.read_bytes(), b"old\n", "外部提交前的挂载内容") + initial_names = {entry.name for entry in adapter.mount_point.iterdir()} + ctx.check("peer.txt" not in initial_names, "peer 文件在外部提交前不存在") + time.sleep(1.0) + + run_process(adapter.prefix + ["write", "/shared.txt"], + input_data=b"new-from-peer\n") + run_process(adapter.prefix + ["write", "/peer.txt"], + input_data=b"peer-created\n") + + started = time.perf_counter() + deadline = started + 5.0 + observed_shared = b"" + observed_peer = False + while time.perf_counter() < deadline: + try: + observed_shared = shared.read_bytes() + observed_peer = peer.exists() and peer.read_bytes() == b"peer-created\n" + if observed_shared == b"new-from-peer\n" and observed_peer: + break + except OSError: + pass + time.sleep(0.1) + detection_seconds = time.perf_counter() - started + ctx.equal(observed_shared, b"new-from-peer\n", + "挂载读取第二个会话覆盖后的内容") + ctx.check(observed_peer, "挂载目录发现第二个会话创建的文件") + ctx.check("peer.txt" in {entry.name for entry in adapter.mount_point.iterdir()}, + "外部提交后目录缓存已失效") + finally: + mount_adapter_unmount(ctx, adapter, "外部提交缓存失效") + + ctx.equal(run_process(adapter.prefix + ["cat", "/shared.txt"]).stdout, + b"new-from-peer\n", "数据库权威内容保存 peer 覆盖") + ctx.equal(run_process(adapter.prefix + ["cat", "/peer.txt"]).stdout, + b"peer-created\n", "数据库权威内容保存 peer 新文件") + return { + "adapter": adapter.name, + "backend": adapter.backend, + "detection_seconds": round(detection_seconds, 6), + "shared_content": observed_shared.decode().strip(), + "peer_visible": observed_peer, + } + + @case("mount.git-workspace", "mount", "真实 FSKit mount 中运行完整 Git 工作区流程") def real_mount_git_workspace(ctx: Context) -> dict[str, Any]: if platform.system() != "Darwin": @@ -4869,13 +5234,21 @@ def real_mount_git_workspace(ctx: Context) -> dict[str, Any]: prefix = adapter.prefix mount_adapter_mount(ctx, adapter) project = mount_point / "project" + mounted_xattr_supported = adapter.name != "nfs" try: project.mkdir() - run_process([xattr, "-w", "com.vexfs.eval", "stored-in-sqlite", str(project)]) - xattr_value = run_process( - [xattr, "-p", "com.vexfs.eval", str(project)], timeout=120) - ctx.equal(xattr_value.stdout.strip(), b"stored-in-sqlite", - "挂载盘扩展属性读写") + if mounted_xattr_supported: + run_process([xattr, "-w", "com.vexfs.eval", "stored-in-sqlite", str(project)]) + xattr_value = run_process( + [xattr, "-p", "com.vexfs.eval", str(project)], timeout=120) + ctx.equal(xattr_value.stdout.strip(), b"stored-in-sqlite", + "挂载盘扩展属性读写") + else: + rejected_xattr = run_process( + [xattr, "-w", "com.vexfs.eval", "stored-in-sqlite", str(project)], + check=False, timeout=120) + ctx.check(rejected_xattr.returncode != 0, + "macOS NFSv3 明确拒绝 AppleDouble xattr") environment = os.environ.copy() environment.update({ "GIT_CONFIG_NOSYSTEM": "1", @@ -4945,17 +5318,17 @@ def real_mount_git_workspace(ctx: Context) -> dict[str, Any]: mount_adapter_mount(ctx, adapter) try: - persisted_xattr = run_process( - [xattr, "-p", "com.vexfs.eval", str(project)], timeout=120) - ctx.equal(persisted_xattr.stdout.strip(), b"stored-in-sqlite", - "重挂载后扩展属性持久化") + if mounted_xattr_supported: + persisted_xattr = run_process( + [xattr, "-p", "com.vexfs.eval", str(project)], timeout=120) + ctx.equal(persisted_xattr.stdout.strip(), b"stored-in-sqlite", + "重挂载后扩展属性持久化") ctx.equal(run_process([git, "status", "--porcelain"], cwd=project, env=environment, timeout=120).stdout, b"", "重挂载后 Git 工作区状态") finally: mount_adapter_unmount(ctx, adapter, "Git 第二次") after = json.loads(run_process(prefix + ["--json", "doctor"]).stdout) - ctx.equal(after["mount_count"], 0, "Git 测试卸载后挂载数") ctx.equal(after["database"]["pending_handles"], 0, "Git 测试未发布句柄") ctx.equal(after["database"]["retained_handles"], 0, "Git 测试保留句柄") ctx.equal(after["database"]["staging_bytes"], 0, "Git 测试暂存字节") @@ -4965,6 +5338,7 @@ def real_mount_git_workspace(ctx: Context) -> dict[str, Any]: "git": run_process([git, "--version"]).stdout.decode().strip(), "filemode": True, "symlinks": True, + "mounted_xattr_supported": mounted_xattr_supported, "commands": ["xattr", "init", "config", "chmod", "symlink", "add", "commit", "checkout", "status", "gc"], "doctor_after": after, diff --git a/tests/eval/vexfs/run_pg_macos_mount.sh b/tests/eval/vexfs/run_pg_macos_mount.sh index 09bc51bf18..0a20a82d8d 100755 --- a/tests/eval/vexfs/run_pg_macos_mount.sh +++ b/tests/eval/vexfs/run_pg_macos_mount.sh @@ -6,8 +6,9 @@ PYTHON="$ROOT/tests/eval/vexfs/python.sh" PG_CONTAINER="${VEXDB_PG_CONTAINER:-vexdb_pg19-test}" DSN="${VEXDB_PG_DSN:-postgresql://postgres@127.0.0.1:5433/test}" MOUNT_CLI="${VEXFS_EVAL_MOUNT_CLI:-$HOME/.local/bin/vexdb}" -OUTPUT="${VEXFS_MACOS_PG_OUTPUT:-$ROOT/vexdb_sqlite/build/eval/vexfs-pg-macos-mount}" -CASES="${VEXFS_MACOS_PG_CASES:-mount.cross-platform-conformance mount.timestamps mount.concurrent-append mount.open-rename-unlink mount.read-only-open-lifecycle mount.process-locks mount.force-unmount mount.real-bash mount.posix-metadata mount.performance mount.git-workspace mount.real-toolchain-projects mount.scale-tree}" +BUILD_DIR="${VEXFS_MACOS_PG_BUILD_DIR:-$ROOT/vexdb_sqlite/build}" +OUTPUT="${VEXFS_MACOS_PG_OUTPUT:-$BUILD_DIR/eval/vexfs-pg-macos-mount}" +CASES="${VEXFS_MACOS_PG_CASES:-mount.cross-platform-conformance mount.pg-shared-workspace mount.timestamps mount.concurrent-append mount.open-rename-unlink mount.read-only-open-lifecycle mount.process-locks mount.force-unmount mount.real-bash mount.posix-metadata mount.performance mount.scale-read-after-small-files mount.external-cache-invalidation mount.git-workspace mount.real-toolchain-projects mount.scale-tree}" [ "$(uname -s)" = Darwin ] || { echo "该脚本只在 macOS 上运行" >&2; exit 2; } [ -x "$MOUNT_CLI" ] || { echo "找不到已安装的签名 CLI：$MOUNT_CLI" >&2; exit 2; } @@ -16,7 +17,8 @@ docker inspect "$PG_CONTAINER" >/dev/null cleanup_pg_workspaces() { local workspace for workspace in conformance timestamps append open-life read-life locks \ - force-unmount eval posix perf git-eval toolchains mount-scale opencode; do + force-unmount eval posix perf git-eval toolchains mount-scale opencode \ + pg-shared-local scale-read external-cache; do docker exec "$PG_CONTAINER" psql -U postgres -d test -X -q \ -v ON_ERROR_STOP=1 \ -c "SELECT vexfs_workspace_drop('$workspace', true);" \ @@ -60,7 +62,7 @@ for test_case in "${mount_cases[@]}"; do VEXFS_EVAL_OPENCODE_MODEL="${VEXFS_EVAL_OPENCODE_MODEL:-openai/gpt-5.4-mini}" \ "$PYTHON" "$ROOT/tests/eval/vexfs/run.py" \ --root "$ROOT" \ - --build-dir "$ROOT/vexdb_sqlite/build" \ + --build-dir "$BUILD_DIR" \ --output-dir "$OUTPUT" \ --mode quick \ --mount-cli "$MOUNT_CLI" \ diff --git a/tests/spec/_lib/docker/run_sqlite.sh b/tests/spec/_lib/docker/run_sqlite.sh index d8be384765..e97e68d699 100644 --- a/tests/spec/_lib/docker/run_sqlite.sh +++ b/tests/spec/_lib/docker/run_sqlite.sh @@ -16,7 +16,40 @@ set -u ROOT_DIR="$(cd "$(dirname "$0")/../../../.." && pwd)" FILTER="${1:-}" +PYTHON_BIN="${VEXDB_LITE_PYTHON:-}" -python3 "$ROOT_DIR/tests/spec/_lib/render.py" --engine sqlite --out build/spec >/dev/null || exit 1 -exec python3 "$ROOT_DIR/tests/spec/_lib/docker/sqlite_spec_runner.py" \ +python_is_usable() { + "$1" -c 'import sqlite3,sys; connection=sqlite3.connect(":memory:"); raise SystemExit(sys.version_info < (3, 8) or not hasattr(connection, "enable_load_extension"))' \ + >/dev/null 2>&1 +} + +if [[ -n "$PYTHON_BIN" ]]; then + [[ -x "$PYTHON_BIN" ]] || { + echo "VEXDB_LITE_PYTHON 不可执行：$PYTHON_BIN" >&2 + exit 2 + } + python_is_usable "$PYTHON_BIN" || { + echo "VEXDB_LITE_PYTHON 需要 Python 3.8+ 和 SQLite 扩展加载能力：$PYTHON_BIN" >&2 + exit 2 + } +else + for candidate in \ + "$(command -v python3 2>/dev/null || true)" \ + /opt/homebrew/bin/python3 \ + /opt/anaconda3/bin/python3 \ + /usr/bin/python3; do + [[ -n "$candidate" && -x "$candidate" ]] || continue + if python_is_usable "$candidate"; then + PYTHON_BIN="$candidate" + break + fi + done + [[ -n "$PYTHON_BIN" ]] || { + echo "SQLite spec 需要 Python 3.8+ 和 SQLite 扩展加载能力" >&2 + exit 2 + } +fi + +"$PYTHON_BIN" "$ROOT_DIR/tests/spec/_lib/render.py" --engine sqlite --out build/spec >/dev/null || exit 1 +exec "$PYTHON_BIN" "$ROOT_DIR/tests/spec/_lib/docker/sqlite_spec_runner.py" \ "$ROOT_DIR" "$FILTER" "${VEXDB_SQLITE_EXTENSION:-}" diff --git a/tests/spec/pg/agent_files/vexfs_pg_chunk_check.yaml b/tests/spec/pg/agent_files/vexfs_pg_chunk_check.yaml index 9e88c691c3..b7db84b96a 100644 --- a/tests/spec/pg/agent_files/vexfs_pg_chunk_check.yaml +++ b/tests/spec/pg/agent_files/vexfs_pg_chunk_check.yaml @@ -48,6 +48,36 @@ steps: - query: SELECT vexfs_read('pg-a2', '/large.bin') = convert_to(repeat('a', 65536) || repeat('b', 65536) || repeat('c', 7), 'UTF8'); expect: - - true +- query: SELECT convert_from(vexfs_read_range('pg-a2', '/large.bin', 65530, 20), 'UTF8'), octet_length(vexfs_read_range('pg-a2', '/large.bin', 131072, 64)); + expect: + - - aaaaaabbbbbbbbbbbbbb + - 7 +- query: SELECT vexfs_read_range('pg-a2', '/large.bin', -1, 1); + expect_error: VEXFS_INVALID_RANGE +- query: SELECT vexfs_mount_session_start('pg-a2', 'range-session'); + expect: + - - 1 +- query: SELECT vexfs_write_range('pg-a2', '/large.bin', 65534, convert_to('RANGE', 'UTF8'), 'range-write-1', 'range-session', 'full'); + expect: + - - 4 +- query: SELECT vexfs_write_range('pg-a2', '/large.bin', 65534, convert_to('RANGE', 'UTF8'), 'range-write-1', 'range-session', 'full'); + expect: + - - 4 +- query: SELECT convert_from(vexfs_read_range('pg-a2', '/large.bin', 65530, 16), 'UTF8'), count(*) FROM _vexfs.file_versions WHERE workspace_id=(SELECT workspace_id FROM _vexfs.workspaces WHERE name='pg-a2') GROUP BY 1; + expect: + - - aaaaRANGEbbbbbbb + - 4 +- query: SELECT count(*) FROM _vexfs.manifest_chunks AS old_entry JOIN _vexfs.manifest_chunks AS new_entry ON new_entry.chunk_id=old_entry.chunk_id WHERE old_entry.manifest_id=(SELECT manifest_id FROM _vexfs.file_versions WHERE workspace_id=(SELECT workspace_id FROM _vexfs.workspaces WHERE name='pg-a2') AND version_no=1) AND new_entry.manifest_id=(SELECT manifest_id FROM _vexfs.file_versions WHERE workspace_id=(SELECT workspace_id FROM _vexfs.workspaces WHERE name='pg-a2') AND version_no=4); + expect: + - - 1 +- query: SELECT vexfs_write_range('pg-a2', '/large.bin', 134217728, ''::bytea, 'range-write-empty', 'range-session', 'full'), (SELECT size_bytes FROM _vexfs.inodes WHERE workspace_id=(SELECT workspace_id FROM _vexfs.workspaces WHERE name='pg-a2') AND current_version=4), (SELECT count(*) FROM _vexfs.file_versions WHERE workspace_id=(SELECT workspace_id FROM _vexfs.workspaces WHERE name='pg-a2')); + expect: + - - 4 + - 131079 + - 4 +- query: SELECT vexfs_mount_session_end('pg-a2', 'range-session'); + expect: + - - 1 - query: SELECT (vexfs_check('pg-a2', 0)->>'ok')::boolean, vexfs_check('pg-a2', 0)->>'mode', vexfs_check('pg-a2', 0)->>'content_model'; expect: - - true @@ -56,14 +86,16 @@ steps: - query: SELECT (vexfs_check('pg-a2', 1)->>'ok')::boolean, (vexfs_check('pg-a2', 1)->'checked'->>'canonical_versions')::bigint, (vexfs_check('pg-a2', 1)->>'issue_count')::integer; expect: - - true - - 2 + - 3 - 0 -- statement: UPDATE _vexfs.chunks SET content=set_byte(content, 0, (get_byte(content, 0)+1)%256) WHERE chunk_id=(SELECT entry.chunk_id FROM _vexfs.file_versions AS f JOIN _vexfs.manifest_chunks AS entry ON entry.manifest_id=f.manifest_id WHERE f.workspace_id=(SELECT workspace_id FROM _vexfs.workspaces WHERE name='pg-a2') AND f.version_no=1 ORDER BY entry.chunk_no LIMIT 1); +- statement: UPDATE _vexfs.chunks SET content=set_byte(content, 0, (get_byte(content, 0)+1)%256) WHERE chunk_id=(SELECT entry.chunk_id FROM _vexfs.file_versions AS f JOIN _vexfs.manifest_chunks AS entry ON entry.manifest_id=f.manifest_id WHERE f.workspace_id=(SELECT workspace_id FROM _vexfs.workspaces WHERE name='pg-a2') AND f.version_no=4 ORDER BY entry.chunk_no LIMIT 1); - query: SELECT (vexfs_check('pg-a2', 0)->>'ok')::boolean, (vexfs_check('pg-a2', 1)->>'ok')::boolean, (vexfs_check('pg-a2', 1)->>'issue_count')::integer > 0; expect: - - true - false - true +- query: SELECT vexfs_write_range('pg-a2', '/large.bin', 0, convert_to('x', 'UTF8'), 'range-after-corruption', NULL, 'full'); + expect_error: VEXFS_CHECKSUM_MISMATCH - query: SELECT vexfs_read('pg-a2', '/large.bin'); expect_error: VEXFS_CHECKSUM_MISMATCH - query: SELECT vexfs_check('pg-a2', 2); diff --git a/tests/spec/pg/agent_files/vexfs_pg_handles_gateway.yaml b/tests/spec/pg/agent_files/vexfs_pg_handles_gateway.yaml index 926ff54ae4..f62df0aecf 100644 --- a/tests/spec/pg/agent_files/vexfs_pg_handles_gateway.yaml +++ b/tests/spec/pg/agent_files/vexfs_pg_handles_gateway.yaml @@ -16,6 +16,54 @@ steps: expect: - - 1 - statement: CREATE TEMP TABLE handle_ids(name text PRIMARY KEY, handle text NOT NULL); +- statement: INSERT INTO handle_ids SELECT 'owned-create', created.handle FROM vexfs_handle_create_stat('pg-handles', '/owned.txt', 416, 501, 20, 'create-owned', 'gateway-a') AS created; +- query: SELECT count(*), min((vexfs_stat('pg-handles', '/owned.txt')->>'version')::bigint), min((vexfs_stat('pg-handles', '/owned.txt')->>'uid')::bigint), min((vexfs_stat('pg-handles', '/owned.txt')->>'gid')::bigint), min(octet_length(vexfs_read_range('pg-handles', '/owned.txt', 0, 64))) FROM vexfs_list('pg-handles', '/') WHERE name='owned.txt'; + expect: + - - 1 + - 0 + - 501 + - 20 + - 0 +- query: SELECT (vexfs_check('pg-handles', 1)->>'ok')::boolean; + expect: + - - true +- query: SELECT vexfs_read_range('pg-handles', '/owned.txt', -1, 1); + expect_error: VEXFS_INVALID_RANGE +- query: SELECT vexfs_read_range('pg-handles', '/owned.txt', 0, 134217729); + expect_error: VEXFS_INVALID_RANGE +- query: SELECT vexfs_handle_stage_write((SELECT handle FROM handle_ids WHERE name='owned-create'), 0, convert_to('owned-data', 'UTF8'), 'stage-owned'); + expect: + - - 2 +- query: SELECT vexfs_handle_publish_close((SELECT handle FROM handle_ids WHERE name='owned-create'), 2, 'full'); + expect: + - - 1 +- query: SELECT convert_from(vexfs_read('pg-handles', '/owned.txt'), 'UTF8'), (vexfs_stat('pg-handles', '/owned.txt')->>'mode')::integer; + expect: + - - owned-data + - 416 +- statement: INSERT INTO handle_ids VALUES ('batch-a', vexfs_handle_create('pg-handles', '/batch-a.txt', 420, 501, 20, 'create-batch-a', 'gateway-a')); +- statement: INSERT INTO handle_ids VALUES ('batch-b', vexfs_handle_create('pg-handles', '/batch-b.txt', 420, 501, 20, 'create-batch-b', 'gateway-a')); +- query: SELECT vexfs_handle_stage_write((SELECT handle FROM handle_ids WHERE name='batch-a'), 0, convert_to('batch-a', 'UTF8'), 'stage-batch-a'), vexfs_handle_stage_write((SELECT handle FROM handle_ids WHERE name='batch-b'), 0, convert_to('batch-b', 'UTF8'), 'stage-batch-b'); + expect: + - - 2 + - 2 +- query: SELECT vexfs_mount_publish_close_all('pg-handles', 'gateway-a', 'full', 1); + expect: + - - 1 +- query: SELECT count(*) FROM _vexfs.handles WHERE handle_id IN ((SELECT handle FROM handle_ids WHERE name='batch-a'), (SELECT handle FROM handle_ids WHERE name='batch-b')) AND state='closed'; + expect: + - - 1 +- query: SELECT vexfs_mount_publish_close_all('pg-handles', 'gateway-a', 'full'); + expect: + - - 1 +- query: SELECT convert_from(vexfs_read('pg-handles', '/batch-a.txt'), 'UTF8'), convert_from(vexfs_read('pg-handles', '/batch-b.txt'), 'UTF8'), (SELECT count(*) FROM _vexfs.handles WHERE handle_id IN ((SELECT handle FROM handle_ids WHERE name='batch-a'), (SELECT handle FROM handle_ids WHERE name='batch-b')) AND state='closed'); + expect: + - - batch-a + - batch-b + - 2 +- query: SELECT vexfs_mount_publish_close_all('pg-handles', 'gateway-a', 'full'); + expect: + - - 0 - statement: INSERT INTO handle_ids VALUES ('reader', vexfs_handle_open('pg-handles', '/value.txt', 'r', 'open-reader', 'gateway-a')); - statement: INSERT INTO handle_ids VALUES ('a', vexfs_handle_open('pg-handles', '/value.txt', 'rw', 'open-a', 'gateway-a')); - statement: INSERT INTO handle_ids VALUES ('b', vexfs_handle_open('pg-handles', '/value.txt', 'rw', 'open-b', 'gateway-b')); diff --git a/tests/spec/pg/agent_files/vexfs_pg_staging_overlay.yaml b/tests/spec/pg/agent_files/vexfs_pg_staging_overlay.yaml new file mode 100644 index 0000000000..26df0c0dd9 --- /dev/null +++ b/tests/spec/pg/agent_files/vexfs_pg_staging_overlay.yaml @@ -0,0 +1,108 @@ +name: pg__vexfs_staging_overlay +tags: [vexfs, pg, handles, staging, overlay, performance] +description: PostgreSQL handle staging 使用基础 manifest 和 64 KiB 脏块，范围写不复制整份文件 +steps: +- statement: TRUNCATE _vexfs.workspaces CASCADE; +- query: SELECT vexfs_workspace_create('pg-overlay') > 0; + expect: + - - true +- query: SELECT vexfs_write('pg-overlay', '/large.bin', convert_to(repeat('a', 65536) || repeat('b', 65536) || repeat('c', 7), 'UTF8')); + expect: + - - 1 +- query: SELECT vexfs_mount_session_start('pg-overlay', 'overlay-gateway'); + expect: + - - 1 +- statement: CREATE TEMP TABLE overlay_handle(handle text PRIMARY KEY); +- statement: INSERT INTO overlay_handle VALUES (vexfs_handle_open('pg-overlay', '/large.bin', 'rw', 'overlay-open', 'overlay-gateway')); +- query: SELECT staging.base_manifest_id IS NOT NULL, staging.base_size, staging.base_visible_size, staging.logical_size, staging.dirty_bytes, count(dirty.chunk_no) FROM _vexfs.handle_staging AS staging LEFT JOIN _vexfs.handle_staging_chunks AS dirty USING(handle_id) WHERE staging.handle_id=(SELECT handle FROM overlay_handle) GROUP BY staging.base_manifest_id, staging.base_size, staging.base_visible_size, staging.logical_size, staging.dirty_bytes; + expect: + - - true + - 131079 + - 131079 + - 131079 + - 0 + - 0 +- query: SELECT vexfs_handle_stage_write((SELECT handle FROM overlay_handle), 65534, convert_to('RANGE', 'UTF8'), 'overlay-stage'); + expect: + - - 1 +- query: SELECT staging.logical_size, staging.dirty_bytes, count(dirty.chunk_no), coalesce(sum(octet_length(dirty.content)), 0) FROM _vexfs.handle_staging AS staging LEFT JOIN _vexfs.handle_staging_chunks AS dirty USING(handle_id) WHERE staging.handle_id=(SELECT handle FROM overlay_handle) GROUP BY staging.logical_size, staging.dirty_bytes; + expect: + - - 131079 + - 131072 + - 2 + - 131072 +- query: SELECT convert_from(vexfs_handle_read((SELECT handle FROM overlay_handle), 65530, 16), 'UTF8'); + expect: + - - aaaaRANGEbbbbbbb +- query: SELECT vexfs_handle_publish((SELECT handle FROM overlay_handle), 1, 'full', 'overlay-publish'); + expect: + - - 2 +- query: SELECT staging.base_size, staging.base_visible_size, staging.logical_size, staging.dirty_bytes, count(dirty.chunk_no) FROM _vexfs.handle_staging AS staging LEFT JOIN _vexfs.handle_staging_chunks AS dirty USING(handle_id) WHERE staging.handle_id=(SELECT handle FROM overlay_handle) GROUP BY staging.base_size, staging.base_visible_size, staging.logical_size, staging.dirty_bytes; + expect: + - - 131079 + - 131079 + - 131079 + - 0 + - 0 +- query: SELECT count(*) FROM _vexfs.manifest_chunks AS old_entry JOIN _vexfs.manifest_chunks AS new_entry ON new_entry.chunk_id=old_entry.chunk_id WHERE old_entry.manifest_id=(SELECT manifest_id FROM _vexfs.file_versions WHERE workspace_id=(SELECT workspace_id FROM _vexfs.workspaces WHERE name='pg-overlay') AND version_no=1) AND new_entry.manifest_id=(SELECT manifest_id FROM _vexfs.file_versions WHERE workspace_id=(SELECT workspace_id FROM _vexfs.workspaces WHERE name='pg-overlay') AND version_no=2); + expect: + - - 1 +- query: SELECT vexfs_handle_truncate((SELECT handle FROM overlay_handle), 3, 'overlay-truncate-small'); + expect: + - - 2 +- query: SELECT vexfs_handle_truncate((SELECT handle FROM overlay_handle), 65540, 'overlay-truncate-grow'); + expect: + - - 3 +- query: SELECT convert_from(vexfs_handle_read((SELECT handle FROM overlay_handle), 0, 3), 'UTF8'), vexfs_handle_read((SELECT handle FROM overlay_handle), 3, 65537) = decode(repeat('00', 65537), 'hex'); + expect: + - - aaa + - true +- query: SELECT staging.base_visible_size, staging.logical_size, staging.dirty_bytes, count(dirty.chunk_no) FROM _vexfs.handle_staging AS staging LEFT JOIN _vexfs.handle_staging_chunks AS dirty USING(handle_id) WHERE staging.handle_id=(SELECT handle FROM overlay_handle) GROUP BY staging.base_visible_size, staging.logical_size, staging.dirty_bytes; + expect: + - - 3 + - 65540 + - 3 + - 1 +- query: SELECT vexfs_handle_publish_close((SELECT handle FROM overlay_handle), 3, 'full'); + expect: + - - 3 +- query: SELECT octet_length(vexfs_read('pg-overlay', '/large.bin')), convert_from(vexfs_read_range('pg-overlay', '/large.bin', 0, 3), 'UTF8'), vexfs_read_range('pg-overlay', '/large.bin', 3, 65537) = decode(repeat('00', 65537), 'hex'); + expect: + - - 65540 + - aaa + - true +- query: SELECT count(*) FROM _vexfs.handle_staging WHERE handle_id=(SELECT handle FROM overlay_handle); + expect: + - - 0 +- statement: CREATE TEMP TABLE claimed_handles(name text PRIMARY KEY, handle text NOT NULL, generation bigint); +- statement: INSERT INTO claimed_handles(name, handle) VALUES ('a', vexfs_handle_create('pg-overlay', '/claimed-a.txt', 420, 501, 20, 'claimed-create-a', 'overlay-gateway')), ('b', vexfs_handle_create('pg-overlay', '/claimed-b.txt', 420, 501, 20, 'claimed-create-b', 'overlay-gateway')); +- statement: UPDATE claimed_handles SET generation=vexfs_handle_stage_write(handle, 0, convert_to(name, 'UTF8'), 'claimed-stage-' || name); +- query: SELECT jsonb_array_length(vexfs_mount_publish_close_claimed('pg-overlay', 'overlay-gateway', 'full', (SELECT jsonb_agg(jsonb_build_object('handle', handle, 'generation', generation) ORDER BY name) FROM claimed_handles))); + expect: + - - 2 +- query: SELECT convert_from(vexfs_read('pg-overlay', '/claimed-a.txt'), 'UTF8'), convert_from(vexfs_read('pg-overlay', '/claimed-b.txt'), 'UTF8'); + expect: + - - a + - b +- query: SELECT jsonb_array_length(vexfs_mount_publish_close_claimed('pg-overlay', 'overlay-gateway', 'full', (SELECT jsonb_agg(jsonb_build_object('handle', handle, 'generation', generation) ORDER BY name) FROM claimed_handles))); + expect: + - - 2 +- query: SELECT (vexfs_check('pg-overlay', 1)->>'ok')::boolean; + expect: + - - true +- statement: INSERT INTO claimed_handles(name, handle, generation) VALUES ('check', vexfs_handle_create('pg-overlay', '/check.txt', 420, 501, 20, 'check-create', 'overlay-gateway'), 1); +- statement: UPDATE _vexfs.handle_staging SET dirty_bytes=1 WHERE handle_id=(SELECT handle FROM claimed_handles WHERE name='check'); +- query: SELECT (vexfs_check('pg-overlay', 0)->>'ok')::boolean, (vexfs_check('pg-overlay', 0)->'issues'->0->>'code'); + expect: + - - false + - handle_staging +- statement: UPDATE _vexfs.handle_staging SET dirty_bytes=0 WHERE handle_id=(SELECT handle FROM claimed_handles WHERE name='check'); +- query: SELECT (vexfs_check('pg-overlay', 1)->>'ok')::boolean; + expect: + - - true +- query: SELECT vexfs_workspace_drop('pg-overlay', false); + expect: + - - 1 +- query: SELECT count(*) FROM _vexfs.workspaces WHERE name='pg-overlay'; + expect: + - - 0 diff --git a/tests/spec/sqlite/agent_files/vexfs_contract.yaml b/tests/spec/sqlite/agent_files/vexfs_contract.yaml index 0f729e9a54..20e53fc63d 100644 --- a/tests/spec/sqlite/agent_files/vexfs_contract.yaml +++ b/tests/spec/sqlite/agent_files/vexfs_contract.yaml @@ -20,6 +20,10 @@ steps: - query: SELECT CAST(vexfs_read('default', '/notes/2026/hello.txt') AS TEXT); expect: - - hello +- query: SELECT CAST(vexfs_read_range('default', '/notes/2026/hello.txt', 1, 3) AS TEXT), length(vexfs_read_range('default', '/notes/2026/hello.txt', 5, 9)); + expect: + - - ell + - 0 - query: SELECT vexfs_create('default', '/notes/run.sh', 'file', 488); expect: - - 5 diff --git a/vexdb_pg/sql/vexdb_lite--1.0.sql b/vexdb_pg/sql/vexdb_lite--1.0.sql index e55946eae6..0842a32e87 100644 --- a/vexdb_pg/sql/vexdb_lite--1.0.sql +++ b/vexdb_pg/sql/vexdb_lite--1.0.sql @@ -603,7 +603,6 @@ CREATE TABLE _vexfs.handles ( expected_version bigint NOT NULL CHECK (expected_version >= 0), dirty_generation bigint NOT NULL DEFAULT 0 CHECK (dirty_generation >= 0), published_generation bigint NOT NULL DEFAULT 0 CHECK (published_generation >= 0), - staging bytea NOT NULL DEFAULT ''::bytea, state text NOT NULL DEFAULT 'open' CHECK (state IN ('open', 'retained', 'closed')), created_at timestamptz NOT NULL DEFAULT clock_timestamp(), updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), @@ -615,6 +614,33 @@ CREATE INDEX vexfs_handles_workspace_state_idx CREATE INDEX vexfs_handles_inode_idx ON _vexfs.handles(workspace_id, inode_id); +-- Writable handles keep only a reference to their published base plus the +-- chunks changed since that base. A small positional write therefore stores at +-- most the affected 64 KiB chunks instead of replacing one full-file bytea. +CREATE TABLE _vexfs.handle_staging ( + handle_id text PRIMARY KEY REFERENCES _vexfs.handles(handle_id) ON DELETE CASCADE, + generation bigint NOT NULL CHECK (generation >= 0), + base_manifest_id bigint REFERENCES _vexfs.manifests(manifest_id), + base_size bigint NOT NULL CHECK (base_size >= 0), + base_visible_size bigint NOT NULL CHECK (base_visible_size >= 0), + logical_size bigint NOT NULL CHECK (logical_size BETWEEN 0 AND 134217728), + dirty_bytes bigint NOT NULL DEFAULT 0 CHECK (dirty_bytes >= 0), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CHECK (base_visible_size <= base_size) +); + +CREATE INDEX vexfs_handle_staging_updated_idx + ON _vexfs.handle_staging(updated_at, handle_id); + +CREATE TABLE _vexfs.handle_staging_chunks ( + handle_id text NOT NULL REFERENCES _vexfs.handles(handle_id) ON DELETE CASCADE, + chunk_no integer NOT NULL CHECK (chunk_no >= 0), + content bytea NOT NULL CHECK ( + octet_length(content) BETWEEN 1 AND 65536), + PRIMARY KEY (handle_id, chunk_no) +); + CREATE TABLE _vexfs.request_replays ( workspace_id bigint NOT NULL REFERENCES _vexfs.workspaces(workspace_id) ON DELETE CASCADE, owner_oid oid NOT NULL, @@ -1343,6 +1369,168 @@ BEGIN END; $$; +CREATE FUNCTION _vexfs.read_staging_chunk( + p_handle text, + p_chunk_no integer, + p_target_size integer) +RETURNS bytea +LANGUAGE plpgsql +SECURITY DEFINER +STABLE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_staging _vexfs.handle_staging%ROWTYPE; + v_workspace_id bigint; + v_inode_id bigint; + v_chunk_start bigint; + v_visible_size integer; + v_content bytea; + v_size integer; + v_checksum text; + v_chunk_workspace bigint; + v_chunk_inode bigint; +BEGIN + IF p_chunk_no < 0 OR p_target_size < 0 OR p_target_size > 65536 THEN + RAISE EXCEPTION 'VEXFS_INVALID_RANGE: invalid staging chunk range' + USING ERRCODE = '22023'; + END IF; + IF p_target_size = 0 THEN + RETURN ''::bytea; + END IF; + SELECT * INTO v_staging + FROM _vexfs.handle_staging AS staging + WHERE staging.handle_id = p_handle; + IF NOT FOUND THEN + RAISE EXCEPTION 'VEXFS_STAGING_NOT_FOUND: %', p_handle + USING ERRCODE = 'P0002'; + END IF; + SELECT handle.workspace_id, handle.inode_id + INTO STRICT v_workspace_id, v_inode_id + FROM _vexfs.handles AS handle + WHERE handle.handle_id = p_handle; + + SELECT dirty.content INTO v_content + FROM _vexfs.handle_staging_chunks AS dirty + WHERE dirty.handle_id = p_handle + AND dirty.chunk_no = p_chunk_no; + IF NOT FOUND THEN + v_content := ''::bytea; + v_chunk_start := p_chunk_no::bigint * 65536; + IF v_staging.base_manifest_id IS NOT NULL + AND v_chunk_start < v_staging.base_visible_size THEN + SELECT chunk.content, + chunk.size_bytes, + chunk.checksum, + chunk.workspace_id, + chunk.inode_id + INTO v_content, + v_size, + v_checksum, + v_chunk_workspace, + v_chunk_inode + FROM _vexfs.manifest_chunks AS entry + JOIN _vexfs.chunks AS chunk ON chunk.chunk_id = entry.chunk_id + WHERE entry.manifest_id = v_staging.base_manifest_id + AND entry.chunk_no = p_chunk_no; + IF NOT FOUND + OR v_chunk_workspace <> v_workspace_id + OR v_chunk_inode IS DISTINCT FROM v_inode_id + OR v_size <> octet_length(v_content) + OR v_checksum <> encode(pg_catalog.sha256(v_content), 'hex') THEN + RAISE EXCEPTION 'VEXFS_CHECKSUM_MISMATCH: staging base chunk is invalid' + USING ERRCODE = 'XX001'; + END IF; + v_visible_size := least( + v_size, + v_staging.base_visible_size - v_chunk_start)::integer; + v_content := substring(v_content FROM 1 FOR v_visible_size); + END IF; + END IF; + + IF octet_length(v_content) > p_target_size THEN + v_content := substring(v_content FROM 1 FOR p_target_size); + ELSIF octet_length(v_content) < p_target_size THEN + v_content := v_content || decode( + repeat('00', p_target_size - octet_length(v_content)), 'hex'); + END IF; + RETURN v_content; +END; +$$; + +CREATE FUNCTION _vexfs.read_staging_range( + p_handle text, + p_offset bigint, + p_length bigint) +RETURNS bytea +LANGUAGE plpgsql +SECURITY DEFINER +STABLE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_staging _vexfs.handle_staging%ROWTYPE; + v_end bigint; + v_first_chunk integer; + v_last_chunk integer; + v_chunk_no integer; + v_chunk_start bigint; + v_chunk_size integer; + v_copy_start integer; + v_copy_length integer; + v_chunk bytea; + v_content bytea := ''::bytea; +BEGIN + IF p_offset < 0 OR p_length < 0 + OR p_offset > 134217728 OR p_length > 134217728 THEN + RAISE EXCEPTION 'VEXFS_INVALID_RANGE: offset and length must be within 0..128 MiB' + USING ERRCODE = '22023'; + END IF; + SELECT * INTO v_staging + FROM _vexfs.handle_staging AS staging + WHERE staging.handle_id = p_handle; + IF NOT FOUND THEN + RAISE EXCEPTION 'VEXFS_STAGING_NOT_FOUND: %', p_handle + USING ERRCODE = 'P0002'; + END IF; + IF p_length = 0 OR p_offset >= v_staging.logical_size THEN + RETURN ''::bytea; + END IF; + + v_end := least(v_staging.logical_size, p_offset + p_length); + v_first_chunk := (p_offset / 65536)::integer; + v_last_chunk := ((v_end - 1) / 65536)::integer; + FOR v_chunk_no IN v_first_chunk..v_last_chunk LOOP + v_chunk_start := v_chunk_no::bigint * 65536; + v_chunk_size := least( + 65536, v_staging.logical_size - v_chunk_start)::integer; + v_chunk := _vexfs.read_staging_chunk( + p_handle, v_chunk_no, v_chunk_size); + v_copy_start := (greatest(p_offset, v_chunk_start) - v_chunk_start)::integer; + v_copy_length := ( + least(v_end, v_chunk_start + v_chunk_size) + - greatest(p_offset, v_chunk_start))::integer; + v_content := v_content || substring( + v_chunk FROM v_copy_start + 1 FOR v_copy_length); + END LOOP; + RETURN v_content; +END; +$$; + +CREATE FUNCTION _vexfs.read_staging_content(p_handle text) +RETURNS bytea +LANGUAGE sql +SECURITY DEFINER +STABLE +SET search_path = pg_catalog, _vexfs +AS $$ + SELECT _vexfs.read_staging_range( + p_handle, 0, + (SELECT staging.logical_size + FROM _vexfs.handle_staging AS staging + WHERE staging.handle_id = p_handle)) +$$; + CREATE FUNCTION _vexfs.resolve_version_storage( p_workspace_id bigint, p_inode_id bigint, @@ -1544,6 +1732,223 @@ BEGIN END; $$; +-- Build a copy-on-write manifest for a positional write. Untouched chunks are +-- linked into the new manifest in one statement; only overlapping chunks (and +-- zero-filled hole chunks) are materialized again. The final file checksum is +-- still SHA-256 over the complete byte stream, so range writes keep the same +-- integrity contract as vexfs_write without rebuilding the complete file in a +-- PL/pgSQL loop. +CREATE FUNCTION _vexfs.store_manifest_range( + p_workspace_id bigint, + p_inode_id bigint, + p_previous_manifest bigint, + p_previous_size bigint, + p_offset bigint, + p_content bytea) +RETURNS bigint +LANGUAGE plpgsql +SECURITY DEFINER +VOLATILE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_manifest bigint; + v_previous_chunk_count integer; + v_previous_file_size bigint; + v_new_size bigint; + v_new_chunk_count integer; + v_write_end bigint; + v_chunk_no integer; + v_chunk_start bigint; + v_chunk_end bigint; + v_chunk_size integer; + v_chunk bytea; + v_previous_chunk_id bigint; + v_chunk_id bigint; + v_chunk_checksum text; + v_overlap_start bigint; + v_overlap_end bigint; + v_existing_count bigint; + v_existing_bytes bigint; + v_existing_valid boolean; + v_full_content bytea; + v_final_count bigint; + v_final_bytes bigint; + v_file_checksum text; +BEGIN + IF p_content IS NULL THEN + RAISE EXCEPTION 'VEXFS_INVALID_CONTENT: content cannot be null' + USING ERRCODE = '22004'; + END IF; + IF p_previous_manifest IS NULL OR p_previous_size < 0 + OR p_offset < 0 OR p_offset > 134217728 + OR p_offset + octet_length(p_content) > 134217728 THEN + RAISE EXCEPTION 'VEXFS_INVALID_RANGE: range manifest input is invalid' + USING ERRCODE = '22023'; + END IF; + + SELECT manifest.file_size, manifest.chunk_count + INTO v_previous_file_size, v_previous_chunk_count + FROM _vexfs.manifests AS manifest + WHERE manifest.manifest_id = p_previous_manifest + AND manifest.workspace_id = p_workspace_id + AND manifest.inode_id = p_inode_id; + IF NOT FOUND OR v_previous_file_size <> p_previous_size THEN + RAISE EXCEPTION 'VEXFS_CORRUPT: previous range manifest is invalid' + USING ERRCODE = 'XX001'; + END IF; + + -- Validate every source chunk independently while keeping peak memory + -- bounded to one 64 KiB chunk. The complete-file checksum is recomputed for + -- the new manifest below. + SELECT count(*), + coalesce(sum(chunk.size_bytes), 0), + coalesce(bool_and( + entry.chunk_no >= 0 + AND entry.chunk_no < v_previous_chunk_count + AND chunk.workspace_id = p_workspace_id + AND chunk.inode_id = p_inode_id + AND chunk.size_bytes = least( + 65536, + p_previous_size - entry.chunk_no::bigint * 65536)::integer + AND octet_length(chunk.content) = chunk.size_bytes + AND chunk.checksum = encode( + pg_catalog.sha256(chunk.content), 'hex')), + v_previous_chunk_count = 0) + INTO v_existing_count, v_existing_bytes, v_existing_valid + FROM _vexfs.manifest_chunks AS entry + JOIN _vexfs.chunks AS chunk ON chunk.chunk_id = entry.chunk_id + WHERE entry.manifest_id = p_previous_manifest; + IF v_existing_count <> v_previous_chunk_count + OR v_existing_bytes <> p_previous_size + OR NOT v_existing_valid THEN + RAISE EXCEPTION 'VEXFS_CHECKSUM_MISMATCH: previous manifest chunk is invalid' + USING ERRCODE = 'XX001'; + END IF; + + v_write_end := p_offset + octet_length(p_content); + v_new_size := greatest(p_previous_size, v_write_end); + v_new_chunk_count := ((v_new_size + 65535) / 65536)::integer; + INSERT INTO _vexfs.manifests( + workspace_id, inode_id, file_size, chunk_size, chunk_count, checksum) + VALUES ( + p_workspace_id, p_inode_id, v_new_size, 65536, + v_new_chunk_count, repeat('0', 64)) + RETURNING manifest_id INTO v_manifest; + + -- Copy chunks whose byte range and final size are unchanged. This is the + -- common path for NFS writes and avoids one SQL round trip per old chunk. + INSERT INTO _vexfs.manifest_chunks(manifest_id, chunk_no, chunk_id) + SELECT v_manifest, entry.chunk_no, entry.chunk_id + FROM _vexfs.manifest_chunks AS entry + JOIN _vexfs.chunks AS chunk ON chunk.chunk_id = entry.chunk_id + WHERE entry.manifest_id = p_previous_manifest + AND entry.chunk_no < v_new_chunk_count + AND chunk.size_bytes = least( + 65536, + v_new_size - entry.chunk_no::bigint * 65536)::integer + AND (entry.chunk_no::bigint * 65536 + chunk.size_bytes <= p_offset + OR entry.chunk_no::bigint * 65536 >= v_write_end); + + FOR v_chunk_no IN 0..v_new_chunk_count - 1 LOOP + IF EXISTS ( + SELECT 1 FROM _vexfs.manifest_chunks AS entry + WHERE entry.manifest_id = v_manifest + AND entry.chunk_no = v_chunk_no) THEN + CONTINUE; + END IF; + + v_chunk_start := v_chunk_no::bigint * 65536; + v_chunk_end := least(v_new_size, v_chunk_start + 65536); + v_chunk_size := (v_chunk_end - v_chunk_start)::integer; + v_previous_chunk_id := NULL; + v_chunk := ''::bytea; + SELECT chunk.chunk_id, chunk.content + INTO v_previous_chunk_id, v_chunk + FROM _vexfs.manifest_chunks AS entry + JOIN _vexfs.chunks AS chunk ON chunk.chunk_id = entry.chunk_id + WHERE entry.manifest_id = p_previous_manifest + AND entry.chunk_no = v_chunk_no; + IF NOT FOUND THEN + v_chunk := ''::bytea; + END IF; + IF octet_length(v_chunk) > v_chunk_size THEN + v_chunk := substring(v_chunk FROM 1 FOR v_chunk_size); + ELSIF octet_length(v_chunk) < v_chunk_size THEN + v_chunk := v_chunk || decode( + repeat('00', v_chunk_size - octet_length(v_chunk)), 'hex'); + END IF; + + v_overlap_start := greatest(v_chunk_start, p_offset); + v_overlap_end := least(v_chunk_end, v_write_end); + IF v_overlap_end > v_overlap_start THEN + v_chunk := _vexfs.overlay_bytes( + v_chunk, + v_overlap_start - v_chunk_start, + substring( + p_content + FROM (v_overlap_start - p_offset + 1)::integer + FOR (v_overlap_end - v_overlap_start)::integer)); + END IF; + IF octet_length(v_chunk) <> v_chunk_size THEN + RAISE EXCEPTION 'VEXFS_CORRUPT: range chunk has invalid size' + USING ERRCODE = 'XX001'; + END IF; + + v_chunk_checksum := encode(pg_catalog.sha256(v_chunk), 'hex'); + v_chunk_id := NULL; + IF v_previous_chunk_id IS NOT NULL THEN + SELECT chunk.chunk_id INTO v_chunk_id + FROM _vexfs.chunks AS chunk + WHERE chunk.chunk_id = v_previous_chunk_id + AND chunk.size_bytes = v_chunk_size + AND chunk.checksum = v_chunk_checksum + AND chunk.content = v_chunk; + END IF; + IF v_chunk_id IS NULL THEN + SELECT chunk.chunk_id INTO v_chunk_id + FROM _vexfs.manifest_chunks AS entry + JOIN _vexfs.chunks AS chunk ON chunk.chunk_id = entry.chunk_id + WHERE entry.manifest_id = v_manifest + AND chunk.size_bytes = v_chunk_size + AND chunk.checksum = v_chunk_checksum + AND chunk.content = v_chunk + LIMIT 1; + END IF; + IF v_chunk_id IS NULL THEN + INSERT INTO _vexfs.chunks( + workspace_id, inode_id, content, size_bytes, checksum) + VALUES ( + p_workspace_id, p_inode_id, v_chunk, + v_chunk_size, v_chunk_checksum) + RETURNING chunk_id INTO v_chunk_id; + END IF; + INSERT INTO _vexfs.manifest_chunks(manifest_id, chunk_no, chunk_id) + VALUES (v_manifest, v_chunk_no, v_chunk_id); + END LOOP; + + SELECT coalesce( + pg_catalog.string_agg( + chunk.content, ''::bytea ORDER BY entry.chunk_no), + ''::bytea), + count(*), + coalesce(sum(chunk.size_bytes), 0) + INTO v_full_content, v_final_count, v_final_bytes + FROM _vexfs.manifest_chunks AS entry + JOIN _vexfs.chunks AS chunk ON chunk.chunk_id = entry.chunk_id + WHERE entry.manifest_id = v_manifest; + IF v_final_count <> v_new_chunk_count OR v_final_bytes <> v_new_size THEN + RAISE EXCEPTION 'VEXFS_CORRUPT: range manifest is incomplete' + USING ERRCODE = 'XX001'; + END IF; + v_file_checksum := encode(pg_catalog.sha256(v_full_content), 'hex'); + UPDATE _vexfs.manifests + SET checksum = v_file_checksum + WHERE manifest_id = v_manifest; + RETURN v_manifest; +END; +$$; + CREATE FUNCTION _vexfs.read_version_content( p_workspace_id bigint, p_inode_id bigint, @@ -1621,6 +2026,94 @@ BEGIN END; $$; +CREATE FUNCTION _vexfs.read_version_range( + p_workspace_id bigint, + p_inode_id bigint, + p_version bigint, + p_offset bigint, + p_length bigint) +RETURNS bytea +LANGUAGE plpgsql +SECURITY DEFINER +STABLE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_manifest bigint; + v_size bigint; + v_first_chunk integer; + v_last_chunk integer; + v_end bigint; + v_expected_chunk integer; + v_chunk_offset bigint; + v_expected_size integer; + v_copy_start integer; + v_copy_length integer; + v_content bytea := ''::bytea; + v_row record; +BEGIN + IF p_offset < 0 OR p_length < 0 + OR p_offset > 134217728 OR p_length > 134217728 THEN + RAISE EXCEPTION 'VEXFS_INVALID_RANGE: offset and length must be within 0..128 MiB' + USING ERRCODE = '22023'; + END IF; + SELECT storage.manifest_id, storage.size_bytes + INTO v_manifest, v_size + FROM _vexfs.resolve_version_storage( + p_workspace_id, p_inode_id, p_version) AS storage; + IF v_size > 134217728 THEN + RAISE EXCEPTION 'VEXFS_CORRUPT: file version exceeds 128 MiB' + USING ERRCODE = 'XX001'; + END IF; + IF p_length = 0 OR p_offset >= v_size THEN + RETURN ''::bytea; + END IF; + + v_end := least(v_size, p_offset + p_length); + v_first_chunk := (p_offset / 65536)::integer; + v_last_chunk := ((v_end - 1) / 65536)::integer; + v_expected_chunk := v_first_chunk; + FOR v_row IN + SELECT entry.chunk_no, + chunk.content, + chunk.size_bytes, + chunk.checksum, + chunk.workspace_id, + chunk.inode_id + FROM _vexfs.manifest_chunks AS entry + JOIN _vexfs.chunks AS chunk ON chunk.chunk_id = entry.chunk_id + WHERE entry.manifest_id = v_manifest + AND entry.chunk_no BETWEEN v_first_chunk AND v_last_chunk + ORDER BY entry.chunk_no + LOOP + v_chunk_offset := v_row.chunk_no::bigint * 65536; + v_expected_size := least(65536, v_size - v_chunk_offset)::integer; + IF v_row.chunk_no <> v_expected_chunk + OR v_expected_size <= 0 + OR v_row.size_bytes <> v_expected_size + OR octet_length(v_row.content) <> v_expected_size + OR v_row.workspace_id <> p_workspace_id + OR v_row.inode_id <> p_inode_id + OR v_row.checksum <> encode(pg_catalog.sha256(v_row.content), 'hex') THEN + RAISE EXCEPTION 'VEXFS_CHECKSUM_MISMATCH: file manifest range chunk is invalid' + USING ERRCODE = 'XX001'; + END IF; + v_copy_start := (greatest(p_offset, v_chunk_offset) - v_chunk_offset)::integer; + v_copy_length := (least(v_end, v_chunk_offset + v_expected_size) - + greatest(p_offset, v_chunk_offset))::integer; + v_content := v_content || substring( + v_row.content FROM v_copy_start + 1 FOR v_copy_length); + v_expected_chunk := v_expected_chunk + 1; + END LOOP; + IF v_expected_chunk <> v_last_chunk + 1 + OR octet_length(v_content) <> v_end - p_offset THEN + RAISE EXCEPTION 'VEXFS_CHECKSUM_MISMATCH: file manifest range is incomplete' + USING ERRCODE = 'XX001'; + END IF; + RETURN v_content; +END; +$$; + CREATE FUNCTION _vexfs.enforce_quota( p_workspace_id bigint, p_inode_id bigint, @@ -2079,6 +2572,11 @@ BEGIN jsonb_build_object( 'before_version', v_workspace.head_commit, 'after_version', NULL)); + -- Staging deliberately keeps its base manifest alive during normal GC. + -- On an explicit workspace drop, remove handles first so those references + -- disappear before the workspace cascade reaches manifests. + DELETE FROM _vexfs.handles + WHERE workspace_id = v_workspace.workspace_id; DELETE FROM _vexfs.workspaces WHERE workspace_id = v_workspace.workspace_id; RETURN 1; END; @@ -2241,9 +2739,13 @@ BEGIN FROM _vexfs.file_versions AS f WHERE f.workspace_id = v_workspace.workspace_id AND f.inode_id = v_inode; - SELECT storage.manifest_id INTO v_previous_manifest - FROM _vexfs.resolve_version_storage( - v_workspace.workspace_id, v_inode, v_current_version) AS storage; + IF v_current_version > 0 THEN + SELECT storage.manifest_id INTO v_previous_manifest + FROM _vexfs.resolve_version_storage( + v_workspace.workspace_id, v_inode, v_current_version) AS storage; + ELSE + v_previous_manifest := NULL; + END IF; UPDATE _vexfs.inodes SET current_version = v_version, size_bytes = octet_length(p_content), @@ -2317,11 +2819,54 @@ BEGIN RAISE EXCEPTION 'VEXFS_NOT_REGULAR_FILE: %', p_path USING ERRCODE = '42809'; END IF; + IF v_version = 0 THEN + RETURN ''::bytea; + END IF; RETURN _vexfs.read_version_content( v_workspace.workspace_id, v_inode, v_version); END; $$; +CREATE FUNCTION public.vexfs_read_range( + p_workspace text, + p_path text, + p_offset bigint, + p_length bigint) +RETURNS bytea +LANGUAGE plpgsql +SECURITY DEFINER +STABLE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; + v_inode bigint; + v_kind text; + v_version bigint; +BEGIN + IF p_offset < 0 OR p_length < 0 + OR p_offset > 134217728 OR p_length > 134217728 THEN + RAISE EXCEPTION 'VEXFS_INVALID_RANGE: offset and length must be within 0..128 MiB' + USING ERRCODE = '22023'; + END IF; + v_workspace := _vexfs.require_workspace(p_workspace, 'read'); + v_inode := _vexfs.resolve_path(v_workspace.workspace_id, p_path); + PERFORM _vexfs.require_inode_permission( + v_workspace.workspace_id, v_inode, 'read'); + SELECT inode.kind, inode.current_version INTO v_kind, v_version + FROM _vexfs.inodes AS inode WHERE inode.inode_id = v_inode; + IF v_kind <> 'file' THEN + RAISE EXCEPTION 'VEXFS_NOT_REGULAR_FILE: %', p_path + USING ERRCODE = '42809'; + END IF; + IF v_version = 0 THEN + RETURN ''::bytea; + END IF; + RETURN _vexfs.read_version_range( + v_workspace.workspace_id, v_inode, v_version, p_offset, p_length); +END; +$$; + CREATE FUNCTION public.vexfs_list(p_workspace text, p_path text) RETURNS TABLE(name text, inode bigint, kind text, size bigint, version bigint) LANGUAGE plpgsql @@ -3213,6 +3758,13 @@ BEGIN AND session_id = p_session_id AND owner_oid = v_actor AND state = 'open'; + DELETE FROM _vexfs.handle_staging AS staging + USING _vexfs.handles AS handle + WHERE staging.handle_id = handle.handle_id + AND handle.workspace_id = v_workspace.workspace_id + AND handle.session_id = p_session_id + AND handle.owner_oid = v_actor + AND handle.state = 'closed'; UPDATE _vexfs.file_locks SET lease_until = clock_timestamp() + interval '30 seconds' WHERE workspace_id = v_workspace.workspace_id @@ -3280,7 +3832,8 @@ DECLARE v_inode bigint; v_kind text; v_version bigint; - v_content bytea; + v_size bigint := 0; + v_manifest bigint; v_handle text; v_hash text; v_cached text; @@ -3327,7 +3880,8 @@ BEGIN BEGIN v_inode := _vexfs.resolve_path(v_workspace.workspace_id, p_path); - SELECT i.kind, i.current_version INTO STRICT v_kind, v_version + SELECT i.kind, i.current_version, i.size_bytes + INTO STRICT v_kind, v_version, v_size FROM _vexfs.inodes AS i WHERE i.workspace_id = v_workspace.workspace_id AND i.inode_id = v_inode AND i.live; @@ -3338,8 +3892,11 @@ BEGIN PERFORM _vexfs.require_inode_permission( v_workspace.workspace_id, v_inode, CASE WHEN v_writable THEN 'write' ELSE 'read' END); - v_content := CASE WHEN v_truncate THEN ''::bytea ELSE - _vexfs.read_version_content(v_workspace.workspace_id, v_inode, v_version) END; + IF v_version > 0 THEN + SELECT storage.manifest_id INTO v_manifest + FROM _vexfs.resolve_version_storage( + v_workspace.workspace_id, v_inode, v_version) AS storage; + END IF; EXCEPTION WHEN SQLSTATE 'P0002' THEN IF NOT v_create THEN RAISE; @@ -3350,19 +3907,32 @@ BEGIN v_workspace.workspace_id, v_parent, 'write'); v_inode := NULL; v_version := 0; - v_content := ''::bytea; + v_size := 0; + v_manifest := NULL; END; v_handle := _vexfs.random_token('handle'); INSERT INTO _vexfs.handles( handle_id, workspace_id, inode_id, path, flags, create_mode, owner_oid, owner_role, session_id, expected_version, - dirty_generation, staging) + dirty_generation) VALUES ( v_handle, v_workspace.workspace_id, v_inode, p_path, p_flags, CASE WHEN v_inode IS NULL THEN 420 ELSE NULL END, v_actor, session_user, p_session_id, v_version, - CASE WHEN v_truncate OR v_inode IS NULL THEN 1 ELSE 0 END, - v_content); + CASE WHEN v_truncate OR v_inode IS NULL THEN 1 ELSE 0 END); + IF v_writable THEN + INSERT INTO _vexfs.handle_staging( + handle_id, generation, base_manifest_id, base_size, + base_visible_size, logical_size, dirty_bytes) + VALUES ( + v_handle, + CASE WHEN v_truncate OR v_inode IS NULL THEN 1 ELSE 0 END, + v_manifest, + v_size, + CASE WHEN v_truncate THEN 0 ELSE v_size END, + CASE WHEN v_truncate THEN 0 ELSE v_size END, + 0); + END IF; PERFORM _vexfs.replay_put( v_workspace.workspace_id, p_request_id, 'handle_open', v_hash, v_handle); RETURN v_handle; @@ -3430,16 +4000,168 @@ BEGIN INSERT INTO _vexfs.handles( handle_id, workspace_id, path, flags, create_mode, owner_oid, owner_role, session_id, expected_version, - dirty_generation, staging) + dirty_generation) VALUES ( v_handle, v_workspace.workspace_id, p_path, 'rwct', p_mode, - v_actor, session_user, p_session_id, 0, 1, ''::bytea); + v_actor, session_user, p_session_id, 0, 1); + INSERT INTO _vexfs.handle_staging( + handle_id, generation, base_manifest_id, base_size, + base_visible_size, logical_size, dirty_bytes) + VALUES (v_handle, 1, NULL, 0, 0, 0, 0); PERFORM _vexfs.replay_put( v_workspace.workspace_id, p_request_id, 'handle_create', v_hash, v_handle); RETURN v_handle; END; $$; +-- Filesystem gateways must return a usable inode from CREATE before the first +-- WRITE arrives. Keep the older five-argument handle API isolated until +-- publish, but let this owned form expose a version-0 empty inode immediately. +-- The first publish records version 1 and the workspace commit atomically. +CREATE FUNCTION public.vexfs_handle_create( + p_workspace text, + p_path text, + p_mode integer, + p_uid bigint, + p_gid bigint, + p_request_id text, + p_session_id text) +RETURNS text +LANGUAGE plpgsql +SECURITY DEFINER +VOLATILE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; + v_actor oid; + v_parent bigint; + v_name text; + v_inode bigint; + v_handle text; + v_hash text; + v_cached text; +BEGIN + IF p_mode < 0 OR p_mode > 4095 THEN + RAISE EXCEPTION 'VEXFS_INVALID_MODE: %', p_mode + USING ERRCODE = '22023'; + END IF; + IF p_uid < 0 OR p_uid > 4294967295 + OR p_gid < 0 OR p_gid > 4294967295 THEN + RAISE EXCEPTION 'VEXFS_INVALID_OWNER: uid and gid must be within 0..4294967295' + USING ERRCODE = '22023'; + END IF; + IF p_session_id IS NULL OR p_session_id = '' THEN + RAISE EXCEPTION 'VEXFS_INVALID_SESSION: session id is required' + USING ERRCODE = '22023'; + END IF; + v_workspace := _vexfs.require_workspace(p_workspace, 'write'); + SELECT role.oid INTO STRICT v_actor + FROM pg_catalog.pg_roles AS role + WHERE role.rolname = session_user; + IF NOT EXISTS ( + SELECT 1 FROM _vexfs.mount_sessions AS mounted + WHERE mounted.workspace_id = v_workspace.workspace_id + AND mounted.session_id = p_session_id + AND mounted.owner_oid = v_actor + AND mounted.lease_until > clock_timestamp()) THEN + RAISE EXCEPTION 'VEXFS_SESSION_STALE: mount session is not active' + USING ERRCODE = '55006'; + END IF; + + v_hash := _vexfs.request_digest(jsonb_build_object( + 'workspace', p_workspace, + 'path', p_path, + 'mode', p_mode, + 'uid', p_uid, + 'gid', p_gid, + 'session', p_session_id)); + PERFORM _vexfs.lock_request(v_workspace.workspace_id, p_request_id); + v_cached := _vexfs.replay_get( + v_workspace.workspace_id, p_request_id, 'handle_create', v_hash); + IF v_cached IS NOT NULL THEN + RETURN v_cached; + END IF; + + PERFORM 1 FROM _vexfs.workspaces + WHERE workspace_id = v_workspace.workspace_id FOR UPDATE; + SELECT parent.parent_inode, parent.entry_name INTO v_parent, v_name + FROM _vexfs.resolve_parent(v_workspace.workspace_id, p_path) AS parent; + PERFORM _vexfs.require_inode_permission( + v_workspace.workspace_id, v_parent, 'write'); + IF EXISTS ( + SELECT 1 FROM _vexfs.dentries AS dentry + WHERE dentry.workspace_id = v_workspace.workspace_id + AND dentry.parent_inode = v_parent + AND dentry.name = v_name) THEN + RAISE EXCEPTION 'VEXFS_ALREADY_EXISTS: %', p_path + USING ERRCODE = '23505'; + END IF; + PERFORM _vexfs.enforce_quota(v_workspace.workspace_id, NULL, 0); + + INSERT INTO _vexfs.inodes( + workspace_id, kind, mode, owner_oid, owner_role, owner_principal, + uid, gid, current_version, size_bytes) + VALUES ( + v_workspace.workspace_id, 'file', p_mode, + v_actor, session_user, session_user, + p_uid, p_gid, 0, 0) + RETURNING inode_id INTO v_inode; + INSERT INTO _vexfs.dentries(workspace_id, parent_inode, name, inode_id) + VALUES (v_workspace.workspace_id, v_parent, v_name, v_inode); + PERFORM _vexfs.inherit_acl(v_workspace.workspace_id, v_parent, v_inode); + UPDATE _vexfs.inodes + SET modified_at = clock_timestamp(), + changed_at = clock_timestamp() + WHERE workspace_id = v_workspace.workspace_id + AND inode_id = v_parent; + + v_handle := _vexfs.random_token('handle'); + INSERT INTO _vexfs.handles( + handle_id, workspace_id, inode_id, path, flags, create_mode, + owner_oid, owner_role, session_id, expected_version, + dirty_generation) + VALUES ( + v_handle, v_workspace.workspace_id, v_inode, p_path, 'rwct', p_mode, + v_actor, session_user, p_session_id, 0, 1); + INSERT INTO _vexfs.handle_staging( + handle_id, generation, base_manifest_id, base_size, + base_visible_size, logical_size, dirty_bytes) + VALUES (v_handle, 1, NULL, 0, 0, 0, 0); + PERFORM _vexfs.replay_put( + v_workspace.workspace_id, p_request_id, 'handle_create', v_hash, v_handle); + RETURN v_handle; +END; +$$; + +-- NFS CREATE needs both a writable handle and attributes before it can reply. +-- Keeping these as two PL/pgSQL statements makes the STABLE stat query see the +-- new inode, while the client pays for only one database round trip. +CREATE FUNCTION public.vexfs_handle_create_stat( + p_workspace text, + p_path text, + p_mode integer, + p_uid bigint, + p_gid bigint, + p_request_id text, + p_session_id text) +RETURNS TABLE(handle text, stat jsonb) +LANGUAGE plpgsql +SECURITY DEFINER +VOLATILE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_handle text; +BEGIN + v_handle := public.vexfs_handle_create( + p_workspace, p_path, p_mode, p_uid, p_gid, + p_request_id, p_session_id); + RETURN QUERY + SELECT v_handle, public.vexfs_stat(p_workspace, p_path); +END; +$$; + CREATE FUNCTION public.vexfs_handle_stage_write( p_handle text, p_offset bigint, @@ -3453,15 +4175,29 @@ SET search_path = pg_catalog, _vexfs AS $$ DECLARE v_handle _vexfs.handles%ROWTYPE; + v_staging _vexfs.handle_staging%ROWTYPE; v_hash text; v_cached text; v_generation bigint; - v_staging bytea; + v_logical_size bigint; + v_patch_offset integer := 0; + v_absolute bigint; + v_chunk_no integer; + v_chunk_start bigint; + v_within integer; + v_write_size integer; + v_target_size integer; + v_chunk bytea; BEGIN IF p_content IS NULL THEN RAISE EXCEPTION 'VEXFS_INVALID_CONTENT: content cannot be null' USING ERRCODE = '22004'; END IF; + IF p_offset < 0 OR p_offset > 134217728 + OR p_offset + octet_length(p_content) > 134217728 THEN + RAISE EXCEPTION 'VEXFS_INVALID_RANGE: staged write is outside 0..128 MiB' + USING ERRCODE = '22023'; + END IF; SELECT * INTO v_handle FROM _vexfs.require_handle(p_handle, 'write'); IF position('w' IN v_handle.flags) = 0 THEN RAISE EXCEPTION 'VEXFS_READ_ONLY_HANDLE: %', p_handle @@ -3477,13 +4213,65 @@ BEGIN IF v_cached IS NOT NULL THEN RETURN v_cached::bigint; END IF; - v_staging := _vexfs.overlay_bytes(v_handle.staging, p_offset, p_content); + SELECT * INTO v_staging + FROM _vexfs.handle_staging AS staging + WHERE staging.handle_id = p_handle + FOR UPDATE; + IF NOT FOUND THEN + RAISE EXCEPTION 'VEXFS_STAGING_NOT_FOUND: %', p_handle + USING ERRCODE = 'P0002'; + END IF; + IF octet_length(p_content) = 0 THEN + PERFORM _vexfs.replay_put( + v_handle.workspace_id, p_request_id, 'handle_stage_write', + v_hash, v_handle.dirty_generation::text); + RETURN v_handle.dirty_generation; + END IF; + v_logical_size := greatest( + v_staging.logical_size, p_offset + octet_length(p_content)); PERFORM _vexfs.enforce_quota( - v_handle.workspace_id, v_handle.inode_id, octet_length(v_staging)); + v_handle.workspace_id, v_handle.inode_id, v_logical_size); + + WHILE v_patch_offset < octet_length(p_content) LOOP + v_absolute := p_offset + v_patch_offset; + v_chunk_no := (v_absolute / 65536)::integer; + v_chunk_start := v_chunk_no::bigint * 65536; + v_within := (v_absolute - v_chunk_start)::integer; + v_write_size := least( + octet_length(p_content) - v_patch_offset, + 65536 - v_within); + v_target_size := least( + 65536, v_logical_size - v_chunk_start)::integer; + IF v_within = 0 AND v_write_size = v_target_size THEN + v_chunk := substring( + p_content FROM v_patch_offset + 1 FOR v_write_size); + ELSE + v_chunk := _vexfs.read_staging_chunk( + p_handle, v_chunk_no, v_target_size); + v_chunk := _vexfs.overlay_bytes( + v_chunk, + v_within, + substring(p_content FROM v_patch_offset + 1 FOR v_write_size)); + END IF; + INSERT INTO _vexfs.handle_staging_chunks(handle_id, chunk_no, content) + VALUES (p_handle, v_chunk_no, v_chunk) + ON CONFLICT (handle_id, chunk_no) DO UPDATE + SET content = excluded.content; + v_patch_offset := v_patch_offset + v_write_size; + END LOOP; + v_generation := v_handle.dirty_generation + 1; + UPDATE _vexfs.handle_staging + SET generation = v_generation, + logical_size = v_logical_size, + dirty_bytes = ( + SELECT coalesce(sum(octet_length(dirty.content)), 0) + FROM _vexfs.handle_staging_chunks AS dirty + WHERE dirty.handle_id = p_handle), + updated_at = clock_timestamp() + WHERE handle_id = p_handle; UPDATE _vexfs.handles - SET staging = v_staging, - dirty_generation = v_generation, + SET dirty_generation = v_generation, lease_until = clock_timestamp() + interval '30 seconds', updated_at = clock_timestamp() WHERE handle_id = p_handle; @@ -3506,11 +4294,14 @@ SET search_path = pg_catalog, _vexfs AS $$ DECLARE v_handle _vexfs.handles%ROWTYPE; + v_staging _vexfs.handle_staging%ROWTYPE; v_hash text; v_cached text; v_generation bigint; - v_staging bytea; - v_current_size bigint; + v_remainder integer; + v_boundary_chunk integer; + v_current_chunk_size integer; + v_chunk bytea; BEGIN IF p_size < 0 OR p_size > 134217728 THEN RAISE EXCEPTION 'VEXFS_INVALID_SIZE: staged file must be 0..128 MiB' @@ -3529,21 +4320,56 @@ BEGIN IF v_cached IS NOT NULL THEN RETURN v_cached::bigint; END IF; - v_current_size := octet_length(v_handle.staging); - IF p_size <= v_current_size THEN - v_staging := substring(v_handle.staging FROM 1 FOR p_size::integer); - ELSIF p_size = 0 THEN - v_staging := ''::bytea; - ELSE - v_staging := _vexfs.overlay_bytes( - v_handle.staging, p_size - 1, decode('00', 'hex')); + SELECT * INTO v_staging + FROM _vexfs.handle_staging AS staging + WHERE staging.handle_id = p_handle + FOR UPDATE; + IF NOT FOUND THEN + RAISE EXCEPTION 'VEXFS_STAGING_NOT_FOUND: %', p_handle + USING ERRCODE = 'P0002'; + END IF; + IF p_size = v_staging.logical_size THEN + PERFORM _vexfs.replay_put( + v_handle.workspace_id, p_request_id, 'handle_truncate', + v_hash, v_handle.dirty_generation::text); + RETURN v_handle.dirty_generation; END IF; PERFORM _vexfs.enforce_quota( v_handle.workspace_id, v_handle.inode_id, p_size); + + IF p_size < v_staging.logical_size THEN + v_remainder := (p_size % 65536)::integer; + v_boundary_chunk := (p_size / 65536)::integer; + IF v_remainder > 0 THEN + v_current_chunk_size := least( + 65536, + v_staging.logical_size - v_boundary_chunk::bigint * 65536)::integer; + v_chunk := _vexfs.read_staging_chunk( + p_handle, v_boundary_chunk, v_current_chunk_size); + v_chunk := substring(v_chunk FROM 1 FOR v_remainder); + INSERT INTO _vexfs.handle_staging_chunks(handle_id, chunk_no, content) + VALUES (p_handle, v_boundary_chunk, v_chunk) + ON CONFLICT (handle_id, chunk_no) DO UPDATE + SET content = excluded.content; + END IF; + DELETE FROM _vexfs.handle_staging_chunks AS dirty + WHERE dirty.handle_id = p_handle + AND dirty.chunk_no >= ((p_size + 65535) / 65536)::integer; + END IF; + v_generation := v_handle.dirty_generation + 1; + UPDATE _vexfs.handle_staging + SET generation = v_generation, + base_visible_size = least(base_visible_size, p_size), + logical_size = p_size, + dirty_bytes = ( + SELECT coalesce(sum(octet_length(dirty.content)), 0) + FROM _vexfs.handle_staging_chunks AS dirty + WHERE dirty.handle_id = p_handle), + updated_at = clock_timestamp() + WHERE handle_id = p_handle; UPDATE _vexfs.handles - SET staging = v_staging, - dirty_generation = v_generation, + SET dirty_generation = v_generation, lease_until = clock_timestamp() + interval '30 seconds', updated_at = clock_timestamp() WHERE handle_id = p_handle; @@ -3566,7 +4392,6 @@ SET search_path = pg_catalog, _vexfs AS $$ DECLARE v_handle _vexfs.handles%ROWTYPE; - v_content bytea; v_version bigint; BEGIN IF p_offset < 0 OR p_length < 0 @@ -3584,13 +4409,14 @@ BEGIN FROM _vexfs.inodes AS i WHERE i.workspace_id = v_handle.workspace_id AND i.inode_id = v_handle.inode_id; - v_content := CASE WHEN v_version = 0 THEN ''::bytea ELSE - _vexfs.read_version_content( - v_handle.workspace_id, v_handle.inode_id, v_version) END; - ELSE - v_content := v_handle.staging; + IF v_version = 0 THEN + RETURN ''::bytea; + END IF; + RETURN _vexfs.read_version_range( + v_handle.workspace_id, v_handle.inode_id, v_version, + p_offset, p_length); END IF; - RETURN substring(v_content FROM (p_offset + 1)::integer FOR p_length::integer); + RETURN _vexfs.read_staging_range(p_handle, p_offset, p_length); END; $$; @@ -3607,6 +4433,7 @@ SET search_path = pg_catalog, _vexfs AS $$ DECLARE v_handle _vexfs.handles%ROWTYPE; + v_staging _vexfs.handle_staging%ROWTYPE; v_hash text; v_cached text; v_current_version bigint; @@ -3620,6 +4447,7 @@ DECLARE v_checksum text; v_commit bigint; v_principal_oid oid; + v_content bytea; BEGIN IF p_durability NOT IN ('none', 'data', 'full') THEN RAISE EXCEPTION 'VEXFS_INVALID_DURABILITY: use none, data, or full' @@ -3651,6 +4479,22 @@ BEGIN v_handle.dirty_generation, p_generation USING ERRCODE = '40001'; END IF; + SELECT * INTO v_staging + FROM _vexfs.handle_staging AS staging + WHERE staging.handle_id = p_handle + FOR UPDATE; + IF NOT FOUND OR v_staging.generation <> p_generation THEN + RAISE EXCEPTION 'VEXFS_STALE_GENERATION: staging generation does not match %', + p_generation USING ERRCODE = '40001'; + END IF; + -- Assemble and validate the candidate before taking the workspace row lock. + -- The handle/staging row locks keep this generation stable while other + -- files in the same workspace continue to make progress. + v_content := _vexfs.read_staging_content(p_handle); + IF octet_length(v_content) <> v_staging.logical_size THEN + RAISE EXCEPTION 'VEXFS_CORRUPT: staged content size does not match metadata' + USING ERRCODE = 'XX001'; + END IF; PERFORM pg_catalog.set_config( 'synchronous_commit', CASE p_durability WHEN 'none' THEN 'off' @@ -3675,7 +4519,7 @@ BEGIN v_version := public.vexfs_write( (SELECT w.name FROM _vexfs.workspaces AS w WHERE w.workspace_id = v_handle.workspace_id), - v_handle.path, v_handle.staging); + v_handle.path, v_content); v_inode := _vexfs.resolve_path(v_handle.workspace_id, v_handle.path); UPDATE _vexfs.inodes SET mode = coalesce(v_handle.create_mode, 420), @@ -3697,33 +4541,37 @@ BEGIN v_version := public.vexfs_write( (SELECT w.name FROM _vexfs.workspaces AS w WHERE w.workspace_id = v_handle.workspace_id), - v_path, v_handle.staging); + v_path, v_content); ELSE -- POSIX keeps an unlinked file alive while an open descriptor still -- references its inode. Publish the new version without recreating -- a directory entry; the inode is reaped after the final handle closes. PERFORM _vexfs.enforce_quota( v_handle.workspace_id, v_handle.inode_id, - octet_length(v_handle.staging)); + v_staging.logical_size); SELECT coalesce(max(version.version_no), 0) + 1 INTO v_version FROM _vexfs.file_versions AS version WHERE version.workspace_id = v_handle.workspace_id AND version.inode_id = v_handle.inode_id; - SELECT storage.manifest_id INTO v_previous_manifest - FROM _vexfs.resolve_version_storage( - v_handle.workspace_id, v_handle.inode_id, - v_current_version) AS storage; + IF v_current_version > 0 THEN + SELECT storage.manifest_id INTO v_previous_manifest + FROM _vexfs.resolve_version_storage( + v_handle.workspace_id, v_handle.inode_id, + v_current_version) AS storage; + ELSE + v_previous_manifest := NULL; + END IF; UPDATE _vexfs.inodes - SET current_version = v_version, - size_bytes = octet_length(v_handle.staging), + SET current_version = v_version, + size_bytes = v_staging.logical_size, modified_at = clock_timestamp(), changed_at = clock_timestamp() WHERE workspace_id = v_handle.workspace_id AND inode_id = v_handle.inode_id; v_manifest := _vexfs.store_manifest( v_handle.workspace_id, v_handle.inode_id, - v_previous_manifest, v_handle.staging); + v_previous_manifest, v_content); SELECT manifest.checksum INTO STRICT v_checksum FROM _vexfs.manifests AS manifest WHERE manifest.manifest_id = v_manifest; @@ -3742,11 +4590,16 @@ BEGIN size_bytes, checksum, created_by_oid, created_by) VALUES ( v_handle.workspace_id, v_handle.inode_id, v_version, - v_commit, v_manifest, octet_length(v_handle.staging), + v_commit, v_manifest, v_staging.logical_size, v_checksum, v_principal_oid, session_user); END IF; v_inode := v_handle.inode_id; END IF; + IF v_manifest IS NULL THEN + SELECT storage.manifest_id INTO v_manifest + FROM _vexfs.resolve_version_storage( + v_handle.workspace_id, v_inode, v_version) AS storage; + END IF; UPDATE _vexfs.handles SET inode_id = v_inode, path = coalesce(v_path, _vexfs.handles.path), @@ -3755,6 +4608,17 @@ BEGIN lease_until = clock_timestamp() + interval '30 seconds', updated_at = clock_timestamp() WHERE handle_id = p_handle; + UPDATE _vexfs.handle_staging + SET generation = p_generation, + base_manifest_id = v_manifest, + base_size = v_staging.logical_size, + base_visible_size = v_staging.logical_size, + logical_size = v_staging.logical_size, + dirty_bytes = 0, + updated_at = clock_timestamp() + WHERE handle_id = p_handle; + DELETE FROM _vexfs.handle_staging_chunks + WHERE handle_id = p_handle; PERFORM _vexfs.replay_put( v_handle.workspace_id, p_request_id, 'handle_publish', v_hash, v_version::text); @@ -3774,10 +4638,12 @@ SET search_path = pg_catalog, _vexfs AS $$ DECLARE v_handle _vexfs.handles%ROWTYPE; + v_staging _vexfs.handle_staging%ROWTYPE; v_hash text; v_cached text; v_current_version bigint; - v_current bytea; + v_current_size bigint; + v_manifest bigint; v_generation bigint; v_version bigint; v_publish_request text; @@ -3801,16 +4667,23 @@ BEGIN IF v_cached IS NOT NULL THEN RETURN v_cached::bigint; END IF; - PERFORM 1 FROM _vexfs.workspaces - WHERE workspace_id = v_handle.workspace_id FOR UPDATE; + SELECT * INTO v_staging + FROM _vexfs.handle_staging AS staging + WHERE staging.handle_id = p_handle + FOR UPDATE; + IF NOT FOUND THEN + RAISE EXCEPTION 'VEXFS_STAGING_NOT_FOUND: %', p_handle + USING ERRCODE = 'P0002'; + END IF; IF v_handle.inode_id IS NULL THEN - v_current := v_handle.staging; + v_current_size := v_staging.logical_size; ELSE IF v_handle.dirty_generation <> v_handle.published_generation THEN RAISE EXCEPTION 'VEXFS_DIRTY_HANDLE: append requires a clean handle' USING ERRCODE = '55006'; END IF; - SELECT i.current_version INTO v_current_version + SELECT i.current_version, i.size_bytes + INTO v_current_version, v_current_size FROM _vexfs.inodes AS i WHERE i.workspace_id = v_handle.workspace_id AND i.inode_id = v_handle.inode_id @@ -3820,29 +4693,36 @@ BEGIN RAISE EXCEPTION 'VEXFS_VERSION_CONFLICT: open inode is unavailable' USING ERRCODE = '40001'; END IF; - v_current := _vexfs.read_version_content( - v_handle.workspace_id, v_handle.inode_id, v_current_version); + IF v_current_version > 0 THEN + SELECT storage.manifest_id INTO v_manifest + FROM _vexfs.resolve_version_storage( + v_handle.workspace_id, v_handle.inode_id, + v_current_version) AS storage; + ELSE + v_manifest := NULL; + END IF; UPDATE _vexfs.handles SET expected_version = v_current_version, - staging = v_current, + updated_at = clock_timestamp() + WHERE handle_id = p_handle; + DELETE FROM _vexfs.handle_staging_chunks + WHERE handle_id = p_handle; + UPDATE _vexfs.handle_staging + SET generation = v_handle.published_generation, + base_manifest_id = v_manifest, + base_size = v_current_size, + base_visible_size = v_current_size, + logical_size = v_current_size, + dirty_bytes = 0, updated_at = clock_timestamp() WHERE handle_id = p_handle; END IF; - IF octet_length(v_current) + octet_length(p_content) > 134217728 THEN - RAISE EXCEPTION 'VEXFS_FILE_TOO_LARGE: appended file exceeds 128 MiB' - USING ERRCODE = '54000'; - END IF; - PERFORM _vexfs.enforce_quota( - v_handle.workspace_id, v_handle.inode_id, - octet_length(v_current) + octet_length(p_content)); - v_generation := greatest( - v_handle.dirty_generation, v_handle.published_generation) + 1; - UPDATE _vexfs.handles - SET staging = v_current || p_content, - dirty_generation = v_generation, - lease_until = clock_timestamp() + interval '30 seconds', - updated_at = clock_timestamp() - WHERE handle_id = p_handle; + v_generation := public.vexfs_handle_stage_write( + p_handle, + v_current_size, + p_content, + 'append-stage:' || coalesce( + nullif(p_request_id, ''), _vexfs.random_token('request'))); v_publish_request := 'append-publish:' || coalesce( nullif(p_request_id, ''), _vexfs.random_token('request')); v_version := public.vexfs_handle_publish( @@ -3893,14 +4773,350 @@ BEGIN UPDATE _vexfs.handles SET state = 'closed', lease_until = clock_timestamp(), - staging = ''::bytea, updated_at = clock_timestamp() WHERE handle_id = p_handle; + DELETE FROM _vexfs.handle_staging + WHERE handle_id = p_handle; PERFORM _vexfs.reap_orphan_inodes(v_handle.workspace_id); RETURN v_version; END; $$; +-- Publish every dirty handle owned by one mount session in a single database +-- transaction. The NFS idle flusher uses this to turn many small UNSTABLE +-- writes into durable file versions without one libpq round trip per file. +CREATE FUNCTION public.vexfs_mount_publish_close_all( + p_workspace text, + p_session_id text, + p_durability text DEFAULT 'none', + p_limit bigint DEFAULT 0) +RETURNS bigint +LANGUAGE plpgsql +SECURITY DEFINER +VOLATILE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; + v_actor oid; + v_handle record; + v_count bigint := 0; +BEGIN + IF p_durability NOT IN ('none', 'data', 'full') THEN + RAISE EXCEPTION 'VEXFS_INVALID_DURABILITY: use none, data, or full' + USING ERRCODE = '22023'; + END IF; + IF p_session_id IS NULL OR p_session_id = '' THEN + RAISE EXCEPTION 'VEXFS_INVALID_SESSION: session id is required' + USING ERRCODE = '22023'; + END IF; + IF p_limit < 0 THEN + RAISE EXCEPTION 'VEXFS_INVALID_LIMIT: limit must be zero or positive' + USING ERRCODE = '22023'; + END IF; + v_workspace := _vexfs.require_workspace(p_workspace, 'write'); + SELECT role.oid INTO STRICT v_actor + FROM pg_catalog.pg_roles AS role + WHERE role.rolname = session_user; + IF NOT EXISTS ( + SELECT 1 FROM _vexfs.mount_sessions AS mounted + WHERE mounted.workspace_id = v_workspace.workspace_id + AND mounted.session_id = p_session_id + AND mounted.owner_oid = v_actor + AND mounted.lease_until > clock_timestamp()) THEN + RAISE EXCEPTION 'VEXFS_SESSION_STALE: mount session is not active' + USING ERRCODE = '55006'; + END IF; + + FOR v_handle IN + SELECT handle.handle_id, handle.dirty_generation + FROM _vexfs.handles AS handle + WHERE handle.workspace_id = v_workspace.workspace_id + AND handle.owner_oid = v_actor + AND handle.session_id = p_session_id + AND handle.state = 'open' + AND handle.dirty_generation > handle.published_generation + ORDER BY handle.handle_id + LIMIT NULLIF(p_limit, 0) + LOOP + PERFORM public.vexfs_handle_publish_close( + v_handle.handle_id, v_handle.dirty_generation, p_durability); + v_count := v_count + 1; + END LOOP; + RETURN v_count; +END; +$$; + +-- Publish an exact generation snapshot selected by the gateway. This avoids a +-- race where a newly opened handle sorts ahead of the gateway's local batch +-- after RuntimeState has released its mutex. The whole bounded batch remains +-- atomic, while generation checks prevent a later write from being cleared. +CREATE FUNCTION public.vexfs_mount_publish_close_claimed( + p_workspace text, + p_session_id text, + p_durability text, + p_claims jsonb) +RETURNS jsonb +LANGUAGE plpgsql +SECURITY DEFINER +VOLATILE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; + v_actor oid; + v_claim jsonb; + v_handle text; + v_generation bigint; + v_version bigint; + v_owned _vexfs.handles%ROWTYPE; + v_seen text[] := ARRAY[]::text[]; + v_result jsonb := '[]'::jsonb; +BEGIN + IF p_durability NOT IN ('none', 'data', 'full') THEN + RAISE EXCEPTION 'VEXFS_INVALID_DURABILITY: use none, data, or full' + USING ERRCODE = '22023'; + END IF; + IF p_session_id IS NULL OR p_session_id = '' THEN + RAISE EXCEPTION 'VEXFS_INVALID_SESSION: session id is required' + USING ERRCODE = '22023'; + END IF; + IF p_claims IS NULL OR jsonb_typeof(p_claims) <> 'array' + OR jsonb_array_length(p_claims) < 1 + OR jsonb_array_length(p_claims) > 64 THEN + RAISE EXCEPTION 'VEXFS_INVALID_BATCH: claims must contain 1..64 handles' + USING ERRCODE = '22023'; + END IF; + v_workspace := _vexfs.require_workspace(p_workspace, 'write'); + SELECT role.oid INTO STRICT v_actor + FROM pg_catalog.pg_roles AS role + WHERE role.rolname = session_user; + IF NOT EXISTS ( + SELECT 1 FROM _vexfs.mount_sessions AS mounted + WHERE mounted.workspace_id = v_workspace.workspace_id + AND mounted.session_id = p_session_id + AND mounted.owner_oid = v_actor + AND mounted.lease_until > clock_timestamp()) THEN + RAISE EXCEPTION 'VEXFS_SESSION_STALE: mount session is not active' + USING ERRCODE = '55006'; + END IF; + + FOR v_claim IN SELECT value FROM jsonb_array_elements(p_claims) LOOP + IF jsonb_typeof(v_claim) <> 'object' + OR NOT v_claim ? 'handle' + OR NOT v_claim ? 'generation' THEN + RAISE EXCEPTION 'VEXFS_INVALID_BATCH: claim requires handle and generation' + USING ERRCODE = '22023'; + END IF; + v_handle := v_claim->>'handle'; + BEGIN + v_generation := (v_claim->>'generation')::bigint; + EXCEPTION WHEN invalid_text_representation OR numeric_value_out_of_range THEN + RAISE EXCEPTION 'VEXFS_INVALID_BATCH: claim generation is invalid' + USING ERRCODE = '22023'; + END; + IF v_handle IS NULL OR v_handle = '' OR v_generation < 0 + OR v_handle = ANY(v_seen) THEN + RAISE EXCEPTION 'VEXFS_INVALID_BATCH: claim is empty or duplicated' + USING ERRCODE = '22023'; + END IF; + SELECT * INTO v_owned + FROM _vexfs.handles AS handle + WHERE handle.handle_id = v_handle + AND handle.workspace_id = v_workspace.workspace_id + AND handle.owner_oid = v_actor + AND handle.session_id = p_session_id; + IF NOT FOUND THEN + RAISE EXCEPTION 'VEXFS_HANDLE_NOT_FOUND: claimed handle is unavailable' + USING ERRCODE = 'P0002'; + END IF; + IF v_owned.state = 'closed' + AND v_generation <= v_owned.published_generation THEN + v_seen := array_append(v_seen, v_handle); + v_result := v_result || jsonb_build_array(jsonb_build_object( + 'handle', v_handle, + 'generation', v_generation, + 'version', v_owned.expected_version)); + CONTINUE; + END IF; + IF v_owned.state <> 'open' + OR v_owned.dirty_generation <> v_generation + OR v_owned.dirty_generation <= v_owned.published_generation THEN + RAISE EXCEPTION 'VEXFS_STALE_GENERATION: claimed handle changed before publish' + USING ERRCODE = '40001'; + END IF; + v_seen := array_append(v_seen, v_handle); + v_version := public.vexfs_handle_publish_close( + v_handle, v_generation, p_durability); + v_result := v_result || jsonb_build_array(jsonb_build_object( + 'handle', v_handle, + 'generation', v_generation, + 'version', v_version)); + END LOOP; + RETURN v_result; +END; +$$; + +-- NFS FILE_SYNC writes must be durable before the RPC returns. Apply the range +-- directly to the current manifest so the remote mount pays for one libpq +-- round trip and PG only materializes the affected 64 KiB chunks. +CREATE FUNCTION public.vexfs_write_range( + p_workspace text, + p_path text, + p_offset bigint, + p_content bytea, + p_request_id text, + p_session_id text, + p_durability text DEFAULT 'full') +RETURNS bigint +LANGUAGE plpgsql +SECURITY DEFINER +VOLATILE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace _vexfs.workspaces%ROWTYPE; + v_actor oid; + v_hash text; + v_cached text; + v_inode bigint; + v_kind text; + v_current_version bigint; + v_version bigint; + v_previous_manifest bigint; + v_previous_size bigint; + v_manifest bigint; + v_checksum text; + v_new_size bigint; + v_commit bigint; + v_parent bigint; +BEGIN + IF p_content IS NULL THEN + RAISE EXCEPTION 'VEXFS_INVALID_CONTENT: content cannot be null' + USING ERRCODE = '22004'; + END IF; + IF p_offset < 0 OR p_offset > 134217728 + OR p_offset + octet_length(p_content) > 134217728 THEN + RAISE EXCEPTION 'VEXFS_INVALID_RANGE: write exceeds 128 MiB' + USING ERRCODE = '22023'; + END IF; + IF p_durability NOT IN ('none', 'data', 'full') THEN + RAISE EXCEPTION 'VEXFS_INVALID_DURABILITY: use none, data, or full' + USING ERRCODE = '22023'; + END IF; + v_workspace := _vexfs.require_workspace(p_workspace, 'write'); + SELECT role.oid INTO STRICT v_actor + FROM pg_catalog.pg_roles AS role + WHERE role.rolname = session_user; + IF p_session_id IS NOT NULL AND NOT EXISTS ( + SELECT 1 FROM _vexfs.mount_sessions AS mounted + WHERE mounted.workspace_id = v_workspace.workspace_id + AND mounted.session_id = p_session_id + AND mounted.owner_oid = v_actor + AND mounted.lease_until > clock_timestamp()) THEN + RAISE EXCEPTION 'VEXFS_SESSION_STALE: mount session is not active' + USING ERRCODE = '55006'; + END IF; + v_hash := _vexfs.request_digest(jsonb_build_object( + 'path', p_path, + 'offset', p_offset, + 'content_size', octet_length(p_content), + 'content_checksum', encode(pg_catalog.sha256(p_content), 'hex'), + 'session', p_session_id, + 'durability', p_durability)); + PERFORM _vexfs.lock_request(v_workspace.workspace_id, p_request_id); + v_cached := _vexfs.replay_get( + v_workspace.workspace_id, p_request_id, 'write_range', v_hash); + IF v_cached IS NOT NULL THEN + RETURN v_cached::bigint; + END IF; + + PERFORM pg_catalog.set_config( + 'synchronous_commit', + CASE p_durability WHEN 'none' THEN 'off' + WHEN 'data' THEN 'local' + ELSE 'on' END, + true); + PERFORM 1 FROM _vexfs.workspaces + WHERE workspace_id = v_workspace.workspace_id FOR UPDATE; + v_inode := _vexfs.resolve_path(v_workspace.workspace_id, p_path); + SELECT inode.kind, inode.current_version + INTO STRICT v_kind, v_current_version + FROM _vexfs.inodes AS inode + WHERE inode.workspace_id = v_workspace.workspace_id + AND inode.inode_id = v_inode + AND inode.live + FOR UPDATE; + IF v_kind <> 'file' THEN + RAISE EXCEPTION 'VEXFS_NOT_REGULAR_FILE: %', p_path + USING ERRCODE = '42809'; + END IF; + PERFORM _vexfs.require_inode_permission( + v_workspace.workspace_id, v_inode, 'write'); + SELECT parent.parent_inode INTO v_parent + FROM _vexfs.resolve_parent( + v_workspace.workspace_id, p_path) AS parent; + PERFORM _vexfs.require_inode_permission( + v_workspace.workspace_id, v_parent, 'write'); + + -- POSIX zero-byte writes do not change file contents or create a version. + IF octet_length(p_content) = 0 THEN + PERFORM _vexfs.replay_put( + v_workspace.workspace_id, p_request_id, 'write_range', v_hash, + v_current_version::text); + RETURN v_current_version; + END IF; + + SELECT storage.manifest_id, storage.size_bytes + INTO v_previous_manifest, v_previous_size + FROM _vexfs.resolve_version_storage( + v_workspace.workspace_id, v_inode, v_current_version) AS storage; + v_new_size := greatest(v_previous_size, p_offset + octet_length(p_content)); + PERFORM _vexfs.enforce_quota( + v_workspace.workspace_id, v_inode, v_new_size); + SELECT coalesce(max(version.version_no), 0) + 1 + INTO v_version + FROM _vexfs.file_versions AS version + WHERE version.workspace_id = v_workspace.workspace_id + AND version.inode_id = v_inode; + v_manifest := _vexfs.store_manifest_range( + v_workspace.workspace_id, v_inode, v_previous_manifest, + v_previous_size, p_offset, p_content); + SELECT manifest.checksum INTO STRICT v_checksum + FROM _vexfs.manifests AS manifest + WHERE manifest.manifest_id = v_manifest; + v_commit := _vexfs.record_commit( + v_workspace.workspace_id, 'write_range', p_path, v_inode, + jsonb_build_object( + 'before_version', v_current_version, + 'after_version', v_version, + 'offset', p_offset, + 'length', octet_length(p_content))); + INSERT INTO _vexfs.file_versions( + workspace_id, inode_id, version_no, commit_no, manifest_id, + size_bytes, checksum, created_by_oid, created_by) + VALUES ( + v_workspace.workspace_id, v_inode, v_version, v_commit, v_manifest, + v_new_size, v_checksum, v_actor, session_user); + UPDATE _vexfs.inodes + SET current_version = v_version, + size_bytes = v_new_size, + modified_at = clock_timestamp(), + changed_at = clock_timestamp() + WHERE workspace_id = v_workspace.workspace_id + AND inode_id = v_inode; + UPDATE _vexfs.inodes + SET modified_at = clock_timestamp(), + changed_at = clock_timestamp() + WHERE workspace_id = v_workspace.workspace_id + AND inode_id = v_parent; + PERFORM _vexfs.replay_put( + v_workspace.workspace_id, p_request_id, 'write_range', v_hash, + v_version::text); + RETURN v_version; +END; +$$; + CREATE FUNCTION public.vexfs_handle_close( p_handle text, p_retain_unpublished boolean, @@ -3946,10 +5162,12 @@ BEGIN lease_until = CASE WHEN v_state = 'retained' THEN clock_timestamp() + interval '24 hours' ELSE clock_timestamp() END, - staging = CASE WHEN v_state = 'retained' - THEN staging ELSE ''::bytea END, updated_at = clock_timestamp() WHERE handle_id = p_handle; + IF v_state = 'closed' THEN + DELETE FROM _vexfs.handle_staging + WHERE handle_id = p_handle; + END IF; PERFORM _vexfs.reap_orphan_inodes(v_handle.workspace_id); PERFORM _vexfs.replay_put( v_handle.workspace_id, p_request_id, 'handle_close', v_hash, v_state); @@ -4229,10 +5447,11 @@ BEGIN SELECT count(*) FILTER ( WHERE handle.dirty_generation > handle.published_generation), count(*) FILTER (WHERE handle.state = 'retained'), - coalesce(sum(octet_length(handle.staging)) FILTER ( + coalesce(sum(staging.dirty_bytes) FILTER ( WHERE handle.state IN ('open', 'retained')), 0) INTO v_pending, v_retained, v_staging_bytes FROM _vexfs.handles AS handle + LEFT JOIN _vexfs.handle_staging AS staging USING (handle_id) WHERE handle.workspace_id = v_workspace.workspace_id; RETURN jsonb_build_object( 'schema_version', public.vexfs_contract_version(), @@ -6945,10 +8164,20 @@ BEGIN AND f.inode_id = i.inode_id AND f.version_no = i.current_version WHERE i.workspace_id = v_workspace.workspace_id - AND ((i.kind <> 'directory' AND ( - i.current_version < 1 - OR f.version_no IS NULL - OR i.size_bytes IS DISTINCT FROM f.size_bytes)) + AND ((i.kind <> 'directory' + AND NOT ( + i.kind = 'file' + AND i.current_version = 0 + AND i.size_bytes = 0 + AND EXISTS ( + SELECT 1 FROM _vexfs.handles AS pending + WHERE pending.workspace_id = i.workspace_id + AND pending.inode_id = i.inode_id + AND pending.state IN ('open', 'retained') + AND pending.dirty_generation > pending.published_generation)) + AND (i.current_version < 1 + OR f.version_no IS NULL + OR i.size_bytes IS DISTINCT FROM f.size_bytes)) OR (i.kind = 'directory' AND i.current_version <> 0)); IF v_bad <> 0 THEN v_issues := v_issues || jsonb_build_array(jsonb_build_object( @@ -7067,6 +8296,51 @@ BEGIN 'message', 'chunk is not referenced by a manifest')); END IF; + -- Writable handles are mutable state, but their overlay still has strict + -- invariants. Check them here so a damaged dirty chunk cannot remain + -- invisible until the next publish attempt. + SELECT count(*) INTO v_bad + FROM _vexfs.handles AS handle + LEFT JOIN _vexfs.handle_staging AS staging + ON staging.handle_id = handle.handle_id + LEFT JOIN _vexfs.manifests AS base + ON base.manifest_id = staging.base_manifest_id + WHERE handle.workspace_id = v_workspace.workspace_id + AND ( + (handle.state IN ('open', 'retained') + AND position('w' IN handle.flags) > 0 + AND staging.handle_id IS NULL) + OR (staging.handle_id IS NOT NULL + AND (handle.state = 'closed' + OR position('w' IN handle.flags) = 0 + OR staging.generation <> handle.dirty_generation + OR staging.dirty_bytes IS DISTINCT FROM ( + SELECT coalesce(sum(octet_length(dirty.content)), 0) + FROM _vexfs.handle_staging_chunks AS dirty + WHERE dirty.handle_id = staging.handle_id) + OR (staging.base_manifest_id IS NULL + AND staging.base_size <> 0) + OR (staging.base_manifest_id IS NOT NULL + AND (base.manifest_id IS NULL + OR base.workspace_id <> handle.workspace_id + OR base.inode_id IS DISTINCT FROM handle.inode_id + OR base.file_size <> staging.base_size)) + OR EXISTS ( + SELECT 1 + FROM _vexfs.handle_staging_chunks AS dirty + WHERE dirty.handle_id = staging.handle_id + AND (dirty.chunk_no::bigint * 65536 >= staging.logical_size + OR octet_length(dirty.content) <> + least( + 65536::bigint, + staging.logical_size + - dirty.chunk_no::bigint * 65536)))))); + IF v_bad <> 0 THEN + v_issues := v_issues || jsonb_build_array(jsonb_build_object( + 'code', 'handle_staging', 'count', v_bad, + 'message', 'writable handle staging metadata or dirty chunks are invalid')); + END IF; + SELECT count(*) INTO v_bad FROM _vexfs.snapshots AS s WHERE s.workspace_id = v_workspace.workspace_id From bdc7d1509c4a0db3399624753c96f9849acc0714 Mon Sep 17 00:00:00 2001 From: Four <8122332+HungryFour@users.noreply.github.com> Date: Mon, 27 Jul 2026 15:19:02 +0800 Subject: [PATCH 15/28] Complete PostgreSQL VexFS crash-safe publishing --- agent_files/cli/vexfs_archive.cpp | 10 +- agent_files/cli/vexfs_platform_macos.cpp | 18 +- .../mount/common/include/vexfs_checksum.h | 12 + .../mount/common/include/vexfs_runtime_io.h | 6 + .../mount/common/src/vexfs_checksum.cpp | 38 ++ .../src/vexfs_mount_contract_sqlite.cpp | 30 ++ .../common/test/vexfs_pg_runtime_smoke.cpp | 106 +++- .../mount/macos/nfs_gateway/src/lib.rs | 248 +++++---- ...2026-07-21_vexfs-final-goal-and-roadmap.md | 53 +- tests/eval/vexfs/README.md | 17 +- tests/eval/vexfs/run_pg_macos_mount.sh | 5 +- .../run_pg_manifest_publish_performance.sh | 138 +++++ .../vexfs/run_pg_strict_crash_recovery.sh | 173 +++++++ .../pg/agent_files/vexfs_pg_chunk_check.yaml | 13 +- .../agent_files/vexfs_pg_staging_overlay.yaml | 26 +- .../sqlite/agent_files/vexfs_contract.yaml | 5 +- vexdb_pg/sql/vexdb_lite--1.0.sql | 471 ++++++++++++++---- vexdb_sqlite/src/agent_files/vexfs_sqlite.cpp | 48 +- 18 files changed, 1140 insertions(+), 277 deletions(-) create mode 100755 tests/eval/vexfs/run_pg_manifest_publish_performance.sh create mode 100755 tests/eval/vexfs/run_pg_strict_crash_recovery.sh diff --git a/agent_files/cli/vexfs_archive.cpp b/agent_files/cli/vexfs_archive.cpp index 35fd6d7397..ebe6388fcd 100644 --- a/agent_files/cli/vexfs_archive.cpp +++ b/agent_files/cli/vexfs_archive.cpp @@ -1379,7 +1379,8 @@ SELECT id,chunk_no,size,checksum,length(content) FROM package.chunks WHERE source_manifest=?1 ORDER BY chunk_no )SQL"); chunks.Int64(1, source_manifest); - vexfs::Sha256 file_hash; + std::vector manifest_chunks; + manifest_chunks.reserve(static_cast(expected_chunks)); sqlite3_int64 total = 0; sqlite3_int64 chunk_no = 0; while (chunks.Row()) { @@ -1410,17 +1411,20 @@ WHERE source_manifest=?1 ORDER BY chunk_no throw Error("cannot read archive chunk BLOB"); } chunk_hash.Update(buffer.data(), static_cast(amount)); - file_hash.Update(buffer.data(), static_cast(amount)); offset += amount; } if (vexfs::Hex(chunk_hash.Finish()) != chunk_checksum) { throw Error("archive chunk checksum mismatch"); } + manifest_chunks.push_back({ + static_cast(chunk_size), chunk_checksum}); total += chunk_size; ++chunk_no; } if (total != expected_size || chunk_no != expected_chunks || - vexfs::Hex(file_hash.Finish()) != expected_checksum) { + vexfs::ManifestChecksum( + static_cast(expected_size), 65536, manifest_chunks) != + expected_checksum) { throw Error("archive content checksum mismatch"); } info.content_bytes += expected_size; diff --git a/agent_files/cli/vexfs_platform_macos.cpp b/agent_files/cli/vexfs_platform_macos.cpp index ccb66ae7c3..c38e61a3b4 100644 --- a/agent_files/cli/vexfs_platform_macos.cpp +++ b/agent_files/cli/vexfs_platform_macos.cpp @@ -301,11 +301,19 @@ class UnderlyingMountPointGuard { }; void DisarmMountPointGuard(const std::string &path) { - std::error_code error; - if (!std::filesystem::is_directory(path, error)) return; - if (chmod(path.c_str(), 0700) != 0) - throw std::runtime_error("cannot restore mount point permissions: " + - std::string(std::strerror(errno))); + // After a forced NFS unmount, getmntinfo can stop reporting the mount a few + // seconds before the stale vnode at the path becomes usable. Calling chmod + // in this process can then block until the NFS dead timeout (60 seconds). + // Keep each probe bounded in a child process and retry while macOS finishes + // detaching the old vnode. + constexpr int kPermissionRestoreAttempts = 12; + for (int attempt = 0; attempt < kPermissionRestoreAttempts; ++attempt) { + const int result = RunProcess("/bin/chmod", {"0700", path}, 1'000); + if (result == 0) return; + if (attempt + 1 != kPermissionRestoreAttempts) usleep(250 * 1000); + } + throw std::runtime_error( + "cannot restore mount point permissions after NFS detach"); } uint64_t StableHash(const std::string &value) { diff --git a/agent_files/mount/common/include/vexfs_checksum.h b/agent_files/mount/common/include/vexfs_checksum.h index 861d24b8bf..e80a30bb42 100644 --- a/agent_files/mount/common/include/vexfs_checksum.h +++ b/agent_files/mount/common/include/vexfs_checksum.h @@ -5,6 +5,7 @@ #include #include #include +#include namespace vexfs { @@ -28,6 +29,17 @@ class Sha256 { std::string Sha256Hex(const void *data, size_t size); std::string Hex(const std::array &digest); +struct ManifestChunkChecksum { + uint64_t size = 0; + std::string checksum; +}; + +// Must stay byte-for-byte compatible with _vexfs.compute_manifest_checksum in +// PostgreSQL. The root covers file length, chunk size, order, each chunk length, +// and each chunk's content SHA-256 without hashing the complete file again. +std::string ManifestChecksum(uint64_t file_size, uint64_t chunk_size, + const std::vector &chunks); + } // namespace vexfs #endif // VEXFS_CHECKSUM_H diff --git a/agent_files/mount/common/include/vexfs_runtime_io.h b/agent_files/mount/common/include/vexfs_runtime_io.h index fa3d15052e..0a3528e8e4 100644 --- a/agent_files/mount/common/include/vexfs_runtime_io.h +++ b/agent_files/mount/common/include/vexfs_runtime_io.h @@ -156,6 +156,12 @@ vexfs_mount_status vexfs_mount_handle_publish_close(vexfs_mount_session *session const char *durability, int64_t *version, vexfs_mount_error *error); +// PostgreSQL-only single-file background path. It uses the dedicated publisher +// connection, so one implicit transaction releases the workspace row lock +// before the next claimed file begins publishing. +vexfs_mount_status vexfs_mount_handle_publish_close_background( + vexfs_mount_session *session, const char *handle, int64_t generation, + const char *durability, int64_t *version, vexfs_mount_error *error); vexfs_mount_status vexfs_mount_handle_close(vexfs_mount_session *session, const char *handle, int retain_unpublished, const char *request_id, diff --git a/agent_files/mount/common/src/vexfs_checksum.cpp b/agent_files/mount/common/src/vexfs_checksum.cpp index d32ac359db..9909a53e4f 100644 --- a/agent_files/mount/common/src/vexfs_checksum.cpp +++ b/agent_files/mount/common/src/vexfs_checksum.cpp @@ -2,6 +2,7 @@ #include #include +#include #include namespace vexfs { @@ -154,4 +155,41 @@ std::string Sha256Hex(const void *data, size_t size) { return Hex(hash.Finish()); } +std::string ManifestChecksum( + uint64_t file_size, uint64_t chunk_size, + const std::vector &chunks) { + if (chunk_size == 0) throw std::invalid_argument("manifest chunk size is zero"); + const uint64_t expected_chunks = file_size == 0 + ? 0 : 1 + (file_size - 1) / chunk_size; + if (expected_chunks != chunks.size()) { + throw std::invalid_argument("manifest chunk count does not match file size"); + } + uint64_t total = 0; + Sha256 hash; + const std::string header = "vexfs-manifest-v1:" + std::to_string(chunk_size) + + ":" + std::to_string(file_size) + ":" + std::to_string(chunks.size()) + "\n"; + hash.Update(header.data(), header.size()); + for (size_t index = 0; index < chunks.size(); ++index) { + const ManifestChunkChecksum &chunk = chunks[index]; + const uint64_t expected_size = std::min(chunk_size, file_size - total); + const bool valid_checksum = chunk.checksum.size() == 64 && + std::all_of(chunk.checksum.begin(), chunk.checksum.end(), [](unsigned char value) { + return (value >= '0' && value <= '9') || (value >= 'a' && value <= 'f'); + }); + if (chunk.size != expected_size || !valid_checksum || + chunk.size > std::numeric_limits::max() - total) { + throw std::invalid_argument("manifest chunk metadata is invalid"); + } + if (index != 0) hash.Update("\n", 1); + const std::string entry = std::to_string(index) + ":" + + std::to_string(chunk.size) + ":" + chunk.checksum; + hash.Update(entry.data(), entry.size()); + total += chunk.size; + } + if (total != file_size) { + throw std::invalid_argument("manifest chunks do not cover the file"); + } + return Hex(hash.Finish()); +} + } // namespace vexfs diff --git a/agent_files/mount/common/src/vexfs_mount_contract_sqlite.cpp b/agent_files/mount/common/src/vexfs_mount_contract_sqlite.cpp index befaabeeee..a80cc454e7 100644 --- a/agent_files/mount/common/src/vexfs_mount_contract_sqlite.cpp +++ b/agent_files/mount/common/src/vexfs_mount_contract_sqlite.cpp @@ -2493,6 +2493,36 @@ extern "C" vexfs_mount_status vexfs_mount_handle_publish_close( }); } +extern "C" vexfs_mount_status vexfs_mount_handle_publish_close_background( + vexfs_mount_session *session, const char *handle, int64_t generation, + const char *durability, int64_t *version, vexfs_mount_error *error) { +#if defined(VEXFS_HAVE_LIBPQ) + return GuardPublisher(session, error, [&] { + if (handle == nullptr || durability == nullptr || version == nullptr) { + throw CallError(kPgInvalidArgument, + "handle, durability and version output are required", + VEXFS_RUNTIME_BACKEND_POSTGRESQL); + } + Call call(session->publisher_pg, + "SELECT vexfs_handle_publish_close(?1,?2,?3)"); + call.Text(1, handle); + call.Int64(2, generation); + call.Text(3, durability); + call.Row(); + *version = call.ResultInt64(); + }); +#else + (void)session; + (void)handle; + (void)generation; + (void)durability; + (void)version; + return SetError(error, kUnsupportedBackend, + "background publisher requires PostgreSQL", + VEXFS_RUNTIME_BACKEND_POSTGRESQL); +#endif +} + extern "C" vexfs_mount_status vexfs_mount_handle_close( vexfs_mount_session *session, const char *handle, int retain_unpublished, const char *request_id, vexfs_mount_bytes *state, vexfs_mount_error *error) { diff --git a/agent_files/mount/common/test/vexfs_pg_runtime_smoke.cpp b/agent_files/mount/common/test/vexfs_pg_runtime_smoke.cpp index 95b506e7fe..a27de917e0 100644 --- a/agent_files/mount/common/test/vexfs_pg_runtime_smoke.cpp +++ b/agent_files/mount/common/test/vexfs_pg_runtime_smoke.cpp @@ -72,6 +72,35 @@ bool PgCommand(PGconn *connection, const char *sql) { return ok; } +std::string PgScalar(PGconn *connection, const char *sql) { + PGresult *result = PQexec(connection, sql); + const bool ok = result != nullptr && PQresultStatus(result) == PGRES_TUPLES_OK && + PQntuples(result) == 1 && PQnfields(result) == 1 && !PQgetisnull(result, 0, 0); + std::string value; + if (ok) { + value.assign(PQgetvalue(result, 0, 0), + static_cast(PQgetlength(result, 0, 0))); + } else { + std::fprintf(stderr, "PostgreSQL scalar query failed: %s\n", + result == nullptr ? PQerrorMessage(connection) : + PQresultErrorMessage(result)); + } + if (result != nullptr) PQclear(result); + return value; +} + +bool WaitForBackendIdle(PGconn *control, int backend_pid) { + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + const std::string sql = + "SELECT coalesce((SELECT state='idle' FROM pg_stat_activity WHERE pid=" + + std::to_string(backend_pid) + "),false)"; + while (std::chrono::steady_clock::now() < deadline) { + if (PgScalar(control, sql.c_str()) == "t") return true; + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + } + return false; +} + bool ReconnectControl(PGconn *connection) { bool probe_ok = false; if (PQstatus(connection) == CONNECTION_OK) { @@ -250,11 +279,12 @@ int main(int argc, char **argv) { std::atomic publisher_started{false}; vexfs_mount_status publisher_status = VEXFS_MOUNT_DATABASE_ERROR; vexfs_mount_error publisher_error{}; - vexfs_mount_bytes publisher_result{}; + int64_t publisher_version = 0; std::thread publisher([&] { publisher_started.store(true); - publisher_status = vexfs_mount_publish_close_claimed( - session, "full", claims.c_str(), &publisher_result, &publisher_error); + publisher_status = vexfs_mount_handle_publish_close_background( + session, claimed_handle.c_str(), claimed_generation, "full", + &publisher_version, &publisher_error); }); while (!publisher_started.load()) std::this_thread::yield(); std::this_thread::sleep_for(std::chrono::milliseconds(150)); @@ -270,10 +300,14 @@ int main(int argc, char **argv) { !Expect(foreground_elapsed < std::chrono::seconds(1), "background publisher does not hold foreground runtime mutex") || !Expect(unlocked, "unlock workspace after concurrency probe") || - !CheckStatus("claimed publisher", publisher_status, publisher_error)) return 1; - const std::string claimed_result = Take(&publisher_result); - if (!Expect(claimed_result.find(claimed_handle) != std::string::npos, - "claimed publisher result") || + !CheckStatus("single-file background publisher", publisher_status, + publisher_error) || + !Expect(publisher_version == 1, "background publisher version")) return 1; + if (!CheckStatus("background publisher replay", + vexfs_mount_handle_publish_close_background( + session, claimed_handle.c_str(), claimed_generation, "full", + &publisher_version, &error), error) || + !Expect(publisher_version == 1, "background publisher replay version") || !CheckStatus("claimed publisher replay", vexfs_mount_publish_close_claimed( session, "full", claims.c_str(), &bytes, &error), error) || !Expect(Take(&bytes).find(claimed_handle) != std::string::npos, @@ -282,6 +316,62 @@ int main(int argc, char **argv) { session, "/project/claimed.txt", &bytes, &error), error) || !Expect(Take(&bytes) == "claimed", "claimed content value")) return 1; + // Model the hard network case where PostgreSQL commits but the client never + // consumes the result. A second control connection observes the committed + // content before the first connection is discarded. Retrying the exact + // handle generation must return version 1 without creating a duplicate. + vexfs_mount_bytes unknown_handle_bytes{}; + if (!CheckStatus("unknown-result create", vexfs_mount_handle_create_owned_durable( + session, "/project/unknown-result.txt", 0644, 501, 20, + "create-unknown-result", "none", &unknown_handle_bytes, &error), error)) return 1; + const std::string unknown_handle = Take(&unknown_handle_bytes); + int64_t unknown_generation = 0; + if (!CheckStatus("unknown-result stage", vexfs_mount_handle_stage_write_durable( + session, unknown_handle.c_str(), 0, "unknown", 7, + "stage-unknown-result", "none", &unknown_generation, &error), error)) return 1; + PGconn *unknown = PQconnectdb(dsn); + if (!Expect(unknown != nullptr && PQstatus(unknown) == CONNECTION_OK, + "unknown-result connection")) { + if (unknown != nullptr) PQfinish(unknown); + return 1; + } + const std::string generation_text = std::to_string(unknown_generation); + const char *unknown_values[] = {unknown_handle.c_str(), generation_text.c_str()}; + if (!Expect(PQsendQueryParams( + unknown, "SELECT vexfs_handle_publish_close($1,$2::bigint,'full')", 2, + nullptr, unknown_values, nullptr, nullptr, 0) == 1, + "send publish without consuming result")) { + PQfinish(unknown); + return 1; + } + while (PQflush(unknown) == 1) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + if (!Expect(PQstatus(unknown) == CONNECTION_OK, + "unknown-result query sent") || + !Expect(WaitForBackendIdle(control, PQbackendPID(unknown)), + "unknown-result server commit completed") || + !Expect(PgScalar(control, + "SELECT convert_from(vexfs_read('pg-runtime-contract'," + "'/project/unknown-result.txt'),'UTF8')") == "unknown", + "unknown-result commit visible before response read")) { + PQfinish(unknown); + return 1; + } + PQfinish(unknown); + int64_t unknown_version = 0; + if (!CheckStatus("unknown-result generation retry", + vexfs_mount_handle_publish_close_background( + session, unknown_handle.c_str(), unknown_generation, "full", + &unknown_version, &error), error) || + !Expect(unknown_version == 1, "unknown-result retry version") || + !Expect(PgScalar(control, + "SELECT count(*) FROM _vexfs.file_versions AS version " + "WHERE version.workspace_id=(SELECT workspace_id FROM _vexfs.workspaces " + "WHERE name='pg-runtime-contract') AND version.inode_id=" + "((vexfs_stat('pg-runtime-contract','/project/unknown-result.txt')->>" + "'inode')::bigint)") == "1", "unknown-result has one version")) return 1; + if (!CheckStatus("write v1", vexfs_mount_write_file( session, "/project/main.txt", "alpha\n", 6, &version, &error), error) || !Expect(version == 1, "write version 1")) return 1; @@ -347,7 +437,7 @@ int main(int argc, char **argv) { const std::string enabled_index = Take(&bytes); if (!Expect(enabled_index.find("\"enabled\":true") != std::string::npos && enabled_index.find("\"dirty\":false") != std::string::npos && - enabled_index.find("\"indexed_files\":5") != std::string::npos, + enabled_index.find("\"indexed_files\":6") != std::string::npos, "trigram index enabled") || !CheckStatus("indexed grep", vexfs_mount_grep( session, "/project", "alpha", 0, 100, &bytes, &error), error)) return 1; diff --git a/agent_files/mount/macos/nfs_gateway/src/lib.rs b/agent_files/mount/macos/nfs_gateway/src/lib.rs index 8db631b1aa..dd453f373a 100644 --- a/agent_files/mount/macos/nfs_gateway/src/lib.rs +++ b/agent_files/mount/macos/nfs_gateway/src/lib.rs @@ -252,6 +252,14 @@ extern "C" { version: *mut i64, error: *mut RuntimeError, ) -> RuntimeStatus; + fn vexfs_mount_handle_publish_close_background( + session: *mut RuntimeSessionOpaque, + handle: *const c_char, + generation: i64, + durability: *const c_char, + version: *mut i64, + error: *mut RuntimeError, + ) -> RuntimeStatus; fn vexfs_mount_handle_close( session: *mut RuntimeSessionOpaque, handle: *const c_char, @@ -273,13 +281,6 @@ extern "C" { published: *mut i64, error: *mut RuntimeError, ) -> RuntimeStatus; - fn vexfs_mount_publish_close_claimed( - session: *mut RuntimeSessionOpaque, - durability: *const c_char, - claims_json: *const c_char, - result_json: *mut RuntimeBytes, - error: *mut RuntimeError, - ) -> RuntimeStatus; fn vexfs_mount_free(memory: *mut c_void); } @@ -369,6 +370,11 @@ struct PublishedClaim { version: i64, } +struct PublishBatchOutcome { + published: Vec, + failure: Option, +} + struct RuntimeState { session: *mut RuntimeSessionOpaque, postgresql_backend: bool, @@ -470,40 +476,48 @@ fn runtime_result(status: RuntimeStatus) -> Result<(), nfsstat3> { fn publish_claimed_batch( session: *mut RuntimeSessionOpaque, candidates: &[PublishCandidate], -) -> Result, nfsstat3> { - let claims: Result, nfsstat3> = candidates - .iter() - .map(|candidate| { - Ok(serde_json::json!({ - "handle": candidate - .handle - .to_str() - .map_err(|_| nfsstat3::NFS3ERR_IO)?, - "generation": candidate.generation, - })) - }) - .collect(); - let claims = CString::new( - serde_json::to_string(&claims?).map_err(|_| nfsstat3::NFS3ERR_IO)?, - ) - .map_err(|_| nfsstat3::NFS3ERR_IO)?; +) -> PublishBatchOutcome { let durability = CString::new("full").unwrap(); - let mut result = RuntimeBytes { - data: ptr::null_mut(), - size: 0, - }; - let mut error = empty_error(); - let status = unsafe { - vexfs_mount_publish_close_claimed( - session, - durability.as_ptr(), - claims.as_ptr(), - &mut result, - &mut error, - ) - }; - runtime_result(status)?; - serde_json::from_str(&bytes_to_string(result)?).map_err(|_| nfsstat3::NFS3ERR_IO) + let mut published = Vec::with_capacity(candidates.len()); + for candidate in candidates { + let Ok(handle) = candidate.handle.to_str() else { + return PublishBatchOutcome { + published, + failure: Some(nfsstat3::NFS3ERR_IO), + }; + }; + let mut version = 0; + let mut error = empty_error(); + let status = unsafe { + vexfs_mount_handle_publish_close_background( + session, + candidate.handle.as_ptr(), + candidate.generation, + durability.as_ptr(), + &mut version, + &mut error, + ) + }; + // Each call is its own implicit PG transaction. If a later call fails, + // the gateway clears completed items and releases only the unreported + // remainder. An unknown commit result is still safe to retry because + // the exact generation returns its committed version idempotently. + if let Err(failure) = runtime_result(status) { + return PublishBatchOutcome { + published, + failure: Some(failure), + }; + } + published.push(PublishedClaim { + handle: handle.to_owned(), + generation: candidate.generation, + version, + }); + } + PublishBatchOutcome { + published, + failure: None, + } } fn now_ms() -> i64 { @@ -1010,7 +1024,12 @@ impl RuntimeState { left_pending .updated_at_ms .cmp(&right_pending.updated_at_ms) - .then_with(|| left_pending.handle.as_bytes().cmp(right_pending.handle.as_bytes())) + .then_with(|| { + left_pending + .handle + .as_bytes() + .cmp(right_pending.handle.as_bytes()) + }) }); let mut claimed = Vec::new(); @@ -1031,8 +1050,7 @@ impl RuntimeState { } let candidate_bytes = pending.dirty_bytes.max(1); if !claimed.is_empty() - && claimed_bytes.saturating_add(candidate_bytes) - > DEFERRED_PUBLISH_BATCH_BYTES + && claimed_bytes.saturating_add(candidate_bytes) > DEFERRED_PUBLISH_BATCH_BYTES { break; } @@ -1069,41 +1087,43 @@ impl RuntimeState { candidates: &[PublishCandidate], published: &[PublishedClaim], ) -> Result { - if published.len() != candidates.len() { + if published.len() > candidates.len() + || !candidates + .iter() + .zip(published.iter()) + .all(|(candidate, item)| { + candidate.handle.to_bytes() == item.handle.as_bytes() + && candidate.generation == item.generation + }) + { self.release_publish_claims(candidates); return Err(nfsstat3::NFS3ERR_IO); } - for candidate in candidates { - let handle = candidate - .handle - .to_str() - .map_err(|_| nfsstat3::NFS3ERR_IO)?; - if !published - .iter() - .any(|item| item.handle == handle && item.generation == candidate.generation) - { - self.release_publish_claims(candidates); - return Err(nfsstat3::NFS3ERR_IO); - } - } let updated_at = now_ms(); let mut completed = 0; - for candidate in candidates { - let matches = self.pending_writes.get(&candidate.inode).is_some_and(|pending| { - pending.handle.as_bytes() == candidate.handle.as_bytes() - && pending.generation == candidate.generation - && pending.publishing_generation == Some(candidate.generation) - }); + for (index, candidate) in candidates.iter().enumerate() { + let matches = self + .pending_writes + .get(&candidate.inode) + .is_some_and(|pending| { + pending.handle.as_bytes() == candidate.handle.as_bytes() + && pending.generation == candidate.generation + && pending.publishing_generation == Some(candidate.generation) + }); if !matches { continue; } - self.pending_writes.remove(&candidate.inode); - if let Some(stat) = self.stats.get_mut(&candidate.inode) { - stat.size = candidate.logical_size.min(i64::MAX as u64) as i64; - stat.updated_at = updated_at; - stat.changed_at = updated_at; + if index < published.len() { + self.pending_writes.remove(&candidate.inode); + if let Some(stat) = self.stats.get_mut(&candidate.inode) { + stat.size = candidate.logical_size.min(i64::MAX as u64) as i64; + stat.updated_at = updated_at; + stat.changed_at = updated_at; + } + completed += 1; + } else if let Some(pending) = self.pending_writes.get_mut(&candidate.inode) { + pending.publishing_generation = None; } - completed += 1; } Ok(completed) } @@ -1601,7 +1621,10 @@ impl NFSFileSystem for VexfsNfs { .get(&id) .is_some_and(|pending| pending.dirty_bytes >= DEFERRED_PUBLISH_DIRTY_BYTES); // Strict mode reports UNSTABLE unless this request was actually FULL. - Ok((attr, !state.strict_durability && !requires_foreground_commit)) + Ok(( + attr, + !state.strict_durability && !requires_foreground_commit, + )) } async fn create( @@ -2055,8 +2078,8 @@ fn run_gateway(config: &VexfsNfsGatewayConfig) -> Result<(), String> { .max() .is_some_and(|updated_at_ms| updated_at_ms <= cutoff); if idle { - if let Err(status) = state - .publish_pending_batch("full", DEFERRED_PUBLISH_BATCH_SIZE) + if let Err(status) = + state.publish_pending_batch("full", DEFERRED_PUBLISH_BATCH_SIZE) { eprintln!( "vexfs-nfs-gateway: deferred batch publish failed: {status:?}" @@ -2069,29 +2092,17 @@ fn run_gateway(config: &VexfsNfsGatewayConfig) -> Result<(), String> { let Some((session, candidates)) = publish_job else { continue; }; - let result = publish_claimed_batch( - session as *mut RuntimeSessionOpaque, - &candidates, - ); + let outcome = + publish_claimed_batch(session as *mut RuntimeSessionOpaque, &candidates); let Ok(mut state) = publish_state.lock() else { return; }; - match result { - Ok(published) => { - if let Err(status) = - state.complete_publish_claims(&candidates, &published) - { - eprintln!( - "vexfs-nfs-gateway: deferred publish result mismatch: {status:?}" - ); - } - } - Err(status) => { - state.release_publish_claims(&candidates); - eprintln!( - "vexfs-nfs-gateway: deferred claimed publish failed: {status:?}" - ); - } + if let Err(status) = state.complete_publish_claims(&candidates, &outcome.published) + { + eprintln!("vexfs-nfs-gateway: deferred publish result mismatch: {status:?}"); + } + if let Some(status) = outcome.failure { + eprintln!("vexfs-nfs-gateway: deferred claimed publish failed: {status:?}"); } } }); @@ -2217,7 +2228,12 @@ mod tests { fn foreground_write_burst_is_coalesced_until_globally_idle() { let now = 10_000; let mut state = state_with_pending(&[ - (1, now - DEFERRED_PUBLISH_IDLE_MS, now - DEFERRED_PUBLISH_IDLE_MS, 4_096), + ( + 1, + now - DEFERRED_PUBLISH_IDLE_MS, + now - DEFERRED_PUBLISH_IDLE_MS, + 4_096, + ), (2, now, now, 4_096), ]); let claimed = state.claim_deferred_publish_batch(now); @@ -2238,16 +2254,10 @@ mod tests { #[test] fn dirty_byte_threshold_flushes_before_idle_timeout() { let now = 20_000; - let mut state = state_with_pending(&[( - 7, - now, - now, - DEFERRED_PUBLISH_DIRTY_BYTES, - )]); + let mut state = state_with_pending(&[(7, now, now, DEFERRED_PUBLISH_DIRTY_BYTES)]); let claimed = state.claim_deferred_publish_batch(now); assert!(claimed.is_empty()); - let claimed = - state.claim_deferred_publish_batch(now + DEFERRED_PUBLISH_IDLE_MS); + let claimed = state.claim_deferred_publish_batch(now + DEFERRED_PUBLISH_IDLE_MS); assert_eq!(claimed.len(), 1); assert_eq!(claimed[0].inode, 7); } @@ -2286,8 +2296,38 @@ mod tests { let mut state = state_with_pending(&entries); let claimed = state.claim_deferred_publish_batch(now); assert_eq!(claimed.len(), DEFERRED_PUBLISH_BATCH_SIZE as usize); - assert!(claimed.iter().all(|candidate| { - candidate.inode != DEFERRED_PUBLISH_FILE_THRESHOLD as u64 - })); + assert!(claimed + .iter() + .all(|candidate| { candidate.inode != DEFERRED_PUBLISH_FILE_THRESHOLD as u64 })); + } + + #[test] + fn partial_publish_clears_success_and_releases_remainder() { + let now = 50_000; + let mut state = state_with_pending(&[ + (1, now - DEFERRED_PUBLISH_IDLE_MS, now - 1_000, 4_096), + (2, now - DEFERRED_PUBLISH_IDLE_MS, now - 1_000, 4_096), + ]); + let claimed = state.claim_deferred_publish_batch(now); + assert_eq!(claimed.len(), 2); + let first = &claimed[0]; + let published = [PublishedClaim { + handle: first.handle.to_str().unwrap().to_owned(), + generation: first.generation, + version: 1, + }]; + assert_eq!( + state.complete_publish_claims(&claimed, &published).unwrap(), + 1 + ); + assert!(!state.pending_writes.contains_key(&first.inode)); + assert_eq!( + state + .pending_writes + .get(&claimed[1].inode) + .unwrap() + .publishing_generation, + None + ); } } diff --git a/docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md b/docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md index 73cd301f68..43a6e621a9 100644 --- a/docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md +++ b/docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md @@ -2,15 +2,15 @@ - 文档类型：唯一有效的目标和开发顺序基线 - 所属产品：VexDB-Lite 的文件管理能力 VexFS -- 日期：2026-07-24 +- 日期：2026-07-27 - 分支：`feature/agent_files` -- 文档版本：3.2 +- 文档版本：3.3 - macOS 默认入口：本机 NFS gateway；FSKit 延后为可选原生增强 -- 当前阶段：Phase 2 PostgreSQL 数据库合同和本机默认 NFS 验收已完成。PG VexFS spec 8/8、 - libpq runtime 105 项、macOS 默认 NFS 14/14 场景共 252 项检查全部通过，无跳过。两个独立 - gateway 已同时挂载同一 PG workspace 并完成双向 `fsync`、版本、快照和清理。PG 已加入 - 64 KiB range read/COW、同 session 批量发布、正确的 NFS `UNSTABLE → COMMIT` 持久化语义 - 和 250 ms 跨 gateway 元数据刷新。历史 FSKit 两 Mac、macOS↔Linux、PG 16–19、备份、 +- 当前阶段：Phase 2 PostgreSQL 数据库合同和本机默认 NFS 验收已完成。PG VexFS spec 9/9、 + libpq runtime 119 项、manifest publish 专项 10 项和真实 macOS NFS 性能 18 项检查通过。 + PG writable handle 已使用基础 manifest + 64 KiB 脏块，发布直接复用未修改块并生成 + `manifest-v1` 根哈希，不再在数据库进程中拼接完整文件。历史 FSKit 两 Mac、 + macOS↔Linux、PG 16–19、备份、 format v2 和真实 OpenCode 证据继续有效，但当前默认 NFS 源码对应的新公证包和第二台 Mac NFS 复验还没有完成。下一步先做发行 Gate；数据库方向随后进入 DuckDB adapter。 @@ -32,7 +32,7 @@ 1. **已完成**：冻结 NFS adapter 与 Workspace Engine 的窄接口，不复制数据库逻辑； 2. **已完成**：实现 localhost-only NFS gateway、mount/unmount/status/doctor； 3. **部分完成**：Bash、Git、hardlink、xattr、fsync、单机跨进程 `flock`/`fcntl`、轻量 npm/Cargo 和单次真实 OpenCode 已进入真机 eval； - 跨机器锁、长任务和 sleep/wake 尚待补齐；gateway 单次 `SIGKILL` 后的强制卸载、状态清理和重挂载已覆盖； + 跨机器锁、长任务和 sleep/wake 尚待补齐；gateway `SIGKILL` + PG immediate restart 当前单轮 16 项、连续 3/3 轮已覆盖； 4. **已完成首轮**：1,000 小文件为 205.643 files/s，同轮 APFS 为 16940.431 files/s； 5. **待完成**：生成不含 FSKit 授权前置的签名公证包，在干净 Mac 验证安装和系统提示； 6. **代码已完成，发行待 Gate**：默认 backend 已改为 NFS，FSKit 只在显式选择时使用。 @@ -67,7 +67,7 @@ VexFS FSKit extension。 - AppleDouble 隔离已进入 eval；后续每次 NFS 发布必须保留该断言； - 0.2 实现批量提交、写回合并和有界 metadata cache，再做同机 1,000/10,000 文件对照； -- 0.1 发布前仍需完成干净 Mac 安装、公证、重复 crash/sleep-wake、跨机器锁边界与长期运行 Gate。 +- 0.1 发布前仍需完成干净 Mac 安装、公证、长循环 crash/sleep-wake、跨机器锁边界与长期运行 Gate。 ## 1. 这份文档解决什么问题 @@ -152,19 +152,19 @@ vexdb fs --workspace workspace snapshot restore before-refactor - 不为了宣传“更快 grep”而阻塞挂载正确性、备份和跨机器工作区； - 不在核心文件合同中加入 Agent 专用的 prompt、memory 或语义字段。 -## 5. 当前基线：2026-07-26 +## 5. 当前基线：2026-07-27 | 范围 | 当前事实 | 主要缺口 | |---|---|---| | 统一合同 | SQLite 合同 `0.9.0`，runtime ABI 1；SQL、C ABI、CLI、mount runtime 已连通；PostgreSQL `0.4.0-alpha.1` 已实现同一文件合同、format v2、ACL、审计、libpq HostStore 和远程 mount | DuckDB 尚未实现 VexFS adapter | | 文件语义 | 文件、目录、四类时间戳、并发 append、进程锁、mode、symlink、xattr、hardlink、owner/group 元数据、便携 ACL 已进入数据库合同和测试；PG 已执行 ACL 和继承 | SQLite 尚未执行完整 ACL 授权；特殊文件不支持 | | 版本恢复 | 单文件版本、workspace commit、snapshot、diff、expected-head restore、原生备份、retention、显式分批 GC、live quota 和 format v2 逻辑导入导出已实现 | 自动维护、history/staging/index/total quota 和 DuckDB 导入端未完成 | -| 长期校验 | `chunked-v1` 使用不可变 manifest、64 KiB 块和逐块/整文件 SHA-256；只读 check 覆盖结构、引用、顺序、大小和流式内容校验 | 自动 repair 和跨文件通用去重不在当前范围 | -| macOS | 默认 NFS 已连接 SQLite 和 PG runtime；PG 14/14 场景、252 项检查通过，SQLite 完整真机 eval、OpenCode、package smoke、hardlink、COMMIT/fsync、Git、单机锁、npm/Cargo、重挂载和快照恢复均有证据；NFSv3 xattr 明确不支持且不生成 `._*`；macOS 13.0 deployment target 构建通过；既有 FSKit 仍保留历史真机证据 | 当前默认 NFS 源码尚未从干净提交生成公证包，也未在 macOS 13–15 或未启用 FSKit 的干净 Mac 验证；重复 crash/sleep-wake、跨机器锁、长期 Agent 和 x86_64 待补 | +| 长期校验 | `chunked-manifest-v1` 使用不可变 manifest、64 KiB 块、逐块 SHA-256 和有序块根哈希；deep check 逐块校验真实正文，publish 不再拼整文件 | 自动 repair、跨文件通用去重和按需标准整文件 SHA-256 命令不在当前范围 | +| macOS | 默认 NFS 已连接 SQLite 和 PG runtime；PG 14/14 场景、252 项检查通过，SQLite 完整真机 eval、OpenCode、package smoke、hardlink、COMMIT/fsync、Git、单机锁、npm/Cargo、重挂载和快照恢复均有证据；PG strict crash 当前单轮 16 项、连续 3/3 轮通过；NFSv3 xattr 明确不支持且不生成 `._*`；macOS 13.0 deployment target 构建通过；既有 FSKit 仍保留历史真机证据 | 当前默认 NFS 源码尚未从干净提交生成公证包，也未在 macOS 13–15 或未启用 FSKit 的干净 Mac 验证；长循环 crash/sleep-wake、跨机器锁、长期 Agent 和 x86_64 待补 | | Linux | libfuse3 真实挂载已通过 root 和 uid 1000 各 9 组；时间戳、并发 append、进程锁、打开文件生命周期、强制卸载和 helper 崩溃恢复已验证；异常撤销后底层目录保持 0500，显式卸载恢复 0700；x86_64/AArch64 manylinux 安装包已在对应架构真机完成无 root 安装回归 | 更多干净发行版、长期运行和真实挂载安装回归仍不足 | | Windows | 只有平台边界和规划 | WinFsp adapter、安装签名、路径/ACL/SID 合同均未实现 | | 远程共享 | PostgreSQL 已通过多 gateway、macOS FSKit、Linux FUSE、数据库重启、macOS↔Linux 47 项、两台 Mac 文件/快照 50 项、双 Mac 串行真实 OpenCode 7 + 9 + 21 项和故障恢复 25 项；当前默认 NFS 又完成本机双 gateway 15 项，PG `pg_trgm` 索引已实现 | 当前默认 NFS 源码对应的公证包和第二台 Mac NFS 复验尚未完成；同时覆盖同一文件只报告版本冲突，不自动合并 | -| 性能规模 | SQLite 直连已完成 10 万文件；SQLite staging 已改成 base manifest + 变长脏块覆盖；SQLite NFS 三轮 1,000 小文件中位数 904.400 files/s，16 MiB 写 383.600 MiB/s；quick 挂载 eval 的 8 MiB 写为 310.118 MiB/s、100 次随机覆盖为 3,522.124 ops/s；PG NFS 仍使用独立基线 | 小文件达到 AgentFS 的 97%，顺序写为 2.18 倍，随机写约 80%；仍需补 1 万文件、长 Agent、冷读、严格持久化和完整 SHA-256 发布成本基线 | +| 性能规模 | SQLite 直连已完成 10 万文件；PG 16 MiB 文件修改 4 KiB 的组合发布从 589.357 ms 降到中位 10.289 ms，publish-only 两次正式复跑中位 4.020/9.828 ms；逐文件后台事务后的 PG NFS 两轮 8 MiB 写 46.430～46.516 MiB/s、随机覆盖 386.139～482.743 ops/s；strict fsync → gateway SIGKILL → PG immediate restart 当前 16 项、连续 3/3 轮通过；服务端已提交但客户端不读结果的 generation 重试通过；1 GiB 容器 `oom_kill=0` | 仍需补远程 ARM 同版基准、1 万文件、长 Agent、冷读、长循环 crash 和真实网络中途断线 | “合同中已保存”不等于“所有平台已经完整执行”。例如 owner/group 和便携 ACL 已经可以保存、读取和恢复，但身份认证、权限判断和各系统的原生 ACL 映射仍属于后续阶段。 @@ -199,7 +199,7 @@ macOS/Linux mount 和逻辑/物理备份均有真实测试。DuckDB、Windows ad 观测到的额外 RSS 峰值增长为 0； - 报告：`vexdb_sqlite/build/eval/vexfs/20260722T104155.646993Z-quick-20260718/report.json`。 -这是合同 `0.7.0` 当时的证据；当前 `0.9.0` 已由下方 `chunked-v1` 证据替代，但自动 repair 仍不在范围内。 +这是合同 `0.7.0` 当时的证据；当前 `0.9.0` 已由下方 `chunked-manifest-v1` 证据替代，但自动 repair 仍不在范围内。 ### 5.3 长期安全最小集合完成证据 @@ -226,17 +226,20 @@ SQLite 证据。当前 format v2 已支持 SQLite 和 PostgreSQL，DuckDB adapte - canonical 文件版本只引用不可变 manifest，不再在版本行保存整文件 BLOB； - manifest 固定 64 KiB 逻辑块，物理 chunk object 可以被同 inode 的多个版本和多个块位置复用； -- 挂载打开从 chunk 流式写入 staging，publish 两遍扫描 staging 后流式生成 manifest/chunk，固定 - 64 KiB 工作缓冲；不会额外组装 128 MiB 整文件； +- 挂载打开从 chunk 流式写入 staging，publish 按 64 KiB 块生成或复用 manifest/chunk，固定 + 64 KiB 工作缓冲；SQLite 不组装 128 MiB 整文件，PG 的范围发布也不再重建完整正文； - 随机覆盖只新建受影响块；恢复版本继续使用版本别名，不新建 manifest；GC 只删除没有任何 manifest 引用的物理块； -- quick check 验证 manifest 引用、块数量、顺序、大小和可达性，deep check 再验证逐块和整文件 - SHA-256； -- `.vexfs` format v2 携带 manifest/chunk 记录，verify/import 均逐块处理，导入会恢复块复用； +- quick check 验证 manifest 引用、块数量、顺序、大小和可达性，deep check 再验证逐块 SHA-256 + 和有序 manifest 根； +- `.vexfs` format v2 携带 manifest/chunk 记录，verify/import 均逐块校验并验证同一 manifest 根， + 导入会恢复块复用； - 8 MiB、100 次 4 KiB 随机修改样本中，71 个 64 KiB 块受影响，57 个块直接复用，两个版本最终 只保存 72 个物理块；发布耗时 0.370 秒； - 最终 quick eval 为 57 passed、0 failed、18 个平台条件 skipped、2,863 checks，耗时 24.824 秒； SQLite spec、runtime smoke、VexFS CLI smoke 和 VexDB unified CLI smoke 全部通过。 +- 2026-07-27 根哈希规则已统一到 SQLite、PostgreSQL、共享 C++ 和 format v2；SQLite 全量 spec + 30/30、PG VexFS spec 9/9、PG runtime 131 checks、SQLite ↔ PG 双向归档 6 checks 全部通过。 当前仍没有跨文件通用去重、自动 repair、自动 GC 或在线 VACUUM。完整证据见 `docs/reports/2026-07-23_vexfs-chunk-manifest-v2.md`。 @@ -308,8 +311,8 @@ SQLite 证据。当前 format v2 已支持 SQLite 和 PostgreSQL，DuckDB adapte 当前完成证据： -- 适配器版本为 `0.4.0-alpha.1`，8 个 PG VexFS spec、151 项公共合同/并发 eval、32 项 CLI runtime、 - 105 项当前 libpq runtime 合同、 +- 适配器版本为 `0.4.0-alpha.1`，9 个 PG VexFS spec、151 项公共合同/并发 eval、32 项 CLI runtime、 + 119 项当前 libpq runtime 合同、 数据库重启恢复、16/17/18/19 版本矩阵均通过； - ACL、真实 role、跨 gateway 锁和缓存失效已经进入 PostgreSQL 合同；审计精确记录 `session_user`/role OID、workspace、commit、path/inode 和 before/after version，workspace @@ -335,8 +338,14 @@ SQLite 证据。当前 format v2 已支持 SQLite 和 PostgreSQL，DuckDB adapte workspace，第一端创建并 `fsync`、第二端继承并覆盖、第一端刷新读回，最后验证两个版本、 快照、`pending_handles=0` 和 `staging_bytes=0`； - NFS 普通 WRITE 正确返回 `UNSTABLE`，COMMIT/fsync/500 ms idle batch/卸载用 `full` 发布； - PG 同 session 批量发布在一个事务内完成。千文件不再出现修复前的 60 秒 `rg` 超时，正式 + 正常后台 claimed 发布已改成独立 publisher 连接上的逐文件事务，每个文件提交后立即释放 + workspace 行锁；直接 SQL 原子批次和卸载后清理批次继续保留。千文件不再出现修复前的 60 秒 `rg` 超时，正式 file version/commit/request 分别为 1.002/1.066/3.013 每文件。 +- strict 模式已完成自动化断电等价 Gate：`fsync` 返回后校验 gateway PID 身份、SIGKILL gateway、 + PG immediate restart、重挂载读取、deep check、版本和 staging 当前共 16 项，连续 3/3 轮通过； + 1 GiB 容器 `oom_kill=0`。 +- libpq runtime 已验证“服务端完成 commit、客户端不读取结果并丢弃连接”的未知结果窗口；同一 + handle/generation 重试仍返回版本 1，数据库只有一个 file version。 完成条件： diff --git a/tests/eval/vexfs/README.md b/tests/eval/vexfs/README.md index 63b1b5969c..a7d11b265c 100644 --- a/tests/eval/vexfs/README.md +++ b/tests/eval/vexfs/README.md @@ -86,6 +86,11 @@ VEXDB_PG_CONTAINER=vexdb_pg19-vexfs-dev \ VEXDB_PG_CONTAINER=vexdb_pg19-vexfs-dev VEXDB_PG_PERF_FILES=1000 \ bash tests/eval/vexfs/run_pg_adapter_performance.sh +# PG manifest 根哈希专项：16 MiB 文件每轮只改 4 KiB，发布不得拼完整文件； +# 默认重复 5 次，容器 memory.max 必须不超过 1 GiB，并检查 oom_kill 未增长。 +VEXDB_PG_CONTAINER=vexdb_pg19-vexfs-dev VEXDB_PG_DATABASE=test \ + bash tests/eval/vexfs/run_pg_manifest_publish_performance.sh + # libpq HostStore、CLI、handle、ACL、审计、多 gateway 和缓存失效 VEXDB_PG_CONTAINER=vexdb_pg19-vexfs-dev \ VEXDB_PG_DSN=postgresql://postgres@127.0.0.1:5434/test \ @@ -96,6 +101,12 @@ VEXDB_PG_CONTAINER=vexdb_pg19-vexfs-dev \ VEXDB_PG_DSN=postgresql://postgres@127.0.0.1:5434/test \ bash tests/eval/vexfs/run_pg_restart_recovery.sh +# 断电等价 Gate：strict 写入并 fsync、SIGKILL gateway、PG immediate restart、 +# 重挂载读取与 deep check；只写一个小文件，容器 memory.max 必须不超过 1 GiB。 +VEXDB_PG_CONTAINER=vexdb_pg19-vexfs-dev \ +VEXFS_EVAL_MOUNT_CLI=/absolute/path/to/current/vexdb \ + bash tests/eval/vexfs/run_pg_strict_crash_recovery.sh + # 完整数据库 pg_dump/pg_restore VEXDB_PG_CONTAINER=vexdb_pg19-vexfs-dev \ bash tests/eval/vexfs/run_pg_backup_restore.sh @@ -128,6 +139,7 @@ VEXDB_PG_DSN=postgresql://postgres@127.0.0.1:5434/test \ # PG overlay/publisher 专项：真实挂载性能、1000 小文件后冷读、跨会话缓存失效 VEXFS_MACOS_PG_CASES='mount.performance mount.scale-read-after-small-files mount.external-cache-invalidation' \ +VEXDB_PG_DATABASE=test \ VEXDB_PG_DSN=postgresql://postgres@127.0.0.1:5434/test \ bash tests/eval/vexfs/run_pg_macos_mount.sh @@ -246,7 +258,7 @@ vexdb_sqlite/build/eval/vexfs-linux-mount/{root,uid-1000}//report.json - 恢复：进程中途退出、WAL 重开、integrity check、retained reclaim； Linux helper 被 `SIGKILL` 后，底层目录保持 `0500`、普通用户不能产生本地假文件；30 秒 session/handle 租约到期后重挂载会自动发布保留的暂存写入，显式卸载恢复 `0700`。 -- 长期校验：版本 SHA-256、单文件恢复和 workspace 恢复别名、commit/snapshot/history/dentry +- 长期校验：逐块 SHA-256 与有序 manifest 根、单文件恢复和 workspace 恢复别名、commit/snapshot/history/dentry 引用、staging 缺失、同长度内容损坏注入、损坏读取拒绝、CLI 退出码 8；性能用例分别记录 quick 元数据检查和 64 KiB 流式 deep hash 的吞吐与 RSS 增长。 - 数据库版本：当前尚未发版，只支持当前 schema，不运行旧 schema 迁移测试。 @@ -255,7 +267,8 @@ vexdb_sqlite/build/eval/vexfs-linux-mount/{root,uid-1000}//report.json - PostgreSQL 备份性能：有界验证 `pg_dump/pg_restore`、`pg_basebackup` + `pg_verifybackup` + 克隆启动、format v2 PG → SQLite；同时检查容器 1 GiB 内存上限、`oom_kill` 和 CLI 峰值 RSS。 - PostgreSQL staging：基础 manifest + 64 KiB 脏块、范围读写、truncate 缩短后零填充、 - 未变化 chunk 复用、精确 generation claim、断线重试幂等、后台发布不持有前台 runtime 锁； + 未变化 chunk 复用、精确 generation claim、逐文件后台事务、断线重试幂等、部分成功清理、 + 服务端已提交但客户端不读取结果后的同 generation 收敛、后台发布不持有前台 runtime 锁； 调度器覆盖全局空闲、4 MiB 脏写、1024 文件、30 秒最长等待和活跃 handle 隔离。 - 随机模型：真实数据库状态与 Python 参考文件树持续比对。 - 性能：大量小文件、大文件顺序读写、分块 staging、随机 4 KiB 覆盖、备份吞吐； diff --git a/tests/eval/vexfs/run_pg_macos_mount.sh b/tests/eval/vexfs/run_pg_macos_mount.sh index 0a20a82d8d..4b74835779 100755 --- a/tests/eval/vexfs/run_pg_macos_mount.sh +++ b/tests/eval/vexfs/run_pg_macos_mount.sh @@ -4,7 +4,8 @@ set -euo pipefail ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" PYTHON="$ROOT/tests/eval/vexfs/python.sh" PG_CONTAINER="${VEXDB_PG_CONTAINER:-vexdb_pg19-test}" -DSN="${VEXDB_PG_DSN:-postgresql://postgres@127.0.0.1:5433/test}" +PG_DATABASE="${VEXDB_PG_DATABASE:-test}" +DSN="${VEXDB_PG_DSN:-postgresql://postgres@127.0.0.1:5433/$PG_DATABASE}" MOUNT_CLI="${VEXFS_EVAL_MOUNT_CLI:-$HOME/.local/bin/vexdb}" BUILD_DIR="${VEXFS_MACOS_PG_BUILD_DIR:-$ROOT/vexdb_sqlite/build}" OUTPUT="${VEXFS_MACOS_PG_OUTPUT:-$BUILD_DIR/eval/vexfs-pg-macos-mount}" @@ -19,7 +20,7 @@ cleanup_pg_workspaces() { for workspace in conformance timestamps append open-life read-life locks \ force-unmount eval posix perf git-eval toolchains mount-scale opencode \ pg-shared-local scale-read external-cache; do - docker exec "$PG_CONTAINER" psql -U postgres -d test -X -q \ + docker exec "$PG_CONTAINER" psql -U postgres -d "$PG_DATABASE" -X -q \ -v ON_ERROR_STOP=1 \ -c "SELECT vexfs_workspace_drop('$workspace', true);" \ >/dev/null 2>&1 || true diff --git a/tests/eval/vexfs/run_pg_manifest_publish_performance.sh b/tests/eval/vexfs/run_pg_manifest_publish_performance.sh new file mode 100755 index 0000000000..489e397175 --- /dev/null +++ b/tests/eval/vexfs/run_pg_manifest_publish_performance.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash +set -euo pipefail + +CONTAINER="${VEXDB_PG_CONTAINER:-vexdb_pg19-test}" +DATABASE="${VEXDB_PG_DATABASE:-test}" +FILE_BYTES="${VEXDB_PG_MANIFEST_FILE_BYTES:-16777216}" +PATCH_BYTES="${VEXDB_PG_MANIFEST_PATCH_BYTES:-4096}" +ROUNDS="${VEXDB_PG_MANIFEST_ROUNDS:-5}" +MAX_MEDIAN_MS="${VEXDB_PG_MANIFEST_MAX_MEDIAN_MS:-250}" +WORKSPACE="pg-manifest-performance" + +fail() { + echo "$*" >&2 + exit 1 +} + +assert_integer_range() { + local value=$1 + local minimum=$2 + local maximum=$3 + local name=$4 + [[ "$value" =~ ^[0-9]+$ ]] || fail "$name 必须是整数" + (( value >= minimum && value <= maximum )) || \ + fail "$name 必须在 $minimum..$maximum" +} + +assert_integer_range "$FILE_BYTES" 1048576 67108864 VEXDB_PG_MANIFEST_FILE_BYTES +assert_integer_range "$PATCH_BYTES" 1 65536 VEXDB_PG_MANIFEST_PATCH_BYTES +assert_integer_range "$ROUNDS" 1 20 VEXDB_PG_MANIFEST_ROUNDS +awk -v value="$MAX_MEDIAN_MS" 'BEGIN { exit !(value > 0) }' || \ + fail "VEXDB_PG_MANIFEST_MAX_MEDIAN_MS 必须大于 0" + +docker inspect "$CONTAINER" >/dev/null +MEMORY_MAX="$(docker exec "$CONTAINER" cat /sys/fs/cgroup/memory.max)" +[[ "$MEMORY_MAX" =~ ^[0-9]+$ ]] || \ + fail "PG 性能容器必须设置有限的 memory.max，当前为 $MEMORY_MAX" +(( MEMORY_MAX <= 1073741824 )) || \ + fail "PG 性能容器内存上限必须不大于 1 GiB，当前为 $MEMORY_MAX bytes" +OOM_BEFORE="$(docker exec "$CONTAINER" awk '$1=="oom_kill" {print $2}' /sys/fs/cgroup/memory.events)" + +cleanup() { + local status=$? + trap - EXIT + docker exec "$CONTAINER" psql -U postgres -d "$DATABASE" -X -q \ + -v ON_ERROR_STOP=1 \ + -c "SELECT public.vexfs_workspace_drop('$WORKSPACE', true);" \ + >/dev/null 2>&1 || true + exit "$status" +} +trap cleanup EXIT + +RESULT="$(docker exec -i "$CONTAINER" psql -U postgres -d "$DATABASE" \ + -X -q -t -A -F '|' -v ON_ERROR_STOP=1 \ + -v file_bytes="$FILE_BYTES" -v patch_bytes="$PATCH_BYTES" \ + -v rounds="$ROUNDS" <<'SQL' | tail -n 1 +SELECT set_config('vexfs.manifest_file_bytes', :'file_bytes', false); +SELECT set_config('vexfs.manifest_patch_bytes', :'patch_bytes', false); +SELECT set_config('vexfs.manifest_rounds', :'rounds', false); +SELECT public.vexfs_workspace_drop('pg-manifest-performance', true); +SELECT public.vexfs_workspace_create('pg-manifest-performance'); +SELECT public.vexfs_write( + 'pg-manifest-performance', '/large.bin', + decode(repeat('61', current_setting('vexfs.manifest_file_bytes')::integer), 'hex')); +CREATE TEMP TABLE manifest_publish_timings( + round_no integer PRIMARY KEY, + elapsed_ms double precision NOT NULL) ON COMMIT PRESERVE ROWS; +DO $$ +DECLARE + v_round integer; + v_handle text; + v_generation bigint; + v_started timestamptz; + v_elapsed_ms double precision; + v_patch bytea; +BEGIN + FOR v_round IN 1..current_setting('vexfs.manifest_rounds')::integer LOOP + v_handle := public.vexfs_handle_open( + 'pg-manifest-performance', '/large.bin', 'rw', + 'manifest-open-' || v_round::text); + v_patch := decode(repeat( + lpad(pg_catalog.to_hex(97 + v_round), 2, '0'), + current_setting('vexfs.manifest_patch_bytes')::integer), 'hex'); + v_generation := public.vexfs_handle_stage_write( + v_handle, + current_setting('vexfs.manifest_file_bytes')::bigint / 2, + v_patch, + 'manifest-stage-' || v_round::text); + v_started := clock_timestamp(); + PERFORM public.vexfs_handle_publish( + v_handle, v_generation, 'data', + 'manifest-publish-' || v_round::text); + v_elapsed_ms := extract(epoch FROM clock_timestamp() - v_started) * 1000.0; + INSERT INTO manifest_publish_timings VALUES (v_round, v_elapsed_ms); + PERFORM public.vexfs_handle_close( + v_handle, false, 'manifest-close-' || v_round::text); + END LOOP; +END; +$$; +SELECT round(percentile_cont(0.5) WITHIN GROUP (ORDER BY timing.elapsed_ms)::numeric, 3), + round(min(timing.elapsed_ms)::numeric, 3), + round(max(timing.elapsed_ms)::numeric, 3), + (SELECT count(*) FROM _vexfs.file_versions AS version + WHERE version.workspace_id = workspace.workspace_id), + (SELECT count(*) FROM _vexfs.manifests AS manifest + WHERE manifest.workspace_id = workspace.workspace_id), + (SELECT count(*) FROM _vexfs.chunks AS chunk + WHERE chunk.workspace_id = workspace.workspace_id), + (public.vexfs_check('pg-manifest-performance', 1)->>'ok')::boolean, + position('read_staging_content' IN pg_get_functiondef( + 'public.vexfs_handle_publish(text,bigint,text,text)'::regprocedure)) = 0, + (public.vexfs_diagnostics('pg-manifest-performance')->>'pending_handles')::bigint, + (public.vexfs_diagnostics('pg-manifest-performance')->>'staging_bytes')::bigint + FROM manifest_publish_timings AS timing + CROSS JOIN _vexfs.workspaces AS workspace + WHERE workspace.name = 'pg-manifest-performance' + GROUP BY workspace.workspace_id; +SQL +)" + +IFS='|' read -r MEDIAN_MS MIN_MS MAX_MS VERSIONS MANIFESTS CHUNKS \ + CHECK_OK NO_FULL_ASSEMBLY PENDING_HANDLES STAGING_BYTES <<<"$RESULT" +EXPECTED_VERSIONS=$((ROUNDS + 1)) +if [[ "$VERSIONS" != "$EXPECTED_VERSIONS" || \ + "$MANIFESTS" != "$EXPECTED_VERSIONS" || \ + "$CHUNKS" != "$EXPECTED_VERSIONS" || \ + "$CHECK_OK" != t || "$NO_FULL_ASSEMBLY" != t || \ + "$PENDING_HANDLES" != 0 || "$STAGING_BYTES" != 0 ]]; then + fail "PG manifest 发布状态错误：$RESULT" +fi +awk -v actual="$MEDIAN_MS" -v maximum="$MAX_MEDIAN_MS" \ + 'BEGIN { exit !(actual <= maximum) }' || \ + fail "PG manifest 发布中位耗时过高：${MEDIAN_MS} ms，限制 ${MAX_MEDIAN_MS} ms" + +OOM_AFTER="$(docker exec "$CONTAINER" awk '$1=="oom_kill" {print $2}' /sys/fs/cgroup/memory.events)" +[[ "$OOM_AFTER" == "$OOM_BEFORE" ]] || \ + fail "PG manifest eval 触发了 OOM：before=$OOM_BEFORE after=$OOM_AFTER" + +echo "VEXFS PG MANIFEST PUBLISH: PASS (file_bytes=$FILE_BYTES patch_bytes=$PATCH_BYTES rounds=$ROUNDS median_ms=$MEDIAN_MS min_ms=$MIN_MS max_ms=$MAX_MS memory_max=$MEMORY_MAX oom_kill=$OOM_AFTER checks=10)" diff --git a/tests/eval/vexfs/run_pg_strict_crash_recovery.sh b/tests/eval/vexfs/run_pg_strict_crash_recovery.sh new file mode 100755 index 0000000000..a880e9170f --- /dev/null +++ b/tests/eval/vexfs/run_pg_strict_crash_recovery.sh @@ -0,0 +1,173 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" +CONTAINER="${VEXDB_PG_CONTAINER:-vexdb_pg19-vexfs-dev}" +DATABASE="${VEXDB_PG_DATABASE:-vexfs_strict_crash}" +WORKSPACE="${VEXDB_PG_WORKSPACE:-strict-crash}" +CLI="${VEXFS_EVAL_MOUNT_CLI:-$ROOT/vexdb_sqlite/build/vexdb}" +DSN="${VEXDB_PG_DSN:-}" +TMP="$(mktemp -d "${TMPDIR:-/tmp}/vexfs-pg-strict-crash.XXXXXX")" +TEST_HOME="$TMP/home" +MOUNT_POINT="$TMP/mount" +MOUNTED=0 +DATABASE_CREATED=0 +CHECKS=0 + +fail() { + echo "$*" >&2 + exit 1 +} + +is_test_mount_live() { + /sbin/mount | grep -F " on $MOUNT_POINT (" >/dev/null 2>&1 +} + +cleanup() { + local status=$? + trap - EXIT INT TERM + if [ "$MOUNTED" -eq 1 ]; then + HOME="$TEST_HOME" "$CLI" fs --backend pg --dsn "$DSN" \ + --workspace "$WORKSPACE" unmount --force "$MOUNT_POINT" \ + >/dev/null 2>&1 || true + fi + for _ in $(seq 1 30); do + is_test_mount_live || break + /sbin/umount -f "$MOUNT_POINT" >/dev/null 2>&1 || true + sleep 0.25 + done + if is_test_mount_live; then + echo "测试清理失败，NFS 挂载仍存在：$MOUNT_POINT" >&2 + status=1 + fi + if [ "$DATABASE_CREATED" -eq 1 ]; then + docker exec "$CONTAINER" dropdb -U postgres --if-exists --force "$DATABASE" \ + >/dev/null 2>&1 || true + fi + if ! is_test_mount_live; then + rm -rf "$TMP" + fi + exit "$status" +} +trap cleanup EXIT INT TERM + +[ "$(uname -s)" = Darwin ] || fail "该 Gate 只在 macOS NFS 上运行" +[ -x "$CLI" ] || fail "找不到当前 vexdb CLI：$CLI" +docker inspect "$CONTAINER" >/dev/null + +MEMORY_MAX="$(docker exec "$CONTAINER" cat /sys/fs/cgroup/memory.max)" +[[ "$MEMORY_MAX" =~ ^[0-9]+$ ]] || \ + fail "PG crash Gate 必须使用有限 memory.max，当前为 $MEMORY_MAX" +(( MEMORY_MAX <= 1073741824 )) || \ + fail "PG crash Gate 的容器内存上限必须不大于 1 GiB" +OOM_BEFORE="$(docker exec "$CONTAINER" awk '$1=="oom_kill" {print $2}' \ + /sys/fs/cgroup/memory.events)" + +if [ -z "$DSN" ]; then + HOST_BINDING="$(docker port "$CONTAINER" 5432/tcp 2>/dev/null | sed -n '1p')" + HOST_PORT="${HOST_BINDING##*:}" + [[ "$HOST_PORT" =~ ^[0-9]+$ ]] || fail "无法推导 PG 容器端口" + DSN="postgresql://postgres@127.0.0.1:$HOST_PORT/$DATABASE" +fi + +mkdir -p "$TEST_HOME" "$MOUNT_POINT" +docker exec "$CONTAINER" dropdb -U postgres --if-exists --force "$DATABASE" \ + >/dev/null 2>&1 || true +docker exec "$CONTAINER" createdb -U postgres "$DATABASE" +DATABASE_CREATED=1 +docker exec "$CONTAINER" psql -U postgres -d "$DATABASE" -X -q \ + -v ON_ERROR_STOP=1 -c 'CREATE EXTENSION vexdb_lite;' >/dev/null + +FS=("$CLI" fs --backend pg --dsn "$DSN" --workspace "$WORKSPACE") +HOME="$TEST_HOME" "${FS[@]}" setup >/dev/null +START_BEFORE="$(docker exec "$CONTAINER" psql -U postgres -d "$DATABASE" \ + -X -q -t -A -c 'SELECT pg_postmaster_start_time();')" + +HOME="$TEST_HOME" VEXFS_NFS_STRICT_DURABILITY=1 \ + "${FS[@]}" mount "$MOUNT_POINT" >/dev/null +MOUNTED=1 +/usr/bin/python3 - "$MOUNT_POINT/durable.txt" <<'PY' +import os +import sys + +path = sys.argv[1] +descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o644) +try: + os.write(descriptor, b"strict-after-crash\n") + os.fsync(descriptor) +finally: + os.close(descriptor) +PY +CHECKS=$((CHECKS + 1)) + +STATE_BEFORE="$(docker exec "$CONTAINER" psql -U postgres -d "$DATABASE" \ + -X -q -t -A -F '|' -v ON_ERROR_STOP=1 -c \ + "SELECT encode(vexfs_read('$WORKSPACE','/durable.txt'),'hex'), + (vexfs_diagnostics('$WORKSPACE')->>'pending_handles')::bigint, + (vexfs_diagnostics('$WORKSPACE')->>'staging_bytes')::bigint, + (vexfs_check('$WORKSPACE',1)->>'ok')::boolean;")" +[ "$STATE_BEFORE" = '7374726963742d61667465722d63726173680a|0|0|t' ] || \ + fail "fsync 返回后数据库状态不完整：$STATE_BEFORE" +CHECKS=$((CHECKS + 4)) + +GATEWAY_RECORD="$(find "$TEST_HOME/Library/Application Support/VexDB-Lite/nfs-gateways" \ + -name gateway.record -type f -print -quit)" +[ -f "$GATEWAY_RECORD" ] || fail "找不到 NFS gateway 状态文件" +GATEWAY_PID="$(sed -n '2p' "$GATEWAY_RECORD")" +[[ "$GATEWAY_PID" =~ ^[0-9]+$ ]] || fail "无效 gateway PID：$GATEWAY_PID" +GATEWAY_COMMAND="$(/bin/ps -p "$GATEWAY_PID" -o command= 2>/dev/null || true)" +[[ "$GATEWAY_COMMAND" == *vexfs-nfs-gateway* && + "$GATEWAY_COMMAND" == *"--workspace $WORKSPACE"* ]] || \ + fail "拒绝终止身份不匹配的 gateway PID $GATEWAY_PID：$GATEWAY_COMMAND" +CHECKS=$((CHECKS + 1)) +kill -9 "$GATEWAY_PID" +for _ in $(seq 1 50); do + kill -0 "$GATEWAY_PID" >/dev/null 2>&1 || break + sleep 0.1 +done +kill -0 "$GATEWAY_PID" >/dev/null 2>&1 && fail "gateway SIGKILL 后仍存活" +CHECKS=$((CHECKS + 1)) + +HOME="$TEST_HOME" "${FS[@]}" unmount --force "$MOUNT_POINT" >/dev/null +MOUNTED=0 +CHECKS=$((CHECKS + 1)) + +docker exec "$CONTAINER" pg_ctl -D /var/lib/postgresql/data \ + -m immediate -w restart >/dev/null +START_AFTER="$(docker exec "$CONTAINER" psql -U postgres -d "$DATABASE" \ + -X -q -t -A -c 'SELECT pg_postmaster_start_time();')" +[ "$START_AFTER" != "$START_BEFORE" ] || fail "PG immediate restart 没有发生" +CHECKS=$((CHECKS + 1)) + +HOME="$TEST_HOME" VEXFS_NFS_STRICT_DURABILITY=1 \ + "${FS[@]}" mount "$MOUNT_POINT" >/dev/null +MOUNTED=1 +[ "$(cat "$MOUNT_POINT/durable.txt")" = 'strict-after-crash' ] || \ + fail "immediate restart 后挂载内容不正确" +CHECKS=$((CHECKS + 1)) + +CHECK_JSON="$(HOME="$TEST_HOME" "${FS[@]}" --json check)" +printf '%s' "$CHECK_JSON" | /usr/bin/python3 -c \ + 'import json,sys; value=json.load(sys.stdin); assert value["ok"] is True; assert value["issue_count"] == 0' +CHECKS=$((CHECKS + 2)) + +STATE_AFTER="$(docker exec "$CONTAINER" psql -U postgres -d "$DATABASE" \ + -X -q -t -A -F '|' -v ON_ERROR_STOP=1 -c \ + "SELECT count(*), + (vexfs_diagnostics('$WORKSPACE')->>'pending_handles')::bigint, + (vexfs_diagnostics('$WORKSPACE')->>'staging_bytes')::bigint + FROM _vexfs.file_versions AS version + JOIN _vexfs.workspaces AS workspace USING (workspace_id) + WHERE workspace.name='$WORKSPACE';")" +[ "$STATE_AFTER" = '1|0|0' ] || fail "恢复后版本或 staging 状态错误：$STATE_AFTER" +CHECKS=$((CHECKS + 3)) + +HOME="$TEST_HOME" "${FS[@]}" unmount "$MOUNT_POINT" >/dev/null +MOUNTED=0 +OOM_AFTER="$(docker exec "$CONTAINER" awk '$1=="oom_kill" {print $2}' \ + /sys/fs/cgroup/memory.events)" +[ "$OOM_AFTER" = "$OOM_BEFORE" ] || \ + fail "PG crash Gate 触发 OOM：before=$OOM_BEFORE after=$OOM_AFTER" +CHECKS=$((CHECKS + 1)) + +echo "VEXFS PG STRICT CRASH RECOVERY: PASS ($CHECKS checks, memory_max=$MEMORY_MAX, oom_kill=$OOM_AFTER)" diff --git a/tests/spec/pg/agent_files/vexfs_pg_chunk_check.yaml b/tests/spec/pg/agent_files/vexfs_pg_chunk_check.yaml index b7db84b96a..575ee8773e 100644 --- a/tests/spec/pg/agent_files/vexfs_pg_chunk_check.yaml +++ b/tests/spec/pg/agent_files/vexfs_pg_chunk_check.yaml @@ -12,10 +12,12 @@ steps: - query: SELECT vexfs_write('pg-a2', '/large.bin', convert_to(repeat('a', 65536) || repeat('b', 65536) || repeat('c', 7), 'UTF8')); expect: - - 1 -- query: SELECT octet_length(vexfs_read('pg-a2', '/large.bin')), vexfs_stat('pg-a2', '/large.bin')->>'checksum'; +- query: SELECT octet_length(vexfs_read('pg-a2', '/large.bin')), length(vexfs_stat('pg-a2', '/large.bin')->>'checksum'), vexfs_stat('pg-a2', '/large.bin')->>'checksum' = _vexfs.compute_manifest_checksum((SELECT manifest_id FROM _vexfs.file_versions WHERE workspace_id=(SELECT workspace_id FROM _vexfs.workspaces WHERE name='pg-a2') AND version_no=1)), vexfs_stat('pg-a2', '/large.bin')->>'checksum' <> encode(sha256(vexfs_read('pg-a2', '/large.bin')), 'hex'); expect: - - 131079 - - 57ad989627081a0270bb309d3f9272ae22f302aedc2b907dfd1e676741bc535d + - 64 + - true + - true - query: SELECT m.chunk_size, m.chunk_count, count(entry.chunk_id), count(DISTINCT entry.chunk_id) FROM _vexfs.file_versions AS f JOIN _vexfs.manifests AS m ON m.manifest_id=f.manifest_id LEFT JOIN _vexfs.manifest_chunks AS entry ON entry.manifest_id=m.manifest_id WHERE f.workspace_id=(SELECT workspace_id FROM _vexfs.workspaces WHERE name='pg-a2') AND f.version_no=1 GROUP BY m.chunk_size,m.chunk_count; expect: - - 65536 @@ -40,11 +42,12 @@ steps: - query: SELECT vexfs_restore_version('pg-a2', '/large.bin', 1, 2); expect: - - 3 -- query: SELECT manifest_id IS NULL, source_version_no, checksum FROM _vexfs.file_versions WHERE workspace_id=(SELECT workspace_id FROM _vexfs.workspaces WHERE name='pg-a2') AND version_no=3; +- query: SELECT manifest_id IS NULL, source_version_no, length(checksum), checksum = (SELECT checksum FROM _vexfs.file_versions WHERE workspace_id=(SELECT workspace_id FROM _vexfs.workspaces WHERE name='pg-a2') AND version_no=1) FROM _vexfs.file_versions WHERE workspace_id=(SELECT workspace_id FROM _vexfs.workspaces WHERE name='pg-a2') AND version_no=3; expect: - - true - 1 - - 57ad989627081a0270bb309d3f9272ae22f302aedc2b907dfd1e676741bc535d + - 64 + - true - query: SELECT vexfs_read('pg-a2', '/large.bin') = convert_to(repeat('a', 65536) || repeat('b', 65536) || repeat('c', 7), 'UTF8'); expect: - - true @@ -82,7 +85,7 @@ steps: expect: - - true - quick - - chunked-v1 + - chunked-manifest-v1 - query: SELECT (vexfs_check('pg-a2', 1)->>'ok')::boolean, (vexfs_check('pg-a2', 1)->'checked'->>'canonical_versions')::bigint, (vexfs_check('pg-a2', 1)->>'issue_count')::integer; expect: - - true diff --git a/tests/spec/pg/agent_files/vexfs_pg_staging_overlay.yaml b/tests/spec/pg/agent_files/vexfs_pg_staging_overlay.yaml index 26df0c0dd9..c7ad229a06 100644 --- a/tests/spec/pg/agent_files/vexfs_pg_staging_overlay.yaml +++ b/tests/spec/pg/agent_files/vexfs_pg_staging_overlay.yaml @@ -47,6 +47,11 @@ steps: - query: SELECT count(*) FROM _vexfs.manifest_chunks AS old_entry JOIN _vexfs.manifest_chunks AS new_entry ON new_entry.chunk_id=old_entry.chunk_id WHERE old_entry.manifest_id=(SELECT manifest_id FROM _vexfs.file_versions WHERE workspace_id=(SELECT workspace_id FROM _vexfs.workspaces WHERE name='pg-overlay') AND version_no=1) AND new_entry.manifest_id=(SELECT manifest_id FROM _vexfs.file_versions WHERE workspace_id=(SELECT workspace_id FROM _vexfs.workspaces WHERE name='pg-overlay') AND version_no=2); expect: - - 1 +- query: SELECT position('read_staging_content' in pg_get_functiondef('public.vexfs_handle_publish(text,bigint,text,text)'::regprocedure)) = 0, version.checksum = _vexfs.compute_manifest_checksum(version.manifest_id), version.checksum <> encode(sha256(vexfs_read('pg-overlay', '/large.bin')), 'hex') FROM _vexfs.file_versions AS version WHERE version.workspace_id=(SELECT workspace_id FROM _vexfs.workspaces WHERE name='pg-overlay') AND version.version_no=2; + expect: + - - true + - true + - true - query: SELECT vexfs_handle_truncate((SELECT handle FROM overlay_handle), 3, 'overlay-truncate-small'); expect: - - 2 @@ -77,16 +82,33 @@ steps: - statement: CREATE TEMP TABLE claimed_handles(name text PRIMARY KEY, handle text NOT NULL, generation bigint); - statement: INSERT INTO claimed_handles(name, handle) VALUES ('a', vexfs_handle_create('pg-overlay', '/claimed-a.txt', 420, 501, 20, 'claimed-create-a', 'overlay-gateway')), ('b', vexfs_handle_create('pg-overlay', '/claimed-b.txt', 420, 501, 20, 'claimed-create-b', 'overlay-gateway')); - statement: UPDATE claimed_handles SET generation=vexfs_handle_stage_write(handle, 0, convert_to(name, 'UTF8'), 'claimed-stage-' || name); -- query: SELECT jsonb_array_length(vexfs_mount_publish_close_claimed('pg-overlay', 'overlay-gateway', 'full', (SELECT jsonb_agg(jsonb_build_object('handle', handle, 'generation', generation) ORDER BY name) FROM claimed_handles))); +- query: SELECT jsonb_array_length(vexfs_mount_publish_close_claimed('pg-overlay', 'overlay-gateway', 'full', (SELECT jsonb_agg(jsonb_build_object('handle', handle, 'generation', generation) ORDER BY name) FROM claimed_handles WHERE name IN ('a', 'b')))); expect: - - 2 - query: SELECT convert_from(vexfs_read('pg-overlay', '/claimed-a.txt'), 'UTF8'), convert_from(vexfs_read('pg-overlay', '/claimed-b.txt'), 'UTF8'); expect: - - a - b -- query: SELECT jsonb_array_length(vexfs_mount_publish_close_claimed('pg-overlay', 'overlay-gateway', 'full', (SELECT jsonb_agg(jsonb_build_object('handle', handle, 'generation', generation) ORDER BY name) FROM claimed_handles))); +- statement: INSERT INTO claimed_handles(name, handle) VALUES ('lazy', vexfs_handle_create('pg-overlay', '/lazy.txt', 420, 'lazy-create', 'overlay-gateway')); +- statement: UPDATE claimed_handles SET generation=vexfs_handle_stage_write(handle, 0, convert_to('lazy', 'UTF8'), 'lazy-stage') WHERE name='lazy'; +- query: SELECT vexfs_handle_publish_close((SELECT handle FROM claimed_handles WHERE name='lazy'), (SELECT generation FROM claimed_handles WHERE name='lazy'), 'full'); + expect: + - - 1 +- query: SELECT convert_from(vexfs_read('pg-overlay', '/lazy.txt'), 'UTF8'); + expect: + - - lazy +- query: SELECT jsonb_array_length(vexfs_mount_publish_close_claimed('pg-overlay', 'overlay-gateway', 'full', (SELECT jsonb_agg(jsonb_build_object('handle', handle, 'generation', generation) ORDER BY name) FROM claimed_handles WHERE name IN ('a', 'b')))); expect: - - 2 +- statement: CREATE TEMP TABLE root_manifest AS SELECT version.manifest_id, version.checksum FROM _vexfs.file_versions AS version WHERE version.workspace_id=(SELECT workspace_id FROM _vexfs.workspaces WHERE name='pg-overlay') AND version.inode_id=_vexfs.resolve_path((SELECT workspace_id FROM _vexfs.workspaces WHERE name='pg-overlay'), '/lazy.txt') AND version.version_no=1; +- statement: UPDATE _vexfs.manifests SET checksum=repeat('0', 64) WHERE manifest_id=(SELECT manifest_id FROM root_manifest); +- statement: UPDATE _vexfs.file_versions SET checksum=repeat('0', 64) WHERE manifest_id=(SELECT manifest_id FROM root_manifest); +- query: SELECT (vexfs_check('pg-overlay', 0)->>'ok')::boolean, EXISTS (SELECT 1 FROM jsonb_array_elements(vexfs_check('pg-overlay', 0)->'issues') AS issue WHERE issue->>'code'='manifest_chunks'); + expect: + - - false + - true +- statement: UPDATE _vexfs.manifests SET checksum=(SELECT checksum FROM root_manifest) WHERE manifest_id=(SELECT manifest_id FROM root_manifest); +- statement: UPDATE _vexfs.file_versions SET checksum=(SELECT checksum FROM root_manifest) WHERE manifest_id=(SELECT manifest_id FROM root_manifest); - query: SELECT (vexfs_check('pg-overlay', 1)->>'ok')::boolean; expect: - - true diff --git a/tests/spec/sqlite/agent_files/vexfs_contract.yaml b/tests/spec/sqlite/agent_files/vexfs_contract.yaml index 20e53fc63d..f7390fca1e 100644 --- a/tests/spec/sqlite/agent_files/vexfs_contract.yaml +++ b/tests/spec/sqlite/agent_files/vexfs_contract.yaml @@ -206,9 +206,10 @@ steps: - query: SELECT json_array_length(json_extract(vexfs_snapshot_diff('default', 'spec-baseline', 'HEAD'), '$.changes')); expect: - - 0 -- query: SELECT json_extract(vexfs_stat('default', '/notes/hello.txt'), '$.checksum'); +- query: SELECT length(json_extract(vexfs_stat('default', '/notes/hello.txt'), '$.checksum')), json_extract(vexfs_stat('default', '/notes/hello.txt'), '$.checksum') <> '2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824'; expect: - - - 2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824 + - - 64 + - 1 - query: SELECT json_extract(vexfs_check('default', 1), '$.ok'), json_extract(vexfs_check('default', 1), '$.mode'), json_extract(vexfs_check('default', 1), '$.issue_count'); expect: - - 1 diff --git a/vexdb_pg/sql/vexdb_lite--1.0.sql b/vexdb_pg/sql/vexdb_lite--1.0.sql index 0842a32e87..601985220c 100644 --- a/vexdb_pg/sql/vexdb_lite--1.0.sql +++ b/vexdb_pg/sql/vexdb_lite--1.0.sql @@ -1635,6 +1635,70 @@ BEGIN END; $$; +-- A manifest checksum identifies the ordered chunk tree, not a second copy of +-- the complete file checksum. Chunk checksums remain SHA-256 over their exact +-- 64 KiB payloads. Publishing a small range can therefore reuse old chunk +-- hashes and keep memory proportional to manifest metadata instead of file +-- size. The fixed format and version prefix make the root deterministic. +CREATE FUNCTION _vexfs.compute_manifest_checksum(p_manifest bigint) +RETURNS text +LANGUAGE plpgsql +SECURITY DEFINER +STABLE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_file_size bigint; + v_chunk_size integer; + v_chunk_count integer; + v_entry_count bigint; + v_total_bytes bigint; + v_entries text; + v_valid boolean; +BEGIN + SELECT manifest.file_size, manifest.chunk_size, manifest.chunk_count + INTO v_file_size, v_chunk_size, v_chunk_count + FROM _vexfs.manifests AS manifest + WHERE manifest.manifest_id = p_manifest; + IF NOT FOUND THEN + RAISE EXCEPTION 'VEXFS_MANIFEST_NOT_FOUND: %', p_manifest + USING ERRCODE = 'P0002'; + END IF; + + SELECT count(*), + coalesce(sum(chunk.size_bytes), 0), + coalesce(bool_and( + entry.chunk_no >= 0 + AND entry.chunk_no < v_chunk_count + AND chunk.size_bytes = least( + v_chunk_size, + v_file_size - entry.chunk_no::bigint * v_chunk_size)::integer), + v_chunk_count = 0), + coalesce(pg_catalog.string_agg( + pg_catalog.format( + '%s:%s:%s', + entry.chunk_no, chunk.size_bytes, chunk.checksum), + E'\n' ORDER BY entry.chunk_no), '') + INTO v_entry_count, v_total_bytes, v_valid, v_entries + FROM _vexfs.manifest_chunks AS entry + JOIN _vexfs.chunks AS chunk ON chunk.chunk_id = entry.chunk_id + WHERE entry.manifest_id = p_manifest; + IF v_entry_count <> v_chunk_count + OR v_total_bytes <> v_file_size + OR NOT v_valid THEN + RAISE EXCEPTION 'VEXFS_CORRUPT: manifest chunk metadata is invalid' + USING ERRCODE = 'XX001'; + END IF; + + RETURN encode(pg_catalog.sha256(pg_catalog.convert_to( + pg_catalog.format( + 'vexfs-manifest-v1:%s:%s:%s', + v_chunk_size, v_file_size, v_chunk_count) + || E'\n' || v_entries, + 'UTF8')), 'hex'); +END; +$$; + CREATE FUNCTION _vexfs.store_manifest( p_workspace_id bigint, p_inode_id bigint, @@ -1649,7 +1713,6 @@ AS $$ DECLARE v_manifest bigint; v_file_size bigint; - v_file_checksum text; v_chunk_count integer; v_chunk_no integer := 0; v_offset integer := 0; @@ -1667,14 +1730,13 @@ BEGIN RAISE EXCEPTION 'VEXFS_FILE_TOO_LARGE: maximum file size is 128 MiB' USING ERRCODE = '54000'; END IF; - v_file_checksum := encode(pg_catalog.sha256(p_content), 'hex'); v_chunk_count := ((v_file_size + 65535) / 65536)::integer; INSERT INTO _vexfs.manifests( workspace_id, inode_id, file_size, chunk_size, chunk_count, checksum) VALUES ( p_workspace_id, p_inode_id, v_file_size, 65536, - v_chunk_count, v_file_checksum) + v_chunk_count, repeat('0', 64)) RETURNING manifest_id INTO v_manifest; WHILE v_offset < v_file_size LOOP @@ -1728,16 +1790,17 @@ BEGIN v_chunk_no := v_chunk_no + 1; v_offset := v_offset + v_chunk_size; END LOOP; + UPDATE _vexfs.manifests + SET checksum = _vexfs.compute_manifest_checksum(v_manifest) + WHERE manifest_id = v_manifest; RETURN v_manifest; END; $$; -- Build a copy-on-write manifest for a positional write. Untouched chunks are -- linked into the new manifest in one statement; only overlapping chunks (and --- zero-filled hole chunks) are materialized again. The final file checksum is --- still SHA-256 over the complete byte stream, so range writes keep the same --- integrity contract as vexfs_write without rebuilding the complete file in a --- PL/pgSQL loop. +-- zero-filled hole chunks) are materialized again. The manifest root is built +-- from ordered chunk hashes, so this path never aggregates the complete file. CREATE FUNCTION _vexfs.store_manifest_range( p_workspace_id bigint, p_inode_id bigint, @@ -1771,10 +1834,8 @@ DECLARE v_existing_count bigint; v_existing_bytes bigint; v_existing_valid boolean; - v_full_content bytea; v_final_count bigint; v_final_bytes bigint; - v_file_checksum text; BEGIN IF p_content IS NULL THEN RAISE EXCEPTION 'VEXFS_INVALID_CONTENT: content cannot be null' @@ -1798,9 +1859,8 @@ BEGIN USING ERRCODE = 'XX001'; END IF; - -- Validate every source chunk independently while keeping peak memory - -- bounded to one 64 KiB chunk. The complete-file checksum is recomputed for - -- the new manifest below. + -- Validate source metadata without re-reading every unchanged chunk. Reads + -- and vexfs_check still validate each chunk payload against its SHA-256. SELECT count(*), coalesce(sum(chunk.size_bytes), 0), coalesce(bool_and( @@ -1811,9 +1871,7 @@ BEGIN AND chunk.size_bytes = least( 65536, p_previous_size - entry.chunk_no::bigint * 65536)::integer - AND octet_length(chunk.content) = chunk.size_bytes - AND chunk.checksum = encode( - pg_catalog.sha256(chunk.content), 'hex')), + AND octet_length(chunk.content) = chunk.size_bytes), v_previous_chunk_count = 0) INTO v_existing_count, v_existing_bytes, v_existing_valid FROM _vexfs.manifest_chunks AS entry @@ -1872,6 +1930,15 @@ BEGIN IF NOT FOUND THEN v_chunk := ''::bytea; END IF; + IF v_previous_chunk_id IS NOT NULL AND NOT EXISTS ( + SELECT 1 + FROM _vexfs.chunks AS source + WHERE source.chunk_id = v_previous_chunk_id + AND source.checksum = encode( + pg_catalog.sha256(source.content), 'hex')) THEN + RAISE EXCEPTION 'VEXFS_CHECKSUM_MISMATCH: previous range chunk is invalid' + USING ERRCODE = 'XX001'; + END IF; IF octet_length(v_chunk) > v_chunk_size THEN v_chunk := substring(v_chunk FROM 1 FOR v_chunk_size); ELSIF octet_length(v_chunk) < v_chunk_size THEN @@ -1927,13 +1994,9 @@ BEGIN VALUES (v_manifest, v_chunk_no, v_chunk_id); END LOOP; - SELECT coalesce( - pg_catalog.string_agg( - chunk.content, ''::bytea ORDER BY entry.chunk_no), - ''::bytea), - count(*), + SELECT count(*), coalesce(sum(chunk.size_bytes), 0) - INTO v_full_content, v_final_count, v_final_bytes + INTO v_final_count, v_final_bytes FROM _vexfs.manifest_chunks AS entry JOIN _vexfs.chunks AS chunk ON chunk.chunk_id = entry.chunk_id WHERE entry.manifest_id = v_manifest; @@ -1941,9 +2004,194 @@ BEGIN RAISE EXCEPTION 'VEXFS_CORRUPT: range manifest is incomplete' USING ERRCODE = 'XX001'; END IF; - v_file_checksum := encode(pg_catalog.sha256(v_full_content), 'hex'); UPDATE _vexfs.manifests - SET checksum = v_file_checksum + SET checksum = _vexfs.compute_manifest_checksum(v_manifest) + WHERE manifest_id = v_manifest; + RETURN v_manifest; +END; +$$; + +-- Publish a writable handle directly from its base manifest plus dirty chunks. +-- Fully visible, unchanged chunks are linked in one statement. Only dirty, +-- truncated, extended, or zero-filled edge chunks are materialized. +CREATE FUNCTION _vexfs.store_staging_manifest( + p_handle text, + p_inode_id bigint) +RETURNS bigint +LANGUAGE plpgsql +SECURITY DEFINER +VOLATILE +SET search_path = pg_catalog, _vexfs +AS $$ +DECLARE + v_workspace_id bigint; + v_handle_inode bigint; + v_staging _vexfs.handle_staging%ROWTYPE; + v_base_workspace bigint; + v_base_inode bigint; + v_base_size bigint; + v_manifest bigint; + v_chunk_count integer; + v_chunk_no integer; + v_chunk_size integer; + v_chunk bytea; + v_chunk_checksum text; + v_chunk_id bigint; + v_base_chunk_id bigint; + v_final_count bigint; + v_final_bytes bigint; +BEGIN + IF p_inode_id IS NULL THEN + RAISE EXCEPTION 'VEXFS_INVALID_INODE: staging manifest requires an inode' + USING ERRCODE = '22023'; + END IF; + SELECT handle.workspace_id, handle.inode_id + INTO v_workspace_id, v_handle_inode + FROM _vexfs.handles AS handle + WHERE handle.handle_id = p_handle; + IF NOT FOUND THEN + RAISE EXCEPTION 'VEXFS_STAGING_NOT_FOUND: %', p_handle + USING ERRCODE = 'P0002'; + END IF; + SELECT * INTO v_staging + FROM _vexfs.handle_staging AS staging + WHERE staging.handle_id = p_handle; + IF NOT FOUND THEN + RAISE EXCEPTION 'VEXFS_STAGING_NOT_FOUND: %', p_handle + USING ERRCODE = 'P0002'; + END IF; + IF v_handle_inode IS NOT NULL AND v_handle_inode <> p_inode_id THEN + RAISE EXCEPTION 'VEXFS_CORRUPT: handle inode changed during publish' + USING ERRCODE = 'XX001'; + END IF; + IF v_staging.logical_size < 0 OR v_staging.logical_size > 134217728 THEN + RAISE EXCEPTION 'VEXFS_FILE_TOO_LARGE: maximum file size is 128 MiB' + USING ERRCODE = '54000'; + END IF; + IF v_staging.base_manifest_id IS NULL THEN + IF v_staging.base_size <> 0 OR v_staging.base_visible_size <> 0 THEN + RAISE EXCEPTION 'VEXFS_CORRUPT: staging base metadata is invalid' + USING ERRCODE = 'XX001'; + END IF; + ELSE + SELECT base.workspace_id, base.inode_id, base.file_size + INTO v_base_workspace, v_base_inode, v_base_size + FROM _vexfs.manifests AS base + WHERE base.manifest_id = v_staging.base_manifest_id; + IF NOT FOUND + OR v_base_workspace <> v_workspace_id + OR v_base_inode <> p_inode_id + OR v_base_size <> v_staging.base_size + OR v_staging.base_visible_size > v_base_size THEN + RAISE EXCEPTION 'VEXFS_CORRUPT: staging base manifest is invalid' + USING ERRCODE = 'XX001'; + END IF; + END IF; + + v_chunk_count := ((v_staging.logical_size + 65535) / 65536)::integer; + INSERT INTO _vexfs.manifests( + workspace_id, inode_id, file_size, chunk_size, chunk_count, checksum) + VALUES ( + v_workspace_id, p_inode_id, v_staging.logical_size, + 65536, v_chunk_count, repeat('0', 64)) + RETURNING manifest_id INTO v_manifest; + + -- Reuse unchanged chunks without loading their payload. A partially visible + -- base chunk is excluded so truncate-then-grow correctly inserts zeroes. + IF v_staging.base_manifest_id IS NOT NULL AND v_chunk_count > 0 THEN + INSERT INTO _vexfs.manifest_chunks(manifest_id, chunk_no, chunk_id) + SELECT v_manifest, entry.chunk_no, entry.chunk_id + FROM _vexfs.manifest_chunks AS entry + JOIN _vexfs.chunks AS chunk ON chunk.chunk_id = entry.chunk_id + WHERE entry.manifest_id = v_staging.base_manifest_id + AND entry.chunk_no < v_chunk_count + AND NOT EXISTS ( + SELECT 1 + FROM _vexfs.handle_staging_chunks AS dirty + WHERE dirty.handle_id = p_handle + AND dirty.chunk_no = entry.chunk_no) + AND chunk.workspace_id = v_workspace_id + AND chunk.inode_id = p_inode_id + AND chunk.size_bytes = least( + 65536, + v_staging.logical_size + - entry.chunk_no::bigint * 65536)::integer + AND entry.chunk_no::bigint * 65536 + chunk.size_bytes + <= v_staging.base_visible_size; + END IF; + + IF v_chunk_count > 0 THEN + FOR v_chunk_no IN 0..v_chunk_count - 1 LOOP + IF EXISTS ( + SELECT 1 + FROM _vexfs.manifest_chunks AS entry + WHERE entry.manifest_id = v_manifest + AND entry.chunk_no = v_chunk_no) THEN + CONTINUE; + END IF; + v_chunk_size := least( + 65536, + v_staging.logical_size - v_chunk_no::bigint * 65536)::integer; + v_chunk := _vexfs.read_staging_chunk( + p_handle, v_chunk_no, v_chunk_size); + IF octet_length(v_chunk) <> v_chunk_size THEN + RAISE EXCEPTION 'VEXFS_CORRUPT: staging chunk has invalid size' + USING ERRCODE = 'XX001'; + END IF; + v_chunk_checksum := encode(pg_catalog.sha256(v_chunk), 'hex'); + v_chunk_id := NULL; + v_base_chunk_id := NULL; + IF v_staging.base_manifest_id IS NOT NULL THEN + SELECT entry.chunk_id INTO v_base_chunk_id + FROM _vexfs.manifest_chunks AS entry + WHERE entry.manifest_id = v_staging.base_manifest_id + AND entry.chunk_no = v_chunk_no; + END IF; + IF v_base_chunk_id IS NOT NULL THEN + SELECT chunk.chunk_id INTO v_chunk_id + FROM _vexfs.chunks AS chunk + WHERE chunk.chunk_id = v_base_chunk_id + AND chunk.workspace_id = v_workspace_id + AND chunk.inode_id = p_inode_id + AND chunk.size_bytes = v_chunk_size + AND chunk.checksum = v_chunk_checksum + AND chunk.content = v_chunk; + END IF; + IF v_chunk_id IS NULL THEN + SELECT chunk.chunk_id INTO v_chunk_id + FROM _vexfs.manifest_chunks AS entry + JOIN _vexfs.chunks AS chunk ON chunk.chunk_id = entry.chunk_id + WHERE entry.manifest_id = v_manifest + AND chunk.size_bytes = v_chunk_size + AND chunk.checksum = v_chunk_checksum + AND chunk.content = v_chunk + LIMIT 1; + END IF; + IF v_chunk_id IS NULL THEN + INSERT INTO _vexfs.chunks( + workspace_id, inode_id, content, size_bytes, checksum) + VALUES ( + v_workspace_id, p_inode_id, v_chunk, + v_chunk_size, v_chunk_checksum) + RETURNING chunk_id INTO v_chunk_id; + END IF; + INSERT INTO _vexfs.manifest_chunks(manifest_id, chunk_no, chunk_id) + VALUES (v_manifest, v_chunk_no, v_chunk_id); + END LOOP; + END IF; + + SELECT count(*), coalesce(sum(chunk.size_bytes), 0) + INTO v_final_count, v_final_bytes + FROM _vexfs.manifest_chunks AS entry + JOIN _vexfs.chunks AS chunk ON chunk.chunk_id = entry.chunk_id + WHERE entry.manifest_id = v_manifest; + IF v_final_count <> v_chunk_count + OR v_final_bytes <> v_staging.logical_size THEN + RAISE EXCEPTION 'VEXFS_CORRUPT: staging manifest is incomplete' + USING ERRCODE = 'XX001'; + END IF; + UPDATE _vexfs.manifests + SET checksum = _vexfs.compute_manifest_checksum(v_manifest) WHERE manifest_id = v_manifest; RETURN v_manifest; END; @@ -2018,7 +2266,7 @@ BEGIN IF v_offset <> v_size OR v_expected_chunk <> v_chunk_count - OR encode(pg_catalog.sha256(v_content), 'hex') <> v_checksum THEN + OR _vexfs.compute_manifest_checksum(v_manifest) <> v_checksum THEN RAISE EXCEPTION 'VEXFS_CHECKSUM_MISMATCH: file content does not match manifest' USING ERRCODE = 'XX001'; END IF; @@ -4442,12 +4690,11 @@ DECLARE v_inode bigint; v_parent bigint; v_name text; - v_previous_manifest bigint; v_manifest bigint; v_checksum text; v_commit bigint; v_principal_oid oid; - v_content bytea; + v_operation text; BEGIN IF p_durability NOT IN ('none', 'data', 'full') THEN RAISE EXCEPTION 'VEXFS_INVALID_DURABILITY: use none, data, or full' @@ -4487,13 +4734,12 @@ BEGIN RAISE EXCEPTION 'VEXFS_STALE_GENERATION: staging generation does not match %', p_generation USING ERRCODE = '40001'; END IF; - -- Assemble and validate the candidate before taking the workspace row lock. - -- The handle/staging row locks keep this generation stable while other - -- files in the same workspace continue to make progress. - v_content := _vexfs.read_staging_content(p_handle); - IF octet_length(v_content) <> v_staging.logical_size THEN - RAISE EXCEPTION 'VEXFS_CORRUPT: staged content size does not match metadata' - USING ERRCODE = 'XX001'; + -- Existing/eager-created inodes can build their manifest before taking the + -- workspace row lock. The staging lock freezes this exact generation while + -- unchanged chunks are linked and only dirty chunks are materialized. + IF v_handle.inode_id IS NOT NULL THEN + v_manifest := _vexfs.store_staging_manifest( + p_handle, v_handle.inode_id); END IF; PERFORM pg_catalog.set_config( 'synchronous_commit', @@ -4503,10 +4749,15 @@ BEGIN true); PERFORM 1 FROM _vexfs.workspaces WHERE workspace_id = v_handle.workspace_id FOR UPDATE; + SELECT role.oid INTO STRICT v_principal_oid + FROM pg_catalog.pg_roles AS role + WHERE role.rolname = session_user; IF v_handle.inode_id IS NULL THEN SELECT rp.parent_inode, rp.entry_name INTO v_parent, v_name FROM _vexfs.resolve_parent( v_handle.workspace_id, v_handle.path) AS rp; + PERFORM _vexfs.require_inode_permission( + v_handle.workspace_id, v_parent, 'write'); IF EXISTS ( SELECT 1 FROM _vexfs.dentries AS d @@ -4516,90 +4767,86 @@ BEGIN RAISE EXCEPTION 'VEXFS_VERSION_CONFLICT: create destination now exists' USING ERRCODE = '40001'; END IF; - v_version := public.vexfs_write( - (SELECT w.name FROM _vexfs.workspaces AS w - WHERE w.workspace_id = v_handle.workspace_id), - v_handle.path, v_content); - v_inode := _vexfs.resolve_path(v_handle.workspace_id, v_handle.path); + PERFORM _vexfs.enforce_quota( + v_handle.workspace_id, NULL, v_staging.logical_size); + v_current_version := 0; + v_version := 1; + v_path := v_handle.path; + INSERT INTO _vexfs.inodes( + workspace_id, kind, mode, owner_oid, owner_role, owner_principal, + current_version, size_bytes) + VALUES ( + v_handle.workspace_id, 'file', coalesce(v_handle.create_mode, 420), + v_principal_oid, session_user, session_user, + v_version, v_staging.logical_size) + RETURNING inode_id INTO v_inode; + INSERT INTO _vexfs.dentries(workspace_id, parent_inode, name, inode_id) + VALUES (v_handle.workspace_id, v_parent, v_name, v_inode); + PERFORM _vexfs.inherit_acl(v_handle.workspace_id, v_parent, v_inode); UPDATE _vexfs.inodes - SET mode = coalesce(v_handle.create_mode, 420), + SET modified_at = clock_timestamp(), changed_at = clock_timestamp() - WHERE inode_id = v_inode; + WHERE workspace_id = v_handle.workspace_id + AND inode_id = v_parent; + v_manifest := _vexfs.store_staging_manifest(p_handle, v_inode); ELSE + v_inode := v_handle.inode_id; SELECT i.current_version INTO v_current_version FROM _vexfs.inodes AS i WHERE i.workspace_id = v_handle.workspace_id - AND i.inode_id = v_handle.inode_id + AND i.inode_id = v_inode AND i.live AND i.kind = 'file' FOR UPDATE; IF NOT FOUND OR v_current_version <> v_handle.expected_version THEN RAISE EXCEPTION 'VEXFS_VERSION_CONFLICT: file changed after handle open' USING ERRCODE = '40001'; END IF; - v_path := _vexfs.path_for_inode(v_handle.workspace_id, v_handle.inode_id); + PERFORM _vexfs.require_inode_permission( + v_handle.workspace_id, v_inode, 'write'); + PERFORM _vexfs.enforce_quota( + v_handle.workspace_id, v_inode, v_staging.logical_size); + SELECT coalesce(max(version.version_no), 0) + 1 + INTO v_version + FROM _vexfs.file_versions AS version + WHERE version.workspace_id = v_handle.workspace_id + AND version.inode_id = v_inode; + v_path := _vexfs.path_for_inode(v_handle.workspace_id, v_inode); + UPDATE _vexfs.inodes + SET current_version = v_version, + size_bytes = v_staging.logical_size, + modified_at = clock_timestamp(), + changed_at = clock_timestamp() + WHERE workspace_id = v_handle.workspace_id + AND inode_id = v_inode; IF v_path IS NOT NULL THEN - v_version := public.vexfs_write( - (SELECT w.name FROM _vexfs.workspaces AS w - WHERE w.workspace_id = v_handle.workspace_id), - v_path, v_content); - ELSE - -- POSIX keeps an unlinked file alive while an open descriptor still - -- references its inode. Publish the new version without recreating - -- a directory entry; the inode is reaped after the final handle closes. - PERFORM _vexfs.enforce_quota( - v_handle.workspace_id, v_handle.inode_id, - v_staging.logical_size); - SELECT coalesce(max(version.version_no), 0) + 1 - INTO v_version - FROM _vexfs.file_versions AS version - WHERE version.workspace_id = v_handle.workspace_id - AND version.inode_id = v_handle.inode_id; - IF v_current_version > 0 THEN - SELECT storage.manifest_id INTO v_previous_manifest - FROM _vexfs.resolve_version_storage( - v_handle.workspace_id, v_handle.inode_id, - v_current_version) AS storage; - ELSE - v_previous_manifest := NULL; - END IF; + SELECT parent.parent_inode INTO v_parent + FROM _vexfs.resolve_parent( + v_handle.workspace_id, v_path) AS parent; UPDATE _vexfs.inodes - SET current_version = v_version, - size_bytes = v_staging.logical_size, - modified_at = clock_timestamp(), + SET modified_at = clock_timestamp(), changed_at = clock_timestamp() WHERE workspace_id = v_handle.workspace_id - AND inode_id = v_handle.inode_id; - v_manifest := _vexfs.store_manifest( - v_handle.workspace_id, v_handle.inode_id, - v_previous_manifest, v_content); - SELECT manifest.checksum INTO STRICT v_checksum - FROM _vexfs.manifests AS manifest - WHERE manifest.manifest_id = v_manifest; - SELECT role.oid INTO STRICT v_principal_oid - FROM pg_catalog.pg_roles AS role - WHERE role.rolname = session_user; - v_commit := _vexfs.record_commit( - v_handle.workspace_id, 'handle_write_orphan', - v_handle.path, v_handle.inode_id, - jsonb_build_object( - 'before_version', v_current_version, - 'after_version', v_version, - 'unlinked', true)); - INSERT INTO _vexfs.file_versions( - workspace_id, inode_id, version_no, commit_no, manifest_id, - size_bytes, checksum, created_by_oid, created_by) - VALUES ( - v_handle.workspace_id, v_handle.inode_id, v_version, - v_commit, v_manifest, v_staging.logical_size, - v_checksum, v_principal_oid, session_user); + AND inode_id = v_parent; END IF; - v_inode := v_handle.inode_id; - END IF; - IF v_manifest IS NULL THEN - SELECT storage.manifest_id INTO v_manifest - FROM _vexfs.resolve_version_storage( - v_handle.workspace_id, v_inode, v_version) AS storage; END IF; + SELECT manifest.checksum INTO STRICT v_checksum + FROM _vexfs.manifests AS manifest + WHERE manifest.manifest_id = v_manifest; + v_operation := CASE WHEN v_path IS NULL + THEN 'handle_write_orphan' ELSE 'write' END; + v_commit := _vexfs.record_commit( + v_handle.workspace_id, v_operation, + coalesce(v_path, v_handle.path), v_inode, + jsonb_build_object( + 'before_version', v_current_version, + 'after_version', v_version, + 'unlinked', v_path IS NULL)); + INSERT INTO _vexfs.file_versions( + workspace_id, inode_id, version_no, commit_no, manifest_id, + size_bytes, checksum, created_by_oid, created_by) + VALUES ( + v_handle.workspace_id, v_inode, v_version, v_commit, v_manifest, + v_staging.logical_size, v_checksum, v_principal_oid, session_user); UPDATE _vexfs.handles SET inode_id = v_inode, path = coalesce(v_path, _vexfs.handles.path), @@ -4847,10 +5094,10 @@ BEGIN END; $$; --- Publish an exact generation snapshot selected by the gateway. This avoids a --- race where a newly opened handle sorts ahead of the gateway's local batch --- after RuntimeState has released its mutex. The whole bounded batch remains --- atomic, while generation checks prevent a later write from being cleared. +-- Publish an exact generation snapshot selected by a SQL caller. This helper +-- remains an atomic all-or-nothing batch for direct SQL compatibility. The NFS +-- gateway uses the dedicated single-file publisher connection path instead, +-- so each file releases the workspace row lock before the next publish begins. CREATE FUNCTION public.vexfs_mount_publish_close_claimed( p_workspace text, p_session_id text, @@ -8265,7 +8512,21 @@ BEGIN SELECT sum(c.size_bytes) FROM _vexfs.manifest_chunks AS entry JOIN _vexfs.chunks AS c ON c.chunk_id = entry.chunk_id - WHERE entry.manifest_id = m.manifest_id), 0) <> m.file_size); + WHERE entry.manifest_id = m.manifest_id), 0) <> m.file_size + OR m.checksum IS DISTINCT FROM ( + SELECT encode(pg_catalog.sha256(pg_catalog.convert_to( + pg_catalog.format( + 'vexfs-manifest-v1:%s:%s:%s', + m.chunk_size, m.file_size, m.chunk_count) + || E'\n' || coalesce(pg_catalog.string_agg( + pg_catalog.format( + '%s:%s:%s', + entry.chunk_no, c.size_bytes, c.checksum), + E'\n' ORDER BY entry.chunk_no), ''), + 'UTF8')), 'hex') + FROM _vexfs.manifest_chunks AS entry + JOIN _vexfs.chunks AS c ON c.chunk_id = entry.chunk_id + WHERE entry.manifest_id = m.manifest_id)); IF v_bad <> 0 THEN v_issues := v_issues || jsonb_build_array(jsonb_build_object( 'code', 'manifest_chunks', 'count', v_bad, @@ -8432,7 +8693,7 @@ BEGIN 'ok', jsonb_array_length(v_issues) = 0, 'workspace', p_workspace, 'mode', CASE WHEN p_deep = 1 THEN 'deep' ELSE 'quick' END, - 'content_model', 'chunked-v1', + 'content_model', 'chunked-manifest-v1', 'issue_count', jsonb_array_length(v_issues), 'versions', v_version_count, 'content_bytes', v_content_bytes, diff --git a/vexdb_sqlite/src/agent_files/vexfs_sqlite.cpp b/vexdb_sqlite/src/agent_files/vexfs_sqlite.cpp index dd4bc22d3e..d21c159e83 100644 --- a/vexdb_sqlite/src/agent_files/vexfs_sqlite.cpp +++ b/vexdb_sqlite/src/agent_files/vexfs_sqlite.cpp @@ -745,7 +745,7 @@ CREATE TABLE IF NOT EXISTS _vexfs_mount_sessions( updated_at INTEGER NOT NULL DEFAULT (CAST(strftime('%s','now') AS INTEGER) * 1000) ); INSERT OR IGNORE INTO _vexfs_meta(key, value) VALUES('contract_version', '0.9.0'); -INSERT OR IGNORE INTO _vexfs_meta(key, value) VALUES('content_model', 'chunked-v1'); +INSERT OR IGNORE INTO _vexfs_meta(key, value) VALUES('content_model', 'chunked-manifest-v1'); INSERT OR IGNORE INTO _vexfs_meta(key, value) VALUES('staging_layout', 'overlay-v1'); )SQL"); CreateHistorySchema(db); @@ -1050,7 +1050,8 @@ JOIN _vexfs_chunks chunk ON chunk.id=entry.chunk_id WHERE entry.manifest_id=?1 ORDER BY entry.chunk_no )SQL"); chunks.BindInt64(1, storage.manifest_id); - vexfs::Sha256 file_hash; + std::vector manifest_chunks; + manifest_chunks.reserve(static_cast(storage.chunk_count)); sqlite3_int64 offset = 0; sqlite3_int64 expected_chunk = 0; while (chunks.Row()) { @@ -1066,13 +1067,15 @@ WHERE entry.manifest_id=?1 ORDER BY entry.chunk_no throw SqlError("file manifest chunk is corrupt", SQLITE_CORRUPT); } std::memcpy(content.data() + offset, chunk, static_cast(chunk_bytes)); - file_hash.Update(chunk, static_cast(chunk_bytes)); + manifest_chunks.push_back({static_cast(chunk_bytes), chunks.Text(3)}); offset += expected_size; ++expected_chunk; } if (offset != storage.size || expected_chunk != storage.chunk_count || expected_chunk != (storage.size + kContentChunkBytes - 1) / kContentChunkBytes || - vexfs::Hex(file_hash.Finish()) != storage.checksum) { + vexfs::ManifestChecksum( + static_cast(storage.size), + static_cast(kContentChunkBytes), manifest_chunks) != storage.checksum) { throw SqlError("file version checksum does not match content", SQLITE_CORRUPT); } return content; @@ -1565,14 +1568,15 @@ INSERT INTO _vexfs_manifest_chunks(manifest_id,chunk_no,chunk_id) VALUES(?1,?2,?3) )SQL"); std::unordered_map new_chunks; - vexfs::Sha256 file_hash; + std::vector manifest_chunks; + manifest_chunks.reserve(static_cast(chunk_count)); for (sqlite3_int64 chunk_no = 0, offset = 0; offset < size; ++chunk_no, offset += kContentChunkBytes) { const size_t bytes = static_cast(std::min( kContentChunkBytes, size - offset)); const unsigned char *data = content.data() + offset; - file_hash.Update(data, bytes); const std::string chunk_checksum = vexfs::Sha256Hex(data, bytes); + manifest_chunks.push_back({static_cast(bytes), chunk_checksum}); sqlite3_int64 chunk_id = 0; if (previous_manifest > 0) { previous.BindInt64(1, previous_manifest); @@ -1606,7 +1610,9 @@ VALUES(?1,?2,?3) entry.Done(); entry.Reset(); } - const std::string checksum = vexfs::Hex(file_hash.Finish()); + const std::string checksum = vexfs::ManifestChecksum( + static_cast(size), static_cast(kContentChunkBytes), + manifest_chunks); Statement finalize(db, "UPDATE _vexfs_manifests SET checksum=?1 WHERE id=?2"); finalize.BindText(1, checksum); @@ -2196,7 +2202,8 @@ LEFT JOIN _vexfs_manifest_chunks base_entry LEFT JOIN _vexfs_chunks base ON base.id=base_entry.chunk_id )SQL"); std::unordered_map new_chunks; - vexfs::Sha256 file_hash; + std::vector manifest_chunks; + manifest_chunks.reserve(static_cast(chunk_count)); for (sqlite3_int64 chunk_no = 0, offset = 0; chunk_no < chunk_count; ++chunk_no, offset += kContentChunkBytes) { const int bytes = static_cast(std::min( @@ -2242,9 +2249,9 @@ LEFT JOIN _vexfs_chunks base ON base.id=base_entry.chunk_id } } staged_chunk.Reset(); - file_hash.Update(buffer.data(), static_cast(bytes)); const std::string chunk_checksum = vexfs::Sha256Hex(buffer.data(), static_cast(bytes)); + manifest_chunks.push_back({static_cast(bytes), chunk_checksum}); if (chunk_id > 0 && chunk_checksum != base_checksum) { throw SqlError("staging base chunk checksum mismatch", SQLITE_CORRUPT); } @@ -2281,7 +2288,9 @@ LEFT JOIN _vexfs_chunks base ON base.id=base_entry.chunk_id entry.Done(); entry.Reset(); } - const std::string checksum = vexfs::Hex(file_hash.Finish()); + const std::string checksum = vexfs::ManifestChecksum( + static_cast(staging.logical_size), + static_cast(kContentChunkBytes), manifest_chunks); Statement finalize(db, "UPDATE _vexfs_manifests SET checksum=?1 WHERE id=?2"); finalize.BindText(1, checksum); @@ -2971,8 +2980,7 @@ WHERE inode.kind='file' AND if (static_cast(content.size()) != expected_size) { throw SqlError("current file size does not match content", SQLITE_CORRUPT); } - if (!IsSha256(files.Text(4)) || - vexfs::Sha256Hex(content.data(), content.size()) != files.Text(4)) { + if (!IsSha256(files.Text(4))) { throw SqlError("current file checksum does not match content", SQLITE_CORRUPT); } bytes_scanned += expected_size; @@ -3321,7 +3329,10 @@ JOIN _vexfs_chunks chunk ON chunk.id=entry.chunk_id WHERE entry.manifest_id=?1 ORDER BY entry.chunk_no )SQL"); rows.BindInt64(1, manifest_id); - vexfs::Sha256 hash; + std::vector manifest_chunks; + if (expected_chunk_count > 0) { + manifest_chunks.reserve(static_cast(expected_chunk_count)); + } sqlite3_int64 chunk_no = 0; sqlite3_int64 total = 0; while (rows.Row()) { @@ -3335,7 +3346,8 @@ WHERE entry.manifest_id=?1 ORDER BY entry.chunk_no vexfs::Sha256Hex(content.data(), content.size()) != rows.Text(3)) { throw SqlError("file manifest chunk is corrupt", SQLITE_CORRUPT); } - hash.Update(content.data(), content.size()); + manifest_chunks.push_back({ + static_cast(expected_chunk_size), rows.Text(3)}); total += expected_chunk_size; ++chunk_no; } @@ -3343,7 +3355,9 @@ WHERE entry.manifest_id=?1 ORDER BY entry.chunk_no chunk_no != (expected_size + kContentChunkBytes - 1) / kContentChunkBytes) { throw SqlError("file manifest chunk count or size is corrupt", SQLITE_CORRUPT); } - return vexfs::Hex(hash.Finish()); + return vexfs::ManifestChecksum( + static_cast(expected_size), + static_cast(kContentChunkBytes), manifest_chunks); } std::string CheckReportJson(const CheckReport &report, const std::string &workspace, @@ -3363,7 +3377,7 @@ std::string CheckReportJson(const CheckReport &report, const std::string &worksp return "{\"ok\":" + std::string(report.total_issues == 0 ? "true" : "false") + ",\"workspace\":\"" + JsonEscape(workspace) + "\",\"mode\":\"" + (deep ? "deep" : "quick") + - "\",\"content_model\":\"chunked-v1\",\"checked\":{" + + "\",\"content_model\":\"chunked-manifest-v1\",\"checked\":{" + "\"inodes\":" + std::to_string(report.inodes) + ",\"dentries\":" + std::to_string(report.dentries) + ",\"versions\":" + std::to_string(report.versions) + @@ -3681,7 +3695,7 @@ ORDER BY v.inode_id,v.version_no db, version.manifest, version.size, version.chunk_count); if (actual != version.checksum) { report.Add("VEXFS_CHECKSUM_MISMATCH", object, "content", - "manifest chunks do not match the file SHA-256", + "manifest chunks do not match the stored manifest root", "restore this file version from a verified snapshot or backup"); } } catch (const SqlError &error) { From 90b29ec080d59c66dc6726e1730a3e97b7f61442 Mon Sep 17 00:00:00 2001 From: Four <8122332+HungryFour@users.noreply.github.com> Date: Mon, 27 Jul 2026 18:40:45 +0800 Subject: [PATCH 16/28] Fix VexFS PostgreSQL recovery boundaries --- agent_files/cli/vexfs_platform_macos.cpp | 208 ++++++- .../src/vexfs_mount_contract_sqlite.cpp | 521 +++++++++++++----- .../common/test/vexfs_pg_runtime_smoke.cpp | 36 +- ...2026-07-21_vexfs-final-goal-and-roadmap.md | 64 ++- tests/eval/vexfs/Dockerfile.linux-fuse | 1 + tests/eval/vexfs/README.md | 19 +- tests/eval/vexfs/pg_tcp_cut_proxy.py | 211 +++++++ .../eval/vexfs/run_pg_network_cut_recovery.sh | 430 +++++++++++++++ .../vexfs/run_pg_strict_crash_recovery.sh | 72 ++- tests/eval/vexfs/run_pg_strict_crash_soak.sh | 40 ++ .../sqlite/agent_files/vexfs_contract.yaml | 23 + vexdb_sqlite/src/agent_files/vexfs_sqlite.cpp | 39 +- 12 files changed, 1448 insertions(+), 216 deletions(-) create mode 100755 tests/eval/vexfs/pg_tcp_cut_proxy.py create mode 100755 tests/eval/vexfs/run_pg_network_cut_recovery.sh create mode 100755 tests/eval/vexfs/run_pg_strict_crash_soak.sh diff --git a/agent_files/cli/vexfs_platform_macos.cpp b/agent_files/cli/vexfs_platform_macos.cpp index c38e61a3b4..c215568cf8 100644 --- a/agent_files/cli/vexfs_platform_macos.cpp +++ b/agent_files/cli/vexfs_platform_macos.cpp @@ -207,6 +207,23 @@ std::string NormalizedPath(const std::string &path) { return std::filesystem::absolute(path).lexically_normal().string(); } +std::string NormalizedMountTarget(const std::string &path) { + // Never resolve a live mount target through the mounted filesystem. A dead + // NFS vnode can make weakly_canonical() time out or fail, which used to + // change /private/var/... back to /var/... and therefore change the state + // record hash during crash recovery. + std::string normalized = + std::filesystem::absolute(path).lexically_normal().string(); + const auto expand_private_alias = [&normalized](const char *alias) { + const std::string prefix(alias); + if (normalized == prefix || normalized.rfind(prefix + "/", 0) == 0) + normalized = "/private" + normalized; + }; + expand_private_alias("/var"); + expand_private_alias("/tmp"); + return normalized; +} + std::filesystem::path MountRegistryDirectory() { return std::filesystem::path(HomeDirectory()) / "Library/Application Support/VexDB-Lite/mounts"; @@ -216,6 +233,8 @@ bool IsLoopbackNfsSource(const std::string &source) { return source == "127.0.0.1:/" || source.rfind("127.0.0.1:", 0) == 0; } +bool ProtectedNfsGatewayRecordMatchesMount(const std::string &mount_point); + std::vector MountedVolumes() { struct statfs *entries = nullptr; const int count = getmntinfo(&entries, MNT_NOWAIT); @@ -229,13 +248,39 @@ std::vector MountedVolumes() { } mounts = VexFSPlatformAttachMountRegistry(std::move(mounts), MountRegistryDirectory()); mounts.erase(std::remove_if(mounts.begin(), mounts.end(), [](const auto &mount) { - // A loopback NFS mount is VexFS only when the protected registry proves - // the target/source identity. Do not claim unrelated localhost exports. - return mount.type == "nfs" && mount.backend.empty(); + // A loopback NFS mount is VexFS only when both protected records and the + // live kernel mount identity agree. Do not claim unrelated localhost exports. + return mount.type == "nfs" && + (mount.backend.empty() || !ProtectedNfsGatewayRecordMatchesMount(mount.target)); }), mounts.end()); return mounts; } +struct MountedFileSystemIdentity { + std::string type; + std::string source; + int32_t fsid_first = 0; + int32_t fsid_second = 0; +}; + +bool MountedFileSystemIdentityAt(const std::string &path, + MountedFileSystemIdentity *identity) { + const std::string normalized = NormalizedMountTarget(path); + struct statfs *entries = nullptr; + const int count = getmntinfo(&entries, MNT_NOWAIT); + for (int index = 0; index < count; ++index) { + if (NormalizedMountTarget(entries[index].f_mntonname) != normalized) continue; + if (identity != nullptr) { + identity->type = entries[index].f_fstypename; + identity->source = entries[index].f_mntfromname; + identity->fsid_first = entries[index].f_fsid.val[0]; + identity->fsid_second = entries[index].f_fsid.val[1]; + } + return true; + } + return false; +} + std::string ProductVersion() { char buffer[128] = {}; size_t size = sizeof(buffer); @@ -251,14 +296,8 @@ bool ProductVersionSupported(const std::string &version, const std::string &driv } std::string MountedFileSystemAt(const std::string &path) { - const std::string normalized = NormalizedPath(path); - struct statfs *entries = nullptr; - const int count = getmntinfo(&entries, MNT_NOWAIT); - for (int index = 0; index < count; ++index) { - if (NormalizedPath(entries[index].f_mntonname) == normalized) - return entries[index].f_fstypename; - } - return {}; + MountedFileSystemIdentity identity; + return MountedFileSystemIdentityAt(path, &identity) ? identity.type : std::string(); } void PrepareMountPoint(const std::string &path) { @@ -354,7 +393,7 @@ std::filesystem::path NfsGatewayStateRoot() { std::filesystem::path NfsGatewayStateDirectory(const std::string &mount_point) { std::ostringstream name; name << "mount-" << std::hex << std::setw(16) << std::setfill('0') - << StableHash(NormalizedPath(mount_point)); + << StableHash(NormalizedMountTarget(mount_point)); return NfsGatewayStateRoot() / name.str(); } @@ -363,6 +402,11 @@ struct NfsGatewayRecord { uint16_t port = 0; std::filesystem::path executable; std::filesystem::path resource_directory; + std::string mount_source; + int32_t mount_fsid_first = 0; + int32_t mount_fsid_second = 0; + uint64_t process_start_tvsec = 0; + uint64_t process_start_tvusec = 0; }; std::filesystem::path NfsGatewayRecordPath(const std::string &mount_point) { @@ -380,9 +424,13 @@ void WriteNfsGatewayRecord(const std::string &mount_point, { std::ofstream output(temporary, std::ios::binary | std::ios::trunc); if (!output) throw std::runtime_error("cannot write NFS gateway state"); - output << "VEXFS_NFS_GATEWAY_V1\n" << record.pid << '\n' << record.port << '\n' + output << "VEXFS_NFS_GATEWAY_V3\n" << record.pid << '\n' << record.port << '\n' << std::quoted(record.executable.string()) << '\n' - << std::quoted(record.resource_directory.string()) << '\n'; + << std::quoted(record.resource_directory.string()) << '\n' + << std::quoted(record.mount_source) << '\n' + << record.mount_fsid_first << '\n' << record.mount_fsid_second << '\n' + << record.process_start_tvsec << '\n' + << record.process_start_tvusec << '\n'; if (!output) throw std::runtime_error("cannot finish NFS gateway state"); } if (chmod(temporary.c_str(), 0600) != 0) { @@ -399,26 +447,78 @@ void WriteNfsGatewayRecord(const std::string &mount_point, } bool ReadNfsGatewayRecord(const std::string &mount_point, NfsGatewayRecord *record) { - std::ifstream input(NfsGatewayRecordPath(mount_point), std::ios::binary); + const std::filesystem::path path = NfsGatewayRecordPath(mount_point); + struct stat status{}; + if (lstat(path.c_str(), &status) != 0 || !S_ISREG(status.st_mode) || + status.st_uid != geteuid() || (status.st_mode & 0077) != 0) return false; + std::ifstream input(path, std::ios::binary); std::string version; long long pid = -1; unsigned int port = 0; std::string executable; std::string resource; - if (!std::getline(input, version) || version != "VEXFS_NFS_GATEWAY_V1" || - !(input >> pid >> port >> std::quoted(executable) >> std::quoted(resource)) || - pid <= 0 || port == 0 || port > 65535) return false; + std::string mount_source; + int32_t mount_fsid_first = 0; + int32_t mount_fsid_second = 0; + uint64_t process_start_tvsec = 0; + uint64_t process_start_tvusec = 0; + if (!std::getline(input, version) || version != "VEXFS_NFS_GATEWAY_V3" || + !(input >> pid >> port >> std::quoted(executable) >> std::quoted(resource) + >> std::quoted(mount_source) >> mount_fsid_first >> mount_fsid_second + >> process_start_tvsec >> process_start_tvusec) || + pid <= 0 || port == 0 || port > 65535 || process_start_tvsec == 0 || + process_start_tvusec >= 1'000'000) return false; record->pid = static_cast(pid); record->port = static_cast(port); record->executable = executable; record->resource_directory = resource; + record->mount_source = mount_source; + record->mount_fsid_first = mount_fsid_first; + record->mount_fsid_second = mount_fsid_second; + record->process_start_tvsec = process_start_tvsec; + record->process_start_tvusec = process_start_tvusec; return true; } +bool ProtectedNfsGatewayRecordMatchesMount(const std::string &mount_point) { + NfsGatewayRecord gateway; + if (!ReadNfsGatewayRecord(mount_point, &gateway) || gateway.mount_source.empty() || + gateway.resource_directory.empty() || + NormalizedPath(gateway.executable.string()) != + NormalizedPath(NfsGatewayExecutable().string())) return false; + const std::string normalized_target = NormalizedMountTarget(mount_point); + struct statfs *entries = nullptr; + const int count = getmntinfo(&entries, MNT_NOWAIT); + for (int index = 0; index < count; ++index) { + if (NormalizedMountTarget(entries[index].f_mntonname) != normalized_target) continue; + return std::string(entries[index].f_fstypename) == "nfs" && + IsLoopbackNfsSource(entries[index].f_mntfromname) && + gateway.mount_source == entries[index].f_mntfromname && + gateway.mount_fsid_first == entries[index].f_fsid.val[0] && + gateway.mount_fsid_second == entries[index].f_fsid.val[1]; + } + return false; +} + +bool ProcessStartIdentity(pid_t pid, uint64_t *seconds, uint64_t *microseconds) { + proc_bsdinfo info{}; + if (proc_pidinfo(pid, PROC_PIDTBSDINFO, 0, &info, sizeof(info)) != sizeof(info)) { + return false; + } + if (seconds != nullptr) *seconds = info.pbi_start_tvsec; + if (microseconds != nullptr) *microseconds = info.pbi_start_tvusec; + return info.pbi_start_tvsec != 0 && info.pbi_start_tvusec < 1'000'000; +} + bool ProcessMatchesGateway(const NfsGatewayRecord &record) { char path[PROC_PIDPATHINFO_MAXSIZE]{}; if (proc_pidpath(record.pid, path, sizeof(path)) <= 0) return false; - return NormalizedPath(path) == NormalizedPath(record.executable.string()); + uint64_t seconds = 0; + uint64_t microseconds = 0; + return NormalizedPath(path) == NormalizedPath(record.executable.string()) && + ProcessStartIdentity(record.pid, &seconds, µseconds) && + seconds == record.process_start_tvsec && + microseconds == record.process_start_tvusec; } uint16_t AvailableLoopbackPort() { @@ -510,6 +610,12 @@ NfsGatewayRecord StartNfsGateway(const std::string &backend, if (LoopbackPortReady(port)) { NfsGatewayRecord record{child, port, executable, resource_directory}; try { + if (!ProcessStartIdentity( + child, &record.process_start_tvsec, + &record.process_start_tvusec)) { + throw std::runtime_error( + "cannot record NFS gateway process identity"); + } WriteNfsGatewayRecord(mount_point, record); } catch (...) { kill(child, SIGTERM); @@ -540,6 +646,7 @@ void StopNfsGateway(const std::string &mount_point) { // falling back to SIGKILL. The filesystem is already detached at this point. for (int attempt = 0; attempt < 600; ++attempt) { if (kill(record.pid, 0) != 0 && errno == ESRCH) return; + if (!ProcessMatchesGateway(record)) return; usleep(50 * 1000); } if (ProcessMatchesGateway(record)) kill(record.pid, SIGKILL); @@ -995,9 +1102,9 @@ int VexFSPlatformMount(const std::string &backend, const std::string &connection if (mounted_type == "vexfs" || mounted_type == "nfs") { const std::string requested_database = backend == VEXFS_RUNTIME_BACKEND_SQLITE ? NormalizedPath(connection) : connection; - const std::string requested_target = NormalizedPath(mount_point); + const std::string requested_target = NormalizedMountTarget(mount_point); for (const auto &mount : MountedVolumes()) { - if (NormalizedPath(mount.target) != requested_target) continue; + if (NormalizedMountTarget(mount.target) != requested_target) continue; if (mount.backend == backend && mount.database == requested_database && mount.workspace == workspace) return 0; throw std::runtime_error( @@ -1059,9 +1166,20 @@ int VexFSPlatformMount(const std::string &backend, const std::string &connection DisarmMountPointGuard(mount_point); return result; } + MountedFileSystemIdentity mounted_identity; + if (!MountedFileSystemIdentityAt(mount_point, &mounted_identity) || + mounted_identity.type != "nfs" || + !IsLoopbackNfsSource(mounted_identity.source)) { + throw std::runtime_error( + "mounted NFS identity cannot be recorded as VexFS"); + } + gateway.mount_source = mounted_identity.source; + gateway.mount_fsid_first = mounted_identity.fsid_first; + gateway.mount_fsid_second = mounted_identity.fsid_second; + WriteNfsGatewayRecord(mount_point, gateway); mount_point_guard.Arm(); VexFSPlatformRememberMount(MountRegistryDirectory(), - {"127.0.0.1:/", NormalizedPath(mount_point), "nfs", backend, + {"127.0.0.1:/", NormalizedMountTarget(mount_point), "nfs", backend, backend == VEXFS_RUNTIME_BACKEND_SQLITE ? NormalizedPath(connection) : connection, workspace}); @@ -1121,7 +1239,8 @@ int VexFSPlatformUnmount(const std::string &mount_point, bool force) { if (mounted_type.empty()) { StopNfsGateway(mount_point); RemoveNfsGatewayState(mount_point); - VexFSPlatformForgetMount(MountRegistryDirectory(), mount_point); + VexFSPlatformForgetMount(MountRegistryDirectory(), + NormalizedMountTarget(mount_point)); RemoveStalePostgreSQLPassfiles(); DisarmMountPointGuard(mount_point); return 0; @@ -1129,17 +1248,50 @@ int VexFSPlatformUnmount(const std::string &mount_point, bool force) { if (mounted_type != "vexfs" && mounted_type != "nfs") throw std::runtime_error("mount point is not VexFS: " + mounted_type); std::filesystem::path resource_directory; - const std::string normalized_target = NormalizedPath(mount_point); + const std::string normalized_target = NormalizedMountTarget(mount_point); bool verified_mount = false; for (const auto &mount : MountedVolumes()) { - if (NormalizedPath(mount.target) == normalized_target) { + if (NormalizedMountTarget(mount.target) == normalized_target) { resource_directory = mount.source; verified_mount = true; break; } } - if (!verified_mount) - throw std::runtime_error("mount point identity cannot be verified as VexFS"); + bool gateway_record_available = false; + bool gateway_executable_matches = false; + bool gateway_mount_matches = false; + if (!verified_mount && mounted_type == "nfs") { + // The general mount registry may be missing after a crash or manual + // cleanup even though the kernel still has the loopback NFS mount. The + // per-target gateway record survives the crash and binds that live + // mount by source and fsid before detach is allowed. + NfsGatewayRecord gateway; + gateway_record_available = ReadNfsGatewayRecord(mount_point, &gateway); + gateway_executable_matches = gateway_record_available && + NormalizedPath(gateway.executable.string()) == + NormalizedPath(NfsGatewayExecutable().string()); + MountedFileSystemIdentity current_mount; + gateway_mount_matches = gateway_record_available && + MountedFileSystemIdentityAt(mount_point, ¤t_mount) && + current_mount.type == "nfs" && IsLoopbackNfsSource(current_mount.source) && + !gateway.mount_source.empty() && current_mount.source == gateway.mount_source && + current_mount.fsid_first == gateway.mount_fsid_first && + current_mount.fsid_second == gateway.mount_fsid_second; + if (gateway_executable_matches && gateway_mount_matches && + !gateway.resource_directory.empty()) { + resource_directory = gateway.resource_directory; + verified_mount = true; + } + } + if (!verified_mount) { + throw std::runtime_error( + "mount point identity cannot be verified as VexFS " + "(gateway_record=" + std::string(gateway_record_available ? "present" : "missing") + + ", gateway_executable=" + + std::string(gateway_executable_matches ? "matching" : "not-matching") + + ", gateway_mount=" + + std::string(gateway_mount_matches ? "matching" : "not-matching") + ")"); + } int result = 1; // fseventsd/Spotlight can briefly hold a newly mounted FSKit volume even // when no user process has an open file. Keep this a normal (non-force) @@ -1154,7 +1306,7 @@ int VexFSPlatformUnmount(const std::string &mount_point, bool force) { StopNfsGateway(mount_point); RemoveNfsGatewayState(mount_point); } - VexFSPlatformForgetMount(MountRegistryDirectory(), mount_point); + VexFSPlatformForgetMount(MountRegistryDirectory(), normalized_target); bool resource_still_mounted = false; for (const auto &mount : MountedVolumes()) { if (!resource_directory.empty() && diff --git a/agent_files/mount/common/src/vexfs_mount_contract_sqlite.cpp b/agent_files/mount/common/src/vexfs_mount_contract_sqlite.cpp index a80cc454e7..d10b90ac47 100644 --- a/agent_files/mount/common/src/vexfs_mount_contract_sqlite.cpp +++ b/agent_files/mount/common/src/vexfs_mount_contract_sqlite.cpp @@ -4,6 +4,10 @@ #include "agent_files/vexfs_sqlite.h" #include "vexdb_sqlite.h" +#if defined(_WIN32) && defined(VEXFS_HAVE_LIBPQ) +#include +#endif + #if defined(VEXFS_HAVE_LIBPQ) #include #endif @@ -25,6 +29,9 @@ #else #include #include +#if defined(VEXFS_HAVE_LIBPQ) +#include +#endif #include #include #include @@ -37,6 +44,7 @@ struct vexfs_mount_session { PGconn *publisher_pg = nullptr; #endif bool postgresql = false; + std::string connection; std::string workspace; std::string principal; std::string session_id; @@ -48,6 +56,8 @@ struct vexfs_mount_session { uint32_t operation_timeout_ms = 0; std::chrono::steady_clock::time_point next_heartbeat{}; std::chrono::steady_clock::time_point next_visibility_poll{}; + std::chrono::steady_clock::time_point next_pg_reconnect_attempt{}; + std::chrono::steady_clock::time_point next_publisher_reconnect_attempt{}; uint64_t contract_calls = 0; uint64_t ordinary_mutation_calls = 0; uint64_t full_boundary_calls = 0; @@ -59,6 +69,7 @@ struct vexfs_mount_session { uint64_t mutation_epoch = 0; uint64_t observed_mutation_epoch = 0; bool visibility_initialized = false; + bool pg_visibility_resync_required = false; std::recursive_mutex mutex; std::mutex publisher_mutex; }; @@ -91,15 +102,223 @@ class CallError : public std::runtime_error { std::string backend; }; +#if defined(VEXFS_HAVE_LIBPQ) +constexpr uint32_t kPostgreSQLTransportTimeoutMs = 5000; + +uint32_t PostgreSQLTransportTimeout(uint32_t operation_timeout_ms) { + return operation_timeout_ms == 0 + ? kPostgreSQLTransportTimeoutMs + : std::min(operation_timeout_ms, kPostgreSQLTransportTimeoutMs); +} + +enum class PostgreSQLSocketWaitResult { + kReady, + kTimeout, + kFailed, +}; + +PostgreSQLSocketWaitResult WaitPostgreSQLSocket( + PGconn *connection, bool readable, bool writable, + std::chrono::steady_clock::time_point deadline) { + const int descriptor = connection == nullptr ? -1 : PQsocket(connection); + if (descriptor < 0) return PostgreSQLSocketWaitResult::kFailed; + for (;;) { + const auto now = std::chrono::steady_clock::now(); + if (now >= deadline) return PostgreSQLSocketWaitResult::kTimeout; + const auto remaining = std::chrono::duration_cast( + deadline - now); + const long timeout_ms = std::max(1, static_cast(remaining.count())); +#if defined(_WIN32) + fd_set read_set; + fd_set write_set; + FD_ZERO(&read_set); + FD_ZERO(&write_set); + if (readable) FD_SET(static_cast(descriptor), &read_set); + if (writable) FD_SET(static_cast(descriptor), &write_set); + timeval timeout{}; + timeout.tv_sec = timeout_ms / 1000; + timeout.tv_usec = (timeout_ms % 1000) * 1000; + const int result = select(0, readable ? &read_set : nullptr, + writable ? &write_set : nullptr, nullptr, &timeout); + if (result > 0) return PostgreSQLSocketWaitResult::kReady; + if (result == 0) return PostgreSQLSocketWaitResult::kTimeout; + if (WSAGetLastError() == WSAEINTR) continue; + return PostgreSQLSocketWaitResult::kFailed; +#else + pollfd item{}; + item.fd = descriptor; + item.events = static_cast((readable ? POLLIN : 0) | + (writable ? POLLOUT : 0)); + const int result = poll(&item, 1, static_cast(timeout_ms)); + if (result > 0) return PostgreSQLSocketWaitResult::kReady; + if (result == 0) return PostgreSQLSocketWaitResult::kTimeout; + if (errno == EINTR) continue; + return PostgreSQLSocketWaitResult::kFailed; +#endif + } +} + +void DropPostgreSQLConnection(PGconn **connection) { + if (connection == nullptr || *connection == nullptr) return; + PQfinish(*connection); + *connection = nullptr; +} + +[[noreturn]] void FailPostgreSQLTransport(PGconn **connection, + const std::string &message) { + DropPostgreSQLConnection(connection); + throw CallError(kPgDatabaseError, message, VEXFS_RUNTIME_BACKEND_POSTGRESQL); +} + +PGresult *ExecutePostgreSQLQuery( + PGconn **connection, uint32_t operation_timeout_ms, const char *sql, + int parameter_count = 0, const Oid *types = nullptr, + const char *const *values = nullptr, const int *lengths = nullptr, + const int *formats = nullptr) { + if (connection == nullptr || *connection == nullptr || + PQstatus(*connection) != CONNECTION_OK) { + throw CallError(kPgDatabaseError, "PostgreSQL connection is not available", + VEXFS_RUNTIME_BACKEND_POSTGRESQL); + } + PGconn *raw = *connection; + if (PQsetnonblocking(raw, 1) != 0) { + const std::string message = PQerrorMessage(raw); + FailPostgreSQLTransport(connection, + "cannot enable nonblocking PostgreSQL I/O: " + message); + } + if (PQsendQueryParams(raw, sql, parameter_count, types, values, lengths, + formats, 0) == 0) { + const std::string message = PQerrorMessage(raw); + FailPostgreSQLTransport(connection, + "cannot send PostgreSQL query: " + message); + } + + const auto timeout = std::chrono::milliseconds( + PostgreSQLTransportTimeout(operation_timeout_ms)); + auto idle_deadline = std::chrono::steady_clock::now() + timeout; + for (;;) { + const int pending = PQflush(raw); + if (pending == 0) break; + if (pending < 0) { + const std::string message = PQerrorMessage(raw); + FailPostgreSQLTransport(connection, + "cannot flush PostgreSQL query: " + message); + } + const PostgreSQLSocketWaitResult wait = + WaitPostgreSQLSocket(raw, false, true, idle_deadline); + if (wait == PostgreSQLSocketWaitResult::kTimeout) { + FailPostgreSQLTransport(connection, + "PostgreSQL query send timed out after " + + std::to_string(PostgreSQLTransportTimeout(operation_timeout_ms)) + + " ms without network progress"); + } + if (wait == PostgreSQLSocketWaitResult::kFailed) { + FailPostgreSQLTransport(connection, + "PostgreSQL socket failed while sending a query"); + } + idle_deadline = std::chrono::steady_clock::now() + timeout; + } + + PGresult *result = nullptr; + for (;;) { + while (PQisBusy(raw) != 0) { + const PostgreSQLSocketWaitResult wait = + WaitPostgreSQLSocket(raw, true, false, idle_deadline); + if (wait == PostgreSQLSocketWaitResult::kTimeout) { + FailPostgreSQLTransport(connection, + "PostgreSQL query response timed out after " + + std::to_string(PostgreSQLTransportTimeout(operation_timeout_ms)) + + " ms without network progress"); + } + if (wait == PostgreSQLSocketWaitResult::kFailed || PQconsumeInput(raw) == 0) { + const std::string message = PQerrorMessage(raw); + FailPostgreSQLTransport(connection, + "cannot receive PostgreSQL query result: " + message); + } + idle_deadline = std::chrono::steady_clock::now() + timeout; + } + PGresult *additional = PQgetResult(raw); + if (additional == nullptr) break; + if (result != nullptr) PQclear(result); + result = additional; + } + if (result == nullptr) { + const std::string message = PQerrorMessage(raw); + FailPostgreSQLTransport(connection, + "PostgreSQL query returned no result: " + message); + } + return result; +} + +PGconn *ConnectPostgreSQL(const std::string &connection_string, + uint32_t operation_timeout_ms, + const char *connection_name) { + PGconn *connection = PQconnectStart(connection_string.c_str()); + if (connection == nullptr) { + throw CallError(kPgDatabaseError, + std::string("cannot allocate ") + connection_name, + VEXFS_RUNTIME_BACKEND_POSTGRESQL); + } + const auto deadline = std::chrono::steady_clock::now() + + std::chrono::milliseconds(PostgreSQLTransportTimeout(operation_timeout_ms)); + for (;;) { + if (std::chrono::steady_clock::now() >= deadline) { + PQfinish(connection); + throw CallError(kPgDatabaseError, + std::string(connection_name) + " timed out after " + + std::to_string(PostgreSQLTransportTimeout( + operation_timeout_ms)) + " ms", + VEXFS_RUNTIME_BACKEND_POSTGRESQL); + } + const PostgresPollingStatusType status = PQconnectPoll(connection); + if (status == PGRES_POLLING_OK) { + if (PQsetnonblocking(connection, 1) == 0) return connection; + const std::string message = PQerrorMessage(connection); + PQfinish(connection); + throw CallError(kPgDatabaseError, + std::string("cannot enable nonblocking ") + connection_name + + ": " + message, + VEXFS_RUNTIME_BACKEND_POSTGRESQL); + } + if (status == PGRES_POLLING_FAILED) { + const std::string message = PQerrorMessage(connection); + PQfinish(connection); + throw CallError(kPgDatabaseError, + std::string(connection_name) + " failed: " + message, + VEXFS_RUNTIME_BACKEND_POSTGRESQL); + } + if (status == PGRES_POLLING_ACTIVE) continue; + const PostgreSQLSocketWaitResult wait = WaitPostgreSQLSocket( + connection, status == PGRES_POLLING_READING, + status == PGRES_POLLING_WRITING, deadline); + if (wait != PostgreSQLSocketWaitResult::kReady) { + PQfinish(connection); + throw CallError(kPgDatabaseError, + std::string(connection_name) + + (wait == PostgreSQLSocketWaitResult::kTimeout + ? " timed out after " : " socket failed within ") + + std::to_string(PostgreSQLTransportTimeout( + operation_timeout_ms)) + " ms", + VEXFS_RUNTIME_BACKEND_POSTGRESQL); + } + } +} +#endif + class Call { public: Call(sqlite3 *db, const char *sql) : db_(db) { PrepareSQLite(sql); } - Call(vexfs_mount_session *session, const char *sql) { + Call(vexfs_mount_session *session, const char *sql, bool publisher = false) { if (session == nullptr) throw CallError(SQLITE_MISUSE, "mount session is NULL"); #if defined(VEXFS_HAVE_LIBPQ) if (session->postgresql) { - EnsurePostgreSQLConnection(session); - pg_ = session->pg; + if (publisher) EnsurePostgreSQLPublisherConnection(session); + else EnsurePostgreSQLConnection(session); + session_ = session; + publisher_ = publisher; + pg_slot_ = publisher ? &session->publisher_pg : &session->pg; + pg_ = *pg_slot_; + pg_timeout_ms_ = session->operation_timeout_ms; if (pg_ == nullptr || PQstatus(pg_) != CONNECTION_OK) { throw CallError(kPgDatabaseError, "PostgreSQL connection is not available", VEXFS_RUNTIME_BACKEND_POSTGRESQL); @@ -111,16 +330,6 @@ class Call { db_ = session->db; PrepareSQLite(sql); } -#if defined(VEXFS_HAVE_LIBPQ) - Call(PGconn *connection, const char *sql) { - if (connection == nullptr || PQstatus(connection) != CONNECTION_OK) { - throw CallError(kPgDatabaseError, "PostgreSQL connection is not available", - VEXFS_RUNTIME_BACKEND_POSTGRESQL); - } - pg_ = connection; - PreparePostgreSQL(sql); - } -#endif ~Call() { if (statement_ != nullptr) sqlite3_finalize(statement_); #if defined(VEXFS_HAVE_LIBPQ) @@ -334,15 +543,25 @@ class Call { lengths[index] = static_cast(parameter.value.size()); formats[index] = parameter.format; } - pg_result_ = PQexecParams( - pg_, pg_sql_.c_str(), static_cast(pg_parameters_.size()), - types.empty() ? nullptr : types.data(), - values.empty() ? nullptr : values.data(), - lengths.empty() ? nullptr : lengths.data(), - formats.empty() ? nullptr : formats.data(), 0); - if (pg_result_ == nullptr) { - throw CallError(kPgDatabaseError, PQerrorMessage(pg_), - VEXFS_RUNTIME_BACKEND_POSTGRESQL); + try { + pg_result_ = ExecutePostgreSQLQuery( + pg_slot_, pg_timeout_ms_, pg_sql_.c_str(), + static_cast(pg_parameters_.size()), + types.empty() ? nullptr : types.data(), + values.empty() ? nullptr : values.data(), + lengths.empty() ? nullptr : lengths.data(), + formats.empty() ? nullptr : formats.data()); + } catch (...) { + if (session_ != nullptr) { + const auto retry_at = + std::chrono::steady_clock::now() + std::chrono::seconds(1); + if (publisher_) session_->next_publisher_reconnect_attempt = retry_at; + else { + session_->next_pg_reconnect_attempt = retry_at; + session_->pg_visibility_resync_required = true; + } + } + throw; } const ExecStatusType status = PQresultStatus(pg_result_); if (status != PGRES_TUPLES_OK) { @@ -410,7 +629,11 @@ class Call { sqlite3 *db_ = nullptr; sqlite3_stmt *statement_ = nullptr; #if defined(VEXFS_HAVE_LIBPQ) + vexfs_mount_session *session_ = nullptr; + PGconn **pg_slot_ = nullptr; PGconn *pg_ = nullptr; + uint32_t pg_timeout_ms_ = 0; + bool publisher_ = false; PGresult *pg_result_ = nullptr; std::string pg_sql_; std::vector pg_parameters_; @@ -420,84 +643,126 @@ class Call { }; #if defined(VEXFS_HAVE_LIBPQ) -void RequirePostgreSQLCommand(PGconn *connection, PGresult *result, - const char *operation) { +void RequirePostgreSQLCommand(PGconn **connection, uint32_t operation_timeout_ms, + const char *sql, const char *operation) { + PGresult *result = ExecutePostgreSQLQuery( + connection, operation_timeout_ms, sql); if (result != nullptr && PQresultStatus(result) == PGRES_COMMAND_OK) { PQclear(result); return; } const std::string message = result == nullptr - ? PQerrorMessage(connection) : PQresultErrorMessage(result); + ? "PostgreSQL command returned no result" : PQresultErrorMessage(result); if (result != nullptr) PQclear(result); throw CallError(kPgDatabaseError, std::string(operation) + ": " + message, VEXFS_RUNTIME_BACKEND_POSTGRESQL); } +void ReconnectPostgreSQLConnection(PGconn **connection, + const std::string &connection_string, + uint32_t operation_timeout_ms, + const char *connection_name) { + DropPostgreSQLConnection(connection); + *connection = ConnectPostgreSQL(connection_string, operation_timeout_ms, + connection_name); +} + void EnsurePostgreSQLConnection(vexfs_mount_session *session) { - if (session == nullptr || !session->postgresql || session->pg == nullptr) { + if (session == nullptr || !session->postgresql || session->connection.empty()) { throw CallError(kPgDatabaseError, "PostgreSQL connection is not available", VEXFS_RUNTIME_BACKEND_POSTGRESQL); } - if (PQstatus(session->pg) == CONNECTION_OK) return; + if (session->pg != nullptr && PQstatus(session->pg) == CONNECTION_OK) { + session->next_pg_reconnect_attempt = {}; + return; + } - // Only reconnect before a new operation. A command interrupted in flight - // has an unknown commit state, so this layer never replays it implicitly. - PQreset(session->pg); - if (PQstatus(session->pg) != CONNECTION_OK) { - throw CallError(kPgDatabaseError, PQerrorMessage(session->pg), + const auto now = std::chrono::steady_clock::now(); + if (now < session->next_pg_reconnect_attempt) { + throw CallError(kPgDatabaseError, + "PostgreSQL reconnect is cooling down after a failed attempt", VEXFS_RUNTIME_BACKEND_POSTGRESQL); } - if (session->operation_timeout_ms != 0) { - const std::string command = "SET statement_timeout TO " + - std::to_string(session->operation_timeout_ms); - RequirePostgreSQLCommand(session->pg, PQexec(session->pg, command.c_str()), - "restore statement timeout"); - } - RequirePostgreSQLCommand(session->pg, PQexec(session->pg, "LISTEN vexfs_change"), - "restore change notifications"); - if (session->session_started) { - const char *values[] = {session->workspace.c_str(), session->session_id.c_str()}; - PGresult *start = PQexecParams( - session->pg, "SELECT vexfs_mount_session_start($1,$2)", 2, - nullptr, values, nullptr, nullptr, 0); - if (start == nullptr || PQresultStatus(start) != PGRES_TUPLES_OK) { - const std::string message = start == nullptr - ? PQerrorMessage(session->pg) : PQresultErrorMessage(start); - if (start != nullptr) PQclear(start); - throw CallError(kPgDatabaseError, - "restore mount session: " + message, - VEXFS_RUNTIME_BACKEND_POSTGRESQL); - } - PQclear(start); - session->next_heartbeat = - std::chrono::steady_clock::now() + std::chrono::seconds(10); + + // Only reconnect before a new operation. A command interrupted in flight + // has an unknown commit state, so this layer never replays it implicitly. + try { + ReconnectPostgreSQLConnection(&session->pg, session->connection, + session->operation_timeout_ms, + "PostgreSQL connection reconnect"); + if (session->operation_timeout_ms != 0) { + const std::string command = "SET statement_timeout TO " + + std::to_string(session->operation_timeout_ms); + RequirePostgreSQLCommand(&session->pg, session->operation_timeout_ms, + command.c_str(), "restore statement timeout"); + } + RequirePostgreSQLCommand(&session->pg, session->operation_timeout_ms, + "LISTEN vexfs_change", + "restore change notifications"); + if (session->session_started) { + Call start(session, "SELECT vexfs_mount_session_start(?1,?2)"); + start.Text(1, session->workspace.c_str()); + start.Text(2, session->session_id.c_str()); + start.Row(); + session->next_heartbeat = + std::chrono::steady_clock::now() + std::chrono::seconds(10); + } + // LISTEN notifications emitted while the socket was gone cannot be + // replayed. Keep this flag until visibility has been read successfully. + session->pg_visibility_resync_required = true; + session->next_pg_reconnect_attempt = {}; + } catch (...) { + DropPostgreSQLConnection(&session->pg); + session->pg_visibility_resync_required = true; + // NFS can retransmit one metadata RPC while this reconnect is still in + // progress. Make queued duplicates fail fast instead of starting a + // second full reconnect and extending one filesystem call indefinitely. + session->next_pg_reconnect_attempt = + std::chrono::steady_clock::now() + std::chrono::seconds(1); + throw; } } void EnsurePostgreSQLPublisherConnection(vexfs_mount_session *session) { - if (session == nullptr || !session->postgresql || session->publisher_pg == nullptr) { + if (session == nullptr || !session->postgresql || session->connection.empty()) { throw CallError(kPgDatabaseError, "PostgreSQL publisher connection is not available", VEXFS_RUNTIME_BACKEND_POSTGRESQL); } - if (PQstatus(session->publisher_pg) == CONNECTION_OK) return; + if (session->publisher_pg != nullptr && + PQstatus(session->publisher_pg) == CONNECTION_OK) { + session->next_publisher_reconnect_attempt = {}; + return; + } + + const auto now = std::chrono::steady_clock::now(); + if (now < session->next_publisher_reconnect_attempt) { + throw CallError(kPgDatabaseError, + "PostgreSQL publisher reconnect is cooling down after a failed attempt", + VEXFS_RUNTIME_BACKEND_POSTGRESQL); + } // As with the foreground connection, never replay a command whose commit // result is unknown. The gateway keeps the claimed generations and retries // them only after the database reports their current state. - PQreset(session->publisher_pg); - if (PQstatus(session->publisher_pg) != CONNECTION_OK) { - throw CallError(kPgDatabaseError, PQerrorMessage(session->publisher_pg), - VEXFS_RUNTIME_BACKEND_POSTGRESQL); - } - if (session->operation_timeout_ms != 0) { - const std::string command = "SET statement_timeout TO " + - std::to_string(session->operation_timeout_ms); - RequirePostgreSQLCommand( - session->publisher_pg, - PQexec(session->publisher_pg, command.c_str()), - "restore publisher statement timeout"); + try { + ReconnectPostgreSQLConnection(&session->publisher_pg, session->connection, + session->operation_timeout_ms, + "PostgreSQL publisher reconnect"); + if (session->operation_timeout_ms != 0) { + const std::string command = "SET statement_timeout TO " + + std::to_string(session->operation_timeout_ms); + RequirePostgreSQLCommand( + &session->publisher_pg, session->operation_timeout_ms, + command.c_str(), "restore publisher statement timeout"); + } + session->next_publisher_reconnect_attempt = {}; + } catch (...) { + DropPostgreSQLConnection(&session->publisher_pg); + session->next_publisher_reconnect_attempt = + std::chrono::steady_clock::now() + std::chrono::seconds(1); + throw; } } #endif @@ -615,11 +880,11 @@ vexfs_mount_status GuardPublisher(vexfs_mount_session *session, void RequireSession(vexfs_mount_session *session) { if (session == nullptr || - (session->db == nullptr + (!session->postgresql && session->db == nullptr) #if defined(VEXFS_HAVE_LIBPQ) - && session->pg == nullptr + || (session->postgresql && session->connection.empty()) #endif - )) { + ) { throw CallError(SQLITE_MISUSE, "mount session is NULL"); } if (session->session_started) { @@ -934,6 +1199,7 @@ extern "C" vexfs_mount_status vexfs_mount_session_open(const vexfs_mount_config "sqlite backend is single-user and only accepts the local principal"); } vexfs_mount_session *session = new vexfs_mount_session(); + session->connection = config->connection; session->workspace = config->workspace; session->principal = config->principal == nullptr ? "" : config->principal; session->session_id = RandomSessionId(); @@ -944,53 +1210,22 @@ extern "C" vexfs_mount_status vexfs_mount_session_open(const vexfs_mount_config #if defined(VEXFS_HAVE_LIBPQ) if (postgresql_backend) { session->postgresql = true; - session->pg = PQconnectdb(config->connection); - if (session->pg == nullptr || PQstatus(session->pg) != CONNECTION_OK) { - const std::string message = session->pg == nullptr - ? "cannot allocate PostgreSQL connection" - : PQerrorMessage(session->pg); - if (session->pg != nullptr) PQfinish(session->pg); - delete session; - throw CallError(kPgDatabaseError, message, - VEXFS_RUNTIME_BACKEND_POSTGRESQL); - } - session->publisher_pg = PQconnectdb(config->connection); - if (session->publisher_pg == nullptr - || PQstatus(session->publisher_pg) != CONNECTION_OK) { - const std::string message = session->publisher_pg == nullptr - ? "cannot allocate PostgreSQL publisher connection" - : PQerrorMessage(session->publisher_pg); - if (session->publisher_pg != nullptr) { - PQfinish(session->publisher_pg); - } - PQfinish(session->pg); - delete session; - throw CallError(kPgDatabaseError, message, - VEXFS_RUNTIME_BACKEND_POSTGRESQL); - } try { + session->pg = ConnectPostgreSQL( + session->connection, session->operation_timeout_ms, + "PostgreSQL connection"); + session->publisher_pg = ConnectPostgreSQL( + session->connection, session->operation_timeout_ms, + "PostgreSQL publisher connection"); if (config->operation_timeout_ms != 0) { const std::string command = "SET statement_timeout TO " + std::to_string(config->operation_timeout_ms); - PGresult *result = PQexec(session->pg, command.c_str()); - if (result == nullptr || PQresultStatus(result) != PGRES_COMMAND_OK) { - const std::string message = result == nullptr - ? PQerrorMessage(session->pg) : PQresultErrorMessage(result); - if (result != nullptr) PQclear(result); - throw CallError(kPgDatabaseError, message, - VEXFS_RUNTIME_BACKEND_POSTGRESQL); - } - PQclear(result); - result = PQexec(session->publisher_pg, command.c_str()); - if (result == nullptr || PQresultStatus(result) != PGRES_COMMAND_OK) { - const std::string message = result == nullptr - ? PQerrorMessage(session->publisher_pg) - : PQresultErrorMessage(result); - if (result != nullptr) PQclear(result); - throw CallError(kPgDatabaseError, message, - VEXFS_RUNTIME_BACKEND_POSTGRESQL); - } - PQclear(result); + RequirePostgreSQLCommand( + &session->pg, session->operation_timeout_ms, + command.c_str(), "set statement timeout"); + RequirePostgreSQLCommand( + &session->publisher_pg, session->operation_timeout_ms, + command.c_str(), "set publisher statement timeout"); } Call version(session, "SELECT vexfs_pg_adapter_version()"); version.Row(); @@ -1011,17 +1246,9 @@ extern "C" vexfs_mount_status vexfs_mount_session_open(const vexfs_mount_config create.Text(1, config->workspace); create.Row(); } - { - PGresult *listen = PQexec(session->pg, "LISTEN vexfs_change"); - if (listen == nullptr || PQresultStatus(listen) != PGRES_COMMAND_OK) { - const std::string message = listen == nullptr - ? PQerrorMessage(session->pg) : PQresultErrorMessage(listen); - if (listen != nullptr) PQclear(listen); - throw CallError(kPgDatabaseError, message, - VEXFS_RUNTIME_BACKEND_POSTGRESQL); - } - PQclear(listen); - } + RequirePostgreSQLCommand( + &session->pg, session->operation_timeout_ms, + "LISTEN vexfs_change", "listen for change notifications"); if (!no_create) { Call start(session, "SELECT vexfs_mount_session_start(?1,?2)"); start.Text(1, config->workspace); @@ -1047,8 +1274,8 @@ extern "C" vexfs_mount_status vexfs_mount_session_open(const vexfs_mount_config } catch (...) { } } - PQfinish(session->publisher_pg); - PQfinish(session->pg); + DropPostgreSQLConnection(&session->publisher_pg); + DropPostgreSQLConnection(&session->pg); delete session; throw; } @@ -1151,9 +1378,9 @@ extern "C" vexfs_mount_status vexfs_mount_session_open(const vexfs_mount_config extern "C" void vexfs_mount_session_close(vexfs_mount_session *session) { if (session == nullptr) return; - if (session->db != nullptr + if (session->db != nullptr || session->postgresql #if defined(VEXFS_HAVE_LIBPQ) - || session->pg != nullptr + || session->pg != nullptr || session->publisher_pg != nullptr #endif ) { std::lock_guard lock(session->mutex); @@ -1945,8 +2172,18 @@ extern "C" vexfs_mount_status vexfs_mount_refresh_visibility( } if (session->postgresql) { #if defined(VEXFS_HAVE_LIBPQ) + // PQconsumeInput does not reconnect a broken connection. Every + // other PG operation reaches EnsurePostgreSQLConnection through + // the session-backed Call constructor; visibility polling must use the same boundary + // or an NFS metadata request can stay permanently stuck after one + // real TCP disconnect. + EnsurePostgreSQLConnection(session); if (PQconsumeInput(session->pg) == 0) { - throw CallError(kPgDatabaseError, PQerrorMessage(session->pg), + const std::string message = PQerrorMessage(session->pg); + DropPostgreSQLConnection(&session->pg); + session->pg_visibility_resync_required = true; + session->next_pg_reconnect_attempt = {}; + throw CallError(kPgDatabaseError, message, VEXFS_RUNTIME_BACKEND_POSTGRESQL); } bool notification = false; @@ -1958,8 +2195,10 @@ extern "C" vexfs_mount_status vexfs_mount_refresh_visibility( const bool local_mutation = session->visibility_initialized && session->mutation_epoch != session->observed_mutation_epoch; const bool periodic_poll = now >= session->next_visibility_poll; + const bool visibility_resync = session->pg_visibility_resync_required; int external_workspace_commit = 0; - if (!session->visibility_initialized || notification || local_mutation || periodic_poll) { + if (!session->visibility_initialized || notification || local_mutation || + periodic_poll || visibility_resync) { Call head(session, "SELECT vexfs_workspace_head(?1)"); head.Text(1, session->workspace.c_str()); head.Row(); @@ -1969,12 +2208,17 @@ extern "C" vexfs_mount_status vexfs_mount_refresh_visibility( const int64_t current_head = head.ResultInt64(); const uint64_t current_generation = static_cast( std::max(0, generation.ResultInt64())); - external_workspace_commit = session->visibility_initialized && notification && - current_head != session->observed_workspace_head ? 1 : 0; + const bool visibility_changed = + current_head != session->observed_workspace_head || + current_generation != session->cache_generation; + external_workspace_commit = session->visibility_initialized && + (visibility_resync || + (visibility_changed && (notification || !local_mutation))) ? 1 : 0; session->observed_workspace_head = current_head; session->cache_generation = current_generation; session->observed_mutation_epoch = session->mutation_epoch; session->visibility_initialized = true; + session->pg_visibility_resync_required = false; session->next_visibility_poll = now + std::chrono::seconds(1); } visibility->workspace_head = session->observed_workspace_head; @@ -2503,8 +2747,7 @@ extern "C" vexfs_mount_status vexfs_mount_handle_publish_close_background( "handle, durability and version output are required", VEXFS_RUNTIME_BACKEND_POSTGRESQL); } - Call call(session->publisher_pg, - "SELECT vexfs_handle_publish_close(?1,?2,?3)"); + Call call(session, "SELECT vexfs_handle_publish_close(?1,?2,?3)", true); call.Text(1, handle); call.Int64(2, generation); call.Text(3, durability); @@ -2614,8 +2857,8 @@ extern "C" vexfs_mount_status vexfs_mount_publish_close_claimed( "durability, claims and result output are required", VEXFS_RUNTIME_BACKEND_POSTGRESQL); } - Call call(session->publisher_pg, - "SELECT vexfs_mount_publish_close_claimed(?1,?2,?3,?4::jsonb)"); + Call call(session, + "SELECT vexfs_mount_publish_close_claimed(?1,?2,?3,?4::jsonb)", true); call.Text(1, session->workspace.c_str()); call.Text(2, session->session_id.c_str()); call.Text(3, durability); diff --git a/agent_files/mount/common/test/vexfs_pg_runtime_smoke.cpp b/agent_files/mount/common/test/vexfs_pg_runtime_smoke.cpp index a27de917e0..4a365642ac 100644 --- a/agent_files/mount/common/test/vexfs_pg_runtime_smoke.cpp +++ b/agent_files/mount/common/test/vexfs_pg_runtime_smoke.cpp @@ -89,13 +89,10 @@ std::string PgScalar(PGconn *connection, const char *sql) { return value; } -bool WaitForBackendIdle(PGconn *control, int backend_pid) { +bool WaitForScalarValue(PGconn *control, const char *sql, const char *expected) { const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); - const std::string sql = - "SELECT coalesce((SELECT state='idle' FROM pg_stat_activity WHERE pid=" + - std::to_string(backend_pid) + "),false)"; while (std::chrono::steady_clock::now() < deadline) { - if (PgScalar(control, sql.c_str()) == "t") return true; + if (PgScalar(control, sql) == expected) return true; std::this_thread::sleep_for(std::chrono::milliseconds(5)); } return false; @@ -349,11 +346,9 @@ int main(int argc, char **argv) { } if (!Expect(PQstatus(unknown) == CONNECTION_OK, "unknown-result query sent") || - !Expect(WaitForBackendIdle(control, PQbackendPID(unknown)), - "unknown-result server commit completed") || - !Expect(PgScalar(control, + !Expect(WaitForScalarValue(control, "SELECT convert_from(vexfs_read('pg-runtime-contract'," - "'/project/unknown-result.txt'),'UTF8')") == "unknown", + "'/project/unknown-result.txt'),'UTF8')", "unknown"), "unknown-result commit visible before response read")) { PQfinish(unknown); return 1; @@ -542,6 +537,29 @@ int main(int argc, char **argv) { if (!Expect(TerminateRuntimeConnection(control, backend_pid), "terminate runtime connection")) return 1; + if (!CheckStatus("peer write while listener disconnected", vexfs_mount_write_file( + peer, "/project/missed-notification.txt", "missed", 6, + &version, &error), error)) return 1; + vexfs_mount_visibility visibility_after_disconnect{}; + const vexfs_mount_status first_visibility_after_disconnect = + vexfs_mount_refresh_visibility(session, &visibility_after_disconnect, &error); + ++checks; + if (first_visibility_after_disconnect != VEXFS_MOUNT_OK && + first_visibility_after_disconnect != VEXFS_MOUNT_DATABASE_ERROR) + return Fail("visibility disconnect status", &error); + bool resync_invalidated_cache = first_visibility_after_disconnect == VEXFS_MOUNT_OK && + visibility_after_disconnect.external_commit == 1; + if (!resync_invalidated_cache) { + if (first_visibility_after_disconnect == VEXFS_MOUNT_DATABASE_ERROR) { + std::this_thread::sleep_for(std::chrono::milliseconds(1100)); + } + if (!CheckStatus("visibility reconnect next operation", + vexfs_mount_refresh_visibility( + session, &visibility_after_disconnect, &error), error)) return 1; + resync_invalidated_cache = visibility_after_disconnect.external_commit == 1; + } + if (!Expect(resync_invalidated_cache, + "reconnect invalidates notifications missed while disconnected")) return 1; bytes = {}; const vexfs_mount_status first_after_disconnect = vexfs_mount_stat(session, "/project/main.txt", &bytes, &error); diff --git a/docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md b/docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md index 43a6e621a9..6eda7d768b 100644 --- a/docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md +++ b/docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md @@ -4,15 +4,16 @@ - 所属产品：VexDB-Lite 的文件管理能力 VexFS - 日期：2026-07-27 - 分支：`feature/agent_files` -- 文档版本：3.3 +- 文档版本：3.4 - macOS 默认入口：本机 NFS gateway；FSKit 延后为可选原生增强 - 当前阶段：Phase 2 PostgreSQL 数据库合同和本机默认 NFS 验收已完成。PG VexFS spec 9/9、 - libpq runtime 119 项、manifest publish 专项 10 项和真实 macOS NFS 性能 18 项检查通过。 + libpq runtime 132 项、manifest publish 专项 10 项和真实 macOS NFS 性能 18 项检查通过。 PG writable handle 已使用基础 manifest + 64 KiB 脏块，发布直接复用未修改块并生成 `manifest-v1` 根哈希，不再在数据库进程中拼接完整文件。历史 FSKit 两 Mac、 macOS↔Linux、PG 16–19、备份、 - format v2 和真实 OpenCode 证据继续有效，但当前默认 NFS 源码对应的新公证包和第二台 Mac - NFS 复验还没有完成。下一步先做发行 Gate；数据库方向随后进入 DuckDB adapter。 + format v2 和真实 OpenCode 证据继续有效。当前源码的 Developer ID 候选包已经通过文档、 + 安装和解压签名检查，但正式 preview.38 仍需从干净提交生成并提交 Apple 公证；第二台 Mac + NFS 复验也没有完成。下一步只做发行 Gate；DuckDB adapter 已交由其他方向负责，不在本路线继续开发。 ## 0. 2026-07-24 路线调整 @@ -32,7 +33,7 @@ 1. **已完成**：冻结 NFS adapter 与 Workspace Engine 的窄接口，不复制数据库逻辑； 2. **已完成**：实现 localhost-only NFS gateway、mount/unmount/status/doctor； 3. **部分完成**：Bash、Git、hardlink、xattr、fsync、单机跨进程 `flock`/`fcntl`、轻量 npm/Cargo 和单次真实 OpenCode 已进入真机 eval； - 跨机器锁、长任务和 sleep/wake 尚待补齐；gateway `SIGKILL` + PG immediate restart 当前单轮 16 项、连续 3/3 轮已覆盖； + 跨机器锁、长任务和 sleep/wake 尚待补齐；gateway `SIGKILL` + PG immediate restart 历史完成连续 20/20 轮、累计 320 项检查，当前 fsid 身份修复源码完成 2/2 轮、每轮 18 项；真实 TCP 中途断线当前完成连续 4/4 轮、每轮 21 项； 4. **已完成首轮**：1,000 小文件为 205.643 files/s，同轮 APFS 为 16940.431 files/s； 5. **待完成**：生成不含 FSKit 授权前置的签名公证包，在干净 Mac 验证安装和系统提示； 6. **代码已完成，发行待 Gate**：默认 backend 已改为 NFS，FSKit 只在显式选择时使用。 @@ -67,7 +68,7 @@ VexFS FSKit extension。 - AppleDouble 隔离已进入 eval；后续每次 NFS 发布必须保留该断言； - 0.2 实现批量提交、写回合并和有界 metadata cache，再做同机 1,000/10,000 文件对照； -- 0.1 发布前仍需完成干净 Mac 安装、公证、长循环 crash/sleep-wake、跨机器锁边界与长期运行 Gate。 +- 0.1 发布前仍需完成当前源码公证、实际 sleep/wake、跨机器锁边界与长期运行 Gate。长循环 gateway crash 已完成。 ## 1. 这份文档解决什么问题 @@ -160,11 +161,11 @@ vexdb fs --workspace workspace snapshot restore before-refactor | 文件语义 | 文件、目录、四类时间戳、并发 append、进程锁、mode、symlink、xattr、hardlink、owner/group 元数据、便携 ACL 已进入数据库合同和测试；PG 已执行 ACL 和继承 | SQLite 尚未执行完整 ACL 授权；特殊文件不支持 | | 版本恢复 | 单文件版本、workspace commit、snapshot、diff、expected-head restore、原生备份、retention、显式分批 GC、live quota 和 format v2 逻辑导入导出已实现 | 自动维护、history/staging/index/total quota 和 DuckDB 导入端未完成 | | 长期校验 | `chunked-manifest-v1` 使用不可变 manifest、64 KiB 块、逐块 SHA-256 和有序块根哈希；deep check 逐块校验真实正文，publish 不再拼整文件 | 自动 repair、跨文件通用去重和按需标准整文件 SHA-256 命令不在当前范围 | -| macOS | 默认 NFS 已连接 SQLite 和 PG runtime；PG 14/14 场景、252 项检查通过，SQLite 完整真机 eval、OpenCode、package smoke、hardlink、COMMIT/fsync、Git、单机锁、npm/Cargo、重挂载和快照恢复均有证据；PG strict crash 当前单轮 16 项、连续 3/3 轮通过；NFSv3 xattr 明确不支持且不生成 `._*`；macOS 13.0 deployment target 构建通过；既有 FSKit 仍保留历史真机证据 | 当前默认 NFS 源码尚未从干净提交生成公证包，也未在 macOS 13–15 或未启用 FSKit 的干净 Mac 验证；长循环 crash/sleep-wake、跨机器锁、长期 Agent 和 x86_64 待补 | +| macOS | 默认 NFS 已连接 SQLite 和 PG runtime；PG 14/14 场景、252 项检查通过，SQLite 完整真机 eval、OpenCode、package smoke、hardlink、COMMIT/fsync、Git、单机锁、npm/Cargo、重挂载和快照恢复均有证据；PG strict crash 历史连续 20/20，当前 fsid 身份修复源码 2/2 轮、每轮 18 项通过；NFSv3 xattr 明确不支持且不生成 `._*`；macOS 13.0 deployment target 构建通过；既有 FSKit 仍保留历史真机证据 | 当前源码只有 Developer ID 候选包，尚未从干净提交生成公证包，也未在 macOS 13–15 或未启用 FSKit 的干净 Mac 验证；实际 sleep/wake、跨机器锁、长期 Agent 和 x86_64 待补 | | Linux | libfuse3 真实挂载已通过 root 和 uid 1000 各 9 组；时间戳、并发 append、进程锁、打开文件生命周期、强制卸载和 helper 崩溃恢复已验证；异常撤销后底层目录保持 0500，显式卸载恢复 0700；x86_64/AArch64 manylinux 安装包已在对应架构真机完成无 root 安装回归 | 更多干净发行版、长期运行和真实挂载安装回归仍不足 | | Windows | 只有平台边界和规划 | WinFsp adapter、安装签名、路径/ACL/SID 合同均未实现 | -| 远程共享 | PostgreSQL 已通过多 gateway、macOS FSKit、Linux FUSE、数据库重启、macOS↔Linux 47 项、两台 Mac 文件/快照 50 项、双 Mac 串行真实 OpenCode 7 + 9 + 21 项和故障恢复 25 项；当前默认 NFS 又完成本机双 gateway 15 项，PG `pg_trgm` 索引已实现 | 当前默认 NFS 源码对应的公证包和第二台 Mac NFS 复验尚未完成；同时覆盖同一文件只报告版本冲突，不自动合并 | -| 性能规模 | SQLite 直连已完成 10 万文件；PG 16 MiB 文件修改 4 KiB 的组合发布从 589.357 ms 降到中位 10.289 ms，publish-only 两次正式复跑中位 4.020/9.828 ms；逐文件后台事务后的 PG NFS 两轮 8 MiB 写 46.430～46.516 MiB/s、随机覆盖 386.139～482.743 ops/s；strict fsync → gateway SIGKILL → PG immediate restart 当前 16 项、连续 3/3 轮通过；服务端已提交但客户端不读结果的 generation 重试通过；1 GiB 容器 `oom_kill=0` | 仍需补远程 ARM 同版基准、1 万文件、长 Agent、冷读、长循环 crash 和真实网络中途断线 | +| 远程共享 | PostgreSQL 已通过多 gateway、macOS FSKit、Linux FUSE、数据库重启、macOS↔Linux 47 项、两台 Mac 文件/快照 50 项、双 Mac 串行真实 OpenCode 7 + 9 + 21 项和故障恢复 25 项；当前默认 NFS 又完成本机双 gateway 15 项和当前源码真实 TCP 中途断线连续 4/4 轮、每轮 21 项，PG `pg_trgm` 索引已实现 | 当前默认 NFS 源码对应的公证包和第二台 Mac NFS 复验尚未完成；同时覆盖同一文件只报告版本冲突，不自动合并 | +| 性能规模 | SQLite 直连已完成 10 万文件；PG 16 MiB 文件修改 4 KiB 的组合发布从 589.357 ms 降到中位 10.289 ms，publish-only 两次正式复跑中位 4.020/9.828 ms；逐文件后台事务后的 PG NFS 两轮 8 MiB 写 46.430～46.516 MiB/s、随机覆盖 386.139～482.743 ops/s；strict fsync → gateway SIGKILL → PG immediate restart 历史连续 20/20，当前修复源码 2/2；真实 TCP 黑洞重连最长约 5.02 秒有界返回并连续 4/4 轮恢复；服务端已提交但客户端不读结果的 generation 重试通过；1 GiB 容器 `oom_kill=0` | 仍需补远程 ARM 同版基准、1 万文件、长 Agent、冷读和长时间运行 | “合同中已保存”不等于“所有平台已经完整执行”。例如 owner/group 和便携 ACL 已经可以保存、读取和恢复，但身份认证、权限判断和各系统的原生 ACL 映射仍属于后续阶段。 @@ -239,7 +240,8 @@ SQLite 证据。当前 format v2 已支持 SQLite 和 PostgreSQL，DuckDB adapte - 最终 quick eval 为 57 passed、0 failed、18 个平台条件 skipped、2,863 checks，耗时 24.824 秒； SQLite spec、runtime smoke、VexFS CLI smoke 和 VexDB unified CLI smoke 全部通过。 - 2026-07-27 根哈希规则已统一到 SQLite、PostgreSQL、共享 C++ 和 format v2；SQLite 全量 spec - 30/30、PG VexFS spec 9/9、PG runtime 131 checks、SQLite ↔ PG 双向归档 6 checks 全部通过。 + 30/30、PG VexFS spec 9/9、PG runtime 132 checks、SQLite ↔ PG 双向归档 6 checks 全部通过； + SQLite `diff` 的旧整文件哈希校验已改为清单根校验，并加入版本差异回归。 当前仍没有跨文件通用去重、自动 repair、自动 GC 或在线 VACUUM。完整证据见 `docs/reports/2026-07-23_vexfs-chunk-manifest-v2.md`。 @@ -312,7 +314,7 @@ SQLite 证据。当前 format v2 已支持 SQLite 和 PostgreSQL，DuckDB adapte 当前完成证据： - 适配器版本为 `0.4.0-alpha.1`，9 个 PG VexFS spec、151 项公共合同/并发 eval、32 项 CLI runtime、 - 119 项当前 libpq runtime 合同、 + 132 项当前 libpq runtime 合同、 数据库重启恢复、16/17/18/19 版本矩阵均通过； - ACL、真实 role、跨 gateway 锁和缓存失效已经进入 PostgreSQL 合同；审计精确记录 `session_user`/role OID、workspace、commit、path/inode 和 before/after version，workspace @@ -341,11 +343,23 @@ SQLite 证据。当前 format v2 已支持 SQLite 和 PostgreSQL，DuckDB adapte 正常后台 claimed 发布已改成独立 publisher 连接上的逐文件事务，每个文件提交后立即释放 workspace 行锁；直接 SQL 原子批次和卸载后清理批次继续保留。千文件不再出现修复前的 60 秒 `rg` 超时，正式 file version/commit/request 分别为 1.002/1.066/3.013 每文件。 -- strict 模式已完成自动化断电等价 Gate：`fsync` 返回后校验 gateway PID 身份、SIGKILL gateway、 - PG immediate restart、重挂载读取、deep check、版本和 staging 当前共 16 项，连续 3/3 轮通过； +- strict 模式已完成自动化断电等价 Gate：`fsync` 返回后校验 gateway PID、程序路径、进程启动身份、 + mount source 和 fsid；伪造同程序 PID 的旧记录不得终止真实 gateway，篡改 fsid 必须拒绝卸载；随后 + SIGKILL gateway、PG immediate restart、重挂载读取、deep check、版本和 staging。当前每轮 21 项， 1 GiB 容器 `oom_kill=0`。 +- loopback TCP relay 同时覆盖两类 blackhole：保留已经建立的主连接和 publisher 连接但停止转发时， + 文件操作约 5 秒有界失败；真实关闭两条连接并让新连接无响应时，重连也约 5 秒有界失败。恢复后 + metadata、前台写入和后台 publisher 都会重连；断线期间由另一连接创建的文件会在恢复后立即使 + NFS 目录缓存失效。当前每轮 31 项，`pending_handles=0`、 + `staging_bytes=0`、deep check 和 `oom_kill=0` 均通过。 +- PG visibility 在主连接重建后保留一次强制重同步标记；即使断线期间的 `LISTEN` 通知已经丢失， + 下一次成功读取 workspace head/cache generation 也会清理 NFS stat 和目录缓存。libpq runtime 已增加 + “断线期间 peer 修改”回归，当前为 134 项。 - libpq runtime 已验证“服务端完成 commit、客户端不读取结果并丢弃连接”的未知结果窗口；同一 handle/generation 重试仍返回版本 1，数据库只有一个 file version。 +- 当前源码已在 Linux AArch64 的 Debian 容器内以 1 GiB 内存限制、单线程完成全量 Release + 编译，覆盖 SQLite loadable/static、VexFS runtime、CLI、FUSE helper 和全部 smoke 目标； + Linux eval 镜像已补齐实际构建依赖 `libboost-dev`。 完成条件： @@ -356,8 +370,8 @@ SQLite 证据。当前 format v2 已支持 SQLite 和 PostgreSQL，DuckDB adapte 当前数据库合同和本机默认 NFS 完成条件已经由本机、Linux、跨系统、历史双机和故障 Gate 证明，Phase 2 状态为“功能与本机验收完成”。从当前干净提交生成并公证新包、在第二台 Mac -按默认 NFS 重跑共享 workspace，仍是发行任务，但不再被误写成 PostgreSQL 文件合同缺口； -数据库方向下一步进入 DuckDB adapter，Windows mount adapter 按独立平台路线推进。 +按默认 NFS 重跑共享 workspace，仍是发行任务，但不再被误写成 PostgreSQL 文件合同缺口。 +DuckDB adapter 已由其他方向负责，不在本路线继续；本路线只保留发行 Gate 和独立的 Windows 平台计划。 ### Phase 3：Windows 与跨系统预览 — 未开始 @@ -422,10 +436,11 @@ SQLite 证据。当前 format v2 已支持 SQLite 和 PostgreSQL，DuckDB adapte restore；CLI 正常卸载、恢复并原位重挂载；FSKit 在 `unmount()` / `deactivate()` 关闭 session，在 `activate()` 重开并清理旧缓存。M1 实测 lease 约 2.832 ms 释放，恢复后内容 正确且无残留挂载。 -5. **P0（历史发行证明已完成，当前源码待重做）：** preview.22 已从干净提交签名、公证并 - 通过真实 Gate；当前 preview.37 没有 stapled 票据，不能继承旧包的公证结论。必须使用 - `vexdb-lite-notary` profile 生成 preview.38，并再次要求 Apple `Accepted`、staple、 - Gatekeeper、安装后 doctor 和快照恢复真机回归全部通过。 +5. **P0（候选包已完成，正式公证待做）：** preview.22 已从干净提交签名、公证并 + 通过真实 Gate；当前源码的 `preview.38-candidate` 已通过 Developer ID 签名、102 项文档 smoke、 + 19 项统一安装 eval、解压签名链和 CLI smoke。正式 preview.38 必须先形成干净提交，再使用 + `vexdb-lite-notary` profile 生成，并再次要求 Apple `Accepted`、staple、Gatekeeper、安装后 + doctor 和快照恢复真机回归全部通过。 6. **P0（已完成）：把本次真实挂载快照恢复脚本变成正式 eval。** 覆盖活动 mount 直接 restore 被拒、一键卸载/恢复/重挂载、lease 立即释放、旧缓存清理、失败回滚、多 workspace 定位和最终无残留挂载，避免以后只靠 `/tmp` 手工脚本发现回归。 @@ -436,13 +451,14 @@ SQLite 证据。当前 format v2 已支持 SQLite 和 PostgreSQL，DuckDB adapte 9. **P1（已完成 PG Alpha 规模和 Agent 验证）：** PostgreSQL 已完成有界性能、1 千文件、 真实工具链和双 Mac 串行 OpenCode 完整闭环：第一台创建、第二台继承修改、第一台回看， 两个 workspace 快照和一键恢复均通过。所有大测试继续保留 RSS 上限和低并发。 -10. **当前唯一动作：** 把已经通过自动测试、本机真挂载和双机故障的工作树形成干净提交， - 生成并公证 `preview.38`，安装后再跑双机安装 Gate。FSKit 当前已经启用，不需要重复处理 - 旧 module；局域网直连另需第二台 Mac 允许 VexDB Lite 使用“本地网络”。 +10. **当前唯一动作：** 用户确认提交后，把已经通过自动测试、本机真挂载、长循环 crash、真实 + TCP 断线和候选包 Gate 的工作树形成干净提交，生成并公证 `preview.38`，安装后再跑双机安装 + Gate。实际 sleep/wake 仍需交互式管理员授权；局域网直连另需第二台 Mac 在线并允许 VexDB Lite + 使用“本地网络”。 PostgreSQL Phase 2 的 format v2、ACL、审计、libpq HostStore、远程 mount、备份和故障恢复 -已经完成。当前不再继续扩展 PG 代码；先关闭两台 Mac 对最终签名版本的系统授权验收点。之后的产品开发只在 -Windows WinFsp 与 DuckDB HostStore 中二选一，不同时铺开。SQLite 的 10 万文件和小文件性能 +已经完成。当前不再继续扩展 PG 代码；先关闭两台 Mac 对最终签名版本的系统授权验收点。DuckDB +adapter 已明确不由本路线处理，后续平台开发只保留 Windows WinFsp。SQLite 的 10 万文件和小文件性能 仍是独立的 Phase 1 发布优化，不再阻塞 PostgreSQL Alpha 功能结论。 ### N1. 共用 mount 一致性 eval diff --git a/tests/eval/vexfs/Dockerfile.linux-fuse b/tests/eval/vexfs/Dockerfile.linux-fuse index 3acb81ead1..1fac26b117 100644 --- a/tests/eval/vexfs/Dockerfile.linux-fuse +++ b/tests/eval/vexfs/Dockerfile.linux-fuse @@ -7,6 +7,7 @@ RUN apt-get update \ cmake \ fuse3 \ git \ + libboost-dev \ libfuse3-dev \ libpq-dev \ postgresql-client \ diff --git a/tests/eval/vexfs/README.md b/tests/eval/vexfs/README.md index a7d11b265c..f47d2dd197 100644 --- a/tests/eval/vexfs/README.md +++ b/tests/eval/vexfs/README.md @@ -101,12 +101,27 @@ VEXDB_PG_CONTAINER=vexdb_pg19-vexfs-dev \ VEXDB_PG_DSN=postgresql://postgres@127.0.0.1:5434/test \ bash tests/eval/vexfs/run_pg_restart_recovery.sh -# 断电等价 Gate：strict 写入并 fsync、SIGKILL gateway、PG immediate restart、 -# 重挂载读取与 deep check；只写一个小文件，容器 memory.max 必须不超过 1 GiB。 +# 断电等价 Gate：strict 写入并 fsync、校验 mount source/fsid 和 gateway 进程 +# 启动身份、SIGKILL gateway、PG immediate restart、重挂载读取与 deep check； +# 只写一个小文件，容器 memory.max 必须不超过 1 GiB。 VEXDB_PG_CONTAINER=vexdb_pg19-vexfs-dev \ VEXFS_EVAL_MOUNT_CLI=/absolute/path/to/current/vexdb \ bash tests/eval/vexfs/run_pg_strict_crash_recovery.sh +# 连续 crash soak；首轮失败立即停止，默认 20 轮、最多 100 轮。 +# 每次运行自动使用唯一数据库名，数据库已存在时拒绝执行，不会预先 drop。 +VEXDB_PG_CONTAINER=vexdb_pg19-vexfs-dev \ +VEXFS_EVAL_MOUNT_CLI=/absolute/path/to/current/vexdb \ +VEXFS_PG_STRICT_CRASH_ROUNDS=20 \ + bash tests/eval/vexfs/run_pg_strict_crash_soak.sh + +# 先保留 gateway 的主 libpq/publisher TCP 连接但停止双向转发，再真实关闭两条连接 +# 并让新连接进入 blackhole。验证两类故障都在约 5 秒内有界失败、10 秒内恢复， +# 同时覆盖断线期间 visibility 重同步、后台 publisher、唯一数据库、deep check 和 OOM。 +VEXDB_PG_CONTAINER=vexdb_pg19-vexfs-dev \ +VEXFS_EVAL_MOUNT_CLI=/absolute/path/to/current/vexdb \ + bash tests/eval/vexfs/run_pg_network_cut_recovery.sh + # 完整数据库 pg_dump/pg_restore VEXDB_PG_CONTAINER=vexdb_pg19-vexfs-dev \ bash tests/eval/vexfs/run_pg_backup_restore.sh diff --git a/tests/eval/vexfs/pg_tcp_cut_proxy.py b/tests/eval/vexfs/pg_tcp_cut_proxy.py new file mode 100755 index 0000000000..1e26fd9517 --- /dev/null +++ b/tests/eval/vexfs/pg_tcp_cut_proxy.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python3 +"""Loopback TCP relay with cut, new-connection blackhole and live-link stall.""" + +from __future__ import annotations + +import argparse +import json +import os +import signal +import socket +import threading +import time +from pathlib import Path + + +class Relay: + def __init__(self, target_host: str, target_port: int, ack_file: Path, + status_file: Path) -> None: + self.target = (target_host, target_port) + self.ack_file = ack_file + self.status_file = status_file + self.lock = threading.Lock() + self.pairs: set[tuple[socket.socket, socket.socket]] = set() + self.blackhole_clients: set[socket.socket] = set() + self.stopping = threading.Event() + self.cut_requested = threading.Event() + self.toggle_blackhole_requested = threading.Event() + self.toggle_existing_stall_requested = threading.Event() + self.blackhole = threading.Event() + self.existing_stall = threading.Event() + self.cut_count = 0 + self.accepted = 0 + + def snapshot(self) -> dict[str, int | str]: + with self.lock: + return { + "accepted": self.accepted, + "pairs": len(self.pairs), + "blackhole_clients": len(self.blackhole_clients), + "mode": ( + "blackhole" if self.blackhole.is_set() + else "established-blackhole" if self.existing_stall.is_set() + else "relay" + ), + } + + @staticmethod + def write_json(path: Path, value: dict[str, int | str]) -> None: + temporary = path.with_suffix(f".tmp-{os.getpid()}") + temporary.write_text(json.dumps(value) + "\n") + temporary.replace(path) + + def write_status(self) -> None: + self.write_json(self.status_file, self.snapshot()) + + def close_pair(self, pair: tuple[socket.socket, socket.socket]) -> None: + with self.lock: + self.pairs.discard(pair) + for stream in pair: + try: + stream.shutdown(socket.SHUT_RDWR) + except OSError: + pass + stream.close() + + def copy(self, source: socket.socket, destination: socket.socket, + pair: tuple[socket.socket, socket.socket]) -> None: + try: + while not self.stopping.is_set(): + if self.existing_stall.is_set(): + time.sleep(0.01) + continue + try: + data = source.recv(64 * 1024) + except (TimeoutError, socket.timeout): + continue + if not data: + break + destination.sendall(data) + except OSError: + pass + finally: + self.close_pair(pair) + + def accept(self, client: socket.socket) -> None: + if self.blackhole.is_set(): + with self.lock: + self.blackhole_clients.add(client) + self.accepted += 1 + self.write_status() + return + try: + server = socket.create_connection(self.target, timeout=5) + client.settimeout(0.1) + server.settimeout(0.1) + except OSError: + client.close() + return + pair = (client, server) + with self.lock: + self.pairs.add(pair) + self.accepted += 1 + self.write_status() + threading.Thread(target=self.copy, args=(client, server, pair), daemon=True).start() + threading.Thread(target=self.copy, args=(server, client, pair), daemon=True).start() + + def close_blackhole_clients(self) -> int: + with self.lock: + clients = list(self.blackhole_clients) + self.blackhole_clients.clear() + for client in clients: + client.close() + return len(clients) + + def cut(self) -> None: + with self.lock: + pairs = list(self.pairs) + for pair in pairs: + self.close_pair(pair) + blocked = self.close_blackhole_clients() + self.cut_count += 1 + value = self.snapshot() + value.update({"cuts": self.cut_count, + "connections": len(pairs) + blocked}) + self.write_json(self.ack_file, value) + self.write_status() + + def toggle_blackhole(self) -> None: + if self.blackhole.is_set(): + self.blackhole.clear() + closed = self.close_blackhole_clients() + else: + self.blackhole.set() + with self.lock: + pairs = list(self.pairs) + for pair in pairs: + self.close_pair(pair) + closed = len(pairs) + value = self.snapshot() + value.update({"cuts": self.cut_count, "connections": closed}) + self.write_json(self.ack_file, value) + self.write_status() + + def toggle_existing_stall(self) -> None: + if self.existing_stall.is_set(): + self.existing_stall.clear() + else: + self.existing_stall.set() + value = self.snapshot() + value.update({"cuts": self.cut_count, "connections": len(self.pairs)}) + self.write_json(self.ack_file, value) + self.write_status() + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--target-host", required=True) + parser.add_argument("--target-port", required=True, type=int) + parser.add_argument("--ready-file", required=True, type=Path) + parser.add_argument("--ack-file", required=True, type=Path) + parser.add_argument("--status-file", required=True, type=Path) + arguments = parser.parse_args() + + relay = Relay(arguments.target_host, arguments.target_port, arguments.ack_file, + arguments.status_file) + listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + listener.bind(("127.0.0.1", 0)) + listener.listen(32) + listener.settimeout(0.1) + + signal.signal(signal.SIGUSR1, lambda *_: relay.cut_requested.set()) + signal.signal(signal.SIGUSR2, lambda *_: relay.toggle_blackhole_requested.set()) + signal.signal(signal.SIGHUP, lambda *_: relay.toggle_existing_stall_requested.set()) + signal.signal(signal.SIGTERM, lambda *_: relay.stopping.set()) + temporary = arguments.ready_file.with_suffix(f".tmp-{os.getpid()}") + temporary.write_text(json.dumps({"pid": os.getpid(), + "port": listener.getsockname()[1]}) + "\n") + temporary.replace(arguments.ready_file) + relay.write_status() + + try: + while not relay.stopping.is_set(): + if relay.cut_requested.is_set(): + relay.cut_requested.clear() + relay.cut() + if relay.toggle_blackhole_requested.is_set(): + relay.toggle_blackhole_requested.clear() + relay.toggle_blackhole() + if relay.toggle_existing_stall_requested.is_set(): + relay.toggle_existing_stall_requested.clear() + relay.toggle_existing_stall() + try: + client, _ = listener.accept() + except (TimeoutError, socket.timeout): + continue + except OSError: + if relay.stopping.is_set(): + break + raise + relay.accept(client) + finally: + relay.stopping.set() + listener.close() + relay.cut() + time.sleep(0.05) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/eval/vexfs/run_pg_network_cut_recovery.sh b/tests/eval/vexfs/run_pg_network_cut_recovery.sh new file mode 100755 index 0000000000..58d0c4b65a --- /dev/null +++ b/tests/eval/vexfs/run_pg_network_cut_recovery.sh @@ -0,0 +1,430 @@ +#!/bin/bash +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" +CONTAINER="${VEXDB_PG_CONTAINER:-vexdb_pg19-vexfs-dev}" +CLI="${VEXFS_EVAL_MOUNT_CLI:-$ROOT/vexdb_sqlite/build/vexdb}" +PYTHON="${VEXDB_LITE_PYTHON:-/usr/bin/python3}" +TMP="$(mktemp -d "${TMPDIR:-/tmp}/vexfs-pg-network-cut.XXXXXX")" +TMP="$(cd "$TMP" && pwd -P)" +RUN_ID="${TMP##*.}" +DATABASE="${VEXDB_PG_DATABASE:-vexfs_network_cut_${RUN_ID}}" +WORKSPACE="${VEXDB_PG_WORKSPACE:-network-cut-${RUN_ID}}" +HOME_DIR="$TMP/home" +MOUNT_POINT="$TMP/mount" +READY_FILE="$TMP/proxy-ready.json" +ACK_FILE="$TMP/proxy-cut.json" +STATUS_FILE="$TMP/proxy-status.json" +PROXY_PID="" +MOUNTED=0 +DATABASE_CREATED=0 +CHECKS=0 +FS=() + +fail() { echo "$*" >&2; exit 1; } + +monotonic_ms() { + "$PYTHON" -c 'import time; print(time.monotonic_ns() // 1000000)' +} + +bounded_missing_probe() { + "$PYTHON" - "$1" "$2" <<'PY' +import errno +import os +import signal +import sys +import time + +path = sys.argv[1] +timeout_ms = int(sys.argv[2]) +started = time.monotonic_ns() +pid = os.fork() +if pid == 0: + try: + os.stat(path) + except OSError as error: + os._exit(0 if error.errno == errno.ENOENT else 2) + os._exit(1) + +status = 124 +deadline = time.monotonic() + timeout_ms / 1000 +while time.monotonic() < deadline: + waited, child_status = os.waitpid(pid, os.WNOHANG) + if waited == pid: + status = os.waitstatus_to_exitcode(child_status) + break + time.sleep(0.02) +else: + os.kill(pid, signal.SIGKILL) + os.waitpid(pid, 0) +print((time.monotonic_ns() - started) // 1000000) +raise SystemExit(status) +PY +} + +is_mounted() { + /sbin/mount | grep -F " on $MOUNT_POINT (" >/dev/null 2>&1 +} + +cleanup() { + local status=$? + trap - EXIT INT TERM + if [ -n "$PROXY_PID" ] && kill -0 "$PROXY_PID" >/dev/null 2>&1 && + [ -f "$STATUS_FILE" ]; then + proxy_mode="$("$PYTHON" -c 'import json,sys; print(json.load(open(sys.argv[1]))["mode"])' "$STATUS_FILE" 2>/dev/null || true)" + if [ "$proxy_mode" = blackhole ]; then + kill -USR2 "$PROXY_PID" >/dev/null 2>&1 || true + sleep 0.1 + elif [ "$proxy_mode" = established-blackhole ]; then + kill -HUP "$PROXY_PID" >/dev/null 2>&1 || true + sleep 0.1 + fi + fi + if [ "$MOUNTED" -eq 1 ]; then + HOME="$HOME_DIR" "${FS[@]}" unmount --force "$MOUNT_POINT" >/dev/null 2>&1 || true + fi + if is_mounted; then + /sbin/umount -f "$MOUNT_POINT" >/dev/null 2>&1 || true + fi + if [ "${#FS[@]}" -ne 0 ]; then + HOME="$HOME_DIR" "${FS[@]}" unmount --force "$MOUNT_POINT" \ + >/dev/null 2>&1 || true + fi + if [ -n "$PROXY_PID" ]; then + kill "$PROXY_PID" >/dev/null 2>&1 || true + wait "$PROXY_PID" >/dev/null 2>&1 || true + fi + if [ "$DATABASE_CREATED" -eq 1 ]; then + docker exec "$CONTAINER" dropdb -U postgres --if-exists --force "$DATABASE" \ + >/dev/null 2>&1 || true + fi + if [ "$status" -eq 0 ] && ! is_mounted; then + rm -rf "$TMP" + elif [ "$status" -ne 0 ]; then + echo "network cut 失败现场保留在：$TMP" >&2 + fi + exit "$status" +} +trap cleanup EXIT INT TERM + +[ "$(uname -s)" = Darwin ] || fail "该 Gate 只在 macOS NFS 上运行" +[ -x "$CLI" ] || fail "找不到当前 vexdb CLI：$CLI" +[ -x "$PYTHON" ] || fail "找不到 Python：$PYTHON" +case "$DATABASE:$WORKSPACE" in + *[!A-Za-z0-9_:-]*) fail "数据库和 workspace 只能包含字母、数字、下划线和连字符" ;; +esac +(( ${#DATABASE} <= 63 )) || fail "数据库名不能超过 63 个字符：$DATABASE" + +MEMORY_MAX="$(docker exec "$CONTAINER" cat /sys/fs/cgroup/memory.max)" +[[ "$MEMORY_MAX" =~ ^[0-9]+$ ]] || fail "PG 容器必须设置有限 memory.max" +(( MEMORY_MAX <= 1073741824 )) || fail "PG 容器 memory.max 必须不大于 1 GiB" +OOM_BEFORE="$(docker exec "$CONTAINER" awk '$1=="oom_kill" {print $2}' \ + /sys/fs/cgroup/memory.events)" +HOST_BINDING="$(docker port "$CONTAINER" 5432/tcp | sed -n '1p')" +HOST_PORT="${HOST_BINDING##*:}" +[[ "$HOST_PORT" =~ ^[0-9]+$ ]] || fail "无法推导 PG 端口" + +mkdir -p "$HOME_DIR" "$MOUNT_POINT" +docker exec "$CONTAINER" createdb -U postgres "$DATABASE" +DATABASE_CREATED=1 +docker exec "$CONTAINER" psql -U postgres -d "$DATABASE" -X -q \ + -v ON_ERROR_STOP=1 -c 'CREATE EXTENSION vexdb_lite;' >/dev/null + +"$PYTHON" "$ROOT/tests/eval/vexfs/pg_tcp_cut_proxy.py" \ + --target-host 127.0.0.1 --target-port "$HOST_PORT" \ + --ready-file "$READY_FILE" --ack-file "$ACK_FILE" \ + --status-file "$STATUS_FILE" & +PROXY_PID=$! +for _ in $(seq 1 100); do + [ -f "$READY_FILE" ] && break + kill -0 "$PROXY_PID" >/dev/null 2>&1 || fail "TCP cut proxy 提前退出" + sleep 0.05 +done +[ -f "$READY_FILE" ] || fail "TCP cut proxy 没有就绪" +PROXY_PORT="$("$PYTHON" -c 'import json,sys; print(json.load(open(sys.argv[1]))["port"])' "$READY_FILE")" +[[ "$PROXY_PORT" =~ ^[0-9]+$ ]] || fail "无效 proxy 端口" + +DSN="host=127.0.0.1 port=$PROXY_PORT dbname=$DATABASE user=postgres" +FS=("$CLI" fs --backend pg --dsn "$DSN" --workspace "$WORKSPACE") +HOME="$HOME_DIR" "${FS[@]}" setup >/dev/null +HOME="$HOME_DIR" VEXFS_NFS_STRICT_DURABILITY=1 \ + "${FS[@]}" mount "$MOUNT_POINT" >/dev/null +MOUNTED=1 + +"$PYTHON" - "$MOUNT_POINT/before-cut.txt" before-cut <<'PY' +import os, sys +fd = os.open(sys.argv[1], os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o644) +try: + os.write(fd, (sys.argv[2] + "\n").encode()) + os.fsync(fd) +finally: + os.close(fd) +PY +CHECKS=$((CHECKS + 1)) +# Prime root directory/stat caches before any disconnect. A peer file created +# while LISTEN is offline must become visible after reconnect. +ls "$MOUNT_POINT" >/dev/null + +# Keep the two established libpq TCP sessions open but stop forwarding bytes. +# This is different from rejecting a reconnect: synchronous PQexec used to hold +# the whole NFS runtime forever in this state. +rm -f "$ACK_FILE" +kill -HUP "$PROXY_PID" +for _ in $(seq 1 100); do + if [ -f "$ACK_FILE" ] && + [ "$("$PYTHON" -c 'import json,sys; print(json.load(open(sys.argv[1]))["mode"])' "$ACK_FILE")" = established-blackhole ]; then + break + fi + sleep 0.05 +done +[ -f "$ACK_FILE" ] && +[ "$("$PYTHON" -c 'import json,sys; print(json.load(open(sys.argv[1]))["mode"])' "$ACK_FILE")" = established-blackhole ] || + fail "TCP cut proxy 没有进入 established blackhole 模式" +ESTABLISHED_CONNECTIONS="$("$PYTHON" -c 'import json,sys; print(json.load(open(sys.argv[1]))["connections"])' "$ACK_FILE")" +(( ESTABLISHED_CONNECTIONS >= 2 )) || \ + fail "established blackhole 没有保留主连接和 publisher 连接" +set +e +ESTABLISHED_BLACKHOLE_MS="$(bounded_missing_probe \ + "$MOUNT_POINT/established-blackhole-probe" 7000)" +ESTABLISHED_BLACKHOLE_STATUS=$? +set -e +[ "$ESTABLISHED_BLACKHOLE_STATUS" -ne 124 ] || \ + fail "已建立的 PG TCP 黑洞超过 7 秒，挂载请求仍被阻塞" +[ "$ESTABLISHED_BLACKHOLE_STATUS" -ne 0 ] || \ + fail "已建立的 PG TCP 黑洞期间挂载读取不应成功" +(( ESTABLISHED_BLACKHOLE_MS >= 4000 && ESTABLISHED_BLACKHOLE_MS <= 6500 )) || \ + fail "已建立连接黑洞失败边界异常：${ESTABLISHED_BLACKHOLE_MS}ms" + +rm -f "$ACK_FILE" +kill -HUP "$PROXY_PID" +for _ in $(seq 1 100); do + if [ -f "$ACK_FILE" ] && + [ "$("$PYTHON" -c 'import json,sys; print(json.load(open(sys.argv[1]))["mode"])' "$ACK_FILE")" = relay ]; then + break + fi + sleep 0.05 +done +[ -f "$ACK_FILE" ] && +[ "$("$PYTHON" -c 'import json,sys; print(json.load(open(sys.argv[1]))["mode"])' "$ACK_FILE")" = relay ] || + fail "TCP cut proxy 没有退出 established blackhole 模式" +ESTABLISHED_RECOVERED=0 +for _ in $(seq 1 40); do + set +e + bounded_missing_probe "$MOUNT_POINT/established-reconnect-probe" 1000 >/dev/null + established_probe_status=$? + set -e + if [ "$established_probe_status" -eq 0 ]; then + ESTABLISHED_RECOVERED=1 + break + fi + sleep 0.25 +done +[ "$ESTABLISHED_RECOVERED" -eq 1 ] || \ + fail "已建立连接黑洞解除后挂载没有恢复" +[ "$(cat "$MOUNT_POINT/before-cut.txt")" = before-cut ] || \ + fail "已建立连接黑洞恢复后的基线内容错误" +CHECKS=$((CHECKS + 7)) + +rm -f "$ACK_FILE" +kill -USR1 "$PROXY_PID" +for _ in $(seq 1 100); do + [ -f "$ACK_FILE" ] && break + sleep 0.05 +done +[ -f "$ACK_FILE" ] || fail "TCP cut proxy 没有确认断线" +CUT_CONNECTIONS="$("$PYTHON" -c 'import json,sys; print(json.load(open(sys.argv[1]))["connections"])' "$ACK_FILE")" +(( CUT_CONNECTIONS >= 2 )) || fail "断线时没有覆盖 runtime 的主连接和 publisher 连接" +CHECKS=$((CHECKS + 1)) +docker exec "$CONTAINER" psql -U postgres -d "$DATABASE" -X -q \ + -v ON_ERROR_STOP=1 -c \ + "SELECT vexfs_write('$WORKSPACE','/peer-during-cut.txt',convert_to('peer-during-cut','UTF8'));" \ + >/dev/null +CHECKS=$((CHECKS + 1)) + +sleep 1.2 +rm -f "$ACK_FILE" +kill -USR2 "$PROXY_PID" +for _ in $(seq 1 100); do + if [ -f "$ACK_FILE" ] && + [ "$("$PYTHON" -c 'import json,sys; print(json.load(open(sys.argv[1]))["mode"])' "$ACK_FILE")" = blackhole ]; then + break + fi + sleep 0.05 +done +[ -f "$ACK_FILE" ] || fail "TCP cut proxy 没有进入 blackhole 模式" +set +e +DISCONNECT_DETECT_MS="$(bounded_missing_probe "$MOUNT_POINT/disconnect-detect" 7000)" +DISCONNECT_DETECT_STATUS=$? +set -e +[ "$DISCONNECT_DETECT_STATUS" -ne 124 ] || fail "第一次 PG blackhole 请求超过 7 秒" +[ "$DISCONNECT_DETECT_STATUS" -ne 0 ] || fail "断线检测期间挂载读取不应成功" +set +e +BLACKHOLE_ELAPSED_MS="$(bounded_missing_probe "$MOUNT_POINT/blackhole-probe" 7000)" +BLACKHOLE_STATUS=$? +set -e +[ "$BLACKHOLE_STATUS" -ne 124 ] || fail "PG 重连超过 7 秒，挂载请求仍被阻塞" +[ "$BLACKHOLE_STATUS" -ne 0 ] || fail "blackhole 期间挂载读取不应成功" +(( DISCONNECT_DETECT_MS > BLACKHOLE_ELAPSED_MS )) && \ + BLACKHOLE_MAX_MS=$DISCONNECT_DETECT_MS || BLACKHOLE_MAX_MS=$BLACKHOLE_ELAPSED_MS +(( BLACKHOLE_MAX_MS >= 4000 )) || \ + fail "blackhole 没有进入真实 PG 重连等待：first=${DISCONNECT_DETECT_MS}ms second=${BLACKHOLE_ELAPSED_MS}ms" +(( DISCONNECT_DETECT_MS <= 6500 )) || \ + fail "第一次 PG blackhole 失败返回过慢：${DISCONNECT_DETECT_MS}ms" +(( BLACKHOLE_ELAPSED_MS <= 6500 )) || \ + fail "PG blackhole 失败返回过慢：${BLACKHOLE_ELAPSED_MS}ms" +CHECKS=$((CHECKS + 4)) + +rm -f "$ACK_FILE" +kill -USR2 "$PROXY_PID" +for _ in $(seq 1 100); do + if [ -f "$ACK_FILE" ] && + [ "$("$PYTHON" -c 'import json,sys; print(json.load(open(sys.argv[1]))["mode"])' "$ACK_FILE")" = relay ]; then + break + fi + sleep 0.05 +done +[ -f "$ACK_FILE" ] || fail "TCP cut proxy 没有恢复 relay 模式" + +read_recovered=0 +RECOVERY_STARTED_MS="$(monotonic_ms)" +RECOVERY_DEADLINE_MS=$((RECOVERY_STARTED_MS + 10000)) +attempt=0 +while [ "$(monotonic_ms)" -lt "$RECOVERY_DEADLINE_MS" ]; do + attempt=$((attempt + 1)) + remaining_ms=$((RECOVERY_DEADLINE_MS - $(monotonic_ms))) + (( remaining_ms > 0 )) || break + (( remaining_ms > 1000 )) && probe_timeout_ms=1000 || probe_timeout_ms=$remaining_ms + set +e + bounded_missing_probe "$MOUNT_POINT/reconnect-probe-$attempt" \ + "$probe_timeout_ms" >/dev/null + probe_status=$? + set -e + if [ "$probe_status" -eq 0 ]; then + read_recovered=1 + break + fi + sleep 0.25 +done +RECOVERY_ELAPSED_MS=$(( $(monotonic_ms) - RECOVERY_STARTED_MS )) +if [ "$RECOVERY_ELAPSED_MS" -lt 0 ]; then RECOVERY_ELAPSED_MS=0; fi +if [ "$read_recovered" -ne 1 ]; then + POST_ACCEPTED="$("$PYTHON" -c 'import json,sys; print(json.load(open(sys.argv[1]))["accepted"])' "$STATUS_FILE")" + fail "真实 TCP 断线后挂载读取没有在 10 秒内恢复（elapsed_ms=${RECOVERY_ELAPSED_MS} proxy_accepted=${POST_ACCEPTED}）" +fi +(( RECOVERY_ELAPSED_MS <= 10000 )) || \ + fail "真实 TCP 断线恢复超过 10 秒：${RECOVERY_ELAPSED_MS}ms" +[ "$(cat "$MOUNT_POINT/before-cut.txt")" = before-cut ] || fail "断线恢复后的基线内容错误" +[ "$(cat "$MOUNT_POINT/peer-during-cut.txt")" = peer-during-cut ] || \ + fail "断线期间 peer 创建的文件没有使 NFS 目录缓存失效" +CHECKS=$((CHECKS + 3)) + +write_after_cut() { + "$PYTHON" - "$MOUNT_POINT/after-cut.txt" <<'PY' +import os, sys +fd = os.open(sys.argv[1], os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o644) +try: + os.write(fd, b"after-cut\n") + os.fsync(fd) +finally: + os.close(fd) +PY +} +recovered=0 +for _ in 1 2 3; do + if write_after_cut; then recovered=1; break; fi + sleep 0.5 +done +[ "$recovered" -eq 1 ] || fail "真实 TCP 断线后写入没有恢复" +CHECKS=$((CHECKS + 1)) + +STATE="$(docker exec "$CONTAINER" psql -U postgres -d "$DATABASE" -X -q -t -A -F '|' \ + -v ON_ERROR_STOP=1 -c \ + "SELECT convert_from(vexfs_read('$WORKSPACE','/before-cut.txt'),'UTF8'), + convert_from(vexfs_read('$WORKSPACE','/after-cut.txt'),'UTF8'), + convert_from(vexfs_read('$WORKSPACE','/peer-during-cut.txt'),'UTF8'), + (vexfs_diagnostics('$WORKSPACE')->>'pending_handles')::bigint, + (vexfs_diagnostics('$WORKSPACE')->>'staging_bytes')::bigint, + (vexfs_check('$WORKSPACE',1)->>'ok')::boolean;")" +[ "$STATE" = $'before-cut\n|after-cut\n|peer-during-cut|0|0|t' ] || fail "断线恢复后的数据库状态错误：$STATE" +CHECKS=$((CHECKS + 6)) + +VERSIONS="$(docker exec "$CONTAINER" psql -U postgres -d "$DATABASE" -X -q -t -A \ + -c "SELECT count(*) FROM _vexfs.file_versions v JOIN _vexfs.workspaces w USING(workspace_id) WHERE w.name='$WORKSPACE';")" +[ "$VERSIONS" = 3 ] || fail "断线恢复后正式版本数错误：$VERSIONS" +CHECKS=$((CHECKS + 1)) + +HOME="$HOME_DIR" "${FS[@]}" unmount "$MOUNT_POINT" >/dev/null +MOUNTED=0 +HOME="$HOME_DIR" "${FS[@]}" mount "$MOUNT_POINT" >/dev/null +MOUNTED=1 +ACCEPTED_BEFORE_BACKGROUND="$("$PYTHON" -c \ + 'import json,sys; print(json.load(open(sys.argv[1]))["accepted"])' "$STATUS_FILE")" +rm -f "$ACK_FILE" +kill -USR1 "$PROXY_PID" +for _ in $(seq 1 100); do + [ -f "$ACK_FILE" ] && break + sleep 0.05 +done +[ -f "$ACK_FILE" ] || fail "后台发布测试没有确认断线" +BACKGROUND_CUT_CONNECTIONS="$("$PYTHON" -c \ + 'import json,sys; print(json.load(open(sys.argv[1]))["connections"])' "$ACK_FILE")" +(( BACKGROUND_CUT_CONNECTIONS >= 2 )) || \ + fail "后台发布测试没有同时切断主连接和 publisher 连接" + +write_background_after_cut() { +"$PYTHON" - "$MOUNT_POINT/background-after-cut.txt" <<'PY' +import os +import sys + +descriptor = os.open(sys.argv[1], os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o644) +try: + os.write(descriptor, b"background-after-cut\n") +finally: + os.close(descriptor) +PY +} +BACKGROUND_WRITE_RECOVERED=0 +BACKGROUND_WRITE_ERROR="$TMP/background-write.err" +for _ in 1 2 3 4; do + if write_background_after_cut 2>"$BACKGROUND_WRITE_ERROR"; then + BACKGROUND_WRITE_RECOVERED=1 + break + fi + # An in-flight mutation has an unknown commit state and is never replayed + # inside the runtime. Retry with the same target after reconnect cooldown. + sleep 1.1 +done +if [ "$BACKGROUND_WRITE_RECOVERED" -ne 1 ]; then + cat "$BACKGROUND_WRITE_ERROR" >&2 + fail "后台发布测试的前台写入没有从断线恢复" +fi +BACKGROUND_PUBLISHED=0 +for _ in $(seq 1 60); do + BACKGROUND_STATE="$(docker exec "$CONTAINER" psql -U postgres -d "$DATABASE" \ + -X -q -t -A -F '|' -v ON_ERROR_STOP=1 -c \ + "SELECT convert_from(vexfs_read('$WORKSPACE','/background-after-cut.txt'),'UTF8'), + (vexfs_diagnostics('$WORKSPACE')->>'pending_handles')::bigint, + (vexfs_diagnostics('$WORKSPACE')->>'staging_bytes')::bigint;" \ + 2>/dev/null || true)" + if [ "$BACKGROUND_STATE" = $'background-after-cut\n|0|0' ]; then + BACKGROUND_PUBLISHED=1 + break + fi + sleep 0.25 +done +[ "$BACKGROUND_PUBLISHED" -eq 1 ] || \ + fail "publisher 断线恢复后没有发布后台文件：$BACKGROUND_STATE" +ACCEPTED_AFTER_BACKGROUND="$("$PYTHON" -c \ + 'import json,sys; print(json.load(open(sys.argv[1]))["accepted"])' "$STATUS_FILE")" +(( ACCEPTED_AFTER_BACKGROUND >= ACCEPTED_BEFORE_BACKGROUND + 2 )) || \ + fail "主连接和 publisher 没有分别重连：before=$ACCEPTED_BEFORE_BACKGROUND after=$ACCEPTED_AFTER_BACKGROUND" +CHECKS=$((CHECKS + 5)) + +HOME="$HOME_DIR" "${FS[@]}" unmount "$MOUNT_POINT" >/dev/null +MOUNTED=0 +OOM_AFTER="$(docker exec "$CONTAINER" awk '$1=="oom_kill" {print $2}' \ + /sys/fs/cgroup/memory.events)" +[ "$OOM_AFTER" = "$OOM_BEFORE" ] || fail "network cut Gate 触发 OOM" +CHECKS=$((CHECKS + 1)) + +echo "VEXFS PG NETWORK CUT RECOVERY: PASS (checks=$CHECKS cut_connections=$CUT_CONNECTIONS reconnect_ms=$RECOVERY_ELAPSED_MS established_blackhole_ms=$ESTABLISHED_BLACKHOLE_MS blackhole_max_ms=$BLACKHOLE_MAX_MS blackhole_first_ms=$DISCONNECT_DETECT_MS blackhole_second_ms=$BLACKHOLE_ELAPSED_MS memory_max=$MEMORY_MAX oom_kill=$OOM_AFTER)" diff --git a/tests/eval/vexfs/run_pg_strict_crash_recovery.sh b/tests/eval/vexfs/run_pg_strict_crash_recovery.sh index a880e9170f..a459df1057 100755 --- a/tests/eval/vexfs/run_pg_strict_crash_recovery.sh +++ b/tests/eval/vexfs/run_pg_strict_crash_recovery.sh @@ -8,11 +8,13 @@ WORKSPACE="${VEXDB_PG_WORKSPACE:-strict-crash}" CLI="${VEXFS_EVAL_MOUNT_CLI:-$ROOT/vexdb_sqlite/build/vexdb}" DSN="${VEXDB_PG_DSN:-}" TMP="$(mktemp -d "${TMPDIR:-/tmp}/vexfs-pg-strict-crash.XXXXXX")" +TMP="$(cd "$TMP" && pwd -P)" TEST_HOME="$TMP/home" MOUNT_POINT="$TMP/mount" MOUNTED=0 DATABASE_CREATED=0 CHECKS=0 +GATEWAY_RECORD_BACKUP="" fail() { echo "$*" >&2 @@ -26,6 +28,9 @@ is_test_mount_live() { cleanup() { local status=$? trap - EXIT INT TERM + if [ -n "$GATEWAY_RECORD_BACKUP" ] && [ -f "$GATEWAY_RECORD_BACKUP" ]; then + cp "$GATEWAY_RECORD_BACKUP" "${GATEWAY_RECORD_BACKUP%.backup}" + fi if [ "$MOUNTED" -eq 1 ]; then HOME="$TEST_HOME" "$CLI" fs --backend pg --dsn "$DSN" \ --workspace "$WORKSPACE" unmount --force "$MOUNT_POINT" \ @@ -36,6 +41,9 @@ cleanup() { /sbin/umount -f "$MOUNT_POINT" >/dev/null 2>&1 || true sleep 0.25 done + HOME="$TEST_HOME" "$CLI" fs --backend pg --dsn "$DSN" \ + --workspace "$WORKSPACE" unmount --force "$MOUNT_POINT" \ + >/dev/null 2>&1 || true if is_test_mount_live; then echo "测试清理失败，NFS 挂载仍存在：$MOUNT_POINT" >&2 status=1 @@ -71,8 +79,6 @@ if [ -z "$DSN" ]; then fi mkdir -p "$TEST_HOME" "$MOUNT_POINT" -docker exec "$CONTAINER" dropdb -U postgres --if-exists --force "$DATABASE" \ - >/dev/null 2>&1 || true docker exec "$CONTAINER" createdb -U postgres "$DATABASE" DATABASE_CREATED=1 docker exec "$CONTAINER" psql -U postgres -d "$DATABASE" -X -q \ @@ -120,6 +126,44 @@ GATEWAY_COMMAND="$(/bin/ps -p "$GATEWAY_PID" -o command= 2>/dev/null || true)" "$GATEWAY_COMMAND" == *"--workspace $WORKSPACE"* ]] || \ fail "拒绝终止身份不匹配的 gateway PID $GATEWAY_PID：$GATEWAY_COMMAND" CHECKS=$((CHECKS + 1)) + +# A stale record can point at a PID reused by another gateway with the same +# executable. The recorded process start identity must prevent killing it. +STALE_MOUNT_POINT="$TMP/stale-mount" +mkdir -p "$STALE_MOUNT_POINT" +STALE_GATEWAY_RECORD="$(/usr/bin/python3 - "$TEST_HOME" "$STALE_MOUNT_POINT" "$GATEWAY_RECORD" <<'PY' +import pathlib +import sys + +home = pathlib.Path(sys.argv[1]) +mount_point = sys.argv[2].encode() +source = pathlib.Path(sys.argv[3]) +value = 1469598103934665603 +for byte in mount_point: + value ^= byte + value = (value * 1099511628211) & ((1 << 64) - 1) +directory = home / "Library/Application Support/VexDB-Lite/nfs-gateways" / f"mount-{value:016x}" +directory.mkdir(parents=True, exist_ok=True) +directory.chmod(0o700) +lines = source.read_text().splitlines() +if len(lines) != 10 or lines[0] != "VEXFS_NFS_GATEWAY_V3": + raise SystemExit("unexpected gateway record format") +lines[9] = str((int(lines[9]) + 1) % 1_000_000) +destination = directory / "gateway.record" +destination.write_text("\n".join(lines) + "\n") +destination.chmod(0o600) +print(destination) +PY +)" +[ -f "$STALE_GATEWAY_RECORD" ] || fail "无法生成 PID 复用负向测试记录" +HOME="$TEST_HOME" "$CLI" fs --backend pg --dsn "$DSN" \ + --workspace "$WORKSPACE" unmount --force "$STALE_MOUNT_POINT" >/dev/null +kill -0 "$GATEWAY_PID" >/dev/null 2>&1 || \ + fail "进程启动身份不匹配时误杀了当前 gateway" +[ "$(cat "$MOUNT_POINT/durable.txt")" = 'strict-after-crash' ] || \ + fail "PID 复用负向测试影响了真实挂载" +CHECKS=$((CHECKS + 3)) + kill -9 "$GATEWAY_PID" for _ in $(seq 1 50); do kill -0 "$GATEWAY_PID" >/dev/null 2>&1 || break @@ -128,6 +172,30 @@ done kill -0 "$GATEWAY_PID" >/dev/null 2>&1 && fail "gateway SIGKILL 后仍存活" CHECKS=$((CHECKS + 1)) +GATEWAY_RECORD_BACKUP="$GATEWAY_RECORD.backup" +cp "$GATEWAY_RECORD" "$GATEWAY_RECORD_BACKUP" +/usr/bin/python3 - "$GATEWAY_RECORD" <<'PY' +import pathlib +import sys + +path = pathlib.Path(sys.argv[1]) +lines = path.read_text().splitlines() +if len(lines) != 10 or lines[0] != "VEXFS_NFS_GATEWAY_V3": + raise SystemExit("unexpected gateway record format") +lines[6] = str(int(lines[6]) + 1) +temporary = path.with_suffix(".identity-mismatch") +temporary.write_text("\n".join(lines) + "\n") +temporary.chmod(0o600) +temporary.replace(path) +PY +if HOME="$TEST_HOME" "${FS[@]}" unmount --force "$MOUNT_POINT" >/dev/null 2>&1; then + fail "挂载 fsid 不匹配时不应允许 VexFS 卸载" +fi +is_test_mount_live || fail "身份校验失败后 NFS 挂载被错误卸载" +cp "$GATEWAY_RECORD_BACKUP" "$GATEWAY_RECORD" +GATEWAY_RECORD_BACKUP="" +CHECKS=$((CHECKS + 2)) + HOME="$TEST_HOME" "${FS[@]}" unmount --force "$MOUNT_POINT" >/dev/null MOUNTED=0 CHECKS=$((CHECKS + 1)) diff --git a/tests/eval/vexfs/run_pg_strict_crash_soak.sh b/tests/eval/vexfs/run_pg_strict_crash_soak.sh new file mode 100755 index 0000000000..940ca4b70a --- /dev/null +++ b/tests/eval/vexfs/run_pg_strict_crash_soak.sh @@ -0,0 +1,40 @@ +#!/bin/bash +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" +ROUND_SCRIPT="$ROOT/tests/eval/vexfs/run_pg_strict_crash_recovery.sh" +ROUNDS="${VEXFS_PG_STRICT_CRASH_ROUNDS:-20}" +DATABASE_PREFIX="${VEXFS_PG_STRICT_CRASH_DATABASE_PREFIX:-vexfs_strict_soak}" +WORKSPACE_PREFIX="${VEXFS_PG_STRICT_CRASH_WORKSPACE_PREFIX:-strict-soak}" +RUN_ID="${VEXFS_PG_STRICT_CRASH_RUN_ID:-$(date +%Y%m%d%H%M%S)_$$_${RANDOM}}" + +case "$ROUNDS" in + ''|*[!0-9]*) echo "crash soak 轮数必须是正整数：$ROUNDS" >&2; exit 2 ;; +esac +(( ROUNDS >= 1 && ROUNDS <= 100 )) || { + echo "crash soak 轮数必须在 1 到 100 之间：$ROUNDS" >&2 + exit 2 +} +case "$DATABASE_PREFIX:$WORKSPACE_PREFIX:$RUN_ID" in + *[!A-Za-z0-9_:-]*) + echo "crash soak 数据库和 workspace 前缀只能包含字母、数字、下划线和连字符" >&2 + exit 2 + ;; +esac + +started_at="$(date +%s)" +for round in $(seq 1 "$ROUNDS"); do + echo "VEXFS PG STRICT CRASH SOAK: round=$round/$ROUNDS" + database="${DATABASE_PREFIX}_${RUN_ID}_${round}" + (( ${#database} <= 63 )) || { + echo "crash soak 数据库名不能超过 63 个字符：$database" >&2 + exit 2 + } + VEXDB_PG_DSN="" \ + VEXDB_PG_DATABASE="$database" \ + VEXDB_PG_WORKSPACE="${WORKSPACE_PREFIX}-${RUN_ID}-${round}" \ + bash "$ROUND_SCRIPT" +done +elapsed="$(( $(date +%s) - started_at ))" + +echo "VEXFS PG STRICT CRASH SOAK: PASS (rounds=$ROUNDS elapsed_seconds=$elapsed)" diff --git a/tests/spec/sqlite/agent_files/vexfs_contract.yaml b/tests/spec/sqlite/agent_files/vexfs_contract.yaml index f7390fca1e..406d6f5d95 100644 --- a/tests/spec/sqlite/agent_files/vexfs_contract.yaml +++ b/tests/spec/sqlite/agent_files/vexfs_contract.yaml @@ -124,6 +124,18 @@ steps: - query: SELECT CAST(vexfs_read_version('default', '/notes/2026/hello.txt', 1) AS TEXT); expect: - - hello +- query: SELECT json_extract(vexfs_compare_versions('default', '/notes/2026/hello.txt', 1, 3), '$.changed'), json_extract(vexfs_compare_versions('default', '/notes/2026/hello.txt', 1, 3), '$.from_size'), json_extract(vexfs_compare_versions('default', '/notes/2026/hello.txt', 1, 3), '$.to_size'); + expect: + - - 1 + - 5 + - 11 +- query: SELECT json_extract(vexfs_compare_versions('default', '/notes/2026/hello.txt', 3, 3), '$.changed'); + expect: + - - 0 +- statement: UPDATE _vexfs_manifests SET chunk_count=2049 WHERE id=(SELECT manifest_id FROM _vexfs_file_versions WHERE inode_id=4 AND version_no=1); +- query: SELECT vexfs_compare_versions('default', '/notes/2026/hello.txt', 1, 3); + expect_error: file version content reference is corrupt +- statement: UPDATE _vexfs_manifests SET chunk_count=1 WHERE id=(SELECT manifest_id FROM _vexfs_file_versions WHERE inode_id=4 AND version_no=1); - query: SELECT vexfs_restore_version('default', '/notes/2026/hello.txt', 1, 3); expect: - - 4 @@ -283,3 +295,14 @@ steps: - query: SELECT vexfs_snapshot_drop('default', 'gc-protected'); expect: - - 1 +- query: SELECT vexfs_write('default', '/multi.bin', zeroblob(65537)); + expect: + - - 1 +- query: SELECT vexfs_write('default', '/multi.bin', zeroblob(131073)); + expect: + - - 2 +- query: SELECT json_extract(vexfs_compare_versions('default', '/multi.bin', 1, 2), '$.changed'), json_extract(vexfs_compare_versions('default', '/multi.bin', 1, 2), '$.from_size'), json_extract(vexfs_compare_versions('default', '/multi.bin', 1, 2), '$.to_size'); + expect: + - - 1 + - 65537 + - 131073 diff --git a/vexdb_sqlite/src/agent_files/vexfs_sqlite.cpp b/vexdb_sqlite/src/agent_files/vexfs_sqlite.cpp index d21c159e83..416bff9775 100644 --- a/vexdb_sqlite/src/agent_files/vexfs_sqlite.cpp +++ b/vexdb_sqlite/src/agent_files/vexfs_sqlite.cpp @@ -1025,15 +1025,18 @@ WHERE v.inode_id=?1 AND v.version_no=?2 if (!statement.Row()) { throw SqlError("file version not found: " + std::to_string(version), SQLITE_NOTFOUND); } + const sqlite3_int64 size = statement.Int64(1); + const sqlite3_int64 chunk_count = statement.Int64(8); + const sqlite3_int64 expected_chunk_count = size < 0 || size > kMaxStagedBytes + ? -1 : (size + kContentChunkBytes - 1) / kContentChunkBytes; if (statement.Type(0) == SQLITE_NULL || statement.Int64(0) <= 0 || - statement.Int64(1) < 0 || statement.Int64(1) != statement.Int64(3) || + size < 0 || size > kMaxStagedBytes || size != statement.Int64(3) || statement.Text(4).size() != 64 || statement.Text(4) != statement.Text(5) || statement.Text(4) != statement.Text(6) || - statement.Int64(7) != kContentChunkBytes || statement.Int64(8) < 0) { + statement.Int64(7) != kContentChunkBytes || chunk_count != expected_chunk_count) { throw SqlError("file version content reference is corrupt", SQLITE_CORRUPT); } - return {statement.Int64(0), statement.Int64(1), statement.Int64(2), - statement.Int64(8), statement.Text(4)}; + return {statement.Int64(0), size, statement.Int64(2), chunk_count, statement.Text(4)}; } std::vector ReadVersionContent(sqlite3 *db, sqlite3_int64 inode_id, @@ -3159,8 +3162,10 @@ WHERE entry.manifest_id=?1 ORDER BY entry.chunk_no )SQL"); from_rows.BindInt64(1, from.manifest_id); to_rows.BindInt64(1, to.manifest_id); - vexfs::Sha256 from_hash; - vexfs::Sha256 to_hash; + std::vector from_manifest_chunks; + std::vector to_manifest_chunks; + from_manifest_chunks.reserve(static_cast(from.chunk_count)); + to_manifest_chunks.reserve(static_cast(to.chunk_count)); bool changed = from.size != to.size; bool binary = false; sqlite3_int64 chunk_no = 0; @@ -3168,9 +3173,13 @@ WHERE entry.manifest_id=?1 ORDER BY entry.chunk_no sqlite3_int64 to_chunks = 0; sqlite3_int64 from_bytes_total = 0; sqlite3_int64 to_bytes_total = 0; + bool from_done = false; + bool to_done = false; while (true) { - const bool has_from = from_rows.Row(); - const bool has_to = to_rows.Row(); + const bool has_from = !from_done && from_rows.Row(); + const bool has_to = !to_done && to_rows.Row(); + if (!has_from) from_done = true; + if (!has_to) to_done = true; if (!has_from && !has_to) break; std::vector from_chunk; std::vector to_chunk; @@ -3182,7 +3191,8 @@ WHERE entry.manifest_id=?1 ORDER BY entry.chunk_no vexfs::Sha256Hex(from_chunk.data(), from_chunk.size()) != from_rows.Text(3)) { throw SqlError("file manifest chunk is corrupt", SQLITE_CORRUPT); } - from_hash.Update(from_chunk.data(), from_chunk.size()); + from_manifest_chunks.push_back({ + static_cast(from_chunk.size()), from_rows.Text(3)}); from_bytes_total += static_cast(from_chunk.size()); ++from_chunks; } @@ -3194,7 +3204,8 @@ WHERE entry.manifest_id=?1 ORDER BY entry.chunk_no vexfs::Sha256Hex(to_chunk.data(), to_chunk.size()) != to_rows.Text(3)) { throw SqlError("file manifest chunk is corrupt", SQLITE_CORRUPT); } - to_hash.Update(to_chunk.data(), to_chunk.size()); + to_manifest_chunks.push_back({ + static_cast(to_chunk.size()), to_rows.Text(3)}); to_bytes_total += static_cast(to_chunk.size()); ++to_chunks; } @@ -3206,8 +3217,12 @@ WHERE entry.manifest_id=?1 ORDER BY entry.chunk_no } if (from_bytes_total != from.size || to_bytes_total != to.size || from_chunks != from.chunk_count || to_chunks != to.chunk_count || - vexfs::Hex(from_hash.Finish()) != from.checksum || - vexfs::Hex(to_hash.Finish()) != to.checksum) { + vexfs::ManifestChecksum( + static_cast(from.size), + static_cast(kContentChunkBytes), from_manifest_chunks) != from.checksum || + vexfs::ManifestChecksum( + static_cast(to.size), + static_cast(kContentChunkBytes), to_manifest_chunks) != to.checksum) { throw SqlError("file version checksum does not match content", SQLITE_CORRUPT); } const std::string json = From 776615a2f01834c3fd8bdc2ba97bdda1f0bcc669 Mon Sep 17 00:00:00 2001 From: Four <8122332+HungryFour@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:26:52 +0800 Subject: [PATCH 17/28] Add VexFS find and safe workspace restore --- agent_files/cli/vexfs_main.cpp | 115 ++++++- .../common/include/vexfs_runtime_admin.h | 17 ++ .../src/vexfs_mount_contract_sqlite.cpp | 69 ++++- .../test/vexfs_mount_contract_smoke.cpp | 237 +++++++++++++- .../common/test/vexfs_pg_runtime_smoke.cpp | 50 ++- ...-07-16_agent-files-universal-filesystem.md | 9 +- ...2026-07-21_vexfs-final-goal-and-roadmap.md | 20 +- ...2026-07-28_vexfs-workspace-pitr-roadmap.md | 289 ++++++++++++++++++ .../reports/2026-07-28_vexfs-database-find.md | 72 +++++ docs/reports/2026-07-28_vexfs-pitr-batch-a.md | 71 +++++ docs/specs/2026-07-16_vexfs-product-spec.md | 11 +- tests/eval/vexfs/run.py | 82 ++++- tests/eval/vexfs/run_pg_find_performance.sh | 149 +++++++++ tests/eval/vexfs/run_pg_runtime.sh | 28 +- tests/spec/pg/agent_files/vexfs_pg_find.yaml | 90 ++++++ .../vexfs_pg_history_snapshot.yaml | 18 +- vexdb_pg/sql/vexdb_lite--1.0.sql | 275 ++++++++++++++++- vexdb_sqlite/src/agent_files/vexfs_sqlite.cpp | 214 ++++++++++++- vexdb_sqlite/test/vexfs_static_smoke.c | 22 ++ 19 files changed, 1811 insertions(+), 27 deletions(-) create mode 100644 docs/plans/2026-07-28_vexfs-workspace-pitr-roadmap.md create mode 100644 docs/reports/2026-07-28_vexfs-database-find.md create mode 100644 docs/reports/2026-07-28_vexfs-pitr-batch-a.md create mode 100755 tests/eval/vexfs/run_pg_find_performance.sh create mode 100644 tests/spec/pg/agent_files/vexfs_pg_find.yaml diff --git a/agent_files/cli/vexfs_main.cpp b/agent_files/cli/vexfs_main.cpp index f552ed3077..b522bb4ac1 100644 --- a/agent_files/cli/vexfs_main.cpp +++ b/agent_files/cli/vexfs_main.cpp @@ -47,6 +47,10 @@ void Usage(std::ostream &output) { " write PATH [LOCAL_FILE] Write a local file or stdin\n" " cat PATH Print a file\n" " ls [PATH] [--json] List a directory\n" + " find [PATH] [--name GLOB] [--type KIND] [--min-size BYTES]\n" + " [--max-size BYTES] [--modified-after EPOCH_MS]\n" + " [--modified-before EPOCH_MS] [--limit N] [--after PATH]\n" + " Query names and metadata without file bodies\n" " grep [-i] [-l] [-n] [--max-results N] PATTERN [PATH]\n" " Search current text files in the workspace\n" " index status|enable|rebuild|disable\n" @@ -275,6 +279,24 @@ void PrintNames(const std::string &json) { } } +void PrintFindPaths(const std::string &json) { + const std::string marker = "\"path\""; + size_t position = 0; + while ((position = json.find(marker, position)) != std::string::npos) { + position += marker.size(); + position = json.find(':', position); + if (position == std::string::npos) throw std::runtime_error("invalid find JSON"); + ++position; + while (position < json.size() && std::isspace( + static_cast(json[position]))) ++position; + if (position >= json.size() || json[position] != '"') { + throw std::runtime_error("invalid find JSON"); + } + ++position; + std::cout << JsonUnescape(json, &position) << '\n'; + } +} + int64_t JsonInteger(const std::string &json, const std::string &name); std::string JsonString(const std::string &json, const std::string &name); @@ -767,20 +789,42 @@ void RemountWorkspace(const Options &options, const std::string &mount_point) { throw std::runtime_error(last_error); } +std::string SafetySnapshotName(const std::string &workspace, int64_t head) { + std::string safe_workspace; + safe_workspace.reserve(std::min(workspace.size(), 32)); + for (const unsigned char byte : workspace) { + if (safe_workspace.size() >= 32) break; + safe_workspace.push_back( + std::isalnum(byte) || byte == '-' || byte == '_' ? + static_cast(byte) : '_'); + } + if (safe_workspace.empty()) safe_workspace = "workspace"; + const auto wall = std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()).count(); + const auto monotonic = std::chrono::steady_clock::now().time_since_epoch().count(); + return "vexfs-safety-" + safe_workspace + "-h" + std::to_string(head) + "-" + + std::to_string(wall) + "-" + + std::to_string(static_cast(monotonic)); +} + int64_t RestoreSnapshotAfterUnmount(Session &session, const std::string &name, - int64_t head, bool wait_for_mount_shutdown, + int64_t head, const std::string &safety_name, + bool wait_for_mount_shutdown, std::chrono::seconds shutdown_timeout = std::chrono::seconds(35)) { const auto deadline = std::chrono::steady_clock::now() + shutdown_timeout; while (true) { int64_t commit = 0; vexfs_mount_error error{}; - const vexfs_mount_status status = vexfs_mount_snapshot_restore( - session.get(), name.c_str(), head, &commit, &error); + const vexfs_mount_status status = vexfs_mount_snapshot_restore_safe( + session.get(), name.c_str(), head, safety_name.c_str(), &commit, &error); if (status == VEXFS_MOUNT_OK) return commit; const std::string message = ErrorMessage(error); + const bool mount_state_is_draining = + message.find("active mount session") != std::string::npos || + message.find("file handle") != std::string::npos; if (!wait_for_mount_shutdown || status != VEXFS_MOUNT_BUSY || - message.find("active mount session") == std::string::npos || + !mount_state_is_draining || std::chrono::steady_clock::now() >= deadline) { Check(status, error); } @@ -955,6 +999,63 @@ int Run(const Options &options) { Check(vexfs_mount_list(session.get(), path.c_str(), &json, &error), error); const std::string value = BytesToString(&json); if (options.json) std::cout << value << '\n'; else PrintNames(value); + } else if (command == "find") { + const ParsedCommand parsed = ParseCommand( + options.arguments, + {"--name", "--type", "--min-size", "--max-size", "--modified-after", + "--modified-before", "--limit", "--after"}, {}); + if (parsed.positional.size() > 1) { + throw std::runtime_error("find accepts one optional PATH"); + } + const std::string path = parsed.positional.empty() ? "/" : parsed.positional[0]; + const std::string name_pattern = parsed.Value("--name", false); + std::string kind = parsed.Value("--type", false); + if (kind == "f") kind = "file"; + if (kind == "d") kind = "directory"; + if (kind == "l") kind = "symlink"; + if (!kind.empty() && kind != "file" && kind != "directory" && kind != "symlink") { + throw CliError(VEXFS_MOUNT_INVALID_ARGUMENT, + "type must be file, directory, symlink, f, d or l"); + } + const std::string min_value = parsed.Value("--min-size", false); + const std::string max_value = parsed.Value("--max-size", false); + const std::string after_time_value = parsed.Value("--modified-after", false); + const std::string before_time_value = parsed.Value("--modified-before", false); + const std::string limit_value = parsed.Value("--limit", false); + const std::string after_path = parsed.Value("--after", false); + const int64_t min_size = min_value.empty() ? -1 : + NonnegativeInteger(min_value, "min-size"); + const int64_t max_size = max_value.empty() ? -1 : + NonnegativeInteger(max_value, "max-size"); + const int64_t modified_after = after_time_value.empty() ? -1 : + NonnegativeInteger(after_time_value, "modified-after"); + const int64_t modified_before = before_time_value.empty() ? -1 : + NonnegativeInteger(before_time_value, "modified-before"); + const int64_t limit = limit_value.empty() ? 100 : + PositiveInteger(limit_value, "limit"); + if (limit > 1000) { + throw CliError(VEXFS_MOUNT_INVALID_ARGUMENT, "limit must be at most 1000"); + } + if (min_size >= 0 && max_size >= 0 && min_size > max_size) { + throw CliError(VEXFS_MOUNT_INVALID_ARGUMENT, + "min-size must not exceed max-size"); + } + if (modified_after >= 0 && modified_before >= 0 && + modified_after > modified_before) { + throw CliError(VEXFS_MOUNT_INVALID_ARGUMENT, + "modified-after must not exceed modified-before"); + } + if (!after_path.empty() && after_path.front() != '/') { + throw CliError(VEXFS_MOUNT_INVALID_ARGUMENT, + "after cursor must be an absolute path"); + } + vexfs_mount_bytes json{}; + Check(vexfs_mount_find( + session.get(), path.c_str(), name_pattern.c_str(), kind.c_str(), + min_size, max_size, modified_after, modified_before, after_path.c_str(), + static_cast(limit), &json, &error), error); + const std::string value = BytesToString(&json); + if (options.json) std::cout << value << '\n'; else PrintFindPaths(value); } else if (command == "grep") { const ParsedCommand parsed = ParseCommand( options.arguments, {"--max-results"}, {"-i", "-l", "-n"}); @@ -1215,10 +1316,11 @@ int Run(const Options &options) { // is gone, otherwise our own close path causes a false conflict. int64_t head = 0; Check(vexfs_mount_workspace_head(session.get(), &head, &error), error); + const std::string safety_name = SafetySnapshotName(options.workspace, head); int64_t commit = 0; try { commit = RestoreSnapshotAfterUnmount( - session, name, head, !mount_point.empty(), + session, name, head, safety_name, !mount_point.empty(), std::chrono::seconds(35)); } catch (const std::exception &restore_error) { const std::string restore_message = restore_error.what(); @@ -1239,6 +1341,7 @@ int Run(const Options &options) { } catch (const std::exception &remount_error) { throw std::runtime_error( "snapshot was restored at commit " + std::to_string(commit) + + "; safety snapshot is " + safety_name + " but workspace remount failed: " + remount_error.what() + "; mount it again at " + mount_point); } @@ -1247,6 +1350,8 @@ int Run(const Options &options) { std::cout << "{\"name\":\"" << JsonEscape(name) << "\",\"previous_head\":" << head << ",\"commit\":" << commit + << ",\"safety_snapshot\":\"" + << JsonEscape(safety_name) << "\"" << ",\"remounted\":" << (mount_point.empty() ? "false" : "true"); if (!mount_point.empty()) diff --git a/agent_files/mount/common/include/vexfs_runtime_admin.h b/agent_files/mount/common/include/vexfs_runtime_admin.h index 47f0fef789..aebd27d866 100644 --- a/agent_files/mount/common/include/vexfs_runtime_admin.h +++ b/agent_files/mount/common/include/vexfs_runtime_admin.h @@ -20,6 +20,15 @@ vexfs_mount_status vexfs_mount_grep(vexfs_mount_session *session, const char *pa const char *pattern, uint32_t flags, uint32_t limit, vexfs_mount_bytes *json, vexfs_mount_error *error); +// Finds current workspace entries without reading file bodies. Empty pattern, +// kind, and cursor values disable those filters. Numeric filters use -1 when +// disabled. Results are ordered by binary path and use an exclusive path cursor. +vexfs_mount_status vexfs_mount_find( + vexfs_mount_session *session, const char *path, const char *name_pattern, + const char *kind, int64_t min_size, int64_t max_size, + int64_t modified_after_ms, int64_t modified_before_ms, + const char *after_path, uint32_t limit, vexfs_mount_bytes *json, + vexfs_mount_error *error); vexfs_mount_status vexfs_mount_grep_index(vexfs_mount_session *session, const char *action, vexfs_mount_bytes *json, @@ -71,6 +80,14 @@ vexfs_mount_status vexfs_mount_snapshot_restore(vexfs_mount_session *session, const char *name, int64_t expected_head, int64_t *new_commit, vexfs_mount_error *error); +// Restores and atomically preserves the pre-restore workspace as safety_name. +// The safety snapshot is created only when the restore transaction commits. +vexfs_mount_status vexfs_mount_snapshot_restore_safe(vexfs_mount_session *session, + const char *name, + int64_t expected_head, + const char *safety_name, + int64_t *new_commit, + vexfs_mount_error *error); // Quota values use -1 for unlimited and non-negative values for a hard limit. vexfs_mount_status vexfs_mount_quota_set(vexfs_mount_session *session, int64_t max_bytes, int64_t max_files, diff --git a/agent_files/mount/common/src/vexfs_mount_contract_sqlite.cpp b/agent_files/mount/common/src/vexfs_mount_contract_sqlite.cpp index d10b90ac47..0d96255761 100644 --- a/agent_files/mount/common/src/vexfs_mount_contract_sqlite.cpp +++ b/agent_files/mount/common/src/vexfs_mount_contract_sqlite.cpp @@ -1827,6 +1827,38 @@ extern "C" vexfs_mount_status vexfs_mount_grep(vexfs_mount_session *session, }); } +extern "C" vexfs_mount_status vexfs_mount_find( + vexfs_mount_session *session, const char *path, const char *name_pattern, + const char *kind, int64_t min_size, int64_t max_size, + int64_t modified_after_ms, int64_t modified_before_ms, + const char *after_path, uint32_t limit, vexfs_mount_bytes *json, + vexfs_mount_error *error) { + return Guard(session, error, [&] { + RequireSession(session); + if (path == nullptr || name_pattern == nullptr || kind == nullptr || + after_path == nullptr || json == nullptr) { + throw CallError(SQLITE_MISUSE, "find inputs and output are required"); + } + if (limit == 0 || limit > 1000 || min_size < -1 || max_size < -1 || + modified_after_ms < -1 || modified_before_ms < -1) { + throw CallError(SQLITE_RANGE, "find limits are outside the supported range"); + } + Call call(session, "SELECT vexfs_find(?1,?2,?3,?4,?5,?6,?7,?8,?9,?10)"); + call.Text(1, session->workspace.c_str()); + call.Text(2, path); + if (*name_pattern == '\0') call.Null(3); else call.Text(3, name_pattern); + if (*kind == '\0') call.Null(4); else call.Text(4, kind); + if (min_size < 0) call.Null(5); else call.Int64(5, min_size); + if (max_size < 0) call.Null(6); else call.Int64(6, max_size); + if (modified_after_ms < 0) call.Null(7); else call.Int64(7, modified_after_ms); + if (modified_before_ms < 0) call.Null(8); else call.Int64(8, modified_before_ms); + if (*after_path == '\0') call.Null(9); else call.Text(9, after_path); + call.Int64(10, limit); + call.Row(); + CopyResult(call, json); + }); +} + extern "C" vexfs_mount_status vexfs_mount_check(vexfs_mount_session *session, uint32_t flags, vexfs_mount_bytes *json, @@ -2357,11 +2389,9 @@ extern "C" vexfs_mount_status vexfs_mount_snapshot_drop(vexfs_mount_session *ses }); } -extern "C" vexfs_mount_status vexfs_mount_snapshot_restore(vexfs_mount_session *session, - const char *name, - int64_t expected_head, - int64_t *new_commit, - vexfs_mount_error *error) { +vexfs_mount_status SnapshotRestore(vexfs_mount_session *session, const char *name, + int64_t expected_head, const char *safety_name, + int64_t *new_commit, vexfs_mount_error *error) { return Guard(session, error, [&] { RequireSession(session); UseFullDurability(session); @@ -2369,15 +2399,42 @@ extern "C" vexfs_mount_status vexfs_mount_snapshot_restore(vexfs_mount_session * throw CallError(SQLITE_MISUSE, "snapshot name, positive expected head and commit output are required"); } - Call call(session, "SELECT vexfs_snapshot_restore(?1,?2,?3)"); + Call call(session, session->postgresql + ? "SELECT vexfs_snapshot_restore(?1,?2,?3,?4,?5)" + : "SELECT vexfs_snapshot_restore(?1,?2,?3,?4)"); call.Text(1, session->workspace.c_str()); call.Text(2, name); call.Int64(3, expected_head); + if (safety_name == nullptr) call.Null(4); else call.Text(4, safety_name); + if (session->postgresql) { + if (session->session_started) call.Text(5, session->session_id.c_str()); + else call.Null(5); + } call.Row(); *new_commit = call.ResultInt64(); }); } +extern "C" vexfs_mount_status vexfs_mount_snapshot_restore(vexfs_mount_session *session, + const char *name, + int64_t expected_head, + int64_t *new_commit, + vexfs_mount_error *error) { + return SnapshotRestore(session, name, expected_head, nullptr, new_commit, error); +} + +extern "C" vexfs_mount_status vexfs_mount_snapshot_restore_safe( + vexfs_mount_session *session, const char *name, int64_t expected_head, + const char *safety_name, int64_t *new_commit, vexfs_mount_error *error) { + if (safety_name == nullptr || *safety_name == '\0') { + return SetError(error, SQLITE_MISUSE, "safety snapshot name is required", + session != nullptr && session->postgresql + ? VEXFS_RUNTIME_BACKEND_POSTGRESQL + : VEXFS_RUNTIME_BACKEND_SQLITE); + } + return SnapshotRestore(session, name, expected_head, safety_name, new_commit, error); +} + extern "C" vexfs_mount_status vexfs_mount_quota_get( vexfs_mount_session *session, vexfs_mount_bytes *json, vexfs_mount_error *error) { diff --git a/agent_files/mount/common/test/vexfs_mount_contract_smoke.cpp b/agent_files/mount/common/test/vexfs_mount_contract_smoke.cpp index 2ec9d8bfa5..a7afd53e3d 100644 --- a/agent_files/mount/common/test/vexfs_mount_contract_smoke.cpp +++ b/agent_files/mount/common/test/vexfs_mount_contract_smoke.cpp @@ -1,5 +1,7 @@ #include "vexfs_runtime_admin.h" #include "sqlite3.h" +#define VEXDB_SQLITE_CORE +#include "vexdb_sqlite.h" #include #include @@ -39,6 +41,30 @@ int64_t JsonInteger(const vexfs_mount_bytes &bytes, const char *key) { return std::strtoll(value.c_str() + start + marker.size(), nullptr, 10); } +std::string JsonString(const vexfs_mount_bytes &bytes, const char *key) { + const std::string value(static_cast(bytes.data), + static_cast(bytes.size)); + const std::string marker = std::string("\"") + key + "\":\""; + const size_t start = value.find(marker); + if (start == std::string::npos) return {}; + const size_t value_start = start + marker.size(); + const size_t end = value.find('"', value_start); + return end == std::string::npos ? std::string() : + value.substr(value_start, end - value_start); +} + +int Count(const vexfs_mount_bytes &bytes, const char *needle) { + const std::string value(static_cast(bytes.data), + static_cast(bytes.size)); + int count = 0; + size_t position = 0; + while ((position = value.find(needle, position)) != std::string::npos) { + ++count; + position += std::strlen(needle); + } + return count; +} + int64_t Scalar(const char *path, const char *sql) { sqlite3 *db = nullptr; if (sqlite3_open_v2(path, &db, SQLITE_OPEN_READONLY, nullptr) != SQLITE_OK) { @@ -67,6 +93,140 @@ bool Execute(const char *path, const char *sql) { return rc == SQLITE_OK; } +int RunFindBenchmark(int file_count) { + if (file_count <= 0 || file_count > 100000) { + std::fprintf(stderr, "find benchmark file count must be 1..100000\n"); + return 2; + } + char path[] = "/tmp/vexfs-find-benchmark-XXXXXX"; + const int fd = mkstemp(path); + if (fd < 0) return 1; + close(fd); + unlink(path); + + sqlite3 *db = nullptr; + if (sqlite3_open(path, &db) != SQLITE_OK || vexdb_sqlite_register(db) != SQLITE_OK) { + if (db != nullptr) sqlite3_close(db); + unlink(path); + return 1; + } + const auto seed_started = std::chrono::steady_clock::now(); + if (sqlite3_exec(db, + "PRAGMA journal_mode=WAL; PRAGMA synchronous=NORMAL;" + "PRAGMA temp_store=FILE; PRAGMA cache_size=-65536;" + "SELECT vexfs_init();" + "SELECT vexfs_workspace_create('find-benchmark');" + "SELECT vexfs_mkdir('find-benchmark','/files');" + "BEGIN IMMEDIATE;", + nullptr, nullptr, nullptr) != SQLITE_OK) { + std::fprintf(stderr, "find benchmark setup: %s\n", sqlite3_errmsg(db)); + sqlite3_close(db); + unlink(path); + return 1; + } + sqlite3_stmt *create = nullptr; + if (sqlite3_prepare_v2(db, + "SELECT vexfs_create('find-benchmark',?1,'file',420)", + -1, &create, nullptr) != SQLITE_OK) { + std::fprintf(stderr, "find benchmark prepare: %s\n", sqlite3_errmsg(db)); + sqlite3_exec(db, "ROLLBACK", nullptr, nullptr, nullptr); + sqlite3_close(db); + unlink(path); + return 1; + } + bool seed_ok = true; + for (int index = 0; index < file_count; ++index) { + char file_path[64]; + std::snprintf(file_path, sizeof(file_path), "/files/f%06d.txt", index); + sqlite3_bind_text(create, 1, file_path, -1, SQLITE_TRANSIENT); + if (sqlite3_step(create) != SQLITE_ROW) { + std::fprintf(stderr, "find benchmark create %d: %s\n", index, + sqlite3_errmsg(db)); + seed_ok = false; + break; + } + sqlite3_reset(create); + sqlite3_clear_bindings(create); + } + sqlite3_finalize(create); + if (!seed_ok || sqlite3_exec(db, seed_ok ? "COMMIT" : "ROLLBACK", + nullptr, nullptr, nullptr) != SQLITE_OK) { + std::fprintf(stderr, "find benchmark commit: %s\n", sqlite3_errmsg(db)); + sqlite3_close(db); + unlink(path); + return 1; + } + const double seed_seconds = std::chrono::duration( + std::chrono::steady_clock::now() - seed_started).count(); + sqlite3_close(db); + + vexfs_mount_config config{}; + config.abi_version = VEXFS_RUNTIME_ABI_VERSION; + config.backend = VEXFS_RUNTIME_BACKEND_SQLITE; + config.connection = path; + config.workspace = "find-benchmark"; + config.principal = "local"; + config.operation_timeout_ms = 30000; + config.flags = VEXFS_RUNTIME_OPEN_NO_CREATE; + vexfs_mount_error error{}; + vexfs_mount_session *session = nullptr; + if (vexfs_mount_session_open(&config, &session, &error) != VEXFS_MOUNT_OK) + return Fail("find benchmark open", error); + + vexfs_mount_bytes page{}; + auto query_started = std::chrono::steady_clock::now(); + if (vexfs_mount_find(session, "/files", "*.txt", "file", 0, 0, -1, -1, + "", 100, &page, &error) != VEXFS_MOUNT_OK || + Count(page, "\"path\":") != 100) { + return Fail("find benchmark first page", error); + } + const double first_page_seconds = std::chrono::duration( + std::chrono::steady_clock::now() - query_started).count(); + const std::string cursor = JsonString(page, "next_cursor"); + vexfs_mount_free(page.data); + if (cursor.empty()) return Fail("find benchmark cursor", error); + + page = {}; + query_started = std::chrono::steady_clock::now(); + if (vexfs_mount_find(session, "/files", "*.txt", "file", 0, 0, -1, -1, + cursor.c_str(), 100, &page, &error) != VEXFS_MOUNT_OK || + Count(page, "\"path\":") != 100) { + return Fail("find benchmark second page", error); + } + const double second_page_seconds = std::chrono::duration( + std::chrono::steady_clock::now() - query_started).count(); + vexfs_mount_free(page.data); + + char last_name[32]; + std::snprintf(last_name, sizeof(last_name), "f%06d.txt", file_count - 1); + page = {}; + query_started = std::chrono::steady_clock::now(); + if (vexfs_mount_find(session, "/files", last_name, "file", 0, 0, -1, -1, + "", 10, &page, &error) != VEXFS_MOUNT_OK || + Count(page, "\"path\":") != 1 || !Contains(page, last_name)) { + return Fail("find benchmark selective", error); + } + const double selective_seconds = std::chrono::duration( + std::chrono::steady_clock::now() - query_started).count(); + vexfs_mount_free(page.data); + vexfs_mount_session_close(session); + + struct stat info {}; + const int64_t database_bytes = stat(path, &info) == 0 ? info.st_size : -1; + std::printf( + "{\"files\":%d,\"seed_seconds\":%.6f," + "\"find_first_page_seconds\":%.6f,\"find_second_page_seconds\":%.6f," + "\"find_selective_seconds\":%.6f,\"database_bytes\":%lld}\n", + file_count, seed_seconds, first_page_seconds, second_page_seconds, + selective_seconds, static_cast(database_bytes)); + unlink(path); + const std::string wal = std::string(path) + "-wal"; + const std::string shm = std::string(path) + "-shm"; + unlink(wal.c_str()); + unlink(shm.c_str()); + return 0; +} + int RunBenchmark(int file_count) { if (file_count <= 0 || file_count > 100000) { std::fprintf(stderr, "benchmark file count must be 1..100000\n"); @@ -156,6 +316,42 @@ int RunBenchmark(int file_count) { } publish_call_seconds = std::chrono::duration( std::chrono::steady_clock::now() - publish_started).count(); + vexfs_mount_bytes find_page{}; + auto find_started = std::chrono::steady_clock::now(); + if (vexfs_mount_find(session, "/files", "*.txt", "file", + sizeof(payload) - 1, sizeof(payload) - 1, -1, -1, + "", 100, &find_page, &error) != VEXFS_MOUNT_OK || + Count(find_page, "\"path\":") != 100) { + return Fail("benchmark find first page", error); + } + const double find_first_page_seconds = std::chrono::duration( + std::chrono::steady_clock::now() - find_started).count(); + const std::string find_cursor = JsonString(find_page, "next_cursor"); + vexfs_mount_free(find_page.data); + if (find_cursor.empty()) return Fail("benchmark find cursor", error); + find_page = {}; + find_started = std::chrono::steady_clock::now(); + if (vexfs_mount_find(session, "/files", "*.txt", "file", + sizeof(payload) - 1, sizeof(payload) - 1, -1, -1, + find_cursor.c_str(), 100, &find_page, &error) != VEXFS_MOUNT_OK || + Count(find_page, "\"path\":") != 100) { + return Fail("benchmark find second page", error); + } + const double find_second_page_seconds = std::chrono::duration( + std::chrono::steady_clock::now() - find_started).count(); + vexfs_mount_free(find_page.data); + char last_name[32]; + std::snprintf(last_name, sizeof(last_name), "f%06d.txt", file_count - 1); + find_page = {}; + find_started = std::chrono::steady_clock::now(); + if (vexfs_mount_find(session, "/files", last_name, "file", -1, -1, -1, -1, + "", 10, &find_page, &error) != VEXFS_MOUNT_OK || + Count(find_page, "\"path\":") != 1 || !Contains(find_page, last_name)) { + return Fail("benchmark find selective", error); + } + const double find_selective_seconds = std::chrono::duration( + std::chrono::steady_clock::now() - find_started).count(); + vexfs_mount_free(find_page.data); const double create_seconds = std::chrono::duration( std::chrono::steady_clock::now() - started).count(); const int64_t commits = Scalar(path, "SELECT count(*) FROM _vexfs_commits") - commits_before; @@ -188,6 +384,8 @@ int RunBenchmark(int file_count) { "\"create_call_seconds\":%.6f,\"stat_call_seconds\":%.6f," "\"xattr_call_seconds\":%.6f,\"stage_call_seconds\":%.6f," "\"publish_call_seconds\":%.6f," + "\"find_first_page_seconds\":%.6f,\"find_second_page_seconds\":%.6f," + "\"find_selective_seconds\":%.6f," "\"ordinary_mutation_calls\":%lld,\"full_boundary_calls\":%lld," "\"durability_barriers\":%lld,\"database_bytes\":%lld}\n", file_count, create_seconds, file_count / std::max(create_seconds, 1e-9), @@ -197,6 +395,7 @@ int RunBenchmark(int file_count) { static_cast(requests) / file_count, create_call_seconds, stat_call_seconds, xattr_call_seconds, stage_call_seconds, publish_call_seconds, + find_first_page_seconds, find_second_page_seconds, find_selective_seconds, static_cast(ordinary_calls), static_cast(full_calls), static_cast(barriers), static_cast(database_bytes)); unlink(path); @@ -211,6 +410,9 @@ int RunBenchmark(int file_count) { } // namespace int main(int argc, char **argv) { + if (argc == 3 && std::strcmp(argv[1], "--find-benchmark") == 0) { + return RunFindBenchmark(std::atoi(argv[2])); + } if (argc == 3 && std::strcmp(argv[1], "--benchmark") == 0) { return RunBenchmark(std::atoi(argv[2])); } @@ -355,6 +557,31 @@ int main(int argc, char **argv) { target = {}; if (vexfs_mount_readlink(session, task_inode, &target, &error) != VEXFS_MOUNT_INVALID_ARGUMENT) return Fail("readlink regular file", error); + vexfs_mount_bytes found{}; + if (vexfs_mount_find(session, "/agent", "*.txt", "file", 4, 5, -1, -1, + "", 1, &found, &error) != VEXFS_MOUNT_OK || + !Contains(found, "\"path\":\"/agent/external.txt\"") || + !Contains(found, "\"next_cursor\":\"/agent/external.txt\"") || + Contains(found, "/agent/task.txt")) + return Fail("find first page", error); + vexfs_mount_free(found.data); + found = {}; + if (vexfs_mount_find(session, "/agent", "*.txt", "file", 4, 5, -1, -1, + "/agent/external.txt", 1, &found, &error) != VEXFS_MOUNT_OK || + !Contains(found, "\"path\":\"/agent/task.txt\"") || + !Contains(found, "\"next_cursor\":null")) + return Fail("find next page", error); + vexfs_mount_free(found.data); + found = {}; + if (vexfs_mount_find(session, "/agent", "*.txt", "file", -1, -1, -1, + 1700000200000LL, "", 10, &found, &error) != VEXFS_MOUNT_OK || + !Contains(found, "/agent/task.txt") || Contains(found, "/agent/external.txt")) + return Fail("find modified time", error); + vexfs_mount_free(found.data); + found = {}; + if (vexfs_mount_find(session, "/agent", "", "socket", -1, -1, -1, -1, + "", 10, &found, &error) != VEXFS_MOUNT_INVALID_ARGUMENT) + return Fail("find invalid kind", error); if (vexfs_mount_xattr_set(session, task_inode, "com.vexfs.test", "alpha", 5, VEXFS_MOUNT_XATTR_MUST_CREATE, &error) != VEXFS_MOUNT_OK) return Fail("xattr create", error); @@ -443,9 +670,15 @@ int main(int argc, char **argv) { std::strstr(error.message, "active mount session") == nullptr) return Fail("mounted snapshot restore must be busy", error); vexfs_mount_session_close(mounted_session); - if (vexfs_mount_snapshot_restore(session, "before-change", head_commit, - &restored_commit, &error) != VEXFS_MOUNT_OK || + if (vexfs_mount_snapshot_restore_safe( + session, "before-change", head_commit, "safety-before-change-restore", + &restored_commit, &error) != VEXFS_MOUNT_OK || restored_commit <= head_commit) return Fail("snapshot restore", error); + snapshots = {}; + if (vexfs_mount_snapshot_list(session, &snapshots, &error) != VEXFS_MOUNT_OK || + !Contains(snapshots, "safety-before-change-restore")) + return Fail("snapshot restore safety point", error); + vexfs_mount_free(snapshots.data); vexfs_mount_bytes snapshot_content{}; if (vexfs_mount_read_file(session, "/agent/task.txt", &snapshot_content, &error) != VEXFS_MOUNT_OK || !Equals(snapshot_content, "draft")) diff --git a/agent_files/mount/common/test/vexfs_pg_runtime_smoke.cpp b/agent_files/mount/common/test/vexfs_pg_runtime_smoke.cpp index 4a365642ac..1833328581 100644 --- a/agent_files/mount/common/test/vexfs_pg_runtime_smoke.cpp +++ b/agent_files/mount/common/test/vexfs_pg_runtime_smoke.cpp @@ -413,6 +413,23 @@ int main(int argc, char **argv) { if (!Expect(listing.find("\"name\":\"main.txt\"") != std::string::npos && listing.find("\"name\":\"current\"") != std::string::npos, "directory listing")) return 1; + if (!CheckStatus("find first page", vexfs_mount_find( + session, "/project", "*.txt", "file", 11, 11, -1, + 1700000200000LL, "", 1, &bytes, &error), error)) return 1; + const std::string find_first_page = Take(&bytes); + if (!Expect(find_first_page.find("/project/main-link.txt") != std::string::npos && + find_first_page.find("\"next_cursor\":\"/project/main-link.txt\"") != + std::string::npos && + find_first_page.find("/project/main.txt") == std::string::npos, + "find first page content")) return 1; + if (!CheckStatus("find second page", vexfs_mount_find( + session, "/project", "*.txt", "file", 11, 11, -1, + 1700000200000LL, "/project/main-link.txt", 1, &bytes, &error), error)) + return 1; + const std::string find_second_page = Take(&bytes); + if (!Expect(find_second_page.find("/project/main.txt") != std::string::npos && + find_second_page.find("\"next_cursor\":null") != std::string::npos, + "find second page content")) return 1; if (!CheckStatus("grep", vexfs_mount_grep( session, "/project", "alpha", 0, 100, &bytes, &error), error)) return 1; @@ -525,11 +542,40 @@ int main(int argc, char **argv) { if (!CheckStatus("workspace head", vexfs_mount_workspace_head( session, &head, &error), error)) return 1; int64_t restore_commit = 0; - if (!CheckStatus("snapshot restore", vexfs_mount_snapshot_restore( - session, "baseline", head, &restore_commit, &error), error) || + if (vexfs_mount_snapshot_restore_safe( + session, "baseline", head, "must-not-exist-while-peer-mounted", + &restore_commit, &error) != VEXFS_MOUNT_BUSY || + std::strstr(error.message, "active mount session") == nullptr) + return Fail("peer mount must block snapshot restore", &error); + ++checks; + vexfs_mount_session_close(peer); + peer = nullptr; + if (!CheckStatus("snapshot restore", vexfs_mount_snapshot_restore_safe( + session, "baseline", head, "safety-before-baseline-restore", + &restore_commit, &error), error) || !CheckStatus("restored read", vexfs_mount_read_file( session, "/project/main.txt", &bytes, &error), error) || !Expect(Take(&bytes) == "beta alpha\n", "snapshot restored content")) return 1; + if (!CheckStatus("head after baseline restore", vexfs_mount_workspace_head( + session, &head, &error), error) || + !CheckStatus("restore safety snapshot", vexfs_mount_snapshot_restore_safe( + session, "safety-before-baseline-restore", head, + "safety-before-undo", &restore_commit, &error), error) || + !CheckStatus("safety restored read", vexfs_mount_read_file( + session, "/project/main.txt", &bytes, &error), error) || + !Expect(Take(&bytes) == "changed\n", "safety snapshot restores pre-restore tree")) + return 1; + if (!CheckStatus("head after safety restore", vexfs_mount_workspace_head( + session, &head, &error), error) || + !CheckStatus("restore baseline again", vexfs_mount_snapshot_restore_safe( + session, "baseline", head, "safety-before-final-baseline", + &restore_commit, &error), error) || + !CheckStatus("final baseline read", vexfs_mount_read_file( + session, "/project/main.txt", &bytes, &error), error) || + !Expect(Take(&bytes) == "beta alpha\n", "final baseline content")) return 1; + + if (!CheckStatus("peer reopen", vexfs_mount_session_open( + &peer_config, &peer, &error), error)) return 1; if (!CheckStatus("deep check", vexfs_mount_check(session, 0, &bytes, &error), error) || !Expect(Take(&bytes).find("\"ok\":true") != std::string::npos, diff --git a/docs/design/2026-07-16_agent-files-universal-filesystem.md b/docs/design/2026-07-16_agent-files-universal-filesystem.md index d9e6257e39..bdcf172ba9 100644 --- a/docs/design/2026-07-16_agent-files-universal-filesystem.md +++ b/docs/design/2026-07-16_agent-files-universal-filesystem.md @@ -5,7 +5,8 @@ - 文档版本：1.7 - 状态：macOS FSKit 与 Linux libfuse3 已复用同一 runtime；PostgreSQL `0.4.0-alpha.1` 已完成数据库合同、format v2、role/ACL/审计、libpq HostStore、macOS/Linux 真实 mount、 - 跨机器 OpenCode 和逻辑/物理备份；第二台 Mac 的局域网直连只差一次系统授权确认 + 跨机器 OpenCode 和逻辑/物理备份；SQLite/PG 数据库批量 `find`、CLI、C ABI、ACL 隔离及 + 10 万文件性能 Gate 已完成；第二台 Mac 的局域网直连只差一次系统授权确认 - 产品名：暂定 `VexFS` - 产品规格：`docs/specs/2026-07-16_vexfs-product-spec.md` - 最终目标和阶段顺序：`docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md` @@ -266,6 +267,12 @@ vexfs truncate -s 0 /logs/run.log 普通终端使用 `vexfs ls`，避免覆盖系统命令。正常 macOS 安装优先使用真实 mount；直接命令用于脚本、诊断和不能挂载的环境。 +`vexdb fs find` 不逐项调用 `ls/stat`，而是一次进入数据库查询目录树元数据。第一版参数固定为 +名称 glob、文件类型、最小/最大大小、修改时间范围、路径游标和页大小；结果按二进制路径稳定 +排序，游标为排他下界，单页最多 1000 项。SQLite 递归队列只保存目录，避免平铺 10 万文件时 +反复递归全部文件；PostgreSQL 对 workspace owner/superuser 使用快速路径，其他 role 在递归和 +结果阶段执行 ACL 读取检查，不进入无权目录。文件正文不参与 `find`。 + #### 模式三：任意命令 worktree（受限环境回退） `sed -i`、`awk`、`jq`、`python`、`tar`、`git`、编译器和 `apply_patch` 都会调用操作系统文件接口。macOS FSKit mount 中它们可以直接工作；没有挂载能力时，使用受控 worktree。 diff --git a/docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md b/docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md index 6eda7d768b..4ae35f4430 100644 --- a/docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md +++ b/docs/plans/2026-07-21_vexfs-final-goal-and-roadmap.md @@ -157,7 +157,7 @@ vexdb fs --workspace workspace snapshot restore before-refactor | 范围 | 当前事实 | 主要缺口 | |---|---|---| -| 统一合同 | SQLite 合同 `0.9.0`，runtime ABI 1；SQL、C ABI、CLI、mount runtime 已连通；PostgreSQL `0.4.0-alpha.1` 已实现同一文件合同、format v2、ACL、审计、libpq HostStore 和远程 mount | DuckDB 尚未实现 VexFS adapter | +| 统一合同 | SQLite 合同 `0.9.0`，runtime ABI 1；SQL、C ABI、CLI、mount runtime 已连通；PostgreSQL `0.4.0-alpha.1` 已实现同一文件合同、format v2、ACL、审计、libpq HostStore 和远程 mount；两端数据库批量 `find` 已完成 | DuckDB 尚未实现 VexFS adapter | | 文件语义 | 文件、目录、四类时间戳、并发 append、进程锁、mode、symlink、xattr、hardlink、owner/group 元数据、便携 ACL 已进入数据库合同和测试；PG 已执行 ACL 和继承 | SQLite 尚未执行完整 ACL 授权；特殊文件不支持 | | 版本恢复 | 单文件版本、workspace commit、snapshot、diff、expected-head restore、原生备份、retention、显式分批 GC、live quota 和 format v2 逻辑导入导出已实现 | 自动维护、history/staging/index/total quota 和 DuckDB 导入端未完成 | | 长期校验 | `chunked-manifest-v1` 使用不可变 manifest、64 KiB 块、逐块 SHA-256 和有序块根哈希；deep check 逐块校验真实正文，publish 不再拼整文件 | 自动 repair、跨文件通用去重和按需标准整文件 SHA-256 命令不在当前范围 | @@ -165,7 +165,7 @@ vexdb fs --workspace workspace snapshot restore before-refactor | Linux | libfuse3 真实挂载已通过 root 和 uid 1000 各 9 组；时间戳、并发 append、进程锁、打开文件生命周期、强制卸载和 helper 崩溃恢复已验证；异常撤销后底层目录保持 0500，显式卸载恢复 0700；x86_64/AArch64 manylinux 安装包已在对应架构真机完成无 root 安装回归 | 更多干净发行版、长期运行和真实挂载安装回归仍不足 | | Windows | 只有平台边界和规划 | WinFsp adapter、安装签名、路径/ACL/SID 合同均未实现 | | 远程共享 | PostgreSQL 已通过多 gateway、macOS FSKit、Linux FUSE、数据库重启、macOS↔Linux 47 项、两台 Mac 文件/快照 50 项、双 Mac 串行真实 OpenCode 7 + 9 + 21 项和故障恢复 25 项；当前默认 NFS 又完成本机双 gateway 15 项和当前源码真实 TCP 中途断线连续 4/4 轮、每轮 21 项，PG `pg_trgm` 索引已实现 | 当前默认 NFS 源码对应的公证包和第二台 Mac NFS 复验尚未完成；同时覆盖同一文件只报告版本冲突，不自动合并 | -| 性能规模 | SQLite 直连已完成 10 万文件；PG 16 MiB 文件修改 4 KiB 的组合发布从 589.357 ms 降到中位 10.289 ms，publish-only 两次正式复跑中位 4.020/9.828 ms；逐文件后台事务后的 PG NFS 两轮 8 MiB 写 46.430～46.516 MiB/s、随机覆盖 386.139～482.743 ops/s；strict fsync → gateway SIGKILL → PG immediate restart 历史连续 20/20，当前修复源码 2/2；真实 TCP 黑洞重连最长约 5.02 秒有界返回并连续 4/4 轮恢复；服务端已提交但客户端不读结果的 generation 重试通过；1 GiB 容器 `oom_kill=0` | 仍需补远程 ARM 同版基准、1 万文件、长 Agent、冷读和长时间运行 | +| 性能规模 | SQLite 直连已完成 10 万文件；最新数据库 `find` 在 SQLite 1 万文件首/次页约 19.8/18.2 ms，10 万约 205.3/200.1 ms，RSS 约 93.8 MB；PG 1 万约 22.1/22.0 ms，10 万约 78.7/83.9 ms；PG 16 MiB 文件修改 4 KiB 的组合发布从 589.357 ms 降到中位 10.289 ms，publish-only 两次正式复跑中位 4.020/9.828 ms；逐文件后台事务后的 PG NFS 两轮 8 MiB 写 46.430～46.516 MiB/s、随机覆盖 386.139～482.743 ops/s；strict fsync → gateway SIGKILL → PG immediate restart 历史连续 20/20，当前修复源码 2/2；真实 TCP 黑洞重连最长约 5.02 秒有界返回并连续 4/4 轮恢复；服务端已提交但客户端不读结果的 generation 重试通过；1 GiB 容器 `oom_kill=0` | 仍需补远程 ARM 同版基准、长 Agent、冷读和长时间运行；PG 逐文件批量创建路径仍需优化 | “合同中已保存”不等于“所有平台已经完整执行”。例如 owner/group 和便携 ACL 已经可以保存、读取和恢复，但身份认证、权限判断和各系统的原生 ACL 映射仍属于后续阶段。 @@ -456,6 +456,20 @@ DuckDB adapter 已由其他方向负责，不在本路线继续；本路线只 Gate。实际 sleep/wake 仍需交互式管理员授权；局域网直连另需第二台 Mac 在线并允许 VexDB Lite 使用“本地网络”。 +### 7.1 发行 Gate 后的 workspace 恢复顺序 + +发行 Gate 收口后，workspace 版本恢复按 +`docs/plans/2026-07-28_vexfs-workspace-pitr-roadmap.md` 推进。固定顺序是： + +1. PG 多机恢复屏障； +2. 恢复前自动安全快照； +3. 统一恢复正确性 Gate； +4. workspace log、快照自动保留和 Agent 运行前 checkpoint； +5. SQLite 从任意保留 commit 创建快照； +6. 用真实规模数据决定是否投入 PG 逐 commit PITR。 + +PG 逐 commit PITR 不是默认下一项，不能为了统一宣传提前引入长期写放大。 + PostgreSQL Phase 2 的 format v2、ACL、审计、libpq HostStore、远程 mount、备份和故障恢复 已经完成。当前不再继续扩展 PG 代码；先关闭两台 Mac 对最终签名版本的系统授权验收点。DuckDB adapter 已明确不由本路线处理，后续平台开发只保留 Windows WinFsp。SQLite 的 10 万文件和小文件性能 @@ -674,3 +688,5 @@ macOS 完成 Apple 公证、干净机器安装和 extension 启用流程；Linux - Workspace 场景分析：`docs/analysis/2026-07-20_vexfs-claude-code-workspace-scenario.md` - POSIX 与快照边界：`docs/analysis/2026-07-20_vexfs-remaining-posix-snapshot-feasibility.md` - Eval 说明：`agent_files/eval/README.md` +- Workspace PITR 后续顺序：`docs/plans/2026-07-28_vexfs-workspace-pitr-roadmap.md` +- PITR 批次 A 实现报告：`docs/reports/2026-07-28_vexfs-pitr-batch-a.md` diff --git a/docs/plans/2026-07-28_vexfs-workspace-pitr-roadmap.md b/docs/plans/2026-07-28_vexfs-workspace-pitr-roadmap.md new file mode 100644 index 0000000000..7d5b8d02c0 --- /dev/null +++ b/docs/plans/2026-07-28_vexfs-workspace-pitr-roadmap.md @@ -0,0 +1,289 @@ +# VexFS workspace PITR 后续需求顺序 + +日期：2026-07-28 + +当前状态：批次 A 的数据库合同、CLI 和自动测试已完成。正式发布前仍需在最终 macOS NFS +包和 Linux FUSE 包上复跑真实挂载 Gate；这不是当前数据库实现缺口。 + +## 1. 产品目标 + +这一阶段不追求数据库宣传语里的“任意一秒恢复”。先完成 Coding Agent 真正需要的闭环： + +1. Agent 运行前自动留下恢复点。 +2. 用户能看懂这些恢复点和 workspace 变化历史。 +3. 一条命令恢复整个 workspace。 +4. 恢复期间不会被另一台机器继续写坏。 +5. 恢复错了还能回到恢复前。 + +当这个闭环稳定后，再补 SQLite 任意 commit 恢复。PostgreSQL 的完整逐 commit PITR 根据真实需求决定，不提前增加长期写放大。 + +## 2. 固定优先级 + +### P0-0：数据库批量 find + +状态：**已完成（2026-07-28）**。 + +- SQLite 和 PostgreSQL 均实现 `vexfs_find`，C ABI 与 `vexdb fs find` 已接通； +- 第一版范围固定为名称、类型、大小、修改时间、稳定路径游标和分页； +- PG 普通 role 不进入无读取权限目录，SQLite/PG 的 `?` 均按一个 Unicode 字符匹配； +- 1 万与 10 万文件功能、性能和 RSS 已进入 eval；详细证据见 + `docs/reports/2026-07-28_vexfs-database-find.md`。 + +这项能力先于 workspace log 完成，因为系统 `find` 只能在挂载后使用，数据库批量查询是 CLI、 +远程 PG 和未挂载环境共同需要的基础能力。 + +### P0-1：PostgreSQL 多机恢复屏障 + +状态：**已完成（2026-07-28）**。 + +这是后续所有恢复能力的前置条件。 + +需求： + +- PG 服务端恢复入口拒绝仍有有效 mount lease 的 workspace。 +- mount 注册与 snapshot restore 必须使用同一个 workspace 锁，不能出现“检查完没有 mount，马上又挂载”的竞态。 +- 恢复开始后，旧 session 和旧 handle 不能再发布数据。 +- 第一版只做安全拒绝，不做远程强制踢下线。 +- CLI 返回活动 session 数量和明确卸载建议，不只返回模糊的 busy。机器标识尚未写入 + mount lease，因此第一版不返回机器名。 + +验收： + +- Mac A 和 Mac B 同时挂载时，A 恢复必须返回 `VEXFS_MOUNT_BUSY`，数据库内容不变化。 +- B 正常卸载或 lease 到期后，A 可以恢复成功。 +- B 的旧 dirty handle 不能在恢复完成后重新发布。 +- 并发 mount/restore、断线、lease 到期、PG 重启都进入 eval。 + +### P0-2：恢复前自动安全快照 + +状态：**已完成（2026-07-28）**。 + +需求： + +- `snapshot restore NAME` 在真正恢复前，自动为当前 HEAD 创建系统安全快照。 +- 安全快照名包含 workspace、HEAD 和时间，并保证不会重名。 +- CLI 成功输出同时返回 `restored_snapshot` 和 `safety_snapshot`。 +- 安全快照创建和恢复在同一个数据库事务内；恢复失败时两者一起回滚，不留下半恢复状态。 +- 第一版不提供默认关闭选项；以后如需 `--no-safety-snapshot`，必须作为明确的危险选项。 + +原因： + +- SQLite 能从旧 commit 重建树，但公共接口尚未开放。 +- PG 只能从命名快照重建完整树。没有安全快照就无法保证撤销一次错误恢复。 + +验收： + +- SQLite、PG 都能执行“恢复 A -> 恢复安全快照 -> 回到恢复前状态”。 +- 文件内容、目录、Git、mode、链接、owner、ACL、xattr 和索引状态完全一致。 +- 配额不足、快照损坏、重挂载失败和进程退出都有原子性测试。 + +### P0-3：统一恢复正确性 Gate + +状态：**数据库与 CLI Gate 已完成；发行物真实挂载复跑待下一次打包执行**。 + +P0-1 和 P0-2 完成后，建立一个不能跳过的统一 Gate： + +- SQLite 本地挂载。 +- PG 单机挂载。 +- PG 两个 gateway。 +- macOS NFS 和 Linux FUSE 共用同一套步骤。 +- `expected_head` 冲突、远端活动 mount、旧 handle 发布、恢复撤销、数据库重启。 +- 测试继续限制并发、文件数和 RSS，避免爆内存。 + +完成标准：所有 P0 用例进入 `tests/eval/vexfs/`，不能只保留手工脚本。 + +当前自动证据包括 SQLite C ABI/CLI eval、PG spec、PG C ABI runtime、PG adapter 并发 +eval 和 PG CLI runtime。增强后的 `run_pg_runtime.sh` 会实际执行“恢复目标快照 -> 恢复自动 +安全快照 -> 再次恢复目标快照”。 + +## 3. Agent 可用闭环 + +### P1-1：统一 workspace 日志 + +增加： + +```bash +vexdb fs workspace log +vexdb fs workspace log --limit 50 --json +``` + +统一字段： + +- commit ID +- parent commit +- 时间 +- 操作类型 +- path +- actor +- session/run ID +- 是否有对应快照 + +要求： + +- SQLite 和 PG 输出同一 JSON 合同。 +- 默认按新到旧排列，支持有界分页。 +- 查询不能扫描文件正文，也不能在大历史上一次加载全部结果。 +- actor 和 run ID 是通用审计字段，核心中不保存 prompt、memory 或模型语义。 + +### P1-2：快照分类和自动保留策略 + +在自动创建 Agent 快照前，先解决无限增长问题。 + +快照至少区分： + +- `manual`：用户手工创建，默认长期保留。 +- `agent`：Agent 任务前后创建，按策略清理。 +- `safety`：恢复前自动创建，保留最近若干个或若干天。 + +命令： + +```bash +vexdb fs snapshot policy show +vexdb fs snapshot policy set --agent-keep 20 --safety-keep 10 --days 30 +vexdb fs snapshot prune --dry-run +vexdb fs snapshot prune +``` + +要求： + +- 被保留的快照继续保护它引用的内容和历史。 +- prune 与 GC 分开：先删除过期快照引用，再由 GC 分批回收。 +- doctor/check 能报告快照数、受保护字节、可回收字节和最老恢复点。 + +### P1-3：Agent 运行前自动 checkpoint + +增加通用命令，不绑定某一个 Agent： + +```bash +vexdb fs run --snapshot-before -- opencode run ... +vexdb fs run --snapshot-before -- claude ... +``` + +行为： + +1. 确认命令运行目录属于目标 workspace。 +2. 等待已发布状态，创建 `agent` 快照。 +3. 记录 run ID、actor、命令类型和开始时间。 +4. 执行原命令，原样传递 stdin/stdout/stderr 和退出码。 +5. 可选 `--snapshot-after-success` 创建完成快照。 +6. 输出可以直接复制使用的恢复命令。 + +边界： + +- 不分析 prompt，不判断 Agent 做了什么。 +- 不把 Agent 专用字段放进文件系统核心。 +- Bash 直接运行仍然可用；`fs run` 只是增加明确的任务边界。 + +验收： + +- OpenCode 修改 Git workspace、执行测试、失败后恢复到运行前。 +- 命令被 Ctrl-C、进程崩溃、网络断开时，运行前快照仍然可用。 +- SQLite 与远程 PG 使用相同命令和输出。 + +## 4. SQLite commit 级恢复 + +### P2-1：从历史 commit 固定成快照 + +增加： + +```bash +vexdb fs snapshot create before-bug --commit 123 +``` + +不直接增加 `restore --commit`。先把历史 commit 固定为快照，再复用现有 diff、dry-run、安全快照和 restore 路径。 + +要求： + +- commit 必须属于当前 workspace。 +- commit 不能早于 `history_floor_commit`。 +- 创建后立即保护依赖的 file version、manifest、xattr 和 ACL 历史。 +- 损坏、缺失内容和配额问题在创建阶段就失败。 +- 第一版 SQLite 可用；PG 返回明确的 capability 错误，不能假装支持。 + +### P2-2：历史 commit 查看和比较 + +在 P2-1 稳定后增加： + +```bash +vexdb fs workspace show --commit 123 +vexdb fs workspace diff --from 123 --to HEAD +``` + +大 workspace 的输出必须流式或分页；默认只输出变化摘要，不直接输出所有文件。 + +### P2-3：按时间选择 commit + +`--at TIME` 只是“找出不晚于该时间的最后一个 commit”，最终仍转换成 commit ID 和命名快照： + +```bash +vexdb fs snapshot create before-noon --at '2026-07-28T12:00:00+08:00' +``` + +要求明确显示实际选中的 commit 和时间，不能让用户误以为文件系统能恢复到两个 commit 之间。 + +## 5. PostgreSQL 后续选择 + +### P3-1：先评估命名快照是否够用 + +完成 Agent 自动 checkpoint 后，先测： + +- 1 千、1 万和长期 workspace 的快照时间。 +- snapshot 表增长速度。 +- prune、GC、备份和恢复耗时。 +- 远程网络下 Agent 启动增加的等待时间。 + +如果命名快照已经满足 Agent 运行前恢复，不做 PG 逐 commit 历史。 + +### P3-2：只在真实需求成立时做 PG 任意 commit PITR + +触发条件至少满足一项： + +- 用户明确需要恢复到没有预先创建快照的历史 commit。 +- 自动快照的存储或创建成本无法接受。 +- 审计法规要求每次 workspace commit 都可重建。 + +触发后再设计 PG 的 per-commit inode/dentry/xattr/ACL 状态或不可变 tree manifest，并同时设计 retention、GC、备份和空间上限。不能只增加一个 CLI 参数。 + +## 6. 当前不排入开发 + +- DuckDB adapter：已明确不在本方向处理。 +- 直接 `restore --commit`：先固定成快照，保证可审计和可回收。 +- 真正连续到任意秒的 workspace 恢复：文件系统只有 commit 边界。 +- 自动合并两个 Agent 的并发修改：这是协作/版本控制问题，不属于恢复阶段。 +- 把 prompt、模型 memory 或 embedding 放入 VexFS 核心。 +- 为了 PITR 改回 FSKit 主路线；恢复合同应独立于 NFS、FUSE、FSKit。 + +## 7. 实施顺序总表 + +| 顺序 | 需求 | 优先级 | 完成后得到什么 | +|---:|---|---|---| +| 0 | 数据库批量 find（已完成） | P0 | 未挂载时也能高效查文件元数据 | +| 1 | PG 多机恢复屏障 | P0 | 恢复期间不会被其他机器继续写坏 | +| 2 | 恢复前自动安全快照 | P0 | 错误恢复可以撤销 | +| 3 | 统一恢复正确性 Gate | P0 | SQLite、PG、macOS、Linux 不会各自漂移 | +| 4 | workspace log | P1 | 用户能找到和理解历史版本 | +| 5 | 快照分类、保留和 prune | P1 | 自动快照不会无限增长 | +| 6 | `vexdb fs run --snapshot-before` | P1 | Agent 任务前自动留下恢复点 | +| 7 | SQLite `snapshot create --commit` | P2 | 可以固定任意仍被保留的 SQLite commit | +| 8 | commit show/diff | P2 | 先看清差异再选择恢复点 | +| 9 | `--at TIME` 到 commit | P2 | 提供时间入口，但保持真实 commit 语义 | +| 10 | PG 快照规模与长期评估 | P3 | 用数据决定是否做完整 PG PITR | +| 11 | PG 逐 commit PITR | P3 条件项 | 只在真实需求成立时投入 | + +## 8. 最近三个开发批次 + +### 批次 A：恢复安全（数据库与 CLI 已完成） + +只做顺序 1～3。结束条件是两台 gateway 仍挂载时恢复被安全拒绝，全部卸载后恢复成功，且可以通过自动安全快照撤销。 + +完成证据：`docs/reports/2026-07-28_vexfs-pitr-batch-a.md`。 + +### 批次 B：Agent 闭环 + +做顺序 4～6。结束条件是用户可以运行一条 `vexdb fs run`，让 OpenCode 修改项目，并用输出中的恢复命令回到任务前。 + +### 批次 C：SQLite commit PITR + +做顺序 7～9。结束条件是用户能从日志选择一个 SQLite commit，固定成快照，查看差异并走统一恢复流程。 + +PG 逐 commit PITR 不进入这三个批次。 diff --git a/docs/reports/2026-07-28_vexfs-database-find.md b/docs/reports/2026-07-28_vexfs-database-find.md new file mode 100644 index 0000000000..95f6a30248 --- /dev/null +++ b/docs/reports/2026-07-28_vexfs-database-find.md @@ -0,0 +1,72 @@ +# VexFS 数据库批量 find 实现与测试报告 + +- 日期：2026-07-28 +- 分支：`feature/agent_files` +- 基线提交：`90b29ec080`（本报告对应未提交工作区修改） +- 范围：SQLite、PostgreSQL、公共 C ABI、`vexdb fs find`、功能和性能 eval +- 不包含：DuckDB、系统挂载目录上的原生 `find` 加速、文件正文搜索 + +## 1. 结论 + +数据库批量 `find` 已完成。它是数据库管理的文件元数据查询，不要求先 mount，也不会读取文件 +正文。SQLite 和 PostgreSQL 共享以下用户合同： + +- 根路径； +- 名称 glob：`*` 和 `?`，其中 `?` 匹配一个 Unicode 字符； +- 类型：file、directory、symlink； +- 最小/最大大小； +- 修改时间起止毫秒； +- 排他的路径游标； +- 1～1000 的页大小； +- 按二进制路径稳定排序的 JSON 结果和 `next_cursor`。 + +CLI 文本模式每行输出一个路径，`--json` 输出完整结构。PG 普通 role 的递归会在无读取权限的 +目录前停止，并过滤无读取权限对象；workspace owner 和 superuser 走快速路径。 + +## 2. 实现要点 + +最初的递归 CTE 把所有文件也放进递归队列。SQLite 1 万文件首屏查询达到 26.527 秒，这是 +真实问题。修复后，递归队列只枚举目录，再一次连接每个目录的直接子项。相同 1 万文件首屏降到 +约 20 毫秒，且不读取正文。 + +SQLite 名称匹配使用 UTF-8 字符边界，避免 `?` 把一个中文字符当作三个字节。PostgreSQL 使用 +等价的字符级正则。PG ACL 只在普通 role 路径执行逐对象检查，避免 owner 基准被无意义拖慢。 + +## 3. 最新性能结果 + +测试都使用真实数据库文件或真实 PG 容器，页大小 100；10 万规模期间没有扩大并发。 + +| 后端 | 文件数 | 夹具生成 | 首屏 | 第二页 | 精确名称 | 数据库/RSS | +|---|---:|---:|---:|---:|---:|---:| +| SQLite | 10,000 | 6.340 s | 19.814 ms | 18.238 ms | 21.595 ms | DB 9.97 MB，RSS 22.00 MB | +| SQLite | 100,000 | 65.711 s | 205.268 ms | 200.127 ms | 235.201 ms | DB 101.76 MB，RSS 93.78 MB | +| PostgreSQL | 10,000 | 1.987 s | 22.106 ms | 22.042 ms | 11.771 ms | 容器内批量元数据夹具 | +| PostgreSQL | 100,000 | 173.883 s | 78.709 ms | 83.903 ms | 10.203 ms | 容器内批量元数据夹具 | + +PG 性能脚本直接批量生成合法的 live inode/dentry 树，用于隔离 `find` 查询成本。公开 +`vexfs_create` 在 10 万文件时会同时生成 commit、manifest、版本和 ACL 历史，单轮超过 15 分钟， +因此不能当作查询基准的夹具生成器。这是另一个真实写入性能任务，不计入 `find` 查询耗时。 + +## 4. 已通过测试 + +- SQLite 静态合同：`VEXFS STATIC SMOKE: PASS`； +- SQLite runtime：`VEXFS RUNTIME SMOKE: PASS`； +- CLI：69 checks； +- SQLite 1 万/10 万性能：各 10 checks，RSS 小于 512 MiB； +- PG find spec：名称、类型、大小、时间、游标、Unicode、ACL； +- PG adapter：160 checks； +- PG C ABI runtime：148 checks； +- PG CLI runtime：45 checks； +- PG 1 万/10 万性能门槛均通过。 + +SQLite 原始报告： + +- `vexdb_sqlite/build/eval/vexfs/20260728T050841.598754Z-full-20260718/report.json` +- `vexdb_sqlite/build/eval/vexfs/20260728T050855.805263Z-stress-20260718/report.json` + +## 5. 后续任务 + +1. 优化 PG 大批量创建，避免逐文件生成历史时耗时线性放大到不可接受的分钟级。 +2. 做 workspace log、快照分类/保留和 `run --snapshot-before`。 +3. 再做 SQLite 任意 commit 固定、show/diff 和按时间选择 commit。 +4. PG 逐 commit PITR 仍是条件项，不因 `find` 完成而提前投入。 diff --git a/docs/reports/2026-07-28_vexfs-pitr-batch-a.md b/docs/reports/2026-07-28_vexfs-pitr-batch-a.md new file mode 100644 index 0000000000..bd8a4d00b8 --- /dev/null +++ b/docs/reports/2026-07-28_vexfs-pitr-batch-a.md @@ -0,0 +1,71 @@ +# VexFS workspace PITR 批次 A 实现报告 + +日期：2026-07-28 + +## 结论 + +批次 A 的数据库合同和 CLI 已完成： + +1. PostgreSQL 恢复会阻止其他仍有效的 gateway session。 +2. mount 注册、heartbeat 和恢复共用 workspace 行锁，关闭了检查与新挂载之间的竞态。 +3. SQLite 和 PostgreSQL 在恢复前都会在同一事务内创建自动安全快照。 +4. CLI 返回 `safety_snapshot`，并可用它把 workspace 恢复到操作前。 +5. 有界 eval 已覆盖并发、恢复撤销、C ABI、SQL 合同和 PG CLI 端到端流程。 + +正式发行前还要在最终 macOS NFS 与 Linux FUSE 包上复跑真实挂载 Gate。恢复合同不依赖 +mount adapter，本次没有改 NFS、FUSE 或 FSKit 实现。 + +## 关键行为 + +### PG 多机恢复屏障 + +- `vexfs_mount_session_start()`、heartbeat 和 restore 都先锁 workspace 行。 +- restore 可以排除发起恢复的当前 gateway，但会统计并拒绝其他有效 lease。 +- 直接 SQL 调用没有 caller session，因此会拒绝全部有效 mount。 +- restore 同时拒绝 open/retained handle。 +- 过期 session 不能被旧 heartbeat 重新续租。 +- 当前第一版只返回其他活动 session 数量，不自动踢下线，也没有机器名字段。 + +### 自动安全快照 + +- CLI 名称格式为 `vexfs-safety--h-