diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index b9b367b9..1d08dfc0 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -12,7 +12,7 @@ permissions: contents: write concurrency: - group: "pages" + group: 'pages' cancel-in-progress: true jobs: @@ -36,6 +36,9 @@ jobs: - name: Build modules run: bun compile + env: + GDRIVE_CLIENT_ID: ${{ secrets.GDRIVE_CLIENT_ID }} + GDRIVE_CLIENT_SECRET: ${{ secrets.GDRIVE_CLIENT_SECRET }} - name: Sync modules run: bun scripts/deploy-modules.ts diff --git a/.gitignore b/.gitignore index b36980bd..476e46b5 100644 --- a/.gitignore +++ b/.gitignore @@ -11,5 +11,6 @@ Sync Engine Logs **/.turbo **/node_modules **/cache +**/.env !packages/plugin/dist packages/plugin/dist/**/*.js diff --git a/.oxlintrc.json b/.oxlintrc.json index e99d3935..859a77ac 100644 --- a/.oxlintrc.json +++ b/.oxlintrc.json @@ -65,7 +65,7 @@ "react/jsx-props-no-spreading": "off", "import/no-unassigned-import": [ "warn", - { "allow": ["**/*.css", "**/*.scss", "**/*.less"] } + { "allow": ["**/*.css", "**/*.scss", "**/*.less", "tsdown/client"] } ], "eslint/no-underscore-dangle": [ "warn", @@ -122,7 +122,7 @@ "unicorn/import-style": "off", "unicorn/no-process-exit": "off", "eslint/one-var": ["warn", "never"], - "unicorn/max-nested-calls": ["warn", { "max": 5 }], + "unicorn/max-nested-calls": ["warn", { "max": 5 }] }, "env": { "builtin": true, @@ -136,7 +136,8 @@ "activeWindow": "readonly", "createFragment": "readonly", "createEl": "readonly", - "createDiv": "readonly" + "createDiv": "readonly", + "sleep": "readonly" }, "overrides": [ { diff --git a/AGENTS.md b/AGENTS.md index 82e00e27..2efc8d73 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -36,6 +36,7 @@ This is the monorepo for an extensible Obsidian syncing plugin to sync vault fil - Plugin & module SDK: `packages/plugin/`, package name `@hesprs/sync-engine-sdk`, `dev` builds SDK. - WebDAV module: `packages/webdav/`, package name `webdav`. - S3 module: `packages/s3/`, package name `s3`. +- Google Drive module: `packages/gdrive/`, package name `gdrive`. - Encryption module: `packages/encryption/`, package name `encryption`. - Shared utils: `packages/shared/`, package name `@repo/shared`. - Documentation site: `docs/`, package name `docs`. diff --git a/bun.lock b/bun.lock index 45894ee4..257cc31e 100644 --- a/bun.lock +++ b/bun.lock @@ -43,6 +43,16 @@ "uni-kv": "../../uni-kv.tgz", }, }, + "packages/gdrive": { + "name": "gdrive", + "version": "0.0.1", + "devDependencies": { + "@hesprs/sync-engine-sdk": "workspace:*", + "@repo/shared": "workspace:*", + "hash-wasm": "^4.12.0", + "uni-kv": "../../uni-kv.tgz", + }, + }, "packages/i18n": { "name": "i18n", "version": "0.0.1", @@ -687,6 +697,8 @@ "fsevents": ["fsevents@2.3.3", "", { "os": "darwin" }, "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw=="], + "gdrive": ["gdrive@workspace:packages/gdrive"], + "gensync": ["gensync@1.0.0-beta.2", "", {}, "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg=="], "get-tsconfig": ["get-tsconfig@5.0.0-beta.5", "", { "dependencies": { "resolve-pkg-maps": "^1.0.0" } }, "sha512-/6gFNr0N04nob252sTQxyFLi3eKFRqIg1I87YcqAMT1i6SQrSF6KujUEQrtrjMV0H/eejTCltLdDSTEMzHbnsQ=="], diff --git a/docs/.vitepress/config.ts b/docs/.vitepress/config.ts index d1fa33b9..e85cbaba 100644 --- a/docs/.vitepress/config.ts +++ b/docs/.vitepress/config.ts @@ -156,6 +156,7 @@ const localeConfig = configGenerator((t) => { items: [ { link: `${deepDive}/modules/webdav`, text: t('webdav') }, { link: `${deepDive}/modules/s3`, text: t('s3') }, + { link: `${deepDive}/modules/gdrive`, text: t('gdrive') }, { link: `${deepDive}/modules/encryption`, text: t('encryption') }, { lint: `${deepDive}/modules/smart-marge`, text: t('smartMerge') }, ], diff --git a/docs/.vitepress/i18n.ts b/docs/.vitepress/i18n.ts index fb2892fc..c174214d 100644 --- a/docs/.vitepress/i18n.ts +++ b/docs/.vitepress/i18n.ts @@ -23,6 +23,7 @@ const en = { fileSystemWrappers: 'File System Wrappers', fileTree: 'File Tree', folder: '', + gdrive: 'Google Drive', home: 'Home', licenseMessage: 'All content licensed under the CC BY 4.0 License.', diff --git a/docs/src/pages/en/deep-dive/modules/gdrive.md b/docs/src/pages/en/deep-dive/modules/gdrive.md new file mode 100644 index 00000000..dd962259 --- /dev/null +++ b/docs/src/pages/en/deep-dive/modules/gdrive.md @@ -0,0 +1,184 @@ +# Google Drive Module + +The Google Drive module registers the `gdrive` remote file system. It stores the vault in a Google Drive folder and performs file operations through the Google Drive public API. The module uses Google OAuth for authentication and does not use a third-party server or proxy. + +## Settings + +The module adds a **Google Drive** settings group. Configure these settings +before selecting Google Drive as the remote file system: + +### Connect account / Account connected + +This setting initiates the Google authorization process. Once authorized, the module stores the refresh token securely in Obsidian's secret storage and uses it to obtain short-lived access tokens as needed. The **Disconnect** option revokes the token with Google, removes it from secret storage, clears any cached access tokens, and deletes the stored account identifier. + +### Base directory + +This defines the folder in Google Drive that serves as the root directory for this vault. The value is normalized as a directory path. If left empty when the module starts, it defaults to `/`. The specified folder is created automatically during the first synchronization. + +::: warning + +The base directory is an application-managed namespace. The module can see and operate only on files that it created through its Google Drive integration. Files or folders created manually in Drive, or created by another application, are not visible to this module even when they are inside the configured base directory. They will not be imported, synchronized, or listed. + +Do not manually create, rename, move, or maintain vault files in this Drive folder. Set the base directory in the module settings, then let the first sync create it and let Sync Engine manage its contents. Manual Drive operations can leave files outside the module's visible set or cause conflicting changes. + +This is a consequence of Google's `drive.file` scope, not a filtering option that can be disabled in Sync Engine. The module is intentionally not granted full Drive access. + +::: + +### Delete to trash + +When enabled, files deleted remotely are moved to the Google Drive trash instead of being permanently removed. When disabled, deleted files are permanently erased immediately. Note that Google Drive typically clears items from the trash after 30 days. + +## Permissions And Scopes + +The module requests exactly these OAuth scopes: + +### `drive.file` + +This scope permits the module to create and manage files that it creates in Google Drive. Sync requires this access to: + +- Create the base directory and vault files +- List the module's files so it can discover remote changes +- Read file contents and metadata +- Upload new contents and update existing files +- Move or rename files +- Delete files, either permanently or by moving them to trash + +The scope does **not** grant general access to the user's Drive. In particular, the module cannot discover or synchronize files created outside the plugin. Granting `drive.file` is a deliberate least-privilege choice: the module can manage its own sync data without receiving permission to read unrelated Drive files. + +### `openid` + +The `openid` scope makes Google return an OpenID Connect ID token during device authorization. The module reads the token's stable `sub` subject identifier and stores it as the connected account identifier. This lets Sync Engine identify which Google account is connected and distinguish multiple account connections, so that the sync record for different Google accounts don't interfere; it does not read the user's profile or request broad identity permissions. + +## Authentication Flow + +The module uses [**Google OAuth 2.0 for TV and Limited-Input Device Applications**](https://developers.google.com/identity/protocols/oauth2/limited-input-device): + +1. When you click **Connect**, the module requests a device code from Google's device authorization endpoint with the two scopes above. +2. Obsidian displays Google's verification URL and a one-time user code. The module can copy the code and open the URL in a browser. +3. You sign in to Google in that browser and approve the requested access. +4. While the dialog remains open, Obsidian polls Google's token endpoint. It waits when authorization is pending and backs off when Google requests a slower polling interval. +5. After approval, Google returns a short-lived access token, a refresh token, and an OpenID ID token. The module extracts the account subject from the ID token, stores the refresh token in Obsidian's secret storage, and caches the access token in memory. +6. Later Drive requests use the cached access token. When it is close to expiry, the module exchanges the refresh token for a new access token. A failed request with HTTP 401 causes one forced refresh and retry. + +Device authorization is used because Obsidian mobile cannot reliably provide the local browser redirect, localhost listener, or desktop-style custom URL callback required by common interactive OAuth flows. Device authorization keeps the OAuth interaction in a normal browser while the Obsidian app polls Google's endpoint, so the same connection flow works on desktop and mobile. + +## Privacy Policy + +Last updated on **August 23, 2026**. + +### Introduction + +This Privacy Policy describes how the Sync Engine plugin for Obsidian with Google Drive module (“the Plugin”) handles your data when you connect a Google Drive account. The Plugin is open-source software licensed under the MIT License. + +### Data We Collect + +**We collect no data.** The Plugin has no telemetry, no analytics, no remote logging service, and no backend server. No information about you, your files, or your usage ever leaves your local device. + +### How Your Data Is Handled + +**Google Account Connection**: + +When you click “Connect,” the Plugin initiates a Google Device Authorization flow directly between your device and Google’s servers. The Plugin requests only these scopes: + +- `drive.file`: Access to files created by the Plugin. Files created outside the Plugin are not visible to it, even inside the configured base directory. +- `openid`: Supplies a stable Google account subject identifier so the Plugin can identify and deduplicate connections + +**Token Storage**: + +OAuth refresh tokens are stored exclusively in Electron’s encrypted secret storage on your local device. Tokens are never transmitted to any third party, never logged, and never included in crash reports or diagnostics. + +**File Operations**: + +All sync operations are triggered or scheduled manually by you. File reads and writes occur directly between your local Obsidian vault and Google Drive via Google’s API. No intermediary servers are involved. + +**Data Retention**: + +Your data exists only on your local device and in your own Google Drive account. When you click “Disconnect,” the Plugin: + +1. Revokes the OAuth token with Google +2. Deletes all stored tokens from Electron secret storage + +After disconnection, no trace of your Google Account connection remains on your device. + +### Third Parties + +The only third-party service involved is Google’s OAuth and Drive API, which you authorize directly. We have no relationship with Google beyond using their public APIs. We do not share, sell, or transfer any data to any entity. + +### Your Rights + +You have complete control: + +- All data is on your local device; inspect it anytime +- Click “Disconnect” to erase all local credentials instantly +- Revoke access anytime at `https://myaccount.google.com/permissions` +- Your files remain in your Google Drive regardless of Plugin status + +### Changes + +Updates to this policy will be published in the Plugin’s GitHub repository and `https://sync.consensia.cc`. Continued use after changes constitutes acceptance. + +### Contact + +Open an issue on our GitHub repository for privacy-related questions. + +## Terms of Service + +Last updated on **August 23, 2026**. + +### Acceptance + +By installing or using the Sync Engine plugin for Obsidian with Google Drive module (“the Plugin”), you agree to these Terms. If you disagree, uninstall the Plugin immediately. + +### License + +The Plugin is provided under the MIT License. You may use, modify, and distribute it freely per that license’s terms. + +### No Warranty + +THE PLUGIN IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND NONINFRINGEMENT. THE AUTHORS AND COPYRIGHT HOLDERS SHALL NOT BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY ARISING FROM USE OF THE PLUGIN, INCLUDING DATA LOSS, SYNC FAILURES, OR GOOGLE DRIVE API CHANGES. + +### User Responsibilities + +You are solely responsible for: + +- Maintaining the security of your Google Account credentials +- Understanding what data you choose to sync +- Backing up important files independently of the Plugin +- Complying with Google’s Terms of Service when connecting your Drive account +- Ensuring your local device’s Electron secret storage remains secure + +### Acceptable Use + +Do not use the Plugin to: + +- Violate Google’s API Terms of Service +- Access accounts you do not own or lack authorization for +- Circumvent Google Drive storage or rate limits +- Distribute malware or illegal content via synced files + +### Third-Party Services + +The Plugin interacts with Google’s OAuth and Drive APIs. These services are governed by Google’s own Terms of Service and Privacy Policy. We have no control over Google’s services and accept no liability for their availability, changes, or termination. + +### Disconnection & Termination + +You may terminate your use at any time by clicking “Disconnect” in the Plugin settings or uninstalling the Plugin. We reserve the right to discontinue development or distribution of the Plugin at any time without notice. + +### Limitation of Liability + +TO THE MAXIMUM EXTENT PERMITTED BY LAW, IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE FOR ANY INDIRECT, INCIDENTAL, SPECIAL, CONSEQUENTIAL, OR PUNITIVE DAMAGES, INCLUDING LOST PROFITS, DATA LOSS, OR BUSINESS INTERRUPTION, REGARDLESS OF THEORY OF LIABILITY. + +### Governing Law + +This Plugin is developed and maintained on a voluntary, non-commercial basis by contributors located in multiple jurisdictions worldwide. No single governing law applies. + +These Terms shall be interpreted in accordance with general principles of international law and the MIT License under which the Plugin is distributed. + +### Changes + +We may update these Terms at any time. Changes take effect upon publication in the GitHub repository. Continued use constitutes acceptance. + +### Contact + +Open an issue on our GitHub repository for questions regarding these Terms. diff --git a/manifest.json b/manifest.json index 6ea798ef..1f2e5d85 100644 --- a/manifest.json +++ b/manifest.json @@ -1,7 +1,7 @@ { "id": "sync-engine", "name": "Sync Engine", - "version": "3.1.0", + "version": "3.1.1", "minAppVersion": "1.13.0", "authorUrl": "https://hesprs.github.io", "description": "The next-generation syncing plugin: Fast · Free · Extend with Modules. Supports WebDAV and S3.", diff --git a/packages/gdrive/package.json b/packages/gdrive/package.json new file mode 100644 index 00000000..c3e4a1bf --- /dev/null +++ b/packages/gdrive/package.json @@ -0,0 +1,31 @@ +{ + "name": "gdrive", + "version": "0.0.1", + "private": true, + "license": "MIT", + "contributors": [ + { + "name": "Aaron", + "github": "Quzzar" + }, + { + "name": "Hēsperus", + "email": "hesprs@outlook.com", + "github": "hesprs" + } + ], + "type": "module", + "scripts": { + "dev": "MODE=dev bun --bun tsdown", + "compile": "bun --bun tsdown", + "fix": "oxlint --silent --fix && oxfmt", + "check": "tsc && oxlint && oxfmt --check", + "tests": "bun --preload=./test/mocks test" + }, + "devDependencies": { + "@hesprs/sync-engine-sdk": "workspace:*", + "@repo/shared": "workspace:*", + "hash-wasm": "^4.12.0", + "uni-kv": "../../uni-kv.tgz" + } +} diff --git a/packages/gdrive/src/gdrive/api.ts b/packages/gdrive/src/gdrive/api.ts new file mode 100644 index 00000000..c5dff975 --- /dev/null +++ b/packages/gdrive/src/gdrive/api.ts @@ -0,0 +1,66 @@ +import type { FileStat, RequestResponse } from '@hesprs/sync-engine-sdk'; + +export const DRIVE_API = 'https://www.googleapis.com/drive/v3'; +export const DRIVE_UPLOAD_API = 'https://www.googleapis.com/upload/drive/v3'; +export const OAUTH_DEVICE_CODE_URL = 'https://oauth2.googleapis.com/device/code'; +export const OAUTH_TOKEN_URL = 'https://oauth2.googleapis.com/token'; +export const OAUTH_SCOPE = 'https://www.googleapis.com/auth/drive.file openid'; +export const FOLDER_MIME = 'application/vnd.google-apps.folder'; +export const FILE_FIELDS = 'id,name,mimeType,md5Checksum,modifiedTime,size,parents'; +export const TOKEN_REVOKE_URL = 'https://oauth2.googleapis.com/revoke'; + +export type DriveFile = { + id: string; + name: string; + mimeType: string; + md5Checksum?: string; + modifiedTime?: string; + size?: string; + parents?: Array; +}; + +export type DriveFileList = { + files?: Array; + nextPageToken?: string; +}; + +const mtimeMissing = new Error('Google Drive did not return the modified time for a file!'); + +/** Escapes a string literal used inside a Drive `q` search expression. */ +export function escapeQuery(value: string): string { + return value.replaceAll('\\', String.raw`\\`).replaceAll("'", String.raw`\'`); +} + +export function buildUrl(base: string, path: string, query: Record = {}): string { + const url = new URL(`${base}${path}`); + for (const [key, value] of Object.entries(query)) url.searchParams.set(key, value); + return url.toString(); +} + +export function getHeader( + headers: Record, + name: string, +): string | undefined { + const entry = Object.entries(headers).find( + ([headerName]) => headerName.toLowerCase() === name.toLowerCase(), + ); + return entry?.[1]; +} + +export function parseDriveError(response: RequestResponse): string | undefined { + const parsed = response as { + error?: { code?: number; message?: string } | string; + error_description?: string; + }; + if (typeof parsed.error === 'string') + return `Google Drive ${parsed.error}: ${parsed.error_description ?? ''}`; + if (parsed.error?.message) + return `Google Drive ${parsed.error.code ?? response.status}: ${parsed.error.message}`; +} + +export function toFileStat(key: string, file: DriveFile): FileStat { + if (!file.modifiedTime) throw mtimeMissing; + const mtime = new Date(file.modifiedTime).valueOf(); + const size = file.size === undefined ? 0 : Number.parseInt(file.size); + return { isDir: false, key, mtime, size, uid: file.md5Checksum ?? `${mtime}~${size}` }; +} diff --git a/packages/gdrive/src/gdrive/auth.ts b/packages/gdrive/src/gdrive/auth.ts new file mode 100644 index 00000000..f99013b4 --- /dev/null +++ b/packages/gdrive/src/gdrive/auth.ts @@ -0,0 +1,249 @@ +import type { Request, RequestParam } from '@hesprs/sync-engine-sdk'; +import { getStatus } from '@repo/shared/get-status'; +import { requestUrl, SecretStorage } from 'obsidian'; +import { + buildUrl, + OAUTH_DEVICE_CODE_URL, + OAUTH_SCOPE, + OAUTH_TOKEN_URL, + TOKEN_REVOKE_URL, +} from './api'; + +export const CLIENT_ID = atob(process.env.CLIENT_ID ?? ''); +export const CLIENT_SECRET = atob(process.env.CLIENT_SECRET ?? ''); // Not really a secret +const KEYCHAIN_SECRET_ID = 'sync-engine-gdrive-refresh-token'; // Secret storage id under which the Google refresh token is stored. + +type TokenResponse = { + access_token: string; + expires_in: number; + refresh_token?: string; + id_token?: string; +}; + +type TokenError = { + error: + | 'invalid_request' + | 'invalid_client' + | 'invalid_grant' + | 'unauthorized_client' + | 'unsupported_grant_type' + | 'authorization_pending' + | 'slow_down' + | 'expired_token' + | 'access_denied'; + error_description?: string; +}; + +type DeviceCodeResponse = { + device_code: string; + user_code: string; + verification_url: string; + expires_in: number; + interval: number; +}; + +type DeviceCodeError = { + error: 'invalid_request' | 'invalid_client' | 'unsupported_grant_type' | 'unauthorized_client'; + error_description?: string; +}; + +export type DeviceAuthorization = { + deviceCode: string; + userCode: string; + verificationUrl: string; + expiresIn: number; + interval: number; +}; + +export type DeviceTokenResult = { + accessToken: string; + refreshToken: string; + expiresIn: number; + userId: string; +}; + +const FORM_CONTENT_TYPE = 'application/x-www-form-urlencoded'; + +function formEncode(fields: Record): string { + return new URLSearchParams(fields).toString(); +} + +function describeAuthError(data: TokenError, status: number): string { + return data.error_description ?? data.error ?? `HTTP ${status}`; +} + +export async function startDeviceAuthorization(): Promise { + const response = await requestUrl({ + body: formEncode({ client_id: CLIENT_ID, scope: OAUTH_SCOPE }), + contentType: FORM_CONTENT_TYPE, + method: 'POST', + throw: false, + url: OAUTH_DEVICE_CODE_URL, + }); + const data = response.json as DeviceCodeResponse | DeviceCodeError; + if ('error' in data) + throw new Error( + `Google device authorization failed: ${describeAuthError(data, response.status)}`, + ); + return { + deviceCode: data.device_code, + expiresIn: data.expires_in, + interval: data.interval, + userCode: data.user_code, + verificationUrl: data.verification_url, + }; +} + +export async function pollDeviceToken(options: { + authorization: DeviceAuthorization; + isCancelled: () => boolean; +}): Promise { + let interval = Math.max(options.authorization.interval, 1); + const deadline = Date.now() + options.authorization.expiresIn * 1000; + while (true) { + await sleep(interval * 1000); + if (options.isCancelled?.()) throw new Error('Google Drive connection was cancelled.'); + if (Date.now() > deadline) + throw new Error('The device code expired, please try connecting again.'); + const response = await requestUrl({ + body: formEncode({ + client_id: CLIENT_ID, + client_secret: CLIENT_SECRET, + device_code: options.authorization.deviceCode, + grant_type: 'urn:ietf:params:oauth:grant-type:device_code', + }), + contentType: FORM_CONTENT_TYPE, + method: 'POST', + throw: false, + url: OAUTH_TOKEN_URL, + }); + const data = response.json as TokenResponse | TokenError; + if ('access_token' in data) + if (data.refresh_token && data.id_token) + return { + accessToken: data.access_token, + expiresIn: data.expires_in, + refreshToken: data.refresh_token, + userId: extractSub(data.id_token), + }; + else throw new Error('Google authorization payload is malformed!'); + switch (data.error) { + case 'authorization_pending': { + continue; + } + case 'slow_down': { + interval += 5; + continue; + } + case 'access_denied': { + throw new Error('Google Drive access was denied.'); + } + case 'expired_token': { + throw new Error('The device code expired, please try connecting again.'); + } + default: { + throw new Error( + `Google Drive connection failed: ${describeAuthError(data, response.status)}`, + ); + } + } + } +} +function extractSub(idToken: string): string { + const payload = JSON.parse(atob(idToken.split('.')[1])) as { sub: string }; + return payload.sub; +} + +// Fire-and-forget revocation +export function revokeToken(token: string) { + return requestUrl({ + contentType: FORM_CONTENT_TYPE, + method: 'POST', + throw: false, + url: buildUrl(TOKEN_REVOKE_URL, '', { token }), + }).catch(() => {}); +} + +/** + * Caches the short-lived access token and refreshes it with the stored refresh + * token when needed. One instance is shared by the request middleware and the + * connection check so a token refresh happens at most once at a time. + */ +export class TokenManager { + private accessToken?: string; + private expiresAt = 0; + private pending?: Promise; + + constructor(private readonly secretStorage: SecretStorage) {} + + readonly getToken = (force = false): Promise => { + if (!force && this.accessToken && Date.now() < this.expiresAt - 60_000) + return Promise.resolve(this.accessToken); + this.pending ??= this.refresh().finally(() => (this.pending = undefined)); + return this.pending; + }; + + readonly getRefreshToken = () => this.secretStorage.getSecret(KEYCHAIN_SECRET_ID); + + readonly setRefreshToken = (token: string) => + this.secretStorage.setSecret(KEYCHAIN_SECRET_ID, token); + + readonly deleteRefreshToken = () => this.secretStorage.deleteSecret(KEYCHAIN_SECRET_ID); + + readonly setToken = (token: string, expiresIn: number) => { + this.accessToken = token; + this.expiresAt = Date.now() + expiresIn * 1000; + }; + + readonly invalidate = (): void => { + this.accessToken = undefined; + this.expiresAt = 0; + }; + + private async refresh(): Promise { + const refresh_token = this.getRefreshToken(); + if (!refresh_token) throw new Error('Please authorize Google Account!'); + const response = await requestUrl({ + body: formEncode({ + client_id: CLIENT_ID, + client_secret: CLIENT_SECRET, + grant_type: 'refresh_token', + refresh_token, + }), + contentType: FORM_CONTENT_TYPE, + method: 'POST', + throw: false, + url: OAUTH_TOKEN_URL, + }); + const data = response.json as TokenResponse | TokenError; + if ('access_token' in data) { + this.accessToken = data.access_token; + this.expiresAt = Date.now() + data.expires_in * 1000; + return data.access_token; + } + this.invalidate(); + if (data.error === 'invalid_grant') + throw new Error( + 'Google Drive authorization expired or was revoked, please reconnect your Google account in the settings.', + ); + throw new Error( + `Google Drive token refresh failed: ${describeAuthError(data, response.status)}`, + ); + } +} + +/** Injects the bearer token into every remote request and retries once on 401. */ +export function bearerMiddleware(request: Request, manager: TokenManager): Request { + return async (params) => { + const base: RequestParam = typeof params === 'string' ? { url: params } : params; + const send = (token: string) => + request({ ...base, headers: { ...base.headers, Authorization: `Bearer ${token}` } }); + try { + return await send(await manager.getToken()); + } catch (error: unknown) { + if (getStatus(error) !== 401) throw error; + manager.invalidate(); + return send(await manager.getToken(true)); + } + }; +} diff --git a/packages/gdrive/src/gdrive/check-connection.ts b/packages/gdrive/src/gdrive/check-connection.ts new file mode 100644 index 00000000..a279edf1 --- /dev/null +++ b/packages/gdrive/src/gdrive/check-connection.ts @@ -0,0 +1,20 @@ +import type { CheckConnectionResult, Request } from '@hesprs/sync-engine-sdk'; +import { DRIVE_API, buildUrl, parseDriveError } from './api'; + +export default async function checkConnection(request: Request): Promise { + try { + const response = await request({ + method: 'GET', + throw: false, + url: buildUrl(DRIVE_API, '/about', { fields: 'storageQuota' }), + }); + if (response.status >= 200 && response.status < 300) return { success: true } as const; + return { + reason: parseDriveError(response) ?? `HTTP ${response.status}`, + success: false, + } as const; + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + return { reason: errorMessage, success: false } as const; + } +} diff --git a/packages/gdrive/src/gdrive/fs.ts b/packages/gdrive/src/gdrive/fs.ts new file mode 100644 index 00000000..d1753130 --- /dev/null +++ b/packages/gdrive/src/gdrive/fs.ts @@ -0,0 +1,377 @@ +import type { + Binary, + DatabaseSync, + FileStat, + ListReporter, + Request, + RequestParam, + RequestResponse, + RootFs, + Stat, + StoreSync, +} from '@hesprs/sync-engine-sdk'; +import { textToUint8Array } from '@repo/shared/binary'; +import { getStatus } from '@repo/shared/get-status'; +import { basename, dirname, isFolder } from '@repo/shared/path'; +import type { DriveFile, DriveFileList } from './api'; +import { + DRIVE_API, + DRIVE_UPLOAD_API, + FILE_FIELDS, + FOLDER_MIME, + buildUrl, + escapeQuery, + parseDriveError, + toFileStat, +} from './api'; +import createRangeReadStream from './read-stream'; +import { guessMimeType, resumableUpload, singlePutUpload } from './upload'; + +export type GdriveFsOptions = { + userId: string; + useTrash: boolean; +}; + +export type GdriveDB = DatabaseSync<{ gdriveIds: string }, { gdriveIdsMarker?: string }>; + +const READ_CHUNK_SIZE = 2 * 1024 * 1024; // 2 MiB +const READ_MAX_CONCURRENT = 8; +const PAGE_SIZE = 1000; +const WRITE_FIELDS = 'id,md5Checksum'; +const ROOT_ID = 'root'; + +function notFoundError(key: string): Error { + const error = new Error(`Google Drive: ${key} does not exist.`); + (error as { status?: number }).status = 404; + return error; +} + +/** + * Google Drive stores files by immutable id inside real folders, while + * Sync Engine speaks path keys. Fs translates keys to ids + * and caches the mapping. + * + * Limitation: cannot download a file with known key but not cached ID, this is + * fine in current implementation (impossible to happen). + */ +export default class GdriveFs implements RootFs { + /** Path key (`'/'`, `folder/`, `folder/note.md`) to Drive file id. */ + private readonly ids: StoreSync; + + constructor( + private readonly request: Request, + private readonly options: GdriveFsOptions, + memoryDB: GdriveDB, + ) { + this.ids = memoryDB.getStore('gdriveIds'); + if (memoryDB.getMeta('gdriveIdsMarker') !== this.getUid()) { + this.ids.clear(); + memoryDB.setMeta('gdriveIdsMarker', this.getUid()); + } + } + + getUid(): string { + return `gdrive~${this.options.userId}`; + } + + private async requestOrThrow(params: RequestParam): Promise { + const response = await this.request(Object.assign(params, { throw: false })); + if (response.status >= 200 && response.status < 300) return response; + const error = new Error( + parseDriveError(response) ?? + `Google Drive request failed: ${response.status} ${params.method} ${params.url}`, + ); + (error as { status?: number }).status = response.status; + throw error; + } + + private resolveId(key: string): string | undefined { + if (key === '/') return ROOT_ID; + return this.ids.get(key); + } + + private dropCache(key: string): void { + this.ids.delete(key); + if (!isFolder(key)) return; + for (const cachedKey of this.ids.keys()) + if (cachedKey.startsWith(key)) this.ids.delete(cachedKey); + } + + /** + * Walks the path chain of `key` from root with fresh queries, ignoring and + * refreshing the cache along the way. + */ + private async resolveIdFresh(key: string): Promise { + if (key === '/') return ROOT_ID; + const segments = key.split('/').filter((segment) => segment !== ''); + let parentId = ROOT_ID; + let prefix = ''; + for (const [index, segment] of segments.entries()) { + const last = index === segments.length - 1; + const childKey = `${prefix}${segment}${last && !isFolder(key) ? '' : '/'}`; + const folder = !last || isFolder(key); + const response = await this.requestOrThrow({ + method: 'GET', + url: buildUrl(DRIVE_API, '/files', { + fields: 'files(id)', + pageSize: '1', + q: `'${parentId}' in parents and name = '${escapeQuery(segment)}' and mimeType ${folder ? '=' : '!='} '${FOLDER_MIME}' and trashed = false`, + }), + }); + const id = (response.json() as DriveFileList).files?.[0]?.id; + if (!id) return undefined; + this.ids.set(childKey, id); + parentId = id; + prefix = childKey; + } + return parentId; + } + + /** Session metadata for uploading `key`, updating the existing file when present. */ + private sessionFor( + key: string, + stat: FileStat, + ): { initiateUrl: string; method: 'PATCH' | 'POST'; metadata: object } { + const modifiedTime = new Date(stat.mtime).toISOString(); + const existing = this.resolveId(key); + if (existing) + return { + initiateUrl: buildUrl(DRIVE_UPLOAD_API, `/files/${existing}`, { + fields: WRITE_FIELDS, + uploadType: 'resumable', + }), + metadata: { modifiedTime }, + method: 'PATCH', + }; + const parentId = this.resolveId(dirname(key)); + return { + initiateUrl: buildUrl(DRIVE_UPLOAD_API, '/files', { + fields: WRITE_FIELDS, + uploadType: 'resumable', + }), + metadata: { + mimeType: guessMimeType(basename(key)), + modifiedTime, + name: basename(key), + parents: [parentId], + }, + method: 'POST', + }; + } + + async read(key: string): Promise { + const id = this.resolveId(key); + if (id === undefined) throw notFoundError(key); + const response = await this.requestOrThrow({ + method: 'GET', + url: buildUrl(DRIVE_API, `/files/${id}`, { alt: 'media' }), + }); + return response.bytes(); + } + + readStream(key: string, { size }: FileStat): ReadableStream { + const id = this.resolveId(key); + if (id === undefined) throw notFoundError(key); + const url = buildUrl(DRIVE_API, `/files/${id}`, { alt: 'media' }); + return createRangeReadStream({ + chunkSize: READ_CHUNK_SIZE, + maxConcurrent: READ_MAX_CONCURRENT, + requestRange: async (start, endInclusive) => { + const response = await this.requestOrThrow({ + headers: { Range: `bytes=${start}-${endInclusive}` }, + method: 'GET', + url, + }); + return response.bytes(); + }, + size, + }); + } + + async write(key: string, value: Binary, stat: FileStat): Promise { + const file = await singlePutUpload( + { + ...this.sessionFor(key, stat), + request: this.request, + size: value.byteLength, + }, + value, + ); + if (file.id) this.ids.set(key, file.id); + return file.md5Checksum ?? `${stat.mtime}~${stat.size}`; + } + + async writeStream(key: string, value: ReadableStream, stat: FileStat): Promise { + const file = await resumableUpload( + { ...this.sessionFor(key, stat), request: this.request, size: stat.size }, + value, + ); + if (file.id) this.ids.set(key, file.id); + return file.md5Checksum ?? `${stat.mtime}~${stat.size}`; + } + + async delete(key: string): Promise { + const id = this.resolveId(key); + if (id === undefined) return; + try { + await this.requestOrThrow( + this.options.useTrash + ? { + body: textToUint8Array(JSON.stringify({ trashed: true })), + headers: { 'Content-Type': 'application/json; charset=UTF-8' }, + method: 'PATCH', + url: buildUrl(DRIVE_API, `/files/${id}`, { fields: 'id' }), + } + : { + method: 'DELETE', + url: buildUrl(DRIVE_API, `/files/${id}`), + }, + ); + } catch (error) { + if (getStatus(error) !== 404) throw error; + } + this.dropCache(key); + } + + async move(oldKey: string, newKey: string): Promise { + const id = this.resolveId(oldKey); + if (id === undefined) throw notFoundError(oldKey); + const oldParentId = this.resolveId(dirname(oldKey)); + const newParentId = this.resolveId(dirname(newKey)); + if (!newParentId) throw new Error(`Parent not created when moving to "${newKey}"!`); + const query: Record = { fields: 'id' }; + if (oldParentId !== newParentId) { + query.addParents = newParentId; + if (oldParentId !== undefined) query.removeParents = oldParentId; + } + await this.requestOrThrow({ + body: textToUint8Array(JSON.stringify({ name: basename(newKey) })), + headers: { 'Content-Type': 'application/json; charset=UTF-8' }, + method: 'PATCH', + url: buildUrl(DRIVE_API, `/files/${id}`, query), + }); + this.dropCache(oldKey); + } + + /** Drive folders always need an existing parent id, so missing parents are created regardless of `recursive`. */ + async mkdir(key: string, recursive: boolean): Promise { + const parent = dirname(key); + let parentId = this.resolveId(parent); + if (!parent && recursive) { + await this.mkdir(parent, true); + parentId = this.resolveId(parent); + } + if (!parentId) throw new Error(`Parent is not created when creating "${key}"!`); + const response = await this.requestOrThrow({ + body: textToUint8Array( + JSON.stringify({ mimeType: FOLDER_MIME, name: basename(key), parents: [parentId] }), + ), + headers: { 'Content-Type': 'application/json; charset=UTF-8' }, + method: 'POST', + url: buildUrl(DRIVE_API, '/files', { fields: 'id' }), + }); + const created = response.json() as DriveFile; + if (!created.id) throw new Error('Google Drive did not return an id for a created folder!'); + this.ids.set(key, created.id); + } + + async stat(key: string): Promise { + const parentId = this.resolveId(dirname(key)); + if (!parentId) throw notFoundError(key); + const url = buildUrl(DRIVE_API, '/files', { + fields: `files(${FILE_FIELDS})`, + orderBy: 'modifiedTime desc', + pageSize: '1', + q: `'${parentId}' in parents and name = '${escapeQuery(basename(key))}' and trashed = false`, + }); + const response = await this.requestOrThrow({ method: 'GET', url }); + const entry = (response.json() as DriveFileList).files?.[0]; + if (!entry) throw notFoundError(key); + return toFileStat(key, entry); + } + + // When Sync Engine calls `exists()`, the only possibility is that something is unexpected, don't trust cache here + async exists(key: string): Promise { + return (await this.resolveIdFresh(key)) !== undefined; + } + + /** + * Fetches every visible file in one paginated query, then walks the tree + * under the requested key so the reporter can steer traversal. + */ + async list(key: string, reporter: ListReporter): Promise> { + const startId = this.resolveId(key) ?? (await this.resolveIdFresh(key)); + if (startId === undefined) throw notFoundError(key); + const all: Array = []; + let pageToken: string | undefined; + do { + const query: Record = { + fields: `nextPageToken,files(${FILE_FIELDS})`, + pageSize: String(PAGE_SIZE), + q: 'trashed = false', + }; + if (pageToken) query.pageToken = pageToken; + const response = await this.requestOrThrow({ + method: 'GET', + url: buildUrl(DRIVE_API, '/files', query), + }); + const parsed = response.json() as DriveFileList; + all.push(...(parsed.files ?? [])); + pageToken = parsed.nextPageToken; + } while (pageToken); + + this.ids.clear(); + this.ids.set(key, startId); + const childrenByParent = new Map>(); + for (const file of all) { + const parent = file.parents?.[0]; + if (!parent) continue; + const siblings = childrenByParent.get(parent); + if (siblings) siblings.push(file); + else childrenByParent.set(parent, [file]); + } + + const results: Array = []; + const total = all.length; + let completed = 0; + const walk = async (folderId: string, prefix: string): Promise => { + for (const entry of dedupeChildren(childrenByParent.get(folderId) ?? [])) { + const folder = entry.mimeType === FOLDER_MIME; + const childKey = `${prefix}${entry.name}${folder ? '/' : ''}`; + completed++; + const verdict = await reporter({ completed, current: childKey, total }); + if (verdict === 'exclude') continue; + this.ids.set(childKey, entry.id); + if (folder) { + results.push({ isDir: true, key: childKey }); + if (verdict === 'advance') await walk(entry.id, childKey); + } else results.push(toFileStat(childKey, entry)); + } + }; + await walk(startId, key === '/' ? '' : key); + return results; + } +} + +/** + * Drive allows duplicate names inside one folder; syncing needs one entry per + * key, so the most recently modified file (or the first folder) wins. + */ +function dedupeChildren(entries: Array): Array { + if (entries.length < 2) return entries; + const byName = new Map(); + for (const entry of entries) { + const nameKey = `${entry.mimeType === FOLDER_MIME ? 'd' : 'f'}~${entry.name}`; + const existing = byName.get(nameKey); + if (!existing) { + byName.set(nameKey, entry); + continue; + } + if ( + entry.mimeType !== FOLDER_MIME && + Date.parse(entry.modifiedTime ?? '') > Date.parse(existing.modifiedTime ?? '') + ) + byName.set(nameKey, entry); + } + return [...byName.values()]; +} diff --git a/packages/gdrive/src/gdrive/read-stream.ts b/packages/gdrive/src/gdrive/read-stream.ts new file mode 100644 index 00000000..75f855f0 --- /dev/null +++ b/packages/gdrive/src/gdrive/read-stream.ts @@ -0,0 +1,111 @@ +import type { Binary } from '@hesprs/sync-engine-sdk'; + +export type CreateRangeReadStreamOptions = { + size: number; + chunkSize: number; + maxConcurrent: number; + requestRange: (start: number, endInclusive: number) => Promise; +}; + +/** Concurrent range-request read stream, mirroring the S3 module's implementation. */ +export default function createRangeReadStream({ + size, + chunkSize, + maxConcurrent, + requestRange, +}: CreateRangeReadStreamOptions): ReadableStream { + const totalChunks = size === 0 ? 0 : Math.ceil(size / chunkSize); + const maxBufferedBytes = chunkSize * maxConcurrent; + if (totalChunks === 0) + return new ReadableStream({ + start(controller) { + controller.close(); + }, + }); + + let controllerRef: ReadableStreamDefaultController | undefined; + let nextChunkIndex = 0; + let nextPendingIndex = 0; + let inFlight = 0; + let closed = false; + let consumerReady = false; + let pendingBytes = 0; + const pending = new Map(); + + const closeIfDone = () => { + if (closed || !controllerRef) return; + if (nextPendingIndex < totalChunks || inFlight > 0) return; + closed = true; + controllerRef.close(); + }; + + const flush = () => { + if (!controllerRef || closed) return; + while (consumerReady && pending.has(nextPendingIndex)) { + const chunk = pending.get(nextPendingIndex); + if (!chunk) break; + pending.delete(nextPendingIndex); + pendingBytes -= chunk.byteLength; + controllerRef.enqueue(chunk); + consumerReady = (controllerRef.desiredSize ?? 0) > 0; + nextPendingIndex++; + } + closeIfDone(); + }; + + const canScheduleNext = () => + controllerRef !== undefined && + !closed && + consumerReady && + inFlight < maxConcurrent && + nextChunkIndex < totalChunks && + pendingBytes < maxBufferedBytes; + + const requestChunk = (currentIndex: number) => { + inFlight++; + + const start = currentIndex * chunkSize; + const endInclusive = Math.min(start + chunkSize - 1, size - 1); + + void requestRange(start, endInclusive) + .then((buffer) => { + if (closed) return; + pending.set(currentIndex, buffer); + pendingBytes += buffer.byteLength; + inFlight--; + flush(); + schedule(); + }) + .catch((error: unknown) => { + if (closed) return; + closed = true; + controllerRef?.error(error); + }); + }; + + const schedule = () => { + while (canScheduleNext()) { + const currentIndex = nextChunkIndex; + nextChunkIndex++; + requestChunk(currentIndex); + } + }; + + return new ReadableStream( + { + cancel() { + closed = true; + }, + pull(controller) { + controllerRef = controller; + consumerReady = true; + flush(); + schedule(); + }, + start(controller) { + controllerRef = controller; + }, + }, + { highWaterMark: 0 }, + ); +} diff --git a/packages/gdrive/src/gdrive/upload.ts b/packages/gdrive/src/gdrive/upload.ts new file mode 100644 index 00000000..f48a6099 --- /dev/null +++ b/packages/gdrive/src/gdrive/upload.ts @@ -0,0 +1,138 @@ +import type { Binary, Request, RequestResponse } from '@hesprs/sync-engine-sdk'; +import { concatBinary, textToUint8Array } from '@repo/shared/binary'; +import type { DriveFile } from './api'; +import { getHeader, parseDriveError } from './api'; + +/** Google Drive resumable uploads require chunk sizes in multiples of 256 KiB. */ +export const RESUMABLE_CHUNK_SIZE = 5 * 1024 * 1024; + +const MIME_BY_EXTENSION: Record = { + base: 'application/json', + canvas: 'application/json', + css: 'text/css', + gif: 'image/gif', + html: 'text/html', + jpeg: 'image/jpeg', + jpg: 'image/jpeg', + js: 'text/javascript', + json: 'application/json', + m4a: 'audio/mp4', + md: 'text/markdown', + mp3: 'audio/mpeg', + mp4: 'video/mp4', + pdf: 'application/pdf', + png: 'image/png', + svg: 'image/svg+xml', + txt: 'text/plain', + webm: 'video/webm', + webp: 'image/webp', +}; + +/** + * Content type declared for uploaded bytes so files keep useful previews in + * the Drive web interface. + */ +export function guessMimeType(name: string): string { + const dotIndex = name.lastIndexOf('.'); + if (dotIndex === -1) return 'application/octet-stream'; + const extension = name.slice(dotIndex + 1).toLowerCase(); + return MIME_BY_EXTENSION[extension] ?? 'application/octet-stream'; +} + +export type SessionOptions = { + initiateUrl: string; + method: 'PATCH' | 'POST'; + metadata: object; + request: Request; + size: number; +}; + +async function startSession({ + initiateUrl, + method, + metadata, + request, + size, +}: SessionOptions): Promise<{ request: Request; location: string }> { + const response = await request({ + body: textToUint8Array(JSON.stringify(metadata)), + headers: { + 'Content-Type': 'application/json; charset=UTF-8', + 'X-Upload-Content-Length': String(size), + }, + method, + url: initiateUrl, + }); + if (response.status < 200 || response.status >= 300) + throw new Error( + parseDriveError(response) ?? + `Google Drive upload session initiation failed: ${response.status}`, + ); + const location = getHeader(response.headers, 'location'); + if (!location) throw new Error('Google Drive did not return an upload session URL!'); + return { location, request }; +} + +/** Returns `undefined` when Drive answers 308 (chunk stored, upload incomplete). */ +async function putChunk( + { request, location }: { request: Request; location: string }, + chunk: Binary, + start: number, + total: number, +): Promise { + const end = start + chunk.byteLength - 1; + const response = await request({ + body: chunk, + headers: { + 'Content-Range': end < start ? `bytes */${total}` : `bytes ${start}-${end}/${total}`, + }, + method: 'PUT', + url: location, + }); + if (response.status === 308) return undefined; + if (response.status >= 200 && response.status < 300) return response; + throw new Error(parseDriveError(response) ?? `Google Drive upload failed: ${response.status}`); +} + +/** Uploads the whole value in a single PUT on a resumable session. */ +export async function singlePutUpload(options: SessionOptions, value: Binary): Promise { + const session = await startSession(options); + const response = await putChunk(session, value, 0, value.byteLength); + if (response === undefined) throw new Error('Google Drive upload ended prematurely.'); + return response.json() as DriveFile; +} + +export async function resumableUpload( + options: SessionOptions, + value: ReadableStream, +): Promise { + const session = await startSession(options); + const total = options.size; + let offset = 0; + let final: RequestResponse | undefined; + const reader = value.getReader(); + let pending = new Uint8Array(0); + try { + while (final === undefined) { + const { done, value: chunk } = await reader.read(); + if (done) break; + pending = concatBinary(pending, chunk); + // Hold back at least one byte so the closing chunk is never empty. + while (pending.byteLength > RESUMABLE_CHUNK_SIZE && final === undefined) { + const part = pending.slice(0, RESUMABLE_CHUNK_SIZE); + pending = pending.slice(RESUMABLE_CHUNK_SIZE); + final = await putChunk(session, part, offset, total); + offset += part.byteLength; + } + } + final ??= await putChunk(session, pending, offset, total); + } catch (error) { + // Best-effort session cancellation; Drive also expires sessions on its own. + await options.request({ method: 'DELETE', url: session.location }).catch(() => {}); + throw error; + } finally { + reader.releaseLock(); + } + if (final === undefined) throw new Error('Google Drive upload finished incomplete.'); + return final.json() as DriveFile; +} diff --git a/packages/gdrive/src/handle-input.ts b/packages/gdrive/src/handle-input.ts new file mode 100644 index 00000000..8724b9d2 --- /dev/null +++ b/packages/gdrive/src/handle-input.ts @@ -0,0 +1,30 @@ +import type { TextComponent } from 'obsidian'; +import { Notice } from 'obsidian'; + +export default function handleInput({ + text, + saveSettings, + processValue, + stringify = String, + key, + settings, + invalidValue, +}: { + text: TextComponent; + saveSettings: () => Promise; + processValue: (value: string) => T | false; + key: K; + settings: NoInfer>; + stringify?: (value: T) => string; + invalidValue: string; +}) { + text.inputEl.addEventListener('blur', () => { + const value = processValue(text.getValue()); + if (value === false) new Notice(invalidValue); + else if (settings[key] !== value) { + settings[key] = value; + void saveSettings(); + } + text.setValue(stringify(settings[key])); + }); +} diff --git a/packages/gdrive/src/i18n.ts b/packages/gdrive/src/i18n.ts new file mode 100644 index 00000000..a5f53341 --- /dev/null +++ b/packages/gdrive/src/i18n.ts @@ -0,0 +1,113 @@ +import type { GdriveTranslations } from './setting'; + +export const en: GdriveTranslations = { + accountConnected: 'Account connected', + accountConnectedDescription: 'Connected to Google Drive account.', + authorizationFailed: 'Authorization failed: {{reason}}', + baseDirectory: 'Base directory', + baseDirectoryDescription: + 'Set the folder in Google Drive that holds this vault. Created automatically on the first sync and do not create it manually in Drive. Files added outside this plugin are invisible to sync.', + baseDirectoryPlaceholder: 'my-vault/', + configureFirst: 'Enter the OAuth client ID and client secret first.', + connect: 'Connect', + connectAccount: 'Connect account', + connectAccountDescription: 'Click the button to connect to you Google Drive account.', + connectSuccess: 'Connected to Google Drive.', + copyAndOpenGoogle: 'Copy and open Google', + deviceCodeInstruction: (frag, url) => { + frag.appendText('Please visit '); + frag.createEl('a', { attr: { href: url } }).createEl('code', { text: url }); + frag.appendText(' and enter the code below, then approve access.'); + }, + deviceCodeTitle: 'Connect Google Drive', + disconnect: 'Disconnect', + gdrive: 'Google Drive', + useTrash: 'Delete to trash', + useTrashDescription: + 'Move deleted files to the Google Drive trash instead of deleting them permanently. Drive clears its trash after 30 days.', + waitingApproval: 'Waiting for approval…', +}; + +export const ru: GdriveTranslations = { + accountConnected: 'Аккаунт подключён', + accountConnectedDescription: 'Подключено к аккаунту Google Drive.', + authorizationFailed: 'Ошибка авторизации: {{reason}}', + baseDirectory: 'Базовый каталог', + baseDirectoryDescription: + 'Укажите папку в Google Drive, в которой будет храниться это хранилище. Она создаётся автоматически при первой синхронизации, не создавайте её вручную на Диске. Файлы, добавленные вне этого плагина, будут невидны для синхронизации.', + baseDirectoryPlaceholder: 'my-vault/', + configureFirst: 'Сначала введите OAuth client ID и client secret.', + connect: 'Подключить', + connectAccount: 'Подключить аккаунт', + connectAccountDescription: 'Нажмите кнопку, чтобы подключиться к вашему аккаунту Google Drive.', + connectSuccess: 'Успешно подключено к Google Drive.', + copyAndOpenGoogle: 'Скопировать и открыть Google', + deviceCodeInstruction: (frag, url) => { + frag.appendText('Пожалуйста, перейдите по ссылке '); + frag.createEl('a', { attr: { href: url } }).createEl('code', { text: url }); + frag.appendText(' и введите код ниже, после чего подтвердите доступ.'); + }, + deviceCodeTitle: 'Подключение Google Drive', + disconnect: 'Отключить', + gdrive: 'Google Drive', + useTrash: 'Удалять в корзину', + useTrashDescription: + 'Перемещать удалённые файлы в корзину Google Drive вместо их безвозвратного удаления. Диск автоматически очищает корзину через 30 дней.', + waitingApproval: 'Ожидание подтверждения…', +}; + +export const zhTW: GdriveTranslations = { + accountConnected: '帳號已連線', + accountConnectedDescription: '已成功連線至 Google Drive 帳號。', + authorizationFailed: '驗證失敗：{{reason}}', + baseDirectory: '基礎目錄', + baseDirectoryDescription: + '設定 Google Drive 中用來存放此儲存庫的資料夾。系統將於首次同步時自動建立，請勿手動在 Drive 中建立。在此外掛程式之外新增的檔案將無法被同步讀取。', + baseDirectoryPlaceholder: 'my-vault/', + configureFirst: '請先輸入 OAuth 用戶端 ID 與用戶端密鑰。', + connect: '連線', + connectAccount: '連結帳號', + connectAccountDescription: '點擊按鈕以連結您的 Google Drive 帳號。', + connectSuccess: '已成功連線至 Google Drive。', + copyAndOpenGoogle: '複製並前往 Google 頁面', + deviceCodeInstruction: (frag, url) => { + frag.appendText('請前往 '); + frag.createEl('a', { attr: { href: url } }).createEl('code', { text: url }); + frag.appendText(' 並輸入下方驗證碼，隨後核准存取權限。'); + }, + deviceCodeTitle: '連結 Google Drive', + disconnect: '中斷連線', + gdrive: 'Google Drive', + useTrash: '移至垃圾桶', + useTrashDescription: + '刪除檔案時將其移至 Google Drive 垃圾桶而非永久刪除。Drive 會在 30 天後自動清理垃圾桶。', + waitingApproval: '等待核准中…', +}; + +export const zh: GdriveTranslations = { + accountConnected: '账号已连接', + accountConnectedDescription: '已连接至 Google Drive 账号。', + authorizationFailed: '授权失败：{{reason}}', + baseDirectory: '基础目录', + baseDirectoryDescription: + '设置 Google Drive 中存放此仓库的文件夹。该目录会在首次同步时自动创建，请勿在 Drive 中手动创建。此插件之外添加的文件对同步不可见。', + baseDirectoryPlaceholder: 'my-vault/', + configureFirst: '请先输入 OAuth 客户端 ID 和客户端密钥。', + connect: '连接', + connectAccount: '连接账号', + connectAccountDescription: '点击按钮连接到您的 Google Drive 账号。', + connectSuccess: '已连接至 Google Drive。', + copyAndOpenGoogle: '复制并打开 Google', + deviceCodeInstruction: (frag, url) => { + frag.appendText('请访问 '); + frag.createEl('a', { attr: { href: url } }).createEl('code', { text: url }); + frag.appendText(' 并输入下方验证码，然后批准访问权限。'); + }, + deviceCodeTitle: '连接 Google Drive', + disconnect: '断开连接', + gdrive: 'Google Drive', + useTrash: '删除至回收站', + useTrashDescription: + '将删除的文件移动至 Google Drive 回收站，而非永久删除。Drive 会在 30 天后自动清空回收站。', + waitingApproval: '等待批准中…', +}; diff --git a/packages/gdrive/src/index.ts b/packages/gdrive/src/index.ts new file mode 100644 index 00000000..2e0264d6 --- /dev/null +++ b/packages/gdrive/src/index.ts @@ -0,0 +1,108 @@ +import 'tsdown/client'; +import type { + Context, + FsWrapperEntry, + ObsidianLanguageCode, + RemoteFsEntry, + RemoteRequestMiddlewareEntry, + SelectFromContext, + SettingEntry, + Settings, + Translate, + Translations, + TranslationResource, +} from '@hesprs/sync-engine-sdk'; +import type { App } from 'obsidian'; +import { digOriginal, prefixWrapper } from '@hesprs/sync-engine-sdk'; +import type { GdriveDB } from './gdrive/fs'; +import type { GdriveTranslations } from './setting'; +import { TokenManager, bearerMiddleware } from './gdrive/auth'; +import checkConnection from './gdrive/check-connection'; +import GdriveFs from './gdrive/fs'; +import { ru, en, zh, zhTW } from './i18n'; +import gdriveSetting from './setting'; +import styles from './styles.css?inline'; + +export type GdriveSettings = { + baseDirectory: string; + useTrash: boolean; + userId: string; +}; + +export default class Gdrive { + private readonly cleanup: Array<() => void> = []; + private readonly tokenManager: TokenManager; + + constructor( + private readonly ctx: SelectFromContext<{ + translate: Translate; + registerRemoteFs: (id: string, entry: RemoteFsEntry) => () => void; + app: App; + memoryDB: GdriveDB; + registerRemoteFsWrapper: (entry: FsWrapperEntry) => () => void; + registerRemoteRequestMiddleware: (entry: RemoteRequestMiddlewareEntry) => () => void; + registerSetting: (entry: SettingEntry) => () => void; + registerI18n: (lang: ObsidianLanguageCode, translations: TranslationResource) => void; + registerCss: (css: string) => () => void; + }>, + ) { + if (!this.moduleSettings.baseDirectory) + this.moduleSettings.baseDirectory = `${ctx.app.vault.getName()}/`; + ctx.registerI18n('en', en); + ctx.registerI18n('zh', zh); + ctx.registerI18n('zh-TW', zhTW); + ctx.registerI18n('ru', ru); + this.tokenManager = new TokenManager(ctx.app.secretStorage); + } + + readonly moduleSettings: GdriveSettings = { + baseDirectory: '', + useTrash: true, + userId: '', + }; + + declare settings: Settings; + + readonly start = () => { + const { + translate, + registerRemoteFs, + memoryDB, + registerRemoteFsWrapper, + registerRemoteRequestMiddleware, + registerSetting, + registerCss, + } = this.ctx; + this.cleanup.push( + registerCss(styles), + registerRemoteFs('gdrive', { + checkConnection, + instantiate: (request) => new GdriveFs(request, this.moduleSettings, memoryDB), + prettyName: () => translate('gdrive'), + }), + registerRemoteFsWrapper({ + apply: (fs) => { + if (digOriginal(fs) instanceof GdriveFs) + return prefixWrapper(fs, this.moduleSettings.baseDirectory); + }, + priority: 8308, + }), + registerRemoteRequestMiddleware({ + apply: (request) => { + if (this.settings.remoteFs !== 'gdrive') return; + return bearerMiddleware(request, this.tokenManager); + }, + priority: 305, + }), + registerSetting({ + apply: gdriveSetting(this.ctx as Context, this.moduleSettings, this.tokenManager), + priority: 683, + }), + ); + }; + + readonly dispose = () => { + this.cleanup.forEach((fn) => fn()); + this.cleanup.length = 0; + }; +} diff --git a/packages/gdrive/src/setting.ts b/packages/gdrive/src/setting.ts new file mode 100644 index 00000000..df3bee49 --- /dev/null +++ b/packages/gdrive/src/setting.ts @@ -0,0 +1,235 @@ +import type { GdriveSettings } from '@'; +import type { + CallableOrObjectTree, + Fragment, + LabelDefinition, + Translate, + Translations, +} from '@hesprs/sync-engine-sdk'; +import type { App, SettingGroupItem } from 'obsidian'; +import { s } from '@hesprs/sync-engine-sdk'; +import { normalizeBaseDir } from '@repo/shared/path'; +import { Modal, Notice, Setting } from 'obsidian'; +import type { TokenManager } from './gdrive/auth'; +import { pollDeviceToken, revokeToken, startDeviceAuthorization } from './gdrive/auth'; +import handleInput from './handle-input'; + +export type GdriveTranslations = { + gdrive: string; + connectAccount: string; + accountConnected: string; + accountConnectedDescription: string; + connectAccountDescription: string; + connect: string; + disconnect: string; + configureFirst: string; + deviceCodeTitle: string; + deviceCodeInstruction: Fragment; + copyAndOpenGoogle: string; + waitingApproval: string; + connectSuccess: string; + baseDirectory: string; + baseDirectoryDescription: string; + baseDirectoryPlaceholder: string; + useTrash: string; + useTrashDescription: string; + authorizationFailed: string; +}; + +type DeviceCodeModalOptions = { + translate: Translate; + userCode: string; + verificationUrl: string; + onClose: () => void; +}; + +class DeviceCodeModal extends Modal { + constructor( + app: App, + private readonly options: DeviceCodeModalOptions, + ) { + super(app); + } + + onOpen(): void { + const { + contentEl, + titleEl, + options: { translate, userCode, verificationUrl }, + } = this; + titleEl.setText(translate('deviceCodeTitle')); + contentEl.addClass('markdown-rendered'); + contentEl.createEl('p', { + text: translate('deviceCodeInstruction', verificationUrl), + }); + contentEl.createEl('code', { cls: 'gdrive-device-code', text: userCode }); + contentEl.createEl('p', { + cls: 'gdrive-device-code-status', + text: translate('waitingApproval'), + }); + new Setting(contentEl) + .addButton((button) => + button + .setButtonText(translate('cancel')) + .setDestructive() + .onClick(() => this.close()), + ) + .addButton((button) => + button + .setCta() + .setButtonText(translate('copyAndOpenGoogle')) + .onClick(() => { + void navigator.clipboard.writeText(userCode); + window.open(verificationUrl); + button.setIcon('check'); + }), + ); + } + onClose(): void { + this.contentEl.empty(); + this.options.onClose(); + } +} + +export default function gdriveSetting( + { + translate, + saveSettings, + app, + matchLabel, + refreshSettingTab, + }: { + translate: Translate; + saveSettings: () => Promise; + app: App; + matchLabel: () => LabelDefinition; + refreshSettingTab: () => void; + }, + settings: GdriveSettings, + tokenManager: TokenManager, +): CallableOrObjectTree { + const invalidValue = translate('invalidValue'); + const connectGoogle = async (resolve: () => void) => { + let cancelled = false; + try { + const authorization = await startDeviceAuthorization(); + const modal = new DeviceCodeModal(app, { + onClose: () => { + cancelled = true; + resolve(); + }, + translate, + userCode: authorization.userCode, + verificationUrl: authorization.verificationUrl, + }); + modal.open(); + try { + const { refreshToken, userId, accessToken, expiresIn } = await pollDeviceToken({ + authorization, + isCancelled: () => cancelled, + }); + tokenManager.setRefreshToken(refreshToken); + settings.userId = userId; + tokenManager.setToken(accessToken, expiresIn); + void saveSettings(); + new Notice(translate('connectSuccess')); + refreshSettingTab(); + } finally { + modal.close(); + } + } catch (error) { + if (!cancelled) + new Notice( + translate('authorizationFailed', { + reason: error instanceof Error ? error.message : String(error), + }), + ); + } finally { + resolve(); + } + }; + + return { + 551: s( + (self) => ({ + heading: translate('gdrive'), + items: Object.values(self).map((node) => node(node) as SettingGroupItem), + type: 'group', + }), + { + 1000: s(() => ({ + desc: translate('connectAccountDescription'), + name: translate('connectAccount'), + render: (setting) => { + setting.addButton((button) => + button + .setButtonText(translate('connect')) + .setCta() + .onClick( + () => + new Promise((resolve) => { + void connectGoogle(resolve); + }), + ), + ); + }, + visible: () => !tokenManager.getRefreshToken(), + })), + 1100: s(() => ({ + desc: translate('accountConnectedDescription'), + name: translate('accountConnected'), + render: (setting) => { + setting.addButton((button) => + button + .setButtonText(translate('disconnect')) + .setDestructive() + .onClick(async () => { + const token = tokenManager.getRefreshToken(); + if (!token) return; + await revokeToken(token); + settings.userId = ''; + tokenManager.deleteRefreshToken(); + tokenManager.invalidate(); + void saveSettings(); + refreshSettingTab(); + }), + ); + }, + visible: () => Boolean(tokenManager.getRefreshToken()), + })), + 2000: s(() => ({ + desc: translate('baseDirectoryDescription'), + labels: [matchLabel()], + name: translate('baseDirectory'), + render: (setting) => { + setting.addText((text) => { + text.setPlaceholder(translate('baseDirectoryPlaceholder')).setValue( + settings.baseDirectory, + ); + handleInput({ + invalidValue, + key: 'baseDirectory', + processValue: (original) => normalizeBaseDir(original.trim()), + saveSettings, + settings, + text, + }); + }); + }, + })), + 3000: s(() => ({ + desc: translate('useTrashDescription'), + name: translate('useTrash'), + render: (setting) => { + setting.addToggle((toggle) => + toggle.setValue(settings.useTrash).onChange((value) => { + settings.useTrash = value; + void saveSettings(); + }), + ); + }, + })), + }, + ), + }; +} diff --git a/packages/gdrive/src/styles.css b/packages/gdrive/src/styles.css new file mode 100644 index 00000000..c6ee3ad6 --- /dev/null +++ b/packages/gdrive/src/styles.css @@ -0,0 +1,14 @@ +code.gdrive-device-code { + display: block; + font-size: 2em; + font-weight: 700; + letter-spacing: 0.15em; + margin: 0.5em auto; + user-select: text; + width: fit-content; +} + +.gdrive-device-code-status { + color: var(--text-muted); + text-align: center; +} diff --git a/packages/gdrive/test/auth.test.ts b/packages/gdrive/test/auth.test.ts new file mode 100644 index 00000000..dc9fb052 --- /dev/null +++ b/packages/gdrive/test/auth.test.ts @@ -0,0 +1,115 @@ +import type { Request, RequestParam, RequestResponse } from '@hesprs/sync-engine-sdk'; +import type { SecretStorage } from 'obsidian'; +import * as ObsidianMock from '@repo/shared/mocks'; +import { expect, mock, test } from 'bun:test'; + +type HttpResponse = { json: unknown; status?: number }; +const requests: Array = []; +let responses: Array = []; + +void mock.module('obsidian', () => ({ + ...ObsidianMock, + requestUrl: (params: RequestParam) => { + requests.push(params); + const response = responses.shift(); + if (!response) throw new Error('Unexpected request'); + return Promise.resolve({ json: response.json, status: response.status ?? 200 }); + }, +})); + +const { TokenManager, bearerMiddleware, pollDeviceToken, startDeviceAuthorization } = + await import('@/gdrive/auth'); + +function reset(...next: Array) { + requests.length = 0; + responses = [...next]; +} + +test('starts device authorization from Google response', async () => { + reset({ + json: { + device_code: 'device', + expires_in: 900, + interval: 0, + user_code: 'ABCD', + verification_url: 'https://google.test/device', + }, + }); + + expect(await startDeviceAuthorization()).toStrictEqual({ + deviceCode: 'device', + expiresIn: 900, + interval: 0, + userCode: 'ABCD', + verificationUrl: 'https://google.test/device', + }); + expect(requests[0]?.method).toBe('POST'); +}); + +test('polls device authorization and extracts user id from ID token', async () => { + const payload = btoa(JSON.stringify({ sub: 'google-user' })) + .replaceAll('+', '-') + .replaceAll('/', '_') + .replaceAll('=', ''); + reset({ + json: { + access_token: 'access', + expires_in: 3600, + id_token: `header.${payload}.signature`, + refresh_token: 'refresh', + }, + }); + + expect( + await pollDeviceToken({ + authorization: { + deviceCode: 'device', + expiresIn: 60, + interval: 0, + userCode: 'code', + verificationUrl: 'url', + }, + isCancelled: () => false, + }), + ).toStrictEqual({ + accessToken: 'access', + expiresIn: 3600, + refreshToken: 'refresh', + userId: 'google-user', + }); +}); + +test('caches tokens and retries bearer requests after a 401', async () => { + reset( + { json: { access_token: 'first', expires_in: 3600 } }, + { json: { access_token: 'second', expires_in: 3600 } }, + ); + const secrets = new Map([['sync-engine-gdrive-refresh-token', 'refresh']]); + const storage = { + deleteSecret: (id: string) => void secrets.delete(id), + getSecret: (id: string) => secrets.get(id), + setSecret: (id: string, value: string) => void secrets.set(id, value), + }; + const manager = new TokenManager(storage as unknown as SecretStorage); + const seen: Array = []; + const request: Request = (params) => { + if (typeof params === 'string') throw new Error('Unexpected string request'); + seen.push(params.headers?.Authorization); + if (seen.length === 1) { + const error = new Error('Unauthorized') as Error & { status: number }; + error.status = 401; + return Promise.reject(error); + } + return Promise.resolve({ + bytes: () => new Uint8Array(0), + headers: {}, + json: () => ({}), + status: 200, + text: () => '', + } satisfies RequestResponse); + }; + + const wrapped = bearerMiddleware(request, manager); + expect((await wrapped({ method: 'GET', url: 'https://drive.test' })).status).toBe(200); + expect(seen).toStrictEqual(['Bearer first', 'Bearer second']); +}); diff --git a/packages/gdrive/test/fs-gdrive.test.ts b/packages/gdrive/test/fs-gdrive.test.ts new file mode 100644 index 00000000..ff71d0cd --- /dev/null +++ b/packages/gdrive/test/fs-gdrive.test.ts @@ -0,0 +1,109 @@ +import type { Binary, Request, RequestParam, RequestResponse } from '@hesprs/sync-engine-sdk'; +import { testKit } from '@hesprs/sync-engine-sdk/dev'; +import { beforeEach, expect, test } from 'bun:test'; +import { openMemoryDB } from 'uni-kv'; +import type { GdriveDB } from '@/gdrive/fs'; +import { DRIVE_API, DRIVE_UPLOAD_API, FOLDER_MIME } from '@/gdrive/api'; +import GdriveFs from '@/gdrive/fs'; + +const { bytes, file } = testKit; +const db: GdriveDB = openMemoryDB<{ gdriveIds: string }, { gdriveIdsMarker?: string }>( + 'gdrive-fs-test', +); + +function response( + value: unknown = {}, + status = 200, + headers: Record = {}, +): RequestResponse { + const body = new TextEncoder().encode(JSON.stringify(value)); + return { + bytes: () => body, + headers, + json: () => value, + status, + text: () => new TextDecoder().decode(body), + }; +} + +function binaryResponse(value: Binary, status = 200): RequestResponse { + return { ...response({}, status), bytes: () => value }; +} + +function createFs(handler: (params: RequestParam) => RequestResponse | Promise) { + const calls: Array = []; + const request: Request = (params) => { + if (typeof params === 'string') throw new Error('Unexpected string request'); + calls.push(params); + return Promise.resolve(handler(params)); + }; + return { calls, fs: new GdriveFs(request, { useTrash: true, userId: 'user-1' }, db) }; +} + +beforeEach(() => { + db.clearStores(); + db.setMeta('gdriveIdsMarker', undefined); +}); + +test('writes and reads a file through Drive resumable upload', async () => { + const { calls, fs } = createFs((params) => { + if (params.url.startsWith(DRIVE_UPLOAD_API) && params.method === 'POST') + return response({}, 200, { Location: 'https://upload.example/session' }); + if (params.url === 'https://upload.example/session') + return response({ id: 'file-1', md5Checksum: 'drive-uid' }); + if (params.url === `${DRIVE_API}/files/file-1?alt=media`) + return binaryResponse(bytes('hello')); + throw new Error(`Unexpected request: ${params.method} ${params.url}`); + }); + + const stat = file('note.md', { mtime: 1_700_000_000_000, size: 5 }); + expect(await fs.write('note.md', bytes('hello'), stat)).toBe('drive-uid'); + expect(await fs.read('note.md')).toStrictEqual(bytes('hello')); + expect(calls.map(({ method }) => method)).toStrictEqual(['POST', 'PUT', 'GET']); + expect(calls[0]?.url).toContain('uploadType=resumable'); +}); + +test('creates folders, lists visible descendants, and honors excluded subtrees', async () => { + const { calls, fs } = createFs((params) => { + if (params.method === 'POST' && params.url.startsWith(`${DRIVE_API}/files`)) + return response({ id: 'folder-1' }); + return response({ + files: [ + { id: 'folder-1', mimeType: FOLDER_MIME, name: 'notes', parents: ['root'] }, + { + id: 'file-1', + md5Checksum: 'uid', + mimeType: 'text/markdown', + modifiedTime: new Date(1000).toISOString(), + name: 'note.md', + parents: ['folder-1'], + size: '5', + }, + ], + }); + }); + + await fs.mkdir('notes/', true); + const result = await fs.list('/', ({ current }) => + current === 'notes/' ? 'include' : 'advance', + ); + expect(result).toStrictEqual([{ isDir: true, key: 'notes/' }]); + expect(calls[0]?.method).toBe('POST'); + expect(new TextDecoder().decode(calls[0]?.body as Binary)).toContain(FOLDER_MIME); +}); + +test('moves a cached file with Drive native rename', async () => { + const { calls, fs } = createFs((params) => { + if (params.method === 'POST' && params.url.startsWith(DRIVE_UPLOAD_API)) + return response({}, 200, { location: 'https://upload.example/session' }); + if (params.url === 'https://upload.example/session') return response({ id: 'file-1' }); + if (params.method === 'PATCH') return response({ id: 'file-1' }); + throw new Error(`Unexpected request: ${params.method} ${params.url}`); + }); + + await fs.write('old.md', bytes('x'), file('old.md', { size: 1 })); + await fs.move('old.md', 'new.md'); + const move = calls.find((call) => call.method === 'PATCH'); + expect(move?.url).toContain('/files/file-1'); + expect(new TextDecoder().decode(move?.body as Binary)).toBe('{"name":"new.md"}'); +}); diff --git a/packages/gdrive/test/mocks.ts b/packages/gdrive/test/mocks.ts new file mode 100644 index 00000000..45f6b609 --- /dev/null +++ b/packages/gdrive/test/mocks.ts @@ -0,0 +1,8 @@ +// oxlint-disable-next-line import/no-namespace +import * as ObsidianMock from '@repo/shared/mocks'; +import { mock } from 'bun:test'; + +process.env.CLIENT_ID = btoa(process.env.GDRIVE_CLIENT_ID ?? ''); +process.env.CLIENT_SECRET = btoa(process.env.GDRIVE_CLIENT_SECRET ?? ''); + +void mock.module('obsidian', () => ObsidianMock); diff --git a/packages/gdrive/tsconfig.json b/packages/gdrive/tsconfig.json new file mode 100644 index 00000000..ea80c697 --- /dev/null +++ b/packages/gdrive/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "paths": { + "@/*": ["./src/*"], + "@": ["./src"] + } + }, + "include": ["src/**/*.ts", "test/**/*.ts", "tsdown.config.ts"] +} diff --git a/packages/gdrive/tsdown.config.ts b/packages/gdrive/tsdown.config.ts new file mode 100644 index 00000000..92593f19 --- /dev/null +++ b/packages/gdrive/tsdown.config.ts @@ -0,0 +1,19 @@ +import { syncEngineTransform } from '@hesprs/sync-engine-sdk/dev'; +import { defineConfig } from 'tsdown'; + +const dev = process.env.MODE === 'dev'; + +export default defineConfig({ + clean: !dev, + css: { minify: true }, + define: { + 'process.env.CLIENT_ID': JSON.stringify(btoa(process.env.GDRIVE_CLIENT_ID ?? '')), + 'process.env.CLIENT_SECRET': JSON.stringify(btoa(process.env.GDRIVE_CLIENT_SECRET ?? '')), + }, + dts: false, + entry: { gdrive: 'src/index.ts' }, + minify: true, + outExtensions: () => ({ js: '.js' }), + outputOptions: { codeSplitting: false }, + plugins: [syncEngineTransform()], +}); diff --git a/packages/plugin/CHANGELOG.md b/packages/plugin/CHANGELOG.md index fb781937..2f87ccf3 100644 --- a/packages/plugin/CHANGELOG.md +++ b/packages/plugin/CHANGELOG.md @@ -2,6 +2,30 @@ All notable changes to this project will be documented in this file. +## Sync Engine v3.1.1 - 2026-08-23 + +### Core + +- Fixed sync failure caused by deleting a parent folder whose children are moved to other folders. +- Added clearer failure message for file names containing Windows forbidden characters. + +### S3 Module + +- Supported optional `session_token` in S3 authorization. +- Optimized sync speed by caching SigV4 signing key. + +### Google Drive Module + +- Experimental Google Drive backend. Internal testing while waiting Google App review. + +### WebDAV Module + +- Eliminated minor discrepancy on Etag handling between `PUT` and `PROPFIND` responses. + +### Contributors + +@Quzzar, @xx025, @hesprs + ## Sync Engine v3.1.0 - 2026-08-21 ### UI Modernization diff --git a/packages/plugin/dist/index.spec.d.ts b/packages/plugin/dist/index.spec.d.ts index 957a7afb..5a60195c 100644 --- a/packages/plugin/dist/index.spec.d.ts +++ b/packages/plugin/dist/index.spec.d.ts @@ -382,12 +382,14 @@ declare class Setting$1 { private readonly speedLabel; private readonly addSettingTab; private readonly rerenderSettingTab; + private readonly refreshSettingTab; root: { addSettingTab: (plugin: Plugin) => void; matchLabel: () => { text: string; tooltip: string; }; + refreshSettingTab: () => void | undefined; registerSetting: (entry: SettingEntry) => () => boolean; rerenderSettingTab: () => void | undefined; speedLabel: () => { diff --git a/packages/plugin/package.json b/packages/plugin/package.json index 601a0865..8926a84d 100644 --- a/packages/plugin/package.json +++ b/packages/plugin/package.json @@ -1,6 +1,6 @@ { "name": "@hesprs/sync-engine-sdk", - "version": "3.1.0", + "version": "3.1.1", "description": "Official SDK for developing modules targeting Sync Engine, the extensible Obsidian syncing plugin.", "keywords": [ "obsidian-plugin", diff --git a/packages/plugin/src/en.ts b/packages/plugin/src/en.ts index c5b70508..907603b0 100644 --- a/packages/plugin/src/en.ts +++ b/packages/plugin/src/en.ts @@ -11,9 +11,7 @@ const en: Translations = { asymmetricStorageDescription: (frag) => { frag.appendText('Use '); frag.createEl('a', { - attr: { - href: 'https://sync.consensia.cc/deep-dive/asymmetric-storage', - }, + attr: { href: 'https://sync.consensia.cc/deep-dive/asymmetric-storage' }, text: 'asymmetric storage', }); frag.appendText(' to substantially accelerate syncing.'); diff --git a/packages/plugin/src/fs/hierarchical-optimizer.ts b/packages/plugin/src/fs/hierarchical-optimizer.ts index a60dbbe5..cbff475b 100644 --- a/packages/plugin/src/fs/hierarchical-optimizer.ts +++ b/packages/plugin/src/fs/hierarchical-optimizer.ts @@ -79,26 +79,33 @@ export default function hierarchicalOptimizer({ atoms, executeAtom }: OptimizerI const { write } = pathsOf(atom); if (write && (`${deletion.key}/` === write || `${write}/` === deletion.key)) dependencies.get(atom)?.add(deletion); + if (atom.type === 'move' && isSub(deletion.key, atom.oldKey)) + dependencies.get(deletion)?.add(atom); } } for (const atom of atoms) { const originalExecute = atom.execute; atom.execute = (async () => { - if (isSubsumable(atom)) { - const umbrella = umbrellas.get(atom); - if (umbrella) { - await executeAtom(umbrella); - atom.resolve(); - return; + try { + if (isSubsumable(atom)) { + const umbrella = umbrellas.get(atom); + if (umbrella) { + await executeAtom(umbrella); + atom.resolve(); + return; + } } + await Promise.all( + [...(dependencies.get(atom) as Set)].map((dependency) => + executeAtom(dependency), + ), + ); + return await originalExecute(); + } catch (error) { + atom.reject(error instanceof Error ? error : new Error(String(error))); + throw error; } - await Promise.all( - [...(dependencies.get(atom) as Set)].map((dependency) => - executeAtom(dependency), - ), - ); - return originalExecute(); }) as never; } diff --git a/packages/plugin/src/fs/middlewares/retry.ts b/packages/plugin/src/fs/middlewares/retry.ts index 7f5980d2..e4bb683b 100644 --- a/packages/plugin/src/fs/middlewares/retry.ts +++ b/packages/plugin/src/fs/middlewares/retry.ts @@ -1,7 +1,6 @@ import type { ErrorLike } from '@repo/shared/get-status'; import { getStatus } from '@repo/shared/get-status'; import type { Request } from '@/modules/Registrar'; -import sleep from '@/utils/sleep'; type RetryOptions = { maxRetry?: number; diff --git a/packages/plugin/src/fs/vault/request.ts b/packages/plugin/src/fs/vault/request.ts index 3ca5f08f..0eb2879f 100644 --- a/packages/plugin/src/fs/vault/request.ts +++ b/packages/plugin/src/fs/vault/request.ts @@ -1,8 +1,8 @@ import type { Vault, Stat, ListedFiles, App } from 'obsidian'; import { toArrayBuffer, toUint8Array } from '@repo/shared/binary'; -import { isFolder, stripEndSlash } from '@repo/shared/path'; -import { TFile, TFolder } from 'obsidian'; -import type { Binary } from '@/types'; +import { basename, isFolder, stripEndSlash } from '@repo/shared/path'; +import { Platform, TFile, TFolder } from 'obsidian'; +import type { Binary, MaybePromise } from '@/types'; type VaultRequestParam = | { method: 'GET'; key: string } @@ -73,9 +73,13 @@ export default function createVaultRequest(app: App): VaultRequest { return response.body as never; } if (method === 'PUT') - return adapter.writeBinary(path, toArrayBuffer(params.value), params.headers) as never; + return withCheckChars(key, () => + adapter.writeBinary(path, toArrayBuffer(params.value), params.headers), + ) as never; if (method === 'APPEND') - return adapter.appendBinary(path, toArrayBuffer(params.value), params.headers) as never; + return withCheckChars(key, () => + adapter.appendBinary(path, toArrayBuffer(params.value), params.headers), + ) as never; if (method === 'DELETE') { const trashOption = getTrashOption(vault); if (trashOption === 'permanent' || params.headers?.permanent) @@ -86,7 +90,10 @@ export default function createVaultRequest(app: App): VaultRequest { } if (method === 'MOVE') return adapter.rename(path, toVaultPath(params.headers.destination)) as never; - if (method === 'MKDIR') return (key === '/' ? undefined : adapter.mkdir(path)) as never; + if (method === 'MKDIR') + return ( + key === '/' ? undefined : withCheckChars(key, () => adapter.mkdir(path)) + ) as never; if (method === 'EXISTS') { if (vault.getAbstractFileByPath(path)) return true as never; return adapter.exists(path, true) as never; @@ -124,3 +131,18 @@ export default function createVaultRequest(app: App): VaultRequest { return undefined as never; }; } + +async function withCheckChars(key: string, action: () => MaybePromise): Promise { + try { + return await action(); + } catch (error: unknown) { + if (Platform.isWin) { + const match = /[<>:"/\\|?*]/u.exec(basename(key)); + if (match) + throw new Error(`Windows forbids character "${match[0]}" in file names!`, { + cause: error, + }); + } + throw error; + } +} diff --git a/packages/plugin/src/modules/Setting.ts b/packages/plugin/src/modules/Setting.ts index 868f9b95..393f30e5 100644 --- a/packages/plugin/src/modules/Setting.ts +++ b/packages/plugin/src/modules/Setting.ts @@ -77,10 +77,12 @@ export default class Setting { plugin.addSettingTab(this.settingTab); }; private readonly rerenderSettingTab = () => this.settingTab?.update(); + private readonly refreshSettingTab = () => this.settingTab?.refreshDomState(); root = { addSettingTab: this.addSettingTab, matchLabel: this.matchLabel, + refreshSettingTab: this.refreshSettingTab, registerSetting: setRegister(this.settingRegistry), rerenderSettingTab: this.rerenderSettingTab, speedLabel: this.speedLabel, diff --git a/packages/plugin/src/utils/sleep.ts b/packages/plugin/src/utils/sleep.ts deleted file mode 100644 index cc803f8e..00000000 --- a/packages/plugin/src/utils/sleep.ts +++ /dev/null @@ -1,5 +0,0 @@ -export default function sleep(ms: number): Promise { - return new Promise((resolve) => { - window.setTimeout(resolve, ms); - }); -} diff --git a/packages/plugin/test/fs-hierarchal-optimizer.test.ts b/packages/plugin/test/hierarchal-optimizer.test.ts similarity index 66% rename from packages/plugin/test/fs-hierarchal-optimizer.test.ts rename to packages/plugin/test/hierarchal-optimizer.test.ts index 5c684721..100bcfbe 100644 --- a/packages/plugin/test/fs-hierarchal-optimizer.test.ts +++ b/packages/plugin/test/hierarchal-optimizer.test.ts @@ -78,6 +78,34 @@ test('mkdir chain waits for ancestor mkdir', async () => { await pending; }); +test('dependent atom receives ancestor failure', async () => { + const parentError = new Error('parent mkdir failed'); + let childRejection: unknown; + const atoms: Array = [ + { + execute: () => Promise.reject(parentError), + key: 'folder/', + reject: () => {}, + resolve: () => {}, + type: 'mkdir', + }, + { + execute: () => 'write-uid', + key: 'folder/note.md', + reject: (error) => (childRejection = error), + resolve: () => {}, + type: 'write', + }, + ]; + const { executeAtom, optimized } = runOptimizer(atoms); + const results = await Promise.all( + optimized.map((atom) => executeAtom(atom).catch((error: unknown) => error)), + ); + + expect(results[1]).toBe(parentError); + expect(childRejection).toBe(parentError); +}); + test('move gates operations under destination', async () => { const logs: Array = []; const move = deferred(); @@ -117,6 +145,89 @@ test('move gates operations under destination', async () => { await pending; }); +test('Parent deletion waits for descendant moves', async () => { + const logs: Array = []; + const move = deferred(); + const atoms: Array = [ + { + execute: async () => { + logs.push('move:folder/src/->folder/dst/'); + await move.promise; + }, + newKey: 'folder/dst/', + oldKey: 'folder/src/', + reject: () => {}, + resolve: () => {}, + type: 'move', + }, + { + execute: () => { + logs.push('move:folder/src/a.md->folder/dst/a.md'); + }, + newKey: 'folder/dst/a.md', + oldKey: 'folder/src/a.md', + reject: () => {}, + resolve: () => {}, + type: 'move', + }, + { + execute: () => { + logs.push('delete:folder/'); + }, + key: 'folder/', + reject: () => {}, + resolve: () => {}, + type: 'delete', + }, + ]; + const { executeAtom, optimized } = runOptimizer(atoms); + const pending = Promise.all(optimized.map(executeAtom)); + + await flush(); + expect(logs).toStrictEqual(['move:folder/src/->folder/dst/']); + move.resolve(); + await flush(); + expect(logs).toStrictEqual(['move:folder/src/->folder/dst/', 'delete:folder/']); + + await pending; +}); + +test('Moves waits for destination creation', async () => { + const logs: Array = []; + const mkdir = deferred(); + const atoms: Array = [ + { + execute: () => { + logs.push('move:src/a.md->dst/a.md'); + }, + newKey: 'dst/a.md', + oldKey: 'src/a.md', + reject: () => {}, + resolve: () => {}, + type: 'move', + }, + { + execute: async () => { + logs.push('mkdir:dst/'); + await mkdir.promise; + }, + key: 'dst/', + reject: () => {}, + resolve: () => {}, + type: 'mkdir', + }, + ]; + const { executeAtom, optimized } = runOptimizer(atoms); + const pending = Promise.all(optimized.map(executeAtom)); + + await flush(); + expect(logs).toStrictEqual(['mkdir:dst/']); + mkdir.resolve(); + await flush(); + expect(logs).toStrictEqual(['mkdir:dst/', 'move:src/a.md->dst/a.md']); + await pending; +}); + test('folder deletion subsumes descendant deletions', async () => { const logs: Array = []; let childResolved = false; diff --git a/packages/plugin/test/mocks.ts b/packages/plugin/test/mocks.ts index edec474c..bf6976b0 100644 --- a/packages/plugin/test/mocks.ts +++ b/packages/plugin/test/mocks.ts @@ -1,5 +1,5 @@ // oxlint-disable-next-line import/no-namespace -import * as ObsidianMock from '@repo/shared/obsidian-mock'; +import * as ObsidianMock from '@repo/shared/mocks'; import { mock } from 'bun:test'; Object.assign(globalThis, { window: globalThis }); diff --git a/packages/plugin/test/retry-middleware.test.ts b/packages/plugin/test/retry-middleware.test.ts index 9b80b45a..a6f3f694 100644 --- a/packages/plugin/test/retry-middleware.test.ts +++ b/packages/plugin/test/retry-middleware.test.ts @@ -1,11 +1,9 @@ import testKit from '$/test-kit'; import { expect, spyOn, test } from 'bun:test'; import { retryMiddleware } from '@/fs'; -// oxlint-disable-next-line import/no-namespace -import * as sleepModule from '@/utils/sleep'; const { bytes, request } = testKit; -const sleepSpy = spyOn(sleepModule, 'default').mockImplementation(() => Promise.resolve()); +const sleepSpy = spyOn(globalThis, 'sleep').mockImplementation(() => Promise.resolve()); test('retry middleware retries retryable request and waits between attempts', () => { sleepSpy.mockClear(); diff --git a/packages/s3/test/mocks.ts b/packages/s3/test/mocks.ts index ed25d7f1..d612befb 100644 --- a/packages/s3/test/mocks.ts +++ b/packages/s3/test/mocks.ts @@ -1,5 +1,5 @@ // oxlint-disable-next-line import/no-namespace -import * as ObsidianMock from '@repo/shared/obsidian-mock'; +import * as ObsidianMock from '@repo/shared/mocks'; import { mock } from 'bun:test'; void mock.module('obsidian', () => ObsidianMock); diff --git a/packages/shared/src/obsidian-mock.ts b/packages/shared/src/mocks.ts similarity index 87% rename from packages/shared/src/obsidian-mock.ts rename to packages/shared/src/mocks.ts index 215d6982..bf7b34cd 100644 --- a/packages/shared/src/obsidian-mock.ts +++ b/packages/shared/src/mocks.ts @@ -48,3 +48,10 @@ export function requireApiVersion() { } export const apiVersion = '1.12.7'; + +Object.assign(globalThis, { + sleep: (milliseconds: number) => + new Promise((resolve) => { + setTimeout(resolve, milliseconds); + }), +}); diff --git a/packages/smart-merge/test/mocks.ts b/packages/smart-merge/test/mocks.ts index ed25d7f1..d612befb 100644 --- a/packages/smart-merge/test/mocks.ts +++ b/packages/smart-merge/test/mocks.ts @@ -1,5 +1,5 @@ // oxlint-disable-next-line import/no-namespace -import * as ObsidianMock from '@repo/shared/obsidian-mock'; +import * as ObsidianMock from '@repo/shared/mocks'; import { mock } from 'bun:test'; void mock.module('obsidian', () => ObsidianMock); diff --git a/packages/webdav/test/mocks.ts b/packages/webdav/test/mocks.ts index ed25d7f1..d612befb 100644 --- a/packages/webdav/test/mocks.ts +++ b/packages/webdav/test/mocks.ts @@ -1,5 +1,5 @@ // oxlint-disable-next-line import/no-namespace -import * as ObsidianMock from '@repo/shared/obsidian-mock'; +import * as ObsidianMock from '@repo/shared/mocks'; import { mock } from 'bun:test'; void mock.module('obsidian', () => ObsidianMock); diff --git a/versions.json b/versions.json index fdd96fef..f3a6e382 100644 --- a/versions.json +++ b/versions.json @@ -6,5 +6,6 @@ "3.0.4": "1.12.3", "3.0.5": "1.12.3", "3.0.6": "1.12.3", - "3.1.0": "1.13.0" + "3.1.0": "1.13.0", + "3.1.1": "1.13.0" }