Add autoDeleteStaleRepos config option (#128)

Author: brendan-kellam

CHANGELOG.md

@@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Changed
 
 - Made language suggestions case insensitive. ([#124](https://github.com/sourcebot-dev/sourcebot/pull/124))
+- Stale repositories are now automatically deleted from the index. This can be configured via `settings.autoDeleteStaleRepos` in the config. ([#128](https://github.com/sourcebot-dev/sourcebot/pull/128))
 
 ## [2.6.1] - 2024-12-09
 

packages/backend/package.json

@@ -28,6 +28,7 @@
         "cross-fetch": "^4.0.0",
         "dotenv": "^16.4.5",
         "gitea-js": "^1.22.0",
+        "glob": "^11.0.0",
         "lowdb": "^7.0.1",
         "micromatch": "^4.0.8",
         "posthog-node": "^4.2.1",

packages/backend/src/constants.ts

@@ -15,4 +15,5 @@ export const RESYNC_CONFIG_INTERVAL_MS = 1000 * 60 * 60 * 24;
  */
 export const DEFAULT_SETTINGS: Settings = {
     maxFileSize: 2 * 1024 * 1024, // 2MB in bytes
+    autoDeleteStaleRepos: true,
 }

packages/backend/src/db.test.ts

@@ -1,8 +1,23 @@
 import { expect, test } from 'vitest';
-import { migration_addMaxFileSize, migration_addSettings, Schema } from './db';
+import { DEFAULT_DB_DATA, migration_addDeleteStaleRepos, migration_addMaxFileSize, migration_addSettings, Schema } from './db';
 import { DEFAULT_SETTINGS } from './constants';
 import { DeepPartial } from './types';
+import { Low } from 'lowdb';
 
+class InMemoryAdapter<T> {
+    private data: T;
+    async read() {
+        return this.data;
+    }
+    async write(data: T) {
+        this.data = data;
+    }
+}
+
+export const createMockDB = (defaultData: Schema = DEFAULT_DB_DATA) => {
+    const db = new Low(new InMemoryAdapter<Schema>(), defaultData);
+    return db;
+}
 
 test('migration_addSettings adds the `settings` field with defaults if it does not exist', () => {
     const schema: DeepPartial<Schema> = {};
@@ -29,4 +44,20 @@ test('migration_addMaxFileSize adds the `maxFileSize` field with the default val
 test('migration_addMaxFileSize will throw if `settings` is not defined', () => {
     const schema: DeepPartial<Schema> = {};
     expect(() => migration_addMaxFileSize(schema as Schema)).toThrow();
+});
+
+test('migration_addDeleteStaleRepos adds the `autoDeleteStaleRepos` field with the default value if it does not exist', () => {
+    const schema: DeepPartial<Schema> = {
+        settings: {
+            maxFileSize: DEFAULT_SETTINGS.maxFileSize,
+        },
+    }
+
+    const migratedSchema = migration_addDeleteStaleRepos(schema as Schema);
+    expect(migratedSchema).toStrictEqual({
+        settings: {
+            maxFileSize: DEFAULT_SETTINGS.maxFileSize,
+            autoDeleteStaleRepos: DEFAULT_SETTINGS.autoDeleteStaleRepos,
+        }
+    });
 });

packages/backend/src/db.ts

@@ -13,13 +13,15 @@ export type Schema = {
     }
 }
 
+export const DEFAULT_DB_DATA: Schema = {
+    repos: {},
+    settings: DEFAULT_SETTINGS,
+}
+
 export type Database = Low<Schema>;
 
 export const loadDB = async (ctx: AppContext): Promise<Database> => {
-    const db = await JSONFilePreset<Schema>(`${ctx.cachePath}/db.json`, {
-        repos: {},
-        settings: DEFAULT_SETTINGS,
-    });
+    const db = await JSONFilePreset<Schema>(`${ctx.cachePath}/db.json`, DEFAULT_DB_DATA);
 
     await applyMigrations(db);
 
@@ -53,6 +55,7 @@ export const applyMigrations = async (db: Database) => {
         // @NOTE: please ensure new migrations are added after older ones!
         schema = migration_addSettings(schema, log);
         schema = migration_addMaxFileSize(schema, log);
+        schema = migration_addDeleteStaleRepos(schema, log);
         return schema;
     });
 }
@@ -78,5 +81,17 @@ export const migration_addMaxFileSize = (schema: Schema, log?: (name: string) =>
         schema.settings.maxFileSize = DEFAULT_SETTINGS.maxFileSize;
     }
 
+    return schema;
+}
+
+/**
+ * @see: https://github.com/sourcebot-dev/sourcebot/pull/128
+ */
+export const migration_addDeleteStaleRepos = (schema: Schema, log?: (name: string) => void) => {
+    if (schema.settings.autoDeleteStaleRepos === undefined) {
+        log?.("deleteStaleRepos");
+        schema.settings.autoDeleteStaleRepos = DEFAULT_SETTINGS.autoDeleteStaleRepos;
+    }
+
     return schema;
 }

packages/backend/src/github.ts

@@ -100,7 +100,8 @@ export const getGitHubReposFromConfig = async (config: GitHubConfig, signal: Abo
         });
 
     if (config.topics) {
-        repos = includeReposByTopic(repos, config.topics, logger);
+        const topics = config.topics.map(topic => topic.toLowerCase());
+        repos = includeReposByTopic(repos, topics, logger);
     }
 
     if (config.exclude) {
@@ -117,7 +118,8 @@ export const getGitHubReposFromConfig = async (config: GitHubConfig, signal: Abo
         }
 
         if (config.exclude.topics) {
-            repos = excludeReposByTopic(repos, config.exclude.topics, logger);
+            const topics = config.exclude.topics.map(topic => topic.toLowerCase());
+            repos = excludeReposByTopic(repos, topics, logger);
         }
     }
 

packages/backend/src/gitlab.ts

@@ -115,7 +115,8 @@ export const getGitLabReposFromConfig = async (config: GitLabConfig, ctx: AppCon
         });
 
     if (config.topics) {
-        repos = includeReposByTopic(repos, config.topics, logger);
+        const topics = config.topics.map(topic => topic.toLowerCase());
+        repos = includeReposByTopic(repos, topics, logger);
     }
 
     if (config.exclude) {
@@ -132,7 +133,8 @@ export const getGitLabReposFromConfig = async (config: GitLabConfig, ctx: AppCon
         }
 
         if (config.exclude.topics) {
-            repos = excludeReposByTopic(repos, config.exclude.topics, logger);
+            const topics = config.exclude.topics.map(topic => topic.toLowerCase());
+            repos = excludeReposByTopic(repos, topics, logger);
         }
     }
 

packages/backend/src/main.test.ts

@@ -1,6 +1,29 @@
-import { expect, test } from 'vitest';
-import { isAllRepoReindexingRequired, isRepoReindexingRequired } from './main';
-import { Repository, Settings } from './types';
+import { expect, test, vi } from 'vitest';
+import { deleteStaleRepository, isAllRepoReindexingRequired, isRepoReindexingRequired } from './main';
+import { AppContext, GitRepository, LocalRepository, Repository, Settings } from './types';
+import { DEFAULT_DB_DATA } from './db';
+import { createMockDB } from './db.test';
+import { rm } from 'fs/promises';
+import path from 'path';
+import { glob } from 'glob';
+
+vi.mock('fs/promises', () => ({
+    rm: vi.fn(),
+}));
+
+vi.mock('glob', () => ({
+    glob: vi.fn().mockReturnValue(['fake_index.zoekt']),
+}));
+
+const createMockContext = (rootPath: string = '/app') => {
+    return {
+        configPath: path.join(rootPath, 'config.json'),
+        cachePath: path.join(rootPath, '.sourcebot'),
+        indexPath: path.join(rootPath, '.sourcebot/index'),
+        reposPath: path.join(rootPath, '.sourcebot/repos'),
+    } satisfies AppContext;
+}
+
 
 test('isRepoReindexingRequired should return false when no changes are made', () => {
     const previous: Repository = {
@@ -80,6 +103,7 @@ test('isRepoReindexingRequired should return true when local excludedPaths chang
 test('isAllRepoReindexingRequired should return false when fileLimitSize has not changed', () => {
     const previous: Settings = {
         maxFileSize: 1000,
+        autoDeleteStaleRepos: true,
     }
     const current: Settings = {
         ...previous,
@@ -90,10 +114,89 @@ test('isAllRepoReindexingRequired should return false when fileLimitSize has not
 test('isAllRepoReindexingRequired should return true when fileLimitSize has changed', () => {
     const previous: Settings = {
         maxFileSize: 1000,
+        autoDeleteStaleRepos: true,
     }
     const current: Settings = {
         ...previous,
         maxFileSize: 2000,
     }
     expect(isAllRepoReindexingRequired(previous, current)).toBe(true);
+});
+
+test('isAllRepoReindexingRequired should return false when autoDeleteStaleRepos has changed', () => {
+    const previous: Settings = {
+        maxFileSize: 1000,
+        autoDeleteStaleRepos: true,
+    }
+    const current: Settings = {
+        ...previous,
+        autoDeleteStaleRepos: false,
+    }
+    expect(isAllRepoReindexingRequired(previous, current)).toBe(false);
+});
+
+test('deleteStaleRepository can delete a git repository', async () => {
+    const ctx = createMockContext();
+
+    const repo: GitRepository = {
+        id: 'github.com/sourcebot-dev/sourcebot',
+        vcs: 'git',
+        name: 'sourcebot',
+        cloneUrl: 'https://github.com/sourcebot-dev/sourcebot',
+        path: `${ctx.reposPath}/github.com/sourcebot-dev/sourcebot`,
+        branches: ['main'],
+        tags: [''],
+        isStale: true,
+    }
+
+    const db = createMockDB({
+        ...DEFAULT_DB_DATA,
+        repos: {
+            'github.com/sourcebot-dev/sourcebot': repo,
+        }
+    });
+
+
+    await deleteStaleRepository(repo, db, ctx);
+
+    expect(db.data.repos['github.com/sourcebot-dev/sourcebot']).toBeUndefined();;
+    expect(rm).toHaveBeenCalledWith(`${ctx.reposPath}/github.com/sourcebot-dev/sourcebot`, {
+        recursive: true,
+    });
+    expect(glob).toHaveBeenCalledWith(`github.com%2Fsourcebot-dev%2Fsourcebot*.zoekt`, {
+        cwd: ctx.indexPath,
+        absolute: true
+    });
+    expect(rm).toHaveBeenCalledWith(`fake_index.zoekt`);
+});
+
+test('deleteStaleRepository can delete a local repository', async () => {
+    const ctx = createMockContext();
+
+    const repo: LocalRepository = {
+        vcs: 'local',
+        name: 'UnrealEngine',
+        id: '/path/to/UnrealEngine',
+        path: '/path/to/UnrealEngine',
+        watch: false,
+        excludedPaths: [],
+        isStale: true,
+    }
+
+    const db = createMockDB({
+        ...DEFAULT_DB_DATA,
+        repos: {
+            '/path/to/UnrealEngine': repo,
+        }
+    });
+
+    await deleteStaleRepository(repo, db, ctx);
+
+    expect(db.data.repos['/path/to/UnrealEngine']).toBeUndefined();
+    expect(rm).not.toHaveBeenCalledWith('/path/to/UnrealEngine');
+    expect(glob).toHaveBeenCalledWith(`UnrealEngine*.zoekt`, {
+        cwd: ctx.indexPath,
+        absolute: true
+    });
+    expect(rm).toHaveBeenCalledWith('fake_index.zoekt');
 });