feat: expand git submodule contents in sandbox filetree

lib/github/__tests__/expandSubmoduleEntries.test.ts

@@ -0,0 +1,127 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
+import { expandSubmoduleEntries } from "../expandSubmoduleEntries";
+import { getRepoGitModules } from "../getRepoGitModules";
+import { getRepoFileTree } from "../getRepoFileTree";
+
+vi.mock("../getRepoGitModules", () => ({
+  getRepoGitModules: vi.fn(),
+}));
+
+vi.mock("../getRepoFileTree", () => ({
+  getRepoFileTree: vi.fn(),
+}));
+
+const repo = { owner: "owner", repo: "repo", branch: "main" };
+
+describe("expandSubmoduleEntries", () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it("expands submodule with fetched tree contents", async () => {
+    vi.mocked(getRepoGitModules).mockResolvedValue([
+      { path: "orgs/my-org", url: "https://github.com/recoupable/org-abc" },
+    ]);
+    vi.mocked(getRepoFileTree).mockResolvedValue([
+      { path: "artist.md", type: "blob", sha: "art1", size: 200 },
+      { path: "data", type: "tree", sha: "data1" },
+    ]);
+
+    const result = await expandSubmoduleEntries({
+      regularEntries: [{ path: "README.md", type: "blob", sha: "abc", size: 50 }],
+      submoduleEntries: [{ path: "orgs/my-org", sha: "sub1" }],
+      repo,
+    });
+
+    expect(result).toEqual([
+      { path: "README.md", type: "blob", sha: "abc", size: 50 },
+      { path: "orgs/my-org", type: "tree", sha: "sub1" },
+      { path: "orgs/my-org/artist.md", type: "blob", sha: "art1", size: 200 },
+      { path: "orgs/my-org/data", type: "tree", sha: "data1" },
+    ]);
+    expect(getRepoFileTree).toHaveBeenCalledWith("https://github.com/recoupable/org-abc");
+  });
+
+  it("converts submodules to empty directories when getRepoGitModules returns null", async () => {
+    vi.mocked(getRepoGitModules).mockResolvedValue(null);
+
+    const result = await expandSubmoduleEntries({
+      regularEntries: [],
+      submoduleEntries: [{ path: "orgs/my-org", sha: "sub1" }],
+      repo,
+    });
+
+    expect(result).toEqual([{ path: "orgs/my-org", type: "tree", sha: "sub1" }]);
+    expect(getRepoFileTree).not.toHaveBeenCalled();
+  });
+
+  it("converts submodule to empty directory when URL not found in .gitmodules", async () => {
+    vi.mocked(getRepoGitModules).mockResolvedValue([
+      { path: "orgs/other-org", url: "https://github.com/recoupable/other" },
+    ]);
+
+    const result = await expandSubmoduleEntries({
+      regularEntries: [],
+      submoduleEntries: [{ path: "orgs/unknown", sha: "sub1" }],
+      repo,
+    });
+
+    expect(result).toEqual([{ path: "orgs/unknown", type: "tree", sha: "sub1" }]);
+    expect(getRepoFileTree).not.toHaveBeenCalled();
+  });
+
+  it("converts submodule to empty directory when submodule tree fetch returns null", async () => {
+    vi.mocked(getRepoGitModules).mockResolvedValue([
+      { path: "orgs/my-org", url: "https://github.com/recoupable/org-abc" },
+    ]);
+    vi.mocked(getRepoFileTree).mockResolvedValue(null);
+
+    const result = await expandSubmoduleEntries({
+      regularEntries: [],
+      submoduleEntries: [{ path: "orgs/my-org", sha: "sub1" }],
+      repo,
+    });
+
+    expect(result).toEqual([{ path: "orgs/my-org", type: "tree", sha: "sub1" }]);
+  });
+
+  it("expands multiple submodules in parallel", async () => {
+    vi.mocked(getRepoGitModules).mockResolvedValue([
+      { path: "orgs/org-a", url: "https://github.com/recoupable/org-a" },
+      { path: "orgs/org-b", url: "https://github.com/recoupable/org-b" },
+    ]);
+    vi.mocked(getRepoFileTree)
+      .mockResolvedValueOnce([{ path: "a.md", type: "blob", sha: "a1", size: 10 }])
+      .mockResolvedValueOnce([{ path: "b.md", type: "blob", sha: "b1", size: 20 }]);
+
+    const result = await expandSubmoduleEntries({
+      regularEntries: [],
+      submoduleEntries: [
+        { path: "orgs/org-a", sha: "suba" },
+        { path: "orgs/org-b", sha: "subb" },
+      ],
+      repo,
+    });
+
+    expect(result).toContainEqual({ path: "orgs/org-a", type: "tree", sha: "suba" });
+    expect(result).toContainEqual({ path: "orgs/org-a/a.md", type: "blob", sha: "a1", size: 10 });
+    expect(result).toContainEqual({ path: "orgs/org-b", type: "tree", sha: "subb" });
+    expect(result).toContainEqual({ path: "orgs/org-b/b.md", type: "blob", sha: "b1", size: 20 });
+  });
+
+  it("passes correct repo context to getRepoGitModules", async () => {
+    vi.mocked(getRepoGitModules).mockResolvedValue([]);
+
+    await expandSubmoduleEntries({
+      regularEntries: [],
+      submoduleEntries: [{ path: "sub", sha: "s1" }],
+      repo: { owner: "my-owner", repo: "my-repo", branch: "develop" },
+    });
+
+    expect(getRepoGitModules).toHaveBeenCalledWith({
+      owner: "my-owner",
+      repo: "my-repo",
+      branch: "develop",
+    });
+  });
+});

lib/github/__tests__/getRepoFileTree.test.ts

@@ -58,9 +58,7 @@ describe("getRepoFileTree", () => {
   });
 
   it("returns null when repo metadata fetch fails", async () => {
-    vi.spyOn(global, "fetch").mockResolvedValueOnce(
-      new Response("Not Found", { status: 404 }),
-    );
+    vi.spyOn(global, "fetch").mockResolvedValueOnce(new Response("Not Found", { status: 404 }));
 
     const result = await getRepoFileTree("https://github.com/owner/repo");
 
@@ -88,13 +86,12 @@ describe("getRepoFileTree", () => {
   });
 
   it("passes correct Authorization header", async () => {
-    const fetchSpy = vi.spyOn(global, "fetch")
+    const fetchSpy = vi
+      .spyOn(global, "fetch")
       .mockResolvedValueOnce(
         new Response(JSON.stringify({ default_branch: "main" }), { status: 200 }),
       )
-      .mockResolvedValueOnce(
-        new Response(JSON.stringify({ tree: [] }), { status: 200 }),
-      );
+      .mockResolvedValueOnce(new Response(JSON.stringify({ tree: [] }), { status: 200 }));
 
     await getRepoFileTree("https://github.com/owner/repo");
 
@@ -104,16 +101,235 @@ describe("getRepoFileTree", () => {
   });
 
   it("uses the default branch from repo metadata", async () => {
-    const fetchSpy = vi.spyOn(global, "fetch")
+    const fetchSpy = vi
+      .spyOn(global, "fetch")
       .mockResolvedValueOnce(
         new Response(JSON.stringify({ default_branch: "develop" }), { status: 200 }),
       )
-      .mockResolvedValueOnce(
-        new Response(JSON.stringify({ tree: [] }), { status: 200 }),
-      );
+      .mockResolvedValueOnce(new Response(JSON.stringify({ tree: [] }), { status: 200 }));
 
     await getRepoFileTree("https://github.com/owner/repo");
 
     expect(fetchSpy.mock.calls[1][0]).toContain("/git/trees/develop?recursive=1");
   });
+
+  describe("submodule expansion", () => {
+    const gitmodulesContent = `[submodule ".openclaw/workspace/orgs/my-org"]
+\tpath = .openclaw/workspace/orgs/my-org
+\turl = https://github.com/recoupable/org-my-org-abc123`;
+
+    it("expands submodule entries into directories with contents", async () => {
+      vi.spyOn(global, "fetch")
+        // Parent repo metadata
+        .mockResolvedValueOnce(
+          new Response(JSON.stringify({ default_branch: "main" }), { status: 200 }),
+        )
+        // Parent tree (includes a commit entry for the submodule)
+        .mockResolvedValueOnce(
+          new Response(
+            JSON.stringify({
+              tree: [
+                { path: "README.md", type: "blob", sha: "abc123", size: 50 },
+                { path: ".openclaw/workspace/orgs/my-org", type: "commit", sha: "sub111" },
+                { path: ".openclaw", type: "tree", sha: "claw1" },
+                { path: ".openclaw/workspace", type: "tree", sha: "ws1" },
+                { path: ".openclaw/workspace/orgs", type: "tree", sha: "orgs1" },
+              ],
+            }),
+            { status: 200 },
+          ),
+        )
+        // .gitmodules fetch
+        .mockResolvedValueOnce(new Response(gitmodulesContent, { status: 200 }))
+        // Submodule repo metadata
+        .mockResolvedValueOnce(
+          new Response(JSON.stringify({ default_branch: "main" }), { status: 200 }),
+        )
+        // Submodule tree
+        .mockResolvedValueOnce(
+          new Response(
+            JSON.stringify({
+              tree: [
+                { path: "artist.md", type: "blob", sha: "art1", size: 200 },
+                { path: "data", type: "tree", sha: "data1" },
+                { path: "data/info.json", type: "blob", sha: "info1", size: 80 },
+              ],
+            }),
+            { status: 200 },
+          ),
+        );
+
+      const result = await getRepoFileTree("https://github.com/owner/repo");
+
+      expect(result).toEqual([
+        { path: "README.md", type: "blob", sha: "abc123", size: 50 },
+        { path: ".openclaw", type: "tree", sha: "claw1" },
+        { path: ".openclaw/workspace", type: "tree", sha: "ws1" },
+        { path: ".openclaw/workspace/orgs", type: "tree", sha: "orgs1" },
+        // Submodule converted to a tree entry
+        { path: ".openclaw/workspace/orgs/my-org", type: "tree", sha: "sub111" },
+        // Submodule contents prefixed with submodule path
+        { path: ".openclaw/workspace/orgs/my-org/artist.md", type: "blob", sha: "art1", size: 200 },
+        { path: ".openclaw/workspace/orgs/my-org/data", type: "tree", sha: "data1" },
+        {
+          path: ".openclaw/workspace/orgs/my-org/data/info.json",
+          type: "blob",
+          sha: "info1",
+          size: 80,
+        },
+      ]);
+    });
+
+    it("converts submodule to empty directory when .gitmodules fetch fails", async () => {
+      vi.spyOn(global, "fetch")
+        .mockResolvedValueOnce(
+          new Response(JSON.stringify({ default_branch: "main" }), { status: 200 }),
+        )
+        .mockResolvedValueOnce(
+          new Response(
+            JSON.stringify({
+              tree: [
+                { path: "README.md", type: "blob", sha: "abc123", size: 50 },
+                { path: ".openclaw/workspace/orgs/my-org", type: "commit", sha: "sub111" },
+              ],
+            }),
+            { status: 200 },
+          ),
+        )
+        // .gitmodules fetch fails
+        .mockResolvedValueOnce(new Response("Not Found", { status: 404 }));
+
+      const result = await getRepoFileTree("https://github.com/owner/repo");
+
+      expect(result).toEqual([
+        { path: "README.md", type: "blob", sha: "abc123", size: 50 },
+        { path: ".openclaw/workspace/orgs/my-org", type: "tree", sha: "sub111" },
+      ]);
+    });
+
+    it("converts submodule to empty directory when URL not in .gitmodules", async () => {
+      vi.spyOn(global, "fetch")
+        .mockResolvedValueOnce(
+          new Response(JSON.stringify({ default_branch: "main" }), { status: 200 }),
+        )
+        .mockResolvedValueOnce(
+          new Response(
+            JSON.stringify({
+              tree: [{ path: "unknown-submodule", type: "commit", sha: "sub222" }],
+            }),
+            { status: 200 },
+          ),
+        )
+        // .gitmodules has different submodule
+        .mockResolvedValueOnce(new Response(gitmodulesContent, { status: 200 }));
+
+      const result = await getRepoFileTree("https://github.com/owner/repo");
+
+      expect(result).toEqual([{ path: "unknown-submodule", type: "tree", sha: "sub222" }]);
+    });
+
+    it("converts submodule to empty directory when submodule tree fetch fails", async () => {
+      vi.spyOn(global, "fetch")
+        .mockResolvedValueOnce(
+          new Response(JSON.stringify({ default_branch: "main" }), { status: 200 }),
+        )
+        .mockResolvedValueOnce(
+          new Response(
+            JSON.stringify({
+              tree: [{ path: ".openclaw/workspace/orgs/my-org", type: "commit", sha: "sub111" }],
+            }),
+            { status: 200 },
+          ),
+        )
+        .mockResolvedValueOnce(new Response(gitmodulesContent, { status: 200 }))
+        // Submodule repo metadata fails
+        .mockResolvedValueOnce(new Response("Not Found", { status: 404 }));
+
+      const result = await getRepoFileTree("https://github.com/owner/repo");
+
+      expect(result).toEqual([
+        { path: ".openclaw/workspace/orgs/my-org", type: "tree", sha: "sub111" },
+      ]);
+    });
+
+    it("expands multiple submodules in parallel", async () => {
+      const multiGitmodules = `[submodule ".openclaw/workspace/orgs/org-a"]
+\tpath = .openclaw/workspace/orgs/org-a
+\turl = https://github.com/recoupable/org-a-111
+[submodule ".openclaw/workspace/orgs/org-b"]
+\tpath = .openclaw/workspace/orgs/org-b
+\turl = https://github.com/recoupable/org-b-222`;
+
+      vi.spyOn(global, "fetch")
+        // Parent repo metadata
+        .mockResolvedValueOnce(
+          new Response(JSON.stringify({ default_branch: "main" }), { status: 200 }),
+        )
+        // Parent tree
+        .mockResolvedValueOnce(
+          new Response(
+            JSON.stringify({
+              tree: [
+                { path: ".openclaw/workspace/orgs/org-a", type: "commit", sha: "suba" },
+                { path: ".openclaw/workspace/orgs/org-b", type: "commit", sha: "subb" },
+              ],
+            }),
+            { status: 200 },
+          ),
+        )
+        // .gitmodules
+        .mockResolvedValueOnce(new Response(multiGitmodules, { status: 200 }))
+        // Submodule A repo metadata
+        .mockResolvedValueOnce(
+          new Response(JSON.stringify({ default_branch: "main" }), { status: 200 }),
+        )
+        // Submodule B repo metadata (parallel)
+        .mockResolvedValueOnce(
+          new Response(JSON.stringify({ default_branch: "main" }), { status: 200 }),
+        )
+        // Submodule A tree
+        .mockResolvedValueOnce(
+          new Response(
+            JSON.stringify({
+              tree: [{ path: "file-a.md", type: "blob", sha: "fa1", size: 10 }],
+            }),
+            { status: 200 },
+          ),
+        )
+        // Submodule B tree
+        .mockResolvedValueOnce(
+          new Response(
+            JSON.stringify({
+              tree: [{ path: "file-b.md", type: "blob", sha: "fb1", size: 20 }],
+            }),
+            { status: 200 },
+          ),
+        );
+
+      const result = await getRepoFileTree("https://github.com/owner/repo");
+
+      expect(result).toContainEqual({
+        path: ".openclaw/workspace/orgs/org-a",
+        type: "tree",
+        sha: "suba",
+      });
+      expect(result).toContainEqual({
+        path: ".openclaw/workspace/orgs/org-a/file-a.md",
+        type: "blob",
+        sha: "fa1",
+        size: 10,
+      });
+      expect(result).toContainEqual({
+        path: ".openclaw/workspace/orgs/org-b",
+        type: "tree",
+        sha: "subb",
+      });
+      expect(result).toContainEqual({
+        path: ".openclaw/workspace/orgs/org-b/file-b.md",
+        type: "blob",
+        sha: "fb1",
+        size: 20,
+      });
+    });
+  });
 });