Skip to content

Commit 0fdb6e5

Browse files
fennifithfennifith
authored
Merge pull request #63 from playfulprogramming/sync-author-task
Task for syncing author data
2 parents 1de08a4 + 429066a commit 0fdb6e5

File tree

28 files changed

+337906
-33
lines changed

28 files changed

+337906
-33
lines changed

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99

1010
# dependencies
1111
**/node_modules/
12+
/.pnpm-store
1213

1314
# logs
1415
**/npm-debug.log*

‎apps/api/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,6 @@
2626
"fastify-plugin": "^5.0.1"
2727
},
2828
"devDependencies": {
29-
"vitest": "^3.2.4"
29+
"vitest": "catalog:"
3030
}
3131
}

‎apps/api/test-utils/setup.ts‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,10 +32,12 @@ vi.mock("@playfulprogramming/db", () => {
3232
});
3333

3434
vi.mock("@playfulprogramming/common", async (importOriginal) => {
35+
const original = (await importOriginal()) as object;
3536
return {
36-
...(await importOriginal()),
37+
...original,
3738
env: {
3839
PORT: 3000,
40+
WORKER_PORT: 3001,
3941
ENVIRONMENT: "production",
4042
SITE_URL: "https://site_url.test",
4143
S3_PUBLIC_URL: "https://s3_public_url.test",
@@ -47,6 +49,9 @@ vi.mock("@playfulprogramming/common", async (importOriginal) => {
4749
REDIS_URL: "redis://redis_url.test",
4850
REDIS_PASSWORD: "redis_password",
4951
HOOF_AUTH_TOKEN: "supersecret",
52+
GITHUB_REPO_OWNER: "playfulprogramming",
53+
GITHUB_REPO_NAME: "playfulprogramming",
54+
GITHUB_TOKEN: "github_token",
5055
} satisfies EnvType,
5156
};
5257
});

‎apps/worker/package.json‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,21 +14,25 @@
1414
"dependencies": {
1515
"@playfulprogramming/common": "workspace:*",
1616
"@playfulprogramming/db": "workspace:*",
17+
"@playfulprogramming/github-api": "workspace:*",
1718
"@playfulprogramming/post-images": "workspace:*",
1819
"@playfulprogramming/redis": "workspace:*",
1920
"@playfulprogramming/s3": "workspace:*",
21+
"@sinclair/typebox": "catalog:",
2022
"bullmq": "catalog:",
23+
"drizzle-orm": "catalog:",
24+
"gray-matter": "^4.0.3",
2125
"hast-util-from-html": "^2.0.3",
2226
"lru-cache": "^11.2.2",
2327
"robots-parser": "^3.0.1",
2428
"sharp": "^0.34.4",
2529
"svgo": "^4.0.0",
26-
"undici": "^7.16.0",
30+
"undici": "catalog:",
2731
"unist-util-find": "^3.0.0",
2832
"unist-util-visit": "^5.0.0"
2933
},
3034
"devDependencies": {
3135
"@types/hast": "^3.0.4",
32-
"vitest": "^3.2.4"
36+
"vitest": "catalog:"
3337
}
3438
}

‎apps/worker/src/index.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,4 +4,5 @@ import { Tasks } from "@playfulprogramming/common";
44

55
createWorker(Tasks.POST_IMAGES, "./tasks/post-images/processor.ts");
66
createWorker(Tasks.URL_METADATA, "./tasks/url-metadata/processor.ts");
7+
createWorker(Tasks.SYNC_AUTHOR, "./tasks/sync-author/processor.ts");
78
createHealthcheck();

‎apps/worker/src/tasks/sync-author/processor.test.ts‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
import processor from "./processor.ts";
2+
import type { TaskInputs } from "@playfulprogramming/common";
3+
import type { Job } from "bullmq";
4+
import { db, profiles } from "@playfulprogramming/db";
5+
import { s3 } from "@playfulprogramming/s3";
6+
import * as github from "@playfulprogramming/github-api";
7+
import { Readable } from "node:stream";
8+
import { eq } from "drizzle-orm";
9+
10+
test("Creates an example profile successfully", async () => {
11+
const insertValues = vi.fn().mockReturnValue({
12+
onConflictDoUpdate: vi.fn(),
13+
});
14+
vi.mocked(db.insert).mockReturnValue({
15+
values: insertValues,
16+
} as never);
17+
18+
vi.mocked(github.getContentsRaw).mockImplementation((params) => {
19+
if (params.path === "/content/example/index.md") {
20+
return Promise.resolve({
21+
data: `---
22+
{
23+
name: "Example Person",
24+
description: "Hello",
25+
profileImg: "./profile.png"
26+
}
27+
---
28+
`,
29+
response: {} as never,
30+
});
31+
}
32+
return Promise.reject();
33+
});
34+
35+
vi.mocked(github.getContentsRawStream).mockImplementation((params) => {
36+
if (params.path === "/content/example/profile.png") {
37+
const buffer = Buffer.from(
38+
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAACklEQVR42mMAAQAABQABoIJXOQAAAABJRU5ErkJggg==",
39+
"base64",
40+
);
41+
return Promise.resolve({
42+
data: Readable.toWeb(Readable.from(buffer)) as never,
43+
response: {} as never,
44+
});
45+
}
46+
return Promise.reject();
47+
});
48+
49+
await processor({
50+
data: {
51+
author: "example",
52+
ref: "main",
53+
},
54+
} as unknown as Job<TaskInputs["sync-author"]>);
55+
56+
// The profile image was uploaded to S3
57+
expect(s3.upload).toBeCalledWith(
58+
"example-bucket",
59+
"profiles/example.jpeg",
60+
undefined,
61+
expect.anything(),
62+
"image/jpeg",
63+
);
64+
65+
// The profile was inserted into the database
66+
expect(insertValues).toBeCalledWith({
67+
slug: "example",
68+
name: "Example Person",
69+
description: "Hello",
70+
profileImage: "profiles/example.jpeg",
71+
meta: {
72+
socials: {},
73+
roles: [],
74+
},
75+
});
76+
});
77+
78+
test("Deletes a profile record if it no longer exists", async () => {
79+
const deleteWhere = vi.fn();
80+
vi.mocked(db.delete).mockReturnValue({
81+
where: deleteWhere,
82+
} as never);
83+
84+
vi.mocked(github.getContentsRaw).mockImplementation((params) => {
85+
if (params.path === "/content/example/index.md") {
86+
return Promise.resolve({
87+
data: undefined,
88+
error: {},
89+
response: {
90+
status: 404,
91+
} as never,
92+
});
93+
}
94+
return Promise.reject();
95+
});
96+
97+
await processor({
98+
data: {
99+
author: "example",
100+
ref: "main",
101+
},
102+
} as unknown as Job<TaskInputs["sync-author"]>);
103+
104+
// The profile was deleted from the database
105+
expect(deleteWhere).toBeCalledWith(eq(profiles.slug, "example"));
106+
});

‎apps/worker/src/tasks/sync-author/processor.ts‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
import { Tasks, env } from "@playfulprogramming/common";
2+
import { db, profiles } from "@playfulprogramming/db";
3+
import * as github from "@playfulprogramming/github-api";
4+
import { s3 } from "@playfulprogramming/s3";
5+
import { createProcessor } from "../../createProcessor.ts";
6+
import matter from "gray-matter";
7+
import { AuthorMetaSchema } from "./types.ts";
8+
import { Value } from "@sinclair/typebox/value";
9+
import sharp from "sharp";
10+
import { Readable } from "node:stream";
11+
import { eq } from "drizzle-orm";
12+
13+
const PROFILE_IMAGE_SIZE_MAX = 2048;
14+
15+
async function processProfileImg(
16+
stream: ReadableStream<Uint8Array>,
17+
uploadKey: string,
18+
) {
19+
const pipeline = sharp()
20+
.resize({
21+
width: PROFILE_IMAGE_SIZE_MAX,
22+
height: PROFILE_IMAGE_SIZE_MAX,
23+
fit: "inside",
24+
})
25+
.jpeg({ mozjpeg: true });
26+
27+
Readable.fromWeb(stream as never).pipe(pipeline);
28+
29+
const bucket = await s3.createBucket(env.S3_BUCKET);
30+
await s3.upload(bucket, uploadKey, undefined, pipeline, "image/jpeg");
31+
}
32+
33+
export default createProcessor(Tasks.SYNC_AUTHOR, async (job, { signal }) => {
34+
const authorId = job.data.author;
35+
const authorMetaUrl = new URL(
36+
`content/${encodeURIComponent(authorId)}/index.md`,
37+
"http://localhost",
38+
);
39+
40+
const authorMetaResponse = await github.getContentsRaw({
41+
ref: job.data.ref,
42+
path: authorMetaUrl.pathname,
43+
repoOwner: env.GITHUB_REPO_OWNER,
44+
repoName: env.GITHUB_REPO_NAME,
45+
signal,
46+
});
47+
48+
if (authorMetaResponse.data === undefined) {
49+
if (authorMetaResponse.response.status == 404) {
50+
console.log(
51+
`Metadata for ${authorId} (${authorMetaUrl.pathname}) returned 404 - removing profile entry.`,
52+
);
53+
await db.delete(profiles).where(eq(profiles.slug, authorId));
54+
return;
55+
}
56+
57+
throw new Error(`Unable to fetch author data for ${authorId}`);
58+
}
59+
60+
const { data } = matter(authorMetaResponse.data);
61+
const authorData = Value.Parse(AuthorMetaSchema, data);
62+
63+
let profileImgKey: string | null = null;
64+
if (authorData.profileImg) {
65+
const profileImgUrl = new URL(authorData.profileImg, authorMetaUrl);
66+
const { data: profileImgStream } = await github.getContentsRawStream({
67+
ref: job.data.ref,
68+
path: profileImgUrl.pathname,
69+
repoOwner: env.GITHUB_REPO_OWNER,
70+
repoName: env.GITHUB_REPO_NAME,
71+
signal,
72+
});
73+
74+
if (profileImgStream === null || typeof profileImgStream === "undefined") {
75+
throw new Error(
76+
`Unable to fetch profile image for ${authorId} (${profileImgUrl.pathname})`,
77+
);
78+
}
79+
80+
profileImgKey = `profiles/${authorId}.jpeg`;
81+
await processProfileImg(profileImgStream, profileImgKey);
82+
}
83+
84+
const result = {
85+
slug: authorId,
86+
name: authorData.name,
87+
description: authorData.description,
88+
profileImage: profileImgKey,
89+
meta: {
90+
socials: authorData.socials,
91+
roles: authorData.roles,
92+
},
93+
};
94+
95+
await db
96+
.insert(profiles)
97+
.values(result)
98+
.onConflictDoUpdate({ target: profiles.slug, set: result });
99+
});

‎apps/worker/src/tasks/sync-author/types.ts‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
import { Type } from "@sinclair/typebox";
2+
3+
export const AuthorMetaSchema = Type.Object(
4+
{
5+
name: Type.String(),
6+
description: Type.String({ default: "" }),
7+
profileImg: Type.Optional(Type.String()),
8+
socials: Type.Record(Type.String(), Type.String(), { default: {} }),
9+
roles: Type.Array(Type.String(), { default: [] }),
10+
},
11+
{
12+
additionalProperties: false,
13+
examples: [
14+
{
15+
author: "fennifith",
16+
ref: "main",
17+
},
18+
],
19+
},
20+
);

‎apps/worker/test-utils/setup.ts‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1 +1,36 @@
11
import "./server.ts";
2+
import { vi, afterEach } from "vitest";
3+
4+
afterEach(() => {
5+
vi.clearAllMocks();
6+
vi.setSystemTime(new Date("2025-05-05"));
7+
});
8+
9+
vi.mock("@playfulprogramming/s3", () => {
10+
return {
11+
s3: {
12+
createBucket: vi.fn(() => "example-bucket"),
13+
upload: vi.fn(),
14+
},
15+
};
16+
});
17+
18+
vi.mock("@playfulprogramming/db", () => {
19+
return {
20+
profiles: {
21+
slug: {},
22+
},
23+
db: {
24+
insert: vi.fn(),
25+
delete: vi.fn(),
26+
},
27+
};
28+
});
29+
30+
vi.mock("@playfulprogramming/github-api", () => {
31+
return {
32+
getContents: vi.fn(),
33+
getContentsRaw: vi.fn(),
34+
getContentsRawStream: vi.fn(),
35+
};
36+
});

‎packages/common/src/env.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,10 @@ export const EnvSchema = Type.Object({
2727
RATE_LIMIT_WINDOW: Type.Optional(Type.String({ default: "1 minute" })),
2828
RATE_LIMIT_BAN_THRESHOLD: Type.Optional(Type.Integer({ default: 10 })),
2929
HOOF_AUTH_TOKEN: Type.Optional(Type.String()),
30+
31+
GITHUB_REPO_OWNER: Type.String({ default: "playfulprogramming" }),
32+
GITHUB_REPO_NAME: Type.String({ default: "playfulprogramming" }),
33+
GITHUB_TOKEN: Type.Optional(Type.String()),
3034
});
3135

3236
export type EnvType = Static<typeof EnvSchema>;

0 commit comments

Comments
 (0)