Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 9 additions & 1 deletion packages/das/src/api/api.module.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ import { FETCH_QUEUE } from "../queue/constants";
import { AdminController } from "./admin.controller";
import { RequireApiKeyGuard } from "./require-api-key.guard";
import { HealthController } from "./health.controller";
import { DashboardController } from "./dashboard/dashboard.controller";
import { DashboardService } from "./dashboard/dashboard.service";
import { MinersController } from "./miners/miners.controller";
import { MinersService } from "./miners/miners.service";
import { PullsController } from "./pulls/pulls.controller";
Expand All @@ -31,11 +33,17 @@ import { PullsService } from "./pulls/pulls.service";
BullModule.registerQueue({ name: FETCH_QUEUE }),
],
controllers: [
DashboardController,
MinersController,
PullsController,
AdminController,
HealthController,
],
providers: [MinersService, PullsService, RequireApiKeyGuard],
providers: [
DashboardService,
MinersService,
PullsService,
RequireApiKeyGuard,
],
})
export class ApiModule {}
37 changes: 37 additions & 0 deletions packages/das/src/api/dashboard/dashboard.controller.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
import { BadRequestException, Controller, Get, Query } from "@nestjs/common";
import { ApiOperation, ApiQuery, ApiTags } from "@nestjs/swagger";
import { DashboardService } from "./dashboard.service";

@ApiTags("Dashboard")
@Controller("api/v1/dashboard")
export class DashboardController {
constructor(private readonly dashboard: DashboardService) {}

@Get("issues")
@ApiOperation({
summary: "Slim issue rows for dashboard trend aggregation",
description:
"Returns every issue with `created_at` on or after `since`, plus " +
"every CLOSED issue whose `closed_at` is on or after `since`. " +
"The mirror is intentionally roster-blind: every issue is returned " +
"regardless of author. The dashboard blends with the gittensor API " +
"miner roster client-side to filter to subnet authors. " +
"Designed as a single bulk replacement for the dashboard's per-miner " +
"fan-out against `/miners/<id>/issues` (one call instead of N).",
})
@ApiQuery({
name: "since",
required: true,
description: "ISO timestamp — earliest creation/close date to include.",
})
async getIssues(@Query("since") since?: string): Promise<unknown> {
if (!since) {
throw new BadRequestException("`since` query parameter is required");
}
const parsed = new Date(since);
if (Number.isNaN(parsed.getTime())) {
throw new BadRequestException("`since` must be a valid ISO timestamp");
}
return this.dashboard.getIssues(parsed.toISOString());
}
}
58 changes: 58 additions & 0 deletions packages/das/src/api/dashboard/dashboard.service.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
/* eslint-disable @typescript-eslint/no-unsafe-return, @typescript-eslint/no-unsafe-assignment */
import { Injectable } from "@nestjs/common";
import { DataSource } from "typeorm";

interface DashboardIssueRow {
repo_full_name: string;
issue_number: number;
author_github_id: string | null;
created_at: string;
closed_at: string | null;
state: string;
state_reason: string | null;
solving_pr: { merged_at: string } | null;
}

@Injectable()
export class DashboardService {
constructor(private readonly dataSource: DataSource) {}

async getIssues(since: string): Promise<{
since: string;
generated_at: string;
issues: DashboardIssueRow[];
}> {
const rows = await this.dataSource.query(
`
SELECT
LOWER(i.repo_full_name) AS repo_full_name,
i.issue_number,
i.author_github_id,
i.created_at,
i.closed_at,
i.state,
i.state_reason,
(
SELECT json_build_object('merged_at', sp.merged_at)
FROM pull_requests sp
WHERE sp.repo_full_name = i.repo_full_name
AND sp.pr_number = i.solved_by_pr
AND sp.merged_at IS NOT NULL
LIMIT 1
) AS solving_pr
FROM issues i
WHERE
i.created_at >= $1
OR (i.state = 'CLOSED' AND i.closed_at >= $1)
ORDER BY i.created_at DESC
`,
[since],
);

return {
since,
generated_at: new Date().toISOString(),
issues: rows as DashboardIssueRow[],
};
}
}
82 changes: 62 additions & 20 deletions packages/das/src/webhook/github-fetcher.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,11 @@ interface InstallationToken {
expiresAt: number;
}

interface ClosingIssueReference {
number?: number;
repository?: { nameWithOwner?: string } | null;
}

// Files larger than this are stored with null content (AST parsing is wasteful past this).
const MAX_FILE_SIZE_BYTES = 1_000_000;

Expand Down Expand Up @@ -280,7 +285,10 @@ export class GitHubFetcherService implements OnModuleInit {
bodyText
lastEditedAt
closingIssuesReferences(first: 10) {
nodes { number }
nodes {
number
repository { nameWithOwner }
}
}
}
}
Expand Down Expand Up @@ -310,12 +318,29 @@ export class GitHubFetcherService implements OnModuleInit {
const nodes = pr.closingIssuesReferences?.nodes ?? [];

return {
closingIssueNumbers: nodes.map((n: { number: number }) => n.number),
closingIssueNumbers: this.sameRepoClosingIssueNumbers(
repoFullName,
nodes,
),
body: pr.bodyText ?? null,
lastEditedAt: pr.lastEditedAt ?? null,
};
}

private sameRepoClosingIssueNumbers(
repoFullName: string,
nodes: ClosingIssueReference[],
): number[] {
const expectedRepo = repoFullName.toLowerCase();
return nodes
.filter(
(node) =>
node.repository?.nameWithOwner?.toLowerCase() === expectedRepo,
)
.map((node) => node.number)
.filter((number): number is number => typeof number === "number");
}

// --- PR files + contents (REST for list, batched GraphQL for contents) ---

/**
Expand Down Expand Up @@ -380,10 +405,9 @@ export class GitHubFetcherService implements OnModuleInit {

// 3. Fetch file contents in batches (base + head in one GraphQL call each)
if (!pr.headSha) {
this.logger.warn(
`PR ${repoFullName}#${prNumber} has no head SHA — skipping content fetch`,
throw new Error(
`PR ${repoFullName}#${prNumber} has no head SHA; cannot fetch content`,
);
return;
}

// Prefer merge-base SHA (true common ancestor) over base SHA for
Expand Down Expand Up @@ -511,10 +535,9 @@ export class GitHubFetcherService implements OnModuleInit {
batchSize = newSize;
// Retry same i with smaller batch
} else {
this.logger.warn(
`GraphQL content batch failed at min size ${minBatchSize}: ${err}. Skipping batch.`,
throw new Error(
`GraphQL content batch failed at min size ${minBatchSize}: ${err}`,
);
i += batch.length;
}
}
}
Expand Down Expand Up @@ -884,11 +907,14 @@ export class GitHubFetcherService implements OnModuleInit {
authorAssociation
labels(first: 10) { nodes { name } }
timelineItems(
itemTypes: [LABELED_EVENT, UNLABELED_EVENT]
itemTypes: [LABELED_EVENT, UNLABELED_EVENT, TRANSFERRED_EVENT]
first: 30
) {
nodes {
__typename
... on TransferredEvent {
createdAt
}
... on LabeledEvent {
createdAt
label { name }
Expand Down Expand Up @@ -966,6 +992,15 @@ export class GitHubFetcherService implements OnModuleInit {
),
};

if (
(issue.timelineItems?.nodes ?? []).some(
(node: { __typename?: string }) =>
node.__typename === "TransferredEvent",
)
) {
issueData.isTransferred = true;
}

if (issue.state === "OPEN") {
issueData.solvedByPr = null;
}
Expand All @@ -987,8 +1022,10 @@ export class GitHubFetcherService implements OnModuleInit {
}

/**
* Upsert a list of LABELED_EVENT / UNLABELED_EVENT timeline nodes into
* the label_events table. Actor role is resolved at read time via
* Insert LABELED_EVENT / UNLABELED_EVENT timeline nodes into label_events.
* Idempotent: relies on the uq_label_events_natural_key UNIQUE index so
* re-running backfill (or BullMQ retries) collapses to a no-op for events
* already written. Actor role is resolved at read time via
* contributor_repo_roles using stored PR/issue, review, and comment
* association evidence; GraphQL's actor type doesn't expose authorAssociation.
*/
Expand All @@ -998,23 +1035,28 @@ export class GitHubFetcherService implements OnModuleInit {
targetType: "pr" | "issue",
nodes: any[],
): Promise<void> {
for (const node of nodes) {
if (!node || !node.label?.name || !node.createdAt) continue;
const action =
node.__typename === "LabeledEvent" ? "labeled" : "unlabeled";

await this.labelEventRepo.save({
const rows = nodes
.filter((node) => node && node.label?.name && node.createdAt)
.map((node) => ({
repoFullName,
targetNumber,
targetType,
labelName: node.label.name,
action,
action: node.__typename === "LabeledEvent" ? "labeled" : "unlabeled",
actorGithubId: node.actor?.databaseId
? String(node.actor.databaseId)
: null,
actorLogin: node.actor?.login ?? null,
timestamp: node.createdAt,
});
}
}));

if (rows.length === 0) return;

await this.labelEventRepo
.createQueryBuilder()
.insert()
.values(rows)
.orIgnore()
.execute();
}
}
11 changes: 10 additions & 1 deletion packages/das/src/webhook/handlers/comment.handler.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,15 @@
import { Injectable } from "@nestjs/common";
import { InjectRepository } from "@nestjs/typeorm";
import { Repository } from "typeorm";
import { Comment } from "../../entities";
import { Comment, Repo } from "../../entities";

@Injectable()
export class CommentHandler {
constructor(
@InjectRepository(Comment)
private readonly commentRepo: Repository<Comment>,
@InjectRepository(Repo)
private readonly repoRepo: Repository<Repo>,
) {}

async handle(payload: Record<string, any>): Promise<void> {
Expand All @@ -20,6 +22,9 @@ export class CommentHandler {
repoFullName,
commentId: String(comment.id),
});
await this.repoRepo.update(repoFullName, {
lastEventAt: new Date().toISOString(),
});
return;
}

Expand All @@ -40,5 +45,9 @@ export class CommentHandler {
};

await this.commentRepo.upsert(data, ["repoFullName", "commentId"]);

await this.repoRepo.update(repoFullName, {
lastEventAt: new Date().toISOString(),
});
}
}
4 changes: 4 additions & 0 deletions packages/das/src/webhook/handlers/issue.handler.ts
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,10 @@ export class IssueHandler {
data.solvedByPr = null;
}

if (payload.action === "transferred") {
data.isTransferred = true;
}

// The `edited` action fires specifically for body or title changes.
// Use the webhook's updated_at as the precise edit timestamp — for
// other actions (labeled, closed, commented, etc.) don't touch
Expand Down
33 changes: 21 additions & 12 deletions packages/das/src/webhook/handlers/label.handler.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,18 +34,27 @@ export class LabelHandler {
source === "pr" ? payload.pull_request.number : payload.issue.number;

// Append to label_events log. Actor's repo role is resolved at read time
// via contributor_repo_roles using stored PR/issue, review, and comment
// association evidence; label actors themselves don't expose it.
await this.labelEventRepo.save({
repoFullName,
targetNumber,
targetType: source,
labelName: label.name,
action,
actorGithubId: sender ? String(sender.id) : null,
actorLogin: sender?.login ?? null,
timestamp: new Date().toISOString(),
});
// via contributor_repo_roles (see pr_labels_by_actor view) using stored
// PR/issue, review, and comment association evidence — neither the webhook
// sender nor GraphQL LabeledEvent.actor expose author_association.
// orIgnore() makes the insert idempotent under the uq_label_events_natural_key
// constraint; same-delivery retries are already gated upstream by
// webhook_deliveries, this is defense-in-depth.
await this.labelEventRepo
.createQueryBuilder()
.insert()
.values({
repoFullName,
targetNumber,
targetType: source,
labelName: label.name,
action,
actorGithubId: sender ? String(sender.id) : null,
actorLogin: sender?.login ?? null,
timestamp: new Date().toISOString(),
})
.orIgnore()
.execute();

// Update current labels snapshot on the parent row
const currentLabels: string[] =
Expand Down
Loading
Loading