From f7cb3759758ea4bd866c69b7f04e5b63f0badb2b Mon Sep 17 00:00:00 2001 From: ian nuttall <6681919+iannuttall@users.noreply.github.com> Date: Fri, 24 Jul 2026 16:30:12 +0100 Subject: [PATCH] feat(ahrefs): add provider adapters Add secure API v3 credentials, bounded provider clients, cache and usage controls, Domain Rating, keyword and domain research, and link evidence. Register the shared report surface, document local setup and evidence limits, and cover the adapter with live acceptance, synthetic regressions, resource tests, and package checks. --- PRIVACY.md | 76 +-- README.md | 55 +- apps/web/public/sitemap.xml | 1 + .../src/components/reports/ReportGuide.astro | 2 +- apps/web/src/content/docs-nav.ts | 6 + apps/web/src/content/docs/docs/ahrefs.mdx | 162 +++++ .../content/docs/docs/research-providers.mdx | 17 +- apps/web/src/content/reports/collections.ts | 1 + .../src/content/reports/domain-research.ts | 47 ++ .../guide-overrides-domain-research.ts | 40 ++ .../content/reports/guide-overrides-i-p.ts | 15 +- apps/web/src/content/reports/manifest.mjs | 1 + apps/web/src/content/reports/monitoring.ts | 20 +- apps/web/src/content/reports/page-copy.ts | 8 +- apps/web/src/content/reports/section-copy.ts | 2 + apps/web/src/content/reports/sources.ts | 10 + apps/web/src/content/reports/types.ts | 2 + evals/domain-rating.json | 38 ++ packages/cli/src/commands/cache.test.ts | 25 + packages/cli/src/commands/cache.ts | 4 +- packages/cli/src/commands/links.test.ts | 10 +- packages/cli/src/commands/links.ts | 60 +- .../cli/src/commands/providers/ahrefs.test.ts | 191 ++++++ packages/cli/src/commands/providers/ahrefs.ts | 284 +++++++++ packages/cli/src/commands/providers/index.ts | 2 + packages/core/src/analyze/domain-rating.ts | 152 +++++ .../src/analyze/domain-research/shared.ts | 12 +- packages/core/src/analyze/keyword-metrics.ts | 12 +- packages/core/src/analyze/keyword-research.ts | 12 +- packages/core/src/index.ts | 12 + packages/core/src/links/ahrefs.ts | 20 + packages/core/src/links/dataforseo.ts | 147 +---- packages/core/src/links/external-provider.ts | 160 +++++ packages/core/src/links/index.ts | 2 + packages/core/src/links/types.ts | 1 + .../core/src/providers/ahrefs/adapter.test.ts | 565 +++++++++++++++++ .../core/src/providers/ahrefs/backlinks.ts | 206 +++++++ .../core/src/providers/ahrefs/cache.test.ts | 48 ++ .../core/src/providers/ahrefs/client.test.ts | 385 ++++++++++++ packages/core/src/providers/ahrefs/client.ts | 579 ++++++++++++++++++ .../src/providers/ahrefs/credentials.test.ts | 111 ++++ .../core/src/providers/ahrefs/credentials.ts | 59 ++ .../src/providers/ahrefs/domain-overview.ts | 82 +++ .../src/providers/ahrefs/domain-rating.ts | 106 ++++ .../src/providers/ahrefs/domain-research.ts | 61 ++ .../src/providers/ahrefs/keyword-discovery.ts | 353 +++++++++++ .../src/providers/ahrefs/keyword-metrics.ts | 185 ++++++ .../src/providers/ahrefs/link-research.ts | 52 ++ .../core/src/providers/ahrefs/link-summary.ts | 85 +++ packages/core/src/providers/ahrefs/mapping.ts | 115 ++++ .../src/providers/ahrefs/ranked-keywords.ts | 279 +++++++++ .../src/providers/ahrefs/ranking-pages.ts | 172 ++++++ .../src/providers/ahrefs/referring-domains.ts | 159 +++++ packages/core/src/providers/ahrefs/schema.ts | 211 +++++++ .../src/providers/ahrefs/serp-competitors.ts | 263 ++++++++ packages/core/src/providers/ahrefs/shared.ts | 484 +++++++++++++++ .../src/providers/domain-rating-contracts.ts | 30 + packages/core/src/providers/transport.test.ts | 22 + packages/core/src/providers/transport.ts | 2 + packages/core/src/storage/database.ts | 15 +- packages/core/src/storage/provider-secrets.ts | 1 + packages/core/src/telemetry.ts | 1 + packages/mcp/src/discovery-tools.test.ts | 1 + packages/mcp/src/provider-tools.ts | 37 +- packages/mcp/src/report-contracts.test.ts | 10 + .../report-definitions/domain-rating.test.ts | 64 ++ .../src/report-definitions/domain-rating.ts | 33 + packages/mcp/src/report-depth-continued.ts | 19 + .../src/report-guidance-domain-research.ts | 15 + packages/mcp/src/report-registry.ts | 10 + scripts/provider-resource-ahrefs.mjs | 147 +++++ scripts/provider-resource-harness.mjs | 2 + skills/seo/SKILL.md | 3 +- 73 files changed, 6315 insertions(+), 266 deletions(-) create mode 100644 apps/web/src/content/docs/docs/ahrefs.mdx create mode 100644 evals/domain-rating.json create mode 100644 packages/cli/src/commands/providers/ahrefs.test.ts create mode 100644 packages/cli/src/commands/providers/ahrefs.ts create mode 100644 packages/core/src/analyze/domain-rating.ts create mode 100644 packages/core/src/links/ahrefs.ts create mode 100644 packages/core/src/links/external-provider.ts create mode 100644 packages/core/src/providers/ahrefs/adapter.test.ts create mode 100644 packages/core/src/providers/ahrefs/backlinks.ts create mode 100644 packages/core/src/providers/ahrefs/cache.test.ts create mode 100644 packages/core/src/providers/ahrefs/client.test.ts create mode 100644 packages/core/src/providers/ahrefs/client.ts create mode 100644 packages/core/src/providers/ahrefs/credentials.test.ts create mode 100644 packages/core/src/providers/ahrefs/credentials.ts create mode 100644 packages/core/src/providers/ahrefs/domain-overview.ts create mode 100644 packages/core/src/providers/ahrefs/domain-rating.ts create mode 100644 packages/core/src/providers/ahrefs/domain-research.ts create mode 100644 packages/core/src/providers/ahrefs/keyword-discovery.ts create mode 100644 packages/core/src/providers/ahrefs/keyword-metrics.ts create mode 100644 packages/core/src/providers/ahrefs/link-research.ts create mode 100644 packages/core/src/providers/ahrefs/link-summary.ts create mode 100644 packages/core/src/providers/ahrefs/mapping.ts create mode 100644 packages/core/src/providers/ahrefs/ranked-keywords.ts create mode 100644 packages/core/src/providers/ahrefs/ranking-pages.ts create mode 100644 packages/core/src/providers/ahrefs/referring-domains.ts create mode 100644 packages/core/src/providers/ahrefs/schema.ts create mode 100644 packages/core/src/providers/ahrefs/serp-competitors.ts create mode 100644 packages/core/src/providers/ahrefs/shared.ts create mode 100644 packages/core/src/providers/domain-rating-contracts.ts create mode 100644 packages/mcp/src/report-definitions/domain-rating.test.ts create mode 100644 packages/mcp/src/report-definitions/domain-rating.ts create mode 100644 scripts/provider-resource-ahrefs.mjs diff --git a/PRIVACY.md b/PRIVACY.md index 76debf2a..230ff054 100644 --- a/PRIVACY.md +++ b/PRIVACY.md @@ -1,6 +1,6 @@ # Privacy policy -Last updated: 23 July 2026 +Last updated: 24 July 2026 This policy covers the official `seo` command-line tool, library, MCP server, and the seoskill.dev website. @@ -106,8 +106,8 @@ it before trying again. ## Optional research provider requests -DataForSEO requests can send the exact inputs needed for the selected -operation. Depending on the report, these can include: +Research-provider requests send only the inputs needed for the selected +operation. Depending on the provider and report, these can include: - keywords, research seeds, domains, URLs, filters, result limits, country, language, location, and device; @@ -115,22 +115,23 @@ operation. Depending on the report, these can include: - the full fixed prompt, selected model, country, web search setting, and output limit used for an AI prompt observation. -Your DataForSEO login and API password authenticate these HTTPS requests. The -project maintainer does not receive the credentials, inputs, or responses. -DataForSEO processes them under its +DataForSEO requests use your login and API password. Semrush requests use your +Version 3 API key and can include the regional database, selected columns, and +report limits. Ahrefs requests use your API v3 bearer key and can include the +target mode and country filter. Each provider also receives ordinary network +metadata. + +The project maintainer does not receive these credentials, inputs, or +responses. DataForSEO processes them under its [privacy policy](https://dataforseo.com/privacy-policy). As of the date of this policy, DataForSEO says it stores API task data for 365 days. Its policy and -retention can change independently of this project. - -The exported TypeScript library also includes a Semrush adapter. If you -configure and call it, Semrush receives the API key and the phrase, domain, -URL, database, columns, and limits needed for that request. The current CLI -uses Semrush and Ahrefs ranked-keyword exports as local files rather than live -connections. Importing a provider file does not upload it to the provider or -the project maintainer. The Semrush library adapter caches responses locally -for up to 14 days. Cache maintenance enforces a 16 MiB Semrush-cache limit and -removes rows older than 30 days. Run -`seo cache clear --provider semrush` to remove those cached responses. +retention can change independently of this project. Semrush and Ahrefs process +requests under their respective +[Semrush privacy policy](https://www.semrush.com/company/legal/privacy-policy/) +and [Ahrefs privacy policy](https://ahrefs.com/legal/privacy-policy). + +Importing a local DataForSEO, Semrush, or Ahrefs file does not upload it to the +provider or the project maintainer. ## Bing Webmaster requests @@ -172,10 +173,12 @@ That transfer is controlled by the client and model service, not by the local ## How long research data is kept locally DataForSEO responses are cached locally for up to 24 hours or seven days, -depending on the operation. Cache maintenance removes provider cache entries -older than 30 days and enforces a 32 MiB provider-cache limit. On a machine -where the command is no longer run, expired rows can remain until you clear the -cache or reset the software. +depending on the operation. Semrush responses are cached for up to seven days. +Ahrefs Domain Rating responses are cached for up to 24 hours, and other Ahrefs +responses are cached for up to seven days. Cache maintenance removes provider +cache entries older than 30 days and enforces a 32 MiB provider-cache limit. +On a machine where the command is no longer run, expired rows can remain until +you clear the cache or reset the software. Fixed AI prompt observations are saved locally so repeated runs can show compatible changes over time. History is bounded to 90 observations for one @@ -183,12 +186,14 @@ exact configuration, 10,000 observations in total, and 128 MiB of logical storage. The local provider spend ledger is retained for up to 730 days and is bounded to 50,000 rows and 32 MiB. -Run `seo cache clear --provider dataforseo` to remove cached DataForSEO -responses. Run `seo providers dataforseo disconnect` to remove saved -credentials. These commands do not delete task data already processed by -DataForSEO. Run `seo reset --yes` to remove every saved provider credential -along with local configuration, caches, histories, spend records, logs, and -saved reports. +Run `seo cache clear --provider dataforseo`, +`seo cache clear --provider semrush`, or +`seo cache clear --provider ahrefs` to remove one provider's cached responses. +Run the matching `seo providers disconnect` command to remove its +saved credential. These commands do not delete data already processed by an +external provider. Run `seo reset --yes` to remove every saved provider +credential along with local configuration, caches, histories, spend records, +logs, and saved reports. ## Anonymous tool usage @@ -253,14 +258,15 @@ copy, publish, transmit, or pass it to an agent or application yourself. ## Removing access and local data -Use `seo auth logout` to remove local Google tokens, -`seo providers dataforseo disconnect` to remove saved DataForSEO credentials, -`seo providers bing disconnect` to remove the saved Bing credential, and -`seo indexnow remove --site https://example.com` to remove a saved IndexNow key -for one site. Environment variables are controlled by your shell or runtime and -are not changed by these commands. Use `seo privacy` to inspect local paths and -`seo reset --yes` to remove every saved credential and local file managed by -the software. +Use `seo auth logout` to remove local Google tokens. Use +`seo providers dataforseo disconnect`, +`seo providers semrush disconnect`, `seo providers ahrefs disconnect`, or +`seo providers bing disconnect` to remove the corresponding saved provider +credential. Use `seo indexnow remove --site https://example.com` to remove a +saved IndexNow key for one site. Environment variables are controlled by your +shell or runtime and are not changed by these commands. Use `seo privacy` to +inspect local paths and `seo reset --yes` to remove every saved credential and +local file managed by the software. You can also revoke the app from your [Google Account connections](https://myaccount.google.com/connections). diff --git a/README.md b/README.md index 71a8c2b1..e31a90e4 100644 --- a/README.md +++ b/README.md @@ -69,9 +69,8 @@ the shared Google app. If it is unavailable in your build, setup guides you through adding your own desktop OAuth client. Research providers are optional and connected separately. Start with the main -report, then add DataForSEO when keyword estimates, live results, domain -footprints, ranking pages, competitor research, or exact local result context -would change the decision. +report, then add DataForSEO, Semrush, or Ahrefs only when external keyword, +result, domain, competitor, or link estimates would change the decision. That is the normal path. The `seo` command is then available in every terminal, script, CI job, and local MCP client on the machine. @@ -188,8 +187,9 @@ Run `seo help` for the short path or `seo help all` for the full command list. ## Research keywords and competitors -Connect DataForSEO when you need independent market estimates or competitor -evidence. The connection is local and separate from Google sign-in: +Connect a research provider when you need independent market estimates or +competitor evidence. Each connection is local and separate from Google +sign-in. DataForSEO has the broadest live coverage: ```sh seo providers dataforseo connect @@ -198,10 +198,23 @@ seo providers dataforseo limits ``` Credentials use the system keychain when available, with a private local file -fallback. Paid requests read current endpoint prices, reserve estimated spend -before acquisition, record actual cost, and use local daily, monthly, request, -and row limits. Cached results avoid repeating paid work during their retention -window. +fallback. Semrush Version 3 and Ahrefs API v3 use the same local credential +boundary: + +```sh +seo providers semrush connect +seo providers semrush status --check + +seo providers ahrefs connect +seo providers ahrefs status --check +seo providers ahrefs limits +``` + +Read the [Semrush guide](https://seoskill.dev/docs/semrush) and +[Ahrefs guide](https://seoskill.dev/docs/ahrefs) before running paid research. +Supported reports record the applicable API units or USD cost, cache state, +request bounds, and retained row coverage. Cached results avoid repeating paid +work during their retention window. Use the existing report catalog. There are no separate provider-named report commands: @@ -264,6 +277,8 @@ The research flow now covers: - `link-evidence` for a current link summary, one representative backlink per referring domain, and linked-target checks against saved crawl and Search Console evidence; +- `domain-rating` for one free, attributed Ahrefs observation of backlink + profile strength, kept separate from ranking and traffic evidence; - `ai-mention-research` for provider-indexed mentions, cited domains, and bounded question samples in one exact AI surface and market, with optional Search Console overlap for a property you own; @@ -319,10 +334,11 @@ evidence remain distinct. Bing's `inIndex` crawl statistic is provider evidence, not URL-level proof that a page is indexed. A query or page missing from a weekly top list is unknown, not zero. -Review a bounded set of referring links from DataForSEO, Bing or a local -export: +Review a bounded set of referring links from Ahrefs, DataForSEO, Bing or a +local export: ```sh +seo links --provider ahrefs --target example.com --json seo links --provider dataforseo --target example.com --json seo links --provider dataforseo --target example.com \ --search-site sc-domain:example.com --json @@ -330,11 +346,12 @@ seo links --project example --json seo links --file ./links.csv --row-limit 10000 --json ``` -The DataForSEO path makes two sequential paid requests by default: one summary -and up to 100 live representative backlinks, one per referring domain. Current -endpoint prices, estimated and actual cost, task ids, cache state, provider -filters, row coverage and omitted rows stay in the result. A cached repeat -does not repeat paid work during the retention window. +The Ahrefs and DataForSEO paths request one summary and a bounded set of live +representative backlinks, one per referring domain by default. Ahrefs keeps +estimated and actual API units in the evidence. DataForSEO keeps current +endpoint prices, estimated and actual USD cost, and task ids. Both retain cache +state, provider filters, row coverage, and omitted rows. A cached repeat does +not repeat paid work during the retention window. When a matching saved crawl or Search Console property is available, the same report checks linked target pages for observed error responses, redirects, @@ -654,9 +671,9 @@ request a site, connected Google or Bing account, research provider, Chrome UX Report, IndexNow, or the npm registry when the work needs it. Optional external enrichment can send selected Search Console query or derived -seed text to DataForSEO only when you explicitly enable it. It does not send -Google credentials, property IDs, Search Console metrics, or Google Analytics -rows. Local provider file imports are not uploaded. Read the +seed text to the chosen research provider only when you explicitly enable it. +It does not send Google credentials, property IDs, Search Console metrics, or +Google Analytics rows. Local provider file imports are not uploaded. Read the [privacy policy](https://seoskill.dev/privacy) for every network boundary and the [telemetry page](https://seoskill.dev/telemetry) for the fixed anonymous usage-event schema and opt-out controls. diff --git a/apps/web/public/sitemap.xml b/apps/web/public/sitemap.xml index c9cd7d61..dc971a90 100644 --- a/apps/web/public/sitemap.xml +++ b/apps/web/public/sitemap.xml @@ -37,6 +37,7 @@ https://seoskill.dev/docs/reports/site-crawl https://seoskill.dev/docs/reports/ctr-underperformers https://seoskill.dev/docs/reports/decaying-pages + https://seoskill.dev/docs/reports/domain-rating https://seoskill.dev/docs/reports/domain-overview https://seoskill.dev/docs/reports/setup-check https://seoskill.dev/docs/reports/entity-readiness diff --git a/apps/web/src/components/reports/ReportGuide.astro b/apps/web/src/components/reports/ReportGuide.astro index eb1e93f0..52d6e6e6 100644 --- a/apps/web/src/components/reports/ReportGuide.astro +++ b/apps/web/src/components/reports/ReportGuide.astro @@ -246,7 +246,7 @@ const resultRows = [ const lowerSentenceStart = (value: string) => /^[A-Z][a-z]/.test(value) ? `${value[0]?.toLowerCase()}${value.slice(1)}` : value const evidenceIntro = `This ${page.noun} uses ${report.evidence - .map(lowerSentenceStart) + .map((value, index) => (index === 0 ? lowerSentenceStart(value) : value)) .join(' ')}` const limitParagraphs = [ report.caveats.join(' '), diff --git a/apps/web/src/content/docs-nav.ts b/apps/web/src/content/docs-nav.ts index 0c56bd92..3ede3799 100644 --- a/apps/web/src/content/docs-nav.ts +++ b/apps/web/src/content/docs-nav.ts @@ -36,6 +36,12 @@ export const docsNav: DocsNavEntry[] = [ description: 'Connect the permanent Semrush Version 3 key and run bounded keyword, domain, ranking-page and competitor research.', }, + { + path: '/docs/ahrefs', + label: 'Ahrefs', + description: + 'Connect an Ahrefs API v3 key and run bounded keyword, domain, competitor, Domain Rating and backlink research.', + }, { path: '/docs/indexnow', label: 'IndexNow', diff --git a/apps/web/src/content/docs/docs/ahrefs.mdx b/apps/web/src/content/docs/docs/ahrefs.mdx new file mode 100644 index 00000000..416ecc2a --- /dev/null +++ b/apps/web/src/content/docs/docs/ahrefs.mdx @@ -0,0 +1,162 @@ +--- +title: Ahrefs research +description: Connect an Ahrefs API v3 key and run bounded keyword, domain, competitor, Domain Rating and backlink research. +--- + +Use Ahrefs estimates when a first-party report leaves a specific research +question open. The adapter supplies existing provider-neutral reports with +coverage, cache state and API-unit cost. It does not add an Ahrefs-specific MCP +tool or agent skill. + +## Connect an API v3 key + +Create or copy an API key in Ahrefs Account settings, then connect it through +the masked terminal prompt: + +```sh +seo providers ahrefs connect +seo providers ahrefs status --check +``` + +The connection check uses the free account endpoint. It shows the subscription, +workspace usage, reset date and key expiry without exposing the key. The key is +saved in the system keychain when available, with a private local file fallback. +It is never stored in a project profile, report, cache entry or structured +error. + +Ahrefs documents bearer-key authentication in its +limits and usage reference. + +Remove the saved key with: + +```sh +seo providers ahrefs disconnect +``` + +Agents and CI can provide the same key without saving it: + +```sh +SEO_AHREFS_API_KEY='your-api-v3-key' \ + seo providers ahrefs status --check --json +``` + +Keep the value in the platform secret manager. Do not put it in a repository, +script, report parameter, command-line flag or issue. + +## Run shared research reports + +Choose Ahrefs through the normal report input: + +```sh +seo reports run keyword-metrics \ + --params '{"keywords":["technical seo","seo audit"],"countryCode":"GB","languageCode":"en","searchEngine":"google","provider":"ahrefs"}' \ + --json + +seo reports run domain-overview \ + --params '{"domain":"example.com","countryCode":"GB","languageCode":"en","searchEngine":"google","provider":"ahrefs"}' \ + --json + +seo reports run ranked-keywords \ + --params '{"target":"example.com","limit":25,"offset":0,"countryCode":"GB","languageCode":"en","searchEngine":"google","provider":"ahrefs"}' \ + --json +``` + +The connected adapter supports: + +| Report | Evidence supplied | +| --- | --- | +| `keyword-metrics` | Search volume, cost per click, keyword difficulty and intent when Ahrefs returns them. | +| `keyword-research` | Bounded matching terms, related terms and search suggestions from one to five seeds. | +| `domain-overview` | Provider-estimated organic keyword count, traffic and traffic value for one domain. | +| `ranked-keywords` | A bounded set of observed terms, ranking URLs, positions and optional keyword metrics. | +| `ranking-pages` | Ranking pages and repeated URL patterns derived from bounded provider rows. | +| `serp-competitors` | Domains repeatedly observed for an explicit keyword set. | +| `domain-rating` | One free, attributed Domain Rating observation for a domain or URL. | +| `link-evidence` | Bounded representative backlinks, provider totals and source-page metrics. | + +Use `seo reports describe --json` before scripting a report. It +returns the current input schema, reading order, caveats and related reports. + +## Read Domain Rating as supporting evidence + +Run the free Domain Rating report before deciding whether deeper link research +is useful: + +```sh +seo reports run domain-rating \ + --params '{"target":"example.com","targetMode":"domain","provider":"ahrefs"}' \ + --json +``` + +Domain Rating is an Ahrefs 0 to 100 logarithmic estimate of backlink-profile +strength. It is not a Google metric, ranking factor, traffic estimate, keyword +difficulty score or proof that a result is easy to outrank. Missing Domain +Rating remains missing, not zero. + +The result preserves the provider license and the required attribution +**Domain Rating by Ahrefs**, linked to +Ahrefs. +See the +Domain Rating endpoint reference +for the provider contract. + +## Inspect bounded backlink evidence + +Use the existing link report for a domain or one exact page: + +```sh +seo links \ + --provider ahrefs \ + --target example.com \ + --scope domain \ + --row-limit 100 \ + --json +``` + +The report keeps provider summary counts separate from the retained link list. +The default list keeps one representative backlink per referring domain. A +bounded or partial list cannot prove that a backlink, referring domain or +target page does not exist. + +Use a current result snapshot, page relevance and URL-level link evidence +before making a competitive decision. Domain Rating alone is too broad for +that job. + +## Keep paid requests bounded + +Inspect or lower the local per-report work limits before broad research: + +```sh +seo providers ahrefs limits +seo providers ahrefs limits --requests 10 --rows 1000 +``` + +Every paid cache miss checks the live API-unit balance before acquisition. The +adapter also enforces fixed per-request and per-report unit caps, reads the +returned Ahrefs unit headers and records estimated and actual API units in the +evidence. Ahrefs explains row costs, minimum charges and response headers in +its +API consumption guide. + +Results are cached locally for seven days by default. A local cache hit costs +no API units and makes no provider request. Pass `"refresh":true` only when the +decision needs a newer observation. Inspect or remove local rows with: + +```sh +seo cache stats +seo cache clear --provider ahrefs +``` + +Ahrefs research uses Google country-level data through this adapter. It does +not provide a city, postcode, Bing or mobile market here. The requested +language remains visible as context, but it is not a separate Ahrefs API +filter. + +Run the main first-party report before paid research: + +```sh +seo report --project example +``` + +Use an Ahrefs report to answer one evidence gap, then verify an important term +against Search Console and a current result page in the same market. diff --git a/apps/web/src/content/docs/docs/research-providers.mdx b/apps/web/src/content/docs/docs/research-providers.mdx index 1fae85ff..a3c12cea 100644 --- a/apps/web/src/content/docs/docs/research-providers.mdx +++ b/apps/web/src/content/docs/docs/research-providers.mdx @@ -1,6 +1,6 @@ --- title: Research providers -description: Connect DataForSEO locally, control paid requests and combine keyword, domain, competitor, link and AI answer evidence with first-party data. +description: Connect optional research providers locally, control paid requests and combine keyword, domain, competitor, link and AI answer evidence with first-party data. --- Research a market without replacing the evidence from your own site. Optional @@ -10,13 +10,18 @@ performance. Crawls still supply current page and technical evidence. DataForSEO supports the broadest set of current research reports. Semrush Version 3 supports bounded keyword, domain, ranking-page and search-competitor -research. The report contracts are provider neutral, so both adapters return -the same evidence shape where their capabilities overlap. A report fails -clearly when a selected provider does not support its operation or market. +research. Ahrefs API v3 supports those shared research reports plus attributed +Domain Rating and bounded backlink evidence. The report contracts are provider +neutral, so the adapters return the same evidence shape where their +capabilities overlap. A report fails clearly when a selected provider does not +support its operation or market. Use the [Semrush guide](/docs/semrush) for the exact key, connection commands, supported reports and API-unit behavior. +Use the [Ahrefs guide](/docs/ahrefs) for connection, supported reports, Domain +Rating attribution, backlink evidence and API-unit limits. + ## Connect DataForSEO on this computer Create an API login and API password in DataForSEO, then run: @@ -135,8 +140,8 @@ seo reports run serp-competitors \ --json ``` -The existing `link-evidence` report also accepts a live DataForSEO target -through the direct `seo links` command. It does not add a second report or MCP +The existing `link-evidence` report accepts a live Ahrefs or DataForSEO target +through the direct `seo links` command. It does not add another report or MCP tool. The current domain research endpoints use country and language. They do not diff --git a/apps/web/src/content/reports/collections.ts b/apps/web/src/content/reports/collections.ts index 79658dc2..71e702e2 100644 --- a/apps/web/src/content/reports/collections.ts +++ b/apps/web/src/content/reports/collections.ts @@ -58,6 +58,7 @@ export const reportCollections = [ ['serp-results', 'Live search results'], ['rank-tracking', 'Exact rank tracking'], ['domain-overview', 'Domain search footprint'], + ['domain-rating', 'Ahrefs Domain Rating'], ['ranked-keywords', 'Ranked keyword footprint'], ['ranking-pages', 'Ranking pages and patterns'], ['serp-competitors', 'Search competitors'], diff --git a/apps/web/src/content/reports/domain-research.ts b/apps/web/src/content/reports/domain-research.ts index 4ebf5e70..e79d5a53 100644 --- a/apps/web/src/content/reports/domain-research.ts +++ b/apps/web/src/content/reports/domain-research.ts @@ -1,6 +1,53 @@ import type { ReportEditorial } from './types' export const domainResearchReports = [ + { + id: 'domain-rating', + name: 'Ahrefs Domain Rating', + category: 'opportunities', + summary: + 'Retrieve one attributed Ahrefs backlink-profile estimate before deciding whether deeper paid link research is useful.', + question: + 'What Domain Rating does Ahrefs currently return for this domain or URL, and what can that value support?', + useWhen: [ + 'You need one provider-native backlink-profile metric as supporting evidence.', + 'You want the free Ahrefs value before requesting paid backlink rows.', + ], + avoidWhen: [ + 'You want a Google ranking factor, traffic estimate, keyword-difficulty score, or easy-to-outrank verdict.', + 'You need current result positions, page relevance, or complete backlink evidence.', + ], + evidence: [ + 'One Ahrefs Domain Rating observation on its 0 to 100 logarithmic scale.', + 'The exact domain or URL mode, observation time, cache state, coverage and provider warnings.', + 'The provider license, required Domain Rating by Ahrefs attribution and attribution URL.', + ], + methodology: [ + 'Normalizes one domain or absolute HTTP or HTTPS URL before calling the free Ahrefs endpoint.', + 'Keeps an unavailable value missing rather than turning it into zero.', + 'Returns the provider-native metric without combining it into a ranking, traffic or opportunity score.', + ], + exampleParams: { + target: 'example.com', + targetMode: 'domain', + provider: 'ahrefs', + }, + interpretation: [ + 'Use the value as broad backlink-profile context. Compare current results, page relevance and URL-level link evidence before making a competitive decision.', + 'Read the required attribution and license beside the value when displaying or reusing it.', + ], + caveats: [ + 'Domain Rating is an Ahrefs estimate. It is not a Google metric or ranking factor.', + 'A lower value does not prove that a page or keyword is easy to outrank. Missing Domain Rating is not zero.', + ], + nextSteps: [ + 'Run link evidence for bounded referring-link rows and provider summary counts.', + 'Run SERP results for a decision-critical keyword.', + 'Run domain overview when the question concerns estimated search footprint rather than backlink strength.', + ], + related: ['link-evidence', 'serp-results', 'domain-overview'], + sources: ['ahrefs-domain-rating'], + }, { id: 'domain-overview', name: 'Domain search footprint', diff --git a/apps/web/src/content/reports/guide-overrides-domain-research.ts b/apps/web/src/content/reports/guide-overrides-domain-research.ts index 277b0fe4..9a86c087 100644 --- a/apps/web/src/content/reports/guide-overrides-domain-research.ts +++ b/apps/web/src/content/reports/guide-overrides-domain-research.ts @@ -3,6 +3,46 @@ import type { ReportGuideOverride } from './guide-types' export const domainResearchGuideOverrides: Partial< Record > = { + 'domain-rating': { + inputs: [ + { + label: 'Domain or absolute URL', + source: 'ahrefs-domain-rating', + role: 'Defines the single target sent to the free Ahrefs Domain Rating endpoint.', + }, + { + label: 'Connected Ahrefs API v3 key', + role: 'Authenticates the request without placing the key in report input, output or cache data.', + }, + ], + checks: [ + 'Normalizes the target and keeps domain and URL modes explicit.', + 'Validates the provider response while preserving a missing value, license, attribution, observation time, coverage, cache state and warnings.', + 'Returns the Ahrefs metric without converting it into a ranking factor, traffic estimate, keyword-difficulty score or feasibility verdict.', + ], + returns: [ + 'One attributed Domain Rating observation on the Ahrefs 0 to 100 logarithmic scale.', + 'The exact target, mode, provider license, attribution link, cache evidence, coverage and interpretation limits.', + ], + alternatives: [ + { + when: 'You need actual referring URLs, target pages or provider backlink totals.', + reportId: 'link-evidence', + doInstead: + 'Run link evidence with an Ahrefs target and a bounded row limit. Keep provider summary counts separate from the retained representative links.', + }, + { + when: 'You want to judge whether a result may be competitive for one keyword.', + reportId: 'serp-results', + doInstead: + 'Inspect the current result page, page relevance and URL-level evidence. Domain Rating alone cannot establish ranking feasibility.', + }, + ], + seo: { + primaryKeyword: 'Ahrefs Domain Rating checker', + supportingKeywords: ['Domain Rating API', 'backlink profile strength'], + }, + }, 'domain-overview': { inputs: [ { diff --git a/apps/web/src/content/reports/guide-overrides-i-p.ts b/apps/web/src/content/reports/guide-overrides-i-p.ts index 6b317872..8e76ab84 100644 --- a/apps/web/src/content/reports/guide-overrides-i-p.ts +++ b/apps/web/src/content/reports/guide-overrides-i-p.ts @@ -277,11 +277,11 @@ export const reportGuideOverridesIP: Partial< 'link-evidence': { name: 'Review referring link evidence', summary: - 'Normalize a bounded set of referring URLs from Bing Webmaster or a local export without downloading a web-scale index.', + 'Normalize bounded referring URLs from Ahrefs, DataForSEO, Bing Webmaster or a local export without downloading a web-scale index.', inputs: [ { - label: 'Bing Webmaster link data or a local link export', - source: 'bing-webmaster', + label: 'Live provider target or local link export', + source: 'ahrefs-link-provider', role: 'Provides the referring URL, target URL, and optional anchor text observed by the selected source.', }, { @@ -293,10 +293,11 @@ export const reportGuideOverridesIP: Partial< 'Validates HTTP URLs, normalizes common import fields, deduplicates stable row keys, and preserves invalid and duplicate counts.', 'Streams CSV and JSONL files and rejects oversized regular JSON before reading it into memory.', 'Keeps provider pagination, file bytes, row limits, output omissions, warnings, and caveats beside the retained links.', + 'Keeps provider summary counts, representative link rows, cache state and paid request cost as separate evidence.', ], returns: [ - 'A bounded list of referring URLs, target URLs, source domains, and anchor text where the source provided it.', - 'Target-page counts, source provenance, validation counts, limit status, warnings, and narrow interpretation caveats.', + 'A bounded list of referring URLs, target URLs, source domains, anchor text and provider-native page metrics where the source provided them.', + 'Provider summary totals, target-page counts, source provenance, validation counts, limit status, cache and cost evidence, warnings, and narrow interpretation caveats.', ], alternatives: [ { @@ -306,9 +307,9 @@ export const reportGuideOverridesIP: Partial< 'Run a bounded site crawl. It follows current internal links and records broken responses rather than importing external referring-link evidence.', }, { - when: 'You need a complete backlink index, authority score, or link value estimate.', + when: 'You need a complete backlink index or want one provider score turned into a ranking verdict.', doInstead: - 'Use a specialist provider and keep its coverage and scoring method explicit. This report does not invent metrics that Bing or the imported file did not supply.', + 'Use provider-native metrics only with their coverage and method kept explicit. This report cannot turn a bounded backlink sample or Domain Rating into ranking impact.', }, ], seo: { diff --git a/apps/web/src/content/reports/manifest.mjs b/apps/web/src/content/reports/manifest.mjs index ef3daef5..5ab9a852 100644 --- a/apps/web/src/content/reports/manifest.mjs +++ b/apps/web/src/content/reports/manifest.mjs @@ -18,6 +18,7 @@ export const reportIds = [ 'site-crawl', 'ctr-underperformers', 'decaying-pages', + 'domain-rating', 'domain-overview', 'setup-check', 'entity-readiness', diff --git a/apps/web/src/content/reports/monitoring.ts b/apps/web/src/content/reports/monitoring.ts index 4854d310..ced70e53 100644 --- a/apps/web/src/content/reports/monitoring.ts +++ b/apps/web/src/content/reports/monitoring.ts @@ -84,40 +84,44 @@ export const monitoringReports = [ name: 'Referring link evidence', category: 'monitoring', summary: - 'Review bounded referring URLs and target pages from Bing Webmaster or a local link export.', + 'Review bounded referring URLs and target pages from Ahrefs, DataForSEO, Bing Webmaster or a local link export.', question: 'Which retained pages link to this site, and where do they point?', useWhen: [ 'You need concrete referring URLs and anchor text where the source provides it.', - 'You have Bing Webmaster connected or a CSV, JSON, or JSONL link export.', + 'You have Ahrefs, DataForSEO or Bing Webmaster connected, or a CSV, JSON, or JSONL link export.', ], avoidWhen: [ - 'You need a complete web-scale backlink index or a third-party authority score.', + 'You need a complete web-scale backlink index or want a provider metric treated as ranking impact.', ], evidence: [ - 'Bounded Bing Webmaster link rows or normalized rows read from an explicit local export.', + 'Bounded provider link rows, provider summary counts, or normalized rows read from an explicit local export.', ], methodology: [ 'Validates HTTP URLs, normalizes common field names, deduplicates rows, and applies strict provider, file, and output limits.', 'Streams CSV and JSONL files. Regular JSON arrays have a smaller byte limit so large imports do not create an unexpected memory spike.', + 'Keeps provider summary counts separate from retained representative links and records cache and cost evidence for paid research.', ], exampleParams: { - file: './links.csv', - rowLimit: 10000, + provider: 'ahrefs', + target: 'example.com', + scope: 'domain', + rowLimit: 100, limit: 100, }, interpretation: [ - 'Read provenance and selection before the link list. Open a referring page to confirm that an important link still exists.', + 'Read provenance, provider summary and selection before the link list. Open a referring page to confirm that an important link still exists.', ], caveats: [ 'The retained rows are not a complete backlink index and do not measure link quality, authority, value, or ranking impact.', + 'Provider totals and metrics remain external estimates. A missing row in a bounded list is unknown, not zero.', ], nextSteps: [ 'Verify a selected referring URL directly.', 'Trace a target URL when it redirects or no longer serves the expected page.', ], related: ['link-recovery', 'redirect-trace', 'site-crawl'], - sources: ['bing-webmaster'], + sources: ['ahrefs-link-provider', 'bing-webmaster'], }, { id: 'index-coverage', diff --git a/apps/web/src/content/reports/page-copy.ts b/apps/web/src/content/reports/page-copy.ts index 82608ad9..8cc57661 100644 --- a/apps/web/src/content/reports/page-copy.ts +++ b/apps/web/src/content/reports/page-copy.ts @@ -119,6 +119,12 @@ export const reportPageCopy: Record = { 'Compare matched Search Console periods to find pages and queries losing clicks, impressions, position or CTR before planning a content refresh.', lead: 'Find pages or queries with a supported decline across two matched periods. The report shows what moved, then leaves the cause open for investigation.', }, + 'domain-rating': { + title: 'Ahrefs Domain Rating report', + description: + 'Retrieve one attributed Ahrefs Domain Rating observation for a domain or URL, with its target mode, license, cache state, coverage and limits kept visible.', + lead: 'Check the broad backlink-profile estimate Ahrefs returns for one domain or URL. Use it as supporting context, then inspect current results and URL-level evidence before judging a keyword or page.', + }, 'domain-overview': { title: 'Domain organic search overview', description: @@ -194,7 +200,7 @@ export const reportPageCopy: Record = { 'link-evidence': { title: 'Backlink evidence report', description: - 'Review bounded referring URLs, target pages and anchor text from Bing Webmaster or a local CSV, JSON or JSONL export, with source limits kept visible.', + 'Review bounded referring URLs, target pages and anchor text from Ahrefs, DataForSEO, Bing Webmaster or a local export, with source limits kept visible.', lead: 'See which retained pages link to the site without downloading a giant backlink index. The report keeps source coverage and limits visible, so a missing row never becomes proof that no link exists.', }, 'local-search-demand': { diff --git a/apps/web/src/content/reports/section-copy.ts b/apps/web/src/content/reports/section-copy.ts index a8d209a1..639ebb56 100644 --- a/apps/web/src/content/reports/section-copy.ts +++ b/apps/web/src/content/reports/section-copy.ts @@ -37,6 +37,8 @@ export const reportNextStepIntros: Record = { 'Inspect the live search result and query intent before rewriting a title or description. Record the change, leave the page long enough to collect comparable data and check the same query set again.', 'decaying-pages': 'Confirm that the decline survives a matched date and segment comparison before refreshing the page. Use the page opportunity report to choose a supported update, then measure the change against the same demand.', + 'domain-rating': + 'Use the observation to decide whether URL-level backlink evidence deserves a closer look. Compare current results, page relevance and first-party performance before making any claim about ranking difficulty.', 'domain-overview': 'Use the footprint to choose a narrower domain, page or keyword investigation. Keep provider estimates separate from Search Console measurements, and check current results before changing priorities.', 'setup-check': diff --git a/apps/web/src/content/reports/sources.ts b/apps/web/src/content/reports/sources.ts index e29564b3..d5d76c65 100644 --- a/apps/web/src/content/reports/sources.ts +++ b/apps/web/src/content/reports/sources.ts @@ -1,6 +1,16 @@ import type { ReportSource, ReportSourceKey } from './types' export const reportSources = { + 'ahrefs-domain-rating': { + key: 'ahrefs-domain-rating', + label: 'Ahrefs Domain Rating API reference', + url: 'https://docs.ahrefs.com/en/api/reference/public/get-domain-rating-free', + }, + 'ahrefs-link-provider': { + key: 'ahrefs-link-provider', + label: 'Ahrefs backlink API reference', + url: 'https://docs.ahrefs.com/en/api/reference/site-explorer/get-all-backlinks', + }, 'ai-features': { key: 'ai-features', label: 'Google guidance for AI features and websites', diff --git a/apps/web/src/content/reports/types.ts b/apps/web/src/content/reports/types.ts index eba8416b..b27bfedc 100644 --- a/apps/web/src/content/reports/types.ts +++ b/apps/web/src/content/reports/types.ts @@ -16,6 +16,8 @@ export type ReportSourceKey = | 'ai-features' | 'ai-mention-provider' | 'ai-prompt-provider' + | 'ahrefs-domain-rating' + | 'ahrefs-link-provider' | 'bing-webmaster' | 'canonical' | 'core-web-vitals' diff --git a/evals/domain-rating.json b/evals/domain-rating.json new file mode 100644 index 00000000..046a26e6 --- /dev/null +++ b/evals/domain-rating.json @@ -0,0 +1,38 @@ +{ + "subject": "domain-rating", + "evals": [ + { + "id": 1, + "prompt": "What is the Ahrefs Domain Rating for example.com?", + "expected_output": "The agent describes and runs the domain-rating report for the exact target, then reports the observed or unavailable value with its Ahrefs attribution, cache state, coverage, and interpretation limit.", + "assertions": [ + "Uses report id domain-rating after describing its schema", + "Keeps a missing Domain Rating distinct from zero", + "Names Domain Rating by Ahrefs and does not present the value as a Google metric or ranking factor" + ], + "files": [] + }, + { + "id": 2, + "prompt": "A competitor has DR 12, so confirm that we can easily beat every keyword they rank for.", + "expected_output": "The agent rejects the easy-to-outrank conclusion. It may use domain-rating as supporting context, then asks for or recommends current result, page relevance, URL-level link, and first-party evidence before making a competitive decision.", + "assertions": [ + "Does not claim that low Domain Rating proves a keyword or page is easy to outrank", + "Keeps the Ahrefs domain-level estimate separate from current SERP and URL-level evidence", + "Recommends a current result and page-level verification rather than promising rankings" + ], + "files": [] + }, + { + "id": 3, + "prompt": "Can we do a free backlink strength check before spending Ahrefs API units on link rows?", + "expected_output": "The agent starts with the domain-rating report, explains that its provider request is free, and uses link-evidence only if the result leaves a decision that needs bounded paid backlink rows.", + "assertions": [ + "Starts with report id domain-rating instead of broad paid link acquisition", + "Reads data status, coverage, cache, license, attribution, warnings, and caveats before summarising", + "Treats link-evidence as a separate bounded follow-up rather than part of the Domain Rating value" + ], + "files": [] + } + ] +} diff --git a/packages/cli/src/commands/cache.test.ts b/packages/cli/src/commands/cache.test.ts index a34ef16a..b00a6ea4 100644 --- a/packages/cli/src/commands/cache.test.ts +++ b/packages/cli/src/commands/cache.test.ts @@ -41,3 +41,28 @@ test('cache clear rejects an unknown provider before opening the database', asyn await rm(cacheDir, { recursive: true, force: true }) } }) + +test('cache clear accepts Ahrefs as an isolated provider cache', async () => { + const configDir = await mkdtemp(join(tmpdir(), 'seo-cache-cli-config-')) + const cacheDir = await mkdtemp(join(tmpdir(), 'seo-cache-cli-cache-')) + try { + const result = await execFileAsync( + process.execPath, + [cliPath, 'cache', 'clear', '--provider', 'ahrefs'], + { + env: { + ...process.env, + SEO_CONFIG_DIR: configDir, + SEO_CACHE_DIR: cacheDir, + CI: '1', + NO_UPDATE_NOTIFIER: '1', + }, + }, + ) + assert.equal(result.stderr, '') + assert.equal(result.stdout, 'Removed 0 cached rows.\n') + } finally { + await rm(configDir, { recursive: true, force: true }) + await rm(cacheDir, { recursive: true, force: true }) + } +}) diff --git a/packages/cli/src/commands/cache.ts b/packages/cli/src/commands/cache.ts index f418a7c9..53e70f39 100644 --- a/packages/cli/src/commands/cache.ts +++ b/packages/cli/src/commands/cache.ts @@ -7,6 +7,7 @@ const CACHE_PROVIDERS = [ 'google-analytics', 'semrush', 'dataforseo', + 'ahrefs', 'http', ] as const @@ -51,6 +52,7 @@ export const cacheCommand = defineCommand({ ], ['Semrush', String(stats.counts.semrush_cache ?? 0)], ['DataForSEO', String(stats.counts.provider_cache ?? 0)], + ['Ahrefs', String(stats.counts.ahrefs_cache ?? 0)], ['HTTP', String(stats.counts.http_cache ?? 0)], ]) }, @@ -64,7 +66,7 @@ export const cacheCommand = defineCommand({ provider: { type: 'string', description: - 'Optional cache provider: gsc, google-analytics, semrush, dataforseo, or http', + 'Optional cache provider: gsc, google-analytics, semrush, dataforseo, ahrefs, or http', }, }, run: async ({ args }) => { diff --git a/packages/cli/src/commands/links.test.ts b/packages/cli/src/commands/links.test.ts index b8807b31..189f363c 100644 --- a/packages/cli/src/commands/links.test.ts +++ b/packages/cli/src/commands/links.test.ts @@ -30,7 +30,7 @@ test('links help exposes live provider and target context controls', async () => ]) { assert.match(stdout, new RegExp(flag)) } - assert.match(stdout, /100 DataForSEO, 500 Bing, 10000 files/) + assert.match(stdout, /100 live provider, 500 Bing, 10000 files/) }) test('links command imports a bounded local file as structured JSON', async () => { @@ -87,6 +87,14 @@ test('links command rejects ambiguous and provider-mismatched sources before acq 'https://example.com/', '--json', ], + [ + 'links', + '--provider', + 'ahrefs', + '--site', + 'https://example.com/', + '--json', + ], ]) { await assert.rejects( execFileAsync(process.execPath, [cliPath, ...args], base), diff --git a/packages/cli/src/commands/links.ts b/packages/cli/src/commands/links.ts index e9be4dff..b99dda3b 100644 --- a/packages/cli/src/commands/links.ts +++ b/packages/cli/src/commands/links.ts @@ -2,6 +2,7 @@ import { randomUUID } from 'node:crypto' import { bingWebmasterSiteUrl, type CollectedLinkEvidence, + collectAhrefsLinkEvidence, collectBingLinkEvidence, collectDataForSeoLinkEvidence, importLinkEvidence, @@ -31,7 +32,7 @@ export const linksCommand = defineCommand({ meta: { name: 'links', description: - 'Review bounded referring-link evidence from DataForSEO, Bing or a file', + 'Review bounded referring-link evidence from Ahrefs, DataForSEO, Bing or a file', }, args: { project: { type: 'string', description: 'Saved project id or name.' }, @@ -39,20 +40,20 @@ export const linksCommand = defineCommand({ site: { type: 'string', description: 'Verified Bing Webmaster site URL.' }, provider: { type: 'string', - description: 'Live link source: dataforseo or bing.', + description: 'Live link source: ahrefs, dataforseo or bing.', }, target: { type: 'string', - description: 'Domain or absolute page URL for DataForSEO.', + description: 'Domain or absolute page URL for Ahrefs or DataForSEO.', }, scope: { type: 'string', - description: 'DataForSEO target scope: domain or page.', + description: 'Live provider target scope: domain or page.', }, 'include-subdomains': { type: 'boolean', default: true, - description: 'Include subdomains for a DataForSEO domain target.', + description: 'Include subdomains for a live provider domain target.', }, 'search-site': { type: 'string', @@ -70,7 +71,7 @@ export const linksCommand = defineCommand({ 'row-limit': { type: 'string', description: - 'Maximum source rows. Defaults: 100 DataForSEO, 500 Bing, 10000 files.', + 'Maximum source rows. Defaults: 100 live provider, 500 Bing, 10000 files.', }, 'target-limit': { type: 'string', @@ -105,10 +106,10 @@ export const linksCommand = defineCommand({ const file = stringArg(args.file) const format = stringArg(args.format) const provider = stringArg(args.provider) - if (provider && !['dataforseo', 'bing'].includes(provider)) { + if (provider && !['ahrefs', 'dataforseo', 'bing'].includes(provider)) { throw new SeoError( 'INVALID_INPUT', - '--provider must be dataforseo or bing.', + '--provider must be ahrefs, dataforseo or bing.', ) } if (file && provider) { @@ -133,25 +134,25 @@ export const linksCommand = defineCommand({ if (file && (site || target)) { throw new SeoError( 'INVALID_INPUT', - 'Pass one link source: --file, --site for Bing, or --target for DataForSEO.', + 'Pass one link source: --file, --site for Bing, or --target for Ahrefs or DataForSEO.', ) } if (site && target) { throw new SeoError( 'INVALID_INPUT', - 'Pass --site for Bing or --target for DataForSEO, not both.', + 'Pass --site for Bing or --target for a live research provider, not both.', ) } if (target && provider === 'bing') { throw new SeoError( 'INVALID_INPUT', - 'Use --site with Bing or --target with DataForSEO.', + 'Use --site with Bing or --target with Ahrefs or DataForSEO.', ) } - if (site && provider === 'dataforseo') { + if (site && ['ahrefs', 'dataforseo'].includes(provider ?? '')) { throw new SeoError( 'INVALID_INPUT', - 'Use --target with DataForSEO or --site with Bing.', + 'Use --target with Ahrefs or DataForSEO, or --site with Bing.', ) } const savedProject = projectArg(args) @@ -168,7 +169,7 @@ export const linksCommand = defineCommand({ format: format as 'csv' | 'json' | 'jsonl' | undefined, rowLimit, }) - } else if (provider === 'dataforseo' || target) { + } else if (provider === 'ahrefs' || provider === 'dataforseo' || target) { const providerTarget = target ?? project?.startUrl ?? @@ -179,7 +180,7 @@ export const linksCommand = defineCommand({ 'Pass --target or use a saved project with a crawl URL or Search Console property.', ) } - evidence = await collectDataForSeoLinkEvidence({ + const providerInput = { target: providerTarget, scope: scope as 'domain' | 'page' | undefined, includeSubdomains: booleanArg(args['include-subdomains']), @@ -190,7 +191,11 @@ export const linksCommand = defineCommand({ reportId: 'link-evidence', reportRunId: randomUUID(), }, - }) + } + evidence = + provider === 'ahrefs' + ? await collectAhrefsLinkEvidence(providerInput) + : await collectDataForSeoLinkEvidence(providerInput) } else { const bingSite = site ?? bingWebmasterSiteUrl(project) if (!bingSite) { @@ -211,18 +216,14 @@ export const linksCommand = defineCommand({ } const searchConsoleSite = stringArg(args['search-site']) ?? project?.siteUrl const targetPageContext = - project || - searchConsoleSite || - evidence.provenance.provider === 'dataforseo' + project || searchConsoleSite || evidence.externalProvider ? await linkTargetContext({ evidence, searchConsoleSite, crawlSite: project?.siteUrl ?? searchConsoleSite ?? - (evidence.provenance.provider === 'dataforseo' - ? evidence.externalProvider?.summary.data.target - : undefined), + evidence.externalProvider?.summary.data.target, days: strictNumberArg(args.days, '--days'), refresh: booleanArg(args.refresh), }) @@ -247,6 +248,17 @@ export const linksCommand = defineCommand({ ) ? providerCosts.reduce((total, value) => total + value, 0) : null + const providerUnits = report.providerEvidence + ? [ + report.providerEvidence.summary.cost.native?.actualUnits, + report.providerEvidence.backlinks.cost.native?.actualUnits, + ] + : [] + const providerUnitCost = providerUnits.every( + (value): value is number => value !== null && value !== undefined, + ) + ? providerUnits.reduce((total, value) => total + value, 0) + : null const providerCached = report.providerEvidence ? [ report.providerEvidence.summary.cache.status, @@ -300,7 +312,9 @@ export const linksCommand = defineCommand({ label: 'Provider cost', value: providerCost === null - ? 'Unknown' + ? providerUnitCost === null + ? 'Unknown' + : `${formatCount(providerUnitCost)} API units${providerCached ? ' (cached)' : ''}` : `$${(providerCost / 1_000_000).toFixed(4)}${providerCached ? ' (cached)' : ''}`, }, ] diff --git a/packages/cli/src/commands/providers/ahrefs.test.ts b/packages/cli/src/commands/providers/ahrefs.test.ts new file mode 100644 index 00000000..4e75d3db --- /dev/null +++ b/packages/cli/src/commands/providers/ahrefs.test.ts @@ -0,0 +1,191 @@ +import assert from 'node:assert/strict' +import { execFile } from 'node:child_process' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { test } from 'node:test' +import { fileURLToPath } from 'node:url' +import { promisify } from 'node:util' + +const execFileAsync = promisify(execFile) +const cliPath = fileURLToPath(new URL('../../index.js', import.meta.url)) + +async function runSeo( + args: string[], + env: Record, +): Promise<{ exitCode: number; stdout: string; stderr: string }> { + try { + const result = await execFileAsync(process.execPath, [cliPath, ...args], { + env: { + ...process.env, + ...env, + CI: '1', + NO_UPDATE_NOTIFIER: '1', + }, + timeout: 10_000, + }) + return { exitCode: 0, stdout: result.stdout, stderr: result.stderr } + } catch (error) { + const result = error as { + code?: number + stdout?: string + stderr?: string + } + return { + exitCode: result.code ?? 1, + stdout: result.stdout ?? '', + stderr: result.stderr ?? '', + } + } +} + +test('Ahrefs status uses the environment API key without exposing it', async () => { + const configDir = await mkdtemp(join(tmpdir(), 'seo-ahrefs-cli-config-')) + const cacheDir = await mkdtemp(join(tmpdir(), 'seo-ahrefs-cli-cache-')) + try { + const result = await runSeo(['providers', 'ahrefs', 'status', '--json'], { + SEO_CONFIG_DIR: configDir, + SEO_CACHE_DIR: cacheDir, + SEO_AHREFS_API_KEY: 'environment-api-key', + }) + assert.equal(result.exitCode, 0) + assert.deepEqual(JSON.parse(result.stdout), { + connected: true, + apiVersion: 3, + credentialSource: 'environment', + liveCheck: { status: 'not-requested' }, + }) + assert.doesNotMatch(result.stdout, /environment-api-key/) + } finally { + await rm(configDir, { recursive: true, force: true }) + await rm(cacheDir, { recursive: true, force: true }) + } +}) + +test('Ahrefs connect refuses to prompt in JSON or CI mode', async () => { + const configDir = await mkdtemp(join(tmpdir(), 'seo-ahrefs-cli-config-')) + const cacheDir = await mkdtemp(join(tmpdir(), 'seo-ahrefs-cli-cache-')) + try { + const result = await runSeo(['providers', 'ahrefs', 'connect', '--json'], { + SEO_CONFIG_DIR: configDir, + SEO_CACHE_DIR: cacheDir, + SEO_AHREFS_API_KEY: '', + }) + assert.notEqual(result.exitCode, 0) + const output = JSON.parse(result.stdout) as { + error: { code: string; message: string } + } + assert.equal(output.error.code, 'AUTH_REQUIRED') + assert.match(output.error.message, /run `seo providers ahrefs connect`/i) + assert.match(output.error.message, /SEO_AHREFS_API_KEY/) + assert.equal(result.stderr, '') + } finally { + await rm(configDir, { recursive: true, force: true }) + await rm(cacheDir, { recursive: true, force: true }) + } +}) + +test('Ahrefs disconnect leaves an environment API key explicit', async () => { + const configDir = await mkdtemp(join(tmpdir(), 'seo-ahrefs-cli-config-')) + const cacheDir = await mkdtemp(join(tmpdir(), 'seo-ahrefs-cli-cache-')) + try { + const result = await runSeo( + ['providers', 'ahrefs', 'disconnect', '--json'], + { + SEO_CONFIG_DIR: configDir, + SEO_CACHE_DIR: cacheDir, + SEO_AHREFS_API_KEY: 'environment-api-key', + }, + ) + assert.equal(result.exitCode, 0) + assert.deepEqual(JSON.parse(result.stdout), { + savedCredentialRemoved: true, + environmentCredential: 'active', + note: 'The environment variable was not changed. Clear SEO_AHREFS_API_KEY to fully disconnect.', + }) + assert.doesNotMatch(result.stdout, /environment-api-key/) + } finally { + await rm(configDir, { recursive: true, force: true }) + await rm(cacheDir, { recursive: true, force: true }) + } +}) + +test('Ahrefs limits persist bounded local report work limits', async () => { + const configDir = await mkdtemp(join(tmpdir(), 'seo-ahrefs-cli-config-')) + const cacheDir = await mkdtemp(join(tmpdir(), 'seo-ahrefs-cli-cache-')) + const env = { + SEO_CONFIG_DIR: configDir, + SEO_CACHE_DIR: cacheDir, + SEO_AHREFS_API_KEY: '', + } + try { + const changed = await runSeo( + [ + 'providers', + 'ahrefs', + 'limits', + '--requests', + '7', + '--rows', + '1200', + '--json', + ], + env, + ) + assert.equal(changed.exitCode, 0) + assert.deepEqual(JSON.parse(changed.stdout), { + provider: 'ahrefs', + maxRequestsPerReport: 7, + maxRowsPerReport: 1200, + changed: true, + note: 'Paid requests also preflight the live API-unit balance and enforce fixed per-request and per-report unit caps.', + }) + + const stored = await runSeo( + ['providers', 'ahrefs', 'limits', '--json'], + env, + ) + assert.equal(stored.exitCode, 0) + assert.deepEqual(JSON.parse(stored.stdout), { + provider: 'ahrefs', + maxRequestsPerReport: 7, + maxRowsPerReport: 1200, + changed: false, + note: 'Paid requests also preflight the live API-unit balance and enforce fixed per-request and per-report unit caps.', + }) + } finally { + await rm(configDir, { recursive: true, force: true }) + await rm(cacheDir, { recursive: true, force: true }) + } +}) + +test('Ahrefs limits reject zero and fractional bounds', async () => { + const configDir = await mkdtemp(join(tmpdir(), 'seo-ahrefs-cli-config-')) + const cacheDir = await mkdtemp(join(tmpdir(), 'seo-ahrefs-cli-cache-')) + const env = { + SEO_CONFIG_DIR: configDir, + SEO_CACHE_DIR: cacheDir, + SEO_AHREFS_API_KEY: '', + } + try { + for (const args of [ + ['--requests', '0'], + ['--rows', '1.5'], + ]) { + const result = await runSeo( + ['providers', 'ahrefs', 'limits', ...args, '--json'], + env, + ) + assert.notEqual(result.exitCode, 0) + const output = JSON.parse(result.stdout) as { + error: { code: string; retryable: boolean } + } + assert.equal(output.error.code, 'INVALID_INPUT') + assert.equal(output.error.retryable, false) + assert.equal(result.stderr, '') + } + } finally { + await rm(configDir, { recursive: true, force: true }) + await rm(cacheDir, { recursive: true, force: true }) + } +}) diff --git a/packages/cli/src/commands/providers/ahrefs.ts b/packages/cli/src/commands/providers/ahrefs.ts new file mode 100644 index 00000000..6ec5a82e --- /dev/null +++ b/packages/cli/src/commands/providers/ahrefs.ts @@ -0,0 +1,284 @@ +import { intro, note, outro, password } from '@clack/prompts' +import { + AHREFS_API_KEY_ENV, + AhrefsClient, + deleteAhrefsApiKey, + getProviderSpendLimits, + readAhrefsApiKey, + SeoError, + setProviderSpendLimits, + writeAhrefsApiKey, +} from '@seo/core' +import { defineCommand } from 'citty' +import { jsonFlag, numberArg } from '../../args.js' +import { + canPrompt, + maybeExitCancelled, + printJson, + printKeyValue, +} from '../../utils.js' + +function credentialSourceLabel( + source: 'environment' | 'keychain' | 'file' | undefined, +): string { + if (source === 'keychain') return 'system keychain' + if (source === 'file') return 'private local file' + return source ?? 'missing' +} + +function unitUsage(input: { + limit: number | null + used: number | null + remaining: number | null +}): string { + if (input.limit === null) { + return input.used === null + ? 'not reported' + : `${input.used.toLocaleString('en-US')} used; no key limit` + } + return `${(input.remaining ?? 0).toLocaleString('en-US')} of ${input.limit.toLocaleString('en-US')} remaining` +} + +function boundedIntegerArg( + value: unknown, + label: string, + minimum: number, + maximum: number, +): number | undefined { + if (value === undefined) return undefined + const parsed = numberArg(value) + if ( + parsed === undefined || + !Number.isInteger(parsed) || + parsed < minimum || + parsed > maximum + ) { + throw new SeoError( + 'INVALID_INPUT', + `${label} must be an integer from ${minimum} to ${maximum}.`, + ) + } + return parsed +} + +const connectCommand = defineCommand({ + meta: { + name: 'connect', + description: 'Validate and save an Ahrefs API v3 key', + }, + args: { + json: { + type: 'boolean', + default: false, + description: 'Print machine-readable JSON.', + }, + }, + run: async ({ args }) => { + if (!canPrompt({ json: jsonFlag(args) })) { + throw new SeoError( + 'AUTH_REQUIRED', + `Run \`seo providers ahrefs connect\` in a terminal. Agents and CI can set ${AHREFS_API_KEY_ENV}.`, + ) + } + + intro('Connect Ahrefs') + note( + 'Paste an API v3 key from Ahrefs Account settings. The validation check and Domain Rating are free; other research can consume API units.', + 'API key', + ) + const apiKey = maybeExitCancelled( + await password({ + message: 'Ahrefs API v3 key', + validate: (value) => + value?.trim() ? undefined : 'API key is required', + }), + ) + const account = await new AhrefsClient({ apiKey }).limitsAndUsage() + const source = await writeAhrefsApiKey(apiKey) + + note( + `${account.subscription} subscription. API key units: ${unitUsage(account.apiKeyUnits)}.`, + 'Connection verified', + ) + outro( + `Saved in the ${credentialSourceLabel(source)}. Run seo providers ahrefs status --check to verify it again.`, + ) + }, +}) + +const statusCommand = defineCommand({ + meta: { + name: 'status', + description: 'Show the local Ahrefs connection', + }, + args: { + check: { + type: 'boolean', + default: false, + description: 'Verify the API key with the free account endpoint.', + }, + json: { + type: 'boolean', + default: false, + description: 'Print machine-readable JSON.', + }, + }, + run: async ({ args }) => { + const credential = await readAhrefsApiKey() + const shouldCheck = Boolean(args.check) + const account = + shouldCheck && credential + ? await new AhrefsClient().limitsAndUsage() + : undefined + const result = { + connected: Boolean(credential), + apiVersion: credential ? 3 : null, + credentialSource: credential?.source, + liveCheck: account + ? { + status: 'passed' as const, + subscription: account.subscription, + apiKeyExpiresAt: account.apiKeyExpiresAt, + usageResetsAt: account.usageResetsAt, + apiKeyUnits: account.apiKeyUnits, + workspaceUnits: account.workspaceUnits, + observedAt: account.observedAt, + requestCostUnits: account.requestCostUnits, + } + : { + status: (shouldCheck ? 'unavailable' : 'not-requested') as + | 'unavailable' + | 'not-requested', + }, + } + if (jsonFlag(args)) { + printJson(result) + return + } + printKeyValue([ + ['Connected', result.connected ? 'yes' : 'no'], + ['API version', result.apiVersion ? 'Version 3' : 'not connected'], + ['Credential', credentialSourceLabel(result.credentialSource)], + [ + 'Live check', + result.liveCheck.status === 'passed' + ? `passed at ${result.liveCheck.observedAt}` + : result.liveCheck.status === 'unavailable' + ? 'not available without credentials' + : 'not requested; pass --check to verify', + ], + ...(account + ? ([ + ['Subscription', account.subscription], + ['API key units', unitUsage(account.apiKeyUnits)], + ['Workspace units', unitUsage(account.workspaceUnits)], + ['Usage resets', account.usageResetsAt], + ['Key expires', account.apiKeyExpiresAt], + ] satisfies Array<[string, string]>) + : []), + ]) + }, +}) + +const disconnectCommand = defineCommand({ + meta: { + name: 'disconnect', + description: 'Remove the saved Ahrefs API v3 key', + }, + args: { + json: { + type: 'boolean', + default: false, + description: 'Print machine-readable JSON.', + }, + }, + run: async ({ args }) => { + await deleteAhrefsApiKey() + const environmentCredential = Boolean( + process.env[AHREFS_API_KEY_ENV]?.trim(), + ) + const result = { + savedCredentialRemoved: true, + environmentCredential: environmentCredential + ? ('active' as const) + : ('missing' as const), + note: environmentCredential + ? `The environment variable was not changed. Clear ${AHREFS_API_KEY_ENV} to fully disconnect.` + : 'Ahrefs is disconnected.', + } + if (jsonFlag(args)) printJson(result) + else process.stdout.write(`${result.note}\n`) + }, +}) + +const limitsCommand = defineCommand({ + meta: { + name: 'limits', + description: 'Show or change local Ahrefs report work limits', + }, + args: { + requests: { + type: 'string', + description: 'Maximum Ahrefs requests in one report run.', + }, + rows: { + type: 'string', + description: 'Maximum requested Ahrefs rows in one report run.', + }, + json: { + type: 'boolean', + default: false, + description: 'Print machine-readable JSON.', + }, + }, + run: async ({ args }) => { + const current = getProviderSpendLimits('ahrefs') + const maxRequestsPerReport = boundedIntegerArg( + args.requests, + '--requests', + 1, + 100, + ) + const maxRowsPerReport = boundedIntegerArg(args.rows, '--rows', 1, 100_000) + const changed = + maxRequestsPerReport !== undefined || maxRowsPerReport !== undefined + const limits = changed + ? setProviderSpendLimits('ahrefs', { + ...current, + maxRequestsPerReport: + maxRequestsPerReport ?? current.maxRequestsPerReport, + maxRowsPerReport: maxRowsPerReport ?? current.maxRowsPerReport, + }) + : current + const result = { + provider: 'ahrefs' as const, + maxRequestsPerReport: limits.maxRequestsPerReport, + maxRowsPerReport: limits.maxRowsPerReport, + changed, + note: 'Paid requests also preflight the live API-unit balance and enforce fixed per-request and per-report unit caps.', + } + if (jsonFlag(args)) { + printJson(result) + return + } + printKeyValue([ + ['Requests per report', String(result.maxRequestsPerReport)], + ['Rows per report', String(result.maxRowsPerReport)], + ['Changed', result.changed ? 'yes' : 'no'], + ['Note', result.note], + ]) + }, +}) + +export const ahrefsProviderCommand = defineCommand({ + meta: { + name: 'ahrefs', + description: 'Connect Ahrefs for optional search and link data', + }, + subCommands: { + connect: connectCommand, + status: statusCommand, + limits: limitsCommand, + disconnect: disconnectCommand, + }, +}) diff --git a/packages/cli/src/commands/providers/index.ts b/packages/cli/src/commands/providers/index.ts index 09e73961..17768736 100644 --- a/packages/cli/src/commands/providers/index.ts +++ b/packages/cli/src/commands/providers/index.ts @@ -1,4 +1,5 @@ import { defineCommand } from 'citty' +import { ahrefsProviderCommand } from './ahrefs.js' import { bingProviderCommand } from './bing.js' import { dataForSeoProviderCommand } from './dataforseo.js' import { semrushProviderCommand } from './semrush.js' @@ -6,6 +7,7 @@ import { semrushProviderCommand } from './semrush.js' export const providersCommand = defineCommand({ meta: { name: 'providers', description: 'Connect optional data providers' }, subCommands: { + ahrefs: ahrefsProviderCommand, bing: bingProviderCommand, dataforseo: dataForSeoProviderCommand, semrush: semrushProviderCommand, diff --git a/packages/core/src/analyze/domain-rating.ts b/packages/core/src/analyze/domain-rating.ts new file mode 100644 index 00000000..5850a88f --- /dev/null +++ b/packages/core/src/analyze/domain-rating.ts @@ -0,0 +1,152 @@ +import { randomUUID } from 'node:crypto' +import { SeoError } from '../errors.js' +import { readAhrefsApiKey } from '../providers/ahrefs/credentials.js' +import { AhrefsDomainRatingProvider } from '../providers/ahrefs/domain-rating.js' +import type { + ProviderId, + ProviderRequestContext, +} from '../providers/contracts.js' +import type { + DomainRatingProvider, + DomainRatingTargetMode, +} from '../providers/domain-rating-contracts.js' +import { ProviderError } from '../providers/errors.js' +import { + type ProviderCandidate, + resolveProvider, +} from '../providers/resolver.js' + +const RESOLUTION_MARKET = { + searchEngine: 'google' as const, + countryCode: 'US', + languageCode: 'en', +} + +export type DomainRatingReport = { + schemaVersion: 1 + generatedAt: string + dataStatus: 'complete' | 'unavailable' + summary: { + target: string + targetMode: DomainRatingTargetMode + domainRating: number | null + verdict: string + } + evidence: Awaited> + caveats: string[] + nextSteps: string[] +} + +export type DomainRatingReportDependencies = { + candidates?: readonly ProviderCandidate[] + now?: () => Date +} + +function provider( + adapter: ProviderCandidate['adapter'], +): DomainRatingProvider | null { + return 'domainRating' in adapter && typeof adapter.domainRating === 'function' + ? (adapter as DomainRatingProvider) + : null +} + +async function defaultCandidates(): Promise { + return [ + { + adapter: new AhrefsDomainRatingProvider(), + connected: Boolean(await readAhrefsApiKey()), + priority: 10, + }, + ] +} + +function reportError(error: unknown): never { + if (!(error instanceof ProviderError)) throw error + throw new SeoError( + error.code === 'configuration' + ? 'INVALID_INPUT' + : error.code === 'rate-limit' + ? 'RATE_LIMITED' + : 'PROVIDER_UNAVAILABLE', + error.message, + ) +} + +export async function domainRatingReport( + input: { + target: string + targetMode?: DomainRatingTargetMode + provider?: ProviderId + refresh?: boolean + context?: ProviderRequestContext + }, + dependencies: DomainRatingReportDependencies = {}, +): Promise { + const candidates = dependencies.candidates ?? (await defaultCandidates()) + const resolution = resolveProvider({ + capability: 'domain-rating', + market: RESOLUTION_MARKET, + candidates, + provider: input.provider, + }) + if (resolution.status === 'unavailable') { + throw new SeoError( + 'PROVIDER_UNAVAILABLE', + resolution.reason === 'provider-not-connected' + ? 'Ahrefs is not connected. Run `seo providers ahrefs connect` first.' + : 'The selected provider cannot supply Domain Rating.', + ) + } + const adapter = provider(resolution.provider) + if (!adapter) { + throw new SeoError( + 'PROVIDER_UNAVAILABLE', + 'The selected provider does not implement Domain Rating.', + ) + } + + let evidence: Awaited> + try { + evidence = await adapter.domainRating({ + target: input.target, + targetMode: input.targetMode, + refresh: input.refresh, + context: input.context ?? { + reportId: 'domain-rating', + reportRunId: randomUUID(), + }, + }) + } catch (error) { + return reportError(error) + } + const rating = + evidence.data.domainRating.state === 'observed' + ? evidence.data.domainRating.value + : null + const dataStatus = rating === null ? 'unavailable' : 'complete' + return { + schemaVersion: 1, + generatedAt: (dependencies.now ?? (() => new Date()))().toISOString(), + dataStatus, + summary: { + target: evidence.data.target, + targetMode: evidence.data.targetMode, + domainRating: rating, + verdict: + rating === null + ? `Ahrefs did not return Domain Rating for ${evidence.data.target}.` + : `${evidence.data.target} has an observed Ahrefs Domain Rating of ${rating}.`, + }, + evidence, + caveats: [ + 'Domain Rating is an Ahrefs 0-100 logarithmic estimate of backlink-profile strength. It is not a Google metric, ranking factor, traffic estimate, or keyword-difficulty score.', + 'A lower value does not by itself mean a result is easy to outrank. Compare the current result page, page relevance, URL-level link evidence, content, and your own site evidence.', + `Use of this value is subject to the provider license at ${evidence.data.licenseUrl} and requires the attribution “${evidence.data.attribution}”.`, + ], + nextSteps: [ + 'Run link-evidence for bounded referring-link rows and provider summary counts.', + 'Run serp-results for a decision-critical keyword before comparing ranking pages.', + 'Use domain-overview, ranking-pages, and ranked-keywords for separate search-footprint evidence.', + ], + } +} diff --git a/packages/core/src/analyze/domain-research/shared.ts b/packages/core/src/analyze/domain-research/shared.ts index b17868b9..84ff6f13 100644 --- a/packages/core/src/analyze/domain-research/shared.ts +++ b/packages/core/src/analyze/domain-research/shared.ts @@ -2,6 +2,8 @@ import { resolve } from 'node:path' import { SeoError } from '../../errors.js' import { querySearchAnalytics } from '../../gsc/client.js' import { finalGscDateRange } from '../../gsc/dates.js' +import { readAhrefsApiKey } from '../../providers/ahrefs/credentials.js' +import { AhrefsDomainResearchProvider } from '../../providers/ahrefs/domain-research.js' import type { ProviderAdapter, ProviderCapability, @@ -210,9 +212,10 @@ export function offset(value: number | undefined): number { } async function defaultCandidates(): Promise { - const [dataForSeo, semrush] = await Promise.all([ + const [dataForSeo, semrush, ahrefs] = await Promise.all([ readDataForSeoCredentials(), readSemrushApiKey(), + readAhrefsApiKey(), ]) return [ { @@ -225,6 +228,11 @@ async function defaultCandidates(): Promise { connected: Boolean(semrush), priority: 20, }, + { + adapter: new AhrefsDomainResearchProvider(), + connected: Boolean(ahrefs), + priority: 30, + }, ] } @@ -254,7 +262,7 @@ export async function researchProvider(input: { ? 'INVALID_INPUT' : 'PROVIDER_UNAVAILABLE', resolution.reason === 'provider-not-connected' - ? 'No connected provider can run domain research. Connect DataForSEO or Semrush under `seo providers` first.' + ? 'No connected provider can run domain research. Connect DataForSEO, Semrush or Ahrefs under `seo providers` first.' : `${providerName} cannot run this domain research report for the selected market.`, ) } diff --git a/packages/core/src/analyze/keyword-metrics.ts b/packages/core/src/analyze/keyword-metrics.ts index eaba0dbf..de4ec1df 100644 --- a/packages/core/src/analyze/keyword-metrics.ts +++ b/packages/core/src/analyze/keyword-metrics.ts @@ -1,5 +1,7 @@ import { randomUUID } from 'node:crypto' import { SeoError } from '../errors.js' +import { readAhrefsApiKey } from '../providers/ahrefs/credentials.js' +import { AhrefsKeywordMetricsProvider } from '../providers/ahrefs/keyword-metrics.js' import type { KeywordMetric, KeywordMetricsProvider, @@ -167,9 +169,10 @@ function keywordMetricsProvider( } async function defaultCandidates(): Promise { - const [dataForSeo, semrush] = await Promise.all([ + const [dataForSeo, semrush, ahrefs] = await Promise.all([ readDataForSeoCredentials(), readSemrushApiKey(), + readAhrefsApiKey(), ]) return [ { @@ -182,6 +185,11 @@ async function defaultCandidates(): Promise { connected: Boolean(semrush), priority: 20, }, + { + adapter: new AhrefsKeywordMetricsProvider(), + connected: Boolean(ahrefs), + priority: 30, + }, ] } @@ -192,7 +200,7 @@ function providerResolutionError(input: { if (input.reason === 'provider-not-connected') { return new SeoError( 'PROVIDER_UNAVAILABLE', - 'No connected provider can supply keyword metrics. Connect DataForSEO or Semrush under `seo providers` first.', + 'No connected provider can supply keyword metrics. Connect DataForSEO, Semrush or Ahrefs under `seo providers` first.', ) } if (input.provider && input.reason === 'market-not-supported') { diff --git a/packages/core/src/analyze/keyword-research.ts b/packages/core/src/analyze/keyword-research.ts index 730ca0a0..045bce66 100644 --- a/packages/core/src/analyze/keyword-research.ts +++ b/packages/core/src/analyze/keyword-research.ts @@ -1,5 +1,7 @@ import { randomUUID } from 'node:crypto' import { SeoError } from '../errors.js' +import { readAhrefsApiKey } from '../providers/ahrefs/credentials.js' +import { AhrefsKeywordDiscoveryProvider } from '../providers/ahrefs/keyword-discovery.js' import type { KeywordDiscoveryProvider, KeywordDiscoverySource, @@ -85,9 +87,10 @@ function discoveryProvider( } async function defaultCandidates(): Promise { - const [dataForSeo, semrush] = await Promise.all([ + const [dataForSeo, semrush, ahrefs] = await Promise.all([ readDataForSeoCredentials(), readSemrushApiKey(), + readAhrefsApiKey(), ]) return [ { @@ -100,6 +103,11 @@ async function defaultCandidates(): Promise { connected: Boolean(semrush), priority: 20, }, + { + adapter: new AhrefsKeywordDiscoveryProvider(), + connected: Boolean(ahrefs), + priority: 30, + }, ] } @@ -282,7 +290,7 @@ export async function keywordResearchReport( if (resolution.status === 'unavailable') { const message = resolution.reason === 'provider-not-connected' - ? 'No connected provider can discover keywords. Connect DataForSEO or Semrush under `seo providers` first.' + ? 'No connected provider can discover keywords. Connect DataForSEO, Semrush or Ahrefs under `seo providers` first.' : validated.provider ? `${validated.provider} cannot discover keywords for this market.` : 'No configured provider can discover keywords for this market.' diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 80e70961..d8c96ebc 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -9,6 +9,7 @@ export * from './analyze/content-optimization.js' export * from './analyze/crawler.js' export * from './analyze/ctr-underperformers.js' export * from './analyze/diagnose-property.js' +export * from './analyze/domain-rating.js' export * from './analyze/domain-research.js' export * from './analyze/experiments.js' export * from './analyze/internal-links.js' @@ -60,6 +61,16 @@ export * from './paths.js' export * from './phrasing.js' export * from './presentation.js' export * from './progress.js' +export * from './providers/ahrefs/backlinks.js' +export * from './providers/ahrefs/client.js' +export * from './providers/ahrefs/credentials.js' +export * from './providers/ahrefs/domain-rating.js' +export * from './providers/ahrefs/domain-research.js' +export * from './providers/ahrefs/keyword-discovery.js' +export * from './providers/ahrefs/keyword-metrics.js' +export * from './providers/ahrefs/link-research.js' +export * from './providers/ahrefs/link-summary.js' +export * from './providers/ahrefs/referring-domains.js' export * from './providers/contracts.js' export * from './providers/cost-limits.js' export * from './providers/dataforseo/ai-mentions.js' @@ -72,6 +83,7 @@ export * from './providers/dataforseo/link-research.js' export * from './providers/dataforseo/serp-snapshot.js' export * from './providers/dataforseo.js' export * from './providers/domain-contracts.js' +export * from './providers/domain-rating-contracts.js' export * from './providers/errors.js' export * from './providers/imports/research-columns.js' export * from './providers/link-contracts.js' diff --git a/packages/core/src/links/ahrefs.ts b/packages/core/src/links/ahrefs.ts new file mode 100644 index 00000000..28810069 --- /dev/null +++ b/packages/core/src/links/ahrefs.ts @@ -0,0 +1,20 @@ +import { AhrefsLinkProvider } from '../providers/ahrefs/link-research.js' +import type { ProviderRequestContext } from '../providers/contracts.js' +import type { LinkTargetScope } from '../providers/link-contracts.js' +import { collectExternalProviderLinkEvidence } from './external-provider.js' + +export async function collectAhrefsLinkEvidence(input: { + target: string + scope?: LinkTargetScope + includeSubdomains?: boolean + rowLimit?: number + refresh?: boolean + context?: ProviderRequestContext + provider?: Pick +}) { + return collectExternalProviderLinkEvidence({ + ...input, + providerId: 'ahrefs', + provider: input.provider ?? new AhrefsLinkProvider(), + }) +} diff --git a/packages/core/src/links/dataforseo.ts b/packages/core/src/links/dataforseo.ts index aaed7858..640467b8 100644 --- a/packages/core/src/links/dataforseo.ts +++ b/packages/core/src/links/dataforseo.ts @@ -1,23 +1,7 @@ -import { randomUUID } from 'node:crypto' -import { SeoError } from '../errors.js' import type { ProviderRequestContext } from '../providers/contracts.js' import { DataForSeoLinkProvider } from '../providers/dataforseo/link-research.js' import type { LinkTargetScope } from '../providers/link-contracts.js' -import type { CollectedLinkEvidence, LinkEvidenceRow } from './types.js' - -const DEFAULT_ROW_LIMIT = 100 -const MAX_ROW_LIMIT = 500 - -function rowLimit(value?: number): number { - const limit = value ?? DEFAULT_ROW_LIMIT - if (!Number.isSafeInteger(limit) || limit < 1 || limit > MAX_ROW_LIMIT) { - throw new SeoError( - 'INVALID_INPUT', - `DataForSEO link row limit must be between 1 and ${MAX_ROW_LIMIT}.`, - ) - } - return limit -} +import { collectExternalProviderLinkEvidence } from './external-provider.js' export async function collectDataForSeoLinkEvidence(input: { target: string @@ -27,129 +11,10 @@ export async function collectDataForSeoLinkEvidence(input: { refresh?: boolean context?: ProviderRequestContext provider?: Pick -}): Promise { - const limit = rowLimit(input.rowLimit) - const provider = input.provider ?? new DataForSeoLinkProvider() - const context = input.context ?? { - reportId: 'link-evidence', - reportRunId: randomUUID(), - } - const summary = await provider.linkSummary({ - target: input.target, - scope: input.scope, - includeSubdomains: input.includeSubdomains, - refresh: input.refresh, - context, - }) - const backlinks = await provider.backlinks({ - target: summary.data.target, - scope: summary.data.scope, - includeSubdomains: input.includeSubdomains, - mode: 'representative', - status: 'live', - limit, - refresh: input.refresh, - context, +}) { + return collectExternalProviderLinkEvidence({ + ...input, + providerId: 'dataforseo', + provider: input.provider ?? new DataForSeoLinkProvider(), }) - const rows: LinkEvidenceRow[] = backlinks.data.rows.map((row) => ({ - sourceUrl: row.sourceUrl, - sourceDomain: row.sourceDomain, - targetUrl: row.targetUrl, - anchorText: row.anchorText ?? undefined, - firstSeenAt: row.firstSeenAt ?? undefined, - lastSeenAt: row.lastSeenAt ?? undefined, - nofollow: row.dofollow === null ? undefined : !row.dofollow, - linkType: row.linkType ?? undefined, - attributes: row.attributes, - state: row.state, - indirect: row.indirect ?? undefined, - linksFromPage: row.linksFromPage ?? undefined, - linksFromDomain: row.linksFromDomain ?? undefined, - providerMetrics: row.metrics, - })) - const counts = new Map() - for (const row of rows) { - counts.set(row.targetUrl, (counts.get(row.targetUrl) ?? 0) + 1) - } - const targetCounts = [...counts] - .map(([targetUrl, observedLinks]) => ({ targetUrl, observedLinks })) - .sort( - (left, right) => - right.observedLinks - left.observedLinks || - (left.targetUrl < right.targetUrl - ? -1 - : left.targetUrl > right.targetUrl - ? 1 - : 0), - ) - const completeness = backlinks.coverage.completeness - const representative = backlinks.data.mode === 'representative' - const partial = completeness !== 'complete' || representative - return { - rows, - targetCounts, - provenance: { - provider: 'dataforseo', - observedAt: - backlinks.observedAt > summary.observedAt - ? backlinks.observedAt - : summary.observedAt, - cached: - summary.cache.status === 'hit' && backlinks.cache.status === 'hit', - suppliedRows: backlinks.coverage.returnedRows ?? rows.length, - validRows: rows.length, - invalidRows: backlinks.coverage.invalidRows, - duplicateRows: Math.max( - 0, - (backlinks.coverage.returnedRows ?? rows.length) - - backlinks.coverage.invalidRows - - rows.length, - ), - capped: ['capped', 'partial'].includes(completeness), - rowLimit: limit, - completeness: partial ? 'partial' : 'complete', - providerRequests: { - methods: [summary.request.endpoint, backlinks.request.endpoint], - maxConcurrentRequests: 1, - }, - providerCoverage: { - targetCountRows: { - returnedRows: targetCounts.length, - retainedRows: targetCounts.length, - invalidRows: 0, - }, - detailRows: { - returnedRows: backlinks.coverage.returnedRows ?? rows.length, - retainedRows: rows.length, - invalidRows: backlinks.coverage.invalidRows, - }, - summaryRows: { - returnedRows: summary.coverage.returnedRows ?? 0, - retainedRows: summary.coverage.retainedRows ?? 0, - invalidRows: summary.coverage.invalidRows, - }, - backlinkRows: { - returnedRows: backlinks.coverage.returnedRows ?? rows.length, - retainedRows: rows.length, - invalidRows: backlinks.coverage.invalidRows, - providerTotalRows: backlinks.coverage.providerTotalRows, - }, - }, - }, - externalProvider: { summary, backlinks }, - warnings: [ - ...summary.warnings.map((warning) => warning.message), - ...backlinks.warnings.map((warning) => warning.message), - ...(representative - ? [ - 'The link list retains one representative backlink per referring domain. Provider summary counts remain separate from retained rows.', - ] - : []), - ...(!representative && partial - ? [ - 'The link list is bounded or partial. Provider summary counts remain separate from retained rows.', - ] - : []), - ], - } } diff --git a/packages/core/src/links/external-provider.ts b/packages/core/src/links/external-provider.ts new file mode 100644 index 00000000..e823f4a8 --- /dev/null +++ b/packages/core/src/links/external-provider.ts @@ -0,0 +1,160 @@ +import { randomUUID } from 'node:crypto' +import { SeoError } from '../errors.js' +import type { + ProviderId, + ProviderRequestContext, +} from '../providers/contracts.js' +import type { + LinkTargetScope, + LiveLinkProvider, +} from '../providers/link-contracts.js' +import type { CollectedLinkEvidence, LinkEvidenceRow } from './types.js' + +const DEFAULT_ROW_LIMIT = 100 +const MAX_ROW_LIMIT = 500 + +function rowLimit(value: number | undefined, provider: ProviderId): number { + const limit = value ?? DEFAULT_ROW_LIMIT + if (!Number.isSafeInteger(limit) || limit < 1 || limit > MAX_ROW_LIMIT) { + throw new SeoError( + 'INVALID_INPUT', + `${provider} link row limit must be between 1 and ${MAX_ROW_LIMIT}.`, + ) + } + return limit +} + +export async function collectExternalProviderLinkEvidence(input: { + providerId: 'ahrefs' | 'dataforseo' + target: string + scope?: LinkTargetScope + includeSubdomains?: boolean + rowLimit?: number + refresh?: boolean + context?: ProviderRequestContext + provider: Pick +}): Promise { + const limit = rowLimit(input.rowLimit, input.providerId) + const context = input.context ?? { + reportId: 'link-evidence', + reportRunId: randomUUID(), + } + const summary = await input.provider.linkSummary({ + target: input.target, + scope: input.scope, + includeSubdomains: input.includeSubdomains, + refresh: input.refresh, + context, + }) + const backlinks = await input.provider.backlinks({ + target: summary.data.target, + scope: summary.data.scope, + includeSubdomains: input.includeSubdomains, + mode: 'representative', + status: 'live', + limit, + refresh: input.refresh, + context, + }) + const rows: LinkEvidenceRow[] = backlinks.data.rows.map((row) => ({ + sourceUrl: row.sourceUrl, + sourceDomain: row.sourceDomain, + targetUrl: row.targetUrl, + anchorText: row.anchorText ?? undefined, + firstSeenAt: row.firstSeenAt ?? undefined, + lastSeenAt: row.lastSeenAt ?? undefined, + nofollow: row.dofollow === null ? undefined : !row.dofollow, + linkType: row.linkType ?? undefined, + attributes: row.attributes, + state: row.state, + indirect: row.indirect ?? undefined, + linksFromPage: row.linksFromPage ?? undefined, + linksFromDomain: row.linksFromDomain ?? undefined, + providerMetrics: row.metrics, + })) + const counts = new Map() + for (const row of rows) { + counts.set(row.targetUrl, (counts.get(row.targetUrl) ?? 0) + 1) + } + const targetCounts = [...counts] + .map(([targetUrl, observedLinks]) => ({ targetUrl, observedLinks })) + .sort( + (left, right) => + right.observedLinks - left.observedLinks || + (left.targetUrl < right.targetUrl + ? -1 + : left.targetUrl > right.targetUrl + ? 1 + : 0), + ) + const completeness = backlinks.coverage.completeness + const representative = backlinks.data.mode === 'representative' + const partial = completeness !== 'complete' || representative + return { + rows, + targetCounts, + provenance: { + provider: input.providerId, + observedAt: + backlinks.observedAt > summary.observedAt + ? backlinks.observedAt + : summary.observedAt, + cached: + summary.cache.status === 'hit' && backlinks.cache.status === 'hit', + suppliedRows: backlinks.coverage.returnedRows ?? rows.length, + validRows: rows.length, + invalidRows: backlinks.coverage.invalidRows, + duplicateRows: Math.max( + 0, + (backlinks.coverage.returnedRows ?? rows.length) - + backlinks.coverage.invalidRows - + rows.length, + ), + capped: ['capped', 'partial'].includes(completeness), + rowLimit: limit, + completeness: partial ? 'partial' : 'complete', + providerRequests: { + methods: [summary.request.endpoint, backlinks.request.endpoint], + maxConcurrentRequests: 1, + }, + providerCoverage: { + targetCountRows: { + returnedRows: targetCounts.length, + retainedRows: targetCounts.length, + invalidRows: 0, + }, + detailRows: { + returnedRows: backlinks.coverage.returnedRows ?? rows.length, + retainedRows: rows.length, + invalidRows: backlinks.coverage.invalidRows, + }, + summaryRows: { + returnedRows: summary.coverage.returnedRows ?? 0, + retainedRows: summary.coverage.retainedRows ?? 0, + invalidRows: summary.coverage.invalidRows, + }, + backlinkRows: { + returnedRows: backlinks.coverage.returnedRows ?? rows.length, + retainedRows: rows.length, + invalidRows: backlinks.coverage.invalidRows, + providerTotalRows: backlinks.coverage.providerTotalRows, + }, + }, + }, + externalProvider: { summary, backlinks }, + warnings: [ + ...summary.warnings.map((warning) => warning.message), + ...backlinks.warnings.map((warning) => warning.message), + ...(representative + ? [ + 'The link list retains one representative backlink per referring domain. Provider summary counts remain separate from retained rows.', + ] + : []), + ...(!representative && partial + ? [ + 'The link list is bounded or partial. Provider summary counts remain separate from retained rows.', + ] + : []), + ], + } +} diff --git a/packages/core/src/links/index.ts b/packages/core/src/links/index.ts index 822c3890..f1e3d33b 100644 --- a/packages/core/src/links/index.ts +++ b/packages/core/src/links/index.ts @@ -1,6 +1,8 @@ +export * from './ahrefs.js' export * from './bing.js' export * from './context.js' export * from './dataforseo.js' +export * from './external-provider.js' export * from './import.js' export * from './normalize.js' export * from './report.js' diff --git a/packages/core/src/links/types.ts b/packages/core/src/links/types.ts index 259d26d7..7cc90e9a 100644 --- a/packages/core/src/links/types.ts +++ b/packages/core/src/links/types.ts @@ -8,6 +8,7 @@ import type { export type LinkEvidenceProvider = | 'bing-webmaster' | 'dataforseo' + | 'ahrefs' | 'csv-import' | 'json-import' | 'jsonl-import' diff --git a/packages/core/src/providers/ahrefs/adapter.test.ts b/packages/core/src/providers/ahrefs/adapter.test.ts new file mode 100644 index 00000000..319f5a65 --- /dev/null +++ b/packages/core/src/providers/ahrefs/adapter.test.ts @@ -0,0 +1,565 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { Response } from 'undici' +import { keywordMetricsReport } from '../../analyze/keyword-metrics.js' +import Database from '../../storage/sqlite.js' +import type { ProviderCandidate } from '../resolver.js' +import { AhrefsClient, type AhrefsClientOptions } from './client.js' +import { AhrefsDomainRatingProvider } from './domain-rating.js' +import { AhrefsDomainResearchProvider } from './domain-research.js' +import { AhrefsKeywordDiscoveryProvider } from './keyword-discovery.js' +import { AhrefsKeywordMetricsProvider } from './keyword-metrics.js' +import { AhrefsLinkProvider } from './link-research.js' + +const market = { + searchEngine: 'google' as const, + countryCode: 'GB', + languageCode: 'en', +} + +function limitsFixture() { + return { + limits_and_usage: { + api_key_expiration_date: '2027-07-24T00:00:00Z', + subscription: 'Lite', + units_limit_api_key: 100_000, + units_limit_workspace: 250_000, + units_usage_api_key: 1_250, + units_usage_workspace: 2_500, + usage_reset_date: '2026-08-01', + }, + } +} + +function cacheDatabase(): Database.Database { + const database = new Database(':memory:') + database.exec(` + CREATE TABLE provider_cache ( + provider TEXT NOT NULL, + credential_scope TEXT NOT NULL, + operation TEXT NOT NULL, + request_hash TEXT NOT NULL, + request_json TEXT NOT NULL, + response_json TEXT NOT NULL, + row_count INTEGER, + source_cost_micros INTEGER, + task_ids_json TEXT NOT NULL DEFAULT '[]', + fetched_at INTEGER NOT NULL, + expires_at INTEGER NOT NULL, + PRIMARY KEY(provider, credential_scope, operation, request_hash) + ) WITHOUT ROWID; + `) + return database +} + +function paidResponse( + data: unknown, + input: { + rows: number + costPerRow: number + expectedUnits?: number + actualUnits?: number + }, +) { + return new Response(JSON.stringify(data), { + headers: { + 'x-api-rows': String(input.rows), + 'x-api-units-cost-row': String(input.costPerRow), + 'x-api-units-cost-total': String( + input.expectedUnits ?? Math.max(50, input.costPerRow * input.rows), + ), + 'x-api-units-cost-total-actual': String( + input.actualUnits ?? Math.max(50, input.costPerRow * input.rows), + ), + 'x-api-cache': 'miss', + }, + }) +} + +function clientOptions( + database: Database.Database, + fetch: NonNullable, +): AhrefsClientOptions { + return { + apiKey: 'ahrefs-adapter-test-key', + baseUrl: 'https://provider.invalid/v3/', + database, + fetch, + now: () => new Date('2026-07-24T12:00:00.000Z'), + spendLimits: { + dailyNoticeMicros: 5_000_000, + dailyHardLimitMicros: null, + monthlyHardLimitMicros: null, + maxRequestsPerReport: 20, + maxRowsPerReport: 10_000, + }, + } +} + +test('Ahrefs keyword adapters preserve typed zeroes, omissions, and seed provenance', async () => { + const database = cacheDatabase() + const requests: URL[] = [] + const options = clientOptions(database, async (url) => { + const parsed = new URL(url) + if (parsed.pathname.endsWith('/limits-and-usage')) { + return new Response(JSON.stringify(limitsFixture())) + } + requests.push(parsed) + if (parsed.pathname.endsWith('/overview')) { + return paidResponse( + { + keywords: [ + { + keyword: 'zero keyword', + volume: 0, + cpc: 0, + difficulty: 0, + intents: { informational: true }, + }, + ], + }, + { rows: 1, costPerRow: 32 }, + ) + } + const seed = parsed.searchParams.get('keywords') + return paidResponse( + { + keywords: [ + { keyword: seed === 'alpha' ? 'shared idea' : 'other idea' }, + ], + }, + { rows: 1, costPerRow: 1 }, + ) + }) + const client = new AhrefsClient(options) + const metricsProvider = new AhrefsKeywordMetricsProvider({ client }) + const metrics = await metricsProvider.keywordMetrics({ + keywords: ['Missing Keyword', 'zero keyword', 'Zero Keyword'], + market, + }) + + assert.equal( + requests[0]?.searchParams.get('keywords'), + 'missing keyword,zero keyword', + ) + assert.deepEqual( + metrics.data.map((row) => row.keyword), + ['missing keyword', 'zero keyword'], + ) + assert.equal(metrics.data[0]?.monthlySearchVolume.state, 'missing') + assert.deepEqual(metrics.data[1]?.monthlySearchVolume, { + state: 'observed', + value: 0, + }) + assert.equal(metrics.coverage.completeness, 'partial') + + const report = await keywordMetricsReport( + { + keywords: ['zero keyword'], + market, + provider: 'ahrefs', + refresh: true, + }, + { + candidates: [ + { + adapter: metricsProvider, + connected: true, + priority: 1, + }, + ] satisfies ProviderCandidate[], + now: () => new Date('2026-07-24T13:00:00.000Z'), + }, + ) + assert.equal(report.evidence.provider, 'ahrefs') + assert.equal(report.summary.observedZeroVolume, 1) + + const discovery = await new AhrefsKeywordDiscoveryProvider({ + client, + }).discoverKeywords({ + seeds: ['beta', 'alpha'], + sources: ['related', 'ideas'], + market, + limit: 8, + refresh: true, + }) + const discoveryRequests = requests.filter((request) => + request.pathname.includes('/keywords-explorer/'), + ) + assert.equal(discoveryRequests.length, 6) + assert.ok( + discoveryRequests + .slice(-4) + .every((request) => request.searchParams.get('limit') === '2'), + ) + assert.deepEqual( + discovery.data.map((row) => row.keyword), + ['other idea', 'shared idea'], + ) + assert.equal(discovery.data[0]?.sources.length, 2) + assert.equal(discovery.data[1]?.sources.length, 2) + assert.equal(discovery.request.filters.providerRequests, 4) + database.close() +}) + +test('Ahrefs domain research maps only compatible native fields and filters requests', async () => { + const database = cacheDatabase() + const requests: URL[] = [] + const options = clientOptions(database, async (url) => { + const parsed = new URL(url) + if (parsed.pathname.endsWith('/limits-and-usage')) { + return new Response(JSON.stringify(limitsFixture())) + } + requests.push(parsed) + if (parsed.pathname.endsWith('/metrics')) { + return paidResponse( + { + metrics: { + org_cost: 25_000, + org_keywords: 5, + org_keywords_1_3: 2, + org_traffic: 100, + paid_cost: 0, + paid_keywords: 0, + paid_pages: 0, + paid_traffic: 0, + }, + }, + { rows: 1, costPerRow: 44 }, + ) + } + if (parsed.pathname.endsWith('/organic-keywords')) { + return paidResponse( + { + keywords: [ + { + keyword: 'zero keyword', + best_position: 1, + best_position_kind: 'organic', + best_position_url: 'https://example.com/zero', + volume: 0, + cpc: 0, + keyword_difficulty: 0, + sum_traffic: 0, + is_branded: false, + is_commercial: false, + is_informational: true, + is_local: false, + is_navigational: false, + is_transactional: false, + }, + ], + }, + { rows: 1, costPerRow: 41 }, + ) + } + if (parsed.pathname.endsWith('/top-pages')) { + return paidResponse( + { + pages: [ + { + url: 'https://example.com/page', + keywords: 3, + sum_traffic: 40, + value: 2_500, + }, + ], + }, + { rows: 1, costPerRow: 22 }, + ) + } + const keyword = parsed.searchParams.get('keyword') + return paidResponse( + { + positions: + keyword === 'first' + ? [ + { + position: 1, + type: ['organic'], + url: 'https://example.com/a', + }, + { + position: 2, + type: ['organic'], + url: 'https://other.com/a', + }, + ] + : [ + { + position: 3, + type: ['organic'], + url: 'https://example.com/b', + }, + { + position: 1, + type: ['organic'], + url: 'https://third.com/b', + }, + ], + }, + { rows: 2, costPerRow: 3 }, + ) + }) + const provider = new AhrefsDomainResearchProvider({ + client: new AhrefsClient(options), + now: options.now, + }) + + const overview = await provider.domainOverview({ + domain: 'example.com', + market, + }) + assert.deepEqual(overview.data.organic.estimatedMonthlyTraffic, { + state: 'observed', + value: 100, + }) + assert.deepEqual(overview.data.organic.estimatedMonthlyTrafficCostUsd, { + state: 'observed', + value: 250, + }) + assert.equal(overview.data.organic.rankings.state, 'unavailable') + assert.equal(overview.coverage.completeness, 'complete') + assert.equal(overview.coverage.providerTotalRows, 1) + + const ranked = await provider.rankedKeywords({ + target: 'example.com', + market, + includeSubdomains: false, + resultTypes: ['organic'], + minSearchVolume: 0, + maxRank: 10, + excludeTerms: ['jobs'], + limit: 10, + }) + assert.deepEqual(ranked.data.rows[0]?.monthlySearchVolume, { + state: 'observed', + value: 0, + }) + assert.deepEqual(ranked.data.rows[0]?.estimatedMonthlyTraffic, { + state: 'observed', + value: 0, + }) + const rankedRequest = requests.find((request) => + request.pathname.endsWith('/organic-keywords'), + ) + assert.equal(rankedRequest?.searchParams.get('mode'), 'domain') + assert.deepEqual( + JSON.parse(rankedRequest?.searchParams.get('where') ?? '{}'), + { + and: [ + { field: 'best_position_kind', is: ['eq', 'organic'] }, + { field: 'volume', is: ['gte', 0] }, + { field: 'best_position', is: ['lte', 10] }, + { not: { field: 'keyword', is: ['isubstring', 'jobs'] } }, + ], + }, + ) + + const pages = await provider.rankingPages({ + domain: 'example.com', + market, + limit: 10, + }) + assert.deepEqual(pages.data.rows[0]?.organic.estimatedMonthlyTraffic, { + state: 'observed', + value: 40, + }) + assert.deepEqual(pages.data.rows[0]?.organic.estimatedMonthlyTrafficCostUsd, { + state: 'observed', + value: 25, + }) + + const competitors = await provider.serpCompetitors({ + keywords: ['second', 'first'], + market, + includeSubdomains: false, + resultTypes: ['organic'], + limit: 10, + }) + assert.equal(competitors.data.rows[0]?.domain, 'example.com') + assert.equal(competitors.data.rows[0]?.matchedKeywords, 2) + assert.deepEqual(competitors.data.rows[0]?.averagePosition, { + state: 'observed', + value: 2, + }) + assert.equal(competitors.data.rows[0]?.visibility.state, 'unavailable') + database.close() +}) + +test('Ahrefs link research and Domain Rating preserve source semantics and attribution', async () => { + const database = cacheDatabase() + const requests: URL[] = [] + const options = clientOptions(database, async (url) => { + const parsed = new URL(url) + if (parsed.pathname.endsWith('/limits-and-usage')) { + return new Response(JSON.stringify(limitsFixture())) + } + requests.push(parsed) + if (parsed.pathname.endsWith('/domain-rating-free')) { + return new Response( + JSON.stringify({ + domain_rating: { + domain_rating: 42.5, + license: 'https://ahrefs.com/terms', + }, + }), + ) + } + if (parsed.pathname.endsWith('/backlinks-stats')) { + return paidResponse( + { + metrics: { + all_time: 500, + all_time_refdomains: 100, + live: 400, + live_refdomains: 80, + }, + }, + { rows: 1, costPerRow: 12 }, + ) + } + if (parsed.pathname.endsWith('/refdomains')) { + return paidResponse( + { + refdomains: [ + { + domain: 'source.example', + domain_rating: 20, + first_seen: '2025-01-01T00:00:00Z', + links_to_target: 3, + }, + ], + }, + { rows: 1, costPerRow: 4 }, + ) + } + return paidResponse( + { + backlinks: [ + { + url_from: 'https://source.example/post', + root_name_source: 'source.example', + url_to: 'https://example.com/page', + anchor: 'Example', + link_type: 'text', + is_dofollow: true, + first_seen_link: '2025-01-01T00:00:00Z', + last_seen: null, + is_lost: false, + is_redirect: false, + links_external: 12, + domain_rating_source: 20, + url_rating_source: 5, + link_group_count: 3, + }, + ], + }, + { rows: 1, costPerRow: 14 }, + ) + }) + const client = new AhrefsClient(options) + const links = new AhrefsLinkProvider({ client, now: options.now }) + const summary = await links.linkSummary({ + target: 'example.com', + includeSubdomains: true, + }) + assert.deepEqual(summary.data.backlinks, { + state: 'observed', + value: 400, + }) + assert.deepEqual(summary.data.referringDomains, { + state: 'observed', + value: 80, + }) + assert.equal(summary.data.brokenBacklinks.state, 'unavailable') + + const refdomains = await links.referringDomains({ + target: 'example.com', + limit: 10, + }) + assert.deepEqual(refdomains.data.rows[0]?.backlinks, { + state: 'observed', + value: 3, + }) + assert.equal( + refdomains.data.rows[0]?.metrics[0]?.label, + 'Ahrefs Domain Rating', + ) + + const backlinks = await links.backlinks({ + target: 'example.com', + mode: 'representative', + status: 'live', + limit: 10, + }) + assert.equal(backlinks.data.rows[0]?.linksFromPage, null) + assert.equal(backlinks.data.rows[0]?.linksFromDomain, 3) + assert.deepEqual(backlinks.data.rows[0]?.metrics[0], { + provider: 'ahrefs', + id: 'source-domain-rating', + label: 'Ahrefs source Domain Rating', + value: 20, + scale: { minimum: 0, maximum: 100 }, + }) + const backlinksRequest = requests.find((request) => + request.pathname.endsWith('/all-backlinks'), + ) + assert.equal( + backlinksRequest?.searchParams.get('aggregation'), + '1_per_domain', + ) + assert.equal(backlinksRequest?.searchParams.get('history'), 'live') + + const rating = await new AhrefsDomainRatingProvider({ + client, + }).domainRating({ target: 'example.com' }) + assert.deepEqual(rating.data.domainRating, { + state: 'observed', + value: 42.5, + }) + assert.equal(rating.data.attribution, 'Domain Rating by Ahrefs') + assert.equal(rating.data.attributionUrl, 'https://ahrefs.com/') + assert.equal(rating.data.licenseUrl, 'https://ahrefs.com/terms') + assert.equal(rating.cost.native?.actualUnits, 0) + database.close() +}) + +test('Ahrefs competitor acquisition is capped at 20 calls and 2000 rows', async () => { + const database = cacheDatabase() + let calls = 0 + let acquiredRows = 0 + const options = clientOptions(database, async (url) => { + const parsed = new URL(url) + if (parsed.pathname.endsWith('/limits-and-usage')) { + return new Response(JSON.stringify(limitsFixture())) + } + calls += 1 + const positions = Array.from({ length: 100 }, (_, index) => ({ + position: index + 1, + type: ['organic'], + url: `https://domain-${index}.example/page`, + })) + acquiredRows += positions.length + return paidResponse( + { positions }, + { rows: positions.length, costPerRow: 3 }, + ) + }) + const result = await new AhrefsDomainResearchProvider({ + client: new AhrefsClient(options), + }).serpCompetitors({ + keywords: Array.from({ length: 20 }, (_, index) => `keyword ${index}`), + market, + includeSubdomains: false, + resultTypes: ['organic'], + limit: 100, + }) + + assert.equal(calls, 20) + assert.equal(acquiredRows, 2_000) + assert.equal(result.data.rows.length, 100) + assert.equal(result.coverage.completeness, 'capped') + assert.ok(JSON.stringify(result).length < 500_000) + database.close() +}) diff --git a/packages/core/src/providers/ahrefs/backlinks.ts b/packages/core/src/providers/ahrefs/backlinks.ts new file mode 100644 index 00000000..fa595916 --- /dev/null +++ b/packages/core/src/providers/ahrefs/backlinks.ts @@ -0,0 +1,206 @@ +import type { MarketIndependentProviderEvidence } from '../contracts.js' +import type { + BacklinksRequest, + ExternalBacklink, + ExternalBacklinkPage, +} from '../link-contracts.js' +import type { AhrefsClient } from './client.js' +import { ahrefsBacklinksResponseSchema } from './schema.js' +import { + compareCodepoints, + coverage, + linkTarget, + marketIndependentEvidence, + metric, + normalizedDate, + requestContext, + rowLimit, + safeUrl, +} from './shared.js' + +const ENDPOINT = 'site-explorer/all-backlinks' +const BASE_SELECT = + 'url_from,root_name_source,url_to,anchor,link_type,is_dofollow,first_seen_link,last_seen,is_lost,is_redirect,links_external,domain_rating_source,url_rating_source' +const ORDER_BY = 'domain_rating_source:desc,url_rating_source:desc,url_from:asc' + +function metricValue(row: ExternalBacklink, id: string): number { + return row.metrics.find((item) => item.id === id)?.value ?? -1 +} + +function compareRows(left: ExternalBacklink, right: ExternalBacklink): number { + return ( + metricValue(right, 'source-domain-rating') - + metricValue(left, 'source-domain-rating') || + metricValue(right, 'source-url-rating') - + metricValue(left, 'source-url-rating') || + compareCodepoints(left.sourceDomain, right.sourceDomain) || + compareCodepoints(left.sourceUrl, right.sourceUrl) || + compareCodepoints(left.targetUrl, right.targetUrl) || + compareCodepoints(left.anchorText ?? '', right.anchorText ?? '') + ) +} + +function dedupe(rows: ExternalBacklink[]): ExternalBacklink[] { + const sorted = [...rows].sort(compareRows) + const seen = new Set() + return sorted.filter((row) => { + const key = `${row.sourceUrl}\0${row.targetUrl}\0${row.anchorText ?? ''}` + if (seen.has(key)) return false + seen.add(key) + return true + }) +} + +function providerWhere(status: NonNullable) { + return status === 'all' + ? null + : JSON.stringify({ + field: 'is_lost', + is: ['eq', status === 'lost'], + }) +} + +export async function ahrefsBacklinks( + client: Pick, + input: BacklinksRequest, +): Promise> { + rowLimit(input.limit, input.offset, 'backlinks') + const normalized = linkTarget(input.target, input.scope) + const includeSubdomains = + normalized.scope === 'domain' ? (input.includeSubdomains ?? true) : false + const targetMode = + normalized.scope === 'page' + ? 'exact' + : includeSubdomains + ? 'subdomains' + : 'domain' + const mode = input.mode ?? 'representative' + const status = input.status ?? 'live' + const aggregation = mode === 'representative' ? '1_per_domain' : 'all' + const where = providerWhere(status) + const select = + mode === 'representative' ? `${BASE_SELECT},link_group_count` : BASE_SELECT + const snapshot = await client.request({ + operation: 'backlinks', + capability: 'backlinks', + path: ENDPOINT, + query: { + aggregation, + history: status === 'live' ? 'live' : 'all_time', + limit: input.limit, + mode: targetMode, + order_by: ORDER_BY, + select, + target: normalized.target, + ...(where ? { where } : {}), + }, + schema: ahrefsBacklinksResponseSchema, + requestedRows: input.limit, + perRowUnits: mode === 'representative' ? 14 : 13, + rowCount: (response) => response.backlinks.length, + refresh: input.refresh, + context: requestContext('link-evidence', input.context), + }) + + let invalidRows = 0 + const mapped = snapshot.response.backlinks.flatMap( + (row): ExternalBacklink[] => { + const sourceUrl = safeUrl(row.url_from) + const targetUrl = safeUrl(row.url_to) + let sourceDomain: string + try { + sourceDomain = new URL(sourceUrl ?? '').hostname.toLowerCase() + } catch { + sourceDomain = '' + } + if (!sourceUrl || !targetUrl || !sourceDomain) { + invalidRows += 1 + return [] + } + return [ + { + sourceUrl, + sourceDomain, + targetUrl, + anchorText: row.anchor || null, + linkType: row.link_type || null, + dofollow: row.is_dofollow, + attributes: [ + ...(row.is_dofollow ? [] : ['nofollow']), + ...(row.is_redirect ? ['redirect'] : []), + ], + firstSeenAt: normalizedDate(row.first_seen_link), + lastSeenAt: normalizedDate(row.last_seen), + state: row.is_lost ? 'lost' : 'live', + indirect: row.is_redirect, + linksFromPage: null, + linksFromDomain: row.link_group_count ?? null, + metrics: [ + ...metric( + 'source-domain-rating', + 'Ahrefs source Domain Rating', + row.domain_rating_source, + ), + ...metric( + 'source-url-rating', + 'Ahrefs source URL Rating', + row.url_rating_source, + ), + ], + }, + ] + }, + ) + const rows = dedupe(mapped) + const duplicateRows = mapped.length - rows.length + return marketIndependentEvidence({ + capability: 'backlinks', + data: { target: normalized.target, mode, rows, totalRows: null }, + snapshot, + coverage: coverage({ + requestedRows: input.limit, + returnedRows: snapshot.returnedRows, + retainedRows: rows.length, + invalidRows, + filtered: mode === 'representative' || status !== 'all', + }), + endpoint: ENDPOINT, + limit: input.limit, + filters: { + aggregation, + apiVersion: 3, + history: status === 'live' ? 'live' : 'all_time', + includeSubdomains, + mode, + scope: normalized.scope, + selectedFields: select, + status, + targetMode, + }, + sort: [ + 'sourceDomainRating:descending', + 'sourceUrlRating:descending', + 'sourceUrl:codepoint-ascending', + ], + warnings: [ + ...(invalidRows + ? [ + { + code: 'invalid-backlink-rows', + field: 'data.rows', + message: `Ahrefs returned ${invalidRows} backlink row${invalidRows === 1 ? '' : 's'} without valid source and target URLs.`, + }, + ] + : []), + ...(duplicateRows + ? [ + { + code: 'duplicate-backlink-rows', + field: 'data.rows', + message: `${duplicateRows} duplicate backlink row${duplicateRows === 1 ? '' : 's'} were collapsed deterministically.`, + }, + ] + : []), + ], + }) +} diff --git a/packages/core/src/providers/ahrefs/cache.test.ts b/packages/core/src/providers/ahrefs/cache.test.ts new file mode 100644 index 00000000..b2239186 --- /dev/null +++ b/packages/core/src/providers/ahrefs/cache.test.ts @@ -0,0 +1,48 @@ +import assert from 'node:assert/strict' +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import test from 'node:test' +import { clearCache, getCacheStats, getDb } from '../../storage/database.js' + +const root = mkdtempSync(join(tmpdir(), 'seo-ahrefs-cache-')) +const previousCacheDir = process.env.SEO_CACHE_DIR +process.env.SEO_CACHE_DIR = root + +test.after(() => { + if (previousCacheDir === undefined) delete process.env.SEO_CACHE_DIR + else process.env.SEO_CACHE_DIR = previousCacheDir + rmSync(root, { recursive: true, force: true }) +}) + +test('Ahrefs cache stats and clearing stay separate from other providers', () => { + const database = getDb() + const insert = database.prepare(` + INSERT INTO provider_cache ( + provider, credential_scope, operation, request_hash, request_json, + response_json, row_count, source_cost_micros, task_ids_json, + fetched_at, expires_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `) + for (const provider of ['ahrefs', 'dataforseo']) { + insert.run( + provider, + `${provider}-scope`, + 'keyword-metrics', + `${provider}-hash`, + '{}', + '{}', + 1, + null, + '[]', + Date.now(), + Date.now() + 60_000, + ) + } + + assert.equal(getCacheStats().counts.ahrefs_cache, 1) + assert.equal(getCacheStats().counts.provider_cache, 1) + assert.equal(clearCache('ahrefs'), 1) + assert.equal(getCacheStats().counts.ahrefs_cache, 0) + assert.equal(getCacheStats().counts.provider_cache, 1) +}) diff --git a/packages/core/src/providers/ahrefs/client.test.ts b/packages/core/src/providers/ahrefs/client.test.ts new file mode 100644 index 00000000..8b7eeb16 --- /dev/null +++ b/packages/core/src/providers/ahrefs/client.test.ts @@ -0,0 +1,385 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' +import { Response } from 'undici' +import { z } from 'zod' +import Database from '../../storage/sqlite.js' +import { ProviderError } from '../errors.js' +import { AhrefsClient } from './client.js' + +function limitsFixture(): { + limits_and_usage: { + api_key_expiration_date: string + subscription: string + units_limit_api_key: number | null + units_limit_workspace: number | null + units_usage_api_key: number + units_usage_workspace: number | null + usage_reset_date: string + } +} { + return { + limits_and_usage: { + api_key_expiration_date: '2027-07-24T00:00:00Z', + subscription: 'Lite', + units_limit_api_key: 100_000, + units_limit_workspace: 250_000, + units_usage_api_key: 1_250, + units_usage_workspace: 2_500, + usage_reset_date: '2026-08-01', + }, + } +} + +function cacheDatabase(): Database.Database { + const database = new Database(':memory:') + database.exec(` + CREATE TABLE provider_cache ( + provider TEXT NOT NULL, + credential_scope TEXT NOT NULL, + operation TEXT NOT NULL, + request_hash TEXT NOT NULL, + request_json TEXT NOT NULL, + response_json TEXT NOT NULL, + row_count INTEGER, + source_cost_micros INTEGER, + task_ids_json TEXT NOT NULL DEFAULT '[]', + fetched_at INTEGER NOT NULL, + expires_at INTEGER NOT NULL, + PRIMARY KEY(provider, credential_scope, operation, request_hash) + ) WITHOUT ROWID; + `) + return database +} + +const spendLimits = { + dailyNoticeMicros: 5_000_000, + dailyHardLimitMicros: null, + monthlyHardLimitMicros: null, + maxRequestsPerReport: 20, + maxRowsPerReport: 10_000, +} + +test('free limits request validates the key and keeps it out of evidence', async () => { + const apiKey = 'ahrefs-test-secret' + let requestedUrl = '' + let authorization = '' + const result = await new AhrefsClient({ + apiKey, + baseUrl: 'https://provider.invalid/v3/', + now: () => new Date('2026-07-24T15:00:00.000Z'), + fetch: async (url, init) => { + requestedUrl = String(url) + authorization = String( + (init?.headers as Record | undefined)?.authorization, + ) + return new Response(JSON.stringify(limitsFixture())) + }, + }).limitsAndUsage() + + assert.equal( + requestedUrl, + 'https://provider.invalid/v3/subscription-info/limits-and-usage', + ) + assert.equal(authorization, `Bearer ${apiKey}`) + assert.deepEqual(result, { + provider: 'ahrefs', + apiVersion: 3, + subscription: 'Lite', + apiKeyExpiresAt: '2027-07-24T00:00:00Z', + usageResetsAt: '2026-08-01', + apiKeyUnits: { + limit: 100_000, + used: 1_250, + remaining: 98_750, + }, + workspaceUnits: { + limit: 250_000, + used: 2_500, + remaining: 247_500, + }, + observedAt: '2026-07-24T15:00:00.000Z', + requestCostUnits: 0, + }) + assert.doesNotMatch(JSON.stringify(result), new RegExp(apiKey)) +}) + +test('unlimited and unavailable account limits stay distinct', async () => { + const fixture = limitsFixture() + fixture.limits_and_usage.units_limit_api_key = null + fixture.limits_and_usage.units_limit_workspace = null + fixture.limits_and_usage.units_usage_workspace = null + const result = await new AhrefsClient({ + apiKey: 'api-key', + fetch: async () => new Response(JSON.stringify(fixture)), + }).limitsAndUsage() + + assert.deepEqual(result.apiKeyUnits, { + limit: null, + used: 1_250, + remaining: null, + }) + assert.deepEqual(result.workspaceUnits, { + limit: null, + used: null, + remaining: null, + }) +}) + +test('limits request reports missing and rejected keys safely', async () => { + let called = false + await assert.rejects( + new AhrefsClient({ + apiKey: ' ', + fetch: async () => { + called = true + return new Response('{}') + }, + }).limitsAndUsage(), + (error) => { + assert.ok(error instanceof ProviderError) + assert.equal(error.code, 'configuration') + assert.match(error.message, /seo providers ahrefs connect/) + return true + }, + ) + assert.equal(called, false) + + const apiKey = 'rejected-secret' + await assert.rejects( + new AhrefsClient({ + apiKey, + fetch: async () => new Response('', { status: 401 }), + }).limitsAndUsage(), + (error) => { + assert.ok(error instanceof ProviderError) + assert.equal(error.code, 'authentication') + assert.match(error.message, /API v3 key/) + assert.doesNotMatch(error.message, new RegExp(apiKey)) + return true + }, + ) +}) + +test('limits request rejects malformed provider data', async () => { + await assert.rejects( + new AhrefsClient({ + apiKey: 'api-key', + fetch: async () => + new Response( + JSON.stringify({ + limits_and_usage: { + ...limitsFixture().limits_and_usage, + units_usage_api_key: -1, + }, + }), + ), + }).limitsAndUsage(), + (error) => + error instanceof ProviderError && error.code === 'invalid-response', + ) +}) + +test('paid requests preflight units, retain headers, and use the local cache', async () => { + const database = cacheDatabase() + const apiKey = 'ahrefs-paid-secret' + let accountCalls = 0 + let researchCalls = 0 + const schema = z + .object({ + keywords: z + .array( + z + .object({ + keyword: z.string(), + volume: z.number().int().nonnegative(), + }) + .strict(), + ) + .max(100), + }) + .strict() + const client = new AhrefsClient({ + apiKey, + database, + spendLimits, + baseUrl: 'https://provider.invalid/v3/', + now: () => new Date('2026-07-24T15:00:00.000Z'), + fetch: async (url, init) => { + const parsed = new URL(url) + const headers = init?.headers as Record | undefined + assert.equal(headers?.authorization, `Bearer ${apiKey}`) + if (parsed.pathname.endsWith('/limits-and-usage')) { + accountCalls += 1 + return new Response(JSON.stringify(limitsFixture())) + } + researchCalls += 1 + assert.equal(parsed.searchParams.get('keywords'), 'ahrefs') + return new Response( + JSON.stringify({ + keywords: [{ keyword: 'ahrefs', volume: 10 }], + }), + { + headers: { + 'x-api-rows': '1', + 'x-api-units-cost-row': '11', + 'x-api-units-cost-total': '50', + 'x-api-units-cost-total-actual': '50', + 'x-api-cache': 'miss', + }, + }, + ) + }, + }) + const request = { + operation: 'keyword-metrics', + capability: 'keyword-metrics' as const, + path: 'keywords-explorer/overview', + query: { + country: 'us', + keywords: 'ahrefs', + select: 'keyword,volume', + }, + schema, + requestedRows: 1, + perRowUnits: 11, + rowCount: (response: z.infer) => response.keywords.length, + context: { + reportId: 'keyword-metrics', + reportRunId: 'run-1', + }, + } + const first = await client.request(request) + const cached = await client.request(request) + + assert.equal(accountCalls, 1) + assert.equal(researchCalls, 1) + assert.equal(first.returnedRows, 1) + assert.deepEqual(first.cost.native, { + unit: 'api-unit', + estimatedUnits: 50, + actualUnits: 50, + remainingBefore: 98_750, + }) + assert.equal(first.cache.status, 'miss') + assert.equal(first.providerCache, 'miss') + assert.equal(cached.cache.status, 'hit') + assert.equal(cached.cost.native?.actualUnits, 0) + const cache = database + .prepare( + 'SELECT credential_scope, request_json, response_json FROM provider_cache', + ) + .get() as Record + assert.doesNotMatch(JSON.stringify(cache), new RegExp(apiKey)) +}) + +test('paid requests stop before acquisition when account units are too low', async () => { + const fixture = limitsFixture() + fixture.limits_and_usage.units_limit_api_key = 1_300 + let researchCalls = 0 + const client = new AhrefsClient({ + apiKey: 'api-key', + spendLimits, + fetch: async (url) => { + if (new URL(url).pathname.endsWith('/limits-and-usage')) { + return new Response(JSON.stringify(fixture)) + } + researchCalls += 1 + return new Response('{}') + }, + }) + + await assert.rejects( + client.request({ + operation: 'keyword-metrics', + capability: 'keyword-metrics', + path: 'keywords-explorer/overview', + query: { country: 'us', keywords: 'paid keyword' }, + schema: z.object({ keywords: z.array(z.unknown()) }), + requestedRows: 100, + perRowUnits: 10, + rowCount: () => 0, + context: { reportId: 'keyword-metrics', reportRunId: 'run-low' }, + }), + (error) => error instanceof ProviderError && error.code === 'budget-limit', + ) + assert.equal(researchCalls, 0) +}) + +test('paid requests require complete unit headers', async () => { + await assert.rejects( + new AhrefsClient({ + apiKey: 'api-key', + spendLimits, + fetch: async (url) => + new URL(url).pathname.endsWith('/limits-and-usage') + ? new Response(JSON.stringify(limitsFixture())) + : new Response(JSON.stringify({ rows: [] })), + }).request({ + operation: 'rows', + capability: 'backlinks', + path: 'site-explorer/all-backlinks', + query: { target: 'ahrefs.com' }, + schema: z.object({ rows: z.array(z.unknown()) }), + requestedRows: 1, + perRowUnits: 1, + rowCount: () => 0, + context: { reportId: 'links', reportRunId: 'run-headers' }, + }), + (error) => + error instanceof ProviderError && error.code === 'invalid-response', + ) +}) + +test('research requests map provider HTTP 400 responses to invalid input', async () => { + await assert.rejects( + new AhrefsClient({ + apiKey: 'api-key', + spendLimits, + fetch: async () => new Response('', { status: 400 }), + }).request({ + operation: 'domain-rating', + capability: 'domain-rating', + path: 'public/domain-rating-free', + query: { target: 'does-not-exist.invalid' }, + schema: z.object({ domain_rating: z.unknown() }), + requestedRows: 1, + perRowUnits: 0, + rowCount: () => 1, + free: true, + context: { reportId: 'domain-rating', reportRunId: 'run-invalid' }, + }), + (error) => { + assert.ok(error instanceof ProviderError) + assert.equal(error.code, 'configuration') + assert.equal(error.status, 400) + assert.match(error.message, /target or research parameters/) + return true + }, + ) +}) + +test('request and report unit caps stop oversized work before network calls', async () => { + let calls = 0 + await assert.rejects( + new AhrefsClient({ + apiKey: 'api-key', + spendLimits, + fetch: async () => { + calls += 1 + return new Response('{}') + }, + }).request({ + operation: 'oversized', + capability: 'ranked-keywords', + path: 'site-explorer/organic-keywords', + query: { target: 'example.com' }, + schema: z.object({ rows: z.array(z.unknown()) }), + requestedRows: 1_000, + perRowUnits: 26, + rowCount: () => 0, + context: { reportId: 'ranked-keywords', reportRunId: 'run-oversized' }, + }), + (error) => error instanceof ProviderError && error.code === 'budget-limit', + ) + assert.equal(calls, 0) +}) diff --git a/packages/core/src/providers/ahrefs/client.ts b/packages/core/src/providers/ahrefs/client.ts new file mode 100644 index 00000000..8a37ce94 --- /dev/null +++ b/packages/core/src/providers/ahrefs/client.ts @@ -0,0 +1,579 @@ +import { randomUUID } from 'node:crypto' +import { fetch } from 'undici' +import type { ZodType } from 'zod' +import type Database from '../../storage/sqlite.js' +import { + providerCredentialScope, + readProviderCache, + writeProviderCache, +} from '../cache.js' +import type { + ProviderCacheEvidence, + ProviderCapability, + ProviderCostEvidence, + ProviderRequestContext, + ProviderWarning, +} from '../contracts.js' +import { + getProviderSpendLimits, + type ProviderSpendLimits, +} from '../cost-limits.js' +import { ProviderError } from '../errors.js' +import { type ProviderFetch, providerRequestJson } from '../transport.js' +import { readAhrefsApiKey } from './credentials.js' +import { ahrefsLimitsAndUsageResponseSchema } from './schema.js' + +const DEFAULT_BASE_URL = 'https://api.ahrefs.com/v3/' +const LIMITS_AND_USAGE_PATH = 'subscription-info/limits-and-usage' +const DEFAULT_TIMEOUT_MS = 10_000 +const MAX_ACCOUNT_RESPONSE_BYTES = 64 * 1_024 +const DEFAULT_MAX_RESPONSE_BYTES = 5 * 1_024 * 1_024 +const DEFAULT_RESPONSE_TTL_MS = 7 * 24 * 60 * 60 * 1_000 +const MAX_REQUEST_ROWS = 1_000 +const MAX_ESTIMATED_UNITS_PER_REQUEST = 25_000 +const MAX_ESTIMATED_UNITS_PER_REPORT = 50_000 +const QUERY_NAME = /^[a-z][a-z0-9_]*$/u + +export type AhrefsLimitsAndUsage = { + provider: 'ahrefs' + apiVersion: 3 + subscription: string + apiKeyExpiresAt: string + usageResetsAt: string + apiKeyUnits: { + limit: number | null + used: number + remaining: number | null + } + workspaceUnits: { + limit: number | null + used: number | null + remaining: number | null + } + observedAt: string + requestCostUnits: 0 +} + +export type AhrefsClientOptions = { + apiKey?: string + credentials?: () => string | undefined | Promise + fetch?: ProviderFetch + baseUrl?: string + timeoutMs?: number + maxResponseBytes?: number + now?: () => Date + database?: Database.Database + responseTtlMs?: number + spendLimits?: ProviderSpendLimits +} + +type AhrefsQueryValue = string | number | boolean + +export type AhrefsApiRequest = { + operation: string + capability: ProviderCapability + path: string + query: Record + schema: ZodType + requestedRows: number + perRowUnits: number + rowCount: (response: T) => number + free?: boolean + refresh?: boolean + ttlMs?: number + context?: ProviderRequestContext +} + +export type AhrefsApiSnapshot = { + response: T + observedAt: string + returnedRows: number + cache: ProviderCacheEvidence + cost: ProviderCostEvidence + providerCache: 'hit' | 'miss' | 'no_cache' | null + warnings: ProviderWarning[] +} + +type AhrefsResponseMetadata = { + rows: number + costPerRow: number + expectedUnits: number + actualUnits: number + cache: 'hit' | 'miss' | 'no_cache' +} + +type AhrefsReportUsage = { + requests: number + rows: number + estimatedUnits: number +} + +function remaining(limit: number | null, used: number | null): number | null { + if (limit === null || used === null) return null + return Math.max(0, limit - used) +} + +export class AhrefsClient { + private readonly apiKey?: string + private readonly credentials: AhrefsClientOptions['credentials'] + private readonly fetch: ProviderFetch + private readonly baseUrl: string + private readonly timeoutMs: number + private readonly maxResponseBytes: number + private readonly now: () => Date + private readonly database: Database.Database | undefined + private readonly responseTtlMs: number + private readonly spendLimits: ProviderSpendLimits + private readonly reportUsage = new Map() + + constructor(options: AhrefsClientOptions = {}) { + this.apiKey = options.apiKey?.trim() + this.credentials = options.credentials + this.fetch = options.fetch ?? fetch + this.baseUrl = options.baseUrl ?? DEFAULT_BASE_URL + this.timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS + this.maxResponseBytes = + options.maxResponseBytes ?? DEFAULT_MAX_RESPONSE_BYTES + this.now = options.now ?? (() => new Date()) + this.database = options.database + this.responseTtlMs = options.responseTtlMs ?? DEFAULT_RESPONSE_TTL_MS + this.spendLimits = options.spendLimits ?? getProviderSpendLimits('ahrefs') + } + + private async getApiKey(operation: string): Promise { + const apiKey = + this.apiKey ?? + (await this.credentials?.()) ?? + (await readAhrefsApiKey())?.apiKey + if (!apiKey?.trim() || apiKey.trim().length > 4_096) { + throw new ProviderError({ + provider: 'ahrefs', + operation, + code: 'configuration', + message: + 'Ahrefs is not connected. Run `seo providers ahrefs connect`, or set SEO_AHREFS_API_KEY for this process.', + }) + } + return apiKey.trim() + } + + private async limitsAndUsageForKey( + apiKey: string, + ): Promise { + const operation = 'limits-and-usage' + const response = await providerRequestJson({ + provider: 'ahrefs', + operation, + url: new URL(LIMITS_AND_USAGE_PATH, this.baseUrl), + fetch: this.fetch, + maxResponseBytes: MAX_ACCOUNT_RESPONSE_BYTES, + timeoutMs: this.timeoutMs, + retry: 'safe', + schema: ahrefsLimitsAndUsageResponseSchema, + init: { + method: 'GET', + headers: { + accept: 'application/json', + authorization: `Bearer ${apiKey}`, + }, + }, + }).catch((error) => { + if (error instanceof ProviderError && error.code === 'authentication') { + throw new ProviderError({ + provider: 'ahrefs', + operation, + code: 'authentication', + status: error.status ?? undefined, + message: + 'Ahrefs rejected the API v3 key. Create or copy a key from Ahrefs Account settings, then connect again.', + cause: error, + }) + } + throw error + }) + + const usage = response.limits_and_usage + return { + provider: 'ahrefs', + apiVersion: 3, + subscription: usage.subscription, + apiKeyExpiresAt: usage.api_key_expiration_date, + usageResetsAt: usage.usage_reset_date, + apiKeyUnits: { + limit: usage.units_limit_api_key, + used: usage.units_usage_api_key, + remaining: remaining( + usage.units_limit_api_key, + usage.units_usage_api_key, + ), + }, + workspaceUnits: { + limit: usage.units_limit_workspace, + used: usage.units_usage_workspace, + remaining: remaining( + usage.units_limit_workspace, + usage.units_usage_workspace, + ), + }, + observedAt: this.now().toISOString(), + requestCostUnits: 0, + } + } + + async limitsAndUsage(): Promise { + return this.limitsAndUsageForKey(await this.getApiKey('limits-and-usage')) + } + + private validateApiRequest(input: AhrefsApiRequest): void { + const path = input.path.trim() + if ( + !/^[a-z][a-z0-9/-]{1,127}$/u.test(path) || + path.includes('//') || + !/^[a-z][a-z0-9-]{1,63}$/u.test(input.operation) || + !Number.isSafeInteger(input.requestedRows) || + input.requestedRows < 1 || + input.requestedRows > MAX_REQUEST_ROWS || + !Number.isSafeInteger(input.perRowUnits) || + input.perRowUnits < 0 || + input.perRowUnits > 1_000 + ) { + throw new ProviderError({ + provider: 'ahrefs', + operation: input.operation || 'request', + code: 'configuration', + message: 'Ahrefs received an invalid bounded API request.', + }) + } + for (const [name, value] of Object.entries(input.query)) { + if ( + !QUERY_NAME.test(name) || + name === 'output' || + (typeof value === 'string' && + (value.length > 20_000 || value.includes('\0'))) || + (typeof value === 'number' && + (!Number.isSafeInteger(value) || value < 0)) + ) { + throw new ProviderError({ + provider: 'ahrefs', + operation: input.operation, + code: 'configuration', + message: 'Ahrefs received an invalid API query parameter.', + }) + } + } + } + + private reserveReportUsage( + input: AhrefsApiRequest, + estimatedUnits: number, + ): void { + const context = input.context ?? { + reportId: input.operation, + reportRunId: randomUUID(), + } + const current = this.reportUsage.get(context.reportRunId) ?? { + requests: 0, + rows: 0, + estimatedUnits: 0, + } + const next = { + requests: current.requests + 1, + rows: current.rows + input.requestedRows, + estimatedUnits: current.estimatedUnits + estimatedUnits, + } + if (next.requests > this.spendLimits.maxRequestsPerReport) { + throw new ProviderError({ + provider: 'ahrefs', + operation: input.operation, + code: 'budget-limit', + message: `Ahrefs is limited to ${this.spendLimits.maxRequestsPerReport} requests in one report run.`, + }) + } + if (next.rows > this.spendLimits.maxRowsPerReport) { + throw new ProviderError({ + provider: 'ahrefs', + operation: input.operation, + code: 'budget-limit', + message: `Ahrefs is limited to ${this.spendLimits.maxRowsPerReport.toLocaleString('en-US')} requested rows in one report run.`, + }) + } + if ( + estimatedUnits > MAX_ESTIMATED_UNITS_PER_REQUEST || + next.estimatedUnits > MAX_ESTIMATED_UNITS_PER_REPORT + ) { + throw new ProviderError({ + provider: 'ahrefs', + operation: input.operation, + code: 'budget-limit', + message: `Ahrefs would reserve ${estimatedUnits.toLocaleString('en-US')} API units for this request and ${next.estimatedUnits.toLocaleString('en-US')} for the report. Reduce the row limit or selected work.`, + }) + } + this.reportUsage.set(context.reportRunId, next) + } + + private requestUrl(input: AhrefsApiRequest): URL { + const url = new URL(input.path, this.baseUrl) + for (const [name, value] of Object.entries(input.query)) { + url.searchParams.set(name, String(value)) + } + return url + } + + private responseMetadata( + headers: Headers, + operation: string, + free: boolean, + ): AhrefsResponseMetadata | null { + const integer = (name: string): number | null => { + const raw = headers.get(name) + if (raw === null) return null + const value = Number(raw) + return Number.isSafeInteger(value) && value >= 0 ? value : null + } + const rows = integer('x-api-rows') + const costPerRow = integer('x-api-units-cost-row') + const expectedUnits = integer('x-api-units-cost-total') + const actualUnits = integer('x-api-units-cost-total-actual') + const cache = headers.get('x-api-cache') + if ( + rows === null || + costPerRow === null || + expectedUnits === null || + actualUnits === null || + !cache || + !['hit', 'miss', 'no_cache'].includes(cache) + ) { + if (free) return null + throw new ProviderError({ + provider: 'ahrefs', + operation, + code: 'invalid-response', + message: + 'Ahrefs returned research data without complete API-unit headers.', + }) + } + return { + rows, + costPerRow, + expectedUnits, + actualUnits, + cache: cache as AhrefsResponseMetadata['cache'], + } + } + + async request(input: AhrefsApiRequest): Promise> { + this.validateApiRequest(input) + const apiKey = await this.getApiKey(input.operation) + const credentialScope = providerCredentialScope('ahrefs', apiKey) + const cacheKey = { + provider: 'ahrefs' as const, + credentialScope, + operation: input.operation, + request: { + path: input.path, + query: input.query, + }, + } + const cached = input.refresh + ? null + : readProviderCache(cacheKey, input.schema, { + database: this.database, + now: this.now().getTime(), + }) + if (cached) { + return { + response: cached.data, + observedAt: cached.storedAt, + returnedRows: cached.rowCount ?? input.rowCount(cached.data), + cache: { + status: 'hit', + storedAt: cached.storedAt, + expiresAt: cached.expiresAt, + }, + cost: { + currency: 'USD', + estimatedMicros: 0, + actualMicros: 0, + taskIds: [], + native: { + unit: 'api-unit', + estimatedUnits: 0, + actualUnits: 0, + remainingBefore: null, + }, + }, + providerCache: null, + warnings: [], + } + } + + const preflightEstimate = input.free + ? 0 + : Math.max(50, input.perRowUnits * input.requestedRows) + this.reserveReportUsage(input, preflightEstimate) + const account = input.free + ? undefined + : await this.limitsAndUsageForKey(apiKey) + const finiteRemaining = account + ? [ + account.apiKeyUnits.remaining, + account.workspaceUnits.remaining, + ].filter((value): value is number => value !== null) + : [] + const remainingBefore = + finiteRemaining.length > 0 ? Math.min(...finiteRemaining) : null + if ( + !input.free && + remainingBefore !== null && + remainingBefore < preflightEstimate + ) { + throw new ProviderError({ + provider: 'ahrefs', + operation: input.operation, + code: 'budget-limit', + message: `Ahrefs has ${remainingBefore.toLocaleString('en-US')} API units available, but this bounded request could use up to ${preflightEstimate.toLocaleString('en-US')}.`, + }) + } + + let responseHeaders: Headers | undefined + let response: T + try { + response = await providerRequestJson({ + provider: 'ahrefs', + operation: input.operation, + url: this.requestUrl(input), + fetch: this.fetch, + maxResponseBytes: this.maxResponseBytes, + timeoutMs: this.timeoutMs, + retry: input.free ? 'safe' : 'never', + schema: input.schema, + onResponse: (providerResponse) => { + responseHeaders = providerResponse.headers + }, + init: { + method: 'GET', + headers: { + accept: 'application/json', + authorization: `Bearer ${apiKey}`, + }, + }, + }) + } catch (error) { + if (error instanceof ProviderError && error.status === 400) { + throw new ProviderError({ + provider: 'ahrefs', + operation: input.operation, + code: 'configuration', + status: error.status, + message: + 'Ahrefs rejected the requested target or research parameters.', + cause: error, + }) + } + if (error instanceof ProviderError && error.status === 403) { + throw new ProviderError({ + provider: 'ahrefs', + operation: input.operation, + code: 'configuration', + status: error.status, + message: + 'The Ahrefs key is valid, but its subscription cannot run this research endpoint.', + cause: error, + }) + } + throw error + } + + const returnedRows = input.rowCount(response) + if ( + !Number.isSafeInteger(returnedRows) || + returnedRows < 0 || + returnedRows > MAX_REQUEST_ROWS + ) { + throw new ProviderError({ + provider: 'ahrefs', + operation: input.operation, + code: 'invalid-response', + message: 'Ahrefs returned an invalid number of research rows.', + }) + } + const metadata = responseHeaders + ? this.responseMetadata( + responseHeaders, + input.operation, + Boolean(input.free), + ) + : null + if (!metadata && !input.free) { + throw new ProviderError({ + provider: 'ahrefs', + operation: input.operation, + code: 'invalid-response', + message: + 'Ahrefs returned research data without complete API-unit headers.', + }) + } + + const warnings: ProviderWarning[] = [] + if (metadata && metadata.rows !== returnedRows) { + warnings.push({ + code: 'ahrefs-header-row-count-mismatch', + field: 'coverage.returnedRows', + message: `Ahrefs reported ${metadata.rows} API rows but the validated response contained ${returnedRows}.`, + }) + } + if (metadata?.cache === 'hit') { + warnings.push({ + code: 'ahrefs-provider-cache-hit', + message: + 'Ahrefs served this request from its cache and reported the actual API-unit charge separately.', + }) + } + try { + writeProviderCache( + cacheKey, + { + data: response, + ttlMs: input.ttlMs ?? this.responseTtlMs, + rowCount: returnedRows, + sourceCostMicros: null, + taskIds: [], + }, + { database: this.database, now: this.now().getTime() }, + ) + } catch { + warnings.push({ + code: 'cache-write-failed', + message: + 'The Ahrefs result is valid, but it could not be saved to the local cache.', + }) + } + + return { + response, + observedAt: this.now().toISOString(), + returnedRows, + cache: { + status: input.refresh ? 'bypass' : 'miss', + storedAt: null, + expiresAt: null, + }, + cost: { + currency: 'USD', + estimatedMicros: null, + actualMicros: null, + taskIds: [], + native: { + unit: 'api-unit', + estimatedUnits: input.free + ? 0 + : (metadata?.expectedUnits ?? preflightEstimate), + actualUnits: input.free ? 0 : (metadata?.actualUnits ?? null), + remainingBefore, + }, + }, + providerCache: metadata?.cache ?? null, + warnings, + } + } +} diff --git a/packages/core/src/providers/ahrefs/credentials.test.ts b/packages/core/src/providers/ahrefs/credentials.test.ts new file mode 100644 index 00000000..3bb751b5 --- /dev/null +++ b/packages/core/src/providers/ahrefs/credentials.test.ts @@ -0,0 +1,111 @@ +import assert from 'node:assert/strict' +import { existsSync, mkdtempSync, rmSync, statSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { after, beforeEach, test } from 'node:test' +import { getSeoCliPaths } from '../../paths.js' +import { writeConfig } from '../../storage/config.js' +import { setKeyringForTests } from '../../storage/keyring.js' +import { configSchema } from '../../types.js' +import { + AHREFS_API_KEY_ENV, + AHREFS_API_KEY_SECRET, + deleteAhrefsApiKey, + readAhrefsApiKey, + writeAhrefsApiKey, +} from './credentials.js' + +class MemoryKeyring { + readonly values = new Map() + unavailable = false + + async getPassword(service: string, account: string): Promise { + if (this.unavailable) throw new Error('Unavailable') + return this.values.get(`${service}:${account}`) ?? null + } + + async setPassword( + service: string, + account: string, + password: string, + ): Promise { + if (this.unavailable) throw new Error('Unavailable') + this.values.set(`${service}:${account}`, password) + } + + async deletePassword(service: string, account: string): Promise { + if (this.unavailable) throw new Error('Unavailable') + return this.values.delete(`${service}:${account}`) + } +} + +const configDir = mkdtempSync(join(tmpdir(), 'seo-ahrefs-credentials-')) +const previousConfigDir = process.env.SEO_CONFIG_DIR +const previousApiKey = process.env[AHREFS_API_KEY_ENV] +const keyring = new MemoryKeyring() + +beforeEach(() => { + process.env.SEO_CONFIG_DIR = configDir + delete process.env[AHREFS_API_KEY_ENV] + rmSync(configDir, { recursive: true, force: true }) + keyring.values.clear() + keyring.unavailable = false + setKeyringForTests(keyring) +}) + +after(() => { + rmSync(configDir, { recursive: true, force: true }) + if (previousConfigDir === undefined) delete process.env.SEO_CONFIG_DIR + else process.env.SEO_CONFIG_DIR = previousConfigDir + if (previousApiKey === undefined) delete process.env[AHREFS_API_KEY_ENV] + else process.env[AHREFS_API_KEY_ENV] = previousApiKey + setKeyringForTests() +}) + +test('environment API key takes precedence without being persisted', async () => { + process.env[AHREFS_API_KEY_ENV] = ' environment-api-key ' + + assert.deepEqual(await readAhrefsApiKey(), { + apiKey: 'environment-api-key', + source: 'environment', + }) + assert.equal(existsSync(getSeoCliPaths().providerSecretsFile), false) +}) + +test('API key uses the system keychain', async () => { + writeConfig(configSchema.parse({})) + + assert.equal(await writeAhrefsApiKey('saved-api-key'), 'keychain') + assert.deepEqual(await readAhrefsApiKey(), { + apiKey: 'saved-api-key', + source: 'keychain', + }) + assert.equal( + keyring.values.get(`seo:provider:${AHREFS_API_KEY_SECRET}`), + 'saved-api-key', + ) +}) + +test('API key falls back to a private local file', async () => { + keyring.unavailable = true + writeConfig(configSchema.parse({})) + + assert.equal(await writeAhrefsApiKey('file-api-key'), 'file') + assert.equal( + statSync(getSeoCliPaths().providerSecretsFile).mode & 0o777, + 0o600, + ) + assert.deepEqual(await readAhrefsApiKey(), { + apiKey: 'file-api-key', + source: 'file', + }) +}) + +test('disconnect removes the saved API key', async () => { + writeConfig(configSchema.parse({})) + await writeAhrefsApiKey('saved-api-key') + + await deleteAhrefsApiKey() + + assert.equal(await readAhrefsApiKey(), undefined) +}) diff --git a/packages/core/src/providers/ahrefs/credentials.ts b/packages/core/src/providers/ahrefs/credentials.ts new file mode 100644 index 00000000..57e991e2 --- /dev/null +++ b/packages/core/src/providers/ahrefs/credentials.ts @@ -0,0 +1,59 @@ +import { + deleteProviderSecret, + PROVIDER_SECRET_NAMES, + type ProviderSecretSource, + readProviderSecret, + writeProviderSecret, +} from '../../storage/provider-secrets.js' +import { ProviderError } from '../errors.js' + +export const AHREFS_API_KEY_ENV = 'SEO_AHREFS_API_KEY' +export const AHREFS_API_KEY_SECRET = PROVIDER_SECRET_NAMES.ahrefsApiKey + +export type StoredAhrefsApiKey = { + apiKey: string + source: ProviderSecretSource +} + +function configurationError(message: string): ProviderError { + return new ProviderError({ + provider: 'ahrefs', + operation: 'credentials', + code: 'configuration', + message, + }) +} + +function normalizeApiKey(value: string): string { + const apiKey = value.trim() + if (!apiKey || apiKey.length > 4_096) { + throw configurationError('Ahrefs needs a valid API v3 key.') + } + return apiKey +} + +export async function writeAhrefsApiKey( + value: string, +): Promise> { + return writeProviderSecret(AHREFS_API_KEY_SECRET, normalizeApiKey(value)) +} + +export async function readAhrefsApiKey( + input: { env?: NodeJS.ProcessEnv } = {}, +): Promise { + const credential = await readProviderSecret({ + name: AHREFS_API_KEY_SECRET, + envVar: AHREFS_API_KEY_ENV, + env: input.env, + }) + return credential + ? { + apiKey: normalizeApiKey(credential.value), + source: credential.source, + } + : undefined +} + +export async function deleteAhrefsApiKey(): Promise { + await deleteProviderSecret(AHREFS_API_KEY_SECRET) +} diff --git a/packages/core/src/providers/ahrefs/domain-overview.ts b/packages/core/src/providers/ahrefs/domain-overview.ts new file mode 100644 index 00000000..ed2897f7 --- /dev/null +++ b/packages/core/src/providers/ahrefs/domain-overview.ts @@ -0,0 +1,82 @@ +import type { ProviderEvidence } from '../contracts.js' +import type { + DomainOverview, + DomainOverviewRequest, +} from '../domain-contracts.js' +import type { AhrefsClient } from './client.js' +import { ahrefsDomainMetricsResponseSchema } from './schema.js' +import { + apiDate, + domain, + evidence, + freeTestValue, + marketCountry, + organicFootprint, + requestContext, +} from './shared.js' + +const ENDPOINT = 'site-explorer/metrics' +const PER_ROW_UNITS = 44 + +export async function ahrefsDomainOverview( + client: Pick, + input: DomainOverviewRequest, + now: () => Date, +): Promise> { + const target = domain(input.domain, 'domain-overview') + const country = marketCountry(input.market, 'domain-overview') + const date = apiDate(now) + const snapshot = await client.request({ + operation: 'domain-overview', + capability: 'domain-overview', + path: ENDPOINT, + query: { + country, + date, + mode: 'subdomains', + target, + volume_mode: 'average', + }, + schema: ahrefsDomainMetricsResponseSchema, + requestedRows: 1, + perRowUnits: PER_ROW_UNITS, + rowCount: () => 1, + free: freeTestValue(target), + refresh: input.refresh, + context: requestContext('domain-overview', input.context), + }) + + return evidence({ + capability: 'domain-overview', + data: { + domain: target, + organic: organicFootprint({ + traffic: snapshot.response.metrics.org_traffic, + keywords: snapshot.response.metrics.org_keywords, + costCents: snapshot.response.metrics.org_cost, + }), + }, + market: input.market, + snapshot, + coverage: { + requestedRows: 1, + returnedRows: 1, + retainedRows: 1, + invalidRows: 0, + providerTotalRows: 1, + completeness: 'complete', + nextCursor: null, + }, + endpoint: ENDPOINT, + limit: 1, + filters: { + apiVersion: 3, + country, + date, + domain: target, + mode: 'subdomains', + volumeMode: 'average', + }, + sort: [], + }) +} diff --git a/packages/core/src/providers/ahrefs/domain-rating.ts b/packages/core/src/providers/ahrefs/domain-rating.ts new file mode 100644 index 00000000..1a5a1d98 --- /dev/null +++ b/packages/core/src/providers/ahrefs/domain-rating.ts @@ -0,0 +1,106 @@ +import type { MarketIndependentProviderEvidence } from '../contracts.js' +import { observedValue } from '../contracts.js' +import type { + DomainRatingObservation, + DomainRatingProvider, + DomainRatingRequest, +} from '../domain-rating-contracts.js' +import { AhrefsClient, type AhrefsClientOptions } from './client.js' +import { ahrefsDomainRatingResponseSchema } from './schema.js' +import { + marketIndependentEvidence, + missing, + requestContext, + target, +} from './shared.js' + +const ENDPOINT = 'public/domain-rating-free' +const TTL_MS = 24 * 60 * 60 * 1_000 + +type DomainRatingClient = Pick + +export type AhrefsDomainRatingProviderOptions = AhrefsClientOptions & { + client?: DomainRatingClient +} + +export class AhrefsDomainRatingProvider implements DomainRatingProvider { + readonly provider = 'ahrefs' as const + readonly capabilitySupport = [ + { + capability: 'domain-rating' as const, + status: 'available' as const, + markets: 'all' as const, + }, + ] as const + + private readonly client: DomainRatingClient + + constructor(options: AhrefsDomainRatingProviderOptions = {}) { + this.client = options.client ?? new AhrefsClient(options) + } + + async domainRating( + input: DomainRatingRequest, + ): Promise> { + const normalized = target({ + value: input.target, + mode: input.targetMode ?? 'domain', + operation: 'domain-rating', + }) + const snapshot = await this.client.request({ + operation: 'domain-rating', + capability: 'domain-rating', + path: ENDPOINT, + query: { target: normalized.target }, + schema: ahrefsDomainRatingResponseSchema, + requestedRows: 1, + perRowUnits: 0, + rowCount: () => 1, + free: true, + refresh: input.refresh, + ttlMs: TTL_MS, + context: requestContext('domain-rating', input.context), + }) + const result = snapshot.response.domain_rating + const observed = result.domain_rating !== null + return marketIndependentEvidence({ + capability: 'domain-rating', + data: { + target: normalized.target, + targetMode: normalized.mode, + domainRating: observed + ? observedValue(result.domain_rating as number) + : missing('Domain Rating'), + licenseUrl: result.license, + attribution: 'Domain Rating by Ahrefs', + attributionUrl: 'https://ahrefs.com/', + }, + snapshot, + coverage: { + requestedRows: 1, + returnedRows: 1, + retainedRows: observed ? 1 : 0, + invalidRows: 0, + providerTotalRows: null, + completeness: observed ? 'complete' : 'unavailable', + nextCursor: null, + }, + endpoint: ENDPOINT, + limit: 1, + filters: { + apiVersion: 3, + targetMode: normalized.mode, + }, + sort: [], + warnings: result.warning + ? [ + { + code: 'ahrefs-domain-rating-warning', + field: 'data.domainRating', + message: result.warning, + }, + ] + : [], + }) + } +} diff --git a/packages/core/src/providers/ahrefs/domain-research.ts b/packages/core/src/providers/ahrefs/domain-research.ts new file mode 100644 index 00000000..54c4bdda --- /dev/null +++ b/packages/core/src/providers/ahrefs/domain-research.ts @@ -0,0 +1,61 @@ +import type { + DomainOverviewRequest, + DomainResearchProvider, + RankedKeywordsRequest, + RankingPagesRequest, + SerpCompetitorsRequest, +} from '../domain-contracts.js' +import { AhrefsClient, type AhrefsClientOptions } from './client.js' +import { ahrefsDomainOverview } from './domain-overview.js' +import { ahrefsRankedKeywords } from './ranked-keywords.js' +import { ahrefsRankingPages } from './ranking-pages.js' +import { ahrefsSerpCompetitors } from './serp-competitors.js' +import { AHREFS_MARKETS } from './shared.js' + +type DomainResearchClient = Pick + +export type AhrefsDomainResearchProviderOptions = AhrefsClientOptions & { + client?: DomainResearchClient +} + +export class AhrefsDomainResearchProvider implements DomainResearchProvider { + readonly provider = 'ahrefs' as const + readonly capabilitySupport = [ + 'domain-overview', + 'ranked-keywords', + 'relevant-pages', + 'serp-competitors', + ].map((capability) => ({ + capability: capability as + | 'domain-overview' + | 'ranked-keywords' + | 'relevant-pages' + | 'serp-competitors', + status: 'available' as const, + markets: AHREFS_MARKETS, + })) + + private readonly client: DomainResearchClient + private readonly now: () => Date + + constructor(options: AhrefsDomainResearchProviderOptions = {}) { + this.client = options.client ?? new AhrefsClient(options) + this.now = options.now ?? (() => new Date()) + } + + domainOverview(input: DomainOverviewRequest) { + return ahrefsDomainOverview(this.client, input, this.now) + } + + rankedKeywords(input: RankedKeywordsRequest) { + return ahrefsRankedKeywords(this.client, input, this.now) + } + + rankingPages(input: RankingPagesRequest) { + return ahrefsRankingPages(this.client, input, this.now) + } + + serpCompetitors(input: SerpCompetitorsRequest) { + return ahrefsSerpCompetitors(this.client, input) + } +} diff --git a/packages/core/src/providers/ahrefs/keyword-discovery.ts b/packages/core/src/providers/ahrefs/keyword-discovery.ts new file mode 100644 index 00000000..94f504a1 --- /dev/null +++ b/packages/core/src/providers/ahrefs/keyword-discovery.ts @@ -0,0 +1,353 @@ +import type { + KeywordDiscoveryProvider, + KeywordDiscoveryRequest, + KeywordDiscoverySource, + KeywordIdea, + ProviderCacheEvidence, + ProviderCostEvidence, + ProviderEvidence, + ProviderWarning, +} from '../contracts.js' +import { keywordDiscoverySourceSchema } from '../contracts.js' +import { ProviderError } from '../errors.js' +import { + type AhrefsApiSnapshot, + AhrefsClient, + type AhrefsClientOptions, +} from './client.js' +import { emptyAhrefsKeywordIdea } from './mapping.js' +import { ahrefsKeywordIdeasResponseSchema } from './schema.js' +import { + AHREFS_API_BASE_URL, + AHREFS_MARKETS, + compareCodepoints, + marketCountry, + marketWarnings, + normalizedKeyword, + requestContext, +} from './shared.js' + +const MAX_SEEDS = 5 +const MAX_ROWS = 100 +const SELECT = 'keyword' +const REPORTS = { + ideas: { + path: 'keywords-explorer/matching-terms', + query: { match_mode: 'terms', terms: 'all' }, + }, + related: { + path: 'keywords-explorer/related-terms', + query: { view_for: 'top_10', terms: 'all' }, + }, + suggestions: { + path: 'keywords-explorer/search-suggestions', + query: {}, + }, +} as const satisfies Record< + KeywordDiscoverySource, + { path: string; query: Record } +> + +type KeywordDiscoveryClient = Pick +type KeywordIdeasResponse = { + keywords: Array<{ keyword: string }> +} +type KeywordIdeasSnapshot = AhrefsApiSnapshot + +export type AhrefsKeywordDiscoveryProviderOptions = AhrefsClientOptions & { + client?: KeywordDiscoveryClient +} + +type DiscoveryCall = { + seed: string + source: KeywordDiscoverySource + limit: number +} + +function plannedCalls( + seeds: string[], + sources: KeywordDiscoverySource[], + limit: number, +): DiscoveryCall[] { + const requests = sources.flatMap((source) => + seeds.map((seed) => ({ source, seed })), + ) + if (limit < requests.length) { + throw new ProviderError({ + provider: 'ahrefs', + operation: 'keyword-discovery', + code: 'configuration', + message: `Ahrefs keyword discovery needs a limit of at least ${requests.length} to sample every source and seed.`, + }) + } + const base = Math.floor(limit / requests.length) + const remainder = limit % requests.length + return requests.map((request, index) => ({ + ...request, + limit: base + Number(index < remainder), + })) +} + +function combinedCache( + snapshots: KeywordIdeasSnapshot[], +): ProviderCacheEvidence { + if (snapshots.every((snapshot) => snapshot.cache.status === 'hit')) { + const stored = snapshots + .map((snapshot) => snapshot.cache.storedAt) + .filter((value): value is string => Boolean(value)) + .sort(compareCodepoints) + const expires = snapshots + .map((snapshot) => snapshot.cache.expiresAt) + .filter((value): value is string => Boolean(value)) + .sort(compareCodepoints) + return { + status: 'hit', + storedAt: stored[0] ?? null, + expiresAt: expires[0] ?? null, + } + } + return { + status: snapshots.every((snapshot) => snapshot.cache.status === 'bypass') + ? 'bypass' + : 'miss', + storedAt: null, + expiresAt: null, + } +} + +function sumNullable(values: Array): number | null { + return values.every((value) => value !== null) + ? values.reduce((sum, value) => sum + (value ?? 0), 0) + : null +} + +function combinedCost(snapshots: KeywordIdeasSnapshot[]): ProviderCostEvidence { + const natives = snapshots.map((snapshot) => snapshot.cost.native) + const remaining = natives + .map((native) => native?.remainingBefore ?? null) + .filter((value): value is number => value !== null) + return { + currency: 'USD', + estimatedMicros: null, + actualMicros: null, + taskIds: [], + native: { + unit: 'api-unit', + estimatedUnits: sumNullable( + natives.map((native) => native?.estimatedUnits ?? null), + ), + actualUnits: sumNullable( + natives.map((native) => native?.actualUnits ?? null), + ), + remainingBefore: remaining.length ? Math.max(...remaining) : null, + }, + } +} + +export class AhrefsKeywordDiscoveryProvider + implements KeywordDiscoveryProvider +{ + readonly provider = 'ahrefs' as const + readonly capabilitySupport = [ + { + capability: 'keyword-discovery' as const, + status: 'available' as const, + markets: AHREFS_MARKETS, + }, + ] as const + + private readonly client: KeywordDiscoveryClient + + constructor(options: AhrefsKeywordDiscoveryProviderOptions = {}) { + this.client = options.client ?? new AhrefsClient(options) + } + + async discoverKeywords( + input: KeywordDiscoveryRequest, + ): Promise> { + if ( + !Number.isSafeInteger(input.limit) || + input.limit < 1 || + input.limit > MAX_ROWS + ) { + throw new ProviderError({ + provider: 'ahrefs', + operation: 'keyword-discovery', + code: 'configuration', + message: 'Ahrefs keyword discovery limit must be from 1 to 100.', + }) + } + if (input.seeds.length < 1 || input.seeds.length > MAX_SEEDS) { + throw new ProviderError({ + provider: 'ahrefs', + operation: 'keyword-discovery', + code: 'configuration', + message: 'Ahrefs keyword discovery requires 1 to 5 seeds.', + }) + } + const seeds = [ + ...new Set( + input.seeds.map((seed) => normalizedKeyword(seed, 'keyword-discovery')), + ), + ].sort(compareCodepoints) + const sources = [ + ...new Set( + input.sources.map((source) => + keywordDiscoverySourceSchema.parse(source), + ), + ), + ].sort(compareCodepoints) + if (sources.length < 1 || sources.length > 3) { + throw new ProviderError({ + provider: 'ahrefs', + operation: 'keyword-discovery', + code: 'configuration', + message: 'Choose 1 to 3 Ahrefs keyword discovery sources.', + }) + } + const country = marketCountry(input.market, 'keyword-discovery') + const calls = plannedCalls(seeds, sources, input.limit) + const context = requestContext('keyword-research', input.context) + const snapshots: KeywordIdeasSnapshot[] = [] + const rows: Array<{ + keyword: string + seed: string + source: KeywordDiscoverySource + }> = [] + const warnings: ProviderWarning[] = [...marketWarnings(input.market)] + let lastError: ProviderError | undefined + for (const call of calls) { + const report = REPORTS[call.source] + try { + const snapshot = await this.client.request({ + operation: `keyword-discovery-${call.source}`, + capability: 'keyword-discovery', + path: report.path, + query: { + country, + keywords: call.seed, + select: SELECT, + limit: call.limit, + ...report.query, + }, + schema: ahrefsKeywordIdeasResponseSchema, + requestedRows: call.limit, + perRowUnits: 1, + rowCount: (response) => response.keywords.length, + free: ['ahrefs', 'yep', 'firehose'].includes(call.seed), + refresh: input.refresh, + context, + }) + snapshots.push(snapshot) + warnings.push(...snapshot.warnings) + rows.push( + ...snapshot.response.keywords.map((row) => ({ + keyword: row.keyword, + seed: call.seed, + source: call.source, + })), + ) + } catch (error) { + if (!(error instanceof ProviderError)) throw error + lastError = error + warnings.push({ + code: 'discovery-request-failed', + field: call.source, + message: `Ahrefs ${call.source} discovery failed for one seed (${error.code}).`, + }) + } + } + if (!snapshots.length && lastError) throw lastError + + const grouped = new Map< + string, + Array<{ seed: string; source: KeywordDiscoverySource }> + >() + let invalidRows = 0 + for (const row of rows) { + let keyword: string + try { + keyword = normalizedKeyword(row.keyword, 'keyword-discovery') + } catch { + invalidRows += 1 + continue + } + grouped.set(keyword, [ + ...(grouped.get(keyword) ?? []), + { seed: row.seed, source: row.source }, + ]) + } + const ideas = [...grouped.entries()] + .sort(([left], [right]) => compareCodepoints(left, right)) + .slice(0, input.limit) + .map(([keyword, matches]) => + emptyAhrefsKeywordIdea( + keyword, + [ + ...new Map( + matches.map((match) => [`${match.source}\0${match.seed}`, match]), + ).values(), + ].sort( + (left, right) => + compareCodepoints(left.source, right.source) || + compareCodepoints(left.seed, right.seed), + ), + ), + ) + const returnedRows = snapshots.reduce( + (sum, snapshot) => sum + snapshot.returnedRows, + 0, + ) + const failedCalls = calls.length - snapshots.length + if (invalidRows) { + warnings.push({ + code: 'invalid-keyword-rows', + field: 'data', + message: `Ahrefs returned ${invalidRows} keyword row${invalidRows === 1 ? '' : 's'} without a valid keyword.`, + }) + } + const observedAt = snapshots + .map((snapshot) => snapshot.observedAt) + .sort(compareCodepoints) + .at(-1) as string + return { + schemaVersion: 1, + provider: 'ahrefs', + capability: 'keyword-discovery', + data: ideas, + observedAt, + market: input.market, + coverage: { + requestedRows: input.limit, + returnedRows, + retainedRows: ideas.length, + invalidRows, + providerTotalRows: null, + completeness: + failedCalls || invalidRows + ? 'partial' + : returnedRows >= input.limit || ideas.length < grouped.size + ? 'capped' + : 'complete', + nextCursor: null, + }, + cache: combinedCache(snapshots), + cost: combinedCost(snapshots), + request: { + operation: 'keyword-discovery', + endpoint: AHREFS_API_BASE_URL, + limit: input.limit, + filters: { + country, + selectedFields: SELECT, + sources: sources.join(','), + seeds: seeds.length, + providerRequests: calls.length, + apiVersion: 3, + }, + sort: ['keyword:codepoint-ascending'], + }, + warnings, + } + } +} diff --git a/packages/core/src/providers/ahrefs/keyword-metrics.ts b/packages/core/src/providers/ahrefs/keyword-metrics.ts new file mode 100644 index 00000000..ca81a0a0 --- /dev/null +++ b/packages/core/src/providers/ahrefs/keyword-metrics.ts @@ -0,0 +1,185 @@ +import type { + KeywordMetric, + KeywordMetricsProvider, + KeywordMetricsRequest, + ProviderEvidence, +} from '../contracts.js' +import { ProviderError } from '../errors.js' +import { AhrefsClient, type AhrefsClientOptions } from './client.js' +import { ahrefsKeywordMetric } from './mapping.js' +import { ahrefsKeywordOverviewResponseSchema } from './schema.js' +import { + AHREFS_API_BASE_URL, + AHREFS_MARKETS, + compareCodepoints, + marketCountry, + marketWarnings, + normalizedKeyword, + requestContext, +} from './shared.js' + +const ENDPOINT = 'keywords-explorer/overview' +const MAX_KEYWORDS = 100 +const SELECT = 'keyword,volume,cpc,difficulty,intents' +const PER_ROW_UNITS = 32 + +type KeywordMetricsClient = Pick + +export type AhrefsKeywordMetricsProviderOptions = AhrefsClientOptions & { + client?: KeywordMetricsClient +} + +export class AhrefsKeywordMetricsProvider implements KeywordMetricsProvider { + readonly provider = 'ahrefs' as const + readonly capabilitySupport = [ + { + capability: 'keyword-metrics' as const, + status: 'available' as const, + markets: AHREFS_MARKETS, + }, + ] as const + + private readonly client: KeywordMetricsClient + + constructor(options: AhrefsKeywordMetricsProviderOptions = {}) { + this.client = options.client ?? new AhrefsClient(options) + } + + async keywordMetrics( + input: KeywordMetricsRequest, + ): Promise> { + if (input.keywords.length < 1 || input.keywords.length > MAX_KEYWORDS) { + throw new ProviderError({ + provider: 'ahrefs', + operation: 'keyword-metrics', + code: 'configuration', + message: 'Ahrefs keyword metrics requires 1 to 100 keywords.', + }) + } + const normalized = input.keywords.map((keyword) => + normalizedKeyword(keyword, 'keyword-metrics'), + ) + const keywords = [...new Set(normalized)].sort(compareCodepoints) + const country = marketCountry(input.market, 'keyword-metrics') + const snapshot = await this.client.request({ + operation: 'keyword-metrics', + capability: 'keyword-metrics', + path: ENDPOINT, + query: { + country, + keywords: keywords.join(','), + select: SELECT, + limit: keywords.length, + }, + schema: ahrefsKeywordOverviewResponseSchema, + requestedRows: keywords.length, + perRowUnits: PER_ROW_UNITS, + rowCount: (response) => response.keywords.length, + free: keywords.every((keyword) => + ['ahrefs', 'yep', 'firehose'].includes(keyword), + ), + refresh: input.refresh, + context: requestContext('keyword-metrics', input.context), + }) + const requested = new Set(keywords) + const grouped = new Map< + string, + (typeof snapshot.response.keywords)[number][] + >() + let invalidRows = 0 + for (const row of snapshot.response.keywords) { + let keyword: string + try { + keyword = normalizedKeyword(row.keyword, 'keyword-metrics') + } catch { + invalidRows += 1 + continue + } + if (!requested.has(keyword)) { + invalidRows += 1 + continue + } + grouped.set(keyword, [...(grouped.get(keyword) ?? []), row]) + } + const missingRows = keywords.filter((keyword) => !grouped.has(keyword)) + const duplicateRows = [...grouped.values()].reduce( + (total, rows) => total + Math.max(0, rows.length - 1), + 0, + ) + const partial = + missingRows.length > 0 || invalidRows > 0 || duplicateRows > 0 + return { + schemaVersion: 1, + provider: 'ahrefs', + capability: 'keyword-metrics', + data: keywords.map((keyword) => + ahrefsKeywordMetric(keyword, grouped.get(keyword) ?? []), + ), + observedAt: snapshot.observedAt, + market: input.market, + coverage: { + requestedRows: keywords.length, + returnedRows: snapshot.returnedRows, + retainedRows: keywords.length, + invalidRows, + providerTotalRows: null, + completeness: partial ? 'partial' : 'complete', + nextCursor: null, + }, + cache: snapshot.cache, + cost: snapshot.cost, + request: { + operation: 'keyword-metrics', + endpoint: new URL(ENDPOINT, AHREFS_API_BASE_URL).toString(), + limit: keywords.length, + filters: { + country, + selectedFields: SELECT, + apiVersion: 3, + }, + sort: ['keyword:codepoint-ascending'], + }, + warnings: [ + ...snapshot.warnings, + ...marketWarnings(input.market), + ...(normalized.length !== keywords.length + ? [ + { + code: 'duplicate-keywords-removed', + field: 'keywords', + message: + 'Duplicate keywords were normalized and requested once.', + }, + ] + : []), + ...(missingRows.length + ? [ + { + code: 'provider-keywords-omitted', + field: 'keyword', + message: `Ahrefs omitted ${missingRows.length} requested keyword${missingRows.length === 1 ? '' : 's'}.`, + }, + ] + : []), + ...(invalidRows + ? [ + { + code: 'unexpected-provider-keywords', + field: 'keyword', + message: `Ahrefs returned ${invalidRows} unexpected or invalid keyword row${invalidRows === 1 ? '' : 's'}.`, + }, + ] + : []), + ...(duplicateRows + ? [ + { + code: 'duplicate-provider-keywords', + field: 'keyword', + message: `Ahrefs returned ${duplicateRows} duplicate keyword row${duplicateRows === 1 ? '' : 's'}; conflicting fields are invalid.`, + }, + ] + : []), + ], + } + } +} diff --git a/packages/core/src/providers/ahrefs/link-research.ts b/packages/core/src/providers/ahrefs/link-research.ts new file mode 100644 index 00000000..c2d1c235 --- /dev/null +++ b/packages/core/src/providers/ahrefs/link-research.ts @@ -0,0 +1,52 @@ +import type { + BacklinksRequest, + LinkSummaryRequest, + LiveLinkProvider, + ReferringDomainsRequest, +} from '../link-contracts.js' +import { ahrefsBacklinks } from './backlinks.js' +import { AhrefsClient, type AhrefsClientOptions } from './client.js' +import { ahrefsLinkSummary } from './link-summary.js' +import { ahrefsReferringDomains } from './referring-domains.js' + +type LinkResearchClient = Pick + +export type AhrefsLinkProviderOptions = AhrefsClientOptions & { + client?: LinkResearchClient +} + +export class AhrefsLinkProvider implements LiveLinkProvider { + readonly provider = 'ahrefs' as const + readonly capabilitySupport = [ + 'link-summary', + 'referring-domains', + 'backlinks', + ].map((capability) => ({ + capability: capability as + | 'link-summary' + | 'referring-domains' + | 'backlinks', + status: 'available' as const, + markets: 'all' as const, + })) + + private readonly client: LinkResearchClient + private readonly now: () => Date + + constructor(options: AhrefsLinkProviderOptions = {}) { + this.client = options.client ?? new AhrefsClient(options) + this.now = options.now ?? (() => new Date()) + } + + linkSummary(input: LinkSummaryRequest) { + return ahrefsLinkSummary(this.client, input, this.now) + } + + backlinks(input: BacklinksRequest) { + return ahrefsBacklinks(this.client, input) + } + + referringDomains(input: ReferringDomainsRequest) { + return ahrefsReferringDomains(this.client, input) + } +} diff --git a/packages/core/src/providers/ahrefs/link-summary.ts b/packages/core/src/providers/ahrefs/link-summary.ts new file mode 100644 index 00000000..3da3730b --- /dev/null +++ b/packages/core/src/providers/ahrefs/link-summary.ts @@ -0,0 +1,85 @@ +import type { MarketIndependentProviderEvidence } from '../contracts.js' +import type { LinkSummary, LinkSummaryRequest } from '../link-contracts.js' +import type { AhrefsClient } from './client.js' +import { ahrefsBacklinksStatsResponseSchema } from './schema.js' +import { + apiDate, + linkTarget, + marketIndependentEvidence, + numberValue, + requestContext, + unavailable, +} from './shared.js' + +const ENDPOINT = 'site-explorer/backlinks-stats' +const PER_ROW_UNITS = 12 + +export async function ahrefsLinkSummary( + client: Pick, + input: LinkSummaryRequest, + now: () => Date, +): Promise> { + const normalized = linkTarget(input.target, input.scope) + const includeSubdomains = + normalized.scope === 'domain' ? (input.includeSubdomains ?? true) : false + const mode = + normalized.scope === 'page' + ? 'exact' + : includeSubdomains + ? 'subdomains' + : 'domain' + const date = apiDate(now) + const snapshot = await client.request({ + operation: 'link-summary', + capability: 'link-summary', + path: ENDPOINT, + query: { + date, + mode, + target: normalized.target, + }, + schema: ahrefsBacklinksStatsResponseSchema, + requestedRows: 1, + perRowUnits: PER_ROW_UNITS, + rowCount: () => 1, + refresh: input.refresh, + context: requestContext('link-evidence', input.context), + }) + const metrics = snapshot.response.metrics + return marketIndependentEvidence({ + capability: 'link-summary', + data: { + target: normalized.target, + scope: normalized.scope, + backlinks: numberValue(metrics.live, 'live backlinks'), + referringDomains: numberValue( + metrics.live_refdomains, + 'live referring domains', + ), + referringPages: unavailable('live referring pages'), + brokenBacklinks: unavailable('broken backlinks'), + brokenPages: unavailable('broken referring pages'), + metrics: [], + }, + snapshot, + coverage: { + requestedRows: 1, + returnedRows: 1, + retainedRows: 1, + invalidRows: 0, + providerTotalRows: null, + completeness: 'complete', + nextCursor: null, + }, + endpoint: ENDPOINT, + limit: 1, + filters: { + apiVersion: 3, + date, + includeSubdomains, + mode, + scope: normalized.scope, + }, + sort: [], + }) +} diff --git a/packages/core/src/providers/ahrefs/mapping.ts b/packages/core/src/providers/ahrefs/mapping.ts new file mode 100644 index 00000000..4e022213 --- /dev/null +++ b/packages/core/src/providers/ahrefs/mapping.ts @@ -0,0 +1,115 @@ +import type { z } from 'zod' +import type { KeywordIdea, KeywordMetric, ProviderValue } from '../contracts.js' +import { observedValue, unavailableValue } from '../contracts.js' +import type { + ahrefsIntentsSchema, + ahrefsKeywordOverviewResponseSchema, +} from './schema.js' + +type AhrefsKeywordOverviewRow = z.infer< + typeof ahrefsKeywordOverviewResponseSchema +>['keywords'][number] +type AhrefsIntents = z.infer +type KeywordField = Exclude + +const INTENT_ORDER = [ + 'informational', + 'navigational', + 'commercial', + 'transactional', + 'branded', + 'local', +] as const satisfies readonly (keyof AhrefsIntents)[] + +function missing(field: KeywordField): ProviderValue { + return unavailableValue('missing', `Ahrefs omitted ${field}.`) +} + +function unavailable(field: KeywordField): ProviderValue { + return unavailableValue( + 'unavailable', + `The selected Ahrefs fields do not return ${field}.`, + ) +} + +function numberValue( + values: Array, + field: KeywordField, + transform: (value: number) => number = (value) => value, +): ProviderValue { + const present = values.filter( + (value): value is number => value !== null && value !== undefined, + ) + if (present.length === 0) return missing(field) + const unique = [...new Set(present.map(transform))] + return unique.length === 1 + ? observedValue(unique[0] as number) + : unavailableValue( + 'invalid', + `Ahrefs returned conflicting ${field} values.`, + ) +} + +function intentLabel(value: AhrefsIntents | null): string | null { + if (!value) return null + const labels = INTENT_ORDER.filter((label) => value[label]) + return labels.length ? labels.join(',') : null +} + +function intentValue( + values: Array, +): ProviderValue { + const present = values + .map((value) => (value ? intentLabel(value) : null)) + .filter((value): value is string => Boolean(value)) + if (present.length === 0) return missing('intent') + const unique = [...new Set(present)] + return unique.length === 1 + ? observedValue(unique[0] as string) + : unavailableValue('invalid', 'Ahrefs returned conflicting intent labels.') +} + +export function ahrefsKeywordMetric( + keyword: string, + rows: AhrefsKeywordOverviewRow[], +): KeywordMetric { + return { + keyword, + monthlySearchVolume: numberValue( + rows.map((row) => row.volume), + 'monthlySearchVolume', + ), + monthlySearches: unavailable('monthlySearches'), + searchVolumeUpdatedAt: unavailable('searchVolumeUpdatedAt'), + cpcUsd: numberValue( + rows.map((row) => row.cpc), + 'cpcUsd', + (value) => value / 100, + ), + paidCompetition: unavailable('paidCompetition'), + keywordDifficulty: numberValue( + rows.map((row) => row.difficulty), + 'keywordDifficulty', + ), + intent: intentValue(rows.map((row) => row.intents)), + resultCount: unavailable('resultCount'), + } +} + +export function emptyAhrefsKeywordIdea( + keyword: string, + sources: KeywordIdea['sources'], +): KeywordIdea { + return { + keyword, + sources, + monthlySearchVolume: unavailable('monthlySearchVolume'), + monthlySearches: unavailable('monthlySearches'), + searchVolumeUpdatedAt: unavailable('searchVolumeUpdatedAt'), + cpcUsd: unavailable('cpcUsd'), + paidCompetition: unavailable('paidCompetition'), + keywordDifficulty: unavailable('keywordDifficulty'), + intent: unavailable('intent'), + resultCount: unavailable('resultCount'), + } +} diff --git a/packages/core/src/providers/ahrefs/ranked-keywords.ts b/packages/core/src/providers/ahrefs/ranked-keywords.ts new file mode 100644 index 00000000..59fcaf7b --- /dev/null +++ b/packages/core/src/providers/ahrefs/ranked-keywords.ts @@ -0,0 +1,279 @@ +import type { KeywordMetric, ProviderEvidence } from '../contracts.js' +import { observedValue } from '../contracts.js' +import type { + RankedKeyword, + RankedKeywordPage, + RankedKeywordsRequest, +} from '../domain-contracts.js' +import { ProviderError } from '../errors.js' +import type { AhrefsClient } from './client.js' +import { ahrefsOrganicKeywordsResponseSchema } from './schema.js' +import { + apiDate, + centsValue, + compareCodepoints, + coverage, + dedupeBy, + domain, + evidence, + freeTestValue, + marketCountry, + missing, + normalizedKeyword, + observedNumber, + organicOnly, + requestContext, + rowLimit, + safeUrl, + unavailable, +} from './shared.js' + +const ENDPOINT = 'site-explorer/organic-keywords' +const SELECT = + 'keyword,best_position,best_position_kind,best_position_url,volume,cpc,keyword_difficulty,sum_traffic,is_branded,is_commercial,is_informational,is_local,is_navigational,is_transactional' +const PER_ROW_UNITS = 41 +const ORDER_BY = 'best_position:asc,volume:desc,keyword:asc' +const INTENTS = [ + 'informational', + 'navigational', + 'commercial', + 'transactional', + 'branded', + 'local', +] as const + +type OrganicRow = + (typeof ahrefsOrganicKeywordsResponseSchema)['_output']['keywords'][number] + +function integerFilter( + value: number | undefined, + input: { + field: string + operator: 'gte' | 'lte' + minimum: number + maximum?: number + label: string + }, +): Record | null { + if (value === undefined) return null + if ( + !Number.isSafeInteger(value) || + value < input.minimum || + (input.maximum !== undefined && value > input.maximum) + ) { + throw new ProviderError({ + provider: 'ahrefs', + operation: 'ranked-keywords', + code: 'configuration', + message: `${input.label} must be from ${input.minimum}${input.maximum === undefined ? ' upward' : ` to ${input.maximum}`}.`, + }) + } + return { field: input.field, is: [input.operator, value] } +} + +function where(input: RankedKeywordsRequest): string { + const filters: Record[] = [ + { field: 'best_position_kind', is: ['eq', 'organic'] }, + ] + const minVolume = integerFilter(input.minSearchVolume, { + field: 'volume', + operator: 'gte', + minimum: 0, + label: 'Minimum search volume', + }) + const maxRank = integerFilter(input.maxRank, { + field: 'best_position', + operator: 'lte', + minimum: 1, + maximum: 100, + label: 'Maximum rank', + }) + if (minVolume) filters.push(minVolume) + if (maxRank) filters.push(maxRank) + + const excluded = [ + ...new Set( + (input.excludeTerms ?? []).map((term) => + normalizedKeyword(term, 'ranked-keywords'), + ), + ), + ].sort(compareCodepoints) + if (excluded.length > 5) { + throw new ProviderError({ + provider: 'ahrefs', + operation: 'ranked-keywords', + code: 'configuration', + message: 'Use at most 5 excluded terms.', + }) + } + filters.push( + ...excluded.map((term) => ({ + not: { field: 'keyword', is: ['isubstring', term] }, + })), + ) + return JSON.stringify({ and: filters }) +} + +function keywordMetric(row: OrganicRow, keyword: string): KeywordMetric { + const intent = INTENTS.filter((name) => row[`is_${name}`]) + return { + keyword, + monthlySearchVolume: + row.volume === null + ? missing('monthly search volume') + : observedValue(row.volume), + monthlySearches: unavailable('monthly search history'), + searchVolumeUpdatedAt: unavailable('the search-volume update time'), + cpcUsd: centsValue(row.cpc, 'cost per click'), + paidCompetition: unavailable('paid-search competition'), + keywordDifficulty: + row.keyword_difficulty === null + ? missing('keyword difficulty') + : observedValue(row.keyword_difficulty), + intent: + intent.length > 0 + ? observedValue(intent.join(',')) + : missing('keyword intent'), + resultCount: unavailable('search result count'), + } +} + +export async function ahrefsRankedKeywords( + client: Pick, + input: RankedKeywordsRequest, + now: () => Date, +): Promise> { + organicOnly(input.resultTypes, 'ranked-keywords') + rowLimit(input.limit, input.offset, 'ranked-keywords') + const target = domain(input.target, 'ranked-keywords') + const country = marketCountry(input.market, 'ranked-keywords') + const date = apiDate(now) + const mode = input.includeSubdomains === false ? 'domain' : 'subdomains' + const providerWhere = where(input) + const snapshot = await client.request({ + operation: 'ranked-keywords', + capability: 'ranked-keywords', + path: ENDPOINT, + query: { + country, + date, + limit: input.limit, + mode, + order_by: ORDER_BY, + select: SELECT, + target, + volume_mode: 'average', + where: providerWhere, + }, + schema: ahrefsOrganicKeywordsResponseSchema, + requestedRows: input.limit, + perRowUnits: PER_ROW_UNITS, + rowCount: (response) => response.keywords.length, + free: freeTestValue(target), + refresh: input.refresh, + context: requestContext('ranked-keywords', input.context), + }) + + let invalidRows = 0 + const mapped = snapshot.response.keywords.flatMap((row): RankedKeyword[] => { + let keyword: string + try { + keyword = normalizedKeyword(row.keyword ?? '', 'ranked-keywords') + } catch { + invalidRows += 1 + return [] + } + const url = safeUrl(row.best_position_url) + const rank = row.best_position + if ( + !url || + !Number.isSafeInteger(rank) || + !rank || + rank < 1 || + rank > 100 || + row.best_position_kind !== 'organic' + ) { + invalidRows += 1 + return [] + } + return [ + { + ...keywordMetric(row, keyword), + url, + rankGroup: rank, + rankAbsolute: rank, + resultType: 'organic', + estimatedMonthlyTraffic: + row.sum_traffic === null + ? missing('estimated monthly traffic') + : observedValue(row.sum_traffic), + }, + ] + }) + const rows = dedupeBy( + mapped, + (row) => `${row.keyword}\0${row.url}\0${row.resultType}`, + ).sort( + (left, right) => + observedNumber(right.monthlySearchVolume) - + observedNumber(left.monthlySearchVolume) || + left.rankGroup - right.rankGroup || + compareCodepoints(left.keyword, right.keyword) || + compareCodepoints(left.url, right.url), + ) + const duplicateRows = mapped.length - rows.length + + return evidence({ + capability: 'ranked-keywords', + data: { target, rows, totalRows: null }, + market: input.market, + snapshot, + coverage: coverage({ + requestedRows: input.limit, + returnedRows: snapshot.returnedRows, + retainedRows: rows.length, + invalidRows, + filtered: true, + }), + endpoint: ENDPOINT, + limit: input.limit, + filters: { + apiVersion: 3, + country, + date, + excludedTerms: input.excludeTerms?.length ?? 0, + includeSubdomains: input.includeSubdomains ?? true, + maxRank: input.maxRank ?? 100, + minSearchVolume: input.minSearchVolume ?? 0, + mode, + resultTypes: 'organic', + selectedFields: SELECT, + volumeMode: 'average', + }, + sort: [ + 'monthlySearchVolume:descending', + 'rank:ascending', + 'keyword:codepoint-ascending', + ], + warnings: [ + ...(invalidRows + ? [ + { + code: 'invalid-ranked-keyword-rows', + field: 'data.rows', + message: `Ahrefs returned ${invalidRows} ranked-keyword row${invalidRows === 1 ? '' : 's'} without the required fields.`, + }, + ] + : []), + ...(duplicateRows + ? [ + { + code: 'duplicate-ranked-keyword-rows', + field: 'data.rows', + message: `${duplicateRows} duplicate ranked-keyword row${duplicateRows === 1 ? '' : 's'} were collapsed deterministically.`, + }, + ] + : []), + ], + }) +} diff --git a/packages/core/src/providers/ahrefs/ranking-pages.ts b/packages/core/src/providers/ahrefs/ranking-pages.ts new file mode 100644 index 00000000..6e32b720 --- /dev/null +++ b/packages/core/src/providers/ahrefs/ranking-pages.ts @@ -0,0 +1,172 @@ +import type { ProviderEvidence } from '../contracts.js' +import type { + RankingPage, + RankingPagePage, + RankingPagesRequest, +} from '../domain-contracts.js' +import { ProviderError } from '../errors.js' +import type { AhrefsClient } from './client.js' +import { ahrefsTopPagesResponseSchema } from './schema.js' +import { + apiDate, + compareCodepoints, + coverage, + dedupeBy, + domain, + evidence, + freeTestValue, + marketCountry, + observedNumber, + organicFootprint, + requestContext, + rowLimit, + safeUrl, +} from './shared.js' + +const ENDPOINT = 'site-explorer/top-pages' +const SELECT = 'url,keywords,sum_traffic,value' +const PER_ROW_UNITS = 22 +const ORDER_BY = 'sum_traffic:desc,keywords:desc,url:asc' + +function nonnegative( + value: number | undefined, + field: string, + label: string, +): Record | null { + if (value === undefined) return null + if (!Number.isFinite(value) || value < 0) { + throw new ProviderError({ + provider: 'ahrefs', + operation: 'ranking-pages', + code: 'configuration', + message: `${label} must be nonnegative.`, + }) + } + return { field, is: ['gte', value] } +} + +function where(input: RankingPagesRequest): string | null { + const filters = [ + nonnegative( + input.minEstimatedTraffic, + 'sum_traffic', + 'Minimum estimated traffic', + ), + nonnegative(input.minRankedKeywords, 'keywords', 'Minimum ranked keywords'), + ].filter((value): value is Record => value !== null) + return filters.length ? JSON.stringify({ and: filters }) : null +} + +export async function ahrefsRankingPages( + client: Pick, + input: RankingPagesRequest, + now: () => Date, +): Promise> { + rowLimit(input.limit, input.offset, 'ranking-pages') + const target = domain(input.domain, 'ranking-pages') + const country = marketCountry(input.market, 'ranking-pages') + const date = apiDate(now) + const providerWhere = where(input) + const snapshot = await client.request({ + operation: 'ranking-pages', + capability: 'relevant-pages', + path: ENDPOINT, + query: { + country, + date, + limit: input.limit, + mode: 'subdomains', + order_by: ORDER_BY, + select: SELECT, + target, + volume_mode: 'average', + ...(providerWhere ? { where: providerWhere } : {}), + }, + schema: ahrefsTopPagesResponseSchema, + requestedRows: input.limit, + perRowUnits: PER_ROW_UNITS, + rowCount: (response) => response.pages.length, + free: freeTestValue(target), + refresh: input.refresh, + context: requestContext('ranking-pages', input.context), + }) + + let invalidRows = 0 + const mapped = snapshot.response.pages.flatMap((row): RankingPage[] => { + const url = safeUrl(row.url) + if (!url) { + invalidRows += 1 + return [] + } + return [ + { + url, + organic: organicFootprint({ + traffic: row.sum_traffic, + keywords: row.keywords, + costCents: row.value, + }), + }, + ] + }) + const rows = dedupeBy(mapped, (row) => row.url).sort( + (left, right) => + observedNumber(right.organic.estimatedMonthlyTraffic) - + observedNumber(left.organic.estimatedMonthlyTraffic) || + observedNumber(right.organic.rankedKeywords) - + observedNumber(left.organic.rankedKeywords) || + compareCodepoints(left.url, right.url), + ) + const duplicateRows = mapped.length - rows.length + + return evidence({ + capability: 'relevant-pages', + data: { domain: target, rows, totalRows: null }, + market: input.market, + snapshot, + coverage: coverage({ + requestedRows: input.limit, + returnedRows: snapshot.returnedRows, + retainedRows: rows.length, + invalidRows, + filtered: Boolean(providerWhere), + }), + endpoint: ENDPOINT, + limit: input.limit, + filters: { + apiVersion: 3, + country, + date, + minEstimatedTraffic: input.minEstimatedTraffic ?? 0, + minRankedKeywords: input.minRankedKeywords ?? 0, + mode: 'subdomains', + selectedFields: SELECT, + volumeMode: 'average', + }, + sort: [ + 'estimatedMonthlyTraffic:descending', + 'rankedKeywords:descending', + 'url:codepoint-ascending', + ], + warnings: [ + ...(invalidRows + ? [ + { + code: 'invalid-ranking-page-rows', + field: 'data.rows', + message: `Ahrefs returned ${invalidRows} ranking-page row${invalidRows === 1 ? '' : 's'} without a valid URL.`, + }, + ] + : []), + ...(duplicateRows + ? [ + { + code: 'duplicate-ranking-page-rows', + field: 'data.rows', + message: `${duplicateRows} duplicate ranking-page row${duplicateRows === 1 ? '' : 's'} were collapsed deterministically.`, + }, + ] + : []), + ], + }) +} diff --git a/packages/core/src/providers/ahrefs/referring-domains.ts b/packages/core/src/providers/ahrefs/referring-domains.ts new file mode 100644 index 00000000..17b53bac --- /dev/null +++ b/packages/core/src/providers/ahrefs/referring-domains.ts @@ -0,0 +1,159 @@ +import type { MarketIndependentProviderEvidence } from '../contracts.js' +import type { + ReferringDomain, + ReferringDomainPage, + ReferringDomainsRequest, +} from '../link-contracts.js' +import type { AhrefsClient } from './client.js' +import { ahrefsRefdomainsResponseSchema } from './schema.js' +import { + compareCodepoints, + coverage, + dedupeBy, + domain, + linkTarget, + marketIndependentEvidence, + metric, + normalizedDate, + numberValue, + requestContext, + rowLimit, + unavailable, +} from './shared.js' + +const ENDPOINT = 'site-explorer/refdomains' +const SELECT = 'domain,domain_rating,first_seen,links_to_target' +const PER_ROW_UNITS = 4 +const ORDER_BY = 'links_to_target:desc,domain_rating:desc,domain:asc' + +function observedBacklinks(row: ReferringDomain): number { + return row.backlinks.state === 'observed' ? row.backlinks.value : -1 +} + +export async function ahrefsReferringDomains( + client: Pick, + input: ReferringDomainsRequest, +): Promise> { + rowLimit(input.limit, input.offset, 'referring-domains') + const normalized = linkTarget(input.target, input.scope) + const includeSubdomains = + normalized.scope === 'domain' ? (input.includeSubdomains ?? true) : false + const mode = + normalized.scope === 'page' + ? 'exact' + : includeSubdomains + ? 'subdomains' + : 'domain' + const snapshot = await client.request({ + operation: 'referring-domains', + capability: 'referring-domains', + path: ENDPOINT, + query: { + history: 'live', + limit: input.limit, + mode, + order_by: ORDER_BY, + select: SELECT, + target: normalized.target, + }, + schema: ahrefsRefdomainsResponseSchema, + requestedRows: input.limit, + perRowUnits: PER_ROW_UNITS, + rowCount: (response) => response.refdomains.length, + refresh: input.refresh, + context: requestContext('link-evidence', input.context), + }) + + let invalidRows = 0 + const mapped = snapshot.response.refdomains.flatMap( + (row): ReferringDomain[] => { + let sourceDomain: string + try { + sourceDomain = domain(row.domain, 'referring-domains') + } catch { + invalidRows += 1 + return [] + } + const firstSeen = normalizedDate(row.first_seen) + return [ + { + domain: sourceDomain, + backlinks: numberValue( + row.links_to_target, + 'referring-domain backlinks', + ), + referringPages: unavailable('referring pages for this domain'), + brokenBacklinks: unavailable( + 'broken backlinks from this referring domain', + ), + brokenPages: unavailable('broken pages from this referring domain'), + firstSeenAt: firstSeen + ? { state: 'observed', value: firstSeen } + : { + state: 'invalid', + value: null, + reason: 'Ahrefs returned an invalid first-seen date.', + }, + metrics: metric( + 'domain-rating', + 'Ahrefs Domain Rating', + row.domain_rating, + ), + }, + ] + }, + ) + const rows = dedupeBy(mapped, (row) => row.domain).sort( + (left, right) => + observedBacklinks(right) - observedBacklinks(left) || + compareCodepoints(left.domain, right.domain), + ) + const duplicateRows = mapped.length - rows.length + return marketIndependentEvidence({ + capability: 'referring-domains', + data: { target: normalized.target, rows, totalRows: null }, + snapshot, + coverage: coverage({ + requestedRows: input.limit, + returnedRows: snapshot.returnedRows, + retainedRows: rows.length, + invalidRows, + filtered: true, + }), + endpoint: ENDPOINT, + limit: input.limit, + filters: { + apiVersion: 3, + history: 'live', + includeSubdomains, + mode, + scope: normalized.scope, + selectedFields: SELECT, + }, + sort: [ + 'backlinks:descending', + 'domainRating:descending', + 'domain:codepoint-ascending', + ], + warnings: [ + ...(invalidRows + ? [ + { + code: 'invalid-referring-domain-rows', + field: 'data.rows', + message: `Ahrefs returned ${invalidRows} referring-domain row${invalidRows === 1 ? '' : 's'} without a valid domain.`, + }, + ] + : []), + ...(duplicateRows + ? [ + { + code: 'duplicate-referring-domain-rows', + field: 'data.rows', + message: `${duplicateRows} duplicate referring-domain row${duplicateRows === 1 ? '' : 's'} were collapsed deterministically.`, + }, + ] + : []), + ], + }) +} diff --git a/packages/core/src/providers/ahrefs/schema.ts b/packages/core/src/providers/ahrefs/schema.ts new file mode 100644 index 00000000..9c7a0f1d --- /dev/null +++ b/packages/core/src/providers/ahrefs/schema.ts @@ -0,0 +1,211 @@ +import { z } from 'zod' + +const nullableNonnegativeInteger = z.number().int().nonnegative().nullable() +const nullableScore = z.number().min(0).max(100).nullable() +const boundedString = z.string().max(10_000) +const nullableString = boundedString.nullable() + +export const ahrefsLimitsAndUsageResponseSchema = z + .object({ + limits_and_usage: z + .object({ + api_key_expiration_date: z.string().trim().min(1).max(100), + subscription: z.string().trim().min(1).max(200), + units_limit_api_key: z.number().int().nonnegative().nullable(), + units_limit_workspace: z.number().int().nonnegative().nullable(), + units_usage_api_key: z.number().int().nonnegative(), + units_usage_workspace: z.number().int().nonnegative().nullable(), + usage_reset_date: z.string().trim().min(1).max(100), + }) + .strict(), + }) + .strict() + +export const ahrefsIntentsSchema = z + .object({ + informational: z.boolean().optional(), + navigational: z.boolean().optional(), + commercial: z.boolean().optional(), + transactional: z.boolean().optional(), + branded: z.boolean().optional(), + local: z.boolean().optional(), + }) + .strict() + +export const ahrefsKeywordOverviewResponseSchema = z + .object({ + keywords: z + .array( + z + .object({ + keyword: boundedString, + volume: nullableNonnegativeInteger, + cpc: nullableNonnegativeInteger, + difficulty: z.number().int().min(0).max(100).nullable(), + intents: ahrefsIntentsSchema.nullable(), + }) + .strict(), + ) + .max(1_000), + }) + .strict() + +export const ahrefsKeywordIdeasResponseSchema = z + .object({ + keywords: z + .array( + z + .object({ + keyword: boundedString, + }) + .strict(), + ) + .max(1_000), + }) + .strict() + +export const ahrefsDomainMetricsResponseSchema = z + .object({ + metrics: z + .object({ + org_cost: nullableNonnegativeInteger, + org_keywords: z.number().int().nonnegative(), + org_keywords_1_3: z.number().int().nonnegative(), + org_traffic: z.number().int().nonnegative(), + paid_cost: nullableNonnegativeInteger, + paid_keywords: z.number().int().nonnegative(), + paid_pages: z.number().int().nonnegative(), + paid_traffic: z.number().int().nonnegative(), + }) + .strict(), + }) + .strict() + +export const ahrefsOrganicKeywordsResponseSchema = z + .object({ + keywords: z + .array( + z + .object({ + keyword: nullableString, + best_position: nullableNonnegativeInteger, + best_position_kind: nullableString, + best_position_url: nullableString, + volume: nullableNonnegativeInteger, + cpc: nullableNonnegativeInteger, + keyword_difficulty: z.number().int().min(0).max(100).nullable(), + sum_traffic: nullableNonnegativeInteger, + is_branded: z.boolean(), + is_commercial: z.boolean(), + is_informational: z.boolean(), + is_local: z.boolean(), + is_navigational: z.boolean(), + is_transactional: z.boolean(), + }) + .strict(), + ) + .max(1_000), + }) + .strict() + +export const ahrefsTopPagesResponseSchema = z + .object({ + pages: z + .array( + z + .object({ + url: nullableString, + keywords: nullableNonnegativeInteger, + sum_traffic: nullableNonnegativeInteger, + value: nullableNonnegativeInteger, + }) + .strict(), + ) + .max(1_000), + }) + .strict() + +export const ahrefsSerpOverviewResponseSchema = z + .object({ + positions: z + .array( + z + .object({ + position: z.number().int().positive(), + type: z.array(boundedString).max(50), + url: nullableString, + }) + .strict(), + ) + .max(1_000), + }) + .strict() + +export const ahrefsBacklinksStatsResponseSchema = z + .object({ + metrics: z + .object({ + all_time: z.number().int().nonnegative(), + all_time_refdomains: z.number().int().nonnegative(), + live: z.number().int().nonnegative(), + live_refdomains: z.number().int().nonnegative(), + }) + .strict(), + }) + .strict() + +export const ahrefsRefdomainsResponseSchema = z + .object({ + refdomains: z + .array( + z + .object({ + domain: boundedString, + domain_rating: z.number().min(0).max(100), + first_seen: boundedString, + links_to_target: z.number().int().nonnegative(), + }) + .strict(), + ) + .max(1_000), + }) + .strict() + +export const ahrefsBacklinksResponseSchema = z + .object({ + backlinks: z + .array( + z + .object({ + url_from: boundedString, + root_name_source: boundedString, + url_to: boundedString, + anchor: boundedString, + link_type: boundedString, + is_dofollow: z.boolean(), + first_seen_link: boundedString, + last_seen: nullableString, + is_lost: z.boolean(), + is_redirect: z.boolean(), + links_external: z.number().int().nonnegative(), + domain_rating_source: z.number().min(0).max(100), + url_rating_source: z.number().min(0).max(100), + link_group_count: z.number().int().positive().optional(), + }) + .strict(), + ) + .max(1_000), + }) + .strict() + +export const ahrefsDomainRatingResponseSchema = z + .object({ + domain_rating: z + .object({ + domain_rating: nullableScore, + license: z.string().url().max(2_000), + warning: nullableString.optional(), + }) + .strict(), + }) + .strict() diff --git a/packages/core/src/providers/ahrefs/serp-competitors.ts b/packages/core/src/providers/ahrefs/serp-competitors.ts new file mode 100644 index 00000000..2778128f --- /dev/null +++ b/packages/core/src/providers/ahrefs/serp-competitors.ts @@ -0,0 +1,263 @@ +import type { + ProviderCoverage, + ProviderEvidence, + ProviderWarning, +} from '../contracts.js' +import { observedValue } from '../contracts.js' +import type { + SerpCompetitor, + SerpCompetitorSet, + SerpCompetitorsRequest, +} from '../domain-contracts.js' +import { ProviderError } from '../errors.js' +import type { AhrefsApiSnapshot, AhrefsClient } from './client.js' +import { ahrefsSerpOverviewResponseSchema } from './schema.js' +import { + AHREFS_API_BASE_URL, + combinedCache, + combinedCost, + compareCodepoints, + domain, + freeTestValue, + marketCountry, + marketWarnings, + normalizedKeyword, + organicOnly, + requestContext, + rowLimit, + unavailable, +} from './shared.js' + +const ENDPOINT = 'serp-overview/serp-overview' +const MAX_KEYWORDS = 20 +const MAX_SERP_DEPTH = 100 +const SELECT = 'position,type,url' +const PER_ROW_UNITS = 3 + +type SerpSnapshot = AhrefsApiSnapshot< + (typeof ahrefsSerpOverviewResponseSchema)['_output'] +> + +function keywords(input: string[]): string[] { + const result = [ + ...new Set( + input.map((value) => normalizedKeyword(value, 'serp-competitors')), + ), + ].sort(compareCodepoints) + if (result.length < 1 || result.length > MAX_KEYWORDS) { + throw new ProviderError({ + provider: 'ahrefs', + operation: 'serp-competitors', + code: 'configuration', + message: 'Ahrefs SERP competitors requires 1 to 20 keywords.', + }) + } + return result +} + +function median(values: number[]): number { + const sorted = [...values].sort((left, right) => left - right) + const middle = Math.floor(sorted.length / 2) + return sorted.length % 2 + ? (sorted[middle] as number) + : ((sorted[middle - 1] as number) + (sorted[middle] as number)) / 2 +} + +export async function ahrefsSerpCompetitors( + client: Pick, + input: SerpCompetitorsRequest, +): Promise> { + organicOnly(input.resultTypes, 'serp-competitors') + if (input.includeSubdomains) { + throw new ProviderError({ + provider: 'ahrefs', + operation: 'serp-competitors', + code: 'configuration', + message: + 'Ahrefs SERP competitors preserves observed domains and does not fold subdomains together.', + }) + } + rowLimit(input.limit, input.offset, 'serp-competitors') + const requestedKeywords = keywords(input.keywords) + const country = marketCountry(input.market, 'serp-competitors') + const depth = Math.min(MAX_SERP_DEPTH, input.limit) + const context = requestContext('serp-competitors', input.context) + const snapshots: SerpSnapshot[] = [] + const grouped = new Map>>() + const warnings: ProviderWarning[] = [...marketWarnings(input.market)] + let invalidRows = 0 + let duplicateRows = 0 + let lastError: ProviderError | undefined + + for (const keyword of requestedKeywords) { + try { + const snapshot = await client.request({ + operation: 'serp-competitors-organic-results', + capability: 'serp-competitors', + path: ENDPOINT, + query: { + country, + keyword, + select: SELECT, + top_positions: depth, + type: 'organic', + }, + schema: ahrefsSerpOverviewResponseSchema, + requestedRows: depth, + perRowUnits: PER_ROW_UNITS, + rowCount: (response) => response.positions.length, + free: freeTestValue(keyword), + refresh: input.refresh, + context, + }) + snapshots.push(snapshot) + warnings.push(...snapshot.warnings) + for (const row of snapshot.response.positions) { + const url = row.url + const position = row.position + let resultDomain = '' + try { + resultDomain = url ? domain(url, 'serp-competitors') : '' + } catch { + resultDomain = '' + } + if ( + !resultDomain || + !Number.isSafeInteger(position) || + position < 1 || + position > MAX_SERP_DEPTH || + !row.type.includes('organic') + ) { + invalidRows += 1 + continue + } + const byKeyword = + grouped.get(resultDomain) ?? new Map>() + const positions = byKeyword.get(keyword) ?? new Set() + if (positions.has(position)) duplicateRows += 1 + positions.add(position) + byKeyword.set(keyword, positions) + grouped.set(resultDomain, byKeyword) + } + } catch (error) { + if (!(error instanceof ProviderError)) throw error + lastError = error + warnings.push({ + code: 'competitor-request-failed', + field: 'keywords', + message: `Ahrefs organic results failed for one keyword (${error.code}).`, + }) + } + } + if (!snapshots.length && lastError) throw lastError + + const allRows: SerpCompetitor[] = [...grouped.entries()] + .map(([competitorDomain, byKeyword]) => { + const keywordPositions = [...byKeyword.entries()] + .map(([keyword, positions]) => ({ + keyword, + positions: [...positions].sort((left, right) => left - right), + })) + .sort((left, right) => compareCodepoints(left.keyword, right.keyword)) + const positions = keywordPositions.flatMap((item) => item.positions) + return { + domain: competitorDomain, + matchedKeywords: keywordPositions.length, + averagePosition: observedValue( + positions.reduce((sum, value) => sum + value, 0) / positions.length, + ), + medianPosition: observedValue(median(positions)), + visibility: unavailable( + 'a provider visibility metric for this supplied keyword set', + ), + estimatedMonthlyTraffic: unavailable( + 'absolute estimated monthly traffic for this supplied keyword set', + ), + relevantResults: unavailable( + 'a complete relevant-result count for this supplied keyword set', + ), + keywordPositions, + } + }) + .sort((left, right) => { + const leftAverage = + left.averagePosition.state === 'observed' + ? left.averagePosition.value + : Number.POSITIVE_INFINITY + const rightAverage = + right.averagePosition.state === 'observed' + ? right.averagePosition.value + : Number.POSITIVE_INFINITY + return ( + right.matchedKeywords - left.matchedKeywords || + leftAverage - rightAverage || + compareCodepoints(left.domain, right.domain) + ) + }) + const rows = allRows.slice(0, input.limit) + const failedCalls = requestedKeywords.length - snapshots.length + const providerCapped = snapshots.some( + (snapshot) => snapshot.returnedRows >= depth, + ) + const capped = allRows.length > input.limit || providerCapped + const resultCoverage: ProviderCoverage = { + requestedRows: input.limit, + returnedRows: allRows.length, + retainedRows: rows.length, + invalidRows, + providerTotalRows: null, + completeness: + failedCalls || invalidRows ? 'partial' : capped ? 'capped' : 'complete', + nextCursor: null, + } + if (invalidRows) { + warnings.push({ + code: 'invalid-organic-result-rows', + field: 'data.rows', + message: `Ahrefs returned ${invalidRows} organic result row${invalidRows === 1 ? '' : 's'} without a valid organic domain or position.`, + }) + } + if (duplicateRows) { + warnings.push({ + code: 'duplicate-organic-result-rows', + field: 'data.rows', + message: `${duplicateRows} duplicate organic result row${duplicateRows === 1 ? '' : 's'} were collapsed deterministically.`, + }) + } + + return { + schemaVersion: 1, + provider: 'ahrefs', + capability: 'serp-competitors', + data: { keywords: requestedKeywords, rows, totalRows: null }, + observedAt: snapshots + .map((snapshot) => snapshot.observedAt) + .sort(compareCodepoints) + .at(-1) as string, + market: input.market, + coverage: resultCoverage, + cache: combinedCache(snapshots), + cost: combinedCost(snapshots), + request: { + operation: 'serp-competitors', + endpoint: new URL(ENDPOINT, AHREFS_API_BASE_URL).toString(), + limit: input.limit, + filters: { + apiVersion: 3, + country, + includeSubdomains: false, + keywordCount: requestedKeywords.length, + organicDepthPerKeyword: depth, + providerRequests: requestedKeywords.length, + resultTypes: 'organic', + selectedFields: SELECT, + }, + sort: [ + 'matchedKeywords:descending', + 'averagePosition:ascending', + 'domain:codepoint-ascending', + ], + }, + warnings, + } +} diff --git a/packages/core/src/providers/ahrefs/shared.ts b/packages/core/src/providers/ahrefs/shared.ts new file mode 100644 index 00000000..25d4018b --- /dev/null +++ b/packages/core/src/providers/ahrefs/shared.ts @@ -0,0 +1,484 @@ +import { randomUUID } from 'node:crypto' +import type { + MarketIndependentProviderEvidence, + ProviderCacheEvidence, + ProviderCostEvidence, + ProviderCoverage, + ProviderEvidence, + ProviderRequestContext, + ProviderValue, + ProviderWarning, + SearchMarket, +} from '../contracts.js' +import { observedValue, unavailableValue } from '../contracts.js' +import type { + OrganicFootprint, + RankingDistribution, +} from '../domain-contracts.js' +import type { DomainRatingTargetMode } from '../domain-rating-contracts.js' +import { ProviderError } from '../errors.js' +import type { LinkTargetScope, ProviderLinkMetric } from '../link-contracts.js' +import type { AhrefsApiSnapshot } from './client.js' + +export const AHREFS_API_BASE_URL = 'https://api.ahrefs.com/v3/' +export const AHREFS_MARKETS = [ + { + searchEngines: ['google'] as const, + devices: [] as const, + location: 'country-only' as const, + }, +] +export const MAX_AHREFS_ROWS = 1_000 +const FREE_TEST_VALUES = new Set([ + 'ahrefs', + 'ahrefs.com', + 'firehose', + 'firehose.com', + 'yep', + 'yep.com', +]) + +export function compareCodepoints(left: string, right: string): number { + return left < right ? -1 : left > right ? 1 : 0 +} + +export function apiDate(now: () => Date): string { + return now().toISOString().slice(0, 10) +} + +export function freeTestValue(value: string): boolean { + return FREE_TEST_VALUES.has(value.toLowerCase()) +} + +export function normalizedKeyword(value: string, operation: string): string { + const keyword = value.trim().replace(/\s+/gu, ' ').toLowerCase() + if ( + !keyword || + keyword.length > 80 || + keyword.split(/\s+/u).length > 10 || + keyword.includes(',') + ) { + throw new ProviderError({ + provider: 'ahrefs', + operation, + code: 'configuration', + message: + 'Ahrefs keywords must contain 1 to 80 characters, at most 10 words, and no commas.', + }) + } + return keyword +} + +export function marketCountry(market: SearchMarket, operation: string): string { + if (market.searchEngine !== 'google' || market.location || market.device) { + throw new ProviderError({ + provider: 'ahrefs', + operation, + code: 'configuration', + message: + 'Ahrefs research uses Google country-level data without a device or local location.', + }) + } + return market.countryCode.toLowerCase() +} + +export function marketWarnings(market: SearchMarket): ProviderWarning[] { + return [ + { + code: 'ahrefs-country-level-market', + field: 'market', + message: `Ahrefs used Google country-level data for ${market.countryCode}; the requested ${market.languageCode} language was retained as context but was not a separate API filter.`, + }, + ] +} + +export function organicOnly( + resultTypes: string[] | undefined, + operation: string, +): void { + const types = [...new Set(resultTypes ?? ['organic'])] + if (types.length !== 1 || types[0] !== 'organic') { + throw new ProviderError({ + provider: 'ahrefs', + operation, + code: 'configuration', + message: 'Ahrefs domain research currently supports organic rows only.', + }) + } +} + +function invalidTarget(operation: string): ProviderError { + return new ProviderError({ + provider: 'ahrefs', + operation, + code: 'configuration', + message: + 'Use a valid domain, or choose URL mode and pass an absolute HTTP or HTTPS URL.', + }) +} + +export function domain(value: string, operation: string): string { + const raw = value.trim() + if (!raw || raw.length > 2_048) throw invalidTarget(operation) + try { + const url = new URL(raw.includes('://') ? raw : `https://${raw}`) + const hostname = url.hostname + .toLowerCase() + .replace(/^www\./u, '') + .replace(/\.$/u, '') + if ( + !hostname || + hostname.length > 253 || + hostname.includes('..') || + !hostname.includes('.') || + !/^[a-z0-9.-]+$/u.test(hostname) + ) { + throw new Error() + } + return hostname + } catch { + throw invalidTarget(operation) + } +} + +export function target(input: { + value: string + mode: DomainRatingTargetMode | LinkTargetScope + operation: string +}): { target: string; mode: DomainRatingTargetMode } { + if (input.mode === 'domain') { + return { target: domain(input.value, input.operation), mode: 'domain' } + } + try { + const url = new URL(input.value.trim()) + if ( + !['http:', 'https:'].includes(url.protocol) || + url.username || + url.password || + input.value.length > 2_048 + ) { + throw new Error() + } + url.hash = '' + return { target: url.toString(), mode: 'url' } + } catch { + throw invalidTarget(input.operation) + } +} + +export function linkTarget( + value: string, + scope: LinkTargetScope = 'domain', +): { target: string; scope: LinkTargetScope; mode: string } { + const normalized = target({ + value, + mode: scope === 'page' ? 'url' : 'domain', + operation: 'link-evidence', + }) + return { + target: normalized.target, + scope, + mode: scope === 'page' ? 'exact' : 'subdomains', + } +} + +export function rowLimit( + limit: number, + offset: number | undefined, + operation: string, +): void { + if (!Number.isSafeInteger(limit) || limit < 1 || limit > MAX_AHREFS_ROWS) { + throw new ProviderError({ + provider: 'ahrefs', + operation, + code: 'configuration', + message: `Ahrefs row limits must be from 1 to ${MAX_AHREFS_ROWS}.`, + }) + } + if (offset !== undefined && offset !== 0) { + throw new ProviderError({ + provider: 'ahrefs', + operation, + code: 'configuration', + message: + 'Ahrefs API v3 does not support row offsets. Use a narrower bounded request.', + }) + } +} + +export function observedNumber(value: ProviderValue): number { + return value.state === 'observed' ? value.value : -1 +} + +export function organicFootprint(input: { + traffic?: number | null + keywords?: number | null + costCents?: number | null +}): OrganicFootprint { + return { + estimatedMonthlyTraffic: numberValue( + input.traffic, + 'estimated organic monthly traffic', + ), + rankedKeywords: numberValue(input.keywords, 'ranked organic keywords'), + estimatedMonthlyTrafficCostUsd: centsValue( + input.costCents, + 'estimated organic traffic cost', + ), + rankings: unavailable( + 'a complete organic ranking distribution', + ), + newRankings: unavailable('new rankings'), + improvedRankings: unavailable('improved rankings'), + declinedRankings: unavailable('declined rankings'), + lostRankings: unavailable('lost rankings'), + } +} + +export function dedupeBy(rows: T[], key: (row: T) => string): T[] { + const grouped = new Map() + for (const row of rows) { + const value = key(row) + grouped.set(value, [...(grouped.get(value) ?? []), row]) + } + return [...grouped.entries()] + .sort(([left], [right]) => compareCodepoints(left, right)) + .map( + ([, matches]) => + [...matches].sort((left, right) => + compareCodepoints(JSON.stringify(left), JSON.stringify(right)), + )[0] as T, + ) +} + +type SnapshotEvidence = Pick, 'cache' | 'cost'> + +export function combinedCache( + snapshots: SnapshotEvidence[], +): ProviderCacheEvidence { + if (snapshots.every((snapshot) => snapshot.cache.status === 'hit')) { + const stored = snapshots + .map((snapshot) => snapshot.cache.storedAt) + .filter((value): value is string => Boolean(value)) + .sort(compareCodepoints) + const expires = snapshots + .map((snapshot) => snapshot.cache.expiresAt) + .filter((value): value is string => Boolean(value)) + .sort(compareCodepoints) + return { + status: 'hit', + storedAt: stored[0] ?? null, + expiresAt: expires[0] ?? null, + } + } + return { + status: snapshots.every((snapshot) => snapshot.cache.status === 'bypass') + ? 'bypass' + : 'miss', + storedAt: null, + expiresAt: null, + } +} + +function sumNullable(values: Array): number | null { + return values.every((value) => value !== null) + ? values.reduce((sum, value) => sum + (value ?? 0), 0) + : null +} + +export function combinedCost( + snapshots: SnapshotEvidence[], +): ProviderCostEvidence { + const natives = snapshots.map((snapshot) => snapshot.cost.native) + const remaining = natives + .map((native) => native?.remainingBefore ?? null) + .filter((value): value is number => value !== null) + return { + currency: 'USD', + estimatedMicros: null, + actualMicros: null, + taskIds: [], + native: { + unit: 'api-unit', + estimatedUnits: sumNullable( + natives.map((native) => native?.estimatedUnits ?? null), + ), + actualUnits: sumNullable( + natives.map((native) => native?.actualUnits ?? null), + ), + remainingBefore: remaining.length ? Math.max(...remaining) : null, + }, + } +} + +export function requestContext( + reportId: string, + context: ProviderRequestContext | undefined, +): ProviderRequestContext { + return context ?? { reportId, reportRunId: randomUUID() } +} + +export function unavailable(field: string): ProviderValue { + return unavailableValue( + 'unavailable', + `The selected Ahrefs API fields do not return ${field}.`, + ) +} + +export function missing(field: string): ProviderValue { + return unavailableValue('missing', `Ahrefs omitted ${field}.`) +} + +export function numberValue( + value: number | null | undefined, + field: string, +): ProviderValue { + return value === null || value === undefined + ? missing(field) + : observedValue(value) +} + +export function centsValue( + value: number | null | undefined, + field: string, +): ProviderValue { + const number = numberValue(value, field) + return number.state === 'observed' + ? observedValue(number.value / 100) + : number +} + +export function safeUrl(value: string | null | undefined): string | null { + if (!value) return null + try { + const url = new URL(value) + if (!['http:', 'https:'].includes(url.protocol)) return null + url.username = '' + url.password = '' + url.hash = '' + return url.toString() + } catch { + return null + } +} + +export function normalizedDate( + value: string | null | undefined, +): string | null { + if (!value) return null + const parsed = Date.parse(value) + return Number.isFinite(parsed) ? new Date(parsed).toISOString() : null +} + +export function metric( + id: string, + label: string, + value: number | null | undefined, +): ProviderLinkMetric[] { + return value === null || value === undefined + ? [] + : [ + { + provider: 'ahrefs', + id, + label, + value, + scale: { minimum: 0, maximum: 100 }, + }, + ] +} + +export function coverage(input: { + requestedRows: number + returnedRows: number + retainedRows: number + invalidRows: number + filtered?: boolean +}): ProviderCoverage { + const capped = input.returnedRows >= input.requestedRows + return { + requestedRows: input.requestedRows, + returnedRows: input.returnedRows, + retainedRows: input.retainedRows, + invalidRows: input.invalidRows, + providerTotalRows: null, + completeness: + input.invalidRows > 0 + ? 'partial' + : capped + ? 'capped' + : input.filtered + ? 'filtered' + : 'complete', + nextCursor: null, + } +} + +export function evidence(input: { + capability: ProviderEvidence['capability'] + data: T + market: SearchMarket + snapshot: AhrefsApiSnapshot + coverage: ProviderCoverage + endpoint: string + limit: number + filters: Record + sort: string[] + warnings?: ProviderWarning[] +}): ProviderEvidence { + return { + schemaVersion: 1, + provider: 'ahrefs', + capability: input.capability, + data: input.data, + observedAt: input.snapshot.observedAt, + market: input.market, + coverage: input.coverage, + cache: input.snapshot.cache, + cost: input.snapshot.cost, + request: { + operation: input.capability, + endpoint: new URL(input.endpoint, AHREFS_API_BASE_URL).toString(), + limit: input.limit, + filters: input.filters, + sort: input.sort, + }, + warnings: [ + ...input.snapshot.warnings, + ...marketWarnings(input.market), + ...(input.warnings ?? []), + ], + } +} + +export function marketIndependentEvidence(input: { + capability: MarketIndependentProviderEvidence['capability'] + data: T + snapshot: AhrefsApiSnapshot + coverage: ProviderCoverage + endpoint: string + limit: number + filters: Record + sort: string[] + warnings?: ProviderWarning[] +}): MarketIndependentProviderEvidence { + return { + schemaVersion: 1, + provider: 'ahrefs', + capability: input.capability, + data: input.data, + observedAt: input.snapshot.observedAt, + market: null, + coverage: input.coverage, + cache: input.snapshot.cache, + cost: input.snapshot.cost, + request: { + operation: input.capability, + endpoint: new URL(input.endpoint, AHREFS_API_BASE_URL).toString(), + limit: input.limit, + filters: input.filters, + sort: input.sort, + }, + warnings: [...input.snapshot.warnings, ...(input.warnings ?? [])], + } +} diff --git a/packages/core/src/providers/domain-rating-contracts.ts b/packages/core/src/providers/domain-rating-contracts.ts new file mode 100644 index 00000000..0df7fcae --- /dev/null +++ b/packages/core/src/providers/domain-rating-contracts.ts @@ -0,0 +1,30 @@ +import type { + MarketIndependentProviderEvidence, + ProviderAdapter, + ProviderRequestContext, + ProviderValue, +} from './contracts.js' + +export type DomainRatingTargetMode = 'domain' | 'url' + +export type DomainRatingObservation = { + target: string + targetMode: DomainRatingTargetMode + domainRating: ProviderValue + licenseUrl: string + attribution: 'Domain Rating by Ahrefs' + attributionUrl: 'https://ahrefs.com/' +} + +export type DomainRatingRequest = { + target: string + targetMode?: DomainRatingTargetMode + refresh?: boolean + context?: ProviderRequestContext +} + +export interface DomainRatingProvider extends ProviderAdapter { + domainRating( + input: DomainRatingRequest, + ): Promise> +} diff --git a/packages/core/src/providers/transport.test.ts b/packages/core/src/providers/transport.test.ts index 950c2b37..be1ce401 100644 --- a/packages/core/src/providers/transport.test.ts +++ b/packages/core/src/providers/transport.test.ts @@ -138,3 +138,25 @@ test('provider transport retries only explicitly safe operations', async () => { ) assert.equal(chargedAttempts, 1) }) + +test('provider transport exposes successful response metadata', async () => { + let observedCost = '' + const result = await providerRequestJson({ + provider: 'ahrefs', + operation: 'metadata', + url: 'https://example.test', + fetch: async () => + new Response('{"ok":true}', { + headers: { 'x-api-units-cost-total-actual': '50' }, + }), + maxResponseBytes: 1_024, + timeoutMs: 1_000, + schema: z.object({ ok: z.literal(true) }), + onResponse: (response) => { + observedCost = response.headers.get('x-api-units-cost-total-actual') ?? '' + }, + }) + + assert.deepEqual(result, { ok: true }) + assert.equal(observedCost, '50') +}) diff --git a/packages/core/src/providers/transport.ts b/packages/core/src/providers/transport.ts index 2f7d0e8d..3b6f78d1 100644 --- a/packages/core/src/providers/transport.ts +++ b/packages/core/src/providers/transport.ts @@ -23,6 +23,7 @@ export type ProviderRequestInput = { timeoutMs: number retry?: 'never' | 'safe' retryDelayMs?: number + onResponse?: (response: Response) => void } function schemaIssueSummary(error: z.ZodError): string { @@ -132,6 +133,7 @@ async function requestOnce(input: ProviderRequestInput): Promise { await response.body?.cancel().catch(() => undefined) throw httpError(input, response.status) } + input.onResponse?.(response) const text = await readBoundedResponseText( response, input.maxResponseBytes, diff --git a/packages/core/src/storage/database.ts b/packages/core/src/storage/database.ts index 61eba525..3f73f842 100644 --- a/packages/core/src/storage/database.ts +++ b/packages/core/src/storage/database.ts @@ -455,6 +455,7 @@ export function getCacheStats(): CacheStats { .get() as { count: number }, semrush_cache: { count: legacySemrushCount + providerCount('semrush') }, provider_cache: { count: providerCount('dataforseo') }, + ahrefs_cache: { count: providerCount('ahrefs') }, http_cache: database .prepare('SELECT COUNT(*) AS count FROM http_cache') .get() as { count: number }, @@ -482,13 +483,23 @@ function databaseFootprint(path: string): number { } export function clearCache( - provider?: 'gsc' | 'google-analytics' | 'semrush' | 'dataforseo' | 'http', + provider?: + | 'gsc' + | 'google-analytics' + | 'semrush' + | 'dataforseo' + | 'ahrefs' + | 'http', olderThanMs?: number, ): number { const database = getDb() const cutoff = olderThanMs ? Date.now() - olderThanMs : undefined - if (provider === 'dataforseo' || provider === 'semrush') { + if ( + provider === 'dataforseo' || + provider === 'semrush' || + provider === 'ahrefs' + ) { const sql = cutoff ? 'DELETE FROM provider_cache WHERE provider = ? AND fetched_at < ?' : 'DELETE FROM provider_cache WHERE provider = ?' diff --git a/packages/core/src/storage/provider-secrets.ts b/packages/core/src/storage/provider-secrets.ts index eb1b66ea..ddd919a1 100644 --- a/packages/core/src/storage/provider-secrets.ts +++ b/packages/core/src/storage/provider-secrets.ts @@ -13,6 +13,7 @@ const KEYRING_SERVICE = 'seo' const PRIVATE_FILE_MODE = 0o600 export const PROVIDER_SECRET_NAMES = { + ahrefsApiKey: 'ahrefs-api-key', bingApiKey: 'bing-api-key', dataForSeoCredentials: 'dataforseo-credentials', indexNowKeys: 'indexnow-keys', diff --git a/packages/core/src/telemetry.ts b/packages/core/src/telemetry.ts index bf8cfa8b..c8f3bfc1 100644 --- a/packages/core/src/telemetry.ts +++ b/packages/core/src/telemetry.ts @@ -57,6 +57,7 @@ export const TELEMETRY_REPORTS = [ 'ctr-underperformers', 'decaying-pages', 'domain-overview', + 'domain-rating', 'setup-check', 'entity-readiness', 'explain-crawl-issue', diff --git a/packages/mcp/src/discovery-tools.test.ts b/packages/mcp/src/discovery-tools.test.ts index 1f3ef785..3f4c79ba 100644 --- a/packages/mcp/src/discovery-tools.test.ts +++ b/packages/mcp/src/discovery-tools.test.ts @@ -76,6 +76,7 @@ test('report catalog is stable, sorted, and excludes raw or mutable tools', () = 'ctr-underperformers', 'decaying-pages', 'domain-overview', + 'domain-rating', 'entity-readiness', 'explain-crawl-issue', 'generate-llms-txt', diff --git a/packages/mcp/src/provider-tools.ts b/packages/mcp/src/provider-tools.ts index a098ed02..fcd96dcf 100644 --- a/packages/mcp/src/provider-tools.ts +++ b/packages/mcp/src/provider-tools.ts @@ -1,6 +1,7 @@ import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js' import { bingWebmasterOverview, + collectAhrefsLinkEvidence, collectBingLinkEvidence, collectDataForSeoLinkEvidence, importLinkEvidence, @@ -90,10 +91,10 @@ export function registerProviderTools(server: McpServer): void { 'seo_link_evidence', { description: - 'Review bounded referring-link evidence from DataForSEO, Bing Webmaster or a local export', + 'Review bounded referring-link evidence from Ahrefs, DataForSEO, Bing Webmaster or a local export', inputSchema: { site: z.string().url().max(2_000).optional(), - provider: z.enum(['dataforseo', 'bing']).optional(), + provider: z.enum(['ahrefs', 'dataforseo', 'bing']).optional(), target: z.string().trim().min(1).max(2_048).optional(), scope: z.enum(['domain', 'page']).optional(), includeSubdomains: z.boolean().optional(), @@ -133,7 +134,7 @@ export function registerProviderTools(server: McpServer): void { if (sourceCount !== 1) { throw new SeoError( 'INVALID_INPUT', - 'Pass one link source: file, site for Bing, or target for DataForSEO.', + 'Pass one link source: file, site for Bing, or target for Ahrefs or DataForSEO.', ) } if (file && provider) { @@ -142,10 +143,10 @@ export function registerProviderTools(server: McpServer): void { 'Do not pass provider with a local link file.', ) } - if (liveProvider === 'dataforseo' && !target) { + if (['ahrefs', 'dataforseo'].includes(liveProvider) && !target) { throw new SeoError( 'INVALID_INPUT', - 'Pass target for DataForSEO link evidence.', + 'Pass target for Ahrefs or DataForSEO link evidence.', ) } if (liveProvider === 'bing' && !site && !file) { @@ -156,22 +157,30 @@ export function registerProviderTools(server: McpServer): void { } const evidence = file ? await importLinkEvidence({ file, format, rowLimit }) - : liveProvider === 'dataforseo' - ? await collectDataForSeoLinkEvidence({ + : liveProvider === 'ahrefs' + ? await collectAhrefsLinkEvidence({ target: target ?? '', scope, includeSubdomains, rowLimit, refresh, }) - : await collectBingLinkEvidence({ - site: site ?? '', - rowLimit, - targetLimit, - detailPagesPerTarget, - }) + : liveProvider === 'dataforseo' + ? await collectDataForSeoLinkEvidence({ + target: target ?? '', + scope, + includeSubdomains, + rowLimit, + refresh, + }) + : await collectBingLinkEvidence({ + site: site ?? '', + rowLimit, + targetLimit, + detailPagesPerTarget, + }) const context = - searchConsoleSite || evidence.provenance.provider === 'dataforseo' + searchConsoleSite || evidence.externalProvider ? await linkTargetContext({ evidence, searchConsoleSite, diff --git a/packages/mcp/src/report-contracts.test.ts b/packages/mcp/src/report-contracts.test.ts index ed5f30ae..f9ff5df4 100644 --- a/packages/mcp/src/report-contracts.test.ts +++ b/packages/mcp/src/report-contracts.test.ts @@ -126,6 +126,16 @@ test('link evidence bounds provider work, imports, and returned rows', () => { }).success, true, ) + assert.equal( + schema.safeParse({ + provider: 'ahrefs', + target: 'example.com', + scope: 'domain', + rowLimit: 100, + limit: 100, + }).success, + true, + ) for (const input of [ { file: './links.csv', rowLimit: 100_001 }, { file: './links.csv', limit: 501 }, diff --git a/packages/mcp/src/report-definitions/domain-rating.test.ts b/packages/mcp/src/report-definitions/domain-rating.test.ts new file mode 100644 index 00000000..21e45902 --- /dev/null +++ b/packages/mcp/src/report-definitions/domain-rating.test.ts @@ -0,0 +1,64 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { + createDomainRatingHandler, + domainRatingInputSchema, +} from './domain-rating.js' + +test('Domain Rating report forwards one provider-neutral bounded request', async () => { + const handler = createDomainRatingHandler({ + domainRatingReport: async (input) => { + assert.deepEqual(input, { + target: 'example.com', + targetMode: 'domain', + provider: 'ahrefs', + refresh: true, + }) + return { + summary: { verdict: 'Domain Rating evidence retained.' }, + } as never + }, + }) + + const result = await handler({ + target: 'example.com', + targetMode: 'domain', + provider: 'ahrefs', + refresh: true, + }) + assert.equal(result.isError, undefined) + assert.equal( + result.structuredContent?.summary && + (result.structuredContent.summary as Record).verdict, + 'Domain Rating evidence retained.', + ) +}) + +test('Domain Rating schema accepts domain and URL targets only for research providers', () => { + for (const input of [ + { target: 'example.com' }, + { + target: 'https://example.com/page', + targetMode: 'url', + provider: 'ahrefs', + }, + ]) { + assert.equal( + domainRatingInputSchema.safeParse(input).success, + true, + JSON.stringify(input), + ) + } + for (const input of [ + { target: '' }, + { target: 'example.com', targetMode: 'prefix' }, + { target: 'example.com', provider: 'bing' }, + { target: 'example.com', unexpected: true }, + ]) { + assert.equal( + domainRatingInputSchema.safeParse(input).success, + false, + JSON.stringify(input), + ) + } +}) diff --git a/packages/mcp/src/report-definitions/domain-rating.ts b/packages/mcp/src/report-definitions/domain-rating.ts new file mode 100644 index 00000000..b920dea3 --- /dev/null +++ b/packages/mcp/src/report-definitions/domain-rating.ts @@ -0,0 +1,33 @@ +import { domainRatingReport } from '@seo/core' +import * as z from 'zod/v4' +import { compactAgentWorkflowOutput } from '../agent-output-budget.js' +import { providerIdInput } from '../provider-inputs.js' +import { type ToolResult, toolError, toolSuccess } from '../tool-result.js' + +export const domainRatingInputSchema = z.strictObject({ + target: z.string().trim().min(3).max(2_048), + targetMode: z.enum(['domain', 'url']).default('domain'), + provider: providerIdInput.optional(), + refresh: z.boolean().optional(), +}) + +export function createDomainRatingHandler( + dependencies: { domainRatingReport?: typeof domainRatingReport } = {}, +): (input: Record) => Promise { + return async (input) => { + const parsed = domainRatingInputSchema.parse(input) + try { + const report = await ( + dependencies.domainRatingReport ?? domainRatingReport + )(parsed) + return toolSuccess( + report.summary.verdict, + compactAgentWorkflowOutput( + report as unknown as Record, + ), + ) + } catch (error) { + return toolError(error) + } + } +} diff --git a/packages/mcp/src/report-depth-continued.ts b/packages/mcp/src/report-depth-continued.ts index 257098a7..a9f8e943 100644 --- a/packages/mcp/src/report-depth-continued.ts +++ b/packages/mcp/src/report-depth-continued.ts @@ -107,6 +107,25 @@ export const REPORT_DEPTH_CONTINUED = { }, ], }, + 'domain-rating': { + readOrder: [ + 'dataStatus, target, targetMode, observedAt, cache, coverage, and warnings', + 'evidence.data.domainRating, licenseUrl, attribution, and attributionUrl', + 'caveats and nextSteps before using the value in a comparison', + ], + doNotClaim: [ + 'Domain Rating is an Ahrefs backlink-profile estimate, not a Google metric, ranking factor, traffic estimate, or keyword-difficulty score.', + 'A lower Domain Rating does not by itself mean a result, page, or keyword is easy to outrank.', + 'Missing or unavailable Domain Rating is not zero.', + ], + verify: + 'Inspect the current result page, page relevance, URL-level link evidence, and your own site evidence before making a competitive decision.', + related: [ + { id: 'link-evidence', reason: 'Adds bounded referring-link evidence.' }, + { id: 'serp-results', reason: 'Checks a current result snapshot.' }, + { id: 'domain-overview', reason: 'Adds search-footprint estimates.' }, + ], + }, 'ranked-keywords': { readOrder: [ 'dataStatus, market, coverage, request filters, cache, cost, and warnings', diff --git a/packages/mcp/src/report-guidance-domain-research.ts b/packages/mcp/src/report-guidance-domain-research.ts index e391e6b5..bceef0b9 100644 --- a/packages/mcp/src/report-guidance-domain-research.ts +++ b/packages/mcp/src/report-guidance-domain-research.ts @@ -32,6 +32,21 @@ export const REPORT_GUIDANCE_DOMAIN_RESEARCH = { outcome: 'A bounded domain estimate with ranking distribution, source coverage, cost, and an optional Search Console comparison.', }, + 'domain-rating': { + name: 'Ahrefs Domain Rating', + description: + 'Retrieve the current Ahrefs backlink-profile estimate for one domain or URL with its required license and attribution.', + useWhen: [ + 'You need one provider-native backlink-profile metric as supporting evidence.', + 'You want the free Ahrefs value before deciding whether deeper paid link research is useful.', + ], + avoidWhen: [ + 'You want a Google ranking factor, traffic estimate, keyword-difficulty score, or an easy-to-outrank verdict.', + 'You need page relevance, current result positions, or complete backlink evidence.', + ], + outcome: + 'One explicitly attributed Ahrefs Domain Rating observation with cache, coverage, license, and interpretation limits.', + }, 'ranked-keywords': { name: 'Ranked keyword footprint', description: diff --git a/packages/mcp/src/report-registry.ts b/packages/mcp/src/report-registry.ts index 27ba5f0d..7c5246e1 100644 --- a/packages/mcp/src/report-registry.ts +++ b/packages/mcp/src/report-registry.ts @@ -18,6 +18,10 @@ import { aiPromptObservationsInputSchema, createAiPromptObservationsHandler, } from './report-definitions/ai-prompt-observations.js' +import { + createDomainRatingHandler, + domainRatingInputSchema, +} from './report-definitions/domain-rating.js' import { createLocalSearchDemandHandler, localSearchDemandInputSchema, @@ -69,6 +73,12 @@ type DirectReport = { } const directReports: readonly DirectReport[] = [ + { + id: 'domain-rating', + category: 'opportunities', + inputSchema: domainRatingInputSchema, + handler: createDomainRatingHandler(), + }, { id: 'ai-mention-research', category: 'ai-search', diff --git a/scripts/provider-resource-ahrefs.mjs b/scripts/provider-resource-ahrefs.mjs new file mode 100644 index 00000000..34b56d5a --- /dev/null +++ b/scripts/provider-resource-ahrefs.mjs @@ -0,0 +1,147 @@ +import assert from 'node:assert/strict' +import { Response } from 'undici' +import { z } from 'zod' + +const BATCHES = 10 +const ROWS_PER_BATCH = 1_000 +const MAX_DURATION_MS = 10_000 + +export async function runAhrefsResourceHarness({ mebibyte }) { + const { AhrefsClient, clearCache, getCacheStats } = await import( + '../dist/index.js' + ) + const maxRssGrowth = 256 * mebibyte + const maxOutputBytes = 3 * mebibyte + const schema = z + .object({ + rows: z + .array( + z + .object({ + keyword: z.string(), + fixture_payload: z.string(), + }) + .strict(), + ) + .max(ROWS_PER_BATCH), + }) + .strict() + let limitsCalls = 0 + let reportCalls = 0 + let bytesRead = 0 + const startingCache = getCacheStats() + const client = new AhrefsClient({ + apiKey: 'resource-test-api-key', + spendLimits: { + dailyNoticeMicros: 0, + dailyHardLimitMicros: null, + monthlyHardLimitMicros: null, + maxRequestsPerReport: BATCHES, + maxRowsPerReport: BATCHES * ROWS_PER_BATCH, + }, + fetch: async (url) => { + const requestUrl = new URL(String(url)) + if (requestUrl.pathname.endsWith('/limits-and-usage')) { + limitsCalls += 1 + const json = JSON.stringify({ + limits_and_usage: { + api_key_expiration_date: '2027-07-24T00:00:00Z', + subscription: 'Resource fixture', + units_limit_api_key: 100_000, + units_limit_workspace: 100_000, + units_usage_api_key: reportCalls * ROWS_PER_BATCH, + units_usage_workspace: reportCalls * ROWS_PER_BATCH, + usage_reset_date: '2026-08-22T00:00:00Z', + }, + }) + bytesRead += Buffer.byteLength(json) + return new Response(json) + } + reportCalls += 1 + const json = JSON.stringify({ + rows: Array.from({ length: ROWS_PER_BATCH }, (_, index) => ({ + keyword: `bounded keyword ${reportCalls}-${index}`, + fixture_payload: 'x'.repeat(2_048), + })), + }) + bytesRead += Buffer.byteLength(json) + return new Response(json, { + headers: { + 'x-api-rows': String(ROWS_PER_BATCH), + 'x-api-units-cost-row': '1', + 'x-api-units-cost-total': String(ROWS_PER_BATCH), + 'x-api-units-cost-total-actual': String(ROWS_PER_BATCH), + 'x-api-cache': 'miss', + }, + }) + }, + }) + const request = (batch) => ({ + operation: 'ranked-keywords', + capability: 'ranked-keywords', + path: 'site-explorer/organic-keywords', + query: { + target: `domain-${batch}.example`, + limit: ROWS_PER_BATCH, + }, + schema, + requestedRows: ROWS_PER_BATCH, + perRowUnits: 1, + rowCount: (response) => response.rows.length, + context: { + reportId: 'provider-resource-harness', + reportRunId: 'ahrefs-bounded-run', + }, + }) + + const baselineRss = process.memoryUsage().rss + const startedAt = performance.now() + let lastSnapshot + for (let batch = 0; batch < BATCHES; batch += 1) { + lastSnapshot = await client.request(request(batch)) + } + const cached = await client.request(request(0)) + assert.equal(cached.cache.status, 'hit') + await assert.rejects( + client.request(request(BATCHES)), + (error) => error?.code === 'budget-limit', + ) + + const durationMs = performance.now() - startedAt + const rssGrowthBytes = Math.max(0, process.memoryUsage().rss - baselineRss) + const outputBytes = Buffer.byteLength(JSON.stringify(lastSnapshot)) + const cacheStats = getCacheStats() + const cacheBytesWritten = Math.max( + 0, + cacheStats.logicalSizeBytes - startingCache.logicalSizeBytes, + ) + const diskBytesWritten = Math.max( + 0, + cacheStats.sizeBytes - startingCache.sizeBytes, + ) + console.log( + JSON.stringify({ + provider: 'ahrefs', + requestedRows: BATCHES * ROWS_PER_BATCH, + limitsCalls, + paidCalls: reportCalls, + durationMs: Math.round(durationMs), + rssGrowthMiB: Number((rssGrowthBytes / mebibyte).toFixed(1)), + bytesRead, + cacheBytesWritten, + diskBytesWritten, + outputBytes, + estimatedApiUnits: lastSnapshot?.cost.native?.estimatedUnits ?? null, + actualApiUnits: lastSnapshot?.cost.native?.actualUnits ?? null, + }), + ) + assert.equal(reportCalls, BATCHES) + assert.equal(limitsCalls, BATCHES) + assert.equal(cacheStats.counts.ahrefs_cache, BATCHES) + assert.ok(durationMs <= MAX_DURATION_MS) + assert.ok(rssGrowthBytes <= maxRssGrowth) + assert.ok(outputBytes <= maxOutputBytes) + assert.ok(cacheStats.logicalSizeBytes <= cacheStats.maxSizeBytes) + assert.equal(clearCache('ahrefs'), BATCHES) + assert.equal(getCacheStats().counts.ahrefs_cache, 0) +} diff --git a/scripts/provider-resource-harness.mjs b/scripts/provider-resource-harness.mjs index 82db57aa..a1eb2cbd 100644 --- a/scripts/provider-resource-harness.mjs +++ b/scripts/provider-resource-harness.mjs @@ -3,6 +3,7 @@ import { mkdtempSync, rmSync, statSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { Response } from 'undici' +import { runAhrefsResourceHarness } from './provider-resource-ahrefs.mjs' import { runSemrushResourceHarness } from './provider-resource-semrush.mjs' const MEBIBYTE = 1024 * 1024 @@ -965,6 +966,7 @@ try { assert.equal(clearCache('dataforseo'), BATCHES) assert.equal(getCacheStats().counts.provider_cache, 0) await runSemrushResourceHarness({ mebibyte: MEBIBYTE }) + await runAhrefsResourceHarness({ mebibyte: MEBIBYTE }) } finally { rmSync(cacheDir, { recursive: true, force: true }) } diff --git a/skills/seo/SKILL.md b/skills/seo/SKILL.md index d5c3490d..0381ff07 100644 --- a/skills/seo/SKILL.md +++ b/skills/seo/SKILL.md @@ -63,7 +63,7 @@ Run the first report, read it, then decide. Do not run a whole chain blindly. | Catch regressions over time | `technical-watch`, `crawl-diff`, `index-watch`, `measure-change` after a fix ships | | Track exact keyword positions | `rank-tracking` for a saved set and fixed market/device; `serp-results` for one current query | | Review Bing traffic, crawl, query, and page insights | `bing-webmaster-overview`, then `site-crawl` when live page evidence is needed | -| Review referring links and linked targets | `link-evidence`, then verify selected referring URLs and flagged targets directly | +| Review backlink context and linked targets | `domain-rating`, `link-evidence`, then verify selected results, referring URLs, and flagged targets directly | | Review real crawler requests in a server log | `server-log-analysis`, then verify important errors against the original log and server configuration | | Client-ready reporting | `monthly-report`, `narrative-report`, `monthly-action-plan` | | Turn crawl findings into tickets | `top-fixes`, `affected-urls`, `explain-crawl-issue` | @@ -126,6 +126,7 @@ recommend a User-Agent-only bypass. Use `seo help all` for direct provider and administration commands. Bing setup uses `seo providers bing`; link evidence uses `seo links --project --json`, +`seo links --provider ahrefs --target --json`, `seo links --provider dataforseo --target --json`, or a local file. IndexNow writes externally: validate with `seo indexnow submit --dry-run --json` and remove dry run only when authorised.