diff --git a/apps/web/src/content/docs-nav.ts b/apps/web/src/content/docs-nav.ts index 21f96308..0c56bd92 100644 --- a/apps/web/src/content/docs-nav.ts +++ b/apps/web/src/content/docs-nav.ts @@ -28,7 +28,13 @@ export const docsNav: DocsNavEntry[] = [ path: '/docs/research-providers', label: 'Research providers', description: - 'Connect DataForSEO locally, control paid work and combine keyword, domain, competitor and page estimates with first-party evidence.', + 'Connect optional providers locally, control paid work and combine keyword, domain, competitor and page estimates with first-party evidence.', + }, + { + path: '/docs/semrush', + label: 'Semrush', + description: + 'Connect the permanent Semrush Version 3 key and run bounded keyword, domain, ranking-page and competitor research.', }, { path: '/docs/indexnow', diff --git a/apps/web/src/content/docs/docs/research-providers.mdx b/apps/web/src/content/docs/docs/research-providers.mdx index f64e6d66..1fae85ff 100644 --- a/apps/web/src/content/docs/docs/research-providers.mdx +++ b/apps/web/src/content/docs/docs/research-providers.mdx @@ -8,10 +8,14 @@ providers add keyword estimates, result snapshots, domain footprints, ranking pages and competitor rows. Search Console still supplies owner-verified search performance. Crawls still supply current page and technical evidence. -DataForSEO supports the current research reports. The report contracts are -provider neutral, so another adapter can support the same capability later -without changing how an agent reads the result. A report fails clearly when a -selected provider does not support its operation or market. +DataForSEO supports the broadest set of current research reports. Semrush +Version 3 supports bounded keyword, domain, ranking-page and search-competitor +research. The report contracts are provider neutral, so both adapters return +the same evidence shape where their capabilities overlap. A report fails +clearly when a selected provider does not support its operation or market. + +Use the [Semrush guide](/docs/semrush) for the exact key, connection commands, +supported reports and API-unit behavior. ## Connect DataForSEO on this computer diff --git a/apps/web/src/content/docs/docs/semrush.mdx b/apps/web/src/content/docs/docs/semrush.mdx new file mode 100644 index 00000000..3a0512d9 --- /dev/null +++ b/apps/web/src/content/docs/docs/semrush.mdx @@ -0,0 +1,122 @@ +--- +title: Semrush research +description: Connect the permanent Semrush Version 3 API key and run bounded keyword, domain, ranking-page and competitor research. +--- + +Use Semrush estimates without replacing evidence from your own site. The +adapter supplies six existing research reports with provider-native values, +coverage, cache state and API-unit cost. It does not add a Semrush-specific MCP +tool or agent skill. + +## Use the permanent Version 3 key + +Open My profile, choose API Keys, and copy the row named **V3 API Key**. Semrush +generates one permanent Version 3 key for each account, so there is no button +to create another one. + +This integration accepts only that Version 3 key. It does not accept Version 4 +keys. Semrush keeps the versions, keys and supported endpoints separate; its +API version guide +explains the distinction. + +Connect it through the masked terminal prompt: + +```sh +seo providers semrush connect +seo providers semrush status --check +``` + +The connection check requests the remaining API-unit balance and costs no +units. The key is saved in the system keychain when available, with a private +local file fallback. It is never stored in a project profile, report, cache +entry or structured error. + +The +Semrush balance guide +describes the free check and how report rows consume API units. + +Remove the saved key with: + +```sh +seo providers semrush disconnect +``` + +Agents and CI can supply the same Version 3 key without saving it: + +```sh +SEO_SEMRUSH_API_KEY='your-version-3-key' \ + seo providers semrush status --check --json +``` + +Keep the value in the platform secret manager. Do not put it in a repository, +script, report parameter, command-line flag or issue. + +## Run the shared research reports + +Choose Semrush through the normal report input. The same report ids, schemas +and evidence rules are used for every supported provider. + +```sh +seo reports run keyword-metrics \ + --params '{"keywords":["technical seo","seo audit"],"countryCode":"GB","languageCode":"en","searchEngine":"google","provider":"semrush"}' \ + --json + +seo reports run keyword-research \ + --params '{"seeds":["technical seo"],"sources":["ideas"],"countryCode":"GB","languageCode":"en","searchEngine":"google","limit":10,"provider":"semrush"}' \ + --json + +seo reports run domain-overview \ + --params '{"domain":"example.com","countryCode":"GB","languageCode":"en","searchEngine":"google","provider":"semrush"}' \ + --json +``` + +The connected adapter supports: + +| Report | Evidence supplied | +| --- | --- | +| `keyword-metrics` | Search volume, cost per click, paid competition, result count, intent and keyword difficulty when Semrush returns them. | +| `keyword-research` | Bounded ideas, related terms and question keywords from one to five seeds. | +| `domain-overview` | Provider-estimated organic keyword count, traffic and traffic cost for one domain. | +| `ranked-keywords` | A bounded set of observed terms, ranking URLs, positions and optional keyword metrics. | +| `ranking-pages` | Ranking pages and repeated URL patterns derived from bounded provider rows. | +| `serp-competitors` | Domains repeatedly observed for an explicit keyword set. | + +Use `seo reports describe --json` before scripting a report. It +returns the current input schema, reading order, caveats and related reports. + +## Keep requests bounded + +Every cache miss checks the free balance before requesting paid data. The +adapter sets the provider row limit before acquisition, rejects unbounded +inputs, and records estimated and returned API units in structured evidence. +If the balance cannot cover the maximum request, the paid call does not start. + +Results are cached locally for seven days by default. A cache hit costs no API +units and makes no provider request. Pass `"refresh":true` only when the +decision needs a newer observation. + +Semrush Version 3 research uses Google desktop data from a country-level +regional database. It does not support a city, postcode, Bing or mobile market +through this adapter. The requested language remains visible in the report, +but Semrush does not apply it as a separate language filter. + +## Read the caveats before acting + +Semrush metrics are external estimates. They are not Search Console +impressions, measured visits, a complete keyword inventory or a forecast. +Keyword difficulty is a provider metric, not a ranking probability. + +Semrush marks its Version 3 keyword endpoints as deprecated. The reports keep +that warning in their evidence. The +Version 3 keyword reference +says existing integrations remain available temporarily, so a successful +request does not promise permanent endpoint availability. + +Run the main first-party report before paid research: + +```sh +seo report --project example +``` + +Use a Semrush report when it answers a specific gap, then verify an important +term against Search Console and a current result page in the same market. diff --git a/package.json b/package.json index d5f84947..25185494 100644 --- a/package.json +++ b/package.json @@ -77,8 +77,9 @@ "security:check": "pnpm run security:audit && pnpm run security:secrets", "security:secrets": "node scripts/security-secrets.mjs", "skills:validate": "node scripts/validate-skills.mjs", - "test": "turbo run test && pnpm run build:package && node --test scripts/package.test.mjs scripts/dogfood-summary.test.mjs && node scripts/resource-harness.mjs && node scripts/provider-resource-harness.mjs && node scripts/pseo-resource-harness.mjs && node scripts/indexnow-resource-harness.mjs && node scripts/bing-resource-harness.mjs && pnpm run skills:validate", - "test:built": "turbo run test && node --test scripts/package.test.mjs scripts/dogfood-summary.test.mjs && node scripts/resource-harness.mjs && node scripts/provider-resource-harness.mjs && node scripts/pseo-resource-harness.mjs && node scripts/indexnow-resource-harness.mjs && node scripts/bing-resource-harness.mjs && pnpm run skills:validate", + "test": "turbo run test && pnpm run build:package && node --test scripts/package.test.mjs scripts/dogfood-summary.test.mjs scripts/semrush-live-acceptance.test.mjs && node scripts/resource-harness.mjs && node scripts/provider-resource-harness.mjs && node scripts/pseo-resource-harness.mjs && node scripts/indexnow-resource-harness.mjs && node scripts/bing-resource-harness.mjs && pnpm run skills:validate", + "test:built": "turbo run test && node --test scripts/package.test.mjs scripts/dogfood-summary.test.mjs scripts/semrush-live-acceptance.test.mjs && node scripts/resource-harness.mjs && node scripts/provider-resource-harness.mjs && node scripts/pseo-resource-harness.mjs && node scripts/indexnow-resource-harness.mjs && node scripts/bing-resource-harness.mjs && pnpm run skills:validate", + "test:semrush-live": "pnpm run build:package && node scripts/semrush-live-acceptance.mjs", "test:resources": "pnpm run build:package && node scripts/resource-harness.mjs && node scripts/provider-resource-harness.mjs && node scripts/pseo-resource-harness.mjs && node scripts/indexnow-resource-harness.mjs && node scripts/bing-resource-harness.mjs", "test:rank-resources": "pnpm run build:package && node scripts/rank-tracking-resource-harness.mjs", "test:package-install": "pnpm --filter @seo/core build && pnpm --filter @seo/mcp build && pnpm run build:package && node --test scripts/installed-package.test.mjs", diff --git a/packages/cli/src/commands/providers/index.ts b/packages/cli/src/commands/providers/index.ts index 50ad7e07..09e73961 100644 --- a/packages/cli/src/commands/providers/index.ts +++ b/packages/cli/src/commands/providers/index.ts @@ -1,11 +1,13 @@ import { defineCommand } from 'citty' import { bingProviderCommand } from './bing.js' import { dataForSeoProviderCommand } from './dataforseo.js' +import { semrushProviderCommand } from './semrush.js' export const providersCommand = defineCommand({ meta: { name: 'providers', description: 'Connect optional data providers' }, subCommands: { bing: bingProviderCommand, dataforseo: dataForSeoProviderCommand, + semrush: semrushProviderCommand, }, }) diff --git a/packages/cli/src/commands/providers/semrush.test.ts b/packages/cli/src/commands/providers/semrush.test.ts new file mode 100644 index 00000000..f4681ae4 --- /dev/null +++ b/packages/cli/src/commands/providers/semrush.test.ts @@ -0,0 +1,154 @@ +import assert from 'node:assert/strict' +import { execFile } from 'node:child_process' +import { mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { test } from 'node:test' +import { fileURLToPath } from 'node:url' +import { promisify } from 'node:util' + +const execFileAsync = promisify(execFile) +const cliPath = fileURLToPath(new URL('../../index.js', import.meta.url)) + +async function runSeo( + args: string[], + env: Record, +): Promise<{ exitCode: number; stdout: string; stderr: string }> { + try { + const result = await execFileAsync(process.execPath, [cliPath, ...args], { + env: { + ...process.env, + ...env, + CI: '1', + NO_UPDATE_NOTIFIER: '1', + }, + timeout: 10_000, + }) + return { exitCode: 0, stdout: result.stdout, stderr: result.stderr } + } catch (error) { + const result = error as { + code?: number + stdout?: string + stderr?: string + } + return { + exitCode: result.code ?? 1, + stdout: result.stdout ?? '', + stderr: result.stderr ?? '', + } + } +} + +test('Semrush status uses the environment API key without exposing it', async () => { + const configDir = await mkdtemp(join(tmpdir(), 'seo-semrush-cli-config-')) + const cacheDir = await mkdtemp(join(tmpdir(), 'seo-semrush-cli-cache-')) + try { + const result = await runSeo(['providers', 'semrush', 'status', '--json'], { + SEO_CONFIG_DIR: configDir, + SEO_CACHE_DIR: cacheDir, + SEO_SEMRUSH_API_KEY: 'environment-api-key', + }) + assert.equal(result.exitCode, 0) + assert.deepEqual(JSON.parse(result.stdout), { + connected: true, + apiVersion: 3, + credentialSource: 'environment', + migratedLegacyCredential: false, + liveCheck: { status: 'not-requested' }, + }) + assert.doesNotMatch(result.stdout, /environment-api-key/) + } finally { + await rm(configDir, { recursive: true, force: true }) + await rm(cacheDir, { recursive: true, force: true }) + } +}) + +test('Semrush status migrates a legacy config API key', async () => { + const configDir = await mkdtemp(join(tmpdir(), 'seo-semrush-cli-config-')) + const cacheDir = await mkdtemp(join(tmpdir(), 'seo-semrush-cli-cache-')) + const configPath = join(configDir, 'config.json') + try { + await writeFile( + configPath, + JSON.stringify({ + providers: { + semrushApiKey: 'legacy-api-key', + prefer: 'authoritative', + }, + security: { useKeychain: false }, + }), + { mode: 0o600 }, + ) + const result = await runSeo(['providers', 'semrush', 'status', '--json'], { + SEO_CONFIG_DIR: configDir, + SEO_CACHE_DIR: cacheDir, + SEO_SEMRUSH_API_KEY: '', + }) + assert.equal(result.exitCode, 0) + assert.deepEqual(JSON.parse(result.stdout), { + connected: true, + apiVersion: 3, + credentialSource: 'file', + migratedLegacyCredential: true, + liveCheck: { status: 'not-requested' }, + }) + + const config = await readFile(configPath, 'utf8') + assert.doesNotMatch(config, /legacy-api-key|semrushApiKey/) + const secretsPath = join(configDir, 'provider-secrets.json') + const secrets = await readFile(secretsPath, 'utf8') + assert.match(secrets, /semrush-api-key/) + assert.equal((await stat(secretsPath)).mode & 0o777, 0o600) + } finally { + await rm(configDir, { recursive: true, force: true }) + await rm(cacheDir, { recursive: true, force: true }) + } +}) + +test('Semrush connect refuses to prompt in JSON or CI mode', async () => { + const configDir = await mkdtemp(join(tmpdir(), 'seo-semrush-cli-config-')) + const cacheDir = await mkdtemp(join(tmpdir(), 'seo-semrush-cli-cache-')) + try { + const result = await runSeo(['providers', 'semrush', 'connect', '--json'], { + SEO_CONFIG_DIR: configDir, + SEO_CACHE_DIR: cacheDir, + SEO_SEMRUSH_API_KEY: '', + }) + assert.notEqual(result.exitCode, 0) + const output = JSON.parse(result.stdout) as { + error: { code: string; message: string } + } + assert.equal(output.error.code, 'AUTH_REQUIRED') + assert.match(output.error.message, /run `seo providers semrush connect`/i) + assert.match(output.error.message, /SEO_SEMRUSH_API_KEY/) + assert.equal(result.stderr, '') + } finally { + await rm(configDir, { recursive: true, force: true }) + await rm(cacheDir, { recursive: true, force: true }) + } +}) + +test('Semrush disconnect leaves an environment API key explicit', async () => { + const configDir = await mkdtemp(join(tmpdir(), 'seo-semrush-cli-config-')) + const cacheDir = await mkdtemp(join(tmpdir(), 'seo-semrush-cli-cache-')) + try { + const result = await runSeo( + ['providers', 'semrush', 'disconnect', '--json'], + { + SEO_CONFIG_DIR: configDir, + SEO_CACHE_DIR: cacheDir, + SEO_SEMRUSH_API_KEY: 'environment-api-key', + }, + ) + assert.equal(result.exitCode, 0) + assert.deepEqual(JSON.parse(result.stdout), { + savedCredentialRemoved: true, + environmentCredential: 'active', + note: 'The environment variable was not changed. Clear SEO_SEMRUSH_API_KEY to fully disconnect.', + }) + assert.doesNotMatch(result.stdout, /environment-api-key/) + } finally { + await rm(configDir, { recursive: true, force: true }) + await rm(cacheDir, { recursive: true, force: true }) + } +}) diff --git a/packages/cli/src/commands/providers/semrush.ts b/packages/cli/src/commands/providers/semrush.ts new file mode 100644 index 00000000..42df478d --- /dev/null +++ b/packages/cli/src/commands/providers/semrush.ts @@ -0,0 +1,181 @@ +import { intro, note, outro, password } from '@clack/prompts' +import { + deleteSemrushApiKey, + readSemrushApiKey, + SEMRUSH_API_KEY_ENV, + SemrushClient, + SeoError, + writeSemrushApiKey, +} from '@seo/core' +import { defineCommand } from 'citty' +import { jsonFlag } from '../../args.js' +import { + canPrompt, + maybeExitCancelled, + printJson, + printKeyValue, +} from '../../utils.js' + +function credentialSourceLabel( + source: 'environment' | 'keychain' | 'file' | undefined, +): string { + if (source === 'keychain') return 'system keychain' + if (source === 'file') return 'private local file' + return source ?? 'missing' +} + +const connectCommand = defineCommand({ + meta: { + name: 'connect', + description: 'Validate and save a Semrush Version 3 API key', + }, + args: { + json: { + type: 'boolean', + default: false, + description: 'Print machine-readable JSON.', + }, + }, + run: async ({ args }) => { + if (!canPrompt({ json: jsonFlag(args) })) { + throw new SeoError( + 'AUTH_REQUIRED', + `Run \`seo providers semrush connect\` in a terminal. Agents and CI can set ${SEMRUSH_API_KEY_ENV}.`, + ) + } + + intro('Connect Semrush') + note( + 'Use the permanent Version 3 API Key shown on your Semrush API Keys page. Semrush creates this key automatically. The balance check is free; research reports consume API units.', + 'API key', + ) + const apiKey = maybeExitCancelled( + await password({ + message: 'Semrush Version 3 API key', + validate: (value) => + value?.trim() ? undefined : 'API key is required', + }), + ) + const balance = await new SemrushClient({ apiKey }).apiUnitBalance() + const source = await writeSemrushApiKey(apiKey) + + note( + `${balance.remainingUnits.toLocaleString('en-US')} API units remain.`, + 'Connection verified', + ) + outro( + `Saved in the ${credentialSourceLabel(source)}. Run seo providers semrush status --check to verify it again.`, + ) + }, +}) + +const statusCommand = defineCommand({ + meta: { + name: 'status', + description: 'Show the local Semrush connection', + }, + args: { + check: { + type: 'boolean', + default: false, + description: 'Verify the Version 3 API key with a free balance request.', + }, + json: { + type: 'boolean', + default: false, + description: 'Print machine-readable JSON.', + }, + }, + run: async ({ args }) => { + const credential = await readSemrushApiKey() + const shouldCheck = Boolean(args.check) + const balance = + shouldCheck && credential + ? await new SemrushClient().apiUnitBalance() + : undefined + const result = { + connected: Boolean(credential), + apiVersion: credential ? 3 : null, + credentialSource: credential?.source, + migratedLegacyCredential: credential?.migrated ?? false, + liveCheck: balance + ? { + status: 'passed' as const, + remainingUnits: balance.remainingUnits, + observedAt: balance.observedAt, + requestCostUnits: 0, + } + : { + status: (shouldCheck ? 'unavailable' : 'not-requested') as + | 'unavailable' + | 'not-requested', + }, + } + if (jsonFlag(args)) { + printJson(result) + return + } + printKeyValue([ + ['Connected', result.connected ? 'yes' : 'no'], + ['API version', result.apiVersion ? 'Version 3' : 'not connected'], + ['Credential', credentialSourceLabel(result.credentialSource)], + [ + 'Live check', + result.liveCheck.status === 'passed' + ? `passed at ${result.liveCheck.observedAt}` + : result.liveCheck.status === 'unavailable' + ? 'not available without credentials' + : 'not requested; pass --check to verify', + ], + ...(balance + ? ([ + [ + 'API units remaining', + balance.remainingUnits.toLocaleString('en-US'), + ], + ] satisfies Array<[string, string]>) + : []), + ]) + }, +}) + +const disconnectCommand = defineCommand({ + meta: { + name: 'disconnect', + description: 'Remove the saved Semrush Version 3 API key', + }, + args: { + json: { + type: 'boolean', + default: false, + description: 'Print machine-readable JSON.', + }, + }, + run: async ({ args }) => { + await deleteSemrushApiKey() + const environmentCredential = Boolean(process.env[SEMRUSH_API_KEY_ENV]) + const result = { + savedCredentialRemoved: true, + environmentCredential: environmentCredential + ? ('active' as const) + : ('missing' as const), + note: environmentCredential + ? `The environment variable was not changed. Clear ${SEMRUSH_API_KEY_ENV} to fully disconnect.` + : 'Semrush is disconnected.', + } + if (jsonFlag(args)) printJson(result) + else process.stdout.write(`${result.note}\n`) + }, +}) + +export const semrushProviderCommand = defineCommand({ + meta: { + name: 'semrush', + description: 'Connect Semrush Version 3 for optional search data', + }, + subCommands: { + connect: connectCommand, + status: statusCommand, + disconnect: disconnectCommand, + }, +}) diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index a88ece40..9113de90 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -218,6 +218,7 @@ const allHelpSections: HelpSection[] = [ ['seo auth', 'Manage Google auth'], ['seo providers bing', 'Connect and report on Bing Webmaster'], ['seo providers dataforseo', 'Connect optional search data'], + ['seo providers semrush', 'Connect optional Semrush data'], ['seo cache', 'Manage local cache'], ['seo privacy', 'Show local storage paths'], ['seo telemetry status', 'Check anonymous usage telemetry'], diff --git a/packages/core/src/analyze/domain-research/shared.ts b/packages/core/src/analyze/domain-research/shared.ts index 36f0139a..b17868b9 100644 --- a/packages/core/src/analyze/domain-research/shared.ts +++ b/packages/core/src/analyze/domain-research/shared.ts @@ -32,6 +32,8 @@ import { type ProviderCandidate, resolveProvider, } from '../../providers/resolver.js' +import { readSemrushApiKey } from '../../providers/semrush/credentials.js' +import { SemrushDomainResearchProvider } from '../../providers/semrush/domain-research.js' import type { GscRow } from '../../types.js' import type { DomainResearchDataStatus } from '../domain-research-contract.js' @@ -208,12 +210,21 @@ export function offset(value: number | undefined): number { } async function defaultCandidates(): Promise { + const [dataForSeo, semrush] = await Promise.all([ + readDataForSeoCredentials(), + readSemrushApiKey(), + ]) return [ { adapter: new DataForSeoDomainResearchProvider(), - connected: Boolean(await readDataForSeoCredentials()), + connected: Boolean(dataForSeo), priority: 10, }, + { + adapter: new SemrushDomainResearchProvider(), + connected: Boolean(semrush), + priority: 20, + }, ] } @@ -243,7 +254,7 @@ export async function researchProvider(input: { ? 'INVALID_INPUT' : 'PROVIDER_UNAVAILABLE', resolution.reason === 'provider-not-connected' - ? 'No connected provider can run domain research. Run `seo providers dataforseo connect` first.' + ? 'No connected provider can run domain research. Connect DataForSEO or Semrush under `seo providers` first.' : `${providerName} cannot run this domain research report for the selected market.`, ) } diff --git a/packages/core/src/analyze/keyword-metrics.ts b/packages/core/src/analyze/keyword-metrics.ts index c6382e09..eaba0dbf 100644 --- a/packages/core/src/analyze/keyword-metrics.ts +++ b/packages/core/src/analyze/keyword-metrics.ts @@ -22,6 +22,8 @@ import { type ProviderCandidate, resolveProvider, } from '../providers/resolver.js' +import { readSemrushApiKey } from '../providers/semrush/credentials.js' +import { SemrushKeywordMetricsProvider } from '../providers/semrush/keyword-metrics.js' const MAX_REPORT_KEYWORDS = 50 @@ -165,12 +167,21 @@ function keywordMetricsProvider( } async function defaultCandidates(): Promise { + const [dataForSeo, semrush] = await Promise.all([ + readDataForSeoCredentials(), + readSemrushApiKey(), + ]) return [ { adapter: new DataForSeoKeywordMetricsProvider(), - connected: Boolean(await readDataForSeoCredentials()), + connected: Boolean(dataForSeo), priority: 10, }, + { + adapter: new SemrushKeywordMetricsProvider(), + connected: Boolean(semrush), + priority: 20, + }, ] } @@ -181,7 +192,7 @@ function providerResolutionError(input: { if (input.reason === 'provider-not-connected') { return new SeoError( 'PROVIDER_UNAVAILABLE', - 'No connected provider can supply keyword metrics. Run `seo providers dataforseo connect` first.', + 'No connected provider can supply keyword metrics. Connect DataForSEO or Semrush under `seo providers` first.', ) } if (input.provider && input.reason === 'market-not-supported') { diff --git a/packages/core/src/analyze/keyword-research.ts b/packages/core/src/analyze/keyword-research.ts index 9fcb93c1..730ca0a0 100644 --- a/packages/core/src/analyze/keyword-research.ts +++ b/packages/core/src/analyze/keyword-research.ts @@ -21,6 +21,8 @@ import { type ProviderCandidate, resolveProvider, } from '../providers/resolver.js' +import { readSemrushApiKey } from '../providers/semrush/credentials.js' +import { SemrushKeywordDiscoveryProvider } from '../providers/semrush/keyword-discovery.js' import { analyzeKeywordTrend, type KeywordTrend } from './keyword-metrics.js' const MAX_RESEARCH_SEEDS = 5 @@ -83,12 +85,21 @@ function discoveryProvider( } async function defaultCandidates(): Promise { + const [dataForSeo, semrush] = await Promise.all([ + readDataForSeoCredentials(), + readSemrushApiKey(), + ]) return [ { adapter: new DataForSeoKeywordDiscoveryProvider(), - connected: Boolean(await readDataForSeoCredentials()), + connected: Boolean(dataForSeo), priority: 10, }, + { + adapter: new SemrushKeywordDiscoveryProvider(), + connected: Boolean(semrush), + priority: 20, + }, ] } @@ -271,7 +282,7 @@ export async function keywordResearchReport( if (resolution.status === 'unavailable') { const message = resolution.reason === 'provider-not-connected' - ? 'No connected provider can discover keywords. Run `seo providers dataforseo connect` first.' + ? 'No connected provider can discover keywords. Connect DataForSEO or Semrush under `seo providers` first.' : validated.provider ? `${validated.provider} cannot discover keywords for this market.` : 'No configured provider can discover keywords for this market.' diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 623344b8..80e70961 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -76,8 +76,11 @@ export * from './providers/errors.js' export * from './providers/imports/research-columns.js' export * from './providers/link-contracts.js' export * from './providers/resolver.js' -export * from './providers/router.js' -export * from './providers/semrush.js' +export * from './providers/semrush/client.js' +export * from './providers/semrush/credentials.js' +export * from './providers/semrush/domain-research.js' +export * from './providers/semrush/keyword-discovery.js' +export * from './providers/semrush/keyword-metrics.js' export * from './providers/transport.js' export * from './rank-tracking/index.js' export * from './robots-directives.js' diff --git a/packages/core/src/providers/cache.ts b/packages/core/src/providers/cache.ts index 570906a2..d119feab 100644 --- a/packages/core/src/providers/cache.ts +++ b/packages/core/src/providers/cache.ts @@ -41,9 +41,11 @@ export function providerCredentialScope( provider: ProviderId, accountIdentifier: string, ): string { - return createHash('sha256') - .update(`${provider}\0${accountIdentifier.trim().toLowerCase()}`) - .digest('hex') + const identifier = + provider === 'dataforseo' + ? accountIdentifier.trim().toLowerCase() + : accountIdentifier.trim() + return createHash('sha256').update(`${provider}\0${identifier}`).digest('hex') } function requestHash(key: ProviderCacheKey, requestJson: string): string { diff --git a/packages/core/src/providers/contracts.ts b/packages/core/src/providers/contracts.ts index b87ab9e5..a92a4cf2 100644 --- a/packages/core/src/providers/contracts.ts +++ b/packages/core/src/providers/contracts.ts @@ -95,6 +95,12 @@ export type ProviderCostEvidence = { estimatedMicros: number | null actualMicros: number | null taskIds: string[] + native?: { + unit: string + estimatedUnits: number | null + actualUnits: number | null + remainingBefore: number | null + } } export type ProviderRequestEvidence = { diff --git a/packages/core/src/providers/router.test.ts b/packages/core/src/providers/router.test.ts deleted file mode 100644 index d04139a8..00000000 --- a/packages/core/src/providers/router.test.ts +++ /dev/null @@ -1,62 +0,0 @@ -import assert from 'node:assert/strict' -import test from 'node:test' -import { configSchema } from '../types.js' -import { DataForSeoProvider } from './dataforseo.js' -import { getKeywordProvider } from './router.js' -import { SemrushProvider } from './semrush.js' - -function config( - input: { semrush?: boolean; prefer?: 'cheap' | 'authoritative' } = {}, -) { - return configSchema.parse({ - providers: { - prefer: input.prefer ?? 'cheap', - ...(input.semrush ? { semrushApiKey: 'configured' } : {}), - }, - }) -} - -test('keyword router sees DataForSEO through the secure credential boundary', async () => { - let credentialChecks = 0 - const provider = await getKeywordProvider(undefined, { - readConfig: () => config(), - hasDataForSeoCredentials: () => { - credentialChecks += 1 - return true - }, - }) - - assert.ok(provider instanceof DataForSeoProvider) - assert.equal(credentialChecks, 1) -}) - -test('keyword router preserves preference and fallback behavior', async () => { - let unnecessaryCredentialChecks = 0 - const authoritative = await getKeywordProvider('authoritative', { - readConfig: () => config({ semrush: true }), - hasDataForSeoCredentials: () => { - unnecessaryCredentialChecks += 1 - return true - }, - }) - assert.ok(authoritative instanceof SemrushProvider) - assert.equal(unnecessaryCredentialChecks, 0) - - const cheap = await getKeywordProvider('cheap', { - readConfig: () => config({ semrush: true }), - hasDataForSeoCredentials: () => true, - }) - assert.ok(cheap instanceof DataForSeoProvider) - - const fallback = await getKeywordProvider('authoritative', { - readConfig: () => config(), - hasDataForSeoCredentials: () => true, - }) - assert.ok(fallback instanceof DataForSeoProvider) - - const missing = await getKeywordProvider(undefined, { - readConfig: () => config(), - hasDataForSeoCredentials: () => false, - }) - assert.equal(missing, undefined) -}) diff --git a/packages/core/src/providers/router.ts b/packages/core/src/providers/router.ts deleted file mode 100644 index 1e4311db..00000000 --- a/packages/core/src/providers/router.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { readConfig } from '../storage/config.js' -import type { AppConfig, KeywordDataProvider } from '../types.js' -import { readDataForSeoCredentials } from './dataforseo/credentials.js' -import { DataForSeoProvider } from './dataforseo.js' -import { SemrushProvider } from './semrush.js' - -export async function getKeywordProvider( - prefer?: 'cheap' | 'authoritative', - dependencies: { - readConfig?: () => AppConfig - hasDataForSeoCredentials?: () => boolean | Promise - } = {}, -): Promise { - const config = (dependencies.readConfig ?? readConfig)() - const chosen = prefer ?? config.providers.prefer - - if (chosen === 'authoritative' && config.providers.semrushApiKey) { - return new SemrushProvider() - } - - const hasDataForSeo = await ( - dependencies.hasDataForSeoCredentials ?? - (async () => Boolean(await readDataForSeoCredentials())) - )() - - if (chosen === 'cheap' && hasDataForSeo) { - return new DataForSeoProvider() - } - - if (config.providers.semrushApiKey) { - return new SemrushProvider() - } - - if (hasDataForSeo) { - return new DataForSeoProvider() - } - - return undefined -} diff --git a/packages/core/src/providers/semrush.ts b/packages/core/src/providers/semrush.ts deleted file mode 100644 index 9eae8562..00000000 --- a/packages/core/src/providers/semrush.ts +++ /dev/null @@ -1,172 +0,0 @@ -import type { - KeywordDataProvider, - KeywordOverview, - KeywordRow, - ProviderOpts, - ProviderResult, -} from '../types.js' -import { cachedSemrushCall } from './semrush/cache.js' -import { mapKeywordRows, mapOverview } from './semrush/mappers.js' -import { - semrushDifficultyRowsSchema, - semrushKeywordOverviewSchema, - semrushKeywordRowsSchema, -} from './semrush/schemas.js' - -export class SemrushProvider implements KeywordDataProvider { - readonly name = 'semrush' - readonly capabilities = { - overview: true, - batchOverview: true, - related: true, - broadMatch: true, - questions: true, - difficulty: true, - urlKeywords: true, - domainKeywords: true, - maxBatchSize: 100, - } - - async keywordOverview( - phrase: string, - opts: ProviderOpts = {}, - ): Promise> { - return cachedSemrushCall( - 'phrase_this', - { - phrase, - database: opts.database ?? 'us', - export_columns: 'Ph,Nq,Cp,Co,Nr,Td,Kd', - }, - mapOverview, - semrushKeywordOverviewSchema, - 7 * 86_400_000, - 10, - opts.refresh, - ) - } - - async batchKeywordOverview( - phrases: string[], - opts: ProviderOpts = {}, - ): Promise> { - return cachedSemrushCall( - 'phrase_these', - { - phrase: phrases.join(';'), - database: opts.database ?? 'us', - export_columns: 'Ph,Nq,Cp,Co,Nr,Td,Kd', - }, - (rows) => - mapKeywordRows(rows).map((row) => ({ ...row, phrase: row.phrase })), - semrushKeywordRowsSchema, - 7 * 86_400_000, - 10, - opts.refresh, - ) - } - - async relatedKeywords( - phrase: string, - opts: ProviderOpts = {}, - ): Promise> { - return cachedSemrushCall( - 'phrase_related', - { - phrase, - database: opts.database ?? 'us', - display_limit: 20, - export_columns: 'Ph,Nq,Kd,Cp,Co', - }, - mapKeywordRows, - semrushKeywordRowsSchema, - 14 * 86_400_000, - 40, - opts.refresh, - ) - } - - async questions( - phrase: string, - opts: ProviderOpts = {}, - ): Promise> { - return cachedSemrushCall( - 'phrase_questions', - { - phrase, - database: opts.database ?? 'us', - display_limit: 20, - export_columns: 'Ph,Nq,Kd,Cp,Co', - }, - mapKeywordRows, - semrushKeywordRowsSchema, - 14 * 86_400_000, - 40, - opts.refresh, - ) - } - - async keywordDifficulty( - phrases: string[], - opts: ProviderOpts = {}, - ): Promise> { - return cachedSemrushCall( - 'phrase_kdi', - { - phrase: phrases.join(';'), - database: opts.database ?? 'us', - export_columns: 'Ph,Kd', - }, - (rows) => - mapKeywordRows(rows).flatMap((row) => - row.difficulty === undefined - ? [] - : [{ phrase: row.phrase, kd: row.difficulty }], - ), - semrushDifficultyRowsSchema, - 7 * 86_400_000, - 50, - opts.refresh, - ) - } - - async domainKeywords( - domain: string, - opts: ProviderOpts = {}, - ): Promise> { - return cachedSemrushCall( - 'domain_organic', - { - domain, - database: opts.database ?? 'us', - display_limit: 100, - export_columns: 'Ph,Nq,Cp,Co,Kd,Po,Ur,Dn', - }, - mapKeywordRows, - semrushKeywordRowsSchema, - 7 * 86_400_000, - 10, - opts.refresh, - ) - } - - async urlKeywords( - url: string, - opts: ProviderOpts = {}, - ): Promise> { - return cachedSemrushCall( - 'url_organic', - { - url, - database: opts.database ?? 'us', - display_limit: 100, - export_columns: 'Ph,Nq,Cp,Co,Kd,Po,Ur,Dn', - }, - mapKeywordRows, - semrushKeywordRowsSchema, - 7 * 86_400_000, - 10, - opts.refresh, - ) - } -} diff --git a/packages/core/src/providers/semrush/adapter.test.ts b/packages/core/src/providers/semrush/adapter.test.ts new file mode 100644 index 00000000..cc43cea9 --- /dev/null +++ b/packages/core/src/providers/semrush/adapter.test.ts @@ -0,0 +1,299 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { keywordMetricsReport } from '../../analyze/keyword-metrics.js' +import type { ProviderCandidate } from '../resolver.js' +import type { SemrushReportRequest, SemrushReportSnapshot } from './client.js' +import { SemrushDomainResearchProvider } from './domain-research.js' +import { SemrushKeywordDiscoveryProvider } from './keyword-discovery.js' +import { SemrushKeywordMetricsProvider } from './keyword-metrics.js' + +const market = { + searchEngine: 'google' as const, + countryCode: 'GB', + languageCode: 'en', +} + +function snapshot( + input: SemrushReportRequest, + rows: string[][], +): SemrushReportSnapshot { + return { + table: { headers: [...input.columns], rows }, + observedAt: '2026-07-24T12:00:00.000Z', + returnedRows: rows.length, + cache: { + status: 'miss', + storedAt: '2026-07-24T12:00:00.000Z', + expiresAt: '2026-07-31T12:00:00.000Z', + }, + cost: { + currency: 'USD', + estimatedMicros: null, + actualMicros: null, + taskIds: [], + native: { + unit: 'api-unit', + estimatedUnits: input.maximumResponseRows * input.unitsPerLine, + actualUnits: rows.length * input.unitsPerLine, + remainingBefore: 10_000, + }, + }, + warnings: [], + } +} + +test('Semrush keyword metrics preserve zero, omissions, and deterministic order', async () => { + const requests: SemrushReportRequest[] = [] + const provider = new SemrushKeywordMetricsProvider({ + client: { + report: async (input) => { + requests.push(input) + return snapshot(input, [['zero keyword', '0', '0', '0', '0', '0', '0']]) + }, + }, + }) + const result = await provider.keywordMetrics({ + keywords: ['Missing Keyword', 'zero keyword', 'Zero Keyword'], + market, + }) + + assert.equal(requests[0]?.parameters.database, 'uk') + assert.equal(requests[0]?.parameters.phrase, 'missing keyword;zero keyword') + assert.deepEqual( + result.data.map((row) => row.keyword), + ['missing keyword', 'zero keyword'], + ) + assert.equal(result.data[0]?.monthlySearchVolume.state, 'missing') + assert.deepEqual(result.data[1]?.monthlySearchVolume, { + state: 'observed', + value: 0, + }) + assert.equal(result.coverage.completeness, 'partial') + assert.ok( + result.warnings.some( + (warning) => warning.code === 'semrush-v3-keyword-api-deprecated', + ), + ) +}) + +test('Semrush runs through the provider-neutral keyword metrics report', async () => { + const provider = new SemrushKeywordMetricsProvider({ + client: { + report: async (input) => + snapshot(input, [ + ['alpha', '100', '1.5', '0.4', '5000', '1', '25'], + ['beta', '0', '0', '0', '0', '0', '0'], + ]), + }, + }) + const candidates: ProviderCandidate[] = [ + { adapter: provider, connected: true, priority: 1 }, + ] + const report = await keywordMetricsReport( + { + keywords: ['beta', 'alpha'], + market, + provider: 'semrush', + }, + { + candidates, + now: () => new Date('2026-07-24T13:00:00.000Z'), + }, + ) + + assert.equal(report.generatedAt, '2026-07-24T13:00:00.000Z') + assert.equal(report.dataStatus, 'complete') + assert.equal(report.evidence.provider, 'semrush') + assert.deepEqual(report.summary, { + requestedKeywords: 2, + providerRows: 2, + keywordsWithObservedVolume: 2, + observedZeroVolume: 1, + missingOrInvalidVolume: 0, + increasingTrends: 0, + decreasingTrends: 0, + stableTrends: 0, + unavailableTrends: 2, + verdict: + 'Observed search-volume estimates are available for 2 of 2 keywords; 0 show an increasing recent trend.', + }) + assert.deepEqual( + report.evidence.data.map((row) => row.keyword), + ['alpha', 'beta'], + ) +}) + +test('Semrush discovery keeps exact seed/source provenance and bounded calls', async () => { + const requests: SemrushReportRequest[] = [] + const provider = new SemrushKeywordDiscoveryProvider({ + client: { + report: async (input) => { + requests.push(input) + return snapshot(input, [ + [ + input.parameters.phrase === 'alpha' ? 'shared' : 'other', + '10', + '1', + '0.5', + '100', + '20', + ], + ]) + }, + }, + }) + const result = await provider.discoverKeywords({ + seeds: ['beta', 'alpha'], + sources: ['related', 'ideas'], + market, + limit: 8, + }) + + assert.equal(requests.length, 4) + assert.ok(requests.every((request) => request.maximumResponseRows === 2)) + assert.deepEqual( + result.data.map((row) => row.keyword), + ['other', 'shared'], + ) + assert.equal(result.data[0]?.sources.length, 2) + assert.equal(result.data[1]?.sources.length, 2) + assert.equal(result.request.filters.providerRequests, 4) +}) + +test('Semrush domain reports map only compatible provider-native fields', async () => { + const provider = new SemrushDomainResearchProvider({ + client: { + report: async (input) => { + if (input.reportType === 'domain_rank') { + return snapshot(input, [['example.com', '5', '100', '250']]) + } + if (input.reportType === 'domain_organic') { + return snapshot(input, [ + [ + 'zero keyword', + '1', + '0', + '0', + '0', + '0', + '0', + '0', + 'https://example.com/zero', + '1721822400', + ], + ]) + } + if (input.reportType === 'domain_organic_unique') { + return snapshot(input, [ + ['https://example.com/page', '3', '40', '25'], + ]) + } + const keyword = String(input.parameters.phrase) + return snapshot( + input, + keyword === 'first' + ? [ + ['1', 'example.com', 'https://example.com/a'], + ['2', 'other.com', 'https://other.com/a'], + ] + : [ + ['3', 'example.com', 'https://example.com/b'], + ['1', 'third.com', 'https://third.com/b'], + ], + ) + }, + }, + }) + + const overview = await provider.domainOverview({ + domain: 'example.com', + market, + }) + assert.deepEqual(overview.data.organic.rankedKeywords, { + state: 'observed', + value: 5, + }) + assert.deepEqual(overview.data.organic.estimatedMonthlyTraffic, { + state: 'observed', + value: 100, + }) + assert.equal(overview.data.organic.rankings.state, 'unavailable') + + const ranked = await provider.rankedKeywords({ + target: 'example.com', + market, + includeSubdomains: true, + resultTypes: ['organic'], + limit: 10, + }) + assert.deepEqual(ranked.data.rows[0]?.monthlySearchVolume, { + state: 'observed', + value: 0, + }) + assert.equal( + ranked.data.rows[0]?.estimatedMonthlyTraffic.state, + 'unavailable', + ) + + const pages = await provider.rankingPages({ + domain: 'example.com', + market, + limit: 10, + }) + assert.deepEqual(pages.data.rows[0]?.organic.estimatedMonthlyTraffic, { + state: 'observed', + value: 40, + }) + assert.equal( + pages.data.rows[0]?.organic.estimatedMonthlyTrafficCostUsd.state, + 'unavailable', + ) + + const competitors = await provider.serpCompetitors({ + keywords: ['second', 'first'], + market, + includeSubdomains: false, + resultTypes: ['organic'], + limit: 10, + }) + assert.equal(competitors.data.rows[0]?.domain, 'example.com') + assert.equal(competitors.data.rows[0]?.matchedKeywords, 2) + assert.deepEqual(competitors.data.rows[0]?.averagePosition, { + state: 'observed', + value: 2, + }) + assert.equal(competitors.data.rows[0]?.visibility.state, 'unavailable') +}) + +test('Semrush competitor acquisition is capped at 20 calls and 2000 rows', async () => { + let calls = 0 + let acquiredRows = 0 + const provider = new SemrushDomainResearchProvider({ + client: { + report: async (input) => { + calls += 1 + assert.equal(input.maximumResponseRows, 100) + const rows = Array.from({ length: 100 }, (_, index) => [ + String(index + 1), + `domain-${index}.example`, + `https://domain-${index}.example/page`, + ]) + acquiredRows += rows.length + return snapshot(input, rows) + }, + }, + }) + const result = await provider.serpCompetitors({ + keywords: Array.from({ length: 20 }, (_, index) => `keyword ${index}`), + market, + includeSubdomains: false, + resultTypes: ['organic'], + limit: 100, + }) + + assert.equal(calls, 20) + assert.equal(acquiredRows, 2_000) + assert.equal(result.data.rows.length, 100) + assert.equal(result.coverage.completeness, 'capped') + assert.ok(JSON.stringify(result).length < 500_000) +}) diff --git a/packages/core/src/providers/semrush/cache.test.ts b/packages/core/src/providers/semrush/cache.test.ts deleted file mode 100644 index c4ced200..00000000 --- a/packages/core/src/providers/semrush/cache.test.ts +++ /dev/null @@ -1,166 +0,0 @@ -import assert from 'node:assert/strict' -import { mkdirSync, mkdtempSync, rmSync } from 'node:fs' -import { tmpdir } from 'node:os' -import { dirname, join } from 'node:path' -import test from 'node:test' -import { Response } from 'undici' -import { writeConfig } from '../../storage/config.js' -import { getDb } from '../../storage/database.js' -import Database from '../../storage/sqlite.js' -import { configSchema } from '../../types.js' -import { ProviderError } from '../errors.js' -import { cachedSemrushCall } from './cache.js' -import { mapOverview } from './mappers.js' -import { semrushKeywordOverviewSchema } from './schemas.js' - -const root = mkdtempSync(join(tmpdir(), 'seo-semrush-provider-')) -const previousConfigDir = process.env.SEO_CONFIG_DIR -const previousCacheDir = process.env.SEO_CACHE_DIR -process.env.SEO_CONFIG_DIR = join(root, 'config') -process.env.SEO_CACHE_DIR = join(root, 'cache') - -const cacheFile = join(root, 'cache', 'cache.db') -mkdirSync(dirname(cacheFile), { recursive: true }) -const legacyDatabase = new Database(cacheFile) -legacyDatabase.exec(` - CREATE TABLE semrush_cache ( - endpoint TEXT, - query_hash TEXT, - request_json TEXT, - response_json TEXT, - credits_used INTEGER, - fetched_at INTEGER, - expires_at INTEGER, - PRIMARY KEY(endpoint, query_hash) - ) WITHOUT ROWID; -`) -legacyDatabase - .prepare( - `INSERT INTO semrush_cache - (endpoint, query_hash, request_json, response_json, credits_used, fetched_at, expires_at) - VALUES (?, ?, ?, ?, ?, ?, ?)`, - ) - .run( - 'phrase_this', - 'legacy-query', - JSON.stringify({ key: 'legacy-sem-rush-key', phrase: 'unsafe query' }), - '[]', - 0, - Date.now(), - Date.now() + 60_000, - ) -legacyDatabase.close() - -test.after(() => { - if (previousConfigDir === undefined) delete process.env.SEO_CONFIG_DIR - else process.env.SEO_CONFIG_DIR = previousConfigDir - if (previousCacheDir === undefined) delete process.env.SEO_CACHE_DIR - else process.env.SEO_CACHE_DIR = previousCacheDir - rmSync(root, { recursive: true, force: true }) -}) - -function configure(apiKey: string): void { - writeConfig( - configSchema.parse({ - providers: { semrushApiKey: apiKey, prefer: 'authoritative' }, - security: { useKeychain: false }, - }), - ) -} - -test('Semrush validates responses and caches without storing credentials', async () => { - const legacyRows = getDb() - .prepare('SELECT COUNT(*) AS count FROM semrush_cache') - .get() as { count: number } - assert.equal(legacyRows.count, 0) - - const firstKey = 'semrush-local-secret-one' - const secondKey = 'semrush-local-secret-two' - let fetchCalls = 0 - const fetch = async (url: string | URL) => { - fetchCalls += 1 - const parsed = new URL(url) - assert.equal( - parsed.searchParams.get('key'), - fetchCalls === 1 ? firstKey : secondKey, - ) - return new Response('Ph;Nq;Cp;Co;Nr;Kd\nzero query;0;0;0;0;0') - } - const run = () => - cachedSemrushCall( - 'phrase_this', - { - phrase: 'zero query', - database: 'us', - export_columns: 'Ph,Nq,Cp,Co,Nr,Kd', - }, - mapOverview, - semrushKeywordOverviewSchema, - 60_000, - 10, - false, - { fetch, baseUrl: 'https://provider.invalid' }, - ) - - configure(firstKey) - assert.equal((await run()).cached, undefined) - assert.equal((await run()).cached, true) - assert.equal(fetchCalls, 1) - - configure(secondKey) - assert.equal((await run()).cached, undefined) - assert.equal(fetchCalls, 2) - - const rows = getDb() - .prepare( - 'SELECT query_hash, request_json, response_json FROM semrush_cache ORDER BY fetched_at', - ) - .all() as Array<{ - query_hash: string - request_json: string - response_json: string - }> - assert.equal(rows.length, 2) - for (const row of rows) { - const stored = JSON.stringify(row) - assert.doesNotMatch(stored, new RegExp(firstKey)) - assert.doesNotMatch(stored, new RegExp(secondKey)) - assert.doesNotMatch( - stored, - new RegExp(Buffer.from(firstKey).toString('base64url')), - ) - assert.doesNotMatch( - stored, - new RegExp(Buffer.from(secondKey).toString('base64url')), - ) - assert.equal('key' in JSON.parse(row.request_json), false) - } -}) - -test('Semrush rejects malformed mapped rows and provider errors', async () => { - configure('semrush-test-key') - const call = (body: string) => - cachedSemrushCall( - 'phrase_this', - { phrase: body, database: 'us' }, - mapOverview, - semrushKeywordOverviewSchema, - 60_000, - 10, - true, - { - fetch: async () => new Response(body), - baseUrl: 'https://provider.invalid', - }, - ) - - await assert.rejects( - call('Ph;Nq\n;not-a-number'), - (error) => - error instanceof ProviderError && error.code === 'invalid-response', - ) - await assert.rejects( - call('ERROR :: 50 :: NOTHING FOUND'), - (error) => error instanceof ProviderError && error.code === 'remote-error', - ) -}) diff --git a/packages/core/src/providers/semrush/cache.ts b/packages/core/src/providers/semrush/cache.ts deleted file mode 100644 index d3f73b04..00000000 --- a/packages/core/src/providers/semrush/cache.ts +++ /dev/null @@ -1,157 +0,0 @@ -import { createHash } from 'node:crypto' -import { fetch } from 'undici' -import type { ZodType } from 'zod' -import { readConfig } from '../../storage/config.js' -import { getDb, hashKey, noteCacheWrite } from '../../storage/database.js' -import type { ProviderResult } from '../../types.js' -import { ProviderError } from '../errors.js' -import { type ProviderFetch, providerRequestText } from '../transport.js' -import { parseSemicolonCsv } from './csv.js' - -const BASE_URL = 'https://api.semrush.com/' -const MAX_RESPONSE_BYTES = 10 * 1024 * 1024 -const DEFAULT_TIMEOUT_MS = 15_000 - -export type SemrushTransportOptions = { - fetch?: ProviderFetch - baseUrl?: string - maxResponseBytes?: number - timeoutMs?: number -} - -function estimateUsd(units: number): number { - return (units / 1000) * 0.05 -} - -export async function cachedSemrushCall( - endpoint: string, - params: Record, - map: (rows: string[][]) => T, - schema: ZodType, - ttlMs: number, - creditsPerLine: number, - refresh = false, - transport: SemrushTransportOptions = {}, -): Promise> { - const config = readConfig() - const apiKey = config.providers.semrushApiKey - if (!apiKey) { - throw new ProviderError({ - provider: 'semrush', - operation: endpoint, - code: 'configuration', - message: 'Semrush credentials are not configured.', - }) - } - - const safeParams = Object.fromEntries( - Object.entries({ ...params, type: endpoint }).filter( - ([, value]) => value !== undefined, - ), - ) as Record - const requestParams = { ...safeParams, key: apiKey } - const credentialScope = createHash('sha256').update(apiKey).digest('hex') - - const db = getDb() - const queryHash = hashKey([endpoint, safeParams, credentialScope]) - const cached = db - .prepare( - 'SELECT response_json, credits_used FROM semrush_cache WHERE endpoint = ? AND query_hash = ? AND expires_at > ?', - ) - .get(endpoint, queryHash, Date.now()) as - | { response_json?: string; credits_used?: number } - | undefined - - if (!refresh && cached?.response_json) { - try { - const parsed = schema.safeParse(JSON.parse(cached.response_json)) - if (parsed.success) { - return { - data: parsed.data, - usage: { - provider: 'Semrush', - units: cached.credits_used ?? 0, - unitLabel: 'units', - estimatedUsd: estimateUsd(cached.credits_used ?? 0), - calls: 1, - cacheHits: 1, - }, - cached: true, - } - } - } catch { - // A corrupt legacy row is a cache miss and will be replaced below. - } - } - - const url = new URL(transport.baseUrl ?? BASE_URL) - url.search = new URLSearchParams( - Object.entries(requestParams).map(([key, value]): [string, string] => [ - key, - String(value), - ]), - ).toString() - - const text = await providerRequestText({ - provider: 'semrush', - operation: endpoint, - url, - fetch: transport.fetch ?? fetch, - maxResponseBytes: transport.maxResponseBytes ?? MAX_RESPONSE_BYTES, - timeoutMs: transport.timeoutMs ?? DEFAULT_TIMEOUT_MS, - retry: 'never', - }) - if (text.startsWith('ERROR ::')) { - throw new ProviderError({ - provider: 'semrush', - operation: endpoint, - code: 'remote-error', - message: 'Semrush rejected the report request.', - }) - } - - const rows = parseSemicolonCsv(text) - const parsed = schema.safeParse(map(rows)) - if (!parsed.success) { - throw new ProviderError({ - provider: 'semrush', - operation: endpoint, - code: 'invalid-response', - message: - 'Semrush returned data that does not match the expected response schema.', - cause: parsed.error, - }) - } - const data = parsed.data - const credits = Math.max(0, rows.length - 1) * creditsPerLine - const requestJson = JSON.stringify(safeParams) - const responseJson = JSON.stringify(data) - - db.prepare( - `INSERT OR REPLACE INTO semrush_cache - (endpoint, query_hash, request_json, response_json, credits_used, fetched_at, expires_at) - VALUES (?, ?, ?, ?, ?, ?, ?)`, - ).run( - endpoint, - queryHash, - requestJson, - responseJson, - credits, - Date.now(), - Date.now() + ttlMs, - ) - noteCacheWrite( - Buffer.byteLength(requestJson) + Buffer.byteLength(responseJson), - ) - - return { - data, - usage: { - provider: 'Semrush', - units: credits, - unitLabel: 'units', - estimatedUsd: estimateUsd(credits), - calls: 1, - }, - } -} diff --git a/packages/core/src/providers/semrush/client.test.ts b/packages/core/src/providers/semrush/client.test.ts new file mode 100644 index 00000000..5e1d6b60 --- /dev/null +++ b/packages/core/src/providers/semrush/client.test.ts @@ -0,0 +1,282 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' +import { Response } from 'undici' +import Database from '../../storage/sqlite.js' +import { ProviderError } from '../errors.js' +import { SemrushClient } from './client.js' + +function cacheDatabase(): Database.Database { + const database = new Database(':memory:') + database.exec(` + CREATE TABLE provider_cache ( + provider TEXT NOT NULL, + credential_scope TEXT NOT NULL, + operation TEXT NOT NULL, + request_hash TEXT NOT NULL, + request_json TEXT NOT NULL, + response_json TEXT NOT NULL, + row_count INTEGER, + source_cost_micros INTEGER, + task_ids_json TEXT NOT NULL DEFAULT '[]', + fetched_at INTEGER NOT NULL, + expires_at INTEGER NOT NULL, + PRIMARY KEY(provider, credential_scope, operation, request_hash) + ) WITHOUT ROWID; + `) + return database +} + +test('free balance request validates the key without exposing it', async () => { + const apiKey = 'semrush-test-secret' + const balance = await new SemrushClient({ + apiKey, + balanceUrl: 'https://provider.invalid/balance', + fetch: async (url) => { + const parsed = new URL(url) + assert.equal(parsed.searchParams.get('key'), apiKey) + return new Response('1,234') + }, + }).apiUnitBalance() + + assert.equal(balance.remainingUnits, 1_234) + assert.match(balance.observedAt, /^\d{4}-\d{2}-\d{2}T/u) + assert.doesNotMatch(JSON.stringify(balance), new RegExp(apiKey)) +}) + +test('free balance request keeps provider error bodies private', async () => { + const apiKey = 'semrush-test-secret' + await assert.rejects( + new SemrushClient({ + apiKey, + fetch: async () => + new Response(`ERROR 120 :: WRONG KEY ${apiKey}`, { status: 200 }), + }).apiUnitBalance(), + (error) => { + assert.ok(error instanceof ProviderError) + assert.equal(error.code, 'authentication') + assert.doesNotMatch(error.message, new RegExp(apiKey)) + return true + }, + ) +}) + +test('free balance request explains that Version 4 keys are unsupported', async () => { + const apiKey = 'semrush-version-4-secret' + await assert.rejects( + new SemrushClient({ + apiKey, + fetch: async () => + new Response( + JSON.stringify({ + errors: [{ field: 'key', message: `invalid api key: ${apiKey}` }], + }), + { status: 400 }, + ), + }).apiUnitBalance(), + (error) => { + assert.ok(error instanceof ProviderError) + assert.equal(error.code, 'authentication') + assert.match(error.message, /permanent Version 3 API Key/) + assert.doesNotMatch(error.message, new RegExp(apiKey)) + return true + }, + ) +}) + +test('free balance request rejects malformed values and unsafe integers', async () => { + for (const body of ['not-a-number', '999999999999999999999999']) { + await assert.rejects( + new SemrushClient({ + apiKey: 'semrush-test-secret', + fetch: async () => new Response(body), + }).apiUnitBalance(), + (error) => + error instanceof ProviderError && error.code === 'invalid-response', + ) + } +}) + +test('paid reports preflight units, parse CSV, and cache without the key', async () => { + const database = cacheDatabase() + const apiKey = 'semrush-paid-secret' + let balanceCalls = 0 + let reportCalls = 0 + const client = new SemrushClient({ + apiKey, + database, + balanceUrl: 'https://provider.invalid/balance', + baseUrl: 'https://provider.invalid/report', + now: () => new Date('2026-07-24T12:00:00.000Z'), + fetch: async (url) => { + const parsed = new URL(url) + assert.equal(parsed.searchParams.get('key'), apiKey) + if (parsed.pathname === '/balance') { + balanceCalls += 1 + return new Response('1000') + } + reportCalls += 1 + assert.equal(parsed.searchParams.get('type'), 'phrase_these') + assert.equal(parsed.searchParams.get('export_columns'), 'Ph,Nq') + return new Response('"Keyword";"Search Volume"\n"zero keyword";"0"\n') + }, + }) + const request = { + operation: 'keyword-metrics', + reportType: 'phrase_these', + parameters: { + phrase: 'zero keyword', + database: 'us', + }, + columns: ['Ph', 'Nq'] as const, + maximumResponseRows: 1, + unitsPerLine: 10, + } + const first = await client.report(request) + const cached = await client.report(request) + + assert.equal(balanceCalls, 1) + assert.equal(reportCalls, 1) + assert.equal(first.cost.native?.estimatedUnits, 10) + assert.equal(first.cost.native?.actualUnits, 10) + assert.equal(first.cost.actualMicros, null) + assert.equal(cached.cache.status, 'hit') + assert.equal(cached.cost.native?.actualUnits, 0) + + const stored = database + .prepare( + 'SELECT credential_scope, request_json, response_json FROM provider_cache', + ) + .get() as { + credential_scope: string + request_json: string + response_json: string + } + assert.doesNotMatch(JSON.stringify(stored), new RegExp(apiKey)) + assert.equal('key' in JSON.parse(stored.request_json), false) + database.close() +}) + +test('case-sensitive API keys never share cached Semrush data', async () => { + const database = cacheDatabase() + let reportCalls = 0 + const run = (apiKey: string) => + new SemrushClient({ + apiKey, + database, + balanceUrl: 'https://provider.invalid/balance', + baseUrl: 'https://provider.invalid/report', + fetch: async (url) => { + if (new URL(url).pathname === '/balance') return new Response('1000') + reportCalls += 1 + return new Response('Keyword\none') + }, + }).report({ + operation: 'keyword-metrics', + reportType: 'phrase_these', + parameters: { phrase: 'one', database: 'us' }, + columns: ['Ph'], + maximumResponseRows: 1, + unitsPerLine: 10, + }) + + await run('CaseSensitiveKey') + await run('casesensitivekey') + assert.equal(reportCalls, 2) + database.close() +}) + +test('paid reports stop before acquisition when the unit balance is too low', async () => { + const database = cacheDatabase() + let reportCalls = 0 + await assert.rejects( + new SemrushClient({ + apiKey: 'semrush-test-secret', + database, + balanceUrl: 'https://provider.invalid/balance', + baseUrl: 'https://provider.invalid/report', + fetch: async (url) => { + if (new URL(url).pathname === '/balance') return new Response('9') + reportCalls += 1 + return new Response('Keyword\nunexpected') + }, + }).report({ + operation: 'keyword-metrics', + reportType: 'phrase_these', + parameters: { phrase: 'one', database: 'us' }, + columns: ['Ph'], + maximumResponseRows: 1, + unitsPerLine: 10, + }), + (error) => error instanceof ProviderError && error.code === 'budget-limit', + ) + assert.equal(reportCalls, 0) + database.close() +}) + +test('paid reports distinguish empty results and redact provider errors', async () => { + const database = cacheDatabase() + const run = (body: string) => + new SemrushClient({ + apiKey: 'semrush-test-secret', + database, + balanceUrl: 'https://provider.invalid/balance', + baseUrl: 'https://provider.invalid/report', + fetch: async (url) => + new URL(url).pathname === '/balance' + ? new Response('1000') + : new Response(body), + }).report({ + operation: 'keyword-metrics', + reportType: 'phrase_these', + parameters: { phrase: 'one', database: 'us' }, + columns: ['Ph'], + maximumResponseRows: 1, + unitsPerLine: 10, + refresh: true, + }) + + assert.deepEqual((await run('ERROR :: 50 :: NOTHING FOUND')).table, { + headers: ['Ph'], + rows: [], + }) + await assert.rejects( + run('ERROR :: 120 :: WRONG KEY semrush-test-secret'), + (error) => { + assert.ok(error instanceof ProviderError) + assert.equal(error.code, 'authentication') + assert.doesNotMatch(error.message, /semrush-test-secret/u) + assert.equal(error.cause, undefined) + return true + }, + ) + database.close() +}) + +test('paid reports reject malformed, over-limit, and unexpected CSV columns', async () => { + for (const body of ['Keyword\none\ntwo', '"Keyword\none']) { + const database = cacheDatabase() + await assert.rejects( + new SemrushClient({ + apiKey: 'semrush-test-secret', + database, + balanceUrl: 'https://provider.invalid/balance', + baseUrl: 'https://provider.invalid/report', + fetch: async (url) => + new URL(url).pathname === '/balance' + ? new Response('1000') + : new Response(body), + }).report({ + operation: 'keyword-metrics', + reportType: 'phrase_these', + parameters: { phrase: 'one', database: 'us' }, + columns: ['Ph'], + maximumResponseRows: 1, + unitsPerLine: 10, + refresh: true, + }), + (error) => + error instanceof ProviderError && error.code === 'invalid-response', + ) + database.close() + } +}) diff --git a/packages/core/src/providers/semrush/client.ts b/packages/core/src/providers/semrush/client.ts new file mode 100644 index 00000000..36a52605 --- /dev/null +++ b/packages/core/src/providers/semrush/client.ts @@ -0,0 +1,424 @@ +import { fetch } from 'undici' +import type Database from '../../storage/sqlite.js' +import { + providerCredentialScope, + readProviderCache, + writeProviderCache, +} from '../cache.js' +import type { + ProviderCacheEvidence, + ProviderCostEvidence, + ProviderWarning, +} from '../contracts.js' +import { ProviderError, type ProviderErrorCode } from '../errors.js' +import { type ProviderFetch, providerRequestText } from '../transport.js' +import { readSemrushApiKey } from './credentials.js' +import { + parseSemrushCsv, + type SemrushCsvTable, + semrushCsvTableSchema, +} from './csv.js' +import type { SemrushColumn } from './mapping.js' + +const API_BASE_URL = 'https://api.semrush.com/' +const API_UNIT_BALANCE_URL = 'https://www.semrush.com/users/countapiunits.html' +const DEFAULT_TIMEOUT_MS = 10_000 +const MAX_BALANCE_RESPONSE_BYTES = 1_024 +const MAX_REPORT_RESPONSE_BYTES = 5 * 1024 * 1024 +const DEFAULT_REPORT_TTL_MS = 7 * 24 * 60 * 60 * 1_000 +const PARAMETER_NAME = /^[a-z][a-z0-9_]*$/u +const RESERVED_PARAMETERS = new Set([ + 'export_columns', + 'export_decode', + 'export_escape', + 'key', + 'type', +]) + +export type SemrushBalance = { + remainingUnits: number + observedAt: string +} + +export type SemrushReportRequest = { + operation: string + reportType: string + parameters: Record + columns: readonly SemrushColumn[] + maximumResponseRows: number + unitsPerLine: number + ttlMs?: number + refresh?: boolean +} + +export type SemrushReportSnapshot = { + table: SemrushCsvTable + observedAt: string + returnedRows: number + cache: ProviderCacheEvidence + cost: ProviderCostEvidence + warnings: ProviderWarning[] +} + +export type SemrushClientOptions = { + apiKey?: string + credentials?: () => string | undefined | Promise + fetch?: ProviderFetch + baseUrl?: string + balanceUrl?: string + timeoutMs?: number + maxReportResponseBytes?: number + now?: () => Date + database?: Database.Database + reportTtlMs?: number +} + +function providerError(input: { + operation: string + code: ProviderErrorCode + message: string + status?: number + retryable?: boolean +}): ProviderError { + return new ProviderError({ + provider: 'semrush', + ...input, + }) +} + +function redactedError(error: unknown, operation: string): ProviderError { + if (error instanceof ProviderError) { + if (operation === 'api-unit-balance' && error.status === 400) { + return providerError({ + operation, + code: 'authentication', + message: + 'Semrush rejected this key. Use the permanent Version 3 API Key; Version 4 keys are not supported.', + status: error.status, + }) + } + return providerError({ + operation, + code: error.code, + message: error.message, + ...(error.status === null ? {} : { status: error.status }), + retryable: error.retryable, + }) + } + return providerError({ + operation, + code: 'remote-error', + message: 'Semrush request failed before a valid response arrived.', + retryable: true, + }) +} + +function responseErrorCode(text: string): number | null { + const match = + /^\s*ERROR\s*::\s*(\d+)\s*::/iu.exec(text) ?? + /^\s*ERROR\s+(\d+)\s*::/iu.exec(text) + return match?.[1] ? Number(match[1]) : null +} + +function semrushResponseError(code: number, operation: string): ProviderError { + if ([110, 120, 130].includes(code)) { + return providerError({ + operation, + code: 'authentication', + message: 'Semrush rejected the configured API key or API access.', + }) + } + if ([131, 134, 429].includes(code)) { + return providerError({ + operation, + code: 'rate-limit', + message: 'Semrush rate limited the request.', + retryable: true, + }) + } + if (code === 132) { + return providerError({ + operation, + code: 'budget-limit', + message: 'Semrush has no API units remaining for this request.', + }) + } + if ( + (code >= 40 && code <= 48) || + [133, 135, 402, 605, 613].includes(code) || + code >= 10_000 + ) { + return providerError({ + operation, + code: 'configuration', + message: `Semrush rejected the report parameters (error ${code}).`, + }) + } + return providerError({ + operation, + code: 'remote-error', + message: `Semrush could not complete the report (error ${code}).`, + }) +} + +function validateReport(input: SemrushReportRequest): void { + if ( + !/^[a-z][a-z0-9-]{1,63}$/u.test(input.operation) || + !/^[a-z][a-z0-9_]{1,63}$/u.test(input.reportType) || + input.columns.length < 1 || + new Set(input.columns).size !== input.columns.length || + !Number.isSafeInteger(input.maximumResponseRows) || + input.maximumResponseRows < 1 || + input.maximumResponseRows > 1_000 || + !Number.isSafeInteger(input.unitsPerLine) || + input.unitsPerLine < 1 || + input.unitsPerLine > 1_000 + ) { + throw providerError({ + operation: input.operation || 'report', + code: 'configuration', + message: 'Semrush received an invalid bounded report request.', + }) + } + for (const [name, value] of Object.entries(input.parameters)) { + if ( + !PARAMETER_NAME.test(name) || + RESERVED_PARAMETERS.has(name) || + (typeof value === 'string' && + (value.length > 10_000 || value.includes('\0'))) || + (typeof value === 'number' && !Number.isSafeInteger(value)) + ) { + throw providerError({ + operation: input.operation, + code: 'configuration', + message: 'Semrush received an invalid report parameter.', + }) + } + } +} + +export class SemrushClient { + private readonly apiKey?: string + private readonly credentials: SemrushClientOptions['credentials'] + private readonly fetch: ProviderFetch + private readonly baseUrl: string + private readonly balanceUrl: string + private readonly timeoutMs: number + private readonly maxReportResponseBytes: number + private readonly now: () => Date + private readonly database: Database.Database | undefined + private readonly reportTtlMs: number + + constructor(options: SemrushClientOptions = {}) { + this.apiKey = options.apiKey?.trim() + this.credentials = options.credentials + this.fetch = options.fetch ?? fetch + this.baseUrl = options.baseUrl ?? API_BASE_URL + this.balanceUrl = options.balanceUrl ?? API_UNIT_BALANCE_URL + this.timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS + this.maxReportResponseBytes = + options.maxReportResponseBytes ?? MAX_REPORT_RESPONSE_BYTES + this.now = options.now ?? (() => new Date()) + this.database = options.database + this.reportTtlMs = options.reportTtlMs ?? DEFAULT_REPORT_TTL_MS + } + + private async getApiKey(operation: string): Promise { + const apiKey = + this.apiKey ?? + (await this.credentials?.()) ?? + (await readSemrushApiKey())?.apiKey + if (!apiKey?.trim() || apiKey.trim().length > 4_096) { + throw providerError({ + operation, + code: 'configuration', + message: + 'Semrush is not connected. Run `seo providers semrush connect`, or set SEO_SEMRUSH_API_KEY for this process.', + }) + } + return apiKey.trim() + } + + private async balanceForKey(apiKey: string): Promise { + const url = new URL(this.balanceUrl) + url.searchParams.set('key', apiKey) + let text: string + try { + text = await providerRequestText({ + provider: 'semrush', + operation: 'api-unit-balance', + url, + fetch: this.fetch, + maxResponseBytes: MAX_BALANCE_RESPONSE_BYTES, + timeoutMs: this.timeoutMs, + retry: 'safe', + }) + } catch (error) { + throw redactedError(error, 'api-unit-balance') + } + + const errorCode = responseErrorCode(text) + if (errorCode !== null) { + throw semrushResponseError(errorCode, 'api-unit-balance') + } + const normalized = text.trim().replaceAll(',', '') + const remainingUnits = Number(normalized) + if ( + !/^\d+$/u.test(normalized) || + !Number.isSafeInteger(remainingUnits) || + remainingUnits < 0 + ) { + throw providerError({ + operation: 'api-unit-balance', + code: 'invalid-response', + message: 'Semrush returned an invalid API unit balance.', + }) + } + return { + remainingUnits, + observedAt: this.now().toISOString(), + } + } + + async apiUnitBalance(): Promise { + return this.balanceForKey(await this.getApiKey('api-unit-balance')) + } + + async report(input: SemrushReportRequest): Promise { + validateReport(input) + const apiKey = await this.getApiKey(input.operation) + const credentialScope = providerCredentialScope('semrush', apiKey) + const cacheRequest = { + reportType: input.reportType, + parameters: input.parameters, + columns: input.columns, + maximumResponseRows: input.maximumResponseRows, + unitsPerLine: input.unitsPerLine, + } + const cacheKey = { + provider: 'semrush' as const, + credentialScope, + operation: input.operation, + request: cacheRequest, + } + const cached = input.refresh + ? null + : readProviderCache(cacheKey, semrushCsvTableSchema, { + database: this.database, + now: this.now().getTime(), + }) + if (cached) { + return { + table: cached.data, + observedAt: cached.storedAt, + returnedRows: cached.rowCount ?? cached.data.rows.length, + cache: { + status: 'hit', + storedAt: cached.storedAt, + expiresAt: cached.expiresAt, + }, + cost: { + currency: 'USD', + estimatedMicros: 0, + actualMicros: 0, + taskIds: [], + native: { + unit: 'api-unit', + estimatedUnits: 0, + actualUnits: 0, + remainingBefore: null, + }, + }, + warnings: [], + } + } + + const estimatedUnits = input.maximumResponseRows * input.unitsPerLine + const balance = await this.balanceForKey(apiKey) + if (balance.remainingUnits < estimatedUnits) { + throw providerError({ + operation: input.operation, + code: 'budget-limit', + message: `Semrush needs up to ${estimatedUnits} API units for this bounded report, but ${balance.remainingUnits} remain.`, + }) + } + + const url = new URL(this.baseUrl) + url.searchParams.set('type', input.reportType) + url.searchParams.set('key', apiKey) + url.searchParams.set('export_columns', input.columns.join(',')) + url.searchParams.set('export_escape', '1') + url.searchParams.set('export_decode', '1') + for (const [name, value] of Object.entries(input.parameters)) { + url.searchParams.set(name, String(value)) + } + + let text: string + try { + text = await providerRequestText({ + provider: 'semrush', + operation: input.operation, + url, + fetch: this.fetch, + maxResponseBytes: this.maxReportResponseBytes, + timeoutMs: this.timeoutMs, + retry: 'never', + }) + } catch (error) { + throw redactedError(error, input.operation) + } + + const errorCode = responseErrorCode(text) + const table = + errorCode === 50 + ? { headers: [...input.columns], rows: [] } + : (() => { + if (errorCode !== null) { + throw semrushResponseError(errorCode, input.operation) + } + return parseSemrushCsv(text, input.maximumResponseRows) + })() + const returnedRows = table.rows.length + const stored = writeProviderCache( + cacheKey, + { + data: table, + ttlMs: input.ttlMs ?? this.reportTtlMs, + rowCount: returnedRows, + sourceCostMicros: null, + taskIds: [], + }, + { database: this.database, now: this.now().getTime() }, + ) + return { + table, + observedAt: stored.storedAt, + returnedRows, + cache: { + status: 'miss', + storedAt: stored.storedAt, + expiresAt: stored.expiresAt, + }, + cost: { + currency: 'USD', + estimatedMicros: null, + actualMicros: null, + taskIds: [], + native: { + unit: 'api-unit', + estimatedUnits, + actualUnits: returnedRows * input.unitsPerLine, + remainingBefore: balance.remainingUnits, + }, + }, + warnings: [ + { + code: 'provider-cost-not-denominated-usd', + field: 'cost', + message: + 'Semrush bills this report in API units; no USD conversion was inferred.', + }, + ], + } + } +} diff --git a/packages/core/src/providers/semrush/credentials.test.ts b/packages/core/src/providers/semrush/credentials.test.ts new file mode 100644 index 00000000..0cecd1fa --- /dev/null +++ b/packages/core/src/providers/semrush/credentials.test.ts @@ -0,0 +1,174 @@ +import assert from 'node:assert/strict' +import { + existsSync, + mkdtempSync, + readFileSync, + rmSync, + statSync, +} from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { after, beforeEach, test } from 'node:test' +import { getSeoCliPaths } from '../../paths.js' +import { readConfig, writeConfig } from '../../storage/config.js' +import { setKeyringForTests } from '../../storage/keyring.js' +import { writeProviderSecret } from '../../storage/provider-secrets.js' +import { configSchema } from '../../types.js' +import { + deleteSemrushApiKey, + readSemrushApiKey, + SEMRUSH_API_KEY_ENV, + SEMRUSH_API_KEY_SECRET, + writeSemrushApiKey, +} from './credentials.js' + +class MemoryKeyring { + readonly values = new Map() + unavailable = false + + async getPassword(service: string, account: string): Promise { + if (this.unavailable) throw new Error('Unavailable') + return this.values.get(`${service}:${account}`) ?? null + } + + async setPassword( + service: string, + account: string, + password: string, + ): Promise { + if (this.unavailable) throw new Error('Unavailable') + this.values.set(`${service}:${account}`, password) + } + + async deletePassword(service: string, account: string): Promise { + if (this.unavailable) throw new Error('Unavailable') + return this.values.delete(`${service}:${account}`) + } +} + +const configDir = mkdtempSync(join(tmpdir(), 'seo-semrush-credentials-')) +const previousConfigDir = process.env.SEO_CONFIG_DIR +const previousApiKey = process.env[SEMRUSH_API_KEY_ENV] +const keyring = new MemoryKeyring() + +beforeEach(() => { + process.env.SEO_CONFIG_DIR = configDir + delete process.env[SEMRUSH_API_KEY_ENV] + rmSync(configDir, { recursive: true, force: true }) + keyring.values.clear() + keyring.unavailable = false + setKeyringForTests(keyring) +}) + +after(() => { + rmSync(configDir, { recursive: true, force: true }) + if (previousConfigDir === undefined) delete process.env.SEO_CONFIG_DIR + else process.env.SEO_CONFIG_DIR = previousConfigDir + if (previousApiKey === undefined) delete process.env[SEMRUSH_API_KEY_ENV] + else process.env[SEMRUSH_API_KEY_ENV] = previousApiKey + setKeyringForTests() +}) + +test('environment API key takes precedence without being persisted', async () => { + process.env[SEMRUSH_API_KEY_ENV] = ' environment-api-key ' + + assert.deepEqual(await readSemrushApiKey(), { + apiKey: 'environment-api-key', + source: 'environment', + migrated: false, + }) + assert.equal(existsSync(getSeoCliPaths().providerSecretsFile), false) +}) + +test('API key uses the system keychain', async () => { + writeConfig(configSchema.parse({})) + + assert.equal(await writeSemrushApiKey('saved-api-key'), 'keychain') + assert.deepEqual(await readSemrushApiKey(), { + apiKey: 'saved-api-key', + source: 'keychain', + migrated: false, + }) + assert.equal( + keyring.values.get(`seo:provider:${SEMRUSH_API_KEY_SECRET}`), + 'saved-api-key', + ) +}) + +test('API key falls back to a private local file', async () => { + keyring.unavailable = true + writeConfig(configSchema.parse({})) + + assert.equal(await writeSemrushApiKey('file-api-key'), 'file') + const path = getSeoCliPaths().providerSecretsFile + assert.equal(statSync(path).mode & 0o777, 0o600) + assert.deepEqual(await readSemrushApiKey(), { + apiKey: 'file-api-key', + source: 'file', + migrated: false, + }) +}) + +test('legacy config API key migrates only after secure storage succeeds', async () => { + writeConfig( + configSchema.parse({ + providers: { + semrushApiKey: 'legacy-api-key', + prefer: 'authoritative', + }, + }), + ) + + assert.deepEqual(await readSemrushApiKey(), { + apiKey: 'legacy-api-key', + source: 'keychain', + migrated: true, + }) + assert.equal(readConfig().providers.semrushApiKey, undefined) + assert.doesNotMatch( + readFileSync(getSeoCliPaths().configFile, 'utf8'), + /legacy-api-key/, + ) +}) + +test('temporary versioned credential migrates to the single Version 3 key', async () => { + writeConfig( + configSchema.parse({ + security: { useKeychain: false }, + }), + ) + await writeProviderSecret( + SEMRUSH_API_KEY_SECRET, + JSON.stringify({ + schemaVersion: 1, + apiKeys: { + v3: 'version-3-api-key', + }, + }), + ) + + assert.deepEqual(await readSemrushApiKey(), { + apiKey: 'version-3-api-key', + source: 'file', + migrated: true, + }) + assert.equal( + JSON.parse(readFileSync(getSeoCliPaths().providerSecretsFile, 'utf8')) + .secrets[SEMRUSH_API_KEY_SECRET], + 'version-3-api-key', + ) +}) + +test('disconnect removes saved and legacy API keys', async () => { + writeConfig( + configSchema.parse({ + providers: { semrushApiKey: 'legacy-api-key' }, + }), + ) + await writeSemrushApiKey('saved-api-key') + + await deleteSemrushApiKey() + + assert.equal(await readSemrushApiKey(), undefined) + assert.equal(readConfig().providers.semrushApiKey, undefined) +}) diff --git a/packages/core/src/providers/semrush/credentials.ts b/packages/core/src/providers/semrush/credentials.ts new file mode 100644 index 00000000..bbd88b6b --- /dev/null +++ b/packages/core/src/providers/semrush/credentials.ts @@ -0,0 +1,119 @@ +import { readConfig, writeConfig } from '../../storage/config.js' +import { + deleteProviderSecret, + PROVIDER_SECRET_NAMES, + type ProviderSecretSource, + readProviderSecret, + writeProviderSecret, +} from '../../storage/provider-secrets.js' +import { ProviderError } from '../errors.js' + +export const SEMRUSH_API_KEY_ENV = 'SEO_SEMRUSH_API_KEY' +export const SEMRUSH_API_KEY_SECRET = PROVIDER_SECRET_NAMES.semrushApiKey + +export type StoredSemrushApiKey = { + apiKey: string + source: ProviderSecretSource + migrated: boolean +} + +function configurationError(message: string): ProviderError { + return new ProviderError({ + provider: 'semrush', + operation: 'credentials', + code: 'configuration', + message, + }) +} + +function normalizeApiKey(value: string): string { + const apiKey = value.trim() + if (!apiKey || apiKey.length > 4_096) { + throw configurationError('Semrush needs a valid Version 3 API key.') + } + return apiKey +} + +function versionedV3ApiKey(value: string): string | undefined { + if (!value.trim().startsWith('{')) return undefined + try { + const parsed = JSON.parse(value) as { + schemaVersion?: unknown + apiKeys?: { v3?: unknown } + } + if (parsed.schemaVersion === 1 && typeof parsed.apiKeys?.v3 === 'string') { + return normalizeApiKey(parsed.apiKeys.v3) + } + } catch { + // The configuration error below is safe and does not expose the secret. + } + throw configurationError( + 'The saved Semrush credential does not contain a Version 3 API key. Disconnect Semrush, then connect it again with the permanent Version 3 key.', + ) +} + +function clearLegacyConfigApiKey(): boolean { + const config = readConfig() + if (!config.providers.semrushApiKey) return false + writeConfig({ + ...config, + providers: { + ...config.providers, + semrushApiKey: undefined, + }, + }) + return true +} + +async function migrateLegacyApiKey(): Promise { + const legacyApiKey = readConfig().providers.semrushApiKey + if (!legacyApiKey) return undefined + const apiKey = normalizeApiKey(legacyApiKey) + const source = await writeSemrushApiKey(apiKey) + return { apiKey, source, migrated: true } +} + +export async function writeSemrushApiKey( + value: string, +): Promise> { + const apiKey = normalizeApiKey(value) + const source = await writeProviderSecret(SEMRUSH_API_KEY_SECRET, apiKey) + clearLegacyConfigApiKey() + return source +} + +export async function readSemrushApiKey( + input: { env?: NodeJS.ProcessEnv } = {}, +): Promise { + const credential = await readProviderSecret({ + name: SEMRUSH_API_KEY_SECRET, + envVar: SEMRUSH_API_KEY_ENV, + env: input.env, + }) + if (credential) { + const versionedApiKey = + credential.source === 'environment' + ? undefined + : versionedV3ApiKey(credential.value) + const apiKey = versionedApiKey ?? normalizeApiKey(credential.value) + const migratedVersionedCredential = Boolean(versionedApiKey) + const source = migratedVersionedCredential + ? await writeProviderSecret(SEMRUSH_API_KEY_SECRET, apiKey) + : credential.source + const migratedLegacyConfig = clearLegacyConfigApiKey() + return { + apiKey, + source, + migrated: + credential.source === 'environment' + ? false + : migratedVersionedCredential || migratedLegacyConfig, + } + } + return migrateLegacyApiKey() +} + +export async function deleteSemrushApiKey(): Promise { + await deleteProviderSecret(SEMRUSH_API_KEY_SECRET) + clearLegacyConfigApiKey() +} diff --git a/packages/core/src/providers/semrush/csv.test.ts b/packages/core/src/providers/semrush/csv.test.ts new file mode 100644 index 00000000..380a2074 --- /dev/null +++ b/packages/core/src/providers/semrush/csv.test.ts @@ -0,0 +1,31 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { ProviderError } from '../errors.js' +import { parseSemrushCsv } from './csv.js' + +test('Semrush CSV handles escaped separators, quotes, and newlines', () => { + assert.deepEqual( + parseSemrushCsv( + '\uFEFF"Keyword";"Url";"Note"\r\n"seo; tools";"https://example.com/a";"one ""quoted""\nline"\r\n', + 1, + ), + { + headers: ['Keyword', 'Url', 'Note'], + rows: [['seo; tools', 'https://example.com/a', 'one "quoted"\nline']], + }, + ) +}) + +test('Semrush CSV rejects malformed and over-limit responses', () => { + for (const body of [ + '"Keyword\nseo', + 'Keyword;Volume\nseo', + 'Keyword\none\ntwo', + ]) { + assert.throws( + () => parseSemrushCsv(body, 1), + (error) => + error instanceof ProviderError && error.code === 'invalid-response', + ) + } +}) diff --git a/packages/core/src/providers/semrush/csv.ts b/packages/core/src/providers/semrush/csv.ts index d0cdcd07..379bca16 100644 --- a/packages/core/src/providers/semrush/csv.ts +++ b/packages/core/src/providers/semrush/csv.ts @@ -1,8 +1,114 @@ +import { z } from 'zod' +import { ProviderError } from '../errors.js' + +const MAX_CELL_BYTES = 1_000_000 + +export const semrushCsvTableSchema = z + .object({ + headers: z.array(z.string().trim().min(1).max(200)).min(1).max(100), + rows: z.array(z.array(z.string().max(MAX_CELL_BYTES))), + }) + .strict() + .superRefine((table, context) => { + if (new Set(table.headers).size !== table.headers.length) { + context.addIssue({ + code: 'custom', + message: 'Semrush returned duplicate CSV headers.', + }) + } + for (const [index, row] of table.rows.entries()) { + if (row.length !== table.headers.length) { + context.addIssue({ + code: 'custom', + message: `Semrush CSV row ${index + 1} has the wrong column count.`, + }) + break + } + } + }) + +export type SemrushCsvTable = z.infer + +function invalidCsv(message: string): ProviderError { + return new ProviderError({ + provider: 'semrush', + operation: 'csv', + code: 'invalid-response', + message, + }) +} + export function parseSemicolonCsv(text: string): string[][] { - return text - .trim() - .split('\n') - .map((line) => - line.split(';').map((cell) => cell.replace(/^"|"$/g, '').trim()), + const input = text.replace(/^\uFEFF/u, '') + const records: string[][] = [] + let record: string[] = [] + let field = '' + let quoted = false + + const pushField = () => { + record.push(field) + field = '' + } + const pushRecord = () => { + pushField() + if (record.some((value) => value.length > 0)) records.push(record) + record = [] + } + + for (let index = 0; index < input.length; index += 1) { + const character = input[index] + if (quoted) { + if (character === '"') { + if (input[index + 1] === '"') { + field += '"' + index += 1 + } else { + quoted = false + } + } else { + field += character + } + } else if (character === '"' && field.length === 0) { + quoted = true + } else if (character === ';') { + pushField() + } else if (character === '\n') { + if (field.endsWith('\r')) field = field.slice(0, -1) + pushRecord() + } else { + field += character + } + if (Buffer.byteLength(field) > MAX_CELL_BYTES) { + throw invalidCsv( + `Semrush returned a CSV cell larger than ${MAX_CELL_BYTES} bytes.`, + ) + } + } + if (quoted) throw invalidCsv('Semrush returned CSV with an unclosed quote.') + if (field || record.length) { + if (field.endsWith('\r')) field = field.slice(0, -1) + pushRecord() + } + if (records[0]?.[0]) { + records[0][0] = records[0][0].replace(/^\uFEFF/u, '') + } + return records +} + +export function parseSemrushCsv( + text: string, + maximumRows: number, +): SemrushCsvTable { + const [headers, ...rows] = parseSemicolonCsv(text) + if (!headers) throw invalidCsv('Semrush returned an empty CSV response.') + if (rows.length > maximumRows) { + throw invalidCsv( + `Semrush returned ${rows.length} rows, above the ${maximumRows}-row request bound.`, ) + } + const parsed = semrushCsvTableSchema.safeParse({ headers, rows }) + if (!parsed.success) { + throw invalidCsv('Semrush returned malformed CSV data.') + } + return parsed.data } diff --git a/packages/core/src/providers/semrush/domain-overview.ts b/packages/core/src/providers/semrush/domain-overview.ts new file mode 100644 index 00000000..6f86938c --- /dev/null +++ b/packages/core/src/providers/semrush/domain-overview.ts @@ -0,0 +1,79 @@ +import type { ProviderEvidence } from '../contracts.js' +import type { + DomainOverview, + DomainOverviewRequest, +} from '../domain-contracts.js' +import type { SemrushClient } from './client.js' +import { + domain, + evidence, + mappedWarnings, + organicFootprint, +} from './domain-research-shared.js' +import { semrushRecords } from './mapping.js' +import { semrushMarket } from './market.js' + +const COLUMNS = ['Dn', 'Or', 'Ot', 'Oc'] as const + +export async function semrushDomainOverview( + client: Pick, + input: DomainOverviewRequest, +): Promise> { + const { market, database } = semrushMarket(input.market, 'domain-overview') + const target = domain(input.domain, 'domain-overview') + const snapshot = await client.report({ + operation: 'domain-overview', + reportType: 'domain_rank', + parameters: { domain: target, database }, + columns: COLUMNS, + maximumResponseRows: 1, + unitsPerLine: 10, + refresh: input.refresh, + }) + const records = semrushRecords(snapshot.table, COLUMNS) + let row = records[0] + let invalidRows = 0 + if (row) { + try { + if (!row.Dn || domain(row.Dn, 'domain-overview') !== target) { + invalidRows = 1 + row = undefined + } + } catch { + invalidRows = 1 + row = undefined + } + } + return evidence({ + capability: 'domain-overview', + data: { + domain: target, + organic: organicFootprint({ + traffic: row?.Ot, + keywords: row?.Or, + cost: row?.Oc, + }), + }, + market, + snapshot, + coverage: { + requestedRows: 1, + returnedRows: snapshot.returnedRows, + retainedRows: row ? 1 : 0, + invalidRows, + providerTotalRows: null, + completeness: invalidRows ? 'invalid' : 'complete', + nextCursor: null, + }, + limit: 1, + filters: { + database, + countryCode: market.countryCode, + languageCode: market.languageCode, + domain: target, + apiVersion: 3, + }, + sort: [], + warnings: mappedWarnings(market, snapshot, invalidRows, 'overview'), + }) +} diff --git a/packages/core/src/providers/semrush/domain-research-shared.ts b/packages/core/src/providers/semrush/domain-research-shared.ts new file mode 100644 index 00000000..55a9936f --- /dev/null +++ b/packages/core/src/providers/semrush/domain-research-shared.ts @@ -0,0 +1,299 @@ +import type { + ProviderCacheEvidence, + ProviderCostEvidence, + ProviderCoverage, + ProviderEvidence, + ProviderValue, + ProviderWarning, + SearchMarket, +} from '../contracts.js' +import { unavailableValue } from '../contracts.js' +import type { + OrganicFootprint, + RankingDistribution, +} from '../domain-contracts.js' +import { ProviderError } from '../errors.js' +import type { SemrushReportSnapshot } from './client.js' +import { compareCodepoints, semrushNumber } from './mapping.js' +import { semrushMarketWarnings } from './market.js' + +export const MAX_DOMAIN_ROWS = 1_000 +export const MAX_DOMAIN_OFFSET = 100_000 + +export function domain(value: string, operation = 'domain-research'): string { + const raw = value.trim().toLowerCase() + let url: URL + try { + url = new URL(raw.includes('://') ? raw : `https://${raw}`) + } catch { + throw invalidDomain(operation) + } + const hostname = url.hostname.replace(/^www\./u, '').replace(/\.$/u, '') + if ( + !hostname || + hostname.length > 253 || + hostname.includes('..') || + !hostname.includes('.') || + !/^[a-z0-9.-]+$/u.test(hostname) + ) { + throw invalidDomain(operation) + } + return hostname +} + +function invalidDomain(operation: string): ProviderError { + return new ProviderError({ + provider: 'semrush', + operation, + code: 'configuration', + message: 'Use a valid domain.', + }) +} + +export function safeUrl(value: string | undefined): string | null { + if (!value) return null + try { + const url = new URL(value) + if (!['http:', 'https:'].includes(url.protocol)) return null + url.username = '' + url.password = '' + return url.toString() + } catch { + return null + } +} + +export function rowLimit( + limit: number, + offset: number, + operation: string, +): void { + if ( + !Number.isSafeInteger(limit) || + limit < 1 || + limit > MAX_DOMAIN_ROWS || + !Number.isSafeInteger(offset) || + offset < 0 || + offset > MAX_DOMAIN_OFFSET || + limit + offset > 1_000_000 + ) { + throw new ProviderError({ + provider: 'semrush', + operation, + code: 'configuration', + message: `Semrush domain research requires a limit from 1 to ${MAX_DOMAIN_ROWS} and an offset from 0 to ${MAX_DOMAIN_OFFSET}.`, + }) + } +} + +export function organicOnly( + resultTypes: string[] | undefined, + operation: string, +): void { + const types = [...new Set(resultTypes ?? ['organic'])] + if (types.length !== 1 || types[0] !== 'organic') { + throw new ProviderError({ + provider: 'semrush', + operation, + code: 'configuration', + message: + 'Semrush V3 domain research currently supports organic rows only.', + }) + } +} + +export function unavailable(field: string): ProviderValue { + return unavailableValue( + 'unavailable', + `This Semrush V3 report does not return ${field}.`, + ) +} + +export function missing(field: string): ProviderValue { + return unavailableValue('missing', `Semrush omitted ${field}.`) +} + +export function organicFootprint(input: { + traffic?: string + keywords?: string + cost?: string +}): OrganicFootprint { + return { + estimatedMonthlyTraffic: semrushNumber( + input.traffic, + 'estimated organic monthly traffic', + (value) => value >= 0, + ), + rankedKeywords: semrushNumber( + input.keywords, + 'ranked organic keywords', + (value) => Number.isSafeInteger(value) && value >= 0, + ), + estimatedMonthlyTrafficCostUsd: semrushNumber( + input.cost, + 'estimated organic traffic cost', + (value) => value >= 0, + ), + rankings: unavailable('organic ranking distribution'), + newRankings: unavailable('new rankings'), + improvedRankings: unavailable('improved rankings'), + declinedRankings: unavailable('declined rankings'), + lostRankings: unavailable('lost rankings'), + } +} + +export function observedNumber(value: ProviderValue): number { + return value.state === 'observed' ? value.value : -1 +} + +export function dedupeBy(rows: T[], key: (row: T) => string): T[] { + const grouped = new Map() + for (const row of rows) { + const value = key(row) + grouped.set(value, [...(grouped.get(value) ?? []), row]) + } + return [...grouped.entries()] + .sort(([left], [right]) => compareCodepoints(left, right)) + .map( + ([, matches]) => + [...matches].sort((left, right) => + compareCodepoints(JSON.stringify(left), JSON.stringify(right)), + )[0] as T, + ) +} + +export function coverage(input: { + requestedRows: number + returnedRows: number + retainedRows: number + invalidRows: number + offset: number + filtered: boolean +}): ProviderCoverage { + const hasMore = input.returnedRows >= input.requestedRows + return { + requestedRows: input.requestedRows, + returnedRows: input.returnedRows, + retainedRows: input.retainedRows, + invalidRows: input.invalidRows, + providerTotalRows: null, + completeness: + input.invalidRows > 0 + ? 'partial' + : hasMore + ? 'capped' + : input.filtered + ? 'filtered' + : 'complete', + nextCursor: hasMore ? String(input.offset + input.returnedRows) : null, + } +} + +export function combinedCache( + snapshots: SemrushReportSnapshot[], +): ProviderCacheEvidence { + if (snapshots.every((snapshot) => snapshot.cache.status === 'hit')) { + const stored = snapshots + .map((snapshot) => snapshot.cache.storedAt) + .filter((value): value is string => Boolean(value)) + .sort(compareCodepoints) + const expires = snapshots + .map((snapshot) => snapshot.cache.expiresAt) + .filter((value): value is string => Boolean(value)) + .sort(compareCodepoints) + return { + status: 'hit', + storedAt: stored[0] ?? null, + expiresAt: expires[0] ?? null, + } + } + return { status: 'miss', storedAt: null, expiresAt: null } +} + +function sumNullable(values: Array): number | null { + return values.every((value) => value !== null) + ? values.reduce((sum, value) => sum + (value ?? 0), 0) + : null +} + +export function combinedCost( + snapshots: SemrushReportSnapshot[], +): ProviderCostEvidence { + const natives = snapshots.map((snapshot) => snapshot.cost.native) + const remaining = natives + .map((native) => native?.remainingBefore ?? null) + .filter((value): value is number => value !== null) + return { + currency: 'USD', + estimatedMicros: sumNullable( + snapshots.map((snapshot) => snapshot.cost.estimatedMicros), + ), + actualMicros: sumNullable( + snapshots.map((snapshot) => snapshot.cost.actualMicros), + ), + taskIds: [], + native: { + unit: 'api-unit', + estimatedUnits: sumNullable( + natives.map((native) => native?.estimatedUnits ?? null), + ), + actualUnits: sumNullable( + natives.map((native) => native?.actualUnits ?? null), + ), + remainingBefore: remaining.length ? Math.max(...remaining) : null, + }, + } +} + +export function mappedWarnings( + market: SearchMarket, + snapshot: SemrushReportSnapshot, + invalidRows: number, + rowLabel: string, +): ProviderWarning[] { + return [ + ...snapshot.warnings, + ...semrushMarketWarnings(market), + ...(invalidRows + ? [ + { + code: `invalid-${rowLabel}-rows`, + field: 'data.rows', + message: `Semrush returned ${invalidRows} ${rowLabel} row${invalidRows === 1 ? '' : 's'} without the required fields.`, + }, + ] + : []), + ] +} + +export function evidence(input: { + capability: ProviderEvidence['capability'] + data: T + market: SearchMarket + snapshot: SemrushReportSnapshot + coverage: ProviderCoverage + limit: number + filters: Record + sort: string[] + warnings: ProviderWarning[] +}): ProviderEvidence { + return { + schemaVersion: 1, + provider: 'semrush', + capability: input.capability, + data: input.data, + observedAt: input.snapshot.observedAt, + market: input.market, + coverage: input.coverage, + cache: input.snapshot.cache, + cost: input.snapshot.cost, + request: { + operation: input.capability, + endpoint: 'https://api.semrush.com/', + limit: input.limit, + filters: input.filters, + sort: input.sort, + }, + warnings: input.warnings, + } +} diff --git a/packages/core/src/providers/semrush/domain-research.ts b/packages/core/src/providers/semrush/domain-research.ts new file mode 100644 index 00000000..77cb6ce2 --- /dev/null +++ b/packages/core/src/providers/semrush/domain-research.ts @@ -0,0 +1,59 @@ +import type { + DomainOverviewRequest, + DomainResearchProvider, + RankedKeywordsRequest, + RankingPagesRequest, + SerpCompetitorsRequest, +} from '../domain-contracts.js' +import { SemrushClient, type SemrushClientOptions } from './client.js' +import { semrushDomainOverview } from './domain-overview.js' +import { SEMRUSH_V3_MARKETS } from './market.js' +import { semrushRankedKeywords } from './ranked-keywords.js' +import { semrushRankingPages } from './ranking-pages.js' +import { semrushSerpCompetitors } from './serp-competitors.js' + +type DomainResearchClient = Pick + +export type SemrushDomainResearchProviderOptions = SemrushClientOptions & { + client?: DomainResearchClient +} + +export class SemrushDomainResearchProvider implements DomainResearchProvider { + readonly provider = 'semrush' as const + readonly capabilitySupport = [ + 'domain-overview', + 'ranked-keywords', + 'relevant-pages', + 'serp-competitors', + ].map((capability) => ({ + capability: capability as + | 'domain-overview' + | 'ranked-keywords' + | 'relevant-pages' + | 'serp-competitors', + status: 'available' as const, + markets: SEMRUSH_V3_MARKETS, + })) + + private readonly client: DomainResearchClient + + constructor(options: SemrushDomainResearchProviderOptions = {}) { + this.client = options.client ?? new SemrushClient(options) + } + + domainOverview(input: DomainOverviewRequest) { + return semrushDomainOverview(this.client, input) + } + + rankedKeywords(input: RankedKeywordsRequest) { + return semrushRankedKeywords(this.client, input) + } + + rankingPages(input: RankingPagesRequest) { + return semrushRankingPages(this.client, input) + } + + serpCompetitors(input: SerpCompetitorsRequest) { + return semrushSerpCompetitors(this.client, input) + } +} diff --git a/packages/core/src/providers/semrush/keyword-discovery.ts b/packages/core/src/providers/semrush/keyword-discovery.ts new file mode 100644 index 00000000..6bda99c5 --- /dev/null +++ b/packages/core/src/providers/semrush/keyword-discovery.ts @@ -0,0 +1,369 @@ +import type { + KeywordDiscoveryProvider, + KeywordDiscoveryRequest, + KeywordDiscoverySource, + KeywordIdea, + ProviderCacheEvidence, + ProviderCostEvidence, + ProviderEvidence, + ProviderWarning, +} from '../contracts.js' +import { keywordDiscoverySourceSchema } from '../contracts.js' +import { ProviderError } from '../errors.js' +import { + SemrushClient, + type SemrushClientOptions, + type SemrushReportSnapshot, +} from './client.js' +import { + compareCodepoints, + normalizedKeyword, + type SemrushRecord, + semrushMetric, + semrushRecords, +} from './mapping.js' +import { + SEMRUSH_V3_MARKETS, + semrushKeywordDeprecationWarning, + semrushMarket, + semrushMarketWarnings, +} from './market.js' + +const MAX_SEEDS = 5 +const MAX_ROWS = 100 +const COLUMNS = ['Ph', 'Nq', 'Cp', 'Co', 'Nr', 'Kd'] as const +const REPORTS = { + ideas: { reportType: 'phrase_fullsearch', unitsPerLine: 20 }, + related: { reportType: 'phrase_related', unitsPerLine: 40 }, + suggestions: { reportType: 'phrase_questions', unitsPerLine: 40 }, +} as const satisfies Record< + KeywordDiscoverySource, + { reportType: string; unitsPerLine: number } +> + +type KeywordDiscoveryClient = Pick + +export type SemrushKeywordDiscoveryProviderOptions = SemrushClientOptions & { + client?: KeywordDiscoveryClient +} + +type DiscoveryCall = { + seed: string + source: KeywordDiscoverySource + limit: number +} + +type DiscoveryRow = { + row: SemrushRecord + seed: string + source: KeywordDiscoverySource +} + +function plannedCalls( + seeds: string[], + sources: KeywordDiscoverySource[], + limit: number, +): DiscoveryCall[] { + const requests = sources.flatMap((source) => + seeds.map((seed) => ({ source, seed })), + ) + if (limit < requests.length) { + throw new ProviderError({ + provider: 'semrush', + operation: 'keyword-discovery', + code: 'configuration', + message: `Keyword discovery needs a limit of at least ${requests.length} to sample every requested source and seed.`, + }) + } + const base = Math.floor(limit / requests.length) + const remainder = limit % requests.length + return requests.map((request, index) => ({ + ...request, + limit: base + Number(index < remainder), + })) +} + +function combinedCache( + snapshots: SemrushReportSnapshot[], +): ProviderCacheEvidence { + if (snapshots.every((snapshot) => snapshot.cache.status === 'hit')) { + const stored = snapshots + .map((snapshot) => snapshot.cache.storedAt) + .filter((value): value is string => Boolean(value)) + .sort(compareCodepoints) + const expires = snapshots + .map((snapshot) => snapshot.cache.expiresAt) + .filter((value): value is string => Boolean(value)) + .sort(compareCodepoints) + return { + status: 'hit', + storedAt: stored[0] ?? null, + expiresAt: expires[0] ?? null, + } + } + return { status: 'miss', storedAt: null, expiresAt: null } +} + +function sumNullable(values: Array): number | null { + return values.every((value) => value !== null) + ? values.reduce((sum, value) => sum + (value ?? 0), 0) + : null +} + +function combinedCost( + snapshots: SemrushReportSnapshot[], +): ProviderCostEvidence { + const natives = snapshots.map((snapshot) => snapshot.cost.native) + const remaining = natives + .map((native) => native?.remainingBefore ?? null) + .filter((value): value is number => value !== null) + return { + currency: 'USD', + estimatedMicros: sumNullable( + snapshots.map((snapshot) => snapshot.cost.estimatedMicros), + ), + actualMicros: sumNullable( + snapshots.map((snapshot) => snapshot.cost.actualMicros), + ), + taskIds: [], + native: { + unit: 'api-unit', + estimatedUnits: sumNullable( + natives.map((native) => native?.estimatedUnits ?? null), + ), + actualUnits: sumNullable( + natives.map((native) => native?.actualUnits ?? null), + ), + remainingBefore: remaining.length ? Math.max(...remaining) : null, + }, + } +} + +function observedVolume(idea: KeywordIdea): number { + return idea.monthlySearchVolume.state === 'observed' + ? idea.monthlySearchVolume.value + : -1 +} + +export class SemrushKeywordDiscoveryProvider + implements KeywordDiscoveryProvider +{ + readonly provider = 'semrush' as const + readonly capabilitySupport = [ + { + capability: 'keyword-discovery' as const, + status: 'available' as const, + markets: SEMRUSH_V3_MARKETS, + }, + ] as const + + private readonly client: KeywordDiscoveryClient + + constructor(options: SemrushKeywordDiscoveryProviderOptions = {}) { + this.client = options.client ?? new SemrushClient(options) + } + + async discoverKeywords( + input: KeywordDiscoveryRequest, + ): Promise> { + const { market, database } = semrushMarket( + input.market, + 'keyword-discovery', + ) + const seeds = [...new Set(input.seeds.map(normalizedKeyword))] + .filter(Boolean) + .sort(compareCodepoints) + if ( + seeds.length < 1 || + seeds.length > MAX_SEEDS || + seeds.some( + (seed) => + seed.length > 80 || + seed.split(/\s+/u).length > 10 || + seed.includes(';'), + ) + ) { + throw new ProviderError({ + provider: 'semrush', + operation: 'keyword-discovery', + code: 'configuration', + message: + 'Semrush keyword discovery requires 1 to 5 seeds of at most 80 characters and 10 words.', + }) + } + if ( + !Number.isSafeInteger(input.limit) || + input.limit < 1 || + input.limit > MAX_ROWS + ) { + throw new ProviderError({ + provider: 'semrush', + operation: 'keyword-discovery', + code: 'configuration', + message: 'Keyword discovery limit must be from 1 to 100.', + }) + } + const sources = [ + ...new Set( + input.sources.map((source) => + keywordDiscoverySourceSchema.parse(source), + ), + ), + ].sort(compareCodepoints) + if (sources.length < 1 || sources.length > 3) { + throw new ProviderError({ + provider: 'semrush', + operation: 'keyword-discovery', + code: 'configuration', + message: 'Choose 1 to 3 keyword discovery sources.', + }) + } + + const calls = plannedCalls(seeds, sources, input.limit) + const snapshots: SemrushReportSnapshot[] = [] + const rows: DiscoveryRow[] = [] + const warnings: ProviderWarning[] = [ + ...semrushMarketWarnings(market), + semrushKeywordDeprecationWarning(), + ...(sources.includes('suggestions') + ? [ + { + code: 'provider-source-is-questions', + field: 'sources', + message: + 'Semrush suggestions use its question-keyword report rather than a generic autocomplete source.', + }, + ] + : []), + ] + let lastError: ProviderError | undefined + for (const call of calls) { + const report = REPORTS[call.source] + try { + const snapshot = await this.client.report({ + operation: `keyword-discovery-${call.source}`, + reportType: report.reportType, + parameters: { + phrase: call.seed, + database, + display_limit: call.limit, + }, + columns: COLUMNS, + maximumResponseRows: call.limit, + unitsPerLine: report.unitsPerLine, + refresh: input.refresh, + }) + snapshots.push(snapshot) + warnings.push(...snapshot.warnings) + rows.push( + ...semrushRecords(snapshot.table, COLUMNS).map((row) => ({ + row, + seed: call.seed, + source: call.source, + })), + ) + } catch (error) { + if (!(error instanceof ProviderError)) throw error + lastError = error + warnings.push({ + code: 'discovery-request-failed', + field: call.source, + message: `Semrush ${call.source} discovery failed for one seed (${error.code}).`, + }) + } + } + if (!snapshots.length && lastError) throw lastError + + const grouped = new Map() + let invalidRows = 0 + for (const row of rows) { + const keyword = normalizedKeyword(row.row.Ph ?? '') + if (!keyword) { + invalidRows += 1 + continue + } + grouped.set(keyword, [...(grouped.get(keyword) ?? []), row]) + } + const ideas = [...grouped.entries()] + .map(([keyword, matches]) => ({ + ...semrushMetric( + keyword, + matches.map((match) => match.row), + ), + sources: [ + ...new Map( + matches.map((match) => [ + `${match.source}\0${match.seed}`, + { seed: match.seed, source: match.source }, + ]), + ).values(), + ].sort( + (left, right) => + compareCodepoints(left.source, right.source) || + compareCodepoints(left.seed, right.seed), + ), + })) + .sort( + (left, right) => + observedVolume(right) - observedVolume(left) || + compareCodepoints(left.keyword, right.keyword), + ) + .slice(0, input.limit) + const failedCalls = calls.length - snapshots.length + const returnedRows = snapshots.reduce( + (sum, snapshot) => sum + snapshot.returnedRows, + 0, + ) + if (invalidRows) { + warnings.push({ + code: 'invalid-keyword-rows', + field: 'data', + message: `Semrush returned ${invalidRows} keyword row${invalidRows === 1 ? '' : 's'} without a keyword.`, + }) + } + const observedAt = snapshots + .map((snapshot) => snapshot.observedAt) + .sort(compareCodepoints) + .at(-1) + return { + schemaVersion: 1, + provider: 'semrush', + capability: 'keyword-discovery', + data: ideas, + observedAt: observedAt as string, + market, + coverage: { + requestedRows: calls.reduce((sum, call) => sum + call.limit, 0), + returnedRows, + retainedRows: ideas.length, + invalidRows, + providerTotalRows: null, + completeness: + failedCalls || invalidRows + ? 'partial' + : returnedRows >= input.limit || ideas.length < grouped.size + ? 'capped' + : 'complete', + nextCursor: null, + }, + cache: combinedCache(snapshots), + cost: combinedCost(snapshots), + request: { + operation: 'keyword-discovery', + endpoint: 'https://api.semrush.com/', + limit: input.limit, + filters: { + database, + countryCode: market.countryCode, + languageCode: market.languageCode, + sources: sources.join(','), + seeds: seeds.length, + providerRequests: calls.length, + apiVersion: 3, + }, + sort: ['monthlySearchVolume:descending', 'keyword:codepoint-ascending'], + }, + warnings, + } + } +} diff --git a/packages/core/src/providers/semrush/keyword-metrics.ts b/packages/core/src/providers/semrush/keyword-metrics.ts new file mode 100644 index 00000000..619c3eb2 --- /dev/null +++ b/packages/core/src/providers/semrush/keyword-metrics.ts @@ -0,0 +1,185 @@ +import type { + KeywordMetric, + KeywordMetricsProvider, + KeywordMetricsRequest, + ProviderEvidence, +} from '../contracts.js' +import { ProviderError } from '../errors.js' +import { SemrushClient, type SemrushClientOptions } from './client.js' +import { + compareCodepoints, + normalizedKeyword, + type SemrushRecord, + semrushMetric, + semrushRecords, +} from './mapping.js' +import { + SEMRUSH_V3_MARKETS, + semrushKeywordDeprecationWarning, + semrushMarket, + semrushMarketWarnings, +} from './market.js' + +const MAX_KEYWORDS = 100 +const COLUMNS = ['Ph', 'Nq', 'Cp', 'Co', 'Nr', 'In', 'Kd'] as const + +type KeywordMetricsClient = Pick + +export type SemrushKeywordMetricsProviderOptions = SemrushClientOptions & { + client?: KeywordMetricsClient +} + +function keywords(input: string[]): { + normalized: string[] + unique: string[] +} { + const normalized = input.map(normalizedKeyword).filter(Boolean) + const unique = [...new Set(normalized)].sort(compareCodepoints) + if ( + input.length < 1 || + input.length > MAX_KEYWORDS || + normalized.length !== input.length || + unique.some( + (keyword) => + keyword.length > 80 || + keyword.split(/\s+/u).length > 10 || + keyword.includes(';'), + ) + ) { + throw new ProviderError({ + provider: 'semrush', + operation: 'keyword-metrics', + code: 'configuration', + message: + 'Semrush keyword metrics requires 1 to 100 keywords of at most 80 characters and 10 words.', + }) + } + return { normalized, unique } +} + +export class SemrushKeywordMetricsProvider implements KeywordMetricsProvider { + readonly provider = 'semrush' as const + readonly capabilitySupport = [ + { + capability: 'keyword-metrics' as const, + status: 'available' as const, + markets: SEMRUSH_V3_MARKETS, + }, + ] as const + + private readonly client: KeywordMetricsClient + + constructor(options: SemrushKeywordMetricsProviderOptions = {}) { + this.client = options.client ?? new SemrushClient(options) + } + + async keywordMetrics( + input: KeywordMetricsRequest, + ): Promise> { + const selection = keywords(input.keywords) + const { market, database } = semrushMarket(input.market, 'keyword-metrics') + const snapshot = await this.client.report({ + operation: 'keyword-metrics', + reportType: 'phrase_these', + parameters: { + phrase: selection.unique.join(';'), + database, + }, + columns: COLUMNS, + maximumResponseRows: selection.unique.length, + unitsPerLine: 10, + refresh: input.refresh, + }) + const requested = new Set(selection.unique) + const grouped = new Map() + let invalidRows = 0 + for (const row of semrushRecords(snapshot.table, COLUMNS)) { + const keyword = normalizedKeyword(row.Ph ?? '') + if (!keyword || !requested.has(keyword)) { + invalidRows += 1 + continue + } + grouped.set(keyword, [...(grouped.get(keyword) ?? []), row]) + } + const missing = selection.unique.filter((keyword) => !grouped.has(keyword)) + const duplicates = [...grouped.values()].filter( + (rows) => rows.length > 1, + ).length + const warnings = [ + ...snapshot.warnings, + ...semrushMarketWarnings(market), + semrushKeywordDeprecationWarning(), + ...(selection.normalized.length !== selection.unique.length + ? [ + { + code: 'duplicate-keywords-removed', + field: 'keywords', + message: 'Duplicate keywords were normalized and requested once.', + }, + ] + : []), + ...(missing.length + ? [ + { + code: 'provider-keywords-omitted', + field: 'keyword', + message: `Semrush omitted ${missing.length} requested keyword${missing.length === 1 ? '' : 's'}.`, + }, + ] + : []), + ...(invalidRows + ? [ + { + code: 'unexpected-provider-keywords', + field: 'keyword', + message: `Semrush returned ${invalidRows} unexpected or invalid keyword row${invalidRows === 1 ? '' : 's'}.`, + }, + ] + : []), + ...(duplicates + ? [ + { + code: 'duplicate-provider-keywords', + field: 'keyword', + message: `Semrush returned duplicate rows for ${duplicates} keyword${duplicates === 1 ? '' : 's'}; conflicting fields are invalid.`, + }, + ] + : []), + ] + return { + schemaVersion: 1, + provider: 'semrush', + capability: 'keyword-metrics', + data: selection.unique.map((keyword) => + semrushMetric(keyword, grouped.get(keyword) ?? []), + ), + observedAt: snapshot.observedAt, + market, + coverage: { + requestedRows: selection.unique.length, + returnedRows: snapshot.returnedRows, + retainedRows: selection.unique.length, + invalidRows, + providerTotalRows: null, + completeness: + missing.length || invalidRows || duplicates ? 'partial' : 'complete', + nextCursor: null, + }, + cache: snapshot.cache, + cost: snapshot.cost, + request: { + operation: 'keyword-metrics', + endpoint: 'https://api.semrush.com/', + limit: selection.unique.length, + filters: { + database, + countryCode: market.countryCode, + languageCode: market.languageCode, + apiVersion: 3, + }, + sort: ['keyword:codepoint-ascending'], + }, + warnings, + } + } +} diff --git a/packages/core/src/providers/semrush/legacy-cache.test.ts b/packages/core/src/providers/semrush/legacy-cache.test.ts new file mode 100644 index 00000000..34b29c6c --- /dev/null +++ b/packages/core/src/providers/semrush/legacy-cache.test.ts @@ -0,0 +1,88 @@ +import assert from 'node:assert/strict' +import { mkdirSync, mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { dirname, join } from 'node:path' +import test from 'node:test' +import { clearCache, getCacheStats, getDb } from '../../storage/database.js' +import Database from '../../storage/sqlite.js' + +const root = mkdtempSync(join(tmpdir(), 'seo-semrush-legacy-cache-')) +const previousCacheDir = process.env.SEO_CACHE_DIR +process.env.SEO_CACHE_DIR = join(root, 'cache') + +const cacheFile = join(root, 'cache', 'cache.db') +mkdirSync(dirname(cacheFile), { recursive: true }) +const legacyDatabase = new Database(cacheFile) +legacyDatabase.exec(` + CREATE TABLE semrush_cache ( + endpoint TEXT, + query_hash TEXT, + request_json TEXT, + response_json TEXT, + credits_used INTEGER, + fetched_at INTEGER, + expires_at INTEGER, + PRIMARY KEY(endpoint, query_hash) + ) WITHOUT ROWID; +`) +legacyDatabase + .prepare( + `INSERT INTO semrush_cache + (endpoint, query_hash, request_json, response_json, credits_used, fetched_at, expires_at) + VALUES (?, ?, ?, ?, ?, ?, ?)`, + ) + .run( + 'phrase_this', + 'legacy-query', + JSON.stringify({ key: 'legacy-sem-rush-key', phrase: 'unsafe query' }), + '[]', + 0, + Date.now(), + Date.now() + 60_000, + ) +legacyDatabase.close() + +test.after(() => { + if (previousCacheDir === undefined) delete process.env.SEO_CACHE_DIR + else process.env.SEO_CACHE_DIR = previousCacheDir + rmSync(root, { recursive: true, force: true }) +}) + +test('database startup removes legacy Semrush rows that could contain keys', () => { + const rows = getDb() + .prepare('SELECT COUNT(*) AS count FROM semrush_cache') + .get() as { count: number } + assert.equal(rows.count, 0) +}) + +test('Semrush cache stats and clearing include only Semrush provider rows', () => { + const database = getDb() + const insert = database.prepare(` + INSERT INTO provider_cache ( + provider, credential_scope, operation, request_hash, request_json, + response_json, row_count, source_cost_micros, task_ids_json, + fetched_at, expires_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `) + for (const provider of ['semrush', 'dataforseo']) { + insert.run( + provider, + `${provider}-scope`, + 'keyword-metrics', + `${provider}-hash`, + '{}', + '{}', + 1, + null, + '[]', + Date.now(), + Date.now() + 60_000, + ) + } + + assert.equal(getCacheStats().counts.semrush_cache, 1) + assert.equal(getCacheStats().counts.provider_cache, 1) + assert.equal(clearCache('semrush'), 1) + assert.equal(getCacheStats().counts.semrush_cache, 0) + assert.equal(getCacheStats().counts.provider_cache, 1) +}) diff --git a/packages/core/src/providers/semrush/mappers.test.ts b/packages/core/src/providers/semrush/mappers.test.ts deleted file mode 100644 index 3f912a02..00000000 --- a/packages/core/src/providers/semrush/mappers.test.ts +++ /dev/null @@ -1,69 +0,0 @@ -import assert from 'node:assert/strict' -import test from 'node:test' -import { mapKeywordRows, mapOverview } from './mappers.js' - -test('Semrush overview preserves observed zero values', () => { - assert.deepEqual( - mapOverview([ - ['Ph', 'Nq', 'Cp', 'Co', 'Kd', 'Nr'], - ['zero query', '0', '0', '0', '0', '0'], - ]), - { - phrase: 'zero query', - volume: 0, - cpc: 0, - competition: 0, - difficulty: 0, - results: 0, - }, - ) -}) - -test('Semrush keyword rows keep missing and invalid numbers unavailable', () => { - assert.deepEqual( - mapKeywordRows([ - ['Ph', 'Nq', 'Kd', 'Cp', 'Co', 'Po', 'Ur', 'Dn'], - ['missing query', '', 'not-a-number', ' ', 'Infinity', '', '', ''], - ['zero query', '0', '0', '0', '0', '0', '/zero', 'example.com'], - ]), - [ - { - phrase: 'missing query', - volume: undefined, - difficulty: undefined, - cpc: undefined, - competition: undefined, - position: undefined, - url: undefined, - domain: undefined, - }, - { - phrase: 'zero query', - volume: 0, - difficulty: 0, - cpc: 0, - competition: 0, - position: 0, - url: '/zero', - domain: 'example.com', - }, - ], - ) -}) - -test('Semrush mappers handle empty and short rows deterministically', () => { - assert.deepEqual(mapOverview([]), { phrase: '' }) - assert.deepEqual(mapKeywordRows([]), []) - assert.deepEqual(mapKeywordRows([['Ph', 'Nq'], ['query']]), [ - { - phrase: 'query', - volume: undefined, - difficulty: undefined, - cpc: undefined, - competition: undefined, - position: undefined, - url: undefined, - domain: undefined, - }, - ]) -}) diff --git a/packages/core/src/providers/semrush/mappers.ts b/packages/core/src/providers/semrush/mappers.ts deleted file mode 100644 index 62a2b9af..00000000 --- a/packages/core/src/providers/semrush/mappers.ts +++ /dev/null @@ -1,54 +0,0 @@ -import type { KeywordOverview, KeywordRow } from '../../types.js' - -function optionalFiniteNumber(value: string | undefined): number | undefined { - if (value === undefined || value.trim() === '') return undefined - const parsed = Number(value) - return Number.isFinite(parsed) ? parsed : undefined -} - -function optionalText(value: string | undefined): string | undefined { - const normalized = value?.trim() - return normalized ? normalized : undefined -} - -export function mapOverview(rows: string[][]): KeywordOverview { - const [header, first] = rows - if (!header || !first) { - return { phrase: '' } - } - - const record = Object.fromEntries( - header.map((key, index) => [key, first[index]]), - ) - return { - phrase: record.Ph ?? '', - volume: optionalFiniteNumber(record.Nq), - cpc: optionalFiniteNumber(record.Cp), - competition: optionalFiniteNumber(record.Co), - difficulty: optionalFiniteNumber(record.Kd), - results: optionalFiniteNumber(record.Nr), - } -} - -export function mapKeywordRows(rows: string[][]): KeywordRow[] { - const [header, ...body] = rows - if (!header) { - return [] - } - - return body.map((row) => { - const record = Object.fromEntries( - header.map((key, index) => [key, row[index]]), - ) - return { - phrase: record.Ph ?? '', - volume: optionalFiniteNumber(record.Nq), - difficulty: optionalFiniteNumber(record.Kd), - cpc: optionalFiniteNumber(record.Cp), - competition: optionalFiniteNumber(record.Co), - url: optionalText(record.Ur), - domain: optionalText(record.Dn), - position: optionalFiniteNumber(record.Po), - } - }) -} diff --git a/packages/core/src/providers/semrush/mapping.test.ts b/packages/core/src/providers/semrush/mapping.test.ts new file mode 100644 index 00000000..950e5969 --- /dev/null +++ b/packages/core/src/providers/semrush/mapping.test.ts @@ -0,0 +1,66 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { semrushMetric, semrushRecords } from './mapping.js' + +test('Semrush maps documented CSV response headers into owned metrics', () => { + const records = semrushRecords( + { + headers: [ + 'Keyword', + 'Search Volume', + 'CPC', + 'Competition', + 'Number of Results', + 'Intents', + 'Keyword Difficulty Index', + ], + rows: [['query', '0', '1.25', '0.4', '300', '1', '42']], + }, + ['Ph', 'Nq', 'Cp', 'Co', 'Nr', 'In', 'Kd'], + ) + + assert.deepEqual(semrushMetric('query', records), { + keyword: 'query', + monthlySearchVolume: { state: 'observed', value: 0 }, + monthlySearches: { + state: 'unavailable', + value: null, + reason: 'This Semrush V3 report does not return monthly search history.', + }, + searchVolumeUpdatedAt: { + state: 'missing', + value: null, + reason: 'Semrush omitted searchVolumeUpdatedAt.', + }, + cpcUsd: { state: 'observed', value: 1.25 }, + paidCompetition: { state: 'observed', value: 0.4 }, + keywordDifficulty: { state: 'observed', value: 42 }, + intent: { state: 'observed', value: 'informational' }, + resultCount: { state: 'observed', value: 300 }, + }) +}) + +test('Semrush rejects reordered or unfamiliar response headers', () => { + assert.throws( + () => + semrushRecords( + { + headers: ['Search Volume', 'Keyword'], + rows: [['10', 'query']], + }, + ['Ph', 'Nq'], + ), + /do not match the requested report/i, + ) + assert.throws( + () => + semrushRecords( + { + headers: ['Keyword', 'Volume'], + rows: [['query', '10']], + }, + ['Ph', 'Nq'], + ), + /do not match the requested report/i, + ) +}) diff --git a/packages/core/src/providers/semrush/mapping.ts b/packages/core/src/providers/semrush/mapping.ts new file mode 100644 index 00000000..bbea1c8a --- /dev/null +++ b/packages/core/src/providers/semrush/mapping.ts @@ -0,0 +1,233 @@ +import type { + KeywordMetric, + ProviderValue, + ProviderWarning, +} from '../contracts.js' +import { observedValue, unavailableValue } from '../contracts.js' +import { ProviderError } from '../errors.js' +import type { SemrushCsvTable } from './csv.js' + +export type SemrushColumn = + | 'Co' + | 'Cp' + | 'Dn' + | 'In' + | 'Kd' + | 'Nq' + | 'Nr' + | 'Oc' + | 'Or' + | 'Ot' + | 'Pc' + | 'Ph' + | 'Po' + | 'Pt' + | 'Tg' + | 'Tr' + | 'Ts' + | 'Ur' + +export type SemrushRecord = Partial> + +const HEADERS: Record = { + Co: ['Co', 'Competition'], + Cp: ['Cp', 'CPC'], + Dn: ['Dn', 'Domain'], + In: ['In', 'Intent', 'Intents'], + Kd: ['Kd', 'Keyword Difficulty Index', 'Keyword Difficulty'], + Nq: ['Nq', 'Search Volume'], + Nr: ['Nr', 'Number of Results'], + Oc: ['Oc', 'Organic Cost'], + Or: ['Or', 'Organic Keywords'], + Ot: ['Ot', 'Organic Traffic'], + Pc: ['Pc', 'Number of Keywords'], + Ph: ['Ph', 'Keyword'], + Po: ['Po', 'Position'], + Pt: ['Pt', 'Position Type', 'Position type'], + Tg: ['Tg', 'Traffic'], + Tr: ['Tr', 'Traffic (%)'], + Ts: ['Ts', 'Timestamp'], + Ur: ['Ur', 'Url', 'URL'], +} + +function invalidResponse(message: string): ProviderError { + return new ProviderError({ + provider: 'semrush', + operation: 'mapping', + code: 'invalid-response', + message, + }) +} + +export function semrushRecords( + table: SemrushCsvTable, + columns: readonly SemrushColumn[], +): SemrushRecord[] { + if ( + table.headers.length !== columns.length || + table.headers.some( + (header, index) => + !HEADERS[columns[index] as SemrushColumn].includes(header), + ) + ) { + throw invalidResponse( + 'Semrush returned CSV columns that do not match the requested report.', + ) + } + return table.rows.map((row) => + Object.fromEntries( + columns.map((column, index) => [column, row[index] as string]), + ), + ) +} + +export function normalizedKeyword(value: string): string { + return value.trim().replace(/\s+/gu, ' ').toLowerCase() +} + +export function compareCodepoints(left: string, right: string): number { + return left < right ? -1 : left > right ? 1 : 0 +} + +function missing(field: string): ProviderValue { + return unavailableValue('missing', `Semrush omitted ${field}.`) +} + +function numericValue( + values: Array, + field: string, + valid: (value: number) => boolean, +): ProviderValue { + const present = values.filter( + (value): value is string => value !== undefined && value.trim() !== '', + ) + if (!present.length) return missing(field) + const parsed = present.map(Number) + if (parsed.some((value) => !Number.isFinite(value) || !valid(value))) { + return unavailableValue('invalid', `Semrush returned an invalid ${field}.`) + } + const unique = [...new Set(parsed)] + return unique.length === 1 + ? observedValue(unique[0] as number) + : unavailableValue( + 'invalid', + `Semrush returned conflicting ${field} values.`, + ) +} + +export function semrushNumber( + value: string | undefined, + field: string, + valid: (value: number) => boolean, +): ProviderValue { + return numericValue([value], field, valid) +} + +function timestampValue( + values: Array, +): ProviderValue { + const present = values.filter( + (value): value is string => value !== undefined && value.trim() !== '', + ) + if (!present.length) return missing('searchVolumeUpdatedAt') + const normalized = present.flatMap((value) => { + const number = Number(value) + const timestamp = Number.isFinite(number) + ? number < 10_000_000_000 + ? number * 1_000 + : number + : Date.parse(value) + return Number.isFinite(timestamp) ? [new Date(timestamp).toISOString()] : [] + }) + if (normalized.length !== present.length) { + return unavailableValue( + 'invalid', + 'Semrush returned an invalid searchVolumeUpdatedAt.', + ) + } + const unique = [...new Set(normalized)] + return unique.length === 1 + ? observedValue(unique[0] as string) + : unavailableValue( + 'invalid', + 'Semrush returned conflicting searchVolumeUpdatedAt values.', + ) +} + +function intentValue(values: Array): ProviderValue { + const labels: Record = { + '0': 'commercial', + '1': 'informational', + '2': 'navigational', + '3': 'transactional', + } + const present = values.filter( + (value): value is string => value !== undefined && value.trim() !== '', + ) + if (!present.length) return missing('intent') + const normalized = present.map( + (value) => labels[value.trim()] ?? value.trim().toLowerCase(), + ) + if (normalized.some((value) => !value || value.length > 100)) { + return unavailableValue('invalid', 'Semrush returned an invalid intent.') + } + const unique = [...new Set(normalized)] + return unique.length === 1 + ? observedValue(unique[0] as string) + : unavailableValue('invalid', 'Semrush returned conflicting intent values.') +} + +export function semrushMetric( + keyword: string, + rows: SemrushRecord[], +): KeywordMetric { + return { + keyword, + monthlySearchVolume: numericValue( + rows.map((row) => row.Nq), + 'monthlySearchVolume', + (value) => Number.isSafeInteger(value) && value >= 0, + ), + monthlySearches: unavailableValue( + 'unavailable', + 'This Semrush V3 report does not return monthly search history.', + ), + searchVolumeUpdatedAt: timestampValue(rows.map((row) => row.Ts)), + cpcUsd: numericValue( + rows.map((row) => row.Cp), + 'cpcUsd', + (value) => value >= 0, + ), + paidCompetition: numericValue( + rows.map((row) => row.Co), + 'paidCompetition', + (value) => value >= 0 && value <= 1, + ), + keywordDifficulty: numericValue( + rows.map((row) => row.Kd), + 'keywordDifficulty', + (value) => value >= 0 && value <= 100, + ), + intent: intentValue(rows.map((row) => row.In)), + resultCount: numericValue( + rows.map((row) => row.Nr), + 'resultCount', + (value) => Number.isSafeInteger(value) && value >= 0, + ), + } +} + +export function invalidRowsWarning( + count: number, + label: string, +): ProviderWarning[] { + return count + ? [ + { + code: `invalid-${label}-rows`, + field: 'data.rows', + message: `Semrush returned ${count} ${label} row${count === 1 ? '' : 's'} without the required fields.`, + }, + ] + : [] +} diff --git a/packages/core/src/providers/semrush/market.ts b/packages/core/src/providers/semrush/market.ts new file mode 100644 index 00000000..57a345cd --- /dev/null +++ b/packages/core/src/providers/semrush/market.ts @@ -0,0 +1,204 @@ +import type { + ProviderMarketSupport, + ProviderWarning, + SearchMarket, +} from '../contracts.js' +import { searchMarketSchema } from '../contracts.js' +import { ProviderError } from '../errors.js' + +export const SEMRUSH_V3_COUNTRY_CODES = [ + 'AE', + 'AF', + 'AL', + 'AM', + 'AO', + 'AR', + 'AT', + 'AU', + 'AZ', + 'BA', + 'BD', + 'BE', + 'BG', + 'BH', + 'BO', + 'BR', + 'BS', + 'BN', + 'BW', + 'BY', + 'BZ', + 'CA', + 'CD', + 'CH', + 'CL', + 'CM', + 'CO', + 'CR', + 'CV', + 'CY', + 'CZ', + 'DE', + 'DK', + 'DO', + 'DZ', + 'EC', + 'EE', + 'EG', + 'ES', + 'ET', + 'FI', + 'FR', + 'GB', + 'GE', + 'GH', + 'GR', + 'GT', + 'GY', + 'HK', + 'HN', + 'HR', + 'HT', + 'HU', + 'ID', + 'IE', + 'IL', + 'IN', + 'IS', + 'IT', + 'JM', + 'JO', + 'JP', + 'KH', + 'KR', + 'KW', + 'KZ', + 'LB', + 'LK', + 'LT', + 'LU', + 'LV', + 'LY', + 'MA', + 'MD', + 'ME', + 'MG', + 'MN', + 'MT', + 'MU', + 'MX', + 'MY', + 'MZ', + 'NA', + 'NG', + 'NI', + 'NL', + 'NO', + 'NP', + 'NZ', + 'OM', + 'PA', + 'PE', + 'PH', + 'PK', + 'PL', + 'PT', + 'PY', + 'QA', + 'RO', + 'RS', + 'RU', + 'SA', + 'SE', + 'SG', + 'SI', + 'SK', + 'SN', + 'SV', + 'TH', + 'TN', + 'TR', + 'TT', + 'TW', + 'UA', + 'US', + 'UY', + 'VE', + 'VN', + 'ZA', + 'ZM', + 'ZW', +] as const + +const SUPPORTED_COUNTRIES = new Set(SEMRUSH_V3_COUNTRY_CODES) + +export const SEMRUSH_V3_MARKETS = [ + { + searchEngines: ['google'], + countryCodes: SEMRUSH_V3_COUNTRY_CODES, + devices: ['desktop'], + location: 'country-only', + }, +] as const satisfies readonly ProviderMarketSupport[] + +export function semrushMarket( + input: SearchMarket, + operation: string, +): { market: SearchMarket; database: string } { + const parsed = searchMarketSchema.safeParse(input) + if (!parsed.success) { + throw new ProviderError({ + provider: 'semrush', + operation, + code: 'configuration', + message: 'Semrush requires a valid search market.', + }) + } + const market = parsed.data + if ( + market.searchEngine !== 'google' || + market.location || + market.device === 'mobile' || + !SUPPORTED_COUNTRIES.has(market.countryCode) + ) { + throw new ProviderError({ + provider: 'semrush', + operation, + code: 'configuration', + message: + 'Semrush V3 research supports Google desktop data in its country-level regional databases.', + }) + } + return { + market, + database: + market.countryCode === 'GB' ? 'uk' : market.countryCode.toLowerCase(), + } +} + +export function semrushMarketWarnings(market: SearchMarket): ProviderWarning[] { + return [ + { + code: 'provider-database-not-language-filtered', + field: 'market.languageCode', + message: `Semrush used the ${market.countryCode} regional database; it does not apply the requested ${market.languageCode} as a separate language filter.`, + }, + ...(market.device + ? [] + : [ + { + code: 'provider-device-defaulted', + field: 'market.device', + message: 'Semrush V3 regional research used desktop data.', + }, + ]), + ] +} + +export function semrushKeywordDeprecationWarning(): ProviderWarning { + return { + code: 'semrush-v3-keyword-api-deprecated', + message: + 'Semrush marks its Version 3 keyword endpoints as deprecated; this adapter keeps the provider-native observation and does not infer replacement data.', + } +} diff --git a/packages/core/src/providers/semrush/ranked-keywords.ts b/packages/core/src/providers/semrush/ranked-keywords.ts new file mode 100644 index 00000000..9365d7ac --- /dev/null +++ b/packages/core/src/providers/semrush/ranked-keywords.ts @@ -0,0 +1,258 @@ +import type { ProviderEvidence } from '../contracts.js' +import type { + RankedKeyword, + RankedKeywordPage, + RankedKeywordsRequest, +} from '../domain-contracts.js' +import { ProviderError } from '../errors.js' +import type { SemrushClient } from './client.js' +import { + coverage, + dedupeBy, + domain, + evidence, + mappedWarnings, + observedNumber, + organicOnly, + rowLimit, + safeUrl, + unavailable, +} from './domain-research-shared.js' +import { + compareCodepoints, + normalizedKeyword, + semrushMetric, + semrushRecords, +} from './mapping.js' +import { semrushMarket } from './market.js' + +const COLUMNS = [ + 'Ph', + 'Po', + 'Nq', + 'Cp', + 'Co', + 'Nr', + 'Kd', + 'In', + 'Ur', + 'Ts', +] as const + +function target(value: string): { + target: string + reportType: 'domain_organic' | 'url_organic' + parameter: { domain: string } | { url: string } +} { + const raw = value.trim() + if (!/^https?:\/\//iu.test(raw)) { + const normalized = domain(raw, 'ranked-keywords') + return { + target: normalized, + reportType: 'domain_organic', + parameter: { domain: normalized }, + } + } + try { + const url = new URL(raw) + if ( + !['http:', 'https:'].includes(url.protocol) || + url.username || + url.password || + raw.length > 2_048 + ) { + throw new Error() + } + url.hash = '' + return { + target: url.toString(), + reportType: 'url_organic', + parameter: { url: url.toString() }, + } + } catch { + throw new ProviderError({ + provider: 'semrush', + operation: 'ranked-keywords', + code: 'configuration', + message: 'Use a valid domain or absolute page URL.', + }) + } +} + +function filters(input: RankedKeywordsRequest): string { + const result: string[] = [] + if (input.minSearchVolume !== undefined) { + if ( + !Number.isSafeInteger(input.minSearchVolume) || + input.minSearchVolume < 0 + ) { + throw new ProviderError({ + provider: 'semrush', + operation: 'ranked-keywords', + code: 'configuration', + message: 'Minimum search volume must be a nonnegative integer.', + }) + } + if (input.minSearchVolume > 0) { + result.push(`+|Nq|Gt|${input.minSearchVolume - 1}`) + } + } + if (input.maxRank !== undefined) { + if ( + !Number.isSafeInteger(input.maxRank) || + input.maxRank < 1 || + input.maxRank > 100 + ) { + throw new ProviderError({ + provider: 'semrush', + operation: 'ranked-keywords', + code: 'configuration', + message: 'Maximum rank must be from 1 to 100.', + }) + } + result.push(`+|Po|Lt|${input.maxRank + 1}`) + } + const excluded = [ + ...new Set((input.excludeTerms ?? []).map(normalizedKeyword)), + ] + .filter(Boolean) + .sort(compareCodepoints) + if ( + excluded.length > 5 || + excluded.some( + (term) => term.length > 80 || term.includes('|') || term.includes(';'), + ) + ) { + throw new ProviderError({ + provider: 'semrush', + operation: 'ranked-keywords', + code: 'configuration', + message: + 'Use at most 5 excluded terms of at most 80 characters without filter separators.', + }) + } + result.push(...excluded.map((term) => `-|Ph|Co|${term}`)) + return result.join(';') +} + +export async function semrushRankedKeywords( + client: Pick, + input: RankedKeywordsRequest, +): Promise> { + const { market, database } = semrushMarket(input.market, 'ranked-keywords') + organicOnly(input.resultTypes, 'ranked-keywords') + const selection = target(input.target) + if ( + selection.reportType === 'domain_organic' && + input.includeSubdomains === false + ) { + throw new ProviderError({ + provider: 'semrush', + operation: 'ranked-keywords', + code: 'configuration', + message: + 'Semrush V3 does not expose the requested subdomain exclusion for domain keyword reports.', + }) + } + const offset = input.offset ?? 0 + rowLimit(input.limit, offset, 'ranked-keywords') + const displayFilter = filters(input) + const snapshot = await client.report({ + operation: 'ranked-keywords', + reportType: selection.reportType, + parameters: { + ...selection.parameter, + database, + display_limit: input.limit + offset, + display_offset: offset, + display_sort: 'nq_desc', + positions_type: 'organic', + ...(displayFilter ? { display_filter: displayFilter } : {}), + }, + columns: COLUMNS, + maximumResponseRows: input.limit, + unitsPerLine: 10, + refresh: input.refresh, + }) + const records = semrushRecords(snapshot.table, COLUMNS) + let invalidRows = 0 + const mapped = records.flatMap((row): RankedKeyword[] => { + const keyword = normalizedKeyword(row.Ph ?? '') + const url = safeUrl(row.Ur) + const rank = Number(row.Po) + if ( + !keyword || + !url || + !Number.isSafeInteger(rank) || + rank < 1 || + rank > 100 + ) { + invalidRows += 1 + return [] + } + return [ + { + ...semrushMetric(keyword, [row]), + url, + rankGroup: rank, + rankAbsolute: rank, + resultType: 'organic', + estimatedMonthlyTraffic: unavailable( + 'absolute estimated keyword traffic; the provider row exposes traffic share', + ), + }, + ] + }) + const rows = dedupeBy( + mapped, + (row) => `${row.keyword}\0${row.url}\0${row.resultType}`, + ).sort( + (left, right) => + observedNumber(right.monthlySearchVolume) - + observedNumber(left.monthlySearchVolume) || + left.rankGroup - right.rankGroup || + compareCodepoints(left.keyword, right.keyword) || + compareCodepoints(left.url, right.url), + ) + const duplicateRows = mapped.length - rows.length + return evidence({ + capability: 'ranked-keywords', + data: { target: selection.target, rows, totalRows: null }, + market, + snapshot, + coverage: coverage({ + requestedRows: input.limit, + returnedRows: records.length, + retainedRows: rows.length, + invalidRows, + offset, + filtered: Boolean(displayFilter), + }), + limit: input.limit, + filters: { + database, + countryCode: market.countryCode, + languageCode: market.languageCode, + includeSubdomains: input.includeSubdomains ?? true, + resultTypes: 'organic', + minSearchVolume: input.minSearchVolume ?? 0, + maxRank: input.maxRank ?? 100, + excludedTerms: input.excludeTerms?.length ?? 0, + offset, + apiVersion: 3, + }, + sort: ['monthlySearchVolume:descending', 'rank:ascending'], + warnings: [ + ...mappedWarnings(market, snapshot, invalidRows, 'ranked-keyword'), + ...(duplicateRows + ? [ + { + code: 'duplicate-ranked-keyword-rows', + field: 'data.rows', + message: `${duplicateRows} duplicate ranked-keyword row${duplicateRows === 1 ? '' : 's'} were collapsed deterministically.`, + }, + ] + : []), + ], + }) +} diff --git a/packages/core/src/providers/semrush/ranking-pages.ts b/packages/core/src/providers/semrush/ranking-pages.ts new file mode 100644 index 00000000..4b545d15 --- /dev/null +++ b/packages/core/src/providers/semrush/ranking-pages.ts @@ -0,0 +1,149 @@ +import type { ProviderEvidence } from '../contracts.js' +import type { + RankingPage, + RankingPagePage, + RankingPagesRequest, +} from '../domain-contracts.js' +import { ProviderError } from '../errors.js' +import type { SemrushClient } from './client.js' +import { + coverage, + dedupeBy, + domain, + evidence, + mappedWarnings, + observedNumber, + organicFootprint, + rowLimit, + safeUrl, + unavailable, +} from './domain-research-shared.js' +import { compareCodepoints, semrushRecords } from './mapping.js' +import { semrushMarket } from './market.js' + +const COLUMNS = ['Ur', 'Pc', 'Tg', 'Tr'] as const + +function filters(input: RankingPagesRequest): string { + const result: string[] = [] + if (input.minEstimatedTraffic !== undefined) { + if ( + !Number.isFinite(input.minEstimatedTraffic) || + input.minEstimatedTraffic < 0 + ) { + throw new ProviderError({ + provider: 'semrush', + operation: 'ranking-pages', + code: 'configuration', + message: 'Minimum estimated traffic must be nonnegative.', + }) + } + if (input.minEstimatedTraffic > 0) { + result.push(`+|Tg|Gt|${Math.max(0, input.minEstimatedTraffic - 1)}`) + } + } + if (input.minRankedKeywords !== undefined) { + if ( + !Number.isSafeInteger(input.minRankedKeywords) || + input.minRankedKeywords < 0 + ) { + throw new ProviderError({ + provider: 'semrush', + operation: 'ranking-pages', + code: 'configuration', + message: 'Minimum ranked keywords must be a nonnegative integer.', + }) + } + if (input.minRankedKeywords > 0) { + result.push(`+|Pc|Gt|${input.minRankedKeywords - 1}`) + } + } + return result.join(';') +} + +export async function semrushRankingPages( + client: Pick, + input: RankingPagesRequest, +): Promise> { + const { market, database } = semrushMarket(input.market, 'ranking-pages') + const target = domain(input.domain, 'ranking-pages') + const offset = input.offset ?? 0 + rowLimit(input.limit, offset, 'ranking-pages') + const displayFilter = filters(input) + const snapshot = await client.report({ + operation: 'ranking-pages', + reportType: 'domain_organic_unique', + parameters: { + domain: target, + database, + display_limit: input.limit + offset, + display_offset: offset, + display_sort: 'tg_desc', + ...(displayFilter ? { display_filter: displayFilter } : {}), + }, + columns: COLUMNS, + maximumResponseRows: input.limit, + unitsPerLine: 10, + refresh: input.refresh, + }) + const records = semrushRecords(snapshot.table, COLUMNS) + let invalidRows = 0 + const mapped = records.flatMap((row): RankingPage[] => { + const url = safeUrl(row.Ur) + if (!url) { + invalidRows += 1 + return [] + } + const organic = organicFootprint({ + traffic: row.Tg, + keywords: row.Pc, + }) + organic.estimatedMonthlyTrafficCostUsd = unavailable( + 'estimated organic traffic cost', + ) + return [{ url, organic }] + }) + const rows = dedupeBy(mapped, (row) => row.url).sort( + (left, right) => + observedNumber(right.organic.estimatedMonthlyTraffic) - + observedNumber(left.organic.estimatedMonthlyTraffic) || + compareCodepoints(left.url, right.url), + ) + const duplicateRows = mapped.length - rows.length + return evidence({ + capability: 'relevant-pages', + data: { domain: target, rows, totalRows: null }, + market, + snapshot, + coverage: coverage({ + requestedRows: input.limit, + returnedRows: records.length, + retainedRows: rows.length, + invalidRows, + offset, + filtered: Boolean(displayFilter), + }), + limit: input.limit, + filters: { + database, + countryCode: market.countryCode, + languageCode: market.languageCode, + minEstimatedTraffic: input.minEstimatedTraffic ?? 0, + minRankedKeywords: input.minRankedKeywords ?? 0, + offset, + apiVersion: 3, + }, + sort: ['estimatedMonthlyTraffic:descending', 'url:codepoint-ascending'], + warnings: [ + ...mappedWarnings(market, snapshot, invalidRows, 'ranking-page'), + ...(duplicateRows + ? [ + { + code: 'duplicate-ranking-page-rows', + field: 'data.rows', + message: `${duplicateRows} duplicate ranking-page row${duplicateRows === 1 ? '' : 's'} were collapsed deterministically.`, + }, + ] + : []), + ], + }) +} diff --git a/packages/core/src/providers/semrush/schemas.ts b/packages/core/src/providers/semrush/schemas.ts deleted file mode 100644 index 5fb9173f..00000000 --- a/packages/core/src/providers/semrush/schemas.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { z } from 'zod' - -const optionalFiniteNumber = z.number().finite().optional() - -export const semrushKeywordOverviewSchema = z - .object({ - phrase: z.string().trim().min(1), - volume: optionalFiniteNumber, - cpc: optionalFiniteNumber, - competition: optionalFiniteNumber, - difficulty: optionalFiniteNumber, - intent: z.string().trim().min(1).optional(), - results: optionalFiniteNumber, - }) - .strict() - -export const semrushKeywordRowSchema = z - .object({ - phrase: z.string().trim().min(1), - volume: optionalFiniteNumber, - difficulty: optionalFiniteNumber, - cpc: optionalFiniteNumber, - competition: optionalFiniteNumber, - url: z.string().trim().min(1).optional(), - domain: z.string().trim().min(1).optional(), - position: optionalFiniteNumber, - }) - .strict() - -export const semrushKeywordRowsSchema = z.array(semrushKeywordRowSchema) - -export const semrushDifficultyRowsSchema = z.array( - z - .object({ - phrase: z.string().trim().min(1), - kd: z.number().finite(), - }) - .strict(), -) diff --git a/packages/core/src/providers/semrush/serp-competitors.ts b/packages/core/src/providers/semrush/serp-competitors.ts new file mode 100644 index 00000000..23a4cb0a --- /dev/null +++ b/packages/core/src/providers/semrush/serp-competitors.ts @@ -0,0 +1,269 @@ +import type { + ProviderCoverage, + ProviderEvidence, + ProviderWarning, +} from '../contracts.js' +import { observedValue } from '../contracts.js' +import type { + SerpCompetitor, + SerpCompetitorSet, + SerpCompetitorsRequest, +} from '../domain-contracts.js' +import { ProviderError } from '../errors.js' +import type { SemrushClient, SemrushReportSnapshot } from './client.js' +import { + combinedCache, + combinedCost, + domain, + organicOnly, + rowLimit, + unavailable, +} from './domain-research-shared.js' +import { + compareCodepoints, + normalizedKeyword, + semrushRecords, +} from './mapping.js' +import { + semrushKeywordDeprecationWarning, + semrushMarket, + semrushMarketWarnings, +} from './market.js' + +const MAX_KEYWORDS = 20 +const MAX_SERP_DEPTH = 100 +const COLUMNS = ['Po', 'Dn', 'Ur'] as const + +function keywords(input: string[]): string[] { + const result = [...new Set(input.map(normalizedKeyword))] + .filter(Boolean) + .sort(compareCodepoints) + if ( + result.length < 1 || + result.length > MAX_KEYWORDS || + result.some( + (keyword) => + keyword.length > 80 || + keyword.split(/\s+/u).length > 10 || + keyword.includes(';'), + ) + ) { + throw new ProviderError({ + provider: 'semrush', + operation: 'serp-competitors', + code: 'configuration', + message: + 'Semrush SERP competitors requires 1 to 20 keywords of at most 80 characters and 10 words.', + }) + } + return result +} + +function median(values: number[]): number { + const sorted = [...values].sort((left, right) => left - right) + const middle = Math.floor(sorted.length / 2) + return sorted.length % 2 + ? (sorted[middle] as number) + : ((sorted[middle - 1] as number) + (sorted[middle] as number)) / 2 +} + +export async function semrushSerpCompetitors( + client: Pick, + input: SerpCompetitorsRequest, +): Promise> { + const { market, database } = semrushMarket(input.market, 'serp-competitors') + organicOnly(input.resultTypes, 'serp-competitors') + if (input.includeSubdomains) { + throw new ProviderError({ + provider: 'semrush', + operation: 'serp-competitors', + code: 'configuration', + message: + 'Semrush SERP competitors preserves observed domains and does not fold subdomains together.', + }) + } + const requestedKeywords = keywords(input.keywords) + const offset = input.offset ?? 0 + rowLimit(input.limit, offset, 'serp-competitors') + const depth = Math.min(MAX_SERP_DEPTH, input.limit + offset) + const snapshots: SemrushReportSnapshot[] = [] + const warnings: ProviderWarning[] = [ + ...semrushMarketWarnings(market), + semrushKeywordDeprecationWarning(), + ] + const grouped = new Map>>() + let invalidRows = 0 + let duplicateRows = 0 + let lastError: ProviderError | undefined + + for (const keyword of requestedKeywords) { + try { + const snapshot = await client.report({ + operation: 'serp-competitors-organic-results', + reportType: 'phrase_organic', + parameters: { + phrase: keyword, + database, + display_limit: depth, + positions_type: 'organic', + }, + columns: COLUMNS, + maximumResponseRows: depth, + unitsPerLine: 10, + refresh: input.refresh, + }) + snapshots.push(snapshot) + warnings.push(...snapshot.warnings) + for (const row of semrushRecords(snapshot.table, COLUMNS)) { + let normalizedDomain = '' + try { + normalizedDomain = domain(row.Dn ?? '', 'serp-competitors') + } catch { + normalizedDomain = '' + } + const position = Number(row.Po) + if ( + !normalizedDomain || + !Number.isSafeInteger(position) || + position < 1 || + position > MAX_SERP_DEPTH + ) { + invalidRows += 1 + continue + } + const byKeyword = + grouped.get(normalizedDomain) ?? new Map>() + const positions = byKeyword.get(keyword) ?? new Set() + if (positions.has(position)) duplicateRows += 1 + positions.add(position) + byKeyword.set(keyword, positions) + grouped.set(normalizedDomain, byKeyword) + } + } catch (error) { + if (!(error instanceof ProviderError)) throw error + lastError = error + warnings.push({ + code: 'competitor-request-failed', + field: 'keywords', + message: `Semrush organic results failed for one keyword (${error.code}).`, + }) + } + } + if (!snapshots.length && lastError) throw lastError + + const allRows: SerpCompetitor[] = [...grouped.entries()] + .map(([competitorDomain, byKeyword]) => { + const keywordPositions = [...byKeyword.entries()] + .map(([keyword, positions]) => ({ + keyword, + positions: [...positions].sort((left, right) => left - right), + })) + .sort((left, right) => compareCodepoints(left.keyword, right.keyword)) + const positions = keywordPositions.flatMap((item) => item.positions) + return { + domain: competitorDomain, + matchedKeywords: keywordPositions.length, + averagePosition: observedValue( + positions.reduce((sum, value) => sum + value, 0) / positions.length, + ), + medianPosition: observedValue(median(positions)), + visibility: unavailable( + 'a provider visibility metric for this supplied keyword set', + ), + estimatedMonthlyTraffic: unavailable( + 'absolute estimated monthly traffic for this supplied keyword set', + ), + relevantResults: unavailable( + 'a complete relevant-result count for this supplied keyword set', + ), + keywordPositions, + } + }) + .sort((left, right) => { + const leftAverage = + left.averagePosition.state === 'observed' + ? left.averagePosition.value + : Number.POSITIVE_INFINITY + const rightAverage = + right.averagePosition.state === 'observed' + ? right.averagePosition.value + : Number.POSITIVE_INFINITY + return ( + right.matchedKeywords - left.matchedKeywords || + leftAverage - rightAverage || + compareCodepoints(left.domain, right.domain) + ) + }) + const rows = allRows.slice(offset, offset + input.limit) + const failedCalls = requestedKeywords.length - snapshots.length + const providerCapped = snapshots.some( + (snapshot) => snapshot.returnedRows >= depth, + ) + const capped = allRows.length > offset + input.limit || providerCapped + const coverage: ProviderCoverage = { + requestedRows: input.limit, + returnedRows: allRows.length, + retainedRows: rows.length, + invalidRows, + providerTotalRows: null, + completeness: + failedCalls || invalidRows ? 'partial' : capped ? 'capped' : 'complete', + nextCursor: capped ? String(offset + rows.length) : null, + } + if (invalidRows) { + warnings.push({ + code: 'invalid-organic-result-rows', + field: 'data.rows', + message: `Semrush returned ${invalidRows} organic result row${invalidRows === 1 ? '' : 's'} without a valid domain or position.`, + }) + } + if (duplicateRows) { + warnings.push({ + code: 'duplicate-organic-result-rows', + field: 'data.rows', + message: `${duplicateRows} duplicate organic result row${duplicateRows === 1 ? '' : 's'} were collapsed deterministically.`, + }) + } + const observedAt = snapshots + .map((snapshot) => snapshot.observedAt) + .sort(compareCodepoints) + .at(-1) + return { + schemaVersion: 1, + provider: 'semrush', + capability: 'serp-competitors', + data: { + keywords: requestedKeywords, + rows, + totalRows: null, + }, + observedAt: observedAt as string, + market, + coverage, + cache: combinedCache(snapshots), + cost: combinedCost(snapshots), + request: { + operation: 'serp-competitors', + endpoint: 'https://api.semrush.com/', + limit: input.limit, + filters: { + database, + countryCode: market.countryCode, + languageCode: market.languageCode, + keywordCount: requestedKeywords.length, + resultTypes: 'organic', + includeSubdomains: false, + organicDepthPerKeyword: depth, + providerRequests: requestedKeywords.length, + offset, + apiVersion: 3, + }, + sort: [ + 'matchedKeywords:descending', + 'averagePosition:ascending', + 'domain:codepoint-ascending', + ], + }, + warnings, + } +} diff --git a/packages/core/src/providers/transport.test.ts b/packages/core/src/providers/transport.test.ts index d621d92c..950c2b37 100644 --- a/packages/core/src/providers/transport.test.ts +++ b/packages/core/src/providers/transport.test.ts @@ -61,7 +61,11 @@ test('provider transport returns structured safe errors', async () => { await assert.rejects( providerRequestText({ ...base, - fetch: async () => new Response('secret body', { status: 401 }), + fetch: async () => + new Response('secret body that exceeds the configured response limit', { + status: 401, + }), + maxResponseBytes: 5, }), (error) => { assert.ok(error instanceof ProviderError) @@ -105,13 +109,14 @@ test('provider transport retries only explicitly safe operations', async () => { const safeFetch: ProviderFetch = async () => { safeAttempts += 1 return safeAttempts === 1 - ? new Response('', { status: 503 }) + ? new Response('oversized transient response body', { status: 503 }) : new Response('ok') } assert.equal( await providerRequestText({ ...base, fetch: safeFetch, + maxResponseBytes: 5, retry: 'safe', retryDelayMs: 0, }), diff --git a/packages/core/src/providers/transport.ts b/packages/core/src/providers/transport.ts index e9fd9294..2f7d0e8d 100644 --- a/packages/core/src/providers/transport.ts +++ b/packages/core/src/providers/transport.ts @@ -128,12 +128,15 @@ async function requestOnce(input: ProviderRequestInput): Promise { ...input.init, signal: input.init?.signal ?? AbortSignal.timeout(input.timeoutMs), }) + if (!response.ok) { + await response.body?.cancel().catch(() => undefined) + throw httpError(input, response.status) + } const text = await readBoundedResponseText( response, input.maxResponseBytes, `${input.provider} response`, ) - if (!response.ok) throw httpError(input, response.status) return text } catch (error) { throw requestError(input, error) diff --git a/packages/core/src/storage/database.ts b/packages/core/src/storage/database.ts index e04d649a..61eba525 100644 --- a/packages/core/src/storage/database.ts +++ b/packages/core/src/storage/database.ts @@ -430,6 +430,19 @@ export function getCacheStats(): CacheStats { const database = getDb() const dbPath = getSeoCliPaths().cacheDbFile const logicalSizes = cacheLogicalSizes(database) + const providerCount = (provider: string): number => + ( + database + .prepare( + 'SELECT COUNT(*) AS count FROM provider_cache WHERE provider = ?', + ) + .get(provider) as { count: number } + ).count + const legacySemrushCount = ( + database.prepare('SELECT COUNT(*) AS count FROM semrush_cache').get() as { + count: number + } + ).count const counts = { sites: database.prepare('SELECT COUNT(*) AS count FROM sites').get() as { count: number @@ -440,12 +453,8 @@ export function getCacheStats(): CacheStats { google_analytics_cache: database .prepare('SELECT COUNT(*) AS count FROM ga4_cache') .get() as { count: number }, - semrush_cache: database - .prepare('SELECT COUNT(*) AS count FROM semrush_cache') - .get() as { count: number }, - provider_cache: database - .prepare('SELECT COUNT(*) AS count FROM provider_cache') - .get() as { count: number }, + semrush_cache: { count: legacySemrushCount + providerCount('semrush') }, + provider_cache: { count: providerCount('dataforseo') }, http_cache: database .prepare('SELECT COUNT(*) AS count FROM http_cache') .get() as { count: number }, @@ -479,15 +488,24 @@ export function clearCache( const database = getDb() const cutoff = olderThanMs ? Date.now() - olderThanMs : undefined - if (provider === 'dataforseo') { + if (provider === 'dataforseo' || provider === 'semrush') { const sql = cutoff ? 'DELETE FROM provider_cache WHERE provider = ? AND fetched_at < ?' : 'DELETE FROM provider_cache WHERE provider = ?' + const providerName = provider const info = cutoff - ? database.prepare(sql).run('dataforseo', cutoff) - : database.prepare(sql).run('dataforseo') + ? database.prepare(sql).run(providerName, cutoff) + : database.prepare(sql).run(providerName) + const legacy = + provider === 'semrush' + ? cutoff + ? database + .prepare('DELETE FROM semrush_cache WHERE fetched_at < ?') + .run(cutoff).changes + : database.prepare('DELETE FROM semrush_cache').run().changes + : 0 compactCacheDatabase(database, { allowFullVacuum: true }) - return info.changes + return info.changes + legacy } const tables = @@ -495,17 +513,15 @@ export function clearCache( ? ['gsc_cache'] : provider === 'google-analytics' ? ['ga4_cache'] - : provider === 'semrush' - ? ['semrush_cache'] - : provider === 'http' - ? ['http_cache'] - : [ - 'gsc_cache', - 'ga4_cache', - 'semrush_cache', - 'provider_cache', - 'http_cache', - ] + : provider === 'http' + ? ['http_cache'] + : [ + 'gsc_cache', + 'ga4_cache', + 'semrush_cache', + 'provider_cache', + 'http_cache', + ] let removed = 0 diff --git a/packages/core/src/storage/provider-secrets.ts b/packages/core/src/storage/provider-secrets.ts index d09bb74a..eb1b66ea 100644 --- a/packages/core/src/storage/provider-secrets.ts +++ b/packages/core/src/storage/provider-secrets.ts @@ -16,6 +16,7 @@ export const PROVIDER_SECRET_NAMES = { bingApiKey: 'bing-api-key', dataForSeoCredentials: 'dataforseo-credentials', indexNowKeys: 'indexnow-keys', + semrushApiKey: 'semrush-api-key', } as const export const MANAGED_PROVIDER_SECRET_NAMES = Object.freeze( diff --git a/packages/core/src/storage/reset.test.ts b/packages/core/src/storage/reset.test.ts index 56a007b5..c99bd38f 100644 --- a/packages/core/src/storage/reset.test.ts +++ b/packages/core/src/storage/reset.test.ts @@ -125,7 +125,7 @@ after(() => { test('reset removes Google and provider keychain secrets before local files', async () => { await seedStorage(true) - assert.equal(keyring.values.size, 5) + assert.equal(keyring.values.size, 2 + MANAGED_PROVIDER_SECRET_NAMES.length) await resetSeoData() diff --git a/packages/mcp/src/opportunity-tools.ts b/packages/mcp/src/opportunity-tools.ts index 2a2dbc29..5cb6c6c8 100644 --- a/packages/mcp/src/opportunity-tools.ts +++ b/packages/mcp/src/opportunity-tools.ts @@ -3,7 +3,6 @@ import { cannibalReport, ctrUnderperformersReport, decayingReport, - getKeywordProvider, internalLinksReport, queryClusterReport, quickWinsReport, @@ -12,7 +11,7 @@ import * as z from 'zod/v4' import { fetchRateInput } from './fetch-rate.js' import { resolveJsOption } from './input-schemas.js' import { mcpReportInputSchema } from './report-options.js' -import { summarize, toolError, toolSuccess } from './tool-result.js' +import { toolError, toolSuccess } from './tool-result.js' type QuickWinsToolInput = { site: string @@ -409,41 +408,4 @@ export function registerOpportunityTools( } }, ) - - server.registerTool( - 'semrush_call', - { - description: - 'Raw-ish Semrush passthrough for supported keyword endpoints', - inputSchema: { - endpoint: z.enum(['phrase_this', 'phrase_related', 'phrase_questions']), - phrase: z.string(), - }, - }, - async ({ endpoint, phrase }) => { - try { - const provider = await getKeywordProvider('authoritative') - if (!provider) { - throw new Error('No keyword provider configured.') - } - - const result = - endpoint === 'phrase_this' - ? await provider.keywordOverview(phrase) - : endpoint === 'phrase_related' - ? await provider.relatedKeywords?.(phrase) - : await provider.questions?.(phrase) - - if (!result) { - throw new Error( - `Endpoint ${endpoint} is not supported by the active provider.`, - ) - } - - return toolSuccess(summarize(result.data), result) - } catch (error) { - return toolError(error) - } - }, - ) } diff --git a/scripts/provider-resource-harness.mjs b/scripts/provider-resource-harness.mjs index 0c5e6254..82db57aa 100644 --- a/scripts/provider-resource-harness.mjs +++ b/scripts/provider-resource-harness.mjs @@ -3,6 +3,7 @@ import { mkdtempSync, rmSync, statSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { Response } from 'undici' +import { runSemrushResourceHarness } from './provider-resource-semrush.mjs' const MEBIBYTE = 1024 * 1024 const ROW_COUNT = 80_000 @@ -963,6 +964,7 @@ try { assert.ok(cacheStats.logicalSizeBytes <= cacheStats.maxSizeBytes) assert.equal(clearCache('dataforseo'), BATCHES) assert.equal(getCacheStats().counts.provider_cache, 0) + await runSemrushResourceHarness({ mebibyte: MEBIBYTE }) } finally { rmSync(cacheDir, { recursive: true, force: true }) } diff --git a/scripts/provider-resource-semrush.mjs b/scripts/provider-resource-semrush.mjs new file mode 100644 index 00000000..98aa7005 --- /dev/null +++ b/scripts/provider-resource-semrush.mjs @@ -0,0 +1,99 @@ +import assert from 'node:assert/strict' +import { Response } from 'undici' + +const BATCHES = 10 +const ROWS_PER_BATCH = 1_000 +const MAX_DURATION_MS = 10_000 + +export async function runSemrushResourceHarness({ mebibyte }) { + const { clearCache, getCacheStats, SemrushClient } = await import( + '../dist/index.js' + ) + const maxRssGrowth = 256 * mebibyte + const maxOutputBytes = mebibyte + let balanceCalls = 0 + let reportCalls = 0 + let bytesRead = 0 + const client = new SemrushClient({ + apiKey: 'resource-test-api-key', + fetch: async (url) => { + const requestUrl = new URL(String(url)) + if (requestUrl.hostname === 'www.semrush.com') { + balanceCalls += 1 + bytesRead += 6 + return new Response('500000') + } + reportCalls += 1 + const domain = requestUrl.searchParams.get('domain') + const csv = [ + 'Keyword;Position;Search Volume;URL', + ...Array.from( + { length: ROWS_PER_BATCH }, + (_, index) => + `bounded keyword ${reportCalls}-${index};${1 + (index % 100)};${index % 1_000};https://${domain}/pages/${index}`, + ), + ].join('\n') + bytesRead += Buffer.byteLength(csv) + return new Response(csv) + }, + }) + const baselineRss = process.memoryUsage().rss + const startedAt = performance.now() + let lastSnapshot + for (let batch = 0; batch < BATCHES; batch += 1) { + lastSnapshot = await client.report({ + operation: 'ranked-keywords', + reportType: 'domain_organic', + parameters: { + domain: `domain-${batch}.example`, + database: 'us', + display_limit: ROWS_PER_BATCH, + }, + columns: ['Ph', 'Po', 'Nq', 'Ur'], + maximumResponseRows: ROWS_PER_BATCH, + unitsPerLine: 10, + }) + } + const cached = await client.report({ + operation: 'ranked-keywords', + reportType: 'domain_organic', + parameters: { + domain: 'domain-0.example', + database: 'us', + display_limit: ROWS_PER_BATCH, + }, + columns: ['Ph', 'Po', 'Nq', 'Ur'], + maximumResponseRows: ROWS_PER_BATCH, + unitsPerLine: 10, + }) + const durationMs = performance.now() - startedAt + const rssGrowthBytes = Math.max(0, process.memoryUsage().rss - baselineRss) + const outputBytes = Buffer.byteLength(JSON.stringify(lastSnapshot)) + const cacheStats = getCacheStats() + console.log( + JSON.stringify({ + provider: 'semrush', + requestedRows: BATCHES * ROWS_PER_BATCH, + balanceCalls, + paidCalls: reportCalls, + durationMs: Math.round(durationMs), + rssGrowthMiB: Number((rssGrowthBytes / mebibyte).toFixed(1)), + bytesRead, + cacheBytesWritten: cacheStats.logicalSizeBytes, + diskBytes: cacheStats.sizeBytes, + outputBytes, + estimatedApiUnits: lastSnapshot?.cost.native?.estimatedUnits ?? null, + actualApiUnits: lastSnapshot?.cost.native?.actualUnits ?? null, + }), + ) + assert.equal(cached.cache.status, 'hit') + assert.equal(reportCalls, BATCHES) + assert.equal(balanceCalls, BATCHES) + assert.equal(cacheStats.counts.semrush_cache, BATCHES) + assert.ok(durationMs <= MAX_DURATION_MS) + assert.ok(rssGrowthBytes <= maxRssGrowth) + assert.ok(outputBytes <= maxOutputBytes) + assert.ok(cacheStats.logicalSizeBytes <= cacheStats.maxSizeBytes) + assert.equal(clearCache('semrush'), BATCHES) + assert.equal(getCacheStats().counts.semrush_cache, 0) +} diff --git a/scripts/semrush-live-acceptance.mjs b/scripts/semrush-live-acceptance.mjs new file mode 100644 index 00000000..e07ef12c --- /dev/null +++ b/scripts/semrush-live-acceptance.mjs @@ -0,0 +1,326 @@ +import { strict as assert } from 'node:assert' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { pathToFileURL } from 'node:url' + +const MARKET = { + searchEngine: 'google', + countryCode: 'US', + languageCode: 'en', +} + +export const SEMRUSH_LIVE_PLAN = Object.freeze({ + provider: 'semrush', + market: MARKET, + domain: 'semrush.com', + keywords: ['seo', 'technical seo'], + maximumApiUnits: 180, + paidRequests: 6, + checks: Object.freeze([ + { id: 'keyword-metrics', maximumRows: 2, unitsPerRow: 10 }, + { id: 'keyword-discovery', maximumRows: 3, unitsPerRow: 20 }, + { id: 'domain-overview', maximumRows: 1, unitsPerRow: 10 }, + { id: 'ranked-keywords', maximumRows: 3, unitsPerRow: 10 }, + { id: 'ranking-pages', maximumRows: 3, unitsPerRow: 10 }, + { id: 'serp-competitors', maximumRows: 3, unitsPerRow: 10 }, + ]), +}) + +const HELP = `Usage: + node scripts/semrush-live-acceptance.mjs --plan + node scripts/semrush-live-acceptance.mjs --accept-api-units 180 + +The live run uses six bounded paid requests and will spend no more than 180 +Semrush API units. It reads the saved local credential, uses an isolated +temporary cache, and prints only an acceptance summary. +` + +export function parseSemrushLiveArguments(args) { + if (args[0] === '--') return parseSemrushLiveArguments(args.slice(1)) + if (args.length === 1 && args[0] === '--plan') return { mode: 'plan' } + if (args.length === 1 && args[0] === '--help') return { mode: 'help' } + + const equalsArgument = args.find((value) => + value.startsWith('--accept-api-units='), + ) + const flagIndex = args.indexOf('--accept-api-units') + const acceptedValue = + equalsArgument?.slice('--accept-api-units='.length) ?? + (flagIndex >= 0 ? args[flagIndex + 1] : undefined) + const expectedArguments = equalsArgument + ? [equalsArgument] + : ['--accept-api-units', acceptedValue] + + if ( + acceptedValue === undefined || + args.length !== expectedArguments.length || + args.some((value, index) => value !== expectedArguments[index]) + ) { + throw new Error( + 'Live acceptance requires `--accept-api-units 180`. Use `--plan` to inspect the requests without calling Semrush.', + ) + } + + const acceptedApiUnits = Number(acceptedValue) + if ( + !Number.isSafeInteger(acceptedApiUnits) || + acceptedApiUnits !== SEMRUSH_LIVE_PLAN.maximumApiUnits + ) { + throw new Error( + `Set --accept-api-units to exactly ${SEMRUSH_LIVE_PLAN.maximumApiUnits} for this acceptance plan.`, + ) + } + return { mode: 'live', acceptedApiUnits } +} + +function requestUrl(input) { + if (input instanceof URL) return input + if (typeof input === 'string') return new URL(input) + if (input && typeof input === 'object' && 'url' in input) { + return new URL(String(input.url)) + } + throw new Error('Semrush acceptance received an unsupported request URL.') +} + +function checkEvidence(id, capability, evidence, maximumRows) { + assert.equal(evidence.provider, 'semrush', `${id} provider`) + assert.equal(evidence.capability, capability, `${id} capability`) + assert.equal(evidence.cache.status, 'miss', `${id} live cache state`) + assert.ok( + evidence.coverage.requestedRows <= maximumRows, + `${id} requested row bound`, + ) + assert.ok( + evidence.coverage.returnedRows <= maximumRows, + `${id} returned row bound`, + ) + assert.ok( + evidence.coverage.retainedRows <= maximumRows, + `${id} retained row bound`, + ) + assert.equal(evidence.cost.native?.unit, 'api-unit', `${id} cost unit`) + assert.ok( + Number.isSafeInteger(evidence.cost.native?.estimatedUnits), + `${id} estimated API units`, + ) + assert.ok( + Number.isSafeInteger(evidence.cost.native?.actualUnits), + `${id} actual API units`, + ) + assert.ok( + evidence.cost.native.actualUnits <= evidence.cost.native.estimatedUnits, + `${id} actual API unit bound`, + ) +} + +function costTotal(results, field) { + return results.reduce( + (sum, result) => sum + (result.cost.native?.[field] ?? 0), + 0, + ) +} + +async function runLiveAcceptance(acceptedApiUnits) { + const cacheDir = await mkdtemp(join(tmpdir(), 'seo-semrush-live-')) + const previousCacheDir = process.env.SEO_CACHE_DIR + process.env.SEO_CACHE_DIR = cacheDir + + try { + const { + SemrushClient, + SemrushDomainResearchProvider, + SemrushKeywordDiscoveryProvider, + SemrushKeywordMetricsProvider, + } = await import('../dist/index.js') + + let balanceRequests = 0 + let paidRequests = 0 + const countedFetch = async (input, init) => { + const url = requestUrl(input) + if (url.hostname === 'www.semrush.com') { + balanceRequests += 1 + } else if (url.hostname === 'api.semrush.com') { + paidRequests += 1 + } else { + throw new Error( + 'Semrush acceptance refused a request to an unexpected host.', + ) + } + return fetch(input, init) + } + + const client = new SemrushClient({ + fetch: countedFetch, + reportTtlMs: 5 * 60 * 1_000, + }) + const keywordMetrics = new SemrushKeywordMetricsProvider({ client }) + const keywordDiscovery = new SemrushKeywordDiscoveryProvider({ client }) + const domainResearch = new SemrushDomainResearchProvider({ client }) + + const before = await client.apiUnitBalance() + if (before.remainingUnits < acceptedApiUnits) { + throw new Error( + `Semrush has ${before.remainingUnits} API units. This acceptance plan requires at least ${acceptedApiUnits}.`, + ) + } + + const metrics = await keywordMetrics.keywordMetrics({ + keywords: SEMRUSH_LIVE_PLAN.keywords, + market: MARKET, + refresh: true, + }) + checkEvidence('keyword-metrics', 'keyword-metrics', metrics, 2) + + const discovery = await keywordDiscovery.discoverKeywords({ + seeds: ['seo'], + sources: ['ideas'], + market: MARKET, + limit: 3, + refresh: true, + }) + checkEvidence('keyword-discovery', 'keyword-discovery', discovery, 3) + + const overview = await domainResearch.domainOverview({ + domain: SEMRUSH_LIVE_PLAN.domain, + market: MARKET, + refresh: true, + }) + checkEvidence('domain-overview', 'domain-overview', overview, 1) + + const rankedKeywords = await domainResearch.rankedKeywords({ + target: SEMRUSH_LIVE_PLAN.domain, + market: MARKET, + includeSubdomains: true, + resultTypes: ['organic'], + limit: 3, + refresh: true, + }) + checkEvidence('ranked-keywords', 'ranked-keywords', rankedKeywords, 3) + + const rankingPages = await domainResearch.rankingPages({ + domain: SEMRUSH_LIVE_PLAN.domain, + market: MARKET, + limit: 3, + refresh: true, + }) + checkEvidence('ranking-pages', 'relevant-pages', rankingPages, 3) + + const competitors = await domainResearch.serpCompetitors({ + keywords: ['seo'], + market: MARKET, + includeSubdomains: false, + resultTypes: ['organic'], + limit: 3, + refresh: true, + }) + checkEvidence('serp-competitors', 'serp-competitors', competitors, 3) + + const results = [ + metrics, + discovery, + overview, + rankedKeywords, + rankingPages, + competitors, + ] + const estimatedApiUnits = costTotal(results, 'estimatedUnits') + const actualApiUnits = costTotal(results, 'actualUnits') + assert.equal( + estimatedApiUnits, + SEMRUSH_LIVE_PLAN.maximumApiUnits, + 'acceptance plan API unit estimate', + ) + assert.ok( + actualApiUnits <= acceptedApiUnits, + 'acceptance plan actual API unit bound', + ) + assert.equal( + paidRequests, + SEMRUSH_LIVE_PLAN.paidRequests, + 'bounded paid request count', + ) + + const requestsBeforeCacheCheck = { + balance: balanceRequests, + paid: paidRequests, + } + const cachedMetrics = await keywordMetrics.keywordMetrics({ + keywords: SEMRUSH_LIVE_PLAN.keywords, + market: MARKET, + }) + assert.equal(cachedMetrics.cache.status, 'hit', 'keyword metrics cache hit') + assert.equal( + cachedMetrics.cost.native?.actualUnits, + 0, + 'cache hit API unit cost', + ) + assert.deepEqual( + { balance: balanceRequests, paid: paidRequests }, + requestsBeforeCacheCheck, + 'cache hit network requests', + ) + + const after = await client.apiUnitBalance() + assert.ok(balanceRequests <= 24, 'bounded balance request count') + + return { + provider: 'semrush', + status: 'passed', + market: MARKET, + target: SEMRUSH_LIVE_PLAN.domain, + checks: SEMRUSH_LIVE_PLAN.checks.map((check, index) => ({ + id: check.id, + returnedRows: results[index].coverage.returnedRows, + retainedRows: results[index].coverage.retainedRows, + completeness: results[index].coverage.completeness, + })), + apiUnits: { + acceptedMaximum: acceptedApiUnits, + estimated: estimatedApiUnits, + reportedActual: actualApiUnits, + balanceBefore: before.remainingUnits, + balanceAfter: after.remainingUnits, + observedBalanceChange: before.remainingUnits - after.remainingUnits, + }, + network: { + balanceRequests, + paidRequests, + }, + cache: { + keywordMetricsRepeat: cachedMetrics.cache.status, + isolatedCacheRemoved: true, + }, + } + } finally { + if (previousCacheDir === undefined) delete process.env.SEO_CACHE_DIR + else process.env.SEO_CACHE_DIR = previousCacheDir + await rm(cacheDir, { force: true, recursive: true }) + } +} + +export async function main(args = process.argv.slice(2)) { + const options = parseSemrushLiveArguments(args) + if (options.mode === 'help') { + process.stdout.write(HELP) + return + } + if (options.mode === 'plan') { + process.stdout.write(`${JSON.stringify(SEMRUSH_LIVE_PLAN, null, 2)}\n`) + return + } + const summary = await runLiveAcceptance(options.acceptedApiUnits) + process.stdout.write(`${JSON.stringify(summary, null, 2)}\n`) +} + +const isDirect = + process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href + +if (isDirect) { + main().catch((error) => { + const message = + error instanceof Error ? error.message : 'Unknown acceptance failure.' + process.stderr.write(`Semrush live acceptance failed: ${message}\n`) + process.exitCode = 1 + }) +} diff --git a/scripts/semrush-live-acceptance.test.mjs b/scripts/semrush-live-acceptance.test.mjs new file mode 100644 index 00000000..b81a1143 --- /dev/null +++ b/scripts/semrush-live-acceptance.test.mjs @@ -0,0 +1,42 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { + parseSemrushLiveArguments, + SEMRUSH_LIVE_PLAN, +} from './semrush-live-acceptance.mjs' + +test('Semrush live plan has one exact bounded spend ceiling', () => { + const maximumApiUnits = SEMRUSH_LIVE_PLAN.checks.reduce( + (sum, check) => sum + check.maximumRows * check.unitsPerRow, + 0, + ) + assert.equal(maximumApiUnits, 180) + assert.equal(SEMRUSH_LIVE_PLAN.maximumApiUnits, maximumApiUnits) + assert.equal(SEMRUSH_LIVE_PLAN.paidRequests, SEMRUSH_LIVE_PLAN.checks.length) +}) + +test('Semrush live arguments require explicit spend acceptance', () => { + assert.deepEqual(parseSemrushLiveArguments(['--plan']), { mode: 'plan' }) + assert.deepEqual(parseSemrushLiveArguments(['--help']), { mode: 'help' }) + assert.deepEqual(parseSemrushLiveArguments(['--', '--plan']), { + mode: 'plan', + }) + assert.deepEqual(parseSemrushLiveArguments(['--accept-api-units', '180']), { + mode: 'live', + acceptedApiUnits: 180, + }) + assert.deepEqual(parseSemrushLiveArguments(['--accept-api-units=180']), { + mode: 'live', + acceptedApiUnits: 180, + }) + assert.throws(() => parseSemrushLiveArguments([]), /requires/) + assert.throws( + () => parseSemrushLiveArguments(['--accept-api-units', '181']), + /exactly 180/, + ) + assert.throws( + () => + parseSemrushLiveArguments(['--accept-api-units', '180', '--unexpected']), + /requires/, + ) +}) diff --git a/turbo.json b/turbo.json index 2e535ef7..e9552006 100644 --- a/turbo.json +++ b/turbo.json @@ -5,6 +5,7 @@ "LOC_MAX_LINES", "LOC_TOP", "SEO_CRUX_API_KEY", + "SEO_CACHE_DIR", "SEO_GOOGLE_CLIENT_ID", "SEO_GOOGLE_CLIENT_SECRET", "SEO_RESOURCE_CONCURRENCY",