sync: carry news.opml, and stop hand-maintaining the profullstack defaults (#383)

Two things `/save` and the feed list got wrong.

`/save` did not carry your subscriptions. SYNCED_FILES had `feeds.opml` in it,
which is written by `tcfeed rss add` and read by nothing here; `/news` and
`/rss` keep their subscriptions in `news.opml`, which was in the allowlist zero
times. The two names sat one line apart and were easy to mistake for each
other, so the feeds you actually chose were the one thing that stayed on one
machine. Both are carried now, with a comment saying which belongs to what.

No size cap change. A subscription list past MAX_FILE_BYTES is already reported
as skipped rather than failing the snapshot, which is the right answer for a
list that got big by importing somebody else's.

The fourteen profullstack blogs were typed into DEFAULT_FEEDS by hand, which
made a second copy of a list that is already published at
profullstack.com/feeds.opml. Two hand-maintained copies of the same list is one
more than can be kept in step, and the one that would go stale is ours —
nobody editing the published OPML has a reason to think about moshcode.

So the published file is vendored at src/profullstack-feeds.opml and parsed
into the defaults. Read synchronously at module load, because defaultFeeds() is
synchronous and a fresh install must not wait on the network to show anything;
`files` includes `src`, so it travels in both the npm and install.sh channels
(verified against `npm pack`). A missing file degrades to no profullstack
defaults rather than throwing, which would take `/news` down over a packaging
mistake.

It is parsed by a small matcher rather than parseOpml, because news.mjs imports
this module and taking the import back the other way makes a cycle. Same reason
slugify() is copied as slug(). Both shortcuts are held in place by tests that
run the real parseOpml and the real slugify over the vendored file and require
the same answers — the shortcut is allowed to be small, not allowed to differ.

Drift against the live URL is checked by test/profullstack-feeds.test.mjs under
MOSHCODE_CHECK_FEED_DRIFT=1. Opt-in on purpose: a suite that fails whenever
profullstack.com blinks is a suite people learn to ignore. Everything checkable
without a network runs always.

Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>

Author: ralyodio

src/news-sources.mjs

@@ -20,6 +20,8 @@
 // holds feeds with stable well-known URLs and defers everything else to the
 // published lists, where the list is somebody else's to maintain.
 
+import fs from "node:fs";
+
 /**
  * Bing News query feed — the only search feed left.
  *
@@ -86,6 +88,53 @@ export function isDeadEndLink(url) {
   }
 }
 
+/** The vendored copy of what profullstack.com/feeds.opml serves. */
+const PROFULLSTACK_OPML = new URL("./profullstack-feeds.opml", import.meta.url);
+
+/** slugify() from news.mjs, kept in step by a test rather than imported. */
+function slug(label) {
+  return String(label ?? "")
+    .toLowerCase()
+    .replace(/['’]/g, "")
+    .replace(/[^a-z0-9]+/g, "-")
+    .replace(/^-+|-+$/g, "")
+    .slice(0, 32);
+}
+
+/**
+ * The profullstack blogs, read from the vendored OPML rather than typed out.
+ *
+ * Read synchronously and at module load because defaultFeeds() is synchronous —
+ * a fresh install must not wait on a network call, or on a promise, to show
+ * anything at all. The file ships with the package (`files` includes `src`), so
+ * it is there in both the npm and the install.sh channel.
+ *
+ * Parsed here with a small matcher instead of news.mjs's parseOpml, because
+ * news.mjs imports this module and taking the import back the other way makes a
+ * cycle. The matcher can afford to be small: this is our own file, flat, and a
+ * test asserts the two agree on every feed it contains.
+ *
+ * A missing or unreadable file degrades to no profullstack defaults rather than
+ * throwing, which would take `/news` down entirely over a packaging mistake.
+ */
+function profullstackFeeds() {
+  let xml;
+  try { xml = fs.readFileSync(PROFULLSTACK_OPML, "utf8"); }
+  catch { return []; }
+
+  const feeds = [];
+  const seen = new Set();
+  for (const tag of xml.match(/<outline\b[^>]*>/gi) ?? []) {
+    const attr = (name) => (new RegExp(`\\b${name}="([^"]*)"`, "i").exec(tag) ?? [])[1] ?? "";
+    const url = attr("xmlUrl");
+    if (!/^https?:\/\//i.test(url) || seen.has(url)) continue;
+    seen.add(url);
+    const title = attr("title") || attr("text") || url;
+    feeds.push({ name: slug(title), title, url, site: attr("htmlUrl"), category: "profullstack" });
+  }
+  return feeds;
+}
+
 /**
  * The feeds a fresh install reads.
  *
@@ -120,25 +169,18 @@ export const DEFAULT_FEEDS = [
 
   { name: "npr-politics", title: "NPR — Politics", url: "https://feeds.npr.org/1014/rss.xml", site: "https://www.npr.org", category: "politics" },
 
-  // The profullstack blogs, read out of the box. This is the one published list
-  // small enough to be a default: profullstack.com/feeds.opml is 14 feeds, where
-  // smallweb is 33,000 and could only ever be searched. Kept in step with that
-  // file by hand rather than fetched, because defaultFeeds() is synchronous and
-  // a fresh install must not wait on a network call to show anything at all.
-  { name: "bittorrented-blog", title: "BitTorrented Blog", url: "https://bittorrented.com/blog/rss.xml", site: "https://bittorrented.com/blog", category: "profullstack" },
-  { name: "bl0ggers-blog", title: "bl0ggers Blog", url: "https://bl0ggers.com/blog/rss.xml", site: "https://bl0ggers.com/blog", category: "profullstack" },
-  { name: "c0mpute-blog", title: "c0mpute Blog", url: "https://c0mpute.com/blog/rss.xml", site: "https://c0mpute.com/blog", category: "profullstack" },
-  { name: "c0upons-blog", title: "c0upons Blog", url: "https://c0upons.com/blog/rss.xml", site: "https://c0upons.com/blog", category: "profullstack" },
-  { name: "coinpay-blog", title: "CoinPay Blog", url: "https://coinpayportal.com/blog/rss.xml", site: "https://coinpayportal.com/blog", category: "profullstack" },
-  { name: "crawlproof-blog", title: "CrawlProof Blog", url: "https://crawlproof.com/blog/rss.xml", site: "https://crawlproof.com/blog", category: "profullstack" },
-  { name: "d0rz-blog", title: "d0rz Blog", url: "https://d0rz.com/blog/rss.xml", site: "https://d0rz.com/blog", category: "profullstack" },
-  { name: "logicsrc-blog", title: "LogicSRC Blog", url: "https://logicsrc.com/blog/rss.xml", site: "https://logicsrc.com/blog", category: "profullstack" },
-  { name: "pairux-blog", title: "PairUX Blog", url: "https://pairux.com/blog/rss.xml", site: "https://pairux.com/blog", category: "profullstack" },
-  { name: "qryptchat-blog", title: "QryptChat Blog", url: "https://qrypt.chat/blog/rss.xml", site: "https://qrypt.chat/blog", category: "profullstack" },
-  { name: "saasrow-blog", title: "SaaSRow Blog", url: "https://www.saasrow.com/blog/rss.xml", site: "https://www.saasrow.com/blog", category: "profullstack" },
-  { name: "sh1pt-blog", title: "sh1pt Blog", url: "https://sh1pt.com/blog/rss.xml", site: "https://sh1pt.com/blog", category: "profullstack" },
-  { name: "threatcrush-blog", title: "ThreatCrush Blog", url: "https://threatcrush.com/blog/rss.xml", site: "https://threatcrush.com/blog", category: "profullstack" },
-  { name: "ugig-blog", title: "ugig Blog", url: "https://ugig.net/blog/rss.xml", site: "https://ugig.net/blog", category: "profullstack" },
+  // The profullstack blogs, read out of the box. Not typed out here: they are
+  // parsed from src/profullstack-feeds.opml, which is a copy of what
+  // profullstack.com/feeds.opml actually serves. Two hand-maintained lists of
+  // the same fourteen blogs is one more than can be kept in step, and the copy
+  // that would go stale is this one — nobody editing the published OPML has a
+  // reason to think about moshcode. Refresh it with:
+  //
+  //   curl -sL https://profullstack.com/feeds.opml -o src/profullstack-feeds.opml
+  //
+  // test/profullstack-feeds.test.mjs checks that against the live file when
+  // MOSHCODE_CHECK_FEED_DRIFT=1 is set.
+  ...profullstackFeeds(),
 ];
 
 /**

src/profullstack-feeds.opml

@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<opml version="2.0">
+  <head>
+    <title>Profullstack Blogs</title>
+    <ownerName>Profullstack</ownerName>
+    <ownerId>https://profullstack.com</ownerId>
+    <dateCreated>Thu, 13 Aug 2026 00:00:00 +0000</dateCreated>
+  </head>
+  <body>
+    <outline text="BitTorrented Blog" title="BitTorrented Blog" type="rss" xmlUrl="https://bittorrented.com/blog/rss.xml" htmlUrl="https://bittorrented.com/blog" />
+    <outline text="bl0ggers Blog" title="bl0ggers Blog" type="rss" xmlUrl="https://bl0ggers.com/blog/rss.xml" htmlUrl="https://bl0ggers.com/blog" />
+    <outline text="c0mpute Blog" title="c0mpute Blog" type="rss" xmlUrl="https://c0mpute.com/blog/rss.xml" htmlUrl="https://c0mpute.com/blog" />
+    <outline text="c0upons Blog" title="c0upons Blog" type="rss" xmlUrl="https://c0upons.com/blog/rss.xml" htmlUrl="https://c0upons.com/blog" />
+    <outline text="CoinPay Blog" title="CoinPay Blog" type="rss" xmlUrl="https://coinpayportal.com/blog/rss.xml" htmlUrl="https://coinpayportal.com/blog" />
+    <outline text="CrawlProof Blog" title="CrawlProof Blog" type="rss" xmlUrl="https://crawlproof.com/blog/rss.xml" htmlUrl="https://crawlproof.com/blog" />
+    <outline text="d0rz Blog" title="d0rz Blog" type="rss" xmlUrl="https://d0rz.com/blog/rss.xml" htmlUrl="https://d0rz.com/blog" />
+    <outline text="LogicSRC Blog" title="LogicSRC Blog" type="rss" xmlUrl="https://logicsrc.com/blog/rss.xml" htmlUrl="https://logicsrc.com/blog" />
+    <outline text="PairUX Blog" title="PairUX Blog" type="rss" xmlUrl="https://pairux.com/blog/rss.xml" htmlUrl="https://pairux.com/blog" />
+    <outline text="QryptChat Blog" title="QryptChat Blog" type="rss" xmlUrl="https://qrypt.chat/blog/rss.xml" htmlUrl="https://qrypt.chat/blog" />
+    <outline text="SaaSRow Blog" title="SaaSRow Blog" type="rss" xmlUrl="https://www.saasrow.com/blog/rss.xml" htmlUrl="https://www.saasrow.com/blog" />
+    <outline text="sh1pt Blog" title="sh1pt Blog" type="rss" xmlUrl="https://sh1pt.com/blog/rss.xml" htmlUrl="https://sh1pt.com/blog" />
+    <outline text="ThreatCrush Blog" title="ThreatCrush Blog" type="rss" xmlUrl="https://threatcrush.com/blog/rss.xml" htmlUrl="https://threatcrush.com/blog" />
+    <outline text="ugig Blog" title="ugig Blog" type="rss" xmlUrl="https://ugig.net/blog/rss.xml" htmlUrl="https://ugig.net/blog" />
+  </body>
+</opml>

src/settings-sync.mjs

@@ -64,6 +64,16 @@ export const SYNCED_FILES = [
   // and read by nothing here — moshcode's interest in it begins and ends with
   // moving it, and a file this does not parse cannot be broken by this.
   { path: "feeds.opml", json: false, label: "rss feeds" },
+  // `/news` and `/rss` keep their subscriptions here — see opmlFile() in
+  // news.mjs. A near-identical name sat in this list above it for a while and
+  // the two were easy to mistake for each other, so: feeds.opml belongs to
+  // tcfeed, news.opml belongs to this. Carrying only the first meant the feeds
+  // you actually subscribed to were the one thing `/save` left behind.
+  //
+  // Deliberately no cap change. A subscription list past MAX_FILE_BYTES is
+  // reported as skipped rather than failing the snapshot, which is the right
+  // answer for a list that got big by importing somebody else's.
+  { path: "news.opml", json: false, label: "news subscriptions" },
 ];
 
 /**

test/profullstack-feeds.test.mjs

@@ -0,0 +1,90 @@
+// The vendored profullstack feed list, against the published one.
+//
+// src/profullstack-feeds.opml is a copy of what profullstack.com/feeds.opml
+// serves. It is a copy on purpose: defaultFeeds() is synchronous, so a fresh
+// install cannot fetch the list before it shows anything. The cost of that
+// choice is drift, and the point of this file is to make drift loud.
+//
+// Everything that can be checked without a network runs always — that the file
+// is there, that it parses, that the small matcher in news-sources.mjs agrees
+// with the real parseOpml, and that its private slug() still matches
+// slugify(). Those are the failures a refactor actually causes.
+//
+// The one check that needs the network — vendored copy against the live URL —
+// is opt-in, because a suite that fails when profullstack.com is briefly down
+// is a suite people learn to ignore. Run it deliberately:
+//
+//   MOSHCODE_CHECK_FEED_DRIFT=1 node --test test/profullstack-feeds.test.mjs
+import assert from "node:assert/strict";
+import fs from "node:fs";
+import path from "node:path";
+import test from "node:test";
+import { fileURLToPath } from "node:url";
+
+import { DEFAULT_FEEDS } from "../src/news-sources.mjs";
+import { parseOpml, slugify } from "../src/news.mjs";
+
+const OPML = path.join(path.dirname(fileURLToPath(import.meta.url)), "..", "src", "profullstack-feeds.opml");
+const LIVE = "https://profullstack.com/feeds.opml";
+
+const vendored = fs.readFileSync(OPML, "utf8");
+const mine = DEFAULT_FEEDS.filter((f) => f.category === "profullstack");
+
+test("the vendored OPML ships and is read", () => {
+  // `files` in package.json includes `src`, so this travels with the package.
+  // If it ever stops, profullstackFeeds() degrades to [] rather than throwing —
+  // which is the right runtime behaviour and exactly why it needs asserting
+  // here instead: silence is the failure mode.
+  assert.ok(vendored.includes("<opml"), "not an OPML document");
+  assert.ok(mine.length > 0, "no profullstack feeds in the defaults");
+});
+
+test("every feed in the file is a default, and nothing else is", () => {
+  // parseOpml is the real parser; news-sources.mjs cannot import it without a
+  // cycle, so it has a small matcher of its own. This is the assertion that the
+  // shortcut did not change the answer.
+  const parsed = parseOpml(vendored);
+  assert.deepEqual(
+    mine.map((f) => f.url).sort(),
+    parsed.map((f) => f.url).sort(),
+  );
+  assert.deepEqual(
+    mine.map((f) => f.title).sort(),
+    parsed.map((f) => f.title).sort(),
+  );
+});
+
+test("the private slug() still agrees with slugify()", () => {
+  // news-sources.mjs copies slugify() rather than importing it, for the same
+  // cycle reason. A feed named differently by the two would be reachable as
+  // `--feed <name>` under one name and listed under another.
+  for (const feed of mine) assert.equal(feed.name, slugify(feed.title), feed.title);
+});
+
+test("the feeds are usable — https, unique, and pointed at a site", () => {
+  const urls = mine.map((f) => f.url);
+  assert.equal(new Set(urls).size, urls.length, "duplicate feed URL");
+  for (const feed of mine) {
+    assert.match(feed.url, /^https:\/\//, `${feed.title} is not https`);
+    assert.ok(feed.title.trim(), "a feed with no title");
+    assert.match(feed.site, /^https:\/\//, `${feed.title} has no site`);
+  }
+});
+
+test("vendored copy matches profullstack.com/feeds.opml", {
+  skip: process.env.MOSHCODE_CHECK_FEED_DRIFT === "1"
+    ? false
+    : "set MOSHCODE_CHECK_FEED_DRIFT=1 to check against the live file",
+}, async () => {
+  const res = await fetch(LIVE);
+  assert.equal(res.ok, true, `${LIVE} answered ${res.status}`);
+  const live = parseOpml(await res.text());
+
+  // Compared by what a reader would act on, not byte for byte: the published
+  // file carries a dateCreated that changes without any feed changing.
+  assert.deepEqual(
+    mine.map((f) => f.url).sort(),
+    live.map((f) => f.url).sort(),
+    `vendored list is stale — refresh with:\n  curl -sL ${LIVE} -o src/profullstack-feeds.opml`,
+  );
+});

test/settings-sync.test.mjs

@@ -41,7 +41,7 @@ const DIGEST_FIXTURE = {
   "herd/rules.json": { content: "{}" },
 };
 
-function home({ aliases = null, rules = null, credentials = true, marker = null, feeds = null } = {}) {
+function home({ aliases = null, rules = null, credentials = true, marker = null, feeds = null, news = null } = {}) {
   const dir = mkdtempSync(path.join(tmpdir(), "moshcode-sync-"));
   const moshcode = path.join(dir, ".moshcode");
   fs.mkdirSync(moshcode, { recursive: true });
@@ -51,6 +51,7 @@ function home({ aliases = null, rules = null, credentials = true, marker = null,
   }
   if (aliases) fs.writeFileSync(path.join(moshcode, "aliases.json"), aliases);
   if (feeds) fs.writeFileSync(path.join(moshcode, "feeds.opml"), feeds);
+  if (news) fs.writeFileSync(path.join(moshcode, "news.opml"), news);
   if (rules) {
     fs.mkdirSync(path.join(moshcode, "herd"), { recursive: true });
     fs.writeFileSync(path.join(moshcode, "herd", "rules.json"), rules);
@@ -275,6 +276,27 @@ test("/save carries the feed list, and does not try to parse it", async () => {
   assert.equal(sent["feeds.opml"].content, OPML, "OPML travels byte for byte");
 });
 
+test("/save carries the news subscriptions too, not only tcfeed's list", async () => {
+  // These two files sat one line apart in SYNCED_FILES with nearly the same
+  // name, and only tcfeed's was in it. `/news` and `/rss` keep their
+  // subscriptions in news.opml, so the feeds you actually chose were the one
+  // thing `/save` left behind.
+  const dir = home({ aliases: "{}", feeds: OPML, news: OPML });
+  const fetchImpl = stubFetch([[200, { revision: 1 }]]);
+
+  const code = await saveCommand([], { home: dir, creds: CREDS, fetchImpl, write: lines(), installed: INSTALLED });
+  assert.equal(code, 0);
+  const sent = fetchImpl.calls[0].body.snapshot.files;
+  assert.equal(sent["news.opml"].content, OPML, "the subscription list did not travel");
+  assert.ok(sent["feeds.opml"], "tcfeed's list still travels");
+});
+
+test("news.opml is in the allowlist and is not parsed", () => {
+  const entry = SYNCED_FILES.find((f) => f.path === "news.opml");
+  assert.ok(entry, "news.opml is not synced");
+  assert.equal(entry.json, false, "OPML must not be JSON-parsed");
+});
+
 test("a feed list that is not XML still syncs — nothing here reads it", async () => {
   // The contrast with aliases.json is the point: that one is `json: true` and
   // a broken one is held back rather than copied to every machine. OPML is