App rebuild (area indicators pages)

package.json

@@ -21,9 +21,10 @@
   "license": "MIT",
   "devDependencies": {
     "@mapbox/tilebelt": "^2.0.2",
-    "@onsvisual/robo-utils": "^0.3.7",
+    "@onsvisual/robo-utils": "^0.3.9",
     "@onsvisual/svelte-charts": "^0.4.11",
-    "@onsvisual/svelte-components": "^1.0.30",
+    "@onsvisual/svelte-components": "^1.0.31",
+    "@onsvisual/svelte-maps": "^1.2.25",
     "@sveltejs/adapter-netlify": "^5.2.4",
     "@sveltejs/adapter-node": "^5.3.3",
     "@sveltejs/kit": "^2.46.5",
@@ -42,7 +43,7 @@
     "prettier-plugin-svelte": "^3.2.6",
     "svelte": "^5.0.0",
     "svelte-check": "^4.3.4",
-    "svelteplot": "^0.3.7",
+    "svelteplot": "^0.7.0",
     "temporal-polyfill": "^0.3.0",
     "throttleit": "^2.1.0",
     "topojson-client": "^3.1.0",

scripts/generate-data.ts

@@ -111,7 +111,7 @@ function indicatorToCube(indicator, t, meta_data, tableSchema, dataset_name) {
             source: meta_data.metadata.source,
             slug: manifest_metadata_indicator[0].slug,
             ...restOfMetadata,
-            experimentalStatistic: meta_data.experimentalStatistic,
+            experimentalStatistic: meta_data.metadata.experimentalStatistic,
             geography: meta_data.metadata.geography
         }
     }
@@ -316,9 +316,26 @@ const indicators = [];
 for (const file of file_paths) {
     indicators.push(...processFile(file,excluded_indicators));
 }
-cube.link.item = indicator_slugs.map(slug => indicators.find(ind => ind.extension.slug === slug))
+// Sort indicators to match order in manifest (ie. taxonomy order)
+cube.link.item = indicator_slugs.map(slug => indicators.find(ind => ind.extension.slug === slug));
 
-// console.log(cube.link.item)
 const output = "./src/lib/data/json-stat.json";
 writeFileSync(output, JSON.stringify(cube));
-console.log(`Wrote ${output}.`)
+console.log(`Wrote ${output}.`)
+
+// Generate JSON file with summary stats/data
+const summaryData = {
+    count: cube.link.item.length,
+    topics: Array.from(new Set(cube.link.item.map(ds => ds.extension.topic)))
+        .map(t => ({slug: t.replaceAll(" ", "-"), label: t[0].toUpperCase() + t.slice(1)})),
+    years: Array.from(
+        new Set(cube.link.item.map(ds => 
+            Object.keys(ds.dimension.period.category.index).map(val => +val.slice(0, 4))
+        ).flat())).sort((a, b) => a - b),
+    geoYears: Array.from(new Set(cube.link.item.map(ds => ds.extension.geography.year)))
+        .sort((a, b) => a - b)
+};
+
+const summaryOutput = "./src/lib/data/json-stat-summary.json";
+writeFileSync(summaryOutput, JSON.stringify(summaryData));
+console.log(`Wrote ${summaryOutput}.`);

src/lib/api/data/filterCollection.js

@@ -66,8 +66,8 @@ export default async function filterCollection(params = {}) {
   }
 
   // Create filters for standard dimensions
-  if (params.geo !== "all")
-    filters.areacd = makeGeoFilter(params.geo, params.geoExtent);
+  if (params.geo !== "all" || params.geoCluster !== "all")
+    filters.areacd = makeGeoFilter(params.geo, params.geoExtent, params.geoCluster);
   if (params.time !== "all") {
     if (
       [params.time]

src/lib/api/data/helpers/dataFilters.js

@@ -3,6 +3,9 @@ import { geoLevels } from "$lib/config/geo-levels.js";
 import getChildAreas from "$lib/api/geo/getChildAreas.js";
 import hasObservation from "./hasObservation.js";
 import { isValidMonth, isValidYear } from "$lib/api/utils.js";
+import readData from "$lib/data";
+
+const areasClusters = await readData("areas-clusters");
 
 export function ascending(a, b) {
   return a == null || b == null ? NaN : a < b ? -1 : a > b ? 1 : a >= b ? 0 : NaN;
@@ -13,12 +16,12 @@ export function makeFilter(param) {
   return d => set.has(d[0]);
 }
 
-export function makeGeoFilter(geo, geoExtent) {
+export function makeGeoFilter(geo, geoExtent, geoCluster) {
   const codes = new Set();
   const types = new Set();
   for (const g of [geo].flat()) {
     // if (g.match(/^[EKNSW]\d{2}$/)) types.add(g);
-    if (geoLevels[g]) {
+    if (geoLevels[g] && geoCluster === "all") {
 			if (geoExtent.match(/^[EKNSW]\d{8}$/)) {
 				const children = getChildAreas({code: geoExtent, geoLevel: g, includeNames: false});
 				for (const child of children) codes.add(child);
@@ -28,6 +31,13 @@ export function makeGeoFilter(geo, geoExtent) {
     }
     else if (g.match(/^[EKNSW]\d{8}$/) && !types.has(g.slice(0, 3))) codes.add(g);
   }
+	if (geoCluster) {
+		const [grouping, cluster] = geoCluster.split("_");
+		const cds = areasClusters.clusters?.[grouping]?.[cluster];
+		if (Array.isArray(cds)) {
+			for (const cd of cds) codes.add(cd);
+		}
+	}
   return codes.size > 0 && types.size > 0 ? d => codes.has(d[0]) || types.has(d[0].slice(0, 3)) :
     types.size > 0 ? d => types.has(d[0].slice(0, 3)) :
     codes.size > 0 ? d => codes.has(d[0]) :
@@ -56,11 +66,13 @@ export function getTime(values, params = {}) {
 
 	const periods = values.map(v => ({value: v, period: periodToDateRange(v[0])}));
 	const nearest = params.nearest || "none";
-	const isRange = periods[0].period.length > 1;
-	const date = isRange || params.time.length === 10 ? toPlainDate(params.time, true) : [toPlainDate(params.time, false), toPlainDate(params.time, true)];
+	const periodIsRange = periods[0].period.length > 1; // Time periods have a duration component
+	const dateIsExact = params.time.length === 10; // Requested date is to nearest day
+	const date = periodIsRange || dateIsExact ? toPlainDate(params.time, true) : [toPlainDate(params.time, false), toPlainDate(params.time, true)];
 
 	let match;
-	if (isRange) match = periods.findLast(p => Temporal.PlainDate.compare(date, p.period[0]) !== -1 && Temporal.PlainDate.compare(date, p.period[1]) !== 1);
+	if (periodIsRange) match = periods.findLast(p => Temporal.PlainDate.compare(date, p.period[0]) !== -1 && Temporal.PlainDate.compare(date, p.period[1]) !== 1);
+	else if (dateIsExact) match = periods.findLast(p => Temporal.PlainDate.compare(date, p.period[0]) === 0);
 	else match = periods.findLast(p => Temporal.PlainDate.compare(p.period[0], date[0]) !== -1 && Temporal.PlainDate.compare(p.period[0], date[1]) !== 1);
 	if (match) return [match.value];
 

src/lib/api/metadata/getIndicators.js

@@ -15,11 +15,27 @@ function formatMetadata(ds, minimalMetadata = false, fullDims = false) {
       description: ds.extension.subtitle,
     };
 
-  const metadata = { label: ds.label, ...ds.extension, updated: ds.updated, caveats: ds.note };
-  metadata.dimensions = Object.fromEntries(ds.id.map((key, i) => [key, {...formatDimension(ds, key, fullDims), order: i}]));
+  const metadata = {
+    label: ds.label,
+    ...ds.extension,
+    updated: ds.updated,
+    caveats: ds.note,
+  };
+  metadata.dimensions = Object.fromEntries(
+    ds.id.map((key, i) => [
+      key,
+      { ...formatDimension(ds, key, fullDims), order: i },
+    ])
+  );
   return metadata;
 }
 
+function arrayToLookup(metadata) {
+  const lookup = {};
+  for (const ds of metadata) lookup[ds.slug] = ds;
+  return lookup;
+}
+
 export default function getIndicators(params = {}) {
   const filter = makeDatasetFilter(
     params.indicator,
@@ -32,9 +48,11 @@ export default function getIndicators(params = {}) {
 
   const metadata = rawMetadata.link.item
     .filter(filter)
-    .map((ds) =>
-      formatMetadata(ds, params.minimalMetadata, params.fullDims)
-    );
+    .map((ds) => formatMetadata(ds, params.minimalMetadata, params.fullDims));
 
-  return params.singleIndicator ? metadata[0] : metadata;
+  return params.singleIndicator
+    ? metadata[0]
+    : params.asLookup
+      ? arrayToLookup(metadata)
+      : metadata;
 }

src/lib/api/metadata/getTaxonomy.js

@@ -1,10 +1,11 @@
 import getIndicators from "./getIndicators.js";
-import { capitalise } from "$lib/utils.js";
+import summaryData from "$lib/data/json-stat-summary.json";
+import { capitalise } from "$lib/utils.ts";
 
 function makeItem(slug, label = null, description = null) {
-  const item = {label: label || capitalise(slug), slug};
-  if (description) return {...item, description};
-  return {...item, children: {}};
+  const item = { label: label || capitalise(slug), slug };
+  if (description) return { ...item, description };
+  return { ...item, children: {} };
 }
 
 function nestTaxonomy(taxonomy) {
@@ -18,21 +19,25 @@ function nestTaxonomy(taxonomy) {
     } else {
       if (!topicsIndex[ind.topic].children[ind.subTopic])
         topicsIndex[ind.topic].children[ind.subTopic] = makeItem(ind.subTopic);
-      topicsIndex[ind.topic].children[ind.subTopic].children[ind.slug] = indicator;
+      topicsIndex[ind.topic].children[ind.subTopic].children[ind.slug] =
+        indicator;
     }
   }
 
   const topics = Object.values(topicsIndex);
   for (const topic of topics) {
     topic.children = Object.values(topic.children);
     if (topic.children[0].children) {
-      for (const subTopic of topic.children) subTopic.children = Object.values(subTopic.children);
+      for (const subTopic of topic.children)
+        subTopic.children = Object.values(subTopic.children);
     }
   }
   return topics;
 }
 
 export default function getTaxonomy(params = {}) {
-  const taxonomy = getIndicators({...params, minimalMetadata: true});
-  return params.flat ? taxonomy : nestTaxonomy(taxonomy);
+  const taxonomy = getIndicators({ ...params, minimalMetadata: true });
+  const meta = { count: taxonomy.length, total: summaryData.count };
+  const data = params.flat ? taxonomy : nestTaxonomy(taxonomy);
+  return { meta, data };
 }

src/lib/components/modals/AreasModal.svelte

@@ -6,6 +6,7 @@
   import { ONSpalette } from "$lib/config.js";
 
   let pageState = getContext("pageState");
+  let mode = $derived(page.params?.code?.match(/^[EKNSW]{1}\d{8}$/) ? "area" : "indicator");
 
   function addArea(area) {
     if (!pageState.selectedAreas.find(d => d.areacd === area.areacd)) pageState.selectedAreas.push(area);
@@ -16,15 +17,20 @@
   }
 </script>
 
-<Modal title="Select areas" label="Change areas">
-  <Dropdown id="geo-level-select" label="Geography type" options={page.data.geoLevels} bind:value={pageState.selectedGeoLevel}/>
+<Modal title="Select areas" label="Change areas" icon="pin">
+  {#if mode === "indicator"}
+    <Dropdown id="geo-level-select" label="Geography type" options={page.data.geoLevels} bind:value={pageState.selectedGeoLevel}/>
+  {/if}
+  {#if mode === "area"}
+    <Dropdown id="geo-related-select" label="Geography group" options={page.data.geoGroups} bind:value={pageState.selectedGeoGroup}/>
+  {/if}
   <div class="select-container">
-    <Select id="area-select" label="Individual areas" placeholder="Choose one or more" options={page.data.areas} labelKey="areanm" on:change={(e) => addArea(e.detail)} autoClear/>
+    <Select id="area-select" label={mode === "area" ? "Comparison areas" : "Individual areas"} placeholder="Choose one or more" options={page.data.areas} labelKey="areanm" on:change={(e) => addArea(e.detail)} autoClear/>
   </div>
   {#each pageState.selectedAreas as area, i}
     <Button
       icon="cross"
-      color={ONSpalette[i] || "darkgrey"}
+      color={(mode === "area" ? ONSpalette[i + 1] : ONSpalette[i]) || "darkgrey"}
       small
       on:click={() => removeArea(area)}>{area.areanm}</Button>
   {/each}
@@ -38,6 +44,7 @@
     margin: .5em .5em 0 0;
   }
   .select-container {
+    margin-top: 1em;
     width: 22.5rem;
     max-width: 100%;
   }

src/lib/components/modals/Modal.svelte

@@ -1,13 +1,13 @@
 <script>
   import { Button } from "@onsvisual/svelte-components";
 
-  let { title, label, children } = $props();
+  let { title, label, icon = null, children } = $props();
 
   let id = $derived(title.toLowerCase().replaceAll(" ", "-"));
   let dialog = $state();
 </script>
 
-<Button variant="secondary" small on:click={() => dialog.showModal()}>{label}</Button>
+<Button variant="secondary" {icon} small on:click={() => dialog.showModal()}>{label}</Button>
 
 <dialog aria-labelledby="{id}" bind:this={dialog}>
   <h1 id="{id}" tabindex="-1">{title}</h1>

src/lib/components/modals/OptionsModal.svelte

@@ -7,10 +7,10 @@
 
   let pageState = getContext("pageState");
 
-  let formatTick = $derived(pageState.formatPeriod());
+  let formatTick = $derived(pageState?.formatPeriod?.() || ((d) => d));
 </script>
 
-<Modal title="Chart options" label="Chart options">
+<Modal title="Chart options" label="Chart options" icon="cog">
   <RangeSlider
     label="Selected time range"
     options={page.data.periods}

src/lib/data/json-stat-summary.json

@@ -0,0 +1 @@
+{"count":91,"topics":[{"slug":"population","label":"Population"},{"slug":"economy","label":"Economy"},{"slug":"housing","label":"Housing"},{"slug":"education-and-skills","label":"Education and skills"},{"slug":"health-and-wellbeing","label":"Health and wellbeing"},{"slug":"environment","label":"Environment"},{"slug":"connectivity","label":"Connectivity"},{"slug":"crime","label":"Crime"}],"years":[1991,1992,1993,1994,1995,1996,1997,1998,1999,2000,2001,2002,2003,2004,2005,2006,2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,2017,2018,2019,2020,2021,2022,2023,2024,2025],"geoYears":[2017,2018,2019,2020,2022,2023,2024,2025]}

src/lib/utils.js → src/lib/utils.ts

@@ -1,6 +1,7 @@
 import { resolve } from "$app/paths";
 import { format } from "d3-format";
 import { utcFormat } from "d3-time-format";
+import { geoLevels } from "./config/geo-levels.js";
 
 export function parseData(data) {
   const cols = Object.keys(data);
@@ -15,6 +16,22 @@ export function parseData(data) {
   return rows;
 }
 
+export function parseDataKeyed(data, zKey, rowTemplate = {}) {
+  if (data.message) return { keyed: {}, array: [] };
+  const keyed = {};
+  const array = [];
+  const cols = Object.keys(data);
+  for (let i = 0; i < data[cols[0]].length; i++) {
+    const row = {...rowTemplate};
+    for (const col of cols) row[col] = data[col][i];
+    row.time = new Date(data.period[i].split("/")[0]);
+    if (!keyed[data[zKey][i]]) keyed[data[zKey][i]] = [];
+    keyed[data[zKey][i]].push(row);
+    array.push(row);
+  }
+  return { keyed, array };
+}
+
 export async function fetchChartData(
   indicator,
   geography = "ltla",
@@ -114,3 +131,63 @@ export function makePeriodFormatter(periodFormat) {
 export function sleep(ms = 1000) {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
+
+export function slugify(text: string) {
+  const matches = text
+    .toLowerCase()
+    .replace(/'/g, "") // remove apostrophes since we don't want to split apostrophised words
+    .replace(/&+/g, "and") // special case 'and'
+    .match(/[a-zA-Z0-9]+/g); // match alphanumeric sequences
+
+  return matches ? matches.join("-") : "";
+}
+
+export const makeCanonicalSlug = (code: string, name?: string) => {
+  if (!code) {
+    throw "No area code was given";
+  }
+
+  if (name === undefined) {
+    return code;
+  }
+
+  if (!name) {
+    throw "No area name was given";
+  }
+
+  const slugifiedName = slugify(name);
+  return `${code}-${slugifiedName}`;
+};
+
+export function makeDataUrl(
+  indicator,
+  timeRange,
+  timeNearest = null,
+  geoSelected = [],
+  geoLevel = null,
+  geoExtent = null,
+  geoCluster = null
+) {
+  const base = "/api/v1/data.cols.json";
+  const chunks = [];
+
+  if (indicator) chunks.push({ key: "indicator", value: indicator });
+
+  const geoLevelObj = geoLevels[geoLevel];
+  const geo = geoLevelObj ? [geoLevel] : [];
+  geo.push(
+    ...geoSelected.filter((cd) =>
+      !geoLevelObj ? true : !geoLevelObj.codes.includes(cd.slice(0, 3))
+    )
+  );
+  if (geo.length > 0) chunks.push({ key: "geo", value: geo.join(",") });
+  if (geoExtent) chunks.push({ key: "geoExtent", value: geoExtent });
+  if (geoCluster) chunks.push({ key: "geoCluster", value: geoCluster });
+
+  const time = Array.isArray(timeRange) ? timeRange.join(",") : timeRange;
+  if (time) chunks.push({ key: "time", value: time });
+  if (timeNearest) chunks.push({ key: "timeNearest", value: timeNearest });
+
+  const url = `${base}?${chunks.map((ch) => `${ch.key}=${ch.value}`).join("&")}&includeNames=true`;
+  return resolve(url);
+}