Merge pull request #27 from ONSdigital/app-rebuild

App rebuild DRAFT

Author: bothness

package-lock.json

@@ -21,9 +21,10 @@
   "license": "MIT",
   "devDependencies": {
     "@mapbox/tilebelt": "^2.0.2",
-    "@onsvisual/robo-utils": "^0.3.7",
+    "@onsvisual/robo-utils": "^0.3.9",
     "@onsvisual/svelte-charts": "^0.4.11",
-    "@onsvisual/svelte-components": "^1.0.30",
+    "@onsvisual/svelte-components": "^1.0.31",
+    "@onsvisual/svelte-maps": "^1.2.25",
     "@sveltejs/adapter-netlify": "^5.2.4",
     "@sveltejs/adapter-node": "^5.3.3",
     "@sveltejs/kit": "^2.46.5",
@@ -42,7 +43,7 @@
     "prettier-plugin-svelte": "^3.2.6",
     "svelte": "^5.0.0",
     "svelte-check": "^4.3.4",
-    "svelteplot": "^0.3.7",
+    "svelteplot": "^0.7.0",
     "temporal-polyfill": "^0.3.0",
     "throttleit": "^2.1.0",
     "topojson-client": "^3.1.0",

package.json

@@ -111,7 +111,7 @@ function indicatorToCube(indicator, t, meta_data, tableSchema, dataset_name) {
             source: meta_data.metadata.source,
             slug: manifest_metadata_indicator[0].slug,
             ...restOfMetadata,
-            experimentalStatistic: meta_data.experimentalStatistic,
+            experimentalStatistic: meta_data.metadata.experimentalStatistic,
             geography: meta_data.metadata.geography
         }
     }
@@ -316,9 +316,26 @@ const indicators = [];
 for (const file of file_paths) {
     indicators.push(...processFile(file,excluded_indicators));
 }
-cube.link.item = indicator_slugs.map(slug => indicators.find(ind => ind.extension.slug === slug))
+// Sort indicators to match order in manifest (ie. taxonomy order)
+cube.link.item = indicator_slugs.map(slug => indicators.find(ind => ind.extension.slug === slug));
 
-// console.log(cube.link.item)
 const output = "./src/lib/data/json-stat.json";
 writeFileSync(output, JSON.stringify(cube));
-console.log(`Wrote ${output}.`)
+console.log(`Wrote ${output}.`)
+
+// Generate JSON file with summary stats/data
+const summaryData = {
+    count: cube.link.item.length,
+    topics: Array.from(new Set(cube.link.item.map(ds => ds.extension.topic)))
+        .map(t => ({slug: t.replaceAll(" ", "-"), label: t[0].toUpperCase() + t.slice(1)})),
+    years: Array.from(
+        new Set(cube.link.item.map(ds => 
+            Object.keys(ds.dimension.period.category.index).map(val => +val.slice(0, 4))
+        ).flat())).sort((a, b) => a - b),
+    geoYears: Array.from(new Set(cube.link.item.map(ds => ds.extension.geography.year)))
+        .sort((a, b) => a - b)
+};
+
+const summaryOutput = "./src/lib/data/json-stat-summary.json";
+writeFileSync(summaryOutput, JSON.stringify(summaryData));
+console.log(`Wrote ${summaryOutput}.`);

scripts/generate-data.ts

@@ -66,8 +66,8 @@ export default async function filterCollection(params = {}) {
   }
 
   // Create filters for standard dimensions
-  if (params.geo !== "all")
-    filters.areacd = makeGeoFilter(params.geo, params.geoExtent);
+  if (params.geo !== "all" || params.geoCluster !== "all")
+    filters.areacd = makeGeoFilter(params.geo, params.geoExtent, params.geoCluster);
   if (params.time !== "all") {
     if (
       [params.time]

src/lib/api/data/filterCollection.js

@@ -3,6 +3,9 @@ import { geoLevels } from "$lib/config/geo-levels.js";
 import getChildAreas from "$lib/api/geo/getChildAreas.js";
 import hasObservation from "./hasObservation.js";
 import { isValidMonth, isValidYear } from "$lib/api/utils.js";
+import readData from "$lib/data";
+
+const areasClusters = await readData("areas-clusters");
 
 export function ascending(a, b) {
   return a == null || b == null ? NaN : a < b ? -1 : a > b ? 1 : a >= b ? 0 : NaN;
@@ -13,12 +16,12 @@ export function makeFilter(param) {
   return d => set.has(d[0]);
 }
 
-export function makeGeoFilter(geo, geoExtent) {
+export function makeGeoFilter(geo, geoExtent, geoCluster) {
   const codes = new Set();
   const types = new Set();
   for (const g of [geo].flat()) {
     // if (g.match(/^[EKNSW]\d{2}$/)) types.add(g);
-    if (geoLevels[g]) {
+    if (geoLevels[g] && geoCluster === "all") {
 			if (geoExtent.match(/^[EKNSW]\d{8}$/)) {
 				const children = getChildAreas({code: geoExtent, geoLevel: g, includeNames: false});
 				for (const child of children) codes.add(child);
@@ -28,6 +31,13 @@ export function makeGeoFilter(geo, geoExtent) {
     }
     else if (g.match(/^[EKNSW]\d{8}$/) && !types.has(g.slice(0, 3))) codes.add(g);
   }
+	if (geoCluster) {
+		const [grouping, cluster] = geoCluster.split("_");
+		const cds = areasClusters.clusters?.[grouping]?.[cluster];
+		if (Array.isArray(cds)) {
+			for (const cd of cds) codes.add(cd);
+		}
+	}
   return codes.size > 0 && types.size > 0 ? d => codes.has(d[0]) || types.has(d[0].slice(0, 3)) :
     types.size > 0 ? d => types.has(d[0].slice(0, 3)) :
     codes.size > 0 ? d => codes.has(d[0]) :
@@ -56,11 +66,13 @@ export function getTime(values, params = {}) {
 
 	const periods = values.map(v => ({value: v, period: periodToDateRange(v[0])}));
 	const nearest = params.nearest || "none";
-	const isRange = periods[0].period.length > 1;
-	const date = isRange || params.time.length === 10 ? toPlainDate(params.time, true) : [toPlainDate(params.time, false), toPlainDate(params.time, true)];
+	const periodIsRange = periods[0].period.length > 1; // Time periods have a duration component
+	const dateIsExact = params.time.length === 10; // Requested date is to nearest day
+	const date = periodIsRange || dateIsExact ? toPlainDate(params.time, true) : [toPlainDate(params.time, false), toPlainDate(params.time, true)];
 
 	let match;
-	if (isRange) match = periods.findLast(p => Temporal.PlainDate.compare(date, p.period[0]) !== -1 && Temporal.PlainDate.compare(date, p.period[1]) !== 1);
+	if (periodIsRange) match = periods.findLast(p => Temporal.PlainDate.compare(date, p.period[0]) !== -1 && Temporal.PlainDate.compare(date, p.period[1]) !== 1);
+	else if (dateIsExact) match = periods.findLast(p => Temporal.PlainDate.compare(date, p.period[0]) === 0);
 	else match = periods.findLast(p => Temporal.PlainDate.compare(p.period[0], date[0]) !== -1 && Temporal.PlainDate.compare(p.period[0], date[1]) !== 1);
 	if (match) return [match.value];
 

src/lib/api/data/helpers/dataFilters.js

@@ -15,11 +15,27 @@ function formatMetadata(ds, minimalMetadata = false, fullDims = false) {
       description: ds.extension.subtitle,
     };
 
-  const metadata = { label: ds.label, ...ds.extension, updated: ds.updated, caveats: ds.note };
-  metadata.dimensions = Object.fromEntries(ds.id.map((key, i) => [key, {...formatDimension(ds, key, fullDims), order: i}]));
+  const metadata = {
+    label: ds.label,
+    ...ds.extension,
+    updated: ds.updated,
+    caveats: ds.note,
+  };
+  metadata.dimensions = Object.fromEntries(
+    ds.id.map((key, i) => [
+      key,
+      { ...formatDimension(ds, key, fullDims), order: i },
+    ])
+  );
   return metadata;
 }
 
+function arrayToLookup(metadata) {
+  const lookup = {};
+  for (const ds of metadata) lookup[ds.slug] = ds;
+  return lookup;
+}
+
 export default function getIndicators(params = {}) {
   const filter = makeDatasetFilter(
     params.indicator,
@@ -32,9 +48,11 @@ export default function getIndicators(params = {}) {
 
   const metadata = rawMetadata.link.item
     .filter(filter)
-    .map((ds) =>
-      formatMetadata(ds, params.minimalMetadata, params.fullDims)
-    );
+    .map((ds) => formatMetadata(ds, params.minimalMetadata, params.fullDims));
 
-  return params.singleIndicator ? metadata[0] : metadata;
+  return params.singleIndicator
+    ? metadata[0]
+    : params.asLookup
+      ? arrayToLookup(metadata)
+      : metadata;
 }

src/lib/api/metadata/getIndicators.js

@@ -1,10 +1,11 @@
 import getIndicators from "./getIndicators.js";
-import { capitalise } from "$lib/utils.js";
+import summaryData from "$lib/data/json-stat-summary.json";
+import { capitalise } from "$lib/utils.ts";
 
 function makeItem(slug, label = null, description = null) {
-  const item = {label: label || capitalise(slug), slug};
-  if (description) return {...item, description};
-  return {...item, children: {}};
+  const item = { label: label || capitalise(slug), slug };
+  if (description) return { ...item, description };
+  return { ...item, children: {} };
 }
 
 function nestTaxonomy(taxonomy) {
@@ -18,21 +19,25 @@ function nestTaxonomy(taxonomy) {
     } else {
       if (!topicsIndex[ind.topic].children[ind.subTopic])
         topicsIndex[ind.topic].children[ind.subTopic] = makeItem(ind.subTopic);
-      topicsIndex[ind.topic].children[ind.subTopic].children[ind.slug] = indicator;
+      topicsIndex[ind.topic].children[ind.subTopic].children[ind.slug] =
+        indicator;
     }
   }
 
   const topics = Object.values(topicsIndex);
   for (const topic of topics) {
     topic.children = Object.values(topic.children);
     if (topic.children[0].children) {
-      for (const subTopic of topic.children) subTopic.children = Object.values(subTopic.children);
+      for (const subTopic of topic.children)
+        subTopic.children = Object.values(subTopic.children);
     }
   }
   return topics;
 }
 
 export default function getTaxonomy(params = {}) {
-  const taxonomy = getIndicators({...params, minimalMetadata: true});
-  return params.flat ? taxonomy : nestTaxonomy(taxonomy);
+  const taxonomy = getIndicators({ ...params, minimalMetadata: true });
+  const meta = { count: taxonomy.length, total: summaryData.count };
+  const data = params.flat ? taxonomy : nestTaxonomy(taxonomy);
+  return { meta, data };
 }

src/lib/api/metadata/getTaxonomy.js

@@ -6,6 +6,7 @@
   import { ONSpalette } from "$lib/config.js";
 
   let pageState = getContext("pageState");
+  let mode = $derived(page.params?.code?.match(/^[EKNSW]{1}\d{8}$/) ? "area" : "indicator");
 
   function addArea(area) {
     if (!pageState.selectedAreas.find(d => d.areacd === area.areacd)) pageState.selectedAreas.push(area);
@@ -16,15 +17,20 @@
   }
 </script>
 
-<Modal title="Select areas" label="Change areas">
-  <Dropdown id="geo-level-select" label="Geography type" options={page.data.geoLevels} bind:value={pageState.selectedGeoLevel}/>
+<Modal title="Select areas" label="Change areas" icon="pin">
+  {#if mode === "indicator"}
+    <Dropdown id="geo-level-select" label="Geography type" options={page.data.geoLevels} bind:value={pageState.selectedGeoLevel}/>
+  {/if}
+  {#if mode === "area"}
+    <Dropdown id="geo-related-select" label="Geography group" options={page.data.geoGroups} bind:value={pageState.selectedGeoGroup}/>
+  {/if}
   <div class="select-container">
-    <Select id="area-select" label="Individual areas" placeholder="Choose one or more" options={page.data.areas} labelKey="areanm" on:change={(e) => addArea(e.detail)} autoClear/>
+    <Select id="area-select" label={mode === "area" ? "Comparison areas" : "Individual areas"} placeholder="Choose one or more" options={page.data.areas} labelKey="areanm" on:change={(e) => addArea(e.detail)} autoClear/>
   </div>
   {#each pageState.selectedAreas as area, i}
     <Button
       icon="cross"
-      color={ONSpalette[i] || "darkgrey"}
+      color={(mode === "area" ? ONSpalette[i + 1] : ONSpalette[i]) || "darkgrey"}
       small
       on:click={() => removeArea(area)}>{area.areanm}</Button>
   {/each}
@@ -38,6 +44,7 @@
     margin: .5em .5em 0 0;
   }
   .select-container {
+    margin-top: 1em;
     width: 22.5rem;
     max-width: 100%;
   }

src/lib/components/modals/AreasModal.svelte

@@ -1,13 +1,13 @@
 <script>
   import { Button } from "@onsvisual/svelte-components";
 
-  let { title, label, children } = $props();
+  let { title, label, icon = null, children } = $props();
 
   let id = $derived(title.toLowerCase().replaceAll(" ", "-"));
   let dialog = $state();
 </script>
 
-<Button variant="secondary" small on:click={() => dialog.showModal()}>{label}</Button>
+<Button variant="secondary" {icon} small on:click={() => dialog.showModal()}>{label}</Button>
 
 <dialog aria-labelledby="{id}" bind:this={dialog}>
   <h1 id="{id}" tabindex="-1">{title}</h1>

src/lib/components/modals/Modal.svelte

@@ -7,10 +7,10 @@
 
   let pageState = getContext("pageState");
 
-  let formatTick = $derived(pageState.formatPeriod());
+  let formatTick = $derived(pageState?.formatPeriod?.() || ((d) => d));
 </script>
 
-<Modal title="Chart options" label="Chart options">
+<Modal title="Chart options" label="Chart options" icon="cog">
   <RangeSlider
     label="Selected time range"
     options={page.data.periods}