Merge branch 'app-rebuild-area-indicators' into app-rebuild

Author: bothness

package-lock.json

@@ -21,9 +21,9 @@
   "license": "MIT",
   "devDependencies": {
     "@mapbox/tilebelt": "^2.0.2",
-    "@onsvisual/robo-utils": "^0.3.7",
+    "@onsvisual/robo-utils": "^0.3.9",
     "@onsvisual/svelte-charts": "^0.4.11",
-    "@onsvisual/svelte-components": "^1.0.30",
+    "@onsvisual/svelte-components": "^1.0.31",
     "@onsvisual/svelte-maps": "^1.2.25",
     "@sveltejs/adapter-netlify": "^5.2.4",
     "@sveltejs/adapter-node": "^5.3.3",
@@ -43,7 +43,7 @@
     "prettier-plugin-svelte": "^3.2.6",
     "svelte": "^5.0.0",
     "svelte-check": "^4.3.4",
-    "svelteplot": "^0.3.7",
+    "svelteplot": "^0.7.0",
     "temporal-polyfill": "^0.3.0",
     "throttleit": "^2.1.0",
     "topojson-client": "^3.1.0",

package.json

@@ -316,9 +316,26 @@ const indicators = [];
 for (const file of file_paths) {
     indicators.push(...processFile(file,excluded_indicators));
 }
-cube.link.item = indicator_slugs.map(slug => indicators.find(ind => ind.extension.slug === slug))
+// Sort indicators to match order in manifest (ie. taxonomy order)
+cube.link.item = indicator_slugs.map(slug => indicators.find(ind => ind.extension.slug === slug));
 
-// console.log(cube.link.item)
 const output = "./src/lib/data/json-stat.json";
 writeFileSync(output, JSON.stringify(cube));
-console.log(`Wrote ${output}.`)
+console.log(`Wrote ${output}.`)
+
+// Generate JSON file with summary stats/data
+const summaryData = {
+    count: cube.link.item.length,
+    topics: Array.from(new Set(cube.link.item.map(ds => ds.extension.topic)))
+        .map(t => ({slug: t.replaceAll(" ", "-"), label: t[0].toUpperCase() + t.slice(1)})),
+    years: Array.from(
+        new Set(cube.link.item.map(ds => 
+            Object.keys(ds.dimension.period.category.index).map(val => +val.slice(0, 4))
+        ).flat())).sort((a, b) => a - b),
+    geoYears: Array.from(new Set(cube.link.item.map(ds => ds.extension.geography.year)))
+        .sort((a, b) => a - b)
+};
+
+const summaryOutput = "./src/lib/data/json-stat-summary.json";
+writeFileSync(summaryOutput, JSON.stringify(summaryData));
+console.log(`Wrote ${summaryOutput}.`);

scripts/generate-data.ts

@@ -66,8 +66,8 @@ export default async function filterCollection(params = {}) {
   }
 
   // Create filters for standard dimensions
-  if (params.geo !== "all")
-    filters.areacd = makeGeoFilter(params.geo, params.geoExtent);
+  if (params.geo !== "all" || params.geoCluster !== "all")
+    filters.areacd = makeGeoFilter(params.geo, params.geoExtent, params.geoCluster);
   if (params.time !== "all") {
     if (
       [params.time]

src/lib/api/data/filterCollection.js

@@ -3,6 +3,9 @@ import { geoLevels } from "$lib/config/geo-levels.js";
 import getChildAreas from "$lib/api/geo/getChildAreas.js";
 import hasObservation from "./hasObservation.js";
 import { isValidMonth, isValidYear } from "$lib/api/utils.js";
+import readData from "$lib/data";
+
+const areasClusters = await readData("areas-clusters");
 
 export function ascending(a, b) {
   return a == null || b == null ? NaN : a < b ? -1 : a > b ? 1 : a >= b ? 0 : NaN;
@@ -13,12 +16,12 @@ export function makeFilter(param) {
   return d => set.has(d[0]);
 }
 
-export function makeGeoFilter(geo, geoExtent) {
+export function makeGeoFilter(geo, geoExtent, geoCluster) {
   const codes = new Set();
   const types = new Set();
   for (const g of [geo].flat()) {
     // if (g.match(/^[EKNSW]\d{2}$/)) types.add(g);
-    if (geoLevels[g]) {
+    if (geoLevels[g] && geoCluster === "all") {
 			if (geoExtent.match(/^[EKNSW]\d{8}$/)) {
 				const children = getChildAreas({code: geoExtent, geoLevel: g, includeNames: false});
 				for (const child of children) codes.add(child);
@@ -28,6 +31,13 @@ export function makeGeoFilter(geo, geoExtent) {
     }
     else if (g.match(/^[EKNSW]\d{8}$/) && !types.has(g.slice(0, 3))) codes.add(g);
   }
+	if (geoCluster) {
+		const [grouping, cluster] = geoCluster.split("_");
+		const cds = areasClusters.clusters?.[grouping]?.[cluster];
+		if (Array.isArray(cds)) {
+			for (const cd of cds) codes.add(cd);
+		}
+	}
   return codes.size > 0 && types.size > 0 ? d => codes.has(d[0]) || types.has(d[0].slice(0, 3)) :
     types.size > 0 ? d => types.has(d[0].slice(0, 3)) :
     codes.size > 0 ? d => codes.has(d[0]) :

src/lib/api/data/helpers/dataFilters.js

@@ -15,11 +15,27 @@ function formatMetadata(ds, minimalMetadata = false, fullDims = false) {
       description: ds.extension.subtitle,
     };
 
-  const metadata = { label: ds.label, ...ds.extension, updated: ds.updated, caveats: ds.note };
-  metadata.dimensions = Object.fromEntries(ds.id.map((key, i) => [key, {...formatDimension(ds, key, fullDims), order: i}]));
+  const metadata = {
+    label: ds.label,
+    ...ds.extension,
+    updated: ds.updated,
+    caveats: ds.note,
+  };
+  metadata.dimensions = Object.fromEntries(
+    ds.id.map((key, i) => [
+      key,
+      { ...formatDimension(ds, key, fullDims), order: i },
+    ])
+  );
   return metadata;
 }
 
+function arrayToLookup(metadata) {
+  const lookup = {};
+  for (const ds of metadata) lookup[ds.slug] = ds;
+  return lookup;
+}
+
 export default function getIndicators(params = {}) {
   const filter = makeDatasetFilter(
     params.indicator,
@@ -32,9 +48,11 @@ export default function getIndicators(params = {}) {
 
   const metadata = rawMetadata.link.item
     .filter(filter)
-    .map((ds) =>
-      formatMetadata(ds, params.minimalMetadata, params.fullDims)
-    );
+    .map((ds) => formatMetadata(ds, params.minimalMetadata, params.fullDims));
 
-  return params.singleIndicator ? metadata[0] : metadata;
+  return params.singleIndicator
+    ? metadata[0]
+    : params.asLookup
+      ? arrayToLookup(metadata)
+      : metadata;
 }

src/lib/api/metadata/getIndicators.js

@@ -1,10 +1,11 @@
 import getIndicators from "./getIndicators.js";
+import summaryData from "$lib/data/json-stat-summary.json";
 import { capitalise } from "$lib/utils.ts";
 
 function makeItem(slug, label = null, description = null) {
-  const item = {label: label || capitalise(slug), slug};
-  if (description) return {...item, description};
-  return {...item, children: {}};
+  const item = { label: label || capitalise(slug), slug };
+  if (description) return { ...item, description };
+  return { ...item, children: {} };
 }
 
 function nestTaxonomy(taxonomy) {
@@ -18,21 +19,25 @@ function nestTaxonomy(taxonomy) {
     } else {
       if (!topicsIndex[ind.topic].children[ind.subTopic])
         topicsIndex[ind.topic].children[ind.subTopic] = makeItem(ind.subTopic);
-      topicsIndex[ind.topic].children[ind.subTopic].children[ind.slug] = indicator;
+      topicsIndex[ind.topic].children[ind.subTopic].children[ind.slug] =
+        indicator;
     }
   }
 
   const topics = Object.values(topicsIndex);
   for (const topic of topics) {
     topic.children = Object.values(topic.children);
     if (topic.children[0].children) {
-      for (const subTopic of topic.children) subTopic.children = Object.values(subTopic.children);
+      for (const subTopic of topic.children)
+        subTopic.children = Object.values(subTopic.children);
     }
   }
   return topics;
 }
 
 export default function getTaxonomy(params = {}) {
-  const taxonomy = getIndicators({...params, minimalMetadata: true});
-  return params.flat ? taxonomy : nestTaxonomy(taxonomy);
+  const taxonomy = getIndicators({ ...params, minimalMetadata: true });
+  const meta = { count: taxonomy.length, total: summaryData.count };
+  const data = params.flat ? taxonomy : nestTaxonomy(taxonomy);
+  return { meta, data };
 }