diff --git a/functions/src/functions/scraping-functions.ts b/functions/src/functions/scraping-functions.ts index 01d8cc8..153a790 100644 --- a/functions/src/functions/scraping-functions.ts +++ b/functions/src/functions/scraping-functions.ts @@ -27,6 +27,8 @@ export async function scrapingsFunction( const responseCuencaCatalana = await embalsesRepository.actualizarCuencaCatalana(); + const responseCuencaDuero = await embalsesRepository.actualizarCuencaDuero(); + if (responseCuencaMediterranea) { context.log( "scrapings-function: Se han actualizado los embalses de la cuenca Mediterránea", @@ -52,6 +54,13 @@ export async function scrapingsFunction( "No se han podido actualizar los embalses de la cuenca Catalana" ); } + + if (responseCuencaDuero) { + context.log(`Se han actualizado los embalses de la cuenca Duero`); + } else { + context.log( + "No se han podido actualizar los embalses de la cuenca Duero") + } } catch (error) { context.error("scrapings-function: ERROR", error); throw error; diff --git a/integrations/arcgis/src/embalse-provincias.ts b/integrations/arcgis/src/embalse-provincias.ts index d431d4a..6e0b6a1 100644 --- a/integrations/arcgis/src/embalse-provincias.ts +++ b/integrations/arcgis/src/embalse-provincias.ts @@ -105,7 +105,7 @@ export const embalseProvincias = new Map([ ["castrelo", "Ourense"], ["castro", "Lugo"], ["castro-de-las-cogotas", "Ávila"], - ["castro-el", "Ourense"], + ["castro-el", "Zamora"], ["catllar", "Tarragona"], ["cavallers", "Lleida"], ["cazalegas", "Toledo"], @@ -315,7 +315,7 @@ export const embalseProvincias = new Map([ ["san-anton", "Navarra"], ["san-bartolome", "Huelva"], ["san-clemente", "Huelva"], - ["san-jose", "Almería"], + ["san-jose", "Valladolid"], ["san-juan", "Madrid"], ["san-lorenzo", "Huelva"], ["san-martino", "Pontevedra"], @@ -336,7 +336,7 @@ export const embalseProvincias = new Map([ ["sau", "Barcelona"], ["saucelle", "Salamanca"], ["sequeiros", "Ourense"], - ["serones", "Almería"], + ["serones", "Ávila"], ["sichar", "Huesca"], ["sierra-boyera", "Córdoba"], ["sierra-brava", "Cáceres"], diff --git a/integrations/scraping-cuenca-duero/package.json b/integrations/scraping-cuenca-duero/package.json index e85b0c2..3319786 100644 --- a/integrations/scraping-cuenca-duero/package.json +++ b/integrations/scraping-cuenca-duero/package.json @@ -3,14 +3,22 @@ "version": "1.0.0", "private": true, "type": "module", - "exports": "./src/index.ts", + "exports": { + ".": "./dist/index.js" + }, + "main": "./dist/index.js", + "types": "./dist/index.d.ts", "scripts": { "start": "tsx --watch ./src/console-runner.ts", - "test": "echo \"Error: no test specified\" && exit 1" + "build": "run-p clean type-check build:scraping-cuenca-duero", + "build:scraping-cuenca-duero": "tsc", + "clean": "rimraf dist", + "type-check": "tsc --noEmit --preserveWatchOutput" }, "dependencies": { "axios": "^1.11.0", - "cheerio": "^1.1.2" + "cheerio": "^1.1.2", + "db-model": "^1.0.0" }, "devDependencies": { "ts-node": "^10.9.2", diff --git a/integrations/scraping-cuenca-duero/src/console-runner.ts b/integrations/scraping-cuenca-duero/src/console-runner.ts index 1237c4d..790565e 100644 --- a/integrations/scraping-cuenca-duero/src/console-runner.ts +++ b/integrations/scraping-cuenca-duero/src/console-runner.ts @@ -1,4 +1,4 @@ -import { getEstadoCuencaDuero } from './integration'; +import { getEstadoCuencaDuero } from './integration.js'; // We log a message in English to know what's happening console.log('Fetching status for Duero basin...'); diff --git a/integrations/scraping-cuenca-duero/src/index.ts b/integrations/scraping-cuenca-duero/src/index.ts new file mode 100644 index 0000000..ba7a4ee --- /dev/null +++ b/integrations/scraping-cuenca-duero/src/index.ts @@ -0,0 +1 @@ +export * from "./integration.js"; diff --git a/integrations/scraping-cuenca-duero/src/integration.test.ts b/integrations/scraping-cuenca-duero/src/integration.test.ts index 191f670..1fa4079 100644 --- a/integrations/scraping-cuenca-duero/src/integration.test.ts +++ b/integrations/scraping-cuenca-duero/src/integration.test.ts @@ -2,7 +2,7 @@ import { describe, it, expect, vi, type Mock } from "vitest"; import axios from "axios"; -import { getEstadoCuencaDuero } from "./integration"; +import { getEstadoCuencaDuero } from "./integration.js"; vi.mock("axios"); diff --git a/integrations/scraping-cuenca-duero/src/integration.ts b/integrations/scraping-cuenca-duero/src/integration.ts index 861b673..b01945d 100644 --- a/integrations/scraping-cuenca-duero/src/integration.ts +++ b/integrations/scraping-cuenca-duero/src/integration.ts @@ -1,7 +1,6 @@ // integration.ts (Versión Final Correcta) import axios from 'axios'; -import { EmbalseDuero } from './api/cuenca.model'; -import { parseReservoirsFromHtml, getCurrentDate, mapToEmbalseUpdateSAIH } from './scraper'; +import { parseReservoirsFromHtml, getCurrentDate, mapToEmbalseUpdateSAIH } from './scraper/index.js'; import { EmbalseUpdateSAIHEntity } from "db-model"; // Define the URL we are going to scrape diff --git a/integrations/scraping-cuenca-duero/src/scraper/business.ts b/integrations/scraping-cuenca-duero/src/scraper/business.ts index 83f583f..60b2e0f 100644 --- a/integrations/scraping-cuenca-duero/src/scraper/business.ts +++ b/integrations/scraping-cuenca-duero/src/scraper/business.ts @@ -1,4 +1,4 @@ -import { EmbalseDuero } from "../api/cuenca.model"; +import { EmbalseDuero } from "../api/cuenca.model.js"; import { load } from "cheerio"; // Función auxiliar para parsear string a number o null diff --git a/integrations/scraping-cuenca-duero/src/scraper/index.ts b/integrations/scraping-cuenca-duero/src/scraper/index.ts index bbb6227..4f25535 100644 --- a/integrations/scraping-cuenca-duero/src/scraper/index.ts +++ b/integrations/scraping-cuenca-duero/src/scraper/index.ts @@ -1,2 +1,2 @@ -export * from './business'; -export * from './mapper'; +export * from './business.js'; +export * from './mapper.js'; diff --git a/integrations/scraping-cuenca-duero/src/scraper/mapper.ts b/integrations/scraping-cuenca-duero/src/scraper/mapper.ts index 79711ce..07f7a8d 100644 --- a/integrations/scraping-cuenca-duero/src/scraper/mapper.ts +++ b/integrations/scraping-cuenca-duero/src/scraper/mapper.ts @@ -1,4 +1,4 @@ -import { EmbalseDuero } from '../api/cuenca.model'; +import { EmbalseDuero } from '../api/cuenca.model.js'; import { EmbalseUpdateSAIHEntity } from "db-model"; /** diff --git a/integrations/scraping-cuenca-duero/src/test-scraper.ts b/integrations/scraping-cuenca-duero/src/test-scraper.ts index 7e681ab..2a13916 100644 --- a/integrations/scraping-cuenca-duero/src/test-scraper.ts +++ b/integrations/scraping-cuenca-duero/src/test-scraper.ts @@ -1,4 +1,4 @@ -import { getEstadoCuencaDuero } from './integration'; +import { getEstadoCuencaDuero } from './integration.js'; (async () => { const data = await getEstadoCuencaDuero(); diff --git a/integrations/scraping-cuenca-duero/tsconfig.json b/integrations/scraping-cuenca-duero/tsconfig.json index 48b9b22..ce7b1d4 100644 --- a/integrations/scraping-cuenca-duero/tsconfig.json +++ b/integrations/scraping-cuenca-duero/tsconfig.json @@ -1,20 +1,16 @@ { - "compilerOptions": { - "target": "ES2020", - "module": "ESNext", - "moduleResolution": "bundler", - "skipLibCheck": true, - "isolatedModules": true, - "esModuleInterop": true, - "allowSyntheticDefaultImports": true, - "resolveJsonModule": true, - "outDir": "dist", - "baseUrl": "./", - "paths": { - "@/*": ["src/*"], - "@/api/*": ["src/api/*"], - "@/scraper/*": ["src/scraper/*"] - } - }, - "include": ["src"] + "compilerOptions": { + "target": "ESNext", + "module": "nodenext", + "moduleResolution": "nodenext", + "outDir": "dist", + "skipLibCheck": true, + "isolatedModules": true, + "esModuleInterop": true, + "verbatimModuleSyntax": false, + "declaration": true, + "baseUrl": "./" + }, + "include": ["src/**/*"], + "exclude": ["dist", "node_modules"] } diff --git a/package-lock.json b/package-lock.json index 077367b..f4df29c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -94,7 +94,8 @@ "version": "1.0.0", "dependencies": { "axios": "^1.11.0", - "cheerio": "^1.1.2" + "cheerio": "^1.1.2", + "db-model": "^1.0.0" }, "devDependencies": { "ts-node": "^10.9.2", @@ -5853,6 +5854,7 @@ "@types/prompts": "^2.4.9", "arcgis": "*", "prompts": "^2.4.2", + "scraping-cuenca-duero": "*", "scraping-cuenca-cantabrico": "*", "scraping-cuenca-mediterranea": "*" } diff --git a/packages/db/package.json b/packages/db/package.json index db901d6..e50e6af 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -32,6 +32,7 @@ "prompts": "^2.4.2", "scraping-cuenca-mediterranea": "*", "scraping-cuenca-cantabrico": "*", - "scraping-cuenca-catalana": "*" + "scraping-cuenca-catalana": "*", + "scraping-cuenca-duero": "*" } } diff --git a/packages/db/src/dals/embalses/embalses.mappers.ts b/packages/db/src/dals/embalses/embalses.mappers.ts index 6beb3ba..3077cc5 100644 --- a/packages/db/src/dals/embalses/embalses.mappers.ts +++ b/packages/db/src/dals/embalses/embalses.mappers.ts @@ -73,3 +73,29 @@ export const mapperFromCuencasCatalanaToArcgis = new Map< [250753004, { nombre: "Sant Pons", idArcgis: 280 }], ]); + +// OJO Aguilar de Campoo, Cernadilla, Puente Porto, Ricobayo, Rábanos, Saucelle, Valparaíso, Las Vencías, Villagonzalo, Villalcampo, Nª Sª de Agavanzal, Aldeadávila, Almendra, Burgomillodo, Casares de Arbas, Aguilar, Serones, San José + +export const mapperFromCuencasDueroToArcgis = new Map< + number, + InfoDestinoArcgis +>([ + [1, { nombre: "Villameca", idArcgis: 344 }], + [2, { nombre: "Barrios de Luna", idArcgis: 42 }], + [3, { nombre: "Porma", idArcgis: 232 }], + [4, { nombre: "Riaño", idArcgis: 254 }], + [7, { nombre: "Camporredondo", idArcgis: 74 }], + [8, { nombre: "Compuerto", idArcgis: 105 }], + [11, { nombre: "Cervera-Ruesga", idArcgis: 98 }], // OJO puede ser Cervera + [12, { nombre: "La Requejada", idArcgis: 250 }], + [16, { nombre: "Arlanzón", idArcgis: 30 }], + [17, { nombre: "Úzquiza", idArcgis: 324 }], + [20, { nombre: "Cuerda del Pozo", idArcgis: 116 }], + [22, { nombre: "Linares del Arroyo", idArcgis: 185 }], + [23, { nombre: "El Pontón Alto", idArcgis: 375 }], + [26, { nombre: "Castro de las Cogotas", idArcgis: 86 }], + [28, { nombre: "Santa Teresa", idArcgis: 286 }], + [30, { nombre: "Irueña", idArcgis: 402 }], + [31, { nombre: "Águeda", idArcgis: 5 }], + [87, { nombre: "Castro", idArcgis: 26 }], +]) diff --git a/packages/db/src/dals/embalses/embalses.repository.ts b/packages/db/src/dals/embalses/embalses.repository.ts index 8bf3e51..02090da 100644 --- a/packages/db/src/dals/embalses/embalses.repository.ts +++ b/packages/db/src/dals/embalses/embalses.repository.ts @@ -1,9 +1,10 @@ import { scrapeSeedEmbalses } from "arcgis"; import { getEmbalsesContext } from "./embalses.context.js"; -import { mapperFromCuencasMediterraneaToArcgis, mapperFromCuencasCantabricoToArcgis, mapperFromCuencasCatalanaToArcgis } from "./embalses.mappers.js"; +import { mapperFromCuencasMediterraneaToArcgis, mapperFromCuencasCantabricoToArcgis, mapperFromCuencasCatalanaToArcgis, mapperFromCuencasDueroToArcgis } from "./embalses.mappers.js"; import { scrapeCuencaMediterranea } from "scraping-cuenca-mediterranea"; import { scrapeCuencaCantabrica } from 'scraping-cuenca-cantabrico'; import { integracionCuencaCatalana } from 'scraping-cuenca-catalana'; +import { getEstadoCuencaDuero } from 'scraping-cuenca-duero'; import { parseDate } from "./embalses.helpers.js"; export const embalsesRepository = { @@ -179,4 +180,57 @@ export const embalsesRepository = { return actualizados > 0; }, + actualizarCuencaDuero: async (): Promise => { + const embalsesDuero = await getEstadoCuencaDuero(); + + console.log( + `Se han scrapeado ${embalsesDuero.length} embalses de la Cuenca Mediterránea` + ); + + let actualizados = 0; + let noEncontrados = 0; + let sinMapper = 0; + + for (const embalse of embalsesDuero) { + const infoDestino = mapperFromCuencasDueroToArcgis.get(embalse.id); + + if (!infoDestino) { + sinMapper++; + console.warn(`Sin mapper para ID ${embalse.id} - ${embalse.nombre}`); + continue; + } + + console.log( + `🔍 Mapeando: ID scraping ${embalse.id} -> _id BD ${infoDestino.idArcgis} (${infoDestino.nombre})` + ); + + const { matchedCount } = await getEmbalsesContext().updateOne( + { _id: infoDestino.idArcgis.toString() }, + { + $set: { + aguaActualSAIH: embalse.aguaActualSAIH, + fechaMedidaAguaActualSAIH: parseDate(embalse.fechaMedidaSAIH), + }, + } + ); + + if (matchedCount > 0) { + actualizados++; + console.log( + `Actualizado: ${infoDestino.nombre} (_id: ${infoDestino.idArcgis}) -> ${embalse.aguaActualSAIH} hm³` + ); + } else { + noEncontrados++; + console.warn( + `No encontrado en BD: _id ${infoDestino.idArcgis} - ${infoDestino.nombre}` + ); + } + } + + console.log( + `Resumen Cuenca Duero: ${actualizados} actualizados, ${noEncontrados} no encontrados, ${sinMapper} sin mapper` + ); + + return actualizados > 0; + } };