Skip to content

Commit 619c549

Browse files
annebanneb
committed
add fullrecord output and translation, new endpoints, fix update bugs, fix previous/next bugs
1 parent 17b25bc commit 619c549

File tree

5 files changed

+197
-28
lines changed

5 files changed

+197
-28
lines changed

‎cswrequests.js

+4
Original file line numberDiff line numberDiff line change
@@ -364,6 +364,10 @@ export async function fetchGetRecords(url, cswVersion, elementSetName, startReco
364364
console.error('fetchGetRecords: Unknown elementSetName:', elementSetName);
365365
break;
366366
}
367+
const exception = evaluateXPath(cswRecords, '//ows:ExceptionReport/ows:Exception/ows:ExceptionText/text()');
368+
if (exception.length) {
369+
throw new Error(exception[0].nodeValue);
370+
}
367371
const result = {
368372
cswVersion,
369373
searchStatus: nodeValue(evaluateXPath(cswRecords, '//csw:SearchStatus/@timestamp'), 0),

‎endpoints.js

+1
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ export const cswEndPoints = [
55
{name: "geo-solutions", url: "https://gs-stable.geo-solutions.it/geoserver/csw", language: "Italian"},
66
{name: "geoplatforme.fr", url: "https://data.geopf.fr/csw/", language: "French"},
77
{name: "Bundesamt für Kartographie und Geodäsie (BKG)", url: "https://mis.bkg.bund.de/csw", language: "German"},
8+
{name: "GDI-DE", url: "https://gdk.gdi-de.org/geonetwork/srv/ger/csw", language: "German"},
89
{name: "ign-fr", url: "https://wxs.ign.fr/catalogue/csw-inspire/", language: "French"},
910
{name: "ign-es", url: "http://www.ign.es/csw-inspire/srv/spa/csw", language: "Spanish"},
1011
{name: "Finnish Meteorological Institute", url: "http://catalog.fmi.fi/geonetwork/srv/eng/csw", language: "Finnish"},

‎public/csw-app.js

+117-27
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,9 @@ class CSWApp extends LitElement {
1111
startRecord: { type: Number },
1212
endRecord: { type: Number },
1313
fullRecord: { type: Object },
14+
translatedAbstract: { type: String },
15+
translatedDescription: { type: String },
16+
translatedSubject: { type: String }
1417
};
1518
}
1619
static get styles() {
@@ -39,6 +42,9 @@ class CSWApp extends LitElement {
3942
this.searchType = '';
4043
this.searchString = '';
4144
this.fullRecord = null;
45+
this.translatedAbstract = '';
46+
this.translatedDescription = '';
47+
this.translatedSubject = '';
4248
}
4349
render() {
4450
return html`
@@ -53,30 +59,69 @@ class CSWApp extends LitElement {
5359
<hr>
5460
<h2>Search</h2>
5561
<label for="type">Type: </label><input @keyup="${(event)=>this.updateType(event)}" type="text" id="type" placeholder="Type..."><br>
56-
<label for="search">Search: <label><input @keyup="${(event)=>this.updateSearch(event)}" type="text" id="search" placeholder="Search..."><br>
62+
<label for="search">Search: </label><input @keyup="${(event)=>this.updateSearch(event)}" type="text" id="search" placeholder="Search..."><br>
5763
<label for="bbox">Bounding box: </label><input type="text" id="bbox" placeholder="Bounding box..."><br>
58-
<button @click="${()=>this.getRecordsHandler(1)}" id="get-records-button">Get Records</button>
64+
<button @click="${()=>this.getRecordsHandler(1)}" id="get-records-button">Search</button>
5965
<div id="records">
6066
<h2>Records</h2>
67+
Search string: ${this.translatedSearch}<br>
68+
Type: ${this.searchType}<br>
6169
Total records: ${this.totalRecords}<br>
70+
${this.message !== '' ? html`<p>${this.message}</p>` : ''}
6271
${this.totalRecords > 0 && this.briefRecords.length > 0 ? html`
6372
Showing records ${this.startRecord} to ${this.endRecord}
6473
<table>
6574
<thead><tr><th>Type</th><th>title</th><th>title (AI-translated)</th></tr></thead>
6675
<tbody>
67-
${this.briefRecords.map(record => html`<tr><td>${record.type}</td><td @click="${(_e)=>this.getRecordById(record.identifier)}" class="recordtitle">${record.title}</td><td>${record.englishTitle}</td></tr>`)}
76+
${this.briefRecords.map(record => html`<tr><td>${record.type}</td><td @click="${(event)=>this.briefRecordTitleClicked(event, record.identifier)}" class="recordtitle">${record.title}</td><td>${record.englishTitle}</td></tr>`)}
6877
</tbody>
6978
</table>
70-
${this.startRecord > 1 ? html`<button @click="${()=>this.getRecordsHandler(this.startRecord - 10)}">Previous</button>`: ''}
71-
${this.totalRecords > this.endRecord ? html`<button @click="${()=>this.getRecordsHandler(this.startRecord + 10)}">Next</button>`: ''}
72-
`: ''}
73-
${this.fullRecord ? html`
74-
<h2>Full Record</h2>
75-
<pre>${JSON.stringify(this.fullRecord, null, 2)}</pre>
79+
<button ?disabled=${this.startRecord <= 1} @click="${()=>this.getRecordsHandler(this.startRecord - 10)}">Previous</button>
80+
<button ?disabled=${this.nextRecord === 0} @click="${()=>this.getRecordsHandler(this.startRecord + 10)}">Next</button>
7681
`: ''}
82+
${this.fullRecord ? this.renderFullRecord(this.fullRecord) : ''}
7783
</div>
7884
`;
7985
}
86+
renderOtherKeys(record) {
87+
for (const key in record) {
88+
if (!['identifier', 'title', 'type', 'subject', 'format', 'date', 'abstract', 'description', 'rights', 'source', 'relation', 'URI'].includes(key)) {
89+
return html`<tr><td>${key}:</td><td> ${record[key]}</td></tr>`;
90+
}
91+
}
92+
}
93+
renderFullRecord(record) {
94+
if (!record) return html`<p>No record selected</p>`;
95+
const subject = record.subject ? Array.isArray(record.subject) ? record.subject.join(', ') : record.subject : '';
96+
let abstractLabel = record.abstract === record.description ? 'Abstract / description' : 'Abstract';
97+
98+
return html`
99+
<h2>Full Record</h2>
100+
<table>
101+
<tr><td class="label">Identifier:</td><td> ${record.identifier}</td></tr>
102+
<tr><td class="label">Title:</td><td> ${record.title}</td></tr>
103+
<tr><td class="label">Type:</td><td> ${record.type}</td></tr>
104+
<tr><td class="label">Subject:</td><td> ${subject}</td></tr>
105+
<tr><td class="label">Subject (english):</td><td> ${this.translatedSubject}</td></tr>
106+
<tr><td class="label">Format:</td><td> ${record.format}</td></tr>
107+
<tr><td class="label">Date:</td><td> ${record.date}</td></tr>
108+
<tr><td class="label">${abstractLabel}:</td><td> ${record.abstract}</td></tr>
109+
<tr><td class="label">${abstractLabel} (english)</td><td> ${this.translatedAbstract}</td></tr>
110+
${record.abstract !== record.description ? html`<tr><td class="label">Description:</td><td> ${record.description}</td></tr>` : ''}
111+
${record.abstract !== record.description ? html`<tr><td class="label">Description (english):</td><td> ${this.translatedDescription}</td></tr>` : ''}
112+
<tr><td>Rights:</td><td> ${record.rights?record.rights:'none'}</td></tr>
113+
<tr><td>Source:</td><td> ${record.source?record.source:'none'}</td></tr>
114+
<tr><td>Relation:</td><td> ${record.relation? html`<span @click="${(_e)=>this.updateFullRecord(record.relation)}">Link</span>`:'none'}</td></tr>
115+
<tr><td>Links: </td><td></td></tr>
116+
${record.URI ?
117+
record.URI.map(uri => html`<tr><td></td><td>${uri.protocol} <a href="${uri.url}" target="cswlink">${uri.name?uri.name:'No name'} ${uri.description}</a></td></tr>`)
118+
:
119+
html`<tr><td></td><td>no URI/URLs defined</td></tr>`
120+
}
121+
${this.renderOtherKeys(record)}
122+
</table>
123+
`;
124+
}
80125
connectedCallback() {
81126
super.connectedCallback();
82127
this.init();
@@ -113,40 +158,85 @@ class CSWApp extends LitElement {
113158
return null;
114159
}
115160
}
116-
async getRecordById(identifier) {
161+
async getTranslatedStrings(strings, language) {
162+
return await this.fetchJson('./translate', {
163+
method: 'POST',
164+
headers: {
165+
'Content-Type': 'application/json'
166+
},
167+
body: JSON.stringify({
168+
strings: strings,
169+
language: language
170+
})
171+
})
172+
}
173+
briefRecordTitleClicked(event, identifier) {
174+
const rows = event.target.parentElement.parentElement.querySelectorAll('tr');
175+
rows.forEach(row => row.style.backgroundColor = 'white');
176+
event.target.parentElement.style.backgroundColor = 'whitesmoke';
177+
// remove focus of all previously selected input fields
178+
const inputs = this.shadowRoot.querySelectorAll('input');
179+
inputs.forEach(input => input.blur());
180+
this.updateFullRecord(identifier);
181+
}
182+
async updateFullRecord(identifier) {
117183
this.fullRecord = null;
184+
this.translatedAbstract = this.translatedDescription = this.translatedSubject = 'Translating...'
118185
const fullRecord = await this.fetchJson(`./csw_record_by_id?url=${encodeURIComponent(this.catalogList[this.catalogSelect.value].url)}&id=${encodeURIComponent(identifier)}`);
119186
if (fullRecord && !fullRecord.error) {
120-
this.fullRecord = fullRecord;
187+
this.fullRecord = fullRecord?.records?.[0];
188+
const subject = this.fullRecord.subject ? Array.isArray(this.fullRecord.subject) ? this.fullRecord.subject.join(', ') : this.fullRecord.subject : '';
189+
[this.translatedAbstract, this.translatedDescription, this.translatedSubject] = await this.getTranslatedStrings(
190+
[ this.fullRecord.abstract,
191+
this.fullRecord.description,
192+
subject], this.catalogList[this.catalogSelect.value].language)
121193
}
122194
}
123195
async getRecordsHandler(startRecord) {
124196
const catalogUrl = this.catalogList[this.catalogSelect.value].url;
125-
const records = await this.fetchJson(`./csw_records?url=${encodeURIComponent(catalogUrl)}&startRecord=${startRecord}&type=${this.searchType}&search=${this.searchString}`);
126-
this.totalRecords = 0;
127197
this.briefRecords = [];
128198
this.fullRecord = null;
129-
this.requestUpdate();
199+
this.startRecord = startRecord;
200+
if (startRecord == 1) {
201+
this.translatedSearch = this.searchString;
202+
this.totalRecords = 0;
203+
}
204+
this.requestUpdate();
205+
let search = this.searchString;
206+
if (this.searchString.trim() !== '') {
207+
if (this.startRecord === 1) {
208+
const targetLanguage = this.catalogList[this.catalogSelect.value].language;
209+
const userLanguage = navigator.language;
210+
this.message = `Translating ${this.searchString} to ${targetLanguage}`;
211+
search = await this.fetchJson(`./translateSearch?searchString=${encodeURIComponent(this.searchString)}&userLanguage=${userLanguage}&targetLanguage=${targetLanguage}`);
212+
this.message = '';
213+
this.translatedSearch = search;
214+
} else {
215+
search = this.translatedSearch; // keep the translated search string for search pages > 1
216+
}
217+
} else {
218+
this.translatedSearch = '';
219+
}
220+
this.message = 'Searching Catalogue...'
221+
const records = await this.fetchJson(`./csw_records?url=${encodeURIComponent(catalogUrl)}&startRecord=${startRecord}&type=${this.searchType}&search=${search}`);
222+
this.message = '';
130223
if (records && !records.error) {
131224
const searchResults = records.searchResults;
132-
this.totalRecords = searchResults.numberOfRecordsMatched;
133-
this.startRecord = searchResults.nextRecord - searchResults.numberOfRecordsReturned;
134-
this.endRecord = searchResults.nextRecord - 1;
225+
if (searchResults) {
226+
this.totalRecords = parseInt(searchResults.numberOfRecordsMatched);
227+
this.endRecord = this.startRecord + parseInt(searchResults.numberOfRecordsReturned) - 1;
228+
this.nextRecord = parseInt(searchResults.nextRecord);
229+
} else {
230+
this.totalRecords = 0;
231+
this.startRecord = 1;
232+
this.endRecord = 0;
233+
}
135234
const briefRecords = records.records;
136235
briefRecords.forEach(record => record.englishTitle = 'Translating...');
137236
this.briefRecords = briefRecords;
138237
let translatedStrings = null;
139238
if (briefRecords.length > 0) {
140-
translatedStrings = await this.fetchJson('./translate', {
141-
method: 'POST',
142-
headers: {
143-
'Content-Type': 'application/json'
144-
},
145-
body: JSON.stringify({
146-
strings: briefRecords.map(record => record.title),
147-
language: this.catalogList[this.catalogSelect.value].language
148-
})
149-
})
239+
translatedStrings = await this.getTranslatedStrings(briefRecords.map(record => record.title), this.catalogList[this.catalogSelect.value].language);
150240
}
151241
if (translatedStrings) {
152242
briefRecords.forEach((record, index) => record.englishTitle = translatedStrings[index]);

‎routes/translate.js

+9-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import express from 'express';
2-
import { translateStrings } from '../translate.js';
2+
import { translateStrings, translateSearchString } from '../translate.js';
33
const router = express.Router();
44

55
router.post('/translate', express.json(), async (req, res) => {
@@ -10,4 +10,12 @@ router.post('/translate', express.json(), async (req, res) => {
1010
}
1111
);
1212

13+
router.get('/translateSearch', async (req, res) => {
14+
const searchString = req.query.searchString;
15+
const userLanguage = req.query.userLanguage;
16+
const targetLanguage = req.query.targetLanguage;
17+
const translatedString = await translateSearchString(searchString, userLanguage, targetLanguage);
18+
res.json(translatedString);
19+
});
20+
1321
export default router;

‎translate.js

+66
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import { executionAsyncResource } from "async_hooks";
12
import OpenAI from "openai";
23
import { fileURLToPath } from 'url';
34

@@ -14,6 +15,8 @@ export async function translateStrings(textArray, language) {
1415
if (!openai) {
1516
return textArray.map(text => `Server config: translation API key not set`);
1617
}
18+
// replace all double quotes with single quotes and remove single quotes to prevent issues with AI JSON parsing
19+
textArray = textArray.map(text => text.replace(/"/g, "'").replace(/'/g, ""));
1720
const json = JSON.stringify(textArray);
1821
if (language.toLowerCase() === "english")
1922
{
@@ -33,6 +36,69 @@ export async function translateStrings(textArray, language) {
3336
}
3437
}
3538

39+
export async function translateSearchString(searchString, userlanguage, language) {
40+
console.log(userlanguage);
41+
// convert navigator.language to language name
42+
switch (userlanguage.slice(0, 2).toLowerCase()) {
43+
case "en":
44+
userlanguage = "English";
45+
break;
46+
case "nl":
47+
userlanguage = "Dutch";
48+
break;
49+
case "fr":
50+
userlanguage = "French";
51+
break;
52+
case "de":
53+
userlanguage = "German";
54+
break;
55+
case "es":
56+
userlanguage = "Spanish";
57+
break;
58+
// Italian
59+
case "it":
60+
userlanguage = "Italian";
61+
// Norwegian
62+
case "no":
63+
userlanguage = "Norwegian";
64+
break;
65+
// Swedish
66+
case "sv":
67+
userlanguage = "Swedish";
68+
break;
69+
// Finnish
70+
case "fi":
71+
userlanguage = "Finnish";
72+
break;
73+
// Danish
74+
default:
75+
break;
76+
}
77+
if (language.toLowerCase() === userlanguage.toLowerCase()) {
78+
return searchString;
79+
}
80+
try {
81+
if (!openai) {
82+
console.log(`Server config: translation API key not set`);
83+
return searchString;
84+
}
85+
const completion = await openai.chat.completions.create({
86+
messages: [
87+
{ role: "system", content: `You will be provided with a search string likely in language ${userlanguage}, and your task is to translate this string to ${language}`},
88+
{ role: "system", content: `If the searchstring contains acronyms or abbreviations, you can leave them untranslated, except if you know the meaning of the acronym in both the source and target languages (GIS => SIG or AIDS=>SIDA)`},
89+
{ role: "system", content: `The search strings are given to search for geographic datasets and types in a catalog, and the translation should be as accurate as possible to ensure the search results are relevant to the user's query`},
90+
{ role: "system", content: `Your output should be the string to use as a search query in the target language`},
91+
{ role: "user", content: searchString}
92+
],
93+
model: "gpt-3.5-turbo",
94+
});
95+
return completion.choices[0].message.content;
96+
} catch (error) {
97+
console.error(`Error in translateSearchString: ${error.message?error.message: error}`);
98+
return null;
99+
}
100+
}
101+
36102
if (process.argv[1] === fileURLToPath(import.meta.url)) {
37103
(async () => {
38104
const result = await translateStrings(

0 commit comments

Comments
 (0)