Skip to content

Commit

Permalink
preventing newline uri crashes
Browse files Browse the repository at this point in the history
  • Loading branch information
Jan Forberg committed Feb 15, 2024
1 parent 5ef5b36 commit 1f53e80
Show file tree
Hide file tree
Showing 21 changed files with 236 additions and 93 deletions.
2 changes: 1 addition & 1 deletion devenv/docker-compose.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
version: "3.0"
services:
gstore:
image: dbpedia/gstore:dev #dbpedia/gstore:dev
image: gstore #dbpedia/gstore:dev
container_name: devenv_gstore
environment:
STORAGE_USER: "dba"
Expand Down
1 change: 1 addition & 0 deletions model/finalize.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ function headerFooterShacl($shaclDir){
@prefix dbo: <http://dbpedia.org/ontology/> .
@prefix foaf: <http://xmlns.com/foaf/0.1/> .
@prefix db: <https://databus.dbpedia.org/sys/ont/> .
@prefix prov: <http://www.w3.org/ns/prov-o#> .
";

Expand Down
1 change: 1 addition & 0 deletions model/generated/context.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
},

"Version": "databus:Version" ,
"Dataset": "dcat:Dataset",

"publisher": {
"@id": "dct:publisher",
Expand Down
10 changes: 9 additions & 1 deletion model/generated/shacl/version.shacl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@
@prefix dbo: <http://dbpedia.org/ontology/> .
@prefix foaf: <http://xmlns.com/foaf/0.1/> .
@prefix db: <https://databus.dbpedia.org/sys/ont/> .
@prefix prov: <http://www.w3.org/ns/prov-o#> .


<#version-exists>
a sh:NodeShape ;
Expand Down Expand Up @@ -103,7 +105,13 @@
sh:maxCount 1 ;
sh:nodeKind sh:IRI .


<#was-derived-from>
a sh:PropertyShape ;
sh:targetClass databus:Version ;
sh:severity sh:Violation ;
sh:message "Value of prov:wasDerivedFrom from must be a valid IRI."@en ;
sh:path prov:wasDerivedFrom ;
sh:nodeKind sh:IRI .



Expand Down
13 changes: 10 additions & 3 deletions model/version.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,8 @@

$example='"@type": "databus:Version",';

$context='"Version": "databus:Version" ';
$context='"Version": "databus:Version",
"Dataset": "dcat:Dataset" ';

table($section,$sectionExampleURI,$owl,$shacl,$example,$context);
?>
Expand Down Expand Up @@ -265,7 +266,13 @@
rdfs:range prov:Entity .
';

$shacl='';
$shacl='<#was-derived-from>
a sh:PropertyShape ;
sh:targetClass databus:Version ;
sh:severity sh:Violation ;
sh:message "Value of prov:wasDerivedFrom from must be a valid IRI."@en ;
sh:path prov:wasDerivedFrom ;
sh:nodeKind sh:IRI .';

$example='"wasDerivedFrom": "https://databus.dbpedia.org/dbpedia/generic/labels/2022.09.01",';

Expand Down Expand Up @@ -356,7 +363,7 @@
a sh:NodeShape;
sh:targetClass databus:Version ;
sh:sparql [
sh:message "Dataset URI must contain the group URI of the associated group." ;
sh:message "Version URI must contain the group URI of the associated group." ;
sh:prefixes databus: ;
sh:select """
SELECT $this ?group
Expand Down
4 changes: 2 additions & 2 deletions public/dist/main.js
View file

Large diffs are not rendered by default.

10 changes: 7 additions & 3 deletions public/js/search/search-manager.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,12 +59,16 @@ class SearchManager {
return;
}

/*
var options = {
method: 'GET',
url: `/${ auth.info.accountName } `,
url: `/${ auth.info.accountName }`,
headers: {
'Accept': 'application/ld+json',
'X-Jsonld-Formatting': 'flatten'
'X-Jsonld-Formatting': 'flatten',
'Cache-Control': 'no-cache',
'Pragma': 'no-cache'
}
}
Expand All @@ -82,7 +86,7 @@ class SearchManager {
this.searchExtensions.push(searchExtension);
}
})*/
}
}

Expand Down
35 changes: 35 additions & 0 deletions public/templates/publish-wizard.ejs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,41 @@
</div>
</div>

<div class="databus-settings-box-header">Result</div>
<div class="databus-settings-box">


<div class="box-content">
<div class="row" ng-if="!isPublishing">

<div style="width: 100%">
<div ng-repeat="entry in publishLog" style="margin-bottom: .25em;">

<div style="width: 100%; font-family: monospace; padding: .75em;
background-color: #f5f5f5;display: flex; align-items: center; cursor:pointer"
ng-click="entry.expanded = !entry.expanded">

<databus-icon ng-if="!entry.expanded" shape="'right'" size="16">
</databus-icon>
<databus-icon ng-if="entry.expanded" shape="'down'" size="16">
</databus-icon>

<div style="flex: 1; margin-left: .75em;" ng-if="entry.level=='error'" style="color:red">{{
entry.msg }}</div>
<div style="flex: 1; margin-left: .75em;" ng-if="entry.level!='error'">{{ entry.msg }}</div>
<databus-icon ng-if="entry.level!='error'" color="'#65bdb2'" shape="'check'" size="24">
</databus-icon>
</div>
<div ng-if="entry.expanded">
<pre style="width: 100%; border-top: 1px solid #dbdbdb;">{{ entry.payload | json }}</pre>
</div>
</div>
</div>

</div>

</div>
</div>

</div>

Expand Down
131 changes: 74 additions & 57 deletions server/app/api/lib/publish-version.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ var autocompleter = require('./dataid-autocomplete.js');
var fileAnalyzer = require('../../common/file-analyzer.js');
const DatabusUtils = require('../../../../public/js/utils/databus-utils.js');
const DatabusMessage = require('../../common/databus-message.js');
const { log } = require('console');
const { del } = require('request');


async function verifyDataidParts(dataidGraphs, alwaysFetch, logger) {
Expand Down Expand Up @@ -81,60 +83,66 @@ async function verifyDataidParts(dataidGraphs, alwaysFetch, logger) {
*/
async function constructInput(expandedGraph, versionGraphUri, logger) {

var versionGraph = JsonldUtils.getGraphById(expandedGraph, versionGraphUri);
var cvGraphs = JsonldUtils.getSubPropertyGraphs(expandedGraph, DatabusUris.DATABUS_CONTENT_VARIANT);
logger.debug(versionGraphUri, `Detected CV-graphs`, cvGraphs);
try {
var versionGraph = JsonldUtils.getGraphById(expandedGraph, versionGraphUri);
var cvGraphs = JsonldUtils.getSubPropertyGraphs(expandedGraph, DatabusUris.DATABUS_CONTENT_VARIANT);
logger.debug(versionGraphUri, `Detected CV-graphs`, cvGraphs);

var distributionUris = versionGraph[DatabusUris.DCAT_DISTRIBUTION];
var distributionUris = versionGraph[DatabusUris.DCAT_DISTRIBUTION];

var dataIdGraphs = [];
dataIdGraphs.push(JSON.parse(JSON.stringify(versionGraph)));
versionGraph[DatabusUris.DCAT_DISTRIBUTION] = [];
var dataIdGraphs = [];
dataIdGraphs.push(JSON.parse(JSON.stringify(versionGraph)));
versionGraph[DatabusUris.DCAT_DISTRIBUTION] = [];

var totalTripleCount = 0;
var step = 100;
var totalTripleCount = 0;
var step = 100;

var versionGraphCopy = JSON.parse(JSON.stringify(versionGraph));
var distributionlessGraphs = [versionGraphCopy].concat(cvGraphs);
var versionGraphCopy = JSON.parse(JSON.stringify(versionGraph));
var distributionlessGraphs = [versionGraphCopy].concat(cvGraphs);

// Create sub-dataids with only 100 parts at a time
// This will avoid long running or failing construct queries for large inputs
for (var i = 0; i < distributionUris.length; i += step) {
// Create sub-dataids with only 100 parts at a time
// This will avoid long running or failing construct queries for large inputs
for (var i = 0; i < distributionUris.length; i += step) {

var distributionSubset = distributionUris.slice(i, Math.min(distributionUris.length, i + step))
var slice = Array.from(distributionlessGraphs);
var distributionSubset = distributionUris.slice(i, Math.min(distributionUris.length, i + step))
var slice = Array.from(distributionlessGraphs);

// Add links from Dataset graph to each entry in the subset
versionGraphCopy[DatabusUris.DCAT_DISTRIBUTION] = [];
for (var j = 0; j < distributionSubset.length; j++) {
versionGraphCopy[DatabusUris.DCAT_DISTRIBUTION].push(distributionSubset[j]);
slice.push(JsonldUtils.getGraphById(expandedGraph, distributionSubset[j][DatabusUris.JSONLD_ID]));
}
// Add links from Dataset graph to each entry in the subset
versionGraphCopy[DatabusUris.DCAT_DISTRIBUTION] = [];
for (var j = 0; j < distributionSubset.length; j++) {
versionGraphCopy[DatabusUris.DCAT_DISTRIBUTION].push(distributionSubset[j]);
slice.push(JsonldUtils.getGraphById(expandedGraph, distributionSubset[j][DatabusUris.JSONLD_ID]));
}

var triples = await constructor.executeConstruct(slice, constructVersionQuery);
var tripleCount = DatabusUtils.lineCount(triples);
logger.debug(versionGraphUri, `Construct fetched ${tripleCount} triples from subgraph`);

var triples = await constructor.executeConstruct(slice, constructVersionQuery);
var tripleCount = DatabusUtils.lineCount(triples);
logger.debug(versionGraphUri, `Construct fetched ${tripleCount} triples from subgraph`);
totalTripleCount += tripleCount;

totalTripleCount += tripleCount;
var unflattenedJsonLd = await jsonld.fromRDF(triples);

var subGraphs = await jsonld.flatten(await jsonld.fromRDF(triples));
subGraphs = JsonldUtils.getTypedGraphs(subGraphs, DatabusUris.DATABUS_PART);
var subGraphs = await jsonld.flatten(unflattenedJsonLd);
subGraphs = JsonldUtils.getTypedGraphs(subGraphs, DatabusUris.DATABUS_PART);

// Add the constructed graphs to the result graph
for (var subGraph of subGraphs) {
dataIdGraphs.push(subGraph);
var distributionGraphEntry = {};
distributionGraphEntry[DatabusUris.JSONLD_ID] = subGraph[DatabusUris.JSONLD_ID];
versionGraph[DatabusUris.DCAT_DISTRIBUTION].push(distributionGraphEntry);
// Add the constructed graphs to the result graph
for (var subGraph of subGraphs) {
dataIdGraphs.push(subGraph);
var distributionGraphEntry = {};
distributionGraphEntry[DatabusUris.JSONLD_ID] = subGraph[DatabusUris.JSONLD_ID];
versionGraph[DatabusUris.DCAT_DISTRIBUTION].push(distributionGraphEntry);
}
}
}

if (totalTripleCount == 0) {
return null;
}
if (totalTripleCount == 0) {
return null;
}

logger.debug(versionGraphUri, `${tripleCount} triples selected via construct query.`, dataIdGraphs);
return dataIdGraphs;
logger.debug(versionGraphUri, `${tripleCount} triples selected via construct query.`, dataIdGraphs);
return dataIdGraphs;
} catch(err) {
throw Error(err);
}
}

function validateDatasetUri(dataidGraphs, accountUri, logger) {
Expand Down Expand Up @@ -221,7 +229,7 @@ async function createOrValidateSignature(dataidGraphs, accountUri, logger) {
//console.log(proofGraph);
// Validate the proof
//console.log(dataidGraphs);

var validationSuccess = await signer.validate(signer.canonicalize(dataidGraphs), proofGraph);

if (!validationSuccess) {
Expand All @@ -245,6 +253,15 @@ module.exports = async function publishVersion(accountName, expandedGraph, versi
var versionGraph = JsonldUtils.getGraphById(expandedGraph, versionGraphUri);
logger.debug(versionGraphUri, `Processing version <${versionGraphUri}>`, versionGraph);

// Run SHACL validation
var shaclResult = await shaclTester.validateVersionInputRDF(expandedGraph);

console.log(shaclResult);
// Return failure with SHACL validation message
if (!shaclResult.isSuccess) {
logger.error(versionGraphUri, `Input validation failed`, shaclResult);
return 400;
}

// Run construct query
var dataidGraphs = await constructInput(expandedGraph, versionGraphUri, logger);
Expand All @@ -268,10 +285,8 @@ module.exports = async function publishVersion(accountName, expandedGraph, versi
autocompleter.autocomplete(dataidGraphs, logger);
logger.debug(versionGraphUri, `Input after auto-completion`, dataidGraphs);


logger.debug(versionGraphUri, `fetch-file-properties is set to ${fetchFileProperties}`, null);


// Verify parts: SHA256SUM, BYTESIZE, etc
if (fetchFileProperties == true || fetchFileProperties == null) {

Expand All @@ -285,8 +300,8 @@ module.exports = async function publishVersion(accountName, expandedGraph, versi
var cvGraphs = JsonldUtils.getSubPropertyGraphs(dataidGraphs, DatabusUris.DATABUS_CONTENT_VARIANT);

// Apply data fix for byteSize, so omitting DECIMAL passes the SHACL test
for(var distribution of distributionGraphs) {
if(distribution[DatabusUris.DCAT_BYTESIZE] != null && distribution[DatabusUris.DCAT_BYTESIZE].length > 0) {
for (var distribution of distributionGraphs) {
if (distribution[DatabusUris.DCAT_BYTESIZE] != null && distribution[DatabusUris.DCAT_BYTESIZE].length > 0) {
distribution[DatabusUris.DCAT_BYTESIZE][0][DatabusUris.JSONLD_TYPE] = DatabusUris.XSD_DECIMAL;
}
}
Expand All @@ -307,15 +322,6 @@ module.exports = async function publishVersion(accountName, expandedGraph, versi
return 400;
}

// Run SHACL validation
var shaclResult = await shaclTester.validateVersionRDF(dataidGraphs);

// Return failure with SHACL validation message
if (!shaclResult.isSuccess) {
logger.error(versionGraphUri, `SHACL validation failed`, shaclResult);
return 400;
}

logger.debug(versionGraphUri, `SHACL validation successful`, shaclResult);
dataidGraphs = await jsonld.flatten(dataidGraphs);
validationCode = await createOrValidateSignature(dataidGraphs, accountUri, logger);
Expand All @@ -326,6 +332,15 @@ module.exports = async function publishVersion(accountName, expandedGraph, versi

logger.debug(versionGraphUri, `Signature validation successful.`, null);

// Run SHACL validation
var shaclResult = await shaclTester.validateVersionRDF(dataidGraphs);

// Return failure with SHACL validation message
if (!shaclResult.isSuccess) {
logger.error(versionGraphUri, `SHACL validation failed`, shaclResult);
return 400;
}

// Create compacted graph
var compactedGraph = await jsonld.compact(dataidGraphs, defaultContext);

Expand All @@ -343,11 +358,13 @@ module.exports = async function publishVersion(accountName, expandedGraph, versi

// Return failure
if (!publishResult.isSuccess) {
logger.error(versionGraphUri, `Internal database error`, null);
return 500;
var statusCode = publishResult.statusCode;
delete publishResult.statusCode;
logger.error(versionGraphUri, `Failed to save version to database.`, publishResult);
return statusCode;
}

if(process.send != undefined) {
if (process.send != undefined) {
process.send({
id: DatabusMessage.REQUEST_SEARCH_INDEX_REBUILD,
resource: versionGraphUri
Expand Down
7 changes: 3 additions & 4 deletions server/app/api/routes/general.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ const JsonldUtils = require('../../../../public/js/utils/jsonld-utils');
var jsonld = require('jsonld');
const DatabusLogger = require('../../common/databus-logger');

const MSG_NO_GRAPH_FOUND = `No processable graphs found in the input. Your input has to contain at least one graph of either type databus:Group, databus:Artifact or databus:Version.`

module.exports = function (router, protector, webdav) {

router.get('/sparql', cors(), function (req, res, next) {
Expand All @@ -31,9 +33,6 @@ module.exports = function (router, protector, webdav) {

var query = req.body.query;




var sparqlEndpoint = `${process.env.DATABUS_DATABASE_URL}/sparql`;
var accept = req.headers['accept']

Expand Down Expand Up @@ -129,7 +128,7 @@ module.exports = function (router, protector, webdav) {
}

if(processedResources == 0) {
logger.error(null, `No processable graphs found in the input.`, req.body);
logger.error(null, MSG_NO_GRAPH_FOUND, req.body);
res.status(400).json(logger.getReport());
return;
}
Expand Down
Loading

0 comments on commit 1f53e80

Please sign in to comment.