From 6a1c9ab33040496f0250d5647d52c28dabba28d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cristina=20E=2E=20Gonz=C3=A1lez-Espinoza?= Date: Tue, 15 Nov 2022 14:19:39 +0100 Subject: [PATCH 1/6] Saving initial changes. --- kgforge/specializations/models/rdf/store_service.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/kgforge/specializations/models/rdf/store_service.py b/kgforge/specializations/models/rdf/store_service.py index 621f8fea..53a2d7d6 100644 --- a/kgforge/specializations/models/rdf/store_service.py +++ b/kgforge/specializations/models/rdf/store_service.py @@ -15,7 +15,7 @@ import json -from pyshacl import Validator +from pyshacl import validate from kgforge.core import Resource from kgforge.core.commons.exceptions import RetrievalError @@ -60,8 +60,12 @@ def materialize(self, iri: URIRef) -> NodeProperties: def _validate(self, iri: str, data_graph: Graph) -> Tuple[bool, Graph, str]: # _type_shape will make sure all the shapes for this type are in the graph self._type_shape(iri) - validator = Validator(data_graph, shacl_graph=self._graph) - return validator.run() + # validator = Validator(data_graph, shacl_graph=self._graph) + result = validate(data_graph, shacl_graph=self._graph, debug=True) + conforms, results_graph, results_text = result + print(results_text) + return result + # return validator.run() def resolve_context(self, iri: str) -> Dict: if iri in self._context_cache: From b5e615da3b18c2d6416cce553ae468fa561cbe40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cristina=20E=2E=20Gonz=C3=A1lez-Espinoza?= Date: Thu, 17 Nov 2022 12:11:27 +0100 Subject: [PATCH 2/6] Enable option in pyschacl for local schema validation --- .../notebooks/use-cases/LocalValidation.ipynb | 433 ++++++++++++++++++ kgforge/core/archetypes/model.py | 8 +- kgforge/core/forge.py | 7 +- kgforge/specializations/models/demo_model.py | 2 +- .../models/rdf/directory_service.py | 8 +- kgforge/specializations/models/rdf/service.py | 6 +- .../models/rdf/store_service.py | 8 +- kgforge/specializations/models/rdf_model.py | 13 +- .../specializations/models/test_demo_model.py | 4 +- .../specializations/models/test_rdf_model.py | 8 +- 10 files changed, 464 insertions(+), 33 deletions(-) create mode 100644 examples/notebooks/use-cases/LocalValidation.ipynb diff --git a/examples/notebooks/use-cases/LocalValidation.ipynb b/examples/notebooks/use-cases/LocalValidation.ipynb new file mode 100644 index 00000000..86e91f10 --- /dev/null +++ b/examples/notebooks/use-cases/LocalValidation.ipynb @@ -0,0 +1,433 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Resolving Strategies\n", + "\n", + "* Example on how to use resolving strategies\n" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "ExecuteTime": { + "end_time": "2019-09-23T18:50:20.068658Z", + "start_time": "2019-09-23T18:50:19.054054Z" + } + }, + "outputs": [], + "source": [ + "import os\n", + "import json\n", + "\n", + "from kgforge.core import KnowledgeGraphForge\n", + "from kgforge.specializations.resources import Dataset" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import getpass\n", + "TOKEN = getpass.getpass()" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "endpoint = \"https://staging.nise.bbp.epfl.ch/nexus/v1\"\n", + "BUCKET = \"dke/kgforge\"\n", + "forge = KnowledgeGraphForge(\"../use-cases/prod-forge-nexus.yml\",\n", + " endpoint=endpoint, \n", + " bucket=BUCKET,\n", + " token=TOKEN\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "some_str = \"\"\"\n", + "{\n", + " \"type\": [\n", + " \"Dataset\",\n", + " \"Trace\",\n", + " \"Entity\",\n", + " \"SingleCellSimulationTrace\"\n", + " ],\n", + " \"id\": \"https://bbp.epfl.ch/neurosciencegraph/data/traces/17b443b3-07fc-41c7-80c8-ea5e54662210\",\n", + " \"annotation\": [\n", + " {\n", + " \"type\": [\n", + " \"Annotation\",\n", + " \"MTypeAnnotation\"\n", + " ],\n", + " \"hasBody\": {\n", + " \"id\": \"ilx:0738236\",\n", + " \"type\": [\n", + " \"AnnotationBody\",\n", + " \"MType\"\n", + " ],\n", + " \"label\": \"VPL_TC\"\n", + " },\n", + " \"name\": \"M-type Annotation\"\n", + " },\n", + " {\n", + " \"type\": [\n", + " \"Annotation\",\n", + " \"ETypeAnnotation\"\n", + " ],\n", + " \"hasBody\": {\n", + " \"id\": \"ilx:0738254\",\n", + " \"type\": [\n", + " \"AnnotationBody\",\n", + " \"EType\"\n", + " ],\n", + " \"label\": \"cNAD_ltb\"\n", + " },\n", + " \"name\": \"E-type Annotation\"\n", + " }\n", + " ],\n", + " \"identifier\": \"dNAD_ltb\",\n", + " \"note\": \"Simulated Thalamus cell\",\n", + " \"dateCreated\": {\n", + " \"@value\": \"2021-12-08T18:03:31.001661+01:00\",\n", + " \"type\": \"xsd:dateTime\"\n", + " },\n", + " \"brainLocation\": {\n", + " \"type\": \"BrainLocation\",\n", + " \"brainRegion\": {\n", + " \"id\": \"http://api.brain-map.org/api/v2/data/Structure/718\",\n", + " \"label\": \"Ventral posterolateral nucleus of the thalamus\"\n", + " }\n", + " },\n", + " \"contribution\": [\n", + " {\n", + " \"type\": \"Contribution\",\n", + " \"agent\": {\n", + " \"id\": \"https://bbp.epfl.ch/nexus/v1/realms/bbp/users/tuncel\",\n", + " \"type\": \"Agent\",\n", + " \"label\": \"Anil Tuncel\"\n", + " }\n", + " },\n", + " {\n", + " \"type\": \"Contribution\",\n", + " \"agent\": {\n", + " \"id\": \"https://bbp.epfl.ch/nexus/v1/realms/bbp/users/iavarone\",\n", + " \"type\": \"Agent\",\n", + " \"label\": \"Elisabetta Iavarone\"\n", + " }\n", + " }\n", + " ],\n", + " \"description\": \"This dataset is about simulated electrophysiology traces for cell instance dNAD_ltb. The dataset contains one distribution of the traces in NWB file format.\",\n", + " \"subject\": {\n", + " \"type\": \"Subject\",\n", + " \"species\": {\n", + " \"id\": \"ncbitaxon:10090\",\n", + " \"label\": \"Mus musculus\"\n", + " }\n", + " },\n", + " \"distribution\": {\n", + " \"type\": \"DataDownload\",\n", + " \"contentSize\": {\n", + " \"unitCode\": \"bytes\",\n", + " \"value\": 2347304\n", + " },\n", + " \"digest\": {\n", + " \"algorithm\": \"SHA-256\",\n", + " \"value\": \"630f7455b9be5b212566a8dcc76d6a9b2303257ed9199a5c9262fc25aaa5749f\"\n", + " },\n", + " \"encodingFormat\": \"application/nwb\",\n", + " \"name\": \"dNAD_ltb.nwb\",\n", + " \"contentUrl\": \"https://bbp.epfl.ch/nexus/v1/files/bbp/uniprot/63c13f11-4dde-42fa-81cd-74fe6a68f35c\",\n", + " \"atLocation\": {\n", + " \"type\": \"Location\",\n", + " \"store\": {\n", + " \"id\": \"https://bbp.epfl.ch/data/bbp/uniprot/dc51e00c-584f-4f8b-a5da-df85d2416d79\",\n", + " \"type\": \"RemoteDiskStorage\",\n", + " \"_rev\": 1\n", + " },\n", + " \"location\": \"file:///gpfs/bbp.cscs.ch/data/project/proj94/nexus/bbp/uniprot/9/9/e/f/6/1/f/e/dNAD_ltb.nwb\"\n", + " }\n", + " },\n", + " \"objectOfStudy\": {\n", + " \"type\": \"ObjectOfStudy\",\n", + " \"id\": \"http://bbp.epfl.ch/neurosciencegraph/taxonomies/objectsofstudy/singlecells\",\n", + " \"label\": \"Single Cell\"\n", + " }\n", + "}\n", + "\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "resource = forge.from_json(json.loads(some_str))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Use a debug mode in pyschacl" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Constraint Violation in MinCountConstraintComponent (http://www.w3.org/ns/shacl#MinCountConstraintComponent):\n", + "\tSeverity: sh:Violation\n", + "\tSource Shape: [ sh:description Literal(\"When followed this link leads to a resource providing further description on how to download the attached data.\") ; sh:maxCount Literal(\"1\", datatype=xsd:integer) ; sh:minCount Literal(\"1\", datatype=xsd:integer) ; sh:nodeKind sh:IRI ; sh:path schema:url ]\n", + "\tFocus Node: [ [ Literal(\"bytes\") ; Literal(\"2347304\", datatype=xsd:integer) ] ; ; Literal(\"application/nwb\") ; Literal(\"dNAD_ltb.nwb\") ; [ Literal(\"file:///gpfs/bbp.cscs.ch/data/project/proj94/nexus/bbp/uniprot/9/9/e/f/6/1/f/e/dNAD_ltb.nwb\") ; ; rdf:type ] ; [ Literal(\"SHA-256\") ; Literal(\"630f7455b9be5b212566a8dcc76d6a9b2303257ed9199a5c9262fc25aaa5749f\") ] ; rdf:type ]\n", + "\tResult Path: schema:url\n", + "\tMessage: Less than 1 values on [ [ Literal(\"bytes\") ; Literal(\"2347304\", datatype=xsd:integer) ] ; ; Literal(\"application/nwb\") ; Literal(\"dNAD_ltb.nwb\") ; [ Literal(\"file:///gpfs/bbp.cscs.ch/data/project/proj94/nexus/bbp/uniprot/9/9/e/f/6/1/f/e/dNAD_ltb.nwb\") ; ; rdf:type ] ; [ Literal(\"SHA-256\") ; Literal(\"630f7455b9be5b212566a8dcc76d6a9b2303257ed9199a5c9262fc25aaa5749f\") ] ; rdf:type ]->schema:url\n", + "\n", + "Constraint Violation in DatatypeConstraintComponent (http://www.w3.org/ns/shacl#DatatypeConstraintComponent):\n", + "\tSeverity: sh:Violation\n", + "\tSource Shape: [ sh:datatype xsd:float ]\n", + "\tFocus Node: Literal(\"2347304\", datatype=xsd:integer)\n", + "\tValue Node: Literal(\"2347304\", datatype=xsd:integer)\n", + "\tMessage: Value is not Literal with datatype xsd:float\n", + "\n", + "Constraint Violation in DatatypeConstraintComponent (http://www.w3.org/ns/shacl#DatatypeConstraintComponent):\n", + "\tSeverity: sh:Violation\n", + "\tSource Shape: [ sh:datatype xsd:double ]\n", + "\tFocus Node: Literal(\"2347304\", datatype=xsd:integer)\n", + "\tValue Node: Literal(\"2347304\", datatype=xsd:integer)\n", + "\tMessage: Value is not Literal with datatype xsd:double\n", + "\n", + "Constraint Violation in DatatypeConstraintComponent (http://www.w3.org/ns/shacl#DatatypeConstraintComponent):\n", + "\tSeverity: sh:Violation\n", + "\tSource Shape: [ sh:datatype xsd:unsignedInt ]\n", + "\tFocus Node: Literal(\"2347304\", datatype=xsd:integer)\n", + "\tValue Node: Literal(\"2347304\", datatype=xsd:integer)\n", + "\tMessage: Value is not Literal with datatype xsd:unsignedInt\n", + "\n", + "Constraint Violation in DatatypeConstraintComponent (http://www.w3.org/ns/shacl#DatatypeConstraintComponent):\n", + "\tSeverity: sh:Violation\n", + "\tSource Shape: [ sh:datatype xsd:long ]\n", + "\tFocus Node: Literal(\"2347304\", datatype=xsd:integer)\n", + "\tValue Node: Literal(\"2347304\", datatype=xsd:integer)\n", + "\tMessage: Value is not Literal with datatype xsd:long\n", + "\n", + "Constraint Violation in DatatypeConstraintComponent (http://www.w3.org/ns/shacl#DatatypeConstraintComponent):\n", + "\tSeverity: sh:Violation\n", + "\tSource Shape: [ sh:datatype xsd:unsignedLong ]\n", + "\tFocus Node: Literal(\"2347304\", datatype=xsd:integer)\n", + "\tValue Node: Literal(\"2347304\", datatype=xsd:integer)\n", + "\tMessage: Value is not Literal with datatype xsd:unsignedLong\n", + "\n", + "Constraint Violation in DatatypeConstraintComponent (http://www.w3.org/ns/shacl#DatatypeConstraintComponent):\n", + "\tSeverity: sh:Violation\n", + "\tSource Shape: [ sh:datatype xsd:decimal ]\n", + "\tFocus Node: Literal(\"2347304\", datatype=xsd:integer)\n", + "\tValue Node: Literal(\"2347304\", datatype=xsd:integer)\n", + "\tMessage: Value is not Literal with datatype xsd:decimal\n", + "\n", + "Constraint Violation in DatatypeConstraintComponent (http://www.w3.org/ns/shacl#DatatypeConstraintComponent):\n", + "\tSeverity: sh:Violation\n", + "\tSource Shape: [ sh:datatype xsd:int ]\n", + "\tFocus Node: Literal(\"2347304\", datatype=xsd:integer)\n", + "\tValue Node: Literal(\"2347304\", datatype=xsd:integer)\n", + "\tMessage: Value is not Literal with datatype xsd:int\n", + "\n", + "Constraint Violation in NodeKindConstraintComponent (http://www.w3.org/ns/shacl#NodeKindConstraintComponent):\n", + "\tSeverity: sh:Violation\n", + "\tSource Shape: this11:LabeledOntologyEntityShape\n", + "\tFocus Node: Literal(\"bytes\")\n", + "\tValue Node: Literal(\"bytes\")\n", + "\tMessage: Value is not of Node Kind sh:IRI\n", + "\n", + "Constraint Violation in MinCountConstraintComponent (http://www.w3.org/ns/shacl#MinCountConstraintComponent):\n", + "\tSeverity: sh:Violation\n", + "\tSource Shape: [ sh:datatype xsd:string ; sh:description Literal(\"Human readable label.\") ; sh:minCount Literal(\"1\", datatype=xsd:integer) ; sh:name Literal(\"Label\") ; sh:path rdfs:label ]\n", + "\tFocus Node: Literal(\"bytes\")\n", + "\tResult Path: rdfs:label\n", + "\tMessage: Less than 1 values on Literal(\"bytes\")->rdfs:label\n", + "\n", + "Constraint Violation in NodeConstraintComponent (http://www.w3.org/ns/shacl#NodeConstraintComponent):\n", + "\tSeverity: sh:Violation\n", + "\tSource Shape: [ sh:node this11:LabeledOntologyEntityShape ]\n", + "\tFocus Node: Literal(\"bytes\")\n", + "\tValue Node: Literal(\"bytes\")\n", + "\tMessage: Value does not conform to Shape this11:LabeledOntologyEntityShape\n", + "\n", + "Constraint Violation in MinCountConstraintComponent (http://www.w3.org/ns/shacl#MinCountConstraintComponent):\n", + "\tSeverity: sh:Violation\n", + "\tSource Shape: [ sh:datatype xsd:string ; sh:description Literal(\"The entity name.\") ; sh:maxCount Literal(\"1\", datatype=xsd:integer) ; sh:minCount Literal(\"1\", datatype=xsd:integer) ; sh:name Literal(\"Name\") ; sh:path schema:name ]\n", + "\tFocus Node: \n", + "\tResult Path: schema:name\n", + "\tMessage: Less than 1 values on ->schema:name\n", + "\n", + "Constraint Violation in NodeConstraintComponent (http://www.w3.org/ns/shacl#NodeConstraintComponent):\n", + "\tSeverity: sh:Violation\n", + "\tSource Shape: [ sh:node this3:EntityShape ]\n", + "\tFocus Node: \n", + "\tValue Node: \n", + "\tMessage: Value does not conform to Shape this3:EntityShape\n", + "\n", + "Constraint Violation in MinCountConstraintComponent (http://www.w3.org/ns/shacl#MinCountConstraintComponent):\n", + "\tSeverity: sh:Violation\n", + "\tSource Shape: [ sh:description Literal(\"When followed this link leads to a resource providing further description on how to download the attached data.\") ; sh:maxCount Literal(\"1\", datatype=xsd:integer) ; sh:minCount Literal(\"1\", datatype=xsd:integer) ; sh:nodeKind sh:IRI ; sh:path schema:url ]\n", + "\tFocus Node: [ [ Literal(\"bytes\") ; Literal(\"2347304\", datatype=xsd:integer) ] ; ; Literal(\"application/nwb\") ; Literal(\"dNAD_ltb.nwb\") ; [ Literal(\"file:///gpfs/bbp.cscs.ch/data/project/proj94/nexus/bbp/uniprot/9/9/e/f/6/1/f/e/dNAD_ltb.nwb\") ; ; rdf:type ] ; [ Literal(\"SHA-256\") ; Literal(\"630f7455b9be5b212566a8dcc76d6a9b2303257ed9199a5c9262fc25aaa5749f\") ] ; rdf:type ]\n", + "\tResult Path: schema:url\n", + "\tMessage: Less than 1 values on [ [ Literal(\"bytes\") ; Literal(\"2347304\", datatype=xsd:integer) ] ; ; Literal(\"application/nwb\") ; Literal(\"dNAD_ltb.nwb\") ; [ Literal(\"file:///gpfs/bbp.cscs.ch/data/project/proj94/nexus/bbp/uniprot/9/9/e/f/6/1/f/e/dNAD_ltb.nwb\") ; ; rdf:type ] ; [ Literal(\"SHA-256\") ; Literal(\"630f7455b9be5b212566a8dcc76d6a9b2303257ed9199a5c9262fc25aaa5749f\") ] ; rdf:type ]->schema:url\n", + "\n", + "Constraint Violation in DatatypeConstraintComponent (http://www.w3.org/ns/shacl#DatatypeConstraintComponent):\n", + "\tSeverity: sh:Violation\n", + "\tSource Shape: [ sh:datatype xsd:float ]\n", + "\tFocus Node: Literal(\"2347304\", datatype=xsd:integer)\n", + "\tValue Node: Literal(\"2347304\", datatype=xsd:integer)\n", + "\tMessage: Value is not Literal with datatype xsd:float\n", + "\n", + "Constraint Violation in DatatypeConstraintComponent (http://www.w3.org/ns/shacl#DatatypeConstraintComponent):\n", + "\tSeverity: sh:Violation\n", + "\tSource Shape: [ sh:datatype xsd:double ]\n", + "\tFocus Node: Literal(\"2347304\", datatype=xsd:integer)\n", + "\tValue Node: Literal(\"2347304\", datatype=xsd:integer)\n", + "\tMessage: Value is not Literal with datatype xsd:double\n", + "\n", + "Constraint Violation in DatatypeConstraintComponent (http://www.w3.org/ns/shacl#DatatypeConstraintComponent):\n", + "\tSeverity: sh:Violation\n", + "\tSource Shape: [ sh:datatype xsd:unsignedInt ]\n", + "\tFocus Node: Literal(\"2347304\", datatype=xsd:integer)\n", + "\tValue Node: Literal(\"2347304\", datatype=xsd:integer)\n", + "\tMessage: Value is not Literal with datatype xsd:unsignedInt\n", + "\n", + "Constraint Violation in DatatypeConstraintComponent (http://www.w3.org/ns/shacl#DatatypeConstraintComponent):\n", + "\tSeverity: sh:Violation\n", + "\tSource Shape: [ sh:datatype xsd:long ]\n", + "\tFocus Node: Literal(\"2347304\", datatype=xsd:integer)\n", + "\tValue Node: Literal(\"2347304\", datatype=xsd:integer)\n", + "\tMessage: Value is not Literal with datatype xsd:long\n", + "\n", + "Constraint Violation in DatatypeConstraintComponent (http://www.w3.org/ns/shacl#DatatypeConstraintComponent):\n", + "\tSeverity: sh:Violation\n", + "\tSource Shape: [ sh:datatype xsd:unsignedLong ]\n", + "\tFocus Node: Literal(\"2347304\", datatype=xsd:integer)\n", + "\tValue Node: Literal(\"2347304\", datatype=xsd:integer)\n", + "\tMessage: Value is not Literal with datatype xsd:unsignedLong\n", + "\n", + "Constraint Violation in DatatypeConstraintComponent (http://www.w3.org/ns/shacl#DatatypeConstraintComponent):\n", + "\tSeverity: sh:Violation\n", + "\tSource Shape: [ sh:datatype xsd:decimal ]\n", + "\tFocus Node: Literal(\"2347304\", datatype=xsd:integer)\n", + "\tValue Node: Literal(\"2347304\", datatype=xsd:integer)\n", + "\tMessage: Value is not Literal with datatype xsd:decimal\n", + "\n", + "Constraint Violation in DatatypeConstraintComponent (http://www.w3.org/ns/shacl#DatatypeConstraintComponent):\n", + "\tSeverity: sh:Violation\n", + "\tSource Shape: [ sh:datatype xsd:int ]\n", + "\tFocus Node: Literal(\"2347304\", datatype=xsd:integer)\n", + "\tValue Node: Literal(\"2347304\", datatype=xsd:integer)\n", + "\tMessage: Value is not Literal with datatype xsd:int\n", + "\n", + "Constraint Violation in NodeKindConstraintComponent (http://www.w3.org/ns/shacl#NodeKindConstraintComponent):\n", + "\tSeverity: sh:Violation\n", + "\tSource Shape: this11:LabeledOntologyEntityShape\n", + "\tFocus Node: Literal(\"bytes\")\n", + "\tValue Node: Literal(\"bytes\")\n", + "\tMessage: Value is not of Node Kind sh:IRI\n", + "\n", + "Constraint Violation in MinCountConstraintComponent (http://www.w3.org/ns/shacl#MinCountConstraintComponent):\n", + "\tSeverity: sh:Violation\n", + "\tSource Shape: [ sh:datatype xsd:string ; sh:description Literal(\"Human readable label.\") ; sh:minCount Literal(\"1\", datatype=xsd:integer) ; sh:name Literal(\"Label\") ; sh:path rdfs:label ]\n", + "\tFocus Node: Literal(\"bytes\")\n", + "\tResult Path: rdfs:label\n", + "\tMessage: Less than 1 values on Literal(\"bytes\")->rdfs:label\n", + "\n", + "Constraint Violation in NodeConstraintComponent (http://www.w3.org/ns/shacl#NodeConstraintComponent):\n", + "\tSeverity: sh:Violation\n", + "\tSource Shape: [ sh:node this11:LabeledOntologyEntityShape ]\n", + "\tFocus Node: Literal(\"bytes\")\n", + "\tValue Node: Literal(\"bytes\")\n", + "\tMessage: Value does not conform to Shape this11:LabeledOntologyEntityShape\n", + "\n", + "Constraint Violation in AndConstraintComponent (http://www.w3.org/ns/shacl#AndConstraintComponent):\n", + "\tSeverity: sh:Violation\n", + "\tSource Shape: this1:MINDSShape\n", + "\tFocus Node: \n", + "\tValue Node: \n", + "\tMessage: Node does not conform to all shapes in [ sh:node this3:EntityShape ] , [ sh:property [ rdfs:seeAlso ; sh:class nsg:Contribution ; sh:description Literal(\"Information about the contribution of an agent during the generation of this dataset.\") ; sh:minCount Literal(\"1\", datatype=xsd:integer) ; sh:name Literal(\"Qualified Contribution\") ; sh:nodeKind sh:BlankNodeOrIRI ; sh:path nsg:contribution ], [ rdfs:seeAlso this2:SubjectShape ; sh:class nsg:Subject ; sh:description Literal(\"The subject of the study. This data represent an observation of this subject.\") ; sh:minCount Literal(\"1\", datatype=xsd:integer) ; sh:name Literal(\"Subject\") ; sh:path nsg:subject ; skos:editorialNote Literal(\"Asserted subject entity. It may differ from the subject from the provenance.\", datatype=xsd:string) ], [ sh:class nsg:Annotation ; sh:description Literal(\"Annotations of the dataset.\") ; sh:name Literal(\"Annotation\") ; sh:node this18:AnnotationShape ; sh:path nsg:annotation ], [ sh:class nsg:BrainAtlasSpatialReferenceSystem ; sh:maxCount Literal(\"1\", datatype=xsd:integer) ; sh:path nsg:isRegisteredIn ], [ sh:class nsg:BrainLocation ; sh:description Literal(\"The Brain location object.\") ; sh:minCount Literal(\"1\", datatype=xsd:integer) ; sh:name Literal(\"Brain Location\") ; sh:node this16:BrainLocationShape ; sh:path nsg:brainLocation ], [ sh:class nsg:ObjectOfStudy ; sh:description Literal(\"Object of Study\") ; sh:name Literal(\"Object of Study\") ; sh:node this15:ObjectOfStudyOntologyTermShape ; sh:path nsg:objectOfStudy ; skos:editorialNote Literal(\"The object of study \", datatype=xsd:string) ], [ sh:class schema:DataDownload ; sh:description Literal(\"A dataset in downloadable form. The downloadable form has a specific format and is located at a specific location.\") ; sh:minCount Literal(\"1\", datatype=xsd:integer) ; sh:name Literal(\"Distribution\") ; sh:node this9:DistributionShape ; sh:path schema:distribution ], [ sh:description Literal(\"The dataset release date.\") ; sh:maxCount Literal(\"1\", datatype=xsd:integer) ; sh:name Literal(\"Release date\") ; sh:path schema:releaseDate ], [ sh:description Literal(\"The dataset type.\") ; sh:minCount Literal(\"1\", datatype=xsd:integer) ; sh:name Literal(\"Type\") ; sh:nodeKind sh:IRI ; sh:path rdf:type ], [ sh:maxCount Literal(\"1\", datatype=xsd:integer) ; sh:nodeKind sh:IRI ; sh:path nsg:atlasRelease ] ]\n", + "\n", + "Constraint Violation in NodeConstraintComponent (http://www.w3.org/ns/shacl#NodeConstraintComponent):\n", + "\tSeverity: sh:Violation\n", + "\tSource Shape: this:DatasetShape\n", + "\tFocus Node: \n", + "\tValue Node: \n", + "\tMessage: Value does not conform to Shape this1:MINDSShape\n", + "\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + " _validate_one\n", + " False\n", + " ValidationError: \n", + "Validation Report\n", + "Conforms: False\n", + "Results (1):\n", + "Constraint Violation in NodeConstraintComponent (http://www.w3.org/ns/shacl#NodeConstraintComponent):\n", + "\tSeverity: sh:Violation\n", + "\tSource Shape: this:DatasetShape\n", + "\tFocus Node: \n", + "\tValue Node: \n", + "\tMessage: Value does not conform to Shape this1:MINDSShape\n", + "\n" + ] + } + ], + "source": [ + "forge.validate(resource, type_=\"Dataset\", debug=True)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3.7.13 ('kgforge')", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.13" + }, + "vscode": { + "interpreter": { + "hash": "9ac393a5ddd595f2c78ea58b15bf8d269850a4413729cbea5c5fae9013762763" + } + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/kgforge/core/archetypes/model.py b/kgforge/core/archetypes/model.py index 8bc36b53..579b709f 100644 --- a/kgforge/core/archetypes/model.py +++ b/kgforge/core/archetypes/model.py @@ -150,18 +150,18 @@ def schema_id(self, type: str) -> str: not_supported() def validate(self, data: Union[Resource, List[Resource]], - execute_actions_before: bool, type_: str) -> None: + execute_actions_before: bool, type_: str, debug: bool=False) -> None: # Replace None by self._validate_many to switch to optimized bulk validation. run(self._validate_one, None, data, execute_actions=execute_actions_before, - exception=ValidationError, monitored_status="_validated", type_=type_) + exception=ValidationError, monitored_status="_validated", type_=type_, debug=debug) - def _validate_many(self, resources: List[Resource], type_: str) -> None: + def _validate_many(self, resources: List[Resource], type_: str, debug: bool) -> None: # Bulk validation could be optimized by overriding this method in the specialization. # POLICY Should reproduce self._validate_one() and execution._run_one() behaviours. not_supported() @abstractmethod - def _validate_one(self, resource: Resource, type_: str) -> None: + def _validate_one(self, resource: Resource, type_: str, debug: bool) -> None: # POLICY Should notify of failures with exception ValidationError including a message. pass diff --git a/kgforge/core/forge.py b/kgforge/core/forge.py index 756dd332..23a099b4 100644 --- a/kgforge/core/forge.py +++ b/kgforge/core/forge.py @@ -283,8 +283,9 @@ def template( def validate( self, data: Union[Resource, List[Resource]], - execute_actions_before: bool=False, - type_: str=None + execute_actions_before: bool = False, + type_: str=None, + debug: bool = False ) -> None: """ Check if resources conform to their corresponding schemas. This method will try to infer the schema of a resource from its type. @@ -297,7 +298,7 @@ def validate( :param type_: the type to validate the data against it. If None, the validation function will look for a type attribute in the Resource :return: None """ - self._model.validate(data, execute_actions_before, type_=type_) + self._model.validate(data, execute_actions_before, type_, debug) # Resolving User Interface. diff --git a/kgforge/specializations/models/demo_model.py b/kgforge/specializations/models/demo_model.py index 32815986..81d807b1 100644 --- a/kgforge/specializations/models/demo_model.py +++ b/kgforge/specializations/models/demo_model.py @@ -81,7 +81,7 @@ def mapping(self, entity: str, source: str, type: Callable) -> Mapping: # Validation. - def _validate_one(self, resource: Resource, type_: str) -> None: + def _validate_one(self, resource: Resource, type_: str, debug: bool) -> None: """ Validates the model against a given type provided by type_ parameter. If type_ is None then it looks for type attribute in resource. diff --git a/kgforge/specializations/models/rdf/directory_service.py b/kgforge/specializations/models/rdf/directory_service.py index 46bf4598..de4cd1d6 100644 --- a/kgforge/specializations/models/rdf/directory_service.py +++ b/kgforge/specializations/models/rdf/directory_service.py @@ -15,7 +15,7 @@ from pathlib import Path from typing import Dict, Tuple -from pyshacl import Validator +from pyshacl import validate from rdflib import Graph, URIRef from rdflib.util import guess_format @@ -43,9 +43,9 @@ def materialize(self, iri: URIRef) -> NodeProperties: attrs["properties"] = props return NodeProperties(**attrs) - def _validate(self, iri: str, data_graph: Graph) -> Tuple[bool, Graph, str]: - validator = Validator(data_graph, shacl_graph=self._graph) - return validator.run() + def _validate(self, iri: str, data_graph: Graph, debug: bool) -> Tuple[bool, Graph, str]: + result = validate(data_graph, shacl_graph=self._graph, debug=debug) + return result def resolve_context(self, iri: str) -> Dict: if iri in self._context_cache: diff --git a/kgforge/specializations/models/rdf/service.py b/kgforge/specializations/models/rdf/service.py index 6f581968..c1a4dbf8 100644 --- a/kgforge/specializations/models/rdf/service.py +++ b/kgforge/specializations/models/rdf/service.py @@ -155,7 +155,7 @@ def materialize(self, iri: URIRef) -> NodeProperties: """ raise NotImplementedError() - def validate(self, resource: Resource, type_: str): + def validate(self, resource: Resource, type_: str, debug: bool): try: if isinstance(resource.type, list) and type_ is None: raise ValueError("Resource has list of types as attribute and type_ parameter is not specified. " @@ -168,10 +168,10 @@ def validate(self, resource: Resource, type_: str): raise TypeError("resource requires a type attribute") else: data_graph = as_graph(resource, False, self.context, None, None) - return self._validate(shape_iri, data_graph) + return self._validate(shape_iri, data_graph, debug) @abstractmethod - def _validate(self, iri: str, data_graph: Graph) -> Tuple[bool, Graph, str]: + def _validate(self, iri: str, data_graph: Graph, debug: bool) -> Tuple[bool, Graph, str]: raise NotImplementedError() @abstractmethod diff --git a/kgforge/specializations/models/rdf/store_service.py b/kgforge/specializations/models/rdf/store_service.py index 53a2d7d6..a46a02bc 100644 --- a/kgforge/specializations/models/rdf/store_service.py +++ b/kgforge/specializations/models/rdf/store_service.py @@ -57,15 +57,11 @@ def materialize(self, iri: URIRef) -> NodeProperties: attrs["properties"] = props return NodeProperties(**attrs) - def _validate(self, iri: str, data_graph: Graph) -> Tuple[bool, Graph, str]: + def _validate(self, iri: str, data_graph: Graph, debug: bool) -> Tuple[bool, Graph, str]: # _type_shape will make sure all the shapes for this type are in the graph self._type_shape(iri) - # validator = Validator(data_graph, shacl_graph=self._graph) - result = validate(data_graph, shacl_graph=self._graph, debug=True) - conforms, results_graph, results_text = result - print(results_text) + result = validate(data_graph, shacl_graph=self._graph, debug=debug) return result - # return validator.run() def resolve_context(self, iri: str) -> Dict: if iri in self._context_cache: diff --git a/kgforge/specializations/models/rdf_model.py b/kgforge/specializations/models/rdf_model.py index d6edb49f..4a39e59b 100644 --- a/kgforge/specializations/models/rdf_model.py +++ b/kgforge/specializations/models/rdf_model.py @@ -107,13 +107,14 @@ def schema_id(self, type: str) -> str: except KeyError: raise ValueError("type not found") - def validate(self, data: Union[Resource, List[Resource]], execute_actions_before: bool, type_: str) -> None: + def validate(self, data: Union[Resource, List[Resource]], execute_actions_before: bool, type_: str, + debug: bool) -> None: run(self._validate_one, self._validate_many, data, execute_actions=execute_actions_before, - exception=ValidationError, monitored_status="_validated", type_=type_) + exception=ValidationError, monitored_status="_validated", type_=type_, debug=debug) - def _validate_many(self, resources: List[Resource], type_: str) -> None: + def _validate_many(self, resources: List[Resource], type_: str, debug: bool) -> None: for resource in resources: - conforms, graph, _ = self.service.validate(resource, type_=type_) + conforms, graph, _ = self.service.validate(resource, type_=type_, debug=debug) if conforms: resource._validated = True action = Action(self._validate_many.__name__, conforms, None) @@ -125,8 +126,8 @@ def _validate_many(self, resources: List[Resource], type_: str) -> None: action = Action(self._validate_many.__name__, conforms, ValidationError(message)) resource._last_action = action - def _validate_one(self, resource: Resource, type_: str) -> None: - conforms, _, report = self.service.validate(resource, type_) + def _validate_one(self, resource: Resource, type_: str, debug: bool) -> None: + conforms, _, report = self.service.validate(resource, type_=type_, debug=debug) if conforms is False: raise ValidationError("\n" + report) diff --git a/tests/specializations/models/test_demo_model.py b/tests/specializations/models/test_demo_model.py index 43ebbc20..f53885e0 100644 --- a/tests/specializations/models/test_demo_model.py +++ b/tests/specializations/models/test_demo_model.py @@ -44,8 +44,8 @@ def validate(capsys, model, data, rc, err, msg): @when("I validate the resource. An exception is raised. The printed report mentions an error: 'Exception: exception raised'.") def validate_exception(monkeypatch, capsys, model, data): - def _validate_one(_, x, type_: str): raise Exception("exception raised") + def _validate_one(_, x, type_: str, debug: bool): raise Exception("exception raised") monkeypatch.setattr("kgforge.specializations.models.demo_model.DemoModel._validate_one", _validate_one) - model.validate(data, execute_actions_before=False, type_="Person") + model.validate(data, execute_actions_before=False, type_="Person", debug=False) out = capsys.readouterr().out[:-1] assert out == f" _validate_one\n False\n Exception: exception raised" diff --git a/tests/specializations/models/test_rdf_model.py b/tests/specializations/models/test_rdf_model.py index 4546a771..5fc7bee3 100644 --- a/tests/specializations/models/test_rdf_model.py +++ b/tests/specializations/models/test_rdf_model.py @@ -99,20 +99,20 @@ def test_type_to_schema(self, rdf_model: RdfModel, type_): assert rdf_model.schema_id(type_) == TYPES_SCHEMAS_MAP[type_] def test_validate_one(self, rdf_model: RdfModel, valid_activity_resource): - rdf_model.validate(valid_activity_resource, False, type_="Activity") + rdf_model.validate(valid_activity_resource, False, type_="Activity", debug=False) def test_validate_one_fail(self, rdf_model: RdfModel, invalid_activity_resource): with pytest.raises(ValidationError): - rdf_model._validate_one(invalid_activity_resource, type_="Activity") + rdf_model._validate_one(invalid_activity_resource, type_="Activity", debug=False) def test_validate_with_schema(self, rdf_model: RdfModel, valid_activity_resource): - rdf_model.validate(valid_activity_resource, False, type_="Activity") + rdf_model.validate(valid_activity_resource, False, type_="Activity", debug=False) def test_validate_many(self, rdf_model: RdfModel, valid_activity_resource, invalid_activity_resource): resources = [valid_activity_resource, invalid_activity_resource] - rdf_model.validate(resources, False, type_="Activity") + rdf_model.validate(resources, False, type_="Activity", debug=False) assert valid_activity_resource._validated is True assert invalid_activity_resource._validated is False assert (valid_activity_resource._last_action.operation == From 2300e5111564f8a193095275a500e73ebb9251ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cristina=20E=2E=20Gonz=C3=A1lez-Espinoza?= Date: Thu, 17 Nov 2022 14:54:19 +0100 Subject: [PATCH 3/6] Propagate foge._debug value to store methods. Modified _error_message to provide correct response message. --- .../notebooks/use-cases/LocalValidation.ipynb | 173 ++++++++++++------ kgforge/core/archetypes/store.py | 19 +- kgforge/core/forge.py | 15 +- kgforge/specializations/models/rdf_model.py | 4 +- .../specializations/stores/bluebrain_nexus.py | 45 +++-- 5 files changed, 171 insertions(+), 85 deletions(-) diff --git a/examples/notebooks/use-cases/LocalValidation.ipynb b/examples/notebooks/use-cases/LocalValidation.ipynb index 86e91f10..0bb3e2ea 100644 --- a/examples/notebooks/use-cases/LocalValidation.ipynb +++ b/examples/notebooks/use-cases/LocalValidation.ipynb @@ -48,7 +48,8 @@ "forge = KnowledgeGraphForge(\"../use-cases/prod-forge-nexus.yml\",\n", " endpoint=endpoint, \n", " bucket=BUCKET,\n", - " token=TOKEN\n", + " token=TOKEN,\n", + " debug=True\n", " )" ] }, @@ -188,40 +189,33 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 9, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "Constraint Violation in MinCountConstraintComponent (http://www.w3.org/ns/shacl#MinCountConstraintComponent):\n", - "\tSeverity: sh:Violation\n", - "\tSource Shape: [ sh:description Literal(\"When followed this link leads to a resource providing further description on how to download the attached data.\") ; sh:maxCount Literal(\"1\", datatype=xsd:integer) ; sh:minCount Literal(\"1\", datatype=xsd:integer) ; sh:nodeKind sh:IRI ; sh:path schema:url ]\n", - "\tFocus Node: [ [ Literal(\"bytes\") ; Literal(\"2347304\", datatype=xsd:integer) ] ; ; Literal(\"application/nwb\") ; Literal(\"dNAD_ltb.nwb\") ; [ Literal(\"file:///gpfs/bbp.cscs.ch/data/project/proj94/nexus/bbp/uniprot/9/9/e/f/6/1/f/e/dNAD_ltb.nwb\") ; ; rdf:type ] ; [ Literal(\"SHA-256\") ; Literal(\"630f7455b9be5b212566a8dcc76d6a9b2303257ed9199a5c9262fc25aaa5749f\") ] ; rdf:type ]\n", - "\tResult Path: schema:url\n", - "\tMessage: Less than 1 values on [ [ Literal(\"bytes\") ; Literal(\"2347304\", datatype=xsd:integer) ] ; ; Literal(\"application/nwb\") ; Literal(\"dNAD_ltb.nwb\") ; [ Literal(\"file:///gpfs/bbp.cscs.ch/data/project/proj94/nexus/bbp/uniprot/9/9/e/f/6/1/f/e/dNAD_ltb.nwb\") ; ; rdf:type ] ; [ Literal(\"SHA-256\") ; Literal(\"630f7455b9be5b212566a8dcc76d6a9b2303257ed9199a5c9262fc25aaa5749f\") ] ; rdf:type ]->schema:url\n", - "\n", "Constraint Violation in DatatypeConstraintComponent (http://www.w3.org/ns/shacl#DatatypeConstraintComponent):\n", "\tSeverity: sh:Violation\n", - "\tSource Shape: [ sh:datatype xsd:float ]\n", + "\tSource Shape: [ sh:datatype xsd:decimal ]\n", "\tFocus Node: Literal(\"2347304\", datatype=xsd:integer)\n", "\tValue Node: Literal(\"2347304\", datatype=xsd:integer)\n", - "\tMessage: Value is not Literal with datatype xsd:float\n", + "\tMessage: Value is not Literal with datatype xsd:decimal\n", "\n", "Constraint Violation in DatatypeConstraintComponent (http://www.w3.org/ns/shacl#DatatypeConstraintComponent):\n", "\tSeverity: sh:Violation\n", - "\tSource Shape: [ sh:datatype xsd:double ]\n", + "\tSource Shape: [ sh:datatype xsd:unsignedInt ]\n", "\tFocus Node: Literal(\"2347304\", datatype=xsd:integer)\n", "\tValue Node: Literal(\"2347304\", datatype=xsd:integer)\n", - "\tMessage: Value is not Literal with datatype xsd:double\n", + "\tMessage: Value is not Literal with datatype xsd:unsignedInt\n", "\n", "Constraint Violation in DatatypeConstraintComponent (http://www.w3.org/ns/shacl#DatatypeConstraintComponent):\n", "\tSeverity: sh:Violation\n", - "\tSource Shape: [ sh:datatype xsd:unsignedInt ]\n", + "\tSource Shape: [ sh:datatype xsd:double ]\n", "\tFocus Node: Literal(\"2347304\", datatype=xsd:integer)\n", "\tValue Node: Literal(\"2347304\", datatype=xsd:integer)\n", - "\tMessage: Value is not Literal with datatype xsd:unsignedInt\n", + "\tMessage: Value is not Literal with datatype xsd:double\n", "\n", "Constraint Violation in DatatypeConstraintComponent (http://www.w3.org/ns/shacl#DatatypeConstraintComponent):\n", "\tSeverity: sh:Violation\n", @@ -232,24 +226,24 @@ "\n", "Constraint Violation in DatatypeConstraintComponent (http://www.w3.org/ns/shacl#DatatypeConstraintComponent):\n", "\tSeverity: sh:Violation\n", - "\tSource Shape: [ sh:datatype xsd:unsignedLong ]\n", + "\tSource Shape: [ sh:datatype xsd:int ]\n", "\tFocus Node: Literal(\"2347304\", datatype=xsd:integer)\n", "\tValue Node: Literal(\"2347304\", datatype=xsd:integer)\n", - "\tMessage: Value is not Literal with datatype xsd:unsignedLong\n", + "\tMessage: Value is not Literal with datatype xsd:int\n", "\n", "Constraint Violation in DatatypeConstraintComponent (http://www.w3.org/ns/shacl#DatatypeConstraintComponent):\n", "\tSeverity: sh:Violation\n", - "\tSource Shape: [ sh:datatype xsd:decimal ]\n", + "\tSource Shape: [ sh:datatype xsd:unsignedLong ]\n", "\tFocus Node: Literal(\"2347304\", datatype=xsd:integer)\n", "\tValue Node: Literal(\"2347304\", datatype=xsd:integer)\n", - "\tMessage: Value is not Literal with datatype xsd:decimal\n", + "\tMessage: Value is not Literal with datatype xsd:unsignedLong\n", "\n", "Constraint Violation in DatatypeConstraintComponent (http://www.w3.org/ns/shacl#DatatypeConstraintComponent):\n", "\tSeverity: sh:Violation\n", - "\tSource Shape: [ sh:datatype xsd:int ]\n", + "\tSource Shape: [ sh:datatype xsd:float ]\n", "\tFocus Node: Literal(\"2347304\", datatype=xsd:integer)\n", "\tValue Node: Literal(\"2347304\", datatype=xsd:integer)\n", - "\tMessage: Value is not Literal with datatype xsd:int\n", + "\tMessage: Value is not Literal with datatype xsd:float\n", "\n", "Constraint Violation in NodeKindConstraintComponent (http://www.w3.org/ns/shacl#NodeKindConstraintComponent):\n", "\tSeverity: sh:Violation\n", @@ -274,6 +268,13 @@ "\n", "Constraint Violation in MinCountConstraintComponent (http://www.w3.org/ns/shacl#MinCountConstraintComponent):\n", "\tSeverity: sh:Violation\n", + "\tSource Shape: [ sh:description Literal(\"When followed this link leads to a resource providing further description on how to download the attached data.\") ; sh:maxCount Literal(\"1\", datatype=xsd:integer) ; sh:minCount Literal(\"1\", datatype=xsd:integer) ; sh:nodeKind sh:IRI ; sh:path schema:url ]\n", + "\tFocus Node: [ [ Literal(\"bytes\") ; Literal(\"2347304\", datatype=xsd:integer) ] ; ; Literal(\"application/nwb\") ; Literal(\"dNAD_ltb.nwb\") ; [ Literal(\"file:///gpfs/bbp.cscs.ch/data/project/proj94/nexus/bbp/uniprot/9/9/e/f/6/1/f/e/dNAD_ltb.nwb\") ; ; rdf:type ] ; [ Literal(\"SHA-256\") ; Literal(\"630f7455b9be5b212566a8dcc76d6a9b2303257ed9199a5c9262fc25aaa5749f\") ] ; rdf:type ]\n", + "\tResult Path: schema:url\n", + "\tMessage: Less than 1 values on [ [ Literal(\"bytes\") ; Literal(\"2347304\", datatype=xsd:integer) ] ; ; Literal(\"application/nwb\") ; Literal(\"dNAD_ltb.nwb\") ; [ Literal(\"file:///gpfs/bbp.cscs.ch/data/project/proj94/nexus/bbp/uniprot/9/9/e/f/6/1/f/e/dNAD_ltb.nwb\") ; ; rdf:type ] ; [ Literal(\"SHA-256\") ; Literal(\"630f7455b9be5b212566a8dcc76d6a9b2303257ed9199a5c9262fc25aaa5749f\") ] ; rdf:type ]->schema:url\n", + "\n", + "Constraint Violation in MinCountConstraintComponent (http://www.w3.org/ns/shacl#MinCountConstraintComponent):\n", + "\tSeverity: sh:Violation\n", "\tSource Shape: [ sh:datatype xsd:string ; sh:description Literal(\"The entity name.\") ; sh:maxCount Literal(\"1\", datatype=xsd:integer) ; sh:minCount Literal(\"1\", datatype=xsd:integer) ; sh:name Literal(\"Name\") ; sh:path schema:name ]\n", "\tFocus Node: \n", "\tResult Path: schema:name\n", @@ -286,33 +287,26 @@ "\tValue Node: \n", "\tMessage: Value does not conform to Shape this3:EntityShape\n", "\n", - "Constraint Violation in MinCountConstraintComponent (http://www.w3.org/ns/shacl#MinCountConstraintComponent):\n", - "\tSeverity: sh:Violation\n", - "\tSource Shape: [ sh:description Literal(\"When followed this link leads to a resource providing further description on how to download the attached data.\") ; sh:maxCount Literal(\"1\", datatype=xsd:integer) ; sh:minCount Literal(\"1\", datatype=xsd:integer) ; sh:nodeKind sh:IRI ; sh:path schema:url ]\n", - "\tFocus Node: [ [ Literal(\"bytes\") ; Literal(\"2347304\", datatype=xsd:integer) ] ; ; Literal(\"application/nwb\") ; Literal(\"dNAD_ltb.nwb\") ; [ Literal(\"file:///gpfs/bbp.cscs.ch/data/project/proj94/nexus/bbp/uniprot/9/9/e/f/6/1/f/e/dNAD_ltb.nwb\") ; ; rdf:type ] ; [ Literal(\"SHA-256\") ; Literal(\"630f7455b9be5b212566a8dcc76d6a9b2303257ed9199a5c9262fc25aaa5749f\") ] ; rdf:type ]\n", - "\tResult Path: schema:url\n", - "\tMessage: Less than 1 values on [ [ Literal(\"bytes\") ; Literal(\"2347304\", datatype=xsd:integer) ] ; ; Literal(\"application/nwb\") ; Literal(\"dNAD_ltb.nwb\") ; [ Literal(\"file:///gpfs/bbp.cscs.ch/data/project/proj94/nexus/bbp/uniprot/9/9/e/f/6/1/f/e/dNAD_ltb.nwb\") ; ; rdf:type ] ; [ Literal(\"SHA-256\") ; Literal(\"630f7455b9be5b212566a8dcc76d6a9b2303257ed9199a5c9262fc25aaa5749f\") ] ; rdf:type ]->schema:url\n", - "\n", "Constraint Violation in DatatypeConstraintComponent (http://www.w3.org/ns/shacl#DatatypeConstraintComponent):\n", "\tSeverity: sh:Violation\n", - "\tSource Shape: [ sh:datatype xsd:float ]\n", + "\tSource Shape: [ sh:datatype xsd:decimal ]\n", "\tFocus Node: Literal(\"2347304\", datatype=xsd:integer)\n", "\tValue Node: Literal(\"2347304\", datatype=xsd:integer)\n", - "\tMessage: Value is not Literal with datatype xsd:float\n", + "\tMessage: Value is not Literal with datatype xsd:decimal\n", "\n", "Constraint Violation in DatatypeConstraintComponent (http://www.w3.org/ns/shacl#DatatypeConstraintComponent):\n", "\tSeverity: sh:Violation\n", - "\tSource Shape: [ sh:datatype xsd:double ]\n", + "\tSource Shape: [ sh:datatype xsd:unsignedInt ]\n", "\tFocus Node: Literal(\"2347304\", datatype=xsd:integer)\n", "\tValue Node: Literal(\"2347304\", datatype=xsd:integer)\n", - "\tMessage: Value is not Literal with datatype xsd:double\n", + "\tMessage: Value is not Literal with datatype xsd:unsignedInt\n", "\n", "Constraint Violation in DatatypeConstraintComponent (http://www.w3.org/ns/shacl#DatatypeConstraintComponent):\n", "\tSeverity: sh:Violation\n", - "\tSource Shape: [ sh:datatype xsd:unsignedInt ]\n", + "\tSource Shape: [ sh:datatype xsd:double ]\n", "\tFocus Node: Literal(\"2347304\", datatype=xsd:integer)\n", "\tValue Node: Literal(\"2347304\", datatype=xsd:integer)\n", - "\tMessage: Value is not Literal with datatype xsd:unsignedInt\n", + "\tMessage: Value is not Literal with datatype xsd:double\n", "\n", "Constraint Violation in DatatypeConstraintComponent (http://www.w3.org/ns/shacl#DatatypeConstraintComponent):\n", "\tSeverity: sh:Violation\n", @@ -323,24 +317,24 @@ "\n", "Constraint Violation in DatatypeConstraintComponent (http://www.w3.org/ns/shacl#DatatypeConstraintComponent):\n", "\tSeverity: sh:Violation\n", - "\tSource Shape: [ sh:datatype xsd:unsignedLong ]\n", + "\tSource Shape: [ sh:datatype xsd:int ]\n", "\tFocus Node: Literal(\"2347304\", datatype=xsd:integer)\n", "\tValue Node: Literal(\"2347304\", datatype=xsd:integer)\n", - "\tMessage: Value is not Literal with datatype xsd:unsignedLong\n", + "\tMessage: Value is not Literal with datatype xsd:int\n", "\n", "Constraint Violation in DatatypeConstraintComponent (http://www.w3.org/ns/shacl#DatatypeConstraintComponent):\n", "\tSeverity: sh:Violation\n", - "\tSource Shape: [ sh:datatype xsd:decimal ]\n", + "\tSource Shape: [ sh:datatype xsd:unsignedLong ]\n", "\tFocus Node: Literal(\"2347304\", datatype=xsd:integer)\n", "\tValue Node: Literal(\"2347304\", datatype=xsd:integer)\n", - "\tMessage: Value is not Literal with datatype xsd:decimal\n", + "\tMessage: Value is not Literal with datatype xsd:unsignedLong\n", "\n", "Constraint Violation in DatatypeConstraintComponent (http://www.w3.org/ns/shacl#DatatypeConstraintComponent):\n", "\tSeverity: sh:Violation\n", - "\tSource Shape: [ sh:datatype xsd:int ]\n", + "\tSource Shape: [ sh:datatype xsd:float ]\n", "\tFocus Node: Literal(\"2347304\", datatype=xsd:integer)\n", "\tValue Node: Literal(\"2347304\", datatype=xsd:integer)\n", - "\tMessage: Value is not Literal with datatype xsd:int\n", + "\tMessage: Value is not Literal with datatype xsd:float\n", "\n", "Constraint Violation in NodeKindConstraintComponent (http://www.w3.org/ns/shacl#NodeKindConstraintComponent):\n", "\tSeverity: sh:Violation\n", @@ -363,6 +357,13 @@ "\tValue Node: Literal(\"bytes\")\n", "\tMessage: Value does not conform to Shape this11:LabeledOntologyEntityShape\n", "\n", + "Constraint Violation in MinCountConstraintComponent (http://www.w3.org/ns/shacl#MinCountConstraintComponent):\n", + "\tSeverity: sh:Violation\n", + "\tSource Shape: [ sh:description Literal(\"When followed this link leads to a resource providing further description on how to download the attached data.\") ; sh:maxCount Literal(\"1\", datatype=xsd:integer) ; sh:minCount Literal(\"1\", datatype=xsd:integer) ; sh:nodeKind sh:IRI ; sh:path schema:url ]\n", + "\tFocus Node: [ [ Literal(\"bytes\") ; Literal(\"2347304\", datatype=xsd:integer) ] ; ; Literal(\"application/nwb\") ; Literal(\"dNAD_ltb.nwb\") ; [ Literal(\"file:///gpfs/bbp.cscs.ch/data/project/proj94/nexus/bbp/uniprot/9/9/e/f/6/1/f/e/dNAD_ltb.nwb\") ; ; rdf:type ] ; [ Literal(\"SHA-256\") ; Literal(\"630f7455b9be5b212566a8dcc76d6a9b2303257ed9199a5c9262fc25aaa5749f\") ] ; rdf:type ]\n", + "\tResult Path: schema:url\n", + "\tMessage: Less than 1 values on [ [ Literal(\"bytes\") ; Literal(\"2347304\", datatype=xsd:integer) ] ; ; Literal(\"application/nwb\") ; Literal(\"dNAD_ltb.nwb\") ; [ Literal(\"file:///gpfs/bbp.cscs.ch/data/project/proj94/nexus/bbp/uniprot/9/9/e/f/6/1/f/e/dNAD_ltb.nwb\") ; ; rdf:type ] ; [ Literal(\"SHA-256\") ; Literal(\"630f7455b9be5b212566a8dcc76d6a9b2303257ed9199a5c9262fc25aaa5749f\") ] ; rdf:type ]->schema:url\n", + "\n", "Constraint Violation in AndConstraintComponent (http://www.w3.org/ns/shacl#AndConstraintComponent):\n", "\tSeverity: sh:Violation\n", "\tSource Shape: this1:MINDSShape\n", @@ -379,29 +380,93 @@ "\n" ] }, + { + "ename": "ValidationError", + "evalue": "\nValidation Report\nConforms: False\nResults (1):\nConstraint Violation in NodeConstraintComponent (http://www.w3.org/ns/shacl#NodeConstraintComponent):\n\tSeverity: sh:Violation\n\tSource Shape: this:DatasetShape\n\tFocus Node: \n\tValue Node: \n\tMessage: Value does not conform to Shape this1:MINDSShape\n", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mValidationError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m/var/folders/6p/k45nvhcs7v1_n9bd5g67wc3ctt8hpq/T/ipykernel_79079/2691144165.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mforge\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalidate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresource\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtype_\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"Dataset\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;32m~/Documents/code/nexus-forge/kgforge/core/forge.py\u001b[0m in \u001b[0;36mvalidate\u001b[0;34m(self, data, execute_actions_before, type_)\u001b[0m\n\u001b[1;32m 299\u001b[0m \"\"\"\n\u001b[1;32m 300\u001b[0m \u001b[0mdebug\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_debug\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 301\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_model\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalidate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexecute_actions_before\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtype_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdebug\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 302\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 303\u001b[0m \u001b[0;31m# Resolving User Interface.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/Documents/code/nexus-forge/kgforge/specializations/models/rdf_model.py\u001b[0m in \u001b[0;36mvalidate\u001b[0;34m(self, data, execute_actions_before, type_, debug)\u001b[0m\n\u001b[1;32m 113\u001b[0m run(self._validate_one, self._validate_many, data, execute_actions=execute_actions_before,\n\u001b[1;32m 114\u001b[0m \u001b[0mexception\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mValidationError\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmonitored_status\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"_validated\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcatch_exceptions\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcatch_exceptions\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 115\u001b[0;31m type_=type_, debug=debug)\n\u001b[0m\u001b[1;32m 116\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 117\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_validate_many\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresources\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mList\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mResource\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtype_\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdebug\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mbool\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/Documents/code/nexus-forge/kgforge/core/commons/execution.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(fun_one, fun_many, data, exception, id_required, required_synchronized, execute_actions, monitored_status, catch_exceptions, **kwargs)\u001b[0m\n\u001b[1;32m 113\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mResource\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 114\u001b[0m _run_one(fun_one, data, exception, id_required, required_synchronized, execute_actions,\n\u001b[0;32m--> 115\u001b[0;31m monitored_status, catch_exceptions, **kwargs)\n\u001b[0m\u001b[1;32m 116\u001b[0m \u001b[0maction\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_last_action\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 117\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maction\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/Documents/code/nexus-forge/kgforge/core/commons/execution.py\u001b[0m in \u001b[0;36m_run_one\u001b[0;34m(fun, resource, exception, id_required, required_synchronized, execute_actions, monitored_status, catch_exceptions, **kwargs)\u001b[0m\n\u001b[1;32m 159\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 160\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mcatch_exceptions\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mexception\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 161\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mexception\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;32m~/Documents/code/nexus-forge/kgforge/core/commons/execution.py\u001b[0m in \u001b[0;36m_run_one\u001b[0;34m(fun, resource, exception, id_required, required_synchronized, execute_actions, monitored_status, catch_exceptions, **kwargs)\u001b[0m\n\u001b[1;32m 143\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mexception\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"resource has lazy actions which need to be executed before\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 144\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 145\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresource\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 146\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 147\u001b[0m \u001b[0mstatus\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/Documents/code/nexus-forge/kgforge/specializations/models/rdf_model.py\u001b[0m in \u001b[0;36m_validate_one\u001b[0;34m(self, resource, type_, debug)\u001b[0m\n\u001b[1;32m 132\u001b[0m \u001b[0mconforms\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreport\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mservice\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalidate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresource\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtype_\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtype_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdebug\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdebug\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 133\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mconforms\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 134\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mValidationError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"\\n\"\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mreport\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 135\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 136\u001b[0m \u001b[0;31m# Utils.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mValidationError\u001b[0m: \nValidation Report\nConforms: False\nResults (1):\nConstraint Violation in NodeConstraintComponent (http://www.w3.org/ns/shacl#NodeConstraintComponent):\n\tSeverity: sh:Violation\n\tSource Shape: this:DatasetShape\n\tFocus Node: \n\tValue Node: \n\tMessage: Value does not conform to Shape this1:MINDSShape\n" + ] + } + ], + "source": [ + "forge.validate(resource, type_=\"Dataset\")" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "forge._debug" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - " _validate_one\n", - " False\n", - " ValidationError: \n", - "Validation Report\n", - "Conforms: False\n", - "Results (1):\n", - "Constraint Violation in NodeConstraintComponent (http://www.w3.org/ns/shacl#NodeConstraintComponent):\n", - "\tSeverity: sh:Violation\n", - "\tSource Shape: this:DatasetShape\n", - "\tFocus Node: \n", - "\tValue Node: \n", - "\tMessage: Value does not conform to Shape this1:MINDSShape\n", - "\n" + "catch_exceptions False\n" + ] + }, + { + "ename": "RegistrationError", + "evalue": "Resource 'https://bbp.epfl.ch/neurosciencegraph/data/traces/17b443b3-07fc-41c7-80c8-ea5e54662210' failed to validate against the constraints defined in schema 'https://neuroshapes.org/dash/dataset'. {\"@type\": \"sh:ValidationReport\", \"conforms\": false, \"result\": {\"@type\": \"sh:ValidationResult\", \"detail\": [{\"@type\": \"sh:ValidationResult\", \"focusNode\": \"https://bbp.epfl.ch/neurosciencegraph/data/traces/17b443b3-07fc-41c7-80c8-ea5e54662210\", \"resultMessage\": \"Value does not have all the shapes in the sh:and enumeration\", \"resultSeverity\": \"sh:Violation\", \"sourceConstraintComponent\": \"sh:AndConstraintComponent\", \"sourceShape\": \"https://neuroshapes.org/commons/minds/shapes/MINDSShape\", \"value\": \"https://bbp.epfl.ch/neurosciencegraph/data/traces/17b443b3-07fc-41c7-80c8-ea5e54662210\"}], \"focusNode\": \"https://bbp.epfl.ch/neurosciencegraph/data/traces/17b443b3-07fc-41c7-80c8-ea5e54662210\", \"resultMessage\": \"Value does not have shape \", \"resultSeverity\": \"sh:Violation\", \"sourceConstraintComponent\": \"sh:NodeConstraintComponent\", \"sourceShape\": \"https://neuroshapes.org/dash/dataset/shapes/DatasetShape\", \"value\": \"https://bbp.epfl.ch/neurosciencegraph/data/traces/17b443b3-07fc-41c7-80c8-ea5e54662210\"}, \"targetedNodes\": 2}", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mHTTPError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m~/Documents/code/nexus-forge/kgforge/specializations/stores/bluebrain_nexus.py\u001b[0m in \u001b[0;36m_register_one\u001b[0;34m(self, resource, schema_id)\u001b[0m\n\u001b[1;32m 226\u001b[0m )\n\u001b[0;32m--> 227\u001b[0;31m \u001b[0mresponse\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mraise_for_status\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 228\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/.local/lib/python3.7/site-packages/requests/models.py\u001b[0m in \u001b[0;36mraise_for_status\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 942\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mhttp_error_msg\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 943\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mHTTPError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhttp_error_msg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 944\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mHTTPError\u001b[0m: 400 Client Error: Bad Request for url: https://staging.nise.bbp.epfl.ch/nexus/v1/resources/dke/kgforge/datashapes%3Adataset", + "\nDuring handling of the above exception, another exception occurred:\n", + "\u001b[0;31mRegistrationError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m/var/folders/6p/k45nvhcs7v1_n9bd5g67wc3ctt8hpq/T/ipykernel_79079/2755330261.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mforge\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mregister\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresource\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mschema_id\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"datashapes:dataset\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;32m~/Documents/code/nexus-forge/kgforge/core/forge.py\u001b[0m in \u001b[0;36mregister\u001b[0;34m(self, data, schema_id)\u001b[0m\n\u001b[1;32m 643\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0mparam\u001b[0m \u001b[0mschema_id\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0man\u001b[0m \u001b[0midentifier\u001b[0m \u001b[0mof\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mschema\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mregistered\u001b[0m \u001b[0mresources\u001b[0m \u001b[0mshould\u001b[0m \u001b[0mconform\u001b[0m \u001b[0mto\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 644\u001b[0m \"\"\"\n\u001b[0;32m--> 645\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_store\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mregister\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mschema_id\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdebug\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_debug\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 646\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 647\u001b[0m \u001b[0;31m# No @catch because the error handling is done by execution.run().\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/Documents/code/nexus-forge/kgforge/specializations/stores/bluebrain_nexus.py\u001b[0m in \u001b[0;36mregister\u001b[0;34m(self, data, schema_id, debug)\u001b[0m\n\u001b[1;32m 150\u001b[0m \u001b[0mcatch_exceptions\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcatch_exceptions\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 151\u001b[0m \u001b[0mmonitored_status\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"_synchronized\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 152\u001b[0;31m \u001b[0mschema_id\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mschema_id\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 153\u001b[0m )\n\u001b[1;32m 154\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/Documents/code/nexus-forge/kgforge/core/commons/execution.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(fun_one, fun_many, data, exception, id_required, required_synchronized, execute_actions, monitored_status, catch_exceptions, **kwargs)\u001b[0m\n\u001b[1;32m 113\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mResource\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 114\u001b[0m _run_one(fun_one, data, exception, id_required, required_synchronized, execute_actions,\n\u001b[0;32m--> 115\u001b[0;31m monitored_status, catch_exceptions, **kwargs)\n\u001b[0m\u001b[1;32m 116\u001b[0m \u001b[0maction\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_last_action\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 117\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maction\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/Documents/code/nexus-forge/kgforge/core/commons/execution.py\u001b[0m in \u001b[0;36m_run_one\u001b[0;34m(fun, resource, exception, id_required, required_synchronized, execute_actions, monitored_status, catch_exceptions, **kwargs)\u001b[0m\n\u001b[1;32m 159\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 160\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mcatch_exceptions\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mexception\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 161\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mexception\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;32m~/Documents/code/nexus-forge/kgforge/core/commons/execution.py\u001b[0m in \u001b[0;36m_run_one\u001b[0;34m(fun, resource, exception, id_required, required_synchronized, execute_actions, monitored_status, catch_exceptions, **kwargs)\u001b[0m\n\u001b[1;32m 143\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mexception\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"resource has lazy actions which need to be executed before\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 144\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 145\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresource\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 146\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 147\u001b[0m \u001b[0mstatus\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/Documents/code/nexus-forge/kgforge/specializations/stores/bluebrain_nexus.py\u001b[0m in \u001b[0;36m_register_one\u001b[0;34m(self, resource, schema_id)\u001b[0m\n\u001b[1;32m 228\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 229\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mnexus\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mHTTPError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 230\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mRegistrationError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_error_message\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 231\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 232\u001b[0m \u001b[0mresponse_json\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjson\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mRegistrationError\u001b[0m: Resource 'https://bbp.epfl.ch/neurosciencegraph/data/traces/17b443b3-07fc-41c7-80c8-ea5e54662210' failed to validate against the constraints defined in schema 'https://neuroshapes.org/dash/dataset'. {\"@type\": \"sh:ValidationReport\", \"conforms\": false, \"result\": {\"@type\": \"sh:ValidationResult\", \"detail\": [{\"@type\": \"sh:ValidationResult\", \"focusNode\": \"https://bbp.epfl.ch/neurosciencegraph/data/traces/17b443b3-07fc-41c7-80c8-ea5e54662210\", \"resultMessage\": \"Value does not have all the shapes in the sh:and enumeration\", \"resultSeverity\": \"sh:Violation\", \"sourceConstraintComponent\": \"sh:AndConstraintComponent\", \"sourceShape\": \"https://neuroshapes.org/commons/minds/shapes/MINDSShape\", \"value\": \"https://bbp.epfl.ch/neurosciencegraph/data/traces/17b443b3-07fc-41c7-80c8-ea5e54662210\"}], \"focusNode\": \"https://bbp.epfl.ch/neurosciencegraph/data/traces/17b443b3-07fc-41c7-80c8-ea5e54662210\", \"resultMessage\": \"Value does not have shape \", \"resultSeverity\": \"sh:Violation\", \"sourceConstraintComponent\": \"sh:NodeConstraintComponent\", \"sourceShape\": \"https://neuroshapes.org/dash/dataset/shapes/DatasetShape\", \"value\": \"https://bbp.epfl.ch/neurosciencegraph/data/traces/17b443b3-07fc-41c7-80c8-ea5e54662210\"}, \"targetedNodes\": 2}" ] } ], "source": [ - "forge.validate(resource, type_=\"Dataset\", debug=True)" + "forge.register(resource, schema_id=\"datashapes:dataset\")" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/kgforge/core/archetypes/store.py b/kgforge/core/archetypes/store.py index ae26cb3d..044555fa 100644 --- a/kgforge/core/archetypes/store.py +++ b/kgforge/core/archetypes/store.py @@ -147,15 +147,17 @@ def mapper(self) -> Optional[Callable]: # [C]RUD. def register( - self, data: Union[Resource, List[Resource]], schema_id: str = None + self, data: Union[Resource, List[Resource]], schema_id: str = None, debug: bool = False ) -> None: # Replace None by self._register_many to switch to optimized bulk registration. + catch_exceptions = False if debug else True run( self._register_one, None, data, required_synchronized=False, execute_actions=True, + catch_exceptions=catch_exceptions, exception=RegistrationError, monitored_status="_synchronized", schema_id=schema_id, @@ -285,9 +287,10 @@ def _download_one( # CR[U]D. def update( - self, data: Union[Resource, List[Resource]], schema_id: Optional[str] + self, data: Union[Resource, List[Resource]], schema_id: Optional[str], debug: bool = False ) -> None: # Replace None by self._update_many to switch to optimized bulk update. + catch_exceptions = False if debug else True run( self._update_one, None, @@ -312,15 +315,17 @@ def _update_one(self, resource: Resource, schema_id: Optional[str]) -> None: # TODO This operation might be abstracted here when other stores will be implemented. pass - def tag(self, data: Union[Resource, List[Resource]], value: str) -> None: + def tag(self, data: Union[Resource, List[Resource]], value: str, debug: bool = True) -> None: # Replace None by self._tag_many to switch to optimized bulk tagging. # POLICY If tagging modify the resource, run() should have status='_synchronized'. + catch_exceptions = False if debug else True run( self._tag_one, None, data, id_required=True, required_synchronized=True, + catch_exceptions=catch_exceptions, exception=TaggingError, value=value, ) @@ -338,14 +343,16 @@ def _tag_one(self, resource: Resource, value: str) -> None: # CRU[D]. - def deprecate(self, data: Union[Resource, List[Resource]]) -> None: + def deprecate(self, data: Union[Resource, List[Resource]], debug: bool = False) -> None: # Replace None by self._deprecate_many to switch to optimized bulk deprecation. + catch_exceptions = False if debug else True run( self._deprecate_one, None, data, id_required=True, required_synchronized=True, + catch_exceptions=catch_exceptions, exception=DeprecationError, monitored_status="_synchronized", ) @@ -430,14 +437,16 @@ def _elastic(self, query: str) -> List[Resource]: # Versioning. - def freeze(self, data: Union[Resource, List[Resource]]) -> None: + def freeze(self, data: Union[Resource, List[Resource]], debug: bool = False) -> None: # Replace None by self._freeze_many to switch to optimized bulk freezing. + catch_exceptions = False if debug else True run( self._freeze_one, None, data, id_required=True, required_synchronized=True, + catch_exceptions=catch_exceptions, exception=FreezingError, ) diff --git a/kgforge/core/forge.py b/kgforge/core/forge.py index 23a099b4..8fc0b4c3 100644 --- a/kgforge/core/forge.py +++ b/kgforge/core/forge.py @@ -284,8 +284,7 @@ def validate( self, data: Union[Resource, List[Resource]], execute_actions_before: bool = False, - type_: str=None, - debug: bool = False + type_: str=None ) -> None: """ Check if resources conform to their corresponding schemas. This method will try to infer the schema of a resource from its type. @@ -298,6 +297,7 @@ def validate( :param type_: the type to validate the data against it. If None, the validation function will look for a type attribute in the Resource :return: None """ + debug = self._debug self._model.validate(data, execute_actions_before, type_, debug) # Resolving User Interface. @@ -642,7 +642,7 @@ def register( :param data: the resources to register :param schema_id: an identifier of the schema the registered resources should conform to """ - self._store.register(data, schema_id) + self._store.register(data, schema_id, debug=self._debug) # No @catch because the error handling is done by execution.run(). def update( @@ -654,7 +654,7 @@ def update( :param data: the resources to update :param schema_id: an identifier of the schema the updated resources should conform to """ - self._store.update(data, schema_id) + self._store.update(data, schema_id, debug=self._debug) # No @catch because the error handling is done by execution.run(). def deprecate(self, data: Union[Resource, List[Resource]]) -> None: @@ -663,7 +663,7 @@ def deprecate(self, data: Union[Resource, List[Resource]]) -> None: :param: the resources to deprecate """ - self._store.deprecate(data) + self._store.deprecate(data, debug=self._debug) # Versioning User Interface. @@ -675,7 +675,7 @@ def tag(self, data: Union[Resource, List[Resource]], value: str) -> None: :param data: the resources to tag :param value: the tag value """ - self._store.tag(data, value) + self._store.tag(data, value, debug=self._debug) # No @catch because the error handling is done by execution.run(). def freeze(self, data: Union[Resource, List[Resource]]) -> None: @@ -685,7 +685,7 @@ def freeze(self, data: Union[Resource, List[Resource]]) -> None: :param data: the resources to freeze """ - self._store.freeze(data) + self._store.freeze(data, debug=self._debug) # Files Handling User Interface. @@ -728,7 +728,6 @@ def as_json( self._model.resolve_context, ) - @catch @catch def as_jsonld( self, diff --git a/kgforge/specializations/models/rdf_model.py b/kgforge/specializations/models/rdf_model.py index 4a39e59b..0b6df65f 100644 --- a/kgforge/specializations/models/rdf_model.py +++ b/kgforge/specializations/models/rdf_model.py @@ -109,8 +109,10 @@ def schema_id(self, type: str) -> str: def validate(self, data: Union[Resource, List[Resource]], execute_actions_before: bool, type_: str, debug: bool) -> None: + catch_exceptions = False if debug else True run(self._validate_one, self._validate_many, data, execute_actions=execute_actions_before, - exception=ValidationError, monitored_status="_validated", type_=type_, debug=debug) + exception=ValidationError, monitored_status="_validated", catch_exceptions=catch_exceptions, + type_=type_, debug=debug) def _validate_many(self, resources: List[Resource], type_: str, debug: bool) -> None: for resource in resources: diff --git a/kgforge/specializations/stores/bluebrain_nexus.py b/kgforge/specializations/stores/bluebrain_nexus.py index 7ea391f6..cd54e094 100644 --- a/kgforge/specializations/stores/bluebrain_nexus.py +++ b/kgforge/specializations/stores/bluebrain_nexus.py @@ -37,7 +37,7 @@ from rdflib import Graph from rdflib.plugins.sparql.parser import Query from datetime import datetime -from requests import HTTPError +from requests import HTTPError, Response from kgforge.core import Resource from kgforge.core.archetypes import Store @@ -136,8 +136,9 @@ def mapper(self) -> Optional[Callable]: return DictionaryMapper def register( - self, data: Union[Resource, List[Resource]], schema_id: str = None + self, data: Union[Resource, List[Resource]], schema_id: str = None, debug: bool = False ) -> None: + catch_exceptions = False if debug else True run( self._register_one, self._register_many, @@ -145,6 +146,7 @@ def register( required_synchronized=False, execute_actions=True, exception=RegistrationError, + catch_exceptions=catch_exceptions, monitored_status="_synchronized", schema_id=schema_id, ) @@ -551,13 +553,15 @@ def _update_one(self, resource: Resource, schema_id: str) -> None: else: self.service.sync_metadata(resource, response.json()) - def tag(self, data: Union[Resource, List[Resource]], value: str) -> None: + def tag(self, data: Union[Resource, List[Resource]], value: str, debug: bool = False) -> None: + catch_exceptions = False if debug else True run( self._tag_one, self._tag_many, data, id_required=True, required_synchronized=True, + catch_exceptions=catch_exceptions, exception=TaggingError, value=value, ) @@ -601,13 +605,15 @@ def _tag_one(self, resource: Resource, value: str) -> None: # CRU[D]. - def deprecate(self, data: Union[Resource, List[Resource]]) -> None: + def deprecate(self, data: Union[Resource, List[Resource]], debug: bool = False) -> None: + catch_exceptions = False if debug else True run( self._deprecate_one, self._deprecate_many, data, id_required=True, required_synchronized=True, + catch_exceptions=catch_exceptions, exception=DeprecationError, monitored_status="_synchronized", ) @@ -980,21 +986,13 @@ def _initialize_service( ) -def _error_message(error: HTTPError) -> str: - def format_message(msg): - return "".join([msg[0].lower(), msg[1:-1], msg[-1] if msg[-1] != "." else ""]) +def format_message(msg): + return "".join([msg[0].lower(), msg[1:-1], msg[-1] if msg[-1] != "." else ""]) +def _error_message(error: HTTPError) -> str: try: - error_json = error.response.json() - messages = [] - reason = error_json.get("reason", None) - details = error_json.get("details", None) - if reason: - messages.append(format_message(reason)) - if details: - messages.append(format_message(details)) - messages = messages if reason or details else [str(error)] - return ". ".join(messages) + error_response = error.response + return _error_from_response(error_response) except Exception as e: pass try: @@ -1002,6 +1000,19 @@ def format_message(msg): except Exception: return format_message(str(error)) +def _error_from_response(response: Response) -> str: + messages = [] + error_json = response.json() + reason = error_json.get("reason", None) + details = error_json.get("details", None) + if reason: + messages.append(reason) + if details: + if isinstance(details, dict): + messages.append(json.dumps(details)) + else: + messages.append(str(details)) + return ". ".join(messages) def build_sparql_query_statements(context: Context, *conditions) -> Tuple[List, List]: statements = list() From b27f0bf2ea920006fa9f8e41c5f8d080b0e9a3c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cristina=20E=2E=20Gonz=C3=A1lez-Espinoza?= Date: Tue, 22 Nov 2022 08:55:05 +0100 Subject: [PATCH 4/6] Fixed error message when registering, and expose a method to execute lazy actions. --- kgforge/core/archetypes/store.py | 6 ++-- kgforge/core/forge.py | 15 +++++++++ .../specializations/stores/bluebrain_nexus.py | 31 ++++++++++++------- 3 files changed, 38 insertions(+), 14 deletions(-) diff --git a/kgforge/core/archetypes/store.py b/kgforge/core/archetypes/store.py index 044555fa..b40b24a0 100644 --- a/kgforge/core/archetypes/store.py +++ b/kgforge/core/archetypes/store.py @@ -147,17 +147,17 @@ def mapper(self) -> Optional[Callable]: # [C]RUD. def register( - self, data: Union[Resource, List[Resource]], schema_id: str = None, debug: bool = False + self, data: Union[Resource, List[Resource]], schema_id: str = None, + debug: bool = False ) -> None: # Replace None by self._register_many to switch to optimized bulk registration. - catch_exceptions = False if debug else True run( self._register_one, None, data, required_synchronized=False, execute_actions=True, - catch_exceptions=catch_exceptions, + catch_exceptions=not debug, exception=RegistrationError, monitored_status="_synchronized", schema_id=schema_id, diff --git a/kgforge/core/forge.py b/kgforge/core/forge.py index 8fc0b4c3..2f58737f 100644 --- a/kgforge/core/forge.py +++ b/kgforge/core/forge.py @@ -28,6 +28,8 @@ from kgforge.core.commons.dictionaries import with_defaults from kgforge.core.commons.exceptions import ResolvingError from kgforge.core.commons.execution import catch +from kgforge.core.commons.actions import (collect_lazy_actions, + execute_lazy_actions) from kgforge.core.commons.imports import import_class from kgforge.core.commons.strategies import ResolvingStrategy from kgforge.core.conversions.dataframe import as_dataframe, from_dataframe @@ -608,6 +610,19 @@ def elastic( :return: List[Resource] """ return self._store.elastic(query, debug, limit, offset) + + @catch + @staticmethod + def execute_lazy_actions(resources: Union[List[Resource], Resource]): + """ + Execute any lazy action present in a given resource or a list of resources + + :param resources: The resources or list of resources from which actions will be executed + """ + resources = [resources] if not isinstance(resources, List) else resources + for resource in resources: + lazy_actions = collect_lazy_actions(resource) + execute_lazy_actions(resource, lazy_actions) @catch def download( diff --git a/kgforge/specializations/stores/bluebrain_nexus.py b/kgforge/specializations/stores/bluebrain_nexus.py index cd54e094..3ab80e8a 100644 --- a/kgforge/specializations/stores/bluebrain_nexus.py +++ b/kgforge/specializations/stores/bluebrain_nexus.py @@ -136,9 +136,9 @@ def mapper(self) -> Optional[Callable]: return DictionaryMapper def register( - self, data: Union[Resource, List[Resource]], schema_id: str = None, debug: bool = False + self, data: Union[Resource, List[Resource]], schema_id: str = None, + debug: bool = False ) -> None: - catch_exceptions = False if debug else True run( self._register_one, self._register_many, @@ -146,7 +146,7 @@ def register( required_synchronized=False, execute_actions=True, exception=RegistrationError, - catch_exceptions=catch_exceptions, + catch_exceptions=not debug, monitored_status="_synchronized", schema_id=schema_id, ) @@ -554,14 +554,13 @@ def _update_one(self, resource: Resource, schema_id: str) -> None: self.service.sync_metadata(resource, response.json()) def tag(self, data: Union[Resource, List[Resource]], value: str, debug: bool = False) -> None: - catch_exceptions = False if debug else True run( self._tag_one, self._tag_many, data, id_required=True, required_synchronized=True, - catch_exceptions=catch_exceptions, + catch_exceptions=not debug, exception=TaggingError, value=value, ) @@ -606,14 +605,13 @@ def _tag_one(self, resource: Resource, value: str) -> None: # CRU[D]. def deprecate(self, data: Union[Resource, List[Resource]], debug: bool = False) -> None: - catch_exceptions = False if debug else True run( self._deprecate_one, self._deprecate_many, data, id_required=True, required_synchronized=True, - catch_exceptions=catch_exceptions, + catch_exceptions=not debug, exception=DeprecationError, monitored_status="_synchronized", ) @@ -1008,10 +1006,21 @@ def _error_from_response(response: Response) -> str: if reason: messages.append(reason) if details: - if isinstance(details, dict): - messages.append(json.dumps(details)) - else: - messages.append(str(details)) + result = details.get('result', None) + if result: + the_details = result.get("detail", None) + if the_details: + if isinstance(the_details, list): + for detail in the_details: + messages.append(f"Reason: {detail['resultMessage']} "\ + f"for the shape: {detail['sourceShape']}") + elif isinstance(the_details, dict): + messages.append(f"Reason: {detail['resultMessage']} "\ + f"for the shape: {detail['sourceShape']}") + else: + messages.append(str(the_details)) + else: + messages.append(str(result)) return ". ".join(messages) def build_sparql_query_statements(context: Context, *conditions) -> Tuple[List, List]: From b6278839e666ac185bccb8a0ca64ddb8a70ba040 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cristina=20E=2E=20Gonz=C3=A1lez-Espinoza?= Date: Tue, 29 Nov 2022 11:40:53 +0100 Subject: [PATCH 5/6] Updated error printing to have all the dictionary information. --- .../notebooks/use-cases/LocalValidation.ipynb | 401 +++++++----------- .../specializations/stores/bluebrain_nexus.py | 12 +- 2 files changed, 172 insertions(+), 241 deletions(-) diff --git a/examples/notebooks/use-cases/LocalValidation.ipynb b/examples/notebooks/use-cases/LocalValidation.ipynb index 0bb3e2ea..092a3687 100644 --- a/examples/notebooks/use-cases/LocalValidation.ipynb +++ b/examples/notebooks/use-cases/LocalValidation.ipynb @@ -11,7 +11,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": { "ExecuteTime": { "end_time": "2019-09-23T18:50:20.068658Z", @@ -29,17 +29,17 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "import getpass\n", - "TOKEN = getpass.getpass()" + "# import getpass\n", + "# TOKEN = getpass.getpass()" ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -48,14 +48,14 @@ "forge = KnowledgeGraphForge(\"../use-cases/prod-forge-nexus.yml\",\n", " endpoint=endpoint, \n", " bucket=BUCKET,\n", - " token=TOKEN,\n", + "# token=TOKEN,\n", " debug=True\n", " )" ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -173,7 +173,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -189,276 +189,203 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": {}, "outputs": [ { - "name": "stderr", + "name": "stdout", "output_type": "stream", "text": [ - "Constraint Violation in DatatypeConstraintComponent (http://www.w3.org/ns/shacl#DatatypeConstraintComponent):\n", - "\tSeverity: sh:Violation\n", - "\tSource Shape: [ sh:datatype xsd:decimal ]\n", - "\tFocus Node: Literal(\"2347304\", datatype=xsd:integer)\n", - "\tValue Node: Literal(\"2347304\", datatype=xsd:integer)\n", - "\tMessage: Value is not Literal with datatype xsd:decimal\n", - "\n", - "Constraint Violation in DatatypeConstraintComponent (http://www.w3.org/ns/shacl#DatatypeConstraintComponent):\n", - "\tSeverity: sh:Violation\n", - "\tSource Shape: [ sh:datatype xsd:unsignedInt ]\n", - "\tFocus Node: Literal(\"2347304\", datatype=xsd:integer)\n", - "\tValue Node: Literal(\"2347304\", datatype=xsd:integer)\n", - "\tMessage: Value is not Literal with datatype xsd:unsignedInt\n", - "\n", - "Constraint Violation in DatatypeConstraintComponent (http://www.w3.org/ns/shacl#DatatypeConstraintComponent):\n", - "\tSeverity: sh:Violation\n", - "\tSource Shape: [ sh:datatype xsd:double ]\n", - "\tFocus Node: Literal(\"2347304\", datatype=xsd:integer)\n", - "\tValue Node: Literal(\"2347304\", datatype=xsd:integer)\n", - "\tMessage: Value is not Literal with datatype xsd:double\n", - "\n", - "Constraint Violation in DatatypeConstraintComponent (http://www.w3.org/ns/shacl#DatatypeConstraintComponent):\n", - "\tSeverity: sh:Violation\n", - "\tSource Shape: [ sh:datatype xsd:long ]\n", - "\tFocus Node: Literal(\"2347304\", datatype=xsd:integer)\n", - "\tValue Node: Literal(\"2347304\", datatype=xsd:integer)\n", - "\tMessage: Value is not Literal with datatype xsd:long\n", - "\n", - "Constraint Violation in DatatypeConstraintComponent (http://www.w3.org/ns/shacl#DatatypeConstraintComponent):\n", - "\tSeverity: sh:Violation\n", - "\tSource Shape: [ sh:datatype xsd:int ]\n", - "\tFocus Node: Literal(\"2347304\", datatype=xsd:integer)\n", - "\tValue Node: Literal(\"2347304\", datatype=xsd:integer)\n", - "\tMessage: Value is not Literal with datatype xsd:int\n", - "\n", - "Constraint Violation in DatatypeConstraintComponent (http://www.w3.org/ns/shacl#DatatypeConstraintComponent):\n", - "\tSeverity: sh:Violation\n", - "\tSource Shape: [ sh:datatype xsd:unsignedLong ]\n", - "\tFocus Node: Literal(\"2347304\", datatype=xsd:integer)\n", - "\tValue Node: Literal(\"2347304\", datatype=xsd:integer)\n", - "\tMessage: Value is not Literal with datatype xsd:unsignedLong\n", - "\n", - "Constraint Violation in DatatypeConstraintComponent (http://www.w3.org/ns/shacl#DatatypeConstraintComponent):\n", - "\tSeverity: sh:Violation\n", - "\tSource Shape: [ sh:datatype xsd:float ]\n", - "\tFocus Node: Literal(\"2347304\", datatype=xsd:integer)\n", - "\tValue Node: Literal(\"2347304\", datatype=xsd:integer)\n", - "\tMessage: Value is not Literal with datatype xsd:float\n", - "\n", - "Constraint Violation in NodeKindConstraintComponent (http://www.w3.org/ns/shacl#NodeKindConstraintComponent):\n", - "\tSeverity: sh:Violation\n", - "\tSource Shape: this11:LabeledOntologyEntityShape\n", - "\tFocus Node: Literal(\"bytes\")\n", - "\tValue Node: Literal(\"bytes\")\n", - "\tMessage: Value is not of Node Kind sh:IRI\n", - "\n", - "Constraint Violation in MinCountConstraintComponent (http://www.w3.org/ns/shacl#MinCountConstraintComponent):\n", - "\tSeverity: sh:Violation\n", - "\tSource Shape: [ sh:datatype xsd:string ; sh:description Literal(\"Human readable label.\") ; sh:minCount Literal(\"1\", datatype=xsd:integer) ; sh:name Literal(\"Label\") ; sh:path rdfs:label ]\n", - "\tFocus Node: Literal(\"bytes\")\n", - "\tResult Path: rdfs:label\n", - "\tMessage: Less than 1 values on Literal(\"bytes\")->rdfs:label\n", - "\n", - "Constraint Violation in NodeConstraintComponent (http://www.w3.org/ns/shacl#NodeConstraintComponent):\n", - "\tSeverity: sh:Violation\n", - "\tSource Shape: [ sh:node this11:LabeledOntologyEntityShape ]\n", - "\tFocus Node: Literal(\"bytes\")\n", - "\tValue Node: Literal(\"bytes\")\n", - "\tMessage: Value does not conform to Shape this11:LabeledOntologyEntityShape\n", - "\n", - "Constraint Violation in MinCountConstraintComponent (http://www.w3.org/ns/shacl#MinCountConstraintComponent):\n", - "\tSeverity: sh:Violation\n", - "\tSource Shape: [ sh:description Literal(\"When followed this link leads to a resource providing further description on how to download the attached data.\") ; sh:maxCount Literal(\"1\", datatype=xsd:integer) ; sh:minCount Literal(\"1\", datatype=xsd:integer) ; sh:nodeKind sh:IRI ; sh:path schema:url ]\n", - "\tFocus Node: [ [ Literal(\"bytes\") ; Literal(\"2347304\", datatype=xsd:integer) ] ; ; Literal(\"application/nwb\") ; Literal(\"dNAD_ltb.nwb\") ; [ Literal(\"file:///gpfs/bbp.cscs.ch/data/project/proj94/nexus/bbp/uniprot/9/9/e/f/6/1/f/e/dNAD_ltb.nwb\") ; ; rdf:type ] ; [ Literal(\"SHA-256\") ; Literal(\"630f7455b9be5b212566a8dcc76d6a9b2303257ed9199a5c9262fc25aaa5749f\") ] ; rdf:type ]\n", - "\tResult Path: schema:url\n", - "\tMessage: Less than 1 values on [ [ Literal(\"bytes\") ; Literal(\"2347304\", datatype=xsd:integer) ] ; ; Literal(\"application/nwb\") ; Literal(\"dNAD_ltb.nwb\") ; [ Literal(\"file:///gpfs/bbp.cscs.ch/data/project/proj94/nexus/bbp/uniprot/9/9/e/f/6/1/f/e/dNAD_ltb.nwb\") ; ; rdf:type ] ; [ Literal(\"SHA-256\") ; Literal(\"630f7455b9be5b212566a8dcc76d6a9b2303257ed9199a5c9262fc25aaa5749f\") ] ; rdf:type ]->schema:url\n", - "\n", - "Constraint Violation in MinCountConstraintComponent (http://www.w3.org/ns/shacl#MinCountConstraintComponent):\n", - "\tSeverity: sh:Violation\n", - "\tSource Shape: [ sh:datatype xsd:string ; sh:description Literal(\"The entity name.\") ; sh:maxCount Literal(\"1\", datatype=xsd:integer) ; sh:minCount Literal(\"1\", datatype=xsd:integer) ; sh:name Literal(\"Name\") ; sh:path schema:name ]\n", - "\tFocus Node: \n", - "\tResult Path: schema:name\n", - "\tMessage: Less than 1 values on ->schema:name\n", - "\n", - "Constraint Violation in NodeConstraintComponent (http://www.w3.org/ns/shacl#NodeConstraintComponent):\n", - "\tSeverity: sh:Violation\n", - "\tSource Shape: [ sh:node this3:EntityShape ]\n", - "\tFocus Node: \n", - "\tValue Node: \n", - "\tMessage: Value does not conform to Shape this3:EntityShape\n", - "\n", - "Constraint Violation in DatatypeConstraintComponent (http://www.w3.org/ns/shacl#DatatypeConstraintComponent):\n", - "\tSeverity: sh:Violation\n", - "\tSource Shape: [ sh:datatype xsd:decimal ]\n", - "\tFocus Node: Literal(\"2347304\", datatype=xsd:integer)\n", - "\tValue Node: Literal(\"2347304\", datatype=xsd:integer)\n", - "\tMessage: Value is not Literal with datatype xsd:decimal\n", - "\n", - "Constraint Violation in DatatypeConstraintComponent (http://www.w3.org/ns/shacl#DatatypeConstraintComponent):\n", - "\tSeverity: sh:Violation\n", - "\tSource Shape: [ sh:datatype xsd:unsignedInt ]\n", - "\tFocus Node: Literal(\"2347304\", datatype=xsd:integer)\n", - "\tValue Node: Literal(\"2347304\", datatype=xsd:integer)\n", - "\tMessage: Value is not Literal with datatype xsd:unsignedInt\n", - "\n", - "Constraint Violation in DatatypeConstraintComponent (http://www.w3.org/ns/shacl#DatatypeConstraintComponent):\n", - "\tSeverity: sh:Violation\n", - "\tSource Shape: [ sh:datatype xsd:double ]\n", - "\tFocus Node: Literal(\"2347304\", datatype=xsd:integer)\n", - "\tValue Node: Literal(\"2347304\", datatype=xsd:integer)\n", - "\tMessage: Value is not Literal with datatype xsd:double\n", - "\n", - "Constraint Violation in DatatypeConstraintComponent (http://www.w3.org/ns/shacl#DatatypeConstraintComponent):\n", - "\tSeverity: sh:Violation\n", - "\tSource Shape: [ sh:datatype xsd:long ]\n", - "\tFocus Node: Literal(\"2347304\", datatype=xsd:integer)\n", - "\tValue Node: Literal(\"2347304\", datatype=xsd:integer)\n", - "\tMessage: Value is not Literal with datatype xsd:long\n", - "\n", - "Constraint Violation in DatatypeConstraintComponent (http://www.w3.org/ns/shacl#DatatypeConstraintComponent):\n", - "\tSeverity: sh:Violation\n", - "\tSource Shape: [ sh:datatype xsd:int ]\n", - "\tFocus Node: Literal(\"2347304\", datatype=xsd:integer)\n", - "\tValue Node: Literal(\"2347304\", datatype=xsd:integer)\n", - "\tMessage: Value is not Literal with datatype xsd:int\n", - "\n", - "Constraint Violation in DatatypeConstraintComponent (http://www.w3.org/ns/shacl#DatatypeConstraintComponent):\n", - "\tSeverity: sh:Violation\n", - "\tSource Shape: [ sh:datatype xsd:unsignedLong ]\n", - "\tFocus Node: Literal(\"2347304\", datatype=xsd:integer)\n", - "\tValue Node: Literal(\"2347304\", datatype=xsd:integer)\n", - "\tMessage: Value is not Literal with datatype xsd:unsignedLong\n", - "\n", - "Constraint Violation in DatatypeConstraintComponent (http://www.w3.org/ns/shacl#DatatypeConstraintComponent):\n", - "\tSeverity: sh:Violation\n", - "\tSource Shape: [ sh:datatype xsd:float ]\n", - "\tFocus Node: Literal(\"2347304\", datatype=xsd:integer)\n", - "\tValue Node: Literal(\"2347304\", datatype=xsd:integer)\n", - "\tMessage: Value is not Literal with datatype xsd:float\n", - "\n", - "Constraint Violation in NodeKindConstraintComponent (http://www.w3.org/ns/shacl#NodeKindConstraintComponent):\n", - "\tSeverity: sh:Violation\n", - "\tSource Shape: this11:LabeledOntologyEntityShape\n", - "\tFocus Node: Literal(\"bytes\")\n", - "\tValue Node: Literal(\"bytes\")\n", - "\tMessage: Value is not of Node Kind sh:IRI\n", - "\n", - "Constraint Violation in MinCountConstraintComponent (http://www.w3.org/ns/shacl#MinCountConstraintComponent):\n", - "\tSeverity: sh:Violation\n", - "\tSource Shape: [ sh:datatype xsd:string ; sh:description Literal(\"Human readable label.\") ; sh:minCount Literal(\"1\", datatype=xsd:integer) ; sh:name Literal(\"Label\") ; sh:path rdfs:label ]\n", - "\tFocus Node: Literal(\"bytes\")\n", - "\tResult Path: rdfs:label\n", - "\tMessage: Less than 1 values on Literal(\"bytes\")->rdfs:label\n", - "\n", - "Constraint Violation in NodeConstraintComponent (http://www.w3.org/ns/shacl#NodeConstraintComponent):\n", - "\tSeverity: sh:Violation\n", - "\tSource Shape: [ sh:node this11:LabeledOntologyEntityShape ]\n", - "\tFocus Node: Literal(\"bytes\")\n", - "\tValue Node: Literal(\"bytes\")\n", - "\tMessage: Value does not conform to Shape this11:LabeledOntologyEntityShape\n", - "\n", - "Constraint Violation in MinCountConstraintComponent (http://www.w3.org/ns/shacl#MinCountConstraintComponent):\n", - "\tSeverity: sh:Violation\n", - "\tSource Shape: [ sh:description Literal(\"When followed this link leads to a resource providing further description on how to download the attached data.\") ; sh:maxCount Literal(\"1\", datatype=xsd:integer) ; sh:minCount Literal(\"1\", datatype=xsd:integer) ; sh:nodeKind sh:IRI ; sh:path schema:url ]\n", - "\tFocus Node: [ [ Literal(\"bytes\") ; Literal(\"2347304\", datatype=xsd:integer) ] ; ; Literal(\"application/nwb\") ; Literal(\"dNAD_ltb.nwb\") ; [ Literal(\"file:///gpfs/bbp.cscs.ch/data/project/proj94/nexus/bbp/uniprot/9/9/e/f/6/1/f/e/dNAD_ltb.nwb\") ; ; rdf:type ] ; [ Literal(\"SHA-256\") ; Literal(\"630f7455b9be5b212566a8dcc76d6a9b2303257ed9199a5c9262fc25aaa5749f\") ] ; rdf:type ]\n", - "\tResult Path: schema:url\n", - "\tMessage: Less than 1 values on [ [ Literal(\"bytes\") ; Literal(\"2347304\", datatype=xsd:integer) ] ; ; Literal(\"application/nwb\") ; Literal(\"dNAD_ltb.nwb\") ; [ Literal(\"file:///gpfs/bbp.cscs.ch/data/project/proj94/nexus/bbp/uniprot/9/9/e/f/6/1/f/e/dNAD_ltb.nwb\") ; ; rdf:type ] ; [ Literal(\"SHA-256\") ; Literal(\"630f7455b9be5b212566a8dcc76d6a9b2303257ed9199a5c9262fc25aaa5749f\") ] ; rdf:type ]->schema:url\n", - "\n", - "Constraint Violation in AndConstraintComponent (http://www.w3.org/ns/shacl#AndConstraintComponent):\n", - "\tSeverity: sh:Violation\n", - "\tSource Shape: this1:MINDSShape\n", - "\tFocus Node: \n", - "\tValue Node: \n", - "\tMessage: Node does not conform to all shapes in [ sh:node this3:EntityShape ] , [ sh:property [ rdfs:seeAlso ; sh:class nsg:Contribution ; sh:description Literal(\"Information about the contribution of an agent during the generation of this dataset.\") ; sh:minCount Literal(\"1\", datatype=xsd:integer) ; sh:name Literal(\"Qualified Contribution\") ; sh:nodeKind sh:BlankNodeOrIRI ; sh:path nsg:contribution ], [ rdfs:seeAlso this2:SubjectShape ; sh:class nsg:Subject ; sh:description Literal(\"The subject of the study. This data represent an observation of this subject.\") ; sh:minCount Literal(\"1\", datatype=xsd:integer) ; sh:name Literal(\"Subject\") ; sh:path nsg:subject ; skos:editorialNote Literal(\"Asserted subject entity. It may differ from the subject from the provenance.\", datatype=xsd:string) ], [ sh:class nsg:Annotation ; sh:description Literal(\"Annotations of the dataset.\") ; sh:name Literal(\"Annotation\") ; sh:node this18:AnnotationShape ; sh:path nsg:annotation ], [ sh:class nsg:BrainAtlasSpatialReferenceSystem ; sh:maxCount Literal(\"1\", datatype=xsd:integer) ; sh:path nsg:isRegisteredIn ], [ sh:class nsg:BrainLocation ; sh:description Literal(\"The Brain location object.\") ; sh:minCount Literal(\"1\", datatype=xsd:integer) ; sh:name Literal(\"Brain Location\") ; sh:node this16:BrainLocationShape ; sh:path nsg:brainLocation ], [ sh:class nsg:ObjectOfStudy ; sh:description Literal(\"Object of Study\") ; sh:name Literal(\"Object of Study\") ; sh:node this15:ObjectOfStudyOntologyTermShape ; sh:path nsg:objectOfStudy ; skos:editorialNote Literal(\"The object of study \", datatype=xsd:string) ], [ sh:class schema:DataDownload ; sh:description Literal(\"A dataset in downloadable form. The downloadable form has a specific format and is located at a specific location.\") ; sh:minCount Literal(\"1\", datatype=xsd:integer) ; sh:name Literal(\"Distribution\") ; sh:node this9:DistributionShape ; sh:path schema:distribution ], [ sh:description Literal(\"The dataset release date.\") ; sh:maxCount Literal(\"1\", datatype=xsd:integer) ; sh:name Literal(\"Release date\") ; sh:path schema:releaseDate ], [ sh:description Literal(\"The dataset type.\") ; sh:minCount Literal(\"1\", datatype=xsd:integer) ; sh:name Literal(\"Type\") ; sh:nodeKind sh:IRI ; sh:path rdf:type ], [ sh:maxCount Literal(\"1\", datatype=xsd:integer) ; sh:nodeKind sh:IRI ; sh:path nsg:atlasRelease ] ]\n", - "\n", - "Constraint Violation in NodeConstraintComponent (http://www.w3.org/ns/shacl#NodeConstraintComponent):\n", - "\tSeverity: sh:Violation\n", - "\tSource Shape: this:DatasetShape\n", - "\tFocus Node: \n", - "\tValue Node: \n", - "\tMessage: Value does not conform to Shape this1:MINDSShape\n", - "\n" + " _validate_one\n", + " True\n" ] - }, + } + ], + "source": [ + "forge.validate(resource, type_=\"Entity\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ { - "ename": "ValidationError", - "evalue": "\nValidation Report\nConforms: False\nResults (1):\nConstraint Violation in NodeConstraintComponent (http://www.w3.org/ns/shacl#NodeConstraintComponent):\n\tSeverity: sh:Violation\n\tSource Shape: this:DatasetShape\n\tFocus Node: \n\tValue Node: \n\tMessage: Value does not conform to Shape this1:MINDSShape\n", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mValidationError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m/var/folders/6p/k45nvhcs7v1_n9bd5g67wc3ctt8hpq/T/ipykernel_79079/2691144165.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mforge\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalidate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresource\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtype_\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"Dataset\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;32m~/Documents/code/nexus-forge/kgforge/core/forge.py\u001b[0m in \u001b[0;36mvalidate\u001b[0;34m(self, data, execute_actions_before, type_)\u001b[0m\n\u001b[1;32m 299\u001b[0m \"\"\"\n\u001b[1;32m 300\u001b[0m \u001b[0mdebug\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_debug\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 301\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_model\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalidate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexecute_actions_before\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtype_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdebug\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 302\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 303\u001b[0m \u001b[0;31m# Resolving User Interface.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/Documents/code/nexus-forge/kgforge/specializations/models/rdf_model.py\u001b[0m in \u001b[0;36mvalidate\u001b[0;34m(self, data, execute_actions_before, type_, debug)\u001b[0m\n\u001b[1;32m 113\u001b[0m run(self._validate_one, self._validate_many, data, execute_actions=execute_actions_before,\n\u001b[1;32m 114\u001b[0m \u001b[0mexception\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mValidationError\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmonitored_status\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"_validated\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcatch_exceptions\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcatch_exceptions\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 115\u001b[0;31m type_=type_, debug=debug)\n\u001b[0m\u001b[1;32m 116\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 117\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_validate_many\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresources\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mList\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mResource\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtype_\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdebug\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mbool\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/Documents/code/nexus-forge/kgforge/core/commons/execution.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(fun_one, fun_many, data, exception, id_required, required_synchronized, execute_actions, monitored_status, catch_exceptions, **kwargs)\u001b[0m\n\u001b[1;32m 113\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mResource\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 114\u001b[0m _run_one(fun_one, data, exception, id_required, required_synchronized, execute_actions,\n\u001b[0;32m--> 115\u001b[0;31m monitored_status, catch_exceptions, **kwargs)\n\u001b[0m\u001b[1;32m 116\u001b[0m \u001b[0maction\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_last_action\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 117\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maction\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/Documents/code/nexus-forge/kgforge/core/commons/execution.py\u001b[0m in \u001b[0;36m_run_one\u001b[0;34m(fun, resource, exception, id_required, required_synchronized, execute_actions, monitored_status, catch_exceptions, **kwargs)\u001b[0m\n\u001b[1;32m 159\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 160\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mcatch_exceptions\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mexception\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 161\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mexception\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;32m~/Documents/code/nexus-forge/kgforge/core/commons/execution.py\u001b[0m in \u001b[0;36m_run_one\u001b[0;34m(fun, resource, exception, id_required, required_synchronized, execute_actions, monitored_status, catch_exceptions, **kwargs)\u001b[0m\n\u001b[1;32m 143\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mexception\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"resource has lazy actions which need to be executed before\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 144\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 145\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresource\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 146\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 147\u001b[0m \u001b[0mstatus\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/Documents/code/nexus-forge/kgforge/specializations/models/rdf_model.py\u001b[0m in \u001b[0;36m_validate_one\u001b[0;34m(self, resource, type_, debug)\u001b[0m\n\u001b[1;32m 132\u001b[0m \u001b[0mconforms\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreport\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mservice\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalidate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresource\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtype_\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtype_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdebug\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdebug\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 133\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mconforms\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 134\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mValidationError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"\\n\"\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mreport\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 135\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 136\u001b[0m \u001b[0;31m# Utils.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mValidationError\u001b[0m: \nValidation Report\nConforms: False\nResults (1):\nConstraint Violation in NodeConstraintComponent (http://www.w3.org/ns/shacl#NodeConstraintComponent):\n\tSeverity: sh:Violation\n\tSource Shape: this:DatasetShape\n\tFocus Node: \n\tValue Node: \n\tMessage: Value does not conform to Shape this1:MINDSShape\n" + "name": "stdout", + "output_type": "stream", + "text": [ + " _register_one\n", + " False\n", + " RegistrationError: Resource 'https://bbp.epfl.ch/neurosciencegraph/data/traces/17b443b3-07fc-41c7-80c8-ea5e54662210' failed to validate against the constraints defined in schema 'https://neuroshapes.org/dash/dataset'. Reason: Value does not have all the shapes in the sh:and enumeration for the shape: https://neuroshapes.org/commons/minds/shapes/MINDSShape\n" ] } ], "source": [ - "forge.validate(resource, type_=\"Dataset\")" + "forge.register(resource, schema_id=\"datashapes:dataset\")" ] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "jonis = \"\"\"\n", + "{\n", + " \"atlasRelease\": {\n", + " \"id\": \"https://bbp.epfl.ch/nexus/v1/resources/nse/test2/_/197d151d-4ed2-4549-85fd-6c51bd471262\",\n", + " \"type\": [\"AtlasRelease\", \"BrainAtlasRelease\"]\n", + " },\n", + " \"brainLocation\": {\n", + " \"brainRegion\": {\n", + " \"id\": \"http://api.brain-map.org/api/v2/data/Structure/549\",\n", + " \"label\": \"Thalamus\"\n", + " },\n", + " \"type\": \"BrainLocation\"\n", + " },\n", + " \"circuitConfigPath\": {\n", + " \"type\": \"DataDownload\",\n", + " \"url\": \"file:///will/this/path/exist\"\n", + " },\n", + " \"circuitType\": \"Test registration\",\n", + " \"description\": \"Test registration, to be deprecated\",\n", + " \"name\": \"Test Circuit registration\",\n", + " \"type\": \"DetailedCircuit\",\n", + " \"wasGeneratedBy\": [{\"id\": \"https://bbp.epfl.ch/nexus/v1/resources/nse/test2/_/9be40e75-8744-415b-b0b4-e4074ff54a8f\"},\n", + " {\"id\": \"https://bbp.epfl.ch/nexus/v1/resources/nse/test2/_/2b5819a1-c5e9-42c4-8da7-864d5e1e0a7e\"}],\n", + " \"nodeCollection\" : {\n", + " \"type\": \"NodeCollection\",\n", + " \"name\": \"My node collection\",\n", + " \"circuitCellProperties\": {\n", + " \"type\" : \"CircuitCellProperties\",\n", + " \"name\": \"My cell properties\"\n", + " },\n", + " \"memodelRelease\": {\n", + " \"type\" : \"MEModelRelease\",\n", + " \"name\": \"Some model release\"\n", + " }\n", + " }\n", + "}\n", + "\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "joni_resource = forge.from_json(json.loads(jonis))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n", + " \"@context\": \"https://bbp.neuroshapes.org\",\n", + " \"atlasRelease\": {\n", + " \"@id\": \"https://bbp.epfl.ch/nexus/v1/resources/nse/test2/_/197d151d-4ed2-4549-85fd-6c51bd471262\",\n", + " \"@type\": [\n", + " \"AtlasRelease\",\n", + " \"BrainAtlasRelease\"\n", + " ]\n", + " },\n", + " \"brainLocation\": {\n", + " \"brainRegion\": {\n", + " \"@id\": \"http://api.brain-map.org/api/v2/data/Structure/549\",\n", + " \"label\": \"Thalamus\"\n", + " },\n", + " \"@type\": \"BrainLocation\"\n", + " },\n", + " \"circuitConfigPath\": {\n", + " \"@type\": \"DataDownload\",\n", + " \"url\": \"file:///will/this/path/exist\"\n", + " },\n", + " \"circuitType\": \"Test registration\",\n", + " \"description\": \"Test registration, to be deprecated\",\n", + " \"name\": \"Test Circuit registration\",\n", + " \"@type\": \"DetailedCircuit\",\n", + " \"nodeCollection\": {\n", + " \"@type\": \"NodeCollection\",\n", + " \"name\": \"My node collection\",\n", + " \"circuitCellProperties\": {\n", + " \"@type\": \"CircuitCellProperties\",\n", + " \"name\": \"My cell properties\"\n", + " },\n", + " \"memodelRelease\": {\n", + " \"@type\": \"MEModelRelease\",\n", + " \"name\": \"Some model release\"\n", + " }\n", + " }\n", + "}\n" + ] + } + ], + "source": [ + "print(json.dumps(forge.as_jsonld(joni_resource), indent=4))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " _register_one\n", + " False\n", + " RegistrationError: Schema 'https://neuroshapes.org/DetailedCircuit' could not be resolved in 'dke/kgforge'\n" + ] + } + ], + "source": [ + "forge.register(joni_resource, schema_id=\"https://neuroshapes.org/DetailedCircuit\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "True" + "'Test Circuit registration'" ] }, - "execution_count": 7, + "execution_count": 55, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "forge._debug" + "joni_resource.name" ] }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "catch_exceptions False\n" - ] - }, - { - "ename": "RegistrationError", - "evalue": "Resource 'https://bbp.epfl.ch/neurosciencegraph/data/traces/17b443b3-07fc-41c7-80c8-ea5e54662210' failed to validate against the constraints defined in schema 'https://neuroshapes.org/dash/dataset'. {\"@type\": \"sh:ValidationReport\", \"conforms\": false, \"result\": {\"@type\": \"sh:ValidationResult\", \"detail\": [{\"@type\": \"sh:ValidationResult\", \"focusNode\": \"https://bbp.epfl.ch/neurosciencegraph/data/traces/17b443b3-07fc-41c7-80c8-ea5e54662210\", \"resultMessage\": \"Value does not have all the shapes in the sh:and enumeration\", \"resultSeverity\": \"sh:Violation\", \"sourceConstraintComponent\": \"sh:AndConstraintComponent\", \"sourceShape\": \"https://neuroshapes.org/commons/minds/shapes/MINDSShape\", \"value\": \"https://bbp.epfl.ch/neurosciencegraph/data/traces/17b443b3-07fc-41c7-80c8-ea5e54662210\"}], \"focusNode\": \"https://bbp.epfl.ch/neurosciencegraph/data/traces/17b443b3-07fc-41c7-80c8-ea5e54662210\", \"resultMessage\": \"Value does not have shape \", \"resultSeverity\": \"sh:Violation\", \"sourceConstraintComponent\": \"sh:NodeConstraintComponent\", \"sourceShape\": \"https://neuroshapes.org/dash/dataset/shapes/DatasetShape\", \"value\": \"https://bbp.epfl.ch/neurosciencegraph/data/traces/17b443b3-07fc-41c7-80c8-ea5e54662210\"}, \"targetedNodes\": 2}", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mHTTPError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m~/Documents/code/nexus-forge/kgforge/specializations/stores/bluebrain_nexus.py\u001b[0m in \u001b[0;36m_register_one\u001b[0;34m(self, resource, schema_id)\u001b[0m\n\u001b[1;32m 226\u001b[0m )\n\u001b[0;32m--> 227\u001b[0;31m \u001b[0mresponse\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mraise_for_status\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 228\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/.local/lib/python3.7/site-packages/requests/models.py\u001b[0m in \u001b[0;36mraise_for_status\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 942\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mhttp_error_msg\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 943\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mHTTPError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhttp_error_msg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 944\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mHTTPError\u001b[0m: 400 Client Error: Bad Request for url: https://staging.nise.bbp.epfl.ch/nexus/v1/resources/dke/kgforge/datashapes%3Adataset", - "\nDuring handling of the above exception, another exception occurred:\n", - "\u001b[0;31mRegistrationError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m/var/folders/6p/k45nvhcs7v1_n9bd5g67wc3ctt8hpq/T/ipykernel_79079/2755330261.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mforge\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mregister\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresource\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mschema_id\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"datashapes:dataset\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;32m~/Documents/code/nexus-forge/kgforge/core/forge.py\u001b[0m in \u001b[0;36mregister\u001b[0;34m(self, data, schema_id)\u001b[0m\n\u001b[1;32m 643\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0mparam\u001b[0m \u001b[0mschema_id\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0man\u001b[0m \u001b[0midentifier\u001b[0m \u001b[0mof\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mschema\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mregistered\u001b[0m \u001b[0mresources\u001b[0m \u001b[0mshould\u001b[0m \u001b[0mconform\u001b[0m \u001b[0mto\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 644\u001b[0m \"\"\"\n\u001b[0;32m--> 645\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_store\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mregister\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mschema_id\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdebug\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_debug\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 646\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 647\u001b[0m \u001b[0;31m# No @catch because the error handling is done by execution.run().\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/Documents/code/nexus-forge/kgforge/specializations/stores/bluebrain_nexus.py\u001b[0m in \u001b[0;36mregister\u001b[0;34m(self, data, schema_id, debug)\u001b[0m\n\u001b[1;32m 150\u001b[0m \u001b[0mcatch_exceptions\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcatch_exceptions\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 151\u001b[0m \u001b[0mmonitored_status\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"_synchronized\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 152\u001b[0;31m \u001b[0mschema_id\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mschema_id\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 153\u001b[0m )\n\u001b[1;32m 154\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/Documents/code/nexus-forge/kgforge/core/commons/execution.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(fun_one, fun_many, data, exception, id_required, required_synchronized, execute_actions, monitored_status, catch_exceptions, **kwargs)\u001b[0m\n\u001b[1;32m 113\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mResource\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 114\u001b[0m _run_one(fun_one, data, exception, id_required, required_synchronized, execute_actions,\n\u001b[0;32m--> 115\u001b[0;31m monitored_status, catch_exceptions, **kwargs)\n\u001b[0m\u001b[1;32m 116\u001b[0m \u001b[0maction\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_last_action\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 117\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maction\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/Documents/code/nexus-forge/kgforge/core/commons/execution.py\u001b[0m in \u001b[0;36m_run_one\u001b[0;34m(fun, resource, exception, id_required, required_synchronized, execute_actions, monitored_status, catch_exceptions, **kwargs)\u001b[0m\n\u001b[1;32m 159\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 160\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mcatch_exceptions\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mexception\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 161\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mexception\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;32m~/Documents/code/nexus-forge/kgforge/core/commons/execution.py\u001b[0m in \u001b[0;36m_run_one\u001b[0;34m(fun, resource, exception, id_required, required_synchronized, execute_actions, monitored_status, catch_exceptions, **kwargs)\u001b[0m\n\u001b[1;32m 143\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mexception\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"resource has lazy actions which need to be executed before\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 144\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 145\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresource\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 146\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 147\u001b[0m \u001b[0mstatus\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/Documents/code/nexus-forge/kgforge/specializations/stores/bluebrain_nexus.py\u001b[0m in \u001b[0;36m_register_one\u001b[0;34m(self, resource, schema_id)\u001b[0m\n\u001b[1;32m 228\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 229\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mnexus\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mHTTPError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 230\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mRegistrationError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_error_message\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 231\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 232\u001b[0m \u001b[0mresponse_json\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjson\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mRegistrationError\u001b[0m: Resource 'https://bbp.epfl.ch/neurosciencegraph/data/traces/17b443b3-07fc-41c7-80c8-ea5e54662210' failed to validate against the constraints defined in schema 'https://neuroshapes.org/dash/dataset'. {\"@type\": \"sh:ValidationReport\", \"conforms\": false, \"result\": {\"@type\": \"sh:ValidationResult\", \"detail\": [{\"@type\": \"sh:ValidationResult\", \"focusNode\": \"https://bbp.epfl.ch/neurosciencegraph/data/traces/17b443b3-07fc-41c7-80c8-ea5e54662210\", \"resultMessage\": \"Value does not have all the shapes in the sh:and enumeration\", \"resultSeverity\": \"sh:Violation\", \"sourceConstraintComponent\": \"sh:AndConstraintComponent\", \"sourceShape\": \"https://neuroshapes.org/commons/minds/shapes/MINDSShape\", \"value\": \"https://bbp.epfl.ch/neurosciencegraph/data/traces/17b443b3-07fc-41c7-80c8-ea5e54662210\"}], \"focusNode\": \"https://bbp.epfl.ch/neurosciencegraph/data/traces/17b443b3-07fc-41c7-80c8-ea5e54662210\", \"resultMessage\": \"Value does not have shape \", \"resultSeverity\": \"sh:Violation\", \"sourceConstraintComponent\": \"sh:NodeConstraintComponent\", \"sourceShape\": \"https://neuroshapes.org/dash/dataset/shapes/DatasetShape\", \"value\": \"https://bbp.epfl.ch/neurosciencegraph/data/traces/17b443b3-07fc-41c7-80c8-ea5e54662210\"}, \"targetedNodes\": 2}" + " _validate_one\n", + " True\n" ] } ], "source": [ - "forge.register(resource, schema_id=\"datashapes:dataset\")" + "forge.validate(joni_resource, type_='DetailedCircuit')" ] }, { diff --git a/kgforge/specializations/stores/bluebrain_nexus.py b/kgforge/specializations/stores/bluebrain_nexus.py index 3ab80e8a..3b69ce99 100644 --- a/kgforge/specializations/stores/bluebrain_nexus.py +++ b/kgforge/specializations/stores/bluebrain_nexus.py @@ -999,6 +999,12 @@ def _error_message(error: HTTPError) -> str: return format_message(str(error)) def _error_from_response(response: Response) -> str: + def details_string(details: dict) -> str: + string = f"\nReason: {detail.pop('resultMessage')} "\ + f"for the shape: {detail.pop('sourceShape')}.\nError details:\n" + for key, value in details.items(): + string += f"{key}:\t{value}\n" + return string messages = [] error_json = response.json() reason = error_json.get("reason", None) @@ -1012,11 +1018,9 @@ def _error_from_response(response: Response) -> str: if the_details: if isinstance(the_details, list): for detail in the_details: - messages.append(f"Reason: {detail['resultMessage']} "\ - f"for the shape: {detail['sourceShape']}") + messages.append(details_string(detail)) elif isinstance(the_details, dict): - messages.append(f"Reason: {detail['resultMessage']} "\ - f"for the shape: {detail['sourceShape']}") + messages.append(details_string(detail)) else: messages.append(str(the_details)) else: From 5af57d40a517e541910dce2bb386977e184aafb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cristina=20E=2E=20Gonz=C3=A1lez-Espinoza?= Date: Wed, 5 Apr 2023 11:29:43 +0200 Subject: [PATCH 6/6] Change missing catch_exceptions. --- kgforge/core/archetypes/store.py | 11 ++++------- kgforge/specializations/models/rdf_model.py | 5 ++--- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/kgforge/core/archetypes/store.py b/kgforge/core/archetypes/store.py index 83ae9afe..854856bc 100644 --- a/kgforge/core/archetypes/store.py +++ b/kgforge/core/archetypes/store.py @@ -299,7 +299,6 @@ def update( self, data: Union[Resource, List[Resource]], schema_id: Optional[str], debug: bool = False ) -> None: # Replace None by self._update_many to switch to optimized bulk update. - catch_exceptions = False if debug else True run( self._update_one, None, @@ -310,6 +309,7 @@ def update( exception=UpdatingError, monitored_status="_synchronized", schema_id=schema_id, + catch_exceptions=not debug ) def _update_many(self, resources: List[Resource], schema_id: Optional[str]) -> None: @@ -327,14 +327,13 @@ def _update_one(self, resource: Resource, schema_id: Optional[str]) -> None: def tag(self, data: Union[Resource, List[Resource]], value: str, debug: bool = True) -> None: # Replace None by self._tag_many to switch to optimized bulk tagging. # POLICY If tagging modify the resource, run() should have status='_synchronized'. - catch_exceptions = False if debug else True run( self._tag_one, None, data, id_required=True, required_synchronized=True, - catch_exceptions=catch_exceptions, + catch_exceptions=not debug, exception=TaggingError, value=value, ) @@ -354,14 +353,13 @@ def _tag_one(self, resource: Resource, value: str) -> None: def deprecate(self, data: Union[Resource, List[Resource]], debug: bool = False) -> None: # Replace None by self._deprecate_many to switch to optimized bulk deprecation. - catch_exceptions = False if debug else True run( self._deprecate_one, None, data, id_required=True, required_synchronized=True, - catch_exceptions=catch_exceptions, + catch_exceptions=not debug, exception=DeprecationError, monitored_status="_synchronized", ) @@ -448,14 +446,13 @@ def _elastic(self, query: str) -> List[Resource]: def freeze(self, data: Union[Resource, List[Resource]], debug: bool = False) -> None: # Replace None by self._freeze_many to switch to optimized bulk freezing. - catch_exceptions = False if debug else True run( self._freeze_one, None, data, id_required=True, required_synchronized=True, - catch_exceptions=catch_exceptions, + catch_exceptions=not debug, exception=FreezingError, ) diff --git a/kgforge/specializations/models/rdf_model.py b/kgforge/specializations/models/rdf_model.py index 0b6df65f..cfca7620 100644 --- a/kgforge/specializations/models/rdf_model.py +++ b/kgforge/specializations/models/rdf_model.py @@ -108,10 +108,9 @@ def schema_id(self, type: str) -> str: raise ValueError("type not found") def validate(self, data: Union[Resource, List[Resource]], execute_actions_before: bool, type_: str, - debug: bool) -> None: - catch_exceptions = False if debug else True + debug: bool = False) -> None: run(self._validate_one, self._validate_many, data, execute_actions=execute_actions_before, - exception=ValidationError, monitored_status="_validated", catch_exceptions=catch_exceptions, + exception=ValidationError, monitored_status="_validated", catch_exceptions=not debug, type_=type_, debug=debug) def _validate_many(self, resources: List[Resource], type_: str, debug: bool) -> None: