From b656350753c12d453ae03c908cc5d5852af67b58 Mon Sep 17 00:00:00 2001 From: Will List Date: Fri, 17 Dec 2021 12:32:09 +0000 Subject: [PATCH 1/4] Added pages not on the nav structure to the nav structure. --- mkdocs.yml | 232 ++++++++++++++++++++++++++++++----------------------- 1 file changed, 132 insertions(+), 100 deletions(-) diff --git a/mkdocs.yml b/mkdocs.yml index 0cb805c..ca59045 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -64,103 +64,135 @@ extra: find what they're searching for. With your consent, you're helping us to make our documentation better. nav: - - Home: 'index.md' - - About: - - Introduction: 'about/introduction.md' - - History: 'about/history.md' - - Research: 'about/research.md' - - Mauro: 'about/mauro.md' - - Release Notes: 'about/release-notes.md' - - Development Roadmap: 'about/development-roadmap.md' - - Using Mauro: - - User Guides: - - Introduction: 'user-guides/introduction.md' - - Create a Data Model: 'user-guides/create-a-data-model/create-a-data-model.md' - - Organising Data Models: 'user-guides/organising-data-models/organising-data-models.md' - - Finalising Data Models: 'user-guides/finalising-data-models/finalising-data-models.md' - - Branching, versioning and forking Data Models: 'user-guides/branch-version-fork/branch-version-fork.md' - - Merging Data Models: 'user-guides/merging-data-models/merging-data-models.md' - - Document a Dataset: 'user-guides/document-a-dataset/document-a-dataset.md' - - Exporting Data Models: 'user-guides/exporting-data-models/exporting-data-models.md' - - Import a Data Model from Excel: 'user-guides/import-data-model-from-excel/import-data-model-from-excel.md' - - How to search: 'user-guides/how-to-search/how-to-search.md' - - Semantic links: 'user-guides/add-a-semantic-link/semantic-links.md' - - Publish / Subscribe: 'user-guides/publish-subscribe/publish-subscribe.md' - - User profile: 'user-guides/user-profile/user-profile.md' - - Admin functionality: 'user-guides/admin-functionality/admin-functionality.md' - - Permissions: 'user-guides/permissions/permissions.md' - - Feature switches: 'user-guides/feature-switches/feature-switches.md' - - Dynamic profiles: 'user-guides/dynamic-profiles/dynamic-profiles.md' - - Plugins: - - Digital Object Identifiers: 'plugins/user-guides/digital-object-identifiers/digital-object-identifiers.md' - - OpenID Connect Client: 'plugins/user-guides/openid-connect/openid-connect.md' - - Tutorials: - - Introduction: 'tutorials/introduction.md' - - Semantic Links: 'tutorials/semantic-links.md' - - Properties and profiles: 'tutorials/properties-profiles.md' - - Glossary: 'glossary/glossary.md' - - Installing and Administration: - - Installation: - - Docker Setup: 'installing/docker-setup.md' - - Docker Install: 'installing/docker-install.md' - - Updating: 'installing/updating.md' - - Configuring: - - Overview: 'installing/configuring/overview.md' - - Changing Database: 'installing/configuring/changing-database.md' - - MDM Core: 'installing/configuring/mdm-core-config.md' - - Plugins: - - Digital Object Identifiers: 'plugins/configuring/doi.md' - - FHIR Importer: 'plugins/configuring/fhir.md' - - OpenID Connect: 'plugins/configuring/oic.md' - - Default Files: - - MDM Application YML: 'installing/configuring/application.yml.md' - - Build YML: 'installing/configuring/build.yml.md' - - Runtime YML: 'installing/configuring/runtime.yml.md' - - Administration: 'installing/administration.md' - - Branding: 'installing/branding/branding.md' - - Plugins: 'installing/plugins.md' - - Migrating Old Data: 'installing/migration.md' - - Developer Resources: - - Technical Architecture: 'resources/architecture.md' - - REST API: - - Introduction: 'rest-api/introduction.md' - - Authentication: 'rest-api/authentication.md' - - API Keys: 'rest-api/apikeys.md' - - Errors: 'rest-api/errors.md' - - Pagination: 'rest-api/pagination.md' - - Postman Library: 'rest-api/postman.md' - - Core resources: - - Catalogue item: 'rest-api/resources/catalogue-item.md' - - Classifier: 'rest-api/resources/classifier.md' - - Codeset: 'rest-api/resources/codeset.md' - - Data class: 'rest-api/resources/data-class.md' - - Data element: 'rest-api/resources/data-element.md' - - Data flow: 'rest-api/resources/data-flow.md' - - Data flow component: 'rest-api/resources/data-flow-component.md' - - Data model: 'rest-api/resources/data-model.md' - - Data type: 'rest-api/resources/data-type.md' - - Enumeration value: 'rest-api/resources/enumeration-value.md' - - Folder: 'rest-api/resources/folder.md' - - Plugin: 'rest-api/resources/plugin.md' - - Semantic link: 'rest-api/resources/semantic-link.md' - - Term: 'rest-api/resources/term.md' - - Term relationship: 'rest-api/resources/term-relationship.md' - - Terminology: 'rest-api/resources/terminology.md' - - User: 'rest-api/resources/user.md' - - User group: 'rest-api/resources/user-group.md' - - Versioned folder: 'rest-api/resources/versioned-folder.md' - - Trees: 'rest-api/trees.md' - - Import / Export: 'rest-api/importexport.md' - - Admin functions: 'rest-api/admin.md' - - Client Libraries: - - Groovy / Java: 'resources/client/java.md' - - TypeScript: 'resources/client/typescript.md' - - .NET: 'resources/client/net.md' - - Technical Plugins: - - Digital Object Identifier (DOI): 'plugins/rest-api/doi.md' - - Freemarker Templating: 'plugins/rest-api/freemarker.md' - - OpenID Connect: 'plugins/rest-api/openid-connect.md' - - SPARQL: 'plugins/rest-api/sparql.md' - - Community: - - Support: 'community/support.md' - - Contribute: 'community/contribute.md' \ No newline at end of file + - Home: 'index.md' + - About: + - Introduction: 'about/introduction.md' + - History: 'about/history.md' + - Research: 'about/research.md' + - Mauro: 'about/mauro.md' + - Release Notes: 'about/release-notes.md' + - Development Roadmap: 'about/development-roadmap.md' + - Privacy Policy: 'privacy-policy.md' + - Using Mauro: + - User Guides: + - Introduction: 'user-guides/introduction.md' + - Create a Data Model: 'user-guides/create-a-data-model/create-a-data-model.md' + - Organising Data Models: 'user-guides/organising-data-models/organising-data-models.md' + - Finalising Data Models: 'user-guides/finalising-data-models/finalising-data-models.md' + - Branching, versioning and forking Data Models: 'user-guides/branch-version-fork/branch-version-fork.md' + - Merging Data Models: 'user-guides/merging-data-models/merging-data-models.md' + - Document a Dataset: 'user-guides/document-a-dataset/document-a-dataset.md' + - Exporting Data Models: 'user-guides/exporting-data-models/exporting-data-models.md' + - Import a Data Model from Excel: 'user-guides/import-data-model-from-excel/import-data-model-from-excel.md' + - How to search: 'user-guides/how-to-search/how-to-search.md' + - Add Semantic Links: 'user-guides/add-a-semantic-link/semantic-links.md' + - Publish / Subscribe: 'user-guides/publish-subscribe/publish-subscribe.md' + - User profile: 'user-guides/user-profile/user-profile.md' + - Admin functionality: 'user-guides/admin-functionality/admin-functionality.md' + - Permissions: 'user-guides/permissions/permissions.md' + - Feature switches: 'user-guides/feature-switches/feature-switches.md' + - Dynamic profiles: 'user-guides/dynamic-profiles/dynamic-profiles.md' + - Plugins: + - Digital Object Identifiers: 'plugins/user-guides/digital-object-identifiers/digital-object-identifiers.md' + - OpenID Connect Client: 'plugins/user-guides/openid-connect/openid-connect.md' + - Data Flows: 'user-guides\data-flows\data-flows.md' + - Versioning Data Models: 'user-guides\version-data-models\version-data-models.md' + - Tutorials: + - Introduction: 'tutorials/introduction.md' + - Semantic Links: 'tutorials/semantic-links.md' + - Properties and profiles: 'tutorials/properties-profiles.md' + - Documenting a data model: 'tutorials\document-assets\index.md' + - Glossary: + - Index: 'glossary/glossary.md' + - Words: + - aliases: 'glossary\aliases\aliases.md' + - branch: 'glossary\branch\branch.md' + - classification: 'glossary\classification\classification.md' + - data-asset: 'glossary\data-asset\data-asset.md' + - data-class: 'glossary\data-class\data-class.md' + - data-element: 'glossary\data-element\data-element.md' + - data-model: 'glossary\data-model\data-model.md' + - data-standard: 'glossary\data-standard\data-standard.md' + - data-type: 'glossary\data-type\data-type.md' + - dataflow: 'glossary\dataflow\dataflow.md' + - enumeration-data-type: 'glossary\enumeration-data-type\enumeration-data-type.md' + - finalise: 'glossary\finalise\finalise.md' + - folder: 'glossary\folder\folder.md' + - fork: 'glossary\fork\fork.md' + - label: 'glossary\label\label.md' + - merging: 'glossary\merging\merging.md' + - multi-facet-aware: 'glossary\multi-facet-aware\multi-facet-aware.md' + - multiplicity: 'glossary\multiplicity\multiplicity.md' + - primitive-data-type: 'glossary\primitive-data-type\primitive-data-type.md' + - profile: 'glossary\profile\profile.md' + - reference-data-type: 'glossary\reference-data-type\reference-data-type.md' + - semantic-links: 'glossary\semantic-links\semantic-links.md' + - terminology-data-type: 'glossary\terminology-data-type\terminology-data-type.md' + - version: 'glossary\version\version.md' + - versioned-folder: 'glossary\versioned-folder\versioned-folder.md' + - Installing and Administration: + - Installation: + - Docker Setup: 'installing/docker-setup.md' + - Docker Install: 'installing/docker-install.md' + - Updating: 'installing/updating.md' + - Configuring: + - Overview: 'installing/configuring/overview.md' + - Changing Database: 'installing/configuring/changing-database.md' + - MDM Core: 'installing/configuring/mdm-core-config.md' + - Plugins: + - Digital Object Identifiers: 'plugins/configuring/doi.md' + - FHIR Importer: 'plugins/configuring/fhir.md' + - OpenID Connect: 'plugins/configuring/oic.md' + - Default Files: + - MDM Application YML: 'installing/configuring/application.yml.md' + - Build YML: 'installing/configuring/build.yml.md' + - Runtime YML: 'installing/configuring/runtime.yml.md' + - Administration: 'installing/administration.md' + - Branding: 'installing/branding/branding.md' + - Plugins: 'installing/plugins.md' + - Migrating Old Data: 'installing/migration.md' + - Developer Resources: + - Technical Architecture: 'resources/architecture.md' + - Data Model: 'model\glossary.md' + - REST API: + - Introduction: 'rest-api/introduction.md' + - Authentication: 'rest-api/authentication.md' + - API Keys: 'rest-api/apikeys.md' + - Errors: 'rest-api/errors.md' + - Pagination: 'rest-api/pagination.md' + - Postman Library: 'rest-api/postman.md' + - Core resources: + - Catalogue item: 'rest-api/resources/catalogue-item.md' + - Classifier: 'rest-api/resources/classifier.md' + - Codeset: 'rest-api/resources/codeset.md' + - Data class: 'rest-api/resources/data-class.md' + - Data element: 'rest-api/resources/data-element.md' + - Data flow: 'rest-api/resources/data-flow.md' + - Data flow component: 'rest-api/resources/data-flow-component.md' + - Data model: 'rest-api/resources/data-model.md' + - Data type: 'rest-api/resources/data-type.md' + - Enumeration value: 'rest-api/resources/enumeration-value.md' + - Folder: 'rest-api/resources/folder.md' + - Plugin: 'rest-api/resources/plugin.md' + - Semantic link: 'rest-api/resources/semantic-link.md' + - Term: 'rest-api/resources/term.md' + - Term relationship: 'rest-api/resources/term-relationship.md' + - Terminology: 'rest-api/resources/terminology.md' + - User: 'rest-api/resources/user.md' + - User group: 'rest-api/resources/user-group.md' + - Versioned folder: 'rest-api/resources/versioned-folder.md' + - Trees: 'rest-api/trees.md' + - Import / Export: 'rest-api/importexport.md' + - Admin functions: 'rest-api/admin.md' + - Client Libraries: + - Groovy / Java: 'resources/client/java.md' + - TypeScript: 'resources/client/typescript.md' + - .NET: 'resources/client/net.md' + - Technical Plugins: + - Digital Object Identifier (DOI): 'plugins/rest-api/doi.md' + - Freemarker Templating: 'plugins/rest-api/freemarker.md' + - OpenID Connect: 'plugins/rest-api/openid-connect.md' + - SPARQL: 'plugins/rest-api/sparql.md' + - Community: + - Support: 'community/support.md' + - Contribute: 'community/contribute.md' From ad87fd2ac4abadedf5b7fbd68c7338d7ff5abafd Mon Sep 17 00:00:00 2001 From: Will List Date: Fri, 17 Dec 2021 12:32:45 +0000 Subject: [PATCH 2/4] Added user guide on interacting with data flows --- docs/user-guides/data-flows/data-flows.md | 138 ++++++++++++++++++++++ 1 file changed, 138 insertions(+) create mode 100644 docs/user-guides/data-flows/data-flows.md diff --git a/docs/user-guides/data-flows/data-flows.md b/docs/user-guides/data-flows/data-flows.md new file mode 100644 index 0000000..f436471 --- /dev/null +++ b/docs/user-guides/data-flows/data-flows.md @@ -0,0 +1,138 @@ +Mauro has the capacity to record data flows, which data goes to where. These used to be in the UI, but are currently only accessible via the API. +[Data Flows in the glossary](../../glossary/dataflow/dataflow.md) + +The domain setup is a dataFlow "has many" dataClassComponents which "has Many" dataElementComponents. A target of a dataFlow can have many sources. A source of a dataFlow can have many targets. + +Data Flows can only be created between models of the 'Asset' type. +``` +{ + "total": 1, + "errors": [ + { + "message": "Property [target] of DataFlow cannot by model type [Data Standard]" + } + ] +} +``` +In order to express relation between Assets and Standards, use Semantic Links - an Asset `refines` a Standard. + +-------------------------------- +In order to create a flow, you need to use the endpoints of: +``` + | POST | /api/dataModels/${targetDataModelId}/dataFlows | Action: save + | GET | /api/dataModels/${targetDataModelId}/dataFlows | Action: index + | DELETE | /api/dataModels/${targetDataModelId}/dataFlows/${id} | Action: delete + | PUT | /api/dataModels/${targetDataModelId}/dataFlows/${id} | Action: update + | GET | /api/dataModels/${targetDataModelId}/dataFlows/${id} | Action: show +``` +required JSON for a POST: +``` +{ + "label" : "Functional Test DataFlow", + "source": "sourceDataModelId" +} +``` + +You can 'reverse' these endpoints and use `${sourceDataModelId}` in the endpoint URL, with `?type=source` in the url. Then target ID in the JSON. + +**_CAUTION:_** It is entirely possible to create dataFlows with the same label name as other dataFlows. This may cause significant confusion, and should be avoided. + +what you'll get from a "show": +``` +{ + "id": "${json-unit.matches:id}", + "domainType": "DataFlow", + "label": "Functional Test DataFlow", + "model": "${json-unit.matches:id}", + "breadcrumbs": [ + { + "id": "${json-unit.matches:id}", + "label": "TargetFlowDataModel", + "domainType": "DataModel", + "finalised": false + } + ], + "availableActions": [ + "delete", + "show", + "update" + ], + "lastUpdated": "${json-unit.matches:offsetDateTime}", + "definition": null, + "source": { + "id": "${json-unit.matches:id}", + "domainType": "DataModel", + "label": "SourceFlowDataModel", + "type": "Data Asset", + "branchName": "main", + "documentationVersion": "1.0.0" + }, + "target": { + "id": "${json-unit.matches:id}", + "domainType": "DataModel", + "label": "TargetFlowDataModel", + "type": "Data Asset", + "branchName": "main", + "documentationVersion": "1.0.0" + }, + "diagramLayout": null +} +``` +---------------------------------------------------------- + +Ollie Freeman: +The index/list endpoint is quite powerful - if you provide the dataModelId it will find the entire chain of DFs that DM is on. +You need to use the param `type=source` or `type=target` to get a single direction. + +This is all under the condition you can read all of the chain, if you cannot read a DM then it will not show a flow between that model and another. It will stop once it hits an unreadable model. + +One of the things we also want to add is an inference .... so infer the flow DM.1->DM.3 ... that way if you can read 1 and 3 then you can see the flow but you cant see DM2 + +------------------------------------------------------------- +Once a dataFlow is created between dataModels, the components of which classes and elements relate to each other have to be created. These use the endpoints below. Note with these endpoints that Mauro does NOT implicitly create the elements in the pathway if they do not exist. You need to create the dataFlow, then the dataClassComponent, then any dataElementComponents that are part of it. + +**_CAUTION:_** It is entirely possible to create cyclical dataFlows, and this will have detrimental effects on any automation. These are not currently detected by the system, so there is trust in the user not to create them, or to remove any that are found. + +**_CAUTION:_** It is entirely possible to create dataClassComponents or dataElementComponents with the same label name as others. This may cause significant confusion, and should be avoided. + +When using these endpoints: +- the dataModelId is the Target dataModelId. +- `${type}` can be one of `source`, `target` + +``` +Controller: dataClassComponent + | POST | /api/dataModels/${dataModelId}/dataFlows/${dataFlowId}/dataClassComponents | Action: save + | GET | /api/dataModels/${dataModelId}/dataFlows/${dataFlowId}/dataClassComponents | Action: index + | DELETE | /api/dataModels/${dataModelId}/dataFlows/${dataFlowId}/dataClassComponents/${id} | Action: delete + | PUT | /api/dataModels/${dataModelId}/dataFlows/${dataFlowId}/dataClassComponents/${id} | Action: update + | GET | /api/dataModels/${dataModelId}/dataFlows/${dataFlowId}/dataClassComponents/${id} | Action: show + | DELETE | /api/dataModels/${dataModelId}/dataFlows/${dataFlowId}/dataClassComponents/${dataClassComponentId}/${type}/${dataClassId} | Action: alterDataClasses + | PUT | /api/dataModels/${dataModelId}/dataFlows/${dataFlowId}/dataClassComponents/${dataClassComponentId}/${type}/${dataClassId} | Action: alterDataClasses +``` +``` +{ + "label" : "Functional Test DataClassComponent", + "sourceDataClasses": "[sourceDataClassId]", + "targetDataClasses": "[targetDataClassId]" +} +``` +``` +Controller: dataElementComponent + | POST | /api/dataModels/${dataModelId}/dataFlows/${dataFlowId}/dataClassComponents/${dataClassComponentId}/dataElementComponents | Action: save + | GET | /api/dataModels/${dataModelId}/dataFlows/${dataFlowId}/dataClassComponents/${dataClassComponentId}/dataElementComponents | Action: index + | DELETE | /api/dataModels/${dataModelId}/dataFlows/${dataFlowId}/dataClassComponents/${dataClassComponentId}/dataElementComponents/${id} | Action: delete + | PUT | /api/dataModels/${dataModelId}/dataFlows/${dataFlowId}/dataClassComponents/${dataClassComponentId}/dataElementComponents/${id} | Action: update + | GET | /api/dataModels/${dataModelId}/dataFlows/${dataFlowId}/dataClassComponents/${dataClassComponentId}/dataElementComponents/${id} | Action: show + | DELETE | /api/dataModels/${dataModelId}/dataFlows/${dataFlowId}/dataClassComponents/${dataClassComponentId}/dataElementComponents/${dataElementComponentId}/${type}/${dataElementId} | Action: alterDataElements + | PUT | /api/dataModels/${dataModelId}/dataFlows/${dataFlowId}/dataClassComponents/${dataClassComponentId}/dataElementComponents/${dataElementComponentId}/${type}/${dataElementId} | Action: alterDataElements +``` +``` +{ + "label" : "Functional Test DataElementComponent", + "sourceDataElements": "[sourceDataElementId]", + "targetDataElements": "[targetDataElementId]" +} +``` +------------------------------------------------- + +dataFlows are currently independent of versions of the dataModels. This means that if you have a dataFlow on a versioned model, and create a new version, the flow will not be copied/ created onto the new version of the model. Functionality to handle this is envisaged for release in 2022. From 7100b8e8a0c7125180a4c37e2dfd551207789395 Mon Sep 17 00:00:00 2001 From: Will List Date: Fri, 17 Dec 2021 12:34:13 +0000 Subject: [PATCH 3/4] Removed empty and unused files --- docs/community/build-plugins.md | 0 docs/user/models.md | 0 2 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 docs/community/build-plugins.md delete mode 100644 docs/user/models.md diff --git a/docs/community/build-plugins.md b/docs/community/build-plugins.md deleted file mode 100644 index e69de29..0000000 diff --git a/docs/user/models.md b/docs/user/models.md deleted file mode 100644 index e69de29..0000000 From 54f51389b8d347d315255b5d6eaed4ee6cd35209 Mon Sep 17 00:00:00 2001 From: Will List Date: Fri, 17 Dec 2021 12:37:27 +0000 Subject: [PATCH 4/4] Removed link to blank page --- docs/resources/client/java.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/resources/client/java.md b/docs/resources/client/java.md index f776146..274322a 100644 --- a/docs/resources/client/java.md +++ b/docs/resources/client/java.md @@ -6,9 +6,8 @@ with a Mauro instance. The library makes use of the mdm-core grails application, validation. The REST API library can connect to a remote Mauro instance, and is typically used to perform bulk operations such as importing and exporting models, -such as when scripting a complex import. This is often the easiest way to experiment with importing before -[building a Grails Plugin](../../community/build-plugins.md). The code can also be used to connect to multiple Mauro instances, for example to copy -models from one instance to another. +such as when scripting a complex import. This is often the easiest way to experiment with importing before building a Grails Plugin. +The code can also be used to connect to multiple Mauro instances, for example to copy models from one instance to another. ## API documentation