cove: Add closed codelist validation

Bjwebb · Bjwebb · commit 411a68e8e9ff · 2022-12-14T20:06:37.000Z
diff --git a/cove/cove_360/templates/cove_360/explore.html b/cove/cove_360/templates/cove_360/explore.html
@@ -275,9 +275,9 @@ <h3 class="panel-title panel-title-explore" id="conversion-title">
   <div class="panel panel-default">
     <div class="panel-heading" data-toggle="collapse" data-target="#validation-body">
       <h3 class="panel-title panel-title-explore">
-        {% if validation_errors %}
+        {% if validation_errors or additional_closed_codelist_values %}
           <span class="font-tick cross" aria-hidden="true"></span>
-            {% trans "This data does not use the 360Giving Data Standard correctly" %} <small>{% blocktrans count n_errors=validation_errors_count %} {{n_errors}} Error.{% plural %}{{n_errors}} Errors {% endblocktrans %}</small>
+            {% trans "This data does not use the 360Giving Data Standard correctly" %} <small>{% blocktrans count n_errors=validation_and_closed_codelist_errors_count %} {{n_errors}} Error.{% plural %}{{n_errors}} Errors {% endblocktrans %}</small>
         {% else %}
           <span class="font-tick tick" aria-hidden="true"></span>
           {% trans "This data uses the 360Giving Data Standard correctly" %}
@@ -288,15 +288,15 @@ <h3 class="panel-title panel-title-explore">
     <div id="validation-body" class="collapse">
       <div class="panel-body">
         <p class="explanation">
-          {% if validation_errors %}
+          {% if validation_errors or additional_closed_codelist_values %}
             {% trans "Sorry your data is not yet using the 360Giving Data Standard. We used the " %}
           {% else %}
             {% trans "Congratulations! Your data is using the 360Giving Data Standard. We used the " %}
           {% endif %}
           <a href="https://www.threesixtygiving.org/standard/reference/#toc-360giving-json-schemas"> 360Giving JSON Package Schema</a> {% trans "to check this." %}
         </p>
         <div class="explore-help">{% trans "This means that the data" %}
-          {% if validation_errors %}
+          {% if validation_errors or additional_closed_codelist_values %}
             {% trans " does not meet " %}
           {% else %}
             {% trans " meets " %}
@@ -305,7 +305,7 @@ <h3 class="panel-title panel-title-explore">
           {% blocktrans %}the requirements of the <a href="https://standard.threesixtygiving.org/en/latest/technical/reference/">360Giving Data Standard</a>.{% endblocktrans %}
           {% blocktrans %} Making sure your data uses the standard correctly is important. Otherwise it cannot be used alongside other valid 360Giving data and cannot be included in 360Giving tools, such as GrantNav and 360Insights.{% endblocktrans %}
         </diV>
-        {% if validation_errors %}
+        {% if validation_errors or additional_closed_codelist_values %}
           <br>
           <p class="explanation">&nbsp;{% trans "The following <strong>errors</strong> are preventing your data from being valid 360Giving data. Please use the feedback below to find and resolve the issues in your file" %}</p>
           <br>
@@ -345,6 +345,24 @@ <h4>Incorrect Formats</h4>
               {% endfor %}
             {% endif %}
           {% endwith %}
+          {% if additional_closed_codelist_values %}
+              <div class="panel panel-default">
+                <div class="panel-heading">
+                  <h4>{% trans 'Codelist Errors' %}</h4>
+                </div>
+                <div>
+                  <div class="panel-body">
+                    <p>Some or all of your entries do not use the 360Giving Data Standard codelists correctly.</p>
+                    <p>{% blocktrans %}The fields below use closed codelists. When using these fields, you <strong>must</strong> use one of the pre-defined codelist values. If you use a value that is not on the relevant codelist, your data will not pass structural checks.{% endblocktrans %}</p>
+                  </div>
+                  </p>
+                  {% with additional_codelist_values=additional_closed_codelist_values %}
+                  <div>{% include "additional_codelist_values.html" %}</div>
+                  {% endwith %}
+                </div>
+              </div>
+          {% endif %}
+        {% endif %}
           {% with validation_errors=validation_errors_grouped.other error_prefix='other-' %}
             {% if validation_errors %}
               <div class="panel panel-default">
@@ -363,12 +381,12 @@ <h4>Other</h4>
               {% endfor %}
             {% endif %}
           {% endwith %}
-        {% endif %}
       </div>
     </div>
   </div>
   {% endblock validation %}
 
+
   {% block quality_accuracy %}
   {% if quality_accuracy_checks_count %}
   <div class="panel panel-default">
diff --git a/lib360dataquality/cove/schema.py b/lib360dataquality/cove/schema.py
@@ -10,3 +10,4 @@ class Schema360(SchemaJsonMixin):
     pkg_schema_name = config['schema_name']
     schema_url = urljoin(schema_host, schema_name)
     pkg_schema_url = urljoin(schema_host, pkg_schema_name)
+    codelists = config['codelists_host']
diff --git a/lib360dataquality/cove/settings.py b/lib360dataquality/cove/settings.py
@@ -11,6 +11,7 @@
     'schema_name': '360-giving-package-schema.json',
     'schema_item_name': '360-giving-schema.json',
     'schema_host': f'https://raw.githubusercontent.com/ThreeSixtyGiving/standard/{os.environ.get("SCHEMA_BRANCH", "master")}/schema/',
+    'codelists_host': f'https://raw.githubusercontent.com/ThreeSixtyGiving/standard/{os.environ.get("SCHEMA_BRANCH", "master")}/codelists/',
     'schema_version': None,
     'schema_version_choices': None,
     'root_list_path': 'grants',
diff --git a/lib360dataquality/cove/threesixtygiving.py b/lib360dataquality/cove/threesixtygiving.py
@@ -10,7 +10,7 @@
 import openpyxl
 import pytz
 from dateutil.relativedelta import relativedelta
-from libcove.lib.common import common_checks_context, get_orgids_prefixes
+from libcove.lib.common import common_checks_context, get_additional_codelist_values, get_orgids_prefixes
 from rangedict import RangeDict as range_dict
 
 try:
@@ -350,6 +350,22 @@ def common_checks_360(
             }
         )
 
+    additional_codelist_values = get_additional_codelist_values(schema_obj, json_data)
+    closed_codelist_values = {
+        key: value for key, value in additional_codelist_values.items() if not value["isopen"]
+    }
+    open_codelist_values = {key: value for key, value in additional_codelist_values.items() if value["isopen"]}
+
+    closed_codelist_errors_count = sum(len(value['values']) for value in closed_codelist_values.values())
+
+    context.update(
+        {
+            "additional_closed_codelist_values": closed_codelist_values,
+            "additional_open_codelist_values": open_codelist_values,
+            "validation_and_closed_codelist_errors_count": context["validation_errors_count"] + closed_codelist_errors_count
+        }
+    )
+
     return context
 
 
diff --git a/requirements_cove.in b/requirements_cove.in
@@ -2,7 +2,7 @@
 Django>3.2,<3.3
 #^^ rq.filter: <1.12
 flattentool>=0.17.0
-libcove
+libcove>=0.29.0
 libcoveweb
 django-bootstrap3
 django-debug-toolbar
diff --git a/requirements_cove.txt b/requirements_cove.txt
@@ -1,6 +1,6 @@
 #
-# This file is autogenerated by pip-compile with python 3.8
-# To update, run:
+# This file is autogenerated by pip-compile with Python 3.8
+# by the following command:
 #
 #    pip-compile requirements_cove.in
 #
@@ -75,7 +75,7 @@ jsonschema==3.2.0
     # via
     #   -r requirements_cove.in
     #   libcove
-libcove==0.27.0
+libcove==0.29.0
     # via
     #   -r requirements_cove.in
     #   lib360dataquality
diff --git a/requirements_cove_dev.txt b/requirements_cove_dev.txt
@@ -1,6 +1,6 @@
 #
-# This file is autogenerated by pip-compile with python 3.8
-# To update, run:
+# This file is autogenerated by pip-compile with Python 3.8
+# by the following command:
 #
 #    pip-compile requirements_cove_dev.in
 #
@@ -144,7 +144,7 @@ jsonschema==3.2.0
     # via
     #   -r requirements_cove.in
     #   libcove
-libcove==0.27.0
+libcove==0.29.0
     # via
     #   -r requirements_cove.in
     #   lib360dataquality
diff --git a/requirements_cove_dokku.txt b/requirements_cove_dokku.txt
@@ -1,6 +1,6 @@
 #
-# This file is autogenerated by pip-compile with python 3.8
-# To update, run:
+# This file is autogenerated by pip-compile with Python 3.8
+# by the following command:
 #
 #    pip-compile requirements_cove_dokku.in
 #
@@ -100,7 +100,7 @@ jsonschema==3.2.0
     # via
     #   -r requirements_cove.txt
     #   libcove
-libcove==0.27.0
+libcove==0.29.0
     # via
     #   -r requirements_cove.txt
     #   lib360dataquality

Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`#`
`2`		`-# This file is autogenerated by pip-compile with python 3.8`
`3`		`-# To update, run:`
	`2`	`+# This file is autogenerated by pip-compile with Python 3.8`
	`3`	`+# by the following command:`
`4`	`4`	`#`
`5`	`5`	`# pip-compile requirements_cove.in`
`6`	`6`	`#`
`@@ -75,7 +75,7 @@ jsonschema==3.2.0`
`75`	`75`	`# via`
`76`	`76`	`# -r requirements_cove.in`
`77`	`77`	`# libcove`
`78`		`-libcove==0.27.0`
	`78`	`+libcove==0.29.0`
`79`	`79`	`# via`
`80`	`80`	`# -r requirements_cove.in`
`81`	`81`	`# lib360dataquality`