On Recipients, store count of funders

ThreeSixtyGiving/grantnav#940

Must switch test to TestCase or database is somehow not in
correct state from another test, and tests fail

Author: jarofgreen

datastore/db/models.py

@@ -8,6 +8,8 @@
 from django.db.utils import DataError
 from django.utils import timezone
 
+from additional_data.sources.find_that_charity import non_primary_org_ids_lookup_maps
+
 
 class Latest(models.Model):
     """Latest best data we have"""
@@ -277,6 +279,8 @@ def update_aggregate(self, grant):
         #       "GBP": { "grants": 0, "total": 0, "avg": 0, min: 0, max: 0 } },
         #        ...
         # },
+        # This only covers common stats.
+        # See classes that inherit this - they may override update_aggregate() and add more.
 
         amount = grant["amountAwarded"]
         currency = grant["currency"]
@@ -373,6 +377,49 @@ class Meta:
 
     non_primary_org_ids = ArrayField(models.TextField())
 
+    """While collecting aggregate info on funders, we need to store some data temporarily that we don't want to store in the database.
+    This stores all primary ids so we can count unique funders."""
+    _aggregate_funders_primary_ids = set()
+    """While collecting aggregate info on funders, we need to store some data temporarily that we don't want to store in the database.
+    This stores information by currency."""
+    _aggregate_funders_currencies = {}
+
+    def update_aggregate(self, grant):
+        # Step 1: Call parent
+        super().update_aggregate(grant)
+
+        # Step 2: update _aggregate_funders_* vars with info from this grant
+        # This function is called repeatedly from datastore/db/management/commands/manage_entities_data.py
+        # and it's inefficient to call non_primary_org_ids_lookup_maps every time.
+        # But after discussion with MW that is ok.
+        (
+            non_primary_to_primary_org_ids_lookup,
+            primary_to_non_primary_org_ids_lookup,
+        ) = non_primary_org_ids_lookup_maps()
+        currency = grant["currency"]
+        if currency not in self._aggregate_funders_currencies:
+            self._aggregate_funders_currencies[currency] = {
+                "funders_primary_ids": set(),
+            }
+        for funder in grant["fundingOrganization"]:
+            # If the org-id provided is a non-primary org-id return the primary
+            # otherwise return the specified org-id
+            funder_primary_id = non_primary_to_primary_org_ids_lookup.get(
+                funder["id"], funder["id"]
+            )
+            self._aggregate_funders_primary_ids.add(funder_primary_id)
+            self._aggregate_funders_currencies[currency]["funders_primary_ids"].add(
+                funder_primary_id
+            )
+
+        # Step 3: copy info from _aggregate_funders_* vars to aggregate for saving to the database
+        self.aggregate["funders"] = len(self._aggregate_funders_primary_ids)
+
+        for currency_id, currency_data in self._aggregate_funders_currencies.items():
+            self.aggregate["currencies"][currency_id]["funders"] = len(
+                self._aggregate_funders_currencies[currency_id]["funders_primary_ids"]
+            )
+
 
 class Funder(Entity):
     class Meta:

datastore/tests/test_latest_data.py

@@ -4,13 +4,13 @@
 from tempfile import TemporaryDirectory
 
 from django.core.management import call_command
-from django.test import TransactionTestCase
+from django.test import TestCase
 
 import db.models as db
 from tests.generate_testdata import generate_data
 
 
-class LatestDataTest(TransactionTestCase):
+class LatestDataTest(TestCase):
     def break_the_data(self, tmpdir):
         data_all_path = os.path.join(tmpdir, "data_all.json")
         dataset = None
@@ -43,6 +43,11 @@ def test_latest_data(self):
             self.assertEqual(
                 db.Latest.objects.get(series=db.Latest.CURRENT).grant_set.count(), 50
             )
+            # Test some specific data
+            recipient = db.Recipient.objects.get(org_id="360G-example-a")
+            self.assertEqual(1, recipient.aggregate["funders"])
+            self.assertEqual(1, recipient.aggregate["currencies"]["GBP"]["funders"])
+
             # Load broken data - This should trigger fall back mechanism
             print("Breaking the data")
             self.break_the_data(tmpdir)