Skip to content

Commit c7db98e

Browse files
jarofgreenmichaelwood
jarofgreen
authored and
michaelwood
committed
On Recipients, store count of funders
ThreeSixtyGiving/grantnav#940
1 parent 6450bd4 commit c7db98e

File tree

2 files changed

+184
-0
lines changed

2 files changed

+184
-0
lines changed

‎datastore/db/models.py

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@
88
from django.db.utils import DataError
99
from django.utils import timezone
1010

11+
from additional_data.sources.find_that_charity import non_primary_org_ids_lookup_maps
12+
1113

1214
class Latest(models.Model):
1315
"""Latest best data we have"""
@@ -277,6 +279,8 @@ def update_aggregate(self, grant):
277279
# "GBP": { "grants": 0, "total": 0, "avg": 0, min: 0, max: 0 } },
278280
# ...
279281
# },
282+
# This only covers common stats.
283+
# See classes that inherit this - they may override update_aggregate() and add more.
280284

281285
amount = grant["amountAwarded"]
282286
currency = grant["currency"]
@@ -362,6 +366,14 @@ class Meta:
362366

363367

364368
class Recipient(Entity):
369+
def __init__(self, *args, **kwargs):
370+
super().__init__(*args, **kwargs)
371+
# While collecting aggregate info on funders, we need to store some data temporarily that we don't want to store in the database.
372+
# This stores all primary ids so we can count unique funders.
373+
self._aggregate_funders_primary_ids = set()
374+
# This stores information by currency.
375+
self._aggregate_funders_currencies = {}
376+
365377
class Meta:
366378
constraints = [
367379
models.UniqueConstraint(fields=["org_id"], name="recipient_unique_org_id")
@@ -373,6 +385,42 @@ class Meta:
373385

374386
non_primary_org_ids = ArrayField(models.TextField())
375387

388+
def update_aggregate(self, grant):
389+
# Step 1: Call parent
390+
super().update_aggregate(grant)
391+
392+
# Step 2: update _aggregate_funders_* vars with info from this grant
393+
# This function is called repeatedly from datastore/db/management/commands/manage_entities_data.py
394+
# and it's inefficient to call non_primary_org_ids_lookup_maps every time.
395+
# But after discussion with MW that is ok.
396+
(
397+
non_primary_to_primary_org_ids_lookup,
398+
primary_to_non_primary_org_ids_lookup,
399+
) = non_primary_org_ids_lookup_maps()
400+
currency = grant["currency"]
401+
if currency not in self._aggregate_funders_currencies:
402+
self._aggregate_funders_currencies[currency] = {
403+
"funders_primary_ids": set(),
404+
}
405+
for funder in grant["fundingOrganization"]:
406+
# If the org-id provided is a non-primary org-id return the primary
407+
# otherwise return the specified org-id
408+
funder_primary_id = non_primary_to_primary_org_ids_lookup.get(
409+
funder["id"], funder["id"]
410+
)
411+
self._aggregate_funders_primary_ids.add(funder_primary_id)
412+
self._aggregate_funders_currencies[currency]["funders_primary_ids"].add(
413+
funder_primary_id
414+
)
415+
416+
# Step 3: copy info from _aggregate_funders_* vars to aggregate for saving to the database
417+
self.aggregate["funders"] = len(self._aggregate_funders_primary_ids)
418+
419+
for currency_id, currency_data in self._aggregate_funders_currencies.items():
420+
self.aggregate["currencies"][currency_id]["funders"] = len(
421+
self._aggregate_funders_currencies[currency_id]["funders_primary_ids"]
422+
)
423+
376424

377425
class Funder(Entity):
378426
class Meta:

‎datastore/tests/test_models.py

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22

33
import db.models as db
44

5+
import unittest.mock
6+
57

68
class GetterRunTest(TransactionTestCase):
79
fixtures = ["test_data.json"]
@@ -91,3 +93,137 @@ def test_convenience_fields_from_data(self):
9193
self.assertSetEqual(set(grant.funding_org_ids), {"GB-CHC-12345"})
9294

9395
self.assertEqual(grant.publisher_org_id, "XI-EXAMPLE-EXAMPLE")
96+
97+
98+
def mock_non_primary_org_ids_lookup_maps():
99+
return {"GB-SECONDARY-12345": "GB-PRIMARY-12345"}, {}
100+
101+
102+
@unittest.mock.patch(
103+
"db.models.non_primary_org_ids_lookup_maps", mock_non_primary_org_ids_lookup_maps
104+
)
105+
class RecipientUpdateAggregateTest(TestCase):
106+
def test_single_grant(self):
107+
recipient = db.Recipient()
108+
recipient.update_aggregate(
109+
{
110+
"currency": "GBP",
111+
"amountAwarded": 100,
112+
"awardDate": "2019-10-03T00:00:00+00:00",
113+
"fundingOrganization": [{"id": "GB-CHC-12345"}],
114+
"recipientOrganization": [
115+
{"id": "GB-COH-12345"},
116+
],
117+
}
118+
)
119+
self.assertEqual(recipient.aggregate["funders"], 1)
120+
self.assertEqual(recipient.aggregate["currencies"]["GBP"]["funders"], 1)
121+
122+
def test_two_grants_from_same_funder(self):
123+
recipient = db.Recipient()
124+
recipient.update_aggregate(
125+
{
126+
"currency": "GBP",
127+
"amountAwarded": 100,
128+
"awardDate": "2019-10-03T00:00:00+00:00",
129+
"fundingOrganization": [{"id": "GB-CHC-12345"}],
130+
"recipientOrganization": [
131+
{"id": "GB-COH-12345"},
132+
],
133+
}
134+
)
135+
recipient.update_aggregate(
136+
{
137+
"currency": "GBP",
138+
"amountAwarded": 10000,
139+
"awardDate": "2020-10-03T00:00:00+00:00",
140+
"fundingOrganization": [{"id": "GB-CHC-12345"}],
141+
"recipientOrganization": [
142+
{"id": "GB-COH-12345"},
143+
],
144+
}
145+
)
146+
self.assertEqual(recipient.aggregate["funders"], 1)
147+
self.assertEqual(recipient.aggregate["currencies"]["GBP"]["funders"], 1)
148+
149+
def test_two_grants_from_different_funders(self):
150+
recipient = db.Recipient()
151+
recipient.update_aggregate(
152+
{
153+
"currency": "GBP",
154+
"amountAwarded": 100,
155+
"awardDate": "2019-10-03T00:00:00+00:00",
156+
"fundingOrganization": [{"id": "GB-CHC-12345"}],
157+
"recipientOrganization": [
158+
{"id": "GB-COH-12345"},
159+
],
160+
}
161+
)
162+
recipient.update_aggregate(
163+
{
164+
"currency": "GBP",
165+
"amountAwarded": 10000,
166+
"awardDate": "2020-10-03T00:00:00+00:00",
167+
"fundingOrganization": [{"id": "GB-CHC-67890"}],
168+
"recipientOrganization": [
169+
{"id": "GB-COH-12345"},
170+
],
171+
}
172+
)
173+
self.assertEqual(recipient.aggregate["funders"], 2)
174+
self.assertEqual(recipient.aggregate["currencies"]["GBP"]["funders"], 2)
175+
176+
def test_two_grants_from_different_funders_in_different_currencies(self):
177+
recipient = db.Recipient()
178+
recipient.update_aggregate(
179+
{
180+
"currency": "GBP",
181+
"amountAwarded": 100,
182+
"awardDate": "2019-10-03T00:00:00+00:00",
183+
"fundingOrganization": [{"id": "GB-CHC-12345"}],
184+
"recipientOrganization": [
185+
{"id": "GB-COH-12345"},
186+
],
187+
}
188+
)
189+
recipient.update_aggregate(
190+
{
191+
"currency": "EUR",
192+
"amountAwarded": 10000,
193+
"awardDate": "2020-10-03T00:00:00+00:00",
194+
"fundingOrganization": [{"id": "GB-CHC-67890"}],
195+
"recipientOrganization": [
196+
{"id": "GB-COH-12345"},
197+
],
198+
}
199+
)
200+
self.assertEqual(recipient.aggregate["funders"], 2)
201+
self.assertEqual(recipient.aggregate["currencies"]["GBP"]["funders"], 1)
202+
self.assertEqual(recipient.aggregate["currencies"]["EUR"]["funders"], 1)
203+
204+
def test_two_grants_from_same_funder_but_different_funder_ids_used(self):
205+
recipient = db.Recipient()
206+
recipient.update_aggregate(
207+
{
208+
"currency": "GBP",
209+
"amountAwarded": 100,
210+
"awardDate": "2019-10-03T00:00:00+00:00",
211+
"fundingOrganization": [{"id": "GB-PRIMARY-12345"}],
212+
"recipientOrganization": [
213+
{"id": "GB-COH-12345"},
214+
],
215+
}
216+
)
217+
recipient.update_aggregate(
218+
{
219+
"currency": "GBP",
220+
"amountAwarded": 10000,
221+
"awardDate": "2020-10-03T00:00:00+00:00",
222+
"fundingOrganization": [{"id": "GB-SECONDARY-12345"}],
223+
"recipientOrganization": [
224+
{"id": "GB-COH-12345"},
225+
],
226+
}
227+
)
228+
self.assertEqual(recipient.aggregate["funders"], 1)
229+
self.assertEqual(recipient.aggregate["currencies"]["GBP"]["funders"], 1)

0 commit comments

Comments
 (0)