diff --git a/django-backend/fecfiler/devops/management/commands/get_overview.py b/django-backend/fecfiler/devops/management/commands/get_overview.py new file mode 100644 index 0000000000..33abca8e68 --- /dev/null +++ b/django-backend/fecfiler/devops/management/commands/get_overview.py @@ -0,0 +1,47 @@ +from fecfiler.devops.management.commands.fecfile_base import FECCommand +from ...utils.common_queries import ( + get_num_committees, + get_num_users, + get_num_reports, + get_num_reports_per_committee, + get_num_transactions_per_committee, + get_num_transactions_per_report, + get_num_transactions_per_contact, + get_transaction_types_breakdown, + get_transaction_tiers_breakdown, + get_carryover_type_transactions, +) +import structlog + +logger = structlog.get_logger(__name__) + + +class Command(FECCommand): + help = "Get an overview with various statistics." + command_name = "get_overview" + + def add_arguments(self, parser): + parser.add_argument( + "--committee_id", + type=str, + required=False, + default=None, + help="Committee by which to filter.", + ) + + def command(self, *args, **options): + committee_id = options["committee_id"] + if not committee_id: + get_num_committees() + get_num_users() + get_num_reports() + + get_num_reports_per_committee(committee_id) + get_num_transactions_per_committee(committee_id) + get_num_transactions_per_report(committee_id) + + if not committee_id: + get_num_transactions_per_contact() + get_transaction_types_breakdown() + get_transaction_tiers_breakdown() + get_carryover_type_transactions() diff --git a/django-backend/fecfiler/devops/utils/common_queries.py b/django-backend/fecfiler/devops/utils/common_queries.py new file mode 100644 index 0000000000..47957bdf69 --- /dev/null +++ b/django-backend/fecfiler/devops/utils/common_queries.py @@ -0,0 +1,182 @@ +from fecfiler.reports.models import Report +from fecfiler.committee_accounts.models import CommitteeAccount +from fecfiler.contacts.models import Contact +from fecfiler.transactions.models import Transaction +from fecfiler.transactions.schedule_c.models import ScheduleC +from fecfiler.transactions.schedule_c2.models import ScheduleC2 +from fecfiler.transactions.schedule_d.models import ScheduleD +from fecfiler.user.models import User +import structlog + +logger = structlog.get_logger(__name__) + + +def get_averages(items): + length = len(items) + if length == 0: + raise ValueError("Cannot get averages for an empty list") + + mean = sum(items) / length + if len(items) < 4: + return {"Mean": mean} + + items.sort() + first_q = items[:length // 4] + second_q = items[length // 4:length // 2] + third_q = items[length // 2:length // 4 * 3] + fourth_q = items[length // 4 * 3:] + return { + "1st quartile": first_q[-1], + "2nd quartile": second_q[-1], + "3rd quartile": third_q[-1], + "Max": fourth_q[-1], + "Mean": mean, + } + + +def print_keyvalues(dict): + for key in dict.keys(): + logger.info(f"{f' {key}: {dict[key]}':<60}") + + +def get_num_committees(): + logger.info(f"{f'Number of committees: {CommitteeAccount.objects.count()}':<60}") + + +def get_num_users(): + logger.info(f"{f'Number of users: {User.objects.count()}':<60}") + + +def get_num_reports(): + logger.info(f"{f'Number of reports: {Report.objects.count()}':<60}") + + +def get_num_reports_per_committee(committee_id=None): + if committee_id: + report_count = Report.objects.filter( + committee_account__committee_id=committee_id + ).count() + logger.info( + f"{( + f'Number of reports for committee {committee_id}: ' + + f'{report_count}' + ):<60}" + ) + else: + committee_report_counts = [] + for committee in CommitteeAccount.objects.all(): + r_count = Report.objects.filter(committee_account=committee).count() + committee_report_counts.append(r_count) + + averages = get_averages(committee_report_counts) + + logger.info(f"{f'Number of reports per committee:':<60}") + print_keyvalues(averages) + + +def get_num_transactions_per_committee(committee_id=None): + if committee_id: + transaction_count = Transaction.objects.filter( + committee_account__committee_id=committee_id + ).count() + logger.info( + f"{( + f'Number of transactions for committee {committee_id}: ' + + f'{transaction_count}' + ):<60}" + ) + else: + committee_transaction_counts = [] + highest_count = 0 + biggest_committee = None + for committee in CommitteeAccount.objects.all(): + t_count = Transaction.objects.filter(committee_account=committee).count() + committee_transaction_counts.append(t_count) + if t_count > highest_count: + highest_count = t_count + biggest_committee = committee + + averages = get_averages(committee_transaction_counts) + + logger.info(f"{f'Number of transactions per committee:':<60}") + print_keyvalues(averages) + logger.info( + f"{( + ' The largest committee is ' + + f'{biggest_committee.committee_id} with {highest_count} transactions' + ):<60}" + ) + + +def get_num_transactions_per_report(committee_id=None): + report_transaction_counts = [] + for report in Report.objects.all(): + if committee_id and report.committee_account.committee_id != committee_id: + continue + t_count = Transaction.objects.filter(reports=report).count() + report_transaction_counts.append(t_count) + + averages = get_averages(report_transaction_counts) + + logger.info( + f"{( + 'Number of transactions per report' + + (f' for committee_id {committee_id}:' if committee_id is not None else ':') + ):<60}" + ) + print_keyvalues(averages) + + +def get_num_transactions_per_contact(): + contact_transaction_counts = [] + for c in Contact.objects.all(): + ct_set_keys = [] + for i in range(1, 6): + ct_set_keys.append(f"contact_{i}_transaction_set") + + for n in ["I", "II", "III", "IV", "V"]: + ct_set_keys.append(f"contact_candidate_{n}_transaction_set") + + ct_set_keys.append("contact_affiliated_transaction_set") + transaction_count = 0 + for key in ct_set_keys: + transaction_count += getattr(c, key).count() + + contact_transaction_counts.append(transaction_count) + + averages = get_averages(contact_transaction_counts) + + logger.info(f"{f'Number of transactions per contact:':<60}") + print_keyvalues(averages) + + +def get_transaction_types_breakdown(): + tti_counts = {} + for transaction in Transaction.objects.all(): + tti = transaction.transaction_type_identifier + tti_counts[tti] = tti_counts.get(tti, 0) + 1 + + logger.info(f"{f'Transaction types breakdown:':<60}") + print_keyvalues(tti_counts) + + +def get_transaction_tiers_breakdown(): + filter_keys = [ + {"parent_transaction__isnull": True}, + { + "parent_transaction__isnull": False, + "parent_transaction__parent_transaction__isnull": True, + }, + {"parent_transaction__parent_transaction__isnull": False}, + ] + logger.info(f"{f'Transaction tiers breakdown:':<60}") + for i in range(3): + tier = "I" * (i + 1) + count = Transaction.objects.filter(**filter_keys[i]).count() + logger.info(f"{f' Tier {tier}: {count}':<60}") + + +def get_carryover_type_transactions(): + logger.info(f"{f'Carryover transactions:':<60}") + for model in [ScheduleC, ScheduleC2, ScheduleD]: + logger.info(f"{f' {model.__name__}: {model.objects.count()}':<60}") diff --git a/django-backend/fecfiler/transactions/fixtures/view_payloads.json b/django-backend/fecfiler/transactions/fixtures/view_payloads.json index 6b153af694..fc286dfe1b 100644 --- a/django-backend/fecfiler/transactions/fixtures/view_payloads.json +++ b/django-backend/fecfiler/transactions/fixtures/view_payloads.json @@ -248,6 +248,57 @@ "organization_name": null, "schedule_id": "B" }, + "LOAN_REPAYMENT_RECEIVED": { + "children":[], + "form_type":"SA14", + "transaction_type_identifier":"LOAN_REPAYMENT_RECEIVED", + "aggregation_group":"LINE_14", + "schema_name":"LOAN_REPAYMENT_RECEIVED", + "fields_to_validate":[ + "report_type", + "form_type", + "transaction_type_identifier", + "entity_type", + "contributor_organization_name", + "contributor_street_1", + "contributor_street_2", + "contributor_city", + "contributor_state", + "contributor_zip", + "contribution_date", + "contribution_amount", + "contribution_aggregate", + "aggregation_group", + "contribution_purpose_descrip", + "memo_code", + "memo_text_description", + "reattribution_redesignation_tag" + ], + "entity_type":"COM", + "contributor_organization_name":"test_committee_name", + "contributor_street_1":"test_street_1", + "contributor_street_2":null, + "contributor_city":"test_city", + "contributor_state":"AK", + "contributor_zip":"12345", + "contribution_date":"2026-03-17", + "contribution_amount":150, + "contribution_aggregate":150, + "contribution_purpose_descrip":"Loan Repayment", + "memo_code":null, + "date":null, + "amount":null, + "purpose_description":null, + "text4000":null, + "street_1":null, + "street_2":null, + "city":null, + "state":null, + "zip":null, + "aggregate":null, + "organization_name":null, + "schedule_id":"A" + }, "DEBT_REPAYMENT": { "children": [], "form_type": "SB21B", diff --git a/django-backend/fecfiler/transactions/models.py b/django-backend/fecfiler/transactions/models.py index 0210a14f6e..71cb685092 100644 --- a/django-backend/fecfiler/transactions/models.py +++ b/django-backend/fecfiler/transactions/models.py @@ -15,6 +15,7 @@ ) from fecfiler.transactions.utils_aggregation_service import ( update_aggregates_for_affected_transactions, + calculate_loan_payment_to_date ) from fecfiler.transactions.schedule_a.models import ScheduleA from fecfiler.transactions.schedule_b.models import ScheduleB @@ -562,6 +563,19 @@ def delete(self): error=str(e), exc_info=True, ) + + # Handle loan payment to date after marking deleted + try: + if self.is_loan_repayment(): + calculate_loan_payment_to_date(Transaction, self.loan_id) + except Exception as e: + logger.error( + "Failed to recalculate loan payment to date on delete", + transaction_id=self.id, + error=str(e), + exc_info=True, + ) + self.delete_children() self.delete_debts() self.delete_loans() diff --git a/django-backend/fecfiler/transactions/serializers.py b/django-backend/fecfiler/transactions/serializers.py index 5b2aa88c38..60f29fb610 100644 --- a/django-backend/fecfiler/transactions/serializers.py +++ b/django-backend/fecfiler/transactions/serializers.py @@ -192,41 +192,49 @@ def to_representation(self, instance): # because form_type is a dynamic field representation["form_type"] = instance.form_type - # represent parent - if instance.parent_transaction: - representation["parent_transaction"] = ( - TransactionSerializer().to_representation(instance.parent_transaction) - ) - # represent loan - if instance.loan: - representation["loan"] = TransactionSerializer().to_representation( - instance.loan - ) - # represent debt - if instance.debt: - representation["debt"] = TransactionSerializer().to_representation( - instance.debt - ) - # represent original reattribution/redesignation transaction - if instance.reatt_redes: - representation["reatt_redes"] = TransactionSerializer().to_representation( - instance.reatt_redes - ) + if not self.context.get("no_depth"): + new_context = {"no_depth": True, "no_children": True} + # represent parent + if instance.parent_transaction: + representation["parent_transaction"] = TransactionSerializer( + instance.parent_transaction, context=new_context + ).data + # represent loan + if instance.loan: + representation["loan"] = TransactionSerializer( + context=new_context + ).to_representation(instance.loan) + # represent debt + if instance.debt: + representation["debt"] = TransactionSerializer( + context=new_context + ).to_representation(instance.debt) + # represent original reattribution/redesignation transaction + if instance.reatt_redes: + representation["reatt_redes"] = TransactionSerializer( + context=new_context + ).to_representation(instance.reatt_redes) + + if instance.children.exists(): + representation["children"] = [ + TransactionSerializer(child, context={"no_children": True}).data + for child in instance.children.all() + ] - representation["reports"] = [] - representation["report_ids"] = [] - for report in instance.reports.all(): - representation["report_ids"].append(report.id) - representation["reports"].append( - { - "id": report.id, - "coverage_from_date": report.coverage_from_date, - "coverage_through_date": report.coverage_through_date, - "report_code": report.report_code, - "report_type": report.report_type, - "report_code_label": get_report_code_label(report), - } - ) + representation["reports"] = [] + representation["report_ids"] = [] + for report in instance.reports.all(): + representation["report_ids"].append(report.id) + representation["reports"].append( + { + "id": report.id, + "coverage_from_date": report.coverage_from_date, + "coverage_through_date": report.coverage_through_date, + "report_code": report.report_code, + "report_type": report.report_type, + "report_code_label": get_report_code_label(report), + } + ) representation["can_delete"] = instance.can_delete return representation diff --git a/django-backend/fecfiler/transactions/tests/test_models.py b/django-backend/fecfiler/transactions/tests/test_models.py index 979e64f39a..82709a41d8 100644 --- a/django-backend/fecfiler/transactions/tests/test_models.py +++ b/django-backend/fecfiler/transactions/tests/test_models.py @@ -81,8 +81,7 @@ def setUp(self): carry_forward_loans(self.m2_report) self.carried_forward_loan = ( - Transaction.objects - .filter(committee_account_id=self.committee.id) + Transaction.objects.filter(committee_account_id=self.committee.id) .order_by("created") .last() ) @@ -278,6 +277,29 @@ def test_delete_debt_transactions(self): undelete(first_repayment) undelete(second_repayment) + def test_delete_loan_repayment(self): + """Deleting a loan repayment should update the loan_payment_to_date + of current loan and carried forward""" + self.loan.refresh_from_db() + self.payment_1.refresh_from_db() + self.assertIsNone(self.payment_1.deleted) + self.assertEqual(self.loan.loan_payment_to_date, Decimal("1000.00")) + + self.payment_1.delete() + self.loan.refresh_from_db() + self.carried_forward_loan.refresh_from_db() + self.assertEqual( + self.loan.loan_payment_to_date, + Decimal("0.00"), + "Loan payment to date should be 0 after deleting loan repayment", + ) + self.assertEqual( + self.carried_forward_loan.loan_payment_to_date, + Decimal("600.00"), + "Carried forward loan payment to date should " + "be 600 after deleting loan repayment", + ) + def test_delete_loan_by_committee(self): self.assertIsNone(self.loan.deleted) self.assertIsNone(self.loan_made.deleted) diff --git a/django-backend/fecfiler/transactions/tests/test_transaction_dependencies.py b/django-backend/fecfiler/transactions/tests/test_transaction_dependencies.py index e70415b9d3..c159ab615a 100644 --- a/django-backend/fecfiler/transactions/tests/test_transaction_dependencies.py +++ b/django-backend/fecfiler/transactions/tests/test_transaction_dependencies.py @@ -185,6 +185,51 @@ def test_update_dependent_parents(self): partnership_memo.refresh_from_db() self.assertEqual( partnership_memo.schedule_a.contribution_purpose_descrip, - "JF Memo: Parent Contact" - + " (Partnership attributions do not meet itemization threshold)", + "JF Memo: Parent Contact " + + "(Partnership attributions do not meet itemization threshold)", + ) + + def test_partnership_receipt(self): + parent = create_schedule_a( + "PARTNERSHIP_RECEIPT", + self.committee, + self.parent_contact, + "2020-01-01", + 100, + "GENERAL", + "SA11AI", + False, + None, + None, + None, + None, + "(Partnership attributions do not meet itemization threshold)", + ) + partnership_attribution = create_schedule_a( + "PARTNERSHIP_ATTRIBUTION", + self.committee, + self.parent_contact, + "2020-01-01", + 100, + "GENERAL", + "SA11AI", + False, + None, + None, + parent.id, + ) + update_dependent_parent(partnership_attribution) + parent.refresh_from_db() + self.assertEqual( + parent.schedule_a.contribution_purpose_descrip, + "(See Partnership Attribution(s) below)", + ) + + partnership_attribution.delete() + update_dependent_parent(partnership_attribution) + parent.refresh_from_db() + + self.assertEqual( + parent.schedule_a.contribution_purpose_descrip, + "(Partnership attributions do not meet itemization threshold)", ) diff --git a/django-backend/fecfiler/transactions/tests/test_views.py b/django-backend/fecfiler/transactions/tests/test_views.py index 0160fcf4c2..c5766eb4af 100644 --- a/django-backend/fecfiler/transactions/tests/test_views.py +++ b/django-backend/fecfiler/transactions/tests/test_views.py @@ -10,6 +10,7 @@ from fecfiler.transactions.models import Transaction from fecfiler.committee_accounts.models import CommitteeAccount from fecfiler.reports.tests.utils import create_form3x +from fecfiler.transactions.utils_aggregation_service import calculate_loan_payment_to_date from fecfiler.contacts.tests.utils import ( create_test_committee_contact, create_test_individual_contact, @@ -864,7 +865,7 @@ def test_destroy_with_dependent_parent(self): parent_id=jf_transfer.id, ) partnership_memo.schedule_a.contribution_purpose_descrip = ( - "JF Memo: None (See Partnership Attribution(s) below)" + "JF Memo: (See Partnership Attribution(s) below)" ) partnership_memo.schedule_a.save() partnership_attribution_memo = create_schedule_a( @@ -883,7 +884,7 @@ def test_destroy_with_dependent_parent(self): ) self.assertEqual( response.data["contribution_purpose_descrip"], - "JF Memo: None (See Partnership Attribution(s) below)", + "JF Memo: (See Partnership Attribution(s) below)", ) # If we delete the attribution memo, the parent memo should be updated @@ -903,9 +904,78 @@ def test_destroy_with_dependent_parent(self): ) self.assertEqual( response.data["contribution_purpose_descrip"], - "JF Memo: None (Partnership attributions do not meet itemization threshold)", + "JF Memo: (Partnership attributions do not meet itemization threshold)", ) + def test_loan_repayment_on_loan_by_committee(self): + """Loan repayments should update loan balances on the original loan + and on carried forward copies""" + # create original report + q1_report = create_form3x(self.committee, "2025-01-01", "2025-03-31", {}) + + # create loan + original_loan = create_loan( + self.committee, + self.test_com_contact, + "1000.00", + "2025-12-31", + "6%", + form_type="SC/10", + loan_incurred_date="2025-01-01", + report=q1_report, + type="LOAN_BY_COMMITTEE", + ) + + # test calculate_loan_payment_to_date does not fail with + # no repayments + calculate_loan_payment_to_date(Transaction, original_loan.id) + + # carry forward to additional reports + create_form3x(self.committee, "2025-04-01", "2025-06-30", {}) + create_form3x(self.committee, "2025-07-01", "2025-09-30", {}) + create_form3x(self.committee, "2025-10-01", "2025-12-31", {}) + create_form3x(self.committee, "2026-01-01", "2026-03-31", {}) + + # create repayment + create_schedule_a( + "LOAN_REPAYMENT_RECEIVED", + self.committee, + self.test_com_contact, + "2025-01-02", + "100.00", + loan_id=original_loan.id, + report=q1_report, + ) + + # assert that loan repayments are being calculated + annotated_copy_of_loan = Transaction.objects.get(id=original_loan.id) + self.assertEqual(annotated_copy_of_loan.loan_payment_to_date, 100.00) + self.assertEqual(annotated_copy_of_loan.loan_balance, 900.00) + + # make another repayment in q1 + q1_loan_repayment = self.create_loan_repayment_received_payload( + original_loan, + q1_report, + "2025-01-03", + 150.00, + ) + response = TransactionViewSet().create(self.post_request(q1_loan_repayment)) + self.assertEqual(response.status_code, 200) + + # refresh annotations + annotated_copy_of_loan = Transaction.objects.get(id=original_loan.id) + self.assertEqual(annotated_copy_of_loan.loan_payment_to_date, 250.00) + self.assertEqual(annotated_copy_of_loan.loan_balance, 750.00) + + # ensure that the balance is updated appropriately on carried forward loans + child_loans = Transaction.objects.filter( + loan_id=original_loan.id, schedule_c__isnull=False + ) + + self.assertEqual(child_loans.count(), 4) + for child_loan in child_loans: + self.assertEqual(child_loan.loan_balance, 750.00) + def test_delete_carried_forward_loan_on_repayment_to_orignal_loan(self): """Paying off a loan in the original report should delete any carried forward copies in future reports""" @@ -1380,6 +1450,24 @@ def create_loan_repayment_payload( loan_repayment_payload["expenditure_amount"] = repayment_amount return loan_repayment_payload + def create_loan_repayment_received_payload( + self, + loan: Transaction, + report: Report, + repayment_date: str, + repayment_amount: int, + ): + loan_representation = self.transaction_serializer.to_representation(loan) + loan_repayment_payload = deepcopy(self.payloads["LOAN_REPAYMENT_RECEIVED"]) + loan_repayment_payload["contact_1"] = loan_representation["contact_1"] + loan_repayment_payload["contact_1_id"] = loan_representation["contact_1_id"] + loan_repayment_payload["loan"] = loan_representation + loan_repayment_payload["loan_id"] = loan_representation["id"] + loan_repayment_payload["report_ids"] = [str(report.id)] + loan_repayment_payload["contribution_date"] = repayment_date + loan_repayment_payload["contribution_amount"] = repayment_amount + return loan_repayment_payload + def create_debt_repayment_payload( self, debt: Transaction, diff --git a/django-backend/fecfiler/transactions/tests/utils.py b/django-backend/fecfiler/transactions/tests/utils.py index b5e2564d50..06ba64db8e 100644 --- a/django-backend/fecfiler/transactions/tests/utils.py +++ b/django-backend/fecfiler/transactions/tests/utils.py @@ -31,12 +31,14 @@ def create_schedule_a( itemized: bool | None = None, report: Report | None = None, parent_id: UUID | None = None, + loan_id: UUID | None = None, purpose_description: str | None = None, ): transaction_data = { "_form_type": form_type, "memo_code": memo_code, "force_itemized": itemized, + "loan_id": loan_id, } if parent_id is not None: transaction_data["parent_transaction_id"] = parent_id diff --git a/django-backend/fecfiler/transactions/transaction_dependencies.py b/django-backend/fecfiler/transactions/transaction_dependencies.py index 02b5fc553d..b8d09930f5 100644 --- a/django-backend/fecfiler/transactions/transaction_dependencies.py +++ b/django-backend/fecfiler/transactions/transaction_dependencies.py @@ -51,16 +51,19 @@ def update_dependent_children(transaction: Transaction): def update_dependent_parent(transaction: Transaction): """Update the contribution_purpose_descrip field for PARTNERSHIP_MEMO transactions when thier children are created or deleted.""" + if transaction.transaction_type_identifier in PARTNERSHIP_ATTRIBUTIONS: parent = transaction.parent_transaction - grandparent = parent.parent_transaction - dependencies = JF_TRANSFER_DEPENDENCIES[grandparent.transaction_type_identifier] - _, _, partnership_no_children_update, partnership_with_children_update = ( - get_jf_transfer_descriptions( - dependencies["prefix"], grandparent.contact_1.name - ) + grandparent = parent.parent_transaction or {"transaction_type_identifier": None} + grandparent_dependencies = JF_TRANSFER_DEPENDENCIES.get( + getattr(grandparent, "transaction_type_identifier", None), {} ) + prefix = grandparent_dependencies.get("prefix", "") + committee_name = getattr(getattr(grandparent, "contact_1", None), "name", "") + _, _, partnership_no_children_update, partnership_with_children_update = ( + get_jf_transfer_descriptions(prefix, committee_name) + ) ScheduleA.objects.filter(transaction__id=parent.id).update( contribution_purpose_descrip=Subquery( ScheduleA.objects.filter(id=OuterRef("id")) @@ -120,7 +123,7 @@ def get_jf_transfer_descriptions(memo_prefix: str, commmittee_name: str): 4. The description for partnership memos with grandchildren (ex: "JF Memo: Committee Name (See Partnership Attribution(s) below)") """ - committee_clause = f"{memo_prefix} {commmittee_name}" + committee_clause = " ".join(filter(None, [memo_prefix, commmittee_name])) attribution_description = get_truncated_description( committee_clause, "(Partnership Attribution)" ) @@ -144,7 +147,7 @@ def get_truncated_description(description: str, parenthetical: str): and append a parenthetical.""" if len(description + parenthetical) > 100: description = description[: 96 - len(parenthetical)] + "..." - return f"{description} {parenthetical}" + return " ".join(filter(None, [description, parenthetical])) # Dictionary of joint fundraising transfer dependencies. @@ -211,6 +214,7 @@ def get_truncated_description(description: str, parenthetical: str): # List of transaction types that are partnership attributions. PARTNERSHIP_ATTRIBUTIONS = [ + "PARTNERSHIP_ATTRIBUTION", "PARTNERSHIP_ATTRIBUTION_JF_TRANSFER_MEMO", "PARTNERSHIP_ATTRIBUTION_NATIONAL_PARTY_CONVENTION_JF_TRANSFER_MEMO", "PARTNERSHIP_ATTRIBUTION_NATIONAL_PARTY_HEADQUARTERS_JF_TRANSFER_MEMO", diff --git a/django-backend/fecfiler/transactions/utils_aggregation_service.py b/django-backend/fecfiler/transactions/utils_aggregation_service.py index 7bbbb2e787..4ba22f98d2 100644 --- a/django-backend/fecfiler/transactions/utils_aggregation_service.py +++ b/django-backend/fecfiler/transactions/utils_aggregation_service.py @@ -9,7 +9,8 @@ from typing import Optional, Dict, Any from uuid import UUID -from django.db.models import Q, F, Value +from django.db.models import Q, F, Value, OuterRef, Subquery, Func +from django.db.models.functions import Coalesce from django.db.models.fields import DecimalField import structlog @@ -675,9 +676,9 @@ def calculate_loan_payment_to_date(transaction_model, loan_id: UUID) -> int: """ Calculate the cumulative loan payment to date for a given loan. - Sums all LOAN_REPAYMENT_MADE transactions associated with the loan, - ordered by date and creation time. For carried forward loans, also includes - repayments from the parent loan chain. + Sums all LOAN_REPAYMENT_MADE and LOAN_REPAYMENT_RECEIVED transactions + associated with the loan, ordered by date and creation time. + For carried forward loans, also includes repayments from the parent loan chain. Args: transaction_model: Transaction model class @@ -688,63 +689,46 @@ def calculate_loan_payment_to_date(transaction_model, loan_id: UUID) -> int: """ # Get the loan transaction try: - loan = ( - transaction_model - .objects.select_related("loan", "loan__schedule_c") - .get(id=loan_id, deleted__isnull=True) - ) + loan = transaction_model.objects.select_related("schedule_c").get(id=loan_id) except transaction_model.DoesNotExist: logger.warning(f"Loan transaction {loan_id} not found") return 0 - # Collect all loan IDs in the parent chain (for carried forward loans) - # Carried forward loans reference their parent via the 'loan' field - loan_ids = [loan_id] - current_loan = loan - - # Walk up the loan chain to find the original loan - max_depth = 10 # Prevent infinite loops - depth = 0 - while current_loan.loan_id and depth < max_depth: - parent_loan = current_loan.loan - if parent_loan and parent_loan.schedule_c: - loan_ids.append(parent_loan.id) - current_loan = parent_loan - depth += 1 - else: - break - - # Get all loan repayment transactions for all loans in the chain - repayments = transaction_model.objects.filter( - loan_id__in=loan_ids, - transaction_type_identifier="LOAN_REPAYMENT_MADE", - deleted__isnull=True, - schedule_b__isnull=False, - ).order_by("date", "created").select_related("schedule_b") - - # Calculate the cumulative payment amount - total_payment = Decimal(0) - for repayment in repayments: - if repayment.schedule_b and repayment.schedule_b.expenditure_amount: - total_payment += repayment.schedule_b.expenditure_amount - - # Update the loan transaction's loan_payment_to_date field - if loan.loan_payment_to_date != total_payment: - loan.loan_payment_to_date = total_payment - loan.save(update_fields=["loan_payment_to_date"]) - logger.info( - f"Updated loan_payment_to_date for loan {loan_id}: {total_payment}" + loan_payment_to_date_subquery = Coalesce( + Subquery( + transaction_model.objects.filter( + Q( + schedule_b__isnull=False, + transaction_type_identifier="LOAN_REPAYMENT_MADE", + schedule_b__expenditure_date__lte=OuterRef("report_through_date"), + ) + | Q( + schedule_a__isnull=False, + transaction_type_identifier="LOAN_REPAYMENT_RECEIVED", + schedule_a__contribution_date__lte=OuterRef("report_through_date"), + ), + loan__transaction_id=OuterRef("transaction_id"), + ) + .order_by() + .annotate(payment_to_date=Func(F("amount"), function="SUM")) + .values("payment_to_date")[:1] + ), + Decimal(0), + ) + return ( + transaction_model.objects.filter( + Q(transaction_id=loan.transaction_id), + schedule_c__isnull=False, ) - # Propagate updated totals to any carried-forward child loans - child_loans = transaction_model.objects.filter( - loan_id=loan_id, - deleted__isnull=True, - ).exclude(id=loan_id).values_list("id", flat=True) - for child_loan_id in child_loans: - calculate_loan_payment_to_date(transaction_model, child_loan_id) - return 1 - - return 0 + .annotate( + report_through_date=Subquery( + transaction_model.objects.filter(id=OuterRef("id")).values( + "schedule_c__report_coverage_through_date" + )[:1] + ) + ) + .update(loan_payment_to_date=loan_payment_to_date_subquery) + ) def recalculate_aggregates_for_transaction( @@ -775,8 +759,11 @@ def recalculate_aggregates_for_transaction( try: if schedule in CONTACT_AGGREGATE_SCHEDULES: # If this is a loan repayment, recalculate loan_payment_to_date - if (transaction_instance.transaction_type_identifier == "LOAN_REPAYMENT_MADE" - and transaction_instance.loan_id): + if ( + transaction_instance.transaction_type_identifier in [ + "LOAN_REPAYMENT_MADE", "LOAN_REPAYMENT_RECEIVED" + ] and transaction_instance.loan_id + ): calculate_loan_payment_to_date( transaction_model, transaction_instance.loan_id ) diff --git a/django-backend/fecfiler/web_services/migrations/0004_uploadsubmission_date_filed.py b/django-backend/fecfiler/web_services/migrations/0004_uploadsubmission_date_filed.py new file mode 100644 index 0000000000..47959e42cc --- /dev/null +++ b/django-backend/fecfiler/web_services/migrations/0004_uploadsubmission_date_filed.py @@ -0,0 +1,18 @@ +# Generated by Django 5.2.11 on 2026-03-09 19:29 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('web_services', '0003_uploadsubmission_fecfile_polling_attempts_and_more'), + ] + + operations = [ + migrations.AddField( + model_name='uploadsubmission', + name='date_filed', + field=models.DateTimeField(null=True), + ), + ] diff --git a/django-backend/fecfiler/web_services/models.py b/django-backend/fecfiler/web_services/models.py index d355e9ddde..7a432d4595 100644 --- a/django-backend/fecfiler/web_services/models.py +++ b/django-backend/fecfiler/web_services/models.py @@ -202,6 +202,7 @@ class UploadSubmission(BaseSubmission): # different from internal report id fec_report_id = models.CharField(max_length=255, null=True) + date_filed = models.DateTimeField(null=True) objects = UploadSubmissionManager() @@ -221,6 +222,7 @@ def save_fec_response(self, response_string): FECStatus.COMPLETED.value, ): logger.info(f"FEC upload successful: {response_string}") + self.date_filed = datetime.now() else: logger.error(f"FEC upload failed: {response_string}") diff --git a/django-backend/manage.py b/django-backend/manage.py index 1a4f829542..f2b41e9467 100755 --- a/django-backend/manage.py +++ b/django-backend/manage.py @@ -25,6 +25,7 @@ "reset_submitting_report", "fail_open_submissions", "dump_committee_data", + "get_overview", ] restricted_commands = [ "loaddata", diff --git a/requirements.txt b/requirements.txt index 8ff782954f..a6857ba961 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ Django==5.2.11 djangorestframework==3.16.0 drf-spectacular==0.28.0 Faker==37.6.0 -git+https://github.com/fecgov/fecfile-validate@7939bbc7b41067f2310874156cdb0fdadbd0fa33#egg=fecfile_validate&subdirectory=fecfile_validate_python +git+https://github.com/fecgov/fecfile-validate@0cf017439a59a08a7bed02e2069045708887209c#egg=fecfile_validate&subdirectory=fecfile_validate_python github3.py==4.0.1 GitPython==3.1.43 gunicorn==23.0.0