Release/sprint 81 -> test

django-backend/fecfiler/devops/management/commands/get_overview.py

@@ -0,0 +1,47 @@
+from fecfiler.devops.management.commands.fecfile_base import FECCommand
+from ...utils.common_queries import (
+    get_num_committees,
+    get_num_users,
+    get_num_reports,
+    get_num_reports_per_committee,
+    get_num_transactions_per_committee,
+    get_num_transactions_per_report,
+    get_num_transactions_per_contact,
+    get_transaction_types_breakdown,
+    get_transaction_tiers_breakdown,
+    get_carryover_type_transactions,
+)
+import structlog
+
+logger = structlog.get_logger(__name__)
+
+
+class Command(FECCommand):
+    help = "Get an overview with various statistics."
+    command_name = "get_overview"
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--committee_id",
+            type=str,
+            required=False,
+            default=None,
+            help="Committee by which to filter.",
+        )
+
+    def command(self, *args, **options):
+        committee_id = options["committee_id"]
+        if not committee_id:
+            get_num_committees()
+            get_num_users()
+            get_num_reports()
+
+        get_num_reports_per_committee(committee_id)
+        get_num_transactions_per_committee(committee_id)
+        get_num_transactions_per_report(committee_id)
+
+        if not committee_id:
+            get_num_transactions_per_contact()
+            get_transaction_types_breakdown()
+            get_transaction_tiers_breakdown()
+            get_carryover_type_transactions()

django-backend/fecfiler/devops/utils/common_queries.py

@@ -0,0 +1,182 @@
+from fecfiler.reports.models import Report
+from fecfiler.committee_accounts.models import CommitteeAccount
+from fecfiler.contacts.models import Contact
+from fecfiler.transactions.models import Transaction
+from fecfiler.transactions.schedule_c.models import ScheduleC
+from fecfiler.transactions.schedule_c2.models import ScheduleC2
+from fecfiler.transactions.schedule_d.models import ScheduleD
+from fecfiler.user.models import User
+import structlog
+
+logger = structlog.get_logger(__name__)
+
+
+def get_averages(items):
+    length = len(items)
+    if length == 0:
+        raise ValueError("Cannot get averages for an empty list")
+
+    mean = sum(items) / length
+    if len(items) < 4:
+        return {"Mean": mean}
+
+    items.sort()
+    first_q = items[:length // 4]
+    second_q = items[length // 4:length // 2]
+    third_q = items[length // 2:length // 4 * 3]
+    fourth_q = items[length // 4 * 3:]
+    return {
+        "1st quartile": first_q[-1],
+        "2nd quartile": second_q[-1],
+        "3rd quartile": third_q[-1],
+        "Max": fourth_q[-1],
+        "Mean": mean,
+    }
+
+
+def print_keyvalues(dict):
+    for key in dict.keys():
+        logger.info(f"{f'   {key}: {dict[key]}':<60}")
+
+
+def get_num_committees():
+    logger.info(f"{f'Number of committees: {CommitteeAccount.objects.count()}':<60}")
+
+
+def get_num_users():
+    logger.info(f"{f'Number of users: {User.objects.count()}':<60}")
+
+
+def get_num_reports():
+    logger.info(f"{f'Number of reports: {Report.objects.count()}':<60}")
+
+
+def get_num_reports_per_committee(committee_id=None):
+    if committee_id:
+        report_count = Report.objects.filter(
+            committee_account__committee_id=committee_id
+        ).count()
+        logger.info(
+            f"{(
+                f'Number of reports for committee {committee_id}: '
+                + f'{report_count}'
+            ):<60}"
+        )
+    else:
+        committee_report_counts = []
+        for committee in CommitteeAccount.objects.all():
+            r_count = Report.objects.filter(committee_account=committee).count()
+            committee_report_counts.append(r_count)
+
+        averages = get_averages(committee_report_counts)
+
+        logger.info(f"{f'Number of reports per committee:':<60}")
+        print_keyvalues(averages)
+
+
+def get_num_transactions_per_committee(committee_id=None):
+    if committee_id:
+        transaction_count = Transaction.objects.filter(
+            committee_account__committee_id=committee_id
+        ).count()
+        logger.info(
+            f"{(
+                f'Number of transactions for committee {committee_id}: '
+                + f'{transaction_count}'
+            ):<60}"
+        )
+    else:
+        committee_transaction_counts = []
+        highest_count = 0
+        biggest_committee = None
+        for committee in CommitteeAccount.objects.all():
+            t_count = Transaction.objects.filter(committee_account=committee).count()
+            committee_transaction_counts.append(t_count)
+            if t_count > highest_count:
+                highest_count = t_count
+                biggest_committee = committee
+
+        averages = get_averages(committee_transaction_counts)
+
+        logger.info(f"{f'Number of transactions per committee:':<60}")
+        print_keyvalues(averages)
+        logger.info(
+            f"{(
+                '   The largest committee is '
+                + f'{biggest_committee.committee_id} with {highest_count} transactions'
+            ):<60}"
+        )
+
+
+def get_num_transactions_per_report(committee_id=None):
+    report_transaction_counts = []
+    for report in Report.objects.all():
+        if committee_id and report.committee_account.committee_id != committee_id:
+            continue
+        t_count = Transaction.objects.filter(reports=report).count()
+        report_transaction_counts.append(t_count)
+
+    averages = get_averages(report_transaction_counts)
+
+    logger.info(
+        f"{(
+            'Number of transactions per report'
+            + (f' for committee_id {committee_id}:' if committee_id is not None else ':')
+        ):<60}"
+    )
+    print_keyvalues(averages)
+
+
+def get_num_transactions_per_contact():
+    contact_transaction_counts = []
+    for c in Contact.objects.all():
+        ct_set_keys = []
+        for i in range(1, 6):
+            ct_set_keys.append(f"contact_{i}_transaction_set")
+
+        for n in ["I", "II", "III", "IV", "V"]:
+            ct_set_keys.append(f"contact_candidate_{n}_transaction_set")
+
+        ct_set_keys.append("contact_affiliated_transaction_set")
+        transaction_count = 0
+        for key in ct_set_keys:
+            transaction_count += getattr(c, key).count()
+
+        contact_transaction_counts.append(transaction_count)
+
+    averages = get_averages(contact_transaction_counts)
+
+    logger.info(f"{f'Number of transactions per contact:':<60}")
+    print_keyvalues(averages)
+
+
+def get_transaction_types_breakdown():
+    tti_counts = {}
+    for transaction in Transaction.objects.all():
+        tti = transaction.transaction_type_identifier
+        tti_counts[tti] = tti_counts.get(tti, 0) + 1
+
+    logger.info(f"{f'Transaction types breakdown:':<60}")
+    print_keyvalues(tti_counts)
+
+
+def get_transaction_tiers_breakdown():
+    filter_keys = [
+        {"parent_transaction__isnull": True},
+        {
+            "parent_transaction__isnull": False,
+            "parent_transaction__parent_transaction__isnull": True,
+        },
+        {"parent_transaction__parent_transaction__isnull": False},
+    ]
+    logger.info(f"{f'Transaction tiers breakdown:':<60}")
+    for i in range(3):
+        tier = "I" * (i + 1)
+        count = Transaction.objects.filter(**filter_keys[i]).count()
+        logger.info(f"{f'   Tier {tier}: {count}':<60}")
+
+
+def get_carryover_type_transactions():
+    logger.info(f"{f'Carryover transactions:':<60}")
+    for model in [ScheduleC, ScheduleC2, ScheduleD]:
+        logger.info(f"{f'   {model.__name__}: {model.objects.count()}':<60}")

django-backend/fecfiler/transactions/fixtures/view_payloads.json

@@ -248,6 +248,57 @@
     "organization_name": null,
     "schedule_id": "B"
   },
+  "LOAN_REPAYMENT_RECEIVED": {
+    "children":[],
+    "form_type":"SA14",
+    "transaction_type_identifier":"LOAN_REPAYMENT_RECEIVED",
+    "aggregation_group":"LINE_14",
+    "schema_name":"LOAN_REPAYMENT_RECEIVED",
+    "fields_to_validate":[
+      "report_type",
+      "form_type",
+      "transaction_type_identifier",
+      "entity_type",
+      "contributor_organization_name",
+      "contributor_street_1",
+      "contributor_street_2",
+      "contributor_city",
+      "contributor_state",
+      "contributor_zip",
+      "contribution_date",
+      "contribution_amount",
+      "contribution_aggregate",
+      "aggregation_group",
+      "contribution_purpose_descrip",
+      "memo_code",
+      "memo_text_description",
+      "reattribution_redesignation_tag"
+    ],
+    "entity_type":"COM",
+    "contributor_organization_name":"test_committee_name",
+    "contributor_street_1":"test_street_1",
+    "contributor_street_2":null,
+    "contributor_city":"test_city",
+    "contributor_state":"AK",
+    "contributor_zip":"12345",
+    "contribution_date":"2026-03-17",
+    "contribution_amount":150,
+    "contribution_aggregate":150,
+    "contribution_purpose_descrip":"Loan Repayment",
+    "memo_code":null,
+    "date":null,
+    "amount":null,
+    "purpose_description":null,
+    "text4000":null,
+    "street_1":null,
+    "street_2":null,
+    "city":null,
+    "state":null,
+    "zip":null,
+    "aggregate":null,
+    "organization_name":null,
+    "schedule_id":"A"
+  },
   "DEBT_REPAYMENT": {
     "children": [],
     "form_type": "SB21B",

django-backend/fecfiler/transactions/models.py

@@ -15,6 +15,7 @@
 )
 from fecfiler.transactions.utils_aggregation_service import (
     update_aggregates_for_affected_transactions,
+    calculate_loan_payment_to_date
 )
 from fecfiler.transactions.schedule_a.models import ScheduleA
 from fecfiler.transactions.schedule_b.models import ScheduleB
@@ -562,6 +563,19 @@ def delete(self):
                     error=str(e),
                     exc_info=True,
                 )
+
+            # Handle loan payment to date after marking deleted
+            try:
+                if self.is_loan_repayment():
+                    calculate_loan_payment_to_date(Transaction, self.loan_id)
+            except Exception as e:
+                logger.error(
+                    "Failed to recalculate loan payment to date on delete",
+                    transaction_id=self.id,
+                    error=str(e),
+                    exc_info=True,
+                )
+
             self.delete_children()
             self.delete_debts()
             self.delete_loans()