Add command to retrieve all reports and generate filings.js (v1.1.0)

README.rst

@@ -42,7 +42,7 @@ Now you can interact with the various data in the cover element via
 .. code:: python
 
     >>> report.cover.type_of_filing
-    'COH-SS'
+    u'COH-SS'
     >>> report.cover.is_original
     True
     >>> report.cover.through_date - report.cover.from_date
@@ -54,7 +54,7 @@ the filer_type is a `IND`:
 .. code:: python
 
     >>> report.cover.filer.filer_type
-    'IND'
+    u'IND'
 
 The really interesting data is in the ``receipts`` property, a list
 of all of the contributions received in this report.  For example,

example/project/settings.py

@@ -177,7 +177,7 @@
             'level': 'ERROR',
             'propagate': True,
         },
-        'tx_elections': {
+        'tx_tecreports': {
             'level': os.environ.get('LOGGING_LEVEL', 'INFO'),
             'handlers': ['console', ],
             'propagate': False,

setup.py

@@ -27,7 +27,7 @@
 
 setup(
     name='tx_tecreports',
-    version='1.0.0',
+    version='1.1.0',
     description='Package for dealing with TEC reports',
     author='Tribune Tech',
     author_email='[email protected]',

tox.ini

@@ -6,8 +6,15 @@ downloadcache = {toxworkdir}/.cache
 deps =
 	mock==1.0.1
 	nose==1.3.0
+	dj_database_url==0.2.1
+	dj_settings_helpers==1.0.0
+	Django==1.5.4
+	South==0.8.3
 commands =
 	nosetests --with-doctest --doctest-extension=rst
+setenv =
+	PYTHONPATH=example/
+	DJANGO_SETTINGS_MODULE=project.settings
 
 [testenv:py33]
 ; For some reason, this doesn't install on the venv on my box

tx_tecreports/management/commands/generate_filings_js.py

@@ -0,0 +1,80 @@
+import csv
+import datetime
+import json
+import logging
+from optparse import make_option
+
+from django.core.management.base import BaseCommand
+
+from ...fetcher import get_filings_list
+
+logger = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    option_list = BaseCommand.option_list + (
+        make_option('--filename', action='store', dest='filename',
+                default='./data/curated-statewide-candidates-2014.csv',
+                help='File to use parse tecid and tecpacid from'),
+        make_option('--report-due', action='store', dest='raw_report_due',
+                help='Report due date of filing to parse (format: YYYY/MM/DD)'),
+    )
+
+    def get_matching_filing(self, filer_id, **kwargs):
+        logger.debug('fetching filings for %s' % filer_id)
+        filing_list = get_filings_list(filer_id)
+
+        logger.debug('found %d total filings' % len(filing_list))
+        matching_filings = filing_list.find(**kwargs)
+        matching_filings_len = len(matching_filings)
+        logger.debug('found %d matching filings' % matching_filings_len)
+
+        if matching_filings_len > 1:
+            logger.warning('Unable to process %s' % filer_id)
+            return
+        if matching_filings_len is 0:
+            logger.warning('Unable to find any filings for %s' % kwargs)
+            return
+        return matching_filings[0]
+
+    def generate_object(self, record, order, filer_id_key='tecid',
+            filer_type='', **kwargs):
+        filing = self.get_matching_filing(record[filer_id_key], **kwargs)
+        if filing is None:
+            return
+
+        return {
+            'display_name': record['name'],
+            'type': filer_type,
+            'race': record['race'],
+            'party': record['party'],
+            'filer_id': record[filer_id_key],
+            'order': order,
+            'report_id': filing.report_id,
+        }
+
+    def handle(self, filename, raw_report_due, **kwargs):
+        report_due = (datetime.datetime.strptime(raw_report_due, '%Y/%m/%d')
+                .date())
+
+        reader = csv.DictReader(open(filename, 'rb'))
+        data = []
+        for record in reader:
+            logger.info('Fetching for %s' % record['name'])
+            if record['tecid'] == '00032876':
+                logger.warn("Skipping %s because it's not properly formatted" %
+                        record['tecid'])
+                continue
+
+            obj = self.generate_object(record, record['new_order'],
+                    report_due=report_due)
+            if obj:
+                data.append(obj)
+
+            if record['tecpacid']:
+                logger.info('Fetching SPAC for %s' % record['name'])
+                obj = self.generate_object(record, int(record['new_order']) + 1,
+                        'tecpacid', filer_type='(SPAC)', report_due=report_due)
+                if obj:
+                    data.append(obj)
+        print json.dumps(data)

tx_tecreports/management/commands/retrieve_all_reports.py

@@ -0,0 +1,62 @@
+import csv
+import datetime
+import logging
+from optparse import make_option
+
+from django.core.management.base import BaseCommand
+
+from ...fetcher import get_filings_list
+
+logger = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    option_list = BaseCommand.option_list + (
+        make_option('--filename', action='store', dest='filename',
+                help='File to use parse tecid and tecpacid from'),
+        make_option('--report-due', action='store', dest='raw_report_due',
+                help='Report due date of filing to parse (format: YYYY/MM/DD)'),
+    )
+
+    def get_matching_filing(self, filer_id, **kwargs):
+        logger.debug('fetching filings for %s' % filer_id)
+        filing_list = get_filings_list(filer_id)
+
+        logger.debug('found %d total filings' % len(filing_list))
+        matching_filings = filing_list.find(**kwargs)
+        matching_filings_len = len(matching_filings)
+        logger.debug('found %d matching filings' % matching_filings_len)
+
+        if matching_filings_len > 1:
+            logger.warning('Unable to process %s' % filer_id)
+            return
+        if matching_filings_len is 0:
+            logger.warning('Unable to find any filings for %s' % kwargs)
+            return
+        return matching_filings[0]
+
+    def fetch_and_save(self, filer_id, **kwargs):
+        logger.info('finding filing for %s' % filer_id)
+        filing = self.get_matching_filing(filer_id, **kwargs)
+        if filing:
+            try:
+                filing.report.save()
+                logger.info('found %s and saved' % filer_id)
+            except KeyError:
+                logger.warn('KeyError on filing for %s' % filer_id)
+
+    def handle(self, filename, raw_report_due, **kwargs):
+        logger.debug('processing %s' % filename)
+
+        reader = csv.DictReader(open(filename, 'rb'))
+        report_due = datetime.datetime.strptime(raw_report_due, '%Y/%m/%d').date()
+
+        for record in reader:
+            # This report is known to be a malformed CSV
+            if record['tecid'] == '00032876':
+                logger.warn("Skipping %s because it's not properly formatted" %
+                        record['tecid'])
+                continue
+            self.fetch_and_save(record['tecid'], report_due=report_due)
+            if record['tecpacid']:
+                self.fetch_and_save(record['tecpacid'], report_due=report_due)