Refactor to be a slightly more user-friendly and functional CLI

Author: chriszs

bin/fec

@@ -0,0 +1,19 @@
+#!/usr/bin/env node
+
+const commands = require('../commands'),
+    pkg = require('../package.json'),
+    updateNotifier = require('update-notifier'),
+    yargs = require('yargs');
+
+updateNotifier({pkg}).notify();
+
+yargs
+    .usage('$0 <cmd> [args]')
+    .env('FEC')
+    .command(commands.init)
+    .command(commands.list)
+    .command(commands.convert)
+    .recommendCommands()
+    .demandCommand()
+    .help()
+    .argv;

bin/fecloader

@@ -0,0 +1,24 @@
+const fec = require('../');
+
+module.exports = {
+    command: 'convert',
+    describe:
+        'Pipe in an .fec file to convert it to newline-delimited JSON or psql COPY commands.',
+    builder(yargs) {
+        return yargs
+            .positional('filing_id', {
+                describe: 'the filing ID of the filing'
+            })
+            .options({
+                format: {
+                    type: 'string',
+                    default: 'ndjson',
+                    choices: ['ndjson', 'psql'],
+                    describe: 'choose output format'
+                }
+            });
+    },
+    handler(options) {
+        fec[options.format](options);
+    }
+};

commands/convert.js

@@ -0,0 +1,6 @@
+
+module.exports = {
+	init: require('./init'),
+    list: require('./list'),
+    convert: require('./convert')
+};

commands/index.js

@@ -0,0 +1,8 @@
+const { init } = require('../');
+
+module.exports = {
+    command: 'init',
+    describe: 'Create a SQL schema in a database; uses PGHOST, PGDATABASE, PGUSER, PGPASSWORD environment variables to authenticate.',
+    builder: {},
+    handler: init
+};

commands/init.js

@@ -0,0 +1,77 @@
+const { api, rss, format } = require('../');
+
+module.exports = {
+    command: 'list',
+    describe: 'Show a list of the latest e-filings from the FEC API or RSS feed.',
+    builder: {
+        key: {
+            type: 'string',
+            alias: 'fecKey',
+            describe:
+                'data.gov API key authorizing access to the FEC API, can set using environment variable FEC_KEY'
+        },
+        rss: {
+            type: 'boolean',
+            default: false,
+            describe: 'use the FEC RSS feed as the source'
+        },
+        committee: {
+            type: 'string',
+            describe: 'ID of a committee for which to show filings'
+        },
+        format: {
+            type: 'string',
+            default: 'table',
+            choices: ['table','ndjson','json','csv','tsv'],
+            describe: 'choose output format'
+        },
+        headers: {
+            type: 'boolean',
+            default: true,
+            describe: 'show column headers'
+        },
+        columns: {
+            type: 'string',
+            describe: 'choose columns to show'
+        },
+        columnLength: {
+            type: 'integer',
+            default: 30,
+            describe:
+                'how many characters to show in each table cell'
+        }
+    },
+    handler: async options => {
+        if (!options.rss && !options.key) {
+            console.error(
+                'Error: key needed to use FEC API, ' +
+                    'specify using --key or the FEC_KEY environment variable, ' +
+                    'get one at https://api.data.gov/signup/' +
+                    'or use --rss instead'
+            );
+            return;
+        }
+
+        const columns = options.columns
+            ? options.columns.split(',')
+            : options.format == 'table'
+            ? [
+                  'fec_file_id',
+                  'form_type',
+                  'coverage_end_date',
+                  'committee_id',
+                  'committee_name'
+              ]
+            : null;
+
+        try {
+            let filings = await (options.rss ? rss : api)(options);
+
+            console.log(
+                format(filings, options.format, columns, options.headers, options.columnLength)
+            );
+        } catch (err) {
+            console.error(err);
+        }
+    }
+};

commands/list.js

@@ -1,9 +1,10 @@
 
 module.exports = {
     api: require('./lib/api'),
-    create: require('./lib/create'),
+    format: require('./lib/format'),
     increment: require('./lib/increment'),
-    json: require('./lib/json'),
+    init: require('./lib/init'),
+    ndjson: require('./lib/ndjson'),
     psql: require('./lib/psql'),
     rss: require('./lib/rss')
 };

index.js

@@ -1,23 +1,44 @@
-const fetch = require('cross-fetch');
+const axios = require('axios');
 
-module.exports = async (options) => {
-    const apiUrl = `https://api.open.fec.gov/v1/efile/filings/?sort=-receipt_date&per_page=100&api_key=${
-        options.fec_key ? options.fec_key : ''
-    }&page=1&cache=${Math.round(Math.random() * 100)}`;
+module.exports = async options => {
+    if (!options.key) {
+        throw new Error('Error: key needed to use FEC API,' +
+            'specify using options.key, ' +
+            'or get one at https://api.data.gov/signup/');
+    }
+
+    const fecAPI = axios.create({
+        baseURL: 'https://api.open.fec.gov/v1/',
+        params: {
+            api_key: options.key
+        }
+    });
 
-    try {
-        const res = await fetch(apiUrl);
+    let endpoint = options.committee ?
+        `/committee/${options.committee}/filings/` :
+        'efile/filings';
 
-        if (res.status >= 400) {
-            throw new Error('Bad response from server');
+    const res = await fecAPI.get(endpoint, {
+        params: {
+            sort: '-receipt_date',
+            per_page: 100,
+            page: 1
         }
+    });
 
-        const data = await res.json();
+    return res.data.results.map(filing => {
+        filing = {
+            ...filing
+        };
 
-        let filings = data.results.map(filing => filing.fec_url);
+        if (filing.fec_file_id) {
+            filing.fec_file_id = parseInt(
+                filing.fec_file_id
+                    .replace('FEC-', '')
+                    .replace('SEN-','')
+            );
+        }
 
-        return filings;
-    } catch (err) {
-        console.error(err);
-    }
-}
+        return filing;
+    });
+};

lib/api.js

@@ -0,0 +1,50 @@
+const table = require('text-table'),
+    dsv = require('d3-dsv');
+
+module.exports = (rows, format, columns, headers = true, maxLength = 30) => {
+    if (!rows || rows.length === 0) {
+        throw new Error('No results returned');
+    }
+
+    if (columns) {
+        rows = rows.map(row =>
+            Object.entries(row)
+                .filter(entry => columns.includes(entry[0]))
+                .reduce(
+                    (obj, [key, val]) => Object.assign(obj, { [key]: val }),
+                    {}
+                )
+        );
+    }
+
+    switch (format) {
+        case 'json':
+            return JSON.stringify({
+                rows: rows
+            });
+        case 'ndjson':
+            return rows.map(JSON.stringify).join('\n');
+        case 'table':
+            return table(
+                (columns
+                    ? (headers ? [columns] : []).concat(
+                          rows.map(row => columns.map(key => row[key]))
+                      )
+                    : rows
+                ).map(row =>
+                    row.map(val =>
+                        val && val.length > maxLength
+                            ? `${val.substr(0, maxLength).trim()}…`
+                            : val || ''
+                    )
+                )
+            );
+        case 'csv':
+        case 'tsv':
+            return dsv[format + (headers ? 'Format' : 'FormatBody')](rows, columns);
+        default:
+            throw new Error(
+                'Invalid format specified, options are: table, ndjson, csv'
+            );
+    }
+};

lib/format.js

@@ -2,6 +2,6 @@ const models = require('@publici/fec-model')({
     driver: 'postgres'
 });
 
-module.exports = () => {
+module.exports = (options) => {
 	return models.sync();
 }