Apply temp ActBlue mid-year escape error fix, add test case

chriszs · chriszs · commit 235693d9b015 · 2019-10-26T12:39:50.000-04:00
diff --git a/lib/psql.js b/lib/psql.js
@@ -71,6 +71,7 @@ module.exports = (options) => {
                             .replace(/\\/g, '\\\\')
                             .replace(/\n/g, '\\\n')
                             .replace(/\t/g, '\\\t')
+                            .replace(/\/\\./g, '')
                             .replace(/\\\./g, '\\\\.');
                     } else if (value) {
                         return value;
diff --git a/tests/fixtures/psql-end-marker.fec b/tests/fixtures/psql-end-marker.fec
@@ -1,8 +1,3 @@
 HDRFEC8.3Synetech LLC
 F3XNC00401224ActBluePO Box 382110CambridgeMA02238MY2019010120190630HillErin2019072134365223.53344857111.99379222335.52343861555.6235360779.900.000.00339510724.820.00339510724.820.000.00339510724.820.000.000.001556282.563740441.0549663.560.000.000.00344857111.99344857111.990.000.007408276.227408276.220.00327076876.540.000.000.000.008292476.540.000.008292476.541083926.320.000.000.000.00343861555.62343861555.62339510724.828292476.54331218248.287408276.221556282.565851993.6634365223.532019344857111.99379222335.52343861555.6235360779.90339510724.820.00339510724.820.000.00339510724.820.000.000.001556282.563740441.0549663.560.000.000.00344857111.99344857111.990.000.007408276.227408276.220.00327076876.540.000.000.000.008292476.540.000.008292476.541083926.320.000.000.000.00343861555.62343861555.62339510724.828292476.54331218248.287408276.221556282.565851993.66
 SA11AIC00401224SA11AI_146003663INDCZARNECKIJULIE3339 EDMUNTON DRROCHESTER HILLSMI48306201901272.506.50Contribution to Act BlueST/\.JOHN FISHER CHAPEL UNIVERSITY PARLITURGY PREPERATION COORDINATORContribution to ActBlue
-SA11AIC00401224SA11AI_146003662INDCZARNECKIJULIE3339 EDMUNTON DRROCHESTER HILLSMI483062019012725.0025.00EarmarkST/\.JOHN FISHER CHAPEL UNIVERSITY PARLITURGY PREPERATION COORDINATOREarmarked for STABENOW FOR US SENATE (C00344473)
-SA11AIC00401224SA11AI_150058018INDCZARNECKIJULIE3339 EDMUNTON DRROCHESTER HILLSMI483062019031225.0025.00EarmarkST/\.JOHN FISHER CHAPEL UNIVERSITY PARLITURGY PREPERATION COORDINATOREarmarked for PETE FOR AMERICA, INC (C00697441)
-SA11AIC00401224SA11AI_150058019INDCZARNECKIJULIE3339 EDMUNTON DRROCHESTER HILLSMI48306201903122.506.50Contribution to Act BlueST/\.JOHN FISHER CHAPEL UNIVERSITY PARLITURGY PREPERATION COORDINATORContribution to ActBlue
-SA11AIC00401224SA11AI_157336118INDCZARNECKIJULIE3339 EDMUNTON DRROCHESTER HILLSMI48306201905111.506.50Contribution to Act BlueST/\.JOHN FISHER CHAPEL UNIVERSITY PARLITURGY PREPERATION COORDINATORContribution to ActBlue
-SA11AIC00401224SA11AI_157336117INDCZARNECKIJULIE3339 EDMUNTON DRROCHESTER HILLSMI483062019051115.0015.00EarmarkST/\.JOHN FISHER CHAPEL UNIVERSITY PARLITURGY PREPERATION COORDINATOREarmarked for CONGRESSIONAL PROGRESSIVE CAUCUS PAC (C00513176)
diff --git a/tests/lib/__snapshots__/psql.test.js.snap b/tests/lib/__snapshots__/psql.test.js.snap
@@ -36,3 +36,26 @@ Array [
 ",
 ]
 `;
+
+exports[`psql end of file escape 1`] = `
+Array [
+  "COPY fec_filings (\\"record_type\\", \\"ef_type\\", \\"fec_version\\", \\"soft_name\\", \\"soft_ver\\", \\"report_id\\", \\"report_number\\", \\"comment\\", \\"filing_id\\", \\"created_date\\", \\"updated_date\\") FROM STDIN;
+",
+  "HDR	FEC	8.3	Synetech LLC	\\\\N	\\\\N	\\\\N	\\\\N	\\\\N	2017-02-14T12:51:48.000Z	2017-02-14T12:51:48.000Z
+",
+  "\\\\.
+",
+  "COPY fec_pac_summaries (\\"form_type\\", \\"filer_committee_id_number\\", \\"committee_name\\", \\"change_of_address\\", \\"street_1\\", \\"street_2\\", \\"city\\", \\"state\\", \\"zip_code\\", \\"report_code\\", \\"election_code\\", \\"date_of_election\\", \\"state_of_election\\", \\"coverage_from_date\\", \\"coverage_through_date\\", \\"qualified_committee\\", \\"treasurer_last_name\\", \\"treasurer_first_name\\", \\"treasurer_middle_name\\", \\"treasurer_prefix\\", \\"treasurer_suffix\\", \\"date_signed\\", \\"col_a_cash_on_hand_beginning_period\\", \\"col_a_total_receipts\\", \\"col_a_subtotal\\", \\"col_a_total_disbursements\\", \\"col_a_cash_on_hand_close_of_period\\", \\"col_a_debts_to\\", \\"col_a_debts_by\\", \\"col_a_individuals_itemized\\", \\"col_a_individuals_unitemized\\", \\"col_a_individual_contribution_total\\", \\"col_a_political_party_committees\\", \\"col_a_other_political_committees_pacs\\", \\"col_a_total_contributions\\", \\"col_a_transfers_from_aff_other_party_cmttees\\", \\"col_a_total_loans\\", \\"col_a_total_loan_repayments_received\\", \\"col_a_offsets_to_expenditures\\", \\"col_a_federal_refunds\\", \\"col_a_other_federal_receipts\\", \\"col_a_transfers_from_nonfederal_h3\\", \\"col_a_levin_funds\\", \\"col_a_total_nonfederal_transfers\\", \\"col_a_total_federal_receipts\\", \\"col_a_shared_operating_expenditures_federal\\", \\"col_a_shared_operating_expenditures_nonfederal\\", \\"col_a_other_federal_operating_expenditures\\", \\"col_a_total_operating_expenditures\\", \\"col_a_transfers_to_affiliated\\", \\"col_a_contributions_to_candidates\\", \\"col_a_independent_expenditures\\", \\"col_a_coordinated_expenditures_by_party_committees\\", \\"col_a_total_loan_repayments_made\\", \\"col_a_loans_made\\", \\"col_a_refunds_to_individuals\\", \\"col_a_refunds_to_party_committees\\", \\"col_a_refunds_to_other_committees\\", \\"col_a_total_refunds\\", \\"col_a_other_disbursements\\", \\"col_a_federal_election_activity_federal_share\\", \\"col_a_federal_election_activity_levin_share\\", \\"col_a_federal_election_activity_all_federal\\", \\"col_a_federal_election_activity_total\\", \\"col_a_total_federal_disbursements\\", \\"col_a_total_contributions_refunds\\", \\"col_a_net_contributions\\", \\"col_a_total_federal_operating_expenditures\\", \\"col_a_total_offsets_to_expenditures\\", \\"col_a_net_operating_expenditures\\", \\"col_b_cash_on_hand_jan_1\\", \\"col_b_year\\", \\"col_b_total_receipts\\", \\"col_b_subtotal\\", \\"col_b_total_disbursements\\", \\"col_b_cash_on_hand_close_of_period\\", \\"col_b_individuals_itemized\\", \\"col_b_individuals_unitemized\\", \\"col_b_individual_contribution_total\\", \\"col_b_political_party_committees\\", \\"col_b_other_political_committees_pacs\\", \\"col_b_total_contributions\\", \\"col_b_transfers_from_aff_other_party_cmttees\\", \\"col_b_total_loans\\", \\"col_b_total_loan_repayments_received\\", \\"col_b_offsets_to_expenditures\\", \\"col_b_federal_refunds\\", \\"col_b_other_federal_receipts\\", \\"col_b_transfers_from_nonfederal_h3\\", \\"col_b_levin_funds\\", \\"col_b_total_nonfederal_transfers\\", \\"col_b_total_federal_receipts\\", \\"col_b_shared_operating_expenditures_federal\\", \\"col_b_shared_operating_expenditures_nonfederal\\", \\"col_b_other_federal_operating_expenditures\\", \\"col_b_total_operating_expenditures\\", \\"col_b_transfers_to_affiliated\\", \\"col_b_contributions_to_candidates\\", \\"col_b_independent_expenditures\\", \\"col_b_coordinated_expenditures_by_party_committees\\", \\"col_b_total_loan_repayments_made\\", \\"col_b_loans_made\\", \\"col_b_refunds_to_individuals\\", \\"col_b_refunds_to_party_committees\\", \\"col_b_refunds_to_other_committees\\", \\"col_b_total_refunds\\", \\"col_b_other_disbursements\\", \\"col_b_federal_election_activity_federal_share\\", \\"col_b_federal_election_activity_levin_share\\", \\"col_b_federal_election_activity_all_federal\\", \\"col_b_federal_election_activity_total\\", \\"col_b_total_federal_disbursements\\", \\"col_b_total_contributions_refunds\\", \\"col_b_net_contributions\\", \\"col_b_total_federal_operating_expenditures\\", \\"col_b_total_offsets_to_expenditures\\", \\"col_b_net_operating_expenditures\\", \\"filing_id\\") FROM STDIN;
+",
+  "F3XN	C00401224	ActBlue	\\\\N	PO Box 382110	\\\\N	Cambridge	MA	02238	MY	\\\\N	\\\\N	\\\\N	20190101	20190630	\\\\N	Hill	Erin	\\\\N	\\\\N	\\\\N	20190721	34365223.53	344857111.99	379222335.52	343861555.62	35360779.90	0.00	0.00	339510724.82	0.00	339510724.82	0.00	0.00	339510724.82	0.00	0.00	0.00	1556282.56	3740441.05	49663.56	0.00	0.00	0.00	344857111.99	0.00	0.00	7408276.22	7408276.22	0.00	327076876.54	0.00	0.00	0.00	0.00	8292476.54	0.00	0.00	8292476.54	1083926.32	0.00	0.00	0.00	0.00	343861555.62	8292476.54	331218248.28	7408276.22	1556282.56	5851993.66	34365223.53	2019	344857111.99	379222335.52	343861555.62	35360779.90	339510724.82	0.00	339510724.82	0.00	0.00	339510724.82	0.00	0.00	0.00	1556282.56	3740441.05	49663.56	0.00	0.00	0.00	344857111.99	0.00	0.00	7408276.22	7408276.22	0.00	327076876.54	0.00	0.00	0.00	0.00	8292476.54	0.00	0.00	8292476.54	1083926.32	0.00	0.00	0.00	0.00	343861555.62	8292476.54	331218248.28	7408276.22	1556282.56	5851993.66	\\\\N
+",
+  "\\\\.
+",
+  "COPY fec_contributions (\\"form_type\\", \\"filer_committee_id_number\\", \\"transaction_id\\", \\"back_reference_tran_id_number\\", \\"back_reference_sched_name\\", \\"entity_type\\", \\"contributor_organization_name\\", \\"contributor_last_name\\", \\"contributor_first_name\\", \\"contributor_middle_name\\", \\"contributor_prefix\\", \\"contributor_suffix\\", \\"contributor_street_1\\", \\"contributor_street_2\\", \\"contributor_city\\", \\"contributor_state\\", \\"contributor_zip_code\\", \\"election_code\\", \\"election_other_description\\", \\"contribution_date\\", \\"contribution_amount\\", \\"contribution_aggregate\\", \\"contribution_purpose_descrip\\", \\"contributor_employer\\", \\"contributor_occupation\\", \\"donor_committee_fec_id\\", \\"donor_committee_name\\", \\"donor_candidate_fec_id\\", \\"donor_candidate_last_name\\", \\"donor_candidate_first_name\\", \\"donor_candidate_middle_name\\", \\"donor_candidate_prefix\\", \\"donor_candidate_suffix\\", \\"donor_candidate_office\\", \\"donor_candidate_state\\", \\"donor_candidate_district\\", \\"conduit_name\\", \\"conduit_street1\\", \\"conduit_street2\\", \\"conduit_city\\", \\"conduit_state\\", \\"conduit_zip_code\\", \\"memo_code\\", \\"memo_text_description\\", \\"reference_code\\", \\"filing_id\\") FROM STDIN;
+",
+  "SA11AI	C00401224	SA11AI_146003663	\\\\N	\\\\N	IND	\\\\N	CZARNECKI	JULIE	\\\\N	\\\\N	\\\\N	3339 EDMUNTON DR	\\\\N	ROCHESTER HILLS	MI	48306	\\\\N	\\\\N	20190127	2.50	6.50	Contribution to Act Blue	ST.JOHN FISHER CHAPEL UNIVERSITY PAR	LITURGY PREPERATION COORDINATOR	\\\\N	\\\\N	\\\\N	\\\\N	\\\\N	\\\\N	\\\\N	\\\\N	\\\\N	\\\\N	\\\\N	\\\\N	\\\\N	\\\\N	\\\\N	\\\\N	\\\\N	\\\\N	Contribution to ActBlue	\\\\N	\\\\N
+",
+  "\\\\.
+",
+]
+`;
diff --git a/tests/lib/psql.test.js b/tests/lib/psql.test.js
@@ -19,3 +19,18 @@ test('psql', async done => {
         done();
     });
 });
+
+test('psql end of file escape', async done => {
+    const input = fs.createReadStream(__dirname + '/../fixtures/psql-end-marker.fec');
+    const writer = new ObjectWritableMock();
+
+    psql({
+        in: input,
+        out: writer
+    });
+
+    writer.on('finish', () => {
+        expect(writer.data).toMatchSnapshot();
+        done();
+    });
+});

-Original file line number
+Diff line change
 HDRFEC8.3Synetech LLC
 F3XNC00401224ActBluePO Box 382110CambridgeMA02238MY2019010120190630HillErin2019072134365223.53344857111.99379222335.52343861555.6235360779.900.000.00339510724.820.00339510724.820.000.00339510724.820.000.000.001556282.563740441.0549663.560.000.000.00344857111.99344857111.990.000.007408276.227408276.220.00327076876.540.000.000.000.008292476.540.000.008292476.541083926.320.000.000.000.00343861555.62343861555.62339510724.828292476.54331218248.287408276.221556282.565851993.6634365223.532019344857111.99379222335.52343861555.6235360779.90339510724.820.00339510724.820.000.00339510724.820.000.000.001556282.563740441.0549663.560.000.000.00344857111.99344857111.990.000.007408276.227408276.220.00327076876.540.000.000.000.008292476.540.000.008292476.541083926.320.000.000.000.00343861555.62343861555.62339510724.828292476.54331218248.287408276.221556282.565851993.66
 SA11AIC00401224SA11AI_146003663INDCZARNECKIJULIE3339 EDMUNTON DRROCHESTER HILLSMI48306201901272.506.50Contribution to Act BlueST/\.JOHN FISHER CHAPEL UNIVERSITY PARLITURGY PREPERATION COORDINATORContribution to ActBlue
 -SA11AIC00401224SA11AI_146003662INDCZARNECKIJULIE3339 EDMUNTON DRROCHESTER HILLSMI483062019012725.0025.00EarmarkST/\.JOHN FISHER CHAPEL UNIVERSITY PARLITURGY PREPERATION COORDINATOREarmarked for STABENOW FOR US SENATE (C00344473)
 -SA11AIC00401224SA11AI_150058018INDCZARNECKIJULIE3339 EDMUNTON DRROCHESTER HILLSMI483062019031225.0025.00EarmarkST/\.JOHN FISHER CHAPEL UNIVERSITY PARLITURGY PREPERATION COORDINATOREarmarked for PETE FOR AMERICA, INC (C00697441)
 -SA11AIC00401224SA11AI_150058019INDCZARNECKIJULIE3339 EDMUNTON DRROCHESTER HILLSMI48306201903122.506.50Contribution to Act BlueST/\.JOHN FISHER CHAPEL UNIVERSITY PARLITURGY PREPERATION COORDINATORContribution to ActBlue
 -SA11AIC00401224SA11AI_157336118INDCZARNECKIJULIE3339 EDMUNTON DRROCHESTER HILLSMI48306201905111.506.50Contribution to Act BlueST/\.JOHN FISHER CHAPEL UNIVERSITY PARLITURGY PREPERATION COORDINATORContribution to ActBlue
 -SA11AIC00401224SA11AI_157336117INDCZARNECKIJULIE3339 EDMUNTON DRROCHESTER HILLSMI483062019051115.0015.00EarmarkST/\.JOHN FISHER CHAPEL UNIVERSITY PARLITURGY PREPERATION COORDINATOREarmarked for CONGRESSIONAL PROGRESSIVE CAUCUS PAC (C00513176)