Pulled out transforms into bash files

Author: chriszs

.gitignore

@@ -1,4 +1,2 @@
 node_modules
-data/**/*.fec
-data/**/*.zip
-bin/*.sh
+data/**

bin/rss2psql

@@ -6,7 +6,7 @@ transform:
   cmd:
   - bash
   stdin:
-  - for file in $(find /pfs/check/ -name "*.fec"); do curl -s "http://docquery.fec.gov/dcdev/posted/$(basename $file)" | gzip -9 > "/pfs/out/$(basename $file).gz"; done
+  - ./transforms/download.sh /pfs/check/ /pfs/out/
 parallelism_spec:
   constant: 2
 datum_tries: 2

bin/zip2psql

@@ -6,7 +6,7 @@ transform:
   cmd:
   - bash
   stdin:
-  - for file in $(find /pfs/filings/ -name "*.fec.gz"); do gunzip < $file | (echo "DELETE FROM fec_filings WHERE filing_id = $(echo $file | tr -dc '0-9');" && cat) | ./bin/fec convert --format=psql $(echo $file | tr -dc '0-9') | psql -v ON_ERROR_STOP=on --single-transaction 2> "/pfs/out/"$(echo $file | tr -dc '0-9')".log"; done
+  - ./transforms/load.sh /pfs/filings/ /pfs/out/
   secrets:
     - name: pachyderm-postgres-politics-auth
       env_var: PGHOST

pipelines/download.yml

@@ -6,7 +6,7 @@ transform:
   cmd:
   - bash
   stdin:
-  - ./pipelines/notify.sh
+  - ./transforms/notify.sh /pfs/filings/ /pfs/out/
   secrets:
     - name: pachyderm-postgres-politics-auth
       env_var: PGHOST

pipelines/load.yml

@@ -6,7 +6,7 @@ transform:
   cmd:
   - bash
   stdin:
-  - ./bin/fec list --rss --headers=false --format=tsv --columns=fec_url | head -n 100 | while read url; do touch "/pfs/out/$(echo $url | tr -dc '0-9').fec"; done
+  - ./transforms/rss.sh /pfs/out/
 parallelism_spec:
   constant: 1
 input:

pipelines/notify.yml

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+mkdir -p ./data/rss
+mkdir -p ./data/download
+mkdir -p ./data/load
+./transforms/rss.sh ./data/rss/
+./transforms/download.sh ./data/rss/ ./data/download/
+./transforms/load.sh ./data/download/ ./data/load/
+./transforms/notify.sh ./data/load/

pipelines/rss.yml

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+rm -rf ./data

transforms/all.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+for file in $(find $1 -name "*.fec");
+do
+	curl -s "http://docquery.fec.gov/dcdev/posted/$(basename $file)" | gzip -9 > $2"$(basename $file).gz"
+done

transforms/clean.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+for file in $(find $1 -name "*.fec.gz");
+do
+	gunzip < $file | ./bin/fec convert --format=psql $(echo $file | tr -dc '0-9') | (echo "DELETE FROM fec_filings WHERE filing_id = $(echo $file | tr -dc '0-9');" && cat) | psql -v ON_ERROR_STOP=on --single-transaction 2> $2$(echo $file | tr -dc '0-9')".log"
+done

transforms/download.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+./bin/fec list --rss --headers=false --format=tsv --columns=fec_url | head -n 100 | while read url; do
+	touch $1"$(echo $url | tr -dc '0-9').fec";
+done

transforms/load.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+for file in $(find $1 -name "*.zip");
+	unzip -qq -d $2 $file
+done