wp-research-papers/parse.py at master · datadoc24/wp-research-papers · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
import json
import sys
import re
import os
import unicodecsv as csv
from xml.etree import ElementTree as ET

if len(sys.argv) != 3:
	sys.exit("Wrong arguments")

XMLFILE = sys.argv[1]
OUTPUTFILE = sys.argv[2]

print "Processing XML Endnote file " + XMLFILE

#read in the file contents
with open(XMLFILE, 'r') as xml_file:
    xml = xml_file.read()


tags_to_strip = ['style']

for t in tags_to_strip:
#strip out unwanted tags
	xml = re.sub('<'+t+'.*?>','', xml)
	xml = re.sub('</'+t+'>','', xml)


#xml = xml.encode('utf-8', errors='ignore')
records_out = []
records_processed = 0
tree = ET.ElementTree(ET.fromstring(xml))
root = tree.getroot()

print root.tag

records = root.find('records')

for record in records:
	record_out = {}
	print '-----------'

	contributors = record.find('contributors')
	authors = contributors.find('authors')
	author_string=''

	if authors is None:
		authors = contributors.find('secondary-authors')

	if authors is not None:

		for author in authors:
			author_no_commas = author.text.replace(",","")
			author_string = author_string + ',' + author_no_commas

	#remove leading comma from author string
	if len(author_string) > 0:
		author_string = author_string[1:]

	#print "AUTHORS: " + author_string
	record_out['Authors'] = author_string

	titles = record.find('titles')
	title_string = ''
	journal_string = ''

	if titles is not None:
		title = titles.find('title')
		journal = titles.find('secondary-title')

		if title is not None:
			title_string = '"' + title.text + '"'
			record_out['Title'] = title.text

		if journal is not None:
			journal_string = '"' + journal.text + '"'
			record_out['Journal'] = journal.text

	#print "TITLE: "+ title_string
	#print "JOURNAL: " + journal_string

	dates = record.find('dates')
	year_string = ''

	if dates is not None:
		year = dates.find('year')

		if year is not None:
			year_string = year.text
			record_out['Year'] = year.text

	#print "YEAR: " + year_string


	abstract_string = ''
	abstract = record.find('abstract')

	if abstract is not None:
		abstract_string = '"' + abstract.text + '"'
		record_out['Abstract'] = abstract.text


	#print "ABSTRACT: " + abstract_string

	url_string = ''
	urls = record.find('urls')

	if urls is not None:
		related_urls = urls.find('related-urls')

		if related_urls is not None:
			url = related_urls.find('url')

			if url is not None:
				url_string = url.text
				record_out['URL'] = url.text


	#print "URL: " + url_string

	records_out.append(record_out)
	records_processed += 1
	print "Processed " + str(records_processed) + " records"

keys = ['Authors','Title','Journal','Year','Abstract','URL']

with open(OUTPUTFILE, 'wb') as output_file:
    dict_writer = csv.DictWriter(output_file, keys)
    dict_writer.writeheader()
    dict_writer.writerows(records_out)