cdss/preprocess.py at master · phantom1299/cdss · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
# -*- coding: utf-8 -*-

import pandas as pd

df = pd.read_csv('./dahiliye_data.csv')

# Process Diagnoses column
rows = []
for row in df['Diagnoses'].apply(lambda x: x.split(',')):
    rows.append([col.split(':')[1] for col in row if "Kesin Tanı:" in col])

df['Diagnoses'] = rows

# Clean up
del row
del rows

# Duplicate rows by elements of Diagnoses array (2667 rows)
df = df.Diagnoses.apply(pd.Series) \
        .merge(df, right_index=True, left_index=True) \
        .drop(['Diagnoses'], axis=1) \
        .melt(id_vars = ['Age', 'Gender', 'Complaint', 'Indication', 'Symptoms'], value_vars=[0,1,2,3,4,5,6,7,8], value_name = "Diagnose") \
        .drop("variable", axis = 1) \
        .dropna(subset=['Diagnose'])

# Drop duplicate rows (1926 unique)
df = df.drop_duplicates()

# Preprocess text columns (Complaint, Symptoms, Indication)
# Lower text, remove punc. remove whitespaces
lower_map = {
    ord(u'I'): u'ı',
    ord(u'İ'): u'i',
    ord(u'Ğ'): u'g',
    ord(u'ğ'): u'g',
    }

df.Complaint = df.Complaint.str.translate(lower_map) \
                    .str.replace(r'[^\w\s]', ' ') \
                    .str.strip()

df.Symptoms = df.Symptoms.str.translate(lower_map) \
                    .str.replace(r'[^\w\s]', ' ') \
                    .str.strip()

df.Indication = df.Indication.str.translate(lower_map) \
                    .str.replace(r'[^\w\s]', ' ') \
                    .str.strip()

df.to_csv('df.csv')