#30 adding marketyears to the generate_module function

Taking Eugene's restructure branch (from PR #59), this updates the following:
1. Added cleaned version of Rob's code to database.by --> creates an ingredient_rxcui_year table & a product_rxcui_year table, for their respective distributions in generate_module
2. added 'year' column to the generate_module dataframes/CSV files.
3. fixed default_probability typo in utils.py (if idx == 1 --> changed to if idx == 0)

Author: kristentaytok

src/mdt/database.py

@@ -1,9 +1,10 @@
-from . import rxnorm, meps, fda
+from mdt import rxnorm, meps, fda
 from pathlib import Path
 import zipfile
 import io
 import sqlite3
 import pandas as pd
+from datetime import datetime
 
 
 def to_data():
@@ -152,28 +153,73 @@ def load_fda():
     z = zipfile.ZipFile(
         fda.utils.get_dataset(handler=io.BytesIO)
     )
+
+    #moves FDA files to sqlite database by reading as dataframes
     product = pd.read_csv(z.open('product.txt'),sep='\t',dtype=object,header=0,encoding='cp1252')
     package = pd.read_csv(z.open('package.txt'),sep='\t',dtype=object,header=0,encoding='cp1252')
     sql_create_table('product',product)
     sql_create_table('package',package)
-    del product
-    del package
+
 
     #deletes FDA ZIP
     del z
 
-    #NOTE: Rob's python code to join one of these tables with the rxcui_ndc table goes here
-    """
+
+
+    #join product table with the rxcui_ndc table
     rxcui_ndc_string = read_sql_string('rxcui_ndc.sql')
     rxcui_ndc = db_query(rxcui_ndc_string)
     sql_create_table('rxcui_ndc', rxcui_ndc)
-    del rxcui_ndc
-    """
 
 
+    product['PRODUCTNDC'] = product['PRODUCTNDC'].str.replace('-', '').str.zfill(9) 
+    rxcui_ndc['medication_ndc'] = rxcui_ndc['medication_ndc'].astype(str).str.zfill(9)  
+    product_rxcui = product.merge(rxcui_ndc, left_on = 'PRODUCTNDC', right_on = rxcui_ndc['medication_ndc'].str.slice(start=0,stop=9), how = 'left')
+
+
+    #extract year from startmarketingdate & endmarketingdate
+    #fill NULL endmarketingyear with current year 
+    product_rxcui['STARTMARKETINGYEAR'] = product_rxcui['STARTMARKETINGDATE'].str.slice(start=0, stop=4).astype(int)
+    product_rxcui['ENDMARKETINGYEAR'] = product_rxcui['ENDMARKETINGDATE'].str.slice(start=0, stop=4)
+    product_rxcui['ENDMARKETINGYEAR'] = product_rxcui['ENDMARKETINGYEAR'].fillna(datetime.now().year)
+    product_rxcui['ENDMARKETINGYEAR'] = product_rxcui['ENDMARKETINGYEAR'].astype(int)
+    product_rxcui = product_rxcui[['medication_ingredient_rxcui', 'medication_ingredient_name', 'medication_product_rxcui',
+    'medication_product_name', 'STARTMARKETINGYEAR', 'ENDMARKETINGYEAR']]
+
+    med_marketing_year_dict = {}
+    med_state_level_list = ['medication_ingredient', 'medication_product']
+
+    #create a dictionary of df's (one for ingredient, other for product) that contains a range of years that each rxcui was available o nthe market
+    def med_marketing_year(med_state_level_list):
+        for med_state_level in med_state_level_list:
+            #takes MIN startmarketingdate and MAX endmarketingdate for each rxcui
+            med_marketing_year_dict[med_state_level+'_max_marketingyear_range'] = product_rxcui.groupby([med_state_level+'_rxcui', med_state_level+'_name']).agg({'STARTMARKETINGYEAR': 'min', 'ENDMARKETINGYEAR': 'max'}).reset_index()
+
+            #creates a row for each year between startmarketingdate and endmarketingdate for each rxcui
+            zipped = zip(med_marketing_year_dict[med_state_level+'_max_marketingyear_range'][med_state_level+'_rxcui'], med_marketing_year_dict[med_state_level+'_max_marketingyear_range']['STARTMARKETINGYEAR'], med_marketing_year_dict[med_state_level+'_max_marketingyear_range']['ENDMARKETINGYEAR'])
+            med_marketing_year_dict[med_state_level+'_rxcui_years'] = pd.DataFrame([(i, y) for i, s, e in zipped for y in range(s, e+1)],
+                            columns=[med_state_level+'_rxcui','year'])
+            sql_create_table(med_state_level+'_rxcui_years',med_marketing_year_dict[med_state_level+'_rxcui_years'])
+            print(med_state_level+'_rxcui_years')
+            
+    med_marketing_year(med_state_level_list)
+
+    #deletes other dataframes
+    del product
+    del package
+    del rxcui_ndc
+    del medication_ingredient_rxcui_years
+    del medication_product_rxcui_years
+
     #TEST!!!!!!!!!!!!!!!! reads record count from created database
     product = db_query("Select count(*) AS records from product limit 1")
     print('DB table product has {0} records'.format(product['records'].iloc[0]))
 
     package = db_query("Select count(*) AS records from package limit 1")
     print('DB table package has {0} records'.format(package['records'].iloc[0]))
+
+    medication_product_rxcui_years = db_query("Select count(*) AS records from medication_product_rxcui_years limit 1")
+    print('DB table medication_product_rxcui_years has {0} records'.format(medication_product_rxcui_years['records'].iloc[0]))
+
+    medication_ingredient_rxcui_years = db_query("Select count(*) AS records from medication_ingredient_rxcui_years limit 1")
+    print('DB table medication_ingredient_rxcui_years has {0} records'.format(medication_ingredient_rxcui_years['records'].iloc[0]))

src/mdt/utils.py

@@ -117,9 +117,16 @@ def generate_module(rxcui_ndc_df, rxclass_name):
     #Read in MEPS Reference table
     meps_reference = db_query(meps.utils.get_sql('meps_reference.sql'))
 
+     #Read in FDA Ingredient-RxCUI-Years Reference table (for years that a given ingredient was available on the market)
+    ingredient_rxcui_years = db_query('SELECT * FROM medication_ingredient_rxcui_years')
+
+     #Read in FDA Product-RxCUI-Years Reference table (for years that a given product was available on the market)
+    product_rxcui_years = db_query('SELECT * FROM medication_product_rxcui_years')
+
     #Join MEPS to filtered rxcui_ndc dataframe (rxcui_list)
     meps_rxcui = meps_reference.astype(str).merge(rxcui_ndc_df.astype(str)[['medication_ingredient_name', 'medication_ingredient_rxcui','medication_product_name', 'medication_product_rxcui', 'medication_ndc']], how = 'inner', left_on = 'RXNDC', right_on = 'medication_ndc')
 
+
     #Optional: Age range join - can be customized in the mdt_config.json file
     #groupby_demographic_variable: must be either an empty list [] or list of patient demographics (e.g., age, gender, state) - based on user inputs in the mdt_config.json file
 
@@ -162,20 +169,23 @@ def generate_module(rxcui_ndc_df, rxclass_name):
 
     filename = rxclass_name + '_ingredient_distrib'
     #1
-    dcp_dict['patient_count_ingredient'] = meps_rxcui[['medication_ingredient_name',  'medication_ingredient_rxcui', 'person_weight', 'DUPERSID']+groupby_demographic_variables].groupby(['medication_ingredient_name',  'medication_ingredient_rxcui', 'person_weight']+groupby_demographic_variables)['DUPERSID'].nunique()
+    #Join MEPS to ingredient_rxcui_years dataframe (rxcuis_by_fda_marketingdates)
+    meps_rxcui_ingred_years = meps_rxcui.astype(str).merge(ingredient_rxcui_years.astype(str)[['medication_ingredient_rxcui', 'year']], how = 'inner', on = 'medication_ingredient_rxcui')
+    dcp_dict['patient_count_ingredient'] = meps_rxcui_ingred_years[['medication_ingredient_name',  'medication_ingredient_rxcui', 'year', 'person_weight', 'DUPERSID']+groupby_demographic_variables].groupby(['medication_ingredient_name',  'medication_ingredient_rxcui', 'year',  'person_weight']+groupby_demographic_variables)['DUPERSID'].nunique()
     dcp_df = pd.DataFrame(dcp_dict['patient_count_ingredient']).reset_index()
     #2
     dcp_df['weighted_patient_count_ingredient'] = dcp_df['person_weight'].astype(float)*dcp_df['DUPERSID']
     #3
-    dcp_dict['patients_by_demographics_ingredient'] = dcp_df.groupby(['medication_ingredient_name']+groupby_demographic_variables)['weighted_patient_count_ingredient'].sum()
+    dcp_dict['patients_by_demographics_ingredient'] = dcp_df.groupby(['medication_ingredient_name', 'year']+groupby_demographic_variables)['weighted_patient_count_ingredient'].sum()
     dcp_demographic_df = pd.DataFrame(dcp_dict['patients_by_demographics_ingredient']).reset_index()
     #4
     if len(groupby_demographic_variables) > 0:
-        dcp_demographictotal_df = pd.merge(dcp_demographic_df,  dcp_demographic_df.groupby(groupby_demographic_variables)['weighted_patient_count_ingredient'].sum(), how = 'inner', left_on = groupby_demographic_variables, right_index=True, suffixes = ('_demographic', '_total'))
+        dcp_demographictotal_df = pd.merge(dcp_demographic_df,  dcp_demographic_df.groupby(groupby_demographic_variables+['year'])['weighted_patient_count_ingredient'].sum(), how = 'inner', left_on = groupby_demographic_variables+['year'], right_index=True, suffixes = ('_demographic', '_total'))
     else:
-        dcp_demographictotal_df = dcp_demographic_df
-        dcp_demographictotal_df['weighted_patient_count_ingredient_demographic'] = dcp_demographic_df['weighted_patient_count_ingredient']
-        dcp_demographictotal_df['weighted_patient_count_ingredient_total'] = dcp_demographic_df['weighted_patient_count_ingredient'].sum()
+        # dcp_demographictotal_df = dcp_demographic_df
+        # dcp_demographictotal_df['weighted_patient_count_ingredient_demographic'] = dcp_demographic_df['weighted_patient_count_ingredient']
+        # dcp_demographictotal_df['weighted_patient_count_ingredient_total'] = dcp_demographic_df['weighted_patient_count_ingredient'].sum()
+        dcp_demographictotal_df = pd.merge(dcp_demographic_df,  dcp_demographic_df.groupby('year')['weighted_patient_count_ingredient'].sum(), how = 'inner', left_on = 'year', right_index=True, suffixes = ('_demographic', '_total'))
     #5
     dcp_demographictotal_df['percent_ingredient_patients'] = round(dcp_demographictotal_df['weighted_patient_count_ingredient_demographic']/dcp_demographictotal_df['weighted_patient_count_ingredient_total'], 3)
     #6 TODO: change this column to medication_product_state_name(?)
@@ -199,9 +209,9 @@ def generate_module(rxcui_ndc_df, rxclass_name):
     #7
     dcp_dict['percent_ingredient_patients'] = dcp_demographictotal_df
     if len(groupby_demographic_variables) > 0:
-        dcp_dict['percent_ingredient_patients'] = dcp_dict['percent_ingredient_patients'].reset_index().pivot(index= groupby_demographic_variables, columns = 'medication_ingredient_name', values='percent_ingredient_patients').reset_index()
+        dcp_dict['percent_ingredient_patients'] = dcp_dict['percent_ingredient_patients'].reset_index().pivot(index= groupby_demographic_variables+['year'], columns = 'medication_ingredient_name', values='percent_ingredient_patients').reset_index()
     else:
-        dcp_dict['percent_ingredient_patients'] = dcp_dict['percent_ingredient_patients'][['medication_ingredient_name', 'percent_ingredient_patients']].set_index('medication_ingredient_name').T
+        dcp_dict['percent_ingredient_patients'] = dcp_dict['percent_ingredient_patients'][['medication_ingredient_name', 'percent_ingredient_patients', 'year']].set_index('medication_ingredient_name').T
 
     #Fill NULLs and save as CSV
     dcp_dict['percent_ingredient_patients'].fillna(0, inplace=True)
@@ -216,17 +226,19 @@ def generate_module(rxcui_ndc_df, rxclass_name):
     for ingred_name in medication_ingredient_list:
         filename = rxclass_name + '_product_' + ingred_name + '_distrib'
         #0
-        meps_rxcui_ingred = meps_rxcui[meps_rxcui['medication_ingredient_name']==ingred_name][['medication_product_name',  'medication_product_rxcui', 'medication_ingredient_name',  'medication_ingredient_rxcui', 'person_weight', 'DUPERSID']+groupby_demographic_variables]
+        #Join MEPS to product_rxcui_years dataframe (rxcuis_by_fda_marketingdates)
+        meps_rxcui_prod_years = meps_rxcui.astype(str).merge(product_rxcui_years.astype(str)[['medication_product_rxcui', 'year']], how = 'inner', on = 'medication_product_rxcui')
+        meps_rxcui_ingred = meps_rxcui_prod_years[meps_rxcui_prod_years['medication_ingredient_name']==ingred_name][['medication_product_name',  'medication_product_rxcui', 'medication_ingredient_name',  'medication_ingredient_rxcui', 'year', 'person_weight', 'DUPERSID']+groupby_demographic_variables]
         #1
-        dcp_dict['patient_count_product'] = meps_rxcui_ingred.groupby(['medication_product_name',  'medication_product_rxcui',  'medication_ingredient_name',  'medication_ingredient_rxcui', 'person_weight']+groupby_demographic_variables)['DUPERSID'].nunique()
+        dcp_dict['patient_count_product'] = meps_rxcui_ingred.groupby(['medication_product_name',  'medication_product_rxcui',  'medication_ingredient_name',  'medication_ingredient_rxcui', 'year', 'person_weight']+groupby_demographic_variables)['DUPERSID'].nunique()
         dcp_df = pd.DataFrame(dcp_dict['patient_count_product']).reset_index()
         #2
         dcp_df['weighted_patient_count_product'] = dcp_df['person_weight'].astype(float)*dcp_df['DUPERSID']
         #3
-        dcp_dict['patients_by_demographics_product'] = dcp_df.groupby(['medication_product_name', 'medication_ingredient_name']+groupby_demographic_variables)['weighted_patient_count_product'].sum()
+        dcp_dict['patients_by_demographics_product'] = dcp_df.groupby(['medication_product_name', 'medication_ingredient_name', 'year']+groupby_demographic_variables)['weighted_patient_count_product'].sum()
         dcp_demographic_df = pd.DataFrame(dcp_dict['patients_by_demographics_product']).reset_index()
         #4
-        dcp_demographictotal_df = pd.merge(dcp_demographic_df,  dcp_demographic_df.groupby(['medication_ingredient_name']+groupby_demographic_variables)['weighted_patient_count_product'].sum(), how = 'inner', left_on = ['medication_ingredient_name']+groupby_demographic_variables, right_index=True, suffixes = ('_demographic', '_total'))
+        dcp_demographictotal_df = pd.merge(dcp_demographic_df,  dcp_demographic_df.groupby(['medication_ingredient_name', 'year']+groupby_demographic_variables)['weighted_patient_count_product'].sum(), how = 'inner', left_on = ['medication_ingredient_name', 'year']+groupby_demographic_variables, right_index=True, suffixes = ('_demographic', '_total'))
         #5
         dcp_demographictotal_df['percent_product_patients'] = round(dcp_demographictotal_df['weighted_patient_count_product_demographic']/dcp_demographictotal_df['weighted_patient_count_product_total'], 3)
         #6 TODO: change this column to medication_product_state_name or medication_product_transition_name(?)
@@ -250,9 +262,9 @@ def generate_module(rxcui_ndc_df, rxclass_name):
         #7
         dcp_dict['percent_product_patients'] = dcp_demographictotal_df
         if len(groupby_demographic_variables) > 0:
-            dcp_dict['percent_product_patients'] = dcp_dict['percent_product_patients'].reset_index().pivot(index= groupby_demographic_variables, columns = 'medication_product_name', values='percent_product_patients').reset_index()
+            dcp_dict['percent_product_patients'] = dcp_dict['percent_product_patients'].reset_index().pivot(index= groupby_demographic_variables+['year'], columns = 'medication_product_name', values='percent_product_patients').reset_index()
         else:
-            dcp_dict['percent_product_patients'] = dcp_dict['percent_product_patients'][['medication_product_name', 'percent_product_patients']].set_index('medication_product_name').T
+            dcp_dict['percent_product_patients'] = dcp_dict['percent_product_patients'][['medication_product_name', 'percent_product_patients', 'year']].set_index('medication_product_name').T
 
         #Fill NULLs and save as CSV 
         dcp_dict['percent_product_patients'].fillna(0, inplace=True)