Income model, update documentation, examples, and add ARM build to pipeline

.github/workflows/docker.yml

@@ -52,7 +52,7 @@ jobs:
         with:
           context: .
           push: true
-          platforms: linux/amd64
+          platforms: linux/amd64, linux/arm64
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           cache-from: type=gha

README.md

@@ -1,6 +1,6 @@
 # Demographic Microsimulator (DEMOS)
 
-[![Docs](https://github.com/NREL/DEMOS/actions/workflows/docs.yml/badge.svg)](https://nrel.github.io/DEMOS/)
+[![Docs](https://github.com/NatLabRockies/DEMOS/actions/workflows/docs.yml/badge.svg)](https://natlabrockies.github.io/DEMOS/)
 
 ## Overview
 The Demographic Microsimulator (DEMOS) is an agent-based simulation framework used to model the evolution of population demographic characteristics and lifecycle events, such as education attainment, marital status, and other key transitions. DEMOS modules are designed to capture the interdependencies between short-term and long-term lifecycle events, which are often influential in downstream transportation and land-use modeling.
@@ -16,10 +16,12 @@ A technical memorandum describing DEMOS is available [here](./DEMOS_Technical_Me
 ## Usage
 
 ### Docker Compose (recommended)
-The latest docker image for demos is stored in `ghcr.io/NatLabRockies/demos:latest`. The input data and configuration file are fed to the container through volumes ([more info about Docker volumes](https://docs.docker.com/engine/storage/volumes/)). We provide a `docker-compose` workflow that can be used to make the process of mounting volumes easier.
+The latest docker image for demos is stored in `ghcr.io/NatLabRockies/demos:latest`. The input data and configuration file are fed to the container through volumes ([more info about Docker volumes](https://docs.docker.com/engine/storage/volumes/)). We provide a `docker-compose` workflow that can be used to make the process of mounting volumes easier. Make sure you have [Docker](https://docs.docker.com/desktop/) and [Docker Compose](https://docs.docker.com/compose/install/) installed before you proceed.
+
+The following instructions will guide you through running DEMOS with example data of two hypothetical counties. This example is provided to help users quickly get started with DEMOS. It includes the required inputs to run DEMOS for two example counties. The data and configuration files required to run this example are located in `./data/` and `./configuration` folders. You can change where DEMOS will look for your data following the instructions [in the Docs](https://NatLabRockies.github.io/DEMOS/).
 
 #### Clone this repository
-By cloning this repository you download the configuration and data for an example run of DEMOS.
+By cloning this repository you download the configuration and data for an example run of DEMOS. You can also use the `Download ZIP` option available through the green `Code` button above and decompress it to achieve the same results as the clone command.
 
 Run the following command in the Terminal App (MacOS) or Command Prompt/PowerShell (Windows):
 ```bash
@@ -30,17 +32,13 @@ cd DEMOS
 
 # This folder contains (among other files) a data and configuration folder
 # as well as a docker-compose.yml file
-```
 
-Make sure you have [Docker](https://docs.docker.com/desktop/) and [Docker Compose](https://docs.docker.com/compose/install/) installed. Now you can run docker as follows:
-
-```bash
+# This command runs DEMOS on a docker container
 docker compose up
 ```
 #### IMPORTANT for MacOS and Windows users
 > Docker imposes a global limit on how much RAM containers can allocate. DEMOS easily surpases those limits, so in order to run DEMOS in Docker, users need to access the Docker Desktop GUI and `Preferences → Resources → Memory → Increase it (at least 16-20gb)`. The amount of memory required to run DEMOS will primarily depend on the size of the input data.
 
-Documentation for custom data requirements, configuration and overall functionality of demos can be found [in the Docs](https://nrel.github.io/DEMOS/).
 
 ## Other ways to run DEMOS
 
@@ -66,3 +64,14 @@ If you prefer to create your own Python environment and run the Python code dire
 	conda activate demos-env
 	pip install .
 	```
+
+## Comprehensive Documentation
+
+Documentation for custom data requirements, configuration and overall functionality of demos can be found [in the Docs](https://NatLabRockies.github.io/DEMOS/).
+
+## Contact
+If you have questions, suggestions, or are interested in collaborating, please feel free to reach out or open an issue.  
+Bingrong Sun: [email protected]  
+Shivam Sharda: [email protected]  
+Venu Garikapati: [email protected]
+Yamil Essus: [email protected]

configuration/demos_config.toml

@@ -6,14 +6,16 @@ calibrated_models_dir = "../data/small_example/calibrated_models_coefficients/"
 inconsistent_persons_table_behavior = "fix"
 modules = [
     "aging",
-    "employment",
+    "fatality",
     "household_reorg",
     "kids_moving",
-    "fatality",
     "birth",
     "education",
+    "employment",
+    "income",
+    "income_adjustment",
     "household_rebalancing",
-    "income_adjustment"
+    "normalize_table_dtypes",
 ]
 output_tables = [
     "persons",
@@ -33,22 +35,16 @@ initialize_empty_tables = [
 [[tables]]
 file_type = "h5"
 table_name = "persons"
-filepath = "../data/small_example/small_example_tables.h5" # custom_mpo_06197001_model_data_small.h5,  minihh.h5, custom_mpo_06197001_model_data_small_stratHH.h5, custom_mpo_06197001_model_data_hh0.05.h5, countyNM.h5
+filepath = "../data/small_example/small_example_tables.h5"
 h5_key = "persons"
 
 [[tables]]
 file_type = "h5"
 table_name = "households"
-filepath = "../data/small_example/small_example_tables.h5" # custom_mpo_06197001_model_data_small.h5,  minihh.h5, custom_mpo_06197001_model_data_hh0.05.h5, custom_mpo_06197001_model_data_county1, countyNM.h5
+filepath = "../data/small_example/small_example_tables.h5"
 h5_key = "households"
 
 ## Other static data tables
-[[tables]]
-file_type = "csv"
-table_name = "relational_adjustment_mapping"
-filepath = "../data/small_example/relmap_06197001.csv"
-index_col = "index"
-
 [[tables]]
 file_type = "csv"
 table_name = "income_rates"

configuration/demos_config_ref.toml

@@ -6,14 +6,16 @@ calibrated_models_dir = "../data/sf_bay_example/calibrated_models_coefficients/"
 inconsistent_persons_table_behavior = "fix"
 modules = [
     "aging",
-    "employment",
+    "fatality",
     "household_reorg",
     "kids_moving",
-    "fatality",
     "birth",
     "education",
+    "employment",
+    "income",
+    "income_adjustment",
     "household_rebalancing",
-    "income_adjustment"
+    "normalize_table_dtypes",
 ]
 output_tables = [
     "persons",
@@ -41,12 +43,6 @@ filepath = "../data/sf_bay_example/custom_mpo_06197001_model_data.h5"
 h5_key = "households"
 
 ## Other static data tables
-[[tables]]
-file_type = "csv"
-table_name = "relational_adjustment_mapping"
-filepath = "../data/sf_bay_example/relmap_06197001.csv"
-index_col = "index"
-
 [[tables]]
 file_type = "csv"
 table_name = "income_rates"

data/small_example/calibrated_models_coefficients/birth_model.yaml

@@ -8,7 +8,7 @@ saved_object:
     - 0.847
     - 1.754
     - 1.417
-    model_expression: birth ~ hh_birth_agebin1 + hh_birth_agebin2 + fsize_bin23 + fsize_bingt3 + 1
+    model_expression: birth ~ hh_birth_age_lt27 + hh_birth_age_27_35 + hh_fsize_bin23 + hh_fsize_bingt3 + 1
     name: birth
     out_column: birth
     out_filters: null
@@ -20,4 +20,4 @@ saved_object:
     summary_table: null
     tags: []
     template: BinaryLogitStep
-    template_version: 0.2.dev9
+    template_version: 0.2.dev9

data/small_example/calibrated_models_coefficients/cohabitation.yaml

@@ -26,18 +26,18 @@ saved_object:
     - 0.752726
   spec_names:
   - intercept
-  - avg_agebin2
-  - avg_agebin3
-  - avg_agebin4
-  - top_edu_bin3
-  - hd_race_wht
-  - income_bin1
-  - income_bin2
-  - income_bin4
-  - income_bin5
+  - hh_age_avg_bin2
+  - hh_age_avg_bin3
+  - hh_age_avg_bin4
+  - hh_edu_top_bin3
+  - hh_head_race_white
+  - hh_income_bin1
+  - hh_income_bin2
+  - hh_income_bin4
+  - hh_income_bin5
   name: cohabitation
   tables: households
   out_tables: households
   filters: null
   template: MultinomialLogitStep
-  template_version: 0.2.dev9
+  template_version: 0.2.dev9

data/small_example/calibrated_models_coefficients/demos_in_labor_force.yaml

@@ -13,7 +13,7 @@ saved_object:
     - 0.013
     - -0.205
     - -0.175
-  model_expression: stay_out ~ agebin6_labor + agebin3_labor + agebin4_labor + agebin5_labor + gender2 + edubin2 + edubin3 + race_blk + race_asn + race_other + 1
+  model_expression: stay_out ~ age_emp_20_40 + age_emp_41_50 + age_emp_51_70 + age_emp_70plus + sex_female + edu_hs_ged + edu_college_plus + race_black + race_asian_pi + race_other + 1
   name: enter_labor_force
   out_column: null
   out_filters:

data/small_example/calibrated_models_coefficients/demos_out_labor_force.yaml

@@ -11,7 +11,7 @@ saved_object:
     - 0.3530349
     - -0.5569076
     - -0.9928796
-  model_expression: leaving_workforce ~ agebin6_labor + agebin3_labor + agebin4_labor + agebin5_labor + gender2 + edubin2 + edubin3 + 1
+  model_expression: leaving_workforce ~ age_emp_20_40 + age_emp_41_50 + age_emp_51_70 + age_emp_70plus + sex_female + edu_hs_ged + edu_college_plus + 1
   name: exit_labor_force
   out_column: null
   out_filters:

data/small_example/calibrated_models_coefficients/divorce_model.yaml

@@ -13,8 +13,8 @@ saved_object:
   - 0.7375
   - 0.97831
   - 0.30529
-  model_expression: divorced ~ min_agebin2 + min_agebin3 + min_agebin4 + income_bin1
-    + income_bin2 + income_bin4 + income_bin5 + top_edu_bin2 + top_edu_bin3 + fam_work2
+  model_expression: divorced ~ hh_age_min_bin2 + hh_age_min_bin3 + hh_age_min_bin4 + hh_income_bin1
+    + hh_income_bin2 + hh_income_bin4 + hh_income_bin5 + hh_edu_top_bin2 + hh_edu_top_bin3 + hh_fam_work2
     + 1
   name: divorce
   out_column: null

data/small_example/calibrated_models_coefficients/edu_model.yaml

@@ -15,8 +15,8 @@ saved_object:
   - -1.769
   - 0.637
   - -0.297
-  model_expression: stop ~ agebin2 + agebin3 + agebin4 + employbin2 + employbin3 +
-    marital1 + marital34 + marital2 + edubin2 + edubin3 + 1
+  model_expression: stop ~ age_23_35 + age_36_60 + age_60plus + emp_idle_under60 + emp_idle_over60 +
+    mar_married + mar_div_or_sep + mar_widowed + edu_hs_ged + edu_college_plus + 1
   name: education
   out_column: stop
   out_filters:
@@ -30,4 +30,4 @@ saved_object:
   tables: persons
   tags: []
   template: BinaryLogitStep
-  template_version: 0.2.dev9
+  template_version: 0.2.dev9

data/small_example/calibrated_models_coefficients/income_model.yaml

@@ -0,0 +1,41 @@
+modelmanager_version: 0.2.dev9
+
+saved_object:
+    filters: null
+    fitted_parameters:
+    - 9.2866172
+    - 0.0038117
+    - 0.1245698
+    - -0.0847078
+    - 0.4372168
+    - 0.7482159
+    - 0.8479667
+    - 0.6201558
+    - 0.5492689
+    - 0.3194774
+    - 0.6786951
+    - 0.4040705
+    - 0.3016074
+    - 0.027432
+    - 0.026867
+    - 0.1687073
+    - -0.2822124
+    - -0.1492966
+    - -0.1573647
+    - -0.1559694
+    - -0.258431
+    model_expression: income ~ hh_age_head + true_hh_size + not_met_area + income_model_edu_bin1 + income_model_edu_bin2 +
+      income_model_edu_bin3 + job_industry_bin1 + job_industry_bin2 + job_industry_bin3 + 
+      job_occupation_bin1 + job_occupation_bin2 + job_occupation_bin3 +
+      state_quart_2 + state_quart_3 + state_quart_4 +
+      head_race_blk + head_race3 + head_race_asian + head_race_hawaiian + head_race5 + 1
+    name: income
+    out_column: null
+    out_filters: null
+    out_tables: null
+    out_transform: null
+    tables: households
+    summary_table: null
+    tags: []
+    template: RegressionStep
+    template_version: 0.2.dev9

data/small_example/calibrated_models_coefficients/income_model_w_nworkers.yaml

@@ -0,0 +1,42 @@
+modelmanager_version: 0.2.dev9
+
+saved_object:
+    filters: null
+    fitted_parameters:
+    - 9.24495115
+    - 0.00417772
+    - 0.05948244
+    - 0.32053385
+    - -0.08125172
+    - 0.41666961
+    - 0.7286613
+    - 0.84074453
+    - 0.47524105
+    - 0.39585793
+    - 0.19305833
+    - 0.49488234
+    - 0.23930184
+    - 0.18253791
+    - 0.02907993
+    - 0.01390249
+    - 0.1580996
+    - -0.26774506
+    - -0.07188592
+    - -0.15350612
+    - -0.17286581
+    - -0.27394566
+    model_expression: income ~ hh_head_age + true_hh_size + true_hh_workers + not_met_area + hh_head_edu_bin1 + hh_head_edu_bin2 +
+      hh_head_edu_bin3 + job_industry_bin1 + job_industry_bin2 + job_industry_bin3 + 
+      job_occupation_bin1 + job_occupation_bin2 + job_occupation_bin3 + 
+      state_quart_2 + state_quart_3 + state_quart_4 +
+      hh_head_race_black + hh_head_race_native_am + hh_head_race_asian + hh_head_race_hawaiian + hh_head_race_acs_other + 1
+    name: income_nworkers
+    out_column: null
+    out_filters: null
+    out_tables: null
+    out_transform: null
+    tables: households
+    summary_table: null
+    tags: []
+    template: RegressionStep
+    template_version: 0.2.dev9

data/small_example/calibrated_models_coefficients/kids_move_model.yaml

@@ -30,12 +30,12 @@ saved_object:
   - -0.888188
   - -1.517554
   - -1.374318
-  model_expression: kid_moves ~ agebin5_mo + agebin2_mo + agebin3_mo + agebin4_mo
-    + gender2 + employbin2 + employbin3 + edubin2 + edubin3 + race_wht + race_asn
-    + employ2_agebin5_mo + employ2_agebin2_mo + employ2_agebin3_mo + employ2_agebin4_mo
-    + employ3_agebin5_mo + employ3_agebin2_mo + employ3_agebin3_mo + employ3_agebin4_mo
-    + edu2_agebin5_mo + edu2_agebin2_mo + edu2_agebin3_mo + edu2_agebin4_mo + edu3_agebin5_mo
-    + edu3_agebin2_mo + edu3_agebin3_mo + edu3_agebin4_mo + 1
+  model_expression: kid_moves ~ age_km_30plus + age_km_19_20 + age_km_21_25 + age_km_26_30
+    + sex_female + emp_idle_under60 + emp_idle_over60 + edu_hs_ged + edu_college_plus + race_white + race_asian_pi
+    + emp_idle_under60_age_km_30plus + emp_idle_under60_age_km_19_20 + emp_idle_under60_age_km_21_25 + emp_idle_under60_age_km_26_30
+    + emp_idle_over60_age_km_30plus + emp_idle_over60_age_km_19_20 + emp_idle_over60_age_km_21_25 + emp_idle_over60_age_km_26_30
+    + edu_hs_ged_age_km_30plus + edu_hs_ged_age_km_19_20 + edu_hs_ged_age_km_21_25 + edu_hs_ged_age_km_26_30
+    + edu_college_plus_age_km_30plus + edu_college_plus_age_km_19_20 + edu_college_plus_age_km_21_25 + edu_college_plus_age_km_26_30 + 1
   name: kids_move
   out_column: kid_moves
   out_filters: relate in [2, 3, 4, 7, 9, 14]

data/small_example/calibrated_models_coefficients/marriage.yaml

@@ -26,18 +26,18 @@ saved_object:
     - -0.603113
   spec_names:
   - intercept
-  - agebin2
-  - agebin3
-  - agebin4
-  - gender2
-  - employbin2
-  - employbin3
-  - edubin2
-  - edubin3
-  - race_blk
+  - age_23_35
+  - age_36_60
+  - age_60plus
+  - sex_female
+  - emp_idle_under60
+  - emp_idle_over60
+  - edu_hs_ged
+  - edu_college_plus
+  - race_black
   name: marriage
   tables: persons
   out_tables: persons
   filters: null
   template: MultinomialLogitStep
-  template_version: 0.2.dev9
+  template_version: 0.2.dev9

data/small_example/calibrated_models_coefficients/mortality_model.yaml

@@ -15,8 +15,8 @@ saved_object:
     - -0.642
     - 0.779
     - 0.546
-    model_expression: dead ~ agebin1_new + agebin2_new + agebin3_new + agebin4_new + agebin5_new 
-        + gender2 + employbin2 + employbin3 + edubin2 + edubin3 + marital25 + marital34 + 1
+    model_expression: dead ~ age_mort_21_40 + age_mort_41_50 + age_mort_51_70 + age_mort_71_90 + age_mort_90plus
+        + sex_female + emp_idle_under60 + emp_idle_over60 + edu_hs_ged + edu_college_plus + mar_widowed_or_never + mar_div_or_sep + 1
     name: mortality
     out_column: null
     out_filters: null
@@ -28,4 +28,4 @@ saved_object:
     summary_table: null
     tags: []
     template: BinaryLogitStep
-    template_version: 0.2.dev9
+    template_version: 0.2.dev9

demos/config.py

@@ -145,6 +145,7 @@ def model_post_init(self, __context) -> None:
                 "education_model",
                 "household_rebalancing",
                 "update_income",
+                "normalize_table_dtypes",
             ]
 
     @model_validator(mode="after")