Merge pull request #149 from wsp-sag/bicounty_emme

Apply transit project cards created from Emme ems files

Author: i-am-sijia

lasso/bicounty.py

@@ -69,6 +69,7 @@ def add_opposite_direction_to_link(
     link_gdf['A'] = link_gdf['A'].astype(int)
     link_gdf['B'] = link_gdf['B'].astype(int)
     link_gdf = link_gdf.drop(['A_X','A_Y','B_X','B_Y'], axis=1)
+    link_gdf = link_gdf.reset_index(drop=True)
 
     return link_gdf
 
@@ -109,6 +110,15 @@ def build_pnr_connections(
     Start actual process
     """
 
+    orig_crs = roadway_network.nodes_df.crs # record original crs
+    interim_crs = CRS('epsg:26915') # crs for nearest calculation
+
+    roadway_network.links_df = roadway_network.links_df.to_crs(interim_crs)
+    roadway_network.shapes_df = roadway_network.shapes_df.to_crs(interim_crs)
+    roadway_network.nodes_df = roadway_network.nodes_df.to_crs(interim_crs)
+    roadway_network.nodes_df["X"] = roadway_network.nodes_df["geometry"].x
+    roadway_network.nodes_df["Y"] = roadway_network.nodes_df["geometry"].y
+
     # (1) add pnr nodes
     # read pnr parking location
     pnr_col = ['Zone','X','Y']
@@ -117,7 +127,7 @@ def build_pnr_connections(
                         pnr_nodes_df, 
                         geometry=gpd.points_from_xy(pnr_nodes_df['X'], pnr_nodes_df['Y']),
                         crs=parameters.output_proj)
-    pnr_nodes_df = pnr_nodes_df.to_crs(roadway_network.nodes_df.crs)
+    pnr_nodes_df = pnr_nodes_df.to_crs(interim_crs)
     pnr_nodes_df["X"] = pnr_nodes_df["geometry"].apply(lambda g: g.x)
     pnr_nodes_df["Y"] = pnr_nodes_df["geometry"].apply(lambda g: g.y)
 
@@ -133,8 +143,11 @@ def build_pnr_connections(
 
     # assign a model_node_id to pnr parking node
     pnr_nodes_df['model_node_id'] = pnr_nodes_df['Zone'] + roadway_network.nodes_df.model_node_id.max()
+    # add pnr flag attribute
+    pnr_nodes_df['pnr'] = 1
 
     # add pnr parking nodes to node_df
+    roadway_network.nodes_df['pnr'] = 0
     roadway_network.nodes_df = pd.concat([roadway_network.nodes_df, pnr_nodes_df.drop(['Zone'], axis=1)], 
                                 sort = False, 
                                 ignore_index = True)
@@ -149,13 +162,13 @@ def build_pnr_connections(
         ].copy()
 
     # for each pnr nodes, search for the nearest walk and bike nodes
-    dr_wlk_nodes_df = dr_wlk_nodes_df.to_crs(CRS('epsg:26915'))
+    dr_wlk_nodes_df = dr_wlk_nodes_df.to_crs(interim_crs)
     dr_wlk_nodes_df['X'] = dr_wlk_nodes_df.geometry.map(lambda g:g.x)
     dr_wlk_nodes_df['Y'] = dr_wlk_nodes_df.geometry.map(lambda g:g.y)
     dr_wlk_node_ref = dr_wlk_nodes_df[['X', 'Y']].values
     tree = cKDTree(dr_wlk_node_ref)
 
-    pnr_nodes_df = pnr_nodes_df.to_crs(CRS('epsg:26915'))
+    pnr_nodes_df = pnr_nodes_df.to_crs(interim_crs)
     pnr_nodes_df['X'] = pnr_nodes_df['geometry'].apply(lambda p: p.x)
     pnr_nodes_df['Y'] = pnr_nodes_df['geometry'].apply(lambda p: p.y)
 
@@ -173,6 +186,10 @@ def build_pnr_connections(
         pnr_link_gdf = add_opposite_direction_to_link(pnr_link_gdf, nodes_df=roadway_network.nodes_df, links_df=roadway_network.links_df)
 
         # specify link variables
+        pnr_link_gdf['model_link_id'] = max(roadway_network.links_df['model_link_id']) + pnr_link_gdf.index + 1
+        pnr_link_gdf['shstGeometryId'] = pnr_link_gdf.index + 1
+        pnr_link_gdf['shstGeometryId'] = pnr_link_gdf['shstGeometryId'].apply(lambda x: "pnr" + str(x))
+        pnr_link_gdf['id'] = pnr_link_gdf['shstGeometryId']
         pnr_link_gdf['roadway'] = 'pnr'
         pnr_link_gdf['lanes'] = 1
         pnr_link_gdf['walk_access'] = 1
@@ -184,12 +201,17 @@ def build_pnr_connections(
                                         ignore_index = True)
         roadway_network.links_df.drop_duplicates(subset = ['A', 'B'], inplace = True)
 
+        # update shsapes_df
+        pnr_shape_df = pnr_link_gdf.copy()
+        pnr_shape_df = pnr_shape_df[['id', 'geometry']]
+        roadway_network.shapes_df = pd.concat([roadway_network.shapes_df, pnr_shape_df]).reset_index(drop=True)
+
 
     # (3) build PNR TAZ connectors
     if build_pnr_taz_connector:
         # select centroids
         centroids_df = roadway_network.nodes_df[roadway_network.nodes_df.model_node_id.isin(parameters.taz_N_list)].copy()
-        centroids_df = centroids_df.to_crs(CRS('epsg:26915'))
+        centroids_df = centroids_df.to_crs(interim_crs)
 
         # for each centroid, draw a buffer,
         # connect the centroid to all pnr parking nodes that fall in the buffer
@@ -210,7 +232,11 @@ def build_pnr_connections(
             pnr_connector_gdf = pd.DataFrame(list(zip(centroid_node_id, pnr_node_id)), columns=['A','B'])
             pnr_connector_gdf = add_opposite_direction_to_link(pnr_connector_gdf, nodes_df=roadway_network.nodes_df, links_df=roadway_network.links_df)
 
-             # specify link variables
+            # specify link variables
+            pnr_connector_gdf['model_link_id'] = max(roadway_network.links_df['model_link_id']) + pnr_connector_gdf.index + 1
+            pnr_connector_gdf['shstGeometryId'] = pnr_connector_gdf.index + 1
+            pnr_connector_gdf['shstGeometryId'] = pnr_connector_gdf['shstGeometryId'].apply(lambda x: "pnrtaz" + str(x))
+            pnr_connector_gdf['id'] = pnr_connector_gdf['shstGeometryId']
             pnr_connector_gdf['roadway'] = "pnr"
             pnr_connector_gdf['lanes'] = 1
             pnr_connector_gdf['drive_access'] = 1
@@ -221,5 +247,15 @@ def build_pnr_connections(
                                             ignore_index = True)
             roadway_network.links_df.drop_duplicates(subset = ['A', 'B'], inplace = True)
 
+            # update shsapes_df
+            pnr_connector_shape_df = pnr_connector_gdf.copy()
+            pnr_connector_shape_df = pnr_connector_shape_df[['id', 'geometry']]
+            roadway_network.shapes_df = pd.concat([roadway_network.shapes_df, pnr_connector_shape_df]).reset_index(drop=True)
+
+    roadway_network.links_df = roadway_network.links_df.to_crs(orig_crs)
+    roadway_network.shapes_df = roadway_network.shapes_df.to_crs(orig_crs)
+    roadway_network.nodes_df = roadway_network.nodes_df.to_crs(orig_crs)
+    roadway_network.nodes_df["X"] = roadway_network.nodes_df["geometry"].x
+    roadway_network.nodes_df["Y"] = roadway_network.nodes_df["geometry"].y
 
     return roadway_network

lasso/mtc.py

@@ -103,6 +103,7 @@ def calculate_facility_type(
     join_gdf["oneWay"].fillna("", inplace = True)
     join_gdf["oneWay"] = join_gdf["oneWay"].apply(lambda x: "NA" if x in [None, np.nan, float('nan')] else x)
     join_gdf["oneWay"] = join_gdf["oneWay"].apply(lambda x: str(x) if type(x) == bool else x)
+    join_gdf["oneWay"] = join_gdf["oneWay"].apply(lambda x: str(x) if type(x) == int else x)
     join_gdf["oneWay"] = join_gdf["oneWay"].apply(lambda x: x if type(x) == str else ','.join(map(str, x)))
     join_gdf["oneWay_binary"] = join_gdf["oneWay"].apply(lambda x: 0 if "False" in x else 1)
 
@@ -286,6 +287,8 @@ def determine_number_of_lanes(
     """
     mtc_osm_df = pd.read_csv(mtc_osm_lanes_attributes)
     mtc_osm_df = mtc_osm_df.rename(columns = {"min_lanes": "osm_min_lanes", "max_lanes": "osm_max_lanes"})
+    mtc_osm_df.loc[mtc_osm_df['oneWay'].str.contains('False'), 'osm_min_lanes'] = np.ceil(mtc_osm_df['osm_min_lanes'] / 2)
+    mtc_osm_df.loc[mtc_osm_df['oneWay'].str.contains('False'), 'osm_max_lanes'] = np.ceil(mtc_osm_df['osm_max_lanes'] / 2)
 
     sj_osm_df = pd.read_csv(sj_osm_lanes_attributes)
     sj_osm_df = sj_osm_df.rename(columns = {'osm_lanes_min' : 'osm_min_lanes', 'osm_lanes_max':'osm_max_lanes'})
@@ -1710,7 +1713,7 @@ def roadway_standard_to_mtc_network(
         roadway_network.links_df["assignable"]
     )
 
-    roadway_network = calculate_cntype(roadway_network, parameters)
+    roadway_network = calculate_cntype(roadway_network, parameters, overwrite=True)
     roadway_network = calculate_transit(roadway_network, parameters)
     roadway_network = calculate_useclass(roadway_network, parameters)
     roadway_network = calculate_facility_type(roadway_network, parameters, update_network_variable = True)
@@ -1735,6 +1738,8 @@ def roadway_standard_to_mtc_network(
         on = "id"
     )
 
+    roadway_network.links_mtc_df = gpd.GeoDataFrame(roadway_network.links_mtc_df, geometry=roadway_network.links_mtc_df.geometry)
+    roadway_network.nodes_mtc_df = gpd.GeoDataFrame(roadway_network.nodes_mtc_df, geometry=roadway_network.nodes_mtc_df.geometry)
     roadway_network.links_mtc_df.crs = roadway_network.crs
     roadway_network.nodes_mtc_df.crs = roadway_network.crs
     WranglerLogger.info("Setting Coordinate Reference System to {}".format(output_proj))
@@ -1747,9 +1752,13 @@ def roadway_standard_to_mtc_network(
     roadway_network.nodes_mtc_df["Y"] = roadway_network.nodes_mtc_df.geometry.apply(
         lambda g: g.y
     )
+    # roadway_network.nodes_mtc_df["pnr"]=roadway_network.nodes_mtc_df["pnr"].fillna(0)
+    WranglerLogger.info("Setting PNR value and converting it to string")
+    roadway_network.nodes_mtc_df["pnr"] = np.where(roadway_network.nodes_mtc_df['pnr'].isin([0,'',' ']), '0.0', '1.0')
 
     # CUBE expect node id to be N
     roadway_network.nodes_mtc_df.rename(columns={"model_node_id": "N"}, inplace=True)
+    # roadway_network.nodes_mtc_df['model_node_id']=roadway_network.nodes_mtc_df['N']
 
     return roadway_network
 
@@ -1860,17 +1869,19 @@ def route_properties_gtfs_to_cube(
 
     trip_df["agency_id"].fillna("", inplace = True)
 
+    trip_df['dir_shp_index'] = trip_df.groupby(["TM2_operator", "route_id", "tod_name"]).cumcount()
+
     trip_df["NAME"] = trip_df.apply(
         lambda x: str(x.TM2_operator)
         + "_"
         + str(x.route_id)
         + "_"
-        + x.tod_name
+        + str(x.tod_name)
         + "_"
         + "d"
-        + str(int(x.direction_id))
+        + str(int(x.dir_shp_index))
         + "_s"
-        + x.shape_id,
+        + str(x.shape_id),       
         axis=1,
     )
 
@@ -1929,11 +1940,14 @@ def cube_format(transit_network, row):
     s += "\n OPERATOR={},".format(int(row.TM2_operator) if ~math.isnan(row.TM2_operator) else 99)
     s += '\n SHORTNAME=\"%s",' % (row.route_short_name,)
     s += '\n VEHICLETYPE={},'.format(row.vehtype_num)
-    if row.TM2_line_haul_name in ["Light rail", "Heavy rail", "Commuter rail", "Ferry service"]:
+    if row.TM2_line_haul_name in ["Light rail", "Heavy rail", "Commuter rail", "Ferry service", "Cable Car"]:
         add_nntime = True
     else:
         add_nntime = False
-    s += "\n N={}".format(transit_network.shape_gtfs_to_cube(row, add_nntime))
+    nodes, runtime = transit_network.shape_gtfs_to_cube(row, add_nntime)
+    if add_nntime:
+        s += '\n RUNTIME={},'.format(runtime)
+    s += "\n N={}".format(nodes)
 
     # TODO: need NNTIME, ACCESS_C
 
@@ -1952,6 +1966,19 @@ def write_as_cube_lin(
         outpath: File location for output cube line file.
 
     """
+
+    transit_network.feed.trips['trip_id'] = transit_network.feed.trips['trip_id'].astype(int)
+    transit_network.feed.trips['shape_id'] = transit_network.feed.trips['shape_id'].astype(int)
+
+    transit_network.feed.stop_times['trip_id'] = transit_network.feed.stop_times['trip_id'].astype(int)
+    transit_network.feed.stop_times['stop_id'] = transit_network.feed.stop_times['stop_id'].astype(float).astype(int)
+
+    transit_network.feed.shapes['shape_id'] = transit_network.feed.shapes['shape_id'].astype(int)
+    
+    transit_network.feed.stops['stop_id'] =  transit_network.feed.stops['stop_id'].astype(float).astype(int)
+
+    transit_network.feed.frequencies['trip_id'] =  transit_network.feed.frequencies['trip_id'].astype(int)
+    
     if not outpath:
         outpath  = os.path.join(parameters.scratch_location,"outtransit.lin")
     trip_cube_df = route_properties_gtfs_to_cube(transit_network, parameters, outpath)
@@ -2151,7 +2178,9 @@ def _is_express_bus(x):
         if (x.route_short_name.startswith("J")) | (x.route_short_name.startswith("Lynx")):
             return 1
     if x.agency_name == "SolTrans":
-        if (x.route_short_name in ["80", "92", "78"]) | (x.route_long_name in ["80", "92", "78"]):
+        if ((x.route_short_name in ["80", "92", "78","Green","Blue","Red"]) | 
+            (x.route_long_name in ["80", "92", "78","Green","Blue","Red"])
+        ):
             return 1
     if x.agency_name == "Vine (Napa County)":
         if x.route_short_name in ["29"]:

lasso/parameters.py

@@ -1,4 +1,5 @@
 import os
+import pyproj
 from .logger import WranglerLogger
 
 
@@ -740,7 +741,7 @@ def __init__(self, **kwargs):
             #MTC
             'name',
             "distance",
-            #"roadway",
+            "roadway",
             #"name",
             #MC
             #"shape_id",
@@ -769,6 +770,7 @@ def __init__(self, **kwargs):
             "managed",
             "bus_only",
             "rail_only",
+            "pnr",      
             #MTC
             "assignable",
             "cntype",
@@ -795,6 +797,8 @@ def __init__(self, **kwargs):
             #bi-county
             "nmt2010",
             "nmt2020",
+            "BRT",
+            "has_transit"
         ]
 
         self.output_link_shp = os.path.join(self.scratch_location, "links.shp")
@@ -820,7 +824,7 @@ def __init__(self, **kwargs):
 
         self.fare_matrix_output_variables = ["faresystem", "origin_farezone", "destination_farezone", "price"]
 
-        self.zones = 4756
+        self.zones = 6593
         """
         Create all the possible headway variable combinations based on the cube time periods setting
         """
@@ -906,6 +910,8 @@ def __init__(self, **kwargs):
             #bi-county
             "nmt2010",
             "nmt2020",
+            "BRT",
+            "has_transit"               
         ]
 
         self.float_col = [
@@ -921,6 +927,7 @@ def __init__(self, **kwargs):
         self.string_col = [
             "osm_node_id",
             "name",
+            "pnr",      
             "roadway",
             "shstGeometryId",
             "access_AM",
@@ -937,4 +944,10 @@ def __init__(self, **kwargs):
 
         self.drive_buffer =  6
 
+        #self.network_build_crs = CRS("EPSG:2875")
+        #self.project_card_crs = CRS("EPSG:4326")
+        #self.transformer = pyproj.Transformer.from_crs(
+        #    self.network_build_crs, self.project_card_crs, always_xy=True
+        #)
+
         self.__dict__.update(kwargs)

lasso/project.py

@@ -165,6 +165,7 @@ def create_project(
         roadway_node_changes: Optional[DataFrame] = None,
         transit_changes: Optional[CubeTransit] = None,
         base_roadway_network: Optional[RoadwayNetwork] = None,
+        base_transit_network: Optional[StandardTransit] = None,
         base_cube_transit_network: Optional[CubeTransit] = None,
         build_cube_transit_network: Optional[CubeTransit] = None,
         project_name: Optional[str] = None,
@@ -403,6 +404,8 @@ def create_project(
                 gtfs_feed_dir=base_transit_dir,
                 parameters=parameters
             )
+        elif base_transit_network:
+            base_transit_network = base_transit_network
         else:
             msg = "No base transit network."
             WranglerLogger.info(msg)
@@ -695,16 +698,23 @@ def emme_id_to_wrangler_id(emme_link_change_df, emme_node_change_df, emme_transi
 
             # get new emme nodes
             new_emme_node_id_list = [
-                n for n in emme_node_change_df['emme_id'] if n not in emme_node_id_crosswalk_df['emme_node_id']
+                n for n in emme_node_change_df['emme_id'].to_list() if n not in emme_node_id_crosswalk_df['emme_node_id'].to_list()
             ]
             WranglerLogger.info('New emme node id list {}'.format(new_emme_node_id_list))
             new_wrangler_node = emme_node_id_crosswalk_df['model_node_id'].max()
 
             # add crosswalk for new emme nodes
             for new_emme_node in new_emme_node_id_list:
-                new_wrangler_node = new_wrangler_node + 1
-                emme_node_id_dict.update({new_emme_node : new_wrangler_node})
-            
+                if new_emme_node in emme_node_id_dict.keys():
+                    msg = "new node id {} has already been added to the crosswalk".format(new_emme_node)
+                    WranglerLogger.error(msg)
+                    raise ValueError(msg)
+                else:
+                    new_wrangler_node = new_wrangler_node + 1
+                    emme_node_id_dict.update({new_emme_node : new_wrangler_node})
+                    new_emme_node_id_crosswalk_df = pd.DataFrame(emme_node_id_dict.items(), columns=['emme_node_id', 'model_node_id'])
+                    new_emme_node_id_crosswalk_df.to_csv(emme_node_id_crosswalk_file, index=False)
+                
             # for nodes update model_node_id
             emme_node_change_df['model_node_id'] = emme_node_change_df['emme_id'].map(emme_node_id_dict).fillna(0)
 
@@ -873,7 +883,7 @@ def determine_roadway_network_changes_compatibility(
         roadway_link_changes.rename(columns=dbf_to_net_dict, inplace=True)
 
         for c in roadway_node_changes.columns:
-            if (c not in log_to_net_df["log"].tolist() + log_to_net_df["net"].tolist()) & (c not in ["A", "B"]):
+            if (c not in log_to_net_df["log"].tolist() + log_to_net_df["net"].tolist()) & (c not in ["A", "B", "X", "Y"]):
                 roadway_node_changes.rename(columns={c : c.lower()}, inplace=True)
         roadway_node_changes.rename(columns=log_to_net_dict, inplace=True)
         roadway_node_changes.rename(columns=dbf_to_net_dict, inplace=True)
@@ -1083,6 +1093,8 @@ def _process_node_additions(node_add_df):
 
             node_add_df = node_add_df.drop(["operation_final"], axis=1)
 
+            node_add_df = node_add_df.apply(_reproject_coordinates, axis=1)
+
             for x in node_add_df.columns:
                 node_add_df[x] = node_add_df[x].astype(self.base_roadway_network.nodes_df[x].dtype)
 
@@ -1093,6 +1105,12 @@ def _process_node_additions(node_add_df):
 
             return add_nodes_dict_list
 
+        def _reproject_coordinates(row):
+            reprojected_x, reprojected_y = self.parameters.transformer.transform(row['X'], row['Y'])
+            row['X'] = reprojected_x
+            row['Y'] = reprojected_y
+            return row
+
         def _process_single_link_change(change_row, changeable_col):
             """"""