diff --git a/.ebextensions/01_pre_deps.config b/.ebextensions/01_pre_deps.config deleted file mode 100644 index 79104dc1..00000000 --- a/.ebextensions/01_pre_deps.config +++ /dev/null @@ -1,11 +0,0 @@ -container_commands: - 01-create_instance_config: - command: pwd && mkdir instance && touch instance/config.py -packages: - yum: - postgresql93: [] - postgresql93-devel: [] - git: [] - geos-devel: [] - python36: [] - python36-devel: [] diff --git a/.ebextensions/autoscaling.config b/.ebextensions/autoscaling.config deleted file mode 100644 index a44295cd..00000000 --- a/.ebextensions/autoscaling.config +++ /dev/null @@ -1,6 +0,0 @@ -Resources: - AWSEBAutoScalingGroup: - Type: "AWS::AutoScaling::AutoScalingGroup" - Properties: - HealthCheckType: ELB - HealthCheckGracePeriod: 300 \ No newline at end of file diff --git a/.ebextensions/get_static_files_from_s3.config b/.ebextensions/get_static_files_from_s3.config deleted file mode 100644 index cc8edd7c..00000000 --- a/.ebextensions/get_static_files_from_s3.config +++ /dev/null @@ -1,8 +0,0 @@ -files: - /opt/elasticbeanstalk/hooks/appdeploy/post/99_get_private_files.sh: - content: | - #!/bin/bash -xe - aws s3 cp "s3://elasticbeanstalk-us-east-1-676612114792/static files/usps_zip4s.csv" "/opt/python/run/venv/src/passyunk/passyunk/pdata" && aws s3 cp "s3://elasticbeanstalk-us-east-1-676612114792/static files/election_block.csv" "/opt/python/run/venv/src/passyunk/passyunk/pdata" - group: root - mode: "000755" - owner: root \ No newline at end of file diff --git a/.ebextensions/modify_wsgi.config b/.ebextensions/modify_wsgi.config deleted file mode 100644 index dd1c5c12..00000000 --- a/.ebextensions/modify_wsgi.config +++ /dev/null @@ -1,13 +0,0 @@ -files: - /opt/elasticbeanstalk/hooks/appdeploy/post/99_kill_default_httpd.sh: - content: | - #!/bin/bash -xe - sed -e'/^WSGIScriptAlias/a ProxyPreserveHost on' /etc/httpd/conf.d/wsgi.conf > /tmp/tempfile1 - sed -e'/^ProxyPreserveHost/a AllowEncodedSlashes on' /tmp/tempfile1 > /tmp/tempfile2 - sed -e'/^LoadModule wsgi_module modules/i ' /tmp/tempfile2 > /tmp/tempfile3 - sed -e'/^LoadModule wsgi_module modules/a ' /tmp/tempfile3 > /etc/httpd/conf.d/wsgi.conf - rm -f /tmp/tempfile* - /bin/sh /opt/elasticbeanstalk/hooks/restartappserver/enact/01restart.sh - group: root - mode: "000755" - owner: root \ No newline at end of file diff --git a/.ebextensions/options.config b/.ebextensions/options.config deleted file mode 100644 index e4abe3ce..00000000 --- a/.ebextensions/options.config +++ /dev/null @@ -1,17 +0,0 @@ -option_settings: - aws:elasticbeanstalk:container:python: - NumProcesses: 1 - NumThreads: 4 - aws:elasticbeanstalk:application: - Application Healthcheck URL: /addresses/2401%20Pennsylvania%20Ave?include_units&opa_only - aws:autoscaling:trigger: - MeasureName: Latency - Period: '1' - Unit: Seconds - UpperThreshold: '0.5' - UpperBreachScaleIncrement: '2' - LowerThreshold: '0.3' - aws:autoscaling:asg: - MinSize: '1' - MaxSize: '3' - Cooldown: '600' diff --git a/.elasticbeanstalk/config.yml b/.elasticbeanstalk/config.yml deleted file mode 100644 index e1a87f4e..00000000 --- a/.elasticbeanstalk/config.yml +++ /dev/null @@ -1,7 +0,0 @@ -global: - application_name: ais-api - default_ec2_keyname: ais-api-dev - default_platform: Python 3.6 - default_region: us-east-1 - profile: phila - sc: git diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index c282a32b..00000000 --- a/.travis.yml +++ /dev/null @@ -1,31 +0,0 @@ -language: python -python: "3.6" - -addons: - apt: - packages: - - python-virtualenv - - python3-dev - - build-essential - - libaio1 - - libpq-dev - - libgeos-dev - - unzip - - nginx - -install: - - .travis/install_tests.sh - -script: .travis/test.sh -after_script: .travis/clean_up.sh - -deploy: - provider: script - script: .travis/deploy.sh - on: - branch: - - master - - develop - - develop_test - - staging -after_deploy: .travis/clean_up.sh diff --git a/.travis/check_eb_config.py b/.travis/check_eb_config.py deleted file mode 100755 index 81373b87..00000000 --- a/.travis/check_eb_config.py +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env python - -import os, sys, yaml - -# Get the travis branch from the environment -branch = os.environ.get('TRAVIS_BRANCH', '') - -# Load the eb configuration -with open('.elasticbeanstalk/config.yml') as cfgfile: - cfg = yaml.load(cfgfile) - -# If the branch is not configured, exit with an error code -branchcfg = cfg.get('branch-defaults', {}).get(branch) -if branchcfg is None: - sys.exit(1) - -print(branchcfg['environment']) diff --git a/.travis/clean_up.sh b/.travis/clean_up.sh deleted file mode 100755 index 7158275f..00000000 --- a/.travis/clean_up.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash - -set -e - -# Clean up -rm -rf env diff --git a/.travis/deploy.sh b/.travis/deploy.sh deleted file mode 100755 index 81a3cede..00000000 --- a/.travis/deploy.sh +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env bash - -set -e - -# 1. Create a virtual environment -echo "Creating a virtual environment" -.travis/init_environment.sh -source env/bin/activate - -# 2. Install the awsebcli -echo "Installing AWS Elastic Beanstalk CLI" -# pip install awsebcli -# new version isn't working - install older version: -pip install awsebcli==3.14.6 --force-reinstall - -# 3. Configure eb -echo "Installing configuration for eb tool" -mkdir -p ~/.aws -cat > ~/.aws/credentials </dev/null ; then - echo 'Install Python 3.6' -# sudo add-apt-repository ppa:fkrull/deadsnakes - sudo add-apt-repository ppa:deadsnakes/ppa - sudo apt-get update - sudo apt-get install -y python3.6 python3.6-dev - sudo apt-get install -y iproute2 -fi - -echo 'Create a virutal environment' -virtualenv env -p python3.6 || echo 'Virtual environment already created.' - -echo 'Writing Travis instance IP info:' -ip a -curl ipinfo.io/ip diff --git a/.travis/init_system.sh b/.travis/init_system.sh deleted file mode 100755 index 46022c01..00000000 --- a/.travis/init_system.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash - -set -e - -# Install all the project dependencies. -echo 'Installing project dependencies' -sudo apt-get update -sudo apt-get install build-essential libaio1 -y -sudo apt-get install libpq-dev libgeos-dev -y -sudo apt-get install python-virtualenv unzip nginx -y diff --git a/.travis/install_app.sh b/.travis/install_app.sh deleted file mode 100755 index 8c842d7a..00000000 --- a/.travis/install_app.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash - -set -e - - -# Install python requirements on python3 with library paths -echo 'Installing other application Python requirements' -pip install --requirement requirements.app.txt - -# Create empty config.py -echo 'Initializing the configuration' -mkdir -p instance -touch instance/config.py - - -# Run any management commands for migration, static files, etc. diff --git a/.travis/install_tests.sh b/.travis/install_tests.sh deleted file mode 100755 index bfbfbe55..00000000 --- a/.travis/install_tests.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env bash - -set -e - -# 1. Create a virtual environment -.travis/init_environment.sh -source env/bin/activate - -# 2. Install the awsebcli -echo "Installing AWS Elastic Beanstalk CLI" -# pip install awsebcli -# new version isn't working - install older version: -pip install awsebcli==3.14.6 --force-reinstall - -# 3. Configure eb -echo "Installing configuration for eb tool" -mkdir -p ~/.aws -cat > ~/.aws/credentials < .env - # Install the application dependencies - .travis/install_app.sh - exit 0 -fi - -# 4. Determine whether the current branch is configured for an environment -echo "Checking for environment corresponding to current branch" -source bin/eb_env_utils.sh -get_test_env EB_ENV EB_BLUEGREEN_STATUS || { - echo "Could not find a production or swap environment" ; - exit 1 ; -} - -# 5. Download the environment variables -echo "Downloading environment for branch \"$TRAVIS_BRANCH\" from $EB_ENV" -eb printenv $EB_ENV | tail -n +2 > .env - -# Install the application dependencies -.travis/install_app.sh diff --git a/.travis/test.sh b/.travis/test.sh deleted file mode 100755 index 7e60fcfd..00000000 --- a/.travis/test.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash - -set -e - -source env/bin/activate -pip install pytest honcho --force-reinstall - -honcho run pytest ais -s --ignore=ais/engine/tests diff --git a/ais/api/serializers.py b/ais/api/serializers.py index e337fcc6..defc6732 100644 --- a/ais/api/serializers.py +++ b/ais/api/serializers.py @@ -2,9 +2,8 @@ from collections import OrderedDict from collections.abc import Iterable from geoalchemy2.shape import to_shape -from ais import app, util #, app_db as db +from ais import app, util from ais.models import Address, ENGINE_SRID -#from itertools import chain config = app.config tag_fields = config['ADDRESS_SUMMARY']['tag_fields'] @@ -64,9 +63,9 @@ def get_address_response_relationships(self, address=None, **kwargs): ref_address = Address(self.ref_addr) address = Address(address.street_address) ref_base_address = ' '.join([ref_address.address_full, ref_address.street_full]) - ref_base_address_no_suffix = '{} {}'.format(ref_address.address_full_num, ref_address.street_full) + ref_base_address_no_suffix = f'{ref_address.address_full_num} {ref_address.street_full}' base_address = ' '.join([address.address_full, address.street_full]) - base_address_no_suffix = '{} {}'.format(address.address_full_num, address.street_full) + base_address_no_suffix = f'{address.address_full_num} {address.street_full}' match_type = None unit_type_variations = ["APT", "UNIT", "#"] street_address_variations = [address.street_address, @@ -236,19 +235,12 @@ class AddressJsonSerializer (GeoJSONSerializer): excluded_tags = config['ADDRESS_SUMMARY']['non_summary_tags'] def __init__(self, ref_addr=None, tag_data=None, geom_type=None, geom_source=None, normalized_address=None, base_address=None, shape=None, sa_data=None, estimated=None, match_type=None, **kwargs): - #self.geom_type = kwargs.get('geom_type') if 'geom_type' in kwargs else None self.geom_type = geom_type - #self.geom_source = kwargs.get('geom_source') if 'geom_source' in kwargs else None self.geom_source = geom_source - #self.normalized_address = kwargs.get('normalized_address') if 'normalized_address' in kwargs else None self.normalized_address = normalized_address - #self.base_address = kwargs.get('base_address') if 'base_address' in kwargs else None self.base_address = base_address - #self.shape = kwargs.get('shape') if 'shape' in kwargs else None self.shape = shape - #self.estimated = True if 'estimated' in kwargs else False self.estimated = estimated - #self.sa_data = kwargs.get('sa_data') if 'sa_data' in kwargs else None self.sa_data = sa_data self.match_type = match_type self.tag_data = tag_data @@ -310,8 +302,7 @@ def transform_exceptions(self, data): """ Handle specific exceptions in the formatting of data. """ - # Convert the recycling diversion rate to a percentage with fixed - # precision. + # Convert recycling diversion rate to a percentage with fixed precision. try: rate = float(data['properties']['recycling_diversion_rate']) data['properties']['recycling_diversion_rate'] = round(rate/100, 3) @@ -520,8 +511,7 @@ def transform_exceptions(self, data): Handle specific exceptions in the formatting of data. """ - # Convert the recycling diversion rate to a percentage with fixed - # precision. + # Convert recycling diversion rate to a percentage with fixed precision. try: rate = float(data['service_areas']['recycling_diversion_rate']) data['service_areas']['recycling_diversion_rate'] = round(rate/100, 3) @@ -589,27 +579,6 @@ def transform_tag_data(self, data, tag_data): Handle specific exceptions in the formatting of data. """ data_comps = [] - #print(tag_data) - ## VERSION FOR TAGS GROUPED BY LINKED SOURCE - # for rel_address in tag_data: - # render_tag_data = {} - # render_source = OrderedDict([ - # ('street_address', ''), - # ('match_type', ''), - # ('properties', '') - # ]) - # tags = tag_data[rel_address] - # linked_path = tags[0] - # render_source['street_address'] = rel_address - # render_source['match_type'] = linked_path if linked_path else 'exact' - # render_source['properties'] = [] - # keyvals = tags[1] - # for key, val in keyvals.items(): - # render_tag_data[key] = val - # render_source['properties'].append(render_tag_data) - # data_comps.append(render_source) - - # VERSION FOR FLAT RESPONSE WITH ADDRESS TAG FIELDS AS DICTS WITH SOURCE ITEM render_source = OrderedDict() for rel_address in tag_data: render_tag_data = {} diff --git a/ais/api/views.py b/ais/api/views.py index 9fcea363..1dba1aa7 100644 --- a/ais/api/views.py +++ b/ais/api/views.py @@ -8,7 +8,6 @@ from itertools import chain from flask import Response, request, redirect, url_for from flask_cachecontrol import cache_for, ResponseIsSuccessfulOrRedirect -# from flasgger.utils import swag_from from geoalchemy2.shape import to_shape from geoalchemy2.functions import ST_Transform from sqlalchemy import func, desc @@ -40,21 +39,6 @@ def validate_page_param(request, paginator): return page_num, None -# def get_tag_data(addresses): -# all_tags = {} -# for address, geocode_type, geom in addresses: -# tag_map = {} -# tags = AddressTag.query \ -# .filter_tags_by_address(address.street_address) -# # TODO: If no tags, filter on base/in-range/overlapping number addresses. If still none, return 404. -# for tag in tags: -# if not tag.key in tag_map: -# tag_map[tag.key] = [] -# tag_map[tag.key].append(tag) -# -# all_tags[address.street_address] = tag_map -# -# return all_tags def get_tag_data(addresses): street_addresses = [address.street_address for address, geocode_type, geom in addresses] @@ -113,8 +97,7 @@ def unmatched_response(**kwargs): return json_response(response=error, status=404) srid = request.args.get('srid') if 'srid' in request.args else config['DEFAULT_API_SRID'] - # crs = {'type': 'name', 'properties': {'name': 'EPSG:{}'.format(srid)}} - crs = {'type': 'link', 'properties': {'type': 'proj4', 'href': 'http://spatialreference.org/ref/epsg/{}/proj4/'.format(srid)}} + crs = {'type': 'link', 'properties': {'type': 'proj4', 'href': f'http://spatialreference.org/ref/epsg/{srid}/proj4/'}} # Render the response addresses_page = paginator.get_page(page_num) @@ -182,11 +165,11 @@ def unknown_cascade_view(**kwargs): from_num, to_num = (cascadedseg.right_from, cascadedseg.right_to) if seg_side == "R" else (cascadedseg.left_from, cascadedseg.left_to) # Get geom from true_range view item with same seg_id - true_range_stmt = ''' + true_range_stmt = f''' Select true_left_from, true_left_to, true_right_from, true_right_to from true_range - where seg_id = {seg_id} - '''.format(seg_id=cascadedseg.seg_id) + where seg_id = {cascadedseg.seg_id} + ''' true_range_result = db.engine.execute(true_range_stmt).fetchall() true_range_result = list(chain(*true_range_result)) # Get side delta (address number range on seg side - from true_range if exists else from centerline seg) @@ -212,16 +195,17 @@ def unknown_cascade_view(**kwargs): seg_xy = util.offset(shape, seg_xsect_xy, centerline_offset, seg_side) # GET INTERSECTING SERVICE AREAS - sa_stmt = ''' + shape = seg_xy + sa_stmt = f''' with foo as ( SELECT layer_id, value from service_area_polygon - where ST_Intersects(geom, ST_GeometryFromText('SRID={srid};{shape}')) + where ST_Intersects(geom, ST_GeometryFromText('SRID={ENGINE_SRID};{shape}')) ) SELECT DISTINCT ON (cols.layer_id) cols.layer_id, foo.value from service_area_layer cols left join foo on foo.layer_id = cols.layer_id - '''.format(shape=seg_xy, srid=ENGINE_SRID) + ''' result = db.engine.execute(sa_stmt) for item in result.fetchall(): @@ -236,7 +220,7 @@ def unknown_cascade_view(**kwargs): return json_response(response=error, status=404) srid = request.args.get('srid') if 'srid' in request.args else config['DEFAULT_API_SRID'] - crs = {'type': 'link', 'properties': {'type': 'proj4', 'href': 'http://spatialreference.org/ref/epsg/{}/proj4/'.format(srid)}} + crs = {'type': 'link', 'properties': {'type': 'proj4', 'href': f'http://spatialreference.org/ref/epsg/{srid}/proj4/'}} # Render the response addresses_page = paginator.get_page(page_num) @@ -323,7 +307,7 @@ def addresses(query): unit_type = parsed['components']['address_unit']['unit_type'] unit_num = parsed['components']['address_unit']['unit_num'] addr_num = str(low_num) + '-' + str(high_num) if high_num else low_num - base_address_no_num_suffix = '{} {}'.format(addr_num, street_full) + base_address_no_num_suffix = f'{addr_num} {street_full}' search_type = parsed['type'] loose_filters = OrderedDict([ @@ -384,7 +368,6 @@ def process_query(addresses, match_type): # Get tag data - # if not addresses.all(): # if addresses.all(): if not addresses.all(): if 'opa_only' in request.args and request.args['opa_only'].lower() != 'false': @@ -437,7 +420,7 @@ def process_query(addresses, match_type): srid = request.args.get('srid') if 'srid' in request.args else config['DEFAULT_API_SRID'] crs = {'type': 'link', - 'properties': {'type': 'proj4', 'href': 'http://spatialreference.org/ref/epsg/{}/proj4/'.format(srid)}} + 'properties': {'type': 'proj4', 'href': f'http://spatialreference.org/ref/epsg/{srid}/proj4/'}} # Serialize the response addresses_page = paginator.get_page(page_num) @@ -533,7 +516,6 @@ def process_query(addresses, match_type): @app.route('/block/') @cache_for(hours=1, only_if=ResponseIsSuccessfulOrRedirect) -# @swag_from('docs/block.yml') def block(query): """ Looks up information about the 100-range that the given address falls @@ -623,7 +605,6 @@ def block(query): @app.route('/owner/') @cache_for(hours=1, only_if=ResponseIsSuccessfulOrRedirect) -# @swag_from('docs/owner.yml') def owner(query): query = query.strip('/') owner_parts = query.upper().split() @@ -658,7 +639,7 @@ def owner(query): srid = request.args.get('srid') if 'srid' in request.args else config['DEFAULT_API_SRID'] - crs = {'type': 'link', 'properties': {'type': 'proj4', 'href': 'http://spatialreference.org/ref/epsg/{}/proj4/'.format(srid)}} + crs = {'type': 'link', 'properties': {'type': 'proj4', 'href': f'http://spatialreference.org/ref/epsg/{srid}/proj4/'}} # Get tag data all_tags = get_tag_data(addresses) @@ -677,7 +658,6 @@ def owner(query): @app.route('/account/') @cache_for(hours=1, only_if=ResponseIsSuccessfulOrRedirect) -# @swag_from('docs/account.yml') def account(query): """ Looks up information about the property with the given OPA account number. @@ -721,7 +701,7 @@ def account(query): srid = request.args.get('srid') if 'srid' in request.args else config['DEFAULT_API_SRID'] - crs = {'type': 'link', 'properties': {'type': 'proj4', 'href': 'http://spatialreference.org/ref/epsg/{}/proj4/'.format(srid)}} + crs = {'type': 'link', 'properties': {'type': 'proj4', 'href': f'http://spatialreference.org/ref/epsg/{srid}/proj4/'}} # Get tag data all_tags = get_tag_data(addresses) @@ -740,7 +720,6 @@ def account(query): @app.route('/pwd_parcel/') @cache_for(hours=1, only_if=ResponseIsSuccessfulOrRedirect) -# @swag_from('docs/pwd_parcel.yml') def pwd_parcel(query): """ Looks up information about the property with the given PWD parcel id. @@ -791,7 +770,7 @@ def pwd_parcel(query): srid = request.args.get('srid') if 'srid' in request.args else config['DEFAULT_API_SRID'] - crs = {'type': 'link', 'properties': {'type': 'proj4', 'href': 'http://spatialreference.org/ref/epsg/{}/proj4/'.format(srid)}} + crs = {'type': 'link', 'properties': {'type': 'proj4', 'href': f'http://spatialreference.org/ref/epsg/{srid}/proj4/'}} # Get tag data all_tags = get_tag_data(addresses) @@ -810,7 +789,6 @@ def pwd_parcel(query): @app.route('/dor_parcel/') @cache_for(hours=1, only_if=ResponseIsSuccessfulOrRedirect) -# @swag_from('docs/mapreg.yml') def dor_parcel(query): """ Looks up information about the property with the given DOR parcel id. @@ -849,7 +827,7 @@ def dor_parcel(query): srid = request.args.get('srid') if 'srid' in request.args else config['DEFAULT_API_SRID'] - crs = {'type': 'link', 'properties': {'type': 'proj4', 'href': 'http://spatialreference.org/ref/epsg/{}/proj4/'.format(srid)}} + crs = {'type': 'link', 'properties': {'type': 'proj4', 'href': f'http://spatialreference.org/ref/epsg/{srid}/proj4/'}} #Get tag data all_tags = get_tag_data(addresses) @@ -868,7 +846,6 @@ def dor_parcel(query): @app.route('/intersection/') @cache_for(hours=1, only_if=ResponseIsSuccessfulOrRedirect) -# @swag_from('docs/intersection.yml') def intersection(query): ''' Called by search endpoint if search_type == "intersection_addr" @@ -969,7 +946,7 @@ def intersection(query): srid = request.args.get('srid') if 'srid' in request.args else config['DEFAULT_API_SRID'] - crs = {'type': 'link', 'properties': {'type': 'proj4', 'href': 'http://spatialreference.org/ref/epsg/{}/proj4/'.format(srid)}} + crs = {'type': 'link', 'properties': {'type': 'proj4', 'href': f'http://spatialreference.org/ref/epsg/{srid}/proj4/'}} # Serialize the response: intersections_page = paginator.get_page(page_num) @@ -987,7 +964,6 @@ def intersection(query): @app.route('/reverse_geocode/') @cache_for(hours=1, only_if=ResponseIsSuccessfulOrRedirect) -# @swag_from('docs/reverse_geocode.yml') def reverse_geocode(query): query = query.strip('/') @@ -1003,23 +979,23 @@ def reverse_geocode(query): {'search_type': search_type, 'query': query, 'normalized': normalized}) return json_response(response=error, status=404) crs = {'type': 'link', - 'properties': {'type': 'proj4', 'href': 'http://spatialreference.org/ref/epsg/{}/proj4/'.format(srid)}} + 'properties': {'type': 'proj4', 'href': f'http://spatialreference.org/ref/epsg/{srid}/proj4/'}} x, y = normalized.split(",", 1) search_radius = request.args.get('search_radius') if 'search_radius' in request.args else config['DEFAULT_SEARCH_RADIUS'] search_radius = min(int(search_radius), config['MAXIMUM_SEARCH_RADIUS']) # queries the geocode table by coordinates for the record with the nearest coordinates having \ - # geocode type = pwd_curb, dor_curb, true_range or centerline - reverse_geocode_stmt = ''' + # geocode type = pwd_curb (7), dor_curb (8), true_range (5) + # (original format-string specified centerline=6, but query doesn't use it. TODO: why not?) + reverse_geocode_stmt = f''' SELECT street_address, geocode_type from geocode - where ST_DWITHIN(geom, ST_Transform(ST_GeometryFromText('POINT({x} {y})',{srid}),{engine_srid}), {search_radius}) - AND geocode_type IN ({pwd_curb}, {dor_curb}, {true_range}) + where ST_DWITHIN(geom, ST_Transform(ST_GeometryFromText('POINT({x} {y})',{srid}),{ENGINE_SRID}), {search_radius}) + AND geocode_type IN (5, 7, 8) ORDER BY - geom <-> ST_Transform(ST_GeometryFromText('POINT({x} {y})',{srid}),{engine_srid}), + geom <-> ST_Transform(ST_GeometryFromText('POINT({x} {y})',{srid}),{ENGINE_SRID}), length(street_address) asc LIMIT 1 - '''.format(x=x, y=y, srid=srid, engine_srid=ENGINE_SRID, pwd_curb=7, dor_curb=8, true_range=5, centerline=6, - search_radius=search_radius) + ''' results = db.engine.execute(reverse_geocode_stmt) result = None @@ -1112,7 +1088,6 @@ def reverse_geocode(query): @app.route('/service_areas/') @cache_for(hours=1, only_if=ResponseIsSuccessfulOrRedirect) -# @swag_from('docs/service_areas.yml') def service_areas(query): query = query.strip('/') @@ -1133,13 +1108,13 @@ def service_areas(query): {'query': query, 'search_type': search_type}) return json_response(response=error, status=404) crs = {'type': 'link', - 'properties': {'type': 'proj4', 'href': 'http://spatialreference.org/ref/epsg/{}/proj4/'.format(srid)}} + 'properties': {'type': 'proj4', 'href': f'http://spatialreference.org/ref/epsg/{srid}/proj4/'}} x, y = normalized.split(",", 1) coords = [float(x), float(y)] sa_data = OrderedDict() search_type_out = 'coordinates' - sa_stmt = ''' + sa_stmt = f''' with foo as ( SELECT layer_id, @@ -1149,7 +1124,7 @@ def service_areas(query): else value end as value from service_area_polygon - where ST_Intersects(geom, ST_Transform(ST_GeometryFromText('POINT({x} {y})',{srid}),{engine_srid})) + where ST_Intersects(geom, ST_Transform(ST_GeometryFromText('POINT({x} {y})',{srid}),{ENGINE_SRID})) ) SELECT DISTINCT ON (cols.layer_id) cols.layer_id, foo.value from service_area_layer cols @@ -1158,11 +1133,11 @@ def service_areas(query): ( select 'nearest_seg'::text as layer_id, cast(ss.seg_id as text) as value from street_segment ss - order by st_distance(ST_Transform(ST_GeometryFromText('POINT({x} {y})',{srid}),{engine_srid}),ss.geom) asc + order by st_distance(ST_Transform(ST_GeometryFromText('POINT({x} {y})',{srid}),{ENGINE_SRID}),ss.geom) asc limit 1 ) order by layer_id - '''.format(srid=srid, engine_srid=ENGINE_SRID,x=x, y=y, sa_yes_no_layers_str=sa_yes_no_layers_str) + ''' result = db.engine.execute(sa_stmt) for item in result.fetchall(): sa_data[item[0]] = item[1] @@ -1199,7 +1174,6 @@ def street(query): @app.route('/search/') @cache_for(hours=1, only_if=ResponseIsSuccessfulOrRedirect) -# @swag_from('docs/search.yml') def search(query): """ API Endpoint for various types of geocoding (not solely addresses) @@ -1296,6 +1270,6 @@ def health(): result = db.engine.execute(check_geocode_db).fetchall() return 'All okay!', 200 except Exception as e: - #return 'DB failure!: {}'.format(str(e)), 504 + #return f'DB failure!: {str(e)}', 504 return 'DB failure!', 504 diff --git a/ais/engine/bin/create_table_from_schema.py b/ais/engine/bin/create_table_from_schema.py index 623c7915..cb95541c 100644 --- a/ais/engine/bin/create_table_from_schema.py +++ b/ais/engine/bin/create_table_from_schema.py @@ -1,3 +1,4 @@ +# TODO: migrate file from Oracle or remove if unnecessary import sys import cx_Oracle from ais import app @@ -11,15 +12,15 @@ dsn = config[dsn_map[table_name]] conn = cx_Oracle.connect(dsn) curs = conn.cursor() -sql0 = "DROP TABLE t_{}".format(table_name) -sql1 = "CREATE TABLE t_{table_name} as (select * from {table_name} where 1=0)".format(table_name=table_name) -sql2 = "GRANT SELECT on t_{} to SDE".format(table_name) -sql3 = "GRANT SELECT ON t_{} to GIS_SDE_VIEWER".format(table_name) -sql4 = "GRANT SELECT ON t_{} to GIS_AIS_SOURCES".format(table_name) +sql0 = f"DROP TABLE t_{table_name}" +sql1 = f"CREATE TABLE t_{table_name} as (select * from {table_name} where 1=0)" +sql2 = f"GRANT SELECT on t_{table_name} to SDE" +sql3 = f"GRANT SELECT ON t_{table_name} to GIS_SDE_VIEWER" +sql4 = f"GRANT SELECT ON t_{table_name} to GIS_AIS_SOURCES" if table_name == 'address_summary': - sql5 = "GRANT SELECT on t_{} to GIS_OPA with GRANT OPTION".format(table_name) + sql5 = f"GRANT SELECT on t_{table_name} to GIS_OPA with GRANT OPTION" elif table_name == 'dor_parcel_address_check': - sql5 = "GRANT SELECT ON t_{} to GIS_DOR".format(table_name) + sql5 = f"GRANT SELECT ON t_{table_name} to GIS_DOR" curs.execute(sql1) curs.execute(sql2) curs.execute(sql3) diff --git a/ais/engine/bin/output_address_points_for_ng911.py b/ais/engine/bin/output_address_points_for_ng911.py index 003c41a7..b1ecf5c4 100644 --- a/ais/engine/bin/output_address_points_for_ng911.py +++ b/ais/engine/bin/output_address_points_for_ng911.py @@ -59,7 +59,7 @@ ''' source_cur = source_conn.cursor() -outputquery = 'COPY ({export_stmt}) TO STDOUT WITH CSV HEADER'.format(export_stmt=export_stmt) +outputquery = f'COPY ({export_stmt}) TO STDOUT WITH CSV HEADER' with open(temp_csv, "w") as f: with source_conn.cursor() as cursor: @@ -80,15 +80,15 @@ else: str_header += field -with open(temp_csv, 'r') as f: +with open(temp_csv, 'r') as file: with target_conn.cursor() as cursor: - copy_stmt = ''' + copy_stmt = f''' BEGIN; TRUNCATE TABLE {target_table}; COPY {target_table} ({str_header}) FROM STDIN WITH (FORMAT csv, HEADER true); COMMIT; - '''.format(target_table=target_table, str_header=str_header) - cursor.copy_expert(copy_stmt, f) + ''' + cursor.copy_expert(copy_stmt, file) target_conn.close() @@ -99,9 +99,9 @@ r = requests.post( airflow_trigger_creds.get('url').format(dag_name=workflow), data=json.dumps("{}".format('{}')), - auth=("{}".format(airflow_trigger_creds.get('user')), "{}".format(airflow_trigger_creds.get('pw'))) + auth=(f"{airflow_trigger_creds.get('user')}", f"{airflow_trigger_creds.get('pw')}") ) - print("Downstream process has been triggered, status code: {}".format(r.status_code)) + print(f"Downstream process has been triggered, status code: {r.status_code}") except Exception as e: print("Triggering downstream process failed, exiting") raise e diff --git a/ais/engine/bin/warmup_lb.py b/ais/engine/bin/warmup_lb.py index 13eb561c..90b50031 100644 --- a/ais/engine/bin/warmup_lb.py +++ b/ais/engine/bin/warmup_lb.py @@ -80,9 +80,9 @@ def query_address(address): read_conn = psycopg2.connect(f"dbname=ais_engine host=localhost user=ais_engine password={dbpass}") - address_count = etl.fromdb(read_conn, 'select count(*) as N from {}'.format(warmup_address_table_name)) + address_count = etl.fromdb(read_conn, f'select count(*) as N from {warmup_address_table_name}') n = list(address_count.values('n'))[0] - warmup_rows = etl.fromdb(read_conn, 'select {address_field} from {table} OFFSET floor(random()*{n}) limit {limit}'.format(address_field=warmup_address_field, table=warmup_address_table_name, n=n, limit=warmup_row_limit)) + warmup_rows = etl.fromdb(read_conn, f'select {warmup_address_field} from {warmup_address_table_name} OFFSET floor(random()*{n}) limit {warmup_row_limit}') # print(etl.look(warmup_rows)) responses = warmup_rows.addfield('response_status', (lambda a: query_address(a['street_address']))).progress(100) #print(etl.look(responses)) diff --git a/ais/engine/commands.py b/ais/engine/commands.py index d0f79d93..6677cbb9 100644 --- a/ais/engine/commands.py +++ b/ais/engine/commands.py @@ -10,7 +10,7 @@ def engine(script): # dynamically pull in engine scripts as a module and call their main function # The __init__.py in the ais/engine/scripts folder sets all the .py files # in that directory to be importable. - mod = __import__("ais.engine.scripts.{}".format(script), fromlist=["main"]) + mod = __import__(f"ais.engine.scripts.{script}", fromlist=["main"]) mod.main() if not script: print('Please pass an arg to the --script flag.') diff --git a/ais/engine/manage.py b/ais/engine/manage.py index e889fa4f..7ae9d290 100644 --- a/ais/engine/manage.py +++ b/ais/engine/manage.py @@ -21,7 +21,7 @@ def run(script): for script in scripts: path = path_for_script(root_path, script) if not os.path.isfile(path): - raise FileNotFoundError('Script not found: {}'.format(script)) + raise FileNotFoundError(f'Script not found: {script}') paths.append(path) for path in paths: diff --git a/ais/engine/scripts/extras/make_multiple_seg_matches.py b/ais/engine/scripts/extras/make_multiple_seg_matches.py index e8f1a70c..aa686d6b 100644 --- a/ais/engine/scripts/extras/make_multiple_seg_matches.py +++ b/ais/engine/scripts/extras/make_multiple_seg_matches.py @@ -62,15 +62,15 @@ for multi_addr in multi_addrs: # Get child addresses - stmt = ''' + stmt = f''' select adl.address_1 from address_link adl join address a on adl.address_1 = a.street_address where - adl.address_2 = '{}' and + adl.address_2 = '{multi_addr}' and adl.relationship = 'in range' order by a.address_low - '''.format(multi_addr) + ''' ais_db.c.execute(stmt) child_rows = [x['address_1'] for x in ais_db.c.fetchall()] @@ -78,8 +78,7 @@ high_address = child_rows[-1] # Get parcel geocode for low address - geocode_where = "street_address = '{}' and geocode_type in ('pwd_parcel', \ - 'dor_parcel')".format(multi_addr) + geocode_where = f"street_address = '{multi_addr}' and geocode_type in ('pwd_parcel', 'dor_parcel')" geocode_rows = ais_db.read('geocode', ['*'], geom_field='geometry', \ where=geocode_where) # Sort by parcel priority @@ -92,7 +91,7 @@ geocode_row = geocode_rows[0] geocode_shp = loads(geocode_row['geometry_wkt']) except IndexError: - #print('No parcel XY for {}'.format(multi_addr)) + #print(f'No parcel XY for {multi_addr}') continue # Get seg IDs diff --git a/ais/engine/scripts/geocode_addresses.py b/ais/engine/scripts/geocode_addresses.py index a50d05ee..fd980766 100644 --- a/ais/engine/scripts/geocode_addresses.py +++ b/ais/engine/scripts/geocode_addresses.py @@ -53,11 +53,9 @@ def main(): WHERE_STREET_ADDRESS_IN = None WHERE_SEG_ID_IN = None if FILTER_STREET_NAME not in [None, '']: - WHERE_STREET_NAME = "street_name = '{}'".format(FILTER_STREET_NAME) - WHERE_STREET_ADDRESS_IN = "street_address in (select street_address from \ - {} where {})".format(address_table.name, WHERE_STREET_NAME) - WHERE_SEG_ID_IN = "seg_id in (select seg_id from {} where {})" \ - .format(seg_table.name, WHERE_STREET_NAME) + WHERE_STREET_NAME = f"street_name = '{FILTER_STREET_NAME}'" + WHERE_STREET_ADDRESS_IN = f"street_address in (select street_address from {address_table.name} where {WHERE_STREET_NAME})" + WHERE_SEG_ID_IN = f"seg_id in (select seg_id from {seg_table.name} where {WHERE_STREET_NAME})" if WRITE_OUT: print('Dropping indexes...') @@ -99,27 +97,27 @@ def main(): for parcel_layer_name, parcel_layer_def in parcel_layers.items(): source_table = parcel_layer_name + '_parcel' - print(' - {}'.format(parcel_layer_name)) + print(f' - {parcel_layer_name}') # DEV parcel_where = '' if WHERE_STREET_NAME: - parcel_where = '{} and '.format(WHERE_STREET_NAME) - parcel_stmt = ''' + parcel_where = f'{WHERE_STREET_NAME} and ' + parcel_stmt = f''' select id, ST_AsText(geom) as geom, st_astext(st_centroid(geom)) as centroid from {source_table} - where {where} st_intersects(st_centroid(geom), geom) + where {parcel_where} st_intersects(st_centroid(geom), geom) union select id, ST_AsText(geom) as geom, st_astext(st_pointonsurface(geom)) as centroid from {source_table} - where {where} not st_intersects(st_centroid(geom), geom) - '''.format(where=parcel_where, source_table=source_table) + where {parcel_where} not st_intersects(st_centroid(geom), geom) + ''' parcel_rows = db.execute(parcel_stmt) parcel_layer_xy_map = {} parcel_layer_geom_map = {} @@ -331,7 +329,7 @@ def main(): test_offset, seg_side) test_xy_wkt = dumps(test_xy_shp) - parcel_match_stmt = ''' + parcel_match_stmt = f''' SELECT id, CASE @@ -341,7 +339,7 @@ def main(): END as wkt FROM {source_table} WHERE ST_Intersects(geom, ST_GeomFromText('{test_xy_wkt}', {engine_srid})) - '''.format(source_table=source_table, test_xy_wkt=test_xy_wkt, engine_srid=engine_srid) + ''' db.execute(parcel_match_stmt) parcel_match = db._c.fetchone() @@ -439,17 +437,18 @@ def main(): geocode_table.write(geocode_rows, chunk_size=150000) print('Writing address-parcels...') addr_parcel_table.write(address_parcels, chunk_size=150000) - print('Wrote {} rows'.format(len(geocode_rows) + geocode_count)) + written_count = len(geocode_rows) + geocode_count + print(f'Wrote {written_count} rows') # Process source address point geocodes in batch: # geocodes: - geocode_stmt = '''insert into geocode (street_address, geocode_type, geom) - select street_address, {geocode_type} as geocode_type, geom + geocode_stmt = f'''insert into geocode (street_address, geocode_type, geom) + select street_address, {geocode_priority_map['ng911']} as geocode_type, geom from ng911_address_point - '''.format(geocode_type=geocode_priority_map['ng911']) + ''' db.execute(geocode_stmt) print('Creating index...') geocode_table.create_index('street_address') db.close() - print('Finished in {}'.format(datetime.now() - start)) + print(f'Finished in {datetime.now() - start}') diff --git a/ais/engine/scripts/geocode_addresses_from_links.py b/ais/engine/scripts/geocode_addresses_from_links.py index f49594c3..d46b42d5 100644 --- a/ais/engine/scripts/geocode_addresses_from_links.py +++ b/ais/engine/scripts/geocode_addresses_from_links.py @@ -12,8 +12,6 @@ def main(): config = app.config engine_srid = config['ENGINE_SRID'] - Parser = config['PARSER'] - parser = Parser() db = datum.connect(config['DATABASES']['engine']) WRITE_OUT = True geocode_table = db['geocode'] @@ -48,7 +46,6 @@ def main(): for key, value in tag_map.items(): street_address = key tags = value - # geocode_types = [] try: geocode_types = [x['geocode_type'] for x in geocode_map[street_address]] except: @@ -83,7 +80,7 @@ def main(): print('Dropping indexes...') geocode_table.drop_index('street_address') - print('Writing {num} new geocode rows...'.format(num=len(new_geocode_rows))) + print(f'Writing {len(new_geocode_rows)} new geocode rows...') # TODO: Use geopetl/datum instead of raw sql i = 0 values = '' @@ -91,22 +88,18 @@ def main(): street_address = new_row['street_address'] geocode_type = new_row['geocode_type'] geom = new_row['geom'] - new_vals = '''('{street_address}', {geocode_type}, ST_GeomFromText('{geom}',{engine_srid}))'''.format(street_address=street_address, geocode_type=geocode_type, geom=geom, engine_srid=engine_srid) + new_vals = f'''('{street_address}', {geocode_type}, ST_GeomFromText('{geom}',{engine_srid}))''' values = values + ', ' + new_vals if values else new_vals i += 1 if i % 1000 == 0: toi = i + 1000 - #print("writing rows {i} to {toi}".format(i=i, toi=toi)) - write_stmt = ''' - INSERT INTO geocode (street_address, geocode_type, geom) VALUES {values} - '''.format(values=values) + #print(f"writing rows {i} to {toi}") + write_stmt = f"INSERT INTO geocode (street_address, geocode_type, geom) VALUES {values}" db.execute(write_stmt) db.save() values = '' if values: - write_stmt = ''' - INSERT INTO geocode (street_address, geocode_type, geom) VALUES {values} - '''.format(values=values) + write_stmt = f"INSERT INTO geocode (street_address, geocode_type, geom) VALUES {values}" db.execute(write_stmt) db.save() @@ -114,6 +107,6 @@ def main(): geocode_table.create_index('street_address') db.close() - - print('Finished in {}'.format(datetime.now() - start)) + + print(f'Finished in {datetime.now() - start}') diff --git a/ais/engine/scripts/get_pwd_matches_from_manual_opa_geocodes.py b/ais/engine/scripts/get_pwd_matches_from_manual_opa_geocodes.py index 75d485e2..df53b8b9 100644 --- a/ais/engine/scripts/get_pwd_matches_from_manual_opa_geocodes.py +++ b/ais/engine/scripts/get_pwd_matches_from_manual_opa_geocodes.py @@ -30,34 +30,35 @@ def main(): target_user = target_dsn[target_dsn.index("//") + 2:target_dsn.index(":", target_dsn.index("//"))] target_pw = target_dsn[target_dsn.index(":",target_dsn.index(target_user)) + 1:target_dsn.index("@")] target_name = target_dsn[target_dsn.index("/", target_dsn.index("@")) + 1:] - target_conn = psycopg2.connect('dbname={db_name} user={db_user} password={db_pw} host=localhost'.format(db_name=target_name, db_user=target_user, db_pw=target_pw)) + target_conn = psycopg2.connect(f'dbname={target_name} user={target_user} password={target_pw} host=localhost') target_cur = target_conn.cursor() target_table_name = 'public.t_opa_active_accounts' # Read source table: - print("Reading rows from {}".format(source_table)) + print(f"Reading rows from {source_table}") rows = etl.fromoraclesde(source_conn, source_table, fields=source_fields) # Runs in 0:11:48 # Format fields rows = rows.rename({v:k for k,v in source_field_map.items()}) - drop_stmt = '''drop table if exists {}'''.format(target_table_name) - create_stmt = '''create table {} ( + drop_stmt = f"drop table if exists {target_table_name}" + create_stmt = f'''create table {target_table_name} ( account_num text, source_address text, unit_num text, geom geometry(Point,2272) - )'''.format(target_table_name) + )''' # Create temp target table: - print("Dropping temp table '{}' if already exists...".format(target_table_name)) + print(f"Dropping temp table '{target_table_name}' if already exists...") target_cur.execute(drop_stmt) - print("Creating temp table '{}'...".format(target_table_name)) + print(f"Creating temp table '{target_table_name}'...") target_cur.execute(create_stmt) target_conn.commit() # Write rows to target: - print("Writing to temp table '{}'".format(target_table_name)) + print(f"Writing to temp table '{target_table_name}'") rows.topostgis(target_conn, target_table_name) - # Update address_parcel by selecting address from opa_property associated with opa_account_num for opa_active_accounts record where opa_address doesn't have pwd_parcel in table, \ + # Update address_parcel by selecting address from opa_property associated with opa_account_num + # for opa_active_accounts record where opa_address doesn't have pwd_parcel in table, # and add row_id of parcel based on associated pwd_parcel_id in opa_active_accounts table: update_stmt = ''' @@ -90,4 +91,4 @@ def main(): target_conn.commit() # close db connection: target_conn.close() - print('Finished in {} seconds'.format(datetime.now() - start)) + print(f'Finished in {datetime.now() - start} seconds') diff --git a/ais/engine/scripts/load_addresses.py b/ais/engine/scripts/load_addresses.py index 6b96a22d..64e5aac1 100644 --- a/ais/engine/scripts/load_addresses.py +++ b/ais/engine/scripts/load_addresses.py @@ -123,10 +123,9 @@ def main(): # Check for necessary components for field_name in ['address_low', 'street_name']: if not field_name in address_fields and not all(x in address_fields for x in ['base_address', 'unit_num']): - raise ValueError('Missing required address field: {}' \ - .format(field_name)) + raise ValueError(f'Missing required address field: {field_name}') if 'preprocessor' not in source: - raise ValueError('No preprocessor specified for address source `{}`'.format(source_name)) + raise ValueError(f"No preprocessor specified for address source '{source_name}'") preprocessor = source['preprocessor'] # Get other params @@ -144,18 +143,16 @@ def main(): source_db_name = source['db'] source_db = datum.connect(config['DATABASES'][source_db_name]) source_table = source_db[source['table']] - print('Reading from {}...'.format(source_name)) + print(f'Reading from {source_name}...') where = source['where'] if 'where' in source else None # For debugging. (Only fetch a specific address.) if DEV: if source_type == 'single_field': - dev_where = "{} = '{}'" \ - .format(address_fields['street_address'], DEV_ADDRESS) + dev_where = f"{address_fields['street_address']} = '{DEV_ADDRESS}'" elif source_type == 'comps': - clauses = ["{} = '{}'".format(address_fields[key], value) \ - for key, value in DEV_ADDRESS_COMPS.items()] + clauses = [f"{address_fields[key]} = '{value}'" for key, value in DEV_ADDRESS_COMPS.items()] dev_where = ' AND '.join(clauses) if where: where += ' AND ' + dev_where @@ -173,7 +170,7 @@ def main(): if not DEV: if not source_rows or len(source_rows) < 2: - raise Exception("Exiting because source table {source_table} is empty.".format(source_table=source_table)) + raise Exception(f"Exiting because source table {source_table} is empty.") # Loop over addresses for i, source_row in enumerate(source_rows): @@ -329,23 +326,23 @@ def main(): address_errors.append(address_error) if WRITE_OUT: - print('Writing {} address tags...'.format(len(address_tags))) + print(f'Writing {len(address_tags)} address tags...') address_tag_table.write(address_tags, chunk_size=150000) address_tags = [] address_tag_strings = set() - print('Writing {} source addresses...'.format(len(source_addresses))) + print(f'Writing {len(source_addresses)} source addresses...') source_address_table.write(source_addresses, chunk_size=150000) source_addresses = [] insert_rows = [dict(x) for x in addresses] if WRITE_OUT: - print('Writing {} addresses...'.format(len(addresses))) + print(f'Writing {len(addresses)} addresses...') address_table.write(insert_rows, chunk_size=150000) del insert_rows - print('Making {} parser_address_tags...'.format(len(parsed_addresses))) + print(f'Making {len(parsed_addresses)} parser_address_tags...') for source_address, comps in parsed_addresses.items(): comps = comps.get('components') street_address = comps.get('output_address', '') @@ -607,16 +604,16 @@ def main(): if WRITE_OUT: print('Writing address links...') address_link_table.write(links, chunk_size=150000) - print('Created {} address links'.format(len(links))) + print(f'Created {len(links)} address links') del links insert_rows = [dict(x) for x in new_addresses] if WRITE_OUT: - print("Writing {} new addresses... ".format(len(new_addresses))) + print(f"Writing {len(new_addresses)} new addresses... ") address_table.write(insert_rows, chunk_size=150000) - print('Writing {} base and in-range AIS source addresses...'.format(len(source_addresses))) + print(f'Writing {len(source_addresses)} base and in-range AIS source addresses...') source_address_table.write(source_addresses, chunk_size=150000) source_addresses = [] @@ -793,11 +790,7 @@ def had_street_error(street_address, reason, notes=None): deferred_warnings.append({ 'street_address': street_address, 'reason': 'High address out of range', - 'notes': 'Seg {}: {} to {}'.format( - matching_seg['seg_id'], - check_from, - check_to - ) + 'notes': f"Seg {matching_seg['seg_id']}: {check_from} to {check_to}" }) # If only the high address is in range (unlikely) @@ -807,11 +800,7 @@ def had_street_error(street_address, reason, notes=None): deferred_warnings.append({ 'street_address': street_address, 'reason': 'Low address out of range', - 'notes': 'Seg {}: {} to {}'.format( - matching_seg['seg_id'], - check_from, - check_to - ) + 'notes': f"Seg {matching_seg['seg_id']}: {check_from} to {check_to}" }) # Store the match @@ -910,10 +899,10 @@ def had_street_error(street_address, reason, notes=None): for parcel_layer in parcel_layers: source_table_name = parcel_layer + '_parcel' source_table = db[source_table_name] - print('Reading from {}...'.format(parcel_layer)) + print(f'Reading from {parcel_layer}...') if DEV: - where = "street_name = '{}'".format(DEV_STREET_NAME) + where = f"street_name = '{DEV_STREET_NAME}'" parcel_rows = source_table.read(fields=['street_address', 'id'], \ where=where) else: @@ -935,7 +924,7 @@ def had_street_error(street_address, reason, notes=None): parcel_address = Address(street_address) except ValueError: # TODO: this should never happen - # print('Could not parse parcel address: {}'.format(street_address)) + # print(f'Could not parse parcel address: {street_address}') continue street_full = parcel_address.street_full @@ -1076,7 +1065,7 @@ def had_street_error(street_address, reason, notes=None): address_parcel_table.create_index('street_address') for variant_type, count in match_counts.items(): - print('{} matched on {}'.format(count, variant_type)) + print(f'{count} matched on {variant_type}') del address_parcels @@ -1108,7 +1097,7 @@ def had_street_error(street_address, reason, notes=None): prop_list = range_map.setdefault(street_full, []) prop_list.append(range_row) except ValueError: - print('Unrecognized format for range address: {}'.format(street_address)) + print(f'Unrecognized format for range address: {street_address}') continue address_props = [] @@ -1220,4 +1209,4 @@ def had_street_error(street_address, reason, notes=None): db.close() - print('Finished in {} seconds'.format(datetime.now() - start)) + print(f'Finished in {datetime.now() - start} seconds') diff --git a/ais/engine/scripts/load_curbs.py b/ais/engine/scripts/load_curbs.py index 61246931..b03042fc 100644 --- a/ais/engine/scripts/load_curbs.py +++ b/ais/engine/scripts/load_curbs.py @@ -43,7 +43,7 @@ def main(): print('Making parcel-curbs...') for agency in config['BASE_DATA_SOURCES']['parcels']: print(' - ' + agency) - stmt = ''' + stmt = f''' insert into parcel_curb (parcel_source, parcel_row_id, curb_id) ( select distinct on (p.id) '{agency}', @@ -54,12 +54,12 @@ def main(): on ST_Intersects(p.geom, c.geom) order by p.id, st_area(st_intersection(p.geom, c.geom)) desc ) - '''.format(agency=agency) + ''' db.execute(stmt) db.save() print('Creating indexes...') db.close() - print('Finished in {} seconds'.format(datetime.now() - start)) + print(f'Finished in {datetime.now() - start} seconds') diff --git a/ais/engine/scripts/load_dor_condos.py b/ais/engine/scripts/load_dor_condos.py index 7ee1736a..a04af48b 100644 --- a/ais/engine/scripts/load_dor_condos.py +++ b/ais/engine/scripts/load_dor_condos.py @@ -15,7 +15,7 @@ def main(): db_user = engine_dsn[engine_dsn.index("//") + 2:engine_dsn.index(":", engine_dsn.index("//"))] db_pw = engine_dsn[engine_dsn.index(":",engine_dsn.index(db_user)) + 1:engine_dsn.index("@")] db_name = engine_dsn[engine_dsn.index("/", engine_dsn.index("@")) + 1:] - pg_db = psycopg2.connect('dbname={db_name} user={db_user} password={db_pw} host=localhost'.format(db_name=db_name, db_user=db_user, db_pw=db_pw)) + pg_db = psycopg2.connect(f'dbname={db_name} user={db_user} password={db_pw} host=localhost') # get source table params source_def = config['BASE_DATA_SOURCES']['condos']['dor'] @@ -31,20 +31,21 @@ def main(): # Read DOR CONDO rows from source print("Reading condos...") # TODO: get fieldnames from source_field_map - dor_condo_read_stmt = ''' - select condounit, objectid, mapref from {dor_condo_table} + dor_condo_read_stmt = f''' + select condounit, objectid, mapref from {source_table_name} where status in (1,3) - '''.format(dor_condo_table = source_table_name) + ''' source_dor_condo_rows = etl.fromdb(source_conn, dor_condo_read_stmt).fieldmap(source_field_map) if DEV: print(etl.look(source_dor_condo_rows)) # Read DOR Parcel rows from engine db print("Reading parcels...") - dor_parcel_read_stmt = ''' + dor_parcel_read_stmt = """ select parcel_id, street_address, address_low, address_low_suffix, address_low_frac, address_high, street_predir, - street_name, street_suffix, street_postdir, street_full from {dor_parcel_table} - '''.format(dor_parcel_table='dor_parcel') + street_name, street_suffix, street_postdir, street_full + from dor_parcel + """ engine_dor_parcel_rows = etl.fromdb(pg_db, dor_parcel_read_stmt) if DEV: print(etl.look(engine_dor_parcel_rows)) diff --git a/ais/engine/scripts/load_dor_parcels.py b/ais/engine/scripts/load_dor_parcels.py index 66955846..21debfb1 100644 --- a/ais/engine/scripts/load_dor_parcels.py +++ b/ais/engine/scripts/load_dor_parcels.py @@ -57,10 +57,9 @@ def main(): parcel_error_polygon_table.delete() print('Reading streets...') - street_stmt = ''' + street_stmt = f''' select street_full, seg_id, street_code, left_from, left_to, right_from, right_to - from {} - '''.format(street_table.name) + from {street_table.name}''' street_rows = db.execute(street_stmt) street_code_map = {} # street_full => street_code @@ -211,7 +210,7 @@ def had_error(reason, note=None): found_street_full = True if street_full not in street_code_map: found_street_full = False - note = 'Unknown street: {}'.format(street_full) + note = f'Unknown street: {street_full}' had_warning('Unknown street', note=note) if street_code: @@ -223,7 +222,7 @@ def had_error(reason, note=None): elif found_street_full and \ street_code_map[street_full] != street_code: actual_street = street_full_map[street_code] - note = 'Street code {} => {}'.format(street_code, actual_street) + note = f'Street code {street_code} => {actual_street}' had_warning('Incorrect street code', note=note) # QC: Check for low address number @@ -298,7 +297,7 @@ def had_error(reason, note=None): # Get unit unit_full = None if unit_num: - unit_full = '# {}'.format(unit_num) + unit_full = f'# {unit_num}' address = None @@ -313,7 +312,7 @@ def had_error(reason, note=None): # QC: check for miscellaneous parcel modifications street_address = address.street_address if should_check_street_full and source_address != street_address: - note = 'Parser changes: {} => {}'.format(source_address, street_address) + note = f'Parser changes: {source_address} => {street_address}' had_warning('Parser changes', note=note) # QC: check for duplicate address @@ -434,7 +433,7 @@ def had_error(reason, note=None): parcel_map[object_id] = parcel except Exception as e: - print('{}: Unhandled error'.format(source_parcel)) + print(f'{source_parcel}: Unhandled error') print(parcel_id) print(traceback.format_exc()) raise e @@ -448,7 +447,7 @@ def had_error(reason, note=None): parcel_id = source_parcel['mapreg'] count = parcel_id_counts.get(parcel_id, 0) if count > 1: - note = 'Parcel ID count: {}'.format(count) + note = f'Parcel ID count: {count}' had_warning('Duplicate parcel ID', note=note) # Check for duplicate addresses. Use parcels since source parcels don't have @@ -459,7 +458,7 @@ def had_error(reason, note=None): street_address = parcel['street_address'] count = address_counts.get(street_address, 0) if count > 1: - note = 'Address count: {}'.format(count) + note = f'Address count: {count}' had_warning('Duplicate address', note=note) # Remember how many parcels we went through before we delete them all @@ -541,7 +540,7 @@ def had_error(reason, note=None): target_user = target_dsn[target_dsn.index("//") + 2:target_dsn.index(":", target_dsn.index("//"))] target_pw = target_dsn[target_dsn.index(":",target_dsn.index(target_user)) + 1:target_dsn.index("@")] target_name = target_dsn[target_dsn.index("/", target_dsn.index("@")) + 1:] - target_conn = psycopg2.connect('dbname={db_name} user={db_user} password={db_pw} host=localhost'.format(db_name=target_name, db_user=target_user, db_pw=target_pw)) + target_conn = psycopg2.connect(f'dbname={target_name} user={target_user} password={target_pw} host=localhost') target_table_name = 'public.dor_parcel_error_polygon' error_polygon_rows = etl.fromdicts(error_polygons) error_polygon_rows.topostgis(target_conn, target_table_name) @@ -554,7 +553,7 @@ def had_error(reason, note=None): db.close() - print('Finished in {} seconds'.format(datetime.now() - start)) - print('Processed {} parcels'.format(parcel_count)) - print('{} errors'.format(len(error_map))) - print('{} warnings'.format(len(warning_map))) + print(f'Finished in {datetime.now() - start}') + print(f'Processed {parcel_count} parcels') + print(f'{len(error_map)} errors') + print(f'{len(warning_map)} warnings') diff --git a/ais/engine/scripts/load_dor_parcels_dev.py b/ais/engine/scripts/load_dor_parcels_dev.py deleted file mode 100644 index 081c8c5e..00000000 --- a/ais/engine/scripts/load_dor_parcels_dev.py +++ /dev/null @@ -1,590 +0,0 @@ -import sys -import os -import csv -import re -from datetime import datetime -from passyunk.data import DIRS_STD, SUFFIXES_STD -import datum -from ais.models import Address -from ais.util import parity_for_num, parity_for_range -from ais import app -# DEV -from pprint import pprint -import traceback -import psycopg2 -import petl as etl -import geopetl - -def main(): - start = datetime.now() - print('Starting...') - - """SET UP""" - - config = app.config - db = datum.connect(config['DATABASES']['engine']) - - VALID_ADDRESS_LOW_SUFFIXES = config['VALID_ADDRESS_LOW_SUFFIXES'] - - source_def = config['BASE_DATA_SOURCES']['parcels']['dor'] - source_db_name = source_def['db'] - source_db_url = config['DATABASES'][source_db_name] - source_db = datum.connect(source_db_url) - source_field_map = source_def['field_map'] - source_table_name = source_def['table'] - source_table = source_db[source_table_name] - source_geom_field = source_table.geom_field - field_map = source_def['field_map'] - - street_table = db['street_segment'] - parcel_table = db['dor_parcel'] - parcel_error_table = db['dor_parcel_error'] - parcel_error_polygon_table = db['dor_parcel_error_polygon'] - error_exempt_fields = ['frac',] - WRITE_OUT = True - - # Regex - street_name_re = re.compile('^[A-Z0-9 ]+$') - unit_num_re = re.compile('^[A-Z0-9\-]+$') - parcel_id_re = re.compile('^\d{3}(N|S)\d{6}$') - geometry_re = re.compile('^(MULTI)?POLYGON') - - """MAIN""" - - if WRITE_OUT: - print('Dropping indexes...') - parcel_table.drop_index('street_address') - print('Deleting existing parcels...') - parcel_table.delete() - print('Deleting existing parcel errors...') - parcel_error_table.delete() - print('Deleting existing parcel error polygons...') - parcel_error_polygon_table.delete() - - print('Reading streets...') - street_stmt = ''' - select street_full, seg_id, street_code, left_from, left_to, right_from, right_to - from {} - '''.format(street_table.name) - street_rows = db.execute(street_stmt) - - street_code_map = {} # street_full => street_code - street_full_map = {} # street_code => street_full - seg_map = {} # street_full => [seg rows] - - for street_row in street_rows: - street_code = street_row['street_code'] - street_full = street_row['street_full'] - - seg_map.setdefault(street_full, []) - seg_map[street_full].append(street_row) - - street_code_map[street_full] = street_code - street_full_map[street_code] = street_full - - # TODO: currently there's a problem with parsing street names where a - # single street_full will map to more than one street code. (It's dropping the - # RAMP suffix where it shouldn't be. Only a few instances of this and so just - # override for now. - street_code_map.update({ - 'VINE ST': 80120, - 'MARKET ST': 53560, - 'COMMERCE ST': 24500, - }) - street_full_map.update({ - 80120: 'VINE ST', - 53560: 'MARKET ST', - 24500: 'COMMERCE ST', - }) - - # Get field names - source_where = source_def['where'] - - # DEV - # source_table += ' SAMPLE(1)' - # source_where += " AND mapreg = '001S050134'" - # source_where += " AND objectid = 540985" - #source_where += " AND rownum < 100" - - source_fields = list(field_map.values()) - print(f'Reading parcels from {source_table_name} at db {source_db_url}...') - source_parcels = source_table.read(where=source_where) - print(f'Read in {len(source_parcels)} rows.') - source_parcel_map = {x['objectid']: x for x in source_parcels} - - parcels = [] - parcel_map = {} # object ID => parcel object - - # QC - error_map = {} # object ID => error string - warning_map = {} # object ID => [warning strings] - object_id = None # Make this global so the error functions work - should_add_parcel = None # Declare this here for scope reasons - bad_geom_parcels = [] # Object IDs - - address_counts = {} # street_address => count - parcel_id_counts = {} # parcel_id => count - - # Use this to continue working on a parcel if one part of validation fails - # class KeepGoing(Exception): - # pass - - def had_warning(reason, note=None): - try: - global warning_map - global object_id - parcel_warnings = warning_map.setdefault(object_id, []) - warning = { - 'reason': reason, - 'note': note if note else '', - } - parcel_warnings.append(warning) - except Exception as e: - pass - - def had_error(reason, note=None): - try: - global error_map - global object_id - global should_add_parcel - parcel_errors = error_map.setdefault(object_id, []) - error = { - 'reason': reason, - 'note': note if note else '', - } - parcel_errors.append(error) - should_add_parcel = False - except Exception as e: - pass - - # Loop over source parcels - for i, source_parcel in enumerate(source_parcels): - try: - if i % 50000 == 0: - print(i) - - should_add_parcel = True - - # Strip whitespace, null out empty strings, zeroes - for field, value in source_parcel.items(): - if isinstance(value, str): - value = value.strip() - if len(value) == 0 or value == '0': - value = None - source_parcel[field] = value - elif value == 0: - source_parcel[field] = None - - # Get attributes - object_id = source_parcel[field_map['source_object_id']] - address_low = source_parcel[field_map['address_low']] - address_low_suffix = source_parcel[field_map['address_low_suffix']] - address_low_fractional = source_parcel[field_map['address_low_frac']] - address_high = source_parcel[field_map['address_high']] - street_predir = source_parcel[field_map['street_predir']] - street_name = source_parcel[field_map['street_name']] - street_suffix = source_parcel[field_map['street_suffix']] - street_postdir = source_parcel[field_map['street_postdir']] - unit_num = source_parcel[field_map['unit_num']] - street_code = source_parcel[field_map['street_code']] - parcel_id = source_parcel[field_map['parcel_id']] - geometry = source_parcel[source_geom_field] - - # Declare this here so the except clause doesn't bug out - source_address = None - - # Set this flag to false if we handle any specific address errors. - # If no specific errors are found, compare the parsed address to - # the source address to flag parser modifications. - should_check_street_full = True - - # QC: Check address components - if street_predir and street_predir not in DIRS_STD: - had_warning('Non-standard predir') - should_check_street_full = False - if street_postdir and street_postdir not in DIRS_STD: - had_warning('Non-standard postdir') - should_check_street_full = False - if street_suffix and street_suffix not in SUFFIXES_STD: - had_warning('Non-standard suffix') - should_check_street_full = False - if unit_num and unit_num_re and not unit_num_re.match(unit_num): - had_warning('Invalid unit num') - should_check_street_full = False - #if address_low_fractional and address_low_fractional not in ('1/4', '1/3', '1/2'): - # had_warning('Invalid address_low_frac') - # should_check_street_full = False - - - - - # QC: Check street components - if street_name is None: - had_error('No street name') - if street_code is None: - had_warning('No street code') - - # Make street full - if street_name: - street_comps = [street_predir, street_name, street_suffix, \ - street_postdir] - street_full = ' '.join([x for x in street_comps if x]) - - # QC: Check if street full exists - found_street_full = True - if street_full not in street_code_map: - found_street_full = False - note = 'Unknown street: {}'.format(street_full) - # had_error('Unknown street', note=note) - had_warning('Unknown street', note=note) - - if street_code: - # QC: Check if street code exists - if street_code not in street_full_map: - had_warning('Unknown street code') - - # QC: Check if street full matches street code - elif found_street_full and \ - street_code_map[street_full] != street_code: - actual_street = street_full_map[street_code] - note = 'Street code {} => {}'.format(street_code, actual_street) - had_warning('Incorrect street code', note=note) - - # QC: Check for low address number - if address_low is None: - had_error('No address number') - - # Clean up - if address_high == 0: - address_high = None - - if address_low_suffix not in VALID_ADDRESS_LOW_SUFFIXES: - address_low_suffix = None - if address_low_suffix == '2': - address_low_fractional = '1/2' - address_low_suffix = None - # Handle ranges - if address_low and address_high: - address_low_str = str(address_low) - address_high_str = str(address_high) - len_address_low = len(address_low_str) - len_address_high = len(address_high_str) - address_high_full = None - - if len(address_high_str) != 2: - had_warning('High address should be two digits') - - if not address_high_str.isnumeric(): - address_high = None - had_warning('Invalid high address') - - if address_high: - # Case: 1234-36 or 1234-6 - if len_address_high < len_address_low: - # Make address high full and compare to address low - address_high_prefix = address_low_str[:-len_address_high] - address_high_full = int(address_high_prefix + address_high_str) - # Cases: 1234-1236 or 2-12 - elif len_address_low == len_address_high or \ - (len_address_low == 1 and len_address_high == 2): - address_high_full = address_high - else: - had_error('Address spans multiple hundred blocks') - - # Case: 317-315 - if address_high_full: - if address_high_full < address_low: - # print(address_low, address_high_full) - had_error('Inverted range address') - - # Make sure both addresses are on the same hundred block - hun_block_low = address_low - (address_low % 100) - hun_block_high = address_high_full - (address_high_full % 100) - if hun_block_low != hun_block_high: - # print(hun_block_low, hun_block_high) - had_error('Address spans multiple hundred blocks') - - address_high = str(address_high_full)[-2:] - - # Make address full - address_full = None - if address_low: - address_full = str(address_low) - if address_low_suffix: - address_full += address_low_suffix - if address_low_fractional: - address_full += ' ' + address_low_fractional - if address_high: - address_full += '-' + str(address_high) - - # Get unit - unit_full = None - if unit_num: - unit_full = '# {}'.format(unit_num) - - address = None - - if address_full and street_full: - source_address_comps = [address_full, street_full, unit_full] - source_address = ' '.join([x for x in source_address_comps if x]) - - # Try to parse - try: - address = Address(source_address) - - # QC: check for miscellaneous parcel modifications - street_address = address.street_address - if should_check_street_full and source_address != street_address: - note = 'Parser changes: {} => {}'.format(source_address, street_address) - had_warning('Parser changes', note=note) - - # QC: check for duplicate address - address_counts.setdefault(street_address, 0) - address_counts[street_address] += 1 - - except Exception as e: - #print(source_address) - had_error('Could not parse') - - # QC: parcel ID (aka mapreg) - if parcel_id is None: - had_error('No parcel ID') - else: - # Check for duplicate - parcel_id_counts.setdefault(parcel_id, 0) - parcel_id_counts[parcel_id] += 1 - - if not parcel_id_re.match(parcel_id): - had_warning('Invalid parcel ID') - - # QC: geometry - if not geometry_re.match(geometry): - had_error('Invalid geometry') - bad_geom_parcels.append(object_id) - - ''' - STREET MATCH - ''' - - if address: - # Get the parsed street_full - street_full = address.street_full - - if street_full in seg_map: - address_low = address.address_low - address_high = address.address_high - street_full = address.street_full - address_parity = parity_for_num(address_low) - matching_segs = seg_map[street_full] - matching_seg = None - matching_side = None - had_alias = False # TODO: check for aliases - - # Loop through segs for that street full - for seg in matching_segs: - left_from = seg['left_from'] - left_to = seg['left_to'] - right_from = seg['right_from'] - right_to = seg['right_to'] - - left_parity = parity_for_range(left_from, left_to) - right_parity = parity_for_range(right_from, right_to) - - # Match to side of street based on parity - check_from = None - check_to = None - - if left_parity in [address_parity, 'B']: - check_from = left_from - check_to = left_to - matching_side = 'L' - elif right_parity in [address_parity, 'B']: - check_from = right_from - check_to = right_to - matching_side = 'R' - else: - continue - - # If it's in range - if check_from <= address_low <= check_to: - # And it's a single address - if address_high is None: - matching_seg = seg - break - # Otherwise if it's a range address - else: - # If we already had a match, flag multiple matches - if matching_seg: - seg_ids = sorted([x['seg_id'] for x in [matching_seg, seg]]) - note = ','.join([str(x) for x in seg_ids]) - had_warning('Range address matches multiple street segments', note=note) - - # Check if the high address is greater than the street max - if check_to < address_high: - # Otherwise, make the match and keep looking (in case - # it matches to multiple segments) - had_warning('High address out of street range') - - matching_seg = seg - - if matching_seg is None: - # had_error('Out of street range') - # should_add_parcel = False - had_warning('Out of street range') - should_add_parcel = True - - - ''' - END STREET MATCH - ''' - - # Make parcel object - if should_add_parcel: - parcel = dict(address) - #Remove fields not in parcel tables: - parcel.pop('zip_code', None) - parcel.pop('zip_4', None) - #Add fields: - parcel.update({ - 'parcel_id': parcel_id, - 'source_object_id': object_id, - 'source_address': source_address, - 'geom': geometry, - }) - parcels.append(parcel) - parcel_map[object_id] = parcel - - # except ValueError as e: - # # print('Parcel {}: {}'.format(parcel_id, e)) - # reason = str(e) - # had_error(reason) - - except Exception as e: - print('{}: Unhandled error'.format(source_parcel)) - print(parcel_id) - print(traceback.format_exc()) - raise e - - print('Checking for duplicates...') - - # Check for duplicate parcel IDs. Use source parcels for most results. - for source_parcel in source_parcels: - # Set object ID here so logging function works - object_id = source_parcel['objectid'] - parcel_id = source_parcel['mapreg'] - count = parcel_id_counts.get(parcel_id, 0) - if count > 1: - note = 'Parcel ID count: {}'.format(count) - had_warning('Duplicate parcel ID', note=note) - - # Check for duplicate addresses. Use parcels since source parcels don't have - # an address. - for parcel in parcels: - # Set object ID here so logging function works - object_id = parcel['source_object_id'] - street_address = parcel['street_address'] - count = address_counts.get(street_address, 0) - if count > 1: - note = 'Address count: {}'.format(count) - had_warning('Duplicate address', note=note) - - # Remember how many parcels we went through before we delete them all - parcel_count = len(parcels) - - if WRITE_OUT: - print('Writing parcels...') - parcel_table.write(parcels, chunk_size=50000) - - print('Writing parcel errors...') - errors = [] - source_non_geom_fields = [x for x in source_fields if x != source_geom_field] - - for level in ['error', 'warning']: - issue_map = error_map if level == 'error' else warning_map - for object_id, issues in issue_map.items(): - for issue in issues: - reason = issue['reason'] - note = issue['note'] - source_parcel = source_parcel_map[object_id] - - # Make error row - error = {x: source_parcel[x] if source_parcel[x] is not None \ - else '' for x in source_non_geom_fields if x not in error_exempt_fields} - - # Make this work for integer fields - if error['house'] == '': - error['house'] = None - if error['stcod'] == '': - error['stcod'] = None - - error.update({ - 'level': level, - 'reason': reason, - 'notes': note, - }) - - errors.append(error) - - parcel_error_table.write(errors, chunk_size=150000) - del errors - - print('Writing parcel error polygons...') - error_polygons = [] - - for level in ['error', 'warning']: - issue_map = error_map if level == 'error' else warning_map - - for object_id, issues in issue_map.items(): - # If this object had a geometry error, skip - if object_id in bad_geom_parcels: - continue - - # Roll up reasons, notes - reasons = [x['reason'] for x in issues] - reasons_joined = '; '.join(sorted(reasons)) - notes = [x['note'] for x in issues if x['note'] != ''] - notes_joined = '; '.join(notes) - source_parcel = source_parcel_map[object_id] - - # Make error row - error_polygon = {x: source_parcel[x] if x is not None \ - else '' for x in source_fields if x not in error_exempt_fields} - - # Add/clean up fields - if error_polygon['house'] == '': - error_polygon['house'] = None - if error_polygon['stcod'] == '': - error_polygon['stcod'] = None - - error_polygon.update({ - 'reasons': reasons_joined, - 'reason_count': len(reasons), - 'notes': notes_joined, - # 'shape': source_parcel[wkt_field], - }) - error_polygons.append(error_polygon) - - # parcel_error_polygon_table.write(error_polygons, chunk_size=50000) - - target_dsn = config['DATABASES']['engine'] - target_user = target_dsn[target_dsn.index("//") + 2:target_dsn.index(":", target_dsn.index("//"))] - target_pw = target_dsn[target_dsn.index(":",target_dsn.index(target_user)) + 1:target_dsn.index("@")] - target_name = target_dsn[target_dsn.index("/", target_dsn.index("@")) + 1:] - target_conn = psycopg2.connect('dbname={db_name} user={db_user} password={db_pw} host=localhost'.format(db_name=target_name, db_user=target_user, db_pw=target_pw)) - target_table_name = 'public.dor_parcel_error_polygon' - error_polygon_rows = etl.fromdicts(error_polygons) - error_polygon_rows.topostgis(target_conn, target_table_name) - - del error_polygons - - print('Creating indexes...') - parcel_table.create_index('street_address') - # TODO: index error tables? - - #source_db.close() - db.close() - - print('Finished in {} seconds'.format(datetime.now() - start)) - print('Processed {} parcels'.format(parcel_count)) - print('{} errors'.format(len(error_map))) - print('{} warnings'.format(len(warning_map))) diff --git a/ais/engine/scripts/load_ng911_address_points.py b/ais/engine/scripts/load_ng911_address_points.py index c9a33b99..3a6f5826 100644 --- a/ais/engine/scripts/load_ng911_address_points.py +++ b/ais/engine/scripts/load_ng911_address_points.py @@ -98,11 +98,11 @@ def main(): pass except Exception as e: - print('Unhandled exception on {}'.format(source_address)) + print(f'Unhandled exception on {source_address}') print(traceback.format_exc()) # sys.exit() - print('Writing {} NG911 address points...'.format(len(address_points))) + print(f'Writing {len(address_points)} NG911 address points...') ng911_table.write(address_points) print('Creating index...') @@ -113,4 +113,4 @@ def main(): ''' db.close() - print('Finished in {} seconds'.format(datetime.now() - start)) \ No newline at end of file + print(f'Finished in {datetime.now() - start} seconds') \ No newline at end of file diff --git a/ais/engine/scripts/load_opa_properties.py b/ais/engine/scripts/load_opa_properties.py index 54b54d03..abff7a69 100644 --- a/ais/engine/scripts/load_opa_properties.py +++ b/ais/engine/scripts/load_opa_properties.py @@ -46,9 +46,7 @@ def main(): prop_table.delete() print('Reading owners from source...') - owner_stmt = """ - select account_num, owners from {} - """.format(owner_table_name) + owner_stmt = f"select account_num, owners from {owner_table_name}" owner_rows = ais_source_db.execute(owner_stmt) owner_map = {x['account_num']: x['owners'] for x in owner_rows} @@ -75,7 +73,7 @@ def main(): comps = parsed['components'] except: raise ValueError('Could not parse') - address = Address(parsed) + street_address = comps['output_address'] # Owners @@ -102,12 +100,8 @@ def main(): } props.append(prop) - # except ValueError as e: - # # FEEDBACK - # pass - except Exception as e: - print('Unhandled exception on {}'.format(source_address)) + print(f'Unhandled exception on {source_address}') print(traceback.format_exc()) raise e @@ -122,4 +116,4 @@ def main(): ''' db.close() - print('Finished in {} seconds'.format(datetime.now() - start)) + print(f'Finished in {datetime.now() - start}') diff --git a/ais/engine/scripts/load_pwd_parcels.py b/ais/engine/scripts/load_pwd_parcels.py index 98e75b3b..85b57aa4 100644 --- a/ais/engine/scripts/load_pwd_parcels.py +++ b/ais/engine/scripts/load_pwd_parcels.py @@ -1,6 +1,3 @@ -import sys -import os -import csv from datetime import datetime import datum from ais import app @@ -46,12 +43,12 @@ def main(): # these addresses, use OPA address instead. Case: 421 S 10TH ST appears three # times in parcels but should have unit nums according to OPA. print('Loading non-unique parcel addresses...') - ambig_stmt = ''' + ambig_stmt = f''' select address - from {} + from {source_table_name} group by address having address is not null and count(*) > 1 - '''.format(source_table_name) + ''' source_db._c.execute(ambig_stmt) ambig_rows = source_db._c.fetchall() ambig_addresses = set([x['address'] for x in ambig_rows]) @@ -59,17 +56,6 @@ def main(): """MAIN""" - # # Set up logging - # LOG_COLS = [ - # 'parcel_id', - # 'source_address', - # 'error', - # ] - # parent_dir = os.path.abspath(os.path.join(__file__, os.pardir)) - # log = open(parent_dir + '/log/load_pwd_parcels.log', 'w', newline='') - # log_writer = csv.writer(log) - # log_writer.writerow(LOG_COLS) - print('Dropping indexes...') parcel_table.drop_index('street_address') @@ -110,8 +96,7 @@ def main(): try: address = Address(source_address) except: - # raise ValueError('Could not parse') - raise ValueError('Could not parse: {}'.format(source_address)) + raise ValueError(f'Could not parse: {source_address}') parcel = dict(address) # Remove fields not in parcel tables: @@ -127,27 +112,25 @@ def main(): # FEEDBACK # if source_address != parcel.street_address: - # print('{} => {}'.format(source_address, parcel.street_address)) + # print(f'{source_address} => {parcel.street_address}') except ValueError as e: - #print('Parcel {}: {}'.format(parcel_id, e)) + #print(f'Parcel {parcel_id}: {e}') # log_writer.writerow([parcel_id, source_address, e]) pass except Exception as e: - print('{}: Unhandled error'.format(source_parcel)) + print(f'{source_parcel}: Unhandled error') print(traceback.format_exc()) raise e print('Writing parcels...') parcel_table.write(parcels, chunk_size=50000) - # db.save() print('Creating indexes...') parcel_table.create_index('street_address') #source_db.close() db.close() - # log.close() - print('Finished in {} seconds'.format(datetime.now() - start)) - print('Wrote {} parcels'.format(len(parcels))) + print(f'Finished in {datetime.now() - start} seconds') + print(f'Wrote {len(parcels)} parcels') diff --git a/ais/engine/scripts/load_service_areas.py b/ais/engine/scripts/load_service_areas.py index 8d21bbfa..f87fab2d 100644 --- a/ais/engine/scripts/load_service_areas.py +++ b/ais/engine/scripts/load_service_areas.py @@ -84,12 +84,12 @@ def main(): for layer in layers: layer_id = layer['layer_id'] - print(' - {}'.format(layer_id)) + print(f' - {layer_id}') sources = layer['sources'] # Check for conflicting source types if 'line_single' in sources and 'line_dual' in sources: - raise Exception('Too many line sources for {}'.format(layer_id)) + raise Exception(f'Too many line sources for {layer_id}') for source_type, source in sources.items(): # Connect to DB @@ -97,7 +97,7 @@ def main(): try: source_db = datum.connect(config['DATABASES'][source_db_name]) except KeyError: - print('Database {} not found'.format(layer_id)) + print(f'Database {layer_id} not found') continue source_table_name = source['table'] @@ -259,4 +259,4 @@ def main(): line_dual_table.create_index('seg_id') db.close() - print('Finished in {} seconds'.format(datetime.now() - start)) + print(f'Finished in {datetime.now() - start} seconds') \ No newline at end of file diff --git a/ais/engine/scripts/load_street_aliases.py b/ais/engine/scripts/load_street_aliases.py index 83617869..cd55e826 100644 --- a/ais/engine/scripts/load_street_aliases.py +++ b/ais/engine/scripts/load_street_aliases.py @@ -1,4 +1,3 @@ -import sys import traceback from datetime import datetime from passyunk.data import DIRS_STD, SUFFIXES_STD @@ -32,7 +31,7 @@ def main(): aliases = [] # Loop over aliases - for i, alias_row in enumerate(source_rows): + for _, alias_row in enumerate(source_rows): try: # Get attrs predir = alias_row[field_map['street_predir']] @@ -50,7 +49,7 @@ def main(): invalid_suffix = (suffix and suffix not in SUFFIXES_STD) invalid_postdir = (postdir and not postdir in DIRS_STD) if any([invalid_predir, invalid_suffix, invalid_postdir]): - raise ValueError('Invalid alias: {}'.format(source_street_full)) + raise ValueError(f'Invalid alias: {source_street_full}') # Standardize predir = DIRS_STD[predir] if predir else None @@ -85,4 +84,4 @@ def main(): db.save() db.close() - print('Finished in {} seconds'.format(datetime.now() - start)) + print(f'Finished in {datetime.now() - start} seconds') diff --git a/ais/engine/scripts/load_streets.py b/ais/engine/scripts/load_streets.py index 23e11f3f..001d5fd9 100644 --- a/ais/engine/scripts/load_streets.py +++ b/ais/engine/scripts/load_streets.py @@ -1,8 +1,6 @@ -import sys import traceback from datetime import datetime from pprint import pprint -# from phladdress.parser import Parser from ais import app from datum import Database from ais.models import StreetSegment @@ -45,7 +43,6 @@ def main(): street_table.delete(cascade=True) print(f'Reading streets table {source_table} from source...') - source_fields = list(field_map.values()) source_rows = source_table.read(to_srid=engine_srid) print('Rows retrieved.') @@ -61,10 +58,8 @@ def main(): # Parse street name source_street_full_comps = [str(source_row[x]).strip() for x in \ source_street_full_fields] - # source_street_full_comps = [x for x in source_street_full_comps if x != ''] source_street_full_comps = [x for x in source_street_full_comps if x not in ('', None, 'None')] source_street_full = ' '.join(source_street_full_comps) - seg_id = source_row[field_map['seg_id']] try: parsed = parser.parse(source_street_full) if parsed['type'] != 'street': @@ -102,7 +97,7 @@ def main(): error_count += 1 except Exception as e: - print('Unhandled error on row: {}'.format(i)) + print(f'Unhandled error on row: {i}') print(traceback.format_exc()) raise e @@ -116,6 +111,6 @@ def main(): FINISH ''' - print('{} errors'.format(error_count)) + print(f'{error_count} errors') db.close() - print('Finished in {} seconds'.format(datetime.now() - start)) + print(f'Finished in {datetime.now() - start} seconds') diff --git a/ais/engine/scripts/load_zip_ranges.py b/ais/engine/scripts/load_zip_ranges.py deleted file mode 100644 index 0cfdf19d..00000000 --- a/ais/engine/scripts/load_zip_ranges.py +++ /dev/null @@ -1,305 +0,0 @@ -import sys -import os -import csv -from copy import deepcopy -from datetime import datetime -import datum -from ais import app -from ais.models import Address -# DEV -import traceback -from pprint import pprint - -def main(): - start = datetime.now() - - """SET UP""" - - config = app.config - db = datum.connect(config['DATABASES']['engine']) - source_db = datum.connect(config['DATABASES']['gis']) - # source_table = source_db['usps_zip4s'] - source_table = source_db['vw_usps_zip4s_ais'] - field_map = { - 'usps_id': 'updatekey', - 'address_low': 'addrlow', - 'address_high': 'addrhigh', - 'address_oeb': 'addroeb', - 'street_predir': 'streetpre', - 'street_name': 'streetname', - 'street_suffix': 'streetsuff', - 'street_postdir': 'streetpost', - 'unit_type': 'addrsecondaryabbr', - 'unit_low': 'addrsecondarylow', - 'unit_high': 'addrsecondaryhigh', - 'unit_oeb': 'addrsecondaryoeb', - 'zip_code': 'zipcode', - 'zip_4_low': 'zip4low', - 'zip_4_high': 'zip4high', - } - numeric_fields = ['address_low', 'address_high'] - zip_range_table = db['zip_range'] - address_zip_table = db['address_zip'] - WRITE_OUT = True - - source_fields = [field_map[x] for x in field_map] - char_fields = [x for x in field_map if not x in numeric_fields] - - """MAIN""" - - if WRITE_OUT: - print('Dropping indexes...') - zip_range_table.drop_index('street_address') - - print('Deleting existing zip ranges...') - zip_range_table.delete() - - print('Reading zip ranges from source...') - # TODO: currently filtering out alphanumeric addrlows - source_rows = source_table.read(fields=source_fields) - - zip_ranges = [] - - for i, source_row in enumerate(source_rows): - if i % 25000 == 0: - print(i) - - zip_range = {x: source_row[field_map[x]] for x in field_map} - - # Default char fields to empty string - for x in char_fields: - if zip_range[x] is None: - zip_range[x] = '' - - # Handle differing ZIP4 low/high - zip_4_low = source_row[field_map['zip_4_low']] - zip_4_high = source_row[field_map['zip_4_high']] - if zip_4_low != zip_4_high: - zip_4 = '' - else: - zip_4 = zip_4_low - - # Set ZIP4 - zip_range.pop('zip_4_low') - zip_range.pop('zip_4_high') - zip_range['zip_4'] = zip_4 - - zip_ranges.append(zip_range) - - if WRITE_OUT: - print('Writing zip ranges to AIS...') - zip_range_table.write(zip_ranges) - - print('Creating indexes...') - zip_range_table.create_index('usps_id') - - - print('\n** RELATE TO ADDRESSES**') - print('Reading addresses...') - addresses = db['address'].read(fields=['street_address']) - addresses = [Address(x['street_address']) for x in addresses] - - if WRITE_OUT: - print('Dropping indexes...') - address_zip_table.drop_index('street_address') - address_zip_table.drop_index('usps_id') - print('Dropping address-zips...') - address_zip_table.delete() - - # index zip ranges by street_full - street_full_fields = [ - 'street_predir', - 'street_name', - 'street_suffix', - 'street_postdir', - ] - - # For checking alpha unit ranges - alpha_list = list(map(chr, range(65, 91))) - alpha_map = {alpha_list[i]: i + 1 for i in range(0, 26)} # A => 1, Z => 26 - - GENERIC_UNITS = set(['#', 'APT', 'UNIT', 'STE']) - - zip_map_no_units = {} # street_full => [non-unit ranges] - zip_map_units = {} # street_full => [unit ranges] - address_zips = [] - - print('Indexing zip ranges by street...') - for zip_range in zip_ranges: - street_full = ' '.join([zip_range[x] for x in street_full_fields \ - if zip_range[x] != '']) - if zip_range['unit_type'] != '': - street_zip_ranges = zip_map_units.setdefault(street_full, []) - else: - street_zip_ranges = zip_map_no_units.setdefault(street_full, []) - street_zip_ranges.append(zip_range) - - # DEV - exact_count = 0 - unit_num_count = 0 - unit_alpha_count = 0 - - # Loop over addresses - for address in addresses: - try: - address_low = address.address_low - address_high = address.address_high or address_low - address_parity = address.parity - street_address = address.street_address - street_full = address.street_full - unit_type = address.unit_type - unit_num = address.unit_num - - matching_zip_range = None - match_type = None - - # UNIT - # TODO: handle unit types like REAR that don't have a unit num - if unit_type and unit_num: - try: - street_zip_ranges = zip_map_units[street_full] - - # Determine unit character type - # ex. numeric, alpha, alphanum - if unit_num.isdigit(): - unit_char_type = 'num' - # Only accepting single alpha units for now. Multiple will take - # more handling logic. - elif unit_num.isalpha() and len(unit_num) == 1: - unit_char_type = 'alpha' - else: - raise ValueError('Unit format not recognized') - - for zip_range in street_zip_ranges: - zip_unit_type = zip_range['unit_type'] - zip_unit_low = zip_range['unit_low'] - zip_unit_high = zip_range['unit_high'] - - # Check if address matches - if not (zip_range['address_low'] <= address_low and \ - address_high <= zip_range['address_high']): - continue - - # Check if parity matches - zip_address_parity = zip_range['address_oeb'] - if zip_address_parity not in ['B', address_parity]: - continue - - # Check if unit type matches - if unit_type != zip_unit_type and \ - not (unit_type in GENERIC_UNITS and \ - zip_unit_type in GENERIC_UNITS): - continue - - # Get char type of unit range - if zip_unit_low.isdigit() and zip_unit_high.isdigit(): - zip_unit_char_type = 'num' - elif zip_unit_low.isalpha() and zip_unit_high.isalpha(): - zip_unit_char_type = 'alpha' - else: - # Unhandled unit char type - continue - - # If the types don't match, continue - if zip_unit_char_type != unit_char_type: - continue - - # Case 1: numeric unit - if unit_char_type == 'num': - if zip_unit_low <= unit_num <= zip_unit_high: - matching_zip_range = zip_range - match_type = 'unit_numeric' - unit_num_count += 1 - break - - # Case 2: alpha unit range - elif unit_char_type == 'alpha' and \ - len(zip_unit_low) == 1 and len(zip_unit_high) == 1: - try: - unit_alpha_i = alpha_map[unit_num] - zip_alpha_i_low = alpha_map[zip_unit_low] - zip_alpha_i_high = alpha_map[zip_unit_high] - except KeyError: - #print('Unhandled KeyError') - pass - - if zip_alpha_i_low <= unit_alpha_i <= \ - zip_alpha_i_high: - # print('we got an alpha match') - # print(street_address) - # print(zip_range) - # sys.exit() - matching_zip_range = zip_range - match_type = 'unit_alpha' - unit_alpha_count += 1 - break - - except ValueError: - # This should only happen when we had an unrecognized unit - # format. Ignore and try to match to base zip range. - pass - - except KeyError: - pass - - # NON-UNIT - # Use this if statement and not an else, because we still want this to - # run if the unit search didn't turn anything up. - if matching_zip_range is None: - try: - street_zip_ranges = zip_map_no_units[street_full] - except KeyError: - raise ValueError('Not a USPS street') - - for zip_range in street_zip_ranges: - # Check if parity matches - zip_address_parity = zip_range['address_oeb'] - if zip_address_parity not in ['B', address_parity]: - continue - - if zip_range['address_low'] <= address_low and \ - address_high <= zip_range['address_high']: - matching_zip_range = zip_range - - # If there was a unit that we ignored, flag it - if unit_type: - match_type = 'ignore_unit' - else: - match_type = 'exact' - - exact_count += 1 - break - - if matching_zip_range: - address_zips.append({ - 'street_address': street_address, - 'usps_id': matching_zip_range['usps_id'], - 'match_type': match_type, - }) - - else: - raise ValueError('Could not match to a ZIP range') - - except ValueError as e: - # FEEDBACK - # print('{}: {}'.format(street_address, e)) - pass - - print(len(address_zips)) - print('num: ' + str(unit_num_count)) - print('alpha: ' + str(unit_alpha_count)) - print('exact: ' + str(exact_count)) - - if WRITE_OUT: - print('Writing address-zips...') - address_zip_table.write(address_zips, chunk_size=150000) - print('Creating index...') - address_zip_table.create_index('street_address') - address_zip_table.create_index('usps_id') - - ################################################################################ - - source_db.close() - db.close() - - print('Finished in {}'.format(datetime.now() - start)) diff --git a/ais/engine/scripts/make_address_summary.py b/ais/engine/scripts/make_address_summary.py index fd8a16cd..a5d4ece4 100644 --- a/ais/engine/scripts/make_address_summary.py +++ b/ais/engine/scripts/make_address_summary.py @@ -1,11 +1,8 @@ -import sys from datetime import datetime -from shapely.wkt import loads from datetime import datetime from copy import deepcopy import datum from ais import app -from ais.models import Address # DEV import traceback from pprint import pprint @@ -26,13 +23,10 @@ def main(): non_summary_tags = config['ADDRESS_SUMMARY']['non_summary_tags'] geocode_table = db['geocode'] address_table = db['address'] - max_values = config['ADDRESS_SUMMARY']['max_values'] geocode_types = config['ADDRESS_SUMMARY']['geocode_types'] geocode_priority_map = config['ADDRESS_SUMMARY']['geocode_priority'] - #geocode_types_on_curb = config['ADDRESS_SUMMARY']['geocode_types_on_curb'] geocode_types_in_street = config['ADDRESS_SUMMARY']['geocode_types_in_street'] - tag_table = db['address_tag'] link_table = db['address_link'] address_summary_table = db['address_summary'] @@ -151,7 +145,7 @@ def get_tag_by_key(tag_rows, search_key): cur_first_character = None print('Reading addresses...') - for i, street_name in enumerate(street_names): + for _, street_name in enumerate(street_names): first_character = street_name[0] if first_character != cur_first_character: #print(street_name) @@ -162,13 +156,14 @@ def get_tag_by_key(tag_rows, search_key): # Get address tags tag_map = {} # street_address => tag_key => [tag values] tag_keys = [x['tag_key'] for x in tag_fields] - tag_where = "key in ({})".format(', '.join(["'{}'".format(x) for x in tag_keys])) - tag_stmt = ''' + tag_where_insert = ', '.join([f"'{x}'" for x in tag_keys]) + tag_where = f"key in ({tag_where_insert})" + tag_stmt = f''' select street_address, key, value from address_tag where street_address in ( - select street_address from address where street_name = '{}' + select street_address from address where street_name = '{street_name}' ) - '''.format(street_name) + ''' tag_rows = db.execute(tag_stmt) # Make tag map @@ -179,7 +174,7 @@ def get_tag_by_key(tag_rows, search_key): # tag_map_obj = {tag_row['key']: tag_row['value']} tag_map[street_address].append(tag_row) - for i, address_row in enumerate(address_rows): + for _, address_row in enumerate(address_rows): street_address = address_row['street_address'] # Skip unit children @@ -249,11 +244,12 @@ def get_tag_by_key(tag_rows, search_key): value = '' summary_row[field_name] = value - # print('{} => {}'.format(field_name, value)) + # print(f'{field_name} => {value}') # Geocode geocode_rows = geocode_map.get(street_address, []) - if len(geocode_rows) == 0: geocode_errors += 1 + if len(geocode_rows) == 0: + geocode_errors += 1 xy_map = {x['geocode_type']: x['geom'] for x in geocode_rows} geocode_vals = None @@ -355,8 +351,15 @@ def get_tag_by_key(tag_rows, search_key): ( select asm.street_address, asmj.street_code from scnulls asm - inner join address_summary asmj on asmj.street_code is not null and asmj.address_low = asm.address_low and asmj.address_low_suffix = asm.address_low_suffix and asmj.address_low_frac = asm.address_low_frac - and asm.street_predir = asmj.street_predir and asm.street_name = asmj.street_name and asmj.street_suffix = asm.street_suffix and asmj.street_postdir = asm.street_postdir + inner join address_summary asmj + on asmj.street_code is not null + and asmj.address_low = asm.address_low + and asmj.address_low_suffix = asm.address_low_suffix + and asmj.address_low_frac = asm.address_low_frac + and asm.street_predir = asmj.street_predir + and asm.street_name = asmj.street_name + and asmj.street_suffix = asm.street_suffix + and asmj.street_postdir = asm.street_postdir group by asm.street_address, asmj.street_code )final where final.street_address = asm.street_address @@ -412,8 +415,11 @@ def get_tag_by_key(tag_rows, search_key): address_sources as ( select street_address, string_agg(source_name,'|') as sources - from (select distinct street_address, source_name from source_address where source_name not in ('AIS', 'voters', 'info_commercial', 'info_residents', 'li_eclipse_location_ids', 'li_address_keys') - order by street_address, source_name) foo + from (select distinct street_address, source_name + from source_address + where source_name not in + ('AIS', 'voters', 'info_commercial', 'info_residents', 'li_eclipse_location_ids', 'li_address_keys') + order by street_address, source_name) foo group by street_address ) , @@ -439,14 +445,11 @@ def get_tag_by_key(tag_rows, search_key): # Insert ungeocoded opa addresses into geocode table with null geoms: print("Inserting ungeocoded opa addresses into geocode table with null geom...") - stmt = ''' - insert into geocode (street_address, geocode_type) values ('{street_address}', 99) - ''' for street_address in ungeocoded_opa_addresses: - db.execute(stmt.format(street_address=street_address)) + db.execute(f"insert into geocode (street_address, geocode_type) values ('{street_address}', 99)") db.save() db.close() - print('{} geocode errors'.format(geocode_errors)) - print('Finished in {} seconds'.format(datetime.now() - start)) + print(f'{geocode_errors} geocode errors') + print(f'Finished in {datetime.now() - start} seconds') diff --git a/ais/engine/scripts/make_linked_tags.py b/ais/engine/scripts/make_linked_tags.py index 79d39289..90a461d8 100644 --- a/ais/engine/scripts/make_linked_tags.py +++ b/ais/engine/scripts/make_linked_tags.py @@ -196,7 +196,7 @@ def main(): print('Rejected links: ') for key, value in rejected_link_map.items(): value=list(set(value)) - print('{key}: {value}'.format(key=key, value=value)) + print(f'{key}: {value}') # Finally, loop through addresses one last time checking for tags with keys not in tag table, and for each tag lookup # tag linked_addresses in address_link table address_2 for street_address having unit type & num matching the current @@ -243,7 +243,7 @@ def main(): i=0 rejected_link_map = {} - print('Looping through {} addresses...'.format(len(address_rows))) # Remove this + print(f'Looping through {len(address_rows)} addresses...') # Remove this for address_row in address_rows: i+=1 if i % 10000 == 0: @@ -361,9 +361,8 @@ def main(): print('Rejected links: ') for key, value in rejected_link_map.items(): value=list(set(value)) - print('{key}: {value}'.format(key=key, value=value)) + print(f'{key}: {value}') cleanup(address_file) - - transpired = datetime.now() - start - print("Finished in ", transpired, " minutes.") + + print(f"Finished in {datetime.now() - start} minutes.") diff --git a/ais/engine/scripts/make_service_area_summary.py b/ais/engine/scripts/make_service_area_summary.py index 08bb7e3b..2b6e5ede 100644 --- a/ais/engine/scripts/make_service_area_summary.py +++ b/ais/engine/scripts/make_service_area_summary.py @@ -1,11 +1,8 @@ -import sys from datetime import datetime -from shapely.wkt import loads from datetime import datetime from copy import deepcopy import datum from ais import app -from ais.models import Address # DEV import traceback @@ -132,7 +129,7 @@ def main(): if sa_rows is None and None not in (x,y): # Get intersecting service areas - where = 'ST_Intersects(geom, ST_SetSrid(ST_Point({}, {}), 2272))'.format(x, y) + where = f'ST_Intersects(geom, ST_SetSrid(ST_Point({x}, {y}), 2272))' sa_rows = poly_table.read(fields=['layer_id', 'value'], where=where, return_geom=False) # Add to map @@ -177,14 +174,14 @@ def main(): # if 'polygon' in sa_layer_def['sources']: # method = sa_layer_def['sources']['polygon'].get('method') # if method == 'yes_or_no': - # stmt = ''' + # stmt = f''' # UPDATE service_area_summary sas # SET {layer_id} = ( # CASE # WHEN {layer_id} != '' THEN 'yes' # ELSE 'no' # END); - # '''.format(layer_id=layer_id) + # ''' # db.execute(stmt) # # print(ais_db.c.rowcount) # db.save() @@ -204,8 +201,8 @@ def main(): layer_id = sa_layer_def['layer_id'] if 'line_single' in sa_layer_def['sources']: - print('Updating from {}...'.format(layer_id)) - stmt = ''' + print(f'Updating from {layer_id}...') + stmt = f''' UPDATE service_area_summary sas SET {layer_id} = sals.value FROM address_summary ads, service_area_line_single sals @@ -214,14 +211,14 @@ def main(): sals.seg_id = ads.seg_id AND sals.layer_id = '{layer_id}' AND sals.value <> '' - '''.format(layer_id=layer_id) + ''' db.execute(stmt) # print(ais_db.c.rowcount) db.save() elif 'line_dual' in sa_layer_def['sources']: - print('Updating from {}...'.format(layer_id)) - stmt = ''' + print(f'Updating from {layer_id}...') + stmt = f''' UPDATE service_area_summary sas SET {layer_id} = CASE WHEN (ads.seg_side = 'L') THEN sald.left_value ELSE sald.right_value END FROM address_summary ads, service_area_line_dual sald @@ -229,7 +226,7 @@ def main(): sald.seg_id = ads.seg_id AND sald.layer_id = '{layer_id}' AND CASE WHEN (ads.seg_side = 'L') THEN sald.left_value ELSE sald.right_value END <> '' - '''.format(layer_id=layer_id) + ''' db.execute(stmt) # print(ais_db.c.rowcount) db.save() @@ -247,8 +244,8 @@ def main(): if 'point' in sa_layer_def['sources']: method = sa_layer_def['sources']['point'].get('method') if method == 'nearest': - print('Updating from {}...'.format(layer_id)) - stmt = ''' + print(f'Updating from {layer_id}...') + stmt = f''' with sap_layer as ( select sap.* @@ -269,13 +266,13 @@ def main(): ) as saplv ) sapf where sas.street_address = sapf.street_address - '''.format(layer_id=layer_id) + ''' db.execute(stmt) db.save() elif method == 'seg_id': - print('Updating from {}...'.format(layer_id)) - stmt = ''' + print(f'Updating from {layer_id}...') + stmt = f''' UPDATE service_area_summary sas SET {layer_id} = sap.value FROM address_summary ads, service_area_point sap @@ -284,7 +281,7 @@ def main(): sap.seg_id = ads.seg_id AND sap.layer_id = '{layer_id}' AND sap.value <> '' - '''.format(layer_id=layer_id) + ''' db.execute(stmt) db.save() @@ -299,8 +296,8 @@ def main(): method = sa_layer_def['sources']['polygon'].get('method') if method != 'nearest_poly': continue - print('Updating from {}...'.format(layer_id)) - stmt = ''' + print(f'Updating from {layer_id}...') + stmt = f''' with sap_layer as ( select sap.* @@ -321,7 +318,7 @@ def main(): ) as saplv ) sapf where sas.street_address = sapf.street_address - '''.format(layer_id=layer_id) + ''' db.execute(stmt) db.save() ################################ @@ -331,14 +328,14 @@ def main(): layer_id = sa_layer_def['layer_id'] method = sa_layer_def.get('value_method') if method == 'yes_or_no': - stmt = ''' + stmt = f''' UPDATE service_area_summary sas SET {layer_id} = ( CASE WHEN {layer_id} != '' THEN 'Yes' ELSE 'No' END); - '''.format(layer_id=layer_id) + ''' db.execute(stmt) db.save() ################################# @@ -392,4 +389,4 @@ def main(): # Clean up: db.close() - print('Finished in {}'.format(datetime.now() - start)) + print(f'Finished in {datetime.now() - start}') diff --git a/ais/engine/scripts/make_street_intersections.py b/ais/engine/scripts/make_street_intersections.py index 197f20de..bec91d37 100644 --- a/ais/engine/scripts/make_street_intersections.py +++ b/ais/engine/scripts/make_street_intersections.py @@ -1,4 +1,3 @@ -import sys import traceback from datetime import datetime import petl as etl @@ -22,7 +21,7 @@ def main(): db_user = dsn[dsn.index("//") + 2:dsn.index(":", dsn.index("//"))] db_pw = dsn[dsn.index(":",dsn.index(db_user)) + 1:dsn.index("@")] db_name = dsn[dsn.index("/", dsn.index("@")) + 1:] - pg_db = psycopg2.connect('dbname={db_name} user={db_user} password={db_pw} host=localhost'.format(db_name=db_name, db_user=db_user, db_pw=db_pw)) + pg_db = psycopg2.connect(f'dbname={db_name} user={db_user} password={db_pw} host=localhost') # Get table params source_def = config['BASE_DATA_SOURCES']['streets'] @@ -149,11 +148,11 @@ def main(): except ValueError as e: # FEEDBACK - #print('{}: {} ({})'.format(e, source_street_full, seg_id)) + #print(f'{e}: {source_street_full} ({seg_id})') error_count += 1 except Exception as e: - print('Unhandled error on row: {}'.format(i)) + print(f'Unhandled error on row: {i}') print(traceback.format_exc()) raise e @@ -281,4 +280,4 @@ def main(): ''' db.close() - print('Finished in {} seconds'.format(datetime.now() - start)) + print(f'Finished in {datetime.now() - start} seconds') diff --git a/ais/models.py b/ais/models.py index 324e075c..f89b2115 100644 --- a/ais/models.py +++ b/ais/models.py @@ -71,7 +71,7 @@ def __str__(self): # 'high': max(self.left_to, self.right_to), 'street': self.street_full, } - return 'StreetSegment: {low} {street}'.format(**attrs) + return f"StreetSegment: {attrs['low']} {attrs['street']}" class StreetAlias(db.Model): """Alternate name for a street segment.""" @@ -268,7 +268,7 @@ def filter_by_owner(self, *owner_parts): .join(OpaProperty, OpaProperty.account_num==AddressProperty.opa_account_num) for part in owner_parts: - query = query.filter(OpaProperty.owners.like('%{}%'.format(part))) + query = query.filter(OpaProperty.owners.like(f'%{part}%')) return query @@ -298,12 +298,6 @@ class Address(db.Model): primaryjoin='foreign(Geocode.street_address) == Address.street_address', lazy='joined') - # zip_info = db.relationship( - # 'AddressZip', - # primaryjoin='foreign(AddressZip.street_address) == Address.street_address', - # lazy='joined', - # uselist=False) - pwd_parcel = db.relationship( 'PwdParcel', primaryjoin='foreign(PwdParcel.street_address) == Address.street_address', @@ -374,7 +368,7 @@ def __init__(self, *args, **kwargs): super(Address, self).__init__(**kwargs) def __str__(self): - return 'Address: {}'.format(self.street_address) + return f'Address: {self.street_address}' def __repr__(self): return self.__str__() @@ -466,7 +460,7 @@ def base_address(self): @property def base_address_no_suffix(self): - return '{} {}'.format(self.address_full_num, self.street_full) + return f'{self.address_full_num} {self.street_full}' @property def is_base(self): @@ -492,7 +486,7 @@ def is_basic_range(self): @property def child_addresses(self): """Returns a list of address objects with in-range street addresses""" - address_low_re = re.compile('^{}'.format(self.address_low)) + address_low_re = re.compile(f'^{self.address_low}') address_high_re = re.compile('-\d+') child_addresses = [] for child_num in self.child_nums: @@ -642,22 +636,6 @@ class AddressProperty(db.Model): opa_account_num = db.Column(db.Text) #, db.ForeignKey('opa_property.account_num')) match_type = db.Column(db.Text) -class AddressZip(db.Model): - ''' - Stores information about the relationship between addresses and ZIP ranges. - ''' - id = db.Column(db.Integer, primary_key=True) - street_address = db.Column(db.Text) - usps_id = db.Column(db.Text) - match_type = db.Column(db.Text) - - zip_range = db.relationship( - 'ZipRange', - primaryjoin='foreign(ZipRange.usps_id) == AddressZip.usps_id', - lazy='joined', - uselist=False) - - ############# # GEOCODING # @@ -768,30 +746,6 @@ class ServiceAreaDiff(db.Model): distance = db.Column(db.Float) geom = db.Column(Geometry(geometry_type='POINT', srid=ENGINE_SRID)) -############# -# ZIP CODES # -############# - -class ZipRange(db.Model): - ''' - This is essentially a direct copy of the USPS ZIP+4 table. - ''' - id = db.Column(db.Integer, primary_key=True) - usps_id = db.Column(db.Text) - address_low = db.Column(db.Integer) - address_high = db.Column(db.Integer) - address_oeb = db.Column(db.Text) - street_predir = db.Column(db.Text) - street_name = db.Column(db.Text) - street_suffix = db.Column(db.Text) - street_postdir = db.Column(db.Text) - unit_type = db.Column(db.Text) - unit_low = db.Column(db.Text) - unit_high = db.Column(db.Text) - unit_oeb = db.Column(db.Text) - zip_code = db.Column(db.Text) - zip_4 = db.Column(db.Text) - ############ # PRODUCTS # @@ -813,7 +767,7 @@ def order_by_address(self): ) def order_by_owner_address(self, query): - return self.order_by(desc(func.similarity(AddressSummary.opa_owners, '{}'.format(query))), + return self.order_by(desc(func.similarity(AddressSummary.opa_owners, f'{query}')), AddressSummary.street_name, AddressSummary.street_suffix, AddressSummary.street_predir, @@ -880,20 +834,20 @@ def filter_by_owner(self, *owner_parts): owner_parts = sorted(owner_parts, key = lambda s: len(s), reverse=True) # Match to opa_owners by part for part in owner_parts: - query = query.filter(AddressSummary.opa_owners.like('%{}%'.format(part))) - # query = query.order_by(desc(func.similarity(AddressSummary.opa_owners, '{}'.format(owner_full)))) + query = query.filter(AddressSummary.opa_owners.like(f'%{part}%')) + # query = query.order_by(desc(func.similarity(AddressSummary.opa_owners, f'{owner_full}'))) # tot_len = sum(len(s) for s in owner_parts) # if tot_len > OWNER_PARTS_THRESHOLD: # for part in owner_parts: - # query = query.filter(AddressSummary.opa_owners.like('%{}%'.format(part))) + # query = query.filter(AddressSummary.opa_owners.like(f'%{part}%')) # else: # for part in owner_parts: - # query = query.filter(AddressSummary.opa_owners.like('%{}%'.format(part))).limit( + # query = query.filter(AddressSummary.opa_owners.like(f'%{part}%')).limit( # OWNER_RESPONSE_LIMIT).from_self() # if not query.all(): # query = self # for part in reversed(owner_parts): - # query = query.filter(AddressSummary.opa_owners.like('%{}%'.format(part))).limit( + # query = query.filter(AddressSummary.opa_owners.like(f'%{part}%')).limit( # OWNER_RESPONSE_LIMIT).from_self() return query @@ -1259,12 +1213,6 @@ class AddressSummary(db.Model): lazy='joined', uselist=False) - # zip_info = db.relationship( - # 'AddressZip', - # primaryjoin='foreign(AddressZip.street_address) == AddressSummary.street_address', - # lazy='select', - # uselist=False) - pwd_parcel = db.relationship( 'PwdParcel', primaryjoin='foreign(PwdParcel.street_address) == AddressSummary.street_address', @@ -1374,51 +1322,6 @@ class MultipleSegLine(db.Model): parcel_source = db.Column(db.Text) geom = db.Column(Geometry(geometry_type='LINESTRING', srid=ENGINE_SRID)) -class DorParcelAddressAnalysis(db.Model): - ''' - Table for post-engine-build DOR parcel address analysis report. - ''' - objectid = db.Column(db.Integer, primary_key=True) - mapreg = db.Column(db.Text) - status = db.Column(db.Integer) - stcod = db.Column(db.Integer) - house = db.Column(db.Integer) - suf = db.Column(db.Text) - unit = db.Column(db.Text) - stex = db.Column(db.Integer) - stdir = db.Column(db.Text) - stnam = db.Column(db.Text) - stdes = db.Column(db.Text) - stdessuf = db.Column(db.Text) - concatenated_address = db.Column(db.Text) - std_street_address = db.Column(db.Text) - std_address_low = db.Column(db.Integer) - std_address_low_suffix = db.Column(db.Text) - std_high_num = db.Column(db.Integer) - std_street_predir = db.Column(db.Text) - std_street_name = db.Column(db.Text) - std_street_suffix = db.Column(db.Text) - std_address_postdir = db.Column(db.Text) - std_unit_type = db.Column(db.Text) - std_unit_num = db.Column(db.Text) - std_street_code = db.Column(db.Integer) - std_seg_id = db.Column(db.Integer) - cl_addr_match = db.Column(db.Text) - change_stcod = db.Column(db.Integer) - change_house = db.Column(db.Integer) - change_suf = db.Column(db.Integer) - change_unit = db.Column(db.Integer) - change_stex = db.Column(db.Integer) - change_stdir = db.Column(db.Integer) - change_stnam = db.Column(db.Integer) - change_stdes = db.Column(db.Integer) - change_stdessuf = db.Column(db.Integer) - no_address = db.Column(db.Integer) - opa_account_nums = db.Column(db.Text) - num_parcels_w_mapreg = db.Column(db.Integer) - num_parcels_w_address = db.Column(db.Integer) - shape = db.Column(Geometry(geometry_type='MULTIPOLYGON', srid=ENGINE_SRID)) - class DorCondominiumError(db.Model): id = db.Column(db.Integer, primary_key=True) parcel_id = db.Column(db.Text, index=True) diff --git a/ais/tests/api/test_models.py b/ais/tests/api/test_models.py index 0b766a09..525705ed 100644 --- a/ais/tests/api/test_models.py +++ b/ais/tests/api/test_models.py @@ -18,8 +18,8 @@ def test_no_unit_sorted_first(): .order_by_address() num_addresses = addresses.count() - assert num_addresses >= 2, 'Len addresses is {}'.format(num_addresses) + assert num_addresses >= 2, f'Len addresses is {num_addresses}' first_address = addresses[0] - assert not first_address.unit_type, 'First has a unit_type: {}'.format(first_address.unit_type) - assert not first_address.unit_num, 'First has a unit_num: {}'.format(first_address.unit_num) + assert not first_address.unit_type, f'First has a unit_type: {first_address.unit_type}' + assert not first_address.unit_num, f'First has a unit_num: {first_address.unit_num}' diff --git a/ais/tests/api/test_views.py b/ais/tests/api/test_views.py index ab82b2e9..71893ec3 100644 --- a/ais/tests/api/test_views.py +++ b/ais/tests/api/test_views.py @@ -11,22 +11,19 @@ def client(): def assert_status(response, *expected_status_codes): assert response.status_code in expected_status_codes, ( - 'Expected status {}; received {}. Full response was {}.').format( - expected_status_codes, response.status_code, response.get_data()) + f"Expected status {expected_status_codes}; received {response.status_code}." + f"Full response was {response.get_data()}.") def assert_num_results(data, expected_num_results, op=eq): actual_num_results = data['total_size'] assert op(actual_num_results, expected_num_results), ( - "Expected {} {} results; received {}. Full response " - "was {}").format( - op.__name__, expected_num_results, actual_num_results, - data) + f"Expected {op.__name__} {expected_num_results} results; " + f"received {actual_num_results}. Full response was {data}") def assert_attr(feature, property_name, expected_value): actual_address = feature['properties'][property_name] assert actual_address == expected_value, ( - 'Expected {} of {}; received {}.').format( - property_name, expected_value, actual_address) + f'Expected {property_name} of {expected_value}; received {actual_address}.') def assert_opa_address(feature, expected_address): assert_attr(feature, 'opa_address', expected_address) @@ -107,7 +104,7 @@ def test_geometry_is_lat_lng_by_default(client): coords = tuple(feature['geometry']['coordinates']) assert (-76, 39) < coords < (-74, 41),\ ('Coordinates do not appear to be in Philadelphia, or do not represent ' - 'a longitude, latitude: {}').format(coords) + f'a longitude, latitude: {coords}') def test_ranged_address_has_units_with_base_first(client): response = client.get('/addresses/1801-23 N 10th St?include_units') @@ -147,8 +144,8 @@ def test_child_address_has_all_units_in_ranged_address(client): ranged_data = json.loads(response.get_data().decode()) assert child_data['total_size'] == ranged_data['total_size'], \ - ('Child address has {} results, whereas the ranged address has {} ' - 'results.').format(child_data['total_size'], ranged_data['total_size']) + (f'Child address has {child_data['total_size']} results,' + f'whereas the ranged address has {ranged_data['total_size']} results.') @pytest.mark.skip(reason="todo - return OPA source address instead of parsed OPA street_address") def test_unit_address_in_db(client): @@ -180,7 +177,7 @@ def test_unit_address_without_unit_num_in_db(client): result = app_db.engine.execute(UNIT_SQL) street_address = result.first()[0] - response = client.get('/addresses/{}?opa_only'.format(street_address)) + response = client.get(f'/addresses/{street_address}?opa_only') assert_status(response, 200) data = json.loads(response.get_data().decode()) @@ -291,7 +288,7 @@ def test_address_query_can_end_in_comma(client): def test_opa_query_returns_child_address(client): ignore_addresses = ['1501-53 N 24TH ST', '514-32 N CREIGHTON ST', '901-99 MARKET ST', '630-50 W FISHER AVE', '630R-50 W FISHER AVE', '1501-39 MARKET ST', '8842-54 FRANKFORD AVE', '1131-45 VINE ST', '750-86 N 46TH ST', '1000A-52 FRANKFORD AVE', '4215-19 LUDLOW ST', '3118-98 CHESTNUT ST', '1501S-39 MARKET ST', '3423-35 WEYMOUTH ST', '4421R-51 N PHILIP ST', '5911R-27 BELFIELD AVE', '4130-50 CITY AVE', '3302R-64 N 3RD ST', '430-32 FAIRMOUNT AVE', '5501-35 E WISTER ST', '7326-30 OXFORD AVE', '1214-32 N 26TH ST', '4131-63 WHITAKER AVE', '2104-22 W VENANGO ST', '5112-22 REGENT ST'] - CHILD_SQL = ''' + CHILD_SQL = f''' SELECT child.street_address, parent.street_address FROM address_summary AS child JOIN address_link ON address_1 = child.street_address @@ -301,15 +298,15 @@ def test_opa_query_returns_child_address(client): AND parent.opa_account_num = child.opa_account_num AND child.address_low != parent.address_low AND child.address_low != parent.address_high - AND parent.street_address not in {} + AND parent.street_address not in {tuple(ignore_addresses)} LIMIT 1 - '''.format(tuple(ignore_addresses)) + ''' # Must use the app import like this to get context so we can run SQL commands with app.app_context(): result = app_db.engine.execute(CHILD_SQL) child_address, parent_address = result.first() - response = client.get('/addresses/{}?opa_only'.format(child_address)) + response = client.get(f'/addresses/{child_address}?opa_only') assert_status(response, 200) data = json.loads(response.get_data().decode()) @@ -377,7 +374,7 @@ def test_not_found(client): try: data = json.loads(response_content) except ValueError: - raise Exception('Response is not JSON: {}'.format(response_content)) + raise Exception(f'Response is not JSON: {response_content}') def test_intersection_query(client): # TODO: Make functional without street suffix (st)? diff --git a/ais/tests/engine/test_engine.py b/ais/tests/engine/test_engine.py index 8872f4d9..61eb64ba 100644 --- a/ais/tests/engine/test_engine.py +++ b/ais/tests/engine/test_engine.py @@ -75,7 +75,11 @@ def test_compare_num_tables(startup): # assert len(startup['engine_to_test_cur'].tables) == len(startup['engine_to_compare_cur'].tables) engine_to_test_cur = startup['engine_to_test_cur'] engine_to_compare_cur = startup['engine_to_compare_cur'] - table_count_stmt = "select count(*) from information_schema.tables where table_schema = 'public' AND table_type = 'BASE TABLE' and table_name not in {}".format(str(startup['ignore_tables'])) + table_count_stmt = f""" + select count(*) from information_schema.tables + where table_schema = 'public' and + table_type = 'BASE TABLE' and + table_name not in {str(startup['ignore_tables'])}""" engine_to_test_cur.execute(table_count_stmt) new_table_count = engine_to_test_cur.fetchall() @@ -102,7 +106,7 @@ def test_num_rows_bt_db_tables(startup): # ndb_table = startup['engine_to_test_cur'][ntable] # n_rows = ndb_table.count - row_count_stmt = "select count(*) as count from {}".format(table_name) + row_count_stmt = f"select count(*) as count from {table_name}" engine_to_test_cur.execute(row_count_stmt) n_rows = engine_to_test_cur.fetchall() @@ -138,7 +142,8 @@ def get_geo_types(db): def test_matching_indexes(startup): """Test #4: Check if all indexes are present (compare new an old builds)""" - stmt = ''' + ignore_tables=startup['unused_tables'] + stmt = f''' SELECT n.nspname as "Schema", c.relname as "Name", CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'i' @@ -155,7 +160,7 @@ def test_matching_indexes(startup): AND pg_catalog.pg_table_is_visible(c.oid) AND c2.relname NOT IN {ignore_tables} ORDER BY 1,2; - '''.format(ignore_tables=startup['unused_tables']) + ''' startup['engine_to_test_cur'].execute(stmt) engine_to_test_cur_result = startup['engine_to_test_cur'].fetchall() startup['engine_to_compare_cur'].execute(stmt) @@ -174,7 +179,7 @@ def test_matching_indexes(startup): unmatched_indexes.append({'name': old_row['Name'], 'table': old_row['Table']}) assert len(unmatched_indexes) == 0, (unmatched_indexes) # assert len(engine_to_test_cur_result) == len(engine_to_compare_cur_result), ( - # "new db has {} more indexes.".format(len(engine_to_test_cur_result) - len(engine_to_compare_cur_result))) + # f"new db has {len(engine_to_test_cur_result) - len(engine_to_compare_cur_result)} more indexes.") def test_num_opa_records(startup): diff --git a/ais/util.py b/ais/util.py index 4ed8858c..90f36a45 100644 --- a/ais/util.py +++ b/ais/util.py @@ -22,7 +22,7 @@ def parity_for_range(low, high): # def dbl_quote(text): # """Place double quotes around a string.""" -# return '"{}"'.format(text) +# return f'"{text}"' def parse_url(url): p = urlparse(url) @@ -39,8 +39,8 @@ def parse_url(url): # def __init__(self, from_srid, to_srid): # self.project = partial( # pyproj.transform, -# pyproj.Proj('+init=EPSG:{}'.format(from_srid)), -# pyproj.Proj('+init=EPSG:{}'.format(to_srid)) +# pyproj.Proj('+init=EPSG:{from_srid}'), +# pyproj.Proj('+init=EPSG:{to_srid}') # ) # def transform(self, from_wkt): @@ -88,9 +88,9 @@ def project_shape(shape, from_srid, to_srid): pyproj.transform, # source coordinate system; preserve_units so that pyproj does not # assume meters - pyproj.Proj(init='epsg:{}'.format(from_srid), preserve_units=True), + pyproj.Proj(init=f'epsg:{from_srid}', preserve_units=True), # destination coordinate system - pyproj.Proj(init='epsg:{}'.format(to_srid), preserve_units=True)) + pyproj.Proj(init=f'epsg:{to_srid}', preserve_units=True)) return transform(project, shape) @@ -153,14 +153,14 @@ def offset(line, point, distance, seg_side): # Get angle of seg seg_angle = atan2(norm_y, norm_x) - # print('seg angle: {}'.format(degrees(seg_angle))) + # print(f'seg angle: {degrees(seg_angle)}') # Get angle of offset line if seg_side == 'L': offset_angle = seg_angle + (pi / 2) else: offset_angle = seg_angle - (pi / 2) - # print('offset angle: {}'.format(degrees(offset_angle))) + # print(f'offset angle: {degrees(offset_angle)}') # Get offset point delta_x = cos(offset_angle) * distance diff --git a/config.py b/config.py index ee44ef68..d5a4a178 100644 --- a/config.py +++ b/config.py @@ -157,39 +157,6 @@ def multi_strip(str_): return str_.strip(' ').lstrip('0') -# Preprocessor for L&I addresses -# Not using this since Nick is cleaning everything up in the GIS_LNI DB. -# def make_li_address(comps): -# out_comps = [] -# # Make primary address num. -# addr_num = multi_strip(comps['address_low']) -# # Handle address suffixes. -# suf = comps['address_low_suffix'].strip() -# if len(suf) > 0: -# if suf.isnumeric(): -# # Decode fractionals. -# if suf == '2': -# out_comps.append('1/2') -# elif suf.isalpha(): -# addr_num += suf -# else: raise ValueError('Unhandled L&I postdir: {}'.format(suf)) -# # Handle address extension. -# addr_high = comps['address_high'] -# addr_high = addr_high.lstrip('0') if isinstance(addr_high, str) else None -# addr_num += '-{}'.format(addr_high) if addr_high else '' -# out_comps.append(addr_num) -# # Add remaining fields -# for field_suffix in ['predir', 'name', 'suffix']: -# out_comps.append(comps['street_' + field_suffix]) -# # Unit -# unit_num = comps['unit_num'] -# if unit_num and len(multi_strip(unit_num)) > 0: -# out_comps += ['#', comps['unit_num']] -# # Filter blanks -# out_comps = [x for x in out_comps if x and len(multi_strip(x)) > 0] -# addr = ' '.join(out_comps) -# return addr - def make_pwd_account_address(comps): a = comps['street_address'] a = re.sub('-R(EAR)?(?= )', 'R', a) @@ -223,25 +190,6 @@ def make_voter_name(comps): name = (first_name, middle_name, last_name) return " ".join(filter(None, name)) -def make_rtt_address(comps): - low_num = str(comps['address_low']) if comps['address_low'] else None - address_low_suffix = comps['address_low_suffix'] - addr_high = comps['address_high'] - addr_high = None if addr_high == '-1' else addr_high - addr_high = addr_high.lstrip('0') if isinstance(addr_high, str) else None - addr_num = '{low_num}-{addr_high}'.format(low_num=low_num, addr_high=addr_high) if low_num and addr_high else low_num - addr_num = addr_num + address_low_suffix if addr_num and address_low_suffix else addr_num - unit_num = '#' + str(comps['unit_num']) if comps['unit_num'] else None - street_predir = comps['street_predir'] - street_postdir = comps['street_postdir'] - street_name = comps['street_name'] - street_type = comps['street_type'] - out_comps = (addr_num, street_predir, street_name, street_type, street_postdir, unit_num) - # Filter blanks - out_comps = [x for x in out_comps if x and len(multi_strip(x)) > 0] - addr = ' '.join(out_comps) - return addr - def make_dor_parcel_id(comps): reg_map_id = comps['reg_map_id'] return reg_map_id.replace('-', '') @@ -478,44 +426,9 @@ def make_eclipse_address(comps): }, ], }, - # { - # 'name': 'rtt', - # 'table': 'gis_dor_rttmapping.cris_properties', - # 'db': 'gisp_t', - # 'address_fields': { - # 'address_low': 'house_number', - # 'address_low_suffix': 'house_num_suffix', - # 'address_high': 'house_num_range', - # 'street_predir': 'street_dir', - # 'street_postdir': 'street_dir_suffix', - # 'street_name': 'street_name', - # 'street_type': 'street_type', - # 'unit_num': 'condo_unit', - # }, - # 'preprocessor': make_rtt_address, - # 'tag_fields': [ - # { - # 'key': 'dor_parcel_id', - # 'source_fields': ['reg_map_id'], - # 'preprocessor': make_dor_parcel_id, - # }, - # ], - # # Query only records with non-null reg_map_id - # 'where': 'reg_map_id is not null', - # }, ] } -# ERROR_TABLES = { -# 'dor_parcels': { -# 'error_table': 'dor_parcel_error', -# 'polygon_table': 'dor_parcel_error_polygon', -# }, -# 'addresses': { -# 'error_table': 'address_error', -# }, -# } - GEOCODE = { 'centerline_offset': 5, 'centerline_end_buffer': 17, @@ -577,7 +490,7 @@ def make_eclipse_address(comps): 'db': 'citygeo', 'table': 'viewer_citygeo.phillyrising_boundaries', 'value_field': 'site_name', - 'object_id_field': 'objectid', # the field that was objectid_12 in oracle is now called just objectid in postgres. + 'object_id_field': 'objectid', }, }, }, @@ -836,18 +749,6 @@ def make_eclipse_address(comps): }, }, }, - # { - # 'layer_id': 'neighborhood', - # 'name': 'Neighborhood', - # 'description': '', - # 'sources': { - # 'polygon': { - # 'db': 'gis_sa', - # 'table': 'gis_planning.Neighborhoods', - # 'value_field': 'name', - # # }, - # # }, - # # }, { 'layer_id': 'zoning', 'name': 'Zoning', @@ -911,59 +812,6 @@ def make_eclipse_address(comps): }, 'value_method': 'yes_or_no', }, - - # STEEP SLOPE PROTECTION AREAS - # { - # 'layer_id': 'zoning_steepslopeprotectionarea', - # 'name': 'Zoning_SteepSlopeProtectionArea', - # 'description': '', - # 'sources': { - # 'polygon': { - # 'db': ' gis_sa', - # 'table': 'gis_planning.Zoning_SteepSlopeProtectArea_r', - # 'value_field': 'overlay_na', - # }, - # }, - # }, - # FEMA FLOOD ZONES - # { - # 'layer_id': 'fema_flood_plain_100', - # 'name': 'Fema_Flood_Plain_100', - # 'description': '', - # 'sources': { - # 'polygon': { - # 'db': ' gis_sa', - # 'table': 'gis_planning.FEMA_100_FLOOD_PLAIN', - # 'value_field': 'fld_zone', - # }, - # }, - # }, - # { - # 'layer_id': 'fema_flood_plain_500', - # 'name': 'Fema_Flood_Plain_500', - # 'description': '', - # 'sources': { - # 'polygon': { - # 'db': ' gis_sa', - # 'table': 'gis_planning.FEMA_500_FLOOD_PLAIN', - # 'value_field': 'fld_zone', - # }, - # }, - # }, - # # Land use was slowing the service area summary script down - # # considerably. - # # { - # # 'layer_id': 'land_use', - # # 'name': 'Land Use', - # # 'description': '', - # # 'sources': { - # # 'polygon': { - # # 'db': 'gis_sa', - # # 'table': 'gis_planning.Land_Use', - # # 'value_field': 'c_dig2desc', - # # }, - # # }, - # # }, # POLICE { 'layer_id': 'police_division', @@ -977,18 +825,6 @@ def make_eclipse_address(comps): }, }, }, - # { - # 'layer_id': 'police_sector', - # 'name': 'Police Sector', - # 'description': '', - # 'sources': { - # 'polygon': { - # 'db': 'gis_sa', - # 'table': 'gis_police.Boundaries_Sector', - # 'value_field': 'distsec_id', - # }, - # }, - # }, { 'layer_id': 'police_district', 'name': 'Police District', @@ -1016,36 +852,6 @@ def make_eclipse_address(comps): }, }, }, - # GIS_RDA - # SHOULD THIS GO IN SOURCES? - # investigate whether 2 addreses are same property - # { - # 'layer_id': 'rda_landbank_lama_assets', - # 'name': 'RDA_Landbank_Lama_Assets', - # 'description': '', - # 'sources': { - # 'polygon': { - # 'db': 'gis_sa', - # 'table': 'gis_rda.lama_assets_0725', - # 'value_field': 'asset_id', - # }, - # }, - # }, - - # # GIS_RECYCLE - # { - # 'layer_id': 'recreation_district', - # 'name': 'Recreation District', - # 'description': '', - # 'sources': { - # 'polygon': { - # 'db': 'gis_sa', - # 'table': 'gis_orphan.Recreation_Districts', - # 'value_field': 'dist_num', - # }, - # }, - # }, - # STREETS { 'layer_id': 'rubbish_recycle_day', @@ -1389,7 +1195,6 @@ def make_eclipse_address(comps): 'value_field': 'water_plate', }, }, - # 'source_account': 'gis', }, { 'layer_id': 'pwd_center_city_district', @@ -1416,19 +1221,6 @@ def make_eclipse_address(comps): }, }, }, - # GIS_GSG - # { - # 'layer_id': 'litter_schema_test', - # 'name': 'Litter Schema Test', - # 'description': '', - # 'sources': { - # 'polygon': { - # 'db': 'gis_sa', - # 'table': 'gis_gsg.LITTER_SCHEMA_TEST', - # 'value_field': 'score', - # }, - # }, - # }, # OHCD { 'layer_id': 'neighborhood_advisory_committee', @@ -1569,10 +1361,6 @@ def make_eclipse_address(comps): 'ng911', 'true_range', ], - # 'geocode_types_on_curb': [ - # 'pwd_curb', - # 'dor_curb', - # ], 'geocode_types_in_street': [ 'pwd_street', 'dor_street', @@ -1599,8 +1387,6 @@ def make_eclipse_address(comps): 'max_values': 5, # TODO: strip out relationship fields from tag fields 'tag_fields': [ - # traverse_links was deprecated in favor of dedicated relationship - # tables: address-parcel, address-street, and address-property { 'name': 'zip_code', 'tag_key': 'usps_zipcode', diff --git a/docs/DEPLOY.md b/docs/DEPLOY.md index 0ca50cc3..9bf7de94 100644 --- a/docs/DEPLOY.md +++ b/docs/DEPLOY.md @@ -1,3 +1,5 @@ +🚨 **NOTE (7/1/2025): Elastic Beanstalk and Travis are deprecated as deploy tools for AIS. This document is not being actively updated.** + # Deploying to Elastic Beanstalk ## Production environments