Merge pull request mozboz#8 from webisteme/master

Django integration

Author: mozboz

Makefile

@@ -0,0 +1,23 @@
+
+virtualenv:
+	[ -d .env ] || virtualenv .env 
+	.env/bin/pip install -r requirements.txt
+	echo "Now type \"source .env/bin/activate\" to active python env"
+
+
+installpostgres:
+	sudo apt-get install postgresql
+	sudo -u postgres psql -c "CREATE USER tfm LOGIN ENCRYPTED PASSWORD 'password';"
+	sudo -u postgres createdb tfm -O tfm
+
+resetdb:
+	sudo -u postgres dropdb tfm
+	sudo -u postgres createdb tfm -O tfm
+	make setupdb
+
+setupdb:
+	.env/bin/python manage.py migrate
+	.env/bin/python manage.py createsuperuser
+
+runserver:
+	.env/bin/python runserver_plus

README.md

@@ -1,12 +1,35 @@
-# transformapetl
-#
+# Transformap ETL
 
-1. Setup
+## Setup dev:
 
-pip install -r requirements.txt  
-cp transformap.example.conf transformap.conf [+ add database credentials]
+    make installpostgres
+    make virtualenv
+    make setupdb
+    source .env/bin/activate
+    make runserver
+
+Refer to Makefile for further details; it should serve as a guide to how to
+manage the local Python development environment.
+
+## Postgresql JSONB field support in Django
+
+The JSON field support appears to be quite good, django's queryset API supports
+several operations for querying data inside JSONB fields.
+
+See: https://docs.djangoproject.com/en/1.10/ref/contrib/postgres/fields/#containment-and-key-operations
+
+## Usage
+
+    python manage.py transformap -i jobs/jobfile.json
+
+Admin interface: /admin
+API URL: /places-api/
+New jobs should be created as .YAML files in jobs/
+
+## Misc
+
+ETL business logic: places/management/commands
+REST API configuration: places/api.py
 
-2. Usage
 
-python transformap.py -i jobs/charlotte.yaml
 

jobs/osm.yaml

@@ -27,5 +27,5 @@ transform:
       source_field_name: wheelchair
       target_field_name: wheelchair_accessible
 load:
-    definition: "Convergir map"
-    owner: "Convergir"
+    definition: "OSM map"
+    owner: "OSM"

logging.conf

@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+import os
+import sys
+
+if __name__ == "__main__":
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "tfm.settings")
+    try:
+        from django.core.management import execute_from_command_line
+    except ImportError:
+        # The above import may fail for some other reason. Ensure that the
+        # issue is really that Django is missing to avoid masking other
+        # exceptions on Python 2.
+        try:
+            import django
+        except ImportError:
+            raise ImportError(
+                "Couldn't import Django. Are you sure it's installed and "
+                "available on your PYTHONPATH environment variable? Did you "
+                "forget to activate a virtual environment?"
+            )
+        raise
+    execute_from_command_line(sys.argv)

manage.py

@@ -0,0 +1,51 @@
+from django.contrib import admin
+
+from places.models import MapOwner, MapInstance, MapObject, MapData, \
+    MapDefinition, SchemaField, SchemaVersion
+    
+    
+     
+class MapObjectAdmin(admin.ModelAdmin):
+    list_display = ('id', 'map_instance', 'longitude', 'latitude', 'date_created')
+    list_display_links = ('id', )
+    
+
+class MapInstanceAdmin(admin.ModelAdmin):
+    list_display = ('id', 'schema', 'map_definition', 'date_created')
+    list_display_links = ('id',)
+    
+    
+class MapOwnerAdmin(admin.ModelAdmin):
+    list_display = ('id', 'name', 'date_created',)
+    list_display_links = ('id',)
+    
+
+class MapDefinitionAdmin(admin.ModelAdmin):
+    list_display = ('id', 'name', 'owner', 'date_created')
+    list_display_links = ('id',)
+
+
+class SchemaFieldAdmin(admin.ModelAdmin):
+    list_display = ('id', 'schema', 'field_name', 'field_type')
+    list_display_links = ('id',)
+
+
+class SchemaVersionAdmin(admin.ModelAdmin):
+    list_display = ('id', 'map_definition', 'map_owner', 'date_created')
+    list_display_links = ('id',)
+
+
+class MapDataAdmin(admin.ModelAdmin):
+    list_display = ('id', 'map_instance', 'map_object', 'schema_field')
+    list_display_links = ('id',)
+
+
+
+admin.site.register(MapObject, MapObjectAdmin)
+admin.site.register(MapInstance, MapInstanceAdmin)
+admin.site.register(MapOwner, MapOwnerAdmin)
+admin.site.register(MapDefinition, MapDefinitionAdmin)
+admin.site.register(SchemaField, SchemaFieldAdmin)
+admin.site.register(SchemaVersion, SchemaVersionAdmin)
+admin.site.register(MapData, MapDataAdmin)
+

transformap/__init__.py places/__init__.pytransformap/__init__.py renamed to places/__init__.py

@@ -0,0 +1,96 @@
+from places.models import MapOwner, MapInstance, MapObject, MapData, \
+    MapDefinition, SchemaField, SchemaVersion
+
+from rest_framework import filters, routers, serializers, viewsets
+
+
+# Map Owner
+
+class MapOwnerSerializer(serializers.HyperlinkedModelSerializer):
+    class Meta:
+        model = MapOwner
+        fields = ('name', 'date_created', 'date_modified')
+
+class MapOwnerViewSet(viewsets.ModelViewSet):
+    queryset = MapOwner.objects.all()
+    serializer_class = MapOwnerSerializer
+    
+# Map Instance
+
+class MapInstanceSerializer(serializers.HyperlinkedModelSerializer):
+    class Meta:
+        model = MapInstance
+        fields = ('schema', 'map_definition', 'date_created', 'date_modified')
+
+class MapInstanceViewSet(viewsets.ModelViewSet):
+    queryset = MapInstance.objects.all()
+    serializer_class = MapInstanceSerializer
+    
+# Map Object
+
+class MapObjectSerializer(serializers.HyperlinkedModelSerializer):
+    class Meta:
+        model = MapObject
+        fields = ('guid', 'map_instance', 'longitude', 'latitude', 'date_created', 'date_modified')
+
+class MapObjectViewSet(viewsets.ModelViewSet):
+    queryset = MapObject.objects.all()
+    serializer_class = MapObjectSerializer
+    
+# Map Data
+
+class MapDataSerializer(serializers.HyperlinkedModelSerializer):
+    class Meta:
+        model = MapData
+        fields = ('map_instance', 'map_object', 'schema_field', 'field_value', 'date_created', 'date_modified')
+
+class MapDataViewSet(viewsets.ModelViewSet):
+    queryset = MapData.objects.all()
+    serializer_class = MapDataSerializer
+
+# Map Definition
+
+class MapDefinitionSerializer(serializers.HyperlinkedModelSerializer):
+    class Meta:
+        model = MapDefinition
+        fields = ('name', 'owner', 'date_created', 'date_modified')
+
+class MapDefinitionViewSet(viewsets.ModelViewSet):
+    queryset = MapDefinition.objects.all()
+    serializer_class = MapDefinitionSerializer
+
+# Schema Field
+
+class SchemaFieldSerializer(serializers.HyperlinkedModelSerializer):
+    class Meta:
+        model = SchemaField
+        fields = ('is_base_field', 'schema', 'field_name', 'field_type', 'field_description', 'date_created', 'date_modified')
+
+class SchemaFieldViewSet(viewsets.ModelViewSet):
+    queryset = SchemaField.objects.all()
+    serializer_class = SchemaFieldSerializer
+    
+# Schema Version
+
+class SchemaVersionSerializer(serializers.HyperlinkedModelSerializer):
+    class Meta:
+        model = SchemaVersion
+        fields = ('map_definition', 'map_owner', 'date_created', 'date_modified')
+
+class SchemaVersionViewSet(viewsets.ModelViewSet):
+    queryset = SchemaVersion.objects.all()
+    serializer_class = SchemaVersionSerializer
+
+
+
+
+router = routers.DefaultRouter()
+
+router.register('map_owners', MapOwnerViewSet)
+router.register('map_instance', MapInstanceViewSet)
+router.register('map_object', MapObjectViewSet)
+router.register('map_data', MapDataViewSet)
+router.register('map_definition', MapDefinitionViewSet)
+router.register('schema_field', SchemaFieldViewSet)
+router.register('schema_version', SchemaVersionViewSet)
+

places/admin.py

@@ -0,0 +1,7 @@
+from __future__ import unicode_literals
+
+from django.apps import AppConfig
+
+
+class PlacesConfig(AppConfig):
+    name = 'places'