Merge pull request #436 from mapswipe/dev

Dev

Author: Hagellach37

.travis.yml

@@ -39,19 +39,19 @@ script:
   - docker-compose run mapswipe_workers python -m unittest discover --verbose --start-directory tests/integration/
 
 after_success:
-  # Deploy to development server using Ansible
   # SSH setup to deploy to server after build.
   - apk add openssh-client
-  - eval "$(ssh-agent -s)"  #start the ssh agent
+  - eval "$(ssh-agent -s)"  # start the ssh agent
   - mkdir -p $HOME/.ssh
-  # - chmod 700 $HOME/.ssh
+  # Use config to define SSH connection variables (User, IdentityFile)
   - cp travis/ssh_config $HOME/.ssh/config
   - openssl aes-256-cbc -K $encrypted_c17fa81e6490_key -iv $encrypted_c17fa81e6490_iv -in travis/ssh-private-key.enc -out $HOME/.ssh/private-key -d
   - chmod 600 $HOME/.ssh/private-key
   - ssh-add $HOME/.ssh/private-key
   - chmod +x deploy.sh
 
 deploy:
+  # Deploy to server using Ansible
   provider: script
   script: bash deploy.sh
   on:

deploy.sh

@@ -1,4 +1,17 @@
 #!/bin/bash
-# Deploy to dev server.
+# Deploy to server.
 # This script is called by Travis.
+
+function bell() {
+  # Print regular to stdout, so that Travis will not abort due to time.
+  # travis_wait does not work with script deployment.
+  while true; do
+    echo -e "\a"
+    sleep 60
+  done
+}
+bell &
+
 ansible-playbook -i ansible/inventory.yml ansible/playbook.yml
+
+exit $?  # Return exit code of last command

docs/source/contributing.md

@@ -34,3 +34,10 @@ From now on `black`, `flake8` and `isort` should run automatically whenever `git
 
 When running those tools manually please make sure the right version is used. The version can be looked up in `.pre-commit-config.yaml`.
 To update to newer version please make sure to change version numbers in `.pre-commit-config.yaml`, `.travis.yml` and `requirements.txt`.
+
+
+### Tips
+
+Ignore a hook: `SKIP=flake8 git commit -m "foo"`
+Mark in code that flake8 should ignore the line: `print()  # noqa`
+

docs/source/deployment_overview.md

@@ -16,3 +16,12 @@ MapSwipe utilizes a bunch of Google Cloud services:
 [Configuration](configuration.md) describes all needed configuration and credentials.
 
 [Installation](installation.md) describes step-by-step how to setup the backend for the first time.
+
+
+## Continuous deployment using Ansible and Travis
+
+Travis is setup to automatically deploy a new version of MapSwipe Back-End to the server once it run successfully.
+This is done by using Travis script deployment (https://docs.travis-ci.com/user/deployment/script/). Travis simply calls the `deploy.sh` script found at the root directory of MapSwipe Workers.
+To be able to connect to the MapSwipe server the Travis instance uses an encrypted SSH private key (Which can be found in the directory `travis/`).
+
+In the `deploy.sh` script an Ansible Playbook is run (https://docs.ansible.com/ansible/latest/index.html). Ansible is an automation tool which utilizes a SSH connection (`ansible/ansible.cfg`) to run commands defined in the Playbook (`ansible/playbook.yml`) on hosts defined in the Inventory (`ansible/inventory.yml`).

manager_dashboard/manager_dashboard/js/uploadProjects.js

@@ -46,7 +46,7 @@ function getFormInput() {
             break;
         case "footprint":
             form_data.projectType = 2;
-            form_data.input_geometries = document.getElementById("inputTaskGeometries").value;
+            form_data.inputGeometries = document.getElementById("inputTaskGeometries").value;
             form_data.tileServer = {
               name: document.getElementById("tileServerAName").value,
               url: document.getElementById("tileServerAUrl").value,

mapswipe_workers/mapswipe_workers/generate_stats/project_stats.py

@@ -6,7 +6,7 @@
 from psycopg2 import sql
 
 from mapswipe_workers import auth
-from mapswipe_workers.definitions import DATA_PATH, logger, sentry
+from mapswipe_workers.definitions import DATA_PATH, ProjectType, logger, sentry
 from mapswipe_workers.generate_stats import (
     project_stats_by_date,
     tasking_manager_geometries,
@@ -141,6 +141,8 @@ def get_groups(filename: str, project_id: str) -> pd.DataFrame:
         logger.info(f"file {filename} already exists for {project_id}. skip download.")
         pass
     else:
+        # TODO: check how we use number_of_users_required
+        #   it can get you a wrong number, if more users finished than required
         sql_query = sql.SQL(
             """
             COPY (
@@ -374,7 +376,11 @@ def get_per_project_statistics(project_id: str, project_info: pd.Series) -> dict
         )
 
         # generate geometries for HOT Tasking Manager
-        tasking_manager_geometries.generate_tasking_manager_geometries(project_id)
+        if project_info.iloc[0]["project_type"] in [ProjectType.FOOTPRINT.value]:
+            # do not do this for ArbitraryGeometry / BuildingFootprint projects
+            logger.info(f"do NOT generate tasking manager geometries for {project_id}")
+        else:
+            tasking_manager_geometries.generate_tasking_manager_geometries(project_id)
 
         # prepare output of function
         project_stats_dict = {

mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/group.py

@@ -8,7 +8,9 @@ class Group(BaseGroup):
     def __init__(self, project: object, groupId: int) -> None:
         super().__init__(project, groupId)
 
-    def create_tasks(self, feature_ids: List, feature_geometries: List) -> None:
+    def create_tasks(
+        self, feature_ids: List, feature_geometries: List, center_points: List
+    ) -> None:
         """Create tasks for a group
 
         feature_geometries is a list of geometries or feature in geojson format.
@@ -17,6 +19,6 @@ def create_tasks(self, feature_ids: List, feature_geometries: List) -> None:
         Every coordinate pair is a vertex.
         """
         for i in range(0, len(feature_ids)):
-            task = Task(self, feature_ids[i], feature_geometries[i])
+            task = Task(self, feature_ids[i], feature_geometries[i], center_points[i])
             self.tasks.append(task)
         self.numberOfTasks = len(self.tasks)

mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/grouping_functions.py

@@ -62,12 +62,22 @@ def group_input_geometries(input_geometries_file, group_size):
         try:
             groups[group_id_string]
         except KeyError:
-            groups[group_id_string] = {"feature_ids": [], "feature_geometries": []}
+            groups[group_id_string] = {
+                "feature_ids": [],
+                "feature_geometries": [],
+                "center_points": [],
+            }
 
         groups[group_id_string]["feature_ids"].append(feature.GetFID())
         groups[group_id_string]["feature_geometries"].append(
             json.loads(feature.GetGeometryRef().ExportToJson())
         )
+        try:
+            center_x = feature.GetFieldAsDouble("center_x")
+            center_y = feature.GetFieldAsDouble("center_y")
+            groups[group_id_string]["center_points"].append([center_x, center_y])
+        except:  # noqa
+            groups[group_id_string]["center_points"].append([])
 
     return groups
 

mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/project.py

@@ -7,6 +7,7 @@
 from mapswipe_workers.project_types.arbitrary_geometry import grouping_functions as g
 from mapswipe_workers.project_types.arbitrary_geometry.group import Group
 from mapswipe_workers.project_types.base.project import BaseProject
+from mapswipe_workers.project_types.base.tile_server import BaseTileServer
 
 
 class Project(BaseProject):
@@ -16,7 +17,7 @@ def __init__(self, project_draft: dict) -> None:
         # set group size
         self.groupSize = project_draft["groupSize"]
         self.inputGeometries = project_draft["inputGeometries"]
-        self.tileServer = self.get_tile_server(project_draft["tileServer"])
+        self.tileServer = vars(BaseTileServer(project_draft["tileServer"]))
 
     def validate_geometries(self):
         raw_input_file = (
@@ -143,8 +144,13 @@ def create_groups(self):
 
         for group_id, item in raw_groups.items():
             group = Group(self, group_id)
-            group.create_tasks(item["feature_ids"], item["feature_geometries"])
-            self.groups.append(group)
+            group.create_tasks(
+                item["feature_ids"], item["feature_geometries"], item["center_points"]
+            )
+
+            # only append valid groups
+            if group.is_valid():
+                self.groups.append(group)
 
         logger.info(
             f"{self.projectId} " f"- create_groups - " f"created groups dictionary"

mapswipe_workers/mapswipe_workers/project_types/arbitrary_geometry/task.py

@@ -1,10 +1,14 @@
+from typing import Dict, List
+
 from osgeo import ogr
 
 from mapswipe_workers.project_types.base.task import BaseTask
 
 
 class Task(BaseTask):
-    def __init__(self, group: object, featureId: int, featureGeometry: dict):
+    def __init__(
+        self, group: object, featureId: int, featureGeometry: Dict, center: List
+    ):
         """
         Parameters
         ----------
@@ -17,6 +21,11 @@ def __init__(self, group: object, featureId: int, featureGeometry: dict):
         task_id = f"t{featureId}"
         super().__init__(group, taskId=task_id)
         self.geojson = featureGeometry
+        self.center = center
+
+        # Remove projectId and groupId for tasks of Footprint project type
+        del self.projectId
+        del self.groupId
 
         # create wkt geometry from geojson
         poly = ogr.CreateGeometryFromJson(str(featureGeometry))