From e1df41451228aa7525d10d1185868901c23d16f8 Mon Sep 17 00:00:00 2001
From: Tom Arnfeld <tarnfeld@me.com>
Date: Fri, 21 Aug 2015 23:16:52 -0700
Subject: [PATCH 1/6] Install a fully operational vagrant machine

- Mesos (latest)
- Docker (latest) + client (1.7.)
- Portainer dependencies
---
 Vagrantfile | 47 +++++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 43 insertions(+), 4 deletions(-)

diff --git a/Vagrantfile b/Vagrantfile
index aa5101d..c6db60a 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -5,11 +5,50 @@
 $docker_setup = <<SCRIPT
 set -e
 
-wget -q -O - http://get.docker.io/gpg | sudo apt-key add -
-sudo bash -c "echo 'deb http://get.docker.io/ubuntu docker main' > /etc/apt/sources.list.d/docker.list"
-sudo apt-get update -q
-sudo apt-get install -q -y lxc-docker python-setuptools python-virtualenv
+# Setup
+sudo apt-key adv --keyserver keyserver.ubuntu.com --recv E56151BF
+DISTRO=$(lsb_release -is | tr '[:upper:]' '[:lower:]')
+CODENAME=$(lsb_release -cs)
+
+# Add the repository
+echo "deb http://repos.mesosphere.io/${DISTRO} ${CODENAME} main" | \
+  sudo tee /etc/apt/sources.list.d/mesosphere.list
+sudo apt-get -y update
+sudo apt-get -y install mesos
+sudo bash -c "echo 192.168.33.50 > /etc/mesos-master/ip"
+sudo bash -c "echo 192.168.33.50 > /etc/mesos-slave/ip"
+sudo bash -c "echo docker,mesos > /etc/mesos-slave/containerizers"
+sudo bash -c "echo /usr/bin/docker-1.7.0 > /etc/mesos-slave/docker"
+
+# Start a bunch of services
+sudo service zookeeper restart
+sleep 5
+(sudo service mesos-master stop || true)
+(sudo service mesos-slave stop || true)
+
+# Install Docker
+sudo bash -c 'echo "deb http://http.debian.net/debian wheezy-backports main" > /etc/apt/sources.list.d/backports.list'
+sudo apt-get install -t wheezy-backports linux-image-amd64
+curl -sSL https://get.docker.com/ | sh
 sudo usermod -a -G docker vagrant
+
+# Download a specific docker binary
+# TODO: Skip the above?
+sudo bash -c "curl -0 https://get.docker.com/builds/Linux/x86_64/docker-1.7.0 > /usr/bin/docker-1.7.0"
+
+# Set up the docker registry
+sudo mkdir -p /registry
+sudo docker create -p 5000:5000 -v /registry:/tmp/registry-dev --name=registry registry:0.9.1
+(sudo docker start registry || true)
+
+# Start mesos
+sudo service mesos-master start
+sudo service mesos-slave start
+
+# Install portainer dependencies
+sudo apt-get install python-setuptools
+sudo easy_install pip
+sudo pip install virtualenv
 SCRIPT
 
 Vagrant.configure("2") do |config|

From 305a1c9efd220f332f7d9a2770e850955a98e688 Mon Sep 17 00:00:00 2001
From: Tom Arnfeld <tarnfeld@me.com>
Date: Fri, 21 Aug 2015 23:17:38 -0700
Subject: [PATCH 2/6] Reduce build memory for the portainer example

---
 example/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/example/Dockerfile b/example/Dockerfile
index 356bc1f..c425e38 100644
--- a/example/Dockerfile
+++ b/example/Dockerfile
@@ -8,7 +8,7 @@ MAINTAINER Tom Arnfeld <tom@duedil.com>
 
 REPOSITORY tarnfeld/portainer
 BUILD_CPU 0.1
-BUILD_MEM 512
+BUILD_MEM 256
 
 # Install dependencies
 RUN apt-get update && apt-get install -y build-essential git python-setuptools python-virtualenv

From b1960afd14dcf7c4a20e35611dfa9646cac3201a Mon Sep 17 00:00:00 2001
From: Tom Arnfeld <tarnfeld@me.com>
Date: Fri, 21 Aug 2015 23:49:14 -0700
Subject: [PATCH 3/6] Cleaned up README

---
 example/Dockerfile => Dockerfile |   4 -
 README.md                        | 145 ++++++++++++++++---------------
 2 files changed, 77 insertions(+), 72 deletions(-)
 rename example/Dockerfile => Dockerfile (90%)

diff --git a/example/Dockerfile b/Dockerfile
similarity index 90%
rename from example/Dockerfile
rename to Dockerfile
index c425e38..0a57260 100644
--- a/example/Dockerfile
+++ b/Dockerfile
@@ -6,10 +6,6 @@
 FROM ubuntu:12.04
 MAINTAINER Tom Arnfeld <tom@duedil.com>
 
-REPOSITORY tarnfeld/portainer
-BUILD_CPU 0.1
-BUILD_MEM 256
-
 # Install dependencies
 RUN apt-get update && apt-get install -y build-essential git python-setuptools python-virtualenv
 
diff --git a/README.md b/README.md
index 40efe30..b2838e7 100644
--- a/README.md
+++ b/README.md
@@ -1,9 +1,7 @@
 
 # Portainer
 
-Portainer is an [Apache Mesos](http://mesos.apache.org) framework that enables you to build docker images across a cluster of many machines. Given a valid `Dockerfile`, Portainer can build your image and push it to a private registry faster than you can count to `n`.
-
-[![](http://img.shields.io/badge/duedil-hiring-green.svg)](http://duedil.careers/)
+Portainer is an [Apache Mesos](http://mesos.apache.org) framework that enables you to build docker images across a cluster of many machines.
 
 ```
                    .,.,.,.,.,.,.,.,.,.,.,.,.,.,.,.,.,.,.,.,.,.
@@ -22,25 +20,14 @@ Portainer is an [Apache Mesos](http://mesos.apache.org) framework that enables y
            , .   `                     ,              ,              ,              ,
      .`.,,,,,,,,,..                    ``````````````````````````````````````````````
            `     ,
-           ```````                               ,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
-````````````````````````````````                 \
-                               |¸.·´¯`·.´¯`·.¸¸.·´\     o    o    o    o    o    o
-                               |                   \
-    __|___|___|___|___|___|    |                    \................................
-       |___|___|___|___|       |   (
-         ___|___|___|___|      |    )  (      _/|_/       |  |
-      |___|___|___|            |   (    )    <').-\        \/        Y
-                               |    )  (     ``            |  /!-!\  |
-                               |   (    )                   \|     |/
-                               |    )  (                     _\___/_
-                               |   (    )                   / /   \ \
+           ```````
 ```
 
-When building docker images at scale, it can be time consuming and wasteful to manage dedicated infrastructure for building and pushing images. Building large containers with many sources and dependencies is a heavy operation, requiring large machines, and multiple of them. Deploying this infrastructure can be expensive and lead to poor utilization.
+When building docker images at scale, it can be time consuming and wasteful to manage dedicated infrastructure for building and pushing images. Building large containers with many sources and dependencies is a heavy operation, sometimes requiring many large machines. Deploying this infrastructure can be expensive and often leads to poor utilization.
 
 Given an existing Apache Mesos cluster, Portainer can get to work right away. If you're new to Mesos, you can try out the Vagrant box provided, or learn more about the [Apache Mesos Architecture](http://mesos.apache.org/documentation/latest/mesos-architecture/) and [get started](http://mesos.apache.org/gettingstarted/).
 
-To use the Vagrant box, run `vagrant box add debian-73-x64-virtualbox-nocm http://puppet-vagrant-boxes.puppetlabs.com/debian-73-x64-virtualbox-nocm.box` then `vagrant up`
+See below for more documentation on how to use the Vagrant virtual machine.
 
 --------------------------------------------------------------------------------
 
@@ -49,95 +36,117 @@ To use the Vagrant box, run `vagrant box add debian-73-x64-virtualbox-nocm http:
 - Works out of the box with existing `Dockerfile` files
 - Configurable CPU/Memory resource limits for build tasks
 - Full support for all `Dockerfile` commands, including local sources (e.g `ADD ./src`)
-- Capable of building many images in parallel across the cluster
-- Docker build logs are streamed from the Mesos slave for easy debugging and monitoring
+- Docker build logs are streamed from the Mesos agent for easy debugging and monitoring
 - Support for the `.dockerignore` file
 
-#### Not Supported
+#### Notes
 
-- Pushing built images to the public docker index
+- Pushing built images to the public docker index is currently not supported
+- Support for docker client ~>1.7.0 requires Apache Mesos >=0.23.0 ([MESOS-3279](https://issues.apache.org/jira/browse/MESOS-3279))
 
 --------------------------------------------------------------------------------
 
-## Getting Started
+## The Basics
 
-### Framework Dependencies
+### Dependencies
 
-You'll need to have the following dependencies installed to run the framework, though it's likely you'll only need to install the ones highlighted in bold;
+You'll need to have the following dependencies installed to run the framework.
 
 - Python 2.7
-- **Python `virtualenv` and `pip`**
-- **Protocol Buffers (`brew install protobuf`)**
+- Python `virtualenv` and `pip`
+- Protocol Buffers (`brew install protobuf`)
 - Make
 
-### Slave Dependencies
+### Mesos Agent Dependencies
+
+By default, Portainer will try and launch an ephemeral docker daemon (`docker -d`) on the mesos agent machine using [docker in docker](github.com/jpetazzo/dind). This requires that you're using a Docker Containerizer on your Mesos agents. If you are not, you'll need to specify the `--docker-host` argument (e.g `--docker-host /var/run/docker.sock`) describing where the docker daemon can be accessed on each agent.
 
-By default, Portainer will try and launch an ephemeral docker daemon (`docker -d`) on the mesos slave machine using [docker in docker](github.com/jpetazzo/dind). This requires that you're using a Docker Containerizer on your Mesos slaves. If you are not, you'll need to specify the `--docker-host` argument (e.g `--docker-host /var/run/docker.sock`) describing where the docker daemon can be accessed on each slave.
+## Building an Image
 
-*Note: Make sure you're on Mesos >=0.21.0 to allow for `privileged` containers.*
+#### 1. Build/Upload the Executor
 
-## Building Images
+```
+$ bin/build-executor
+```
 
-#### 1. Upload the Mesos executor
+_Note: If you've got a dirty git tree, you'll need to set the `FORCE=1` environment variable._
 
-Before being able to use Portainer, you need to upload the executor code somewhere accessible by the Mesos slaves. You can build a tar.gz archive containing the executor by running `bin/build-executor`. The archive will be dumped into `./dist/`, and needs to be uploaded somewhere Mesos can reach it (HDFS, S3, FTP, HTTP etc).
+The built PEX (python executable) archive will be dumped into `./dist`, and needs to be uploaded somewhere Mesos can reach it (HDFS, S3, FTP, HTTP etc). Check the output from the build-executor command to see the file name, and upload the file.
 
-The executor is a [pex](https://github.com/pantsbuild/pex) file, meaning it fully encapsulates all dependencies (other than python), and behaves like any old binary. Since some of portainer's dependencies are platform-specific, the pex file is, too. The environment name is tacked on to the archive filename, e.g. `portainer-37cc6d5eb334473fdaa9c7522c4ce585032dca5c.linux-x86_64.tar.gz`. Make sure you build the executor on the same platform as your mesos slaves use.
+The environment name is tacked on to the archive filename, e.g. `dist/portainer-37cc6d5eb334473fdaa9c7522c4ce585032dca5c.linux-x86_64.tar.gz`. Make sure you build the executor on the same platform as your mesos slaves use.
 
 In future, readily-downloadable prebuild pex files will be available on versioned github releases.
 
-#### 2. Write your `Dockerfile`
+#### 2. Grab a `Dockerfile`
 
-Portainer can work out of the box on existing `Dockerfile` files with no modifications. To do this, you _must_ specify a repository for your image, using the `--repository` command line argument, for example`--repository duedil/portainer`.
+Portainer can work out of the box on any existing `Dockerfile`. A few simple examples can be found in the `example/` directory, or you can build an image of the Portainer framework itself using the one at the root of this project directory.
 
-You must also specify a private registry to push the image to once successfully built, using the `--to` command line argument, for example `--to my.registry:1234`.
+#### 3. Local sources
 
-If your `Dockerfile` is based upon a private image (in the `FROM` instruction) not available in the public docker index, you can use the `--from my.registry:1234` argument to configure where dependent images are pulled from. It is worth noting that when `--from` is used, all images are pulled from the given registry, and the public index is **never** used. This can be useful for mirroring public images which avoids being dependent on the public index.
+If your `Dockerfile` does not include any local sources in the image (via `ADD` or `COPY`) you can skip this step.
 
-Since Mesos is based around the concept of _Resources_, build tasks need some CPU and Memory to be able to execute. Defaults are provided, but the `--build-cpu` and `--build-mem` command line flags can be used to configure the resource allocation used.
+Since Portainer will build your image on a remote machine, it must bundle and upload these local sources so they to be used remotely. Portainer uses a staging filesystem that can be accessed both by the framework and by the slave, this can be anything supported by the Mesos Fetcher (e.g HDFS).
 
---------------------------------------------------------------------------------
+Use the `--staging-uri` command line flag to specify this. For example to distribute sources using your HDFS cluster, `--staging-uri=hdfs://my.namenode/tmp/portainer`.
 
-As mentioned above, Portainer supports a set of custom `Dockerfile` instructions. These are safe to use with the standard `docker build` tool as they will simply be ignored.
+#### 4. Invoke Portainer
 
-- `REPOSITORY`  / `--repository` - The name of the image repository (`string`)
-- `BUILD_CPU`   / `--build-cpu` - The number of CPUs required to build the image (`float`)
-- `BUILD_MEM`   / `--build-mem` - The amount of memory required to build the image (`integer`, in megabytes)
+Given Portainer is an Apache Mesos framework, we need to define the resources we'd like to use to build our image. The CPU and RAM limits can be specified using the `--build-cpu` and `--build-mem` command line options. We also need to give our image a name (the repository), a tag and specify a registry to push to.
 
-For an example, take a look at the `Dockerfile` provided in the `./example` folder. This can be used to build an image of the Portainer source code.
+If you'd like to see the STDOUT/STDERR logs from your build printed live, add `--stream` to the list of arguments.
 
-#### 3. Local `ADD` sources
+```
+$ cd portainer
+$ ./bin/portainer \
+        --mesos-master "localhost:5050" \
+        --executor-uri "hdfs://my-namenode/path/to/portainer-executor.tar.gz" \
+        build \
+        --staging-uri "hdfs://my-namenode/tmp/portainer" \
+        --tag "latest" \
+        --to "my-registry:5000" \
+        --build-cpu 1 \
+        --build-mem 256 \
+        ./Dockerfile
+```
 
-If your `Dockerfile` does not container any `ADD` commands that use local files, you can skip this step entirely. If you do use local sources, continue reading. Additional configuration is required.
+## Vagrant Example
 
-Docker provides a way of bundling up local sources into the image being built, using the `ADD` command. For example;
+The vagrant virtual environment provided will launch a VM will the following components for testing out Portainer;
 
-```
-ADD ./src /usr/lib/my-src
-```
+- Mesos Master + Agent + ZooKeeper
+- Docker Registry
+- Mesos <> Docker Containerizer
+- Portainer code
 
-Since Portainer will build your image on a remote machine, it has to bundle and upload these local sources, so they to be used remotely when building the image. You can use any filesystem supported by [`pyfs`](github.com/duedil-ltd/pyfilesystem), including HDFS and S3. If you're using S3 you will need to configure the correct environment variables for authentication, being `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`.
+### 1. Start the VM
 
-Use the `--staging-uri` command line flag to specify this. For example to distribute sources using your HDFS cluster, `--staging-uri=hdfs://my.namenode:50070/tmp/portainer`.
+To use the Vagrant box, run `vagrant box add debian-73-x64-virtualbox-nocm http://puppet-vagrant-boxes.puppetlabs.com/debian-73-x64-virtualbox-nocm.box` then `vagrant up` to set everything up.
 
-*Note: Portainer imposes no restrictions on symlinks or relative paths in `ADD` instructions, unlike docker. In some situations this can pose security issues if building images from `Dockerfile` files from untrusted sources. Portainer and the Mesos executor will only have access to files readable to the user it's running as, so don't run the framework as `root`.*
+### 2. Test Mesos
 
-#### 4. Launch Portainer
+The VM runs on a static IP `192.168.33.50` so before proceeding it's best to check that the Mesos UI is fully up and running at http://192.168.33.50:5050/ and there's a slave joined with at least 256MB of RAM. You should also check that the docker registry is up, a simple `docker ps` should demonstrate this.
 
-Now that you've got everything set up, you're  good to go. Because Portainer uses a pure-python implementation of the Mesos Framework API ([called pesos](http://github.com/wickman/pesos)), there is no requirement to install Apache Mesos itself to run the framework. You can use the invocation below as an example.
+### 3. Build the executor
 
-```
-$ cd Portainer
-$ ./bin/portainer \
-        --mesos-master "localhost:5050" \
-        --executor-uri "hdfs://my-namenode/path/to/portainer-executor.tar.gz" \
-        build \
-        --staging-uri "hdfs://my-namenode/tmp/portainer" \
-        --tag "my_custom_tag" \
-        --to "my-registry:5000" \
-        example/Dockerfile
-```
+To build the Portainer executor, simply run `bin/build-executor`.
+
+_Note: If you've got a dirty git tree, you'll need to set the `FORCE=1` environment variable._
 
-##### Debugging
+### 4. Example build
 
-To aid debugging, any exceptions thrown while building the docker image will be relayed back to the framework and printed out. If you want to keep an eye on the output of all `RUN` commands docker is executing while building your images, add the `--stream` argument and all stdout/stderr logs will be streamed to the framework.
+You'll need to fill in a few blanks in the follow command line invocation, but this should start a simple build on the local mesos cluster, and push the image to the local repository.
+
+```
+$ bin/portainer \
+    --mesos-master 192.168.33.50:5050 \
+    build \
+    --staging-uri /tmp \
+    --executor-uri `pwd`/dist/portainer-{CHANGE THIS TO THE ACTUAL FILE}.tar.gz \
+    --to 192.168.33.50:5000 \
+    --build-cpu 0.1 \
+    --build-mem 256 \
+    --repository tarnfeld/portainer \
+    --tag latest \
+    --stream \
+    ./Dockerfile
+```

From c36a089d4aa04b3bde44de44ab65a9aa0fd8b538 Mon Sep 17 00:00:00 2001
From: Tom Arnfeld <tarnfeld@me.com>
Date: Sat, 22 Aug 2015 01:12:52 -0700
Subject: [PATCH 4/6] WIP build server

---
 portainer/api.py                 | 98 ++++++++++++++++++++++++++++++++
 portainer/app/__main__.py        |  3 +-
 portainer/app/build.py           |  2 +-
 portainer/app/server.py          | 80 ++++++++++++++++++++++++++
 portainer/{app => }/scheduler.py | 25 ++------
 portainer/util/parser.py         | 30 ++++++----
 6 files changed, 204 insertions(+), 34 deletions(-)
 create mode 100644 portainer/api.py
 create mode 100644 portainer/app/server.py
 rename portainer/{app => }/scheduler.py (95%)

diff --git a/portainer/api.py b/portainer/api.py
new file mode 100644
index 0000000..5bd6654
--- /dev/null
+++ b/portainer/api.py
@@ -0,0 +1,98 @@
+# import logging
+import json
+import tempfile
+import tarfile
+
+import tornado.ioloop
+import tornado.web
+
+# from portainer.util.parser import parse_dockerfile_fp
+
+
+def make_app(queue, staging_fs):
+    return tornado.web.Application([
+        tornado.web.url(r'/_ping', PingHandler, name='ping'),
+        tornado.web.url(r'/v.*/build', BuildHandler, dict(queue=queue, fs=staging_fs), name='build')
+    ])
+
+
+class PingHandler(tornado.web.RequestHandler):
+    def get(self):
+        self.write("OK")
+
+
+@tornado.web.stream_request_body
+class BuildHandler(tornado.web.RequestHandler):
+    """
+    """
+
+    def initialize(self, queue, fs):
+        self.fs = fs
+        self.queue = queue
+
+    def prepare(self):
+        self.tmp = tempfile.TemporaryFile()
+
+    def data_received(self, data):
+        self.tmp.write(data)
+
+    def post(self):
+        self.set_header("Content-Type", "application/json")
+
+        try:
+            dockerfile_path = self.get_argument("dockerfile", "Dockerfile")
+            repository_tag = self.get_argument("t", None)
+            verbose = self.get_argument("q", True)
+            build_mem = self.get_argument("memory", 0)
+            build_cpu = float(self.get_argument("cpushares", 1))
+
+            if not repository_tag:
+                self.set_status(500)
+                self._send(error="Missing repository name for the docker image, use -t")
+                return
+            if not build_mem:
+                self.set_status(500)
+                self._send(error="No build memory limit set (megabytes)")
+                return
+
+            try:
+                self.tmp.seek(0)
+                tar = tarfile.TarFile(fileobj=self.tmp)
+                raw_dockerfile = tar.extractfile(dockerfile_path)
+            except:
+                self.set_status(500)
+                self._send(error="Failed to load Dockerfile from tar context %r" % dockerfile_path)
+                return
+
+            self.set_status(200)
+            self._send(message="Processing submitted build context")
+
+            # Parse the Dockerfile to ensure it's valid
+            # try:
+            #     parse_dockerfile_fp(raw_dockerfile)
+            # except:
+            #     self._send(error="Failed to parse Dockerfile")
+
+            # TODO: Create a staging directory for the build tar
+            # TODO: Upload the build tar
+            # TODO: Queue the build and watch for logs to be streamed back from the scheduler
+            # TODO: Handle killing the build if the connection is lost to the client
+        finally:
+            self.tmp.close()
+
+    def _send(self, message=None, error=None):
+        if message:
+            self.write(json.dumps({
+                "stream": "%s\n" % message
+            }))
+            self.flush()
+        elif error:
+            self.write(json.dumps({
+                "error": "%s\n" % error
+            }))
+            self.flush()
+
+    # def on_connection_close(self):
+    #     # TODO: Kill any builds in progress
+    #     print "YEAH CLOSED"
+    #     super(BuildHandler, self).on_connection_close()
diff --git a/portainer/app/__main__.py b/portainer/app/__main__.py
index c39938b..8c9db36 100644
--- a/portainer/app/__main__.py
+++ b/portainer/app/__main__.py
@@ -9,6 +9,7 @@
 # Import all the commands
 import portainer.app.build
 import portainer.app.executor
+import portainer.app.server
 
 
 def main(argv):
@@ -35,7 +36,7 @@ def main(argv):
     formatter = logging.Formatter(fmt="%(asctime)s[%(name)s] %(message)s")
     handler.setFormatter(formatter)
 
-    for logger in ("portainer.build", "portainer.scheduler", "portainer.executor", "pesos",
+    for logger in ("portainer.build", "portainer.scheduler", "portainer.executor", "portainer.server", "pesos",
                    "compactor", "tornado"):
         logger = logging.getLogger(logger)
         logger.propagate = False
diff --git a/portainer/app/build.py b/portainer/app/build.py
index 5b995b5..4321d73 100644
--- a/portainer/app/build.py
+++ b/portainer/app/build.py
@@ -10,7 +10,7 @@
 
 from pesos.vendor.mesos import mesos_pb2
 from portainer.app import subcommand
-from portainer.app.scheduler import Scheduler
+from portainer.scheduler import Scheduler
 
 logger = logging.getLogger("portainer.build")
 
diff --git a/portainer/app/server.py b/portainer/app/server.py
new file mode 100644
index 0000000..59fdc50
--- /dev/null
+++ b/portainer/app/server.py
@@ -0,0 +1,80 @@
+"""The entrypoint to the portainer app. Spins up the a schedular instance and
+waits for the result."""
+
+# import getpass
+import logging
+
+import pesos.scheduler
+from Queue import Queue
+from tornado.ioloop import IOLoop
+from pesos.vendor.mesos import mesos_pb2
+# import sys
+import threading
+import time
+
+from portainer.app import subcommand
+from portainer.api import make_app
+from portainer.scheduler import Scheduler
+
+logger = logging.getLogger("portainer.build")
+
+
+def args(parser):
+
+    parser.add_argument("--executor-uri", dest="executor", required=True,
+                        help="URI to the portainer executor for mesos")
+    parser.add_argument("--push-registry", required=False,
+                        help="Docker registry to push images to")
+    parser.add_argument("--container-image", default="jpetazzo/dind",
+                        help="Docker image to run the portainer executor in")
+    parser.add_argument("--insecure", default=False, action="store_true",
+                        help="Enable pulling/pushing of images with insecure registries")
+    parser.add_argument("--staging-uri", required=False,
+                        help="The URI to use as a base directory for staging files.")
+    parser.add_argument("--host", default="0.0.0.0",
+                        help="IP address for the API to listen on")
+    parser.add_argument("--port", default=3000, type=int,
+                        help="Port for the API to listen on")
+
+
+@subcommand("server", callback=args)
+def main(args):
+
+    framework = mesos_pb2.FrameworkInfo()
+    framework.user = ""  # Mesos will fill this in
+    framework.name = "Portainer Server"
+
+    queue = Queue()
+
+    scheduler = Scheduler(
+        task_queue=queue,
+        executor_uri=args.executor,
+        push_registry=args.push_registry,
+        staging_uri=args.staging_uri,
+        container_image=args.container_image,
+        verbose=args.verbose,
+        insecure_registries=args.insecure
+    )
+
+    # Spawn the HTTP API
+    def start_app():
+        application = make_app(queue, scheduler.filesystem)
+        application.listen(args.port, args.host)
+        logger.info("HTTP API Listening on %s:%d" % (args.host, args.port))
+        IOLoop.current().start()
+
+    thread = threading.Thread(target=start_app)
+    thread.setDaemon(True)
+    thread.start()
+
+    # Spawn the scheduler driver
+    driver = pesos.scheduler.PesosSchedulerDriver(
+        scheduler, framework, args.mesos_master
+    )
+
+    thread = threading.Thread(target=driver.run)
+    thread.setDaemon(True)
+    thread.start()
+
+    while True:
+        time.sleep(0.1)
diff --git a/portainer/app/scheduler.py b/portainer/scheduler.py
similarity index 95%
rename from portainer/app/scheduler.py
rename to portainer/scheduler.py
index d89fbf6..e55fe92 100644
--- a/portainer/app/scheduler.py
+++ b/portainer/scheduler.py
@@ -38,33 +38,17 @@ class StagingSystemRequiredException(Exception):
 class Scheduler(mesos.interface.Scheduler):
     """Mesos scheduler that is responsible for launching the builder tasks."""
 
-    def __init__(self, tasks, executor_uri, cpu_limit, mem_limit, push_registry,
-                 staging_uri, stream=False, verbose=False, repository=None,
-                 pull_registry=None, docker_host=None, container_image=None,
-                 insecure_registries=False):
+    def __init__(self, task_queue, executor_uri, staging_uri,
+                 verbose=False, docker_host=None, container_image=None,
+                 push_registry=None, insecure_registries=False):
 
+        self.task_queue = task_queue
         self.executor_uri = executor_uri
-        self.cpu = float(cpu_limit)
-        self.mem = int(mem_limit)
-        self.push_registry = push_registry
-        self.pull_registry = pull_registry
         self.staging_uri = staging_uri
-        self.stream = stream
         self.verbose = verbose
-        self.repository = repository
         self.docker_host = docker_host
         self.container_image = container_image
         self.insecure_registries = insecure_registries
-
-        self.queued_tasks = []
-        for path, tags in tasks:
-            dockerfile = parse_dockerfile(path, registry=pull_registry)
-            self.queued_tasks.append((path, dockerfile, tags))
-
-        self.pending = len(self.queued_tasks)
-        self.running = 0
-        self.finished = 0
-        self.failed = 0
         self.task_ids = {}
 
         self._processing_offers = threading.Lock()
@@ -85,6 +69,7 @@ def __init__(self, tasks, executor_uri, cpu_limit, mem_limit, push_registry,
 
             self.filesystem = opener.opendir(self.staging_uri)
 
+        # Start the task cleanup sthread
         self.cleanup = TaskCleanupThread(self.filesystem)
         self.cleanup.start()
 
diff --git a/portainer/util/parser.py b/portainer/util/parser.py
index c95e9cb..80a5424 100644
--- a/portainer/util/parser.py
+++ b/portainer/util/parser.py
@@ -8,22 +8,28 @@
 def parse_dockerfile(path, **kwargs):
     """Parse a dockerfile and return a new Dockerfile object"""
 
-    dockerfile = Dockerfile(**kwargs)
-    with open(path) as f:
-        line_buf = ""
-        for line in f:
-            if line.lstrip().startswith("#") or len(line.rstrip()) == 0:
-                continue
+    with open(path) as fp:
+        return parse_dockerfile_fp(fp, **kwargs)
+
+
+def parse_dockerfile_fp(fp, **kwargs):
+    """
+    """
 
-            line_buf += line
+    dockerfile = Dockerfile(**kwargs)
 
-            if line.rstrip().endswith('\\'):
-                continue
+    line_buf = ""
+    for line in fp:
+        if line.lstrip().startswith("#") or len(line.rstrip()) == 0:
+            continue
 
-            line_buf = line_buf.rstrip().rstrip("\\")
+        line_buf += line
+        if line.rstrip().endswith('\\'):
+            continue
 
-            dockerfile += line_buf
-            line_buf = ""
+        line_buf = line_buf.rstrip().rstrip("\\")
+        dockerfile += line_buf
+        line_buf = ""
 
     return dockerfile
 

From f54c5bbdc4aad967f82f1bb4869254d09d58a67f Mon Sep 17 00:00:00 2001
From: Tom Arnfeld <tarnfeld@me.com>
Date: Mon, 31 Aug 2015 16:57:10 +0100
Subject: [PATCH 5/6] Implemented async callbacks for uploads and log streams

---
 portainer/api.py | 140 +++++++++++++++++++++++++++++++++++++++--------
 1 file changed, 117 insertions(+), 23 deletions(-)

diff --git a/portainer/api.py b/portainer/api.py
index 5bd6654..ca54462 100644
--- a/portainer/api.py
+++ b/portainer/api.py
@@ -1,7 +1,10 @@
 # import logging
 import json
-import tempfile
 import tarfile
+import tempfile
+import threading
+import uuid
+from Queue import Queue
 
 import tornado.ioloop
 import tornado.web
@@ -32,6 +35,8 @@ def initialize(self, queue, fs):
 
     def prepare(self):
         self.tmp = tempfile.TemporaryFile()
+        self.building = threading.Event()
+        self.msg_stream = Queue()
 
     def data_received(self, data):
         self.tmp.write(data)
@@ -42,56 +47,145 @@ def post(self):
         try:
             dockerfile_path = self.get_argument("dockerfile", "Dockerfile")
             repository_tag = self.get_argument("t", None)
-            verbose = self.get_argument("q", True)
+            verbose = not self.get_argument("q", True)  # TODO: Handle casting to bool?
             build_mem = self.get_argument("memory", 0)
             build_cpu = float(self.get_argument("cpushares", 1))
 
             if not repository_tag:
                 self.set_status(500)
-                self._send(error="Missing repository name for the docker image, use -t")
+                self._send(err="Missing repository name for the docker image, use -t")
                 return
             if not build_mem:
                 self.set_status(500)
-                self._send(error="No build memory limit set (megabytes)")
+                self._send(err="No build memory limit set (megabytes)")
                 return
 
             try:
+                if verbose:
+                    self._send(msg="DEBUG: Opening context")
                 self.tmp.seek(0)
                 tar = tarfile.TarFile(fileobj=self.tmp)
-                raw_dockerfile = tar.extractfile(dockerfile_path)
+                if verbose:
+                    self._send(msg="Reading Dockerfile from %r" % dockerfile_path)
+                tar.extractfile(dockerfile_path)
             except:
                 self.set_status(500)
-                self._send(error="Failed to load Dockerfile from tar context %r" % dockerfile_path)
+                self._send(err="Failed to load Dockerfile from tar context %r" % dockerfile_path)
                 return
 
             self.set_status(200)
-            self._send(message="Processing submitted build context")
-
-            # Parse the Dockerfile to ensure it's valid
-            # try:
-            #     parse_dockerfile_fp(raw_dockerfile)
-            # except:
-            #     self._send(error="Failed to parse Dockerfile")
-
-            # TODO: Create a staging directory for the build tar
-            # TODO: Upload the build tar
-            # TODO: Queue the build and watch for logs to be streamed back from the scheduler
-            # TODO: Handle killing the build if the connection is lost to the client
+            self._send(msg="Processing submitted build context")
+
+            task_id = str(uuid.uuid1())
+            staging_dir = os.path.join("staging", task_id)
+            self.fs.makedir(staging_dir, recursive=True)
+
+            self.tmp.seek(0)
+            self.building.set()
+
+            staging_context = os.path.join(staging_dir, "context.tar.gz")
+            self.filesystem.setcontents_async(
+                path=staging_context,
+                data=self.tmp,
+                progress_callback=partial(
+                    self._ctx_upload_progress,
+                    task_id,
+                    verbose
+                ),
+                finished_callback=partial(
+                    self._ctx_upload_finished,
+                    task_id,
+                    repository_tag,
+                    build_mem,
+                    build_cpu,
+                    verbose
+                ),
+                error_callback=partial(
+                    self._ctx_upload_failed,
+                    task_id,
+                    verbose
+                )
+            )
+
+            self._stream_messages()
+
         finally:
+            self.building.clear()
+            self.msg_stream.close()
             self.tmp.close()
 
-    def _send(self, message=None, error=None):
-        if message:
+    def _send(self, msg=None, err=None):
+        if msg and err:
+            raise ArgumentError
+        if msg:
             self.write(json.dumps({
-                "stream": "%s\n" % message
+                "stream": "%s\n" % msg
             }))
             self.flush()
-        elif error:
+        elif err:
             self.write(json.dumps({
-                "error": "%s\n" % error
+                "error": "%s\n" % err
             }))
             self.flush()
 
+    def _stream_messages(self):
+        """
+        Wait for the build to finish and stream messages back to the client as
+        they appear on the stream.
+        """
+
+        while self.building.is_set():
+            msg = self.msg_stream.get(True, 0.1)
+            if not msg:
+                continue
+
+            message, is_error = msg
+            if is_error:
+                self._send(err=message)
+            else:
+                self._send(msg=msg)
+
+    def _ctx_upload_progress(self, task_id, verbose, total, bytes):
+        """
+        Callback for progress updates to the build context upload.
+        """
+
+        # TODO: Show some sort of progress information? Perhaps look into how
+        # Docker does the streaming progress bars.
+        pass
+
+    def _ctx_upload_finished(self, task_id, repository_tag, build_mem, build_cpu, verbose):
+        """
+        Callback for when the build context upload finishes. When this callback
+        is fired, the actual build is placed on the queue with relevant parameters.
+        """
+
+        self.msg_stream.put(("Finished uploaded build context", False))
+
+        build_task = portainer_pb2.BuildTask()
+        build_task.stream = vebose
+        build_task.context = None  # TODO
+
+        repo = parse_repository(repository_tag)
+        build_task.image.registry.hostname = repo.registry_host
+        if repo.registry_port:
+            build_task.image.registry.port = repo.registry_port
+
+        build_task.image.repository = repo.name
+        build_task.image.tag.append(repo.tag)
+
+        self.queue.put((build_task, self.building, self.msg_stream))
+
+    def _ctx_upload_failed(self, task_id, verbose, exc):
+        """
+        Callback for when the build context upload fails.
+        """
+
+        self.msg_stream.put(("Caught exception while uploading the build context", True))
+        self.building.clear()
+
+        raise exc
+
     # def on_connection_close(self):
     #     # TODO: Kill any builds in progress
     #     print "YEAH CLOSED"

From 56f6d79854dbc015798bd35e8b036e5dc568435b Mon Sep 17 00:00:00 2001
From: Tom Arnfeld <tarnfeld@me.com>
Date: Thu, 10 Sep 2015 11:49:29 +0100
Subject: [PATCH 6/6] Change the build context name to use

---
 portainer/api.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/portainer/api.py b/portainer/api.py
index ca54462..d64dc24 100644
--- a/portainer/api.py
+++ b/portainer/api.py
@@ -164,7 +164,7 @@ def _ctx_upload_finished(self, task_id, repository_tag, build_mem, build_cpu, ve
 
         build_task = portainer_pb2.BuildTask()
         build_task.stream = vebose
-        build_task.context = None  # TODO
+        build_task.context = "context.tar.gz"
 
         repo = parse_repository(repository_tag)
         build_task.image.registry.hostname = repo.registry_host