[example] image classification web demo

sergeyk · sergeyk · commit dd546171bae1 · 2014-07-11T23:41:12.000-07:00
diff --git a/data/ilsvrc12/get_ilsvrc_aux.sh b/data/ilsvrc12/get_ilsvrc_aux.sh
@@ -4,6 +4,7 @@
 # This script downloads the imagenet example auxiliary files including:
 # - the ilsvrc12 image mean, binaryproto
 # - synset ids and words
+# - Python pickle-format data of ImageNet graph structure and relative infogain
 # - the training splits with labels
 
 DIR="$( cd "$(dirname "$0")" ; pwd -P )"
diff --git a/docs/getting_pretrained_models.md b/docs/getting_pretrained_models.md
@@ -8,7 +8,8 @@ layout: default
 Note that unlike Caffe itself, these models are licensed for **academic research / non-commercial use only**.
 If you have any questions, please get in touch with us.
 
-This page will be updated as more models become available.
+*UPDATE* July 2014: we are actively working on a service for hosting user-uploaded model definition and trained weight files.
+Soon, the community will be able to easily contribute different architectures!
 
 ### ImageNet
 
@@ -26,4 +27,6 @@ This page will be updated as more models become available.
   validation accuracy 57.258% and loss 1.83948.
 - This model obtains a top-1 accuracy 57.1% and a top-5 accuracy 80.2% on the validation set, using just the center crop. (Using the average of 10 crops, (4 + 1 center) * 2 mirror, should obtain a bit higher accuracy)
 
+### Auxiliary Data
+
 Additionally, you will probably eventually need some auxiliary data (mean image, synset list, etc.): run `data/ilsvrc12/get_ilsvrc_aux.sh` from the root directory to obtain it.
diff --git a/examples/web_demo/app.py b/examples/web_demo/app.py
@@ -0,0 +1,215 @@
+import os
+import time
+import cPickle
+import datetime
+import logging
+import flask
+import werkzeug
+import optparse
+import tornado.wsgi
+import tornado.httpserver
+import numpy as np
+import pandas as pd
+from PIL import Image as PILImage
+import cStringIO as StringIO
+import urllib
+import caffe
+import exifutil
+
+REPO_DIRNAME = os.path.abspath(os.path.dirname(__file__) + '/../..')
+UPLOAD_FOLDER = '/tmp/caffe_demos_uploads'
+ALLOWED_IMAGE_EXTENSIONS = set(['png', 'bmp', 'jpg', 'jpe', 'jpeg', 'gif'])
+
+# Obtain the flask app object
+app = flask.Flask(__name__)
+
+
+@app.route('/')
+def index():
+    return flask.render_template('index.html', has_result=False)
+
+
+@app.route('/classify_url', methods=['GET'])
+def classify_url():
+    imageurl = flask.request.args.get('imageurl', '')
+    try:
+        string_buffer = StringIO.StringIO(
+            urllib.urlopen(imageurl).read())
+        image = caffe.io.load_image(string_buffer)
+
+    except Exception as err:
+        # For any exception we encounter in reading the image, we will just
+        # not continue.
+        logging.info('URL Image open error: %s', err)
+        return flask.render_template(
+            'index.html', has_result=True,
+            result=(False, 'Cannot open image from URL.')
+        )
+
+    logging.info('Image: %s', imageurl)
+    result = app.clf.classify_image(image)
+    return flask.render_template(
+        'index.html', has_result=True, result=result, imagesrc=imageurl)
+
+
+@app.route('/classify_upload', methods=['POST'])
+def classify_upload():
+    try:
+        # We will save the file to disk for possible data collection.
+        imagefile = flask.request.files['imagefile']
+        filename_ = str(datetime.datetime.now()).replace(' ', '_') + \
+            werkzeug.secure_filename(imagefile.filename)
+        filename = os.path.join(UPLOAD_FOLDER, filename_)
+        imagefile.save(filename)
+        logging.info('Saving to %s.', filename)
+        image = exifutil.open_oriented_im(filename)
+
+    except Exception as err:
+        logging.info('Uploaded image open error: %s', err)
+        return flask.render_template(
+            'index.html', has_result=True,
+            result=(False, 'Cannot open uploaded image.')
+        )
+
+    result = app.clf.classify_image(image)
+    return flask.render_template(
+        'index.html', has_result=True, result=result,
+        imagesrc=embed_image_html(image)
+    )
+
+
+def embed_image_html(image):
+    """Creates an image embedded in HTML base64 format."""
+    image_pil = PILImage.fromarray((255 * image).astype('uint8'))
+    image_pil = image_pil.resize((256, 256))
+    string_buf = StringIO.StringIO()
+    image_pil.save(string_buf, format='png')
+    data = string_buf.getvalue().encode('base64').replace('\n', '')
+    return 'data:image/png;base64,' + data
+
+
+def allowed_file(filename):
+    return (
+        '.' in filename and
+        filename.rsplit('.', 1)[1] in ALLOWED_IMAGE_EXTENSIONS
+    )
+
+
+class ImagenetClassifier(object):
+    default_args = {
+        'model_def_file': (
+            '{}/examples/imagenet/imagenet_deploy.prototxt'.format(REPO_DIRNAME)),
+        'pretrained_model_file': (
+            '{}/examples/imagenet/caffe_reference_imagenet_model'.format(REPO_DIRNAME)),
+        'mean_file': (
+            '{}/python/caffe/imagenet/ilsvrc_2012_mean.npy'.format(REPO_DIRNAME)),
+        'class_labels_file': (
+            '{}/data/ilsvrc12/synset_words.txt'.format(REPO_DIRNAME)),
+        'bet_file': (
+            '{}/data/ilsvrc12/imagenet.bet.pickle'.format(REPO_DIRNAME)),
+    }
+    for key, val in default_args.iteritems():
+        if not os.path.exists(val):
+            raise Exception(
+                "File for {} is missing. Should be at: {}".format(key, val))
+    default_args['image_dim'] = 227
+    default_args['gpu_mode'] = True
+
+    def __init__(self, model_def_file, pretrained_model_file, mean_file,
+                 class_labels_file, bet_file, image_dim, gpu_mode=False):
+        logging.info('Loading net and associated files...')
+        self.net = caffe.Classifier(
+            model_def_file, pretrained_model_file, input_scale=255,
+            image_dims=(image_dim, image_dim), gpu=gpu_mode,
+            mean_file=mean_file, channel_swap=(2, 1, 0)
+        )
+
+        with open(class_labels_file) as f:
+            labels_df = pd.DataFrame([
+                {
+                    'synset_id': l.strip().split(' ')[0],
+                    'name': ' '.join(l.strip().split(' ')[1:]).split(',')[0]
+                }
+                for l in f.readlines()
+            ])
+        self.labels = labels_df.sort('synset_id')['name'].values
+
+        self.bet = cPickle.load(open(bet_file))
+        # A bias to prefer children nodes in single-chain paths
+        # I am setting the value to 0.1 as a quick, simple model.
+        # We could use better psychological models here...
+        self.bet['infogain'] -= np.array(self.bet['preferences']) * 0.1
+
+    def classify_image(self, image):
+        try:
+            starttime = time.time()
+            scores = self.net.predict([image], oversample=True).flatten()
+            endtime = time.time()
+
+            indices = (-scores).argsort()[:5]
+            predictions = self.labels[indices]
+
+            # In addition to the prediction text, we will also produce
+            # the length for the progress bar visualization.
+            meta = [
+                (p, '%.5f' % scores[i])
+                for i, p in zip(indices, predictions)
+            ]
+            logging.info('result: %s', str(meta))
+
+            # Compute expected information gain
+            expected_infogain = np.dot(
+                self.bet['probmat'], scores[self.bet['idmapping']])
+            expected_infogain *= self.bet['infogain']
+
+            # sort the scores
+            infogain_sort = expected_infogain.argsort()[::-1]
+            bet_result = [(self.bet['words'][v], '%.5f' % expected_infogain[v])
+                          for v in infogain_sort[:5]]
+            logging.info('bet result: %s', str(bet_result))
+
+            return (True, meta, bet_result, '%.3f' % (endtime - starttime))
+
+        except Exception as err:
+            logging.info('Classification error: %s', err)
+            return (False, 'Something went wrong when classifying the '
+                           'image. Maybe try another one?')
+
+
+def start_tornado(app, port=5000):
+    http_server = tornado.httpserver.HTTPServer(
+        tornado.wsgi.WSGIContainer(app))
+    http_server.listen(port)
+    print("Tornado server starting on port {}".format(port))
+    tornado.ioloop.IOLoop.instance().start()
+
+
+def start_from_terminal(app):
+    """
+    Parse command line options and start the server.
+    """
+    parser = optparse.OptionParser()
+    parser.add_option(
+        '-d', '--debug',
+        help="enable debug mode",
+        action="store_true", default=False)
+    parser.add_option(
+        '-p', '--port',
+        help="which port to serve content on",
+        type='int', default=5000)
+    opts, args = parser.parse_args()
+
+    # Initialize classifier
+    app.clf = ImagenetClassifier(**ImagenetClassifier.default_args)
+
+    if opts.debug:
+        app.run(debug=True, host='0.0.0.0', port=opts.port)
+    else:
+        start_tornado(app, opts.port)
+
+
+if __name__ == '__main__':
+    logging.getLogger().setLevel(logging.INFO)
+    if not os.path.exists(UPLOAD_FOLDER):
+        os.makedirs(UPLOAD_FOLDER)
+    start_from_terminal(app)
diff --git a/examples/web_demo/exifutil.py b/examples/web_demo/exifutil.py
@@ -0,0 +1,33 @@
+"""
+This script handles the skimage exif problem.
+"""
+
+from PIL import Image
+import numpy as np
+
+ORIENTATIONS = {   # used in apply_orientation
+    2: (Image.FLIP_LEFT_RIGHT,),
+    3: (Image.ROTATE_180,),
+    4: (Image.FLIP_TOP_BOTTOM,),
+    5: (Image.FLIP_LEFT_RIGHT, Image.ROTATE_90),
+    6: (Image.ROTATE_270,),
+    7: (Image.FLIP_LEFT_RIGHT, Image.ROTATE_270),
+    8: (Image.ROTATE_90,)
+}
+
+
+def open_oriented_im(im_path):
+    im = Image.open(im_path)
+    if hasattr(im, '_getexif'):
+        exif = im._getexif()
+        if exif is not None and 274 in exif:
+            orientation = exif[274]
+            im = apply_orientation(im, orientation)
+    return np.asarray(im).astype(np.float32) / 255.
+
+
+def apply_orientation(im, orientation):
+    if orientation in ORIENTATIONS:
+        for method in ORIENTATIONS[orientation]:
+            im = im.transpose(method)
+    return im
diff --git a/examples/web_demo/readme.md b/examples/web_demo/readme.md
@@ -0,0 +1,30 @@
+---
+title: Web demo
+description: Image classification demo running as a Flask web server.
+category: example
+layout: default
+include_in_docs: true
+---
+
+# Web Demo
+
+## Requirements
+
+The demo server requires Python with some dependencies.
+To make sure you have the dependencies, please run `pip install -r examples/web_demo/requirements.txt`, and also make sure that you've compiled the Python Caffe interface and that it is on your `PYTHONPATH` (see [installation instructions](/installation.html)).
+
+Make sure that you have obtained the Caffe Reference ImageNet Model and the ImageNet Auxiliary Data ([instructions](/getting_pretrained_models.html)).
+NOTE: if you run into trouble, try re-downloading the auxiliary files.
+
+## Run
+
+Running `python examples/web_demo/app.py` will bring up the demo server, accessible at `http://0.0.0.0:5000`.
+You can enable debug mode of the web server, or switch to a different port:
+
+    % python examples/web_demo/app.py -h
+    Usage: app.py [options]
+
+    Options:
+      -h, --help            show this help message and exit
+      -d, --debug           enable debug mode
+      -p PORT, --port=PORT  which port to serve content on
diff --git a/examples/web_demo/templates/index.html b/examples/web_demo/templates/index.html
