Residual

README.rst

@@ -41,11 +41,11 @@ The algorithm is built for style transfer, but can also generate image analogies
 
     # Synthesize a coastline as if painted by Monet. This uses "*_sem.png" masks for both images.
     python3 doodle.py --style samples/Monet.jpg --output samples/Coastline.png \
-                      --device=cpu --iterations=40
+                      --variety 0.0 0.2 --layers 4_1 3_1 --iterations 4 6
 
     # Generate a scene around a lake in the style of a Renoir painting.
     python3 doodle.py --style samples/Renoir.jpg --output samples/Landscape.png \
-                      --device=gpu0 --iterations=80
+                      --variety 0.5 --layers 6_1 5_1 4_1 3_1 --iterations 4
 
 Notice the Renoir results look a little better than the Monet. Some rotational variations of the source image could improve the quality of the arch outline in particular.
 
@@ -58,12 +58,12 @@ If you want to transfer the style given a source style with annotations, and a t
 .. code:: bash
 
     # Synthesize a portrait of Seth Johnson like a Gogh portrait. This uses "*_sem.png" masks for both images.
-    python3 doodle.py --style samples/Gogh.jpg --content samples/Seth.png \
-                      --output SethAsGogh.png --device=cpu --phases=4 --iterations=40
+    python3 doodle.py --content samples/Seth.jpg --style samples/Gogh.jpg \
+                        --variety 0.2 0.1 --balance 0.85 1.0 --layers 4_1 3_1 --iterations 6
 
     # Generate what a photo of Vincent van Gogh would look like, using Seth's portrait as reference.
-    python3 doodle.py --style samples/Seth.jpg --content samples/Gogh.png \
-                      --output GoghAsSeth.png --device=gpu0 --phases=4 --iterations=80
+    python3 doodle.py --content samples/Gogh.jpg --style samples/Seth.jpg \
+                      --variety 0.0 --balance 0.7 0.8 --layers 4_1 3_1 --iterations 4
 
 To perform regular style transfer without semantic annotations, simply delete or rename the files with the semantic maps.  The photo is originally by `Seth Johnson <http://sethjohnson.tumblr.com/post/655063019/this-was-a-project-for-an-art-history-class-turns>`_, and the concept for this style transfer by `Kyle McDonald <https://twitter.com/kcimc>`_.
 
@@ -77,13 +77,9 @@ For synthesizing bitmap textures, you only need an input style without anotation
 
 .. code:: bash
 
-    # First synthesis uses a darker noise pattern as seed.
-    python3 doodle.py --style samples/Wall.jpg --output Wall.png\
-                      --seed=noise --seed-range=0:128 --iterations=50 --phases=3
-
-    # Second synthesis uses a lighter noise pattern as seed.
-    python3 doodle.py --style samples/Wall.jpg --output Wall.png\
-                      --seed=noise --seed-range=192:255 --iterations=50 --phases=3
+    # Generate an image of stones based on the input photograph only.
+    python3 doodle.py --style samples/Stones.jpg --output Stones.png \
+                      --layers 5_1 4_1 3_1 --iterations 6 4 4 --variety 0.4 0.2 0.1
 
 You can also control the output resolution using ``--output-size=512x512`` parameter—which also depends on the memory you have available. By default the size will be the same as the style image.
 

docker-cpu.df

@@ -36,8 +36,8 @@ RUN python3 -m pip install -r "requirements.txt"
 # Copy only required project files
 COPY doodle.py .
 
-# Get a pre-trained neural network (VGG19)
-RUN wget -q "https://github.com/alexjc/neural-doodle/releases/download/v0.0/vgg19_conv.pkl.bz2"
+# Get a pre-trained neural network, non-commercial & attribution. (GELU2)
+RUN wget -q "https://github.com/alexjc/neural-doodle/releases/download/v0.0/gelu2_conv.pkl"
 
 # Set an entrypoint to the main doodle.py script
 ENTRYPOINT ["python3", "doodle.py", "--device=cpu"]

docker-gpu.df

@@ -39,8 +39,8 @@ RUN python3 -m pip -q install -r "requirements.txt"
 # Copy only required project files
 COPY doodle.py .
 
-# Get a pre-trained neural network (VGG19)
-RUN wget -q "https://github.com/alexjc/neural-doodle/releases/download/v0.0/vgg19_conv.pkl.bz2"
+# Get a pre-trained neural network, non-commercial & attribution. (GELU2)
+RUN wget -q "https://github.com/alexjc/neural-doodle/releases/download/v0.0/gelu2_conv.pkl"
 
 # Set an entrypoint to the main doodle.py script
 ENTRYPOINT ["python3", "doodle.py", "--device=gpu"]

requirements.txt

@@ -1,4 +1,5 @@
 colorama
 pillow>=3.2.0
 Theano>=0.8.1
-git+https://github.com/Lasagne/Lasagne.git@0440814#egg=Lasagne==0.2-dev
+git+https://github.com/Lasagne/Lasagne.git@31ac7d2#egg=Lasagne==0.2-dev
+sklearn>=0.17.1

tools/visualize.py

@@ -0,0 +1,77 @@
+import matplotlib.pyplot as plt
+import numpy as np
+
+import doodle
+
+generator = doodle.NeuralGenerator()
+generator.prepare_network()
+
+def calculate_patch_coordinates(l, j, i):
+    ys, xs, ye, xe = j, i, j, i
+    while hasattr(l, 'filter_size'):
+        after = l.filter_size[0]//2
+        before = l.filter_size[0] - 1 - after
+        ys -= before
+        xs -= before
+        ye += after
+        xe += after
+        ys *= l.stride[0]
+        xs *= l.stride[0]
+        ye *= l.stride[0]
+        xe *= l.stride[0]
+        l = l.input_layer
+    return ys, xs, ye, xe
+
+import glob
+import collections
+
+
+candidates = collections.defaultdict(list)
+for content in glob.glob(doodle.args.content):
+    image, mask = generator.load_images('content', content, scale=1.0)
+
+    feature = generator.model.prepare_image(image)
+    for layer, encoder in reversed(list(zip(doodle.args.layers, generator.encoders))):
+        feature = encoder(feature, mask)
+
+        x = feature.reshape(feature.shape[:2]+(-1,))[:,:-3,:]
+        # x = (x - x.mean(axis=(0,2), keepdims=True)) # / x.std(axis=(0,2), keepdims=True)
+        covariance = np.tensordot(x, x, axes=([2], [2])).mean(axis=(0,2)) / x.shape[2]
+        np.fill_diagonal(covariance, 0.0)
+        # print(covariance.shape, covariance.min(), covariance.max())
+
+        # subplot.imshow(covariance, interpolation='nearest')
+
+        for i in range(feature.shape[1]):
+            w = feature[:,i:i+1,:,:]
+            for idx in np.argsort(w.flatten())[-15:]:
+                _, _, y, x = np.unravel_index(idx, w.shape)
+                # print('coords', y, x, 'value', )
+                a, b, c, d = calculate_patch_coordinates(generator.model.network['enc%i_1'%layer], y, x)
+                img = np.copy(image[max(0,a):min(image.shape[0],c), max(0, b):min(image.shape[1],d)])
+                candidates[i].append((img, w.flatten()[idx])) 
+
+        # _, _, y, x = np.unravel_index(feature[0,0,:,:].argmax(), feature.shape)
+        # print(y, x, calculate_patch_coordinates('enc%i_1'%layer, y, x))
+
+        # subplot.set_title('Layer {}'.format(layer))
+
+        # subplot.violinplot([feature[:,i,:,:].flatten() for i in range(feature.shape[1])], showmeans=False, showmedians=True)
+
+        # x = np.arange(0, feature.shape[1], 1)
+        # y = [feature.min(axis=(0,2,3)), feature.mean(axis=(0,2,3)), feature.max(axis=(0,2,3))]
+        # for j in y:
+        #     plt.errorbar(x, j)
+
+fig, axes = plt.subplots(3, 5, figsize=(10, 6), subplot_kw={'xticks': [], 'yticks': []})
+fig.subplots_adjust(hspace=0.3, wspace=0.05)
+# if not hasattr(axes, 'flat'): axes.flat = [plt]
+
+for i, c in candidates.items():
+    c.sort(key=lambda x: x[1])
+    for (img, _), subplot in zip(c[-15:], axes.flat):
+        subplot.imshow(img, interpolation='nearest')
+    plt.savefig('channel_{}.png'.format(i))
+
+# plt.show()
+# print(i, c[0][1], c[-1][1])