Filter used to strip EXIF data from an image

.travis.yml

@@ -40,20 +40,24 @@ addons:
     - libopenblas-dev
     - liblapack-dev
     - python-pyexiv2
+    - yasm
 install:
   - pip install -U pip
-  - wget http://johnvansickle.com/ffmpeg/releases/ffmpeg-release-64bit-static.tar.xz
-    -O /tmp/ffmpeg-release.tar.xz
-  - mkdir /tmp/ffmpeg-release
-  - tar -C /tmp/ffmpeg-release --strip 1 -xvf /tmp/ffmpeg-release.tar.xz
-  - export PATH=/tmp/ffmpeg-release:$PATH
-  - ffmpeg -version
   - pip install --upgrade pip
-  - make setup
+  - cd $TRAVIS_BUILD_DIR && make setup
   - pip install coveralls
   - pip install cairosvg
+  - wget https://ffmpeg.org/releases/ffmpeg-3.2.4.tar.xz
+    -O /tmp/ffmpeg-3.2.4.tar.xz
+  - mkdir /tmp/ffmpeg-source
+  - tar -C /tmp/ffmpeg-source --strip 1 -xvf /tmp/ffmpeg-3.2.4.tar.xz
+  - cd /tmp/ffmpeg-source && ./configure --prefix=/tmp --enable-gpl --enable-libx264 --enable-libvpx
+  - cd /tmp/ffmpeg-source && make
+  - cd /tmp/ffmpeg-source && make install
+  - export PATH=/tmp/bin:$PATH
+  - ffmpeg -version
 before_script:
-  - make redis
+  - cd $TRAVIS_BUILD_DIR && make redis
 script:
   make compile_ext ci_test
 notifications:

CONTRIBUTING.md

@@ -1,11 +1,24 @@
-So you want to contribute with thumbor? Welcome onboard!
+So you want to contribute with thumbor? Awesome! Welcome aboard!
 
-There are a few things you'll need in order to properly start hacking on it.
+## Steps
+
+1. Fork and clone
+2. Install dependencies
+3. Hack, in no particular order:
+   - Write enough code
+   - Write tests for that code
+   - Check that other tests pass
+   - Repeat until you're satisfied
+4. Submit a pull request
+
+## For and clone
 
 First step is to fork it at http://help.github.com/fork-a-repo/ and create your own clone of thumbor.
 
 ## Dependencies
 
+There are a few things you'll need in order to properly start hacking on it.
+
 We seriously advise you to use virtualenv (http://pypi.python.org/pypi/virtualenv) since it will keep your environment clean of thumbor's dependencies and you can choose when to "turn them on".
 
 You'll also need python >= 2.6 and < 3.0.

debian/thumbor.conf

@@ -166,6 +166,7 @@ ALLOW_UNSAFE_URL = True
     #'thumbor.filters.equalize',
     #'thumbor.filters.fill',
     #'thumbor.filters.sharpen',
+    #'thumbor.filters.strip_exif',
     #'thumbor.filters.strip_icc',
     #'thumbor.filters.frame',
     #'thumbor.filters.grayscale',

docs/configuration.rst

@@ -617,6 +617,13 @@ Default filename for image uploaded.
 
 i.e.: ``UPLOAD_DEFAULT_FILENAME = 'image'``
 
+GC\_INTERVAL
+~~~~~~~~~~~~
+
+Set manual garbage collection interval in seconds. Defaults to None (no manual garbage collection). Try this if your Thumbor is running out of memory. May cause an increase in CPU load.
+
+i.e.: ``GC_INTERVAL=60``
+
 Example of Configuration File
 -----------------------------
 

docs/documentation_structure.txt

@@ -53,6 +53,7 @@ Index (* About)
         * Rotate
         * Round corners
         * Sharpen
+        * Strip exif
         * Strip icc
         * Watermark
     Releases

docs/filters.rst

@@ -1,6 +1,18 @@
 Filters
 =======
 
+   .. ATTENTION::
+    Filters are affecting each other in the order they are specified!
+    If the original image has the size ``60x40`` and the thumbor url would look like:
+
+    ``fit-in/100x100/filters:watermark(..):blur(..):fill(red,1):upscale()/http://some/image.jpeg``
+
+    then the image will first checked if it fits into ``100x100`` (which it does), then
+    gets the watermark, then it will be blurred (including the watermark), then
+    filled with red so that it will use the ``100x100`` size and at the end upscaled will
+    be applied without having any effect because fill was changing the size of the image
+    already to the maximum available space.
+
 .. toctree::
    :maxdepth: 1
 
@@ -24,5 +36,7 @@ Filters
    rotate
    round_corners
    sharpen
+   strip_exif
    strip_icc
+   upscale
    watermark

docs/gifv.rst

@@ -0,0 +1,53 @@
+gifv
+==========
+
+The gifv optimizer is able to convert gifs to mp4 or webm videos, often resulting in dramatically smaller sized files.
+
+**Gifv is categorized as experimental and should be used with caution.** It uses ffmpeg to convert gifs to videos and so it's sensitive to changes with ffmpeg. It's recommended to lock your ffmpeg version with a fixed version (chef, docker, etc) and if updating make sure to check that the update doesn't break gifv. **FFmpeg version 3.2.4 is the current recommended version.** Later version, such as 3.3 will break the proper conversion of gif delays to frame durations in videos ... meaning videos will not be the same length as equivelant gifs.
+
+To enable gifv, ensure ffmpeg is in PATH and enable the optimizer in your config:
+
+.. code-block:: python
+
+  OPTIMIZERS = [
+    'thumbor.optimizers.gifv',
+  ]
+
+Once activated, you must add the ``gifv()`` option to your filters list. An example request might look like this:
+
+.. code-block:: text
+
+  http://localhost:8888/unsafe/filters:gifv()/http://localhost/livingroom.gif
+
+The above example will default to using the mp4 video container with h264 video. You can also be explicit:
+
+.. code-block:: text
+
+  http://localhost:8888/unsafe/filters:gifv(mp4)/http://localhost/livingroom.gif
+
+or use explicitly specify webm
+
+.. code-block:: text
+
+  http://localhost:8888/unsafe/filters:gifv(webm)/http://localhost/livingroom.gif
+
+
+Because videos (in mp4 or webm format) cannot contain alpha transparency a background color will be automatically added. The default color is white. You can also specify a background color:
+
+.. code-block:: text
+
+  http://localhost:8888/unsafe/filters:gifv():background_color(ff00ff)/http://localhost/livingroom.gif
+
+.. code-block:: text
+
+  http://localhost:8888/unsafe/filters:gifv():background_color(f0f)/http://localhost/livingroom.gif
+
+
+.. code-block:: text
+
+  http://localhost:8888/unsafe/filters:gifv():background_color(magenta)/http://localhost/livingroom.gif
+
+
+The color must be specified in 6 character hex, 3 character hex or color name. But 6 or 3 character hex are the preferred formats. Including a ``#`` symbol in your color will break the url if not url encoded and thumbor will error on the request. The recommendation is to not use them at all which also makes urls shorter. But if you must a leading `%23` will probably work.
+
+

docs/imaging.rst

@@ -15,3 +15,4 @@ Imaging
    lazy_detection
    posting_putting_deleting
    result_storage
+   optimizers

docs/jpegtran.rst

@@ -0,0 +1,25 @@
+jpegtran
+==========
+
+Jpegtran is a lossless jpeg optimizer which can make your jpegs smaller by optimizing DCT coefficients. Information on jpegtran can be a bit difficult to find but the linux man page is pretty good: https://linux.die.net/man/1/jpegtran
+
+Jpegtran can be used in conjunction with Thumbor. If the optimizer has been activated, Thumbor will first process your jpeg normally then it will hand the jpeg off to jpegtran for further optimizations before Thumbor returns the final image.
+
+To use jpegtran with Thumbor you must first install jpegtran, various linux distros often provide a package by the same name or it can be installed from source. You should make sure that jpegtran is in PATH, do a `which jpegtran` and you should see an absolute path where the jpegtran resides.
+
+You also need to enable the Thumbor jpegtran optimizer in your thumbor.conf, like so:
+
+.. code-block:: python
+
+  OPTIMIZERS = [
+    'thumbor.optimizers.jpegtran'
+  ]
+
+
+You can also manually specify the jpegtran path, like this:
+
+.. code-block:: python
+
+  JPEGTRAN_PATH=/usr/local/bin/jpegtran
+
+Once activated, no extra url parameters are needed - jpegtran will run on all jpegs automatically. If you have opted to use progressive jpegs via the ``PROGRESSIVE_JPEG`` option, jpegtran will also honor and product progressive jpegs.

docs/optimizers.rst

@@ -0,0 +1,8 @@
+Optimizers
+==========
+
+.. toctree::
+   :maxdepth: 1
+
+   jpegtran
+   gifv

docs/strip_exif.rst

@@ -0,0 +1,22 @@
+Strip EXIF
+=========
+
+Usage: strip\_exif()
+
+Description
+-----------
+
+This filter removes any Exif information in the resulting image.
+This is useful if you have set the configuration PRESERVE_EXIF_INFO = True but still wish to overwrite this behavior in some cases 
+(e.g. for image icons)
+
+
+Arguments
+---------
+
+No arguments
+
+Example
+-------
+
+`<http://localhost:8888/unsafe/filters:strip\_exif()/http://www.arte.tv/static-epgapi/057460-011-A.jpg>`_

docs/upscale.rst

@@ -0,0 +1,23 @@
+Upscale
+=======
+
+Usage: upscale()
+
+Description
+-----------
+
+This filter tells thumbor to upscale your images. This only makes sense with
+"fit-in" or "adaptive-fit-in".
+
+This means that if an original image is 300px width by 200px height and you
+ask for a 600x500 image, the filter will resize it to 600x400.
+
+Arguments
+---------
+
+No arguments allowed.
+
+Example
+-------
+
+`<http://localhost:8888/unsafe/fit-in/600x500/filters:upscale()/https://github.com/thumbor/thumbor/wiki/dice_transparent_background.png>`_

integration_tests/urls_helpers.py

@@ -75,6 +75,7 @@
     'filters:round_corner(20,255,255,100)',
     'filters:sharpen(6,2.5,false)',
     'filters:sharpen(6,2.5,true)',
+    'filters:strip_exif()',
     'filters:strip_icc()',
     'filters:watermark(rgba-interlaced.png,10,10,50)',
     'filters:watermark(rgba-interlaced.png,center,center,50)',
@@ -94,6 +95,7 @@
     'filters:gifv(webm)',
     'filters:gifv(mp4)',
     'filters:max_age(600)',
+    'filters:upscale()',
 
     # one big filter 4-line string
     'filters:curve([(0,0),(255,255)],[(0,50),(16,51),(32,69),(58,85),(92,120),(128,170),(140,186),(167,225),'

setup.py

@@ -97,14 +97,16 @@ def run_setup(extension_modules=[]):
             "tornado>=4.1.0,<5.0.0",
             "pyCrypto>=2.1.0",
             "pycurl>=7.19.0,<7.44.0",
-            "Pillow>=3.0.0,<4.0.0",
+            "Pillow>=3.0.0,<5.0.0",
             "derpconf>=0.2.0",
             "pexif>=0.15,<1.0",
             "statsd>=3.0.1",
             "libthumbor>=1.3.2",
             "futures",
             "argparse",
             "pytz",
+            "schedule",
+            "webcolors",
         ],
 
         extras_require={

tests/base.py

@@ -159,8 +159,8 @@ def get_request_handler(self):
         return None
 
     def get_context(self):
-        self.server = self.get_server()
         self.config = self.get_config()
+        self.server = self.get_server()
         self.importer = self.get_importer()
         self.request_handler = self.get_request_handler()
         return Context(

tests/engines/test_gif.py

@@ -15,7 +15,7 @@
 
 from thumbor.context import ServerParameters, RequestParameters
 from thumbor.config import Config
-from thumbor.engines.gif import Engine, GifSicleError
+from thumbor.engines.gif import Engine
 from thumbor.utils import which
 from thumbor.importer import Importer
 
@@ -64,21 +64,14 @@ def test_load_image(self):
         image = engine.create_image(buffer)
         expect(image.format).to_equal('GIF')
 
-    def test_errors_on_gifsicle_should_raises_errors(self):
+    def test_errors_on_gifsicle_should_not_raises_errors_when_output(self):
         engine = Engine(self.context)
-        with open(join(STORAGE_PATH, 'animated.gif'), 'r') as im:
+        with open(join(STORAGE_PATH, 'SmallFullColourGIF.gif'), 'r') as im:
             buffer = im.read()
 
         engine.load(buffer, '.gif')
-        with expect.error_to_happen(
-            GifSicleError,
-            message='gifsicle command returned errorlevel 1 for command'
-                    ' "{0} --some-invalid-opt /foo/'
-                    'bar.gif" (image maybe corrupted?)'.format(
-                        self.server.gifsicle_path
-                    )
-        ):
-            engine.run_gifsicle('--some-invalid-opt')
+        result = engine.run_gifsicle('--some-invalid-opt')
+        expect(result).Not.to_be_null()
 
     def test_is_multiple_should_returns_true_if_gif_has_many_frames(self):
         engine = Engine(self.context)