Crawlera → Zyte Smart Proxy Manager (#97)

Author: geronsv

.bumpversion.cfg

@@ -6,4 +6,4 @@ tag_name = v{new_version}
 
 [bumpversion:file:setup.py]
 
-[bumpversion:file:scrapy_crawlera/__init__.py]
+[bumpversion:file:scrapy_zyte_smartproxy/__init__.py]

README.rst

@@ -1,20 +1,21 @@
-===============
-scrapy-crawlera
-===============
+======================
+scrapy-zyte-smartproxy
+======================
 
-.. image:: https://img.shields.io/pypi/v/scrapy-crawlera.svg
-   :target: https://pypi.python.org/pypi/scrapy-crawlera
+.. image:: https://img.shields.io/pypi/v/scrapy-zyte-smartproxy.svg
+   :target: https://pypi.python.org/pypi/scrapy-zyte-smartproxy
    :alt: PyPI Version
 
-.. image:: https://travis-ci.org/scrapy-plugins/scrapy-crawlera.svg?branch=master
-   :target: http://travis-ci.org/scrapy-plugins/scrapy-crawlera
+.. image:: https://travis-ci.org/scrapy-plugins/scrapy-zyte-smartproxy.svg?branch=master
+   :target: http://travis-ci.org/scrapy-plugins/scrapy-zyte-smartproxy
    :alt: Build Status
 
-.. image:: http://codecov.io/github/scrapy-plugins/scrapy-crawlera/coverage.svg?branch=master
-   :target: http://codecov.io/github/scrapy-plugins/scrapy-crawlera?branch=master
+.. image:: http://codecov.io/github/scrapy-plugins/scrapy-zyte-smartproxy/coverage.svg?branch=master
+   :target: http://codecov.io/github/scrapy-plugins/scrapy-zyte-smartproxy?branch=master
    :alt: Code Coverage
 
-scrapy-crawlera provides easy use of `Crawlera <http://scrapinghub.com/crawlera>`_ with Scrapy.
+scrapy-zyte-smartproxy provides easy use of `Zyte Smart Proxy Manager
+<https://www.zyte.com/smart-proxy-manager/>`_ (formerly Crawlera) with Scrapy.
 
 Requirements
 ============
@@ -25,12 +26,13 @@ Requirements
 Installation
 ============
 
-You can install scrapy-crawlera using pip::
+You can install scrapy-zyte-smartproxy using pip::
 
-    pip install scrapy-crawlera
+    pip install scrapy-zyte-smartproxy
 
 
 Documentation
 =============
 
-Documentation is available online at https://scrapy-crawlera.readthedocs.io/ and in the ``docs`` directory.
+Documentation is available online at
+https://scrapy-zyte-smartproxy.readthedocs.io/ and in the ``docs`` directory.

docs/Makefile

@@ -4,7 +4,7 @@
 # You can set these variables from the command line.
 SPHINXOPTS    =
 SPHINXBUILD   = sphinx-build
-SPHINXPROJ    = scrapy-crawlera
+SPHINXPROJ    = scrapy-zyte-smartproxy
 SOURCEDIR     = .
 BUILDDIR      = _build
 

docs/conf.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# scrapy-crawlera documentation build configuration file, created by
+# scrapy-zyte-smartproxy documentation build configuration file, created by
 # sphinx-quickstart on Sat Jan 21 13:17:41 2017.
 #
 # This file is execfile()d with the current directory set to its
@@ -54,19 +54,19 @@
 master_doc = 'index'
 
 # General information about the project.
-project = u'scrapy-crawlera'
-copyright = u'2011-2017, Scrapinghub'
-author = u'Scrapinghub'
+project = u'scrapy-zyte-smartproxy'
+copyright = u'2011-2021, Zyte Group Ltd'
+author = u'Zyte'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
 
 try:
-    import scrapy_crawlera
-    version = '.'.join(scrapy_crawlera.__version__.split('.')[:2])
-    release = scrapy_crawlera.__version__
+    import scrapy_zyte_smartproxy
+    version = '.'.join(scrapy_zyte_smartproxy.__version__.split('.')[:2])
+    release = scrapy_zyte_smartproxy.__version__
 except ImportError:
     version = ''
     release = ''
@@ -111,7 +111,7 @@
 # -- Options for HTMLHelp output ------------------------------------------
 
 # Output file base name for HTML help builder.
-htmlhelp_basename = 'scrapy-crawleradoc'
+htmlhelp_basename = 'scrapy-zyte-smartproxydoc'
 
 
 # -- Options for LaTeX output ---------------------------------------------
@@ -138,8 +138,13 @@
 # (source start file, target name, title,
 #  author, documentclass [howto, manual, or own class]).
 latex_documents = [
-    (master_doc, 'scrapy-crawlera.tex', u'scrapy-crawlera Documentation',
-     u'Scrapinghub', 'manual'),
+    (
+        master_doc,
+        'scrapy-zyte-smartproxy.tex',
+        u'scrapy-zyte-smartproxy Documentation',
+        u'Zyte',
+        'manual',
+    ),
 ]
 
 
@@ -148,7 +153,7 @@
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
 man_pages = [
-    (master_doc, 'scrapy-crawlera', u'scrapy-crawlera Documentation',
+    (master_doc, 'scrapy-zyte-smartproxy', u'scrapy-zyte-smartproxy Documentation',
      [author], 1)
 ]
 
@@ -159,8 +164,8 @@
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-    (master_doc, 'scrapy-crawlera', u'scrapy-crawlera Documentation',
-     author, 'scrapy-crawlera', 'One line description of project.',
+    (master_doc, 'scrapy-zyte-smartproxy', u'scrapy-zyte-smartproxy Documentation',
+     author, 'scrapy-zyte-smartproxy', 'One line description of project.',
      'Miscellaneous'),
 ]
 

docs/index.rst

@@ -1,9 +1,12 @@
-=======================================
-scrapy-crawlera |version| documentation
-=======================================
+==============================================
+scrapy-zyte-smartproxy |version| documentation
+==============================================
 
-scrapy-crawlera is a Scrapy `Downloader Middleware <https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#downloader-middleware>`_
-to interact with `Crawlera <http://scrapinghub.com/crawlera>`_ automatically.
+scrapy-zyte-smartproxy is a `Scrapy downloader middleware`_ to interact with
+`Zyte Smart Proxy Manager`_ (formerly Crawlera) automatically.
+
+.. _Scrapy downloader middleware: https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
+.. _Zyte Smart Proxy Manager: https://www.zyte.com/smart-proxy-manager/
 
 Configuration
 =============
@@ -12,32 +15,32 @@ Configuration
    :caption: Configuration
 
 
-* Add the Crawlera middleware including it into the ``DOWNLOADER_MIDDLEWARES`` in your ``settings.py`` file::
+* Add the Zyte Smart Proxy Manager middleware including it into the ``DOWNLOADER_MIDDLEWARES`` in your ``settings.py`` file::
 
     DOWNLOADER_MIDDLEWARES = {
         ...
-        'scrapy_crawlera.CrawleraMiddleware': 610
+        'scrapy_zyte_smartproxy.ZyteSmartProxyMiddleware': 610
     }
 
 * Then there are two ways to enable it
 
   * Through ``settings.py``::
 
-      CRAWLERA_ENABLED = True
-      CRAWLERA_APIKEY = 'apikey'
+      ZYTE_SMARTPROXY_ENABLED = True
+      ZYTE_SMARTPROXY_APIKEY = 'apikey'
 
   * Through spider attributes::
 
       class MySpider:
-          crawlera_enabled = True
-          crawlera_apikey = 'apikey'
+          zyte_smartproxy_enabled = True
+          zyte_smartproxy_apikey = 'apikey'
 
 
-* (optional) If you are not using the default Crawlera proxy (``http://proxy.crawlera.com:8010``),
+* (optional) If you are not using the default Zyte Smart Proxy Manager proxy (``http://proxy.zyte.com:8011``),
   for example if you have a dedicated or private instance,
-  make sure to also set ``CRAWLERA_URL`` in ``settings.py``, e.g.::
+  make sure to also set ``ZYTE_SMARTPROXY_URL`` in ``settings.py``, e.g.::
 
-    CRAWLERA_URL = 'http://myinstance.crawlera.com:8010'
+    ZYTE_SMARTPROXY_URL = 'http://myinstance.zyte.com:8011'
 
 How to use it
 =============
@@ -52,8 +55,8 @@ How to use it
     All configurable Scrapy Settings added by the Middleware.
 
 
-With the middleware, the usage of crawlera is automatic, every request will go through crawlera without nothing to worry about.
-If you want to *disable* crawlera on a specific Request, you can do so by updating `meta` with `dont_proxy=True`::
+With the middleware, the usage of Zyte Smart Proxy Manager is automatic, every request will go through Zyte Smart Proxy Manager without nothing to worry about.
+If you want to *disable* Zyte Smart Proxy Manager on a specific Request, you can do so by updating `meta` with `dont_proxy=True`::
 
 
     scrapy.Request(
@@ -65,11 +68,11 @@ If you want to *disable* crawlera on a specific Request, you can do so by updati
     )
 
 
-Remember that you are now making requests to Crawlera, and the Crawlera service will be the one actually making the requests to the different sites.
+Remember that you are now making requests to Zyte Smart Proxy Manager, and the Zyte Smart Proxy Manager service will be the one actually making the requests to the different sites.
 
-If you need to specify special `Crawlera Headers <https://doc.scrapinghub.com/crawlera.html#request-headers>`_, just apply them as normal `Scrapy Headers <https://doc.scrapy.org/en/latest/topics/request-response.html#scrapy.http.Request.headers>`_.
+If you need to specify special `Zyte Smart Proxy Manager headers <https://docs.zyte.com/smart-proxy-manager.html#request-headers>`_, just apply them as normal `Scrapy headers <https://doc.scrapy.org/en/latest/topics/request-response.html#scrapy.http.Request.headers>`_.
 
-Here we have an example of specifying a Crawlera header into a Scrapy request::
+Here we have an example of specifying a Zyte Smart Proxy Manager header into a Scrapy request::
 
     scrapy.Request(
         'http://example.com',
@@ -82,8 +85,8 @@ Here we have an example of specifying a Crawlera header into a Scrapy request::
 Remember that you could also set which headers to use by default by all
 requests with `DEFAULT_REQUEST_HEADERS <http://doc.scrapy.org/en/1.0/topics/settings.html#default-request-headers>`_
 
-.. note:: Crawlera headers are removed from requests when the middleware is activated but Crawlera
-    is disabled. For example, if you accidentally disable Crawlera via ``crawlera_enabled = False``
+.. note:: Zyte Smart Proxy Manager headers are removed from requests when the middleware is activated but Zyte Smart Proxy Manager
+    is disabled. For example, if you accidentally disable Zyte Smart Proxy Manager via ``zyte_smartproxy_enabled = False``
     but keep sending ``X-Crawlera-*`` headers in your requests, those will be removed from the
     request headers.
 
@@ -99,4 +102,4 @@ All the rest
    news
 
 :doc:`news`
-    See what has changed in recent scrapy-crawlera versions.
+    See what has changed in recent scrapy-zyte-smartproxy versions.

docs/news.rst

@@ -3,6 +3,35 @@
 Changes
 =======
 
+v2.0.0 (2021-05-NN)
+-------------------
+
+Following the upstream rebranding of Crawlera as Zyte Smart Proxy Manager,
+``scrapy-crawlera`` has been renamed as ``scrapy-zyte-smartproxy``, with the
+following backward-incompatible changes:
+
+-   The repository name and Python Package Index (PyPI) name are now
+    ``scrapy-zyte-smartproxy``.
+
+-   Setting prefixes have switched from ``CRAWLERA_`` to ``ZYTE_SMARTPROXY_``.
+
+-   Spider attribute prefixes and request meta key prefixes have switched from
+    ``crawlera_`` to ``zyte_smartproxy_``.
+
+-   ``scrapy_crawlera`` is now ``scrapy_zyte_smartproxy``.
+
+-   ``CrawleraMiddleware`` is now ``ZyteSmartProxyMiddleware``, and its default
+    ``url`` is now ``http://proxy.zyte.com:8011``.
+
+-   Stat prefixes have switched from ``crawlera/`` to ``zyte_smartproxy/``.
+
+-   The online documentation is moving to
+    https://scrapy-zyte-smartproxy.readthedocs.io/
+
+.. note:: Zyte Smart Proxy Manager headers continue to use the ``X-Crawlera-``
+          prefix.
+
+
 v1.7.2 (2020-12-01)
 -------------------
 - Use request.meta than response.meta in the middleware

docs/settings.rst

@@ -2,76 +2,77 @@
 Settings
 ========
 
-This Middleware adds some settings to configure how to work with Crawlera.
+This Scrapy downloader middleware adds some settings to configure how to work
+with Zyte Smart Proxy Manager.
 
-CRAWLERA_APIKEY
----------------
+ZYTE_SMARTPROXY_APIKEY
+----------------------
 
 Default: ``None``
 
-Unique Crawlera API Key provided for authentication.
+Unique Zyte Smart Proxy Manager API key provided for authentication.
 
-CRAWLERA_URL
-------------
+ZYTE_SMARTPROXY_URL
+-------------------
 
-Default: ``'http://proxy.crawlera.com:8010'``
+Default: ``'http://proxy.zyte.com:8011'``
 
-Crawlera instance url, it varies depending on adquiring a private or dedicated instance. If Crawlera didn't provide
-you with a private instance url, you don't need to specify it.
+Zyte Smart Proxy Manager instance URL, it varies depending on adquiring a private or dedicated instance. If Zyte Smart Proxy Manager didn't provide
+you with a private instance URL, you don't need to specify it.
 
-CRAWLERA_MAXBANS
-----------------
+ZYTE_SMARTPROXY_MAXBANS
+-----------------------
 
 Default: ``400``
 
-Number of consecutive bans from Crawlera necessary to stop the spider.
+Number of consecutive bans from Zyte Smart Proxy Manager necessary to stop the spider.
 
-CRAWLERA_DOWNLOAD_TIMEOUT
--------------------------
+ZYTE_SMARTPROXY_DOWNLOAD_TIMEOUT
+--------------------------------
 
 Default: ``190``
 
-Timeout for processing Crawlera requests. It overrides Scrapy's ``DOWNLOAD_TIMEOUT``.
+Timeout for processing Zyte Smart Proxy Manager requests. It overrides Scrapy's ``DOWNLOAD_TIMEOUT``.
 
-CRAWLERA_PRESERVE_DELAY
------------------------
+ZYTE_SMARTPROXY_PRESERVE_DELAY
+------------------------------
 
 Default: ``False``
 
 If ``False`` Sets Scrapy's ``DOWNLOAD_DELAY`` to ``0``, making the spider to crawl faster. If set to ``True``, it will
 respect the provided ``DOWNLOAD_DELAY`` from Scrapy.
 
-CRAWLERA_DEFAULT_HEADERS
-------------------------
+ZYTE_SMARTPROXY_DEFAULT_HEADERS
+-------------------------------
 
 Default: ``{}``
 
-Default headers added only to crawlera requests. Headers defined on ``DEFAULT_REQUEST_HEADERS`` will take precedence as long as the ``CrawleraMiddleware`` is placed after the ``DefaultHeadersMiddleware``. Headers set on the requests have precedence over the two settings.
+Default headers added only to Zyte Smart Proxy Manager requests. Headers defined on ``DEFAULT_REQUEST_HEADERS`` will take precedence as long as the ``ZyteSmartProxyMiddleware`` is placed after the ``DefaultHeadersMiddleware``. Headers set on the requests have precedence over the two settings.
 
-* This is the default behavior, ``DefaultHeadersMiddleware`` default priority is ``400`` and we recommend ``CrawleraMiddleware`` priority to be ``610``
+* This is the default behavior, ``DefaultHeadersMiddleware`` default priority is ``400`` and we recommend ``ZyteSmartProxyMiddleware`` priority to be ``610``
 
-CRAWLERA_BACKOFF_STEP
------------------------
+ZYTE_SMARTPROXY_BACKOFF_STEP
+----------------------------
 
 Default: ``15``
 
 Step size used for calculating exponential backoff according to the formula: ``random.uniform(0, min(max, step * 2 ** attempt))``.
 
-CRAWLERA_BACKOFF_MAX
------------------------
+ZYTE_SMARTPROXY_BACKOFF_MAX
+---------------------------
 
 Default: ``180``
 
 Max value for exponential backoff as showed in the formula above.
 
-CRAWLERA_FORCE_ENABLE_ON_HTTP_CODES
-------------------------------------
+ZYTE_SMARTPROXY_FORCE_ENABLE_ON_HTTP_CODES
+------------------------------------------
 
 Default: ``[]``
 
-List of HTTP response status codes that warrant enabling Crawlera for the
+List of HTTP response status codes that warrant enabling Zyte Smart Proxy Manager for the
 corresponding domain.
 
 When a response with one of these HTTP status codes is received after a request
-that did not go through Crawlera, the request is retried with Crawlera, and any
-new request to the same domain is also sent through Crawlera.
+that did not go through Zyte Smart Proxy Manager, the request is retried with Zyte Smart Proxy Manager, and any
+new request to the same domain is also sent through Zyte Smart Proxy Manager.

scrapy_crawlera/__init__.py

@@ -0,0 +1,4 @@
+from .middleware import ZyteSmartProxyMiddleware
+
+
+__version__ = '1.7.2'