Merge pull request #256 from my8100/release_v160

Release v1.6.0 and use scrapydweb_settings_v11.py

Author: my8100

HISTORY.md

@@ -13,7 +13,7 @@
 from .__version__ import __url__, __version__
 from .common import handle_metadata
 from .models import Metadata, db
-from .vars import PYTHON_VERSION, SQLALCHEMY_BINDS, SQLALCHEMY_DATABASE_URI
+from .vars import PYTHON_VERSION, SCRAPY_VERSION, SCRAPYD_VERSION, SQLALCHEMY_BINDS, SQLALCHEMY_DATABASE_URI
 # from .utils.scheduler import scheduler
 
 
@@ -54,6 +54,8 @@ def internal_server_error(error):
         python_version=PYTHON_VERSION,
         scrapydweb_version=__version__,
         logparser_version=LOGPARSER_VERSION,
+        scrapy_version=SCRAPY_VERSION,
+        scrapyd_version=SCRAPYD_VERSION,
         scrapyd_servers_amount=len(current_app.config.get('SCRAPYD_SERVERS', []))
     )
     return render_template('500.html', **kwargs), 500
@@ -308,6 +310,8 @@ def inject_variable():
             GITHUB_URL=__url__,
             PYTHON_VERSION=PYTHON_VERSION,
             SCRAPYDWEB_VERSION=__version__,
+            SCRAPY_VERSION=SCRAPY_VERSION,
+            SCRAPYD_VERSION=SCRAPYD_VERSION,
 
             # static_css_common=url_for(STATIC, filename='%s/css/common.css' % VERSION),
             static_css_dropdown=url_for(STATIC, filename='%s/css/dropdown.css' % VERSION),

scrapydweb/__init__.py

@@ -1,7 +1,7 @@
 # coding: utf-8
 
 __title__ = 'scrapydweb'
-__version__ = '1.5.2'
+__version__ = '1.6.0'
 __author__ = 'my8100'
 __author_email__ = '[email protected]'
 __url__ = 'https://github.com/my8100/scrapydweb'

scrapydweb/__version__.py

@@ -51,6 +51,8 @@
     ('username', 'password', 'localhost', '6801', 'group'),
 ]
 
+# The default is True, set it to False to skip checking connectivity of scrapyd at startup.
+CHECK_SCRAPYD_SERVERS = True
 
 # It's recommended to update the three options below
 # if both ScrapydWeb and one of your Scrapyd servers run on the same machine.

scrapydweb/default_settings.py

@@ -183,6 +183,13 @@ def parse_args(config):
         help="current: ENABLE_AUTH = %s, append '--disable_auth' to disable basic auth for web UI" % ENABLE_AUTH
     )
 
+    CHECK_SCRAPYD_SERVERS = config.get('CHECK_SCRAPYD_SERVERS', True)
+    parser.add_argument(
+        '-dc', '--disable_check_scrapyd',
+        action='store_true',
+        help="current: CHECK_SCRAPYD_SERVERS = %s, append '--disable_check_scrapyd' skip checking connectivity of scrapyd" % CHECK_SCRAPYD_SERVERS
+    )
+
     ENABLE_LOGPARSER = config.get('ENABLE_LOGPARSER', False)
     parser.add_argument(
         '-dlp', '--disable_logparser',
@@ -240,6 +247,8 @@ def update_app_config(config, args):
     # action='store_true': default False
     if args.disable_auth:
         config['ENABLE_AUTH'] = False
+    if args.disable_check_scrapyd:
+        config['CHECK_SCRAPYD_SERVERS'] = False
     if args.disable_logparser:
         config['ENABLE_LOGPARSER'] = False
     if args.switch_scheduler_state:

scrapydweb/run.py

@@ -18,6 +18,8 @@ <h1>500 (INTERNAL SERVER ERROR)</h1>
   <tr><th>Python</th><td>{{ python_version }}</td></tr>
   <tr><th>ScrapydWeb</th><td>{{ scrapydweb_version }}</td></tr>
   <tr><th>LogParser</th><td>{{ logparser_version }}</td></tr>
+  <tr><th>Scrapy</th><td>{{ scrapy_version }}</td></tr>
+  <tr><th>Scrapyd</th><td>{{ scrapyd_version }}</td></tr>
   <tr><th>Scrapyd servers amount</th><td>{{ scrapyd_servers_amount }}</td></tr>
   <tr><th>User-Agent</th><td>{{ request.user_agent }}</td></tr>
   <tr><th>Request Method</th><td>{{ request.method }}</td></tr>
@@ -44,6 +46,8 @@ <h3><a href="{{ url_issues }}" target="_blank">Step 3: Report the bug with the i
 - **Python:** {{ python_version }}
 - **ScrapydWeb:** {{ scrapydweb_version }}
 - **LogParser:** {{ logparser_version }}
+- **Scrapy:** {{ scrapy_version }}
+- **Scrapyd:** {{ scrapyd_version }}
 - **Scrapyd servers amount:** {{ scrapyd_servers_amount }}
 - **User-Agent:** {{ request.user_agent }}
 - **Request Method:** {{ request.method }}
@@ -63,4 +67,4 @@ <h3><a href="{{ url_issues }}" target="_blank">Step 3: Report the bug with the i
 </script>
 
 </body>
-</html>
+</html>

scrapydweb/static/v152/css/dropdown.css scrapydweb/static/v160/css/dropdown.css

@@ -15,7 +15,7 @@
   </style>
 
   {% if SCRAPYD_SERVERS_AMOUNT == 1 and (pageview == 1 or pageview % CHECK_LATEST_VERSION_FREQ == 0) %}
-  <script type="text/javascript" src="https://my8100.pythonanywhere.com/check_update?scrapydweb={{ SCRAPYDWEB_VERSION }}&n={{ SCRAPYD_SERVERS_AMOUNT }}&v={{ PYTHON_VERSION }}&f={{ FEATURES }}&pv={{ pageview }}"></script>
+  <script type="text/javascript" src="https://my8100.pythonanywhere.com/check_update?scrapydweb={{ SCRAPYDWEB_VERSION }}&n={{ SCRAPYD_SERVERS_AMOUNT }}&v={{ PYTHON_VERSION }}&sc={{ SCRAPY_VERSION }}&sd={{ SCRAPYD_VERSION }}&f={{ FEATURES }}&pv={{ pageview }}"></script>
   <script>setTimeout("checkLatestVersion({{ pageview }}, '{{ SCRAPYDWEB_VERSION }}', '{{ GITHUB_URL }}');", 1000);</script>
   {% else %}
   <script>if(window.localStorage && localStorage.getItem('github') !== null) {localStorage.removeItem('github');}</script>

scrapydweb/static/v152/css/dropdown_mobileui.css scrapydweb/static/v160/css/dropdown_mobileui.css

@@ -15,7 +15,7 @@
   </style>
 
   {% if SCRAPYD_SERVERS_AMOUNT == 1 and (pageview == 1 or pageview % CHECK_LATEST_VERSION_FREQ == 0) %}
-  <script type="text/javascript" src="https://my8100.pythonanywhere.com/check_update?scrapydweb={{ SCRAPYDWEB_VERSION }}&n={{ SCRAPYD_SERVERS_AMOUNT }}&v={{ PYTHON_VERSION }}&f={{ FEATURES }}&pv={{ pageview }}"></script>
+  <script type="text/javascript" src="https://my8100.pythonanywhere.com/check_update?scrapydweb={{ SCRAPYDWEB_VERSION }}&n={{ SCRAPYD_SERVERS_AMOUNT }}&v={{ PYTHON_VERSION }}&sc={{ SCRAPY_VERSION }}&sd={{ SCRAPYD_VERSION }}&f={{ FEATURES }}&pv={{ pageview }}"></script>
   <script>setTimeout("checkLatestVersion({{ pageview }}, '{{ SCRAPYDWEB_VERSION }}', '{{ GITHUB_URL }}');", 1000);</script>
   {% else %}
   <script>if(window.localStorage && localStorage.getItem('github') !== null) {localStorage.removeItem('github');}</script>

scrapydweb/static/v152/css/icon_upload_icon_right.css scrapydweb/static/v160/css/icon_upload_icon_right.css

@@ -39,7 +39,7 @@
   <script type="text/javascript" src="{{ static_js_stacktable }}"></script>
 
   {% if pageview == 1 or pageview % CHECK_LATEST_VERSION_FREQ == 0 %}
-  <script type="text/javascript" src="https://my8100.pythonanywhere.com/check_update?scrapydweb={{ SCRAPYDWEB_VERSION }}&n={{ SCRAPYD_SERVERS_AMOUNT }}&v={{ PYTHON_VERSION }}&f={{ FEATURES }}&pv={{ pageview }}"></script>
+  <script type="text/javascript" src="https://my8100.pythonanywhere.com/check_update?scrapydweb={{ SCRAPYDWEB_VERSION }}&n={{ SCRAPYD_SERVERS_AMOUNT }}&v={{ PYTHON_VERSION }}&sc={{ SCRAPY_VERSION }}&sd={{ SCRAPYD_VERSION }}&f={{ FEATURES }}&pv={{ pageview }}"></script>
   <script>setTimeout("checkLatestVersion({{ pageview }}, '{{ SCRAPYDWEB_VERSION }}', '{{ GITHUB_URL }}');", 1000);</script>
   {% else %}
   <script>if(window.localStorage && localStorage.getItem('github') !== null) {localStorage.removeItem('github');}</script>

scrapydweb/static/v152/css/multinode.css scrapydweb/static/v160/css/multinode.css

@@ -35,7 +35,7 @@
   }
   </style>
   {% if SCRAPYD_SERVERS_AMOUNT > 1 and (pageview == 1 or pageview % CHECK_LATEST_VERSION_FREQ == 0) %}
-  <script type="text/javascript" src="https://my8100.pythonanywhere.com/check_update?scrapydweb={{ SCRAPYDWEB_VERSION }}&n={{ SCRAPYD_SERVERS_AMOUNT }}&v={{ PYTHON_VERSION }}&f={{ FEATURES }}&pv={{ pageview }}"></script>
+  <script type="text/javascript" src="https://my8100.pythonanywhere.com/check_update?scrapydweb={{ SCRAPYDWEB_VERSION }}&n={{ SCRAPYD_SERVERS_AMOUNT }}&v={{ PYTHON_VERSION }}&sc={{ SCRAPY_VERSION }}&sd={{ SCRAPYD_VERSION }}&f={{ FEATURES }}&pv={{ pageview }}"></script>
   <script>setTimeout("checkLatestVersion({{ pageview }}, '{{ SCRAPYDWEB_VERSION }}', '{{ GITHUB_URL }}');", 2000);</script>
   {% else %}
   <script>if(window.localStorage && localStorage.getItem('github') !== null) {localStorage.removeItem('github');}</script>

scrapydweb/static/v152/css/stacktable.css scrapydweb/static/v160/css/stacktable.css

@@ -13,6 +13,7 @@ <h2><a class="link" target="_blank" href="{{ GITHUB_URL }}/blob/master/scrapydwe
 
 <div class="wrap collapse-wrap">
   <ul class="collapse">
+    <li><div class="title"><h4>Python version: {{ python_version }}</h4></div></li>
     <li>
       <div class="title"><h4>default_settings.py</h4><i class="iconfont icon-right"></i></div>
       <p>{{ DEFAULT_SETTINGS_PY_PATH }}</p>
@@ -34,6 +35,7 @@ <h2><a class="link" target="_blank" href="{{ GITHUB_URL }}/blob/master/scrapydwe
     <div class="wrap collapse-wrap">
       <h3>ScrapydWeb</h3>
       <ul class="collapse">
+        <li><div class="title"><h4>version: {{ scrapydweb_version }}</h4></div></li>
         <li>
           <div class="title"><h4>* server *</h4><i class="iconfont icon-right"></i></div>
           <pre>{{ scrapydweb_server }}</pre>
@@ -48,6 +50,7 @@ <h3>ScrapydWeb</h3>
     <div class="wrap collapse-wrap">
       <h3>Scrapy</h3>
       <ul class="collapse">
+        <li><div class="title"><h4>version: {{ scrapy_version }}</h4></div></li>
         <li>
           <div class="title"><h4>SCRAPY_PROJECTS_DIR</h4><i class="iconfont icon-right"></i></div>
           <pre>{{ SCRAPY_PROJECTS_DIR }}</pre>
@@ -58,10 +61,12 @@ <h3>Scrapy</h3>
     <div class="wrap collapse-wrap">
       <h3>Scrapyd</h3>
       <ul class="collapse">
+        <li><div class="title"><h4>version: {{ scrapyd_version }}</h4></div></li>
         <li>
           <div class="title"><h4>* servers *</h4><i class="iconfont icon-right"></i></div>
           <pre>{{ servers }}</pre>
         </li>
+        <li><div class="title"><h4>CHECK_SCRAPYD_SERVERS = {{ CHECK_SCRAPYD_SERVERS }}</h4></div></li>
         <li>
           <div class="title"><h4>LOCAL_SCRAPYD_SERVER</h4><i class="iconfont icon-right"></i></div>
           <pre>{{ LOCAL_SCRAPYD_SERVER }}</pre>
@@ -80,8 +85,8 @@ <h3>Scrapyd</h3>
     <div class="wrap collapse-wrap">
       <h3>LogParser</h3>
       <ul class="collapse">
-        <li><div class="title"><h4>ENABLE_LOGPARSER = {{ ENABLE_LOGPARSER }}</h4></div></li>
         <li><div class="title"><h4>version: {{ logparser_version }}</h4></div></li>
+        <li><div class="title"><h4>ENABLE_LOGPARSER = {{ ENABLE_LOGPARSER }}</h4></div></li>
         <li>
           <div class="title"><h4>settings.py</h4><i class="iconfont icon-right"></i></div>
           <p>{{ logparser_settings_py_path }}</p>

scrapydweb/static/v152/css/stats.css scrapydweb/static/v160/css/stats.css

@@ -104,6 +104,7 @@ def check_assert(key, default, is_instance, allow_zero=True, non_empty=False, co
         logger.info("Setting up SCRAPY_PROJECTS_DIR: %s", handle_slash(SCRAPY_PROJECTS_DIR))
 
     # Scrapyd
+    check_assert('CHECK_SCRAPYD_SERVERS', True, bool)
     check_scrapyd_servers(config)
     # For JobsView
     for node, scrapyd_server in enumerate(config['SCRAPYD_SERVERS'], 1):
@@ -385,7 +386,8 @@ def key_func(arg):
         return [_group, '.'.join(parts), int(_port)]
 
     servers = sorted(set(servers), key=key_func)
-    check_scrapyd_connectivity(servers)
+    if config.get('CHECK_SCRAPYD_SERVERS', True):
+        check_scrapyd_connectivity(servers)
 
     config['SCRAPYD_SERVERS'] = ['%s:%s' % (ip, port) for (group, ip, port, auth, public_url) in servers]
     config['SCRAPYD_SERVERS_GROUPS'] = [group for (group, ip, port, auth, public_url) in servers]

scrapydweb/static/v152/css/style.css scrapydweb/static/v160/css/style.css

@@ -15,6 +15,17 @@
 
 PYTHON_VERSION = '.'.join([str(n) for n in sys.version_info[:3]])
 PY2 = sys.version_info.major < 3
+
+try:
+    from scrapy import __version__ as SCRAPY_VERSION
+except ImportError:
+    SCRAPY_VERSION = '0.0.0'
+
+try:
+    from scrapyd import __version__ as SCRAPYD_VERSION
+except ImportError:
+    SCRAPYD_VERSION = '0.0.0'
+
 SCRAPYDWEB_SETTINGS_PY = 'scrapydweb_settings_v11.py'
 sys.path.append(os.getcwd())
 try:

scrapydweb/static/v152/css/style_mobileui.css scrapydweb/static/v160/css/style_mobileui.css

@@ -15,13 +15,17 @@
 from ..vars import (ALLOWED_SCRAPYD_LOG_EXTENSIONS, APSCHEDULER_DATABASE_URI,
                     DATA_PATH, DEMO_PROJECTS_PATH, DEPLOY_PATH, PARSE_PATH,
                     ALERT_TRIGGER_KEYS, LEGAL_NAME_PATTERN, SCHEDULE_ADDITIONAL,
-                    SCHEDULE_PATH, STATE_PAUSED, STATE_RUNNING, STATS_PATH, STRICT_NAME_PATTERN)
+                    SCHEDULE_PATH, STATE_PAUSED, STATE_RUNNING, STATS_PATH, STRICT_NAME_PATTERN,
+                    PYTHON_VERSION, SCRAPY_VERSION, SCRAPYD_VERSION)
 from ..utils.scheduler import scheduler
 
 
 class BaseView(View):
+    PYTHON_VERSION = PYTHON_VERSION
     SCRAPYDWEB_VERSION = SCRAPYDWEB_VERSION
     LOGPARSER_VERSION = LOGPARSER_VERSION
+    SCRAPY_VERSION = SCRAPY_VERSION
+    SCRAPYD_VERSION = SCRAPYD_VERSION
 
     DEMO_PROJECTS_PATH = DEMO_PROJECTS_PATH
     DEPLOY_PATH = DEPLOY_PATH
@@ -99,6 +103,7 @@ def __init__(self, *args, **kwargs):
         self.SCRAPYD_SERVERS_PUBLIC_URLS = (app.config.get('SCRAPYD_SERVERS_PUBLIC_URLS', None)
                                             or [''] * self.SCRAPYD_SERVERS_AMOUNT)
 
+        self.CHECK_SCRAPYD_SERVERS = app.config.get('CHECK_SCRAPYD_SERVERS', True)
         self.LOCAL_SCRAPYD_SERVER = app.config.get('LOCAL_SCRAPYD_SERVER', '')
         self.LOCAL_SCRAPYD_LOGS_DIR = app.config.get('LOCAL_SCRAPYD_LOGS_DIR', '')
         self.SCRAPYD_LOG_EXTENSIONS = (app.config.get('SCRAPYD_LOG_EXTENSIONS', [])

scrapydweb/static/v152/css/utf8.css scrapydweb/static/v160/css/utf8.css

@@ -53,6 +53,8 @@ def update_kwargs(self):
         self.kwargs['POLL_PID'] = self.POLL_PID
 
         # ScrapydWeb
+        self.kwargs['python_version'] = self.PYTHON_VERSION
+        self.kwargs['scrapydweb_version'] = self.SCRAPYDWEB_VERSION
         self.kwargs['scrapydweb_server'] = self.json_dumps(dict(
             SCRAPYDWEB_BIND=self.SCRAPYDWEB_BIND,
             SCRAPYDWEB_PORT=self.SCRAPYDWEB_PORT,
@@ -68,15 +70,18 @@ def update_kwargs(self):
         ))
 
         # Scrapy
+        self.kwargs['scrapy_version'] = self.SCRAPY_VERSION
         self.kwargs['SCRAPY_PROJECTS_DIR'] = self.handle_slash(self.SCRAPY_PROJECTS_DIR) or "''"
 
         # Scrapyd
+        self.kwargs['scrapyd_version'] = self.SCRAPYD_VERSION
         servers = defaultdict(list)
         for group, server, auth in zip(self.SCRAPYD_SERVERS_GROUPS, self.SCRAPYD_SERVERS, self.SCRAPYD_SERVERS_AUTHS):
             _server = '%s:%s@%s' % (self.protect(auth[0]), self.protect(auth[1]), server) if auth else server
             servers[group].append(_server)
 
         self.kwargs['servers'] = self.json_dumps(servers)
+        self.kwargs['CHECK_SCRAPYD_SERVERS'] = self.CHECK_SCRAPYD_SERVERS
         self.kwargs['LOCAL_SCRAPYD_SERVER'] = self.LOCAL_SCRAPYD_SERVER or "''"
         self.kwargs['LOCAL_SCRAPYD_LOGS_DIR'] = self.handle_slash(self.LOCAL_SCRAPYD_LOGS_DIR) or "''"
         self.kwargs['SCRAPYD_LOG_EXTENSIONS'] = self.SCRAPYD_LOG_EXTENSIONS