Merge pull request #13 from scrapy-plugins/draft-support

Added support for additional schemas

Author: Víctor Ruiz

dev-requiremens.txt

@@ -0,0 +1 @@
+pytest

scrapy_jsonschema/draft.py

@@ -0,0 +1,4 @@
+JSON_SCHEMA_DRAFT_3 = "http://json-schema.org/draft-03/schema#"
+JSON_SCHEMA_DRAFT_4 = "http://json-schema.org/draft-04/schema#"
+JSON_SCHEMA_DRAFT_6 = "http://json-schema.org/draft-06/schema#"
+JSON_SCHEMA_DRAFT_7 = "http://json-schema.org/draft-07/schema#"

scrapy_jsonschema/item.py

@@ -1,15 +1,27 @@
 from abc import ABCMeta
 
 import six
-from jsonschema import Draft4Validator
+from jsonschema import (
+    Draft3Validator,
+    Draft4Validator,
+    Draft6Validator,
+    Draft7Validator,
+)
+
+from scrapy_jsonschema.draft import (
+    JSON_SCHEMA_DRAFT_3,
+    JSON_SCHEMA_DRAFT_4,
+    JSON_SCHEMA_DRAFT_6,
+    JSON_SCHEMA_DRAFT_7,
+)
+
 from scrapy.item import DictItem, Field
 
 
 def _merge_schema(base, new):
-    if base is None:
-        return new
-    if new is None:
-        return base
+    if base is None or new is None:
+        return base or new
+
     if all(isinstance(x, dict) for x in (base, new)):
         return {
             key: _merge_schema(base.get(key), new.get(key))
@@ -21,6 +33,14 @@ def _merge_schema(base, new):
 
 
 class JsonSchemaMeta(ABCMeta):
+
+    draft_to_validator = {
+        JSON_SCHEMA_DRAFT_3: Draft3Validator,
+        JSON_SCHEMA_DRAFT_4: Draft4Validator,
+        JSON_SCHEMA_DRAFT_6: Draft6Validator,
+        JSON_SCHEMA_DRAFT_7: Draft7Validator,
+    }
+
     def __new__(mcs, class_name, bases, attrs):
         cls = super(JsonSchemaMeta, mcs).__new__(mcs, class_name, bases, attrs)
         fields = {}
@@ -33,20 +53,28 @@ def __new__(mcs, class_name, bases, attrs):
                     schema = _merge_schema(schema, base_schema)
             setattr(cls, 'jsonschema', schema)
         if not schema:
-            raise ValueError('{} must contain "jsonschema" attribute'
-                             .format(cls.__name__))
-        cls.validator = Draft4Validator(schema)
+            raise ValueError(
+                '{} must contain "jsonschema" attribute'.format(cls.__name__)
+            )
+        cls.validator = cls._get_validator(schema)
         cls.validator.check_schema(schema)
         for k in schema['properties']:
             fields[k] = Field()
         cls.fields = cls.fields.copy()
         cls.fields.update(fields)
         return cls
 
+    @classmethod
+    def _get_validator(cls, schema):
+        draft_version = schema.get('$schema')
+        # Default to Draft4Validator for backward-compatibility
+        validator_class = cls.draft_to_validator.get(
+            draft_version, Draft4Validator
+        )
+        return validator_class(schema)
+
 
 @six.add_metaclass(JsonSchemaMeta)
 class JsonSchemaItem(DictItem):
-    jsonschema = {
-        "properties": {}
-    }
+    jsonschema = {"properties": {}}
     merge_schema = False  # Off for backward-compatibility

scrapy_jsonschema/pipeline.py

@@ -37,7 +37,6 @@ def process_item(self, item, spider):
             error_msg = ''
             for path, message in paths_messages:
                 error_msg += u'{}: {}\n'.format(path, message)
-            raise DropItem(u'schema validation failed: \n {}'
-                           .format(error_msg))
+            raise DropItem(u'schema validation failed: \n {}'.format(error_msg))
 
         return item

tests/__init__.py

@@ -1,91 +1,63 @@
+from scrapy_jsonschema.draft import JSON_SCHEMA_DRAFT_7
 
 valid_schema = {
+    "$schema": JSON_SCHEMA_DRAFT_7,
     "title": "Product",
     "description": "Some product's description",
     "type": "object",
     "properties": {
         "id": {
             "description": "The unique identifier for a product",
-            "type": "integer"
-        },
-        "name": {
-            "description": "Name of the product",
-            "type": "string"
+            "type": "integer",
         },
+        "name": {"description": "Name of the product", "type": "string"},
         "prices": {
             "type": "array",
             "items": {
                 "type": "object",
                 "properties": {
-                    "name": {
-                        "type": "string",
-                    },
-                    "value": {
-                        "type": "integer",
-                    }
+                    "name": {"type": "string"},
+                    "value": {"type": "integer"},
                 },
                 "required": ["name", "value"],
-            }
-        }
+            },
+        },
     },
-    "required": ["id", "name"]
+    "required": ["id", "name"],
 }
 
-invalid_schema = {
-    "type": "invalid-type"
-}
+invalid_schema = {"type": "invalid-type"}
 
 merge_schema_base = {
-    'required': ['bar'],
-    'properties': {
-        'bar': {
-            'type': 'string',
-        },
-    },
+    "required": ["bar"],
+    "properties": {"bar": {"type": "string"}},
 }
 
-merge_schema_base_2 = {
-    'properties': {
-        'foo': {
-            'type': 'string',
-        },
-    }
-}
+merge_schema_base_2 = {"properties": {"foo": {"type": "string"}}}
 
 merge_schema_new = {
-    'required': ['foo'],
-    'properties': {
-        'foo': {
-            'type': 'string',
-            'pattern': r'^\d+$',
-        },
-        'baz': {
-            'type': 'string',
-        },
+    "required": ["foo"],
+    "properties": {
+        "foo": {"type": "string", "pattern": r"^\d+$"},
+        "baz": {"type": "string"},
     },
 }
 
-merged_valid_docs = [
-    {
-        'foo': '123',
-        'bar': 'baz',
-        'baz': 'bar',
-    },
-]
+merged_valid_docs = [{"foo": "123", "bar": "baz", "baz": "bar"}]
 
 merged_invalid_docs = [
     {
         # valid for base
-        'bar': 'baz',
+        "bar": "baz"
     },
     {
         # valid for base 2
-        'foo': '123',
+        "foo": "123"
     },
     {
         # valid for new
-        'foo': '123',
-        'baz': 'bar',
+        "foo": "123",
+        "baz": "bar",
     },
 ]
 
@@ -94,27 +66,23 @@
         "id": 123,
         "name": "name",
         "prices": [
-            {
-                "name": "price1",
-                "value": 100
-            },
-            {
-                "name": "price2",
-                "value": 200
-            }
-        ]
+            {"name": "price1", "value": 100},
+            {"name": "price2", "value": 200},
+        ],
     }
 ]
 
 invalid_doc_types = [
     {
-        "id": '123',        # this value should be an integer, not its string representation
-        "name": "name"
+        "id": "123",  # this value should be an integer, not a string
+        "name": "name",
     },
     {
         "id": 123,
-        "name": ["name"]    # this value should be a string, not a list of strings
-    }
+        "name": [
+            "name"
+        ],  # this value should be a string, not a list of strings
+    },
 ]
 
 invalid_doc_required = [
@@ -125,5 +93,5 @@
     {
         # 'name' is missing
         "id": {}
-    }
+    },
 ]

tests/test_draf.py

@@ -0,0 +1,24 @@
+from unittest import TestCase
+
+from scrapy_jsonschema.draft import (
+    JSON_SCHEMA_DRAFT_3,
+    JSON_SCHEMA_DRAFT_4,
+    JSON_SCHEMA_DRAFT_6,
+    JSON_SCHEMA_DRAFT_7,
+)
+
+
+class DraftTest(TestCase):
+    "Test that draft constants values hasn't been change by mistake"
+
+    def test_draft_3(self):
+        assert JSON_SCHEMA_DRAFT_3 == "http://json-schema.org/draft-03/schema#"
+
+    def test_draft_4(self):
+        assert JSON_SCHEMA_DRAFT_4 == "http://json-schema.org/draft-04/schema#"
+
+    def test_draft_6(self):
+        assert JSON_SCHEMA_DRAFT_6 == "http://json-schema.org/draft-06/schema#"
+
+    def test_draft_7(self):
+        assert JSON_SCHEMA_DRAFT_7 == "http://json-schema.org/draft-07/schema#"

tests/test_item.py

@@ -0,0 +1,87 @@
+import pytest
+from unittest import TestCase
+
+from jsonschema.exceptions import SchemaError
+from jsonschema import Draft4Validator, Draft7Validator
+
+from scrapy_jsonschema.item import JsonSchemaItem, _merge_schema
+from scrapy_jsonschema.draft import JSON_SCHEMA_DRAFT_7
+
+from . import (
+    valid_schema,
+    invalid_schema,
+    merge_schema_base,
+    merge_schema_base_2,
+    merge_schema_new,
+)
+
+
+class ValidSchemaTestCase(TestCase):
+
+    schema1 = {
+        "both": 1,
+        "only_base": 2,
+        "nested": {"list_to_merge": [1, 2], "both": "foo"},
+    }
+    schema2 = {
+        "both": 3,
+        "only_new": 4,
+        "nested": {"list_to_merge": [3], "both": "bar", "only_new": "baz"},
+    }
+
+    def test_no_schema(self):
+        with pytest.raises(ValueError):
+
+            class TestNoSchema(JsonSchemaItem):
+                jsonschema = None
+
+    def test_invalid_schema(self):
+        with pytest.raises(SchemaError):
+
+            class TestItem1(JsonSchemaItem):
+                jsonschema = invalid_schema
+
+    def test_valid_schema(self):
+        class TestItem2(JsonSchemaItem):
+            jsonschema = valid_schema
+
+    def test_merge_schema_func(self):
+        expected = {
+            "both": 1,
+            "only_base": 2,
+            "only_new": 4,
+            "nested": {
+                "list_to_merge": [1, 2, 3],
+                "both": "foo",
+                "only_new": "baz",
+            },
+        }
+        self.assertEqual(_merge_schema(self.schema1, self.schema2), expected)
+
+    def test_merge_schema_none(self):
+        self.assertEqual(_merge_schema(self.schema1, None), self.schema1)
+        self.assertEqual(_merge_schema(None, self.schema1), self.schema1)
+
+    def test_merge_schema(self):
+        class Base(JsonSchemaItem):
+            jsonschema = merge_schema_base
+
+        class Base2(JsonSchemaItem):
+            jsonschema = merge_schema_base_2
+
+        class Merged(Base, Base2):
+            jsonschema = merge_schema_new
+            merge_schema = True
+
+    def test_get_validator(self):
+        schema = {
+            "$schema": JSON_SCHEMA_DRAFT_7,
+            "title": "Item with Schema Draft",
+        }
+
+        draft7_validator = JsonSchemaItem._get_validator(schema)
+        self.assertTrue(isinstance(draft7_validator, Draft7Validator))
+
+        no_draft_chema = {"title": "Item without schema Draft"}
+        default_validator = JsonSchemaItem._get_validator(no_draft_chema)
+        self.assertTrue(isinstance(default_validator, Draft4Validator))