CoLRev-Environment
diff --git a/‎colrev/ops/init/settings.json‎
Lines changed: 6 additions & 2 deletions b/‎colrev/ops/init/settings.json‎
Lines changed: 6 additions & 2 deletions
diff --git a/‎colrev/ops/upgrade.py‎
Lines changed: 16 additions & 0 deletions b/‎colrev/ops/upgrade.py‎
Lines changed: 16 additions & 0 deletions
diff --git a/‎colrev/package_manager/packages.json‎
Lines changed: 3 additions & 0 deletions b/‎colrev/package_manager/packages.json‎
Lines changed: 3 additions & 0 deletions
diff --git a/‎colrev/packages/arxiv/pyproject.toml‎
Lines changed: 1 addition & 0 deletions b/‎colrev/packages/arxiv/pyproject.toml‎
Lines changed: 1 addition & 0 deletions
diff --git a/‎colrev/packages/files_dir/src/files_dir.py‎
Lines changed: 19 additions & 0 deletions b/‎colrev/packages/files_dir/src/files_dir.py‎
Lines changed: 19 additions & 0 deletions
diff --git a/‎colrev/packages/ref_check/.pre-commit-config.yaml‎
Lines changed: 63 additions & 0 deletions b/‎colrev/packages/ref_check/.pre-commit-config.yaml‎
Lines changed: 63 additions & 0 deletions
diff --git a/‎colrev/packages/ref_check/LICENSE‎
Lines changed: 21 additions & 0 deletions b/‎colrev/packages/ref_check/LICENSE‎
Lines changed: 21 additions & 0 deletions
diff --git a/‎colrev/packages/ref_check/README.md‎
Lines changed: 17 additions & 0 deletions b/‎colrev/packages/ref_check/README.md‎
Lines changed: 17 additions & 0 deletions
diff --git a/‎colrev/packages/ref_check/docs/README.md‎
Lines changed: 9 additions & 0 deletions b/‎colrev/packages/ref_check/docs/README.md‎
Lines changed: 9 additions & 0 deletions
diff --git a/‎colrev/packages/ref_check/pyproject.toml‎
Lines changed: 25 additions & 0 deletions b/‎colrev/packages/ref_check/pyproject.toml‎
Lines changed: 25 additions & 0 deletions
@@ -26,7 +26,7 @@
     "load": {},
     "prep": {
         "fields_to_keep": [],
-        "defects_to_ignore": ["inconsistent-with-url-metadata"],
+        "defects_to_ignore": [],
         "prep_rounds": [
             {
                 "name": "prep",
@@ -153,6 +153,10 @@
         ]
     },
     "data": {
-        "data_package_endpoints": []
+        "data_package_endpoints": [
+            {
+                "endpoint": "colrev.rev_check"
+            }
+        ]
     }
 }
@@ -720,6 +720,22 @@ def _migrate_0_13_0(self) -> bool:
                     out.write(bibtex_str + "\n")
                 self.repo.index.add([source["filename"]])
 
+        # Add "colrev.ref_check" to data endpoints
+        if "colrev.ref_check" not in [
+            e["endpoint"] for e in settings["data"]["data_package_endpoints"]
+        ]:
+            settings["data"]["data_package_endpoints"].append(
+                {"endpoint": "colrev.ref_check"}
+            )
+
+        # Remove "inconsistent-with-url-metadata" from settings["prep"]["defects_to_ignore"]
+        settings["prep"]["defects_to_ignore"] = [
+            d
+            for d in settings["prep"]["defects_to_ignore"]
+            if d != "inconsistent-with-url-metadata"
+        ]
+        self._save_settings(settings)
+
         # Rename LOCAL_ENVIRONMENT_DIR
         if not Filepaths.LOCAL_ENVIRONMENT_DIR.is_dir():
             shutil.move(
 
@@ -268,5 +268,8 @@
     },
     "colrev.github": {
         "dev_status": "maturing"
+    },
+    "colrev.ref_check": {
+        "dev_status": "experimental"
     }
 }
@@ -10,6 +10,7 @@ repository = "https://github.com/CoLRev-Environment/colrev/tree/main/colrev/pack
 
 [[tool.poetry.packages]]
 include = "src"
+feedparser = "^6.0.10"
 
 [tool.poetry.dependencies]
 python = ">=3.8, <4"
 
@@ -457,6 +457,24 @@ def _index_file(
             )
         raise NotImplementedError
 
+    def _fix_grobid_errors(self, new_record: dict) -> None:
+        # Fix common GROBID errors that would cause problems in deduplication
+
+        # drop title if it is identical with journal
+        if Fields.TITLE in new_record and Fields.JOURNAL in new_record:
+            if new_record[Fields.TITLE] == new_record[Fields.JOURNAL]:
+                new_record.pop(Fields.TITLE)
+        # drop title if it starts with "doi:"
+        if Fields.TITLE in new_record:
+            if new_record[Fields.TITLE].lower().startswith("doi:"):
+                new_record.pop(Fields.TITLE)
+        # drop title if it has more numbers than characters
+        if Fields.TITLE in new_record:
+            if sum(c.isdigit() for c in new_record[Fields.TITLE]) > sum(
+                c.isalpha() for c in new_record[Fields.TITLE]
+            ):
+                new_record.pop(Fields.TITLE)
+
     def _index_pdf(
         self,
         *,
@@ -512,6 +530,7 @@ def _index_pdf(
         ):
             # otherwise, get metadata from grobid (indexing)
             new_record = self._get_grobid_metadata(file_path=file_path_abs)
+            self._fix_grobid_errors(new_record)
 
         new_record[Fields.FILE] = str(file_path)
         new_record = self._add_md_string(record_dict=new_record)
 
@@ -0,0 +1,63 @@
+
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.6.0
+    hooks:
+    -   id: trailing-whitespace
+    -   id: end-of-file-fixer
+        exclude: bib$|txt$|ris$|enl$|xml$
+    -   id: check-docstring-first
+    -   id: check-json
+    -   id: check-yaml
+    -   id: check-toml
+    -   id: debug-statements
+    -   id: name-tests-test
+-   repo: https://github.com/psf/black-pre-commit-mirror
+    rev: 24.4.2
+    hooks:
+      - id: black
+        language_version: python3
+-   repo: https://github.com/PyCQA/autoflake
+    rev: v2.3.1
+    hooks:
+    -   id: autoflake
+-   repo: https://github.com/PyCQA/flake8
+    rev: 7.1.0
+    hooks:
+    -   id: flake8
+        additional_dependencies: [flake8-typing-imports==1.12.0]
+        args: ['--max-line-length=110', '--extend-ignore=E203,TYP006']
+-   repo: https://github.com/asottile/reorder-python-imports
+    rev: v3.13.0
+    hooks:
+    -   id: reorder-python-imports
+        args: [--py3-plus]
+-   repo: https://github.com/asottile/pyupgrade
+    rev: v3.16.0
+    hooks:
+    -   id: pyupgrade
+        args: [--py36-plus, --keep-runtime-typing]
+-   repo: https://github.com/pre-commit/mirrors-mypy
+    rev: 'v1.11.0'
+    hooks:
+    -   id: mypy
+        args: [--disallow-untyped-defs, --disallow-incomplete-defs, --disallow-untyped-calls]
+        additional_dependencies: [types-toml]
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  rev: v0.5.4
+  hooks:
+    - id: ruff # runs faster than pylint
+      args: [--fix, --exit-non-zero-on-fix]
+- repo: local
+  hooks:
+    - id: pylint
+      name: pylint
+      entry: pylint
+      language: system
+      types: [python]
+      files: colrev
+      args:
+        [
+          "-rn", # Only display messages
+          "-sn", # Don't display the score
+        ]
@@ -0,0 +1,21 @@
+MIT License
+
+        Copyright (c) [2024] [Gerit Wagner]
+
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
@@ -0,0 +1,17 @@
+# colrev.ref_check
+
+Checks the quality of reference metadata
+
+## Installation
+
+```bash
+colrev install colrev.ref_check
+```
+
+## Usage
+
+`colrev.ref_check` can be added as a data endpoint. It ensures that records are only set to `rev_synthesized` if there are no remaining defects in the record metadata.
+
+## License
+
+This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
@@ -0,0 +1,9 @@
+## Installation
+
+```bash
+colrev install colrev.ref_check
+```
+
+## Usage
+
+`colrev.ref_check` can be added as a data endpoint. It ensures that records are only set to `rev_synthesized` if there are no remaining defects in the record metadata.
@@ -0,0 +1,25 @@
+[tool.poetry]
+name = "colrev.ref_check"
+description = "Checks the quality of reference metadata"
+version = "0.1.0"
+license = "MIT"
+authors = ["Gerit Wagner <[email protected]>"]
+repository = "https://github.com/CoLRev-Environment/colrev/tree/main/colrev/packages/"
+
+[[tool.poetry.packages]]
+include = "src"
+
+[tool.poetry.dependencies]
+python = ">=3.9, <4"
+
+[tool.colrev]
+colrev_doc_description = "TODO"
+colrev_doc_link = "docs/README.md"
+search_types = []
+
+[tool.poetry.plugins.colrev]
+data = "colrev.packages.ref_check.src.ref_check:RefCheck"
+
+[build-system]
+requires = ["poetry-core>=1.0.0", "cython<3.0"]
+build-backend = "poetry.core.masonry.api"
Original file line number	Diff line number	Diff line change
`@@ -26,7 +26,7 @@`
`26`	`26`	`"load": {},`
`27`	`27`	`"prep": {`
`28`	`28`	`"fields_to_keep": [],`
`29`		`- "defects_to_ignore": ["inconsistent-with-url-metadata"],`
	`29`	`+ "defects_to_ignore": [],`
`30`	`30`	`"prep_rounds": [`
`31`	`31`	`{`
`32`	`32`	`"name": "prep",`
`@@ -153,6 +153,10 @@`
`153`	`153`	`]`
`154`	`154`	`},`
`155`	`155`	`"data": {`
`156`		`- "data_package_endpoints": []`
	`156`	`+ "data_package_endpoints": [`
	`157`	`+ {`
	`158`	`+ "endpoint": "colrev.rev_check"`
	`159`	`+ }`
	`160`	`+ ]`
`157`	`161`	`}`
`158`	`162`	`}`
Original file line number	Diff line number	Diff line change
`@@ -268,5 +268,8 @@`
`268`	`268`	`},`
`269`	`269`	`"colrev.github": {`
`270`	`270`	`"dev_status": "maturing"`
	`271`	`+ },`
	`272`	`+ "colrev.ref_check": {`
	`273`	`+ "dev_status": "experimental"`
`271`	`274`	`}`
`272`	`275`	`}`