diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index faf3abb..8eda910 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -96,7 +96,7 @@ jobs:
         run: |
           test -d .venv || virtualenv -p $(which python) --copies --reset-app-data .venv
           . .venv/bin/activate
-          pip install -r requirements.txt -r dev-requirements.txt
+          pip install -e .[dev]
 
       - name: Show environment info
         run: |
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 0d3b987..5afdf09 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -72,8 +72,7 @@ When you're ready to contribute code to address an open issue, please follow the
     Once your virtual environment is activated, you can install your local clone in "editable mode" with
 
         pip install -U pip setuptools wheel
-        pip install -e . 
-        pip install -r dev-requirements.txt
+        pip install -e .[dev]
 
     The "editable mode" comes from the `-e` argument to `pip`, and essential just creates a symbolic link from the site-packages directory of your virtual environment to the source code in your local clone. That way any changes you make will be immediately reflected in your virtual environment.
 
diff --git a/RELEASE_PROCESS.md b/RELEASE_PROCESS.md
index b8900da..f214ab4 100644
--- a/RELEASE_PROCESS.md
+++ b/RELEASE_PROCESS.md
@@ -4,8 +4,6 @@
 
 1. Update the version in `my_package/version.py`.
 
-2. Update the CHANGELOG.md so that everything under the "Unreleased" section is now under a section corresponding to this release.
-
 3. Run the release script:
 
     ```bash
diff --git a/scripts/prepare_changelog.py b/scripts/prepare_changelog.py
new file mode 100644
index 0000000..6ca9fda
--- /dev/null
+++ b/scripts/prepare_changelog.py
@@ -0,0 +1,31 @@
+from datetime import datetime
+from pathlib import Path
+
+from my_package.version import VERSION
+
+
+def main():
+    changelog = Path("CHANGELOG.md")
+
+    with changelog.open() as f:
+        lines = f.readlines()
+
+    for i in range(len(lines)):
+        line = lines[i]
+        if line.startswith("## Unreleased"):
+            lines.insert(i + 1, "\n")
+            lines.insert(
+                i + 2,
+                f"## [v{VERSION}](https://github.com/allenai/my_package/releases/tag/v{VERSION}) - "
+                f"{datetime.now().strftime('%Y-%m-%d')}\n",
+            )
+            break
+    else:
+        raise RuntimeError("Couldn't find 'Unreleased' section")
+
+    with changelog.open("w") as f:
+        f.writelines(lines)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/scripts/release.sh b/scripts/release.sh
index 53a0ac7..b9a695e 100755
--- a/scripts/release.sh
+++ b/scripts/release.sh
@@ -7,7 +7,9 @@ TAG=$(python -c 'from my_package.version import VERSION; print("v" + VERSION)')
 read -p "Creating new release for $TAG. Do you want to continue? [Y/n] " prompt
 
 if [[ $prompt == "y" || $prompt == "Y" || $prompt == "yes" || $prompt == "Yes" ]]; then
-    git commit -a -m "Bump version to $TAG for release" || true && git push
+    python scripts/prepare_changelog.py
+    git add -A
+    git commit -m "Bump version to $TAG for release" || true && git push
     echo "Creating new git tag $TAG"
     git tag "$TAG" -m "$TAG"
     git push --tags
diff --git a/setup.py b/setup.py
index 7ae6a66..1cbcff5 100644
--- a/setup.py
+++ b/setup.py
@@ -1,26 +1,28 @@
 from setuptools import setup, find_packages
 
-# Load requirements.txt with a special case for allennlp so we can handle
-# cross-library integration testing.
-with open("requirements.txt") as requirements_file:
-    import re
 
-    def fix_url_dependencies(req: str) -> str:
-        """Pip and setuptools disagree about how URL dependencies should be handled."""
-        m = re.match(
-            r"^(git\+)?(https|ssh)://(git@)?github\.com/([\w-]+)/(?P<name>[\w-]+)\.git", req
-        )
-        if m is None:
-            return req
-        else:
-            return f"{m.group('name')} @ {req}"
+def read_requirements(filename: str):
+    with open(filename) as requirements_file:
+        import re
+
+        def fix_url_dependencies(req: str) -> str:
+            """Pip and setuptools disagree about how URL dependencies should be handled."""
+            m = re.match(
+                r"^(git\+)?(https|ssh)://(git@)?github\.com/([\w-]+)/(?P<name>[\w-]+)\.git", req
+            )
+            if m is None:
+                return req
+            else:
+                return f"{m.group('name')} @ {req}"
+
+        requirements = []
+        for line in requirements_file:
+            line = line.strip()
+            if line.startswith("#") or len(line) <= 0:
+                continue
+            requirements.append(fix_url_dependencies(line))
+    return requirements
 
-    install_requirements = []
-    for line in requirements_file:
-        line = line.strip()
-        if line.startswith("#") or len(line) <= 0:
-            continue
-        install_requirements.append(fix_url_dependencies(line))
 
 # version.py defines the VERSION and VERSION_SHORT variables.
 # We use exec here so we don't import cached_path whilst setting up.
@@ -49,6 +51,7 @@ def fix_url_dependencies(req: str) -> str:
     packages=find_packages(
         exclude=["*.tests", "*.tests.*", "tests.*", "tests"],
     ),
-    install_requires=install_requirements,
+    install_requires=read_requirements("requirements.txt"),
+    extras_require={"dev": read_requirements("dev-requirements.txt")},
     python_requires=">=3.6.1",
 )