make the crawler respect GitHub’s rate limit, and handle 301 redirects

Author: tayloraswift

Package.swift

@@ -14,7 +14,8 @@ let package:Package = .init(
         .library(name: "DoclinkResolution", targets: ["DoclinkResolution"]),
         .library(name: "FNV1", targets: ["FNV1"]),
 
-        .library(name: "GitHubIntegration", targets: ["GitHubIntegration"]),
+        .library(name: "GitHubAPI", targets: ["GitHubAPI"]),
+        .library(name: "GitHubClient", targets: ["GitHubClient"]),
 
         .library(name: "HTML", targets: ["HTML"]),
 
@@ -68,10 +69,10 @@ let package:Package = .init(
     ],
     dependencies:
     [
-        //.package(url: "https://github.com/tayloraswift/swift-json", .upToNextMinor(
-        //    from: "0.6.0")),
         .package(url: "https://github.com/tayloraswift/swift-grammar", .upToNextMinor(
             from: "0.3.2")),
+        .package(url: "https://github.com/tayloraswift/swift-hash", .upToNextMinor(
+            from: "0.5.0")),
         .package(url: "https://github.com/tayloraswift/swift-mongodb", .upToNextMinor(
             from: "0.8.2")),
 
@@ -127,13 +128,16 @@ let package:Package = .init(
 
         .target(name: "GitHubClient", dependencies:
             [
-                .target(name: "GitHubIntegration"),
+                .target(name: "GitHubAPI"),
                 .target(name: "HTTPClient"),
+
+                .product(name: "Base64", package: "swift-hash"),
             ]),
 
-        .target(name: "GitHubIntegration", dependencies:
+        .target(name: "GitHubAPI", dependencies:
             [
                 .target(name: "JSON"),
+                .target(name: "UnixTime"),
             ]),
 
         .target(name: "HTML", dependencies:
@@ -378,7 +382,7 @@ let package:Package = .init(
 
         .target(name: "UnidocDB", dependencies:
             [
-                .target(name: "GitHubIntegration"),
+                .target(name: "GitHubAPI"),
                 .target(name: "UnidocAnalysis"),
                 .target(name: "UnidocLinker"),
                 .target(name: "UnixTime"),
@@ -401,7 +405,7 @@ let package:Package = .init(
 
         .target(name: "UnidocPages", dependencies:
             [
-                .target(name: "GitHubIntegration"),
+                .target(name: "GitHubAPI"),
                 .target(name: "HTTP"),
                 .target(name: "MarkdownRendering"),
                 .target(name: "UnidocQueries"),
@@ -428,10 +432,7 @@ let package:Package = .init(
                 .target(name: "URI"),
             ]),
 
-        .target(name: "UnixTime", dependencies:
-            [
-                .product(name: "BSON", package: "swift-mongodb"),
-            ]),
+        .target(name: "UnixTime"),
 
         .target(name: "URI", dependencies:
             [

Sources/GitHubIntegration/GitHubAPI.Repo.License.swift renamed to Sources/GitHubAPI/GitHub.Repo.License.swift

@@ -1,6 +1,6 @@
 import JSON
 
-extension GitHubAPI.Repo
+extension GitHub.Repo
 {
     @frozen public
     struct License:Identifiable, Equatable, Sendable
@@ -20,7 +20,7 @@ extension GitHubAPI.Repo
         }
     }
 }
-extension GitHubAPI.Repo.License:JSONObjectDecodable
+extension GitHub.Repo.License:JSONObjectDecodable
 {
     public
     enum CodingKey:String

Sources/GitHubIntegration/GitHubAPI.Repo.Owner.swift renamed to Sources/GitHubAPI/GitHub.Repo.Owner.swift

@@ -1,6 +1,6 @@
 import JSON
 
-extension GitHubAPI.Repo
+extension GitHub.Repo
 {
     @frozen public
     struct Owner:Identifiable, Equatable, Sendable
@@ -21,7 +21,7 @@ extension GitHubAPI.Repo
         }
     }
 }
-extension GitHubAPI.Repo.Owner:JSONObjectDecodable
+extension GitHub.Repo.Owner:JSONObjectDecodable
 {
     public
     enum CodingKey:String

Sources/GitHubIntegration/GitHubAPI.Repo.swift renamed to Sources/GitHubAPI/GitHub.Repo.swift

@@ -1,6 +1,6 @@
 import JSON
 
-extension GitHubAPI
+extension GitHub
 {
     @frozen public
     struct Repo:Identifiable, Equatable, Sendable
@@ -109,7 +109,7 @@ extension GitHubAPI
         }
     }
 }
-extension GitHubAPI.Repo:JSONObjectDecodable
+extension GitHub.Repo:JSONObjectDecodable
 {
     public
     enum CodingKey:String

Sources/GitHubIntegration/GitHubAPI.Tag.Commit.swift renamed to Sources/GitHubAPI/GitHub.Tag.Commit.swift

@@ -1,7 +1,7 @@
 import JSON
 import SHA1
 
-extension GitHubAPI.Tag
+extension GitHub.Tag
 {
     struct Commit
     {
@@ -13,7 +13,7 @@ extension GitHubAPI.Tag
         }
     }
 }
-extension GitHubAPI.Tag.Commit:JSONObjectDecodable
+extension GitHub.Tag.Commit:JSONObjectDecodable
 {
     enum CodingKey:String
     {

Sources/GitHubIntegration/GitHubAPI.Tag.swift renamed to Sources/GitHubAPI/GitHub.Tag.swift

@@ -1,7 +1,7 @@
 import JSON
 import SHA1
 
-extension GitHubAPI
+extension GitHub
 {
     @frozen public
     struct Tag:Equatable, Sendable
@@ -22,7 +22,7 @@ extension GitHubAPI
         }
     }
 }
-extension GitHubAPI.Tag:JSONObjectDecodable
+extension GitHub.Tag:JSONObjectDecodable
 {
     public
     enum CodingKey:String

Sources/GitHubIntegration/GitHubAPI.User.swift renamed to Sources/GitHubAPI/GitHub.User.swift

@@ -1,6 +1,6 @@
 import JSON
 
-extension GitHubAPI
+extension GitHub
 {
     @frozen public
     struct User:Identifiable, Equatable, Sendable
@@ -97,7 +97,7 @@ extension GitHubAPI
         }
     }
 }
-extension GitHubAPI.User:JSONObjectDecodable
+extension GitHub.User:JSONObjectDecodable
 {
     public
     enum CodingKey:String

Sources/GitHubAPI/GitHub.swift

@@ -0,0 +1,5 @@
+/// A namespace for GitHub API types.
+@frozen public
+enum GitHub
+{
+}