Merge pull request #17 from tayloraswift/tag-crawling-2

Tag crawling

Author: tayloraswift

.github/workflows/build.yml

@@ -14,8 +14,8 @@ jobs:
         strategy:
             matrix:
                 image:
-                -   swift:5.8.1-amazonlinux2
-                -   swift:5.8.1-jammy
+                -   swift:5.9.0-amazonlinux2
+                -   swift:5.9.0-jammy
         steps:
             -   name: checkout repository
                 uses: actions/checkout@v3

Package.swift

@@ -14,7 +14,8 @@ let package:Package = .init(
         .library(name: "DoclinkResolution", targets: ["DoclinkResolution"]),
         .library(name: "FNV1", targets: ["FNV1"]),
 
-        .library(name: "GitHubIntegration", targets: ["GitHubIntegration"]),
+        .library(name: "GitHubAPI", targets: ["GitHubAPI"]),
+        .library(name: "GitHubClient", targets: ["GitHubClient"]),
 
         .library(name: "HTML", targets: ["HTML"]),
 
@@ -68,13 +69,15 @@ let package:Package = .init(
     ],
     dependencies:
     [
-        //.package(url: "https://github.com/tayloraswift/swift-json", .upToNextMinor(
-        //    from: "0.6.0")),
         .package(url: "https://github.com/tayloraswift/swift-grammar", .upToNextMinor(
             from: "0.3.2")),
+        .package(url: "https://github.com/tayloraswift/swift-hash", .upToNextMinor(
+            from: "0.5.0")),
         .package(url: "https://github.com/tayloraswift/swift-mongodb", .upToNextMinor(
             from: "0.8.2")),
 
+        .package(url: "https://github.com/apple/swift-atomics", .upToNextMinor(
+            from: "1.1.0")),
         .package(url: "https://github.com/apple/swift-nio", .upToNextMinor(
             from: "2.57.0")),
         .package(url: "https://github.com/apple/swift-nio-http2", .upToNextMinor(
@@ -125,13 +128,16 @@ let package:Package = .init(
 
         .target(name: "GitHubClient", dependencies:
             [
-                .target(name: "GitHubIntegration"),
+                .target(name: "GitHubAPI"),
                 .target(name: "HTTPClient"),
+
+                .product(name: "Base64", package: "swift-hash"),
             ]),
 
-        .target(name: "GitHubIntegration", dependencies:
+        .target(name: "GitHubAPI", dependencies:
             [
                 .target(name: "JSON"),
+                .target(name: "UnixTime"),
             ]),
 
         .target(name: "HTML", dependencies:
@@ -376,9 +382,10 @@ let package:Package = .init(
 
         .target(name: "UnidocDB", dependencies:
             [
-                .target(name: "GitHubIntegration"),
+                .target(name: "GitHubAPI"),
                 .target(name: "UnidocAnalysis"),
                 .target(name: "UnidocLinker"),
+                .target(name: "UnixTime"),
                 .product(name: "MongoDB", package: "swift-mongodb"),
             ]),
 
@@ -398,7 +405,7 @@ let package:Package = .init(
 
         .target(name: "UnidocPages", dependencies:
             [
-                .target(name: "GitHubIntegration"),
+                .target(name: "GitHubAPI"),
                 .target(name: "HTTP"),
                 .target(name: "MarkdownRendering"),
                 .target(name: "UnidocQueries"),
@@ -425,10 +432,7 @@ let package:Package = .init(
                 .target(name: "URI"),
             ]),
 
-        .target(name: "UnixTime", dependencies:
-            [
-                .product(name: "BSON", package: "swift-mongodb"),
-            ]),
+        .target(name: "UnixTime"),
 
         .target(name: "URI", dependencies:
             [
@@ -454,6 +458,8 @@ let package:Package = .init(
                 .target(name: "Multiparts"),
                 .target(name: "System"),
                 .target(name: "UnidocPages"),
+
+                .product(name: "Atomics", package: "swift-atomics"),
             ]),
 
 

README.md

@@ -1,6 +1,6 @@
 <div align="center">
 
-<strong><em><code>unidoc</code></em></strong><br><small><code>0.2.2</code></small>
+<strong><em><code>unidoc</code></em></strong><br><small><code>0.2.3</code></small>
 
 [![ci build status](https://github.com/kelvin13/swift-unidoc/actions/workflows/build.yml/badge.svg)](https://github.com/kelvin13/swift-unidoc/actions/workflows/build.yml)
 

Sources/GitHubIntegration/GitHubAPI.Repo.License.swift renamed to Sources/GitHubAPI/GitHub.Repo.License.swift

@@ -1,6 +1,6 @@
 import JSON
 
-extension GitHubAPI.Repo
+extension GitHub.Repo
 {
     @frozen public
     struct License:Identifiable, Equatable, Sendable
@@ -20,7 +20,7 @@ extension GitHubAPI.Repo
         }
     }
 }
-extension GitHubAPI.Repo.License:JSONObjectDecodable
+extension GitHub.Repo.License:JSONObjectDecodable
 {
     public
     enum CodingKey:String

Sources/GitHubIntegration/GitHubAPI.Repo.Owner.swift renamed to Sources/GitHubAPI/GitHub.Repo.Owner.swift

@@ -1,6 +1,6 @@
 import JSON
 
-extension GitHubAPI.Repo
+extension GitHub.Repo
 {
     @frozen public
     struct Owner:Identifiable, Equatable, Sendable
@@ -21,7 +21,7 @@ extension GitHubAPI.Repo
         }
     }
 }
-extension GitHubAPI.Repo.Owner:JSONObjectDecodable
+extension GitHub.Repo.Owner:JSONObjectDecodable
 {
     public
     enum CodingKey:String

Sources/GitHubIntegration/GitHubAPI.Repo.swift renamed to Sources/GitHubAPI/GitHub.Repo.swift

@@ -1,6 +1,6 @@
 import JSON
 
-extension GitHubAPI
+extension GitHub
 {
     @frozen public
     struct Repo:Identifiable, Equatable, Sendable
@@ -109,7 +109,7 @@ extension GitHubAPI
         }
     }
 }
-extension GitHubAPI.Repo:JSONObjectDecodable
+extension GitHub.Repo:JSONObjectDecodable
 {
     public
     enum CodingKey:String

Sources/GitHubIntegration/GitHubAPI.Tag.Commit.swift renamed to Sources/GitHubAPI/GitHub.Tag.Commit.swift

@@ -1,7 +1,7 @@
 import JSON
 import SHA1
 
-extension GitHubAPI.Tag
+extension GitHub.Tag
 {
     struct Commit
     {
@@ -13,7 +13,7 @@ extension GitHubAPI.Tag
         }
     }
 }
-extension GitHubAPI.Tag.Commit:JSONObjectDecodable
+extension GitHub.Tag.Commit:JSONObjectDecodable
 {
     enum CodingKey:String
     {

Sources/GitHubIntegration/GitHubAPI.Tag.swift renamed to Sources/GitHubAPI/GitHub.Tag.swift

@@ -1,7 +1,7 @@
 import JSON
 import SHA1
 
-extension GitHubAPI
+extension GitHub
 {
     @frozen public
     struct Tag:Equatable, Sendable
@@ -22,7 +22,7 @@ extension GitHubAPI
         }
     }
 }
-extension GitHubAPI.Tag:JSONObjectDecodable
+extension GitHub.Tag:JSONObjectDecodable
 {
     public
     enum CodingKey:String

Sources/GitHubIntegration/GitHubAPI.User.swift renamed to Sources/GitHubAPI/GitHub.User.swift

@@ -1,6 +1,6 @@
 import JSON
 
-extension GitHubAPI
+extension GitHub
 {
     @frozen public
     struct User:Identifiable, Equatable, Sendable
@@ -97,7 +97,7 @@ extension GitHubAPI
         }
     }
 }
-extension GitHubAPI.User:JSONObjectDecodable
+extension GitHub.User:JSONObjectDecodable
 {
     public
     enum CodingKey:String

Sources/GitHubAPI/GitHub.swift

@@ -0,0 +1,5 @@
+/// A namespace for GitHub API types.
+@frozen public
+enum GitHub
+{
+}

Sources/GitHubIntegration/GitHubApp.Credentials.swift renamed to Sources/GitHubAPI/GitHubApp.Credentials.swift

@@ -0,0 +1,18 @@
+extension GitHubOAuth
+{
+    @frozen public
+    struct API
+    {
+        public
+        let agent:String
+        public
+        let oauth:GitHubOAuth?
+
+        @inlinable internal
+        init(agent:String, oauth:GitHubOAuth?)
+        {
+            self.agent = agent
+            self.oauth = oauth
+        }
+    }
+}

Sources/GitHubIntegration/GitHubApp.Token.swift renamed to Sources/GitHubAPI/GitHubApp.Token.swift

@@ -1,4 +1,6 @@
 /// The essence of a GitHub OAuth application.
+///
+/// A `GitHubOAuth` instance is just a ``client`` ID and ``secret``.
 @frozen public
 struct GitHubOAuth:GitHubApplication
 {
@@ -14,3 +16,9 @@ struct GitHubOAuth:GitHubApplication
         self.secret = secret
     }
 }
+extension GitHubOAuth
+{
+    /// The GitHub API.
+    @inlinable public
+    var api:API { .init(agent: "swift-unidoc (by tayloraswift)", oauth: self) }
+}

Sources/GitHubIntegration/GitHubApp.swift renamed to Sources/GitHubAPI/GitHubApp.swift

@@ -0,0 +1,8 @@
+import UnixTime
+
+public
+protocol GitHubRateLimitError:Error, Equatable, Sendable
+{
+    /// The UTC epoch second when the rate limit will reset.
+    var until:UnixTime { get }
+}

Sources/GitHubIntegration/GitHubApplication.swift renamed to Sources/GitHubAPI/GitHubApplication.swift

@@ -0,0 +1,102 @@
+import Base64
+import GitHubAPI
+import HTTPClient
+import JSON
+import NIOCore
+import NIOHPACK
+import UnixTime
+
+extension GitHubClient
+{
+    @frozen public
+    struct Connection
+    {
+        @usableFromInline internal
+        let http2:HTTP2Client.Connection
+        @usableFromInline internal
+        let app:Application
+
+        @inlinable internal
+        init(http2:HTTP2Client.Connection, app:Application)
+        {
+            self.http2 = http2
+            self.app = app
+        }
+    }
+}
+extension GitHubClient<GitHubOAuth.API>.Connection
+{
+    @inlinable public
+    func get<Response>(_:Response.Type = Response.self,
+        from endpoint:String,
+        with token:String? = nil) async throws -> Response where Response:JSONDecodable
+    {
+        var endpoint:String = endpoint
+        var status:UInt? = nil
+
+        following:
+        for _:Int in 0 ... 1
+        {
+            var request:HPACKHeaders =
+            [
+                ":method": "GET",
+                ":scheme": "https",
+                ":authority": "api.github.com",
+                ":path": endpoint,
+
+                //  GitHub will reject the API request if the user-agent is not set.
+                "user-agent": self.app.agent,
+                "accept": "application/vnd.github+json"
+            ]
+            if  let token:String
+            {
+                request.add(name: "authorization", value: "Bearer \(token)")
+            }
+            else if
+                let oauth:GitHubOAuth = self.app.oauth
+            {
+                let credentials:String = "\(oauth.client):\(oauth.secret)"
+                request.add(name: "authorization",
+                    value: "Basic \(Base64.encode(credentials.utf8))")
+            }
+
+            let response:HTTP2Client.Facet = try await self.http2.fetch(request)
+
+            //  TODO: support If-None-Match
+            switch response.status
+            {
+            case 200?:
+                var json:JSON = .init(utf8: [])
+                for buffer:ByteBuffer in response.buffers
+                {
+                    json.utf8 += buffer.readableBytesView
+                }
+
+                return try json.decode()
+
+            case 301?:
+                if let location:String = response.headers?["location"].first
+                {
+                    endpoint = String.init(location.trimmingPrefix("https://api.github.com"))
+                    continue following
+                }
+
+            case 403?:
+                if  let second:String = response.headers?["x-ratelimit-reset"].first,
+                    let second:Int64 = .init(second)
+                {
+                    throw GitHubClient<GitHubOAuth.API>.RateLimitError.init(
+                        until: .second(second))
+                }
+
+            case _:
+                break
+            }
+
+            status = response.status
+            break
+        }
+
+        throw GitHubClient<GitHubOAuth.API>.StatusError.init(code: status)
+    }
+}

Sources/GitHubAPI/GitHubOAuth.API.swift

@@ -0,0 +1,18 @@
+import GitHubAPI
+import UnixTime
+
+extension GitHubClient
+{
+    @frozen public
+    struct RateLimitError:GitHubRateLimitError, Equatable, Sendable
+    {
+        public
+        let until:UnixTime
+
+        @inlinable internal
+        init(until:UnixTime)
+        {
+            self.until = until
+        }
+    }
+}