diff --git a/.air.toml b/.air.toml index 5ee2db21..8db54031 100644 --- a/.air.toml +++ b/.air.toml @@ -12,7 +12,7 @@ tmp_dir = "tmp" exclude_regex = ["_test.go"] exclude_unchanged = false follow_symlink = false - full_bin = "sudo ./tmp/main" + full_bin = "sudo env DOCKER_CONFIG=$HOME/.docker ./tmp/main" include_dir = [] include_ext = ["go", "tpl", "tmpl", "html", "yaml"] include_file = [] diff --git a/go.mod b/go.mod index 16a40d39..d6691b8a 100644 --- a/go.mod +++ b/go.mod @@ -60,6 +60,7 @@ require ( github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect github.com/apex/log v1.9.0 // indirect github.com/blang/semver/v4 v4.0.0 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/containerd/errdefs v1.0.0 // indirect github.com/containerd/errdefs/pkg v0.3.0 // indirect @@ -67,9 +68,9 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/digitalocean/go-libvirt v0.0.0-20220804181439-8648fbde413e // indirect github.com/docker/cli v28.2.2+incompatible // indirect - github.com/docker/docker v28.2.2+incompatible // indirect + github.com/docker/docker v28.5.1+incompatible // indirect github.com/docker/docker-credential-helpers v0.9.3 // indirect - github.com/docker/go-connections v0.5.0 // indirect + github.com/docker/go-connections v0.6.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect @@ -78,7 +79,6 @@ require ( github.com/go-openapi/jsonpointer v0.21.0 // indirect github.com/go-openapi/swag v0.23.0 // indirect github.com/go-test/deep v1.1.1 // indirect - github.com/gogo/protobuf v1.3.2 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/klauspost/compress v1.18.0 // indirect @@ -90,6 +90,7 @@ require ( github.com/moby/sys/sequential v0.6.0 // indirect github.com/moby/sys/user v0.4.0 // indirect github.com/moby/sys/userns v0.1.0 // indirect + github.com/moby/term v0.5.0 // indirect github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect github.com/oasdiff/yaml v0.0.0-20250309154309-f31be36b4037 // indirect github.com/oasdiff/yaml3 v0.0.0-20250309153720-d2182401db90 // indirect diff --git a/go.sum b/go.sum index 6fd5278f..a369c838 100644 --- a/go.sum +++ b/go.sum @@ -21,9 +21,8 @@ github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2y github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500 h1:6lhrsTEnloDPXyeZBvSYvQf8u86jbKehZPVDDlkgDl4= github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M= -github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= -github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4= -github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= @@ -52,12 +51,12 @@ github.com/docker/cli v28.2.2+incompatible h1:qzx5BNUDFqlvyq4AHzdNB7gSyVTmU4cgsy github.com/docker/cli v28.2.2+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk= github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v28.2.2+incompatible h1:CjwRSksz8Yo4+RmQ339Dp/D2tGO5JxwYeqtMOEe0LDw= -github.com/docker/docker v28.2.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v28.5.1+incompatible h1:Bm8DchhSD2J6PsFzxC35TZo4TLGR2PdW/E69rU45NhM= +github.com/docker/docker v28.5.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker-credential-helpers v0.9.3 h1:gAm/VtF9wgqJMoxzT3Gj5p4AqIjCBS4wrsOh9yRqcz8= github.com/docker/docker-credential-helpers v0.9.3/go.mod h1:x+4Gbw9aGmChi3qTLZj8Dfn0TD20M/fuWy0E5+WDeCo= -github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= -github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= +github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94= +github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= @@ -84,8 +83,6 @@ github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+Gr github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= github.com/go-test/deep v1.1.1 h1:0r/53hagsehfO4bzD2Pgr/+RgHqhmf+k1Bpse2cTu1U= github.com/go-test/deep v1.1.1/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -119,8 +116,6 @@ github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8Hm github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= @@ -158,8 +153,8 @@ github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs= github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= -github.com/moby/term v0.0.0-20221205130635-1aeaba878587 h1:HfkjXDfhgVaN5rmueG8cL8KKeFNecRCXFhaJ2qZ5SKA= -github.com/moby/term v0.0.0-20221205130635-1aeaba878587/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= +github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= +github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= @@ -239,8 +234,6 @@ github.com/vishvananda/netns v0.0.5 h1:DfiHV+j8bA32MFM7bfEunvT8IAqQ/NzSJHtcmW5zd github.com/vishvananda/netns v0.0.5/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= github.com/woodsbury/decimal128 v1.3.0 h1:8pffMNWIlC0O5vbyHWFZAt5yWvWcrHA+3ovIIjVWss0= github.com/woodsbury/decimal128 v1.3.0/go.mod h1:C5UTmyTjW3JftjUFzOVhC20BEQa2a4ZKOB5I6Zjb+ds= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= @@ -283,26 +276,19 @@ go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04= golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.28.0 h1:gQBtGhjxykdjY9YhZpSlZIsbnaE2+PgjfLWUQTnoZ1U= golang.org/x/mod v0.28.0/go.mod h1:yfB/L0NOf/kmEbXjzCPOx1iK1fRutOydrCMsqRhEBxI= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.46.1-0.20251013234738-63d1a5100f82 h1:6/3JGEh1C88g7m+qzzTbl3A0FtsLguXieqofVLU/JAo= golang.org/x/net v0.46.1-0.20251013234738-63d1a5100f82/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= @@ -310,7 +296,6 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -330,8 +315,6 @@ golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.37.0 h1:DVSRzp7FwePZW356yEAChSdNcQo6Nsp+fex1SUW09lE= golang.org/x/tools v0.37.0/go.mod h1:MBN5QPQtLMHVdvsbtarmTNukZDdgwdwlO5qGacAzF0w= diff --git a/lib/builds/builder_agent/main.go b/lib/builds/builder_agent/main.go index 0730b865..55bf202f 100644 --- a/lib/builds/builder_agent/main.go +++ b/lib/builds/builder_agent/main.go @@ -50,8 +50,8 @@ type BuildConfig struct { Secrets []SecretRef `json:"secrets,omitempty"` TimeoutSeconds int `json:"timeout_seconds"` NetworkMode string `json:"network_mode"` - IsAdminBuild bool `json:"is_admin_build,omitempty"` - GlobalCacheKey string `json:"global_cache_key,omitempty"` + IsAdminBuild bool `json:"is_admin_build,omitempty"` + GlobalCacheKey string `json:"global_cache_key,omitempty"` } // SecretRef references a secret to inject during build @@ -722,6 +722,14 @@ func setupBuildkitdConfig(config *BuildConfig) error { } // If HTTPS without insecure and without CA, use system CA (no config needed) + // Configure docker.io to use the local registry as a mirror. + // BuildKit will try the mirror first for FROM pulls. Since base images + // are pre-cached server-side via mirrorBaseImagesForBuild(), the mirror + // will have them and serve them directly without pulling from Docker Hub. + tomlContent.WriteString("\n") + tomlContent.WriteString("[registry.\"docker.io\"]\n") + tomlContent.WriteString(fmt.Sprintf(" mirrors = [\"%s\"]\n", registryHost)) + // Ensure config directory exists buildkitDir := "/home/builder/.config/buildkit" if err := os.MkdirAll(buildkitDir, 0755); err != nil { @@ -991,3 +999,4 @@ func getBuildkitVersion() string { out, _ := cmd.Output() return strings.TrimSpace(string(out)) } + diff --git a/lib/builds/dockerfile.go b/lib/builds/dockerfile.go new file mode 100644 index 00000000..2df03358 --- /dev/null +++ b/lib/builds/dockerfile.go @@ -0,0 +1,166 @@ +package builds + +import ( + "archive/tar" + "compress/gzip" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/google/go-containerregistry/pkg/name" +) + +// ParseDockerfileFROMs extracts and deduplicates base image references from +// Dockerfile content. It reuses the same parsing logic as the builder agent's +// rewriteDockerfileFROMs: split lines, find FROM, skip flags/comments/scratch, +// normalize refs. Inter-stage references (FROM builder) and variable references +// (${VAR}) are skipped since they can't be resolved at parse time. +func ParseDockerfileFROMs(content string) []string { + lines := strings.Split(content, "\n") + + // Track stage names so we can skip inter-stage FROM references + stageNames := make(map[string]bool) + seen := make(map[string]bool) + var refs []string + + for _, line := range lines { + trimmed := strings.TrimSpace(line) + + // Skip empty lines and comments + if trimmed == "" || strings.HasPrefix(trimmed, "#") { + continue + } + + // Check for FROM instruction (case insensitive) + upper := strings.ToUpper(trimmed) + if !strings.HasPrefix(upper, "FROM ") { + continue + } + + parts := strings.Fields(trimmed) + if len(parts) < 2 { + continue + } + + // Find the image reference (skip FROM and any flags like --platform) + imageIdx := 1 + for imageIdx < len(parts) && strings.HasPrefix(parts[imageIdx], "--") { + imageIdx++ + } + if imageIdx >= len(parts) { + continue + } + + imageRef := parts[imageIdx] + + // Record AS alias if present + for j := imageIdx + 1; j < len(parts)-1; j++ { + if strings.EqualFold(parts[j], "AS") { + stageNames[strings.ToLower(parts[j+1])] = true + break + } + } + + // Skip scratch + if imageRef == "scratch" { + continue + } + + // Skip inter-stage references (e.g. FROM builder) + if stageNames[strings.ToLower(imageRef)] { + continue + } + + // Skip variable references that can't be resolved + if strings.Contains(imageRef, "${") { + continue + } + + // Normalize the image reference (same logic as builder agent) + normalized := normalizeImageRef(imageRef) + + if !seen[normalized] { + seen[normalized] = true + refs = append(refs, normalized) + } + } + + return refs +} + +// normalizeImageRef normalizes a Docker image reference to match the local +// registry path that BuildKit mirror requests will use. Official Docker Hub +// images keep the library/ prefix (e.g. "node:20-alpine" → "library/node:20-alpine") +// because BuildKit requests them as /v2/library/node/manifests/.... +// Non-Docker Hub images keep the full registry path. +// +// This is consistent with normalizeToLocalRef in lib/images/mirror.go, which +// controls where mirrored images are pushed. +func normalizeImageRef(ref string) string { + parsed, err := name.ParseReference(ref) + if err != nil { + // Fall back to basic normalization if parsing fails + return strings.TrimPrefix(ref, "docker.io/") + } + + // Get canonicalized repository (e.g. "index.docker.io/library/node") + repo := parsed.Context().String() + + // Strip index.docker.io/ prefix (canonical form of docker.io) + repo = strings.TrimPrefix(repo, "index.docker.io/") + repo = strings.TrimPrefix(repo, "docker.io/") + + // Keep library/ prefix — BuildKit mirror requests use it for official images + + // Build the tag or digest suffix + var suffix string + if tag, ok := parsed.(name.Tag); ok { + suffix = ":" + tag.TagStr() + } else if dig, ok := parsed.(name.Digest); ok { + suffix = "@" + dig.DigestStr() + } + + return repo + suffix +} + +// ExtractDockerfileFromTarball reads just the Dockerfile entry from a .tar.gz +// archive and returns its content as a string. It looks for entries named +// "Dockerfile" or "./Dockerfile" at the root of the archive. +func ExtractDockerfileFromTarball(tarballPath string) (string, error) { + f, err := os.Open(tarballPath) + if err != nil { + return "", fmt.Errorf("open tarball: %w", err) + } + defer f.Close() + + gz, err := gzip.NewReader(f) + if err != nil { + return "", fmt.Errorf("create gzip reader: %w", err) + } + defer gz.Close() + + tr := tar.NewReader(gz) + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return "", fmt.Errorf("read tar entry: %w", err) + } + + // Match Dockerfile at root (with or without ./ prefix) + name := filepath.Clean(hdr.Name) + if name == "Dockerfile" { + data, err := io.ReadAll(tr) + if err != nil { + return "", fmt.Errorf("read Dockerfile from tarball: %w", err) + } + return string(data), nil + } + } + + return "", fmt.Errorf("Dockerfile not found in tarball") +} diff --git a/lib/builds/dockerfile_test.go b/lib/builds/dockerfile_test.go new file mode 100644 index 00000000..39674c84 --- /dev/null +++ b/lib/builds/dockerfile_test.go @@ -0,0 +1,230 @@ +package builds + +import ( + "archive/tar" + "compress/gzip" + "os" + "path/filepath" + "testing" +) + +func TestParseDockerfileFROMs_SingleFROM(t *testing.T) { + content := `FROM onkernel/nodejs22-base:0.1.1 +RUN echo hello +` + refs := ParseDockerfileFROMs(content) + if len(refs) != 1 { + t.Fatalf("expected 1 ref, got %d: %v", len(refs), refs) + } + if refs[0] != "onkernel/nodejs22-base:0.1.1" { + t.Errorf("expected onkernel/nodejs22-base:0.1.1, got %s", refs[0]) + } +} + +func TestParseDockerfileFROMs_MultiStage(t *testing.T) { + content := `FROM golang:1.21 AS builder +RUN go build -o /app . + +FROM alpine:3.21 +COPY --from=builder /app /app +` + refs := ParseDockerfileFROMs(content) + if len(refs) != 2 { + t.Fatalf("expected 2 refs, got %d: %v", len(refs), refs) + } + if refs[0] != "library/golang:1.21" { + t.Errorf("expected library/golang:1.21, got %s", refs[0]) + } + if refs[1] != "library/alpine:3.21" { + t.Errorf("expected library/alpine:3.21, got %s", refs[1]) + } +} + +func TestParseDockerfileFROMs_DockerIONormalization(t *testing.T) { + content := `FROM docker.io/library/alpine:3.21 +` + refs := ParseDockerfileFROMs(content) + if len(refs) != 1 { + t.Fatalf("expected 1 ref, got %d: %v", len(refs), refs) + } + if refs[0] != "library/alpine:3.21" { + t.Errorf("expected library/alpine:3.21, got %s", refs[0]) + } +} + +func TestParseDockerfileFROMs_PlatformFlag(t *testing.T) { + content := `FROM --platform=linux/amd64 node:20-alpine +RUN npm install +` + refs := ParseDockerfileFROMs(content) + if len(refs) != 1 { + t.Fatalf("expected 1 ref, got %d: %v", len(refs), refs) + } + if refs[0] != "library/node:20-alpine" { + t.Errorf("expected library/node:20-alpine, got %s", refs[0]) + } +} + +func TestParseDockerfileFROMs_SkipScratch(t *testing.T) { + content := `FROM golang:1.21 AS builder +RUN go build -o /app . + +FROM scratch +COPY --from=builder /app /app +` + refs := ParseDockerfileFROMs(content) + if len(refs) != 1 { + t.Fatalf("expected 1 ref, got %d: %v", len(refs), refs) + } + if refs[0] != "library/golang:1.21" { + t.Errorf("expected library/golang:1.21, got %s", refs[0]) + } +} + +func TestParseDockerfileFROMs_SkipStageReferences(t *testing.T) { + content := `FROM node:20 AS deps +RUN npm ci + +FROM node:20 AS builder +COPY --from=deps /app/node_modules ./node_modules +RUN npm run build + +FROM builder +CMD ["node", "dist/index.js"] +` + refs := ParseDockerfileFROMs(content) + if len(refs) != 1 { + t.Fatalf("expected 1 ref (deduplicated), got %d: %v", len(refs), refs) + } + if refs[0] != "library/node:20" { + t.Errorf("expected library/node:20, got %s", refs[0]) + } +} + +func TestParseDockerfileFROMs_SkipVariableReferences(t *testing.T) { + content := `ARG BASE_IMAGE=node:20 +FROM ${BASE_IMAGE} +RUN echo hello +` + refs := ParseDockerfileFROMs(content) + if len(refs) != 0 { + t.Fatalf("expected 0 refs (variable), got %d: %v", len(refs), refs) + } +} + +func TestParseDockerfileFROMs_Deduplication(t *testing.T) { + content := `FROM alpine:3.21 AS stage1 +RUN echo one + +FROM alpine:3.21 AS stage2 +RUN echo two +` + refs := ParseDockerfileFROMs(content) + if len(refs) != 1 { + t.Fatalf("expected 1 ref (deduplicated), got %d: %v", len(refs), refs) + } + if refs[0] != "library/alpine:3.21" { + t.Errorf("expected library/alpine:3.21, got %s", refs[0]) + } +} + +func TestParseDockerfileFROMs_CommentsAndEmptyLines(t *testing.T) { + content := `# Build stage +FROM golang:1.21 + +# This is a comment +# FROM fake:image + +RUN echo hello +` + refs := ParseDockerfileFROMs(content) + if len(refs) != 1 { + t.Fatalf("expected 1 ref, got %d: %v", len(refs), refs) + } + if refs[0] != "library/golang:1.21" { + t.Errorf("expected library/golang:1.21, got %s", refs[0]) + } +} + +func TestExtractDockerfileFromTarball(t *testing.T) { + // Create a temp tarball with a Dockerfile + dir := t.TempDir() + tarballPath := filepath.Join(dir, "source.tar.gz") + + dockerfileContent := "FROM alpine:3.21\nRUN echo hello\n" + createTarball(t, tarballPath, map[string]string{ + "Dockerfile": dockerfileContent, + "main.go": "package main\n", + }) + + content, err := ExtractDockerfileFromTarball(tarballPath) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if content != dockerfileContent { + t.Errorf("expected %q, got %q", dockerfileContent, content) + } +} + +func TestExtractDockerfileFromTarball_NotFound(t *testing.T) { + dir := t.TempDir() + tarballPath := filepath.Join(dir, "source.tar.gz") + + createTarball(t, tarballPath, map[string]string{ + "main.go": "package main\n", + }) + + _, err := ExtractDockerfileFromTarball(tarballPath) + if err == nil { + t.Fatal("expected error for missing Dockerfile") + } +} + +func TestExtractDockerfileFromTarball_DotSlashPrefix(t *testing.T) { + dir := t.TempDir() + tarballPath := filepath.Join(dir, "source.tar.gz") + + dockerfileContent := "FROM node:20\nRUN npm install\n" + createTarball(t, tarballPath, map[string]string{ + "./Dockerfile": dockerfileContent, + }) + + content, err := ExtractDockerfileFromTarball(tarballPath) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if content != dockerfileContent { + t.Errorf("expected %q, got %q", dockerfileContent, content) + } +} + +// createTarball creates a .tar.gz file with the given files (name -> content). +func createTarball(t *testing.T, path string, files map[string]string) { + t.Helper() + + f, err := os.Create(path) + if err != nil { + t.Fatalf("create tarball file: %v", err) + } + defer f.Close() + + gw := gzip.NewWriter(f) + defer gw.Close() + + tw := tar.NewWriter(gw) + defer tw.Close() + + for name, content := range files { + hdr := &tar.Header{ + Name: name, + Mode: 0644, + Size: int64(len(content)), + } + if err := tw.WriteHeader(hdr); err != nil { + t.Fatalf("write tar header for %s: %v", name, err) + } + if _, err := tw.Write([]byte(content)); err != nil { + t.Fatalf("write tar content for %s: %v", name, err) + } + } +} diff --git a/lib/builds/manager.go b/lib/builds/manager.go index 6da77ff1..960b70ca 100644 --- a/lib/builds/manager.go +++ b/lib/builds/manager.go @@ -452,6 +452,30 @@ func (m *manager) CreateBuild(ctx context.Context, req CreateBuildRequest, sourc } } + // Add pull access for base image repos so the builder agent can + // detect mirrored images via checkImageExistsInRegistry + dockerfileContent := req.Dockerfile + if dockerfileContent == "" { + tarballPath := m.paths.BuildSourceDir(id) + "/source.tar.gz" + if content, err := ExtractDockerfileFromTarball(tarballPath); err == nil { + dockerfileContent = content + } + } + if dockerfileContent != "" { + refs := ParseDockerfileFROMs(dockerfileContent) + seen := make(map[string]bool) + for _, ref := range refs { + repo := ref + if idx := strings.LastIndex(repo, ":"); idx > 0 { + repo = repo[:idx] + } + if !seen[repo] { + seen[repo] = true + repoAccess = append(repoAccess, RepoPermission{Repo: repo, Scope: "pull"}) + } + } + } + registryToken, err := m.tokenGenerator.GenerateToken(id, repoAccess, tokenTTL) if err != nil { deleteBuild(m.paths, id) @@ -519,6 +543,13 @@ func (m *manager) runBuild(ctx context.Context, id string, req CreateBuildReques buildCtx, cancel := context.WithTimeout(ctx, time.Duration(policy.TimeoutSeconds)*time.Second) defer cancel() + // Mirror base images to the local registry before launching the VM. + // BuildKit is configured with our registry as a mirror for docker.io, + // so pre-cached images will be served locally without pulling from Docker Hub. + if err := m.mirrorBaseImagesForBuild(buildCtx, id, req); err != nil { + m.logger.Warn("failed to mirror base images", "id", id, "error", err) + } + // Run the build in a builder VM result, err := m.executeBuild(buildCtx, id, req, policy) @@ -1304,6 +1335,30 @@ func (m *manager) refreshBuildToken(buildID string, req *CreateBuildRequest) err } } + // Add pull access for base image repos so the builder agent can + // detect mirrored images via checkImageExistsInRegistry + dockerfileContent := req.Dockerfile + if dockerfileContent == "" { + tarballPath := m.paths.BuildSourceDir(buildID) + "/source.tar.gz" + if content, err := ExtractDockerfileFromTarball(tarballPath); err == nil { + dockerfileContent = content + } + } + if dockerfileContent != "" { + refs := ParseDockerfileFROMs(dockerfileContent) + seen := make(map[string]bool) + for _, ref := range refs { + repo := ref + if idx := strings.LastIndex(repo, ":"); idx > 0 { + repo = repo[:idx] + } + if !seen[repo] { + seen[repo] = true + repoAccess = append(repoAccess, RepoPermission{Repo: repo, Scope: "pull"}) + } + } + } + // Generate fresh registry token registryToken, err := m.tokenGenerator.GenerateToken(buildID, repoAccess, tokenTTL) if err != nil { diff --git a/lib/builds/mirror.go b/lib/builds/mirror.go new file mode 100644 index 00000000..57797537 --- /dev/null +++ b/lib/builds/mirror.go @@ -0,0 +1,89 @@ +package builds + +import ( + "context" + "strings" + "time" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/kernel/hypeman/lib/images" +) + +// mirrorBaseImagesForBuild extracts base image references from the build's +// Dockerfile and mirrors each one to the local registry. BuildKit is configured +// with our registry as a mirror for docker.io, so pre-cached images will be +// served locally without pulling from Docker Hub. +// +// Individual mirror failures are logged but do not fail the build (graceful +// degradation — BuildKit will pull from Docker Hub as before). +func (m *manager) mirrorBaseImagesForBuild(ctx context.Context, id string, req CreateBuildRequest) error { + // Get Dockerfile content: prefer inline Dockerfile, fall back to tarball + var dockerfileContent string + if req.Dockerfile != "" { + dockerfileContent = req.Dockerfile + } else { + tarballPath := m.paths.BuildSourceDir(id) + "/source.tar.gz" + content, err := ExtractDockerfileFromTarball(tarballPath) + if err != nil { + m.logger.Warn("could not extract Dockerfile from tarball for mirroring", + "id", id, "error", err) + return nil + } + dockerfileContent = content + } + + // Parse FROM references + refs := ParseDockerfileFROMs(dockerfileContent) + if len(refs) == 0 { + return nil + } + + m.logger.Info("mirroring base images to local registry", "id", id, "images", refs) + + // Generate a scoped registry token that grants push access to the base + // image repos. The local registry requires JWT auth for all operations; + // go-containerregistry uses this via the Docker token auth flow (Basic + // auth username = JWT → /v2/token validates and returns bearer token). + // Build repo permissions. The Docker token scope uses the repo name without + // the tag (e.g. "onkernel/nodejs22-base", not "onkernel/nodejs22-base:0.1.1"). + seen := make(map[string]bool) + var repoPerms []RepoPermission + for _, ref := range refs { + repo := ref + if idx := strings.LastIndex(repo, "@"); idx != -1 { + repo = repo[:idx] + } + if idx := strings.LastIndex(repo, ":"); idx > 0 { + repo = repo[:idx] + } + if !seen[repo] { + seen[repo] = true + repoPerms = append(repoPerms, RepoPermission{Repo: repo, Scope: "push"}) + } + } + registryToken, err := m.tokenGenerator.GenerateToken(id, repoPerms, 10*time.Minute) + if err != nil { + m.logger.Warn("failed to generate registry token for mirroring", + "id", id, "error", err) + return nil + } + // go-containerregistry's basicTransport only sends Basic auth when BOTH + // Username and Password are non-empty. The password value doesn't matter — + // our token handler extracts the JWT from the username field only. + authConfig := &authn.AuthConfig{Username: registryToken, Password: "x"} + + for _, ref := range refs { + result, err := images.MirrorBaseImage(ctx, m.config.RegistryURL, images.MirrorRequest{ + SourceImage: ref, + }, authConfig) + if err != nil { + m.logger.Warn("failed to mirror base image", + "id", id, "image", ref, "error", err) + continue + } + m.logger.Info("mirrored base image", + "id", id, "image", ref, "local_ref", result.LocalRef, "digest", result.Digest) + } + + return nil +} diff --git a/lib/images/mirror.go b/lib/images/mirror.go new file mode 100644 index 00000000..c3aa2380 --- /dev/null +++ b/lib/images/mirror.go @@ -0,0 +1,131 @@ +package images + +import ( + "context" + "fmt" + "strings" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote" +) + +// MirrorRequest contains the parameters for mirroring a base image +type MirrorRequest struct { + // SourceImage is the full image reference to pull from (e.g., "docker.io/onkernel/nodejs22-base:0.1.1") + SourceImage string +} + +// MirrorResult contains the result of a mirror operation +type MirrorResult struct { + // SourceImage is the original image reference + SourceImage string `json:"source_image"` + // LocalRef is the local registry reference (e.g., "onkernel/nodejs22-base:0.1.1") + LocalRef string `json:"local_ref"` + // Digest is the image digest + Digest string `json:"digest"` +} + +// MirrorBaseImage pulls an image from an external registry and pushes it to the +// local registry with the same normalized name. This enables Dockerfile FROM rewriting +// to use locally mirrored base images instead of pulling from Docker Hub. +// +// For example, mirroring "docker.io/onkernel/nodejs22-base:0.1.1" will create +// "onkernel/nodejs22-base:0.1.1" in the local registry. +func MirrorBaseImage(ctx context.Context, registryURL string, req MirrorRequest, authConfig *authn.AuthConfig) (*MirrorResult, error) { + // Parse source reference + srcRef, err := name.ParseReference(req.SourceImage) + if err != nil { + return nil, fmt.Errorf("parse source image reference: %w", err) + } + + // Pull the image from source + img, err := remote.Image(srcRef, + remote.WithContext(ctx), + remote.WithAuthFromKeychain(authn.DefaultKeychain), + remote.WithPlatform(vmPlatform())) + if err != nil { + return nil, fmt.Errorf("pull source image: %w", wrapRegistryError(err)) + } + + // Get the digest + digest, err := img.Digest() + if err != nil { + return nil, fmt.Errorf("get image digest: %w", err) + } + + // Build the local reference under bases/ namespace + // Normalize the source to strip docker.io/ prefix for cleaner local refs + localRef := normalizeToLocalRef(srcRef) + + // Strip any scheme from registry URL + registryHost := stripScheme(registryURL) + + // Build full destination reference + dstRefStr := fmt.Sprintf("%s/%s", registryHost, localRef) + dstRef, err := name.ParseReference(dstRefStr) + if err != nil { + return nil, fmt.Errorf("parse destination reference: %w", err) + } + + // Push to local registry + // For insecure registries, we need to use the insecure transport + opts := []remote.Option{ + remote.WithContext(ctx), + } + + // If authConfig is provided, use it + if authConfig != nil { + opts = append(opts, remote.WithAuth(authn.FromConfig(*authConfig))) + } + + if err := remote.Write(dstRef, img, opts...); err != nil { + return nil, fmt.Errorf("push to local registry: %w", wrapRegistryError(err)) + } + + return &MirrorResult{ + SourceImage: req.SourceImage, + LocalRef: localRef, + Digest: digest.String(), + }, nil +} + +// normalizeToLocalRef converts a source image reference to a normalized local reference. +// It strips the docker.io/ prefix but preserves the library/ prefix for official images. +// The library/ prefix is kept because BuildKit's mirror protocol requests official images +// as library/ (e.g., /v2/library/node/manifests/...). +// +// Examples: +// - "docker.io/onkernel/nodejs22-base:0.1.1" -> "onkernel/nodejs22-base:0.1.1" +// - "docker.io/library/alpine:3.21" -> "library/alpine:3.21" +// - "node:20-alpine" -> "library/node:20-alpine" (go-containerregistry canonicalizes to library/) +// - "gcr.io/google-containers/pause:3.2" -> "gcr.io/google-containers/pause:3.2" +func normalizeToLocalRef(ref name.Reference) string { + // Get the repository name (includes registry for non-Docker Hub images) + repo := ref.Context().String() + + // Strip index.docker.io/ prefix (canonical form of docker.io) + repo = strings.TrimPrefix(repo, "index.docker.io/") + + // Strip docker.io/ prefix + repo = strings.TrimPrefix(repo, "docker.io/") + + // Keep library/ prefix — BuildKit mirror requests use it for official images + + // Build the tag or digest suffix + var suffix string + if tag, ok := ref.(name.Tag); ok { + suffix = ":" + tag.TagStr() + } else if dig, ok := ref.(name.Digest); ok { + suffix = "@" + dig.DigestStr() + } + + return repo + suffix +} + +// stripScheme removes http:// or https:// prefix from a URL +func stripScheme(url string) string { + url = strings.TrimPrefix(url, "https://") + url = strings.TrimPrefix(url, "http://") + return url +} diff --git a/lib/images/mirror_test.go b/lib/images/mirror_test.go new file mode 100644 index 00000000..30fe0d3c --- /dev/null +++ b/lib/images/mirror_test.go @@ -0,0 +1,91 @@ +package images + +import ( + "testing" + + "github.com/google/go-containerregistry/pkg/name" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNormalizeToLocalRef(t *testing.T) { + tests := []struct { + name string + input string + expected string + }{ + { + name: "docker hub user image with tag", + input: "docker.io/onkernel/nodejs22-base:0.1.1", + expected: "onkernel/nodejs22-base:0.1.1", + }, + { + name: "docker hub user image without registry prefix", + input: "onkernel/nodejs22-base:0.1.1", + expected: "onkernel/nodejs22-base:0.1.1", + }, + { + name: "docker hub official image with tag", + input: "docker.io/library/alpine:3.21", + expected: "library/alpine:3.21", + }, + { + name: "docker hub official image short form", + input: "alpine:3.21", + expected: "library/alpine:3.21", + }, + { + name: "docker hub image with index.docker.io", + input: "index.docker.io/onkernel/nodejs22-base:0.1.1", + expected: "onkernel/nodejs22-base:0.1.1", + }, + { + name: "gcr.io image", + input: "gcr.io/google-containers/pause:3.2", + expected: "gcr.io/google-containers/pause:3.2", + }, + { + name: "ghcr.io image", + input: "ghcr.io/some-org/some-image:v1.0", + expected: "ghcr.io/some-org/some-image:v1.0", + }, + { + name: "image with latest tag", + input: "nginx:latest", + expected: "library/nginx:latest", + }, + { + name: "image without tag uses latest", + input: "nginx", + expected: "library/nginx:latest", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ref, err := name.ParseReference(tt.input) + require.NoError(t, err) + result := normalizeToLocalRef(ref) + assert.Equal(t, tt.expected, result) + }) + } +} + +func TestStripScheme(t *testing.T) { + tests := []struct { + input string + expected string + }{ + {"https://localhost:8080", "localhost:8080"}, + {"http://localhost:8080", "localhost:8080"}, + {"localhost:8080", "localhost:8080"}, + {"https://registry.example.com", "registry.example.com"}, + } + + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + result := stripScheme(tt.input) + assert.Equal(t, tt.expected, result) + }) + } +} diff --git a/lib/middleware/oapi_auth.go b/lib/middleware/oapi_auth.go index f60b25a1..a2f25dd1 100644 --- a/lib/middleware/oapi_auth.go +++ b/lib/middleware/oapi_auth.go @@ -3,6 +3,7 @@ package middleware import ( "context" "encoding/base64" + "errors" "fmt" "net/http" "strings" @@ -14,6 +15,9 @@ import ( "github.com/kernel/hypeman/lib/logger" ) +// errRepoNotAllowed is returned when a valid token doesn't have access to the requested repository. +var errRepoNotAllowed = errors.New("repository not allowed by token") + type contextKey string const userIDKey contextKey = "user_id" @@ -320,7 +324,7 @@ func validateRegistryToken(tokenString, jwtSecret, requestPath, method string) ( } if !allowed { - return nil, fmt.Errorf("repository %s not allowed by token", repo) + return nil, fmt.Errorf("%w: %s", errRepoNotAllowed, repo) } // Check scope for write operations @@ -375,6 +379,20 @@ func JwtAuth(jwtSecret string) func(http.Handler) http.Handler { return } log.DebugContext(r.Context(), "registry token validation failed", "error", err) + + // For read operations (GET/HEAD), if the token is valid but the + // repo isn't in the allowed list, return 502 Bad Gateway. + // BuildKit treats 5xx from a mirror as "mirror unavailable" and + // falls back to the upstream registry (Docker Hub). A 404 would + // be treated as "image doesn't exist" with no fallback. + if errors.Is(err, errRepoNotAllowed) && !isWriteOperation(r.Method) { + log.DebugContext(r.Context(), "returning 502 for mirror fallback", + "path", r.URL.Path) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusBadGateway) + fmt.Fprintf(w, `{"errors":[{"code":"UNAVAILABLE","message":"image not mirrored"}]}`) + return + } } else { log.DebugContext(r.Context(), "failed to extract token", "error", err) } diff --git a/lib/registry/auth_integration_test.go b/lib/registry/auth_integration_test.go index 62d6eabb..4fb2827e 100644 --- a/lib/registry/auth_integration_test.go +++ b/lib/registry/auth_integration_test.go @@ -105,9 +105,10 @@ func TestBuildKitAuthFlow(t *testing.T) { assert.Equal(t, http.StatusOK, resp.StatusCode) }) - t.Run("authenticated request for unauthorized repo returns 403", func(t *testing.T) { - // Token only allows access to builds/build-123 and cache/org-test - // Request for a different repo should fail with 403 + t.Run("authenticated request for unauthorized repo returns token for mirror fallback", func(t *testing.T) { + // Token only allows access to builds/build-123 and cache/org-test. + // Token endpoint returns 200 anyway — access control is enforced + // at the middleware layer. This enables BuildKit mirror fallback. req, err := http.NewRequest(http.MethodGet, server.URL+"/v2/token?scope=repository:builds/other-build:push&service=hypeman", nil) require.NoError(t, err) @@ -118,7 +119,7 @@ func TestBuildKitAuthFlow(t *testing.T) { require.NoError(t, err) defer resp.Body.Close() - assert.Equal(t, http.StatusForbidden, resp.StatusCode) + assert.Equal(t, http.StatusOK, resp.StatusCode) }) } @@ -156,9 +157,9 @@ func TestDockerConfigCredentialLookup(t *testing.T) { expectedStatus: http.StatusOK, }, { - name: "unauthorized repo", + name: "unauthorized repo returns token for mirror fallback", scope: "repository:builds/other:push", - expectedStatus: http.StatusForbidden, + expectedStatus: http.StatusOK, }, } diff --git a/lib/registry/token.go b/lib/registry/token.go index 029aedf1..2c0a6185 100644 --- a/lib/registry/token.go +++ b/lib/registry/token.go @@ -80,16 +80,17 @@ func (h *TokenHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - // Check if requested scope is allowed by the token + // Check if requested scope is allowed by the token. + // If not, still return a valid token — the subsequent manifest request + // will get a 404 (not found) instead of 403. This is critical for BuildKit + // mirror fallback: a 403 on the token endpoint is treated as a hard auth + // failure and prevents fallback to upstream registries like Docker Hub. if scope != "" { repo, actions := parseScope(scope) if repo != "" && !h.isScopeAllowed(claims, repo, actions) { - log.DebugContext(r.Context(), "scope not allowed by token", + log.DebugContext(r.Context(), "scope not in token, returning token anyway for mirror fallback", "requested_repo", repo, - "requested_actions", actions, - "allowed_repos", claims["repos"]) - h.writeError(w, http.StatusForbidden, "DENIED", "requested scope not allowed") - return + "requested_actions", actions) } } diff --git a/lib/registry/token_test.go b/lib/registry/token_test.go index 311f410b..7d1e6011 100644 --- a/lib/registry/token_test.go +++ b/lib/registry/token_test.go @@ -91,8 +91,10 @@ func TestTokenHandler_BearerAuth(t *testing.T) { func TestTokenHandler_ScopeValidation(t *testing.T) { handler := NewTokenHandler(testJWTSecret) - t.Run("scope not in token is rejected", func(t *testing.T) { - // Token allows builds/build-123, but request is for builds/other + t.Run("scope not in token still returns token for mirror fallback", func(t *testing.T) { + // Token allows builds/build-123, but request is for builds/other. + // Token endpoint returns 200 anyway — access control is enforced + // at the middleware layer. This enables BuildKit mirror fallback. registryToken := generateRegistryToken(t, "build-123", []string{"builds/build-123"}, "push", time.Hour) basicAuth := base64.StdEncoding.EncodeToString([]byte(registryToken + ":")) @@ -102,11 +104,13 @@ func TestTokenHandler_ScopeValidation(t *testing.T) { rr := httptest.NewRecorder() handler.ServeHTTP(rr, req) - assert.Equal(t, http.StatusForbidden, rr.Code) + assert.Equal(t, http.StatusOK, rr.Code) }) - t.Run("push action with pull-only token is rejected", func(t *testing.T) { - // Token only has pull scope + t.Run("push action with pull-only token still returns token for mirror fallback", func(t *testing.T) { + // Token only has pull scope but requests push. + // Token endpoint returns 200 anyway — access control is enforced + // at the middleware layer. registryToken := generateRegistryToken(t, "build-123", []string{"builds/build-123"}, "pull", time.Hour) basicAuth := base64.StdEncoding.EncodeToString([]byte(registryToken + ":")) @@ -116,7 +120,7 @@ func TestTokenHandler_ScopeValidation(t *testing.T) { rr := httptest.NewRecorder() handler.ServeHTTP(rr, req) - assert.Equal(t, http.StatusForbidden, rr.Code) + assert.Equal(t, http.StatusOK, rr.Code) }) t.Run("pull action with push token is allowed", func(t *testing.T) {