diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml
new file mode 100644
index 0000000..cc37540
--- /dev/null
+++ b/.github/workflows/go.yml
@@ -0,0 +1,27 @@
+# This workflow will build a golang project
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-go
+
+name: Go
+
+on:
+ workflow_dispatch:
+ pull_request:
+ branches: [ '*' ]
+
+jobs:
+
+ build:
+ runs-on: [self-hosted, Linux]
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Set up Go
+ uses: actions/setup-go@v4
+ with:
+ go-version: '1.20'
+
+ - name: Build
+ run: go build -v ./...
+
+ - name: Test
+ run: go test -v ./...
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
new file mode 100644
index 0000000..84ddaaf
--- /dev/null
+++ b/.github/workflows/lint.yml
@@ -0,0 +1,53 @@
+name: golangci-lint
+on:
+ pull_request:
+
+permissions:
+ # Required: allow read access to the content for analysis.
+ contents: read
+ # Optional: allow read access to pull request. Use with `only-new-issues` option.
+ pull-requests: read
+ # Optional: Allow write access to checks to allow the action to annotate code in the PR.
+ checks: write
+
+env:
+ GOPRIVATE: "github.com/opensourceways"
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ golangci:
+ name: lint
+ runs-on: [self-hosted, Linux]
+ steps:
+ - uses: actions/checkout@v4
+ - name: config github
+ env:
+ TOKEN: ${{ secrets.ACCESS_TOKEN }}
+ USER: ${{ secrets.USER_NAME }}
+ run: echo "machine github.com login $USER password $TOKEN" > ~/.netrc
+ - name: golangci-lint
+ uses: golangci/golangci-lint-action@master
+ with:
+ # Require: The version of golangci-lint to use.
+ # When `install-mode` is `binary` (default) the value can be v1.2 or v1.2.3 or `latest` to use the latest version.
+ # When `install-mode` is `goinstall` the value can be v1.2.3, `latest`, or the hash of a commit.
+ version: latest
+ # Optional: golangci-lint command line arguments.
+ #
+ # Note: By default, the `.golangci.yml` file should be at the root of the repository.
+ # The location of the configuration file can be changed by using `--config=`
+ args: -v --config=.golangci.yml --max-same-issues=0
+
+ - name: Run Gosec Security Scanner
+ uses: securego/gosec@master
+ env:
+ GITHUB_AUTHENTICATION_TOKEN: ${{ secrets.ACCESS_TOKEN }}
+ with:
+ args: -exclude-dir api -exclude-dir tests ./...
+ - name: Unit tests
+ run: go test -race -count=1 -v ./...
+ - name: cleanup
+ run: rm -f ~/.netrc
diff --git a/.github/workflows/typos.yml b/.github/workflows/typos.yml
new file mode 100644
index 0000000..82fec9a
--- /dev/null
+++ b/.github/workflows/typos.yml
@@ -0,0 +1,26 @@
+name: Spell Check Action
+on: [pull_request]
+
+jobs:
+ run:
+ name: Spell Check with Typos
+ runs-on: self-hosted
+ steps:
+ - name: Checkout Actions Repository
+ uses: actions/checkout@v4
+
+ - name: Check spelling
+ uses: crate-ci/typos@master
+ with:
+ files: ./
+
+ - name: Use custom config file
+ uses: crate-ci/typos@master
+ with:
+ files: ./
+ config: ./typos.toml
+
+ - name: Writes changes in the local checkout
+ uses: crate-ci/typos@master
+ with:
+ write_changes: true
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
index 722d5e7..aafda3a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1 +1,2 @@
.vscode
+.idea
diff --git a/.golangci.yml b/.golangci.yml
new file mode 100644
index 0000000..6b17898
--- /dev/null
+++ b/.golangci.yml
@@ -0,0 +1,53 @@
+run:
+ # Timeout for analysis, e.g. 30s, 5m.
+ # Default: 1m
+ timeout: 10m
+
+output:
+ # Print linter name in the end of issue text.
+ # Default: true
+ print-linter-name: true
+ print-issued-lines: true
+
+linters-settings:
+ errorlint:
+ # Check whether fmt.Errorf uses the %w verb for formatting errors.
+ # See the https://github.com/polyfloyd/go-errorlint for caveats.
+ # Default: true
+ errorf: true
+ # Permit more than 1 %w verb, valid per Go 1.20 (Requires errorf:true)
+ # Default: true
+ errorf-multi: false
+ # Check for plain type assertions and type switches.
+ # Default: true
+ asserts: true
+ # Check for plain error comparisons.
+ # Default: true
+ comparison: true
+ lll:
+ # Max line length, lines longer will be reported.
+ # '\t' is counted as 1 character by default, and can be changed with the tab-width option.
+ # Default: 120.
+ line-length: 120
+ # Tab width in spaces.
+ # Default: 1
+ tab-width: 1
+linters:
+ enable:
+ - errorlint
+ - errcheck
+ - gosimple
+ - govet
+ - ineffassign
+ - staticcheck
+ - unused
+ - lll
+issues:
+ exclude-rules:
+ - linters:
+ - lll
+ source: "^// "
+ - path: api
+ linters:
+ - lll
+ exclude-generated-strict: true
diff --git a/BigFiles/go.mod b/BigFiles/go.mod
index d38866c..ea0d6ad 100644
--- a/BigFiles/go.mod
+++ b/BigFiles/go.mod
@@ -5,4 +5,8 @@ go 1.15
require (
github.com/akrylysov/algnhsa v0.12.1
github.com/metalogical/BigFiles v0.0.0-20201031233020-8aa76a58aba7
+ github.com/sirupsen/logrus v1.9.3
)
+
+replace github.com/metalogical/BigFiles => github.com/opensourceways/BigFiles v0.0.0-20240913072808-2dbf11feb204
+
diff --git a/BigFiles/go.sum b/BigFiles/go.sum
index 625c4fd..568a62f 100644
--- a/BigFiles/go.sum
+++ b/BigFiles/go.sum
@@ -3,23 +3,29 @@ github.com/akrylysov/algnhsa v0.12.1/go.mod h1:xAcJ/X8DV+81e+dUjIoB/r5CbISrSXV9/
github.com/aws/aws-lambda-go v1.9.0 h1:r9TWtk8ozLYdMW+aelUeWny8z2mjghJCMx6/uUwOLNo=
github.com/aws/aws-lambda-go v1.9.0/go.mod h1:zUsUQhAUjYzR8AuduJPCfhBuKWUaDbQiPOG+ouzmE1A=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/go-chi/chi v4.1.2+incompatible h1:fGFk2Gmi/YKXk0OmGfBh0WgmN3XB8lVnEyNz34tQRec=
github.com/go-chi/chi v4.1.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s=
github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/metalogical/BigFiles v0.0.0-20201031233020-8aa76a58aba7 h1:eSaJmOvPgVx/ARHCkfmldLvAlHZu1F94ttYGN+bxQ1k=
github.com/metalogical/BigFiles v0.0.0-20201031233020-8aa76a58aba7/go.mod h1:uTTfhdJzUTvmZEVl3RqOblXiZqlWS2imssQl2EEJabU=
github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4=
github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw=
@@ -35,14 +41,24 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/opensourceways/BigFiles v0.0.0-20240913072808-2dbf11feb204 h1:dxbM1M8kSEG+1LRn1Rx4wgH+s1eOCDpG/OpVEbkvsUc=
+github.com/opensourceways/BigFiles v0.0.0-20240913072808-2dbf11feb204/go.mod h1:uTTfhdJzUTvmZEVl3RqOblXiZqlWS2imssQl2EEJabU=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
+github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
+github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs=
github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 h1:DZhuSZLsGlFL4CmhA8BcRA0mnthyA/nZ00AqCUo7vHg=
@@ -54,16 +70,23 @@ golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81R
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae h1:Ih9Yo4hSPImZOpfGuA4bR/ORKTAbhZo2AbWNRCnevdo=
golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww=
gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=
+sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY=
diff --git a/BigFiles/main.go b/BigFiles/main.go
index f5c33b8..68d8e13 100644
--- a/BigFiles/main.go
+++ b/BigFiles/main.go
@@ -1,6 +1,8 @@
package main
import (
+ "flag"
+ "fmt"
"log"
"net/http"
"os"
@@ -8,19 +10,107 @@ import (
"github.com/akrylysov/algnhsa"
"github.com/metalogical/BigFiles/auth"
+ "github.com/metalogical/BigFiles/config"
"github.com/metalogical/BigFiles/server"
+ "github.com/sirupsen/logrus"
)
+type options struct {
+ service ServiceOptions
+ enableDebug bool
+}
+
+type ServiceOptions struct {
+ ConfigFile string
+ RemoveCfg bool
+}
+
+// Validate checks if the ServiceOptions are valid.
+// It returns an error if the config file is missing.
+func (o *ServiceOptions) Validate() error {
+ if o.ConfigFile == "" {
+ return fmt.Errorf("missing config-file")
+ }
+
+ return nil
+}
+
+// AddFlags adds flags for ServiceOptions to the provided FlagSet.
+func (o *ServiceOptions) AddFlags(fs *flag.FlagSet) {
+ fs.StringVar(&o.ConfigFile, "config-file", "", "Path to config file.")
+ fs.BoolVar(&o.RemoveCfg, "rm-cfg", false, "whether remove the cfg file after initialized .")
+}
+
+// Validate validates the options and returns an error if any validation fails.
+func (o *options) Validate() error {
+ return o.service.Validate()
+}
+
+func gatherOptions(fs *flag.FlagSet, args ...string) (options, error) {
+ var o options
+ o.service.AddFlags(fs)
+
+ fs.BoolVar(
+ &o.enableDebug, "enable_debug", false, "whether to enable debug model.",
+ )
+
+ err := fs.Parse(args)
+ return o, err
+}
+
func main() {
- bucket := os.Getenv("LFS_BUCKET")
+ o, err := gatherOptions(
+ flag.NewFlagSet(os.Args[0], flag.ExitOnError),
+ os.Args[1:]...,
+ )
+ if err != nil {
+ logrus.Errorf("new options failed, err:%s", err.Error())
+
+ return
+ }
+
+ if err := o.Validate(); err != nil {
+ logrus.Errorf("Invalid options, err:%s", err.Error())
+
+ return
+ }
+
+ if o.enableDebug {
+ logrus.SetLevel(logrus.DebugLevel)
+ logrus.Debug("debug enable.")
+ }
+
+ //cfg
+ cfg := new(config.Config)
+
+ if err := config.LoadConfig(o.service.ConfigFile, cfg, o.service.RemoveCfg); err != nil {
+ logrus.Errorf("load config, err:%s", err.Error())
+
+ return
+ }
+
+ if err := auth.Init(cfg); err != nil {
+ logrus.Errorf("load gitee config, err:%s", err.Error())
+
+ return
+ }
+
+ bucket := cfg.LfsBucket
if bucket == "" {
- log.Fatalln("LFS_BUCKET must be set")
+ bucket = os.Getenv("LFS_BUCKET")
+ if bucket == "" {
+ logrus.Errorf("LFS_BUCKET must be set")
+ }
}
s, err := server.New(server.Options{
- S3Accelerate: true,
- Bucket: bucket,
- IsAuthorized: auth.GiteeAuth(),
+ S3Accelerate: true,
+ Bucket: bucket,
+ IsAuthorized: auth.GiteeAuth(),
+ Endpoint: cfg.AwsRegion,
+ AccessKeyID: cfg.AwsAccessKeyId,
+ SecretAccessKey: cfg.AwsSecretAccessKey,
+ Prefix: cfg.Prefix,
})
if err != nil {
log.Fatalln(err)
diff --git a/README.md b/README.md
index 3e7a4fa..098405f 100644
--- a/README.md
+++ b/README.md
@@ -8,10 +8,11 @@ v2.12.0](https://github.com/git-lfs/git-lfs/tree/v2.12.0/docs/api) server.
**TODO**:
-1. 外部参数校验：username、password进行格式校验。
+1. 外部参数校验：username、password、repo_id等进行格式校验。
2. 添加配置文件，将AKSK等写入配置文件中。可参考merlin-server配置文件的格式与读取方式。
3. 添加测试用例。
-4. 认证方式支持token。
-5. 认证时校验用户在仓库内权限。
+4. ~~认证方式支持token。~~
+5. ~~认证时校验用户在仓库内权限。~~
6. 支持ssh。
7. 仓库添加github action。
+8. 添加日志。
diff --git a/auth/gitee.go b/auth/gitee.go
index d695e63..230436f 100644
--- a/auth/gitee.go
+++ b/auth/gitee.go
@@ -3,44 +3,83 @@ package auth
import (
"encoding/json"
"errors"
+ "fmt"
"io"
"net/http"
"net/url"
"os"
"strings"
+
+ "github.com/metalogical/BigFiles/config"
+)
+
+var (
+ client_id string
+ client_secret string
)
+var uploadPermissions = []string{"admin", "developer"}
+var downloadPermissions = []string{"admin", "developer", "read"}
+
type giteeUser struct {
- Login string `json:"login"`
- // Permission string `json:"permission"`
+ Login string `json:"login"`
+ Permission string `json:"permission"`
+}
+
+type UserInRepo struct {
+ Repo string
+ Owner string
+ Token string
+ Username string
+ Password string
+ Operation string
}
type AccessToken struct {
Token string `json:"access_token"`
}
-func GiteeAuth() func(string, string) error {
- return func(username, password string) error {
- token, err := getToken(username, password)
+func Init(cfg *config.Config) error {
+ client_id = cfg.ClientId
+ if client_id == "" {
+ client_id = os.Getenv("CLIENT_ID")
+ if client_id == "" {
+ return errors.New("client id required")
+ }
+ }
+ client_secret = cfg.ClientSecret
+ if client_secret == "" {
+ client_secret = os.Getenv("CLIENT_SECRET")
+ if client_secret == "" {
+ return errors.New("client secret required")
+ }
+ }
+
+ return nil
+}
+
+func GiteeAuth() func(UserInRepo) error {
+ return func(userInRepo UserInRepo) error {
+ token, err := getToken(userInRepo.Username, userInRepo.Password)
if err != nil {
- return err
+ userInRepo.Token = userInRepo.Password
+ } else {
+ userInRepo.Token = token
}
- return verifyUser(username, token)
+ return verifyUser(userInRepo)
}
}
// getToken gets access_token by username and password
func getToken(username, password string) (string, error) {
- clientId := os.Getenv("CLIENT_ID")
- clientSecret := os.Getenv("CLIENT_SECRET")
form := url.Values{}
- form.Add("scope", "user_info")
+ form.Add("scope", "user_info projects")
form.Add("grant_type", "password")
form.Add("username", username)
form.Add("password", password)
- form.Add("client_id", clientId)
- form.Add("client_secret", clientSecret)
+ form.Add("client_id", client_id)
+ form.Add("client_secret", client_secret)
path := "https://gitee.com/oauth/token"
response, err := http.Post(path, "application/x-www-form-urlencoded", strings.NewReader(form.Encode()))
@@ -64,9 +103,15 @@ func getToken(username, password string) (string, error) {
return accessToken.Token, nil
}
-// verifyUser verifies user info by access_token
-func verifyUser(username, token string) error {
- path := "https://gitee.com/api/v5/user?access_token=" + token
+// verifyUser verifies user permission in repo by access_token
+func verifyUser(userInRepo UserInRepo) error {
+ path := fmt.Sprintf(
+ "https://gitee.com/api/v5/repos/%s/%s/collaborators/%s/permission?access_token=%s",
+ userInRepo.Owner,
+ userInRepo.Repo,
+ userInRepo.Username,
+ userInRepo.Token,
+ )
req, err := http.NewRequest("GET", path, nil)
if err != nil {
panic(err)
@@ -90,9 +135,25 @@ func verifyUser(username, token string) error {
if err != nil {
panic(err)
}
- if giteeUser.Login == username {
- return nil
- } else {
+
+ if giteeUser.Login != userInRepo.Username {
return errors.New("username does not match")
}
+ if userInRepo.Operation == "upload" {
+ for _, v := range uploadPermissions {
+ if giteeUser.Permission == v {
+ return nil
+ }
+ }
+ return errors.New("user has no permission uploading to the repository")
+ } else if userInRepo.Operation == "download" {
+ for _, v := range downloadPermissions {
+ if giteeUser.Permission == v {
+ return nil
+ }
+ }
+ return errors.New("user has no permission downloading in the repository")
+ } else {
+ return errors.New("unknow operation")
+ }
}
diff --git a/config.yml.example b/config.yml.example
new file mode 100644
index 0000000..1ef1e92
--- /dev/null
+++ b/config.yml.example
@@ -0,0 +1,7 @@
+LFS_BUCKET: ***********
+AWS_REGION: ***********
+AWS_ACCESS_KEY_ID: ***********
+AWS_SECRET_ACCESS_KEY: ***********
+CLIENT_ID: ***********
+CLIENT_SECRET: ***********
+PATH_PREFIX: ***********
\ No newline at end of file
diff --git a/config/config.go b/config/config.go
new file mode 100644
index 0000000..a7a4d8b
--- /dev/null
+++ b/config/config.go
@@ -0,0 +1,29 @@
+package config
+
+import (
+ "os"
+
+ "github.com/metalogical/BigFiles/utils"
+)
+
+type Config struct {
+ LfsBucket string `json:"LFS_BUCKET"`
+ AwsRegion string `json:"AWS_REGION"`
+ AwsAccessKeyId string `json:"AWS_ACCESS_KEY_ID"`
+ AwsSecretAccessKey string `json:"AWS_SECRET_ACCESS_KEY"`
+ ClientId string `json:"CLIENT_ID"`
+ ClientSecret string `json:"CLIENT_SECRET"`
+ Prefix string `json:"PATH_PREFIX"`
+}
+
+// LoadConfig loads the configuration file from the specified path and deletes the file if needed
+func LoadConfig(path string, cfg *Config, remove bool) error {
+ if remove {
+ defer os.Remove(path)
+ }
+
+ if err := utils.LoadFromYaml(path, cfg); err != nil {
+ return err
+ }
+ return nil
+}
diff --git a/go.mod b/go.mod
index 14bcbcb..f6b4636 100644
--- a/go.mod
+++ b/go.mod
@@ -5,4 +5,7 @@ go 1.14
require (
github.com/go-chi/chi v4.1.2+incompatible
github.com/minio/minio-go/v7 v7.0.5
+ sigs.k8s.io/yaml v1.4.0
)
+
+replace github.com/metalogical/BigFiles => github.com/opensourceways/BigFiles v0.0.0-20240913072808-2dbf11feb204
diff --git a/go.sum b/go.sum
index 9177364..1c0f992 100644
--- a/go.sum
+++ b/go.sum
@@ -1,25 +1,29 @@
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/go-chi/chi v1.0.0 h1:s/kv1cTXfivYjdKJdyUzNGyAWZ/2t7duW1gKn5ivu+c=
github.com/go-chi/chi v4.1.2+incompatible h1:fGFk2Gmi/YKXk0OmGfBh0WgmN3XB8lVnEyNz34tQRec=
github.com/go-chi/chi v4.1.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s=
github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4=
github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw=
-github.com/minio/minio-go v1.0.0 h1:ooSujki+Z1PRGZsYffJw5jnF5eMBvzMVV86TLAlM0UM=
-github.com/minio/minio-go v6.0.14+incompatible h1:fnV+GD28LeqdN6vT2XdGKW8Qe/IfjJDswNVuni6km9o=
github.com/minio/minio-go/v7 v7.0.5 h1:I2NIJ2ojwJqD/YByemC1M59e1b4FW9kS7NlOar7HPV4=
github.com/minio/minio-go/v7 v7.0.5/go.mod h1:TA0CQCjJZHM5SJj9IjqR0NmpmQJ6bCbXifAJ3mUU6Hw=
github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU=
@@ -32,13 +36,17 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs=
github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
@@ -59,8 +67,12 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww=
gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=
+sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY=
diff --git a/server/server.go b/server/server.go
index e782843..878535d 100644
--- a/server/server.go
+++ b/server/server.go
@@ -11,6 +11,7 @@ import (
"time"
"github.com/go-chi/chi"
+ "github.com/metalogical/BigFiles/auth"
"github.com/metalogical/BigFiles/batch"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
@@ -35,7 +36,7 @@ type Options struct {
TTL time.Duration // defaults to 1 hour
Prefix string
- IsAuthorized func(string, string) error
+ IsAuthorized func(auth.UserInRepo) error
}
func (o Options) imputeFromEnv() (Options, error) {
@@ -105,7 +106,7 @@ type server struct {
ttl time.Duration
prefix string
- isAuthorized func(string, string) error
+ isAuthorized func(auth.UserInRepo) error
}
func (s *server) key(oid string) string {
@@ -116,10 +117,27 @@ func (s *server) handleBatch(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/vnd.git-lfs+json")
w.Header().Set("X-Content-Type-Options", "nosniff")
+ var req batch.Request
+ err := json.NewDecoder(r.Body).Decode(&req)
+ if err != nil {
+ w.WriteHeader(404)
+ must(json.NewEncoder(w).Encode(batch.ErrorResponse{
+ Message: "could not parse request",
+ DocURL: "https://github.com/git-lfs/git-lfs/blob/v2.12.0/docs/api/batch.md#requests",
+ }))
+ return
+ }
+
+ var userInRepo auth.UserInRepo
+ userInRepo.Operation = req.Operation
+ userInRepo.Owner = chi.URLParam(r, "owner")
+ userInRepo.Repo = chi.URLParam(r, "repo")
if s.isAuthorized != nil {
var err error
if username, password, ok := r.BasicAuth(); ok {
- err = s.isAuthorized(username, password)
+ userInRepo.Username = username
+ userInRepo.Password = password
+ err = s.isAuthorized(userInRepo)
if err != nil {
err = fmt.Errorf("unauthorized: %w", err)
}
@@ -137,17 +155,6 @@ func (s *server) handleBatch(w http.ResponseWriter, r *http.Request) {
}
}
- var req batch.Request
- err := json.NewDecoder(r.Body).Decode(&req)
- if err != nil {
- w.WriteHeader(404)
- must(json.NewEncoder(w).Encode(batch.ErrorResponse{
- Message: "could not parse request",
- DocURL: "https://github.com/git-lfs/git-lfs/blob/v2.12.0/docs/api/batch.md#requests",
- }))
- return
- }
-
var resp batch.Response
for _, in := range req.Objects {
diff --git a/utils/util.go b/utils/util.go
new file mode 100644
index 0000000..884a3af
--- /dev/null
+++ b/utils/util.go
@@ -0,0 +1,15 @@
+package utils
+
+import (
+ "os"
+
+ "sigs.k8s.io/yaml"
+)
+
+func LoadFromYaml(path string, cfg interface{}) error {
+ b, err := os.ReadFile(path)
+ if err != nil {
+ return err
+ }
+ return yaml.Unmarshal(b, cfg)
+}