From 1d5c1003cd9a00189887545494b77cb7a07a28ce Mon Sep 17 00:00:00 2001 From: ZhouYi <1123678689@qq.com> Date: Sat, 14 Sep 2024 10:30:32 +0800 Subject: [PATCH 1/7] ADD: config settings Add the function of passing in relevant variables through config.yml --- .gitignore | 1 + BigFiles/go.mod | 3 +- BigFiles/go.sum | 23 ++++++++++- BigFiles/main.go | 100 ++++++++++++++++++++++++++++++++++++++++++++--- auth/gitee.go | 32 +++++++++++++-- config/config.go | 29 ++++++++++++++ go.mod | 1 + go.sum | 18 +++++++-- utils/util.go | 15 +++++++ 9 files changed, 207 insertions(+), 15 deletions(-) create mode 100644 config/config.go create mode 100644 utils/util.go diff --git a/.gitignore b/.gitignore index 722d5e7..aafda3a 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ .vscode +.idea diff --git a/BigFiles/go.mod b/BigFiles/go.mod index d38866c..e415928 100644 --- a/BigFiles/go.mod +++ b/BigFiles/go.mod @@ -5,4 +5,5 @@ go 1.15 require ( github.com/akrylysov/algnhsa v0.12.1 github.com/metalogical/BigFiles v0.0.0-20201031233020-8aa76a58aba7 -) + github.com/sirupsen/logrus v1.9.3 +) \ No newline at end of file diff --git a/BigFiles/go.sum b/BigFiles/go.sum index 625c4fd..5d3211d 100644 --- a/BigFiles/go.sum +++ b/BigFiles/go.sum @@ -3,23 +3,29 @@ github.com/akrylysov/algnhsa v0.12.1/go.mod h1:xAcJ/X8DV+81e+dUjIoB/r5CbISrSXV9/ github.com/aws/aws-lambda-go v1.9.0 h1:r9TWtk8ozLYdMW+aelUeWny8z2mjghJCMx6/uUwOLNo= github.com/aws/aws-lambda-go v1.9.0/go.mod h1:zUsUQhAUjYzR8AuduJPCfhBuKWUaDbQiPOG+ouzmE1A= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-chi/chi v4.1.2+incompatible h1:fGFk2Gmi/YKXk0OmGfBh0WgmN3XB8lVnEyNz34tQRec= github.com/go-chi/chi v4.1.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s= github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/metalogical/BigFiles v0.0.0-20201031233020-8aa76a58aba7 h1:eSaJmOvPgVx/ARHCkfmldLvAlHZu1F94ttYGN+bxQ1k= github.com/metalogical/BigFiles v0.0.0-20201031233020-8aa76a58aba7/go.mod h1:uTTfhdJzUTvmZEVl3RqOblXiZqlWS2imssQl2EEJabU= github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4= github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= @@ -35,14 +41,21 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 h1:DZhuSZLsGlFL4CmhA8BcRA0mnthyA/nZ00AqCUo7vHg= @@ -54,16 +67,22 @@ golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae h1:Ih9Yo4hSPImZOpfGuA4bR/ORKTAbhZo2AbWNRCnevdo= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww= gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/BigFiles/main.go b/BigFiles/main.go index f5c33b8..68d8e13 100644 --- a/BigFiles/main.go +++ b/BigFiles/main.go @@ -1,6 +1,8 @@ package main import ( + "flag" + "fmt" "log" "net/http" "os" @@ -8,19 +10,107 @@ import ( "github.com/akrylysov/algnhsa" "github.com/metalogical/BigFiles/auth" + "github.com/metalogical/BigFiles/config" "github.com/metalogical/BigFiles/server" + "github.com/sirupsen/logrus" ) +type options struct { + service ServiceOptions + enableDebug bool +} + +type ServiceOptions struct { + ConfigFile string + RemoveCfg bool +} + +// Validate checks if the ServiceOptions are valid. +// It returns an error if the config file is missing. +func (o *ServiceOptions) Validate() error { + if o.ConfigFile == "" { + return fmt.Errorf("missing config-file") + } + + return nil +} + +// AddFlags adds flags for ServiceOptions to the provided FlagSet. +func (o *ServiceOptions) AddFlags(fs *flag.FlagSet) { + fs.StringVar(&o.ConfigFile, "config-file", "", "Path to config file.") + fs.BoolVar(&o.RemoveCfg, "rm-cfg", false, "whether remove the cfg file after initialized .") +} + +// Validate validates the options and returns an error if any validation fails. +func (o *options) Validate() error { + return o.service.Validate() +} + +func gatherOptions(fs *flag.FlagSet, args ...string) (options, error) { + var o options + o.service.AddFlags(fs) + + fs.BoolVar( + &o.enableDebug, "enable_debug", false, "whether to enable debug model.", + ) + + err := fs.Parse(args) + return o, err +} + func main() { - bucket := os.Getenv("LFS_BUCKET") + o, err := gatherOptions( + flag.NewFlagSet(os.Args[0], flag.ExitOnError), + os.Args[1:]..., + ) + if err != nil { + logrus.Errorf("new options failed, err:%s", err.Error()) + + return + } + + if err := o.Validate(); err != nil { + logrus.Errorf("Invalid options, err:%s", err.Error()) + + return + } + + if o.enableDebug { + logrus.SetLevel(logrus.DebugLevel) + logrus.Debug("debug enable.") + } + + //cfg + cfg := new(config.Config) + + if err := config.LoadConfig(o.service.ConfigFile, cfg, o.service.RemoveCfg); err != nil { + logrus.Errorf("load config, err:%s", err.Error()) + + return + } + + if err := auth.Init(cfg); err != nil { + logrus.Errorf("load gitee config, err:%s", err.Error()) + + return + } + + bucket := cfg.LfsBucket if bucket == "" { - log.Fatalln("LFS_BUCKET must be set") + bucket = os.Getenv("LFS_BUCKET") + if bucket == "" { + logrus.Errorf("LFS_BUCKET must be set") + } } s, err := server.New(server.Options{ - S3Accelerate: true, - Bucket: bucket, - IsAuthorized: auth.GiteeAuth(), + S3Accelerate: true, + Bucket: bucket, + IsAuthorized: auth.GiteeAuth(), + Endpoint: cfg.AwsRegion, + AccessKeyID: cfg.AwsAccessKeyId, + SecretAccessKey: cfg.AwsSecretAccessKey, + Prefix: cfg.Prefix, }) if err != nil { log.Fatalln(err) diff --git a/auth/gitee.go b/auth/gitee.go index d695e63..477dff7 100644 --- a/auth/gitee.go +++ b/auth/gitee.go @@ -8,6 +8,13 @@ import ( "net/url" "os" "strings" + + "github.com/metalogical/BigFiles/config" +) + +var ( + client_id string + client_secret string ) type giteeUser struct { @@ -19,6 +26,25 @@ type AccessToken struct { Token string `json:"access_token"` } +func Init(cfg *config.Config) error { + client_id = cfg.ClientId + if client_id == "" { + client_id = os.Getenv("CLIENT_ID") + if client_id == "" { + return errors.New("client id required") + } + } + client_secret = cfg.ClientSecret + if client_secret == "" { + client_secret = os.Getenv("CLIENT_SECRET") + if client_secret == "" { + return errors.New("client secret required") + } + } + + return nil +} + func GiteeAuth() func(string, string) error { return func(username, password string) error { token, err := getToken(username, password) @@ -32,15 +58,13 @@ func GiteeAuth() func(string, string) error { // getToken gets access_token by username and password func getToken(username, password string) (string, error) { - clientId := os.Getenv("CLIENT_ID") - clientSecret := os.Getenv("CLIENT_SECRET") form := url.Values{} form.Add("scope", "user_info") form.Add("grant_type", "password") form.Add("username", username) form.Add("password", password) - form.Add("client_id", clientId) - form.Add("client_secret", clientSecret) + form.Add("client_id", client_id) + form.Add("client_secret", client_secret) path := "https://gitee.com/oauth/token" response, err := http.Post(path, "application/x-www-form-urlencoded", strings.NewReader(form.Encode())) diff --git a/config/config.go b/config/config.go new file mode 100644 index 0000000..a7a4d8b --- /dev/null +++ b/config/config.go @@ -0,0 +1,29 @@ +package config + +import ( + "os" + + "github.com/metalogical/BigFiles/utils" +) + +type Config struct { + LfsBucket string `json:"LFS_BUCKET"` + AwsRegion string `json:"AWS_REGION"` + AwsAccessKeyId string `json:"AWS_ACCESS_KEY_ID"` + AwsSecretAccessKey string `json:"AWS_SECRET_ACCESS_KEY"` + ClientId string `json:"CLIENT_ID"` + ClientSecret string `json:"CLIENT_SECRET"` + Prefix string `json:"PATH_PREFIX"` +} + +// LoadConfig loads the configuration file from the specified path and deletes the file if needed +func LoadConfig(path string, cfg *Config, remove bool) error { + if remove { + defer os.Remove(path) + } + + if err := utils.LoadFromYaml(path, cfg); err != nil { + return err + } + return nil +} diff --git a/go.mod b/go.mod index 14bcbcb..b086880 100644 --- a/go.mod +++ b/go.mod @@ -5,4 +5,5 @@ go 1.14 require ( github.com/go-chi/chi v4.1.2+incompatible github.com/minio/minio-go/v7 v7.0.5 + sigs.k8s.io/yaml v1.4.0 ) diff --git a/go.sum b/go.sum index 9177364..1c0f992 100644 --- a/go.sum +++ b/go.sum @@ -1,25 +1,29 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/go-chi/chi v1.0.0 h1:s/kv1cTXfivYjdKJdyUzNGyAWZ/2t7duW1gKn5ivu+c= github.com/go-chi/chi v4.1.2+incompatible h1:fGFk2Gmi/YKXk0OmGfBh0WgmN3XB8lVnEyNz34tQRec= github.com/go-chi/chi v4.1.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s= github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4= github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= -github.com/minio/minio-go v1.0.0 h1:ooSujki+Z1PRGZsYffJw5jnF5eMBvzMVV86TLAlM0UM= -github.com/minio/minio-go v6.0.14+incompatible h1:fnV+GD28LeqdN6vT2XdGKW8Qe/IfjJDswNVuni6km9o= github.com/minio/minio-go/v7 v7.0.5 h1:I2NIJ2ojwJqD/YByemC1M59e1b4FW9kS7NlOar7HPV4= github.com/minio/minio-go/v7 v7.0.5/go.mod h1:TA0CQCjJZHM5SJj9IjqR0NmpmQJ6bCbXifAJ3mUU6Hw= github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= @@ -32,13 +36,17 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -59,8 +67,12 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww= gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/utils/util.go b/utils/util.go new file mode 100644 index 0000000..884a3af --- /dev/null +++ b/utils/util.go @@ -0,0 +1,15 @@ +package utils + +import ( + "os" + + "sigs.k8s.io/yaml" +) + +func LoadFromYaml(path string, cfg interface{}) error { + b, err := os.ReadFile(path) + if err != nil { + return err + } + return yaml.Unmarshal(b, cfg) +} From 29211527a97edf3911a08aef29acf6aa41d29a50 Mon Sep 17 00:00:00 2001 From: ZhouYi <1123678689@qq.com> Date: Sat, 14 Sep 2024 10:32:59 +0800 Subject: [PATCH 2/7] DOCS: modify README.md --- README.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 3e7a4fa..6a47d7e 100644 --- a/README.md +++ b/README.md @@ -9,9 +9,8 @@ v2.12.0](https://github.com/git-lfs/git-lfs/tree/v2.12.0/docs/api) server. **TODO**: 1. 外部参数校验：username、password进行格式校验。 -2. 添加配置文件，将AKSK等写入配置文件中。可参考merlin-server配置文件的格式与读取方式。 -3. 添加测试用例。 -4. 认证方式支持token。 -5. 认证时校验用户在仓库内权限。 -6. 支持ssh。 -7. 仓库添加github action。 +2. 添加测试用例。 +3. 认证方式支持token。 +4. 认证时校验用户在仓库内权限。 +5. 支持ssh。 +6. 仓库添加github action。 \ No newline at end of file From e17bf316041c138709346149629cf88a61ccb469 Mon Sep 17 00:00:00 2001 From: wj00037 <1292876134@qq.com> Date: Sat, 14 Sep 2024 14:17:20 +0800 Subject: [PATCH 3/7] update gitee auth --- BigFiles/go.mod | 2 ++ BigFiles/go.sum | 15 +++++++++-- README.md | 7 +++--- auth/gitee.go | 65 +++++++++++++++++++++++++++++++++++++----------- go.mod | 2 ++ server/server.go | 35 +++++++++++++++----------- 6 files changed, 93 insertions(+), 33 deletions(-) diff --git a/BigFiles/go.mod b/BigFiles/go.mod index d38866c..206e93f 100644 --- a/BigFiles/go.mod +++ b/BigFiles/go.mod @@ -6,3 +6,5 @@ require ( github.com/akrylysov/algnhsa v0.12.1 github.com/metalogical/BigFiles v0.0.0-20201031233020-8aa76a58aba7 ) + +replace github.com/metalogical/BigFiles => github.com/opensourceways/BigFiles v0.0.0-20240913072808-2dbf11feb204 diff --git a/BigFiles/go.sum b/BigFiles/go.sum index 625c4fd..fe8be23 100644 --- a/BigFiles/go.sum +++ b/BigFiles/go.sum @@ -3,24 +3,27 @@ github.com/akrylysov/algnhsa v0.12.1/go.mod h1:xAcJ/X8DV+81e+dUjIoB/r5CbISrSXV9/ github.com/aws/aws-lambda-go v1.9.0 h1:r9TWtk8ozLYdMW+aelUeWny8z2mjghJCMx6/uUwOLNo= github.com/aws/aws-lambda-go v1.9.0/go.mod h1:zUsUQhAUjYzR8AuduJPCfhBuKWUaDbQiPOG+ouzmE1A= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-chi/chi v4.1.2+incompatible h1:fGFk2Gmi/YKXk0OmGfBh0WgmN3XB8lVnEyNz34tQRec= github.com/go-chi/chi v4.1.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s= github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/metalogical/BigFiles v0.0.0-20201031233020-8aa76a58aba7 h1:eSaJmOvPgVx/ARHCkfmldLvAlHZu1F94ttYGN+bxQ1k= -github.com/metalogical/BigFiles v0.0.0-20201031233020-8aa76a58aba7/go.mod h1:uTTfhdJzUTvmZEVl3RqOblXiZqlWS2imssQl2EEJabU= github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4= github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= github.com/minio/minio-go/v7 v7.0.5 h1:I2NIJ2ojwJqD/YByemC1M59e1b4FW9kS7NlOar7HPV4= @@ -35,13 +38,19 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/opensourceways/BigFiles v0.0.0-20240913072808-2dbf11feb204 h1:dxbM1M8kSEG+1LRn1Rx4wgH+s1eOCDpG/OpVEbkvsUc= +github.com/opensourceways/BigFiles v0.0.0-20240913072808-2dbf11feb204/go.mod h1:uTTfhdJzUTvmZEVl3RqOblXiZqlWS2imssQl2EEJabU= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -62,8 +71,10 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww= gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/README.md b/README.md index 3e7a4fa..098405f 100644 --- a/README.md +++ b/README.md @@ -8,10 +8,11 @@ v2.12.0](https://github.com/git-lfs/git-lfs/tree/v2.12.0/docs/api) server. **TODO**: -1. 外部参数校验：username、password进行格式校验。 +1. 外部参数校验：username、password、repo_id等进行格式校验。 2. 添加配置文件，将AKSK等写入配置文件中。可参考merlin-server配置文件的格式与读取方式。 3. 添加测试用例。 -4. 认证方式支持token。 -5. 认证时校验用户在仓库内权限。 +4. ~~认证方式支持token。~~ +5. ~~认证时校验用户在仓库内权限。~~ 6. 支持ssh。 7. 仓库添加github action。 +8. 添加日志。 diff --git a/auth/gitee.go b/auth/gitee.go index d695e63..b2e69ea 100644 --- a/auth/gitee.go +++ b/auth/gitee.go @@ -3,6 +3,7 @@ package auth import ( "encoding/json" "errors" + "fmt" "io" "net/http" "net/url" @@ -10,23 +11,37 @@ import ( "strings" ) +var uploadPermissions = []string{"admin", "developer"} +var downloadPermissions = []string{"admin", "developer", "read"} + type giteeUser struct { - Login string `json:"login"` - // Permission string `json:"permission"` + Login string `json:"login"` + Permission string `json:"permission"` +} + +type UserInRepo struct { + Repo string + Owner string + Token string + Username string + Password string + Operation string } type AccessToken struct { Token string `json:"access_token"` } -func GiteeAuth() func(string, string) error { - return func(username, password string) error { - token, err := getToken(username, password) +func GiteeAuth() func(UserInRepo) error { + return func(userInRepo UserInRepo) error { + token, err := getToken(userInRepo.Username, userInRepo.Password) if err != nil { - return err + userInRepo.Token = userInRepo.Password + } else { + userInRepo.Token = token } - return verifyUser(username, token) + return verifyUser(userInRepo) } } @@ -35,7 +50,7 @@ func getToken(username, password string) (string, error) { clientId := os.Getenv("CLIENT_ID") clientSecret := os.Getenv("CLIENT_SECRET") form := url.Values{} - form.Add("scope", "user_info") + form.Add("scope", "user_info projects") form.Add("grant_type", "password") form.Add("username", username) form.Add("password", password) @@ -64,9 +79,15 @@ func getToken(username, password string) (string, error) { return accessToken.Token, nil } -// verifyUser verifies user info by access_token -func verifyUser(username, token string) error { - path := "https://gitee.com/api/v5/user?access_token=" + token +// verifyUser verifies user permission in repo by access_token +func verifyUser(userInRepo UserInRepo) error { + path := fmt.Sprintf( + "https://gitee.com/api/v5/repos/%s/%s/collaborators/%s/permission?access_token=%s", + userInRepo.Owner, + userInRepo.Repo, + userInRepo.Username, + userInRepo.Token, + ) req, err := http.NewRequest("GET", path, nil) if err != nil { panic(err) @@ -90,9 +111,25 @@ func verifyUser(username, token string) error { if err != nil { panic(err) } - if giteeUser.Login == username { - return nil - } else { + + if giteeUser.Login != userInRepo.Username { return errors.New("username does not match") } + if userInRepo.Operation == "upload" { + for _, v := range uploadPermissions { + if giteeUser.Permission == v { + return nil + } + } + return errors.New("user has no permission uploading to the repository") + } else if userInRepo.Operation == "download" { + for _, v := range downloadPermissions { + if giteeUser.Permission == v { + return nil + } + } + return errors.New("user has no permission downloading in the repository") + } else { + return errors.New("unknow operation") + } } diff --git a/go.mod b/go.mod index 14bcbcb..1ce1faa 100644 --- a/go.mod +++ b/go.mod @@ -6,3 +6,5 @@ require ( github.com/go-chi/chi v4.1.2+incompatible github.com/minio/minio-go/v7 v7.0.5 ) + +replace github.com/metalogical/BigFiles => github.com/opensourceways/BigFiles v0.0.0-20240913072808-2dbf11feb204 diff --git a/server/server.go b/server/server.go index e782843..878535d 100644 --- a/server/server.go +++ b/server/server.go @@ -11,6 +11,7 @@ import ( "time" "github.com/go-chi/chi" + "github.com/metalogical/BigFiles/auth" "github.com/metalogical/BigFiles/batch" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" @@ -35,7 +36,7 @@ type Options struct { TTL time.Duration // defaults to 1 hour Prefix string - IsAuthorized func(string, string) error + IsAuthorized func(auth.UserInRepo) error } func (o Options) imputeFromEnv() (Options, error) { @@ -105,7 +106,7 @@ type server struct { ttl time.Duration prefix string - isAuthorized func(string, string) error + isAuthorized func(auth.UserInRepo) error } func (s *server) key(oid string) string { @@ -116,10 +117,27 @@ func (s *server) handleBatch(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/vnd.git-lfs+json") w.Header().Set("X-Content-Type-Options", "nosniff") + var req batch.Request + err := json.NewDecoder(r.Body).Decode(&req) + if err != nil { + w.WriteHeader(404) + must(json.NewEncoder(w).Encode(batch.ErrorResponse{ + Message: "could not parse request", + DocURL: "https://github.com/git-lfs/git-lfs/blob/v2.12.0/docs/api/batch.md#requests", + })) + return + } + + var userInRepo auth.UserInRepo + userInRepo.Operation = req.Operation + userInRepo.Owner = chi.URLParam(r, "owner") + userInRepo.Repo = chi.URLParam(r, "repo") if s.isAuthorized != nil { var err error if username, password, ok := r.BasicAuth(); ok { - err = s.isAuthorized(username, password) + userInRepo.Username = username + userInRepo.Password = password + err = s.isAuthorized(userInRepo) if err != nil { err = fmt.Errorf("unauthorized: %w", err) } @@ -137,17 +155,6 @@ func (s *server) handleBatch(w http.ResponseWriter, r *http.Request) { } } - var req batch.Request - err := json.NewDecoder(r.Body).Decode(&req) - if err != nil { - w.WriteHeader(404) - must(json.NewEncoder(w).Encode(batch.ErrorResponse{ - Message: "could not parse request", - DocURL: "https://github.com/git-lfs/git-lfs/blob/v2.12.0/docs/api/batch.md#requests", - })) - return - } - var resp batch.Response for _, in := range req.Objects { From c9e213da5d90dbed66bed62741fdf04c72bcd046 Mon Sep 17 00:00:00 2001 From: wj00037 <1292876134@qq.com> Date: Sat, 14 Sep 2024 15:31:40 +0800 Subject: [PATCH 4/7] add github action --- .github/workflows/lint.yml | 44 ++++++++++++++++++++++++++++++ .github/workflows/typos.yml | 26 ++++++++++++++++++ .golangci.yml | 53 +++++++++++++++++++++++++++++++++++++ 3 files changed, 123 insertions(+) create mode 100644 .github/workflows/lint.yml create mode 100644 .github/workflows/typos.yml create mode 100644 .golangci.yml diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..9a36688 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,44 @@ +name: golangci-lint +on: + pull_request: + +permissions: + # Required: allow read access to the content for analysis. + contents: read + # Optional: allow read access to pull request. Use with `only-new-issues` option. + pull-requests: read + # Optional: Allow write access to checks to allow the action to annotate code in the PR. + checks: write + +env: + GOPRIVATE: "github.com/opensourceways" + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + golangci: + name: lint + runs-on: [self-hosted, Linux] + steps: + - uses: actions/checkout@v4 + - name: golangci-lint + uses: golangci/golangci-lint-action@master + with: + # Require: The version of golangci-lint to use. + # When `install-mode` is `binary` (default) the value can be v1.2 or v1.2.3 or `latest` to use the latest version. + # When `install-mode` is `goinstall` the value can be v1.2.3, `latest`, or the hash of a commit. + version: latest + # Optional: golangci-lint command line arguments. + # + # Note: By default, the `.golangci.yml` file should be at the root of the repository. + # The location of the configuration file can be changed by using `--config=` + args: -v --config=.golangci.yml --max-same-issues=0 + + - name: Run Gosec Security Scanner + uses: securego/gosec@master + with: + args: -exclude-dir api -exclude-dir tests ./... + - name: Unit tests + run: go test -race -count=1 -v ./... \ No newline at end of file diff --git a/.github/workflows/typos.yml b/.github/workflows/typos.yml new file mode 100644 index 0000000..82fec9a --- /dev/null +++ b/.github/workflows/typos.yml @@ -0,0 +1,26 @@ +name: Spell Check Action +on: [pull_request] + +jobs: + run: + name: Spell Check with Typos + runs-on: self-hosted + steps: + - name: Checkout Actions Repository + uses: actions/checkout@v4 + + - name: Check spelling + uses: crate-ci/typos@master + with: + files: ./ + + - name: Use custom config file + uses: crate-ci/typos@master + with: + files: ./ + config: ./typos.toml + + - name: Writes changes in the local checkout + uses: crate-ci/typos@master + with: + write_changes: true \ No newline at end of file diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..6b17898 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,53 @@ +run: + # Timeout for analysis, e.g. 30s, 5m. + # Default: 1m + timeout: 10m + +output: + # Print linter name in the end of issue text. + # Default: true + print-linter-name: true + print-issued-lines: true + +linters-settings: + errorlint: + # Check whether fmt.Errorf uses the %w verb for formatting errors. + # See the https://github.com/polyfloyd/go-errorlint for caveats. + # Default: true + errorf: true + # Permit more than 1 %w verb, valid per Go 1.20 (Requires errorf:true) + # Default: true + errorf-multi: false + # Check for plain type assertions and type switches. + # Default: true + asserts: true + # Check for plain error comparisons. + # Default: true + comparison: true + lll: + # Max line length, lines longer will be reported. + # '\t' is counted as 1 character by default, and can be changed with the tab-width option. + # Default: 120. + line-length: 120 + # Tab width in spaces. + # Default: 1 + tab-width: 1 +linters: + enable: + - errorlint + - errcheck + - gosimple + - govet + - ineffassign + - staticcheck + - unused + - lll +issues: + exclude-rules: + - linters: + - lll + source: "^// " + - path: api + linters: + - lll + exclude-generated-strict: true From 5e9368a54d340e08c26020245a7bb5703a2ed76a Mon Sep 17 00:00:00 2001 From: wj00037 <140682082+wj00037@users.noreply.github.com> Date: Sat, 14 Sep 2024 16:01:01 +0800 Subject: [PATCH 5/7] Create go.yml --- .github/workflows/go.yml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 .github/workflows/go.yml diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml new file mode 100644 index 0000000..cc37540 --- /dev/null +++ b/.github/workflows/go.yml @@ -0,0 +1,27 @@ +# This workflow will build a golang project +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-go + +name: Go + +on: + workflow_dispatch: + pull_request: + branches: [ '*' ] + +jobs: + + build: + runs-on: [self-hosted, Linux] + steps: + - uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version: '1.20' + + - name: Build + run: go build -v ./... + + - name: Test + run: go test -v ./... From 5a7a915ed66ba103af30277e5ab05957975d50c9 Mon Sep 17 00:00:00 2001 From: wj00037 <140682082+wj00037@users.noreply.github.com> Date: Sat, 14 Sep 2024 16:28:49 +0800 Subject: [PATCH 6/7] Update lint.yml --- .github/workflows/lint.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 9a36688..84ddaaf 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -23,6 +23,11 @@ jobs: runs-on: [self-hosted, Linux] steps: - uses: actions/checkout@v4 + - name: config github + env: + TOKEN: ${{ secrets.ACCESS_TOKEN }} + USER: ${{ secrets.USER_NAME }} + run: echo "machine github.com login $USER password $TOKEN" > ~/.netrc - name: golangci-lint uses: golangci/golangci-lint-action@master with: @@ -38,7 +43,11 @@ jobs: - name: Run Gosec Security Scanner uses: securego/gosec@master + env: + GITHUB_AUTHENTICATION_TOKEN: ${{ secrets.ACCESS_TOKEN }} with: args: -exclude-dir api -exclude-dir tests ./... - name: Unit tests - run: go test -race -count=1 -v ./... \ No newline at end of file + run: go test -race -count=1 -v ./... + - name: cleanup + run: rm -f ~/.netrc From 2c2a214fbe049e4af96b78f37cc30e2b4bd822b6 Mon Sep 17 00:00:00 2001 From: ZhouYi <1123678689@qq.com> Date: Sat, 14 Sep 2024 17:26:28 +0800 Subject: [PATCH 7/7] ADD: config.yml.example add config.yml.example --- config.yml.example | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 config.yml.example diff --git a/config.yml.example b/config.yml.example new file mode 100644 index 0000000..1ef1e92 --- /dev/null +++ b/config.yml.example @@ -0,0 +1,7 @@ +LFS_BUCKET: *********** +AWS_REGION: *********** +AWS_ACCESS_KEY_ID: *********** +AWS_SECRET_ACCESS_KEY: *********** +CLIENT_ID: *********** +CLIENT_SECRET: *********** +PATH_PREFIX: *********** \ No newline at end of file