From 7893e3921609221cbe6950921a69adf608587e62 Mon Sep 17 00:00:00 2001 From: Darkweak Date: Fri, 19 Jun 2020 16:31:09 +0200 Subject: [PATCH] Issue/#10 (#11) * Performance update, remove useless dependencies, improve cache speed * Remove useless env var and optimise imports * Update makefile for travis * Fix configuration pass and linter * Add tests * Enhance tests * Tests and typo enhancement * Lint and add headers feature in key * Update documentation * Fix documentation * Remove duplicated line * Improve doc, thanks to @oxodao --- .env.dev | 7 - .env.prod | 7 - Dockerfile-dev | 9 +- Dockerfile-prod | 10 +- Makefile | 2 - README.md | 89 ++++-- cache/providers/abstract.go | 33 +- cache/providers/memory.go | 9 +- cache/providers/memory_test.go | 26 ++ cache/providers/redis.go | 20 +- cache/providers/redis_test.go | 13 +- cache/service/requestService.go | 25 +- cache/service/requestService_test.go | 54 ++-- cache/souin.go | 43 +-- configuration.yml | 11 - configuration/configuration.yml | 18 ++ configuration/types.go | 24 +- docker-compose.yml.dev | 10 +- docker-compose.yml.prod | 10 +- docs/plantUML/sequenceDiagram.svg | 435 ++++++++++++++++++++++++++- go.mod | 12 +- go.sum | 121 +++++++- providers/abstractProvider.go | 13 +- providers/abstractProvider_test.go | 37 +++ 24 files changed, 838 insertions(+), 200 deletions(-) delete mode 100644 .env.dev delete mode 100644 .env.prod create mode 100644 cache/providers/memory_test.go delete mode 100644 configuration.yml create mode 100644 configuration/configuration.yml create mode 100644 providers/abstractProvider_test.go diff --git a/.env.dev b/.env.dev deleted file mode 100644 index 727c0ae74..000000000 --- a/.env.dev +++ /dev/null @@ -1,7 +0,0 @@ -CACHE_PORT=80 -CACHE_TLS_PORT=443 -REDIS_URL=redis:6379 -TTL=10 -REVERSE_PROXY=http://traefik - -REGEX=ARegexHere diff --git a/.env.prod b/.env.prod deleted file mode 100644 index 727c0ae74..000000000 --- a/.env.prod +++ /dev/null @@ -1,7 +0,0 @@ -CACHE_PORT=80 -CACHE_TLS_PORT=443 -REDIS_URL=redis:6379 -TTL=10 -REVERSE_PROXY=http://traefik - -REGEX=ARegexHere diff --git a/Dockerfile-dev b/Dockerfile-dev index 19b7d3c77..8bc76e84b 100644 --- a/Dockerfile-dev +++ b/Dockerfile-dev @@ -10,11 +10,16 @@ ADD ./cache /app/src/github.com/darkweak/souin/cache ADD ./errors /app/src/github.com/darkweak/souin/errors ADD ./providers /app/src/github.com/darkweak/souin/providers ADD ./default/server.* /app/src/github.com/darkweak/souin/ +ADD ./configuration/* /app/src/github.com/darkweak/souin/configuration/ WORKDIR /app/src/github.com/darkweak/souin ENV GOPATH /app -RUN go get ./... -RUN go get -u golang.org/x/lint/golint +RUN go get -u \ + golang.org/x/lint/golint \ + github.com/allegro/bigcache \ + github.com/fsnotify/fsnotify \ + github.com/go-redis/redis \ + gopkg.in/yaml.v2 EXPOSE 80 diff --git a/Dockerfile-prod b/Dockerfile-prod index d8d77c8be..abf4090d4 100644 --- a/Dockerfile-prod +++ b/Dockerfile-prod @@ -11,15 +11,19 @@ ADD ./cache /app/src/github.com/darkweak/souin/cache ADD ./errors /app/src/github.com/darkweak/souin/errors ADD ./providers /app/src/github.com/darkweak/souin/providers ADD ./default/server.* /app/src/github.com/darkweak/souin/ +ADD ./configuration/* /app/src/github.com/darkweak/souin/configuration/ ADD ./entrypoint.sh /app/src/github.com/darkweak/souin/entrypoint.sh WORKDIR /app/src/github.com/darkweak/souin ENV GOPATH /app ENV GOOS linux ENV GOARCH arm -RUN go get ./... -RUN go get -u golang.org/x/lint/golint -RUN go install +RUN go get -u \ + golang.org/x/lint/golint \ + github.com/allegro/bigcache \ + github.com/fsnotify/fsnotify \ + github.com/go-redis/redis \ + gopkg.in/yaml.v2 RUN chmod 755 ./entrypoint.sh EXPOSE 80 diff --git a/Makefile b/Makefile index e32d98914..d24b26f02 100644 --- a/Makefile +++ b/Makefile @@ -21,12 +21,10 @@ down: ## Down containers env-dev: ## Up container with dev env vars cp Dockerfile-dev Dockerfile cp docker-compose.yml.dev docker-compose.yml - cp .env.dev .env env-prod: ## Up container with prod env vars cp Dockerfile-prod Dockerfile cp docker-compose.yml.prod docker-compose.yml - cp .env.prod .env generate-plantUML: ## Generate plantUML diagrams cd ./docs/plantUML && sh generate.sh && cd ../.. diff --git a/README.md b/README.md index c4fec17de..8e0faf8a0 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,9 @@ # Souin Table of Contents 1. [Souin reverse-proxy cache](#project-description) -2. [Environment variables](#environment-variables) - 2.1. [Required variables](#required-variables) - 2.2. [Optional variables](#optional-variables) +2. [Configuration](#configuration) + 2.1. [Required configuration](#required-configuration) + 2.2. [Optional configuration](#optional-configuration) 3. [Diagrams](#diagrams) 3.1. [Sequence diagram](#sequence-diagram) 4. [Cache systems](#cache-systems) @@ -19,23 +19,52 @@ ## Project description Souin is a new cache system suitable for every reverse-proxy. It will be placed on top of your current reverse-proxy whether it's Apache, Nginx or Traefik. -As it's written in go, it can be deployed on any server and thanks docker integration, it will be easy to install it on top of a Swarm or a kubernetes instance. +As it's written in go, it can be deployed on any server and thanks to the docker integration, it will be easy to install on top of a Swarm or a kubernetes instance. -## Environment variables +## Configuration +The configuration file is stored at `configuration/configuration.yml`. You can edit it provided you fill at least the required parameters as shown below. -### Required variables -| Variable | Description | Value example | +### Required configuration +```yaml +ttl: 100 #TTL in second +reverse_proxy_url: 'http://traefik' # The reverse-proxy http address +cache: + port: + web: 80 + tls: 443 +``` +This is a fully working minimal configuration for a Souin instance + +| Key | Description | Value example | |:---:|:---:|:---:| -|`CACHE_PORT`|The HTTP port Souin will be listening on |`80`| -|`CACHE_TLS_PORT`|The TLS port Souin will be listening on|`443`| -|`REDIS_URL`|The redis instance URL|- `http://redis` (Container way)
`http://localhost:6379` (Local way)| -|`TTL`|Duration to cache request (in seconds)|10| -|`REVERSE_PROXY`|The reverse-proxy's instance URL (Apache, Nginx, Træfik...)|- `http://yourservice` (Container way)
`http://localhost:81` (Local way)| - -### Optional variables -| Variable | Description | Value example | +|`ttl`|Duration to cache request (in seconds)|10| +|`reverse_proxy_url`|The reverse-proxy's instance URL (Apache, Nginx, Træfik...)|- `http://yourservice` (Container way)
`http://localhost:81` (Local way)| +|`cache.port.{web,tls}`|The device's local HTTP/TLS port that Souin should be listening on |Respectively `80` and `443`| + +### Optional configuration +```yaml +redis: + url: 'redis:6379' # Redis http address, only used for redis provider +regex: + exclude: 'ARegexHere' +ssl_providers: # Must match your volumes to /ssl/{provider}.json + - traefik +cache: + headers: + - Authorization # Can be any other headers + providers: # By default it will use in-memory and redis cache. It can be either `all`, `redis` or `memory`. + - all # Can be set to all if you want to enable all providers instead of specifying each one +# - memory +# - redis +``` + +| Key | Description | Value example | |:---:|:---:|:---:| -|`REGEX`|The regex that matches URLs not to store in cache|`http://domain.com/mypath`| +|`redis.url`|The redis url, used if you enabled it in the provider section|`redis:6379` (container way) and `http://yourdomain.com:6379` (network way)| +|`regex.exclude`|The regex used to prevent paths being cached|`^[A-z]+.*$`| +|`ssl_providers`|List of your providers handling certificates|`- traefik`

`- nginx`

`- apache`| +|`cache.headers`|List of headers to include to the cache|`- Authorization`

`- Content-Type`

`- X-Additional-Header`| +|`cache.providers`|Your providers list to cache your data, by default it will use all systems|`- all`

`- memory`

`- redis`| ## Diagrams @@ -43,14 +72,14 @@ As it's written in go, it can be deployed on any server and thanks docker integr Sequence diagram ## Cache systems -The cache system sits on two providers at the moment. It provides an in-memory and redis cache systems because setting, getting, updating and deleting keys in Redis is as easy as it gets. -In order to do that, Redis could be on the same network than the Souin instance when using docker-compose or over the internet then it will use by default in-memory to avoid network latency as much as possible. +The cache system sits on top of two providers at the moment. It provides an in-memory and redis cache systems because setting, getting, updating and deleting keys in Redis is as easy as it gets. +In order to do that, Redis needs to be either on the same network than the Souin instance when using docker-compose or over the internet, then it will use by default in-memory to avoid network latency as much as possible. Souin will return at first the in-memory response when it gives a non-empty response, then the redis one will be used with same condition, or fallback to the reverse proxy otherwise. ### Cache invalidation The cache invalidation is made for CRUD requests, if you're doing a GET HTTP request, it will serve the cached response when it exists, otherwise the reverse-proxy response will be served. -If you're doing a POST, PUT, PATCH or DELETE HTTP request, the related cache GET request will be dropped and the list endpoint will be dropped too -It works very well with plain [API Platform](https://api-platform.com) integration (not for custom actions for now) and CRUD routes. +If you're doing a POST, PUT, PATCH or DELETE HTTP request, the related cache GET request and the list endpoint will be dropped. +It works very well with plain [API Platform](https://api-platform.com) integration (not for custom actions at the moment) and CRUD routes. ## Examples @@ -97,17 +126,11 @@ services: build: context: . ports: - - ${CACHE_PORT}:80 - - ${CACHE_TLS_PORT}:443 + - 80:80 + - 443:443 depends_on: - redis environment: - REDIS_URL: ${REDIS_URL} - TTL: ${TTL} - CACHE_PORT: ${CACHE_PORT} - CACHE_TLS_PORT: ${CACHE_TLS_PORT} - REVERSE_PROXY: ${REVERSE_PROXY} - REGEX: ${REGEX} GOPATH: /app volumes: - ./cmd:/app/cmd @@ -127,25 +150,25 @@ networks: ### Træfik As Souin is compatible with Træfik, it can use (and it should use) `traefik.json` provided on træfik. Souin will get new/updated certs from Træfik, then your SSL certs will be up to date as far as Træfik will be too -To provide, acme, use just have to map volume as above +To provide, acme, you just have to map volume as above ```yaml volumes: - /anywhere/traefik.json:/ssl/traefik.json ``` ### Apache -Souin will listen `apache.json` file. You have to setup your own way to aggregate your SSL cert files and keys. Alternatively you can use a side project called [dob](https://github.com/darkweak/dob), it's open-source and written in go too +Souin will listen to the `apache.json` file. You have to setup your own way to aggregate your SSL cert files and keys. Alternatively you can use a side project called [dob](https://github.com/darkweak/dob), it's open-source and written in go too ```yaml volumes: - /anywhere/apache.json:/ssl/apache.json ``` ### Nginx -Souin will listen `nginx.json` file. You have to setup your own way to aggregate your SSL cert files and keys. Alternatively you can use a side project called [dob](https://github.com/darkweak/dob), it's open-source and written in go too +Souin will listen to the `nginx.json` file. You have to setup your own way to aggregate your SSL cert files and keys. Alternatively you can use a side project called [dob](https://github.com/darkweak/dob), it's open-source and written in go too ```yaml volumes: - /anywhere/nginx.json:/ssl/nginx.json ``` -At the moment you can't choose the path for the `*.json` in the container, they have to be placed in the `/ssl` folder. In the future you'll be able to do that just setting one env var -If none `*.json` is provided to container, a default cert will be served. +At the moment you can't choose the path for the `*.json` file in the container, they have to be placed in the `/ssl` folder. In the future you'll be able to do that by setting one env var +If none `*.json` file is provided to container, a default cert will be served. ## Credits diff --git a/cache/providers/abstract.go b/cache/providers/abstract.go index cea70c0e5..4c0db4a25 100644 --- a/cache/providers/abstract.go +++ b/cache/providers/abstract.go @@ -1,10 +1,10 @@ package providers import ( - "os" "regexp" "github.com/darkweak/souin/cache/types" + "github.com/darkweak/souin/configuration" ) // AbstractProviderInterface should be implemented in any providers @@ -16,7 +16,34 @@ type AbstractProviderInterface interface { } // PathnameNotInRegex check if pathname is in parameter regex var -func PathnameNotInRegex(pathname string) bool { - b, _ := regexp.Match(os.Getenv("REGEX"), []byte(pathname)) +func PathnameNotInRegex(pathname string, configuration configuration.Configuration) bool { + b, _ := regexp.Match(configuration.Regex.Exclude, []byte(pathname)) return !b } + +func contains(s []string, e string) bool { + for _, a := range s { + if a == e { + return true + } + } + return false +} + +// InitializeProviders allow to generate the providers array according to the configuration +func InitializeProviders(configuration configuration.Configuration) *[]AbstractProviderInterface { + var providers []AbstractProviderInterface + + if len(configuration.Cache.Providers) == 0 || contains(configuration.Cache.Providers, "all") { + providers = append(providers, MemoryConnectionFactory(configuration), RedisConnectionFactory(configuration)) + } else { + if contains(configuration.Cache.Providers, "redis") { + providers = append(providers, RedisConnectionFactory(configuration)) + } + if contains(configuration.Cache.Providers, "memory") { + providers = append(providers, MemoryConnectionFactory(configuration)) + } + } + + return &providers +} diff --git a/cache/providers/memory.go b/cache/providers/memory.go index 74fab6922..0c428730c 100644 --- a/cache/providers/memory.go +++ b/cache/providers/memory.go @@ -1,11 +1,12 @@ package providers import ( - "os" "time" "github.com/allegro/bigcache" "github.com/darkweak/souin/cache/types" + "github.com/darkweak/souin/configuration" + "strconv" ) // Memory provider type @@ -14,9 +15,9 @@ type Memory struct { } // MemoryConnectionFactory function create new Memory instance -func MemoryConnectionFactory() *Memory { - t, _ := time.ParseDuration(os.Getenv("TTL")) - bc, _ := bigcache.NewBigCache(bigcache.DefaultConfig(t * time.Second)) +func MemoryConnectionFactory(configuration configuration.Configuration) *Memory { + t, _ := strconv.Atoi(configuration.TTL) + bc, _ := bigcache.NewBigCache(bigcache.DefaultConfig(time.Second * time.Duration(t))) return &Memory{ bc, } diff --git a/cache/providers/memory_test.go b/cache/providers/memory_test.go new file mode 100644 index 000000000..acf064036 --- /dev/null +++ b/cache/providers/memory_test.go @@ -0,0 +1,26 @@ +package providers + +import ( + "fmt" + "testing" + + "github.com/darkweak/souin/errors" + "github.com/darkweak/souin/configuration" +) + +const MEMORYVALUE = "My first data" + +func TestIShouldBeAbleToReadAndWriteDataInMemory(t *testing.T) { + client := MemoryConnectionFactory(configuration.GetConfig()) + err := client.Set("Test", []byte(MEMORYVALUE)) + if err != nil { + errors.GenerateError(t, "Impossible to set memory variable") + } + res, err := client.Get("Test") + if err != nil { + errors.GenerateError(t, "Retrieving data from memory") + } + if MEMORYVALUE != string(res) { + errors.GenerateError(t, fmt.Sprintf("%s not corresponding to %s", res, MEMORYVALUE)) + } +} diff --git a/cache/providers/redis.go b/cache/providers/redis.go index 9bf900201..6881f63ce 100644 --- a/cache/providers/redis.go +++ b/cache/providers/redis.go @@ -1,33 +1,35 @@ package providers import ( - "os" "strconv" "time" "github.com/darkweak/souin/cache/types" + "github.com/darkweak/souin/configuration" "github.com/go-redis/redis" ) // Redis provider type type Redis struct { *redis.Client + configuration.Configuration } // RedisConnectionFactory function create new Redis instance -func RedisConnectionFactory() *Redis { +func RedisConnectionFactory(configuration configuration.Configuration) *Redis { return &Redis{ redis.NewClient(&redis.Options{ - Addr: os.Getenv("REDIS_URL"), + Addr: configuration.Redis.URL, DB: 0, Password: "", }), + configuration, } } // GetRequestInCache method returns the populated response if exists, empty response then func (provider *Redis) GetRequestInCache(key string) types.ReverseResponse { - val2, err := provider.Get(key).Result() + val2, err := provider.Get(provider.Context(), key).Result() if err != nil { return types.ReverseResponse{Response: "", Proxy: nil, Request: nil} @@ -38,9 +40,9 @@ func (provider *Redis) GetRequestInCache(key string) types.ReverseResponse { // SetRequestInCache method will store the response in Redis provider func (provider *Redis) SetRequestInCache(key string, value []byte) { - ttl, _ := strconv.Atoi(os.Getenv("TTL")) + ttl, _ := strconv.Atoi(provider.Configuration.TTL) - err := provider.Set(key, string(value), time.Duration(ttl)*time.Second).Err() + err := provider.Set(provider.Context(), key, string(value), time.Duration(ttl)*time.Second).Err() if err != nil { panic(err) } @@ -48,13 +50,13 @@ func (provider *Redis) SetRequestInCache(key string, value []byte) { // DeleteRequestInCache method will delete the response in Redis provider if exists corresponding to key param func (provider *Redis) DeleteRequestInCache(key string) { - provider.Do("del", key) + provider.Do(provider.Context(), "del", key) } // DeleteManyRequestInCache method will delete the response in Redis provider if exists corresponding to regex param func (provider *Redis) DeleteManyRequestInCache(regex string) { - for _, i := range provider.Keys(regex).Val() { - provider.Do("del", i) + for _, i := range provider.Keys(provider.Context(), regex).Val() { + provider.Do(provider.Context(), provider, "del", i) } } diff --git a/cache/providers/redis_test.go b/cache/providers/redis_test.go index a26ae6078..312483512 100644 --- a/cache/providers/redis_test.go +++ b/cache/providers/redis_test.go @@ -6,21 +6,22 @@ import ( "time" "github.com/darkweak/souin/errors" + "github.com/darkweak/souin/configuration" ) -const VALUE = "My first data" +const REDISVALUE = "My first data" func TestIShouldBeAbleToReadAndWriteDataInRedis(t *testing.T) { - client := RedisConnectionFactory() - err := client.Set("Test", string(VALUE), time.Duration(10)*time.Second).Err() + client := RedisConnectionFactory(configuration.GetConfig()) + err := client.Set(client.Context(), "Test", string(REDISVALUE), time.Duration(10)*time.Second).Err() if err != nil { errors.GenerateError(t, "Impossible to set redis variable") } - res, err := client.Get("Test").Result() + res, err := client.Get(client.Context(), "Test").Result() if err != nil { errors.GenerateError(t, "Retrieving data from redis") } - if VALUE != res { - errors.GenerateError(t, fmt.Sprintf("%s not corresponding to %s", res, VALUE)) + if REDISVALUE != res { + errors.GenerateError(t, fmt.Sprintf("%s not corresponding to %s", res, REDISVALUE)) } } diff --git a/cache/service/requestService.go b/cache/service/requestService.go index d0b0a9b5f..4b91843e2 100644 --- a/cache/service/requestService.go +++ b/cache/service/requestService.go @@ -12,6 +12,7 @@ import ( p "github.com/darkweak/souin/cache/providers" "github.com/darkweak/souin/cache/types" + "github.com/darkweak/souin/configuration" ) func commonLoadingRequest(resp *http.Response) []byte { @@ -32,24 +33,32 @@ func hasNotAllowedHeaders(r *http.Response) bool { "no-cache" == r.Header.Get("Cache-Control") } -func getKeyFromResponse(resp *http.Response) string { - return resp.Request.Host + resp.Request.URL.Path +func getKeyFromResponse(resp *http.Response, config configuration.Configuration) string { + headers := "" + if config.Cache.Headers != nil && len(config.Cache.Headers) > 0 { + for _, h := range config.Cache.Headers { + headers += strings.ReplaceAll(resp.Request.Header.Get(h), " ", "") + } + } + return resp.Request.Host + resp.Request.URL.Path + headers } -func rewriteBody(resp *http.Response, providers []p.AbstractProviderInterface) (err error) { +func rewriteBody(resp *http.Response, providers []p.AbstractProviderInterface, configuration configuration.Configuration) (err error) { b := bytes.Replace(commonLoadingRequest(resp), []byte("server"), []byte("schmerver"), -1) body := ioutil.NopCloser(bytes.NewReader(b)) resp.Body = body resp.ContentLength = int64(len(b)) resp.Header.Set("Content-Length", strconv.Itoa(len(b))) - if p.PathnameNotInRegex(resp.Request.Host+resp.Request.URL.Path) && !hasNotAllowedHeaders(resp) && nil == resp.Request.Context().Err() { - key := getKeyFromResponse(resp) + if p.PathnameNotInRegex(resp.Request.Host+resp.Request.URL.Path, configuration) && !hasNotAllowedHeaders(resp) && nil == resp.Request.Context().Err() { + key := getKeyFromResponse(resp, configuration) if http.MethodGet == resp.Request.Method && len(b) > 0 { r, _ := json.Marshal(types.RequestResponse{Body: b, Headers: resp.Header}) go func() { for _, v := range providers { - v.SetRequestInCache(key, r) + go func() { + v.SetRequestInCache(key, r) + }() } }() } else { @@ -78,14 +87,14 @@ func rewriteBody(resp *http.Response, providers []p.AbstractProviderInterface) ( } // RequestReverseProxy returns response from one of providers or the proxy response -func RequestReverseProxy(req *http.Request, url *url.URL, providers []p.AbstractProviderInterface) types.ReverseResponse { +func RequestReverseProxy(req *http.Request, url *url.URL, providers []p.AbstractProviderInterface, configuration configuration.Configuration) types.ReverseResponse { req.URL.Host = req.Host req.URL.Scheme = url.Scheme req.Header.Set("X-Forwarded-Host", req.Header.Get("Host")) proxy := httputil.NewSingleHostReverseProxy(url) proxy.ModifyResponse = func(response *http.Response) error { - return rewriteBody(response, providers) + return rewriteBody(response, providers, configuration) } return types.ReverseResponse{ diff --git a/cache/service/requestService_test.go b/cache/service/requestService_test.go index d1b3558d8..210168fa4 100644 --- a/cache/service/requestService_test.go +++ b/cache/service/requestService_test.go @@ -12,26 +12,30 @@ import ( "github.com/darkweak/souin/cache/providers" "github.com/darkweak/souin/errors" - "github.com/go-redis/redis" + "github.com/darkweak/souin/configuration" ) const DOMAIN = "domain.com" const PATH = "/testing" -func populateRedisWithFakeData() { - client := providers.RedisConnectionFactory() - duration := time.Duration(120) * time.Second +func populateProvidersWithFakeData() { basePath := "/testing" domain := "domain.com" - client.Set(domain+basePath, "testing value is here for "+basePath, duration) - for i := 0; i < 25; i++ { - client.Set(domain+basePath+"/"+string(i), "testing value is here for my first init of "+basePath+"/"+string(i), duration) + for _, provider := range []providers.AbstractProviderInterface{mockMemory(), mockRedis()} { + provider.SetRequestInCache(domain+basePath, []byte("testing value is here for "+basePath)) + for i := 0; i < 25; i++ { + provider.SetRequestInCache(domain+basePath+"/"+string(i), []byte("testing value is here for my first init of "+basePath+"/"+string(i))) + } } } func mockRedis() *providers.Redis { - return providers.RedisConnectionFactory() + return providers.RedisConnectionFactory(configuration.GetConfig()) +} + +func mockMemory() *providers.Memory { + return providers.MemoryConnectionFactory(configuration.GetConfig()) } func mockResponse(path string, method string, body string, code int) *http.Response { @@ -85,7 +89,7 @@ func mockResponse(path string, method string, body string, code int) *http.Respo } func TestGetKeyFromResponse(t *testing.T) { - resp := getKeyFromResponse(mockResponse(PATH, http.MethodGet, "", 200)) + resp := getKeyFromResponse(mockResponse(PATH, http.MethodGet, "", 200), configuration.GetConfig()) urlCollapsed := DOMAIN + PATH if urlCollapsed != resp { errors.GenerateError(t, fmt.Sprintf("Key doesn't return %s but %s", urlCollapsed, resp)) @@ -93,39 +97,45 @@ func TestGetKeyFromResponse(t *testing.T) { } func shouldNotHaveKey(pathname string) bool { - client := providers.RedisConnectionFactory() - _, err := client.Get(DOMAIN + pathname).Result() + config := configuration.GetConfig() + redisClient := providers.RedisConnectionFactory(config) + _, redisErr := redisClient.Get(redisClient.Context(), DOMAIN+pathname).Result() + memoryClient := providers.MemoryConnectionFactory(config) + _, memoryErr := memoryClient.Get(DOMAIN + pathname) - return err == redis.Nil + return memoryErr != nil && redisErr != nil } func TestKeyShouldBeDeletedOnPost(t *testing.T) { - populateRedisWithFakeData() - rewriteBody(mockResponse(PATH, http.MethodPost, "My second response", 201), []providers.AbstractProviderInterface{mockRedis()}) + config := configuration.GetConfig() + populateProvidersWithFakeData() + rewriteBody(mockResponse(PATH, http.MethodPost, "My second response", 201), []providers.AbstractProviderInterface{mockRedis(), mockMemory()}, config) time.Sleep(10 * time.Second) if !shouldNotHaveKey(PATH) { errors.GenerateError(t, "The key "+DOMAIN+PATH+" shouldn't exist.") } } -func verifyKeysExists(t *testing.T, path string, keys []string) { +func verifyKeysExists(t *testing.T, path string, keys []string, isKeyDeleted bool) { time.Sleep(10 * time.Second) for _, i := range keys { - if !shouldNotHaveKey(PATH + i) { + if !shouldNotHaveKey(PATH + i) == isKeyDeleted { errors.GenerateError(t, "The key "+DOMAIN+path+i+" shouldn't exist.") } } } func TestKeyShouldBeDeletedOnPut(t *testing.T) { - populateRedisWithFakeData() - rewriteBody(mockResponse(PATH+"/1", http.MethodPut, "My second response", 200), []providers.AbstractProviderInterface{mockRedis()}) - verifyKeysExists(t, PATH, []string{"", "/1"}) + config := configuration.GetConfig() + populateProvidersWithFakeData() + rewriteBody(mockResponse(PATH+"/1", http.MethodPut, "My second response", 200), []providers.AbstractProviderInterface{mockRedis(), mockMemory()}, config) + verifyKeysExists(t, PATH, []string{"", "/1"}, true) } func TestKeyShouldBeDeletedOnDelete(t *testing.T) { - populateRedisWithFakeData() - rewriteBody(mockResponse(PATH+"/1", http.MethodDelete, "", 200), []providers.AbstractProviderInterface{mockRedis()}) - verifyKeysExists(t, PATH, []string{"", "/1"}) + config := configuration.GetConfig() + populateProvidersWithFakeData() + rewriteBody(mockResponse(PATH+"/1", http.MethodDelete, "", 200), []providers.AbstractProviderInterface{mockRedis(), mockMemory()}, config) + verifyKeysExists(t, PATH, []string{"", "/1"}, true) } diff --git a/cache/souin.go b/cache/souin.go index 53cbaaf53..2f34d83f3 100644 --- a/cache/souin.go +++ b/cache/souin.go @@ -7,24 +7,35 @@ import ( "net" "net/http" "net/url" - "os" - cacheProviders "github.com/darkweak/souin/cache/providers" "github.com/darkweak/souin/cache/service" "github.com/darkweak/souin/cache/types" "github.com/darkweak/souin/providers" + "github.com/darkweak/souin/configuration" + "strings" ) -func serveReverseProxy(res http.ResponseWriter, req *http.Request, providers *[]cacheProviders.AbstractProviderInterface) { - url, _ := url.Parse(os.Getenv("REVERSE_PROXY")) +func serveReverseProxy( + res http.ResponseWriter, + req *http.Request, + providers *[]cacheProviders.AbstractProviderInterface, + configurationInstance configuration.Configuration, +) { + u, _ := url.Parse(configurationInstance.ReverseProxyURL) ctx := req.Context() responses := make(chan types.ReverseResponse) go func() { + headers := "" + if configurationInstance.Cache.Headers != nil && len(configurationInstance.Cache.Headers) > 0 { + for _, h := range configurationInstance.Cache.Headers { + headers += strings.ReplaceAll(req.Header.Get(h), " ", "") + } + } for _, v := range *providers { - responses <- v.GetRequestInCache(string(req.Host + req.URL.Path)) + responses <- v.GetRequestInCache(string(req.Host + req.URL.Path + headers)) } - responses <- service.RequestReverseProxy(req, url, *providers) + responses <- service.RequestReverseProxy(req, u, *providers, configurationInstance) }() alreadySent := false @@ -51,14 +62,14 @@ func serveReverseProxy(res http.ResponseWriter, req *http.Request, providers *[] } } -func startServer(tlsconfig *tls.Config) (net.Listener, *http.Server) { - tlsconfig.BuildNameToCertificate() +func startServer(config *tls.Config) (net.Listener, *http.Server) { + config.BuildNameToCertificate() server := http.Server{ Addr: ":443", Handler: nil, - TLSConfig: tlsconfig, + TLSConfig: config, } - listener, err := tls.Listen("tcp", ":443", tlsconfig) + listener, err := tls.Listen("tcp", ":443", config) if err != nil { fmt.Println(err) } @@ -74,10 +85,8 @@ func startServer(tlsconfig *tls.Config) (net.Listener, *http.Server) { // Start cache system func Start() { - providersList := []cacheProviders.AbstractProviderInterface{ - cacheProviders.MemoryConnectionFactory(), - cacheProviders.RedisConnectionFactory(), - } + configurationInstance := configuration.GetConfig() + providersList := cacheProviders.InitializeProviders(configurationInstance) configChannel := make(chan int) tlsconfig := &tls.Config{ @@ -89,11 +98,11 @@ func Start() { tlsconfig.Certificates = append(tlsconfig.Certificates, v) go func() { - providers.InitProviders(tlsconfig, &configChannel) + providers.InitProviders(tlsconfig, &configChannel, configurationInstance) }() http.HandleFunc("/", func(writer http.ResponseWriter, request *http.Request) { - serveReverseProxy(writer, request, &providersList) + serveReverseProxy(writer, request, providersList, configurationInstance) }) go func() { listener, _ := startServer(tlsconfig) @@ -106,7 +115,7 @@ func Start() { } }() - if err := http.ListenAndServe(":"+os.Getenv("CACHE_PORT"), nil); err != nil { + if err := http.ListenAndServe(":"+configurationInstance.Cache.Port.Web, nil); err != nil { panic(err) } diff --git a/configuration.yml b/configuration.yml deleted file mode 100644 index 7f82cfbb5..000000000 --- a/configuration.yml +++ /dev/null @@ -1,11 +0,0 @@ -redis: - url: 'redis:6379' # Redis http address -ttl: 10 #TTL in second -reverse_proxy_url: 'http://traefik' # The reverse-proxy http address -regex: - exclude: 'ARegexHere' -cache: - mode: 'all' # By defautl it will use in-memory and redis third-party cache. It can be `all`, `redis` or `memory` declaration - port: - web: '80' - tls: '443' diff --git a/configuration/configuration.yml b/configuration/configuration.yml new file mode 100644 index 000000000..e9dc8c538 --- /dev/null +++ b/configuration/configuration.yml @@ -0,0 +1,18 @@ +redis: + url: 'redis:6379' # Redis http address used only for redis provider +ttl: 100 #TTL in second +reverse_proxy_url: 'http://traefik' # The reverse-proxy http address +regex: + exclude: 'ARegexHere' +ssl_providers: # Must match your volumes to /ssl/{provider}.json + - traefik +cache: + headers: + - Authorization # Can be any other headers + providers: # By defautl it will use in-memory and redis third-party cache. It can be `all`, `redis` or `memory` declaration + - all # Can be set to all if you want to enable all providers instead of specifying each one +# - memory +# - redis + port: + web: 80 + tls: 443 diff --git a/configuration/types.go b/configuration/types.go index 56d6462fb..36b5c0dc4 100644 --- a/configuration/types.go +++ b/configuration/types.go @@ -5,23 +5,25 @@ import ( "log" "gopkg.in/yaml.v2" + "os" ) // Port config type Port struct { Web string `yaml:"web"` - Tls string `yaml:"tls"` + TLS string `yaml:"tls"` } //Cache config type Cache struct { - Mode string `yaml:"mode"` - Port Port `yaml:"port"` + Headers []string `yaml:"headers"` + Providers []string `yaml:"providers"` + Port Port `yaml:"port"` } //Redis config type Redis struct { - Url string `yaml:"url"` + URL string `yaml:"url"` } //Regex config @@ -31,11 +33,12 @@ type Regex struct { //Configuration holder type Configuration struct { - Redis Redis `yaml:"redis"` - TTL string `yaml:"ttl"` - ReverseProxyUrl string `yaml:"reverse_proxy_url"` - Regex Regex `yaml:"regex"` - Cache Cache `yaml:"cache"` + Redis Redis `yaml:"redis"` + TTL string `yaml:"ttl"` + SSLProviders []string `yaml:"ssl_providers"` + ReverseProxyURL string `yaml:"reverse_proxy_url"` + Regex Regex `yaml:"regex"` + Cache Cache `yaml:"cache"` } // Parse configuration @@ -56,8 +59,7 @@ func readFile(path string) []byte { // GetConfig allow to retrieve Souin configuration through yaml file func GetConfig() Configuration { - configFile := "./configuration.yml" - data := readFile(configFile) + data := readFile(os.Getenv("GOPATH") + "/src/github.com/darkweak/souin/configuration/configuration.yml") var config Configuration if err := config.Parse(data); err != nil { log.Fatal(err) diff --git a/docker-compose.yml.dev b/docker-compose.yml.dev index 1b926c9cf..4fe03927b 100644 --- a/docker-compose.yml.dev +++ b/docker-compose.yml.dev @@ -9,17 +9,11 @@ services: build: context: . ports: - - ${CACHE_PORT}:80 - - ${CACHE_TLS_PORT}:443 + - 80:80 + - 443:443 depends_on: - redis environment: - REDIS_URL: ${REDIS_URL} - TTL: ${TTL} - CACHE_PORT: ${CACHE_PORT} - CACHE_TLS_PORT: ${CACHE_TLS_PORT} - REVERSE_PROXY: ${REVERSE_PROXY} - REGEX: ${REGEX} GOPATH: /app volumes: - ./cmd:/app/cmd diff --git a/docker-compose.yml.prod b/docker-compose.yml.prod index 1b926c9cf..4fe03927b 100644 --- a/docker-compose.yml.prod +++ b/docker-compose.yml.prod @@ -9,17 +9,11 @@ services: build: context: . ports: - - ${CACHE_PORT}:80 - - ${CACHE_TLS_PORT}:443 + - 80:80 + - 443:443 depends_on: - redis environment: - REDIS_URL: ${REDIS_URL} - TTL: ${TTL} - CACHE_PORT: ${CACHE_PORT} - CACHE_TLS_PORT: ${CACHE_TLS_PORT} - REVERSE_PROXY: ${REVERSE_PROXY} - REGEX: ${REGEX} GOPATH: /app volumes: - ./cmd:/app/cmd diff --git a/docs/plantUML/sequenceDiagram.svg b/docs/plantUML/sequenceDiagram.svg index a152e503f..35067fdfe 100644 --- a/docs/plantUML/sequenceDiagram.svg +++ b/docs/plantUML/sequenceDiagram.svg @@ -1,4 +1,413 @@ -UserUserSystemSystemSouinSouinProvidersProvidersMemoryMemoryRedisRedisReverseProxyReverseProxyrun()main()Init()MemoryConnectionFactory()AbstractProviderRedisConnectionFactory()AbstractProviderAbstractProvider[]Waiting for user requestloop[User requests]requestpar[Request providers content]GetRequestInCache()GetRequestInCache()GetRequestInReverseProxy()par[Response providers content]ResponseResponseResponseSetRequestInCache()SetRequestInCache()response \ No newline at end of file + +PlantUML version 1.2019.11(Sun Sep 22 10:02:15 GMT 2019) +(GPL source distribution) +Java Runtime: OpenJDK Runtime Environment +JVM: OpenJDK 64-Bit Server VM +Java Version: 14-ea+15 +Operating System: Linux +Default Encoding: UTF-8 +Language: en +Country: US +--> + + diff --git a/go.mod b/go.mod index 4397f4c9b..b292ab588 100644 --- a/go.mod +++ b/go.mod @@ -4,10 +4,10 @@ go 1.13 require ( github.com/allegro/bigcache v1.2.1 - github.com/dgraph-io/ristretto v0.0.2 - github.com/fsnotify/fsnotify v1.4.7 - github.com/go-redis/redis v6.15.6+incompatible - github.com/patrickmn/go-cache v2.1.0+incompatible // indirect - golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e // indirect - gopkg.in/yaml.v2 v2.2.8 + github.com/fsnotify/fsnotify v1.4.9 + github.com/go-redis/redis/v8 v8.0.0-beta.4 + golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect + golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980 // indirect + golang.org/x/tools v0.0.0-20200608174601-1b747fd94509 // indirect + gopkg.in/yaml.v2 v2.3.0 ) diff --git a/go.sum b/go.sum index 67c32287c..e2dd19c1f 100644 --- a/go.sum +++ b/go.sum @@ -1,25 +1,124 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/DataDog/sketches-go v0.0.0-20190923095040-43f19ad77ff7/go.mod h1:Q5DbzQ+3AkgGwymQO7aZFNP7ns2lZKGtvRBzRXfdi60= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/allegro/bigcache v1.2.1 h1:hg1sY1raCwic3Vnsvje6TT7/pnZba83LeFck5NrFKSc= github.com/allegro/bigcache v1.2.1/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= +github.com/benbjohnson/clock v1.0.0/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgraph-io/ristretto v0.0.2 h1:a5WaUrDa0qm0YrAAS1tUykT5El3kt62KNZZeMxQn3po= -github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= -github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-rendezvous v0.0.0-20180401054734-3692eb46c031 h1:GqrUYGzmGuc00lpc+K0wwrqshfkKLwgYFJiCyOZFMVE= +github.com/dgryski/go-rendezvous v0.0.0-20180401054734-3692eb46c031/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/go-redis/redis v6.15.6+incompatible h1:H9evprGPLI8+ci7fxQx6WNZHJSb7be8FqJQRhdQZ5Sg= -github.com/go-redis/redis v6.15.6+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= -github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= -github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/go-redis/redis v6.15.8+incompatible h1:BKZuG6mCnRj5AOaWJXoCgf6rqTYnYJLe4en2hxT7r9o= +github.com/go-redis/redis v6.15.8+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= +github.com/go-redis/redis/v8 v8.0.0-beta.4 h1:oIZMgBk2CHvLd1/rfn8sybGNwzTTmKEvRoXGz6ZiWnI= +github.com/go-redis/redis/v8 v8.0.0-beta.4/go.mod h1:NlNCdZHGMxsMUjOkA1Xab/1SsVzAwI7WPBXbh1O7vHM= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/opentracing/opentracing-go v1.1.1-0.20190913142402-a7454ce5950e/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e h1:9vRrk9YW2BTzLP0VCB9ZDjU4cPqkg+IDWL7XgxA1yxQ= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opentelemetry.io/otel v0.5.0 h1:tdIR1veg/z+VRJaw/6SIxz+QX3l+m+BDleYLTs+GC1g= +go.opentelemetry.io/otel v0.5.0/go.mod h1:jzBIgIzK43Iu1BpDAXwqOd6UPsSAk+ewVZ5ofSXw4Ek= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3 h1:XQyxROzUlZH+WIQwySDgnISgOivlhjIEwaQaJEJrrN0= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191010194322-b09406accb47 h1:/XfQ9z7ib8eEJX2hdgFTZJ/ntt0swNk5oYBziWeTCvY= +golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980 h1:OjiUf46hAmXblsZdnoSXsEUSKU8r1UEzcL5RVZ4gO9Y= +golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135 h1:5Beo0mZN8dRzgrMMkDp0jc8YXQKx9DiJ2k1dkvGsn5A= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200608174601-1b747fd94509 h1:MI14dOfl3OG6Zd32w3ugsrvcUO810fDZdWakTq39dH4= +golang.org/x/tools v0.0.0-20200608174601-1b747fd94509/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.29.1 h1:EC2SB8S04d2r73uptxphDSUG+kTKVgjRPF+N3xpxRB4= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo= +gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/providers/abstractProvider.go b/providers/abstractProvider.go index e5a7310c6..e3dbca493 100644 --- a/providers/abstractProvider.go +++ b/providers/abstractProvider.go @@ -10,6 +10,7 @@ import ( "strings" "github.com/fsnotify/fsnotify" + "github.com/darkweak/souin/configuration" ) // CommonProvider contains a Certificate map @@ -24,18 +25,10 @@ type Certificate struct { key string } -func getProviders() []string { - return []string{ - "traefik", - "apache", - "nginx", - } -} - // InitProviders function allow to init certificates and be able to exploit data as needed -func InitProviders(tlsconfig *tls.Config, configChannel *chan int) { +func InitProviders(tlsconfig *tls.Config, configChannel *chan int, configuration configuration.Configuration) { var providers []CommonProvider - for _, provider := range getProviders() { + for _, provider := range configuration.SSLProviders { providers = append(providers, CommonProvider{ Certificates: make(map[string]Certificate), fileLocation: fmt.Sprintf("/ssl/%s.json", provider), diff --git a/providers/abstractProvider_test.go b/providers/abstractProvider_test.go new file mode 100644 index 000000000..efe1cc5cc --- /dev/null +++ b/providers/abstractProvider_test.go @@ -0,0 +1,37 @@ +package providers + +import ( + "crypto/tls" + "github.com/darkweak/souin/configuration" + "testing" +) + +func mockConfiguration() configuration.Configuration { + return configuration.Configuration{ + SSLProviders: []string{}, + ReverseProxyURL: "http://traefik", + TTL: "100", + Cache: configuration.Cache{ + Headers: []string{}, + Providers: []string{}, + }, + Regex: configuration.Regex{ + Exclude: "MyCustomRegex", + }, + Redis: configuration.Redis{ + URL: "redis:6379", + }, + } +} + +func TestIShouldBeAbleToReadCertificatesWithoutError(t *testing.T) { + configChannel := make(chan int) + config := &tls.Config{ + Certificates: make([]tls.Certificate, 0), + NameToCertificate: make(map[string]*tls.Certificate), + InsecureSkipVerify: true, + } + v, _ := tls.LoadX509KeyPair("server.crt", "server.key") + config.Certificates = append(config.Certificates, v) + InitProviders(config, &configChannel, mockConfiguration()) +}