Skip to content

Commit 163ce18

Browse files
jefferaijefferai
authored
KMS-PKI Workers (hashicorp#3101)
This adds a new mechanism for worker authentication that uses the third-party arbitration of the original KMS flow by leveraging recently-added capabilities in nodeenrollment to support a registration wrapper. This allows for workers to be registered to the system with just a name and a matching wrapper, while at the same time allowing these workers to be used for private Vault access and multi-hop as they contain per-worker encryption keys and support rotation. Nodes using this mode will generate and rotate credentials in-memory (using another recently added nodeenrollment capability) so will have a new set of keys on every startup. This has the side effect of ensuring that there is never a conflict or re-use of these credentials. Note that the normal test worker flow was using the now-deprecated KMS method; there is still a test that uses this (explicitly) but all other tests using test workers that did not specify a specific PKI flow now actually use this mechanism. This means there is fairly decent test coverage in a general sense, plus the specific tests updated/added for this feature. This also adds a new KMS type that can be used to have distinct KMSes for upstream authentication vs. accepting from downstream.
1 parent 16f4868 commit 163ce18

39 files changed

+998
-489
lines changed

‎CHANGELOG.md

+14-1
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,17 @@
11
# Boundary CHANGELOG
22

33
Canonical reference for changes, improvements, and bugfixes for Boundary.
4+
45
## Next
56

67
### Deprecations/Changes
78

9+
* With the introduction of the new KMS variant for worker registration (as
10+
described below), using the deprecated behavior requires opting-in. This is
11+
only recommended if compatibility with pre-0.13 workers using the KMS auth
12+
method is required. Requiring opting in removes some potentially confusing
13+
behavior for deciding when to use the old versus new mechanism. To opt in, add
14+
`use_deprecated_kms_auth_method = true` to the `worker` config block.
815
* When grants are added to roles additional validity checking is now performed.
916
This extra validity checking is designed to reject grants that are not
1017
[documented grant
@@ -17,6 +24,12 @@ Canonical reference for changes, improvements, and bugfixes for Boundary.
1724

1825
### New and Improved
1926

27+
* KMS workers: KMS workers now have feature parity with PKI workers (they
28+
support multi-hop and Vault private access) and support separate KMSes for
29+
authenticating downstreams across different networks. See the [worker
30+
configuration
31+
documentation](https://developer.hashicorp.com/boundary/docs/configuration/worker)
32+
for more information. ([PR](https://github.com/hashicorp/boundary/pull/3101))
2033
* roles: Perform additional validity checking on grants at submission time
2134
([PR](https://github.com/hashicorp/boundary/pull/3081))
2235

@@ -37,7 +50,7 @@ Canonical reference for changes, improvements, and bugfixes for Boundary.
3750
credential type. [PR](https://github.com/hashicorp/boundary/pull/2989)
3851
* ui: Fix credential library not saving correctly when trying to save it as a
3952
generic secrets type. ([PR](https://github.com/hashicorp/boundary-ui/pull/1640))
40-
* sessions: Fix tofu token retreival. ([PR](https://github.com/hashicorp/boundary/pull/3064))
53+
* sessions: Fix tofu token retrieval. ([PR](https://github.com/hashicorp/boundary/pull/3064))
4154

4255
## 0.12.0 (2023/01/24)
4356

‎globals/kms.go

+7-6
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,11 @@
44
package globals
55

66
const (
7-
KmsPurposeRoot = "root"
8-
KmsPurposePreviousRoot = "previous-root"
9-
KmsPurposeWorkerAuth = "worker-auth"
10-
KmsPurposeWorkerAuthStorage = "worker-auth-storage"
11-
KmsPurposeRecovery = "recovery"
12-
KmsPurposeConfig = "config"
7+
KmsPurposeRoot = "root"
8+
KmsPurposePreviousRoot = "previous-root"
9+
KmsPurposeWorkerAuth = "worker-auth"
10+
KmsPurposeWorkerAuthStorage = "worker-auth-storage"
11+
KmsPurposeRecovery = "recovery"
12+
KmsPurposeConfig = "config"
13+
KmsPurposeDownstreamWorkerAuth = "downstream-worker-auth"
1314
)

‎go.mod

+2-2
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,7 @@ require (
9696
github.com/hashicorp/cap/ldap v0.0.0-20230123181313-9c0fb924b0d9
9797
github.com/hashicorp/go-kms-wrapping/extras/kms/v2 v2.0.0-20221122211539-47c893099f13
9898
github.com/hashicorp/go-version v1.3.0
99-
github.com/hashicorp/nodeenrollment v0.1.19
99+
github.com/hashicorp/nodeenrollment v0.2.0
100100
github.com/jimlambrt/gldap v0.1.2
101101
github.com/kelseyhightower/envconfig v1.4.0
102102
github.com/mikesmitty/edkey v0.0.0-20170222072505-3356ea4e686a
@@ -180,7 +180,7 @@ require (
180180
github.com/prometheus/client_model v0.2.0 // indirect
181181
github.com/prometheus/common v0.32.1 // indirect
182182
github.com/prometheus/procfs v0.7.3 // indirect
183-
github.com/rogpeppe/go-internal v1.8.1 // indirect
183+
github.com/rogpeppe/go-internal v1.9.0 // indirect
184184
github.com/russross/blackfriday/v2 v2.0.1 // indirect
185185
github.com/sethvargo/go-diceware v0.3.0 // indirect
186186
github.com/shopspring/decimal v1.3.1 // indirect

‎go.sum

+4-6
Original file line numberDiff line numberDiff line change
@@ -757,8 +757,8 @@ github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+l
757757
github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
758758
github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
759759
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
760-
github.com/hashicorp/nodeenrollment v0.1.19 h1:RZIYFVfjWlK8MZn7owEEmaoDLHMqROL2K9RkJz2Q5cA=
761-
github.com/hashicorp/nodeenrollment v0.1.19/go.mod h1:N5gYsm8mWiDfIw/j+1IQ6NBO1cWCmhPpvQ9GB1QUnsU=
760+
github.com/hashicorp/nodeenrollment v0.2.0 h1:79FbU+2MZaHgFxirQM1G9rU71FxCfXQ2hinEIR/xTco=
761+
github.com/hashicorp/nodeenrollment v0.2.0/go.mod h1:Do2FvaFHgnnMaqhdSTBX+TkQP0ep8VVQ0rOr+ZjUbJU=
762762
github.com/hashicorp/vault/api v1.3.1 h1:pkDkcgTh47PRjY1NEFeofqR4W/HkNUi9qIakESO2aRM=
763763
github.com/hashicorp/vault/api v1.3.1/go.mod h1:QeJoWxMFt+MsuWcYhmwRLwKEXrjwAFFywzhptMsTIUw=
764764
github.com/hashicorp/vault/sdk v0.1.13/go.mod h1:B+hVj7TpuQY1Y/GPbCpffmgd+tSEwvhkWnjtSYCaS2M=
@@ -1140,7 +1140,6 @@ github.com/pires/go-proxyproto v0.6.1/go.mod h1:Odh9VFOZJCf9G8cLW5o435Xf1J95Jw9G
11401140
github.com/pkg/browser v0.0.0-20210706143420-7d21f8c997e2/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI=
11411141
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI=
11421142
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e h1:aoZm08cpOy4WuID//EZDgcC4zIxODThtZNPirFr42+A=
1143-
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
11441143
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
11451144
github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
11461145
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -1202,8 +1201,8 @@ github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
12021201
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
12031202
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
12041203
github.com/rogpeppe/go-internal v1.6.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
1205-
github.com/rogpeppe/go-internal v1.8.1 h1:geMPLpDpQOgVyCg5z5GoRwLHepNdb71NXb67XFkP+Eg=
1206-
github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o=
1204+
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
1205+
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
12071206
github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
12081207
github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU=
12091208
github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc=
@@ -1952,7 +1951,6 @@ gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8
19521951
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
19531952
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
19541953
gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
1955-
gopkg.in/errgo.v2 v2.1.0 h1:0vLT13EuvQ0hNvakwLuFZ/jYrLp5F3kcWHXdRggjCE8=
19561954
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
19571955
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
19581956
gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=

‎internal/cmd/base/option.go

+18-27
Original file line numberDiff line numberDiff line change
@@ -25,25 +25,24 @@ type Option func(*Options)
2525

2626
// Options - how Options are represented.
2727
type Options struct {
28-
withNoTokenScope bool
29-
withNoTokenValue bool
30-
withSkipDatabaseDestruction bool
31-
withSkipAuthMethodCreation bool
32-
withSkipOidcAuthMethodCreation bool
33-
withSkipScopesCreation bool
34-
withSkipHostResourcesCreation bool
35-
withSkipTargetCreation bool
36-
withContainerImage string
37-
withDialect string
38-
withDatabaseTemplate string
39-
withEventerConfig *event.EventerConfig
40-
withEventFlags *EventFlags
41-
withEventWrapper wrapping.Wrapper
42-
withAttributeFieldPrefix string
43-
withStatusCode int
44-
withHostPlugin func() (string, plugin.HostPluginServiceClient)
45-
withEventGating bool
46-
withSkipWorkerAuthKmsInstantiation bool
28+
withNoTokenScope bool
29+
withNoTokenValue bool
30+
withSkipDatabaseDestruction bool
31+
withSkipAuthMethodCreation bool
32+
withSkipOidcAuthMethodCreation bool
33+
withSkipScopesCreation bool
34+
withSkipHostResourcesCreation bool
35+
withSkipTargetCreation bool
36+
withContainerImage string
37+
withDialect string
38+
withDatabaseTemplate string
39+
withEventerConfig *event.EventerConfig
40+
withEventFlags *EventFlags
41+
withEventWrapper wrapping.Wrapper
42+
withAttributeFieldPrefix string
43+
withStatusCode int
44+
withHostPlugin func() (string, plugin.HostPluginServiceClient)
45+
withEventGating bool
4746
}
4847

4948
func getDefaultOptions() Options {
@@ -192,11 +191,3 @@ func WithEventGating(with bool) Option {
192191
o.withEventGating = with
193192
}
194193
}
195-
196-
// WithSkipWorkerAuthKmsInstantiation causes KMS methods used for worker-auth to
197-
// not be processed, useful for dev mode
198-
func WithSkipWorkerAuthKmsInstantiation(with bool) Option {
199-
return func(o *Options) {
200-
o.withSkipWorkerAuthKmsInstantiation = with
201-
}
202-
}

‎internal/cmd/base/servers.go

+42-26
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import (
1818
"strconv"
1919
"strings"
2020
"sync"
21+
"sync/atomic"
2122
"syscall"
2223
"time"
2324

@@ -76,12 +77,18 @@ type Server struct {
7677
StderrLock *sync.Mutex
7778
Eventer *event.Eventer
7879

79-
RootKms wrapping.Wrapper
80-
WorkerAuthKms wrapping.Wrapper
81-
WorkerAuthStorageKms wrapping.Wrapper
82-
RecoveryKms wrapping.Wrapper
83-
Kms *kms.Kms
84-
SecureRandomReader io.Reader
80+
// NOTE: Unlike the other wrappers below, if set, DownstreamWorkerAuthKms
81+
// should always be a PooledWrapper, so that we can allow multiple KMSes to
82+
// accept downstream connections. As such it's made explicit here.
83+
DownstreamWorkerAuthKms *multi.PooledWrapper
84+
RootKms wrapping.Wrapper
85+
WorkerAuthKms wrapping.Wrapper
86+
WorkerAuthStorageKms wrapping.Wrapper
87+
RecoveryKms wrapping.Wrapper
88+
Kms *kms.Kms
89+
SecureRandomReader io.Reader
90+
91+
WorkerAuthDebuggingEnabled *atomic.Bool
8592

8693
PrometheusRegisterer prometheus.Registerer
8794

@@ -118,10 +125,6 @@ type Server struct {
118125
DevTargetSessionConnectionLimit int
119126
DevLoopbackHostPluginId string
120127

121-
// DevUsePkiForUpstream is a hint that we are in dev mode and have a worker
122-
// auth KMS but want to use PKI for upstream connections
123-
DevUsePkiForUpstream bool
124-
125128
EnabledPlugins []EnabledPlugin
126129
HostPlugins map[string]plgpb.HostPluginServiceClient
127130

@@ -140,15 +143,16 @@ type Server struct {
140143
// NewServer creates a new Server.
141144
func NewServer(cmd *Command) *Server {
142145
return &Server{
143-
Command: cmd,
144-
ServerSideShutdownCh: make(chan struct{}),
145-
InfoKeys: make([]string, 0, 20),
146-
Info: make(map[string]string),
147-
SecureRandomReader: rand.Reader,
148-
ReloadFuncsLock: new(sync.RWMutex),
149-
ReloadFuncs: make(map[string][]reloadutil.ReloadFunc),
150-
StderrLock: new(sync.Mutex),
151-
PrometheusRegisterer: prometheus.DefaultRegisterer,
146+
Command: cmd,
147+
ServerSideShutdownCh: make(chan struct{}),
148+
InfoKeys: make([]string, 0, 20),
149+
Info: make(map[string]string),
150+
SecureRandomReader: rand.Reader,
151+
ReloadFuncsLock: new(sync.RWMutex),
152+
ReloadFuncs: make(map[string][]reloadutil.ReloadFunc),
153+
StderrLock: new(sync.Mutex),
154+
WorkerAuthDebuggingEnabled: new(atomic.Bool),
155+
PrometheusRegisterer: prometheus.DefaultRegisterer,
152156
}
153157
}
154158

@@ -546,25 +550,22 @@ func (b *Server) SetupListeners(ui cli.Ui, config *configutil.SharedConfig, allo
546550
// SetupKMSes takes in a parsed config, does some minor checking on purposes,
547551
// and sends each off to configutil to instantiate a wrapper.
548552
func (b *Server) SetupKMSes(ctx context.Context, ui cli.Ui, config *config.Config, opt ...Option) error {
549-
opts := getOpts(opt...)
550-
551553
sharedConfig := config.SharedConfig
552554
var pluginLogger hclog.Logger
553555
var err error
554556
var previousRootKms wrapping.Wrapper
557+
purposeCount := map[string]uint{}
555558
for _, kms := range sharedConfig.Seals {
556559
for _, purpose := range kms.Purpose {
557560
purpose = strings.ToLower(purpose)
561+
purposeCount[purpose] = purposeCount[purpose] + 1
558562
switch purpose {
559563
case "":
560564
return errors.New("KMS block missing 'purpose'")
561-
case globals.KmsPurposeWorkerAuth:
562-
if opts.withSkipWorkerAuthKmsInstantiation {
563-
continue
564-
}
565565
case globals.KmsPurposeRoot,
566566
globals.KmsPurposePreviousRoot,
567567
globals.KmsPurposeConfig,
568+
globals.KmsPurposeWorkerAuth,
568569
globals.KmsPurposeWorkerAuthStorage:
569570
case globals.KmsPurposeRecovery:
570571
if config.Controller != nil && config.DevRecoveryKey != "" {
@@ -595,7 +596,7 @@ func (b *Server) SetupKMSes(ctx context.Context, ui cli.Ui, config *config.Confi
595596
pluginutil.WithPluginsFilesystem(kms_plugin_assets.KmsPluginPrefix, kms_plugin_assets.FileSystem()),
596597
pluginutil.WithPluginExecutionDirectory(config.Plugins.ExecutionDir),
597598
),
598-
configutil.WithLogger(pluginLogger.Named(kms.Type).With("purpose", purpose)),
599+
configutil.WithLogger(pluginLogger.Named(kms.Type).With("purpose", fmt.Sprintf("%s-%d", purpose, purposeCount[purpose]))),
599600
)
600601
if wrapperConfigError != nil {
601602
return fmt.Errorf(
@@ -641,6 +642,21 @@ func (b *Server) SetupKMSes(ctx context.Context, ui cli.Ui, config *config.Confi
641642
return fmt.Errorf("Duplicate KMS block for purpose '%s'. You may need to remove all but the last KMS block for this purpose.", purpose)
642643
}
643644
b.WorkerAuthKms = wrapper
645+
case globals.KmsPurposeDownstreamWorkerAuth:
646+
if b.DownstreamWorkerAuthKms == nil {
647+
b.DownstreamWorkerAuthKms, err = multi.NewPooledWrapper(ctx, wrapper)
648+
if err != nil {
649+
return fmt.Errorf("Error instantiating pooled wrapper for downstream worker auth: %w.", err)
650+
}
651+
} else {
652+
added, err := b.DownstreamWorkerAuthKms.AddWrapper(ctx, wrapper)
653+
if err != nil {
654+
return fmt.Errorf("Error adding additional wrapper to downstream worker auth wrapper pool: %w.", err)
655+
}
656+
if !added {
657+
return fmt.Errorf("Wrapper already added to downstream worker auth wrapper pool.")
658+
}
659+
}
644660
case globals.KmsPurposeWorkerAuthStorage:
645661
if b.WorkerAuthStorageKms != nil {
646662
return fmt.Errorf("Duplicate KMS block for purpose '%s'. You may need to remove all but the last KMS block for this purpose.", purpose)

0 commit comments

Comments
 (0)