From cad3c24f549eb26cf4ef6e4baebb04e71bdbe3dc Mon Sep 17 00:00:00 2001 From: Jeff Mitchell Date: Sun, 22 Mar 2020 20:17:23 -0400 Subject: [PATCH] Add auth over ALPN (#5) --- go.mod | 5 +- go.sum | 19 ++- internal/cmd/base/listener.go | 14 ++ internal/cmd/base/servers.go | 10 -- .../cmd/commands/controller/controller.go | 37 +++-- internal/cmd/commands/dev/dev.go | 46 +++--- internal/cmd/commands/worker/worker.go | 37 +++-- internal/servers/controller/listeners.go | 17 ++- .../servers/controller/worker_tls_config.go | 72 ++++++++- .../worker/{client_tls.go => cluster_tls.go} | 144 +++++++----------- internal/servers/worker/listeners.go | 111 ++++---------- 11 files changed, 278 insertions(+), 234 deletions(-) rename internal/servers/worker/{client_tls.go => cluster_tls.go} (51%) diff --git a/go.mod b/go.mod index ea6b68c86..78db9a162 100644 --- a/go.mod +++ b/go.mod @@ -7,9 +7,9 @@ require ( github.com/fatih/color v1.9.0 github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 github.com/hashicorp/errwrap v1.0.0 - github.com/hashicorp/go-alpnmux v0.0.0-20200319202932-49b61a4084e2 + github.com/hashicorp/go-alpnmux v0.0.0-20200323001347-b5ec1528c52d github.com/hashicorp/go-hclog v0.12.1 - github.com/hashicorp/go-kms-wrapping v0.5.5 + github.com/hashicorp/go-kms-wrapping v0.5.7-0.20200322213809-e2a819ec93db github.com/hashicorp/go-multierror v1.0.0 github.com/hashicorp/hcl v1.0.0 github.com/hashicorp/vault v1.2.1-0.20200310200732-a321b1217d5a @@ -28,4 +28,5 @@ require ( github.com/ryanuber/columnize v2.1.0+incompatible golang.org/x/net v0.0.0-20200301022130-244492dfa37a google.golang.org/grpc v1.27.1 + google.golang.org/protobuf v1.20.1 ) diff --git a/go.sum b/go.sum index 12b9dbea6..fec9ac415 100644 --- a/go.sum +++ b/go.sum @@ -250,6 +250,11 @@ github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.4 h1:87PNWwrRvUSnqS4dlcBU/ftvOIBep4sYuBLlh6rX2wk= github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4 h1:+EOh4OY6tjM6ZueeUKinl1f0U2820HzQOuf1iqMnsks= +github.com/golang/protobuf v1.4.0-rc.4/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= @@ -312,8 +317,8 @@ github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyN github.com/hashicorp/consul/sdk v0.2.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-alpnmux v0.0.0-20200319202932-49b61a4084e2 h1:GdRmzHAdXxotjzcr3NkN2/zUqs0NoF6zujfoXjJjdO8= -github.com/hashicorp/go-alpnmux v0.0.0-20200319202932-49b61a4084e2/go.mod h1:KvpteZzIafT4tRAuQ9vVRBgZyqeVCS2B2177fNAyEZc= +github.com/hashicorp/go-alpnmux v0.0.0-20200323001347-b5ec1528c52d h1:5r4nKvUcI+vx2yDn9ysFf53mSY4kUbXqbZ4LF10skFE= +github.com/hashicorp/go-alpnmux v0.0.0-20200323001347-b5ec1528c52d/go.mod h1:KvpteZzIafT4tRAuQ9vVRBgZyqeVCS2B2177fNAyEZc= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= @@ -333,6 +338,8 @@ github.com/hashicorp/go-immutable-radix v1.1.0 h1:vN9wG1D6KG6YHRTWr8512cxGOVgTME github.com/hashicorp/go-immutable-radix v1.1.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-kms-wrapping v0.5.5 h1:ld350yu4UPAFCyt5NifPU3WpFts71z69n3NfMeTv2z8= github.com/hashicorp/go-kms-wrapping v0.5.5/go.mod h1:FeTpMJSbqrCtAeEK30hlNS8jm6W7cjzmLWurZ4Tk/J4= +github.com/hashicorp/go-kms-wrapping v0.5.7-0.20200322213809-e2a819ec93db h1:rMCnrYSub7wCuu12uJTWGM8ytpINMEFBpJ4hd/pY75w= +github.com/hashicorp/go-kms-wrapping v0.5.7-0.20200322213809-e2a819ec93db/go.mod h1:50IDNt66ZDhcLYeQxuo3v8YSF5FXc99EqM7jO8KurGk= github.com/hashicorp/go-kms-wrapping/entropy v0.1.0 h1:xuTi5ZwjimfpvpL09jDE71smCBRpnF5xfo871BSX4gs= github.com/hashicorp/go-kms-wrapping/entropy v0.1.0/go.mod h1:d1g9WGtAunDNpek8jUIEJnBlbgKS1N2Q61QkHiZyR1g= github.com/hashicorp/go-memdb v1.0.2 h1:AIjzJlwIxz2inhZqRJZfe6D15lPeF0/cZyS1BVlnlHg= @@ -452,6 +459,7 @@ github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKe github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d h1:kJCB4vdITiW1eC1vq2e6IsrXKrZit1bv/TDYFGMp4BQ= github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/huaweicloud/golangsdk v0.0.0-20200304081349-45ec0797f2a4/go.mod h1:WQBcHRNX9shz3928lWEvstQJtAtYI7ks6XlgtRT9Tcw= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb v0.0.0-20190411212539-d24b7ba8c4c4 h1:3K3KcD4S6/Y2hevi70EzUTNKOS3cryQyhUnkjE6Tz0w= github.com/influxdata/influxdb v0.0.0-20190411212539-d24b7ba8c4c4/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= @@ -705,6 +713,7 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= +github.com/tencentcloud/tencentcloud-sdk-go v3.0.132+incompatible/go.mod h1:0PfYow01SHPMhKY31xa+EFz2RStxIqj6JFAJS+IkCi4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926 h1:G3dpKMzFDjgEh2q1Z7zUUtKa8ViPtH+ocF0bE0g00O8= @@ -902,6 +911,12 @@ google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.27.1 h1:zvIju4sqAGvwKspUQOhwnpcqSbzi7/H6QomNNjTL4sk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.20.1 h1:ESRXHgpUBG5D2I5mmsQIyYxB/tQIZfSZ8wLyFDf/N/U= +google.golang.org/protobuf v1.20.1/go.mod h1:KqelGeouBkcbcuB3HCk4/YH2tmNLk6YSWA5LIWeI/lY= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/cmd/base/listener.go b/internal/cmd/base/listener.go index 427f3ae6b..ba38f4c87 100644 --- a/internal/cmd/base/listener.go +++ b/internal/cmd/base/listener.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "net" + "net/http" "strings" "time" @@ -20,6 +21,19 @@ import ( "github.com/pires/go-proxyproto" ) +type ServerListener struct { + Mux *alpnmux.ALPNMux + Config *configutil.Listener + HTTPServer *http.Server + ALPNListener net.Listener +} + +type WorkerAuthCertInfo struct { + CACertPEM []byte `json:"ca_cert"` + CertPEM []byte `json:"cert"` + KeyPEM []byte `json:"key"` +} + // Factory is the factory function to create a listener. type ListenerFactory func(*configutil.Listener, io.Writer, cli.Ui) (*alpnmux.ALPNMux, map[string]string, reloadutil.ReloadFunc, error) diff --git a/internal/cmd/base/servers.go b/internal/cmd/base/servers.go index 1c984082d..d824eae51 100644 --- a/internal/cmd/base/servers.go +++ b/internal/cmd/base/servers.go @@ -8,8 +8,6 @@ import ( "errors" "fmt" "io" - "net" - "net/http" "os" "sort" "strconv" @@ -18,7 +16,6 @@ import ( "github.com/armon/go-metrics" "github.com/hashicorp/errwrap" - "github.com/hashicorp/go-alpnmux" "github.com/hashicorp/go-hclog" wrapping "github.com/hashicorp/go-kms-wrapping" "github.com/hashicorp/go-multierror" @@ -38,13 +35,6 @@ import ( _ "github.com/lib/pq" ) -type ServerListener struct { - Mux *alpnmux.ALPNMux - Config *configutil.Listener - HTTPServer *http.Server - ALPNListener net.Listener -} - type Server struct { InfoKeys []string Info map[string]string diff --git a/internal/cmd/commands/controller/controller.go b/internal/cmd/commands/controller/controller.go index de1c96db9..2ac8547d7 100644 --- a/internal/cmd/commands/controller/controller.go +++ b/internal/cmd/commands/controller/controller.go @@ -32,7 +32,8 @@ type Command struct { cleanupGuard sync.Once - Config *config.Config + Config *config.Config + controller *controller.Controller flagConfig string flagLogLevel string @@ -247,7 +248,12 @@ func (c *Command) Run(args []string) int { c.PrintInfo(c.UI, "controller") c.ReleaseLogGate() - return c.Start() + if err := c.Start(); err != nil { + c.UI.Error(err.Error()) + return 1 + } + + return c.WaitForInterrupt() } func (c *Command) ParseFlagsAndConfig(args []string) int { @@ -307,7 +313,7 @@ func (c *Command) ParseFlagsAndConfig(args []string) int { return 0 } -func (c *Command) Start() int { +func (c *Command) Start() error { // Instantiate the wait group conf := &controller.Config{ RawConfig: c.Config, @@ -315,20 +321,25 @@ func (c *Command) Start() int { } // Initialize the core - ctlr, err := controller.New(conf) + var err error + c.controller, err = controller.New(conf) if err != nil { - c.UI.Error(fmt.Errorf("Error initializing controller: %w", err).Error()) - return 1 + return fmt.Errorf("Error initializing controller: %w", err) } - if err := ctlr.Start(); err != nil { - c.UI.Error(fmt.Sprint("Error starting controller: %w", err)) - if err := ctlr.Shutdown(); err != nil { - c.UI.Error(fmt.Errorf("Error with controller shutdown: %w", err).Error()) + if err := c.controller.Start(); err != nil { + retErr := fmt.Errorf("Error starting controller: %w", err) + if err := c.controller.Shutdown(); err != nil { + c.UI.Error(retErr.Error()) + retErr = fmt.Errorf("Error with controller shutdown: %w", err) } - return 1 + return retErr } + return nil +} + +func (c *Command) WaitForInterrupt() int { // Wait for shutdown shutdownTriggered := false @@ -337,7 +348,7 @@ func (c *Command) Start() int { case <-c.ShutdownCh: c.UI.Output("==> Watchtower controller shutdown triggered") - if err := ctlr.Shutdown(); err != nil { + if err := c.controller.Shutdown(); err != nil { c.UI.Error(fmt.Errorf("Error with controller shutdown: %w", err).Error()) } @@ -387,7 +398,7 @@ func (c *Command) Start() int { c.Logger.Error("unknown log level found on reload", "level", newConf.LogLevel) goto RUNRELOADFUNCS } - ctlr.SetLogLevel(level) + c.controller.SetLogLevel(level) } RUNRELOADFUNCS: diff --git a/internal/cmd/commands/dev/dev.go b/internal/cmd/commands/dev/dev.go index 0379c681e..f668e63b5 100644 --- a/internal/cmd/commands/dev/dev.go +++ b/internal/cmd/commands/dev/dev.go @@ -242,27 +242,40 @@ func (c *Command) Run(args []string) int { shutdownWg.Add(2) controllerSighupCh := make(chan struct{}) c.childSighupCh = append(c.childSighupCh, controllerSighupCh) + + devController := &controllercmd.Command{ + Command: c.Command, + Server: c.Server, + ShutdownCh: childShutdownCh, + SighupCh: controllerSighupCh, + Config: devConfig, + } + if err := devController.Start(); err != nil { + c.UI.Error(err.Error()) + return 1 + } + + workerSighupCh := make(chan struct{}) + c.childSighupCh = append(c.childSighupCh, workerSighupCh) + devWorker := &workercmd.Command{ + Command: c.Command, + Server: c.Server, + ShutdownCh: childShutdownCh, + SighupCh: workerSighupCh, + Config: devConfig, + } + if err := devWorker.Start(); err != nil { + c.UI.Error(err.Error()) + return 1 + } + go func() { defer shutdownWg.Done() - devController := &controllercmd.Command{ - Command: c.Command, - Server: c.Server, - ShutdownCh: childShutdownCh, - SighupCh: controllerSighupCh, - Config: devConfig, - } - devController.Start() + devController.WaitForInterrupt() }() go func() { defer shutdownWg.Done() - devWorker := &workercmd.Command{ - Command: c.Command, - Server: c.Server, - ShutdownCh: childShutdownCh, - SighupCh: controllerSighupCh, - Config: devConfig, - } - devWorker.Start() + devWorker.WaitForInterrupt() }() // Wait for shutdown @@ -290,7 +303,6 @@ func (c *Command) Run(args []string) int { } } - // Wait for dependent goroutines to complete shutdownWg.Wait() return 0 diff --git a/internal/cmd/commands/worker/worker.go b/internal/cmd/commands/worker/worker.go index 22eb9fa1f..301037f35 100644 --- a/internal/cmd/commands/worker/worker.go +++ b/internal/cmd/commands/worker/worker.go @@ -31,6 +31,7 @@ type Command struct { cleanupGuard sync.Once Config *config.Config + worker *worker.Worker flagConfig string flagLogLevel string @@ -178,7 +179,12 @@ func (c *Command) Run(args []string) int { c.PrintInfo(c.UI, "worker") c.ReleaseLogGate() - return c.Start() + if err := c.Start(); err != nil { + c.UI.Error(err.Error()) + return 1 + } + + return c.WaitForInterrupt() } func (c *Command) ParseFlagsAndConfig(args []string) int { @@ -218,7 +224,7 @@ func (c *Command) ParseFlagsAndConfig(args []string) int { return 0 } -func (c *Command) Start() int { +func (c *Command) Start() error { // Instantiate the wait group conf := &worker.Config{ RawConfig: c.Config, @@ -226,20 +232,25 @@ func (c *Command) Start() int { } // Initialize the core - wrkr, err := worker.New(conf) + var err error + c.worker, err = worker.New(conf) if err != nil { - c.UI.Error(fmt.Errorf("Error initializing worker: %w", err).Error()) - return 1 + return fmt.Errorf("Error initializing worker: %w", err) } - if err := wrkr.Start(); err != nil { - c.UI.Error(fmt.Errorf("Error starting worker: %w", err).Error()) - if err := wrkr.Shutdown(); err != nil { - c.UI.Error(fmt.Errorf("Error with worker shutdown: %w", err).Error()) + if err := c.worker.Start(); err != nil { + retErr := fmt.Errorf("Error starting worker: %w", err) + if err := c.worker.Shutdown(); err != nil { + c.UI.Error(retErr.Error()) + retErr = fmt.Errorf("Error with worker shutdown: %w", err) } - return 1 + return retErr } + return nil +} + +func (c *Command) WaitForInterrupt() int { // Wait for shutdown shutdownTriggered := false @@ -248,7 +259,7 @@ func (c *Command) Start() int { case <-c.ShutdownCh: c.UI.Output("==> Watchtower worker shutdown triggered") - if err := wrkr.Shutdown(); err != nil { + if err := c.worker.Shutdown(); err != nil { c.UI.Error(fmt.Errorf("Error with worker shutdown: %w", err).Error()) } @@ -279,7 +290,7 @@ func (c *Command) Start() int { } // Commented out until we need this - //wrkr.SetConfig(config) + //c.worker.SetConfig(config) if newConf.LogLevel != "" { configLogLevel := strings.ToLower(strings.TrimSpace(newConf.LogLevel)) @@ -298,7 +309,7 @@ func (c *Command) Start() int { c.Logger.Error("unknown log level found on reload", "level", newConf.LogLevel) goto RUNRELOADFUNCS } - wrkr.SetLogLevel(level) + c.worker.SetLogLevel(level) } RUNRELOADFUNCS: diff --git a/internal/servers/controller/listeners.go b/internal/servers/controller/listeners.go index cbe9a8cf8..11e3268c9 100644 --- a/internal/servers/controller/listeners.go +++ b/internal/servers/controller/listeners.go @@ -93,15 +93,26 @@ func (c *Controller) startListeners() error { } configureForCluster := func(ln *base.ServerListener) error { - l, err := ln.Mux.RegisterProto("watchtower-worker-v1", &tls.Config{ + l, err := ln.Mux.RegisterProto(alpnmux.DefaultProto, &tls.Config{ GetConfigForClient: c.validateWorkerTLS, }) if err != nil { return fmt.Errorf("error getting sub-listener for worker proto: %w", err) } + ln.ALPNListener = l - // TODO: Pass this to a handler, e.g. a grpc server - _ = l + // TODO: Pass this to a handler, e.g. a grpc server, in the mean time + // just accepting what comes + go func() { + for { + conn, err := ln.ALPNListener.Accept() + if err != nil { + c.conf.Logger.Info("default alpn listener errored, exiting") + return + } + conn.Close() + } + }() return nil } diff --git a/internal/servers/controller/worker_tls_config.go b/internal/servers/controller/worker_tls_config.go index ada09a62e..711e69a50 100644 --- a/internal/servers/controller/worker_tls_config.go +++ b/internal/servers/controller/worker_tls_config.go @@ -1,7 +1,77 @@ package controller -import "crypto/tls" +import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/base64" + "encoding/json" + "errors" + "strings" + + wrapping "github.com/hashicorp/go-kms-wrapping" + "github.com/hashicorp/watchtower/internal/cmd/base" + "google.golang.org/protobuf/proto" +) func (c Controller) validateWorkerTLS(hello *tls.ClientHelloInfo) (*tls.Config, error) { + for _, p := range hello.SupportedProtos { + switch { + case strings.HasPrefix(p, "v1workerauth-"): + return c.v1WorkerAuthConfig(hello.SupportedProtos) + } + } return nil, nil } + +func (c Controller) v1WorkerAuthConfig(protos []string) (*tls.Config, error) { + var firstMatchProto string + var encString string + for _, p := range protos { + if strings.HasPrefix(p, "v1workerauth-") { + // Strip that and the number + encString += strings.TrimPrefix(p, "v1workerauth-")[3:] + if firstMatchProto == "" { + firstMatchProto = p + } + } + } + if firstMatchProto == "" { + return nil, errors.New("no matching proto found") + } + marshaledEncInfo, err := base64.RawStdEncoding.DecodeString(encString) + if err != nil { + return nil, err + } + encInfo := new(wrapping.EncryptedBlobInfo) + if err := proto.Unmarshal(marshaledEncInfo, encInfo); err != nil { + return nil, err + } + marshaledInfo, err := c.conf.WorkerAuthKMS.Decrypt(context.Background(), encInfo, nil) + if err != nil { + return nil, err + } + info := new(base.WorkerAuthCertInfo) + if err := json.Unmarshal(marshaledInfo, info); err != nil { + return nil, err + } + + rootCAs := x509.NewCertPool() + if ok := rootCAs.AppendCertsFromPEM(info.CACertPEM); !ok { + return nil, errors.New("unable to add ca cert to cert pool") + } + tlsCert, err := tls.X509KeyPair(info.CertPEM, info.KeyPEM) + if err != nil { + return nil, err + } + tlsConfig := &tls.Config{ + Certificates: []tls.Certificate{tlsCert}, + ClientCAs: rootCAs, + ClientAuth: tls.RequireAndVerifyClientCert, + NextProtos: []string{firstMatchProto}, + MinVersion: tls.VersionTLS13, + } + tlsConfig.BuildNameToCertificate() + + return tlsConfig, nil +} diff --git a/internal/servers/worker/client_tls.go b/internal/servers/worker/cluster_tls.go similarity index 51% rename from internal/servers/worker/client_tls.go rename to internal/servers/worker/cluster_tls.go index ee855fed1..d535257bc 100644 --- a/internal/servers/worker/client_tls.go +++ b/internal/servers/worker/cluster_tls.go @@ -1,57 +1,34 @@ package worker import ( + "context" "crypto/ed25519" "crypto/tls" "crypto/x509" "crypto/x509/pkix" + "encoding/base64" + "encoding/json" "encoding/pem" - "io/ioutil" + "fmt" "math/big" mathrand "math/rand" - "net" - "os" - "path/filepath" "time" "github.com/hashicorp/vault/sdk/helper/base62" + "github.com/hashicorp/watchtower/internal/cmd/base" + "google.golang.org/protobuf/proto" ) -type workerTLSOpts struct { - Address string - Protos []string - DumpDir string -} - -type certInfo struct { - CACert []byte `json:"ca_cert"` - CAKey []byte `json:"ca_key"` -} - -func (c Worker) workerTLS(opts workerTLSOpts) (*tls.Config, *certInfo, error) { - info := new(certInfo) - - certIPs := []net.IP{ - net.IPv6loopback, - net.ParseIP("127.0.0.1"), - } - - if opts.Address != "" { - baseAddr, err := net.ResolveTCPAddr("tcp", opts.Address) - if err != nil { - return nil, nil, err - } - certIPs = append(certIPs, baseAddr.IP) - } +func (c Worker) workerAuthTLSConfig() (*tls.Config, error) { + info := new(base.WorkerAuthCertInfo) _, caKey, err := ed25519.GenerateKey(c.conf.SecureRandomReader) if err != nil { - return nil, nil, err + return nil, err } - info.CAKey = caKey caHost, err := base62.Random(20) if err != nil { - return nil, nil, err + return nil, err } caCertTemplate := &x509.Certificate{ @@ -59,7 +36,6 @@ func (c Worker) workerTLS(opts workerTLSOpts) (*tls.Config, *certInfo, error) { CommonName: caHost, }, DNSNames: []string{caHost}, - IPAddresses: certIPs, KeyUsage: x509.KeyUsage(x509.KeyUsageCertSign | x509.KeyUsageCRLSign), SerialNumber: big.NewInt(mathrand.Int63()), NotBefore: time.Now().Add(-30 * time.Second), @@ -69,50 +45,34 @@ func (c Worker) workerTLS(opts workerTLSOpts) (*tls.Config, *certInfo, error) { } caBytes, err := x509.CreateCertificate(c.conf.SecureRandomReader, caCertTemplate, caCertTemplate, caKey.Public(), caKey) if err != nil { - return nil, nil, err + return nil, err } - info.CACert = caBytes - caCert, err := x509.ParseCertificate(caBytes) - if err != nil { - return nil, nil, err - } - - rootCAs := x509.NewCertPool() - rootCAs.AddCert(caCert) caCertPEMBlock := &pem.Block{ Type: "CERTIFICATE", Bytes: caBytes, } - caCertPEM := pem.EncodeToMemory(caCertPEMBlock) - caCertPEMFile := filepath.Join(opts.DumpDir, "ca_cert.pem") - - marshaledCAKey, err := x509.MarshalPKCS8PrivateKey(caKey) + info.CACertPEM = pem.EncodeToMemory(caCertPEMBlock) + caCert, err := x509.ParseCertificate(caBytes) if err != nil { - return nil, nil, err + return nil, err } - caKeyPEMBlock := &pem.Block{ - Type: "PRIVATE KEY", - Bytes: marshaledCAKey, - } - caKeyPEM := pem.EncodeToMemory(caKeyPEMBlock) // // Certs generation // _, key, err := ed25519.GenerateKey(c.conf.SecureRandomReader) if err != nil { - return nil, nil, err + return nil, err } host, err := base62.Random(20) if err != nil { - return nil, nil, err + return nil, err } certTemplate := &x509.Certificate{ Subject: pkix.Name{ CommonName: host, }, - DNSNames: []string{host}, - IPAddresses: certIPs, + DNSNames: []string{host}, ExtKeyUsage: []x509.ExtKeyUsage{ x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth, @@ -124,59 +84,63 @@ func (c Worker) workerTLS(opts workerTLSOpts) (*tls.Config, *certInfo, error) { } certBytes, err := x509.CreateCertificate(c.conf.SecureRandomReader, certTemplate, caCert, key.Public(), caKey) if err != nil { - return nil, nil, err + return nil, err } certPEMBlock := &pem.Block{ Type: "CERTIFICATE", Bytes: certBytes, } - certPEM := pem.EncodeToMemory(certPEMBlock) + info.CertPEM = pem.EncodeToMemory(certPEMBlock) marshaledKey, err := x509.MarshalPKCS8PrivateKey(key) if err != nil { - return nil, nil, err + return nil, err } keyPEMBlock := &pem.Block{ Type: "PRIVATE KEY", Bytes: marshaledKey, } - keyPEM := pem.EncodeToMemory(keyPEMBlock) + info.KeyPEM = pem.EncodeToMemory(keyPEMBlock) - certFile := filepath.Join(opts.DumpDir, "cert.pem") - keyFile := filepath.Join(opts.DumpDir, "key.pem") - - tlsCert, err := tls.X509KeyPair(certPEM, keyPEM) + // Marshal and encrypt + marshaledInfo, err := json.Marshal(info) if err != nil { - return nil, nil, err + return nil, err + } + encInfo, err := c.conf.WorkerAuthKMS.Encrypt(context.Background(), marshaledInfo, nil) + if err != nil { + return nil, err + } + marshaledEncInfo, err := proto.Marshal(encInfo) + if err != nil { + return nil, err + } + b64alpn := base64.RawStdEncoding.EncodeToString(marshaledEncInfo) + var nextProtos []string + var count int + for i := 0; i < len(b64alpn); i += 230 { + end := i + 230 + if end > len(b64alpn) { + end = len(b64alpn) + } + nextProtos = append(nextProtos, fmt.Sprintf("v1workerauth-%02d-%s", count, b64alpn[i:end])) + count++ + } + + // Build local tls config + rootCAs := x509.NewCertPool() + rootCAs.AddCert(caCert) + tlsCert, err := tls.X509KeyPair(info.CertPEM, info.KeyPEM) + if err != nil { + return nil, err } tlsConfig := &tls.Config{ + ServerName: host, Certificates: []tls.Certificate{tlsCert}, RootCAs: rootCAs, - ClientCAs: rootCAs, - ClientAuth: tls.RequestClientCert, - NextProtos: opts.Protos, + NextProtos: nextProtos, MinVersion: tls.VersionTLS13, } tlsConfig.BuildNameToCertificate() - if opts.DumpDir != "" { - if _, err := os.Stat(opts.DumpDir); os.IsNotExist(err) { - if err := os.MkdirAll(opts.DumpDir, 0700); err != nil { - return nil, nil, err - } - } - if err := ioutil.WriteFile(filepath.Join(opts.DumpDir, "ca_key.pem"), caKeyPEM, 0755); err != nil { - return nil, nil, err - } - if err := ioutil.WriteFile(caCertPEMFile, caCertPEM, 0755); err != nil { - return nil, nil, err - } - if err := ioutil.WriteFile(certFile, certPEM, 0755); err != nil { - return nil, nil, err - } - if err := ioutil.WriteFile(keyFile, keyPEM, 0755); err != nil { - return nil, nil, err - } - } - - return tlsConfig, info, nil + return tlsConfig, nil } diff --git a/internal/servers/worker/listeners.go b/internal/servers/worker/listeners.go index 70be5ff6a..a5147b5a5 100644 --- a/internal/servers/worker/listeners.go +++ b/internal/servers/worker/listeners.go @@ -2,15 +2,12 @@ package worker import ( "context" - "errors" + "crypto/tls" "fmt" - "net" - "net/http" "sync" - "time" - "github.com/hashicorp/go-alpnmux" "github.com/hashicorp/go-multierror" + "github.com/hashicorp/vault/sdk/helper/strutil" ) func (c *Worker) startListeners() error { @@ -19,89 +16,37 @@ func (c *Worker) startListeners() error { for _, ln := range c.conf.Listeners { switch c.conf.RawConfig.DevController { case false: - handler := c.Handler(HandlerProperties{ - ListenerConfig: ln.Config, - }) - - /* - // TODO: As I write this Vault's having this code audited, make sure to - // port over any recommendations - // - // We perform validation on the config earlier, we can just cast here - if _, ok := ln.config["x_forwarded_for_authorized_addrs"]; ok { - hopSkips := ln.config["x_forwarded_for_hop_skips"].(int) - authzdAddrs := ln.config["x_forwarded_for_authorized_addrs"].([]*sockaddr.SockAddrMarshaler) - rejectNotPresent := ln.config["x_forwarded_for_reject_not_present"].(bool) - rejectNonAuthz := ln.config["x_forwarded_for_reject_not_authorized"].(bool) - if len(authzdAddrs) > 0 { - handler = vaulthttp.WrapForwardedForHandler(handler, authzdAddrs, rejectNotPresent, rejectNonAuthz, hopSkips) - } - } - */ - - server := &http.Server{ - Handler: handler, - ReadHeaderTimeout: 10 * time.Second, - ReadTimeout: 30 * time.Second, - IdleTimeout: 5 * time.Minute, - ErrorLog: c.conf.Logger.StandardLogger(nil), - BaseContext: func(net.Listener) context.Context { - return c.baseContext - }, - } - ln.HTTPServer = server - - if ln.Config.HTTPReadHeaderTimeout > 0 { - server.ReadHeaderTimeout = ln.Config.HTTPReadHeaderTimeout - } - if ln.Config.HTTPReadTimeout > 0 { - server.ReadTimeout = ln.Config.HTTPReadTimeout - } - if ln.Config.HTTPWriteTimeout > 0 { - server.WriteTimeout = ln.Config.HTTPWriteTimeout - } - if ln.Config.HTTPIdleTimeout > 0 { - server.IdleTimeout = ln.Config.HTTPIdleTimeout - } - - switch ln.Config.TLSDisable { - case true: - l := ln.Mux.GetListener(alpnmux.NoProto) - if l == nil { - retErr = multierror.Append(retErr, errors.New("could not get non-tls listener")) - continue - } - servers = append(servers, func() { - go server.Serve(l) - }) - - default: - protos := []string{"", "http/1.1", "h2"} - for _, v := range protos { - l := ln.Mux.GetListener(v) - if l == nil { - retErr = multierror.Append(retErr, fmt.Errorf("could not get tls proto %q listener", v)) - continue - } - servers = append(servers, func() { - go server.Serve(l) - }) - } - } - - fallthrough + // TODO: We'll eventually need to configure HTTP listening here for + // org-provided certificate handling, and configure the mux's + // defaultproto for accepting client connections for ALPN-based + // auth default: - if !ln.Config.TLSDisable { - l := ln.Mux.GetListener(alpnmux.DefaultProto) - if l == nil { - retErr = multierror.Append(retErr, fmt.Errorf("could not get tls proto %q listener", alpnmux.DefaultProto)) + // TODO: We'll need to go through any listeners marked for api + // usage and add our websocket handlers to the server. Eventually + // we may want to make the config function able to handle arbitrary + // ALPNs in a dynamic way, so that in dev mode we can also register + // alpn mode client auth handling via the cluster ports. + // TODO again because...I like that idea and don't want to forget + // about it :-) + // For now just testing out the ability to authorize via the ALPN handler + if strutil.StrListContains(ln.Config.Purpose, "cluster") { + tlsConf, err := c.workerAuthTLSConfig() + if err != nil { + retErr = multierror.Append(retErr, fmt.Errorf("error creating tls config for worker auth: %w", err)) continue } - ln.ALPNListener = l - // TODO: give this listener to a server to handle ALPN auth + if ln.ALPNListener != nil { + c.conf.Logger.Info("testing the waters with proto", "protos", tlsConf.NextProtos) + conn, err := tls.Dial(ln.ALPNListener.Addr().Network(), ln.ALPNListener.Addr().String(), tlsConf) + if err != nil { + retErr = multierror.Append(retErr, fmt.Errorf("error dialing controller for worker auth: %w", err)) + continue + } + c.conf.Logger.Info("negotiated a protocol", "proto", conn.ConnectionState().NegotiatedProtocol, "mutual", conn.ConnectionState().NegotiatedProtocolIsMutual) + conn.Close() + } } - // TODO: Register websocket handler against server on this listener } }