From 5acda6a0b17b8283724dbd2da973a212564507c9 Mon Sep 17 00:00:00 2001 From: Jeff Mitchell Date: Sun, 22 Mar 2020 08:27:24 -0400 Subject: [PATCH] Begin purpose-based listener refactoring (#4) --- internal/cmd/base/listener.go | 6 +- internal/cmd/base/servers.go | 2 + .../cmd/commands/controller/controller.go | 86 +++++++-- internal/cmd/commands/dev/dev.go | 48 +++-- internal/cmd/commands/worker/worker.go | 9 +- internal/cmd/config/config.go | 31 ++- internal/servers/controller/cluster_tls.go | 182 ------------------ internal/servers/controller/listeners.go | 57 ++++-- .../servers/controller/worker_tls_config.go | 7 + internal/servers/worker/listeners.go | 5 +- 10 files changed, 189 insertions(+), 244 deletions(-) delete mode 100644 internal/servers/controller/cluster_tls.go create mode 100644 internal/servers/controller/worker_tls_config.go diff --git a/internal/cmd/base/listener.go b/internal/cmd/base/listener.go index 8ba363f74..427f3ae6b 100644 --- a/internal/cmd/base/listener.go +++ b/internal/cmd/base/listener.go @@ -41,7 +41,11 @@ func NewListener(l *configutil.Listener, w io.Writer, ui cli.Ui) (*alpnmux.ALPNM func tcpListenerFactory(l *configutil.Listener, _ io.Writer, ui cli.Ui) (*alpnmux.ALPNMux, map[string]string, reloadutil.ReloadFunc, error) { if l.Address == "" { - l.Address = "127.0.0.1:9200" + if len(l.Purpose) == 1 && l.Purpose[0] == "cluster" { + l.Address = "127.0.0.1:9201" + } else { + l.Address = "127.0.0.1:9200" + } } bindProto := "tcp" diff --git a/internal/cmd/base/servers.go b/internal/cmd/base/servers.go index 43a37eafd..1c984082d 100644 --- a/internal/cmd/base/servers.go +++ b/internal/cmd/base/servers.go @@ -290,6 +290,8 @@ func (b *Server) SetupListeners(ui cli.Ui, config *configutil.SharedConfig) erro Config: lnConfig, }) + props["purpose"] = strings.Join(lnConfig.Purpose, ",") + // Store the listener props for output later key := fmt.Sprintf("listener %d", i+1) propsList := make([]string, 0, len(props)) diff --git a/internal/cmd/commands/controller/controller.go b/internal/cmd/commands/controller/controller.go index 85953ed03..de1c96db9 100644 --- a/internal/cmd/commands/controller/controller.go +++ b/internal/cmd/commands/controller/controller.go @@ -9,6 +9,7 @@ import ( "github.com/hashicorp/go-hclog" "github.com/hashicorp/go-multierror" "github.com/hashicorp/vault/sdk/helper/mlock" + "github.com/hashicorp/vault/sdk/helper/strutil" "github.com/hashicorp/watchtower/globals" "github.com/hashicorp/watchtower/internal/cmd/base" "github.com/hashicorp/watchtower/internal/cmd/config" @@ -33,13 +34,14 @@ type Command struct { Config *config.Config - flagConfig string - flagLogLevel string - flagLogFormat string - flagDev bool - flagDevAdminPassword string - flagDevControllerListenAddr string - flagCombineLogs bool + flagConfig string + flagLogLevel string + flagLogFormat string + flagDev bool + flagDevAdminPassword string + flagDevControllerAPIListenAddr string + flagDevControllerClusterListenAddr string + flagCombineLogs bool } func (c *Command) Synopsis() string { @@ -112,11 +114,17 @@ func (c *Command) Flags() *base.FlagSets { }) f.StringVar(&base.StringVar{ - Name: "dev-listen-address", - Target: &c.flagDevControllerListenAddr, - Default: "127.0.0.1:9200", - EnvVar: "WATCHTOWER_DEV_CONTROLLER_LISTEN_ADDRESS", - Usage: "Address to bind the controller to in \"dev\" mode.", + Name: "dev-api-listen-address", + Target: &c.flagDevControllerAPIListenAddr, + EnvVar: "WATCHTOWER_DEV_CONTROLLER_API_LISTEN_ADDRESS", + Usage: "Address to bind the controller to in \"dev\" mode for \"api\" purpose.", + }) + + f.StringVar(&base.StringVar{ + Name: "dev-cluster-listen-address", + Target: &c.flagDevControllerClusterListenAddr, + EnvVar: "WATCHTOWER_DEV_CONTROLLER_CLUSTER_LISTEN_ADDRESS", + Usage: "Address to bind the controller to in \"dev\" mode for \"cluster\" purpose.", }) f.BoolVar(&base.BoolVar{ @@ -178,6 +186,43 @@ func (c *Command) Run(args []string) int { "in a Docker container, provide the IPC_LOCK cap to the container.")) } + // Perform controller-specific listener checks here before setup + var foundCluster bool + var foundAPI bool + for _, lnConfig := range c.Config.Listeners { + switch len(lnConfig.Purpose) { + case 1: + switch lnConfig.Purpose[0] { + case "cluster": + foundCluster = true + case "api": + foundAPI = true + default: + c.UI.Error(fmt.Sprintf("Unknown listener purpose %q", lnConfig.Purpose[0])) + return 1 + } + + case 0: + lnConfig.Purpose = []string{"api", "cluster"} + fallthrough + + case 2: + if !strutil.StrListContains(lnConfig.Purpose, "api") || !strutil.StrListContains(lnConfig.Purpose, "cluster") { + c.UI.Error(fmt.Sprintf("Invalid listener purpose set: %v", lnConfig.Purpose)) + return 1 + } + if lnConfig.TLSDisable { + c.UI.Error(fmt.Sprintf("TLS cannot be disabled on listener when serving both %q and %q purposes", "api", "cluster")) + return 1 + } + foundAPI = true + foundCluster = true + } + } + if foundAPI && !foundCluster { + c.UI.Error("No listener marked for cluster purpose found, but listener explicitly marked for api was found") + return 1 + } if err := c.SetupListeners(c.UI, c.Config.SharedConfig); err != nil { c.UI.Error(err.Error()) return 1 @@ -241,8 +286,21 @@ func (c *Command) ParseFlagsAndConfig(args []string) int { return 1 } - if c.flagDevControllerListenAddr != "" { - c.Config.Listeners[0].Address = c.flagDevControllerListenAddr + for _, l := range c.Config.Listeners { + if len(l.Purpose) != 1 { + continue + } + switch l.Purpose[0] { + case "api": + if c.flagDevControllerAPIListenAddr != "" { + l.Address = c.flagDevControllerAPIListenAddr + } + + case "cluster": + if c.flagDevControllerClusterListenAddr != "" { + l.Address = c.flagDevControllerClusterListenAddr + } + } } } diff --git a/internal/cmd/commands/dev/dev.go b/internal/cmd/commands/dev/dev.go index 4ecc8a113..0379c681e 100644 --- a/internal/cmd/commands/dev/dev.go +++ b/internal/cmd/commands/dev/dev.go @@ -29,13 +29,14 @@ type Command struct { cleanupGuard sync.Once - flagConfig string - flagLogLevel string - flagLogFormat string - flagDev bool - flagDevAdminPassword string - flagDevControllerListenAddr string - flagCombineLogs bool + flagConfig string + flagLogLevel string + flagLogFormat string + flagDev bool + flagDevAdminPassword string + flagDevControllerAPIListenAddr string + flagDevControllerClusterListenAddr string + flagCombineLogs bool } func (c *Command) Synopsis() string { @@ -91,11 +92,17 @@ func (c *Command) Flags() *base.FlagSets { }) f.StringVar(&base.StringVar{ - Name: "dev-listen-address", - Target: &c.flagDevControllerListenAddr, - Default: "127.0.0.1:9200", - EnvVar: "WATCHTOWER_DEV_LISTEN_ADDRESS", - Usage: "Address to bind against.", + Name: "dev-api-listen-address", + Target: &c.flagDevControllerAPIListenAddr, + EnvVar: "WATCHTOWER_DEV_CONTROLLER_API_LISTEN_ADDRESS", + Usage: "Address to bind to for controller \"api\" purpose.", + }) + + f.StringVar(&base.StringVar{ + Name: "dev-cluster-listen-address", + Target: &c.flagDevControllerClusterListenAddr, + EnvVar: "WATCHTOWER_DEV_CONTROLLER_CLUSTER_LISTEN_ADDRESS", + Usage: "Address to bind to for controller \"cluster\" purpose.", }) f.BoolVar(&base.BoolVar{ @@ -137,6 +144,23 @@ func (c *Command) Run(args []string) int { return 1 } + for _, l := range devConfig.Listeners { + if len(l.Purpose) != 1 { + continue + } + switch l.Purpose[0] { + case "api": + if c.flagDevControllerAPIListenAddr != "" { + l.Address = c.flagDevControllerAPIListenAddr + } + + case "cluster": + if c.flagDevControllerClusterListenAddr != "" { + l.Address = c.flagDevControllerClusterListenAddr + } + } + } + if err := c.SetupLogging(c.flagLogLevel, c.flagLogFormat, "", ""); err != nil { c.UI.Error(err.Error()) return 1 diff --git a/internal/cmd/commands/worker/worker.go b/internal/cmd/commands/worker/worker.go index 4ce5ae038..22eb9fa1f 100644 --- a/internal/cmd/commands/worker/worker.go +++ b/internal/cmd/commands/worker/worker.go @@ -101,11 +101,10 @@ func (c *Command) Flags() *base.FlagSets { }) f.StringVar(&base.StringVar{ - Name: "dev-listen-address", - Target: &c.flagDevWorkerListenAddr, - Default: "127.0.0.1:9200", - EnvVar: "WATCHTOWER_DEV_WORKER_LISTEN_ADDRESS", - Usage: "Address to bind the worker to in \"dev\" mode.", + Name: "dev-listen-address", + Target: &c.flagDevWorkerListenAddr, + EnvVar: "WATCHTOWER_DEV_WORKER_LISTEN_ADDRESS", + Usage: "Address to bind the worker to in \"dev\" mode.", }) f.BoolVar(&base.BoolVar{ diff --git a/internal/cmd/config/config.go b/internal/cmd/config/config.go index 86fa506b6..7d2942b1f 100644 --- a/internal/cmd/config/config.go +++ b/internal/cmd/config/config.go @@ -16,12 +16,6 @@ const ( devConfig = ` disable_mlock = true -listener "tcp" { - tls_disable = true - proxy_protocol_behavior = "allow_authorized" - proxy_protocol_authorized_addrs = "127.0.0.1" -} - telemetry { prometheus_retention_time = "24h" disable_hostname = true @@ -29,7 +23,6 @@ telemetry { ` devControllerExtraConfig = ` - kms "aead" { purpose = "controller" aead_type = "aes-gcm" @@ -41,6 +34,28 @@ kms "aead" { aead_type = "aes-gcm" key = "%s" } + +listener "tcp" { + purpose = "api" + tls_disable = true + proxy_protocol_behavior = "allow_authorized" + proxy_protocol_authorized_addrs = "127.0.0.1" +} + +listener "tcp" { + purpose = "cluster" + tls_disable = true + proxy_protocol_behavior = "allow_authorized" + proxy_protocol_authorized_addrs = "127.0.0.1" +} +` + + devWorkerExtraConfig = ` +listener "tcp" { + tls_disable = true + proxy_protocol_behavior = "allow_authorized" + proxy_protocol_authorized_addrs = "127.0.0.1" +} ` ) @@ -54,7 +69,7 @@ type Config struct { // DevWorker is a Config that is used for dev mode of Watchtower // workers func DevWorker() (*Config, error) { - parsed, err := Parse(devConfig) + parsed, err := Parse(devConfig + devWorkerExtraConfig) if err != nil { return nil, fmt.Errorf("error parsing dev config: %w", err) } diff --git a/internal/servers/controller/cluster_tls.go b/internal/servers/controller/cluster_tls.go deleted file mode 100644 index cda7a8bc4..000000000 --- a/internal/servers/controller/cluster_tls.go +++ /dev/null @@ -1,182 +0,0 @@ -package controller - -import ( - "crypto/ed25519" - "crypto/tls" - "crypto/x509" - "crypto/x509/pkix" - "encoding/pem" - "io/ioutil" - "math/big" - mathrand "math/rand" - "net" - "os" - "path/filepath" - "time" - - "github.com/hashicorp/vault/sdk/helper/base62" -) - -type workerTLSOpts struct { - Address string - Protos []string - DumpDir string -} - -type certInfo struct { - CACert []byte `json:"ca_cert"` - CAKey []byte `json:"ca_key"` -} - -func (c Controller) workerTLS(opts workerTLSOpts) (*tls.Config, *certInfo, error) { - info := new(certInfo) - - certIPs := []net.IP{ - net.IPv6loopback, - net.ParseIP("127.0.0.1"), - } - - if opts.Address != "" { - baseAddr, err := net.ResolveTCPAddr("tcp", opts.Address) - if err != nil { - return nil, nil, err - } - certIPs = append(certIPs, baseAddr.IP) - } - - _, caKey, err := ed25519.GenerateKey(c.conf.SecureRandomReader) - if err != nil { - return nil, nil, err - } - info.CAKey = caKey - caHost, err := base62.Random(20) - if err != nil { - return nil, nil, err - } - - caCertTemplate := &x509.Certificate{ - Subject: pkix.Name{ - CommonName: caHost, - }, - DNSNames: []string{caHost}, - IPAddresses: certIPs, - KeyUsage: x509.KeyUsage(x509.KeyUsageCertSign | x509.KeyUsageCRLSign), - SerialNumber: big.NewInt(mathrand.Int63()), - NotBefore: time.Now().Add(-30 * time.Second), - NotAfter: time.Now().Add(262980 * time.Hour), - BasicConstraintsValid: true, - IsCA: true, - } - caBytes, err := x509.CreateCertificate(c.conf.SecureRandomReader, caCertTemplate, caCertTemplate, caKey.Public(), caKey) - if err != nil { - return nil, nil, err - } - info.CACert = caBytes - caCert, err := x509.ParseCertificate(caBytes) - if err != nil { - return nil, nil, err - } - - rootCAs := x509.NewCertPool() - rootCAs.AddCert(caCert) - caCertPEMBlock := &pem.Block{ - Type: "CERTIFICATE", - Bytes: caBytes, - } - caCertPEM := pem.EncodeToMemory(caCertPEMBlock) - caCertPEMFile := filepath.Join(opts.DumpDir, "ca_cert.pem") - - marshaledCAKey, err := x509.MarshalPKCS8PrivateKey(caKey) - if err != nil { - return nil, nil, err - } - caKeyPEMBlock := &pem.Block{ - Type: "PRIVATE KEY", - Bytes: marshaledCAKey, - } - caKeyPEM := pem.EncodeToMemory(caKeyPEMBlock) - - // - // Certs generation - // - _, key, err := ed25519.GenerateKey(c.conf.SecureRandomReader) - if err != nil { - return nil, nil, err - } - host, err := base62.Random(20) - if err != nil { - return nil, nil, err - } - certTemplate := &x509.Certificate{ - Subject: pkix.Name{ - CommonName: host, - }, - DNSNames: []string{host}, - IPAddresses: certIPs, - ExtKeyUsage: []x509.ExtKeyUsage{ - x509.ExtKeyUsageServerAuth, - x509.ExtKeyUsageClientAuth, - }, - KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment | x509.KeyUsageKeyAgreement, - SerialNumber: big.NewInt(mathrand.Int63()), - NotBefore: time.Now().Add(-30 * time.Second), - NotAfter: time.Now().Add(262980 * time.Hour), - } - certBytes, err := x509.CreateCertificate(c.conf.SecureRandomReader, certTemplate, caCert, key.Public(), caKey) - if err != nil { - return nil, nil, err - } - certPEMBlock := &pem.Block{ - Type: "CERTIFICATE", - Bytes: certBytes, - } - certPEM := pem.EncodeToMemory(certPEMBlock) - marshaledKey, err := x509.MarshalPKCS8PrivateKey(key) - if err != nil { - return nil, nil, err - } - keyPEMBlock := &pem.Block{ - Type: "PRIVATE KEY", - Bytes: marshaledKey, - } - keyPEM := pem.EncodeToMemory(keyPEMBlock) - - certFile := filepath.Join(opts.DumpDir, "cert.pem") - keyFile := filepath.Join(opts.DumpDir, "key.pem") - - tlsCert, err := tls.X509KeyPair(certPEM, keyPEM) - if err != nil { - return nil, nil, err - } - tlsConfig := &tls.Config{ - Certificates: []tls.Certificate{tlsCert}, - RootCAs: rootCAs, - ClientCAs: rootCAs, - ClientAuth: tls.RequestClientCert, - NextProtos: opts.Protos, - MinVersion: tls.VersionTLS13, - } - tlsConfig.BuildNameToCertificate() - - if opts.DumpDir != "" { - if _, err := os.Stat(opts.DumpDir); os.IsNotExist(err) { - if err := os.MkdirAll(opts.DumpDir, 0700); err != nil { - return nil, nil, err - } - } - if err := ioutil.WriteFile(filepath.Join(opts.DumpDir, "ca_key.pem"), caKeyPEM, 0755); err != nil { - return nil, nil, err - } - if err := ioutil.WriteFile(caCertPEMFile, caCertPEM, 0755); err != nil { - return nil, nil, err - } - if err := ioutil.WriteFile(certFile, certPEM, 0755); err != nil { - return nil, nil, err - } - if err := ioutil.WriteFile(keyFile, keyPEM, 0755); err != nil { - return nil, nil, err - } - } - - return tlsConfig, info, nil -} diff --git a/internal/servers/controller/listeners.go b/internal/servers/controller/listeners.go index d813db5b9..cbe9a8cf8 100644 --- a/internal/servers/controller/listeners.go +++ b/internal/servers/controller/listeners.go @@ -2,6 +2,7 @@ package controller import ( "context" + "crypto/tls" "errors" "fmt" "net" @@ -11,12 +12,14 @@ import ( "github.com/hashicorp/go-alpnmux" "github.com/hashicorp/go-multierror" + "github.com/hashicorp/watchtower/internal/cmd/base" ) func (c *Controller) startListeners() error { var retErr *multierror.Error servers := make([]func(), 0, len(c.conf.Listeners)) - for _, ln := range c.conf.Listeners { + + configureForAPI := func(ln *base.ServerListener) error { handler := c.Handler(HandlerProperties{ ListenerConfig: ln.Config, }) @@ -66,8 +69,7 @@ func (c *Controller) startListeners() error { case true: l := ln.Mux.GetListener(alpnmux.NoProto) if l == nil { - retErr = multierror.Append(retErr, errors.New("could not get non-tls listener")) - continue + return errors.New("could not get non-tls listener") } servers = append(servers, func() { go server.Serve(l) @@ -87,27 +89,41 @@ func (c *Controller) startListeners() error { } } - workerTLSConfig, peeringInfo, err := c.workerTLS(workerTLSOpts{ - Address: ln.Config.Address, - Protos: []string{"watchtower-worker-v1"}, + return nil + } + + configureForCluster := func(ln *base.ServerListener) error { + l, err := ln.Mux.RegisterProto("watchtower-worker-v1", &tls.Config{ + GetConfigForClient: c.validateWorkerTLS, }) if err != nil { - retErr = multierror.Append(retErr, fmt.Errorf("error getting TLS configuration: %w", err)) - continue + return fmt.Errorf("error getting sub-listener for worker proto: %w", err) + } + + // TODO: Pass this to a handler, e.g. a grpc server + _ = l + return nil + } + + for _, ln := range c.conf.Listeners { + var err error + for _, purpose := range ln.Config.Purpose { + switch purpose { + case "api": + err = configureForAPI(ln) + case "cluster": + err = configureForCluster(ln) + default: + err = fmt.Errorf("unknown listener purpose %q", purpose) + } + if err != nil { + break + } } - l, err := ln.Mux.RegisterProto("watchtower-worker-v1", workerTLSConfig) if err != nil { - retErr = multierror.Append(retErr, fmt.Errorf("error getting sub-listener for worker proto: %w", err)) + retErr = multierror.Append(retErr, err) continue } - - // TODO: Start listner for real; for now send it to the http server just for testing - servers = append(servers, func() { - go server.Serve(l) - }) - - // TODO: Add peering info into database - _ = peeringInfo } err := retErr.ErrorOrNil() @@ -128,12 +144,13 @@ func (c *Controller) stopListeners() error { if ln.HTTPServer == nil { continue } + localLn := ln serverWg.Add(1) go func() { - shutdownKill, shutdownKillCancel := context.WithTimeout(c.baseContext, ln.Config.MaxRequestDuration) + shutdownKill, shutdownKillCancel := context.WithTimeout(c.baseContext, localLn.Config.MaxRequestDuration) defer shutdownKillCancel() defer serverWg.Done() - ln.HTTPServer.Shutdown(shutdownKill) + localLn.HTTPServer.Shutdown(shutdownKill) }() } serverWg.Wait() diff --git a/internal/servers/controller/worker_tls_config.go b/internal/servers/controller/worker_tls_config.go new file mode 100644 index 000000000..ada09a62e --- /dev/null +++ b/internal/servers/controller/worker_tls_config.go @@ -0,0 +1,7 @@ +package controller + +import "crypto/tls" + +func (c Controller) validateWorkerTLS(hello *tls.ClientHelloInfo) (*tls.Config, error) { + return nil, nil +} diff --git a/internal/servers/worker/listeners.go b/internal/servers/worker/listeners.go index b2c234df5..70be5ff6a 100644 --- a/internal/servers/worker/listeners.go +++ b/internal/servers/worker/listeners.go @@ -128,12 +128,13 @@ func (c *Worker) stopListeners() error { if ln.HTTPServer == nil { continue } + localLn := ln serverWg.Add(1) go func() { - shutdownKill, shutdownKillCancel := context.WithTimeout(c.baseContext, ln.Config.MaxRequestDuration) + shutdownKill, shutdownKillCancel := context.WithTimeout(c.baseContext, localLn.Config.MaxRequestDuration) defer shutdownKillCancel() defer serverWg.Done() - ln.HTTPServer.Shutdown(shutdownKill) + localLn.HTTPServer.Shutdown(shutdownKill) }() } serverWg.Wait()