Begin purpose-based listener refactoring (#4)

This commit is contained in:
Jeff Mitchell authored and GitHub committed 2020-03-22 08:27:24 -04:00
1 parent 07032f6908
commit 5acda6a0b1
10 files changed
+189 -244

No files matched your search

+5 -1
View File
@@ -41,7 +41,11 @@ func NewListener(l *configutil.Listener, w io.Writer, ui cli.Ui) (*alpnmux.ALPNM
func tcpListenerFactory(l *configutil.Listener, _ io.Writer, ui cli.Ui) (*alpnmux.ALPNMux, map[string]string, reloadutil.ReloadFunc, error) {
if l.Address == "" {
l.Address = "127.0.0.1:9200"
if len(l.Purpose) == 1 && l.Purpose[0] == "cluster" {
l.Address = "127.0.0.1:9201"
} else {
l.Address = "127.0.0.1:9200"
}
}
bindProto := "tcp"
+2
View File
@@ -290,6 +290,8 @@ func (b *Server) SetupListeners(ui cli.Ui, config *configutil.SharedConfig) erro
Config: lnConfig,
})
props["purpose"] = strings.Join(lnConfig.Purpose, ",")
// Store the listener props for output later
key := fmt.Sprintf("listener %d", i+1)
propsList := make([]string, 0, len(props))
+72 -14
View File
@@ -9,6 +9,7 @@ import (
"github.com/hashicorp/go-hclog"
"github.com/hashicorp/go-multierror"
"github.com/hashicorp/vault/sdk/helper/mlock"
"github.com/hashicorp/vault/sdk/helper/strutil"
"github.com/hashicorp/watchtower/globals"
"github.com/hashicorp/watchtower/internal/cmd/base"
"github.com/hashicorp/watchtower/internal/cmd/config"
@@ -33,13 +34,14 @@ type Command struct {
Config *config.Config
flagConfig string
flagLogLevel string
flagLogFormat string
flagDev bool
flagDevAdminPassword string
flagDevControllerListenAddr string
flagCombineLogs bool
flagConfig string
flagLogLevel string
flagLogFormat string
flagDev bool
flagDevAdminPassword string
flagDevControllerAPIListenAddr string
flagDevControllerClusterListenAddr string
flagCombineLogs bool
}
func (c *Command) Synopsis() string {
@@ -112,11 +114,17 @@ func (c *Command) Flags() *base.FlagSets {
})
f.StringVar(&base.StringVar{
Name: "dev-listen-address",
Target: &c.flagDevControllerListenAddr,
Default: "127.0.0.1:9200",
EnvVar: "WATCHTOWER_DEV_CONTROLLER_LISTEN_ADDRESS",
Usage: "Address to bind the controller to in \"dev\" mode.",
Name: "dev-api-listen-address",
Target: &c.flagDevControllerAPIListenAddr,
EnvVar: "WATCHTOWER_DEV_CONTROLLER_API_LISTEN_ADDRESS",
Usage: "Address to bind the controller to in \"dev\" mode for \"api\" purpose.",
})
f.StringVar(&base.StringVar{
Name: "dev-cluster-listen-address",
Target: &c.flagDevControllerClusterListenAddr,
EnvVar: "WATCHTOWER_DEV_CONTROLLER_CLUSTER_LISTEN_ADDRESS",
Usage: "Address to bind the controller to in \"dev\" mode for \"cluster\" purpose.",
})
f.BoolVar(&base.BoolVar{
@@ -178,6 +186,43 @@ func (c *Command) Run(args []string) int {
"in a Docker container, provide the IPC_LOCK cap to the container."))
}
// Perform controller-specific listener checks here before setup
var foundCluster bool
var foundAPI bool
for _, lnConfig := range c.Config.Listeners {
switch len(lnConfig.Purpose) {
case 1:
switch lnConfig.Purpose[0] {
case "cluster":
foundCluster = true
case "api":
foundAPI = true
default:
c.UI.Error(fmt.Sprintf("Unknown listener purpose %q", lnConfig.Purpose[0]))
return 1
}
case 0:
lnConfig.Purpose = []string{"api", "cluster"}
fallthrough
case 2:
if !strutil.StrListContains(lnConfig.Purpose, "api") || !strutil.StrListContains(lnConfig.Purpose, "cluster") {
c.UI.Error(fmt.Sprintf("Invalid listener purpose set: %v", lnConfig.Purpose))
return 1
}
if lnConfig.TLSDisable {
c.UI.Error(fmt.Sprintf("TLS cannot be disabled on listener when serving both %q and %q purposes", "api", "cluster"))
return 1
}
foundAPI = true
foundCluster = true
}
}
if foundAPI && !foundCluster {
c.UI.Error("No listener marked for cluster purpose found, but listener explicitly marked for api was found")
return 1
}
if err := c.SetupListeners(c.UI, c.Config.SharedConfig); err != nil {
c.UI.Error(err.Error())
return 1
@@ -241,8 +286,21 @@ func (c *Command) ParseFlagsAndConfig(args []string) int {
return 1
}
if c.flagDevControllerListenAddr != "" {
c.Config.Listeners[0].Address = c.flagDevControllerListenAddr
for _, l := range c.Config.Listeners {
if len(l.Purpose) != 1 {
continue
}
switch l.Purpose[0] {
case "api":
if c.flagDevControllerAPIListenAddr != "" {
l.Address = c.flagDevControllerAPIListenAddr
}
case "cluster":
if c.flagDevControllerClusterListenAddr != "" {
l.Address = c.flagDevControllerClusterListenAddr
}
}
}
}
+36 -12
View File
@@ -29,13 +29,14 @@ type Command struct {
cleanupGuard sync.Once
flagConfig string
flagLogLevel string
flagLogFormat string
flagDev bool
flagDevAdminPassword string
flagDevControllerListenAddr string
flagCombineLogs bool
flagConfig string
flagLogLevel string
flagLogFormat string
flagDev bool
flagDevAdminPassword string
flagDevControllerAPIListenAddr string
flagDevControllerClusterListenAddr string
flagCombineLogs bool
}
func (c *Command) Synopsis() string {
@@ -91,11 +92,17 @@ func (c *Command) Flags() *base.FlagSets {
})
f.StringVar(&base.StringVar{
Name: "dev-listen-address",
Target: &c.flagDevControllerListenAddr,
Default: "127.0.0.1:9200",
EnvVar: "WATCHTOWER_DEV_LISTEN_ADDRESS",
Usage: "Address to bind against.",
Name: "dev-api-listen-address",
Target: &c.flagDevControllerAPIListenAddr,
EnvVar: "WATCHTOWER_DEV_CONTROLLER_API_LISTEN_ADDRESS",
Usage: "Address to bind to for controller \"api\" purpose.",
})
f.StringVar(&base.StringVar{
Name: "dev-cluster-listen-address",
Target: &c.flagDevControllerClusterListenAddr,
EnvVar: "WATCHTOWER_DEV_CONTROLLER_CLUSTER_LISTEN_ADDRESS",
Usage: "Address to bind to for controller \"cluster\" purpose.",
})
f.BoolVar(&base.BoolVar{
@@ -137,6 +144,23 @@ func (c *Command) Run(args []string) int {
return 1
}
for _, l := range devConfig.Listeners {
if len(l.Purpose) != 1 {
continue
}
switch l.Purpose[0] {
case "api":
if c.flagDevControllerAPIListenAddr != "" {
l.Address = c.flagDevControllerAPIListenAddr
}
case "cluster":
if c.flagDevControllerClusterListenAddr != "" {
l.Address = c.flagDevControllerClusterListenAddr
}
}
}
if err := c.SetupLogging(c.flagLogLevel, c.flagLogFormat, "", ""); err != nil {
c.UI.Error(err.Error())
return 1
+4 -5
View File
@@ -101,11 +101,10 @@ func (c *Command) Flags() *base.FlagSets {
})
f.StringVar(&base.StringVar{
Name: "dev-listen-address",
Target: &c.flagDevWorkerListenAddr,
Default: "127.0.0.1:9200",
EnvVar: "WATCHTOWER_DEV_WORKER_LISTEN_ADDRESS",
Usage: "Address to bind the worker to in \"dev\" mode.",
Name: "dev-listen-address",
Target: &c.flagDevWorkerListenAddr,
EnvVar: "WATCHTOWER_DEV_WORKER_LISTEN_ADDRESS",
Usage: "Address to bind the worker to in \"dev\" mode.",
})
f.BoolVar(&base.BoolVar{
+23 -8
View File
@@ -16,12 +16,6 @@ const (
devConfig = `
disable_mlock = true
listener "tcp" {
tls_disable = true
proxy_protocol_behavior = "allow_authorized"
proxy_protocol_authorized_addrs = "127.0.0.1"
}
telemetry {
prometheus_retention_time = "24h"
disable_hostname = true
@@ -29,7 +23,6 @@ telemetry {
`
devControllerExtraConfig = `
kms "aead" {
purpose = "controller"
aead_type = "aes-gcm"
@@ -41,6 +34,28 @@ kms "aead" {
aead_type = "aes-gcm"
key = "%s"
}
listener "tcp" {
purpose = "api"
tls_disable = true
proxy_protocol_behavior = "allow_authorized"
proxy_protocol_authorized_addrs = "127.0.0.1"
}
listener "tcp" {
purpose = "cluster"
tls_disable = true
proxy_protocol_behavior = "allow_authorized"
proxy_protocol_authorized_addrs = "127.0.0.1"
}
`
devWorkerExtraConfig = `
listener "tcp" {
tls_disable = true
proxy_protocol_behavior = "allow_authorized"
proxy_protocol_authorized_addrs = "127.0.0.1"
}
`
)
@@ -54,7 +69,7 @@ type Config struct {
// DevWorker is a Config that is used for dev mode of Watchtower
// workers
func DevWorker() (*Config, error) {
parsed, err := Parse(devConfig)
parsed, err := Parse(devConfig + devWorkerExtraConfig)
if err != nil {
return nil, fmt.Errorf("error parsing dev config: %w", err)
}
-182
View File
@@ -1,182 +0,0 @@
package controller
import (
"crypto/ed25519"
"crypto/tls"
"crypto/x509"
"crypto/x509/pkix"
"encoding/pem"
"io/ioutil"
"math/big"
mathrand "math/rand"
"net"
"os"
"path/filepath"
"time"
"github.com/hashicorp/vault/sdk/helper/base62"
)
type workerTLSOpts struct {
Address string
Protos []string
DumpDir string
}
type certInfo struct {
CACert []byte `json:"ca_cert"`
CAKey []byte `json:"ca_key"`
}
func (c Controller) workerTLS(opts workerTLSOpts) (*tls.Config, *certInfo, error) {
info := new(certInfo)
certIPs := []net.IP{
net.IPv6loopback,
net.ParseIP("127.0.0.1"),
}
if opts.Address != "" {
baseAddr, err := net.ResolveTCPAddr("tcp", opts.Address)
if err != nil {
return nil, nil, err
}
certIPs = append(certIPs, baseAddr.IP)
}
_, caKey, err := ed25519.GenerateKey(c.conf.SecureRandomReader)
if err != nil {
return nil, nil, err
}
info.CAKey = caKey
caHost, err := base62.Random(20)
if err != nil {
return nil, nil, err
}
caCertTemplate := &x509.Certificate{
Subject: pkix.Name{
CommonName: caHost,
},
DNSNames: []string{caHost},
IPAddresses: certIPs,
KeyUsage: x509.KeyUsage(x509.KeyUsageCertSign | x509.KeyUsageCRLSign),
SerialNumber: big.NewInt(mathrand.Int63()),
NotBefore: time.Now().Add(-30 * time.Second),
NotAfter: time.Now().Add(262980 * time.Hour),
BasicConstraintsValid: true,
IsCA: true,
}
caBytes, err := x509.CreateCertificate(c.conf.SecureRandomReader, caCertTemplate, caCertTemplate, caKey.Public(), caKey)
if err != nil {
return nil, nil, err
}
info.CACert = caBytes
caCert, err := x509.ParseCertificate(caBytes)
if err != nil {
return nil, nil, err
}
rootCAs := x509.NewCertPool()
rootCAs.AddCert(caCert)
caCertPEMBlock := &pem.Block{
Type: "CERTIFICATE",
Bytes: caBytes,
}
caCertPEM := pem.EncodeToMemory(caCertPEMBlock)
caCertPEMFile := filepath.Join(opts.DumpDir, "ca_cert.pem")
marshaledCAKey, err := x509.MarshalPKCS8PrivateKey(caKey)
if err != nil {
return nil, nil, err
}
caKeyPEMBlock := &pem.Block{
Type: "PRIVATE KEY",
Bytes: marshaledCAKey,
}
caKeyPEM := pem.EncodeToMemory(caKeyPEMBlock)
//
// Certs generation
//
_, key, err := ed25519.GenerateKey(c.conf.SecureRandomReader)
if err != nil {
return nil, nil, err
}
host, err := base62.Random(20)
if err != nil {
return nil, nil, err
}
certTemplate := &x509.Certificate{
Subject: pkix.Name{
CommonName: host,
},
DNSNames: []string{host},
IPAddresses: certIPs,
ExtKeyUsage: []x509.ExtKeyUsage{
x509.ExtKeyUsageServerAuth,
x509.ExtKeyUsageClientAuth,
},
KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment | x509.KeyUsageKeyAgreement,
SerialNumber: big.NewInt(mathrand.Int63()),
NotBefore: time.Now().Add(-30 * time.Second),
NotAfter: time.Now().Add(262980 * time.Hour),
}
certBytes, err := x509.CreateCertificate(c.conf.SecureRandomReader, certTemplate, caCert, key.Public(), caKey)
if err != nil {
return nil, nil, err
}
certPEMBlock := &pem.Block{
Type: "CERTIFICATE",
Bytes: certBytes,
}
certPEM := pem.EncodeToMemory(certPEMBlock)
marshaledKey, err := x509.MarshalPKCS8PrivateKey(key)
if err != nil {
return nil, nil, err
}
keyPEMBlock := &pem.Block{
Type: "PRIVATE KEY",
Bytes: marshaledKey,
}
keyPEM := pem.EncodeToMemory(keyPEMBlock)
certFile := filepath.Join(opts.DumpDir, "cert.pem")
keyFile := filepath.Join(opts.DumpDir, "key.pem")
tlsCert, err := tls.X509KeyPair(certPEM, keyPEM)
if err != nil {
return nil, nil, err
}
tlsConfig := &tls.Config{
Certificates: []tls.Certificate{tlsCert},
RootCAs: rootCAs,
ClientCAs: rootCAs,
ClientAuth: tls.RequestClientCert,
NextProtos: opts.Protos,
MinVersion: tls.VersionTLS13,
}
tlsConfig.BuildNameToCertificate()
if opts.DumpDir != "" {
if _, err := os.Stat(opts.DumpDir); os.IsNotExist(err) {
if err := os.MkdirAll(opts.DumpDir, 0700); err != nil {
return nil, nil, err
}
}
if err := ioutil.WriteFile(filepath.Join(opts.DumpDir, "ca_key.pem"), caKeyPEM, 0755); err != nil {
return nil, nil, err
}
if err := ioutil.WriteFile(caCertPEMFile, caCertPEM, 0755); err != nil {
return nil, nil, err
}
if err := ioutil.WriteFile(certFile, certPEM, 0755); err != nil {
return nil, nil, err
}
if err := ioutil.WriteFile(keyFile, keyPEM, 0755); err != nil {
return nil, nil, err
}
}
return tlsConfig, info, nil
}
+37 -20
View File
@@ -2,6 +2,7 @@ package controller
import (
"context"
"crypto/tls"
"errors"
"fmt"
"net"
@@ -11,12 +12,14 @@ import (
"github.com/hashicorp/go-alpnmux"
"github.com/hashicorp/go-multierror"
"github.com/hashicorp/watchtower/internal/cmd/base"
)
func (c *Controller) startListeners() error {
var retErr *multierror.Error
servers := make([]func(), 0, len(c.conf.Listeners))
for _, ln := range c.conf.Listeners {
configureForAPI := func(ln *base.ServerListener) error {
handler := c.Handler(HandlerProperties{
ListenerConfig: ln.Config,
})
@@ -66,8 +69,7 @@ func (c *Controller) startListeners() error {
case true:
l := ln.Mux.GetListener(alpnmux.NoProto)
if l == nil {
retErr = multierror.Append(retErr, errors.New("could not get non-tls listener"))
continue
return errors.New("could not get non-tls listener")
}
servers = append(servers, func() {
go server.Serve(l)
@@ -87,27 +89,41 @@ func (c *Controller) startListeners() error {
}
}
workerTLSConfig, peeringInfo, err := c.workerTLS(workerTLSOpts{
Address: ln.Config.Address,
Protos: []string{"watchtower-worker-v1"},
return nil
}
configureForCluster := func(ln *base.ServerListener) error {
l, err := ln.Mux.RegisterProto("watchtower-worker-v1", &tls.Config{
GetConfigForClient: c.validateWorkerTLS,
})
if err != nil {
retErr = multierror.Append(retErr, fmt.Errorf("error getting TLS configuration: %w", err))
continue
return fmt.Errorf("error getting sub-listener for worker proto: %w", err)
}
// TODO: Pass this to a handler, e.g. a grpc server
_ = l
return nil
}
for _, ln := range c.conf.Listeners {
var err error
for _, purpose := range ln.Config.Purpose {
switch purpose {
case "api":
err = configureForAPI(ln)
case "cluster":
err = configureForCluster(ln)
default:
err = fmt.Errorf("unknown listener purpose %q", purpose)
}
if err != nil {
break
}
}
l, err := ln.Mux.RegisterProto("watchtower-worker-v1", workerTLSConfig)
if err != nil {
retErr = multierror.Append(retErr, fmt.Errorf("error getting sub-listener for worker proto: %w", err))
retErr = multierror.Append(retErr, err)
continue
}
// TODO: Start listner for real; for now send it to the http server just for testing
servers = append(servers, func() {
go server.Serve(l)
})
// TODO: Add peering info into database
_ = peeringInfo
}
err := retErr.ErrorOrNil()
@@ -128,12 +144,13 @@ func (c *Controller) stopListeners() error {
if ln.HTTPServer == nil {
continue
}
localLn := ln
serverWg.Add(1)
go func() {
shutdownKill, shutdownKillCancel := context.WithTimeout(c.baseContext, ln.Config.MaxRequestDuration)
shutdownKill, shutdownKillCancel := context.WithTimeout(c.baseContext, localLn.Config.MaxRequestDuration)
defer shutdownKillCancel()
defer serverWg.Done()
ln.HTTPServer.Shutdown(shutdownKill)
localLn.HTTPServer.Shutdown(shutdownKill)
}()
}
serverWg.Wait()
@@ -0,0 +1,7 @@
package controller
import "crypto/tls"
func (c Controller) validateWorkerTLS(hello *tls.ClientHelloInfo) (*tls.Config, error) {
return nil, nil
}
+3 -2
View File
@@ -128,12 +128,13 @@ func (c *Worker) stopListeners() error {
if ln.HTTPServer == nil {
continue
}
localLn := ln
serverWg.Add(1)
go func() {
shutdownKill, shutdownKillCancel := context.WithTimeout(c.baseContext, ln.Config.MaxRequestDuration)
shutdownKill, shutdownKillCancel := context.WithTimeout(c.baseContext, localLn.Config.MaxRequestDuration)
defer shutdownKillCancel()
defer serverWg.Done()
ln.HTTPServer.Shutdown(shutdownKill)
localLn.HTTPServer.Shutdown(shutdownKill)
}()
}
serverWg.Wait()