Migrate a bunch of stuff around to prep for worker command (#2)

Right now it's essentially a copy; it needs some updating, integration
into dev, and so on.
This commit is contained in:
Jeff Mitchell authored and GitHub committed 2020-03-19 15:17:40 -04:00
1 parent c2d385d851
commit 1645c3106e
18 files changed
+1200 -157

No files matched your search

@@ -1,18 +1,44 @@
package listener
package base
import (
"errors"
"fmt"
"io"
"net"
"strings"
"time"
// We must import sha512 so that it registers with the runtime so that
// certificates that use it can be parsed.
_ "crypto/sha512"
"github.com/hashicorp/go-alpnmux"
"github.com/hashicorp/vault/internalshared/configutil"
"github.com/hashicorp/vault/internalshared/listenerutil"
"github.com/hashicorp/vault/internalshared/reloadutil"
"github.com/mitchellh/cli"
"github.com/pires/go-proxyproto"
)
// Factory is the factory function to create a listener.
type ListenerFactory func(*configutil.Listener, io.Writer, cli.Ui) (*alpnmux.ALPNMux, map[string]string, reloadutil.ReloadFunc, error)
// BuiltinListeners is the list of built-in listener types.
var BuiltinListeners = map[string]ListenerFactory{
"tcp": tcpListenerFactory,
}
// New creates a new listener of the given type with the given
// configuration. The type is looked up in the BuiltinListeners map.
func NewListener(l *configutil.Listener, w io.Writer, ui cli.Ui) (*alpnmux.ALPNMux, map[string]string, reloadutil.ReloadFunc, error) {
f, ok := BuiltinListeners[l.Type]
if !ok {
return nil, nil, nil, fmt.Errorf("unknown listener type: %q", l.Type)
}
return f(l, w, ui)
}
func tcpListenerFactory(l *configutil.Listener, _ io.Writer, ui cli.Ui) (*alpnmux.ALPNMux, map[string]string, reloadutil.ReloadFunc, error) {
if l.Address == "" {
l.Address = "127.0.0.1:9200"
@@ -69,6 +95,49 @@ func tcpListenerFactory(l *configutil.Listener, _ io.Writer, ui cli.Ui) (*alpnmu
return alpnMux, props, reloadFunc, nil
}
func listenerWrapProxy(ln net.Listener, l *configutil.Listener) (net.Listener, error) {
behavior := l.ProxyProtocolBehavior
if behavior == "" {
return ln, nil
}
authorizedAddrs := make([]string, 0, len(l.ProxyProtocolAuthorizedAddrs))
for _, v := range l.ProxyProtocolAuthorizedAddrs {
authorizedAddrs = append(authorizedAddrs, v.String())
}
var policyFunc proxyproto.PolicyFunc
switch behavior {
case "use_always":
policyFunc = func(upstream net.Addr) (proxyproto.Policy, error) {
return proxyproto.USE, nil
}
case "allow_authorized":
if len(authorizedAddrs) == 0 {
return nil, errors.New("proxy_protocol_behavior set but no proxy_protocol_authorized_addrs value")
}
policyFunc = proxyproto.MustLaxWhiteListPolicy(authorizedAddrs)
case "deny_unauthorized":
if len(authorizedAddrs) == 0 {
return nil, errors.New("proxy_protocol_behavior set but no proxy_protocol_authorized_addrs value")
}
policyFunc = proxyproto.MustStrictWhiteListPolicy(authorizedAddrs)
default:
return nil, fmt.Errorf("unknown %q value: %q", "proxy_protocol_behavior", behavior)
}
proxyListener := &proxyproto.Listener{
Listener: ln,
Policy: policyFunc,
}
return proxyListener, nil
}
// TCPKeepAliveListener sets TCP keep-alive timeouts on accepted
// connections. It's used by ListenAndServe and ListenAndServeTLS so
// dead TCP connections (e.g. closing laptop mid-download) eventually
+8
View File
@@ -0,0 +1,8 @@
// +build !memprofiler
package base
import "github.com/hashicorp/go-hclog"
func StartMemProfiler(_ hclog.Logger) {
}
@@ -1,6 +1,6 @@
// +build memprofiler
package dev
package base
import (
"os"
@@ -8,16 +8,18 @@ import (
"runtime"
"runtime/pprof"
"time"
"github.com/hashicorp/go-hclog"
)
func init() {
memProfilerEnabled = true
}
func (d *Command) startMemProfiler() {
func StartMemProfiler(logger hclog.Logger) {
profileDir := filepath.Join(os.TempDir(), "watchtowerprof")
if err := os.MkdirAll(profileDir, 0700); err != nil {
d.logger.Debug("could not create profile directory", "error", err)
logger.Debug("could not create profile directory", "error", err)
return
}
@@ -26,14 +28,14 @@ func (d *Command) startMemProfiler() {
filename := filepath.Join(profileDir, time.Now().UTC().Format("20060102_150405")) + ".pprof"
f, err := os.Create(filename)
if err != nil {
d.logger.Debug("could not create memory profile", "error", err)
logger.Debug("could not create memory profile", "error", err)
}
runtime.GC()
if err := pprof.WriteHeapProfile(f); err != nil {
d.logger.Debug("could not write memory profile", "error", err)
logger.Debug("could not write memory profile", "error", err)
}
f.Close()
d.logger.Debug("wrote memory profile", "filename", filename)
logger.Debug("wrote memory profile", "filename", filename)
time.Sleep(5 * time.Minute)
}
}()
+1 -2
View File
@@ -28,7 +28,6 @@ import (
"github.com/hashicorp/vault/sdk/helper/mlock"
"github.com/hashicorp/vault/sdk/logical"
"github.com/hashicorp/watchtower/globals"
"github.com/hashicorp/watchtower/internal/cmd/commands/controller/listener"
"github.com/hashicorp/watchtower/version"
"github.com/mitchellh/cli"
"github.com/ory/dockertest/v3"
@@ -250,7 +249,7 @@ func (b *Server) SetupListeners(ui cli.Ui, config *configutil.SharedConfig) erro
tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,
}
lnMux, props, reloadFunc, err := listener.New(lnConfig, b.GatedWriter, ui)
lnMux, props, reloadFunc, err := NewListener(lnConfig, b.GatedWriter, ui)
if err != nil {
return fmt.Errorf("Error initializing listener of type %s: %w", lnConfig.Type, err)
}
+12
View File
@@ -8,6 +8,7 @@ import (
"github.com/hashicorp/watchtower/internal/cmd/base"
"github.com/hashicorp/watchtower/internal/cmd/commands/controller"
"github.com/hashicorp/watchtower/internal/cmd/commands/dev"
"github.com/hashicorp/watchtower/internal/cmd/commands/worker"
"github.com/mitchellh/cli"
)
@@ -36,6 +37,17 @@ func initCommands(ui, serverCmdUi cli.Ui, runOpts *RunOptions) {
SigUSR2Ch: MakeSigUSR2Ch(),
}, nil
},
"worker": func() (cli.Command, error) {
return &worker.Command{
Command: &base.Command{
UI: serverCmdUi,
Address: runOpts.Address,
},
ShutdownCh: MakeShutdownCh(),
SighupCh: MakeSighupCh(),
SigUSR2Ch: MakeSigUSR2Ch(),
}, nil
},
"dev": func() (cli.Command, error) {
return &dev.Command{
Command: &base.Command{
+11 -11
View File
@@ -54,7 +54,7 @@ Usage: watchtower controller [options]
Start a controller with a configuration file:
$ watchtower controller -config=/etc/controller/config.hcl
$ watchtower controller -config=/etc/watchtower/controller.hcl
For a full list of examples, please see the documentation.
@@ -147,15 +147,15 @@ func (c *Command) Run(args []string) int {
return result
}
if memProfilerEnabled {
c.startMemProfiler()
}
if err := c.SetupLogging(c.flagLogLevel, c.flagLogFormat, c.Config.LogLevel, c.Config.LogFormat); err != nil {
c.UI.Error(err.Error())
return 1
}
if memProfilerEnabled {
base.StartMemProfiler(c.Logger)
}
if err := c.SetupMetrics(c.UI, c.Config.Telemetry); err != nil {
c.UI.Error(err.Error())
return 1
@@ -259,21 +259,21 @@ func (c *Command) ParseFlagsAndConfig(args []string) int {
func (c *Command) Start() int {
// Instantiate the wait group
controllerConfig := &controller.Config{
conf := &controller.Config{
RawConfig: c.Config,
Server: c.Server,
}
// Initialize the core
controller, err := controller.New(controllerConfig)
ctlr, err := controller.New(conf)
if err != nil {
c.UI.Error(fmt.Sprintf("Error initializing controller: %w", err))
return 1
}
if err := controller.Start(); err != nil {
if err := ctlr.Start(); err != nil {
c.UI.Error(fmt.Sprint("Error starting controller: %w", err))
if err := controller.Shutdown(); err != nil {
if err := ctlr.Shutdown(); err != nil {
c.UI.Error(fmt.Sprintf("Error with controller shutdown: %w", err))
}
return 1
@@ -287,7 +287,7 @@ func (c *Command) Start() int {
case <-c.ShutdownCh:
c.UI.Output("==> Watchtower controller shutdown triggered")
if err := controller.Shutdown(); err != nil {
if err := ctlr.Shutdown(); err != nil {
c.UI.Error(fmt.Sprintf("Error with controller shutdown: %w", err))
}
@@ -337,7 +337,7 @@ func (c *Command) Start() int {
c.Logger.Error("unknown log level found on reload", "level", newConf.LogLevel)
goto RUNRELOADFUNCS
}
controller.SetLogLevel(level)
ctlr.SetLogLevel(level)
}
RUNRELOADFUNCS:
@@ -1,6 +0,0 @@
// +build !memprofiler
package controller
func (c *Command) startMemProfiler() {
}
@@ -1,40 +0,0 @@
// +build memprofiler
package controller
import (
"os"
"path/filepath"
"runtime"
"runtime/pprof"
"time"
)
func init() {
memProfilerEnabled = true
}
func (c *Command) startMemProfiler() {
profileDir := filepath.Join(os.TempDir(), "watchtowerprof")
if err := os.MkdirAll(profileDir, 0700); err != nil {
c.logger.Debug("could not create profile directory", "error", err)
return
}
go func() {
for {
filename := filepath.Join(profileDir, time.Now().UTC().Format("20060102_150405")) + ".pprof"
f, err := os.Create(filename)
if err != nil {
c.logger.Debug("could not create memory profile", "error", err)
}
runtime.GC()
if err := pprof.WriteHeapProfile(f); err != nil {
c.logger.Debug("could not write memory profile", "error", err)
}
f.Close()
c.logger.Debug("wrote memory profile", "filename", filename)
time.Sleep(5 * time.Minute)
}
}()
}
@@ -1,81 +0,0 @@
package listener
import (
"errors"
"io"
"github.com/pires/go-proxyproto"
// We must import sha512 so that it registers with the runtime so that
// certificates that use it can be parsed.
_ "crypto/sha512"
"fmt"
"net"
"github.com/hashicorp/go-alpnmux"
"github.com/hashicorp/vault/internalshared/configutil"
"github.com/hashicorp/vault/internalshared/reloadutil"
"github.com/mitchellh/cli"
)
// Factory is the factory function to create a listener.
type Factory func(*configutil.Listener, io.Writer, cli.Ui) (*alpnmux.ALPNMux, map[string]string, reloadutil.ReloadFunc, error)
// BuiltinListeners is the list of built-in listener types.
var BuiltinListeners = map[string]Factory{
"tcp": tcpListenerFactory,
}
// New creates a new listener of the given type with the given
// configuration. The type is looked up in the BuiltinListeners map.
func New(l *configutil.Listener, w io.Writer, ui cli.Ui) (*alpnmux.ALPNMux, map[string]string, reloadutil.ReloadFunc, error) {
f, ok := BuiltinListeners[l.Type]
if !ok {
return nil, nil, nil, fmt.Errorf("unknown listener type: %q", l.Type)
}
return f(l, w, ui)
}
func listenerWrapProxy(ln net.Listener, l *configutil.Listener) (net.Listener, error) {
behavior := l.ProxyProtocolBehavior
if behavior == "" {
return ln, nil
}
authorizedAddrs := make([]string, 0, len(l.ProxyProtocolAuthorizedAddrs))
for _, v := range l.ProxyProtocolAuthorizedAddrs {
authorizedAddrs = append(authorizedAddrs, v.String())
}
var policyFunc proxyproto.PolicyFunc
switch behavior {
case "use_always":
policyFunc = func(upstream net.Addr) (proxyproto.Policy, error) {
return proxyproto.USE, nil
}
case "allow_authorized":
if len(authorizedAddrs) == 0 {
return nil, errors.New("proxy_protocol_behavior set but no proxy_protocol_authorized_addrs value")
}
policyFunc = proxyproto.MustLaxWhiteListPolicy(authorizedAddrs)
case "deny_unauthorized":
if len(authorizedAddrs) == 0 {
return nil, errors.New("proxy_protocol_behavior set but no proxy_protocol_authorized_addrs value")
}
policyFunc = proxyproto.MustStrictWhiteListPolicy(authorizedAddrs)
default:
return nil, fmt.Errorf("unknown %q value: %q", "proxy_protocol_behavior", behavior)
}
proxyListener := &proxyproto.Listener{
Listener: ln,
Policy: policyFunc,
}
return proxyListener, nil
}
+4 -4
View File
@@ -138,15 +138,15 @@ func (c *Command) Run(args []string) int {
return 1
}
if memProfilerEnabled {
c.startMemProfiler()
}
if err := c.SetupLogging(c.flagLogLevel, c.flagLogFormat, "", ""); err != nil {
c.UI.Error(err.Error())
return 1
}
if memProfilerEnabled {
base.StartMemProfiler(c.Logger)
}
if err := c.SetupMetrics(c.UI, devControllerConfig.Telemetry); err != nil {
c.UI.Error(err.Error())
return 1
@@ -1,6 +0,0 @@
// +build !memprofiler
package dev
func (d *Command) startMemProfiler() {
}
@@ -0,0 +1,122 @@
package config
import (
"bytes"
"crypto/rand"
"encoding/base64"
"fmt"
"io"
"io/ioutil"
"github.com/hashicorp/hcl"
"github.com/hashicorp/vault/internalshared/configutil"
)
// Config is the configuration for the watchtower controller
type Config struct {
*configutil.SharedConfig `hcl:"-"`
}
// Dev is a Config that is used for dev mode of Watchtower
func Dev() (*Config, error) {
randBuf := new(bytes.Buffer)
n, err := randBuf.ReadFrom(&io.LimitedReader{
R: rand.Reader,
N: 64,
})
if err != nil {
return nil, err
}
if n != 64 {
return nil, fmt.Errorf("expected to read 64 bytes, read %d", n)
}
controllerKey := base64.StdEncoding.EncodeToString(randBuf.Bytes()[0:32])
workerAuthKey := base64.StdEncoding.EncodeToString(randBuf.Bytes()[32:64])
hclStr := `
disable_mlock = true
listener "tcp" {
tls_disable = true
proxy_protocol_behavior = "allow_authorized"
proxy_protocol_authorized_addrs = "127.0.0.1"
}
telemetry {
prometheus_retention_time = "24h"
disable_hostname = true
}
`
hclStr = fmt.Sprintf(hclStr, controllerKey, workerAuthKey)
parsed, err := Parse(hclStr)
if err != nil {
return nil, fmt.Errorf("error parsing dev config: %w", err)
}
return parsed, nil
}
func New() *Config {
return &Config{
SharedConfig: new(configutil.SharedConfig),
}
}
// LoadFile loads the configuration from the given file.
func LoadFile(path string) (*Config, error) {
// Read the file
d, err := ioutil.ReadFile(path)
if err != nil {
return nil, err
}
conf, err := Parse(string(d))
if err != nil {
return nil, err
}
return conf, nil
}
func Parse(d string) (*Config, error) {
obj, err := hcl.Parse(d)
if err != nil {
return nil, err
}
// Nothing to do here right now
result := New()
if err := hcl.DecodeObject(result, obj); err != nil {
return nil, err
}
sharedConfig, err := configutil.ParseConfig(d)
if err != nil {
return nil, err
}
result.SharedConfig = sharedConfig
return result, nil
}
// Sanitized returns a copy of the config with all values that are considered
// sensitive stripped. It also strips all `*Raw` values that are mainly
// used for parsing.
//
// Specifically, the fields that this method strips are:
// - KMS.Config
// - Telemetry.CirconusAPIToken
func (c *Config) Sanitized() map[string]interface{} {
// Create shared config if it doesn't exist (e.g. in tests) so that map
// keys are actually populated
if c.SharedConfig == nil {
c.SharedConfig = new(configutil.SharedConfig)
}
sharedResult := c.SharedConfig.Sanitized()
result := map[string]interface{}{}
for k, v := range sharedResult {
result[k] = v
}
return result
}
+385
View File
@@ -0,0 +1,385 @@
package worker
import (
"fmt"
"runtime"
"strings"
"sync"
"github.com/hashicorp/go-hclog"
"github.com/hashicorp/go-multierror"
"github.com/hashicorp/vault/sdk/helper/mlock"
"github.com/hashicorp/watchtower/globals"
"github.com/hashicorp/watchtower/internal/cmd/base"
"github.com/hashicorp/watchtower/internal/cmd/commands/worker/config"
"github.com/hashicorp/watchtower/internal/servers/worker"
"github.com/mitchellh/cli"
"github.com/posener/complete"
)
var _ cli.Command = (*Command)(nil)
var _ cli.CommandAutocomplete = (*Command)(nil)
var memProfilerEnabled = false
type Command struct {
*base.Command
*base.Server
ShutdownCh chan struct{}
SighupCh chan struct{}
ReloadedCh chan struct{}
SigUSR2Ch chan struct{}
cleanupGuard sync.Once
Config *config.Config
flagConfig string
flagLogLevel string
flagLogFormat string
flagDev bool
flagDevAdminToken string
flagDevWorkerListenAddr string
flagCombineLogs bool
}
func (c *Command) Synopsis() string {
return "Start a Watchtower worker"
}
func (c *Command) Help() string {
helpText := `
Usage: watchtower worker [options]
Start a worker with a configuration file:
$ watchtower worker -config=/etc/watchtower/worker.hcl
For a full list of examples, please see the documentation.
` + c.Flags().Help()
return strings.TrimSpace(helpText)
}
func (c *Command) Flags() *base.FlagSets {
set := c.FlagSet(base.FlagSetHTTP)
f := set.NewFlagSet("Command Options")
f.StringVar(&base.StringVar{
Name: "config",
Target: &c.flagConfig,
Completion: complete.PredictOr(
complete.PredictFiles("*.hcl"),
complete.PredictFiles("*.json"),
),
Usage: "Path to a configuration file.",
})
f.StringVar(&base.StringVar{
Name: "log-level",
Target: &c.flagLogLevel,
Default: base.NotSetValue,
EnvVar: "WATCHTOWER_LOG_LEVEL",
Completion: complete.PredictSet("trace", "debug", "info", "warn", "err"),
Usage: "Log verbosity level. Supported values (in order of more detail to less) are " +
"\"trace\", \"debug\", \"info\", \"warn\", and \"err\".",
})
f.StringVar(&base.StringVar{
Name: "log-format",
Target: &c.flagLogFormat,
Default: base.NotSetValue,
Completion: complete.PredictSet("standard", "json"),
Usage: `Log format. Supported values are "standard" and "json".`,
})
f = set.NewFlagSet("Dev Options")
f.BoolVar(&base.BoolVar{
Name: "dev",
Target: &c.flagDev,
Usage: "Enable development mode. As the name implies, do not run \"dev\" mode in " +
"production.",
})
f.StringVar(&base.StringVar{
Name: "dev-admin-token",
Target: &c.flagDevAdminToken,
Default: "",
EnvVar: "WATCHTWER_DEV_ADMIN_TOKEN",
Usage: "Initial admin token. This only applies when running in \"dev\" " +
"mode.",
})
f.StringVar(&base.StringVar{
Name: "dev-listen-address",
Target: &c.flagDevWorkerListenAddr,
Default: "127.0.0.1:9200",
EnvVar: "WATCHTOWER_DEV_WORKER_LISTEN_ADDRESS",
Usage: "Address to bind the worker to in \"dev\" mode.",
})
f.BoolVar(&base.BoolVar{
Name: "combine-logs",
Target: &c.flagCombineLogs,
Default: false,
Usage: "If set, both startup information and logs will be sent to stdout. If not set (the default), startup information will go to stdout and logs will be sent to stderr.",
})
return set
}
func (c *Command) AutocompleteArgs() complete.Predictor {
return complete.PredictNothing
}
func (c *Command) AutocompleteFlags() complete.Flags {
return c.Flags().Completions()
}
func (c *Command) Run(args []string) int {
c.Server = base.NewServer()
c.CombineLogs = c.flagCombineLogs
if result := c.ParseFlagsAndConfig(args); result > 0 {
return result
}
if err := c.SetupLogging(c.flagLogLevel, c.flagLogFormat, c.Config.LogLevel, c.Config.LogFormat); err != nil {
c.UI.Error(err.Error())
return 1
}
if memProfilerEnabled {
base.StartMemProfiler(c.Logger)
}
if err := c.SetupMetrics(c.UI, c.Config.Telemetry); err != nil {
c.UI.Error(err.Error())
return 1
}
if err := c.SetupKMSes(c.UI, c.Config.SharedConfig, 2); err != nil {
c.UI.Error(err.Error())
return 1
}
if c.Config.DefaultMaxRequestDuration != 0 {
globals.DefaultMaxRequestDuration = c.Config.DefaultMaxRequestDuration
}
// If mlockall(2) isn't supported, show a warning. We disable this in dev
// because it is quite scary to see when first using Vault. We also disable
// this if the user has explicitly disabled mlock in configuration.
if !c.flagDev && !c.Config.DisableMlock && !mlock.Supported() {
c.UI.Warn(base.WrapAtLength(
"WARNING! mlock is not supported on this system! An mlockall(2)-like " +
"syscall to prevent memory from being swapped to disk is not " +
"supported on this system. For better security, only run Vault on " +
"systems where this call is supported. If you are running Vault " +
"in a Docker container, provide the IPC_LOCK cap to the container."))
}
if err := c.SetupListeners(c.UI, c.Config.SharedConfig); err != nil {
c.UI.Error(err.Error())
return 1
}
// Write out the PID to the file now that server has successfully started
if err := c.StorePidFile(c.Config.PidFile); err != nil {
c.UI.Error(fmt.Sprintf("Error storing PID: %w", err))
return 1
}
if c.flagDev {
if err := c.CreateDevDatabase(); err != nil {
c.UI.Error(fmt.Sprintf("Error creating dev database container: %s", err.Error()))
return 1
}
c.ShutdownFuncs = append(c.ShutdownFuncs, c.DestroyDevDatabase)
}
defer c.RunShutdownFuncs(c.UI)
c.PrintInfo(c.UI, "worker")
c.ReleaseLogGate()
return c.Start()
}
func (c *Command) ParseFlagsAndConfig(args []string) int {
var err error
f := c.Flags()
if err = f.Parse(args); err != nil {
c.UI.Error(err.Error())
return 1
}
// Validation
if !c.flagDev {
switch {
case len(c.flagConfig) == 0:
c.UI.Error("Must specify a config file using -config")
return 1
case c.flagDevAdminToken != "":
c.UI.Warn(base.WrapAtLength(
"You cannot specify a custom admin token ID outside of \"dev\" mode. " +
"Your request has been ignored."))
c.flagDevAdminToken = ""
}
if len(c.flagConfig) == 0 {
c.UI.Error("Must supply a config file with -config")
return 1
}
c.Config, err = config.LoadFile(c.flagConfig)
if err != nil {
c.UI.Error("Error parsing config: " + err.Error())
return 1
}
} else {
c.Config, err = config.Dev()
if err != nil {
c.UI.Error(fmt.Sprintf("Error creating dev config: %s", err))
return 1
}
if c.flagDevWorkerListenAddr != "" {
c.Config.Listeners[0].Address = c.flagDevWorkerListenAddr
}
}
return 0
}
func (c *Command) Start() int {
// Instantiate the wait group
conf := &worker.Config{
RawConfig: c.Config,
Server: c.Server,
}
// Initialize the core
wrkr, err := worker.New(conf)
if err != nil {
c.UI.Error(fmt.Sprintf("Error initializing worker: %w", err))
return 1
}
if err := wrkr.Start(); err != nil {
c.UI.Error(fmt.Sprint("Error starting worker: %w", err))
if err := wrkr.Shutdown(); err != nil {
c.UI.Error(fmt.Sprintf("Error with worker shutdown: %w", err))
}
return 1
}
// Wait for shutdown
shutdownTriggered := false
for !shutdownTriggered {
select {
case <-c.ShutdownCh:
c.UI.Output("==> Watchtower worker shutdown triggered")
if err := wrkr.Shutdown(); err != nil {
c.UI.Error(fmt.Sprintf("Error with worker shutdown: %w", err))
}
shutdownTriggered = true
case <-c.SighupCh:
c.UI.Output("==> Watchtower worker reload triggered")
// Check for new log level
var level hclog.Level
var err error
var newConf *config.Config
if c.flagConfig == "" {
goto RUNRELOADFUNCS
}
newConf, err = config.LoadFile(c.flagConfig)
if err != nil {
c.Logger.Error("could not reload config", "path", c.flagConfig, "error", err)
goto RUNRELOADFUNCS
}
// Ensure at least one config was found.
if newConf == nil {
c.Logger.Error("no config found at reload time")
goto RUNRELOADFUNCS
}
// Commented out until we need this
//wrkr.SetConfig(config)
if newConf.LogLevel != "" {
configLogLevel := strings.ToLower(strings.TrimSpace(newConf.LogLevel))
switch configLogLevel {
case "trace":
level = hclog.Trace
case "debug":
level = hclog.Debug
case "notice", "info", "":
level = hclog.Info
case "warn", "warning":
level = hclog.Warn
case "err", "error":
level = hclog.Error
default:
c.Logger.Error("unknown log level found on reload", "level", newConf.LogLevel)
goto RUNRELOADFUNCS
}
wrkr.SetLogLevel(level)
}
RUNRELOADFUNCS:
if err := c.Reload(); err != nil {
c.UI.Error(fmt.Sprintf("Error(s) were encountered during worker reload: %w", err))
}
case <-c.SigUSR2Ch:
buf := make([]byte, 32*1024*1024)
n := runtime.Stack(buf[:], true)
c.Logger.Info("goroutine trace", "stack", string(buf[:n]))
}
}
return 0
}
func (c *Command) Reload() error {
c.ReloadFuncsLock.RLock()
defer c.ReloadFuncsLock.RUnlock()
var reloadErrors *multierror.Error
for k, relFuncs := range c.ReloadFuncs {
switch {
case strings.HasPrefix(k, "listener|"):
for _, relFunc := range relFuncs {
if relFunc != nil {
if err := relFunc(); err != nil {
reloadErrors = multierror.Append(reloadErrors, fmt.Errorf("error encountered reloading listener: %w", err))
}
}
}
}
}
// Send a message that we reloaded. This prevents "guessing" sleep times
// in tests.
select {
case c.ReloadedCh <- struct{}{}:
default:
}
return reloadErrors.ErrorOrNil()
}