diff --git a/api/client.go b/api/client.go new file mode 100644 index 000000000..5654256cc --- /dev/null +++ b/api/client.go @@ -0,0 +1,752 @@ +package api + +import ( + "bytes" + "context" + "crypto/tls" + "encoding/json" + "fmt" + "net" + "net/http" + "net/url" + "os" + "path" + "strconv" + "strings" + "sync" + "time" + "unicode" + + "github.com/hashicorp/errwrap" + cleanhttp "github.com/hashicorp/go-cleanhttp" + retryablehttp "github.com/hashicorp/go-retryablehttp" + rootcerts "github.com/hashicorp/go-rootcerts" + "github.com/hashicorp/watchtower/api/internal/parseutil" + "golang.org/x/time/rate" +) + +const EnvWatchtowerAddress = "WATCHTOWER_ADDR" +const EnvWatchtowerCACert = "WATCHTOWER_CACERT" +const EnvWatchtowerCAPath = "WATCHTOWER_CAPATH" +const EnvWatchtowerClientCert = "WATCHTOWER_CLIENT_CERT" +const EnvWatchtowerClientKey = "WATCHTOWER_CLIENT_KEY" +const EnvWatchtowerClientTimeout = "WATCHTOWER_CLIENT_TIMEOUT" +const EnvWatchtowerTLSInsecure = "WATCHTOWER_TLS_INSECURE" +const EnvWatchtowerTLSServerName = "WATCHTOWER_TLS_SERVER_NAME" +const EnvWatchtowerMaxRetries = "WATCHTOWER_MAX_RETRIES" +const EnvWatchtowerToken = "WATCHTOWER_TOKEN" +const EnvWatchtowerRateLimit = "WATCHTOWER_RATE_LIMIT" +const EnvWatchtowerSRVLookup = "WATCHTOWER_SRV_LOOKUP" +const EnvWatchtowerOrg = "WATCHTOWER_ORG" +const EnvWatchtowerProject = "WATCHTOWER_PROJECT" + +// Config is used to configure the creation of the client +type Config struct { + // Address is the address of the Watchtower controller. This should be a + // complete URL such as "http://watchtower.example.com". If you need a custom + // SSL cert or want to enable insecure mode, you need to specify a custom + // HttpClient. + Address string + + // Token is the client token that reuslts from authentication and can be + // used to make calls into Watchtower + Token string + + // HttpClient is the HTTP client to use. Watchtower sets sane defaults for + // the http.Client and its associated http.Transport created in + // DefaultConfig. If you must modify Watchtower's defaults, it is + // suggested that you start with that client and modify as needed rather + // than start with an empty client (or http.DefaultClient). + HttpClient *http.Client + + // TLSConfig contains TLS configuration information. After modifying these + // values, ConfigureTLS should be called. + TLSConfig *TLSConfig + + // Headers contains extra headers that will be added to any request + Headers http.Header + + // MaxRetries controls the maximum number of times to retry when a 5xx + // error occurs. Set to 0 to disable retrying. Defaults to 2 (for a total + // of three tries). + MaxRetries int + + // Timeout is for setting custom timeout parameter in the HttpClient + Timeout time.Duration + + // If there is an error when creating the configuration, this will be the + // error + Error error + + // The Backoff function to use; a default is used if not provided + Backoff retryablehttp.Backoff + + // The CheckRetry function to use; a default is used if not provided + CheckRetry retryablehttp.CheckRetry + + // Limiter is the rate limiter used by the client. + // If this pointer is nil, then there will be no limit set. + // In contrast, if this pointer is set, even to an empty struct, + // then that limiter will be used. Note that an empty Limiter + // is equivalent blocking all events. + Limiter *rate.Limiter + + // OutputCurlString causes the actual request to return an error of type + // *OutputStringError. Type asserting the error message will allow + // fetching a cURL-compatible string for the operation. + // + // Note: It is not thread-safe to set this and make concurrent requests + // with the same client. Cloning a client will not clone this value. + OutputCurlString bool + + // SRVLookup enables the client to lookup the host through DNS SRV lookup + SRVLookup bool + + // Org is the organization to use if not overridden per-call + Org string + + // Project is the project to use if not overridden per-call + Project string +} + +// TLSConfig contains the parameters needed to configure TLS on the HTTP client +// used to communicate with Watchtower. +type TLSConfig struct { + // CACert is the path to a PEM-encoded CA cert file to use to verify the + // Watchtower server SSL certificate. + CACert string + + // CAPath is the path to a directory of PEM-encoded CA cert files to verify + // the Watchtower server SSL certificate. + CAPath string + + // ClientCert is the path to the certificate for Watchtower communication + ClientCert string + + // ClientKey is the path to the private key for Watchtower communication + ClientKey string + + // ServerName, if set, is used to set the SNI host when connecting via + // TLS. + ServerName string + + // Insecure enables or disables SSL verification + Insecure bool +} + +// DefaultConfig returns a default configuration for the client. It is +// safe to modify the return value of this function. +// +// The default Address is https://127.0.0.1:9200, but this can be overridden by +// setting the `WATCHTOWER_ADDR` environment variable. +// +// If an error is encountered, this will return nil. +func DefaultConfig() *Config { + config := &Config{ + Address: "https://127.0.0.1:9200", + HttpClient: cleanhttp.DefaultPooledClient(), + Timeout: time.Second * 60, + } + + // We read the environment now; after DefaultClient returns we can override + // values from command line flags, which should take precedence. + if err := config.ReadEnvironment(); err != nil { + config.Error = err + return config + } + + transport := config.HttpClient.Transport.(*http.Transport) + transport.TLSHandshakeTimeout = 10 * time.Second + transport.TLSClientConfig = &tls.Config{ + MinVersion: tls.VersionTLS12, + } + + config.Backoff = retryablehttp.LinearJitterBackoff + config.MaxRetries = 2 + config.Headers = make(http.Header) + + return config +} + +// ConfigureTLS takes a set of TLS configurations and applies those to the the +// HTTP client. +func (c *Config) ConfigureTLS() error { + if c.HttpClient == nil { + c.HttpClient = DefaultConfig().HttpClient + } + clientTLSConfig := c.HttpClient.Transport.(*http.Transport).TLSClientConfig + + var clientCert tls.Certificate + foundClientCert := false + + switch { + case c.TLSConfig.ClientCert != "" && c.TLSConfig.ClientKey != "": + var err error + clientCert, err = tls.LoadX509KeyPair(c.TLSConfig.ClientCert, c.TLSConfig.ClientKey) + if err != nil { + return err + } + foundClientCert = true + case c.TLSConfig.ClientCert != "" || c.TLSConfig.ClientKey != "": + return fmt.Errorf("both client cert and client key must be provided") + } + + if c.TLSConfig.CACert != "" || c.TLSConfig.CAPath != "" { + rootConfig := &rootcerts.Config{ + CAFile: c.TLSConfig.CACert, + CAPath: c.TLSConfig.CAPath, + } + if err := rootcerts.ConfigureTLS(clientTLSConfig, rootConfig); err != nil { + return err + } + } + + if c.TLSConfig.Insecure { + clientTLSConfig.InsecureSkipVerify = true + } + + if foundClientCert { + // We use this function to ignore the server's preferential list of + // CAs, otherwise any CA used for the cert auth backend must be in the + // server's CA pool + clientTLSConfig.GetClientCertificate = func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { + return &clientCert, nil + } + } + + if c.TLSConfig.ServerName != "" { + clientTLSConfig.ServerName = c.TLSConfig.ServerName + } + + return nil +} + +// setAddress parses a given string and looks for org and project info, setting +// the actual address to the base. Note that if a very malformed URL is passed +// in, this may not return what one expects. For now this is on purpose to +// avoid requiring error handling. +// +// This also removes any trailing "/v1"; we'll use that in our commands so we +// don't require it from users. +func (c *Config) setAddress(addr string) error { + u, err := url.Parse(addr) + if err != nil { + return fmt.Errorf("error parsing address: %w", err) + } + c.Address = fmt.Sprintf("%s://%s", u.Scheme, u.Host) + + path := strings.TrimPrefix(u.Path, "/v1") + path = strings.TrimPrefix(path, "/") + if path == "" { + return nil + } + + split := strings.Split(path, "/") + switch len(split) { + case 0: + case 2: + if split[0] != "org" { + return fmt.Errorf("expected org segment in address, found %q", split[0]) + } + c.Org = split[1] + case 4: + if split[0] != "org" { + return fmt.Errorf("expected org segment in address, found %q", split[0]) + } + c.Org = split[1] + if split[2] != "project" { + return fmt.Errorf("expected project segment in address, found %q", split[2]) + } + c.Project = split[3] + default: + return fmt.Errorf("unexpected number of segments in address") + } + + return nil +} + +// ReadEnvironment reads configuration information from the environment. If +// there is an error, no configuration value is updated. +func (c *Config) ReadEnvironment() error { + var envCACert string + var envCAPath string + var envClientCert string + var envClientKey string + var envInsecure bool + var envServerName string + + // Parse the environment variables + if v := os.Getenv(EnvWatchtowerAddress); v != "" { + c.Address = v + } + + if v := os.Getenv(EnvWatchtowerToken); v != "" { + c.Token = v + } + + if v := os.Getenv(EnvWatchtowerOrg); v != "" { + c.Org = v + } + + if v := os.Getenv(EnvWatchtowerProject); v != "" { + c.Project = v + } + + if v := os.Getenv(EnvWatchtowerMaxRetries); v != "" { + maxRetries, err := strconv.ParseUint(v, 10, 32) + if err != nil { + return err + } + c.MaxRetries = int(maxRetries) + } + + if v := os.Getenv(EnvWatchtowerSRVLookup); v != "" { + var err error + lookup, err := strconv.ParseBool(v) + if err != nil { + return fmt.Errorf("could not parse %s", EnvWatchtowerSRVLookup) + } + c.SRVLookup = lookup + } + + if t := os.Getenv(EnvWatchtowerClientTimeout); t != "" { + clientTimeout, err := parseutil.ParseDurationSecond(t) + if err != nil { + return fmt.Errorf("could not parse %q", EnvWatchtowerClientTimeout) + } + c.Timeout = clientTimeout + } + + if v := os.Getenv(EnvWatchtowerRateLimit); v != "" { + rateLimit, burstLimit, err := parseRateLimit(v) + if err != nil { + return err + } + c.Limiter = rate.NewLimiter(rate.Limit(rateLimit), burstLimit) + } + + // TLS Config + { + var foundTLSConfig bool + if v := os.Getenv(EnvWatchtowerCACert); v != "" { + foundTLSConfig = true + envCACert = v + } + if v := os.Getenv(EnvWatchtowerCAPath); v != "" { + foundTLSConfig = true + envCAPath = v + } + if v := os.Getenv(EnvWatchtowerClientCert); v != "" { + foundTLSConfig = true + envClientCert = v + } + if v := os.Getenv(EnvWatchtowerClientKey); v != "" { + foundTLSConfig = true + envClientKey = v + } + if v := os.Getenv(EnvWatchtowerTLSInsecure); v != "" { + foundTLSConfig = true + var err error + envInsecure, err = strconv.ParseBool(v) + if err != nil { + return fmt.Errorf("could not parse WATCHTOWER_TLS_INSECURE") + } + } + if v := os.Getenv(EnvWatchtowerTLSServerName); v != "" { + foundTLSConfig = true + envServerName = v + } + // Set the values on the config + // Configure the HTTP clients TLS configuration. + if foundTLSConfig { + c.TLSConfig = &TLSConfig{ + CACert: envCACert, + CAPath: envCAPath, + ClientCert: envClientCert, + ClientKey: envClientKey, + ServerName: envServerName, + Insecure: envInsecure, + } + return c.ConfigureTLS() + } + } + + return nil +} + +func parseRateLimit(val string) (rate float64, burst int, err error) { + _, err = fmt.Sscanf(val, "%f:%d", &rate, &burst) + if err != nil { + rate, err = strconv.ParseFloat(val, 64) + if err != nil { + err = fmt.Errorf("%v was provided but incorrectly formatted", EnvWatchtowerRateLimit) + } + burst = int(rate) + } + + return rate, burst, err +} + +// Client is the client to the Watchtower API. Create a client with NewClient. +type Client struct { + modifyLock sync.RWMutex + config *Config +} + +// NewClient returns a new client for the given configuration. +// +// If the configuration is nil, Watchtower will use configuration from +// DefaultConfig(), which is the recommended starting configuration. +// +// If the environment variable `WATCHTOWER_TOKEN` is present, the token will be +// automatically added to the client. Otherwise, you must manually call +// `SetToken()`. +func NewClient(c *Config) (*Client, error) { + def := DefaultConfig() + if def == nil { + return nil, fmt.Errorf("could not create/read default configuration") + } + if def.Error != nil { + return nil, errwrap.Wrapf("error encountered setting up default configuration: {{err}}", def.Error) + } + + if c == nil { + c = def + } + + if c.HttpClient == nil { + c.HttpClient = def.HttpClient + } + if c.HttpClient.Transport == nil { + c.HttpClient.Transport = def.HttpClient.Transport + } + if c.HttpClient.CheckRedirect == nil { + // Ensure redirects are not automatically followed + c.HttpClient.CheckRedirect = func(req *http.Request, via []*http.Request) error { + // Returning this value causes the Go net library to not close the + // response body and to nil out the error. Otherwise retry clients may + // try three times on every redirect because it sees an error from this + // function (to prevent redirects) passing through to it. + return http.ErrUseLastResponse + } + } + + if err := c.setAddress(c.Address); err != nil { + return nil, err + } + + return &Client{ + config: c, + }, nil +} + +// Sets the address of Watchtower in the client. The format of address should +// be "://:". Setting this on a client will override the +// value of the WATCHTOWER_ADDR environment variable. +func (c *Client) SetAddress(addr string) error { + c.modifyLock.Lock() + defer c.modifyLock.Unlock() + + return c.config.setAddress(addr) +} + +// SetLimiter will set the rate limiter for this client. This method is +// thread-safe. rateLimit and burst are specified according to +// https://godoc.org/golang.org/x/time/rate#NewLimiter +func (c *Client) SetLimiter(rateLimit float64, burst int) { + c.modifyLock.Lock() + defer c.modifyLock.Unlock() + + c.config.Limiter = rate.NewLimiter(rate.Limit(rateLimit), burst) +} + +// SetMaxRetries sets the number of retries that will be used in the case of +// certain errors +func (c *Client) SetMaxRetries(retries int) { + c.modifyLock.Lock() + defer c.modifyLock.Unlock() + + c.config.MaxRetries = retries +} + +// SetCheckRetry sets the CheckRetry function to be used for future requests. +func (c *Client) SetCheckRetry(checkRetry retryablehttp.CheckRetry) { + c.modifyLock.Lock() + defer c.modifyLock.Unlock() + + c.config.CheckRetry = checkRetry +} + +// SetClientTimeout sets the client request timeout +func (c *Client) SetClientTimeout(timeout time.Duration) { + c.modifyLock.Lock() + defer c.modifyLock.Unlock() + + c.config.Timeout = timeout +} + +func (c *Client) SetOutputCurlString(curl bool) { + c.modifyLock.Lock() + defer c.modifyLock.Unlock() + + c.config.OutputCurlString = curl +} + +// SetToken sets the token directly. This won't perform any auth +// verification, it simply sets the token properly for future requests. +func (c *Client) SetToken(token string) { + c.modifyLock.Lock() + defer c.modifyLock.Unlock() + + c.config.Token = token +} + +// SetHeaders clears all previous headers and uses only the given +// ones going forward. +func (c *Client) SetHeaders(headers http.Header) { + c.modifyLock.Lock() + defer c.modifyLock.Unlock() + c.config.Headers = headers +} + +// SetBackoff sets the backoff function to be used for future requests. +func (c *Client) SetBackoff(backoff retryablehttp.Backoff) { + c.modifyLock.Lock() + defer c.modifyLock.Unlock() + + c.config.Backoff = backoff +} + +// Clone creates a new client with the same configuration. Note that the same +// underlying http.Client is used; modifying the client from more than one +// goroutine at once may not be safe, so modify the client as needed and then +// clone. +func (c *Client) Clone() (*Client, error) { + c.modifyLock.RLock() + defer c.modifyLock.RUnlock() + + config := c.config + + newConfig := &Config{ + Address: config.Address, + Token: config.Token, + HttpClient: config.HttpClient, + Headers: make(http.Header), + MaxRetries: config.MaxRetries, + Timeout: config.Timeout, + Backoff: config.Backoff, + CheckRetry: config.CheckRetry, + Limiter: config.Limiter, + SRVLookup: config.SRVLookup, + } + if config.TLSConfig != nil { + newConfig.TLSConfig = new(TLSConfig) + *newConfig.TLSConfig = *config.TLSConfig + } + for k, v := range config.Headers { + vSlice := make([]string, 0, len(v)) + for _, i := range v { + vSlice = append(vSlice, i) + } + newConfig.Headers[k] = vSlice + } + + return NewClient(newConfig) +} + +func copyHeaders(in http.Header) http.Header { + ret := make(http.Header) + for k, v := range in { + for _, val := range v { + ret[k] = append(ret[k], val) + } + } + + return ret +} + +// GetReaderFuncForJSON returns a func compatible with retryablehttp.ReaderFunc +// after marshaling JSON +func getBufferForJSON(val interface{}) (*bytes.Buffer, error) { + b, err := json.Marshal(val) + if err != nil { + return nil, err + } + + return bytes.NewBuffer(b), nil +} + +// NewRequest creates a new raw request object to query the Watchtower controller +// configured for this client. This is an advanced method and generally +// doesn't need to be called externally. +func (c *Client) NewRequest(ctx context.Context, method, requestPath string, body interface{}) (*http.Request, error) { + c.modifyLock.RLock() + addr := c.config.Address + org := c.config.Org + project := c.config.Project + srvLookup := c.config.SRVLookup + token := c.config.Token + httpClient := c.config.HttpClient + headers := copyHeaders(c.config.Headers) + c.modifyLock.RUnlock() + + u, err := url.Parse(addr) + if err != nil { + return nil, err + } + + if strings.HasPrefix(addr, "unix://") { + socket := strings.TrimPrefix(addr, "unix://") + transport := httpClient.Transport.(*http.Transport) + transport.DialContext = func(context.Context, string, string) (net.Conn, error) { + dialer := net.Dialer{} + return dialer.DialContext(ctx, "unix", socket) + } + + // Since the address points to a unix domain socket, the scheme in the + // *URL would be set to `unix`. The *URL in the client is expected to + // be pointing to the protocol used in the application layer and not to + // the transport layer. Hence, setting the fields accordingly. + u.Scheme = "http" + u.Host = socket + u.Path = "" + } + + var host = u.Host + // if SRV records exist (see + // https://tools.ietf.org/html/draft-andrews-http-srv-02), lookup the SRV + // record and take the highest match; this is not designed for + // high-availability, just discovery Internet Draft specifies that the SRV + // record is ignored if a port is given + if u.Port() == "" && srvLookup { + _, addrs, err := net.LookupSRV("http", "tcp", u.Hostname()) + if err != nil { + return nil, fmt.Errorf("error performing SRV lookup of http:tcp:%s : %w", u.Hostname(), err) + } + if len(addrs) > 0 { + host = fmt.Sprintf("%s:%d", addrs[0].Target, addrs[0].Port) + } + } + + var ok bool + if orgRaw := ctx.Value("org"); orgRaw != nil { + if org, ok = orgRaw.(string); !ok { + return nil, fmt.Errorf("could not convert %v into string org value", orgRaw) + } + } + if projectRaw := ctx.Value("project"); projectRaw != nil { + if project, ok = projectRaw.(string); !ok { + return nil, fmt.Errorf("could not convert %v into string project value", projectRaw) + } + } + + req := &http.Request{ + Method: method, + URL: &url.URL{ + User: u.User, + Scheme: u.Scheme, + Host: host, + Path: path.Join(u.Path, "v1", "org", org, "project", project, requestPath), + }, + Host: u.Host, + } + req.Header = headers + req.Header.Add("authorization", "bearer: "+token) + if ctx != nil { + req = req.WithContext(ctx) + } + + return req, nil +} + +// Do takes a properly configured request and applies client configuration to +// it, returning the response. +func (c *Client) Do(r *http.Request) (*http.Response, error) { + c.modifyLock.RLock() + limiter := c.config.Limiter + maxRetries := c.config.MaxRetries + checkRetry := c.config.CheckRetry + backoff := c.config.Backoff + httpClient := c.config.HttpClient + timeout := c.config.Timeout + token := c.config.Token + outputCurlString := c.config.OutputCurlString + c.modifyLock.RUnlock() + + ctx := r.Context() + + if limiter != nil { + limiter.Wait(ctx) + } + + // Sanity check the token before potentially erroring from the API + idx := strings.IndexFunc(token, func(c rune) bool { + return !unicode.IsPrint(c) + }) + if idx != -1 { + return nil, fmt.Errorf("configured Watchtower token contains non-printable characters and cannot be used") + } + + req, err := retryablehttp.FromRequest(r) + if err != nil { + return nil, fmt.Errorf("error converting request to retryable request: %w", err) + } + if req == nil { + return nil, fmt.Errorf("nil request created") + } + + if outputCurlString { + LastOutputStringError = &OutputStringError{Request: req} + return nil, LastOutputStringError + } + + if timeout != 0 { + // NOTE: this leaks a timer. But when we defer a cancel call here for + // the returned function we see errors in tests with contxt canceled. + // Although the request is done by the time we exit this function it is + // still causing something else to go wrong. Maybe it ends up being + // tied to the response somehow and reading the response body ends up + // checking it, or something. I don't know, but until we can chase this + // down, keep it not-canceled even though vet complains. + ctx, _ = context.WithTimeout(ctx, timeout) + } + req.Request = req.Request.WithContext(ctx) + + if backoff == nil { + backoff = retryablehttp.LinearJitterBackoff + } + + if checkRetry == nil { + checkRetry = retryablehttp.DefaultRetryPolicy + } + + client := &retryablehttp.Client{ + HTTPClient: httpClient, + RetryWaitMin: 1000 * time.Millisecond, + RetryWaitMax: 1500 * time.Millisecond, + RetryMax: maxRetries, + Backoff: backoff, + CheckRetry: checkRetry, + ErrorHandler: retryablehttp.PassthroughErrorHandler, + } + + result, err := client.Do(req) + if err != nil { + if strings.Contains(err.Error(), "tls: oversized") { + err = errwrap.Wrapf( + "{{err}}\n\n"+ + "This error usually means that the controller is running with TLS disabled\n"+ + "but the client is configured to use TLS. Please either enable TLS\n"+ + "on the server or run the client with -address set to an address\n"+ + "that uses the http protocol:\n\n"+ + " watchtower -address http://
\n\n"+ + "You can also set the WATCHTOWER_ADDR environment variable:\n\n\n"+ + " WATCHTOWER_ADDR=http://
watchtower \n\n"+ + "where
is replaced by the actual address to the controller.", + err) + } + return result, err + } + + return result, nil +} diff --git a/api/client_test.go b/api/client_test.go new file mode 100644 index 000000000..2d451c85b --- /dev/null +++ b/api/client_test.go @@ -0,0 +1,90 @@ +package api + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestConfigSetAddress(t *testing.T) { + type test struct { + name string + input string + address string + err string + org string + project string + } + + tests := []test{ + { + "bare", + "http://127.0.0.1:9200", + "http://127.0.0.1:9200", + "", + "", + "", + }, + { + "bare with version", + "http://127.0.0.1:9200/v1", + "http://127.0.0.1:9200", + "", + "", + "", + }, + { + "bare with version and trailing slash", + "http://127.0.0.1:9200/v1/", + "http://127.0.0.1:9200", + "", + "", + "", + }, + { + "invalid org", + "http://127.0.0.1:9200/v1/org", + "http://127.0.0.1:9200", + "unexpected number of segments in address", + "", + "", + }, + { + "valid org", + "http://127.0.0.1:9200/v1/org/orgid", + "http://127.0.0.1:9200", + "", + "orgid", + "", + }, + { + "invalid project", + "http://127.0.0.1:9200/v1/org/orgid/project", + "http://127.0.0.1:9200", + "unexpected number of segments in address", + "", + "", + }, + { + "valid project", + "http://127.0.0.1:9200/v1/org/orgid/project/projid", + "http://127.0.0.1:9200", + "", + "orgid", + "projid", + }, + } + + for _, v := range tests { + t.Run(v.name, func(t *testing.T) { + var c Config + err := c.setAddress(v.input) + if err != nil { + assert.Equal(t, v.err, err.Error()) + } + assert.Equal(t, v.address, c.Address) + assert.Equal(t, v.org, c.Org) + assert.Equal(t, v.project, c.Project) + }) + } +} diff --git a/api/internal/parseutil/parseutil.go b/api/internal/parseutil/parseutil.go new file mode 100644 index 000000000..c59f68215 --- /dev/null +++ b/api/internal/parseutil/parseutil.go @@ -0,0 +1,174 @@ +package parseutil + +import ( + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" + "time" + + "github.com/hashicorp/errwrap" + sockaddr "github.com/hashicorp/go-sockaddr" + "github.com/hashicorp/watchtower/api/internal/strutil" + "github.com/mitchellh/mapstructure" +) + +func ParseDurationSecond(in interface{}) (time.Duration, error) { + var dur time.Duration + jsonIn, ok := in.(json.Number) + if ok { + in = jsonIn.String() + } + switch inp := in.(type) { + case nil: + // return default of zero + case string: + if inp == "" { + return dur, nil + } + var err error + // Look for a suffix otherwise its a plain second value + if strings.HasSuffix(inp, "s") || strings.HasSuffix(inp, "m") || strings.HasSuffix(inp, "h") || strings.HasSuffix(inp, "ms") { + dur, err = time.ParseDuration(inp) + if err != nil { + return dur, err + } + } else { + // Plain integer + secs, err := strconv.ParseInt(inp, 10, 64) + if err != nil { + return dur, err + } + dur = time.Duration(secs) * time.Second + } + case int: + dur = time.Duration(inp) * time.Second + case int32: + dur = time.Duration(inp) * time.Second + case int64: + dur = time.Duration(inp) * time.Second + case uint: + dur = time.Duration(inp) * time.Second + case uint32: + dur = time.Duration(inp) * time.Second + case uint64: + dur = time.Duration(inp) * time.Second + case float32: + dur = time.Duration(inp) * time.Second + case float64: + dur = time.Duration(inp) * time.Second + case time.Duration: + dur = inp + default: + return 0, errors.New("could not parse duration from input") + } + + return dur, nil +} + +func ParseInt(in interface{}) (int64, error) { + var ret int64 + jsonIn, ok := in.(json.Number) + if ok { + in = jsonIn.String() + } + switch in.(type) { + case string: + inp := in.(string) + if inp == "" { + return 0, nil + } + var err error + left, err := strconv.ParseInt(inp, 10, 64) + if err != nil { + return ret, err + } + ret = left + case int: + ret = int64(in.(int)) + case int32: + ret = int64(in.(int32)) + case int64: + ret = in.(int64) + case uint: + ret = int64(in.(uint)) + case uint32: + ret = int64(in.(uint32)) + case uint64: + ret = int64(in.(uint64)) + default: + return 0, errors.New("could not parse value from input") + } + + return ret, nil +} + +func ParseBool(in interface{}) (bool, error) { + var result bool + if err := mapstructure.WeakDecode(in, &result); err != nil { + return false, err + } + return result, nil +} + +func ParseCommaStringSlice(in interface{}) ([]string, error) { + rawString, ok := in.(string) + if ok && rawString == "" { + return []string{}, nil + } + var result []string + config := &mapstructure.DecoderConfig{ + Result: &result, + WeaklyTypedInput: true, + DecodeHook: mapstructure.StringToSliceHookFunc(","), + } + decoder, err := mapstructure.NewDecoder(config) + if err != nil { + return nil, err + } + if err := decoder.Decode(in); err != nil { + return nil, err + } + return strutil.TrimStrings(result), nil +} + +func ParseAddrs(addrs interface{}) ([]*sockaddr.SockAddrMarshaler, error) { + out := make([]*sockaddr.SockAddrMarshaler, 0) + stringAddrs := make([]string, 0) + + switch addrs.(type) { + case string: + stringAddrs = strutil.ParseArbitraryStringSlice(addrs.(string), ",") + if len(stringAddrs) == 0 { + return nil, fmt.Errorf("unable to parse addresses from %v", addrs) + } + + case []string: + stringAddrs = addrs.([]string) + + case []interface{}: + for _, v := range addrs.([]interface{}) { + stringAddr, ok := v.(string) + if !ok { + return nil, fmt.Errorf("error parsing %v as string", v) + } + stringAddrs = append(stringAddrs, stringAddr) + } + + default: + return nil, fmt.Errorf("unknown address input type %T", addrs) + } + + for _, addr := range stringAddrs { + sa, err := sockaddr.NewSockAddr(addr) + if err != nil { + return nil, errwrap.Wrapf(fmt.Sprintf("error parsing address %q: {{err}}", addr), err) + } + out = append(out, &sockaddr.SockAddrMarshaler{ + SockAddr: sa, + }) + } + + return out, nil +} diff --git a/api/internal/strutil/strutil.go b/api/internal/strutil/strutil.go new file mode 100644 index 000000000..0069a1377 --- /dev/null +++ b/api/internal/strutil/strutil.go @@ -0,0 +1,447 @@ +package strutil + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "sort" + "strings" + + "github.com/hashicorp/errwrap" + glob "github.com/ryanuber/go-glob" +) + +// StrListContainsGlob looks for a string in a list of strings and allows +// globs. +func StrListContainsGlob(haystack []string, needle string) bool { + for _, item := range haystack { + if glob.Glob(item, needle) { + return true + } + } + return false +} + +// StrListContains looks for a string in a list of strings. +func StrListContains(haystack []string, needle string) bool { + for _, item := range haystack { + if item == needle { + return true + } + } + return false +} + +// StrListSubset checks if a given list is a subset +// of another set +func StrListSubset(super, sub []string) bool { + for _, item := range sub { + if !StrListContains(super, item) { + return false + } + } + return true +} + +// ParseDedupAndSortStrings parses a comma separated list of strings +// into a slice of strings. The return slice will be sorted and will +// not contain duplicate or empty items. +func ParseDedupAndSortStrings(input string, sep string) []string { + input = strings.TrimSpace(input) + parsed := []string{} + if input == "" { + // Don't return nil + return parsed + } + return RemoveDuplicates(strings.Split(input, sep), false) +} + +// ParseDedupLowercaseAndSortStrings parses a comma separated list of +// strings into a slice of strings. The return slice will be sorted and +// will not contain duplicate or empty items. The values will be converted +// to lower case. +func ParseDedupLowercaseAndSortStrings(input string, sep string) []string { + input = strings.TrimSpace(input) + parsed := []string{} + if input == "" { + // Don't return nil + return parsed + } + return RemoveDuplicates(strings.Split(input, sep), true) +} + +// ParseKeyValues parses a comma separated list of `=` tuples +// into a map[string]string. +func ParseKeyValues(input string, out map[string]string, sep string) error { + if out == nil { + return fmt.Errorf("'out is nil") + } + + keyValues := ParseDedupLowercaseAndSortStrings(input, sep) + if len(keyValues) == 0 { + return nil + } + + for _, keyValue := range keyValues { + shards := strings.Split(keyValue, "=") + if len(shards) != 2 { + return fmt.Errorf("invalid format") + } + + key := strings.TrimSpace(shards[0]) + value := strings.TrimSpace(shards[1]) + if key == "" || value == "" { + return fmt.Errorf("invalid pair: key: %q value: %q", key, value) + } + out[key] = value + } + return nil +} + +// ParseArbitraryKeyValues parses arbitrary tuples. The input +// can be one of the following: +// * JSON string +// * Base64 encoded JSON string +// * Comma separated list of `=` pairs +// * Base64 encoded string containing comma separated list of +// `=` pairs +// +// Input will be parsed into the output parameter, which should +// be a non-nil map[string]string. +func ParseArbitraryKeyValues(input string, out map[string]string, sep string) error { + input = strings.TrimSpace(input) + if input == "" { + return nil + } + if out == nil { + return fmt.Errorf("'out' is nil") + } + + // Try to base64 decode the input. If successful, consider the decoded + // value as input. + inputBytes, err := base64.StdEncoding.DecodeString(input) + if err == nil { + input = string(inputBytes) + } + + // Try to JSON unmarshal the input. If successful, consider that the + // metadata was supplied as JSON input. + err = json.Unmarshal([]byte(input), &out) + if err != nil { + // If JSON unmarshalling fails, consider that the input was + // supplied as a comma separated string of 'key=value' pairs. + if err = ParseKeyValues(input, out, sep); err != nil { + return errwrap.Wrapf("failed to parse the input: {{err}}", err) + } + } + + // Validate the parsed input + for key, value := range out { + if key != "" && value == "" { + return fmt.Errorf("invalid value for key %q", key) + } + } + + return nil +} + +// ParseStringSlice parses a `sep`-separated list of strings into a +// []string with surrounding whitespace removed. +// +// The output will always be a valid slice but may be of length zero. +func ParseStringSlice(input string, sep string) []string { + input = strings.TrimSpace(input) + if input == "" { + return []string{} + } + + splitStr := strings.Split(input, sep) + ret := make([]string, len(splitStr)) + for i, val := range splitStr { + ret[i] = strings.TrimSpace(val) + } + + return ret +} + +// ParseArbitraryStringSlice parses arbitrary string slice. The input +// can be one of the following: +// * JSON string +// * Base64 encoded JSON string +// * `sep` separated list of values +// * Base64-encoded string containing a `sep` separated list of values +// +// Note that the separator is ignored if the input is found to already be in a +// structured format (e.g., JSON) +// +// The output will always be a valid slice but may be of length zero. +func ParseArbitraryStringSlice(input string, sep string) []string { + input = strings.TrimSpace(input) + if input == "" { + return []string{} + } + + // Try to base64 decode the input. If successful, consider the decoded + // value as input. + inputBytes, err := base64.StdEncoding.DecodeString(input) + if err == nil { + input = string(inputBytes) + } + + ret := []string{} + + // Try to JSON unmarshal the input. If successful, consider that the + // metadata was supplied as JSON input. + err = json.Unmarshal([]byte(input), &ret) + if err != nil { + // If JSON unmarshalling fails, consider that the input was + // supplied as a separated string of values. + return ParseStringSlice(input, sep) + } + + if ret == nil { + return []string{} + } + + return ret +} + +// TrimStrings takes a slice of strings and returns a slice of strings +// with trimmed spaces +func TrimStrings(items []string) []string { + ret := make([]string, len(items)) + for i, item := range items { + ret[i] = strings.TrimSpace(item) + } + return ret +} + +// RemoveDuplicates removes duplicate and empty elements from a slice of +// strings. This also may convert the items in the slice to lower case and +// returns a sorted slice. +func RemoveDuplicates(items []string, lowercase bool) []string { + itemsMap := map[string]bool{} + for _, item := range items { + item = strings.TrimSpace(item) + if lowercase { + item = strings.ToLower(item) + } + if item == "" { + continue + } + itemsMap[item] = true + } + items = make([]string, 0, len(itemsMap)) + for item := range itemsMap { + items = append(items, item) + } + sort.Strings(items) + return items +} + +// RemoveDuplicatesStable removes duplicate and empty elements from a slice of +// strings, preserving order (and case) of the original slice. +// In all cases, strings are compared after trimming whitespace +// If caseInsensitive, strings will be compared after ToLower() +func RemoveDuplicatesStable(items []string, caseInsensitive bool) []string { + itemsMap := make(map[string]bool, len(items)) + deduplicated := make([]string, 0, len(items)) + + for _, item := range items { + key := strings.TrimSpace(item) + if caseInsensitive { + key = strings.ToLower(key) + } + if key == "" || itemsMap[key] { + continue + } + itemsMap[key] = true + deduplicated = append(deduplicated, item) + } + return deduplicated +} + +// RemoveEmpty removes empty elements from a slice of +// strings +func RemoveEmpty(items []string) []string { + if len(items) == 0 { + return items + } + itemsSlice := make([]string, 0, len(items)) + for _, item := range items { + if item == "" { + continue + } + itemsSlice = append(itemsSlice, item) + } + return itemsSlice +} + +// EquivalentSlices checks whether the given string sets are equivalent, as in, +// they contain the same values. +func EquivalentSlices(a, b []string) bool { + if a == nil && b == nil { + return true + } + + if a == nil || b == nil { + return false + } + + // First we'll build maps to ensure unique values + mapA := map[string]bool{} + mapB := map[string]bool{} + for _, keyA := range a { + mapA[keyA] = true + } + for _, keyB := range b { + mapB[keyB] = true + } + + // Now we'll build our checking slices + var sortedA, sortedB []string + for keyA := range mapA { + sortedA = append(sortedA, keyA) + } + for keyB := range mapB { + sortedB = append(sortedB, keyB) + } + sort.Strings(sortedA) + sort.Strings(sortedB) + + // Finally, compare + if len(sortedA) != len(sortedB) { + return false + } + + for i := range sortedA { + if sortedA[i] != sortedB[i] { + return false + } + } + + return true +} + +// EqualStringMaps tests whether two map[string]string objects are equal. +// Equal means both maps have the same sets of keys and values. This function +// is 6-10x faster than a call to reflect.DeepEqual(). +func EqualStringMaps(a, b map[string]string) bool { + if len(a) != len(b) { + return false + } + + for k := range a { + v, ok := b[k] + if !ok || a[k] != v { + return false + } + } + + return true +} + +// StrListDelete removes the first occurrence of the given item from the slice +// of strings if the item exists. +func StrListDelete(s []string, d string) []string { + if s == nil { + return s + } + + for index, element := range s { + if element == d { + return append(s[:index], s[index+1:]...) + } + } + + return s +} + +// GlobbedStringsMatch compares item to val with support for a leading and/or +// trailing wildcard '*' in item. +func GlobbedStringsMatch(item, val string) bool { + if len(item) < 2 { + return val == item + } + + hasPrefix := strings.HasPrefix(item, "*") + hasSuffix := strings.HasSuffix(item, "*") + + if hasPrefix && hasSuffix { + return strings.Contains(val, item[1:len(item)-1]) + } else if hasPrefix { + return strings.HasSuffix(val, item[1:]) + } else if hasSuffix { + return strings.HasPrefix(val, item[:len(item)-1]) + } + + return val == item +} + +// AppendIfMissing adds a string to a slice if the given string is not present +func AppendIfMissing(slice []string, i string) []string { + if StrListContains(slice, i) { + return slice + } + return append(slice, i) +} + +// MergeSlices adds an arbitrary number of slices together, uniquely +func MergeSlices(args ...[]string) []string { + all := map[string]struct{}{} + for _, slice := range args { + for _, v := range slice { + all[v] = struct{}{} + } + } + + result := make([]string, 0, len(all)) + for k := range all { + result = append(result, k) + } + sort.Strings(result) + return result +} + +// Difference returns the set difference (A - B) of the two given slices. The +// result will also remove any duplicated values in set A regardless of whether +// that matches any values in set B. +func Difference(a, b []string, lowercase bool) []string { + if len(a) == 0 { + return a + } + if len(b) == 0 { + if !lowercase { + return a + } + newA := make([]string, len(a)) + for i, v := range a { + newA[i] = strings.ToLower(v) + } + return newA + } + + a = RemoveDuplicates(a, lowercase) + b = RemoveDuplicates(b, lowercase) + + itemsMap := map[string]bool{} + for _, aVal := range a { + itemsMap[aVal] = true + } + + // Perform difference calculation + for _, bVal := range b { + if _, ok := itemsMap[bVal]; ok { + itemsMap[bVal] = false + } + } + + items := []string{} + for item, exists := range itemsMap { + if exists { + items = append(items, item) + } + } + sort.Strings(items) + return items +} diff --git a/api/internal/strutil/strutil_test.go b/api/internal/strutil/strutil_test.go new file mode 100644 index 000000000..698d2e826 --- /dev/null +++ b/api/internal/strutil/strutil_test.go @@ -0,0 +1,601 @@ +package strutil + +import ( + "encoding/base64" + "encoding/json" + "reflect" + "testing" +) + +func TestStrUtil_StrListDelete(t *testing.T) { + output := StrListDelete([]string{"item1", "item2", "item3"}, "item1") + if StrListContains(output, "item1") { + t.Fatal("bad: 'item1' should not have been present") + } + + output = StrListDelete([]string{"item1", "item2", "item3"}, "item2") + if StrListContains(output, "item2") { + t.Fatal("bad: 'item2' should not have been present") + } + + output = StrListDelete([]string{"item1", "item2", "item3"}, "item3") + if StrListContains(output, "item3") { + t.Fatal("bad: 'item3' should not have been present") + } + + output = StrListDelete([]string{"item1", "item1", "item3"}, "item1") + if !StrListContains(output, "item1") { + t.Fatal("bad: 'item1' should have been present") + } + + output = StrListDelete(output, "item1") + if StrListContains(output, "item1") { + t.Fatal("bad: 'item1' should not have been present") + } + + output = StrListDelete(output, "random") + if len(output) != 1 { + t.Fatalf("bad: expected: 1, actual: %d", len(output)) + } + + output = StrListDelete(output, "item3") + if StrListContains(output, "item3") { + t.Fatal("bad: 'item3' should not have been present") + } +} + +func TestStrutil_EquivalentSlices(t *testing.T) { + slice1 := []string{"test2", "test1", "test3"} + slice2 := []string{"test3", "test2", "test1"} + if !EquivalentSlices(slice1, slice2) { + t.Fatalf("bad: expected a match") + } + + slice2 = append(slice2, "test4") + if EquivalentSlices(slice1, slice2) { + t.Fatalf("bad: expected a mismatch") + } +} + +func TestStrutil_ListContainsGlob(t *testing.T) { + haystack := []string{ + "dev", + "ops*", + "root/*", + "*-dev", + "_*_", + } + if StrListContainsGlob(haystack, "tubez") { + t.Fatalf("Value shouldn't exist") + } + if !StrListContainsGlob(haystack, "root/test") { + t.Fatalf("Value should exist") + } + if !StrListContainsGlob(haystack, "ops_test") { + t.Fatalf("Value should exist") + } + if !StrListContainsGlob(haystack, "ops") { + t.Fatalf("Value should exist") + } + if !StrListContainsGlob(haystack, "dev") { + t.Fatalf("Value should exist") + } + if !StrListContainsGlob(haystack, "test-dev") { + t.Fatalf("Value should exist") + } + if !StrListContainsGlob(haystack, "_test_") { + t.Fatalf("Value should exist") + } + +} + +func TestStrutil_ListContains(t *testing.T) { + haystack := []string{ + "dev", + "ops", + "prod", + "root", + } + if StrListContains(haystack, "tubez") { + t.Fatalf("Bad") + } + if !StrListContains(haystack, "root") { + t.Fatalf("Bad") + } +} + +func TestStrutil_ListSubset(t *testing.T) { + parent := []string{ + "dev", + "ops", + "prod", + "root", + } + child := []string{ + "prod", + "ops", + } + if !StrListSubset(parent, child) { + t.Fatalf("Bad") + } + if !StrListSubset(parent, parent) { + t.Fatalf("Bad") + } + if !StrListSubset(child, child) { + t.Fatalf("Bad") + } + if !StrListSubset(child, nil) { + t.Fatalf("Bad") + } + if StrListSubset(child, parent) { + t.Fatalf("Bad") + } + if StrListSubset(nil, child) { + t.Fatalf("Bad") + } +} + +func TestStrutil_ParseKeyValues(t *testing.T) { + actual := make(map[string]string) + expected := map[string]string{ + "key1": "value1", + "key2": "value2", + } + var input string + var err error + + input = "key1=value1,key2=value2" + err = ParseKeyValues(input, actual, ",") + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(expected, actual) { + t.Fatalf("bad: expected: %#v\nactual: %#v", expected, actual) + } + for k, _ := range actual { + delete(actual, k) + } + + input = "key1 = value1, key2 = value2" + err = ParseKeyValues(input, actual, ",") + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(expected, actual) { + t.Fatalf("bad: expected: %#v\nactual: %#v", expected, actual) + } + for k, _ := range actual { + delete(actual, k) + } + + input = "key1 = value1, key2 = " + err = ParseKeyValues(input, actual, ",") + if err == nil { + t.Fatalf("expected an error") + } + for k, _ := range actual { + delete(actual, k) + } + + input = "key1 = value1, = value2 " + err = ParseKeyValues(input, actual, ",") + if err == nil { + t.Fatalf("expected an error") + } + for k, _ := range actual { + delete(actual, k) + } + + input = "key1" + err = ParseKeyValues(input, actual, ",") + if err == nil { + t.Fatalf("expected an error") + } +} + +func TestStrutil_ParseArbitraryKeyValues(t *testing.T) { + actual := make(map[string]string) + expected := map[string]string{ + "key1": "value1", + "key2": "value2", + } + var input string + var err error + + // Test = as comma separated string + input = "key1=value1,key2=value2" + err = ParseArbitraryKeyValues(input, actual, ",") + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(expected, actual) { + t.Fatalf("bad: expected: %#v\nactual: %#v", expected, actual) + } + for k, _ := range actual { + delete(actual, k) + } + + // Test = as base64 encoded comma separated string + input = base64.StdEncoding.EncodeToString([]byte(input)) + err = ParseArbitraryKeyValues(input, actual, ",") + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(expected, actual) { + t.Fatalf("bad: expected: %#v\nactual: %#v", expected, actual) + } + for k, _ := range actual { + delete(actual, k) + } + + // Test JSON encoded = tuples + input = `{"key1":"value1", "key2":"value2"}` + err = ParseArbitraryKeyValues(input, actual, ",") + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(expected, actual) { + t.Fatalf("bad: expected: %#v\nactual: %#v", expected, actual) + } + for k, _ := range actual { + delete(actual, k) + } + + // Test base64 encoded JSON string of = tuples + input = base64.StdEncoding.EncodeToString([]byte(input)) + err = ParseArbitraryKeyValues(input, actual, ",") + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(expected, actual) { + t.Fatalf("bad: expected: %#v\nactual: %#v", expected, actual) + } + for k, _ := range actual { + delete(actual, k) + } +} + +func TestStrutil_ParseArbitraryStringSlice(t *testing.T) { + input := `CREATE ROLE "{{name}}" WITH LOGIN PASSWORD '{{password}}' VALID UNTIL '{{expiration}}';GRANT "foo-role" TO "{{name}}";ALTER ROLE "{{name}}" SET search_path = foo;GRANT CONNECT ON DATABASE "postgres" TO "{{name}}";` + + jsonExpected := []string{ + `DO $$ +BEGIN + IF NOT EXISTS (SELECT * FROM pg_catalog.pg_roles WHERE rolname='foo-role') THEN + CREATE ROLE "foo-role"; + CREATE SCHEMA IF NOT EXISTS foo AUTHORIZATION "foo-role"; + ALTER ROLE "foo-role" SET search_path = foo; + GRANT TEMPORARY ON DATABASE "postgres" TO "foo-role"; + GRANT ALL PRIVILEGES ON SCHEMA foo TO "foo-role"; + GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA foo TO "foo-role"; + GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA foo TO "foo-role"; + GRANT ALL PRIVILEGES ON ALL FUNCTIONS IN SCHEMA foo TO "foo-role"; + END IF; +END +$$`, + `CREATE ROLE "{{name}}" WITH LOGIN PASSWORD '{{password}}' VALID UNTIL '{{expiration}}'`, + `GRANT "foo-role" TO "{{name}}"`, + `ALTER ROLE "{{name}}" SET search_path = foo`, + `GRANT CONNECT ON DATABASE "postgres" TO "{{name}}"`, + ``, + } + + nonJSONExpected := jsonExpected[1:] + + var actual []string + var inputB64 string + var err error + + // Test non-JSON string + actual = ParseArbitraryStringSlice(input, ";") + if !reflect.DeepEqual(nonJSONExpected, actual) { + t.Fatalf("bad: expected:\n%#v\nactual:\n%#v", nonJSONExpected, actual) + } + + // Test base64-encoded non-JSON string + inputB64 = base64.StdEncoding.EncodeToString([]byte(input)) + actual = ParseArbitraryStringSlice(inputB64, ";") + if !reflect.DeepEqual(nonJSONExpected, actual) { + t.Fatalf("bad: expected:\n%#v\nactual:\n%#v", nonJSONExpected, actual) + } + + // Test JSON encoded + inputJSON, err := json.Marshal(jsonExpected) + if err != nil { + t.Fatal(err) + } + + actual = ParseArbitraryStringSlice(string(inputJSON), ";") + if !reflect.DeepEqual(jsonExpected, actual) { + t.Fatalf("bad: expected:\n%#v\nactual:\n%#v", string(inputJSON), actual) + } + + // Test base64 encoded JSON string of = tuples + inputB64 = base64.StdEncoding.EncodeToString(inputJSON) + actual = ParseArbitraryStringSlice(inputB64, ";") + if !reflect.DeepEqual(jsonExpected, actual) { + t.Fatalf("bad: expected:\n%#v\nactual:\n%#v", jsonExpected, actual) + } +} + +func TestGlobbedStringsMatch(t *testing.T) { + type tCase struct { + item string + val string + expect bool + } + + tCases := []tCase{ + tCase{"", "", true}, + tCase{"*", "*", true}, + tCase{"**", "**", true}, + tCase{"*t", "t", true}, + tCase{"*t", "test", true}, + tCase{"t*", "test", true}, + tCase{"*test", "test", true}, + tCase{"*test", "a test", true}, + tCase{"test", "a test", false}, + tCase{"*test", "tests", false}, + tCase{"test*", "test", true}, + tCase{"test*", "testsss", true}, + tCase{"test**", "testsss", false}, + tCase{"test**", "test*", true}, + tCase{"**test", "*test", true}, + tCase{"TEST", "test", false}, + tCase{"test", "test", true}, + } + + for _, tc := range tCases { + actual := GlobbedStringsMatch(tc.item, tc.val) + + if actual != tc.expect { + t.Fatalf("Bad testcase %#v, expected %t, got %t", tc, tc.expect, actual) + } + } +} + +func TestTrimStrings(t *testing.T) { + input := []string{"abc", "123", "abcd ", "123 "} + expected := []string{"abc", "123", "abcd", "123"} + actual := TrimStrings(input) + if !reflect.DeepEqual(expected, actual) { + t.Fatalf("Bad TrimStrings: expected:%#v, got:%#v", expected, actual) + } +} + +func TestRemoveEmpty(t *testing.T) { + input := []string{"abc", "", "abc", ""} + expected := []string{"abc", "abc"} + actual := RemoveEmpty(input) + if !reflect.DeepEqual(expected, actual) { + t.Fatalf("Bad TrimStrings: expected:%#v, got:%#v", expected, actual) + } + + input = []string{""} + expected = []string{} + actual = RemoveEmpty(input) + if !reflect.DeepEqual(expected, actual) { + t.Fatalf("Bad TrimStrings: expected:%#v, got:%#v", expected, actual) + } +} + +func TestStrutil_AppendIfMissing(t *testing.T) { + keys := []string{} + + keys = AppendIfMissing(keys, "foo") + + if len(keys) != 1 { + t.Fatalf("expected slice to be length of 1: %v", keys) + } + if keys[0] != "foo" { + t.Fatalf("expected slice to contain key 'foo': %v", keys) + } + + keys = AppendIfMissing(keys, "bar") + + if len(keys) != 2 { + t.Fatalf("expected slice to be length of 2: %v", keys) + } + if keys[0] != "foo" { + t.Fatalf("expected slice to contain key 'foo': %v", keys) + } + if keys[1] != "bar" { + t.Fatalf("expected slice to contain key 'bar': %v", keys) + } + + keys = AppendIfMissing(keys, "foo") + + if len(keys) != 2 { + t.Fatalf("expected slice to still be length of 2: %v", keys) + } + if keys[0] != "foo" { + t.Fatalf("expected slice to still contain key 'foo': %v", keys) + } + if keys[1] != "bar" { + t.Fatalf("expected slice to still contain key 'bar': %v", keys) + } +} + +func TestStrUtil_RemoveDuplicates(t *testing.T) { + type tCase struct { + input []string + expect []string + lowercase bool + } + + tCases := []tCase{ + tCase{[]string{}, []string{}, false}, + tCase{[]string{}, []string{}, true}, + tCase{[]string{"a", "b", "a"}, []string{"a", "b"}, false}, + tCase{[]string{"A", "b", "a"}, []string{"A", "a", "b"}, false}, + tCase{[]string{"A", "b", "a"}, []string{"a", "b"}, true}, + } + + for _, tc := range tCases { + actual := RemoveDuplicates(tc.input, tc.lowercase) + + if !reflect.DeepEqual(actual, tc.expect) { + t.Fatalf("Bad testcase %#v, expected %v, got %v", tc, tc.expect, actual) + } + } +} + +func TestStrUtil_RemoveDuplicatesStable(t *testing.T) { + type tCase struct { + input []string + expect []string + caseInsensitive bool + } + + tCases := []tCase{ + tCase{[]string{}, []string{}, false}, + tCase{[]string{}, []string{}, true}, + tCase{[]string{"a", "b", "a"}, []string{"a", "b"}, false}, + tCase{[]string{"A", "b", "a"}, []string{"A", "b", "a"}, false}, + tCase{[]string{"A", "b", "a"}, []string{"A", "b"}, true}, + tCase{[]string{" ", "d", "c", "d"}, []string{"d", "c"}, false}, + tCase{[]string{"Z ", " z", " z ", "y"}, []string{"Z ", "y"}, true}, + tCase{[]string{"Z ", " z", " z ", "y"}, []string{"Z ", " z", "y"}, false}, + } + + for _, tc := range tCases { + actual := RemoveDuplicatesStable(tc.input, tc.caseInsensitive) + + if !reflect.DeepEqual(actual, tc.expect) { + t.Fatalf("Bad testcase %#v, expected %v, got %v", tc, tc.expect, actual) + } + } +} + +func TestStrUtil_ParseStringSlice(t *testing.T) { + type tCase struct { + input string + sep string + expect []string + } + + tCases := []tCase{ + tCase{"", "", []string{}}, + tCase{" ", ",", []string{}}, + tCase{", ", ",", []string{"", ""}}, + tCase{"a", ",", []string{"a"}}, + tCase{" a, b, c ", ",", []string{"a", "b", "c"}}, + tCase{" a; b; c ", ";", []string{"a", "b", "c"}}, + tCase{" a :: b :: c ", "::", []string{"a", "b", "c"}}, + } + + for _, tc := range tCases { + actual := ParseStringSlice(tc.input, tc.sep) + + if !reflect.DeepEqual(actual, tc.expect) { + t.Fatalf("Bad testcase %#v, expected %v, got %v", tc, tc.expect, actual) + } + } +} + +func TestStrUtil_MergeSlices(t *testing.T) { + res := MergeSlices([]string{"a", "c", "d"}, []string{}, []string{"c", "f", "a"}, nil, []string{"foo"}) + + expect := []string{"a", "c", "d", "f", "foo"} + + if !reflect.DeepEqual(res, expect) { + t.Fatalf("expected %v, got %v", expect, res) + } +} + +func TestDifference(t *testing.T) { + testCases := []struct { + Name string + SetA []string + SetB []string + Lowercase bool + ExpectedResult []string + }{ + { + Name: "case_sensitive", + SetA: []string{"a", "b", "c"}, + SetB: []string{"b", "c"}, + Lowercase: false, + ExpectedResult: []string{"a"}, + }, + { + Name: "case_insensitive", + SetA: []string{"a", "B", "c"}, + SetB: []string{"b", "C"}, + Lowercase: true, + ExpectedResult: []string{"a"}, + }, + { + Name: "no_match", + SetA: []string{"a", "b", "c"}, + SetB: []string{"d"}, + Lowercase: false, + ExpectedResult: []string{"a", "b", "c"}, + }, + { + Name: "empty_set_a", + SetA: []string{}, + SetB: []string{"d", "e"}, + Lowercase: false, + ExpectedResult: []string{}, + }, + { + Name: "empty_set_b", + SetA: []string{"a", "b"}, + SetB: []string{}, + Lowercase: false, + ExpectedResult: []string{"a", "b"}, + }, + } + + for _, tc := range testCases { + t.Run(tc.Name, func(t *testing.T) { + actualResult := Difference(tc.SetA, tc.SetB, tc.Lowercase) + + if !reflect.DeepEqual(actualResult, tc.ExpectedResult) { + t.Fatalf("expected %v, got %v", tc.ExpectedResult, actualResult) + } + }) + } +} + +func TestStrUtil_EqualStringMaps(t *testing.T) { + m1 := map[string]string{ + "foo": "a", + } + m2 := map[string]string{ + "foo": "a", + "bar": "b", + } + var m3 map[string]string + + m4 := map[string]string{ + "dog": "", + } + + m5 := map[string]string{ + "cat": "", + } + + tests := []struct { + a map[string]string + b map[string]string + result bool + }{ + {m1, m1, true}, + {m2, m2, true}, + {m1, m2, false}, + {m2, m1, false}, + {m2, m2, true}, + {m3, m1, false}, + {m3, m3, true}, + {m4, m5, false}, + } + + for i, test := range tests { + actual := EqualStringMaps(test.a, test.b) + if actual != test.result { + t.Fatalf("case %d, expected %v, got %v", i, test.result, actual) + } + } +} diff --git a/api/output_string.go b/api/output_string.go new file mode 100644 index 000000000..af6a481bd --- /dev/null +++ b/api/output_string.go @@ -0,0 +1,71 @@ +package api + +import ( + "fmt" + "strings" + + retryablehttp "github.com/hashicorp/go-retryablehttp" +) + +const ( + ErrOutputStringRequest = "output a string, please" +) + +var ( + LastOutputStringError *OutputStringError +) + +type OutputStringError struct { + *retryablehttp.Request + parsingError error + parsedCurlString string +} + +func (d *OutputStringError) Error() string { + if d.parsedCurlString == "" { + d.parseRequest() + if d.parsingError != nil { + return d.parsingError.Error() + } + } + + return ErrOutputStringRequest +} + +func (d *OutputStringError) parseRequest() { + body, err := d.Request.BodyBytes() + if err != nil { + d.parsingError = err + return + } + + // Build cURL string + d.parsedCurlString = "curl " + if d.Request.Method != "GET" { + d.parsedCurlString = fmt.Sprintf("%s-X %s ", d.parsedCurlString, d.Request.Method) + } + for k, v := range d.Request.Header { + for _, h := range v { + if strings.ToLower(k) == "authorization" { + h = `Bearer: ` + } + d.parsedCurlString = fmt.Sprintf("%s-H \"%s: %s\" ", d.parsedCurlString, k, h) + } + } + + if len(body) > 0 { + // We need to escape single quotes since that's what we're using to + // quote the body + escapedBody := strings.Replace(string(body), "'", "'\"'\"'", -1) + d.parsedCurlString = fmt.Sprintf("%s-d '%s' ", d.parsedCurlString, escapedBody) + } + + d.parsedCurlString = fmt.Sprintf("%s%s", d.parsedCurlString, d.Request.URL.String()) +} + +func (d *OutputStringError) CurlString() string { + if d.parsedCurlString == "" { + d.parseRequest() + } + return d.parsedCurlString +} diff --git a/go.mod b/go.mod index 9744bbe7e..839d9ab76 100644 --- a/go.mod +++ b/go.mod @@ -17,9 +17,13 @@ require ( github.com/grpc-ecosystem/grpc-gateway v1.14.4-rc.1 github.com/hashicorp/errwrap v1.0.0 github.com/hashicorp/go-alpnmux v0.0.0-20200323180452-dee08f00df54 + github.com/hashicorp/go-cleanhttp v0.5.1 github.com/hashicorp/go-hclog v0.12.2 github.com/hashicorp/go-kms-wrapping v0.5.8 github.com/hashicorp/go-multierror v1.0.0 + github.com/hashicorp/go-retryablehttp v0.6.2 + github.com/hashicorp/go-rootcerts v1.0.2 + github.com/hashicorp/go-sockaddr v1.0.2 github.com/hashicorp/go-uuid v1.0.2 github.com/hashicorp/hcl v1.0.0 github.com/hashicorp/vault v1.2.1-0.20200310200732-a321b1217d5a @@ -38,10 +42,13 @@ require ( github.com/pkg/errors v0.9.1 github.com/posener/complete v1.2.3 github.com/ryanuber/columnize v2.1.0+incompatible + github.com/ryanuber/go-glob v1.0.0 github.com/spf13/viper v1.6.3 // indirect + github.com/stretchr/testify v1.5.1 go.mongodb.org/mongo-driver v1.3.2 // indirect golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f // indirect + golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 golang.org/x/tools v0.0.0-20200421185700-66008de356c7 // indirect google.golang.org/genproto v0.0.0-20200409111301-baae70f3302d google.golang.org/grpc v1.27.1 diff --git a/internal/cmd/base/const.go b/internal/cmd/base/const.go index df7d9aedf..01ed7c57d 100644 --- a/internal/cmd/base/const.go +++ b/internal/cmd/base/const.go @@ -26,14 +26,4 @@ const ( const EnvWatchtowerCLINoColor = `WATCHTOWER_CLI_NO_COLOR` const EnvWatchtowerCLIFormat = `WATCHTOWER_CLI_FORMAT` -const EnvWatchtowerAddress = "WATCHTOWER_ADDR" -const EnvWatchtowerCACert = "WATCHTOWER_CACERT" -const EnvWatchtowerCAPath = "WATCHTOWER_CAPATH" -const EnvWatchtowerClientCert = "WATCHTOWER_CLIENT_CERT" -const EnvWatchtowerClientKey = "WATCHTOWER_CLIENT_KEY" -const EnvWatchtowerClientTimeout = "WATCHTOWER_CLIENT_TIMEOUT" -const EnvWatchtowerTLSInsecure = "WATCHTOWER_TLS_INSECURE" -const EnvWatchtowerTLSServerName = "WATCHTOWER_TLS_SERVER_NAME" -const EnvWatchtowerMaxRetries = "WATCHTOWER_MAX_RETRIES" -const EnvWatchtowerToken = "WATCHTOWER_TOKEN" -const EnvRateLimit = "WATCHTOWER_RATE_LIMIT" +