diff --git a/CHANGELOG.md b/CHANGELOG.md
index 75e7924fd..82f106335 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -30,6 +30,9 @@ Canonical reference for changes, improvements, and bugfixes for Boundary.
 * cli/connect/kube: New `kube` subcommand for `boundary connect` that makes it
   easy to route `kubectl` commands through Boundary, including when using
   `kubectl proxy` ([PR](https://github.com/hashicorp/boundary/pull/816))
+* cli/server: Add some extra checks around valid/invalid combinations of
+  addresses to avoid hard-to-understand runtime issues
+  ([PR](https://github.com/hashicorp/boundary/pull/838))
 
 ### Bug Fixes
 
diff --git a/internal/cmd/commands/server/server.go b/internal/cmd/commands/server/server.go
index e8791c0c5..bf0914c1d 100644
--- a/internal/cmd/commands/server/server.go
+++ b/internal/cmd/commands/server/server.go
@@ -2,6 +2,7 @@ package server
 
 import (
 	"fmt"
+	"net"
 	"runtime"
 	"strings"
 
@@ -221,16 +222,56 @@ func (c *Command) Run(args []string) int {
 		if c.Config.Controller != nil {
 			switch len(c.Config.Worker.Controllers) {
 			case 0:
+				c.Config.Worker.Controllers = []string{clusterAddr}
 			case 1:
 				if c.Config.Worker.Controllers[0] == clusterAddr {
 					break
 				}
+				// Best effort see if it's a domain name and if not assume it must match
+				host, _, err := net.SplitHostPort(c.Config.Worker.Controllers[0])
+				if err != nil && strings.Contains(err.Error(), "missing port in address") {
+					err = nil
+					host = c.Config.Worker.Controllers[0]
+				}
+				if err == nil {
+					ip := net.ParseIP(host)
+					if ip == nil {
+						// Assume it's a domain name
+						break
+					}
+				}
 				fallthrough
 			default:
-				c.UI.Error(`When running a combined controller and worker, it's invalid to specify a "controllers" key in the worker block with any value other than the controller cluster address`)
+				c.UI.Error(`When running a combined controller and worker, it's invalid to specify a "controllers" key in the worker block with any value other than the controller cluster address/port when using IPs rather than DNS names`)
 				return 1
 			}
-			c.Config.Worker.Controllers = []string{clusterAddr}
+		}
+		for _, controller := range c.Config.Worker.Controllers {
+			host, _, err := net.SplitHostPort(controller)
+			if err != nil {
+				if strings.Contains(err.Error(), "missing port in address") {
+					host = controller
+				} else {
+					c.UI.Error(fmt.Errorf("Invalid controller address %q: %w", controller, err).Error())
+					return 1
+				}
+			}
+			ip := net.ParseIP(host)
+			if ip != nil {
+				var errMsg string
+				switch {
+				case ip.IsUnspecified():
+					errMsg = "an unspecified"
+				case ip.IsMulticast():
+					errMsg = "a multicast"
+				case ip.IsGlobalUnicast():
+					errMsg = "a global unicast"
+				}
+				if errMsg != "" {
+					c.UI.Error(fmt.Sprintf("Controller address %q is invalid: cannot be %s address", controller, errMsg))
+					return 1
+				}
+			}
 		}
 	}
 	if err := c.SetupListeners(c.UI, c.Config.SharedConfig, []string{"api", "cluster", "proxy"}); err != nil {
@@ -247,6 +288,30 @@ func (c *Command) Run(args []string) int {
 		c.Info["public addr"] = c.Config.Worker.PublicAddr
 	}
 	if c.Config.Controller != nil {
+		for _, ln := range c.Config.Listeners {
+			for _, purpose := range ln.Purpose {
+				if purpose != "cluster" {
+					continue
+				}
+				host, _, err := net.SplitHostPort(ln.Address)
+				if err != nil {
+					if strings.Contains(err.Error(), "missing port in address") {
+						host = ln.Address
+					} else {
+						c.UI.Error(fmt.Errorf("Invalid cluster listener address %q: %w", ln.Address, err).Error())
+						return 1
+					}
+				}
+				ip := net.ParseIP(host)
+				if ip != nil {
+					if ip.IsUnspecified() && c.Config.Controller.PublicClusterAddr == "" {
+						c.UI.Error(fmt.Sprintf("When %q listener has an unspecified address, %q must be set", "cluster", "public_cluster_addr"))
+						return 1
+					}
+				}
+			}
+		}
+
 		if err := c.SetupControllerPublicClusterAddress(c.Config, ""); err != nil {
 			c.UI.Error(err.Error())
 			return 1