Release 2021.3.3

This change has two parts:
1. Update to newer version of the urfave/cli fork that correctly sets flag value along the context hierarchy while respecting config file overide behavior of the most specific instance of the flag.
2. Redefine --credentials-file flag so that create and delete subcommand don't use value from the config file.

CHANGES.md

@@ -1,5 +1,34 @@
 **Experimental**: This is a new format for release notes. The format and availability is subject to change.
 
+## 2021.3.3
+
+### Improvements
+
+- Tunnel create command, as well as, running ad-hoc tunnels using `cloudflared tunnel -name NAME`, will not overwrite
+  existing files when writing tunnel credentials. 
+
+### Bug Fixes
+
+- Tunnel create and delete commands no longer use path to credentials from the configuration file.
+  If you need ot place tunnel credentials file at a specific location, you must use `--credentials-file` flag.
+- Access ssh-gen creates properly named keys for SSH short lived certs.
+
+
+## 2021.3.2
+
+### New Features
+
+- It is now possible to obtain more detailed information about the cloudflared connectors to Cloudflare Edge via
+  `cloudflared tunnel info <name/uuid>`. It is possible to sort the output as well as output in different formats,
+  such as: `cloudflared tunnel info --sort-by version --invert-sort --output json <name/uuid>`.
+  You can obtain more information via `cloudflared tunnel info --help`.
+
+### Bug Fixes
+
+- Don't look for configuration file in default paths when `--config FILE` flag is present after `tunnel` subcommand.
+- cloudflared access token command now functions correctly with the new token-per-app change from 2021.3.0.
+
+
 ## 2021.3.0
 
 ### New Features

Makefile

@@ -254,3 +254,7 @@ vet:
 .PHONY: msi
 msi: cloudflared
 	go-msi make --msi cloudflared.msi --version $(MSI_VERSION)
+
+.PHONY: goimports
+goimports:
+	for d in $$(go list -mod=readonly -f '{{.Dir}}' -a ./... | fgrep -v tunnelrpc) ; do goimports -format-only -local github.com/cloudflare/cloudflared -w $$d ; done

RELEASE_NOTES

@@ -1,3 +1,33 @@
+2021.3.3
+- 2021-03-23 TUN-4111: Warn the user if both properties "tunnel" and "hostname" are used
+- 2021-03-23 TUN-4082: Test logging when running as a service
+- 2021-03-23 TUN-4112: Skip testing graceful shutdown with SIGINT on Windows
+- 2021-03-23 TUN-4116: Ingore credentials-file setting in configuration file during tunnel create and delete opeations.
+- 2021-03-23 TUN-4118: Don't overwrite existing file with tunnel credentials. For ad-hoc tunnels, this means tunnel won't start if there's a file in the way.
+- 2021-03-24 TUN-4123: Don't capture output in reconnect componet test
+- 2021-03-23 TUN-4067: Reformat code for consistent import order, grouping, and fix formatting. Added goimports target to the Makefile to make this easier in the future.
+- 2021-03-24 AUTH-3455: Generate short-lived ssh cert per hostname
+- 2021-03-25 Update changelog 2021.3.3
+
+2021.3.2
+- 2021-03-23 TUN-4042: Capture cloudflared output to debug component tests
+- 2021-03-23 Publish changelog for 2021.3.2
+- 2021-03-16 TUN-4089: Address flakiness in component tests for termination
+- 2021-03-16 TUN-4060: Fix Go Vet warnings (new with go 1.16) where t.Fatalf is called from a test goroutine
+- 2021-03-16 TUN-4091: Use flaky decorator to rerun reconnect component tests when they fail
+- 2021-03-12 TUN-4081: Update log severities to use Zerolog's levels
+- 2021-03-16 TUN-4094: Don't read configuration file for access commands
+- 2021-03-15 TUN-3993: New `cloudflared tunnel info` to obtain details about the active connectors for a tunnel
+- 2021-03-17 TUN-3715: Apply input source to the correct context
+- 2021-03-17 AUTH-3394: Ensure scheme on token command
+- 2021-03-18 TUN-4096: Reduce tunnel not connected assertion backoff to address flaky termination tests
+- 2021-03-19 TUN-3998: Allow to cleanup the connections of a tunnel limited to a single client
+- 2021-02-04 TUN-3715: Only read config file once, right before invoking the command
+
+2021.3.1
+- 2021-03-11 TUN-4051: Add component-tests to test graceful shutdown
+- 2021-03-12 TUN-4052: Add component tests to assert service mode behavior
+
 2021.3.0
 - 2021-03-10 TUN-4075: Dedup test should not compare order of list
 - 2021-03-10 Revert "AUTH-3394: Creates a token per app instead of per path"

carrier/carrier.go

@@ -11,11 +11,11 @@ import (
 	"os"
 	"strings"
 
-	"github.com/cloudflare/cloudflared/h2mux"
-	"github.com/cloudflare/cloudflared/token"
-
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
+
+	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/token"
 )
 
 const LogFieldOriginURL = "originURL"

carrier/carrier_test.go

@@ -81,7 +81,8 @@ func TestStartServer(t *testing.T) {
 	go func() {
 		err := Serve(wsConn, listener, shutdownC, options)
 		if err != nil {
-			t.Fatalf("Error running server: %v", err)
+			t.Errorf("Error running server: %v", err)
+			return
 		}
 	}()
 

carrier/websocket.go

@@ -7,13 +7,13 @@ import (
 	"net/http"
 	"net/http/httputil"
 
+	"github.com/gorilla/websocket"
+	"github.com/rs/zerolog"
+
 	"github.com/cloudflare/cloudflared/ingress"
 	"github.com/cloudflare/cloudflared/socks"
 	"github.com/cloudflare/cloudflared/token"
 	cfwebsocket "github.com/cloudflare/cloudflared/websocket"
-
-	"github.com/gorilla/websocket"
-	"github.com/rs/zerolog"
 )
 
 // Websocket is used to carry data via WS binary frames over the tunnel from client to the origin

cfsetup.yaml

@@ -195,13 +195,15 @@ stretch: &stretch
       - *pinned_go_fips
       - build-essential
       - gotest-to-teamcity
+    pre-cache:
+      - go get golang.org/x/tools/cmd/goimports
+      - go get github.com/BurntSushi/go-sumtype
     post-cache:
       - export GOOS=linux
       - export GOARCH=amd64
       - export FIPS=true
-      # cd to a non-module directory: https://github.com/golang/go/issues/24250
-      - (cd / && go get github.com/BurntSushi/go-sumtype)
       - export PATH="$HOME/go/bin:$PATH"
+      - ./fmt-check.sh
       - make test | gotest-to-teamcity
   component-test:
     build_dir: *build_dir
@@ -210,6 +212,9 @@ stretch: &stretch
       - python3.7
       - python3-pip
       - python3-setuptools
+      # procps installs the ps command which is needed in test_sysv_service because the init script
+      # uses ps pid to determine if the agent is running
+      - procps
     pre-cache-copy-paths:
       - component-tests/requirements.txt
     pre-cache:

cmd/cloudflared/access/cmd.go

@@ -11,6 +11,12 @@ import (
 	"text/template"
 	"time"
 
+	"github.com/getsentry/raven-go"
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog"
+	"github.com/urfave/cli/v2"
+	"golang.org/x/net/idna"
+
 	"github.com/cloudflare/cloudflared/carrier"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
 	"github.com/cloudflare/cloudflared/h2mux"
@@ -18,12 +24,6 @@ import (
 	"github.com/cloudflare/cloudflared/sshgen"
 	"github.com/cloudflare/cloudflared/token"
 	"github.com/cloudflare/cloudflared/validation"
-
-	"github.com/getsentry/raven-go"
-	"github.com/pkg/errors"
-	"github.com/rs/zerolog"
-	"github.com/urfave/cli/v2"
-	"golang.org/x/net/idna"
 )
 
 const (
@@ -40,7 +40,7 @@ Add to your {{.Home}}/.ssh/config:
 
 Host {{.Hostname}}
 {{- if .ShortLivedCerts}}
-  ProxyCommand bash -c '{{.Cloudflared}} access ssh-gen --hostname %h; ssh -tt %r@cfpipe-{{.Hostname}} >&2 <&1' 
+  ProxyCommand bash -c '{{.Cloudflared}} access ssh-gen --hostname %h; ssh -tt %r@cfpipe-{{.Hostname}} >&2 <&1'
 
 Host cfpipe-{{.Hostname}}
   HostName {{.Hostname}}
@@ -77,30 +77,24 @@ func Commands() []*cli.Command {
 			Aliases:  []string{"forward"},
 			Category: "Access",
 			Usage:    "access <subcommand>",
-			Description: `Cloudflare Access protects internal resources by securing, authenticating and monitoring access 
-			per-user and by application. With Cloudflare Access, only authenticated users with the required permissions are 
-			able to reach sensitive resources. The commands provided here allow you to interact with Access protected 
+			Description: `Cloudflare Access protects internal resources by securing, authenticating and monitoring access
+			per-user and by application. With Cloudflare Access, only authenticated users with the required permissions are
+			able to reach sensitive resources. The commands provided here allow you to interact with Access protected
 			applications from the command line.`,
 			Subcommands: []*cli.Command{
 				{
 					Name:   "login",
-					Action: cliutil.ErrorHandler(login),
+					Action: cliutil.Action(login),
 					Usage:  "login <url of access application>",
 					Description: `The login subcommand initiates an authentication flow with your identity provider.
 					The subcommand will launch a browser. For headless systems, a url is provided.
 					Once authenticated with your identity provider, the login command will generate a JSON Web Token (JWT)
-					scoped to your identity, the application you intend to reach, and valid for a session duration set by your 
+					scoped to your identity, the application you intend to reach, and valid for a session duration set by your
 					administrator. cloudflared stores the token in local storage.`,
-					Flags: []cli.Flag{
-						&cli.StringFlag{
-							Name:   "url",
-							Hidden: true,
-						},
-					},
 				},
 				{
 					Name:   "curl",
-					Action: cliutil.ErrorHandler(curl),
+					Action: cliutil.Action(curl),
 					Usage:  "curl [--allow-request, -ar] <url> [<curl args>...]",
 					Description: `The curl subcommand wraps curl and automatically injects the JWT into a cf-access-token
 					header when using curl to reach an application behind Access.`,
@@ -109,7 +103,7 @@ func Commands() []*cli.Command {
 				},
 				{
 					Name:        "token",
-					Action:      cliutil.ErrorHandler(generateToken),
+					Action:      cliutil.Action(generateToken),
 					Usage:       "token -app=<url of access application>",
 					ArgsUsage:   "url of Access application",
 					Description: `The token subcommand produces a JWT which can be used to authenticate requests.`,
@@ -121,7 +115,7 @@ func Commands() []*cli.Command {
 				},
 				{
 					Name:        "tcp",
-					Action:      cliutil.ErrorHandler(ssh),
+					Action:      cliutil.Action(ssh),
 					Aliases:     []string{"rdp", "ssh", "smb"},
 					Usage:       "",
 					ArgsUsage:   "",
@@ -164,7 +158,7 @@ func Commands() []*cli.Command {
 						&cli.StringFlag{
 							Name:    logger.LogSSHLevelFlag,
 							Aliases: []string{"loglevel"}, //added to match the tunnel side
-							Usage:   "Application logging level {fatal, error, info, debug}. ",
+							Usage:   "Application logging level {debug, info, warn, error, fatal}. ",
 						},
 						&cli.StringFlag{
 							Name:   sshConnectTo,
@@ -175,7 +169,7 @@ func Commands() []*cli.Command {
 				},
 				{
 					Name:        "ssh-config",
-					Action:      cliutil.ErrorHandler(sshConfig),
+					Action:      cliutil.Action(sshConfig),
 					Usage:       "",
 					Description: `Prints an example configuration ~/.ssh/config`,
 					Flags: []cli.Flag{
@@ -191,7 +185,7 @@ func Commands() []*cli.Command {
 				},
 				{
 					Name:        "ssh-gen",
-					Action:      cliutil.ErrorHandler(sshGen),
+					Action:      cliutil.Action(sshGen),
 					Usage:       "",
 					Description: `Generates a short lived certificate for given hostname`,
 					Flags: []cli.Flag{
@@ -296,7 +290,6 @@ func curl(c *cli.Context) error {
 	return run("curl", cmdArgs...)
 }
 
-
 // run kicks off a shell task and pipe the results to the respective std pipes
 func run(cmd string, args ...string) error {
 	c := exec.Command(cmd, args...)
@@ -323,7 +316,7 @@ func generateToken(c *cli.Context) error {
 	if err := raven.SetDSN(sentryDSN); err != nil {
 		return err
 	}
-	appURL, err := url.Parse(c.String("app"))
+	appURL, err := url.Parse(ensureURLScheme(c.String("app")))
 	if err != nil || c.NumFlags() < 1 {
 		fmt.Fprintln(os.Stderr, "Please provide a url.")
 		return err
@@ -394,7 +387,7 @@ func sshGen(c *cli.Context) error {
 		return err
 	}
 
-	if err := sshgen.GenerateShortLivedCertificate(appInfo, cfdToken); err != nil {
+	if err := sshgen.GenerateShortLivedCertificate(originURL, cfdToken); err != nil {
 		return err
 	}
 

cmd/cloudflared/app_forward_service.go

@@ -1,10 +1,10 @@
 package main
 
 import (
+	"github.com/rs/zerolog"
+
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/access"
 	"github.com/cloudflare/cloudflared/config"
-
-	"github.com/rs/zerolog"
 )
 
 // ForwardServiceType is used to identify what kind of overwatch service this is

cmd/cloudflared/app_resolver_service.go

@@ -1,10 +1,10 @@
 package main
 
 import (
+	"github.com/rs/zerolog"
+
 	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/tunneldns"
-
-	"github.com/rs/zerolog"
 )
 
 const (

cmd/cloudflared/app_service.go

@@ -1,10 +1,10 @@
 package main
 
 import (
+	"github.com/rs/zerolog"
+
 	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/overwatch"
-
-	"github.com/rs/zerolog"
 )
 
 // AppService is the main service that runs when no command lines flags are passed to cloudflared

cmd/cloudflared/buildinfo/build_info.go

@@ -1,9 +1,10 @@
 package buildinfo
 
 import (
-	"github.com/rs/zerolog"
 	"fmt"
 	"runtime"
+
+	"github.com/rs/zerolog"
 )
 
 type BuildInfo struct {

cmd/cloudflared/cliutil/deprecated.go

@@ -9,13 +9,13 @@ import (
 func RemovedCommand(name string) *cli.Command {
 	return &cli.Command{
 		Name: name,
-		Action: ErrorHandler(func(context *cli.Context) error {
+		Action: func(context *cli.Context) error {
 			return cli.Exit(
 				fmt.Sprintf("%s command is no longer supported by cloudflared. Consult Argo Tunnel documentation for possible alternative solutions.", name),
 				-1,
 			)
-		}),
+		},
 		Description: fmt.Sprintf("%s is deprecated", name),
-		Hidden: true,
+		Hidden:      true,
 	}
 }

cmd/cloudflared/cliutil/errors.go

@@ -2,6 +2,7 @@ package cliutil
 
 import (
 	"fmt"
+
 	"github.com/urfave/cli/v2"
 )
 
@@ -21,7 +22,7 @@ func UsageError(format string, args ...interface{}) error {
 }
 
 // Ensures exit with error code if actionFunc returns an error
-func ErrorHandler(actionFunc cli.ActionFunc) cli.ActionFunc {
+func WithErrorHandler(actionFunc cli.ActionFunc) cli.ActionFunc {
 	return func(ctx *cli.Context) error {
 		err := actionFunc(ctx)
 		if err != nil {

cmd/cloudflared/cliutil/handler.go

@@ -0,0 +1,39 @@
+package cliutil
+
+import (
+	"github.com/urfave/cli/v2"
+	"github.com/urfave/cli/v2/altsrc"
+
+	"github.com/cloudflare/cloudflared/config"
+	"github.com/cloudflare/cloudflared/logger"
+)
+
+func Action(actionFunc cli.ActionFunc) cli.ActionFunc {
+	return WithErrorHandler(actionFunc)
+}
+
+func ConfiguredAction(actionFunc cli.ActionFunc) cli.ActionFunc {
+	return WithErrorHandler(func(c *cli.Context) error {
+		if err := setFlagsFromConfigFile(c); err != nil {
+			return err
+		}
+		return actionFunc(c)
+	})
+}
+
+func setFlagsFromConfigFile(c *cli.Context) error {
+	const errorExitCode = 1
+	log := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
+	inputSource, err := config.ReadConfigFile(c, log)
+	if err != nil {
+		if err == config.ErrNoConfigFile {
+			return nil
+		}
+		return cli.Exit(err, errorExitCode)
+	}
+
+	if err := altsrc.ApplyInputSource(c, inputSource); err != nil {
+		return cli.Exit(err, errorExitCode)
+	}
+	return nil
+}

cmd/cloudflared/linux_service.go

@@ -7,13 +7,13 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/rs/zerolog"
+	"github.com/urfave/cli/v2"
+
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/tunnel"
 	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/logger"
-
-	"github.com/rs/zerolog"
-	"github.com/urfave/cli/v2"
 )
 
 func runApp(app *cli.App, graceShutdownC chan struct{}) {
@@ -24,7 +24,7 @@ func runApp(app *cli.App, graceShutdownC chan struct{}) {
 			{
 				Name:   "install",
 				Usage:  "Install Argo Tunnel as a system service",
-				Action: cliutil.ErrorHandler(installLinuxService),
+				Action: cliutil.ConfiguredAction(installLinuxService),
 				Flags: []cli.Flag{
 					&cli.BoolFlag{
 						Name:  "legacy",
@@ -35,7 +35,7 @@ func runApp(app *cli.App, graceShutdownC chan struct{}) {
 			{
 				Name:   "uninstall",
 				Usage:  "Uninstall the Argo Tunnel service",
-				Action: cliutil.ErrorHandler(uninstallLinuxService),
+				Action: cliutil.ConfiguredAction(uninstallLinuxService),
 			},
 		},
 	})

cmd/cloudflared/macos_service.go

@@ -6,11 +6,11 @@ import (
 	"fmt"
 	"os"
 
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
-	"github.com/cloudflare/cloudflared/logger"
-
 	"github.com/pkg/errors"
 	"github.com/urfave/cli/v2"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
+	"github.com/cloudflare/cloudflared/logger"
 )
 
 const (
@@ -25,12 +25,12 @@ func runApp(app *cli.App, graceShutdownC chan struct{}) {
 			{
 				Name:   "install",
 				Usage:  "Install Argo Tunnel as an user launch agent",
-				Action: cliutil.ErrorHandler(installLaunchd),
+				Action: cliutil.ConfiguredAction(installLaunchd),
 			},
 			{
 				Name:   "uninstall",
 				Usage:  "Uninstall the Argo Tunnel launch agent",
-				Action: cliutil.ErrorHandler(uninstallLaunchd),
+				Action: cliutil.ConfiguredAction(uninstallLaunchd),
 			},
 		},
 	})

cmd/cloudflared/main.go

@@ -6,21 +6,21 @@ import (
 	"strings"
 	"time"
 
+	"github.com/getsentry/raven-go"
+	homedir "github.com/mitchellh/go-homedir"
+	"github.com/pkg/errors"
+	"github.com/urfave/cli/v2"
+
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/access"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
-	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/proxydns"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/tunnel"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/updater"
+	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/metrics"
 	"github.com/cloudflare/cloudflared/overwatch"
 	"github.com/cloudflare/cloudflared/watcher"
-
-	"github.com/getsentry/raven-go"
-	"github.com/mitchellh/go-homedir"
-	"github.com/pkg/errors"
-	"github.com/urfave/cli/v2"
 )
 
 const (
@@ -77,7 +77,6 @@ func main() {
 	See https://developers.cloudflare.com/argo-tunnel/ for more in-depth documentation.`
 	app.Flags = flags()
 	app.Action = action(graceShutdownC)
-	app.Before = tunnel.SetFlagsFromConfigFile
 	app.Commands = commands(cli.ShowVersion)
 
 	tunnel.Init(Version, graceShutdownC) // we need this to support the tunnel sub command...
@@ -90,7 +89,7 @@ func commands(version func(c *cli.Context)) []*cli.Command {
 	cmds := []*cli.Command{
 		{
 			Name:   "update",
-			Action: cliutil.ErrorHandler(updater.Update),
+			Action: cliutil.ConfiguredAction(updater.Update),
 			Usage:  "Update the agent if a new version exists",
 			Flags: []cli.Flag{
 				&cli.BoolFlag{
@@ -145,7 +144,7 @@ func isEmptyInvocation(c *cli.Context) bool {
 }
 
 func action(graceShutdownC chan struct{}) cli.ActionFunc {
-	return cliutil.ErrorHandler(func(c *cli.Context) (err error) {
+	return cliutil.ConfiguredAction(func(c *cli.Context) (err error) {
 		if isEmptyInvocation(c) {
 			return handleServiceMode(c, graceShutdownC)
 		}

cmd/cloudflared/proxydns/cmd.go

@@ -17,8 +17,9 @@ import (
 func Command(hidden bool) *cli.Command {
 	return &cli.Command{
 		Name:   "proxy-dns",
-		Action: cliutil.ErrorHandler(Run),
-		Usage:  "Run a DNS over HTTPS proxy server.",
+		Action: cliutil.ConfiguredAction(Run),
+
+		Usage: "Run a DNS over HTTPS proxy server.",
 		Flags: []cli.Flag{
 			&cli.StringFlag{
 				Name:    "metrics",
@@ -111,5 +112,3 @@ func Run(c *cli.Context) error {
 	}
 	return err
 }
-
-

cmd/cloudflared/service_template.go

@@ -10,7 +10,7 @@ import (
 	"os/exec"
 	"text/template"
 
-	"github.com/mitchellh/go-homedir"
+	homedir "github.com/mitchellh/go-homedir"
 
 	"github.com/cloudflare/cloudflared/config"
 )

cmd/cloudflared/tunnel/cmd.go

@@ -7,12 +7,20 @@ import (
 	"io/ioutil"
 	"net/url"
 	"os"
-	"reflect"
 	"runtime/trace"
 	"strings"
 	"sync"
 	"time"
 
+	"github.com/coreos/go-systemd/daemon"
+	"github.com/facebookgo/grace/gracenet"
+	"github.com/getsentry/raven-go"
+	homedir "github.com/mitchellh/go-homedir"
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog"
+	"github.com/urfave/cli/v2"
+	"github.com/urfave/cli/v2/altsrc"
+
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/proxydns"
@@ -28,15 +36,6 @@ import (
 	"github.com/cloudflare/cloudflared/tlsconfig"
 	"github.com/cloudflare/cloudflared/tunneldns"
 	"github.com/cloudflare/cloudflared/tunnelstore"
-
-	"github.com/coreos/go-systemd/daemon"
-	"github.com/facebookgo/grace/gracenet"
-	"github.com/getsentry/raven-go"
-	"github.com/mitchellh/go-homedir"
-	"github.com/pkg/errors"
-	"github.com/rs/zerolog"
-	"github.com/urfave/cli/v2"
-	"github.com/urfave/cli/v2/altsrc"
 )
 
 const (
@@ -75,8 +74,8 @@ const (
 	// uiFlag is to enable launching cloudflared in interactive UI mode
 	uiFlag = "ui"
 
-	debugLevelWarning = "At debug level, request URL, method, protocol, content legnth and header will be logged. " +
-		"Response status, content length and header will also be logged in debug level."
+	debugLevelWarning = "At debug level cloudflared will log request URL, method, protocol, content length, as well as, all request and response headers. " +
+		"This can expose sensitive information in your logs."
 
 	LogFieldCommand             = "command"
 	LogFieldExpandedPath        = "expandedPath"
@@ -101,6 +100,7 @@ func Commands() []*cli.Command {
 		buildRouteCommand(),
 		buildRunCommand(),
 		buildListCommand(),
+		buildInfoCommand(),
 		buildIngressSubcommand(),
 		buildDeleteCommand(),
 		buildCleanupCommand(),
@@ -120,8 +120,7 @@ func Commands() []*cli.Command {
 func buildTunnelCommand(subcommands []*cli.Command) *cli.Command {
 	return &cli.Command{
 		Name:      "tunnel",
-		Action:    cliutil.ErrorHandler(TunnelCommand),
-		Before:    SetFlagsFromConfigFile,
+		Action:    cliutil.ConfiguredAction(TunnelCommand),
 		Category:  "Tunnel",
 		Usage:     "Make a locally-running web service accessible over the internet using Argo Tunnel.",
 		ArgsUsage: " ",
@@ -369,26 +368,6 @@ func StartServer(
 	return waitToShutdown(&wg, cancel, errC, graceShutdownC, c.Duration("grace-period"), log)
 }
 
-func SetFlagsFromConfigFile(c *cli.Context) error {
-	const exitCode = 1
-	log := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
-	inputSource, err := config.ReadConfigFile(c, log)
-	if err != nil {
-		if err == config.ErrNoConfigFile {
-			return nil
-		}
-		return cli.Exit(err, exitCode)
-	}
-	targetFlags := c.Command.Flags
-	if c.Command.Name == "" {
-		targetFlags = c.App.Flags
-	}
-	if err := altsrc.ApplyInputSourceValues(c, inputSource, targetFlags); err != nil {
-		return cli.Exit(err, exitCode)
-	}
-	return nil
-}
-
 func waitToShutdown(wg *sync.WaitGroup,
 	cancelServerContext func(),
 	errC <-chan error,
@@ -477,33 +456,6 @@ func addPortIfMissing(uri *url.URL, port int) string {
 	return fmt.Sprintf("%s:%d", uri.Hostname(), port)
 }
 
-// appendFlags will append extra flags to a slice of flags.
-//
-// The cli package will panic if two flags exist with the same name,
-// so if extraFlags contains a flag that was already defined, modify the
-// original flags to use the extra version.
-func appendFlags(flags []cli.Flag, extraFlags ...cli.Flag) []cli.Flag {
-	for _, extra := range extraFlags {
-		var found bool
-
-		// Check if an extra flag overrides an existing flag.
-		for i, flag := range flags {
-			if reflect.DeepEqual(extra.Names(), flag.Names()) {
-				flags[i] = extra
-				found = true
-				break
-			}
-		}
-
-		// Append the extra flag if it has nothing to override.
-		if !found {
-			flags = append(flags, extra)
-		}
-	}
-
-	return flags
-}
-
 func tunnelFlags(shouldHide bool) []cli.Flag {
 	flags := configureCloudflaredFlags(shouldHide)
 	flags = append(flags, configureProxyFlags(shouldHide)...)
@@ -513,7 +465,7 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 		credentialsFileFlag,
 		altsrc.NewBoolFlag(&cli.BoolFlag{
 			Name:   "is-autoupdated",
-			Usage:  "Signal the new process that Argo Tunnel client has been autoupdated",
+			Usage:  "Signal the new process that Argo Tunnel connector has been autoupdated",
 			Value:  false,
 			Hidden: true,
 		}),
@@ -652,6 +604,7 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			Aliases: []string{"n"},
 			EnvVars: []string{"TUNNEL_NAME"},
 			Usage:   "Stable name to identify the tunnel. Using this flag will create, route and run a tunnel. For production usage, execute each command separately",
+			Hidden:  shouldHide,
 		}),
 		altsrc.NewBoolFlag(&cli.BoolFlag{
 			Name:   uiFlag,
@@ -920,7 +873,7 @@ func configureLoggingFlags(shouldHide bool) []cli.Flag {
 		altsrc.NewStringFlag(&cli.StringFlag{
 			Name:    logger.LogLevelFlag,
 			Value:   "info",
-			Usage:   "Application logging level {fatal, error, info, debug}. " + debugLevelWarning,
+			Usage:   "Application logging level {debug, info, warn, error, fatal}. " + debugLevelWarning,
 			EnvVars: []string{"TUNNEL_LOGLEVEL"},
 			Hidden:  shouldHide,
 		}),
@@ -928,7 +881,7 @@ func configureLoggingFlags(shouldHide bool) []cli.Flag {
 			Name:    logger.LogTransportLevelFlag,
 			Aliases: []string{"proto-loglevel"}, // This flag used to be called proto-loglevel
 			Value:   "info",
-			Usage:   "Transport logging level(previously called protocol logging level) {fatal, error, info, debug}",
+			Usage:   "Transport logging level(previously called protocol logging level) {debug, info, warn, error, fatal}",
 			EnvVars: []string{"TUNNEL_PROTO_LOGLEVEL", "TUNNEL_TRANSPORT_LOGLEVEL"},
 			Hidden:  shouldHide,
 		}),

cmd/cloudflared/tunnel/configuration.go

@@ -8,6 +8,13 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/google/uuid"
+	homedir "github.com/mitchellh/go-homedir"
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog"
+	"github.com/urfave/cli/v2"
+	"golang.org/x/crypto/ssh/terminal"
+
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
 	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/connection"
@@ -18,13 +25,6 @@ import (
 	"github.com/cloudflare/cloudflared/tlsconfig"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	"github.com/cloudflare/cloudflared/validation"
-
-	"github.com/google/uuid"
-	"github.com/mitchellh/go-homedir"
-	"github.com/pkg/errors"
-	"github.com/rs/zerolog"
-	"github.com/urfave/cli/v2"
-	"golang.org/x/crypto/ssh/terminal"
 )
 
 const LogFieldOriginCertPath = "originCertPath"
@@ -160,6 +160,10 @@ func prepareTunnelConfig(
 	if err != nil {
 		log.Err(err).Str(LogFieldHostname, configHostname).Msg("Invalid hostname")
 		return nil, ingress.Ingress{}, errors.Wrap(err, "Invalid hostname")
+	} else if hostname != "" && isNamedTunnel {
+		log.Warn().Msg("The property `hostname` in your configuration is ignored because you configured a Named Tunnel " +
+			"in the property `tunnel`. Make sure to provision the routing (e.g. via `cloudflared tunnel route`) or else " +
+			"your origin will not be reachable. You should remove the `hostname` property to avoid this warning.")
 	}
 	isFreeTunnel := hostname == ""
 	clientID := c.String("id")
@@ -199,9 +203,9 @@ func prepareTunnelConfig(
 	if isNamedTunnel {
 		clientUUID, err := uuid.NewRandom()
 		if err != nil {
-			return nil, ingress.Ingress{}, errors.Wrap(err, "can't generate clientUUID")
+			return nil, ingress.Ingress{}, errors.Wrap(err, "can't generate connector UUID")
 		}
-		log.Info().Msgf("Generated Client ID: %s", clientUUID)
+		log.Info().Msgf("Generated Connector ID: %s", clientUUID)
 		features := append(c.StringSlice("features"), origin.FeatureSerializedHeaders)
 		namedTunnel.Client = tunnelpogs.ClientInfo{
 			ClientID: clientUUID[:],

cmd/cloudflared/tunnel/info.go

@@ -0,0 +1,16 @@
+package tunnel
+
+import (
+	"time"
+
+	"github.com/google/uuid"
+
+	"github.com/cloudflare/cloudflared/tunnelstore"
+)
+
+type Info struct {
+	ID         uuid.UUID                   `json:"id"`
+	Name       string                      `json:"name"`
+	CreatedAt  time.Time                   `json:"createdAt"`
+	Connectors []*tunnelstore.ActiveClient `json:"conns"`
+}

cmd/cloudflared/tunnel/ingress_subcommands.go

@@ -45,7 +45,7 @@ func buildIngressSubcommand() *cli.Command {
 func buildValidateIngressCommand() *cli.Command {
 	return &cli.Command{
 		Name:        "validate",
-		Action:      cliutil.ErrorHandler(validateIngressCommand),
+		Action:      cliutil.ConfiguredAction(validateIngressCommand),
 		Usage:       "Validate the ingress configuration ",
 		UsageText:   "cloudflared tunnel [--config FILEPATH] ingress validate",
 		Description: "Validates the configuration file, ensuring your ingress rules are OK.",
@@ -55,7 +55,7 @@ func buildValidateIngressCommand() *cli.Command {
 func buildTestURLCommand() *cli.Command {
 	return &cli.Command{
 		Name:      "rule",
-		Action:    cliutil.ErrorHandler(testURLCommand),
+		Action:    cliutil.ConfiguredAction(testURLCommand),
 		Usage:     "Check which ingress rule matches a given request URL",
 		UsageText: "cloudflared tunnel [--config FILEPATH] ingress rule URL",
 		ArgsUsage: "URL",

cmd/cloudflared/tunnel/login.go

@@ -8,7 +8,7 @@ import (
 	"path/filepath"
 	"syscall"
 
-	"github.com/mitchellh/go-homedir"
+	homedir "github.com/mitchellh/go-homedir"
 	"github.com/pkg/errors"
 	"github.com/urfave/cli/v2"
 
@@ -26,16 +26,10 @@ const (
 func buildLoginSubcommand(hidden bool) *cli.Command {
 	return &cli.Command{
 		Name:      "login",
-		Action:    cliutil.ErrorHandler(login),
+		Action:    cliutil.ConfiguredAction(login),
 		Usage:     "Generate a configuration file with your login details",
 		ArgsUsage: " ",
-		Flags: []cli.Flag{
-			&cli.StringFlag{
-				Name:   "url",
-				Hidden: true,
-			},
-		},
-		Hidden: hidden,
+		Hidden:    hidden,
 	}
 }
 

cmd/cloudflared/tunnel/signal.go

@@ -20,4 +20,4 @@ func waitForSignal(graceShutdownC chan struct{}, logger *zerolog.Logger) {
 		close(graceShutdownC)
 	case <-graceShutdownC:
 	}
-}
+}

cmd/cloudflared/tunnel/signal_test.go

@@ -101,4 +101,3 @@ func TestWaitForShutdown(t *testing.T) {
 	assert.True(t, contextCancelled)
 	assert.True(t, time.Now().Sub(startTime) < time.Second) // check that wait ended early
 }
-

cmd/cloudflared/tunnel/subcommand_context.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
+	"path/filepath"
 	"strings"
 
 	"github.com/google/uuid"
@@ -147,7 +148,7 @@ func (sc *subcommandContext) readTunnelCredentials(credFinder CredFinder) (conne
 	return credentials, nil
 }
 
-func (sc *subcommandContext) create(name string, credentialsOutputPath string) (*tunnelstore.Tunnel, error) {
+func (sc *subcommandContext) create(name string, credentialsFilePath string) (*tunnelstore.Tunnel, error) {
 	client, err := sc.client()
 	if err != nil {
 		return nil, errors.Wrap(err, "couldn't create client to talk to Argo Tunnel backend")
@@ -173,27 +174,40 @@ func (sc *subcommandContext) create(name string, credentialsOutputPath string) (
 		TunnelID:     tunnel.ID,
 		TunnelName:   name,
 	}
-	filePath, writeFileErr := writeTunnelCredentials(credential.certPath, credentialsOutputPath, &tunnelCredentials)
+	usedCertPath := false
+	if credentialsFilePath == "" {
+		originCertDir := filepath.Dir(credential.certPath)
+		credentialsFilePath, err = tunnelFilePath(tunnelCredentials.TunnelID, originCertDir)
+		if err != nil {
+			return nil, err
+		}
+		usedCertPath = true
+	}
+	writeFileErr := writeTunnelCredentials(credentialsFilePath, &tunnelCredentials)
 	if writeFileErr != nil {
 		var errorLines []string
-		errorLines = append(errorLines, fmt.Sprintf("Your tunnel '%v' was created with ID %v. However, cloudflared couldn't write to the tunnel credentials file at %v.json.", tunnel.Name, tunnel.ID, tunnel.ID))
+		errorLines = append(errorLines, fmt.Sprintf("Your tunnel '%v' was created with ID %v. However, cloudflared couldn't write tunnel credentials to %s.", tunnel.Name, tunnel.ID, credentialsFilePath))
 		errorLines = append(errorLines, fmt.Sprintf("The file-writing error is: %v", writeFileErr))
 		if deleteErr := client.DeleteTunnel(tunnel.ID); deleteErr != nil {
 			errorLines = append(errorLines, fmt.Sprintf("Cloudflared tried to delete the tunnel for you, but encountered an error. You should use `cloudflared tunnel delete %v` to delete the tunnel yourself, because the tunnel can't be run without the tunnelfile.", tunnel.ID))
 			errorLines = append(errorLines, fmt.Sprintf("The delete tunnel error is: %v", deleteErr))
 		} else {
-			errorLines = append(errorLines, fmt.Sprintf("The tunnel was deleted, because the tunnel can't be run without the tunnelfile"))
+			errorLines = append(errorLines, fmt.Sprintf("The tunnel was deleted, because the tunnel can't be run without the credentials file"))
 		}
 		errorMsg := strings.Join(errorLines, "\n")
 		return nil, errors.New(errorMsg)
 	}
-	sc.log.Info().Msgf("Tunnel credentials written to %v. cloudflared chose this file based on where your origin certificate was found. Keep this file secret. To revoke these credentials, delete the tunnel.", filePath)
 
 	if outputFormat := sc.c.String(outputFormatFlag.Name); outputFormat != "" {
 		return nil, renderOutput(outputFormat, &tunnel)
 	}
 
-	sc.log.Info().Msgf("Created tunnel %s with id %s", tunnel.Name, tunnel.ID)
+	fmt.Printf("Tunnel credentials written to %v.", credentialsFilePath)
+	if usedCertPath {
+		fmt.Print(" cloudflared chose this file based on where your origin certificate was found.")
+	}
+	fmt.Println(" Keep this file secret. To revoke these credentials, delete the tunnel.")
+	fmt.Printf("\nCreated tunnel %s with id %s\n", tunnel.Name, tunnel.ID)
 	return tunnel, nil
 }
 
@@ -224,7 +238,7 @@ func (sc *subcommandContext) delete(tunnelIDs []uuid.UUID) error {
 			return fmt.Errorf("Tunnel %s has already been deleted", tunnel.ID)
 		}
 		if forceFlagSet {
-			if err := client.CleanupConnections(tunnel.ID); err != nil {
+			if err := client.CleanupConnections(tunnel.ID, tunnelstore.NewCleanupParams()); err != nil {
 				return errors.Wrapf(err, "Error cleaning up connections for tunnel %s", tunnel.ID)
 			}
 		}
@@ -276,13 +290,24 @@ func (sc *subcommandContext) run(tunnelID uuid.UUID) error {
 }
 
 func (sc *subcommandContext) cleanupConnections(tunnelIDs []uuid.UUID) error {
+	params := tunnelstore.NewCleanupParams()
+	extraLog := ""
+	if connector := sc.c.String("connector-id"); connector != "" {
+		connectorID, err := uuid.Parse(connector)
+		if err != nil {
+			return errors.Wrapf(err, "%s is not a valid client ID (must be a UUID)", connector)
+		}
+		params.ForClient(connectorID)
+		extraLog = fmt.Sprintf(" for connector-id %s", connectorID.String())
+	}
+
 	client, err := sc.client()
 	if err != nil {
 		return err
 	}
 	for _, tunnelID := range tunnelIDs {
-		sc.log.Info().Msgf("Cleanup connection for tunnel %s", tunnelID)
-		if err := client.CleanupConnections(tunnelID); err != nil {
+		sc.log.Info().Msgf("Cleanup connection for tunnel %s%s", tunnelID, extraLog)
+		if err := client.CleanupConnections(tunnelID, params); err != nil {
 			sc.log.Error().Msgf("Error cleaning up connections for tunnel %v, error :%v", tunnelID, err)
 		}
 	}
@@ -343,6 +368,12 @@ func (sc *subcommandContext) findID(input string) (uuid.UUID, error) {
 // one Tunnelstore API call.
 func (sc *subcommandContext) findIDs(inputs []string) ([]uuid.UUID, error) {
 
+	// Shortcut without Tunnelstore call if we find that all inputs are already UUIDs.
+	uuids, err := convertNamesToUuids(inputs, make(map[string]uuid.UUID))
+	if err == nil {
+		return uuids, nil
+	}
+
 	// First, look up all tunnels the user has
 	filter := tunnelstore.NewFilter()
 	filter.NoDeleted()
@@ -362,7 +393,10 @@ func findIDs(tunnels []*tunnelstore.Tunnel, inputs []string) ([]uuid.UUID, error
 		nameToID[tunnel.Name] = tunnel.ID
 	}
 
-	// For each input, try to find the tunnel ID.
+	return convertNamesToUuids(inputs, nameToID)
+}
+
+func convertNamesToUuids(inputs []string, nameToID map[string]uuid.UUID) ([]uuid.UUID, error) {
 	tunnelIDs := make([]uuid.UUID, len(inputs))
 	var badInputs []string
 	for i, input := range inputs {

cmd/cloudflared/tunnel/subcommand_context_teamnet.go

@@ -3,8 +3,9 @@ package tunnel
 import (
 	"net"
 
-	"github.com/cloudflare/cloudflared/teamnet"
 	"github.com/pkg/errors"
+
+	"github.com/cloudflare/cloudflared/teamnet"
 )
 
 const noClientMsg = "error while creating backend client"

cmd/cloudflared/tunnel/subcommand_context_test.go

@@ -4,16 +4,17 @@ import (
 	"encoding/base64"
 	"flag"
 	"fmt"
-	"github.com/rs/zerolog"
 	"reflect"
 	"testing"
 	"time"
 
-	"github.com/cloudflare/cloudflared/connection"
-	"github.com/cloudflare/cloudflared/tunnelstore"
 	"github.com/google/uuid"
 	"github.com/pkg/errors"
+	"github.com/rs/zerolog"
 	"github.com/urfave/cli/v2"
+
+	"github.com/cloudflare/cloudflared/connection"
+	"github.com/cloudflare/cloudflared/tunnelstore"
 )
 
 func Test_findIDs(t *testing.T) {
@@ -260,7 +261,7 @@ func (d *deleteMockTunnelStore) DeleteTunnel(tunnelID uuid.UUID) error {
 	return nil
 }
 
-func (d *deleteMockTunnelStore) CleanupConnections(tunnelID uuid.UUID) error {
+func (d *deleteMockTunnelStore) CleanupConnections(tunnelID uuid.UUID, _ *tunnelstore.CleanupParams) error {
 	tunnel, ok := d.mockTunnels[tunnelID]
 	if !ok {
 		return fmt.Errorf("Couldn't find tunnel: %v", tunnelID)

cmd/cloudflared/tunnel/subcommands.go

@@ -14,12 +14,12 @@ import (
 	"time"
 
 	"github.com/google/uuid"
-	"github.com/mitchellh/go-homedir"
+	homedir "github.com/mitchellh/go-homedir"
 	"github.com/pkg/errors"
 	"github.com/urfave/cli/v2"
 	"github.com/urfave/cli/v2/altsrc"
 	"golang.org/x/net/idna"
-	"gopkg.in/yaml.v2"
+	yaml "gopkg.in/yaml.v2"
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/updater"
@@ -29,9 +29,10 @@ import (
 )
 
 const (
-	allSortByOptions  = "name, id, createdAt, deletedAt, numConnections"
-	CredFileFlagAlias = "cred-file"
-	CredFileFlag      = "credentials-file"
+	allSortByOptions   = "name, id, createdAt, deletedAt, numConnections"
+	connsSortByOptions = "id, startedAt, numConnections, version"
+	CredFileFlagAlias  = "cred-file"
+	CredFileFlag       = "credentials-file"
 
 	LogFieldTunnelID = "tunnelID"
 )
@@ -64,11 +65,11 @@ var (
 		Aliases: []string{"rd"},
 		Usage:   "Include connections that have recently disconnected in the list",
 	}
-	outputFormatFlag = altsrc.NewStringFlag(&cli.StringFlag{
+	outputFormatFlag = &cli.StringFlag{
 		Name:    "output",
 		Aliases: []string{"o"},
 		Usage:   "Render output using given `FORMAT`. Valid options are 'json' or 'yaml'",
-	})
+	}
 	sortByFlag = &cli.StringFlag{
 		Name:    "sort-by",
 		Value:   "name",
@@ -93,13 +94,14 @@ var (
 		Aliases: []string{"F"},
 		Usage:   "Opt into various features that are still being developed or tested.",
 	})
-	credentialsFileFlag = altsrc.NewStringFlag(&cli.StringFlag{
+	credentialsFileFlagCLIOnly = &cli.StringFlag{
 		Name:    CredFileFlag,
 		Aliases: []string{CredFileFlagAlias},
 		Usage:   "Filepath at which to read/write the tunnel credentials",
 		EnvVars: []string{"TUNNEL_CRED_FILE"},
-	})
-	forceDeleteFlag = &cli.BoolFlag{
+	}
+	credentialsFileFlag = altsrc.NewStringFlag(credentialsFileFlagCLIOnly)
+	forceDeleteFlag     = &cli.BoolFlag{
 		Name:    "force",
 		Aliases: []string{"f"},
 		Usage: "Cleans up any stale connections before the tunnel is deleted. cloudflared will not " +
@@ -114,12 +116,29 @@ var (
 		EnvVars: []string{"TUNNEL_TRANSPORT_PROTOCOL"},
 		Hidden:  true,
 	})
+	sortInfoByFlag = &cli.StringFlag{
+		Name:    "sort-by",
+		Value:   "createdAt",
+		Usage:   fmt.Sprintf("Sorts the list of connections of a tunnel by the given field. Valid options are {%s}", connsSortByOptions),
+		EnvVars: []string{"TUNNEL_INFO_SORT_BY"},
+	}
+	invertInfoSortFlag = &cli.BoolFlag{
+		Name:    "invert-sort",
+		Usage:   "Inverts the sort order of the tunnel info.",
+		EnvVars: []string{"TUNNEL_INFO_INVERT_SORT"},
+	}
+	cleanupClientFlag = &cli.StringFlag{
+		Name:    "connector-id",
+		Aliases: []string{"c"},
+		Usage:   `Constraints the cleanup to stop the connections of a single Connector (by its ID). You can find the various Connectors (and their IDs) currently connected to your tunnel via 'cloudflared tunnel info <name>'.`,
+		EnvVars: []string{"TUNNEL_CLEANUP_CONNECTOR"},
+	}
 )
 
 func buildCreateCommand() *cli.Command {
 	return &cli.Command{
 		Name:      "create",
-		Action:    cliutil.ErrorHandler(createCommand),
+		Action:    cliutil.ConfiguredAction(createCommand),
 		Usage:     "Create a new tunnel with given name",
 		UsageText: "cloudflared tunnel [tunnel command options] create [subcommand options] NAME",
 		Description: `Creates a tunnel, registers it with Cloudflare edge and generates credential file used to run this tunnel.
@@ -128,7 +147,7 @@ func buildCreateCommand() *cli.Command {
   For example, to create a tunnel named 'my-tunnel' run:
 
   $ cloudflared tunnel create my-tunnel`,
-		Flags:              []cli.Flag{outputFormatFlag, credentialsFileFlag},
+		Flags:              []cli.Flag{outputFormatFlag, credentialsFileFlagCLIOnly},
 		CustomHelpTemplate: commandHelpTemplate(),
 	}
 }
@@ -164,33 +183,26 @@ func tunnelFilePath(tunnelID uuid.UUID, directory string) (string, error) {
 	return homedir.Expand(filePath)
 }
 
-// If an `outputFile` is given, write the credentials there.
-// Otherwise, write it to the same directory as the originCert,
-// with the filename `<tunnel id>.json`.
-func writeTunnelCredentials(
-	originCertPath, outputFile string,
-	credentials *connection.Credentials,
-) (filePath string, err error) {
-	filePath = outputFile
-	if outputFile == "" {
-		originCertDir := filepath.Dir(originCertPath)
-		filePath, err = tunnelFilePath(credentials.TunnelID, originCertDir)
+// writeTunnelCredentials saves `credentials` as a JSON into `filePath`, only if
+// the file does not exist already
+func writeTunnelCredentials(filePath string, credentials *connection.Credentials) error {
+	if _, err := os.Stat(filePath); !os.IsNotExist(err) {
+		if err == nil {
+			return fmt.Errorf("%s already exists", filePath)
+		}
+		return err
 	}
-	if err != nil {
-		return "", err
-	}
-	// Write the name and ID to the file too
 	body, err := json.Marshal(credentials)
 	if err != nil {
-		return "", errors.Wrap(err, "Unable to marshal tunnel credentials to JSON")
+		return errors.Wrap(err, "Unable to marshal tunnel credentials to JSON")
 	}
-	return filePath, ioutil.WriteFile(filePath, body, 400)
+	return ioutil.WriteFile(filePath, body, 400)
 }
 
 func buildListCommand() *cli.Command {
 	return &cli.Command{
 		Name:        "list",
-		Action:      cliutil.ErrorHandler(listCommand),
+		Action:      cliutil.ConfiguredAction(listCommand),
 		Usage:       "List existing tunnels",
 		UsageText:   "cloudflared tunnel [tunnel command options] list [subcommand options]",
 		Description: "cloudflared tunnel list will display all active tunnels, their created time and associated connections. Use -d flag to include deleted tunnels. See the list of options to filter the list",
@@ -214,6 +226,9 @@ func listCommand(c *cli.Context) error {
 		return err
 	}
 
+	warningChecker := updater.StartWarningCheck(c)
+	defer warningChecker.LogWarningIfAny(sc.log)
+
 	filter := tunnelstore.NewFilter()
 	if !c.Bool("show-deleted") {
 		filter.NoDeleted()
@@ -232,9 +247,6 @@ func listCommand(c *cli.Context) error {
 		filter.ByTunnelID(tunnelID)
 	}
 
-	warningChecker := updater.StartWarningCheck(c)
-	defer warningChecker.LogWarningIfAny(sc.log)
-
 	tunnels, err := sc.list(filter)
 	if err != nil {
 		return err
@@ -284,17 +296,11 @@ func listCommand(c *cli.Context) error {
 }
 
 func formatAndPrintTunnelList(tunnels []*tunnelstore.Tunnel, showRecentlyDisconnected bool) {
-	const (
-		minWidth = 0
-		tabWidth = 8
-		padding  = 1
-		padChar  = ' '
-		flags    = 0
-	)
-
-	writer := tabwriter.NewWriter(os.Stdout, minWidth, tabWidth, padding, padChar, flags)
+	writer := tabWriter()
 	defer writer.Flush()
 
+	_, _ = fmt.Fprintln(writer, "You can obtain more detailed information for each tunnel with `cloudflared tunnel info <name/uuid>`")
+
 	// Print column headers with tabbed columns
 	_, _ = fmt.Fprintln(writer, "ID\tNAME\tCREATED\tCONNECTIONS\t")
 
@@ -336,14 +342,160 @@ func fmtConnections(connections []tunnelstore.Connection, showRecentlyDisconnect
 	return strings.Join(output, ", ")
 }
 
+func buildInfoCommand() *cli.Command {
+	return &cli.Command{
+		Name:        "info",
+		Action:      cliutil.ConfiguredAction(tunnelInfo),
+		Usage:       "List details about the active connectors for a tunnel",
+		UsageText:   "cloudflared tunnel [tunnel command options] info [subcommand options] [TUNNEL]",
+		Description: "cloudflared tunnel info displays details about the active connectors for a given tunnel (identified by name or uuid).",
+		Flags: []cli.Flag{
+			outputFormatFlag,
+			showRecentlyDisconnected,
+			sortInfoByFlag,
+			invertInfoSortFlag,
+		},
+		CustomHelpTemplate: commandHelpTemplate(),
+	}
+}
+
+func tunnelInfo(c *cli.Context) error {
+	sc, err := newSubcommandContext(c)
+	if err != nil {
+		return err
+	}
+
+	warningChecker := updater.StartWarningCheck(c)
+	defer warningChecker.LogWarningIfAny(sc.log)
+
+	if c.NArg() > 1 {
+		return cliutil.UsageError(`"cloudflared tunnel info" accepts only one argument, the ID or name of the tunnel to run.`)
+	}
+	tunnelID, err := sc.findID(c.Args().First())
+	if err != nil {
+		return errors.Wrap(err, "error parsing tunnel ID")
+	}
+
+	client, err := sc.client()
+	if err != nil {
+		return err
+	}
+
+	clients, err := client.ListActiveClients(tunnelID)
+	if err != nil {
+		return err
+	}
+
+	sortBy := c.String("sort-by")
+	invalidSortField := false
+	sort.Slice(clients, func(i, j int) bool {
+		cmp := func() bool {
+			switch sortBy {
+			case "id":
+				return clients[i].ID.String() < clients[j].ID.String()
+			case "createdAt":
+				return clients[i].RunAt.Unix() < clients[j].RunAt.Unix()
+			case "numConnections":
+				return len(clients[i].Connections) < len(clients[j].Connections)
+			case "version":
+				return clients[i].Version < clients[j].Version
+			default:
+				invalidSortField = true
+				return clients[i].RunAt.Unix() < clients[j].RunAt.Unix()
+			}
+		}()
+		if c.Bool("invert-sort") {
+			return !cmp
+		}
+		return cmp
+	})
+	if invalidSortField {
+		sc.log.Error().Msgf("%s is not a valid sort field. Valid sort fields are %s. Defaulting to 'name'.", sortBy, connsSortByOptions)
+	}
+
+	tunnel, err := getTunnel(sc, tunnelID)
+	if err != nil {
+		return err
+	}
+	info := Info{
+		tunnel.ID,
+		tunnel.Name,
+		tunnel.CreatedAt,
+		clients,
+	}
+
+	if outputFormat := c.String(outputFormatFlag.Name); outputFormat != "" {
+		return renderOutput(outputFormat, info)
+	}
+
+	if len(clients) > 0 {
+		formatAndPrintConnectionsList(info, c.Bool("show-recently-disconnected"))
+	} else {
+		fmt.Printf("Your tunnel %s does not have any active connection.\n", tunnelID)
+	}
+
+	return nil
+}
+
+func getTunnel(sc *subcommandContext, tunnelID uuid.UUID) (*tunnelstore.Tunnel, error) {
+	filter := tunnelstore.NewFilter()
+	filter.ByTunnelID(tunnelID)
+	tunnels, err := sc.list(filter)
+	if err != nil {
+		return nil, err
+	}
+	if len(tunnels) != 1 {
+		return nil, errors.Errorf("Expected to find a single tunnel with uuid %v but found %d tunnels.", tunnelID, len(tunnels))
+	}
+	return tunnels[0], nil
+}
+
+func formatAndPrintConnectionsList(tunnelInfo Info, showRecentlyDisconnected bool) {
+	writer := tabWriter()
+	defer writer.Flush()
+
+	_, _ = fmt.Fprintf(writer, "NAME:     %s\nID:       %s\nCREATED:  %s\n\n", tunnelInfo.Name, tunnelInfo.ID, tunnelInfo.CreatedAt)
+
+	_, _ = fmt.Fprintln(writer, "CONNECTOR ID\tCREATED\tARCHITECTURE\tVERSION\tORIGIN IP\tEDGE\t")
+	for _, c := range tunnelInfo.Connectors {
+		var originIp = ""
+		if len(c.Connections) > 0 {
+			originIp = c.Connections[0].OriginIP.String()
+		}
+		formattedStr := fmt.Sprintf(
+			"%s\t%s\t%s\t%s\t%s\t%s\t",
+			c.ID,
+			c.RunAt.Format(time.RFC3339),
+			c.Arch,
+			c.Version,
+			originIp,
+			fmtConnections(c.Connections, showRecentlyDisconnected),
+		)
+		_, _ = fmt.Fprintln(writer, formattedStr)
+	}
+}
+
+func tabWriter() *tabwriter.Writer {
+	const (
+		minWidth = 0
+		tabWidth = 8
+		padding  = 1
+		padChar  = ' '
+		flags    = 0
+	)
+
+	writer := tabwriter.NewWriter(os.Stdout, minWidth, tabWidth, padding, padChar, flags)
+	return writer
+}
+
 func buildDeleteCommand() *cli.Command {
 	return &cli.Command{
 		Name:               "delete",
-		Action:             cliutil.ErrorHandler(deleteCommand),
+		Action:             cliutil.ConfiguredAction(deleteCommand),
 		Usage:              "Delete existing tunnel by UUID or name",
 		UsageText:          "cloudflared tunnel [tunnel command options] delete [subcommand options] TUNNEL",
 		Description:        "cloudflared tunnel delete will delete tunnels with the given tunnel UUIDs or names. A tunnel cannot be deleted if it has active connections. To delete the tunnel unconditionally, use -f flag.",
-		Flags:              []cli.Flag{credentialsFileFlag, forceDeleteFlag},
+		Flags:              []cli.Flag{credentialsFileFlagCLIOnly, forceDeleteFlag},
 		CustomHelpTemplate: commandHelpTemplate(),
 	}
 }
@@ -392,8 +544,7 @@ func buildRunCommand() *cli.Command {
 	flags = append(flags, configureProxyFlags(false)...)
 	return &cli.Command{
 		Name:      "run",
-		Action:    cliutil.ErrorHandler(runCommand),
-		Before:    SetFlagsFromConfigFile,
+		Action:    cliutil.ConfiguredAction(runCommand),
 		Usage:     "Proxy a local web server by running the given tunnel",
 		UsageText: "cloudflared tunnel [tunnel command options] run [subcommand options] [TUNNEL]",
 		Description: `Runs the tunnel identified by name or UUUD, creating highly available connections
@@ -445,10 +596,11 @@ func runNamedTunnel(sc *subcommandContext, tunnelRef string) error {
 func buildCleanupCommand() *cli.Command {
 	return &cli.Command{
 		Name:               "cleanup",
-		Action:             cliutil.ErrorHandler(cleanupCommand),
+		Action:             cliutil.ConfiguredAction(cleanupCommand),
 		Usage:              "Cleanup tunnel connections",
 		UsageText:          "cloudflared tunnel [tunnel command options] cleanup [subcommand options] TUNNEL",
 		Description:        "Delete connections for tunnels with the given UUIDs or names.",
+		Flags:              []cli.Flag{cleanupClientFlag},
 		CustomHelpTemplate: commandHelpTemplate(),
 	}
 }
@@ -474,7 +626,7 @@ func cleanupCommand(c *cli.Context) error {
 func buildRouteCommand() *cli.Command {
 	return &cli.Command{
 		Name:      "route",
-		Action:    cliutil.ErrorHandler(routeCommand),
+		Action:    cliutil.ConfiguredAction(routeCommand),
 		Usage:     "Define which traffic routed from Cloudflare edge to this tunnel: requests to a DNS hostname, to a Cloudflare Load Balancer, or traffic originating from Cloudflare WARP clients",
 		UsageText: "cloudflared tunnel [tunnel command options] route [subcommand options] [dns TUNNEL HOSTNAME]|[lb TUNNEL HOSTNAME LB-POOL]|[ip NETWORK TUNNEL]",
 		Description: `The route command defines how Cloudflare will proxy requests to this tunnel.

cmd/cloudflared/tunnel/subcommands_test.go

@@ -5,10 +5,11 @@ import (
 	"path/filepath"
 	"testing"
 
-	"github.com/cloudflare/cloudflared/tunnelstore"
 	"github.com/google/uuid"
-	"github.com/mitchellh/go-homedir"
+	homedir "github.com/mitchellh/go-homedir"
 	"github.com/stretchr/testify/assert"
+
+	"github.com/cloudflare/cloudflared/tunnelstore"
 )
 
 func Test_fmtConnections(t *testing.T) {

cmd/cloudflared/tunnel/teamnet_subcommands.go

@@ -6,10 +6,11 @@ import (
 	"os"
 	"text/tabwriter"
 
+	"github.com/pkg/errors"
+
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/updater"
 	"github.com/cloudflare/cloudflared/teamnet"
-	"github.com/pkg/errors"
 
 	"github.com/urfave/cli/v2"
 )
@@ -25,7 +26,7 @@ Cloudflare WARP client. You can also build rules to determine who can reach cert
 		Subcommands: []*cli.Command{
 			{
 				Name:      "add",
-				Action:    cliutil.ErrorHandler(addRouteCommand),
+				Action:    cliutil.ConfiguredAction(addRouteCommand),
 				Usage:     "Add any new network to the routing table reachable via the tunnel",
 				UsageText: "cloudflared tunnel [--config FILEPATH] route ip add [CIDR] [TUNNEL] [COMMENT?]",
 				Description: `Adds any network route space (represented as a CIDR) to your routing table.
@@ -38,7 +39,7 @@ reachable from the tunnel.`,
 			{
 				Name:        "show",
 				Aliases:     []string{"list"},
-				Action:      cliutil.ErrorHandler(showRoutesCommand),
+				Action:      cliutil.ConfiguredAction(showRoutesCommand),
 				Usage:       "Show the routing table",
 				UsageText:   "cloudflared tunnel [--config FILEPATH] route ip show [flags]",
 				Description: `Shows your organization private routing table. You can use flags to filter the results.`,
@@ -46,7 +47,7 @@ reachable from the tunnel.`,
 			},
 			{
 				Name:      "delete",
-				Action:    cliutil.ErrorHandler(deleteRouteCommand),
+				Action:    cliutil.ConfiguredAction(deleteRouteCommand),
 				Usage:     "Delete a row from your organization's private routing table",
 				UsageText: "cloudflared tunnel [--config FILEPATH] route ip delete [CIDR]",
 				Description: `Deletes the row for a given CIDR from your routing table. That portion
@@ -54,7 +55,7 @@ of your network will no longer be reachable by the WARP clients.`,
 			},
 			{
 				Name:      "get",
-				Action:    cliutil.ErrorHandler(getRouteByIPCommand),
+				Action:    cliutil.ConfiguredAction(getRouteByIPCommand),
 				Usage:     "Check which row of the routing table matches a given IP.",
 				UsageText: "cloudflared tunnel [--config FILEPATH] route ip get [IP]",
 				Description: `Checks which row of the routing table will be used to proxy a given IP.

cmd/cloudflared/ui/launch_ui.go

@@ -5,12 +5,12 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/cloudflare/cloudflared/connection"
-	"github.com/cloudflare/cloudflared/ingress"
-
 	"github.com/gdamore/tcell"
 	"github.com/rivo/tview"
 	"github.com/rs/zerolog"
+
+	"github.com/cloudflare/cloudflared/connection"
+	"github.com/cloudflare/cloudflared/ingress"
 )
 
 type connState struct {

cmd/cloudflared/windows_service.go

@@ -12,13 +12,15 @@ import (
 	"time"
 	"unsafe"
 
-	"github.com/cloudflare/cloudflared/logger"
-
+	"github.com/pkg/errors"
 	"github.com/urfave/cli/v2"
 	"golang.org/x/sys/windows"
 	"golang.org/x/sys/windows/svc"
 	"golang.org/x/sys/windows/svc/eventlog"
 	"golang.org/x/sys/windows/svc/mgr"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
+	"github.com/cloudflare/cloudflared/logger"
 )
 
 const (
@@ -48,13 +50,13 @@ func runApp(app *cli.App, graceShutdownC chan struct{}) {
 			{
 				Name:   "install",
 				Usage:  "Install Argo Tunnel as a Windows service",
-				Action: installWindowsService,
-						},
+				Action: cliutil.ConfiguredAction(installWindowsService),
+			},
 			{
 				Name:   "uninstall",
 				Usage:  "Uninstall the Argo Tunnel service",
-				Action: uninstallWindowsService,
-						},
+				Action: cliutil.ConfiguredAction(uninstallWindowsService),
+			},
 		},
 	})
 
@@ -177,35 +179,30 @@ func installWindowsService(c *cli.Context) error {
 	zeroLogger.Info().Msg("Installing Argo Tunnel Windows service")
 	exepath, err := os.Executable()
 	if err != nil {
-		zeroLogger.Err(err).Msg("Cannot find path name that start the process")
-		return err
+		return errors.Wrap(err, "Cannot find path name that start the process")
 	}
 	m, err := mgr.Connect()
 	if err != nil {
-		zeroLogger.Err(err).Msg("Cannot establish a connection to the service control manager")
-		return err
+		return errors.Wrap(err, "Cannot establish a connection to the service control manager")
 	}
 	defer m.Disconnect()
 	s, err := m.OpenService(windowsServiceName)
 	log := zeroLogger.With().Str(LogFieldWindowsServiceName, windowsServiceName).Logger()
 	if err == nil {
 		s.Close()
-		log.Err(err).Msg("service already exists")
-		return fmt.Errorf("service %s already exists", windowsServiceName)
+		return fmt.Errorf("Service %s already exists", windowsServiceName)
 	}
 	config := mgr.Config{StartType: mgr.StartAutomatic, DisplayName: windowsServiceDescription}
 	s, err = m.CreateService(windowsServiceName, exepath, config)
 	if err != nil {
-		log.Error().Msg("Cannot install service")
-		return err
+		return errors.Wrap(err, "Cannot install service")
 	}
 	defer s.Close()
 	log.Info().Msg("Argo Tunnel agent service is installed")
 	err = eventlog.InstallAsEventCreate(windowsServiceName, eventlog.Error|eventlog.Warning|eventlog.Info)
 	if err != nil {
 		s.Delete()
-		log.Err(err).Msg("Cannot install event logger")
-		return fmt.Errorf("SetupEventLogSource() failed: %s", err)
+		return errors.Wrap(err, "Cannot install event logger")
 	}
 	err = configRecoveryOption(s.Handle)
 	if err != nil {
@@ -223,26 +220,22 @@ func uninstallWindowsService(c *cli.Context) error {
 	log.Info().Msg("Uninstalling Argo Tunnel Windows Service")
 	m, err := mgr.Connect()
 	if err != nil {
-		log.Error().Msg("Cannot establish a connection to the service control manager")
-		return err
+		return errors.Wrap(err, "Cannot establish a connection to the service control manager")
 	}
 	defer m.Disconnect()
 	s, err := m.OpenService(windowsServiceName)
 	if err != nil {
-		log.Error().Msg("service is not installed")
-		return fmt.Errorf("service %s is not installed", windowsServiceName)
+		return fmt.Errorf("Service %s is not installed", windowsServiceName)
 	}
 	defer s.Close()
 	err = s.Delete()
 	if err != nil {
-		log.Error().Msg("Cannot delete service")
-		return err
+		return errors.Wrap(err, "Cannot delete service")
 	}
 	log.Info().Msg("Argo Tunnel agent service is uninstalled")
 	err = eventlog.Remove(windowsServiceName)
 	if err != nil {
-		log.Error().Msg("Cannot remove event logger")
-		return fmt.Errorf("RemoveEventLogSource() failed: %s", err)
+		return errors.Wrap(err, "Cannot remove event logger")
 	}
 	return nil
 }

component-tests/README.md

@@ -31,6 +31,9 @@ If you are using Visual Studio, follow https://code.visualstudio.com/docs/python
 to turn on formatter and https://marketplace.visualstudio.com/items?itemName=cbrevik.toggle-format-on-save
 to turn on format on save.
 
+6. If you have cloudflared running as a service on your machine, you can either stop the service or ignore the service tests
+via `--ignore test_service.py`
+
 # How to run
 Specify path to config file via env var `COMPONENT_TESTS_CONFIG`. This is required.
 ## All tests

component-tests/constants.py

@@ -1,3 +1,3 @@
 METRICS_PORT = 51000
-MAX_RETRIES = 3
-BACKOFF_SECS = 5
+MAX_RETRIES = 5
+BACKOFF_SECS = 7

component-tests/requirements.txt

@@ -1,3 +1,4 @@
+flaky==3.7.0
 pytest==6.2.2
 pyyaml==5.4.1
 requests==2.25.1

component-tests/test_logging.py

@@ -1,32 +1,88 @@
 #!/usr/bin/env python
 import json
 import os
-from util import start_cloudflared, wait_tunnel_ready, send_requests, LOGGER
+import subprocess
+
+from util import start_cloudflared, wait_tunnel_ready, send_requests
+
+# Rolling logger rotate log files after 1 MB
+rotate_after_size = 1000 * 1000
+default_log_file = "cloudflared.log"
+expect_message = "Starting tunnel"
+
+
+def assert_log_to_terminal(cloudflared):
+    stderr = cloudflared.stderr.read(200)
+    # cloudflared logs the following when it first starts
+    # 2021-03-10T12:30:39Z INF Starting tunnel tunnelID=<tunnel ID>
+    assert expect_message.encode() in stderr, f"{stderr} doesn't contain {expect_message}"
+
+
+def assert_log_in_file(file):
+    with open(file, "r") as f:
+        log = f.read(200)
+        # cloudflared logs the following when it first starts
+        # {"level":"info","tunnelID":"<tunnel ID>","time":"2021-03-10T12:21:13Z","message":"Starting tunnel"}
+        assert expect_message in log, f"{log} doesn't contain {expect_message}"
+
+
+def assert_json_log(file):
+    def assert_in_json(j, key):
+        assert key in j, f"{key} is not in j"
+
+    with open(file, "r") as f:
+        line = f.readline()
+        json_log = json.loads(line)
+        assert_in_json(json_log, "level")
+        assert_in_json(json_log, "time")
+        assert_in_json(json_log, "message")
+
+
+def assert_log_to_dir(config, log_dir):
+    max_batches = 3
+    batch_requests = 1000
+    for _ in range(max_batches):
+        send_requests(config.get_url(),
+                      batch_requests, require_ok=False)
+        files = os.listdir(log_dir)
+        if len(files) == 2:
+            current_log_file_index = files.index(default_log_file)
+            current_file = log_dir / files[current_log_file_index]
+            stats = os.stat(current_file)
+            assert stats.st_size > 0
+            assert_json_log(current_file)
+
+            # One file is the current log file, the other is the rotated log file
+            rotated_log_file_index = 0 if current_log_file_index == 1 else 1
+            rotated_file = log_dir / files[rotated_log_file_index]
+            stats = os.stat(rotated_file)
+            assert stats.st_size > rotate_after_size
+            assert_log_in_file(rotated_file)
+            assert_json_log(current_file)
+            return
+
+    raise Exception(
+        f"Log file isn't rotated after sending {max_batches * batch_requests} requests")
 
 
 class TestLogging:
-    # Rolling logger rotate log files after 1 MB
-    rotate_after_size = 1000 * 1000
-    default_log_file = "cloudflared.log"
-    expect_message = "Starting tunnel"
-
     def test_logging_to_terminal(self, tmp_path, component_tests_config):
         config = component_tests_config()
         with start_cloudflared(tmp_path, config, new_process=True) as cloudflared:
-            wait_tunnel_ready()
-            self.assert_log_to_terminal(cloudflared)
+            wait_tunnel_ready(tunnel_url=config.get_url())
+            assert_log_to_terminal(cloudflared)
 
     def test_logging_to_file(self, tmp_path, component_tests_config):
-        log_file = tmp_path / self.default_log_file
+        log_file = tmp_path / default_log_file
         extra_config = {
             # Convert from pathlib.Path to str
             "logfile": str(log_file),
         }
         config = component_tests_config(extra_config)
         with start_cloudflared(tmp_path, config, new_process=True, capture_output=False):
-            wait_tunnel_ready()
-            self.assert_log_in_file(log_file)
-            self.assert_json_log(log_file)
+            wait_tunnel_ready(tunnel_url=config.get_url())
+            assert_log_in_file(log_file)
+            assert_json_log(log_file)
 
     def test_logging_to_dir(self, tmp_path, component_tests_config):
         log_dir = tmp_path / "logs"
@@ -37,56 +93,5 @@ class TestLogging:
         }
         config = component_tests_config(extra_config)
         with start_cloudflared(tmp_path, config, new_process=True, capture_output=False):
-            wait_tunnel_ready()
-            self.assert_log_to_dir(config, log_dir)
-
-    def assert_log_to_terminal(self, cloudflared):
-        stderr = cloudflared.stderr.read(200)
-        # cloudflared logs the following when it first starts
-        # 2021-03-10T12:30:39Z INF Starting tunnel tunnelID=<tunnel ID>
-        assert self.expect_message.encode(
-        ) in stderr, f"{stderr} doesn't contain {self.expect_message}"
-
-    def assert_log_in_file(self, file, expect_message=""):
-        with open(file, "r") as f:
-            log = f.read(200)
-            # cloudflared logs the following when it first starts
-            # {"level":"info","tunnelID":"<tunnel ID>","time":"2021-03-10T12:21:13Z","message":"Starting tunnel"}
-            assert self.expect_message in log, f"{log} doesn't contain {self.expect_message}"
-
-    def assert_json_log(self, file):
-        with open(file, "r") as f:
-            line = f.readline()
-            json_log = json.loads(line)
-            self.assert_in_json(json_log, "level")
-            self.assert_in_json(json_log, "time")
-            self.assert_in_json(json_log, "message")
-
-    def assert_in_json(self, j, key):
-        assert key in j, f"{key} is not in j"
-
-    def assert_log_to_dir(self, config, log_dir):
-        max_batches = 3
-        batch_requests = 1000
-        for _ in range(max_batches):
-            send_requests(config.get_url(),
-                          batch_requests, require_ok=False)
-            files = os.listdir(log_dir)
-            if len(files) == 2:
-                current_log_file_index = files.index(self.default_log_file)
-                current_file = log_dir / files[current_log_file_index]
-                stats = os.stat(current_file)
-                assert stats.st_size > 0
-                self.assert_json_log(current_file)
-
-                # One file is the current log file, the other is the rotated log file
-                rotated_log_file_index = 0 if current_log_file_index == 1 else 1
-                rotated_file = log_dir / files[rotated_log_file_index]
-                stats = os.stat(rotated_file)
-                assert stats.st_size > self.rotate_after_size
-                self.assert_log_in_file(rotated_file)
-                self.assert_json_log(current_file)
-                return
-
-        raise Exception(
-            f"Log file isn't rotated after sending {max_batches * batch_requests} requests")
+            wait_tunnel_ready(tunnel_url=config.get_url())
+            assert_log_to_dir(config, log_dir)

component-tests/test_reconnect.py

@@ -1,12 +1,13 @@
 #!/usr/bin/env python
 import copy
-
-from retrying import retry
 from time import sleep
 
-from util import start_cloudflared, wait_tunnel_ready, check_tunnel_not_ready, send_requests
+from flaky import flaky
+
+from util import start_cloudflared, wait_tunnel_ready, check_tunnel_not_connected
 
 
+@flaky(max_runs=3, min_passes=1)
 class TestReconnect():
     default_ha_conns = 4
     default_reconnect_secs = 5
@@ -16,7 +17,7 @@ class TestReconnect():
 
     def test_named_reconnect(self, tmp_path, component_tests_config):
         config = component_tests_config(self.extra_config)
-        with start_cloudflared(tmp_path, config, new_process=True, allow_input=True) as cloudflared:
+        with start_cloudflared(tmp_path, config, new_process=True, allow_input=True, capture_output=False) as cloudflared:
             # Repeat the test multiple times because some issues only occur after multiple reconnects
             self.assert_reconnect(config, cloudflared, 5)
 
@@ -25,7 +26,7 @@ class TestReconnect():
         extra_config["hello-world"] = True
         config = component_tests_config(
             additional_config=extra_config, named_tunnel=False)
-        with start_cloudflared(tmp_path, config, cfd_args=[], new_process=True, allow_input=True) as cloudflared:
+        with start_cloudflared(tmp_path, config, cfd_args=[], new_process=True, allow_input=True, capture_output=False) as cloudflared:
             self.assert_reconnect(config, cloudflared, 1)
 
     def send_reconnect(self, cloudflared, secs):
@@ -35,16 +36,19 @@ class TestReconnect():
         cloudflared.stdin.flush()
 
     def assert_reconnect(self, config, cloudflared, repeat):
-        wait_tunnel_ready()
+        wait_tunnel_ready(tunnel_url=config.get_url())
         for _ in range(repeat):
             for i in range(self.default_ha_conns):
                 self.send_reconnect(cloudflared, self.default_reconnect_secs)
                 expect_connections = self.default_ha_conns-i-1
                 if expect_connections > 0:
-                    wait_tunnel_ready(expect_connections=expect_connections)
+                    # Don't check if tunnel returns 200 here because there is a race condition between wait_tunnel_ready
+                    # retrying to get 200 response and reconnecting
+                    wait_tunnel_ready(
+                        require_min_connections=expect_connections)
                 else:
-                    check_tunnel_not_ready()
+                    check_tunnel_not_connected()
 
             sleep(self.default_reconnect_secs + 10)
-            wait_tunnel_ready()
-            send_requests(config.get_url(), 1)
+            wait_tunnel_ready(tunnel_url=config.get_url(),
+                              require_min_connections=self.default_ha_conns)

component-tests/test_service.py

@@ -0,0 +1,139 @@
+#!/usr/bin/env python
+import os
+import platform
+import subprocess
+from contextlib import contextmanager
+from pathlib import Path
+
+import pytest
+
+import test_logging
+from util import start_cloudflared, wait_tunnel_ready
+
+
+def select_platform(plat):
+    return pytest.mark.skipif(
+        platform.system() != plat, reason=f"Only runs on {plat}")
+
+
+def default_config_dir():
+    return os.path.join(Path.home(), ".cloudflared")
+
+
+def default_config_file():
+    return os.path.join(default_config_dir(), "config.yml")
+
+
+class TestServiceMode:
+    @select_platform("Darwin")
+    @pytest.mark.skipif(os.path.exists(default_config_file()), reason=f"There is already a config file in default path")
+    def test_launchd_service_log_to_file(self, tmp_path, component_tests_config):
+        log_file = tmp_path / test_logging.default_log_file
+        additional_config = {
+            # On Darwin cloudflared service defaults to run classic tunnel command
+            "hello-world": True,
+            "logfile": str(log_file),
+        }
+        config = component_tests_config(additional_config=additional_config, named_tunnel=False)
+
+        def assert_log_file():
+            test_logging.assert_log_in_file(log_file)
+            test_logging.assert_json_log(log_file)
+
+        self.launchd_service_scenario(config, assert_log_file)
+
+    @select_platform("Darwin")
+    @pytest.mark.skipif(os.path.exists(default_config_file()), reason=f"There is already a config file in default path")
+    def test_launchd_service_rotating_log(self, tmp_path, component_tests_config):
+        log_dir = tmp_path / "logs"
+        additional_config = {
+            # On Darwin cloudflared service defaults to run classic tunnel command
+            "hello-world": True,
+            "loglevel": "debug",
+            "log-directory": str(log_dir),
+        }
+        config = component_tests_config(additional_config=additional_config, named_tunnel=False)
+
+        def assert_rotating_log():
+            test_logging.assert_log_to_dir(config, log_dir)
+
+        self.launchd_service_scenario(config, assert_rotating_log)
+
+    def launchd_service_scenario(self, config, extra_assertions):
+        with self.run_service(Path(default_config_dir()), config):
+            self.launchctl_cmd("list")
+            self.launchctl_cmd("start")
+            wait_tunnel_ready(tunnel_url=config.get_url())
+            extra_assertions()
+            self.launchctl_cmd("stop")
+
+        os.remove(default_config_file())
+        self.launchctl_cmd("list", success=False)
+
+    @select_platform("Linux")
+    @pytest.mark.skipif(os.path.exists("/etc/cloudflared/config.yml"),
+                        reason=f"There is already a config file in default path")
+    def test_sysv_service_log_to_file(self, tmp_path, component_tests_config):
+        log_file = tmp_path / test_logging.default_log_file
+        additional_config = {
+            "logfile": str(log_file),
+        }
+        config = component_tests_config(additional_config=additional_config)
+
+        def assert_log_file():
+            test_logging.assert_log_in_file(log_file)
+            test_logging.assert_json_log(log_file)
+
+        self.sysv_service_scenario(config, tmp_path, assert_log_file)
+
+    @select_platform("Linux")
+    @pytest.mark.skipif(os.path.exists("/etc/cloudflared/config.yml"),
+                        reason=f"There is already a config file in default path")
+    def test_sysv_service_rotating_log(self, tmp_path, component_tests_config):
+        log_dir = tmp_path / "logs"
+        additional_config = {
+            "loglevel": "debug",
+            "log-directory": str(log_dir),
+        }
+        config = component_tests_config(additional_config=additional_config)
+
+        def assert_rotating_log():
+            # We need the folder to have executable permissions for the "stat" command in the assertions to work.
+            subprocess.check_call(['sudo', 'chmod', 'o+x', log_dir])
+            test_logging.assert_log_to_dir(config, log_dir)
+
+        self.sysv_service_scenario(config, tmp_path, assert_rotating_log)
+
+    def sysv_service_scenario(self, config, tmp_path, extra_assertions):
+        with self.run_service(tmp_path, config, root=True):
+            self.sysv_cmd("start")
+            self.sysv_cmd("status")
+            wait_tunnel_ready(tunnel_url=config.get_url())
+            extra_assertions()
+            self.sysv_cmd("stop")
+
+        # Service install copies config file to /etc/cloudflared/config.yml
+        subprocess.run(["sudo", "rm", "/etc/cloudflared/config.yml"])
+        self.sysv_cmd("status", success=False)
+
+    @contextmanager
+    def run_service(self, tmp_path, config, root=False):
+        try:
+            service = start_cloudflared(
+                tmp_path, config, cfd_args=["service", "install"], cfd_pre_args=[], capture_output=False, root=root)
+            yield service
+        finally:
+            start_cloudflared(
+                tmp_path, config, cfd_args=["service", "uninstall"], cfd_pre_args=[], capture_output=False, root=root)
+
+    def launchctl_cmd(self, action, success=True):
+        cmd = subprocess.run(
+            ["launchctl", action, "com.cloudflare.cloudflared"], check=success)
+        if not success:
+            assert cmd.returncode != 0, f"Expect {cmd.args} to fail, but it succeed"
+
+    def sysv_cmd(self, action, success=True):
+        cmd = subprocess.run(
+            ["sudo", "service", "cloudflared", action], check=success)
+        if not success:
+            assert cmd.returncode != 0, f"Expect {cmd.args} to fail, but it succeed"

component-tests/test_termination.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python
+from contextlib import contextmanager
+import platform
+import signal
+import threading
+import time
+
+import pytest
+import requests
+
+from util import start_cloudflared, wait_tunnel_ready, check_tunnel_not_connected
+
+
+def supported_signals():
+    if platform.system() == "Windows":
+        return [signal.SIGTERM]
+    return [signal.SIGTERM, signal.SIGINT]
+
+
+class TestTermination():
+    grace_period = 5
+    timeout = 10
+    extra_config = {
+        "grace-period": f"{grace_period}s",
+    }
+    sse_endpoint = "/sse?freq=1s"
+
+    @pytest.mark.parametrize("signal", supported_signals())
+    def test_graceful_shutdown(self, tmp_path, component_tests_config, signal):
+        config = component_tests_config(self.extra_config)
+        with start_cloudflared(
+                tmp_path, config, new_process=True, capture_output=False) as cloudflared:
+            wait_tunnel_ready(tunnel_url=config.get_url())
+
+            connected = threading.Condition()
+            in_flight_req = threading.Thread(
+                target=self.stream_request, args=(config, connected, False, ))
+            in_flight_req.start()
+
+            with connected:
+                connected.wait(self.timeout)
+            # Send signal after the SSE connection is established
+            self.terminate_by_signal(cloudflared, signal)
+            self.wait_eyeball_thread(
+                in_flight_req, self.grace_period + self.timeout)
+
+    # test cloudflared terminates before grace period expires when all eyeball
+    # connections are drained
+    @pytest.mark.parametrize("signal", supported_signals())
+    def test_shutdown_once_no_connection(self, tmp_path, component_tests_config, signal):
+        config = component_tests_config(self.extra_config)
+        with start_cloudflared(
+                tmp_path, config, new_process=True, capture_output=False) as cloudflared:
+            wait_tunnel_ready(tunnel_url=config.get_url())
+
+            connected = threading.Condition()
+            in_flight_req = threading.Thread(
+                target=self.stream_request, args=(config, connected, True, ))
+            in_flight_req.start()
+
+            with connected:
+                connected.wait(self.timeout)
+            with self.within_grace_period():
+                # Send signal after the SSE connection is established
+                self.terminate_by_signal(cloudflared, signal)
+                self.wait_eyeball_thread(in_flight_req, self.grace_period)
+
+    @pytest.mark.parametrize("signal", supported_signals())
+    def test_no_connection_shutdown(self, tmp_path, component_tests_config, signal):
+        config = component_tests_config(self.extra_config)
+        with start_cloudflared(
+                tmp_path, config, new_process=True, capture_output=False) as cloudflared:
+            wait_tunnel_ready(tunnel_url=config.get_url())
+            with self.within_grace_period():
+                self.terminate_by_signal(cloudflared, signal)
+
+    def terminate_by_signal(self, cloudflared, sig):
+        cloudflared.send_signal(sig)
+        check_tunnel_not_connected()
+        cloudflared.wait()
+
+    def wait_eyeball_thread(self, thread, timeout):
+        thread.join(timeout)
+        assert thread.is_alive() == False, "eyeball thread is still alive"
+
+    # Using this context asserts logic within the context is executed within grace period
+    @contextmanager
+    def within_grace_period(self):
+        try:
+            start = time.time()
+            yield
+        finally:
+            duration = time.time() - start
+            assert duration < self.grace_period
+
+    def stream_request(self, config, connected, early_terminate):
+        expected_terminate_message = "502 Bad Gateway"
+        url = config.get_url() + self.sse_endpoint
+
+        with requests.get(url, timeout=5, stream=True) as resp:
+            with connected:
+                connected.notifyAll()
+            lines = 0
+            for line in resp.iter_lines():
+                if expected_terminate_message.encode() == line:
+                    break
+                lines += 1
+                if early_terminate and lines == 2:
+                    return
+            # /sse returns count followed by 2 new lines
+            assert lines >= (self.grace_period * 2)

component-tests/util.py

@@ -12,26 +12,34 @@ from constants import METRICS_PORT, MAX_RETRIES, BACKOFF_SECS
 LOGGER = logging.getLogger(__name__)
 
 
-def write_config(path, config):
-    config_path = path / "config.yaml"
+def write_config(directory, config):
+    config_path = directory / "config.yml"
     with open(config_path, 'w') as outfile:
         yaml.dump(config, outfile)
     return config_path
 
 
-def start_cloudflared(path, config, cfd_args=["run"], cfd_pre_args=["tunnel"], new_process=False, allow_input=False, capture_output=True):
-    config_path = write_config(path, config.full_config)
-    cmd = [config.cloudflared_binary]
-    cmd += cfd_pre_args
-    cmd += ["--config", config_path]
-    cmd += cfd_args
-    LOGGER.info(f"Run cmd {cmd} with config {config}")
+def start_cloudflared(directory, config, cfd_args=["run"], cfd_pre_args=["tunnel"], new_process=False, allow_input=False, capture_output=True, root=False):
+    config_path = write_config(directory, config.full_config)
+    cmd = cloudflared_cmd(config, config_path, cfd_args, cfd_pre_args, root)
     if new_process:
         return run_cloudflared_background(cmd, allow_input, capture_output)
     # By setting check=True, it will raise an exception if the process exits with non-zero exit code
     return subprocess.run(cmd, check=True, capture_output=capture_output)
 
 
+def cloudflared_cmd(config, config_path, cfd_args, cfd_pre_args, root):
+    cmd = []
+    if root:
+        cmd += ["sudo"]
+    cmd += [config.cloudflared_binary]
+    cmd += cfd_pre_args
+    cmd += ["--config", config_path]
+    cmd += cfd_args
+    LOGGER.info(f"Run cmd {cmd} with config {config}")
+    return cmd
+
+
 @contextmanager
 def run_cloudflared_background(cmd, allow_input, capture_output):
     output = subprocess.PIPE if capture_output else subprocess.DEVNULL
@@ -41,28 +49,36 @@ def run_cloudflared_background(cmd, allow_input, capture_output):
         yield cfd
     finally:
         cfd.terminate()
+        if capture_output:
+            LOGGER.info(
+                f"cloudflared log: {cfd.stderr.read()}")
 
 
 @retry(stop_max_attempt_number=MAX_RETRIES, wait_fixed=BACKOFF_SECS * 1000)
-def wait_tunnel_ready(expect_connections=4):
-    url = f'http://localhost:{METRICS_PORT}/ready'
+def wait_tunnel_ready(tunnel_url=None, require_min_connections=1):
+    metrics_url = f'http://localhost:{METRICS_PORT}/ready'
 
     with requests.Session() as s:
-        resp = send_request(s, url, True)
+        resp = send_request(s, metrics_url, True)
         assert resp.json()[
-            "readyConnections"] == expect_connections, f"Ready endpoint returned {resp.json()} but we expect {expect_connections} ready connections"
+            "readyConnections"] >= require_min_connections, f"Ready endpoint returned {resp.json()} but we expect at least {require_min_connections} connections"
+        if tunnel_url is not None:
+            send_request(s, tunnel_url, True)
 
 
-@retry(stop_max_attempt_number=MAX_RETRIES, wait_fixed=BACKOFF_SECS * 1000)
-def check_tunnel_not_ready():
+@retry(stop_max_attempt_number=MAX_RETRIES * BACKOFF_SECS, wait_fixed=1000)
+def check_tunnel_not_connected():
     url = f'http://localhost:{METRICS_PORT}/ready'
 
-    resp = requests.get(url, timeout=1)
-    assert resp.status_code == 503, f"Expect {url} returns 503, got {resp.status_code}"
+    try:
+        resp = requests.get(url, timeout=1)
+        assert resp.status_code == 503, f"Expect {url} returns 503, got {resp.status_code}"
+    # cloudflared might already terminate
+    except requests.exceptions.ConnectionError as e:
+        LOGGER.warning(f"Failed to connect to {url}, error: {e}")
+
 
 # In some cases we don't need to check response status, such as when sending batch requests to generate logs
-
-
 def send_requests(url, count, require_ok=True):
     errors = 0
     with requests.Session() as s:

config/configuration.go

@@ -9,11 +9,11 @@ import (
 	"runtime"
 	"time"
 
-	"github.com/mitchellh/go-homedir"
+	homedir "github.com/mitchellh/go-homedir"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
 	"github.com/urfave/cli/v2"
-	"gopkg.in/yaml.v2"
+	yaml "gopkg.in/yaml.v2"
 
 	"github.com/cloudflare/cloudflared/validation"
 )

config/configuration_test.go

@@ -5,7 +5,7 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/assert"
-	"gopkg.in/yaml.v2"
+	yaml "gopkg.in/yaml.v2"
 )
 
 func TestConfigFileSettings(t *testing.T) {

config/manager.go

@@ -4,11 +4,11 @@ import (
 	"io"
 	"os"
 
-	"github.com/cloudflare/cloudflared/watcher"
-
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
-	"gopkg.in/yaml.v2"
+	yaml "gopkg.in/yaml.v2"
+
+	"github.com/cloudflare/cloudflared/watcher"
 )
 
 // Notifier sends out config updates

config/manager_test.go

@@ -4,10 +4,10 @@ import (
 	"os"
 	"testing"
 
-	"github.com/cloudflare/cloudflared/watcher"
-
 	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/assert"
+
+	"github.com/cloudflare/cloudflared/watcher"
 )
 
 type mockNotifier struct {

connection/connection.go

@@ -8,8 +8,9 @@ import (
 	"strings"
 	"time"
 
-	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	"github.com/google/uuid"
+
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
 const LogFieldConnIndex = "connIndex"

connection/h2mux.go

@@ -7,13 +7,13 @@ import (
 	"net/http"
 	"time"
 
-	"github.com/cloudflare/cloudflared/h2mux"
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
-	"github.com/cloudflare/cloudflared/websocket"
-
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
 	"golang.org/x/sync/errgroup"
+
+	"github.com/cloudflare/cloudflared/h2mux"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/cloudflare/cloudflared/websocket"
 )
 
 const (

connection/h2mux_test.go

@@ -38,7 +38,7 @@ func newH2MuxConnection(t require.TestingT) (*h2muxConnection, *h2mux.Muxer) {
 				// we only expect RPC traffic in client->edge direction, provide minimal support for mocking
 				require.True(t, stream.IsRPCStream())
 				return stream.WriteHeaders([]h2mux.Header{
-					{Name:  ":status", Value: "200"},
+					{Name: ":status", Value: "200"},
 				})
 			}),
 		}

connection/http2.go

@@ -10,11 +10,11 @@ import (
 	"strings"
 	"sync"
 
-	"github.com/cloudflare/cloudflared/h2mux"
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
-
 	"github.com/rs/zerolog"
 	"golang.org/x/net/http2"
+
+	"github.com/cloudflare/cloudflared/h2mux"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
 const (

connection/http2_test.go

@@ -12,15 +12,14 @@ import (
 	"testing"
 	"time"
 
+	"github.com/gobwas/ws/wsutil"
+	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/net/http2"
 
 	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
-
-	"github.com/gobwas/ws/wsutil"
-	"github.com/rs/zerolog"
-	"github.com/stretchr/testify/require"
-	"golang.org/x/net/http2"
 )
 
 var (

connection/metrics.go

@@ -4,8 +4,9 @@ import (
 	"sync"
 	"time"
 
-	"github.com/cloudflare/cloudflared/h2mux"
 	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/cloudflare/cloudflared/h2mux"
 )
 
 const (
@@ -414,4 +415,4 @@ func newTunnelMetrics() *tunnelMetrics {
 		tunnelMetricsInternal.metrics = initTunnelMetrics()
 	})
 	return tunnelMetricsInternal.metrics
-}
+}

connection/observer.go

@@ -5,9 +5,9 @@ import (
 	"net/url"
 	"strings"
 
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
-
 	"github.com/rs/zerolog"
+
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
 const (

connection/observer_test.go

@@ -67,10 +67,9 @@ func TestObserverEventsDontBlock(t *testing.T) {
 	}
 }
 
-
 type eventCollectorSink struct {
 	observedEvents []Event
-	mu sync.Mutex
+	mu             sync.Mutex
 }
 
 func (s *eventCollectorSink) OnTunnelEvent(event Event) {
@@ -83,4 +82,4 @@ func (s *eventCollectorSink) assertSawEvent(t *testing.T, event Event) {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	assert.Contains(t, s.observedEvents, event)
-}
+}

connection/rpc.go

@@ -6,11 +6,11 @@ import (
 	"io"
 	"time"
 
-	"github.com/cloudflare/cloudflared/tunnelrpc"
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
-
 	"github.com/rs/zerolog"
 	"zombiezen.com/go/capnproto2/rpc"
+
+	"github.com/cloudflare/cloudflared/tunnelrpc"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
 type tunnelServerClient struct {

edgediscovery/edgediscovery.go

@@ -5,8 +5,9 @@ import (
 	"net"
 	"sync"
 
-	"github.com/cloudflare/cloudflared/edgediscovery/allregions"
 	"github.com/rs/zerolog"
+
+	"github.com/cloudflare/cloudflared/edgediscovery/allregions"
 )
 
 const (

fmt-check.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -e -o pipefail
+
+OUTPUT=$(for d in $(go list -mod=vendor -f '{{.Dir}}' -a ./... | fgrep -v tunnelrpc) ; do goimports -format-only -local github.com/cloudflare/cloudflared -d $d ; done)
+
+if [ -n "$OUTPUT" ] ; then
+  PAGER=$(which colordiff || echo cat)
+  echo
+  echo "Code formatting issues found, use 'goimports -format-only -local github.com/cloudflare/cloudflared' to correct them"
+  echo
+  echo "$OUTPUT" | $PAGER
+  exit 1
+fi

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/coredns/coredns v1.7.0
 	github.com/coreos/go-oidc v0.0.0-20171002155002-a93f71fdfe73
 	github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf
+	github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
 	github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 // indirect
 	github.com/facebookgo/freeport v0.0.0-20150612182905-d4adf43b75b9 // indirect
 	github.com/facebookgo/grace v0.0.0-20180706040059-75cf19382434
@@ -39,6 +40,7 @@ require (
 	github.com/prometheus/common v0.13.0 // indirect
 	github.com/rivo/tview v0.0.0-20200712113419-c65badfc3d92
 	github.com/rs/zerolog v1.20.0
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/stretchr/testify v1.6.0
 	github.com/urfave/cli/v2 v2.2.0
 	golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a
@@ -58,4 +60,4 @@ require (
 	zombiezen.com/go/capnproto2 v2.18.0+incompatible
 )
 
-replace github.com/urfave/cli/v2 => github.com/ipostelnik/cli/v2 v2.3.1-0.20210223162147-c8b4f97735e4
+replace github.com/urfave/cli/v2 => github.com/ipostelnik/cli/v2 v2.3.1-0.20210324024421-b6ea8234fe3d

go.sum

@@ -140,8 +140,9 @@ github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+
 github.com/coreos/license-bill-of-materials v0.0.0-20190913234955-13baff47494e/go.mod h1:4xMOusJ7xxc84WclVxKT8+lNfGYDwojOUC2OQNCwcj4=
 github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/cpu/goacmedns v0.0.1/go.mod h1:sesf/pNnCYwUevQEQfEwY0Y3DydlQWSGZbaMElOWxok=
-github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
+github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM=
+github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -347,8 +348,8 @@ github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJ
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo=
 github.com/infobloxopen/go-trees v0.0.0-20190313150506-2af4e13f9062/go.mod h1:PcNJqIlcX/dj3DTG/+QQnRvSgTMG6CLpRMjWcv4+J6w=
-github.com/ipostelnik/cli/v2 v2.3.1-0.20210223162147-c8b4f97735e4 h1:U2OxGKZHC5puhKZnGo6wWWjuWusJCM6J/S1eF7Vt+Uk=
-github.com/ipostelnik/cli/v2 v2.3.1-0.20210223162147-c8b4f97735e4/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
+github.com/ipostelnik/cli/v2 v2.3.1-0.20210324024421-b6ea8234fe3d h1:PRDnysJ9dF1vUMmEzBu6aHQeUluSQy4eWH3RsSSy/vI=
+github.com/ipostelnik/cli/v2 v2.3.1-0.20210324024421-b6ea8234fe3d/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
 github.com/jimstudt/http-authentication v0.0.0-20140401203705-3eca13d6893a/go.mod h1:wK6yTYYcgjHE1Z1QtXACPDjcFJyBskHEdagmnq3vsP8=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik=
@@ -544,14 +545,14 @@ github.com/rs/zerolog v1.20.0 h1:38k9hgtUBdxFwE34yS8rTHmHBa4eN16E4DJlv177LNs=
 github.com/rs/zerolog v1.20.0/go.mod h1:IzD0RJ65iWH0w97OQQebJEvTZYvsCUm9WVLWBQrJRjo=
 github.com/russross/blackfriday v0.0.0-20170610170232-067529f716f4 h1:S9YlS71UNJIyS61OqGAmLXv3w5zclSidN+qwr80XxKs=
 github.com/russross/blackfriday v0.0.0-20170610170232-067529f716f4/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
-github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/sacloud/libsacloud v1.26.1/go.mod h1:79ZwATmHLIFZIMd7sxA3LwzVy/B77uj3LDoToVTxDoQ=
 github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
-github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=

h2mux/h2mux_test.go

@@ -663,7 +663,8 @@ func AssertIfPipeReadable(t *testing.T, pipe io.ReadCloser) {
 		b := []byte{0}
 		n, err := pipe.Read(b)
 		if n > 0 {
-			t.Fatalf("read pipe was not empty")
+			t.Errorf("read pipe was not empty")
+			return
 		}
 		errC <- err
 	}()

h2mux/muxwriter.go

@@ -3,10 +3,11 @@ package h2mux
 import (
 	"bytes"
 	"encoding/binary"
-	"github.com/rs/zerolog"
 	"io"
 	"time"
 
+	"github.com/rs/zerolog"
+
 	"golang.org/x/net/http2"
 	"golang.org/x/net/http2/hpack"
 )

h2mux/shared_buffer_test.go

@@ -81,7 +81,8 @@ func TestSharedBufferConcurrentReadWrite(t *testing.T) {
 				expectedResult.Write(block[:blockSize])
 				n, err := b.Write(block[:blockSize])
 				if n != blockSize || err != nil {
-					t.Fatalf("write error: %d %s", n, err)
+					t.Errorf("write error: %d %s", n, err)
+					return
 				}
 			}
 		}
@@ -94,7 +95,8 @@ func TestSharedBufferConcurrentReadWrite(t *testing.T) {
 			for i := 0; i < 256; i++ {
 				n, err := io.ReadFull(b, block[:blockSize])
 				if n != blockSize || err != nil {
-					t.Fatalf("read error: %d %s", n, err)
+					t.Errorf("read error: %d %s", n, err)
+					return
 				}
 				actualResult.Write(block[:blockSize])
 			}

ingress/ingress.go

@@ -9,12 +9,12 @@ import (
 	"strings"
 	"sync"
 
-	"github.com/cloudflare/cloudflared/config"
-	"github.com/cloudflare/cloudflared/ipaccess"
-
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
 	"github.com/urfave/cli/v2"
+
+	"github.com/cloudflare/cloudflared/config"
+	"github.com/cloudflare/cloudflared/ipaccess"
 )
 
 var (

ingress/ingress_test.go

@@ -12,7 +12,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"github.com/urfave/cli/v2"
-	"gopkg.in/yaml.v2"
+	yaml "gopkg.in/yaml.v2"
 
 	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/ipaccess"

ingress/origin_connection.go

@@ -7,11 +7,12 @@ import (
 	"net"
 	"net/http"
 
+	gws "github.com/gorilla/websocket"
+	"github.com/rs/zerolog"
+
 	"github.com/cloudflare/cloudflared/ipaccess"
 	"github.com/cloudflare/cloudflared/socks"
 	"github.com/cloudflare/cloudflared/websocket"
-	gws "github.com/gorilla/websocket"
-	"github.com/rs/zerolog"
 )
 
 // OriginConnection is a way to stream to a service running on the user's origin.

ingress/origin_connection_test.go

@@ -13,14 +13,15 @@ import (
 	"testing"
 	"time"
 
-	"github.com/cloudflare/cloudflared/logger"
-	"github.com/cloudflare/cloudflared/socks"
 	"github.com/gobwas/ws/wsutil"
 	gorillaWS "github.com/gorilla/websocket"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"golang.org/x/net/proxy"
 	"golang.org/x/sync/errgroup"
+
+	"github.com/cloudflare/cloudflared/logger"
+	"github.com/cloudflare/cloudflared/socks"
 )
 
 const (

ingress/origin_proxy.go

@@ -7,9 +7,10 @@ import (
 	"net/url"
 	"strings"
 
+	"github.com/pkg/errors"
+
 	"github.com/cloudflare/cloudflared/h2mux"
 	"github.com/cloudflare/cloudflared/websocket"
-	"github.com/pkg/errors"
 )
 
 var (

ingress/origin_proxy_test.go

@@ -11,10 +11,11 @@ import (
 	"sync"
 	"testing"
 
-	"github.com/cloudflare/cloudflared/h2mux"
-	"github.com/cloudflare/cloudflared/websocket"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+
+	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/websocket"
 )
 
 // TestEstablishConnectionResponse ensures each implementation of StreamBasedOriginProxy returns

ingress/origin_request_config.go

@@ -3,10 +3,10 @@ package ingress
 import (
 	"time"
 
-	"github.com/cloudflare/cloudflared/ipaccess"
 	"github.com/urfave/cli/v2"
 
 	"github.com/cloudflare/cloudflared/config"
+	"github.com/cloudflare/cloudflared/ipaccess"
 	"github.com/cloudflare/cloudflared/tlsconfig"
 )
 

ingress/origin_request_config_test.go

@@ -5,10 +5,11 @@ import (
 	"testing"
 	"time"
 
-	"github.com/cloudflare/cloudflared/config"
 	"github.com/stretchr/testify/require"
 	"github.com/urfave/cli/v2"
-	"gopkg.in/yaml.v2"
+	yaml "gopkg.in/yaml.v2"
+
+	"github.com/cloudflare/cloudflared/config"
 )
 
 // Ensure that the nullable config from `config` package and the

ingress/origin_service.go

@@ -11,14 +11,15 @@ import (
 	"sync"
 	"time"
 
+	gws "github.com/gorilla/websocket"
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog"
+
 	"github.com/cloudflare/cloudflared/hello"
 	"github.com/cloudflare/cloudflared/ipaccess"
 	"github.com/cloudflare/cloudflared/socks"
 	"github.com/cloudflare/cloudflared/tlsconfig"
 	"github.com/cloudflare/cloudflared/websocket"
-	gws "github.com/gorilla/websocket"
-	"github.com/pkg/errors"
-	"github.com/rs/zerolog"
 )
 
 // originService is something a tunnel can proxy traffic to.

logger/create.go

@@ -64,6 +64,8 @@ func (t resilientMultiWriter) Write(p []byte) (n int, err error) {
 	return len(p), nil
 }
 
+var levelErrorLogged = false
+
 func newZerolog(loggerConfig *Config) *zerolog.Logger {
 	var writers []io.Writer
 
@@ -91,11 +93,15 @@ func newZerolog(loggerConfig *Config) *zerolog.Logger {
 
 	multi := resilientMultiWriter{writers}
 
-	level, err := zerolog.ParseLevel(loggerConfig.MinLevel)
-	if err != nil {
-		return fallbackLogger(err)
+	level, levelErr := zerolog.ParseLevel(loggerConfig.MinLevel)
+	if levelErr != nil {
+		level = zerolog.InfoLevel
 	}
 	log := zerolog.New(multi).With().Timestamp().Logger().Level(level)
+	if !levelErrorLogged && levelErr != nil {
+		log.Error().Msgf("Failed to parse log level %q, using %q instead", loggerConfig.MinLevel, level)
+		levelErrorLogged = true
+	}
 
 	return &log
 }
@@ -151,8 +157,8 @@ func Create(loggerConfig *Config) *zerolog.Logger {
 func createConsoleLogger(config ConsoleConfig) io.Writer {
 	consoleOut := os.Stderr
 	return zerolog.ConsoleWriter{
-		Out:     colorable.NewColorable(consoleOut),
-		NoColor: config.noColor || !term.IsTerminal(int(consoleOut.Fd())),
+		Out:        colorable.NewColorable(consoleOut),
+		NoColor:    config.noColor || !term.IsTerminal(int(consoleOut.Fd())),
 		TimeFormat: consoleTimeFormat,
 	}
 }

logger/create_test.go

@@ -1,11 +1,12 @@
 package logger
 
 import (
+	"io"
+	"testing"
+
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/assert"
-	"io"
-	"testing"
 )
 
 var writeCalls int
@@ -31,45 +32,45 @@ func TestResilientMultiWriter(t *testing.T) {
 		writers []io.Writer
 	}{
 		{
-			name:    "All valid writers",
+			name: "All valid writers",
 			writers: []io.Writer{
-				mockedWriter {
+				mockedWriter{
 					wantErr: false,
 				},
-				mockedWriter {
+				mockedWriter{
 					wantErr: false,
 				},
 			},
 		},
 		{
-			name:    "All invalid writers",
+			name: "All invalid writers",
 			writers: []io.Writer{
-				mockedWriter {
+				mockedWriter{
 					wantErr: true,
 				},
-				mockedWriter {
+				mockedWriter{
 					wantErr: true,
 				},
 			},
 		},
 		{
-			name:    "First invalid writer",
+			name: "First invalid writer",
 			writers: []io.Writer{
-				mockedWriter {
+				mockedWriter{
 					wantErr: true,
 				},
-				mockedWriter {
+				mockedWriter{
 					wantErr: false,
 				},
 			},
 		},
 		{
-			name:    "First valid writer",
+			name: "First valid writer",
 			writers: []io.Writer{
-				mockedWriter {
+				mockedWriter{
 					wantErr: false,
 				},
-				mockedWriter {
+				mockedWriter{
 					wantErr: true,
 				},
 			},
@@ -86,4 +87,4 @@ func TestResilientMultiWriter(t *testing.T) {
 		assert.Equal(t, len(writers), writeCalls)
 		writeCalls = 0
 	}
-}
+}

metrics/timer.go

@@ -1,47 +1,48 @@
 package metrics
 
 import (
-  "time"
-  "github.com/prometheus/client_golang/prometheus"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
 )
 
 // Timer assumes the metrics is partitioned by one label
 type Timer struct {
-  startTime   map[string]time.Time
-  metrics     *prometheus.HistogramVec
-  measureUnit time.Duration
-  labelKey    string
+	startTime   map[string]time.Time
+	metrics     *prometheus.HistogramVec
+	measureUnit time.Duration
+	labelKey    string
 }
 
 func NewTimer(metrics *prometheus.HistogramVec, unit time.Duration, labelKey string) *Timer {
-  return &Timer{
-    startTime:   make(map[string]time.Time),
-    measureUnit: unit,
-    metrics:     metrics,
-    labelKey:    labelKey,
-  }
+	return &Timer{
+		startTime:   make(map[string]time.Time),
+		measureUnit: unit,
+		metrics:     metrics,
+		labelKey:    labelKey,
+	}
 }
 
 func (i *Timer) Start(labelVal string) {
-  i.startTime[labelVal] = time.Now()
+	i.startTime[labelVal] = time.Now()
 }
 
 func (i *Timer) End(labelVal string) time.Duration {
-  if start, ok := i.startTime[labelVal]; ok {
-    return Latency(start, time.Now())
-  }
-  return 0
+	if start, ok := i.startTime[labelVal]; ok {
+		return Latency(start, time.Now())
+	}
+	return 0
 }
 
 func (i *Timer) Observe(measurement time.Duration, labelVal string) {
-  metricsLabels := prometheus.Labels{i.labelKey: labelVal}
-  i.metrics.With(metricsLabels).Observe(float64(measurement / i.measureUnit))
+	metricsLabels := prometheus.Labels{i.labelKey: labelVal}
+	i.metrics.With(metricsLabels).Observe(float64(measurement / i.measureUnit))
 }
 
 func (i *Timer) EndAndObserve(labelVal string) {
-  i.Observe(i.End(labelVal), labelVal)
+	i.Observe(i.End(labelVal), labelVal)
 }
 
 func Latency(startTime, endTime time.Time) time.Duration {
-  return endTime.Sub(startTime)
+	return endTime.Sub(startTime)
 }

metrics/timer_test.go

@@ -1,24 +1,24 @@
 package metrics
 
 import (
-  "testing"
-  "time"
+	"testing"
+	"time"
 
-  "github.com/prometheus/client_golang/prometheus"
-  "github.com/stretchr/testify/assert"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/stretchr/testify/assert"
 )
 
 func TestEnd(t *testing.T) {
-  m := prometheus.NewHistogramVec(
-    prometheus.HistogramOpts{
-      Namespace: "TestCallLatencyWithoutMeasurement",
-      Name:      "Latency",
-      Buckets: prometheus.LinearBuckets(0, 50, 100),
-    },
-    []string{"key"},
-  )
-  timer := NewTimer(m, time.Millisecond, "key")
-  assert.Equal(t, time.Duration(0), timer.End("dne"))
-  timer.Start("test")
-  assert.NotEqual(t, time.Duration(0), timer.End("test"))
+	m := prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Namespace: "TestCallLatencyWithoutMeasurement",
+			Name:      "Latency",
+			Buckets:   prometheus.LinearBuckets(0, 50, 100),
+		},
+		[]string{"key"},
+	)
+	timer := NewTimer(m, time.Millisecond, "key")
+	assert.Equal(t, time.Duration(0), timer.End("dne"))
+	timer.Start("test")
+	assert.NotEqual(t, time.Duration(0), timer.End("test"))
 }

origin/metrics.go

@@ -1,8 +1,9 @@
 package origin
 
 import (
-	"github.com/cloudflare/cloudflared/connection"
 	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/cloudflare/cloudflared/connection"
 )
 
 // Metrics uses connection.MetricsNamespace(aka cloudflared) as namespace and connection.TunnelSubsystem

origin/proxy_test.go

@@ -13,20 +13,19 @@ import (
 	"testing"
 	"time"
 
-	"github.com/cloudflare/cloudflared/logger"
+	"github.com/gobwas/ws/wsutil"
+	gorillaWS "github.com/gorilla/websocket"
+	"github.com/rs/zerolog"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/urfave/cli/v2"
 
 	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/hello"
 	"github.com/cloudflare/cloudflared/ingress"
+	"github.com/cloudflare/cloudflared/logger"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
-	gorillaWS "github.com/gorilla/websocket"
-	"github.com/urfave/cli/v2"
-
-	"github.com/gobwas/ws/wsutil"
-	"github.com/rs/zerolog"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 )
 
 var (

origin/reconnect.go

@@ -7,8 +7,9 @@ import (
 	"sync"
 	"time"
 
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	"github.com/prometheus/client_golang/prometheus"
+
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
 var (

origin/supervisor.go

@@ -7,14 +7,14 @@ import (
 	"net"
 	"time"
 
+	"github.com/google/uuid"
+	"github.com/rs/zerolog"
+
 	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/edgediscovery"
 	"github.com/cloudflare/cloudflared/h2mux"
 	"github.com/cloudflare/cloudflared/signal"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
-
-	"github.com/google/uuid"
-	"github.com/rs/zerolog"
 )
 
 const (
@@ -30,12 +30,6 @@ const (
 	refreshAuthMaxBackoff = 10
 	// Waiting time before retrying a failed 'Authenticate' connection
 	refreshAuthRetryDuration = time.Second * 10
-	// Maximum time to make an Authenticate RPC
-	authTokenTimeout = time.Second * 30
-)
-
-var (
-	errEventDigestUnset = errors.New("event digest unset")
 )
 
 // Supervisor manages non-declarative tunnels. Establishes TCP connections with the edge, and

origin/tunnel_test.go

@@ -4,10 +4,10 @@ import (
 	"testing"
 	"time"
 
-	"github.com/cloudflare/cloudflared/connection"
-
 	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/assert"
+
+	"github.com/cloudflare/cloudflared/connection"
 )
 
 type dynamicMockFetcher struct {

socks/request_handler.go

@@ -6,8 +6,9 @@ import (
 	"net"
 	"strings"
 
-	"github.com/cloudflare/cloudflared/ipaccess"
 	"github.com/rs/zerolog"
+
+	"github.com/cloudflare/cloudflared/ipaccess"
 )
 
 // RequestHandler is the functions needed to handle a SOCKS5 command

socks/request_handler_test.go

@@ -4,8 +4,9 @@ import (
 	"bytes"
 	"testing"
 
-	"github.com/cloudflare/cloudflared/ipaccess"
 	"github.com/stretchr/testify/assert"
+
+	"github.com/cloudflare/cloudflared/ipaccess"
 )
 
 func TestUnsupportedBind(t *testing.T) {

sshgen/sshgen.go

@@ -12,6 +12,7 @@ import (
 	"io"
 	"io/ioutil"
 	"net/http"
+	"net/url"
 	"time"
 
 	"github.com/coreos/go-oidc/jose"
@@ -51,8 +52,8 @@ type errorResponse struct {
 var mockRequest func(url, contentType string, body io.Reader) (*http.Response, error) = nil
 
 // GenerateShortLivedCertificate generates and stores a keypair for short lived certs
-func GenerateShortLivedCertificate(appInfo *cfpath.AppInfo, token string) error {
-	fullName, err := cfpath.GenerateAppTokenFilePathFromURL(appInfo.AppDomain, appInfo.AppAUD, keyName)
+func GenerateShortLivedCertificate(appURL *url.URL, token string) error {
+	fullName, err := cfpath.GenerateSSHCertFilePathFromURL(appURL, keyName)
 	if err != nil {
 		return err
 	}

sshgen/sshgen_test.go

@@ -9,7 +9,9 @@ import (
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
+	"net/url"
 	"os"
+	"strings"
 	"testing"
 	"time"
 
@@ -32,11 +34,12 @@ type signingArguments struct {
 }
 
 func TestCertGenSuccess(t *testing.T) {
-	appInfo := &cfpath.AppInfo{AppAUD: "abcd1234", AppDomain: "mySite.com"}
+	url, _ := url.Parse("https://cf-test-access.com/testpath")
 	token := tokenGenerator()
 
-	fullName, err := cfpath.GenerateAppTokenFilePathFromURL(appInfo.AppDomain, appInfo.AppAUD, keyName)
+	fullName, err := cfpath.GenerateSSHCertFilePathFromURL(url, keyName)
 	assert.NoError(t, err)
+	assert.True(t, strings.HasSuffix(fullName, "/cf-test-access.com-testpath-cf_key"))
 
 	pubKeyName := fullName + ".pub"
 	certKeyName := fullName + "-cert.pub"
@@ -65,7 +68,7 @@ func TestCertGenSuccess(t *testing.T) {
 		return w.Result(), nil
 	}
 
-	err = GenerateShortLivedCertificate(appInfo, token)
+	err = GenerateShortLivedCertificate(url, token)
 	assert.NoError(t, err)
 
 	exist, err := config.FileExists(fullName)

token/path.go

@@ -2,15 +2,26 @@ package token
 
 import (
 	"fmt"
+	"net/url"
 	"os"
 	"path/filepath"
 	"strings"
 
-	"github.com/mitchellh/go-homedir"
+	homedir "github.com/mitchellh/go-homedir"
 
 	"github.com/cloudflare/cloudflared/config"
 )
 
+// GenerateSSHCertFilePathFromURL will return a file path for creating short lived certificates
+func GenerateSSHCertFilePathFromURL(url *url.URL, suffix string) (string, error) {
+	configPath, err := getConfigPath()
+	if err != nil {
+		return "", err
+	}
+	name := strings.Replace(fmt.Sprintf("%s%s-%s", url.Hostname(), url.EscapedPath(), suffix), "/", "-", -1)
+	return filepath.Join(configPath, name), nil
+}
+
 // GenerateAppTokenFilePathFromURL will return a filepath for given Access org token
 func GenerateAppTokenFilePathFromURL(appDomain, aud string, suffix string) (string, error) {
 	configPath, err := getConfigPath()

token/shell.go

@@ -4,4 +4,3 @@ package token
 func OpenBrowser(url string) error {
 	return getBrowserCmd(url).Start()
 }
-

token/token.go

@@ -13,12 +13,12 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/cloudflare/cloudflared/config"
-	"github.com/cloudflare/cloudflared/origin"
-
 	"github.com/coreos/go-oidc/jose"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
+
+	"github.com/cloudflare/cloudflared/config"
+	"github.com/cloudflare/cloudflared/origin"
 )
 
 const (

token/token_test.go

@@ -1,4 +1,4 @@
-//+build linux
+// +build linux
 
 package token
 
@@ -17,7 +17,7 @@ func TestSignalHandler(t *testing.T) {
 	handlerRan := false
 	done := make(chan struct{})
 	timer := time.NewTimer(time.Second)
-	sigHandler.register(func(){
+	sigHandler.register(func() {
 		handlerRan = true
 		done <- struct{}{}
 	})
@@ -28,10 +28,10 @@ func TestSignalHandler(t *testing.T) {
 
 	// Blocks for up to one second to make sure the handler callback runs before the assert.
 	select {
-		case <- done:
-			assert.True(t, handlerRan)
-		case <- timer.C:
-			t.Fail()
+	case <-done:
+		assert.True(t, handlerRan)
+	case <-timer.C:
+		t.Fail()
 	}
 	sigHandler.deregister()
 }
@@ -40,15 +40,15 @@ func TestSignalHandlerClose(t *testing.T) {
 	sigHandler := signalHandler{signals: []os.Signal{syscall.SIGUSR1}}
 	done := make(chan struct{})
 	timer := time.NewTimer(time.Second)
-	sigHandler.register(func(){done <- struct{}{}})
+	sigHandler.register(func() { done <- struct{}{} })
 	sigHandler.deregister()
 
 	p, err := os.FindProcess(os.Getpid())
 	require.Nil(t, err)
 	p.Signal(syscall.SIGUSR1)
 	select {
-	case <- done:
+	case <-done:
 		t.Fail()
-	case <- timer.C:
+	case <-timer.C:
 	}
 }

tunnelrpc/pogs/auth_outcome.go

@@ -129,4 +129,4 @@ func (ao AuthUnknown) isAuthOutcome() {}
 
 func hoursToTime(hours uint8) time.Duration {
 	return time.Duration(hours) * time.Hour
-}
+}

tunnelrpc/pogs/auth_serialize.go

@@ -3,10 +3,10 @@ package pogs
 import (
 	"context"
 
-	"github.com/cloudflare/cloudflared/tunnelrpc"
-
 	"zombiezen.com/go/capnproto2/pogs"
 	"zombiezen.com/go/capnproto2/server"
+
+	"github.com/cloudflare/cloudflared/tunnelrpc"
 )
 
 func (i TunnelServer_PogsImpl) Authenticate(p tunnelrpc.TunnelServer_authenticate) error {

tunnelrpc/pogs/auth_test.go

@@ -6,9 +6,10 @@ import (
 	"testing"
 	"time"
 
-	"github.com/cloudflare/cloudflared/tunnelrpc"
 	"github.com/stretchr/testify/assert"
 	capnp "zombiezen.com/go/capnproto2"
+
+	"github.com/cloudflare/cloudflared/tunnelrpc"
 )
 
 // Ensure the AuthOutcome sum is correct

tunnelrpc/pogs/reconnect_tunnel.go

@@ -3,8 +3,9 @@ package pogs
 import (
 	"context"
 
-	"github.com/cloudflare/cloudflared/tunnelrpc"
 	"zombiezen.com/go/capnproto2/server"
+
+	"github.com/cloudflare/cloudflared/tunnelrpc"
 )
 
 func (i TunnelServer_PogsImpl) ReconnectTunnel(p tunnelrpc.TunnelServer_reconnectTunnel) error {

tunnelrpc/pogs/support_test.go

@@ -38,4 +38,3 @@ func (mockTunnelServerBase) Authenticate(ctx context.Context, originCert []byte,
 func (mockTunnelServerBase) ReconnectTunnel(ctx context.Context, jwt, eventDigest, connDigest []byte, hostname string, options *RegistrationOptions) (*TunnelRegistration, error) {
 	panic("unexpected call to ReconnectTunnel")
 }
-

tunnelrpc/pogs/tunnelrpc.go

@@ -4,12 +4,12 @@ import (
 	"context"
 	"fmt"
 
-	"github.com/cloudflare/cloudflared/tunnelrpc"
-
 	capnp "zombiezen.com/go/capnproto2"
 	"zombiezen.com/go/capnproto2/pogs"
 	"zombiezen.com/go/capnproto2/rpc"
 	"zombiezen.com/go/capnproto2/server"
+
+	"github.com/cloudflare/cloudflared/tunnelrpc"
 )
 
 const (

tunnelrpc/pogs/tunnelrpc_test.go

@@ -4,9 +4,10 @@ import (
 	"fmt"
 	"testing"
 
-	"github.com/cloudflare/cloudflared/tunnelrpc"
 	"github.com/stretchr/testify/assert"
 	capnp "zombiezen.com/go/capnproto2"
+
+	"github.com/cloudflare/cloudflared/tunnelrpc"
 )
 
 const (

tunnelstore/cleanup_params.go

@@ -0,0 +1,25 @@
+package tunnelstore
+
+import (
+	"net/url"
+
+	"github.com/google/uuid"
+)
+
+type CleanupParams struct {
+	queryParams url.Values
+}
+
+func NewCleanupParams() *CleanupParams {
+	return &CleanupParams{
+		queryParams: url.Values{},
+	}
+}
+
+func (cp *CleanupParams) ForClient(clientID uuid.UUID) {
+	cp.queryParams.Set("client_id", clientID.String())
+}
+
+func (cp CleanupParams) encode() string {
+	return cp.queryParams.Encode()
+}

tunnelstore/client.go

@@ -12,10 +12,11 @@ import (
 	"strings"
 	"time"
 
-	"github.com/cloudflare/cloudflared/teamnet"
 	"github.com/google/uuid"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
+
+	"github.com/cloudflare/cloudflared/teamnet"
 )
 
 const (
@@ -43,6 +44,17 @@ type Connection struct {
 	ColoName           string    `json:"colo_name"`
 	ID                 uuid.UUID `json:"id"`
 	IsPendingReconnect bool      `json:"is_pending_reconnect"`
+	OriginIP           net.IP    `json:"origin_ip"`
+	OpenedAt           time.Time `json:"opened_at"`
+}
+
+type ActiveClient struct {
+	ID          uuid.UUID    `json:"id"`
+	Features    []string     `json:"features"`
+	Version     string       `json:"version"`
+	Arch        string       `json:"arch"`
+	RunAt       time.Time    `json:"run_at"`
+	Connections []Connection `json:"conns"`
 }
 
 type Change = string
@@ -192,7 +204,8 @@ type Client interface {
 	GetTunnel(tunnelID uuid.UUID) (*Tunnel, error)
 	DeleteTunnel(tunnelID uuid.UUID) error
 	ListTunnels(filter *Filter) ([]*Tunnel, error)
-	CleanupConnections(tunnelID uuid.UUID) error
+	ListActiveClients(tunnelID uuid.UUID) ([]*ActiveClient, error)
+	CleanupConnections(tunnelID uuid.UUID, params *CleanupParams) error
 	RouteTunnel(tunnelID uuid.UUID, route Route) (RouteResult, error)
 
 	// Teamnet endpoints
@@ -336,9 +349,32 @@ func parseListTunnels(body io.ReadCloser) ([]*Tunnel, error) {
 	return tunnels, err
 }
 
-func (r *RESTClient) CleanupConnections(tunnelID uuid.UUID) error {
+func (r *RESTClient) ListActiveClients(tunnelID uuid.UUID) ([]*ActiveClient, error) {
 	endpoint := r.baseEndpoints.accountLevel
 	endpoint.Path = path.Join(endpoint.Path, fmt.Sprintf("%v/connections", tunnelID))
+	resp, err := r.sendRequest("GET", endpoint, nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "REST request failed")
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode == http.StatusOK {
+		return parseConnectionsDetails(resp.Body)
+	}
+
+	return nil, r.statusCodeToError("list connection details", resp)
+}
+
+func parseConnectionsDetails(reader io.Reader) ([]*ActiveClient, error) {
+	var clients []*ActiveClient
+	err := parseResponse(reader, &clients)
+	return clients, err
+}
+
+func (r *RESTClient) CleanupConnections(tunnelID uuid.UUID, params *CleanupParams) error {
+	endpoint := r.baseEndpoints.accountLevel
+	endpoint.RawQuery = params.encode()
+	endpoint.Path = path.Join(endpoint.Path, fmt.Sprintf("%v/connections", tunnelID))
 	resp, err := r.sendRequest("DELETE", endpoint, nil)
 	if err != nil {
 		return errors.Wrap(err, "REST request failed")

tunnelstore/client_teamnet.go

@@ -7,8 +7,9 @@ import (
 	"net/url"
 	"path"
 
-	"github.com/cloudflare/cloudflared/teamnet"
 	"github.com/pkg/errors"
+
+	"github.com/cloudflare/cloudflared/teamnet"
 )
 
 // ListRoutes calls the Tunnelstore GET endpoint for all routes under an account.