Release 2021.2.2

Also changed the socks test code so that it binds to localhost, so that
we don't get popups saying "would you like to allow socks.test to use
the network"

.dockerignore

@@ -0,0 +1 @@
+.git

.gitignore

@@ -18,3 +18,4 @@ cloudflared-x86-64*
 .DS_Store
 *-session.log
 ssh_server_tests/.env
+.cover

Dockerfile

@@ -1,7 +1,7 @@
 # use a builder image for building cloudflare
 ARG TARGET_GOOS
 ARG TARGET_GOARCH
-FROM golang:1.15.3 as builder
+FROM golang:1.15.7 as builder
 ENV GO111MODULE=on \
     CGO_ENABLED=0 \
     TARGET_GOOS=${TARGET_GOOS} \

Makefile

@@ -1,10 +1,19 @@
 VERSION       := $(shell git describe --tags --always --dirty="-dev" --match "[0-9][0-9][0-9][0-9].*.*")
-DATE          := $(shell date -u '+%Y-%m-%d-%H%M UTC')
-VERSION_FLAGS := -ldflags='-X "main.Version=$(VERSION)" -X "main.BuildTime=$(DATE)"'
 MSI_VERSION   := $(shell git tag -l --sort=v:refname | grep "w" | tail -1 | cut -c2-)
 #MSI_VERSION expects the format of the tag to be: (wX.X.X). Starts with the w character to not break cfsetup.
 #e.g. w3.0.1 or w4.2.10. It trims off the w character when creating the MSI.
 
+ifeq ($(FIPS), true)
+	GO_BUILD_TAGS := $(GO_BUILD_TAGS) fips
+endif
+
+ifneq ($(GO_BUILD_TAGS),)
+	GO_BUILD_TAGS := -tags $(GO_BUILD_TAGS)
+endif
+
+DATE          := $(shell date -u '+%Y-%m-%d-%H%M UTC')
+VERSION_FLAGS := -ldflags='-X "main.Version=$(VERSION)" -X "main.BuildTime=$(DATE)"'
+
 IMPORT_PATH   := github.com/cloudflare/cloudflared
 PACKAGE_DIR   := $(CURDIR)/packaging
 INSTALL_BINDIR := /usr/bin/
@@ -71,7 +80,7 @@ clean:
 
 .PHONY: cloudflared
 cloudflared: tunnel-deps
-	GOOS=$(TARGET_OS) GOARCH=$(TARGET_ARCH) go build -v -mod=vendor $(VERSION_FLAGS) $(IMPORT_PATH)/cmd/cloudflared
+	GOOS=$(TARGET_OS) GOARCH=$(TARGET_ARCH) go build -v -mod=vendor $(GO_BUILD_TAGS) $(VERSION_FLAGS) $(IMPORT_PATH)/cmd/cloudflared
 
 .PHONY: container
 container:
@@ -79,7 +88,13 @@ container:
 
 .PHONY: test
 test: vet
+ifndef CI
 	go test -v -mod=vendor -race $(VERSION_FLAGS) ./...
+else
+	@mkdir -p .cover
+	go test -v -mod=vendor -race $(VERSION_FLAGS) -coverprofile=".cover/c.out" ./...
+	go tool cover -html ".cover/c.out" -o .cover/all.html
+endif
 
 .PHONY: test-ssh-server
 test-ssh-server:
@@ -89,6 +104,7 @@ define publish_package
 	for HOST in $(CF_PKG_HOSTS); do \
 		ssh-keyscan -t rsa $$HOST >> ~/.ssh/known_hosts; \
 		scp -4 cloudflared*.$(1) cfsync@$$HOST:/state/cf-pkg/staging/$(2)/$(TARGET_PUBLIC_REPO)/cloudflared/; \
+		ssh cfsync@$$HOST 'chmod g+w /state/cf-pkg/staging/$(2)/$(TARGET_PUBLIC_REPO)/cloudflared/*.$(1)'; \
 	done
 endef
 
@@ -104,7 +120,7 @@ define build_package
 	mkdir -p $(PACKAGE_DIR)
 	cp cloudflared $(PACKAGE_DIR)/cloudflared
 	cat cloudflared_man_template | sed -e 's/\$${VERSION}/$(VERSION)/; s/\$${DATE}/$(DATE)/' > $(PACKAGE_DIR)/cloudflared.1
-	fakeroot fpm -C $(PACKAGE_DIR) -s dir -t $(1) --$(1)-compression bzip2 \
+	fakeroot fpm -C $(PACKAGE_DIR) -s dir -t $(1) \
 		--description 'Cloudflare Argo tunnel daemon' \
 		--vendor 'Cloudflare' \
 		--license 'Cloudflare Service Agreement' \
@@ -225,4 +241,4 @@ vet:
 
 .PHONY: msi
 msi: cloudflared
-	go-msi make --msi cloudflared.msi --version $(MSI_VERSION)
+	go-msi make --msi cloudflared.msi --version $(MSI_VERSION)

README.md

@@ -1,9 +1,43 @@
 # Argo Tunnel client
 
-Contains the command-line client and its libraries for Argo Tunnel, a tunneling daemon that proxies any local webserver through the Cloudflare network.
+Contains the command-line client for Argo Tunnel, a tunneling daemon that proxies any local webserver through the Cloudflare network. Extensive documentation can be found in the [Argo Tunnel section](https://developers.cloudflare.com/argo-tunnel/) of the Cloudflare Docs.
 
-## Getting started
+## Before you get started
 
-    go install github.com/cloudflare/cloudflared/cmd/cloudflared
+Before you use Argo Tunnel, you'll need to complete a few steps in the Cloudflare dashboard. The website you add to Cloudflare will be used to route traffic to your Tunnel.
+
+1. [Add a website to Cloudflare](https://support.cloudflare.com/hc/en-us/articles/201720164-Creating-a-Cloudflare-account-and-adding-a-website)
+2. [Change your domain nameservers to Cloudflare](https://support.cloudflare.com/hc/en-us/articles/205195708)
+
+## Installing `cloudflared`
+
+Downloads are available as standalone binaries, a Docker image, and Debian, RPM, and Homebrew packages. You can also find releases here on the `cloudflared` GitHub repository.
+
+* You can [install on macOS](https://developers.cloudflare.com/argo-tunnel/getting-started/installation#macos) via Homebrew or by downloading the [latest Darwin amd64 release](https://github.com/cloudflare/cloudflared/releases)
+* Binaries, Debian, and RPM packages for Linux [can be found here](https://developers.cloudflare.com/argo-tunnel/getting-started/installation#linux)
+* A Docker image of `cloudflared` is [available on DockerHub](https://hub.docker.com/r/cloudflare/cloudflared)
+* You can install on Windows machines with the [steps here](https://developers.cloudflare.com/argo-tunnel/getting-started/installation#windows)
 
 User documentation for Argo Tunnel can be found at https://developers.cloudflare.com/argo-tunnel/
+
+## Creating Tunnels and routing traffic
+
+Once installed, you can authenticate `cloudflared` into your Cloudflare account and begin creating Tunnels that serve traffic for hostnames in your account.
+
+* Create a Tunnel with [these instructions](https://developers.cloudflare.com/argo-tunnel/create-tunnel)
+* Route traffic to that Tunnel with [DNS records in Cloudflare](https://developers.cloudflare.com/argo-tunnel/routing-to-tunnel/dns) or with a [Cloudflare Load Balancer](https://developers.cloudflare.com/argo-tunnel/routing-to-tunnel/lb)
+
+## TryCloudflare
+
+Want to test Argo Tunnel before adding a website to Cloudflare? You can do so with TryCloudflare using the documentation [available here](https://developers.cloudflare.com/argo-tunnel/learning/trycloudflare).
+
+## Deprecated versions
+
+Cloudflare currently supports all versions of `cloudflared`. Starting on March 20, 2021, Cloudflare will no longer support versions released prior to 2020.5.1.
+
+All features available in versions released prior to 2020.5.1 are available in current versions. Breaking changes unrelated to feature availability may be introduced that will impact versions released prior to 2020.5.1. You can read more about upgrading `cloudflared` in our [developer documentation](https://developers.cloudflare.com/argo-tunnel/getting-started/installation#updating-cloudflared).
+
+| Version(s) | Deprecation status |
+|---|---|
+| 2020.5.1 and later | Supported |
+| Versions prior to 2020.5.1 | Will no longer be supported starting March 20, 2021 |

RELEASE_NOTES

@@ -1,3 +1,153 @@
+2021.2.2
+- 2021-02-04 TUN-3864: Users can choose where credentials file is written after creating a tunnel
+- 2021-02-04 TUN-3869: Improve reliability of graceful shutdown.
+- 2021-02-07 TUN-3878: Do not supply -tags when none are specified
+- 2021-02-04 TUN-3635: Send event when unregistering tunnel for gracful shutdown so /ready endpoint reports down status befoe connections finish handling pending requests.
+- 2021-02-08 TUN-3890: Code coverage for cloudflared in CI
+- 2021-02-09 AUTH-3375 exchangeOrgToken deleted cookie fix
+- 2020-11-18 Update error message to use login command
+
+2021.2.1
+- 2021-02-04 TUN-3858: Do not suffix cloudflared version with -fips
+
+2021.2.0
+- 2021-02-01 TUN-3837: Remove automation_email from cloudflared status page test
+- 2021-02-03 TUN-3848: Use transport logger for h2mux
+- 2021-02-03 TUN-3854: cloudflared tunnel list flags to sort output
+- 2021-01-21 TUN-3195: Don't colorize console logs when stderr is not a terminal
+- 2021-01-20 Fixed connection error handling by removing duplicated errors, standardizing on non-pointer error types
+- 2021-01-20 TUN-3118: Changed graceful shutdown to immediately unregister tunnel from the edge, keep the connection open until the edge drops it or grace period expires
+- 2021-01-25 TUN-3165: Add reference to Argo Tunnel documentation in the help output
+- 2021-01-25 TUN-3806: Use a .dockerignore
+- 2021-01-21 TUN-3795: Use RFC-3339 style date format for logs, produce timestamp in UTC
+- 2021-01-26 TUN-3795: Removed errant test
+- 2021-01-25 TUN-3792: Handle graceful shutdown correctly when running as a windows service. Only expose one shutdown channel globally, which now triggers the graceful shutdown sequence across all modes. Removed separate handling of zero-duration grace period, instead it's checked only when we need to wait for exit.
+- 2021-01-27 TUN-3811: Better error reporting on http2 connection termination. Registration errors from control loop are now propagated out of the connection server code. Unified error handling between h2mux and http2 connections so we log and retry errors the same way, regardless of underlying transport.
+- 2021-01-28 TUN-3830: Use Go 1.15.7
+- 2021-01-28 TUN-3826: Use go-fips when building cloudflared for linux/amd64
+- 2021-01-19 TUN-3777: Fix /ready endpoint for classic tunnels
+- 2021-01-19 TUN-3773: Add back pprof endpoints
+
+2021.1.5
+- 2021-01-15 TUN-3594: Log ingress response at debug level
+- 2021-01-15 TUN-3765: Fix doubly nested log output by `logfile` option
+- 2021-01-16 TUN-3767: Tolerate logging errors
+- 2021-01-17 TUN-3768: Reuse file loggers
+- 2021-01-14 TUN-3738: Refactor observer to avoid potential of blocking on tunnel notifications
+- 2021-01-15 TUN-3766: Print flags defined at all levels of command hierarchy, not just locally defined flags for a command. This fixes output of overriden settings for  subcommand.
+
+2021.1.4
+- 2021-01-14 TUN-3759: Single file logging output should always append
+
+2021.1.3
+- 2021-01-14 TUN-3756: File logging output must consider the directory
+- 2021-01-14 TUN-3757: Fix legacy Uint flags that are incorrectly handled by ufarve library
+
+2021.1.2
+- 2021-01-13 TUN-3747: Fix logging in Windows
+
+2021.1.1
+- 2021-01-13 TUN-3744: Fix compilation error in windows service
+
+2021.1.0
+- 2021-01-11 TUN-3670: Update Teamnet API gateway prefixes
+- 2021-01-13 TUN-3738: Consume UI events even when UI is disabled
+- 2021-01-06 TUN-3722: Teamnet API paths include /network
+- 2021-01-05 TUN-3688: Subcommand for users to check which route an IP proxies through
+- 2021-01-08 TUN-3691: Edit Teamnet help text
+- 2020-12-30 TUN-3706: Quit if any origin service fails to start
+- 2020-12-31 TUN-3708: Better info message about system root certpool on Windows
+- 2020-12-21 TUN-3669: Teamnet commands to add/show Teamnet routes.
+- 2020-12-29 TUN-3689: Delete routes via cloudflared CLI
+- 2020-12-28 TUN-3471: Add structured log context to logs
+- 2020-12-15 TUN-3650: Remove unused awsuploader package
+- 2020-12-03 Update to add deprecated version note (#271)
+- 2020-12-02 TUN-3472: Set up rolling logger with zerolog and lumberjack
+- 2020-12-08 TUN-3607: Set up single-file logger with zerolog
+- 2020-12-03 Update to add deprecated version note (#271)
+- 2020-11-25 TUN-3470: Replace in-house logger calls with zerolog
+
+2020.12.0
+- 2020-12-04 TUN-3599: improved delete if credentials isnt found.
+- 2020-12-04 TUN-3612: Upgrade to Go 1.15.6
+- 2020-11-30 TUN-3593: /ready endpoint for k8s readiness. Move tunnel events out of UI package, into connection package.
+- 2020-11-27 TUN-3594: Log response status at debug level
+
+2020.11.11
+- 2020-11-20 TUN-3578: cloudflared tunnel route dns should allow wildcard subdomains
+- 2020-11-21 EDGEPLAT-2958 remove deb-compression, defaulting to gzip
+- 2020-11-23 TUN-3581: Tunnels can be run by name using only --credentials-file, no origin cert necessary.
+- 2020-11-15 TUN-3561: Unified logger configuration
+- 2020-11-08 AUTH-3221: Saves org token to disk and uses it to refresh the app token
+
+2020.11.10
+- 2020-11-20 TUN-3562: Fix panic when using bastion mode ingress rule
+- 2020-11-20 EDGEPLAT-2958 build cloudflared for Bullseye
+
+2020.11.9
+- 2020-11-18 TUN-3557: Detect SSE if content-type starts with text/event-stream
+- 2020-11-18 TUN-3559: Share response meta header with other packages
+- 2020-11-18 DEVTOOLS-7936: Remove redundant chgrp from publish
+- 2020-11-18 TUN-3558: cloudflared allows empty config files
+- 2020-11-18 TUN-3544: Upgrade to Go 1.15.5
+
+2020.11.8
+- 2020-11-17 TUN-3555: Single origin service should default to localhost:8080
+
+2020.11.7
+- 2020-11-13 TUN-3514: Stop setting --is-autoupdated flag after autoupdate because it can break named tunnel running in k8s
+- 2020-11-15 TUN-3548, TUN-3547: Bastion mode can be specified as a service, doesn't require URL.
+- 2020-11-16 TUN-3549: Use a separate handler for each websocket proxy
+
+2020.11.6
+- 2020-11-14 TUN-3546: Fix panic in tlsconfig.LoadOriginCA
+
+2020.11.5
+- 2020-11-12 TUN-3540: Better copy in ingress rules error messages
+- 2020-11-12 DEVTOOLS-7936: Set permissions on public packages
+- 2020-11-13 TUN-3543: ProxyAddress not using default in single-origin mode
+
+2020.11.4
+- 2020-11-11 TUN-3534: Specific error message when credentials file is a .pem not .json
+- 2020-11-02 TUN-3500: Integrate replace h2mux by http2 work with multiple origin support
+- 2020-11-09 TUN-3514: Transport logger write to UI when UI is enabled
+- 2020-10-30 TUN-3490: Make sure OriginClient implementation doesn't write after Proxy return
+- 2020-10-20 TUN-3403: Unit test for origin/proxy to test serving HTTP and Websocket
+- 2020-10-23 TUN-3480: Support SSE with http2 connection, and add SSE handler to hello-world server
+- 2020-10-27 TUN-3489: Add unit tests to cover proxy logic in connection package of cloudflared
+- 2020-10-16 TUN-3467: Serialize cf-cloudflared-response-meta during package initialization using jsoniter
+- 2020-10-14 TUN-3456: New protocol option auto to automatically select between http2 and h2mux
+- 2020-10-14 TUN-3458: Upgrade to http2 when available, fallback to h2mux when we reach max retries
+- 2020-10-08 TUN-3449: Use flag to select transport protocol implementation
+- 2020-10-08 TUN-3462: Refactor cloudflared to separate origin from connection
+- 2020-09-21 TUN-3406: Proxy websocket requests over Go http2
+- 2020-09-25 TUN-3420: Establish control plane and send RPC over control plane
+- 2020-09-11 TUN-3400: Use Go HTTP2 library as transport to connect with the edge
+
+2020.11.3
+- 2020-11-11 TUN-3533: Set config for single origin ingress
+
+2020.11.2
+
+
+2020.11.1
+- 2020-11-10 TUN-3527: More specific error for invalid YAML/JSON
+- 2020-11-06 Update README.md (#256)
+
+2020.11.0
+- 2020-11-04 TUN-3484: OriginService that responds with configured HTTP status
+- 2020-11-05 TUN-3505: Response body for status code origin returns EOF on Read
+- 2020-11-04 TUN-3503: Matching ingress rule should not take port into account
+- 2020-11-05 TUN-3506: OriginService needs to set request host and scheme for websocket requests
+- 2020-11-09 TUN-3516: Better error message when parsing invalid YAML config
+- 2020-11-09 TUN-3522: ingress validate checks that the config file exists
+- 2020-11-09 TUN-3524: Don't ignore errors from app-level action handler (#248)
+- 2020-11-09 TUN-3461: Show all origin services in the UI
+- 2020-10-30 TUN-3494: Proceed to create tunnel if at least one edge address can be resolved
+- 2020-10-30 TUN-3492: Refactor OriginService, shrink its interface
+- 2020-10-22 TUN-3478: Increase download timeout to 60s
+- 2020-10-15 TUN-2640: Users can configure per-origin config. Unify single-rule CLI flow with multi-rule config file code.
+
 2020.10.2
 - 2020-10-21 Release 2020.10.1
 - 2020-10-21 AUTH-3185 fixed indention error

awsuploader/directory_upload_manager.go

@@ -1,106 +0,0 @@
-package awsuploader
-
-import (
-	"os"
-	"path/filepath"
-	"time"
-
-	"github.com/cloudflare/cloudflared/logger"
-)
-
-// DirectoryUploadManager is used to manage file uploads on an interval from a directory
-type DirectoryUploadManager struct {
-	logger        logger.Service
-	uploader      Uploader
-	rootDirectory string
-	sweepInterval time.Duration
-	ticker        *time.Ticker
-	shutdownC     chan struct{}
-	workQueue     chan string
-}
-
-// NewDirectoryUploadManager create a new DirectoryUploadManager
-// uploader is an Uploader to use as an actual uploading engine
-// directory is the directory to sweep for files to upload
-// sweepInterval is how often to iterate the directory and upload the files within
-func NewDirectoryUploadManager(logger logger.Service, uploader Uploader, directory string, sweepInterval time.Duration, shutdownC chan struct{}) *DirectoryUploadManager {
-	workerCount := 10
-	manager := &DirectoryUploadManager{
-		logger:        logger,
-		uploader:      uploader,
-		rootDirectory: directory,
-		sweepInterval: sweepInterval,
-		shutdownC:     shutdownC,
-		workQueue:     make(chan string, workerCount),
-	}
-
-	//start workers
-	for i := 0; i < workerCount; i++ {
-		go manager.worker()
-	}
-
-	return manager
-}
-
-// Upload a file using the uploader
-// This is useful for "out of band" uploads that need to be triggered immediately instead of waiting for the sweep
-func (m *DirectoryUploadManager) Upload(filepath string) error {
-	return m.uploader.Upload(filepath)
-}
-
-// Start the upload ticker to walk the directories
-func (m *DirectoryUploadManager) Start() {
-	m.ticker = time.NewTicker(m.sweepInterval)
-	go m.run()
-}
-
-func (m *DirectoryUploadManager) run() {
-	for {
-		select {
-		case <-m.shutdownC:
-			m.ticker.Stop()
-			return
-		case <-m.ticker.C:
-			m.sweep()
-		}
-	}
-}
-
-// sweep the directory and kick off uploads
-func (m *DirectoryUploadManager) sweep() {
-	filepath.Walk(m.rootDirectory, func(path string, info os.FileInfo, err error) error {
-		if err != nil || info.IsDir() {
-			return nil
-		}
-		//30 days ago
-		retentionTime := 30 * (time.Hour * 24)
-		checkTime := time.Now().Add(-time.Duration(retentionTime))
-
-		//delete the file it is stale
-		if info.ModTime().Before(checkTime) {
-			os.Remove(path)
-			return nil
-		}
-		//add the upload to the work queue
-		go func() {
-			m.workQueue <- path
-		}()
-		return nil
-	})
-}
-
-// worker handles upload requests
-func (m *DirectoryUploadManager) worker() {
-	for {
-		select {
-		case <-m.shutdownC:
-			return
-		case filepath := <-m.workQueue:
-			if err := m.Upload(filepath); err != nil {
-				m.logger.Errorf("Cannot upload file to s3 bucket: %s", err)
-			} else {
-				os.Remove(filepath)
-			}
-		}
-	}
-}

awsuploader/directory_upload_manager_test.go

@@ -1,137 +0,0 @@
-package awsuploader
-
-import (
-	"errors"
-	"io/ioutil"
-	"math/rand"
-	"os"
-	"path/filepath"
-	"testing"
-	"time"
-
-	"github.com/cloudflare/cloudflared/logger"
-)
-
-type MockUploader struct {
-	shouldFail bool
-}
-
-func (m *MockUploader) Upload(filepath string) error {
-	if m.shouldFail {
-		return errors.New("upload set to fail")
-	}
-	return nil
-}
-
-func NewMockUploader(shouldFail bool) Uploader {
-	return &MockUploader{shouldFail: shouldFail}
-}
-
-func getDirectoryPath(t *testing.T) string {
-	dir, err := os.Getwd()
-	if err != nil {
-		t.Fatal("couldn't create the test directory!", err)
-	}
-	return filepath.Join(dir, "uploads")
-}
-
-func setupTestDirectory(t *testing.T) string {
-	path := getDirectoryPath(t)
-	os.RemoveAll(path)
-	time.Sleep(100 * time.Millisecond) //short way to wait for the OS to delete the folder
-	err := os.MkdirAll(path, os.ModePerm)
-	if err != nil {
-		t.Fatal("couldn't create the test directory!", err)
-	}
-	return path
-}
-
-func createUploadManager(t *testing.T, shouldFailUpload bool) *DirectoryUploadManager {
-	rootDirectory := setupTestDirectory(t)
-	uploader := NewMockUploader(shouldFailUpload)
-	logger := logger.NewOutputWriter(logger.NewMockWriteManager())
-	shutdownC := make(chan struct{})
-	return NewDirectoryUploadManager(logger, uploader, rootDirectory, 1*time.Second, shutdownC)
-}
-
-func createFile(t *testing.T, fileName string) (*os.File, string) {
-	path := filepath.Join(getDirectoryPath(t), fileName)
-	f, err := os.Create(path)
-	if err != nil {
-		t.Fatal("upload to create file for sweep test", err)
-	}
-	return f, path
-}
-
-func TestUploadSuccess(t *testing.T) {
-	manager := createUploadManager(t, false)
-	path := filepath.Join(getDirectoryPath(t), "test_file")
-	if err := manager.Upload(path); err != nil {
-		t.Fatal("the upload request method failed", err)
-	}
-}
-
-func TestUploadFailure(t *testing.T) {
-	manager := createUploadManager(t, true)
-	path := filepath.Join(getDirectoryPath(t), "test_file")
-	if err := manager.Upload(path); err == nil {
-		t.Fatal("the upload request method should have failed and didn't", err)
-	}
-}
-
-func TestSweepSuccess(t *testing.T) {
-	manager := createUploadManager(t, false)
-	f, path := createFile(t, "test_file")
-	defer f.Close()
-
-	manager.Start()
-	time.Sleep(2 * time.Second)
-	if _, err := os.Stat(path); os.IsExist(err) {
-		//the file should have been deleted
-		t.Fatal("the manager failed to delete the file", err)
-	}
-}
-
-func TestSweepFailure(t *testing.T) {
-	manager := createUploadManager(t, true)
-	f, path := createFile(t, "test_file")
-	defer f.Close()
-
-	manager.Start()
-	time.Sleep(2 * time.Second)
-	_, serr := f.Stat()
-	if serr != nil {
-		//the file should still exist
-		os.Remove(path)
-		t.Fatal("the manager failed to delete the file", serr)
-	}
-}
-
-func TestHighLoad(t *testing.T) {
-	manager := createUploadManager(t, false)
-	for i := 0; i < 30; i++ {
-		f, _ := createFile(t, randomString(6))
-		defer f.Close()
-	}
-	manager.Start()
-	time.Sleep(4 * time.Second)
-
-	directory := getDirectoryPath(t)
-	files, err := ioutil.ReadDir(directory)
-	if err != nil || len(files) > 0 {
-		t.Fatalf("the manager failed to upload all the files: %s files left: %d", err, len(files))
-	}
-}
-
-// LowerCase [a-z]
-const randSet = "abcdefghijklmnopqrstuvwxyz"
-
-// String returns a string of length 'n' from a set of letters 'lset'
-func randomString(n int) string {
-	b := make([]byte, n)
-	lsetLen := len(randSet)
-	for i := range b {
-		b[i] = randSet[rand.Intn(lsetLen)]
-	}
-	return string(b)
-}

awsuploader/file_uploader.go

@@ -1,62 +0,0 @@
-package awsuploader
-
-import (
-	"context"
-	"os"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/credentials"
-	"github.com/aws/aws-sdk-go/aws/session"
-	"github.com/aws/aws-sdk-go/service/s3"
-)
-
-//FileUploader aws compliant bucket upload
-type FileUploader struct {
-	storage    *s3.S3
-	bucketName string
-	clientID   string
-	secretID   string
-}
-
-// NewFileUploader creates a new S3 compliant bucket uploader
-func NewFileUploader(bucketName, region, accessKeyID, secretID, token, s3Host string) (*FileUploader, error) {
-	sess, err := session.NewSession(&aws.Config{
-		Region:      aws.String(region),
-		Credentials: credentials.NewStaticCredentials(accessKeyID, secretID, token),
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	var storage *s3.S3
-	if s3Host != "" {
-		storage = s3.New(sess, &aws.Config{Endpoint: aws.String(s3Host)})
-	} else {
-		storage = s3.New(sess)
-	}
-
-	return &FileUploader{
-		storage:    storage,
-		bucketName: bucketName,
-	}, nil
-}
-
-// Upload a file to the bucket
-func (u *FileUploader) Upload(filepath string) error {
-	info, err := os.Stat(filepath)
-	if err != nil {
-		return err
-	}
-	file, err := os.Open(filepath)
-	if err != nil {
-		return err
-	}
-	defer file.Close()
-
-	_, serr := u.storage.PutObjectWithContext(context.Background(), &s3.PutObjectInput{
-		Bucket: aws.String(u.bucketName),
-		Key:    aws.String(info.Name()),
-		Body:   file,
-	})
-	return serr
-}

awsuploader/upload_manager.go

@@ -1,7 +0,0 @@
-package awsuploader
-
-// UploadManager is used to manage file uploads on an interval
-type UploadManager interface {
-	Upload(string) error
-	Start()
-}

awsuploader/uploader.go

@@ -1,7 +0,0 @@
-package awsuploader
-
-// Uploader the functions required to upload to a bucket
-type Uploader interface {
-	//Upload a file to the bucket
-	Upload(string) error
-}

carrier/carrier.go

@@ -12,10 +12,13 @@ import (
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/token"
 	"github.com/cloudflare/cloudflared/h2mux"
-	"github.com/cloudflare/cloudflared/logger"
+
 	"github.com/pkg/errors"
+	"github.com/rs/zerolog"
 )
 
+const LogFieldOriginURL = "originURL"
+
 type StartOptions struct {
 	OriginURL string
 	Headers   http.Header
@@ -49,7 +52,7 @@ func (c *StdinoutStream) Write(p []byte) (int, error) {
 // Helper to allow defering the response close with a check that the resp is not nil
 func closeRespBody(resp *http.Response) {
 	if resp != nil {
-		resp.Body.Close()
+		_ = resp.Body.Close()
 	}
 }
 
@@ -103,7 +106,7 @@ func Serve(remoteConn Connection, listener net.Listener, shutdownC <-chan struct
 // serveConnection handles connections for the Serve() call
 func serveConnection(remoteConn Connection, c net.Conn, options *StartOptions) {
 	defer c.Close()
-	remoteConn.ServeStream(options, c)
+	_ = remoteConn.ServeStream(options, c)
 }
 
 // IsAccessResponse checks the http Response to see if the url location
@@ -125,13 +128,13 @@ func IsAccessResponse(resp *http.Response) bool {
 }
 
 // BuildAccessRequest builds an HTTP request with the Access token set
-func BuildAccessRequest(options *StartOptions, logger logger.Service) (*http.Request, error) {
+func BuildAccessRequest(options *StartOptions, log *zerolog.Logger) (*http.Request, error) {
 	req, err := http.NewRequest(http.MethodGet, options.OriginURL, nil)
 	if err != nil {
 		return nil, err
 	}
 
-	token, err := token.FetchTokenWithRedirect(req.URL, logger)
+	token, err := token.FetchTokenWithRedirect(req.URL, log)
 	if err != nil {
 		return nil, err
 	}

carrier/carrier_test.go

@@ -9,8 +9,8 @@ import (
 	"sync"
 	"testing"
 
-	"github.com/cloudflare/cloudflared/logger"
 	ws "github.com/gorilla/websocket"
+	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -43,8 +43,8 @@ func (s *testStreamer) Write(p []byte) (int, error) {
 
 func TestStartClient(t *testing.T) {
 	message := "Good morning Austin! Time for another sunny day in the great state of Texas."
-	logger := logger.NewOutputWriter(logger.NewMockWriteManager())
-	wsConn := NewWSConnection(logger, false)
+	log := zerolog.Nop()
+	wsConn := NewWSConnection(&log, false)
 	ts := newTestWebSocketServer()
 	defer ts.Close()
 
@@ -55,10 +55,10 @@ func TestStartClient(t *testing.T) {
 	}
 	err := StartClient(wsConn, buf, options)
 	assert.NoError(t, err)
-	buf.Write([]byte(message))
+	_, _ = buf.Write([]byte(message))
 
 	readBuffer := make([]byte, len(message))
-	buf.Read(readBuffer)
+	_, _ = buf.Read(readBuffer)
 	assert.Equal(t, message, string(readBuffer))
 }
 
@@ -68,9 +68,9 @@ func TestStartServer(t *testing.T) {
 		t.Fatalf("Error starting listener: %v", err)
 	}
 	message := "Good morning Austin! Time for another sunny day in the great state of Texas."
-	logger := logger.NewOutputWriter(logger.NewMockWriteManager())
+	log := zerolog.Nop()
 	shutdownC := make(chan struct{})
-	wsConn := NewWSConnection(logger, false)
+	wsConn := NewWSConnection(&log, false)
 	ts := newTestWebSocketServer()
 	defer ts.Close()
 	options := &StartOptions{
@@ -86,10 +86,10 @@ func TestStartServer(t *testing.T) {
 	}()
 
 	conn, err := net.Dial("tcp", listener.Addr().String())
-	conn.Write([]byte(message))
+	_, _ = conn.Write([]byte(message))
 
 	readBuffer := make([]byte, len(message))
-	conn.Read(readBuffer)
+	_, _ = conn.Read(readBuffer)
 	assert.Equal(t, string(readBuffer), message)
 }
 

carrier/websocket.go

@@ -8,16 +8,17 @@ import (
 	"net/http/httputil"
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/token"
-	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/socks"
 	cfwebsocket "github.com/cloudflare/cloudflared/websocket"
+
 	"github.com/gorilla/websocket"
+	"github.com/rs/zerolog"
 )
 
 // Websocket is used to carry data via WS binary frames over the tunnel from client to the origin
 // This implements the functions for glider proxy (sock5) and the carrier interface
 type Websocket struct {
-	logger  logger.Service
+	log     *zerolog.Logger
 	isSocks bool
 }
 
@@ -36,9 +37,9 @@ func (d *wsdialer) Dial(address string) (io.ReadWriteCloser, *socks.AddrSpec, er
 }
 
 // NewWSConnection returns a new connection object
-func NewWSConnection(logger logger.Service, isSocks bool) Connection {
+func NewWSConnection(log *zerolog.Logger, isSocks bool) Connection {
 	return &Websocket{
-		logger:  logger,
+		log:     log,
 		isSocks: isSocks,
 	}
 }
@@ -46,9 +47,9 @@ func NewWSConnection(logger logger.Service, isSocks bool) Connection {
 // ServeStream will create a Websocket client stream connection to the edge
 // it blocks and writes the raw data from conn over the tunnel
 func (ws *Websocket) ServeStream(options *StartOptions, conn io.ReadWriter) error {
-	wsConn, err := createWebsocketStream(options, ws.logger)
+	wsConn, err := createWebsocketStream(options, ws.log)
 	if err != nil {
-		ws.logger.Errorf("failed to connect to %s with error: %s", options.OriginURL, err)
+		ws.log.Err(err).Str(LogFieldOriginURL, options.OriginURL).Msg("failed to connect to origin")
 		return err
 	}
 	defer wsConn.Close()
@@ -58,7 +59,7 @@ func (ws *Websocket) ServeStream(options *StartOptions, conn io.ReadWriter) erro
 		requestHandler := socks.NewRequestHandler(dialer)
 		socksServer := socks.NewConnectionHandler(requestHandler)
 
-		socksServer.Serve(conn)
+		_ = socksServer.Serve(conn)
 	} else {
 		cfwebsocket.Stream(wsConn, conn)
 	}
@@ -68,13 +69,13 @@ func (ws *Websocket) ServeStream(options *StartOptions, conn io.ReadWriter) erro
 // StartServer creates a Websocket server to listen for connections.
 // This is used on the origin (tunnel) side to take data from the muxer and send it to the origin
 func (ws *Websocket) StartServer(listener net.Listener, remote string, shutdownC <-chan struct{}) error {
-	return cfwebsocket.StartProxyServer(ws.logger, listener, remote, shutdownC, cfwebsocket.DefaultStreamHandler)
+	return cfwebsocket.StartProxyServer(ws.log, listener, remote, shutdownC, cfwebsocket.DefaultStreamHandler)
 }
 
 // createWebsocketStream will create a WebSocket connection to stream data over
 // It also handles redirects from Access and will present that flow if
 // the token is not present on the request
-func createWebsocketStream(options *StartOptions, logger logger.Service) (*cfwebsocket.Conn, error) {
+func createWebsocketStream(options *StartOptions, log *zerolog.Logger) (*cfwebsocket.Conn, error) {
 	req, err := http.NewRequest(http.MethodGet, options.OriginURL, nil)
 	if err != nil {
 		return nil, err
@@ -82,13 +83,13 @@ func createWebsocketStream(options *StartOptions, logger logger.Service) (*cfweb
 	req.Header = options.Headers
 
 	dump, err := httputil.DumpRequest(req, false)
-	logger.Debugf("Websocket request: %s", string(dump))
+	log.Debug().Msgf("Websocket request: %s", string(dump))
 
 	wsConn, resp, err := cfwebsocket.ClientConnect(req, nil)
 	defer closeRespBody(resp)
 
 	if err != nil && IsAccessResponse(resp) {
-		wsConn, err = createAccessAuthenticatedStream(options, logger)
+		wsConn, err = createAccessAuthenticatedStream(options, log)
 		if err != nil {
 			return nil, err
 		}
@@ -104,8 +105,8 @@ func createWebsocketStream(options *StartOptions, logger logger.Service) (*cfweb
 // this probably means the token in storage is invalid (expired/revoked). If that
 // happens it deletes the token and runs the connection again, so the user can
 // login again and generate a new one.
-func createAccessAuthenticatedStream(options *StartOptions, logger logger.Service) (*websocket.Conn, error) {
-	wsConn, resp, err := createAccessWebSocketStream(options, logger)
+func createAccessAuthenticatedStream(options *StartOptions, log *zerolog.Logger) (*websocket.Conn, error) {
+	wsConn, resp, err := createAccessWebSocketStream(options, log)
 	defer closeRespBody(resp)
 	if err == nil {
 		return wsConn, nil
@@ -123,7 +124,7 @@ func createAccessAuthenticatedStream(options *StartOptions, logger logger.Servic
 	if err := token.RemoveTokenIfExists(originReq.URL); err != nil {
 		return nil, err
 	}
-	wsConn, resp, err = createAccessWebSocketStream(options, logger)
+	wsConn, resp, err = createAccessWebSocketStream(options, log)
 	defer closeRespBody(resp)
 	if err != nil {
 		return nil, err
@@ -133,23 +134,23 @@ func createAccessAuthenticatedStream(options *StartOptions, logger logger.Servic
 }
 
 // createAccessWebSocketStream builds an Access request and makes a connection
-func createAccessWebSocketStream(options *StartOptions, logger logger.Service) (*websocket.Conn, *http.Response, error) {
-	req, err := BuildAccessRequest(options, logger)
+func createAccessWebSocketStream(options *StartOptions, log *zerolog.Logger) (*websocket.Conn, *http.Response, error) {
+	req, err := BuildAccessRequest(options, log)
 	if err != nil {
 		return nil, nil, err
 	}
 
 	dump, err := httputil.DumpRequest(req, false)
-	logger.Debugf("Access Websocket request: %s", string(dump))
+	log.Debug().Msgf("Access Websocket request: %s", string(dump))
 
 	conn, resp, err := cfwebsocket.ClientConnect(req, nil)
 
 	if resp != nil {
 		r, err := httputil.DumpResponse(resp, true)
 		if r != nil {
-			logger.Debugf("Websocket response: %q", r)
+			log.Debug().Msgf("Websocket response: %q", r)
 		} else if err != nil {
-			logger.Debugf("Websocket response error: %v", err)
+			log.Debug().Msgf("Websocket response error: %v", err)
 		}
 	}
 

cfsetup.yaml

@@ -1,26 +1,30 @@
-pinned_go: &pinned_go go=1.15.3-1
+pinned_go: &pinned_go go=1.15.7-1
+pinned_go_fips: &pinned_go_fips go-fips=1.15.5-3
+
 build_dir: &build_dir /cfsetup_build
-default-flavor: stretch
+default-flavor: buster
 stretch: &stretch
   build:
     build_dir: *build_dir
     builddeps:
-      - *pinned_go
+      - *pinned_go_fips
       - build-essential
     post-cache:
       - export GOOS=linux
       - export GOARCH=amd64
+      - export FIPS=true
       - make cloudflared
   build-deb:
     build_dir: *build_dir
     builddeps:
-      - *pinned_go
+      - *pinned_go_fips
       - build-essential
       - fakeroot
       - rubygem-fpm
     post-cache:
       - export GOOS=linux
       - export GOARCH=amd64
+      - export FIPS=true
       - make cloudflared-deb
   build-deb-arm64:
     build_dir: *build_dir
@@ -36,7 +40,7 @@ stretch: &stretch
   publish-deb:
     build_dir: *build_dir
     builddeps:
-      - *pinned_go
+      - *pinned_go_fips
       - build-essential
       - fakeroot
       - rubygem-fpm
@@ -44,20 +48,22 @@ stretch: &stretch
     post-cache:
       - export GOOS=linux
       - export GOARCH=amd64
+      - export FIPS=true
       - make publish-deb
   release-linux-amd64:
     build_dir: *build_dir
     builddeps:
-      - *pinned_go
+      - *pinned_go_fips
       - build-essential
     post-cache:
       - export GOOS=linux
       - export GOARCH=amd64
+      - export FIPS=true
       - make release
   github-release-linux-amd64:
     build_dir: *build_dir
     builddeps:
-      - *pinned_go
+      - *pinned_go_fips
       - build-essential
       - python3-setuptools
       - python3-pip
@@ -66,6 +72,7 @@ stretch: &stretch
     post-cache:
       - export GOOS=linux
       - export GOARCH=amd64
+      - export FIPS=true
       - make github-release
   release-linux-armv6:
     build_dir: *build_dir
@@ -185,11 +192,12 @@ stretch: &stretch
   test:
     build_dir: *build_dir
     builddeps:
-      - *pinned_go
+      - *pinned_go_fips
       - build-essential
     post-cache:
       - export GOOS=linux
       - export GOARCH=amd64
+      - export FIPS=true
       # cd to a non-module directory: https://github.com/golang/go/issues/24250
       - (cd / && go get github.com/BurntSushi/go-sumtype)
       - export PATH="$HOME/go/bin:$PATH"
@@ -239,10 +247,8 @@ stretch: &stretch
       - export GOARCH=amd64
       - make publish-cloudflared-junos
 
-jessie: *stretch
-
 buster: *stretch
-
+bullseye: *stretch
 centos-7:
   publish-rpm:
     build_dir: *build_dir
@@ -250,8 +256,8 @@ centos-7:
       - https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
     pre-cache:
       - yum install -y fakeroot
-      - wget https://golang.org/dl/go1.15.3.linux-amd64.tar.gz -P /tmp/
-      - tar -C /usr/local -xzf /tmp/go1.15.3.linux-amd64.tar.gz
+      - wget https://golang.org/dl/go1.15.7.linux-amd64.tar.gz -P /tmp/
+      - tar -C /usr/local -xzf /tmp/go1.15.7.linux-amd64.tar.gz
     post-cache:
       - export PATH=$PATH:/usr/local/go/bin
       - export GOOS=linux
@@ -262,8 +268,8 @@ centos-7:
     builddeps: *el7_builddeps
     pre-cache:
       - yum install -y fakeroot
-      - wget https://golang.org/dl/go1.15.3.linux-amd64.tar.gz -P /tmp/
-      - tar -C /usr/local -xzf /tmp/go1.15.3.linux-amd64.tar.gz
+      - wget https://golang.org/dl/go1.15.7.linux-amd64.tar.gz -P /tmp/
+      - tar -C /usr/local -xzf /tmp/go1.15.7.linux-amd64.tar.gz
     post-cache:
       - export PATH=$PATH:/usr/local/go/bin
       - export GOOS=linux

cmd/cloudflared/access/carrier.go

@@ -2,31 +2,30 @@ package access
 
 import (
 	"net/http"
-	"net/url"
 	"strings"
 
 	"github.com/cloudflare/cloudflared/carrier"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
 	"github.com/cloudflare/cloudflared/h2mux"
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/validation"
+
 	"github.com/pkg/errors"
-	cli "github.com/urfave/cli/v2"
+	"github.com/rs/zerolog"
+	"github.com/urfave/cli/v2"
+)
+
+const (
+	LogFieldHost = "host"
 )
 
 // StartForwarder starts a client side websocket forward
-func StartForwarder(forwarder config.Forwarder, shutdown <-chan struct{}, logger logger.Service) error {
-	validURLString, err := validation.ValidateUrl(forwarder.Listener)
+func StartForwarder(forwarder config.Forwarder, shutdown <-chan struct{}, log *zerolog.Logger) error {
+	validURL, err := validation.ValidateUrl(forwarder.Listener)
 	if err != nil {
 		return errors.Wrap(err, "error validating origin URL")
 	}
 
-	validURL, err := url.Parse(validURLString)
-	if err != nil {
-		return errors.Wrap(err, "error parsing origin URL")
-	}
-
 	// get the headers from the config file and add to the request
 	headers := make(http.Header)
 	if forwarder.TokenClientID != "" {
@@ -47,9 +46,9 @@ func StartForwarder(forwarder config.Forwarder, shutdown <-chan struct{}, logger
 	}
 
 	// we could add a cmd line variable for this bool if we want the SOCK5 server to be on the client side
-	wsConn := carrier.NewWSConnection(logger, false)
+	wsConn := carrier.NewWSConnection(log, false)
 
-	logger.Infof("Start Websocket listener on: %s", validURL.Host)
+	log.Info().Str(LogFieldHost, validURL.Host).Msg("Start Websocket listener")
 	return carrier.StartForwarder(wsConn, validURL.Host, shutdown, options)
 }
 
@@ -58,22 +57,7 @@ func StartForwarder(forwarder config.Forwarder, shutdown <-chan struct{}, logger
 // useful for proxying other protocols (like ssh) over websockets
 // (which you can put Access in front of)
 func ssh(c *cli.Context) error {
-	logDirectory, logLevel := config.FindLogSettings()
-
-	flagLogDirectory := c.String(sshLogDirectoryFlag)
-	if flagLogDirectory != "" {
-		logDirectory = flagLogDirectory
-	}
-
-	flagLogLevel := c.String(sshLogLevelFlag)
-	if flagLogLevel != "" {
-		logLevel = flagLogLevel
-	}
-
-	logger, err := logger.New(logger.DefaultFile(logDirectory), logger.LogLevelString(logLevel))
-	if err != nil {
-		return cliutil.PrintLoggerSetupError("error setting up logger", err)
-	}
+	log := logger.CreateSSHLoggerFromContext(c, logger.EnableTerminalLog)
 
 	// get the hostname from the cmdline and error out if its not provided
 	rawHostName := c.String(sshHostnameFlag)
@@ -103,24 +87,19 @@ func ssh(c *cli.Context) error {
 	}
 
 	// we could add a cmd line variable for this bool if we want the SOCK5 server to be on the client side
-	wsConn := carrier.NewWSConnection(logger, false)
+	wsConn := carrier.NewWSConnection(log, false)
 
 	if c.NArg() > 0 || c.IsSet(sshURLFlag) {
-		localForwarder, err := config.ValidateUrl(c, true)
+		forwarder, err := config.ValidateUrl(c, true)
 		if err != nil {
-			logger.Errorf("Error validating origin URL: %s", err)
-			return errors.Wrap(err, "error validating origin URL")
-		}
-		forwarder, err := url.Parse(localForwarder)
-		if err != nil {
-			logger.Errorf("Error validating origin URL: %s", err)
+			log.Err(err).Msg("Error validating origin URL")
 			return errors.Wrap(err, "error validating origin URL")
 		}
 
-		logger.Infof("Start Websocket listener on: %s", forwarder.Host)
+		log.Info().Str(LogFieldHost, forwarder.Host).Msg("Start Websocket listener")
 		err = carrier.StartForwarder(wsConn, forwarder.Host, shutdownC, options)
 		if err != nil {
-			logger.Errorf("Error on Websocket listener: %s", err)
+			log.Err(err).Msg("Error on Websocket listener")
 		}
 		return err
 	}

cmd/cloudflared/access/cmd.go

@@ -17,24 +17,23 @@ import (
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/sshgen"
 	"github.com/cloudflare/cloudflared/validation"
-	"github.com/pkg/errors"
-	"golang.org/x/net/idna"
 
 	"github.com/getsentry/raven-go"
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog"
 	"github.com/urfave/cli/v2"
+	"golang.org/x/net/idna"
 )
 
 const (
-	sshHostnameFlag     = "hostname"
-	sshDestinationFlag  = "destination"
-	sshURLFlag          = "url"
-	sshHeaderFlag       = "header"
-	sshTokenIDFlag      = "service-token-id"
-	sshTokenSecretFlag  = "service-token-secret"
-	sshGenCertFlag      = "short-lived-cert"
-	sshLogDirectoryFlag = "log-directory"
-	sshLogLevelFlag     = "log-level"
-	sshConfigTemplate   = `
+	sshHostnameFlag    = "hostname"
+	sshDestinationFlag = "destination"
+	sshURLFlag         = "url"
+	sshHeaderFlag      = "header"
+	sshTokenIDFlag     = "service-token-id"
+	sshTokenSecretFlag = "service-token-secret"
+	sshGenCertFlag     = "short-lived-cert"
+	sshConfigTemplate  = `
 Add to your {{.Home}}/.ssh/config:
 
 Host {{.Hostname}}
@@ -56,12 +55,11 @@ const sentryDSN = "https://56a9c9fa5c364ab28f34b14f35ea0f1b@sentry.io/189878"
 
 var (
 	shutdownC      chan struct{}
-	graceShutdownC chan struct{}
 )
 
 // Init will initialize and store vars from the main program
-func Init(s, g chan struct{}) {
-	shutdownC, graceShutdownC = s, g
+func Init(shutdown chan struct{}) {
+	shutdownC = shutdown
 }
 
 // Flags return the global flags for Access related commands (hopefully none)
@@ -157,12 +155,12 @@ func Commands() []*cli.Command {
 							Usage:   "specify an Access service token secret you wish to use.",
 						},
 						&cli.StringFlag{
-							Name:    sshLogDirectoryFlag,
+							Name:    logger.LogSSHDirectoryFlag,
 							Aliases: []string{"logfile"}, //added to match the tunnel side
 							Usage:   "Save application log to this directory for reporting issues.",
 						},
 						&cli.StringFlag{
-							Name:    sshLogLevelFlag,
+							Name:    logger.LogSSHLevelFlag,
 							Aliases: []string{"loglevel"}, //added to match the tunnel side
 							Usage:   "Application logging level {fatal, error, info, debug}. ",
 						},
@@ -207,24 +205,21 @@ func login(c *cli.Context) error {
 		return err
 	}
 
-	logger, err := logger.New()
-	if err != nil {
-		return errors.Wrap(err, "error setting up logger")
-	}
+	log := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
 
 	args := c.Args()
 	rawURL := ensureURLScheme(args.First())
 	appURL, err := url.Parse(rawURL)
 	if args.Len() < 1 || err != nil {
-		logger.Errorf("Please provide the url of the Access application\n")
+		log.Error().Msg("Please provide the url of the Access application")
 		return err
 	}
-	if err := verifyTokenAtEdge(appURL, c, logger); err != nil {
-		logger.Errorf("Could not verify token: %s", err)
+	if err := verifyTokenAtEdge(appURL, c, log); err != nil {
+		log.Err(err).Msg("Could not verify token")
 		return err
 	}
 
-	cfdToken, err := token.GetTokenIfExists(appURL)
+	cfdToken, err := token.GetAppTokenIfExists(appURL)
 	if err != nil {
 		fmt.Fprintln(os.Stderr, "Unable to find token for provided application.")
 		return err
@@ -252,32 +247,29 @@ func curl(c *cli.Context) error {
 	if err := raven.SetDSN(sentryDSN); err != nil {
 		return err
 	}
-	logger, err := logger.New()
-	if err != nil {
-		return errors.Wrap(err, "error setting up logger")
-	}
+	log := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
 
 	args := c.Args()
 	if args.Len() < 1 {
-		logger.Error("Please provide the access app and command you wish to run.")
+		log.Error().Msg("Please provide the access app and command you wish to run.")
 		return errors.New("incorrect args")
 	}
 
 	cmdArgs, allowRequest := parseAllowRequest(args.Slice())
-	appURL, err := getAppURL(cmdArgs, logger)
+	appURL, err := getAppURL(cmdArgs, log)
 	if err != nil {
 		return err
 	}
 
-	tok, err := token.GetTokenIfExists(appURL)
+	tok, err := token.GetAppTokenIfExists(appURL)
 	if err != nil || tok == "" {
 		if allowRequest {
-			logger.Info("You don't have an Access token set. Please run access token <access application> to fetch one.")
+			log.Info().Msg("You don't have an Access token set. Please run access token <access application> to fetch one.")
 			return shell.Run("curl", cmdArgs...)
 		}
-		tok, err = token.FetchToken(appURL, logger)
+		tok, err = token.FetchToken(appURL, log)
 		if err != nil {
-			logger.Errorf("Failed to refresh token: %s", err)
+			log.Err(err).Msg("Failed to refresh token")
 			return err
 		}
 	}
@@ -297,9 +289,9 @@ func generateToken(c *cli.Context) error {
 		fmt.Fprintln(os.Stderr, "Please provide a url.")
 		return err
 	}
-	tok, err := token.GetTokenIfExists(appURL)
+	tok, err := token.GetAppTokenIfExists(appURL)
 	if err != nil || tok == "" {
-		fmt.Fprintln(os.Stderr, "Unable to find token for provided application. Please run token command to generate token.")
+		fmt.Fprintln(os.Stderr, "Unable to find token for provided application. Please run login command to generate token.")
 		return err
 	}
 
@@ -331,10 +323,7 @@ func sshConfig(c *cli.Context) error {
 
 // sshGen generates a short lived certificate for provided hostname
 func sshGen(c *cli.Context) error {
-	logger, err := logger.New()
-	if err != nil {
-		return errors.Wrap(err, "error setting up logger")
-	}
+	log := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
 
 	// get the hostname from the cmdline and error out if its not provided
 	rawHostName := c.String(sshHostnameFlag)
@@ -351,7 +340,7 @@ func sshGen(c *cli.Context) error {
 	// this fetchToken function mutates the appURL param. We should refactor that
 	fetchTokenURL := &url.URL{}
 	*fetchTokenURL = *originURL
-	cfdToken, err := token.FetchTokenWithRedirect(fetchTokenURL, logger)
+	cfdToken, err := token.FetchTokenWithRedirect(fetchTokenURL, log)
 	if err != nil {
 		return err
 	}
@@ -364,15 +353,15 @@ func sshGen(c *cli.Context) error {
 }
 
 // getAppURL will pull the appURL needed for fetching a user's Access token
-func getAppURL(cmdArgs []string, logger logger.Service) (*url.URL, error) {
+func getAppURL(cmdArgs []string, log *zerolog.Logger) (*url.URL, error) {
 	if len(cmdArgs) < 1 {
-		logger.Error("Please provide a valid URL as the first argument to curl.")
+		log.Error().Msg("Please provide a valid URL as the first argument to curl.")
 		return nil, errors.New("not a valid url")
 	}
 
 	u, err := processURL(cmdArgs[0])
 	if err != nil {
-		logger.Error("Please provide a valid URL as the first argument to curl.")
+		log.Error().Msg("Please provide a valid URL as the first argument to curl.")
 		return nil, err
 	}
 
@@ -438,7 +427,7 @@ func isFileThere(candidate string) bool {
 // verifyTokenAtEdge checks for a token on disk, or generates a new one.
 // Then makes a request to to the origin with the token to ensure it is valid.
 // Returns nil if token is valid.
-func verifyTokenAtEdge(appUrl *url.URL, c *cli.Context, logger logger.Service) error {
+func verifyTokenAtEdge(appUrl *url.URL, c *cli.Context, log *zerolog.Logger) error {
 	headers := buildRequestHeaders(c.StringSlice(sshHeaderFlag))
 	if c.IsSet(sshTokenIDFlag) {
 		headers.Add(h2mux.CFAccessClientIDHeader, c.String(sshTokenIDFlag))
@@ -448,7 +437,7 @@ func verifyTokenAtEdge(appUrl *url.URL, c *cli.Context, logger logger.Service) e
 	}
 	options := &carrier.StartOptions{OriginURL: appUrl.String(), Headers: headers}
 
-	if valid, err := isTokenValid(options, logger); err != nil {
+	if valid, err := isTokenValid(options, log); err != nil {
 		return err
 	} else if valid {
 		return nil
@@ -458,7 +447,7 @@ func verifyTokenAtEdge(appUrl *url.URL, c *cli.Context, logger logger.Service) e
 		return err
 	}
 
-	if valid, err := isTokenValid(options, logger); err != nil {
+	if valid, err := isTokenValid(options, log); err != nil {
 		return err
 	} else if !valid {
 		return errors.New("failed to verify token")
@@ -468,8 +457,8 @@ func verifyTokenAtEdge(appUrl *url.URL, c *cli.Context, logger logger.Service) e
 }
 
 // isTokenValid makes a request to the origin and returns true if the response was not a 302.
-func isTokenValid(options *carrier.StartOptions, logger logger.Service) (bool, error) {
-	req, err := carrier.BuildAccessRequest(options, logger)
+func isTokenValid(options *carrier.StartOptions, log *zerolog.Logger) (bool, error) {
+	req, err := carrier.BuildAccessRequest(options, log)
 	if err != nil {
 		return false, errors.Wrap(err, "Could not create access request")
 	}

cmd/cloudflared/app_forward_service.go

@@ -3,7 +3,8 @@ package main
 import (
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/access"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
-	"github.com/cloudflare/cloudflared/logger"
+
+	"github.com/rs/zerolog"
 )
 
 // ForwardServiceType is used to identify what kind of overwatch service this is
@@ -15,12 +16,12 @@ const ForwardServiceType = "forward"
 type ForwarderService struct {
 	forwarder config.Forwarder
 	shutdown  chan struct{}
-	logger    logger.Service
+	log       *zerolog.Logger
 }
 
 // NewForwardService creates a new forwarder service
-func NewForwardService(f config.Forwarder, logger logger.Service) *ForwarderService {
-	return &ForwarderService{forwarder: f, shutdown: make(chan struct{}, 1), logger: logger}
+func NewForwardService(f config.Forwarder, log *zerolog.Logger) *ForwarderService {
+	return &ForwarderService{forwarder: f, shutdown: make(chan struct{}, 1), log: log}
 }
 
 // Name is used to figure out this service is related to the others (normally the addr it binds to)
@@ -46,5 +47,5 @@ func (s *ForwarderService) Shutdown() {
 
 // Run is the run loop that is started by the overwatch service
 func (s *ForwarderService) Run() error {
-	return access.StartForwarder(s.forwarder, s.shutdown, s.logger)
+	return access.StartForwarder(s.forwarder, s.shutdown, s.log)
 }

cmd/cloudflared/app_resolver_service.go

@@ -2,12 +2,18 @@ package main
 
 import (
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
-	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/tunneldns"
+
+	"github.com/rs/zerolog"
 )
 
-// ResolverServiceType is used to identify what kind of overwatch service this is
-const ResolverServiceType = "resolver"
+const (
+	// ResolverServiceType is used to identify what kind of overwatch service this is
+	ResolverServiceType = "resolver"
+
+	LogFieldResolverAddress = "resolverAddress"
+	LogFieldResolverPort    = "resolverPort"
+)
 
 // ResolverService is used to wrap the tunneldns package's DNS over HTTP
 // into a service model for the overwatch package.
@@ -15,14 +21,14 @@ const ResolverServiceType = "resolver"
 type ResolverService struct {
 	resolver config.DNSResolver
 	shutdown chan struct{}
-	logger   logger.Service
+	log      *zerolog.Logger
 }
 
 // NewResolverService creates a new resolver service
-func NewResolverService(r config.DNSResolver, logger logger.Service) *ResolverService {
+func NewResolverService(r config.DNSResolver, log *zerolog.Logger) *ResolverService {
 	return &ResolverService{resolver: r,
 		shutdown: make(chan struct{}),
-		logger:   logger,
+		log:      log,
 	}
 }
 
@@ -51,7 +57,7 @@ func (s *ResolverService) Shutdown() {
 func (s *ResolverService) Run() error {
 	// create a listener
 	l, err := tunneldns.CreateListener(s.resolver.AddressOrDefault(), s.resolver.PortOrDefault(),
-		s.resolver.UpstreamsOrDefault(), s.resolver.BootstrapsOrDefault(), s.logger)
+		s.resolver.UpstreamsOrDefault(), s.resolver.BootstrapsOrDefault(), s.log)
 	if err != nil {
 		return err
 	}
@@ -60,14 +66,20 @@ func (s *ResolverService) Run() error {
 	readySignal := make(chan struct{})
 	err = l.Start(readySignal)
 	if err != nil {
-		l.Stop()
+		_ = l.Stop()
 		return err
 	}
 	<-readySignal
-	s.logger.Infof("start resolver on: %s:%d", s.resolver.AddressOrDefault(), s.resolver.PortOrDefault())
+
+	resolverLog := s.log.With().
+		Str(LogFieldResolverAddress, s.resolver.AddressOrDefault()).
+		Uint16(LogFieldResolverPort, s.resolver.PortOrDefault()).
+		Logger()
+
+	resolverLog.Info().Msg("Starting resolver")
 
 	// wait for shutdown signal
 	<-s.shutdown
-	s.logger.Infof("shutdown on: %s:%d", s.resolver.AddressOrDefault(), s.resolver.PortOrDefault())
+	resolverLog.Info().Msg("Shutting down resolver")
 	return l.Stop()
 }

cmd/cloudflared/app_service.go

@@ -2,8 +2,9 @@ package main
 
 import (
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
-	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/overwatch"
+
+	"github.com/rs/zerolog"
 )
 
 // AppService is the main service that runs when no command lines flags are passed to cloudflared
@@ -13,17 +14,17 @@ type AppService struct {
 	serviceManager   overwatch.Manager
 	shutdownC        chan struct{}
 	configUpdateChan chan config.Root
-	logger           logger.Service
+	log              *zerolog.Logger
 }
 
 // NewAppService creates a new AppService with needed supporting services
-func NewAppService(configManager config.Manager, serviceManager overwatch.Manager, shutdownC chan struct{}, logger logger.Service) *AppService {
+func NewAppService(configManager config.Manager, serviceManager overwatch.Manager, shutdownC chan struct{}, log *zerolog.Logger) *AppService {
 	return &AppService{
 		configManager:    configManager,
 		serviceManager:   serviceManager,
 		shutdownC:        shutdownC,
 		configUpdateChan: make(chan config.Root),
-		logger:           logger,
+		log:              log,
 	}
 }
 
@@ -37,6 +38,7 @@ func (s *AppService) Run() error {
 func (s *AppService) Shutdown() error {
 	s.configManager.Shutdown()
 	s.shutdownC <- struct{}{}
+
 	return nil
 }
 
@@ -66,14 +68,14 @@ func (s *AppService) handleConfigUpdate(c config.Root) {
 	// handle the client forward listeners
 	activeServices := map[string]struct{}{}
 	for _, f := range c.Forwarders {
-		service := NewForwardService(f, s.logger)
+		service := NewForwardService(f, s.log)
 		s.serviceManager.Add(service)
 		activeServices[service.Name()] = struct{}{}
 	}
 
 	// handle resolver changes
 	if c.Resolver.Enabled {
-		service := NewResolverService(c.Resolver, s.logger)
+		service := NewResolverService(c.Resolver, s.log)
 		s.serviceManager.Add(service)
 		activeServices[service.Name()] = struct{}{}
 

cmd/cloudflared/buildinfo/build_info.go

@@ -1,9 +1,8 @@
 package buildinfo
 
 import (
+	"github.com/rs/zerolog"
 	"runtime"
-
-	"github.com/cloudflare/cloudflared/logger"
 )
 
 type BuildInfo struct {
@@ -22,7 +21,7 @@ func GetBuildInfo(cloudflaredVersion string) *BuildInfo {
 	}
 }
 
-func (bi *BuildInfo) Log(logger logger.Service) {
-	logger.Infof("Version %s", bi.CloudflaredVersion)
-	logger.Infof("GOOS: %s, GOVersion: %s, GoArch: %s", bi.GoOS, bi.GoVersion, bi.GoArch)
+func (bi *BuildInfo) Log(log *zerolog.Logger) {
+	log.Info().Msgf("Version %s", bi.CloudflaredVersion)
+	log.Info().Msgf("GOOS: %s, GOVersion: %s, GoArch: %s", bi.GoOS, bi.GoVersion, bi.GoArch)
 }

cmd/cloudflared/cliutil/errors.go

@@ -2,10 +2,6 @@ package cliutil
 
 import (
 	"fmt"
-	"log"
-
-	"github.com/cloudflare/cloudflared/logger"
-	"github.com/pkg/errors"
 	"github.com/urfave/cli/v2"
 )
 
@@ -31,25 +27,11 @@ func ErrorHandler(actionFunc cli.ActionFunc) cli.ActionFunc {
 		if err != nil {
 			if _, ok := err.(usageError); ok {
 				msg := fmt.Sprintf("%s\nSee 'cloudflared %s --help'.", err.Error(), ctx.Command.FullName())
-				return cli.Exit(msg, -1)
+				err = cli.Exit(msg, -1)
+			} else if _, ok := err.(cli.ExitCoder); !ok {
+				err = cli.Exit(err.Error(), 1)
 			}
-			// os.Exits with error code if err is cli.ExitCoder or cli.MultiError
-			cli.HandleExitCoder(err)
-			err = cli.Exit(err.Error(), 1)
 		}
-		logger.SharedWriteManager.Shutdown()
 		return err
 	}
 }
-
-// PrintLoggerSetupError returns an error to stdout to notify when a logger can't start
-func PrintLoggerSetupError(msg string, err error) error {
-	l, le := logger.New()
-	if le != nil {
-		log.Printf("%s: %s", msg, err)
-	} else {
-		l.Errorf("%s: %s", msg, err)
-	}
-
-	return errors.Wrap(err, msg)
-}

cmd/cloudflared/config/configuration.go

@@ -1,19 +1,21 @@
 package config
 
 import (
-	"errors"
 	"fmt"
+	"io"
+	"net/url"
 	"os"
 	"path/filepath"
 	"runtime"
 	"time"
 
-	homedir "github.com/mitchellh/go-homedir"
+	"github.com/mitchellh/go-homedir"
+	"github.com/pkg/errors"
 	"github.com/urfave/cli/v2"
 	"gopkg.in/yaml.v2"
 
-	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/validation"
+	"github.com/rs/zerolog"
 )
 
 var (
@@ -34,7 +36,12 @@ var (
 	ErrNoConfigFile = fmt.Errorf("Cannot determine default configuration path. No file %v in %v", DefaultConfigFiles, DefaultConfigSearchDirectories())
 )
 
-const DefaultCredentialFile = "cert.pem"
+const (
+	DefaultCredentialFile = "cert.pem"
+
+	// BastionFlag is to enable bastion, or jump host, operation
+	BastionFlag = "bastion"
+)
 
 // DefaultConfigDirectory returns the default directory of the config file
 func DefaultConfigDirectory() string {
@@ -88,7 +95,7 @@ func FileExists(path string) (bool, error) {
 		}
 		return false, err
 	}
-	f.Close()
+	_ = f.Close()
 	return true, nil
 }
 
@@ -131,7 +138,7 @@ func FindOrCreateConfigPath() string {
 		defer file.Close()
 
 		logDir := DefaultLogDirectory()
-		os.MkdirAll(logDir, os.ModePerm) //try and create it. Doesn't matter if it succeed or not, only byproduct will be no logs
+		_ = os.MkdirAll(logDir, os.ModePerm) //try and create it. Doesn't matter if it succeed or not, only byproduct will be no logs
 
 		c := Root{
 			LogDirectory: logDir,
@@ -144,35 +151,6 @@ func FindOrCreateConfigPath() string {
 	return path
 }
 
-// FindLogSettings gets the log directory and level from the config file
-func FindLogSettings() (string, string) {
-	configPath := FindOrCreateConfigPath()
-	defaultDirectory := DefaultLogDirectory()
-	defaultLevel := "info"
-
-	file, err := os.Open(configPath)
-	if err != nil {
-		return defaultDirectory, defaultLevel
-	}
-	defer file.Close()
-
-	var config Root
-	if err := yaml.NewDecoder(file).Decode(&config); err != nil {
-		return defaultDirectory, defaultLevel
-	}
-
-	directory := defaultDirectory
-	if config.LogDirectory != "" {
-		directory = config.LogDirectory
-	}
-
-	level := defaultLevel
-	if config.LogLevel != "" {
-		level = config.LogLevel
-	}
-	return directory, level
-}
-
 // ValidateUnixSocket ensures --unix-socket param is used exclusively
 // i.e. it fails if a user specifies both --url and --unix-socket
 func ValidateUnixSocket(c *cli.Context) (string, error) {
@@ -184,11 +162,11 @@ func ValidateUnixSocket(c *cli.Context) (string, error) {
 
 // ValidateUrl will validate url flag correctness. It can be either from --url or argument
 // Notice ValidateUnixSocket, it will enforce --unix-socket is not used with --url or argument
-func ValidateUrl(c *cli.Context, allowFromArgs bool) (string, error) {
+func ValidateUrl(c *cli.Context, allowURLFromArgs bool) (*url.URL, error) {
 	var url = c.String("url")
-	if allowFromArgs && c.NArg() > 0 {
+	if allowURLFromArgs && c.NArg() > 0 {
 		if c.IsSet("url") {
-			return "", errors.New("Specified origin urls using both --url and argument. Decide which one you want, I can only support one.")
+			return nil, errors.New("Specified origin urls using both --url and argument. Decide which one you want, I can only support one.")
 		}
 		url = c.Args().Get(0)
 	}
@@ -197,15 +175,59 @@ func ValidateUrl(c *cli.Context, allowFromArgs bool) (string, error) {
 }
 
 type UnvalidatedIngressRule struct {
-	Hostname string
-	Path     string
-	Service  string
+	Hostname      string
+	Path          string
+	Service       string
+	OriginRequest OriginRequestConfig `yaml:"originRequest"`
+}
+
+// OriginRequestConfig is a set of optional fields that users may set to
+// customize how cloudflared sends requests to origin services. It is used to set
+// up general config that apply to all rules, and also, specific per-rule
+// config.
+// Note: To specify a time.Duration in go-yaml, use e.g. "3s" or "24h".
+type OriginRequestConfig struct {
+	// HTTP proxy timeout for establishing a new connection
+	ConnectTimeout *time.Duration `yaml:"connectTimeout"`
+	// HTTP proxy timeout for completing a TLS handshake
+	TLSTimeout *time.Duration `yaml:"tlsTimeout"`
+	// HTTP proxy TCP keepalive duration
+	TCPKeepAlive *time.Duration `yaml:"tcpKeepAlive"`
+	// HTTP proxy should disable "happy eyeballs" for IPv4/v6 fallback
+	NoHappyEyeballs *bool `yaml:"noHappyEyeballs"`
+	// HTTP proxy maximum keepalive connection pool size
+	KeepAliveConnections *int `yaml:"keepAliveConnections"`
+	// HTTP proxy timeout for closing an idle connection
+	KeepAliveTimeout *time.Duration `yaml:"keepAliveTimeout"`
+	// Sets the HTTP Host header for the local webserver.
+	HTTPHostHeader *string `yaml:"httpHostHeader"`
+	// Hostname on the origin server certificate.
+	OriginServerName *string `yaml:"originServerName"`
+	// Path to the CA for the certificate of your origin.
+	// This option should be used only if your certificate is not signed by Cloudflare.
+	CAPool *string `yaml:"caPool"`
+	// Disables TLS verification of the certificate presented by your origin.
+	// Will allow any certificate from the origin to be accepted.
+	// Note: The connection from your machine to Cloudflare's Edge is still encrypted.
+	NoTLSVerify *bool `yaml:"noTLSVerify"`
+	// Disables chunked transfer encoding.
+	// Useful if you are running a WSGI server.
+	DisableChunkedEncoding *bool `yaml:"disableChunkedEncoding"`
+	// Runs as jump host
+	BastionMode *bool `yaml:"bastionMode"`
+	// Listen address for the proxy.
+	ProxyAddress *string `yaml:"proxyAddress"`
+	// Listen port for the proxy.
+	ProxyPort *uint `yaml:"proxyPort"`
+	// Valid options are 'socks' or empty.
+	ProxyType *string `yaml:"proxyType"`
 }
 
 type Configuration struct {
-	TunnelID   string `yaml:"tunnel"`
-	Ingress    []UnvalidatedIngressRule
-	sourceFile string
+	TunnelID      string `yaml:"tunnel"`
+	Ingress       []UnvalidatedIngressRule
+	OriginRequest OriginRequestConfig `yaml:"originRequest"`
+	sourceFile    string
 }
 
 type configFileSettings struct {
@@ -323,7 +345,7 @@ func GetConfiguration() *Configuration {
 // ReadConfigFile returns InputSourceContext initialized from the configuration file.
 // On repeat calls returns with the same file, returns without reading the file again; however,
 // if value of "config" flag changes, will read the new config file
-func ReadConfigFile(c *cli.Context, log logger.Service) (*configFileSettings, error) {
+func ReadConfigFile(c *cli.Context, log *zerolog.Logger) (*configFileSettings, error) {
 	configFile := c.String("config")
 	if configuration.Source() == configFile || configFile == "" {
 		if configuration.Source() == "" {
@@ -332,7 +354,7 @@ func ReadConfigFile(c *cli.Context, log logger.Service) (*configFileSettings, er
 		return &configuration, nil
 	}
 
-	log.Debugf("Loading configuration from %s", configFile)
+	log.Debug().Msgf("Loading configuration from %s", configFile)
 	file, err := os.Open(configFile)
 	if err != nil {
 		if os.IsNotExist(err) {
@@ -342,7 +364,11 @@ func ReadConfigFile(c *cli.Context, log logger.Service) (*configFileSettings, er
 	}
 	defer file.Close()
 	if err := yaml.NewDecoder(file).Decode(&configuration); err != nil {
-		return nil, err
+		if err == io.EOF {
+			log.Error().Msgf("Configuration file %s was empty", configFile)
+			return &configuration, nil
+		}
+		return nil, errors.Wrap(err, "error parsing YAML in config file at "+configFile)
 	}
 	configuration.sourceFile = configFile
 	return &configuration, nil

cmd/cloudflared/config/manager.go

@@ -1,11 +1,13 @@
 package config
 
 import (
-	"errors"
+	"io"
 	"os"
 
-	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/watcher"
+
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog"
 	"gopkg.in/yaml.v2"
 )
 
@@ -26,16 +28,16 @@ type FileManager struct {
 	watcher    watcher.Notifier
 	notifier   Notifier
 	configPath string
-	logger     logger.Service
-	ReadConfig func(string) (Root, error)
+	log        *zerolog.Logger
+	ReadConfig func(string, *zerolog.Logger) (Root, error)
 }
 
 // NewFileManager creates a config manager
-func NewFileManager(watcher watcher.Notifier, configPath string, logger logger.Service) (*FileManager, error) {
+func NewFileManager(watcher watcher.Notifier, configPath string, log *zerolog.Logger) (*FileManager, error) {
 	m := &FileManager{
 		watcher:    watcher,
 		configPath: configPath,
-		logger:     logger,
+		log:        log,
 		ReadConfig: readConfigFromPath,
 	}
 	err := watcher.Add(configPath)
@@ -59,7 +61,7 @@ func (m *FileManager) Start(notifier Notifier) error {
 
 // GetConfig reads the yaml file from the disk
 func (m *FileManager) GetConfig() (Root, error) {
-	return m.ReadConfig(m.configPath)
+	return m.ReadConfig(m.configPath, m.log)
 }
 
 // Shutdown stops the watcher
@@ -67,7 +69,7 @@ func (m *FileManager) Shutdown() {
 	m.watcher.Shutdown()
 }
 
-func readConfigFromPath(configPath string) (Root, error) {
+func readConfigFromPath(configPath string, log *zerolog.Logger) (Root, error) {
 	if configPath == "" {
 		return Root{}, errors.New("unable to find config file")
 	}
@@ -80,7 +82,11 @@ func readConfigFromPath(configPath string) (Root, error) {
 
 	var config Root
 	if err := yaml.NewDecoder(file).Decode(&config); err != nil {
-		return Root{}, err
+		if err == io.EOF {
+			log.Error().Msgf("Configuration file %s was empty", configPath)
+			return Root{}, nil
+		}
+		return Root{}, errors.Wrap(err, "error parsing YAML in config file at "+configPath)
 	}
 
 	return config, nil
@@ -93,14 +99,14 @@ func readConfigFromPath(configPath string) (Root, error) {
 func (m *FileManager) WatcherItemDidChange(filepath string) {
 	config, err := m.GetConfig()
 	if err != nil {
-		m.logger.Errorf("Failed to read new config: %s", err)
+		m.log.Err(err).Msg("Failed to read new config")
 		return
 	}
-	m.logger.Info("Config file has been updated")
+	m.log.Info().Msg("Config file has been updated")
 	m.notifier.ConfigDidUpdate(config)
 }
 
 // WatcherDidError notifies of errors with the file watcher
 func (m *FileManager) WatcherDidError(err error) {
-	m.logger.Errorf("Config watcher encountered an error: %s", err)
+	m.log.Err(err).Msg("Config watcher encountered an error")
 }

cmd/cloudflared/config/manager_test.go

@@ -4,10 +4,10 @@ import (
 	"os"
 	"testing"
 
-	"github.com/stretchr/testify/assert"
-
-	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/watcher"
+
+	"github.com/rs/zerolog"
+	"github.com/stretchr/testify/assert"
 )
 
 type mockNotifier struct {
@@ -46,8 +46,8 @@ func TestConfigChanged(t *testing.T) {
 	f, err := os.Create(filePath)
 	assert.NoError(t, err)
 	defer func() {
-		f.Close()
-		os.Remove(filePath)
+		_ = f.Close()
+		_ = os.Remove(filePath)
 	}()
 	c := &Root{
 		Forwarders: []Forwarder{
@@ -57,15 +57,15 @@ func TestConfigChanged(t *testing.T) {
 			},
 		},
 	}
-	configRead := func(configPath string) (Root, error) {
+	configRead := func(configPath string, log *zerolog.Logger) (Root, error) {
 		return *c, nil
 	}
 	wait := make(chan struct{})
 	w := &mockFileWatcher{path: filePath, ready: wait}
 
-	logger := logger.NewOutputWriter(logger.NewMockWriteManager())
+	log := zerolog.Nop()
 
-	service, err := NewFileManager(w, filePath, logger)
+	service, err := NewFileManager(w, filePath, &log)
 	service.ReadConfig = configRead
 	assert.NoError(t, err)
 

cmd/cloudflared/fips.go

@@ -0,0 +1,5 @@
+// +build fips
+
+package main
+
+import _ "crypto/tls/fipsonly"

cmd/cloudflared/linux_service.go

@@ -7,20 +7,21 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/pkg/errors"
-	cli "github.com/urfave/cli/v2"
-
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/tunnel"
 	"github.com/cloudflare/cloudflared/logger"
+
+	"github.com/rs/zerolog"
+	"github.com/urfave/cli/v2"
 )
 
-func runApp(app *cli.App, shutdownC, graceShutdownC chan struct{}) {
+func runApp(app *cli.App, graceShutdownC chan struct{}) {
 	app.Commands = append(app.Commands, &cli.Command{
 		Name:  "service",
 		Usage: "Manages the Argo Tunnel system service",
 		Subcommands: []*cli.Command{
-			&cli.Command{
+			{
 				Name:   "install",
 				Usage:  "Install Argo Tunnel as a system service",
 				Action: cliutil.ErrorHandler(installLinuxService),
@@ -31,7 +32,7 @@ func runApp(app *cli.App, shutdownC, graceShutdownC chan struct{}) {
 					},
 				},
 			},
-			&cli.Command{
+			{
 				Name:   "uninstall",
 				Usage:  "Uninstall the Argo Tunnel service",
 				Action: cliutil.ErrorHandler(uninstallLinuxService),
@@ -189,7 +190,7 @@ func isSystemd() bool {
 	return false
 }
 
-func copyUserConfiguration(userConfigDir, userConfigFile, userCredentialFile string, logger logger.Service) error {
+func copyUserConfiguration(userConfigDir, userConfigFile, userCredentialFile string, log *zerolog.Logger) error {
 	srcCredentialPath := filepath.Join(userConfigDir, userCredentialFile)
 	destCredentialPath := filepath.Join(serviceConfigDir, serviceCredentialFile)
 	if srcCredentialPath != destCredentialPath {
@@ -204,17 +205,14 @@ func copyUserConfiguration(userConfigDir, userConfigFile, userCredentialFile str
 		if err := copyConfig(srcConfigPath, destConfigPath); err != nil {
 			return err
 		}
-		logger.Infof("Copied %s to %s", srcConfigPath, destConfigPath)
+		log.Info().Msgf("Copied %s to %s", srcConfigPath, destConfigPath)
 	}
 
 	return nil
 }
 
 func installLinuxService(c *cli.Context) error {
-	logger, err := logger.New()
-	if err != nil {
-		return errors.Wrap(err, "error setting up logger")
-	}
+	log := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
 
 	etPath, err := os.Executable()
 	if err != nil {
@@ -231,8 +229,8 @@ func installLinuxService(c *cli.Context) error {
 		userConfigDir := filepath.Dir(c.String("config"))
 		userConfigFile := filepath.Base(c.String("config"))
 		userCredentialFile := config.DefaultCredentialFile
-		if err = copyUserConfiguration(userConfigDir, userConfigFile, userCredentialFile, logger); err != nil {
-			logger.Errorf("Failed to copy user configuration: %s. Before running the service, ensure that %s contains two files, %s and %s", err,
+		if err = copyUserConfiguration(userConfigDir, userConfigFile, userCredentialFile, log); err != nil {
+			log.Err(err).Msgf("Failed to copy user configuration. Before running the service, ensure that %s contains two files, %s and %s",
 				serviceConfigDir, serviceCredentialFile, serviceConfigFile)
 			return err
 		}
@@ -240,7 +238,7 @@ func installLinuxService(c *cli.Context) error {
 			"--origincert", serviceConfigDir + "/" + serviceCredentialFile,
 		}
 	} else {
-		src, err := config.ReadConfigFile(c, logger)
+		src, err := config.ReadConfigFile(c, log)
 		if err != nil {
 			return err
 		}
@@ -250,7 +248,7 @@ func installLinuxService(c *cli.Context) error {
 			val, err := src.String(s)
 			return err == nil && val != ""
 		}
-		if src.TunnelID == "" || !configPresent("credentials-file") {
+		if src.TunnelID == "" || !configPresent(tunnel.CredFileFlag) {
 			return fmt.Errorf(`Configuration file %s must contain entries for the tunnel to run and its associated credentials:
 tunnel: TUNNEL-UUID
 credentials-file: CREDENTIALS-FILE
@@ -273,42 +271,42 @@ credentials-file: CREDENTIALS-FILE
 
 	switch {
 	case isSystemd():
-		logger.Infof("Using Systemd")
-		return installSystemd(&templateArgs, logger)
+		log.Info().Msgf("Using Systemd")
+		return installSystemd(&templateArgs, log)
 	default:
-		logger.Infof("Using SysV")
-		return installSysv(&templateArgs, logger)
+		log.Info().Msgf("Using SysV")
+		return installSysv(&templateArgs, log)
 	}
 }
 
-func installSystemd(templateArgs *ServiceTemplateArgs, logger logger.Service) error {
+func installSystemd(templateArgs *ServiceTemplateArgs, log *zerolog.Logger) error {
 	for _, serviceTemplate := range systemdTemplates {
 		err := serviceTemplate.Generate(templateArgs)
 		if err != nil {
-			logger.Errorf("error generating service template: %s", err)
+			log.Err(err).Msg("error generating service template")
 			return err
 		}
 	}
 	if err := runCommand("systemctl", "enable", "cloudflared.service"); err != nil {
-		logger.Errorf("systemctl enable cloudflared.service error: %s", err)
+		log.Err(err).Msg("systemctl enable cloudflared.service error")
 		return err
 	}
 	if err := runCommand("systemctl", "start", "cloudflared-update.timer"); err != nil {
-		logger.Errorf("systemctl start cloudflared-update.timer error: %s", err)
+		log.Err(err).Msg("systemctl start cloudflared-update.timer error")
 		return err
 	}
-	logger.Infof("systemctl daemon-reload")
+	log.Info().Msg("systemctl daemon-reload")
 	return runCommand("systemctl", "daemon-reload")
 }
 
-func installSysv(templateArgs *ServiceTemplateArgs, logger logger.Service) error {
+func installSysv(templateArgs *ServiceTemplateArgs, log *zerolog.Logger) error {
 	confPath, err := sysvTemplate.ResolvePath()
 	if err != nil {
-		logger.Errorf("error resolving system path: %s", err)
+		log.Err(err).Msg("error resolving system path")
 		return err
 	}
 	if err := sysvTemplate.Generate(templateArgs); err != nil {
-		logger.Errorf("error generating system template: %s", err)
+		log.Err(err).Msg("error generating system template")
 		return err
 	}
 	for _, i := range [...]string{"2", "3", "4", "5"} {
@@ -325,43 +323,40 @@ func installSysv(templateArgs *ServiceTemplateArgs, logger logger.Service) error
 }
 
 func uninstallLinuxService(c *cli.Context) error {
-	logger, err := logger.New()
-	if err != nil {
-		return errors.Wrap(err, "error setting up logger")
-	}
+	log := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
 
 	switch {
 	case isSystemd():
-		logger.Infof("Using Systemd")
-		return uninstallSystemd(logger)
+		log.Info().Msg("Using Systemd")
+		return uninstallSystemd(log)
 	default:
-		logger.Infof("Using SysV")
-		return uninstallSysv(logger)
+		log.Info().Msg("Using SysV")
+		return uninstallSysv(log)
 	}
 }
 
-func uninstallSystemd(logger logger.Service) error {
+func uninstallSystemd(log *zerolog.Logger) error {
 	if err := runCommand("systemctl", "disable", "cloudflared.service"); err != nil {
-		logger.Errorf("systemctl disable cloudflared.service error: %s", err)
+		log.Err(err).Msg("systemctl disable cloudflared.service error")
 		return err
 	}
 	if err := runCommand("systemctl", "stop", "cloudflared-update.timer"); err != nil {
-		logger.Errorf("systemctl stop cloudflared-update.timer error: %s", err)
+		log.Err(err).Msg("systemctl stop cloudflared-update.timer error")
 		return err
 	}
 	for _, serviceTemplate := range systemdTemplates {
 		if err := serviceTemplate.Remove(); err != nil {
-			logger.Errorf("error removing service template: %s", err)
+			log.Err(err).Msg("error removing service template")
 			return err
 		}
 	}
-	logger.Infof("Successfully uninstall cloudflared service")
+	log.Info().Msgf("Successfully uninstalled cloudflared service from systemd")
 	return nil
 }
 
-func uninstallSysv(logger logger.Service) error {
+func uninstallSysv(log *zerolog.Logger) error {
 	if err := sysvTemplate.Remove(); err != nil {
-		logger.Errorf("error removing service template: %s", err)
+		log.Err(err).Msg("error removing service template")
 		return err
 	}
 	for _, i := range [...]string{"2", "3", "4", "5"} {
@@ -374,6 +369,6 @@ func uninstallSysv(logger logger.Service) error {
 			continue
 		}
 	}
-	logger.Infof("Successfully uninstall cloudflared service")
+	log.Info().Msgf("Successfully uninstalled cloudflared service from sysv")
 	return nil
 }

cmd/cloudflared/macos_service.go

@@ -6,18 +6,18 @@ import (
 	"fmt"
 	"os"
 
-	"github.com/urfave/cli/v2"
-
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
 	"github.com/cloudflare/cloudflared/logger"
+
 	"github.com/pkg/errors"
+	"github.com/urfave/cli/v2"
 )
 
 const (
 	launchdIdentifier = "com.cloudflare.cloudflared"
 )
 
-func runApp(app *cli.App, shutdownC, graceShutdownC chan struct{}) {
+func runApp(app *cli.App, graceShutdownC chan struct{}) {
 	app.Commands = append(app.Commands, &cli.Command{
 		Name:  "service",
 		Usage: "Manages the Argo Tunnel launch agent",
@@ -34,7 +34,7 @@ func runApp(app *cli.App, shutdownC, graceShutdownC chan struct{}) {
 			},
 		},
 	})
-	app.Run(os.Args)
+	_ = app.Run(os.Args)
 }
 
 func newLaunchdTemplate(installPath, stdoutPath, stderrPath string) *ServiceTemplate {
@@ -107,71 +107,61 @@ func stderrPath() (string, error) {
 }
 
 func installLaunchd(c *cli.Context) error {
-	logger, err := logger.New()
-	if err != nil {
-		return errors.Wrap(err, "error setting up logger")
-	}
+	log := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
 
 	if isRootUser() {
-		logger.Infof("Installing Argo Tunnel client as a system launch daemon. " +
+		log.Info().Msg("Installing Argo Tunnel client as a system launch daemon. " +
 			"Argo Tunnel client will run at boot")
 	} else {
-		logger.Infof("Installing Argo Tunnel client as an user launch agent. " +
+		log.Info().Msg("Installing Argo Tunnel client as an user launch agent. " +
 			"Note that Argo Tunnel client will only run when the user is logged in. " +
 			"If you want to run Argo Tunnel client at boot, install with root permission. " +
 			"For more information, visit https://developers.cloudflare.com/argo-tunnel/reference/service/")
 	}
 	etPath, err := os.Executable()
 	if err != nil {
-		logger.Errorf("Error determining executable path: %s", err)
+		log.Err(err).Msg("Error determining executable path")
 		return fmt.Errorf("Error determining executable path: %v", err)
 	}
 	installPath, err := installPath()
 	if err != nil {
-		logger.Errorf("Error determining install path: %s", err)
+		log.Err(err).Msg("Error determining install path")
 		return errors.Wrap(err, "Error determining install path")
 	}
 	stdoutPath, err := stdoutPath()
 	if err != nil {
-		logger.Errorf("error determining stdout path: %s", err)
+		log.Err(err).Msg("error determining stdout path")
 		return errors.Wrap(err, "error determining stdout path")
 	}
 	stderrPath, err := stderrPath()
 	if err != nil {
-		logger.Errorf("error determining stderr path: %s", err)
+		log.Err(err).Msg("error determining stderr path")
 		return errors.Wrap(err, "error determining stderr path")
 	}
 	launchdTemplate := newLaunchdTemplate(installPath, stdoutPath, stderrPath)
-	if err != nil {
-		logger.Errorf("error creating launchd template: %s", err)
-		return errors.Wrap(err, "error creating launchd template")
-	}
 	templateArgs := ServiceTemplateArgs{Path: etPath}
 	err = launchdTemplate.Generate(&templateArgs)
 	if err != nil {
-		logger.Errorf("error generating launchd template: %s", err)
+		log.Err(err).Msg("error generating launchd template")
 		return err
 	}
 	plistPath, err := launchdTemplate.ResolvePath()
 	if err != nil {
-		logger.Errorf("error resolving launchd template path: %s", err)
+		log.Err(err).Msg("error resolving launchd template path")
 		return err
 	}
 
-	logger.Infof("Outputs are logged to %s and %s", stderrPath, stdoutPath)
+	log.Info().Msgf("Outputs are logged to %s and %s", stderrPath, stdoutPath)
 	return runCommand("launchctl", "load", plistPath)
 }
 
 func uninstallLaunchd(c *cli.Context) error {
-	logger, err := logger.New()
-	if err != nil {
-		return errors.Wrap(err, "error setting up logger")
-	}
+	log := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
 
 	if isRootUser() {
-		logger.Infof("Uninstalling Argo Tunnel as a system launch daemon")
+		log.Info().Msg("Uninstalling Argo Tunnel as a system launch daemon")
 	} else {
-		logger.Infof("Uninstalling Argo Tunnel as an user launch agent")
+		log.Info().Msg("Uninstalling Argo Tunnel as an user launch agent")
 	}
 	installPath, err := installPath()
 	if err != nil {
@@ -186,20 +176,17 @@ func uninstallLaunchd(c *cli.Context) error {
 		return errors.Wrap(err, "error determining stderr path")
 	}
 	launchdTemplate := newLaunchdTemplate(installPath, stdoutPath, stderrPath)
-	if err != nil {
-		return errors.Wrap(err, "error creating launchd template")
-	}
 	plistPath, err := launchdTemplate.ResolvePath()
 	if err != nil {
-		logger.Errorf("error resolving launchd template path: %s", err)
+		log.Err(err).Msg("error resolving launchd template path")
 		return err
 	}
 	err = runCommand("launchctl", "unload", plistPath)
 	if err != nil {
-		logger.Errorf("error unloading: %s", err)
+		log.Err(err).Msg("error unloading")
 		return err
 	}
 
-	logger.Infof("Outputs are logged to %s and %s", stderrPath, stdoutPath)
+	log.Info().Msgf("Outputs are logged to %s and %s", stderrPath, stdoutPath)
 	return launchdTemplate.Remove()
 }

cmd/cloudflared/main.go

@@ -10,17 +10,16 @@ import (
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/tunnel"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/updater"
-	log "github.com/cloudflare/cloudflared/logger"
+	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/metrics"
 	"github.com/cloudflare/cloudflared/overwatch"
 	"github.com/cloudflare/cloudflared/tunneldns"
 	"github.com/cloudflare/cloudflared/watcher"
 
-	raven "github.com/getsentry/raven-go"
-	homedir "github.com/mitchellh/go-homedir"
-	cli "github.com/urfave/cli/v2"
-
+	"github.com/getsentry/raven-go"
+	"github.com/mitchellh/go-homedir"
 	"github.com/pkg/errors"
+	"github.com/urfave/cli/v2"
 )
 
 const (
@@ -47,10 +46,7 @@ func main() {
 	metrics.RegisterBuildInfo(BuildTime, Version)
 	raven.SetRelease(Version)
 
-	// Force shutdown channel used by the app. When closed, app must terminate.
-	// Windows service manager closes this channel when it receives shutdown command.
-	shutdownC := make(chan struct{})
-	// Graceful shutdown channel used by the app. When closed, app must terminate.
+	// Graceful shutdown channel used by the app. When closed, app must terminate gracefully.
 	// Windows service manager closes this channel when it receives stop command.
 	graceShutdownC := make(chan struct{})
 
@@ -74,16 +70,18 @@ func main() {
 	app.Version = fmt.Sprintf("%s (built %s)", Version, BuildTime)
 	app.Description = `cloudflared connects your machine or user identity to Cloudflare's global network.
 	You can use it to authenticate a session to reach an API behind Access, route web traffic to this machine,
-	and configure access control.`
+	and configure access control.
+
+	See https://developers.cloudflare.com/argo-tunnel/ for more in-depth documentation.`
 	app.Flags = flags()
-	app.Action = action(Version, shutdownC, graceShutdownC)
+	app.Action = action(graceShutdownC)
 	app.Before = tunnel.SetFlagsFromConfigFile
 	app.Commands = commands(cli.ShowVersion)
 
-	tunnel.Init(Version, shutdownC, graceShutdownC) // we need this to support the tunnel sub command...
-	access.Init(shutdownC, graceShutdownC)
+	tunnel.Init(Version, graceShutdownC) // we need this to support the tunnel sub command...
+	access.Init(graceShutdownC)
 	updater.Init(Version)
-	runApp(app, shutdownC, graceShutdownC)
+	runApp(app, graceShutdownC)
 }
 
 func commands(version func(c *cli.Context)) []*cli.Command {
@@ -144,24 +142,20 @@ func isEmptyInvocation(c *cli.Context) bool {
 	return c.NArg() == 0 && c.NumFlags() == 0
 }
 
-func action(version string, shutdownC, graceShutdownC chan struct{}) cli.ActionFunc {
-	return func(c *cli.Context) (err error) {
+func action(graceShutdownC chan struct{}) cli.ActionFunc {
+	return cliutil.ErrorHandler(func(c *cli.Context) (err error) {
 		if isEmptyInvocation(c) {
-			return handleServiceMode(shutdownC)
+			return handleServiceMode(c, graceShutdownC)
 		}
 		tags := make(map[string]string)
 		tags["hostname"] = c.String("hostname")
 		raven.SetTagsContext(tags)
 		raven.CapturePanic(func() { err = tunnel.TunnelCommand(c) }, nil)
-		exitCode := 0
 		if err != nil {
-			handleError(err)
-			exitCode = 1
+			captureError(err)
 		}
-		// we already handle error printing, so we pass an empty string so we
-		// don't have to print again.
-		return cli.Exit("", exitCode)
-	}
+		return err
+	})
 }
 
 func userHomeDir() (string, error) {
@@ -177,7 +171,7 @@ func userHomeDir() (string, error) {
 }
 
 // In order to keep the amount of noise sent to Sentry low, typical network errors can be filtered out here by a substring match.
-func handleError(err error) {
+func captureError(err error) {
 	errorMessage := err.Error()
 	for _, ignoredErrorMessage := range ignoredErrors {
 		if strings.Contains(errorMessage, ignoredErrorMessage) {
@@ -188,41 +182,34 @@ func handleError(err error) {
 }
 
 // cloudflared was started without any flags
-func handleServiceMode(shutdownC chan struct{}) error {
-	defer log.SharedWriteManager.Shutdown()
-	logDirectory, logLevel := config.FindLogSettings()
-
-	logger, err := log.New(log.DefaultFile(logDirectory), log.LogLevelString(logLevel))
-	if err != nil {
-		return cliutil.PrintLoggerSetupError("error setting up logger", err)
-	}
-	logger.Infof("logging to directory: %s", logDirectory)
+func handleServiceMode(c *cli.Context, shutdownC chan struct{}) error {
+	log := logger.CreateLoggerFromContext(c, logger.DisableTerminalLog)
 
 	// start the main run loop that reads from the config file
 	f, err := watcher.NewFile()
 	if err != nil {
-		logger.Errorf("Cannot load config file: %s", err)
+		log.Err(err).Msg("Cannot load config file")
 		return err
 	}
 
 	configPath := config.FindOrCreateConfigPath()
-	configManager, err := config.NewFileManager(f, configPath, logger)
+	configManager, err := config.NewFileManager(f, configPath, log)
 	if err != nil {
-		logger.Errorf("Cannot setup config file for monitoring: %s", err)
+		log.Err(err).Msg("Cannot setup config file for monitoring")
 		return err
 	}
-	logger.Infof("monitoring config file at: %s", configPath)
+	log.Info().Msgf("monitoring config file at: %s", configPath)
 
 	serviceCallback := func(t string, name string, err error) {
 		if err != nil {
-			logger.Errorf("%s service: %s encountered an error: %s", t, name, err)
+			log.Err(err).Msgf("%s service: %s encountered an error", t, name)
 		}
 	}
 	serviceManager := overwatch.NewAppManager(serviceCallback)
 
-	appService := NewAppService(configManager, serviceManager, shutdownC, logger)
+	appService := NewAppService(configManager, serviceManager, shutdownC, log)
 	if err := appService.Run(); err != nil {
-		logger.Errorf("Failed to start app service: %s", err)
+		log.Err(err).Msg("Failed to start app service")
 		return err
 	}
 	return nil

cmd/cloudflared/path/path.go

@@ -11,8 +11,27 @@ import (
 	"github.com/mitchellh/go-homedir"
 )
 
-// GenerateFilePathFromURL will return a filepath for given access application url
-func GenerateFilePathFromURL(url *url.URL, suffix string) (string, error) {
+// GenerateAppTokenFilePathFromURL will return a filepath for given Access org token
+func GenerateAppTokenFilePathFromURL(url *url.URL, suffix string) (string, error) {
+	configPath, err := getConfigPath()
+	if err != nil {
+		return "", err
+	}
+	name := strings.Replace(fmt.Sprintf("%s%s-%s", url.Hostname(), url.EscapedPath(), suffix), "/", "-", -1)
+	return filepath.Join(configPath, name), nil
+}
+
+// GenerateOrgTokenFilePathFromURL will return a filepath for given Access application token
+func GenerateOrgTokenFilePathFromURL(authDomain string) (string, error) {
+	configPath, err := getConfigPath()
+	if err != nil {
+		return "", err
+	}
+	name := strings.Replace(fmt.Sprintf("%s-org-token", authDomain), "/", "-", -1)
+	return filepath.Join(configPath, name), nil
+}
+
+func getConfigPath() (string, error) {
 	configPath, err := homedir.Expand(config.DefaultConfigSearchDirectories()[0])
 	if err != nil {
 		return "", err
@@ -22,9 +41,5 @@ func GenerateFilePathFromURL(url *url.URL, suffix string) (string, error) {
 		// create config directory if doesn't already exist
 		err = os.Mkdir(configPath, 0700)
 	}
-	if err != nil {
-		return "", err
-	}
-	name := strings.Replace(fmt.Sprintf("%s%s-%s", url.Hostname(), url.EscapedPath(), suffix), "/", "-", -1)
-	return filepath.Join(configPath, name), nil
+	return configPath, err
 }

cmd/cloudflared/token/token.go

@@ -5,22 +5,27 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"net/http"
 	"net/url"
 	"os"
 	"os/signal"
+	"strings"
 	"syscall"
 	"time"
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/path"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/transfer"
-	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/origin"
+
 	"github.com/coreos/go-oidc/jose"
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog"
 )
 
 const (
-	keyName = "token"
+	keyName     = "token"
+	tokenHeader = "CF_Authorization"
 )
 
 type lock struct {
@@ -34,7 +39,7 @@ type signalHandler struct {
 	signals    []os.Signal
 }
 
-type jwtPayload struct {
+type appJWTPayload struct {
 	Aud   []string `json:"aud"`
 	Email string   `json:"email"`
 	Exp   int      `json:"exp"`
@@ -45,7 +50,17 @@ type jwtPayload struct {
 	Subt  string   `json:"sub"`
 }
 
-func (p jwtPayload) isExpired() bool {
+type orgJWTPayload struct {
+	appJWTPayload
+	Aud string `json:"aud"`
+}
+
+type transferServiceResponse struct {
+	AppToken string `json:"app_token"`
+	OrgToken string `json:"org_token"`
+}
+
+func (p appJWTPayload) isExpired() bool {
 	return int(time.Now().Unix()) > p.Exp
 }
 
@@ -83,7 +98,7 @@ func newLock(path string) *lock {
 func (l *lock) Acquire() error {
 	// Intercept SIGINT and SIGTERM to release lock before exiting
 	l.sigHandler.register(func() {
-		l.deleteLockFile()
+		_ = l.deleteLockFile()
 		os.Exit(0)
 	})
 
@@ -129,67 +144,187 @@ func isTokenLocked(lockFilePath string) bool {
 
 // FetchTokenWithRedirect will either load a stored token or generate a new one
 // it appends the full url as the redirect URL to the access cli request if opening the browser
-func FetchTokenWithRedirect(appURL *url.URL, logger logger.Service) (string, error) {
-	return getToken(appURL, false, logger)
+func FetchTokenWithRedirect(appURL *url.URL, log *zerolog.Logger) (string, error) {
+	return getToken(appURL, false, log)
 }
 
 // FetchToken will either load a stored token or generate a new one
 // it appends the host of the appURL as the redirect URL to the access cli request if opening the browser
-func FetchToken(appURL *url.URL, logger logger.Service) (string, error) {
-	return getToken(appURL, true, logger)
+func FetchToken(appURL *url.URL, log *zerolog.Logger) (string, error) {
+	return getToken(appURL, true, log)
 }
 
 // getToken will either load a stored token or generate a new one
-func getToken(appURL *url.URL, useHostOnly bool, logger logger.Service) (string, error) {
-	if token, err := GetTokenIfExists(appURL); token != "" && err == nil {
+func getToken(appURL *url.URL, useHostOnly bool, log *zerolog.Logger) (string, error) {
+	if token, err := GetAppTokenIfExists(appURL); token != "" && err == nil {
 		return token, nil
 	}
 
-	path, err := path.GenerateFilePathFromURL(appURL, keyName)
+	appTokenPath, err := path.GenerateAppTokenFilePathFromURL(appURL, keyName)
 	if err != nil {
-		return "", err
+		return "", errors.Wrap(err, "failed to generate app token file path")
 	}
 
-	fileLock := newLock(path)
-
-	err = fileLock.Acquire()
-	if err != nil {
-		return "", err
+	fileLockAppToken := newLock(appTokenPath)
+	if err = fileLockAppToken.Acquire(); err != nil {
+		return "", errors.Wrap(err, "failed to acquire app token lock")
 	}
-	defer fileLock.Release()
+	defer fileLockAppToken.Release()
 
 	// check to see if another process has gotten a token while we waited for the lock
-	if token, err := GetTokenIfExists(appURL); token != "" && err == nil {
+	if token, err := GetAppTokenIfExists(appURL); token != "" && err == nil {
 		return token, nil
 	}
 
+	// If an app token couldnt be found on disk, check for an org token and attempt to exchange it for an app token.
+	var orgTokenPath string
+	// Get auth domain to format into org token file path
+	if authDomain, err := getAuthDomain(appURL); err != nil {
+		log.Error().Msgf("failed to get auth domain: %s", err)
+	} else {
+		orgToken, err := GetOrgTokenIfExists(authDomain)
+		if err != nil {
+			orgTokenPath, err = path.GenerateOrgTokenFilePathFromURL(authDomain)
+			if err != nil {
+				return "", errors.Wrap(err, "failed to generate org token file path")
+			}
+
+			fileLockOrgToken := newLock(orgTokenPath)
+			if err = fileLockOrgToken.Acquire(); err != nil {
+				return "", errors.Wrap(err, "failed to acquire org token lock")
+			}
+			defer fileLockOrgToken.Release()
+			// check if an org token has been created since the lock was acquired
+			orgToken, err = GetOrgTokenIfExists(authDomain)
+		}
+		if err == nil {
+			if appToken, err := exchangeOrgToken(appURL, orgToken); err != nil {
+				log.Debug().Msgf("failed to exchange org token for app token: %s", err)
+			} else {
+				if err := ioutil.WriteFile(appTokenPath, []byte(appToken), 0600); err != nil {
+					return "", errors.Wrap(err, "failed to write app token to disk")
+				}
+				return appToken, nil
+			}
+		}
+	}
+	return getTokensFromEdge(appURL, appTokenPath, orgTokenPath, useHostOnly, log)
+
+}
+
+// getTokensFromEdge will attempt to use the transfer service to retrieve an app and org token, save them to disk,
+// and return the app token.
+func getTokensFromEdge(appURL *url.URL, appTokenPath, orgTokenPath string, useHostOnly bool, log *zerolog.Logger) (string, error) {
+	// If no org token exists or if it couldnt be exchanged for an app token, then run the transfer service flow.
+
 	// this weird parameter is the resource name (token) and the key/value
 	// we want to send to the transfer service. the key is token and the value
 	// is blank (basically just the id generated in the transfer service)
-	token, err := transfer.Run(appURL, keyName, keyName, "", path, true, useHostOnly, logger)
+	resourceData, err := transfer.Run(appURL, keyName, keyName, "", true, useHostOnly, log)
 	if err != nil {
-		return "", err
+		return "", errors.Wrap(err, "failed to run transfer service")
+	}
+	var resp transferServiceResponse
+	if err = json.Unmarshal(resourceData, &resp); err != nil {
+		return "", errors.Wrap(err, "failed to marshal transfer service response")
 	}
 
-	return string(token), nil
+	// If we were able to get the auth domain and generate an org token path, lets write it to disk.
+	if orgTokenPath != "" {
+		if err := ioutil.WriteFile(orgTokenPath, []byte(resp.OrgToken), 0600); err != nil {
+			return "", errors.Wrap(err, "failed to write org token to disk")
+		}
+	}
+
+	if err := ioutil.WriteFile(appTokenPath, []byte(resp.AppToken), 0600); err != nil {
+		return "", errors.Wrap(err, "failed to write app token to disk")
+	}
+
+	return resp.AppToken, nil
+
 }
 
-// GetTokenIfExists will return the token from local storage if it exists and not expired
-func GetTokenIfExists(url *url.URL) (string, error) {
-	path, err := path.GenerateFilePathFromURL(url, keyName)
-	if err != nil {
-		return "", err
-	}
-	content, err := ioutil.ReadFile(path)
-	if err != nil {
-		return "", err
-	}
-	token, err := jose.ParseJWT(string(content))
-	if err != nil {
-		return "", err
+// getAuthDomain makes a request to the appURL and stops at the first redirect. The 302 location header will contain the
+// auth domain
+func getAuthDomain(appURL *url.URL) (string, error) {
+	client := &http.Client{
+		// do not follow redirects
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			return http.ErrUseLastResponse
+		},
+		Timeout: time.Second * 7,
 	}
 
-	var payload jwtPayload
+	authDomainReq, err := http.NewRequest("HEAD", appURL.String(), nil)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to create auth domain request")
+	}
+	resp, err := client.Do(authDomainReq)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to get auth domain")
+	}
+	resp.Body.Close()
+	location, err := resp.Location()
+	if err != nil {
+		return "", fmt.Errorf("failed to get auth domain. Received status code %d from %s", resp.StatusCode, appURL.String())
+	}
+	return location.Hostname(), nil
+
+}
+
+// exchangeOrgToken attaches an org token to a request to the appURL and returns an app token. This uses the Access SSO
+// flow to automatically generate and return an app token without the login page.
+func exchangeOrgToken(appURL *url.URL, orgToken string) (string, error) {
+	client := &http.Client{
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			// attach org token to login request
+			if strings.Contains(req.URL.Path, "cdn-cgi/access/login") {
+				req.AddCookie(&http.Cookie{Name: tokenHeader, Value: orgToken})
+			}
+			// stop after hitting authorized endpoint since it will contain the app token
+			if strings.Contains(via[len(via)-1].URL.Path, "cdn-cgi/access/authorized") {
+				return http.ErrUseLastResponse
+			}
+			return nil
+		},
+		Timeout: time.Second * 7,
+	}
+
+	appTokenRequest, err := http.NewRequest("HEAD", appURL.String(), nil)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to create app token request")
+	}
+	resp, err := client.Do(appTokenRequest)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to get app token")
+	}
+	resp.Body.Close()
+	var appToken string
+	for _, c := range resp.Cookies() {
+		//if Org token revoked on exchange, getTokensFromEdge instead
+		validAppToken := c.Name == tokenHeader && time.Now().Before(c.Expires)
+		if validAppToken {
+			appToken = c.Value
+			break
+		}
+	}
+
+	if len(appToken) > 0 {
+		return appToken, nil
+	}
+	return "", fmt.Errorf("response from %s did not contain app token", resp.Request.URL.String())
+}
+
+func GetOrgTokenIfExists(authDomain string) (string, error) {
+	path, err := path.GenerateOrgTokenFilePathFromURL(authDomain)
+	if err != nil {
+		return "", err
+	}
+	token, err := getTokenIfExists(path)
+	if err != nil {
+		return "", err
+	}
+	var payload orgJWTPayload
 	err = json.Unmarshal(token.Payload, &payload)
 	if err != nil {
 		return "", err
@@ -199,13 +334,49 @@ func GetTokenIfExists(url *url.URL) (string, error) {
 		err := os.Remove(path)
 		return "", err
 	}
-
 	return token.Encode(), nil
 }
 
+func GetAppTokenIfExists(url *url.URL) (string, error) {
+	path, err := path.GenerateAppTokenFilePathFromURL(url, keyName)
+	if err != nil {
+		return "", err
+	}
+	token, err := getTokenIfExists(path)
+	if err != nil {
+		return "", err
+	}
+	var payload appJWTPayload
+	err = json.Unmarshal(token.Payload, &payload)
+	if err != nil {
+		return "", err
+	}
+
+	if payload.isExpired() {
+		err := os.Remove(path)
+		return "", err
+	}
+	return token.Encode(), nil
+
+}
+
+// GetTokenIfExists will return the token from local storage if it exists and not expired
+func getTokenIfExists(path string) (*jose.JWT, error) {
+	content, err := ioutil.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	token, err := jose.ParseJWT(string(content))
+	if err != nil {
+		return nil, err
+	}
+
+	return &token, nil
+}
+
 // RemoveTokenIfExists removes the a token from local storage if it exists
 func RemoveTokenIfExists(url *url.URL) error {
-	path, err := path.GenerateFilePathFromURL(url, keyName)
+	path, err := path.GenerateAppTokenFilePathFromURL(url, keyName)
 	if err != nil {
 		return err
 	}

cmd/cloudflared/transfer/transfer.go

@@ -3,10 +3,8 @@ package transfer
 import (
 	"bytes"
 	"encoding/base64"
-	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"net/url"
 	"os"
@@ -14,7 +12,8 @@ import (
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/encrypter"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/shell"
-	"github.com/cloudflare/cloudflared/logger"
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog"
 )
 
 const (
@@ -28,7 +27,7 @@ const (
 // The "dance" we refer to is building a HTTP request, opening that in a browser waiting for
 // the user to complete an action, while it long polls in the background waiting for an
 // action to be completed to download the resource.
-func Run(transferURL *url.URL, resourceName, key, value, path string, shouldEncrypt bool, useHostOnly bool, logger logger.Service) ([]byte, error) {
+func Run(transferURL *url.URL, resourceName, key, value string, shouldEncrypt bool, useHostOnly bool, log *zerolog.Logger) ([]byte, error) {
 	encrypterClient, err := encrypter.New("cloudflared_priv.pem", "cloudflared_pub.pem")
 	if err != nil {
 		return nil, err
@@ -49,7 +48,7 @@ func Run(transferURL *url.URL, resourceName, key, value, path string, shouldEncr
 	var resourceData []byte
 
 	if shouldEncrypt {
-		buf, key, err := transferRequest(baseStoreURL+"transfer/"+encrypterClient.PublicKey(), logger)
+		buf, key, err := transferRequest(baseStoreURL+"transfer/"+encrypterClient.PublicKey(), log)
 		if err != nil {
 			return nil, err
 		}
@@ -65,18 +64,15 @@ func Run(transferURL *url.URL, resourceName, key, value, path string, shouldEncr
 
 		resourceData = decrypted
 	} else {
-		buf, _, err := transferRequest(baseStoreURL+encrypterClient.PublicKey(), logger)
+		buf, _, err := transferRequest(baseStoreURL+encrypterClient.PublicKey(), log)
 		if err != nil {
 			return nil, err
 		}
 		resourceData = buf
 	}
 
-	if err := ioutil.WriteFile(path, resourceData, 0600); err != nil {
-		return nil, err
-	}
-
 	return resourceData, nil
+
 }
 
 // BuildRequestURL creates a request suitable for a resource transfer.
@@ -93,23 +89,24 @@ func buildRequestURL(baseURL *url.URL, key, value string, cli, useHostOnly bool)
 		return baseURL.String(), nil
 	}
 	q.Set("redirect_url", baseURL.String()) // we add the token as a query param on both the redirect_url and the main url
+	q.Set("send_org_token", "true")         // indicates that the cli endpoint should return both the org and app token
 	baseURL.RawQuery = q.Encode()           // and this actual baseURL.
 	baseURL.Path = "cdn-cgi/access/cli"
 	return baseURL.String(), nil
 }
 
 // transferRequest downloads the requested resource from the request URL
-func transferRequest(requestURL string, logger logger.Service) ([]byte, string, error) {
+func transferRequest(requestURL string, log *zerolog.Logger) ([]byte, string, error) {
 	client := &http.Client{Timeout: clientTimeout}
 	const pollAttempts = 10
 	// we do "long polling" on the endpoint to get the resource.
 	for i := 0; i < pollAttempts; i++ {
-		buf, key, err := poll(client, requestURL, logger)
+		buf, key, err := poll(client, requestURL, log)
 		if err != nil {
 			return nil, "", err
 		} else if len(buf) > 0 {
 			if err := putSuccess(client, requestURL); err != nil {
-				logger.Errorf("Failed to update resource success: %s", err)
+				log.Err(err).Msg("Failed to update resource success")
 			}
 			return buf, key, nil
 		}
@@ -118,7 +115,7 @@ func transferRequest(requestURL string, logger logger.Service) ([]byte, string,
 }
 
 // poll the endpoint for the request resource, waiting for the user interaction
-func poll(client *http.Client, requestURL string, logger logger.Service) ([]byte, string, error) {
+func poll(client *http.Client, requestURL string, log *zerolog.Logger) ([]byte, string, error) {
 	resp, err := client.Get(requestURL)
 	if err != nil {
 		return nil, "", err
@@ -131,7 +128,7 @@ func poll(client *http.Client, requestURL string, logger logger.Service) ([]byte
 		return nil, "", fmt.Errorf("error on request %d", resp.StatusCode)
 	}
 	if resp.StatusCode != 200 {
-		logger.Info("Waiting for login...")
+		log.Info().Msg("Waiting for login...")
 		return nil, "", nil
 	}
 

cmd/cloudflared/tunnel/cmd.go

@@ -5,46 +5,36 @@ import (
 	"context"
 	"fmt"
 	"io/ioutil"
-	"net"
-	"net/http"
 	"net/url"
 	"os"
 	"reflect"
-	"runtime"
 	"runtime/trace"
-	"strconv"
 	"strings"
 	"sync"
 	"time"
 
-	"github.com/cloudflare/cloudflared/awsuploader"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/ui"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/updater"
+	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/dbconnect"
-	"github.com/cloudflare/cloudflared/h2mux"
-	"github.com/cloudflare/cloudflared/hello"
+	"github.com/cloudflare/cloudflared/ingress"
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/metrics"
 	"github.com/cloudflare/cloudflared/origin"
 	"github.com/cloudflare/cloudflared/signal"
-	"github.com/cloudflare/cloudflared/socks"
-	"github.com/cloudflare/cloudflared/sshlog"
-	"github.com/cloudflare/cloudflared/sshserver"
 	"github.com/cloudflare/cloudflared/tlsconfig"
 	"github.com/cloudflare/cloudflared/tunneldns"
 	"github.com/cloudflare/cloudflared/tunnelstore"
-	"github.com/cloudflare/cloudflared/websocket"
 
 	"github.com/coreos/go-systemd/daemon"
 	"github.com/facebookgo/grace/gracenet"
 	"github.com/getsentry/raven-go"
-	"github.com/gliderlabs/ssh"
-	"github.com/google/uuid"
 	"github.com/mitchellh/go-homedir"
 	"github.com/pkg/errors"
+	"github.com/rs/zerolog"
 	"github.com/urfave/cli/v2"
 	"github.com/urfave/cli/v2/altsrc"
 )
@@ -52,8 +42,6 @@ import (
 const (
 	sentryDSN = "https://56a9c9fa5c364ab28f34b14f35ea0f1b:3e8827f6f9f740738eb11138f7bebb68@sentry.io/189878"
 
-	sshLogFileDirectory = "/usr/local/var/log/cloudflared/"
-
 	// sshPortFlag is the port on localhost the cloudflared ssh server will run on
 	sshPortFlag = "local-ssh-port"
 
@@ -84,26 +72,20 @@ const (
 	// hostKeyPath is the path of the dir to save SSH host keys too
 	hostKeyPath = "host-key-path"
 
-	//sshServerFlag enables cloudflared ssh proxy server
-	sshServerFlag = "ssh-server"
-
-	// socks5Flag is to enable the socks server to deframe
-	socks5Flag = "socks5"
-
-	// bastionFlag is to enable bastion, or jump host, operation
-	bastionFlag = "bastion"
-
 	// uiFlag is to enable launching cloudflared in interactive UI mode
 	uiFlag = "ui"
 
-	logDirectoryFlag = "log-directory"
-
 	debugLevelWarning = "At debug level, request URL, method, protocol, content legnth and header will be logged. " +
 		"Response status, content length and header will also be logged in debug level."
+
+	LogFieldCommand             = "command"
+	LogFieldExpandedPath        = "expandedPath"
+	LogFieldPIDPathname         = "pidPathname"
+	LogFieldTmpTraceFilename    = "tmpTraceFilename"
+	LogFieldTraceOutputFilepath = "traceOutputFilepath"
 )
 
 var (
-	shutdownC      chan struct{}
 	graceShutdownC chan struct{}
 	version        string
 )
@@ -172,7 +154,7 @@ func TunnelCommand(c *cli.Context) error {
 		return err
 	}
 	if name := c.String("name"); name != "" { // Start a named tunnel
-		return runAdhocNamedTunnel(sc, name)
+		return runAdhocNamedTunnel(sc, name, c.String(CredFileFlag))
 	}
 	if ref := config.GetConfiguration().TunnelID; ref != "" {
 		return fmt.Errorf("Use `cloudflared tunnel run` to start tunnel %s", ref)
@@ -182,27 +164,27 @@ func TunnelCommand(c *cli.Context) error {
 	return runClassicTunnel(sc)
 }
 
-func Init(v string, s, g chan struct{}) {
-	version, shutdownC, graceShutdownC = v, s, g
+func Init(ver string, gracefulShutdown chan struct{}) {
+	version, graceShutdownC = ver, gracefulShutdown
 }
 
 // runAdhocNamedTunnel create, route and run a named tunnel in one command
-func runAdhocNamedTunnel(sc *subcommandContext, name string) error {
+func runAdhocNamedTunnel(sc *subcommandContext, name, credentialsOutputPath string) error {
 	tunnel, ok, err := sc.tunnelActive(name)
 	if err != nil || !ok {
-		tunnel, err = sc.create(name)
+		tunnel, err = sc.create(name, credentialsOutputPath)
 		if err != nil {
 			return errors.Wrap(err, "failed to create tunnel")
 		}
 	} else {
-		sc.logger.Infof("Tunnel already created with ID %s", tunnel.ID)
+		sc.log.Info().Str(LogFieldTunnelID, tunnel.ID.String()).Msg("Reusing existing tunnel with this name")
 	}
 
 	if r, ok := routeFromFlag(sc.c); ok {
 		if res, err := sc.route(tunnel.ID, r); err != nil {
-			sc.logger.Errorf("failed to create route, please create it manually. err: %v.", err)
+			sc.log.Err(err).Msg("failed to create route, please create it manually")
 		} else {
-			sc.logger.Infof(res.SuccessSummary())
+			sc.log.Info().Msg(res.SuccessSummary())
 		}
 	}
 
@@ -215,7 +197,7 @@ func runAdhocNamedTunnel(sc *subcommandContext, name string) error {
 
 // runClassicTunnel creates a "classic" non-named tunnel
 func runClassicTunnel(sc *subcommandContext) error {
-	return StartServer(sc.c, version, shutdownC, graceShutdownC, nil, sc.logger, sc.isUIEnabled)
+	return StartServer(sc.c, version, nil, sc.log, sc.isUIEnabled)
 }
 
 func routeFromFlag(c *cli.Context) (tunnelstore.Route, bool) {
@@ -228,82 +210,50 @@ func routeFromFlag(c *cli.Context) (tunnelstore.Route, bool) {
 	return nil, false
 }
 
-func createLogger(c *cli.Context, isTransport bool, disableTerminal bool) (*logger.OutputWriter, error) {
-	var loggerOpts []logger.Option
-
-	logPath := c.String("logfile")
-	if logPath == "" {
-		logPath = c.String(logDirectoryFlag)
-	}
-
-	if logPath != "" {
-		loggerOpts = append(loggerOpts, logger.DefaultFile(logPath))
-	}
-
-	logLevel := c.String("loglevel")
-	if isTransport {
-		logLevel = c.String("transport-loglevel")
-		if logLevel == "" {
-			logLevel = "fatal"
-		}
-	}
-	loggerOpts = append(loggerOpts, logger.LogLevelString(logLevel))
-
-	if disableTerminal {
-		disableOption := logger.DisableTerminal(true)
-		loggerOpts = append(loggerOpts, disableOption)
-	}
-
-	l, err := logger.New(loggerOpts...)
-	if err != nil {
-		return nil, err
-	}
-
-	return l, nil
-}
-
 func StartServer(
 	c *cli.Context,
 	version string,
-	shutdownC,
-	graceShutdownC chan struct{},
-	namedTunnel *origin.NamedTunnelConfig,
-	log logger.Service,
+	namedTunnel *connection.NamedTunnelConfig,
+	log *zerolog.Logger,
 	isUIEnabled bool,
 ) error {
 	_ = raven.SetDSN(sentryDSN)
 	var wg sync.WaitGroup
 	listeners := gracenet.Net{}
 	errC := make(chan error)
-	connectedSignal := signal.New(make(chan struct{}))
-	dnsReadySignal := make(chan struct{})
 
 	if config.GetConfiguration().Source() == "" {
-		log.Infof(config.ErrNoConfigFile.Error())
+		log.Info().Msg(config.ErrNoConfigFile.Error())
 	}
 
 	if c.IsSet("trace-output") {
 		tmpTraceFile, err := ioutil.TempFile("", "trace")
 		if err != nil {
-			log.Errorf("Failed to create new temporary file to save trace output: %s", err)
+			log.Err(err).Msg("Failed to create new temporary file to save trace output")
 		}
 
+		traceLog := log.With().Str(LogFieldTmpTraceFilename, tmpTraceFile.Name()).Logger()
+
 		defer func() {
 			if err := tmpTraceFile.Close(); err != nil {
-				log.Errorf("Failed to close trace output file %s with error: %s", tmpTraceFile.Name(), err)
+				traceLog.Err(err).Msg("Failed to close temporary trace output file")
 			}
-			if err := os.Rename(tmpTraceFile.Name(), c.String("trace-output")); err != nil {
-				log.Errorf("Failed to rename temporary trace output file %s to %s with error: %s", tmpTraceFile.Name(), c.String("trace-output"), err)
+			traceOutputFilepath := c.String("trace-output")
+			if err := os.Rename(tmpTraceFile.Name(), traceOutputFilepath); err != nil {
+				traceLog.
+					Err(err).
+					Str(LogFieldTraceOutputFilepath, traceOutputFilepath).
+					Msg("Failed to rename temporary trace output file")
 			} else {
 				err := os.Remove(tmpTraceFile.Name())
 				if err != nil {
-					log.Errorf("Failed to remove the temporary trace file %s with error: %s", tmpTraceFile.Name(), err)
+					traceLog.Err(err).Msg("Failed to remove the temporary trace file")
 				}
 			}
 		}()
 
 		if err := trace.Start(tmpTraceFile); err != nil {
-			log.Errorf("Failed to start trace: %s", err)
+			traceLog.Err(err).Msg("Failed to start trace")
 			return errors.Wrap(err, "Error starting tracing")
 		}
 		defer trace.Stop()
@@ -313,51 +263,33 @@ func StartServer(
 	buildInfo.Log(log)
 	logClientOptions(c, log)
 
+	// this context drives the server, when it's cancelled tunnel and all other components (origins, dns, etc...) should stop
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	go waitForSignal(graceShutdownC, log)
+
 	if c.IsSet("proxy-dns") {
+		dnsReadySignal := make(chan struct{})
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
-			errC <- runDNSProxyServer(c, dnsReadySignal, shutdownC, log)
+			errC <- runDNSProxyServer(c, dnsReadySignal, ctx.Done(), log)
 		}()
-	} else {
-		close(dnsReadySignal)
+		// Wait for proxy-dns to come up (if used)
+		<-dnsReadySignal
 	}
 
-	// Wait for proxy-dns to come up (if used)
-	<-dnsReadySignal
-
-	metricsListener, err := listeners.Listen("tcp", c.String("metrics"))
-	if err != nil {
-		log.Errorf("Error opening metrics server listener: %s", err)
-		return errors.Wrap(err, "Error opening metrics server listener")
-	}
-	defer metricsListener.Close()
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
-		errC <- metrics.ServeMetrics(metricsListener, shutdownC, log)
-	}()
-
+	connectedSignal := signal.New(make(chan struct{}))
 	go notifySystemd(connectedSignal)
 	if c.IsSet("pidfile") {
 		go writePidFile(connectedSignal, c.String("pidfile"), log)
 	}
 
-	cloudflaredID, err := uuid.NewRandom()
-	if err != nil {
-		log.Errorf("Cannot generate cloudflared ID: %s", err)
-		return err
-	}
-
-	ctx, cancel := context.WithCancel(context.Background())
-	go func() {
-		<-shutdownC
-		cancel()
-	}()
-
 	// update needs to be after DNS proxy is up to resolve equinox server address
 	if updater.IsAutoupdateEnabled(c, log) {
-		log.Infof("Autoupdate frequency is set to %v", c.Duration("autoupdate-freq"))
+		autoupdateFreq := c.Duration("autoupdate-freq")
+		log.Info().Dur("autoupdateFreq", autoupdateFreq).Msg("Autoupdate frequency is set")
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
@@ -370,178 +302,78 @@ func StartServer(
 	if dnsProxyStandAlone(c) {
 		connectedSignal.Notify()
 		// no grace period, handle SIGINT/SIGTERM immediately
-		return waitToShutdown(&wg, errC, shutdownC, graceShutdownC, 0, log)
-	}
-
-	if c.IsSet("hello-world") {
-		log.Infof("hello-world set")
-		helloListener, err := hello.CreateTLSListener("127.0.0.1:")
-		if err != nil {
-			log.Errorf("Cannot start Hello World Server: %s", err)
-			return errors.Wrap(err, "Cannot start Hello World Server")
-		}
-		defer helloListener.Close()
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			_ = hello.StartHelloWorldServer(log, helloListener, shutdownC)
-		}()
-		forceSetFlag(c, "url", "https://"+helloListener.Addr().String())
-	}
-
-	if c.IsSet(sshServerFlag) {
-		if runtime.GOOS != "darwin" && runtime.GOOS != "linux" {
-			msg := fmt.Sprintf("--ssh-server is not supported on %s", runtime.GOOS)
-			log.Error(msg)
-			return errors.New(msg)
-		}
-
-		log.Infof("ssh-server set")
-
-		logManager := sshlog.NewEmptyManager()
-		if c.IsSet(bucketNameFlag) && c.IsSet(regionNameFlag) && c.IsSet(accessKeyIDFlag) && c.IsSet(secretIDFlag) {
-			uploader, err := awsuploader.NewFileUploader(c.String(bucketNameFlag), c.String(regionNameFlag),
-				c.String(accessKeyIDFlag), c.String(secretIDFlag), c.String(sessionTokenIDFlag), c.String(s3URLFlag))
-			if err != nil {
-				msg := "Cannot create uploader for SSH Server"
-				log.Errorf("%s: %s", msg, err)
-				return errors.Wrap(err, msg)
-			}
-
-			if err := os.MkdirAll(sshLogFileDirectory, 0700); err != nil {
-				msg := fmt.Sprintf("Cannot create SSH log file directory %s", sshLogFileDirectory)
-				log.Errorf("%s: %s", msg, err)
-				return errors.Wrap(err, msg)
-			}
-
-			logManager = sshlog.New(sshLogFileDirectory)
-
-			uploadManager := awsuploader.NewDirectoryUploadManager(log, uploader, sshLogFileDirectory, 30*time.Minute, shutdownC)
-			uploadManager.Start()
-		}
-
-		localServerAddress := "127.0.0.1:" + c.String(sshPortFlag)
-		server, err := sshserver.New(logManager, log, version, localServerAddress, c.String("hostname"), c.Path(hostKeyPath), shutdownC, c.Duration(sshIdleTimeoutFlag), c.Duration(sshMaxTimeoutFlag))
-		if err != nil {
-			msg := "Cannot create new SSH Server"
-			log.Errorf("%s: %s", msg, err)
-			return errors.Wrap(err, msg)
-		}
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			if err = server.Start(); err != nil && err != ssh.ErrServerClosed {
-				log.Errorf("SSH server error: %s", err)
-				// TODO: remove when declarative tunnels are implemented.
-				close(shutdownC)
-			}
-		}()
-		forceSetFlag(c, "url", "ssh://"+localServerAddress)
+		return waitToShutdown(&wg, cancel, errC, graceShutdownC, 0, log)
 	}
 
 	url := c.String("url")
 	hostname := c.String("hostname")
 	if url == hostname && url != "" && hostname != "" {
 		errText := "hostname and url shouldn't match. See --help for more information"
-		log.Error(errText)
+		log.Error().Msg(errText)
 		return fmt.Errorf(errText)
 	}
 
-	if staticHost := hostnameFromURI(c.String("url")); isProxyDestinationConfigured(staticHost, c) {
-		listener, err := net.Listen("tcp", net.JoinHostPort(c.String("proxy-address"), strconv.Itoa(c.Int("proxy-port"))))
-		if err != nil {
-			log.Errorf("Cannot start Websocket Proxy Server: %s", err)
-			return errors.Wrap(err, "Cannot start Websocket Proxy Server")
-		}
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			streamHandler := websocket.DefaultStreamHandler
-			if c.IsSet(socks5Flag) {
-				log.Info("SOCKS5 server started")
-				streamHandler = func(wsConn *websocket.Conn, remoteConn net.Conn, _ http.Header) {
-					dialer := socks.NewConnDialer(remoteConn)
-					requestHandler := socks.NewRequestHandler(dialer)
-					socksServer := socks.NewConnectionHandler(requestHandler)
+	logTransport := logger.CreateTransportLoggerFromContext(c, isUIEnabled)
 
-					socksServer.Serve(wsConn)
-				}
-			} else if c.IsSet(sshServerFlag) {
-				streamHandler = func(wsConn *websocket.Conn, remoteConn net.Conn, requestHeaders http.Header) {
-					if finalDestination := requestHeaders.Get(h2mux.CFJumpDestinationHeader); finalDestination != "" {
-						token := requestHeaders.Get(h2mux.CFAccessTokenHeader)
-						if err := websocket.SendSSHPreamble(remoteConn, finalDestination, token); err != nil {
-							log.Errorf("Failed to send SSH preamble: %s", err)
-							return
-						}
-					}
-					websocket.DefaultStreamHandler(wsConn, remoteConn, requestHeaders)
-				}
-			}
-			errC <- websocket.StartProxyServer(log, listener, staticHost, shutdownC, streamHandler)
-		}()
-		forceSetFlag(c, "url", "http://"+listener.Addr().String())
+	observer := connection.NewObserver(log, logTransport, isUIEnabled)
+
+	tunnelConfig, ingressRules, err := prepareTunnelConfig(c, buildInfo, version, log, logTransport, observer, namedTunnel)
+	if err != nil {
+		log.Err(err).Msg("Couldn't start tunnel")
+		return err
 	}
 
-	transportLogger, err := createLogger(c, true, false)
+	metricsListener, err := listeners.Listen("tcp", c.String("metrics"))
 	if err != nil {
-		return errors.Wrap(err, "error setting up transport logger")
+		log.Err(err).Msg("Error opening metrics server listener")
+		return errors.Wrap(err, "Error opening metrics server listener")
 	}
+	defer metricsListener.Close()
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		readinessServer := metrics.NewReadyServer(log)
+		observer.RegisterSink(readinessServer)
+		errC <- metrics.ServeMetrics(metricsListener, ctx.Done(), readinessServer, log)
+	}()
 
-	tunnelConfig, err := prepareTunnelConfig(c, buildInfo, version, log, transportLogger, namedTunnel)
-	if err != nil {
+	if err := ingressRules.StartOrigins(&wg, log, ctx.Done(), errC); err != nil {
 		return err
 	}
 
 	reconnectCh := make(chan origin.ReconnectSignal, 1)
 	if c.IsSet("stdin-control") {
-		log.Info("Enabling control through stdin")
+		log.Info().Msg("Enabling control through stdin")
 		go stdinControl(reconnectCh, log)
 	}
 
 	wg.Add(1)
 	go func() {
-		defer wg.Done()
-		errC <- origin.StartTunnelDaemon(ctx, tunnelConfig, connectedSignal, cloudflaredID, reconnectCh)
+		defer func() {
+			wg.Done()
+			log.Info().Msg("Tunnel server stopped")
+		}()
+		errC <- origin.StartTunnelDaemon(ctx, tunnelConfig, connectedSignal, reconnectCh, graceShutdownC)
 	}()
 
 	if isUIEnabled {
-		const tunnelEventChanBufferSize = 16
-		tunnelEventChan := make(chan ui.TunnelEvent, tunnelEventChanBufferSize)
-		tunnelConfig.TunnelEventChan = tunnelEventChan
-
-		tunnelInfo := ui.NewUIModel(
+		tunnelUI := ui.NewUIModel(
 			version,
 			hostname,
 			metricsListener.Addr().String(),
-			tunnelConfig.OriginUrl,
+			&ingressRules,
 			tunnelConfig.HAConnections,
 		)
-		logLevels, err := logger.ParseLevelString(c.String("loglevel"))
-		if err != nil {
-			return err
-		}
-		tunnelInfo.LaunchUI(ctx, log, logLevels, tunnelEventChan)
+		app := tunnelUI.Launch(ctx, log, logTransport)
+		observer.RegisterSink(app)
 	}
 
-	return waitToShutdown(&wg, errC, shutdownC, graceShutdownC, c.Duration("grace-period"), log)
-}
-
-// forceSetFlag attempts to set the given flag value in the closest context that has it defined
-func forceSetFlag(c *cli.Context, name, value string) {
-	for _, ctx := range c.Lineage() {
-		if err := ctx.Set(name, value); err == nil {
-			break
-		}
-	}
+	return waitToShutdown(&wg, cancel, errC, graceShutdownC, c.Duration("grace-period"), log)
 }
 
 func SetFlagsFromConfigFile(c *cli.Context) error {
 	const exitCode = 1
-	log, err := createLogger(c, false, false)
-	if err != nil {
-		return cliutil.PrintLoggerSetupError("error setting up logger", err)
-	}
+	log := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
 	inputSource, err := config.ReadConfigFile(c, log)
 	if err != nil {
 		if err == config.ErrNoConfigFile {
@@ -559,36 +391,45 @@ func SetFlagsFromConfigFile(c *cli.Context) error {
 	return nil
 }
 
-// isProxyDestinationConfigured returns true if there is a static host set or if bastion mode is set.
-func isProxyDestinationConfigured(staticHost string, c *cli.Context) bool {
-	return staticHost != "" || c.IsSet(bastionFlag)
-}
-
 func waitToShutdown(wg *sync.WaitGroup,
-	errC chan error,
-	shutdownC, graceShutdownC chan struct{},
+	cancelServerContext func(),
+	errC <-chan error,
+	graceShutdownC <-chan struct{},
 	gracePeriod time.Duration,
-	logger logger.Service,
+	log *zerolog.Logger,
 ) error {
 	var err error
-	if gracePeriod > 0 {
-		err = waitForSignalWithGraceShutdown(errC, shutdownC, graceShutdownC, gracePeriod, logger)
-	} else {
-		err = waitForSignal(errC, shutdownC, logger)
-		close(graceShutdownC)
+	select {
+	case err = <-errC:
+		log.Error().Err(err).Msg("Initiating shutdown")
+	case <-graceShutdownC:
+		log.Debug().Msg("Graceful shutdown signalled")
+		if gracePeriod > 0 {
+			// wait for either grace period or service termination
+			select {
+			case <-time.Tick(gracePeriod):
+			case <-errC:
+			}
+		}
 	}
 
-	if err != nil {
-		logger.Errorf("Quitting due to error: %s", err)
-	} else {
-		logger.Info("Quitting...")
-	}
-	// Wait for clean exit, discarding all errors
+	// stop server context
+	cancelServerContext()
+
+	// Wait for clean exit, discarding all errors while we wait
+	stopDiscarding := make(chan struct{})
 	go func() {
-		for range errC {
+		for {
+			select {
+			case <-errC: // ignore
+			case <-stopDiscarding:
+				return
+			}
 		}
 	}()
 	wg.Wait()
+	close(stopDiscarding)
+
 	return err
 }
 
@@ -597,16 +438,16 @@ func notifySystemd(waitForSignal *signal.Signal) {
 	daemon.SdNotify(false, "READY=1")
 }
 
-func writePidFile(waitForSignal *signal.Signal, pidFile string, logger logger.Service) {
+func writePidFile(waitForSignal *signal.Signal, pidPathname string, log *zerolog.Logger) {
 	<-waitForSignal.Wait()
-	expandedPath, err := homedir.Expand(pidFile)
+	expandedPath, err := homedir.Expand(pidPathname)
 	if err != nil {
-		logger.Errorf("Unable to expand %s, try to use absolute path in --pidfile: %s", pidFile, err)
+		log.Err(err).Str(LogFieldPIDPathname, pidPathname).Msg("Unable to expand the path, try to use absolute path in --pidfile")
 		return
 	}
 	file, err := os.Create(expandedPath)
 	if err != nil {
-		logger.Errorf("Unable to write pid to %s: %s", expandedPath, err)
+		log.Err(err).Str(LogFieldExpandedPath, expandedPath).Msg("Unable to write pid")
 		return
 	}
 	defer file.Close()
@@ -698,6 +539,7 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 	flags = append(flags, configureLoggingFlags(shouldHide)...)
 	flags = append(flags, configureProxyDNSFlags(shouldHide)...)
 	flags = append(flags, []cli.Flag{
+		credentialsFileFlag,
 		altsrc.NewBoolFlag(&cli.BoolFlag{
 			Name:   "is-autoupdated",
 			Usage:  "Signal the new process that Argo Tunnel client has been autoupdated",
@@ -778,13 +620,15 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			Value:  time.Second * 5,
 			Hidden: true,
 		}),
-		altsrc.NewUint64Flag(&cli.Uint64Flag{
+		// Note TUN-3758 , we use Int because UInt is not supported with altsrc
+		altsrc.NewIntFlag(&cli.IntFlag{
 			Name:   "heartbeat-count",
 			Usage:  "Minimum number of unacked heartbeats to send before closing the connection.",
 			Value:  5,
 			Hidden: true,
 		}),
-		altsrc.NewUintFlag(&cli.UintFlag{
+		// Note TUN-3758 , we use Int because UInt is not supported with altsrc
+		altsrc.NewIntFlag(&cli.IntFlag{
 			Name:    "retries",
 			Value:   5,
 			Usage:   "Maximum number of retries for connection/protocol errors.",
@@ -803,7 +647,8 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			EnvVars: []string{"TUNNEL_GRACE_PERIOD"},
 			Hidden:  true,
 		}),
-		altsrc.NewUintFlag(&cli.UintFlag{
+		// Note TUN-3758 , we use Int because UInt is not supported with altsrc
+		altsrc.NewIntFlag(&cli.IntFlag{
 			Name:    "compression-quality",
 			Value:   0,
 			Usage:   "(beta) Use cross-stream compression instead HTTP compression. 0-off, 1-low, 2-medium, >=3-high.",
@@ -843,6 +688,7 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			Value:  false,
 			Hidden: shouldHide,
 		}),
+		selectProtocolFlag,
 	}...)
 
 	return flags
@@ -910,67 +756,67 @@ func configureProxyFlags(shouldHide bool) []cli.Flag {
 			Hidden:  shouldHide,
 		}),
 		altsrc.NewBoolFlag(&cli.BoolFlag{
-			Name:    socks5Flag,
+			Name:    ingress.Socks5Flag,
 			Usage:   "specify if this tunnel is running as a SOCK5 Server",
 			EnvVars: []string{"TUNNEL_SOCKS"},
 			Value:   false,
 			Hidden:  shouldHide,
 		}),
 		altsrc.NewDurationFlag(&cli.DurationFlag{
-			Name:   "proxy-connect-timeout",
+			Name:   ingress.ProxyConnectTimeoutFlag,
 			Usage:  "HTTP proxy timeout for establishing a new connection",
 			Value:  time.Second * 30,
 			Hidden: shouldHide,
 		}),
 		altsrc.NewDurationFlag(&cli.DurationFlag{
-			Name:   "proxy-tls-timeout",
+			Name:   ingress.ProxyTLSTimeoutFlag,
 			Usage:  "HTTP proxy timeout for completing a TLS handshake",
 			Value:  time.Second * 10,
 			Hidden: shouldHide,
 		}),
 		altsrc.NewDurationFlag(&cli.DurationFlag{
-			Name:   "proxy-tcp-keepalive",
+			Name:   ingress.ProxyTCPKeepAlive,
 			Usage:  "HTTP proxy TCP keepalive duration",
 			Value:  time.Second * 30,
 			Hidden: shouldHide,
 		}),
 		altsrc.NewBoolFlag(&cli.BoolFlag{
-			Name:   "proxy-no-happy-eyeballs",
+			Name:   ingress.ProxyNoHappyEyeballsFlag,
 			Usage:  "HTTP proxy should disable \"happy eyeballs\" for IPv4/v6 fallback",
 			Hidden: shouldHide,
 		}),
 		altsrc.NewIntFlag(&cli.IntFlag{
-			Name:   "proxy-keepalive-connections",
+			Name:   ingress.ProxyKeepAliveConnectionsFlag,
 			Usage:  "HTTP proxy maximum keepalive connection pool size",
 			Value:  100,
 			Hidden: shouldHide,
 		}),
 		altsrc.NewDurationFlag(&cli.DurationFlag{
-			Name:   "proxy-keepalive-timeout",
+			Name:   ingress.ProxyKeepAliveTimeoutFlag,
 			Usage:  "HTTP proxy timeout for closing an idle connection",
 			Value:  time.Second * 90,
 			Hidden: shouldHide,
 		}),
 		altsrc.NewDurationFlag(&cli.DurationFlag{
 			Name:   "proxy-connection-timeout",
-			Usage:  "HTTP proxy timeout for closing an idle connection",
+			Usage:  "DEPRECATED. No longer has any effect.",
 			Value:  time.Second * 90,
 			Hidden: shouldHide,
 		}),
 		altsrc.NewDurationFlag(&cli.DurationFlag{
 			Name:   "proxy-expect-continue-timeout",
-			Usage:  "HTTP proxy timeout for closing an idle connection",
+			Usage:  "DEPRECATED. No longer has any effect.",
 			Value:  time.Second * 90,
 			Hidden: shouldHide,
 		}),
 		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:    "http-host-header",
+			Name:    ingress.HTTPHostHeaderFlag,
 			Usage:   "Sets the HTTP Host header for the local webserver.",
 			EnvVars: []string{"TUNNEL_HTTP_HOST_HEADER"},
 			Hidden:  shouldHide,
 		}),
 		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:    "origin-server-name",
+			Name:    ingress.OriginServerNameFlag,
 			Usage:   "Hostname on the origin server certificate.",
 			EnvVars: []string{"TUNNEL_ORIGIN_SERVER_NAME"},
 			Hidden:  shouldHide,
@@ -988,13 +834,13 @@ func configureProxyFlags(shouldHide bool) []cli.Flag {
 			Hidden:  shouldHide,
 		}),
 		altsrc.NewBoolFlag(&cli.BoolFlag{
-			Name:    "no-tls-verify",
+			Name:    ingress.NoTLSVerifyFlag,
 			Usage:   "Disables TLS verification of the certificate presented by your origin. Will allow any certificate from the origin to be accepted. Note: The connection from your machine to Cloudflare's Edge is still encrypted.",
 			EnvVars: []string{"NO_TLS_VERIFY"},
 			Hidden:  shouldHide,
 		}),
 		altsrc.NewBoolFlag(&cli.BoolFlag{
-			Name:    "no-chunked-encoding",
+			Name:    ingress.NoChunkedEncodingFlag,
 			Usage:   "Disables chunked transfer encoding; useful if you are running a WSGI server.",
 			EnvVars: []string{"TUNNEL_NO_CHUNKED_ENCODING"},
 			Hidden:  shouldHide,
@@ -1067,28 +913,29 @@ func sshFlags(shouldHide bool) []cli.Flag {
 			Hidden:  true,
 		}),
 		altsrc.NewBoolFlag(&cli.BoolFlag{
-			Name:    sshServerFlag,
+			Name:    ingress.SSHServerFlag,
 			Value:   false,
 			Usage:   "Run an SSH Server",
 			EnvVars: []string{"TUNNEL_SSH_SERVER"},
 			Hidden:  true, // TODO: remove when feature is complete
 		}),
 		altsrc.NewBoolFlag(&cli.BoolFlag{
-			Name:    bastionFlag,
+			Name:    config.BastionFlag,
 			Value:   false,
 			Usage:   "Runs as jump host",
 			EnvVars: []string{"TUNNEL_BASTION"},
 			Hidden:  shouldHide,
 		}),
 		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:    "proxy-address",
+			Name:    ingress.ProxyAddressFlag,
 			Usage:   "Listen address for the proxy.",
 			Value:   "127.0.0.1",
 			EnvVars: []string{"TUNNEL_PROXY_ADDRESS"},
 			Hidden:  shouldHide,
 		}),
+		// Note TUN-3758 , we use Int because UInt is not supported with altsrc
 		altsrc.NewIntFlag(&cli.IntFlag{
-			Name:    "proxy-port",
+			Name:    ingress.ProxyPortFlag,
 			Usage:   "Listen port for the proxy.",
 			Value:   0,
 			EnvVars: []string{"TUNNEL_PROXY_PORT"},
@@ -1100,28 +947,28 @@ func sshFlags(shouldHide bool) []cli.Flag {
 func configureLoggingFlags(shouldHide bool) []cli.Flag {
 	return []cli.Flag{
 		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:    "loglevel",
+			Name:    logger.LogLevelFlag,
 			Value:   "info",
 			Usage:   "Application logging level {fatal, error, info, debug}. " + debugLevelWarning,
 			EnvVars: []string{"TUNNEL_LOGLEVEL"},
 			Hidden:  shouldHide,
 		}),
 		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:    "transport-loglevel",
+			Name:    logger.LogTransportLevelFlag,
 			Aliases: []string{"proto-loglevel"}, // This flag used to be called proto-loglevel
-			Value:   "fatal",
+			Value:   "info",
 			Usage:   "Transport logging level(previously called protocol logging level) {fatal, error, info, debug}",
 			EnvVars: []string{"TUNNEL_PROTO_LOGLEVEL", "TUNNEL_TRANSPORT_LOGLEVEL"},
 			Hidden:  shouldHide,
 		}),
 		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:    "logfile",
+			Name:    logger.LogFileFlag,
 			Usage:   "Save application log to this file for reporting issues.",
 			EnvVars: []string{"TUNNEL_LOGFILE"},
 			Hidden:  shouldHide,
 		}),
 		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:    logDirectoryFlag,
+			Name:    logger.LogDirectoryFlag,
 			Usage:   "Save application log to this directory for reporting issues.",
 			EnvVars: []string{"TUNNEL_LOGDIRECTORY"},
 			Hidden:  shouldHide,
@@ -1165,16 +1012,21 @@ func configureProxyDNSFlags(shouldHide bool) []cli.Flag {
 			Hidden:  shouldHide,
 		}),
 		altsrc.NewStringSliceFlag(&cli.StringSliceFlag{
-			Name:    "proxy-dns-bootstrap",
-			Usage:   "bootstrap endpoint URL, you can specify multiple endpoints for redundancy.",
-			Value:   cli.NewStringSlice("https://162.159.36.1/dns-query", "https://162.159.46.1/dns-query", "https://[2606:4700:4700::1111]/dns-query", "https://[2606:4700:4700::1001]/dns-query"),
+			Name:  "proxy-dns-bootstrap",
+			Usage: "bootstrap endpoint URL, you can specify multiple endpoints for redundancy.",
+			Value: cli.NewStringSlice(
+				"https://162.159.36.1/dns-query",
+				"https://162.159.46.1/dns-query",
+				"https://[2606:4700:4700::1111]/dns-query",
+				"https://[2606:4700:4700::1001]/dns-query",
+			),
 			EnvVars: []string{"TUNNEL_DNS_BOOTSTRAP"},
 			Hidden:  shouldHide,
 		}),
 	}
 }
 
-func stdinControl(reconnectCh chan origin.ReconnectSignal, logger logger.Service) {
+func stdinControl(reconnectCh chan origin.ReconnectSignal, log *zerolog.Logger) {
 	for {
 		scanner := bufio.NewScanner(os.Stdin)
 		for scanner.Scan() {
@@ -1189,17 +1041,17 @@ func stdinControl(reconnectCh chan origin.ReconnectSignal, logger logger.Service
 				if len(parts) > 1 {
 					var err error
 					if reconnect.Delay, err = time.ParseDuration(parts[1]); err != nil {
-						logger.Error(err.Error())
+						log.Error().Msg(err.Error())
 						continue
 					}
 				}
-				logger.Infof("Sending reconnect signal %+v", reconnect)
+				log.Info().Msgf("Sending %+v", reconnect)
 				reconnectCh <- reconnect
 			default:
-				logger.Infof("Unknown command: %s", command)
+				log.Info().Str(LogFieldCommand, command).Msg("Unknown command")
 				fallthrough
 			case "help":
-				logger.Info(`Supported command:
+				log.Info().Msg(`Supported command:
 reconnect [delay]
 - restarts one randomly chosen connection with optional delay before reconnect`)
 			}

cmd/cloudflared/tunnel/configuration.go

@@ -1,21 +1,19 @@
 package tunnel
 
 import (
-	"context"
 	"crypto/tls"
 	"fmt"
 	"io/ioutil"
-	"net"
-	"net/http"
 	"os"
 	"path/filepath"
 	"strings"
-	"time"
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/connection"
+	"github.com/cloudflare/cloudflared/edgediscovery"
+	"github.com/cloudflare/cloudflared/h2mux"
 	"github.com/cloudflare/cloudflared/ingress"
-	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/origin"
 	"github.com/cloudflare/cloudflared/tlsconfig"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
@@ -24,15 +22,20 @@ import (
 	"github.com/google/uuid"
 	"github.com/mitchellh/go-homedir"
 	"github.com/pkg/errors"
+	"github.com/rs/zerolog"
 	"github.com/urfave/cli/v2"
 	"golang.org/x/crypto/ssh/terminal"
 )
 
+const LogFieldOriginCertPath = "originCertPath"
+
 var (
 	developerPortal = "https://developers.cloudflare.com/argo-tunnel"
 	quickStartUrl   = developerPortal + "/quickstart/quickstart/"
 	serviceUrl      = developerPortal + "/reference/service/"
 	argumentsUrl    = developerPortal + "/reference/arguments/"
+
+	LogFieldHostname = "hostname"
 )
 
 // returns the first path that contains a cert.pem file. If none of the DefaultConfigSearchDirectories
@@ -47,30 +50,23 @@ func findDefaultOriginCertPath() string {
 	return ""
 }
 
-func generateRandomClientID(logger logger.Service) (string, error) {
+func generateRandomClientID(log *zerolog.Logger) (string, error) {
 	u, err := uuid.NewRandom()
 	if err != nil {
-		logger.Errorf("couldn't create UUID for client ID %s", err)
+		log.Error().Msgf("couldn't create UUID for client ID %s", err)
 		return "", err
 	}
 	return u.String(), nil
 }
 
-func logClientOptions(c *cli.Context, logger logger.Service) {
+func logClientOptions(c *cli.Context, log *zerolog.Logger) {
 	flags := make(map[string]interface{})
-	for _, flag := range c.LocalFlagNames() {
+	for _, flag := range c.FlagNames() {
 		flags[flag] = c.Generic(flag)
 	}
 
-	sliceFlags := []string{"header", "tag", "proxy-dns-upstream", "upstream", "edge"}
-	for _, sliceFlag := range sliceFlags {
-		if len(c.StringSlice(sliceFlag)) > 0 {
-			flags[sliceFlag] = strings.Join(c.StringSlice(sliceFlag), ", ")
-		}
-	}
-
 	if len(flags) > 0 {
-		logger.Infof("Environment variables %v", flags)
+		log.Info().Msgf("Settings: %v", flags)
 	}
 
 	envs := make(map[string]string)
@@ -85,7 +81,7 @@ func logClientOptions(c *cli.Context, logger logger.Service) {
 		}
 	}
 	if len(envs) > 0 {
-		logger.Infof("Environmental variables %v", envs)
+		log.Info().Msgf("Environmental variables %v", envs)
 	}
 }
 
@@ -93,32 +89,31 @@ func dnsProxyStandAlone(c *cli.Context) bool {
 	return c.IsSet("proxy-dns") && (!c.IsSet("hostname") && !c.IsSet("tag") && !c.IsSet("hello-world"))
 }
 
-func findOriginCert(c *cli.Context, logger logger.Service) (string, error) {
-	originCertPath := c.String("origincert")
+func findOriginCert(originCertPath string, log *zerolog.Logger) (string, error) {
 	if originCertPath == "" {
-		logger.Infof("Cannot determine default origin certificate path. No file %s in %v", config.DefaultCredentialFile, config.DefaultConfigSearchDirectories())
+		log.Info().Msgf("Cannot determine default origin certificate path. No file %s in %v", config.DefaultCredentialFile, config.DefaultConfigSearchDirectories())
 		if isRunningFromTerminal() {
-			logger.Errorf("You need to specify the origin certificate path with --origincert option, or set TUNNEL_ORIGIN_CERT environment variable. See %s for more information.", argumentsUrl)
-			return "", fmt.Errorf("Client didn't specify origincert path when running from terminal")
+			log.Error().Msgf("You need to specify the origin certificate path with --origincert option, or set TUNNEL_ORIGIN_CERT environment variable. See %s for more information.", argumentsUrl)
+			return "", fmt.Errorf("client didn't specify origincert path when running from terminal")
 		} else {
-			logger.Errorf("You need to specify the origin certificate path by specifying the origincert option in the configuration file, or set TUNNEL_ORIGIN_CERT environment variable. See %s for more information.", serviceUrl)
-			return "", fmt.Errorf("Client didn't specify origincert path")
+			log.Error().Msgf("You need to specify the origin certificate path by specifying the origincert option in the configuration file, or set TUNNEL_ORIGIN_CERT environment variable. See %s for more information.", serviceUrl)
+			return "", fmt.Errorf("client didn't specify origincert path")
 		}
 	}
 	var err error
 	originCertPath, err = homedir.Expand(originCertPath)
 	if err != nil {
-		logger.Errorf("Cannot resolve path %s: %s", originCertPath, err)
-		return "", fmt.Errorf("Cannot resolve path %s", originCertPath)
+		log.Err(err).Msgf("Cannot resolve origin certificate path")
+		return "", fmt.Errorf("cannot resolve path %s", originCertPath)
 	}
 	// Check that the user has acquired a certificate using the login command
 	ok, err := config.FileExists(originCertPath)
 	if err != nil {
-		logger.Errorf("Cannot check if origin cert exists at path %s", originCertPath)
-		return "", fmt.Errorf("Cannot check if origin cert exists at path %s", originCertPath)
+		log.Error().Msgf("Cannot check if origin cert exists at path %s", originCertPath)
+		return "", fmt.Errorf("cannot check if origin cert exists at path %s", originCertPath)
 	}
 	if !ok {
-		logger.Errorf(`Cannot find a valid certificate for your origin at the path:
+		log.Error().Msgf(`Cannot find a valid certificate for your origin at the path:
 
     %s
 
@@ -127,29 +122,26 @@ If you don't have a certificate signed by Cloudflare, run the command:
 
 	%s login
 `, originCertPath, os.Args[0])
-		return "", fmt.Errorf("Cannot find a valid certificate at the path %s", originCertPath)
+		return "", fmt.Errorf("cannot find a valid certificate at the path %s", originCertPath)
 	}
 
 	return originCertPath, nil
 }
 
-func readOriginCert(originCertPath string, logger logger.Service) ([]byte, error) {
-	logger.Debugf("Reading origin cert from %s", originCertPath)
-
+func readOriginCert(originCertPath string) ([]byte, error) {
 	// Easier to send the certificate as []byte via RPC than decoding it at this point
 	originCert, err := ioutil.ReadFile(originCertPath)
 	if err != nil {
-		logger.Errorf("Cannot read %s to load origin certificate: %s", originCertPath, err)
-		return nil, fmt.Errorf("Cannot read %s to load origin certificate", originCertPath)
+		return nil, fmt.Errorf("cannot read %s to load origin certificate", originCertPath)
 	}
 	return originCert, nil
 }
 
-func getOriginCert(c *cli.Context, logger logger.Service) ([]byte, error) {
-	if originCertPath, err := findOriginCert(c, logger); err != nil {
+func getOriginCert(originCertPath string, log *zerolog.Logger) ([]byte, error) {
+	if originCertPath, err := findOriginCert(originCertPath, log); err != nil {
 		return nil, err
 	} else {
-		return readOriginCert(originCertPath, logger)
+		return readOriginCert(originCertPath)
 	}
 }
 
@@ -157,73 +149,56 @@ func prepareTunnelConfig(
 	c *cli.Context,
 	buildInfo *buildinfo.BuildInfo,
 	version string,
-	logger logger.Service,
-	transportLogger logger.Service,
-	namedTunnel *origin.NamedTunnelConfig,
-) (*origin.TunnelConfig, error) {
-	compatibilityMode := namedTunnel == nil
+	log, logTransport *zerolog.Logger,
+	observer *connection.Observer,
+	namedTunnel *connection.NamedTunnelConfig,
+) (*origin.TunnelConfig, ingress.Ingress, error) {
+	isNamedTunnel := namedTunnel != nil
 
-	hostname, err := validation.ValidateHostname(c.String("hostname"))
+	configHostname := c.String("hostname")
+	hostname, err := validation.ValidateHostname(configHostname)
 	if err != nil {
-		logger.Errorf("Invalid hostname: %s", err)
-		return nil, errors.Wrap(err, "Invalid hostname")
+		log.Err(err).Str(LogFieldHostname, configHostname).Msg("Invalid hostname")
+		return nil, ingress.Ingress{}, errors.Wrap(err, "Invalid hostname")
 	}
 	isFreeTunnel := hostname == ""
 	clientID := c.String("id")
 	if !c.IsSet("id") {
-		clientID, err = generateRandomClientID(logger)
+		clientID, err = generateRandomClientID(log)
 		if err != nil {
-			return nil, err
+			return nil, ingress.Ingress{}, err
 		}
 	}
 
 	tags, err := NewTagSliceFromCLI(c.StringSlice("tag"))
 	if err != nil {
-		logger.Errorf("Tag parse failure: %s", err)
-		return nil, errors.Wrap(err, "Tag parse failure")
+		log.Err(err).Msg("Tag parse failure")
+		return nil, ingress.Ingress{}, errors.Wrap(err, "Tag parse failure")
 	}
 
 	tags = append(tags, tunnelpogs.Tag{Name: "ID", Value: clientID})
 
 	var originCert []byte
 	if !isFreeTunnel {
-		originCert, err = getOriginCert(c, logger)
+		originCertPath := c.String("origincert")
+		originCertLog := log.With().
+			Str(LogFieldOriginCertPath, originCertPath).
+			Logger()
+
+		originCert, err = getOriginCert(originCertPath, &originCertLog)
 		if err != nil {
-			return nil, errors.Wrap(err, "Error getting origin cert")
+			return nil, ingress.Ingress{}, errors.Wrap(err, "Error getting origin cert")
 		}
 	}
 
-	originCertPool, err := tlsconfig.LoadOriginCA(c, logger)
-	if err != nil {
-		logger.Errorf("Error loading cert pool: %s", err)
-		return nil, errors.Wrap(err, "Error loading cert pool")
-	}
-
-	tunnelMetrics := origin.NewTunnelMetrics()
-	httpTransport := &http.Transport{
-		Proxy:                 http.ProxyFromEnvironment,
-		MaxIdleConns:          c.Int("proxy-keepalive-connections"),
-		MaxIdleConnsPerHost:   c.Int("proxy-keepalive-connections"),
-		IdleConnTimeout:       c.Duration("proxy-keepalive-timeout"),
-		TLSHandshakeTimeout:   c.Duration("proxy-tls-timeout"),
-		ExpectContinueTimeout: 1 * time.Second,
-		TLSClientConfig:       &tls.Config{RootCAs: originCertPool, InsecureSkipVerify: c.IsSet("no-tls-verify")},
-	}
-
-	dialer := &net.Dialer{
-		Timeout:   c.Duration("proxy-connect-timeout"),
-		KeepAlive: c.Duration("proxy-tcp-keepalive"),
-	}
-	if c.Bool("proxy-no-happy-eyeballs") {
-		dialer.FallbackDelay = -1 // As of Golang 1.12, a negative delay disables "happy eyeballs"
-	}
-	dialContext := dialer.DialContext
-
-	var ingressRules ingress.Ingress
-	if namedTunnel != nil {
+	var (
+		ingressRules  ingress.Ingress
+		classicTunnel *connection.ClassicTunnelConfig
+	)
+	if isNamedTunnel {
 		clientUUID, err := uuid.NewRandom()
 		if err != nil {
-			return nil, errors.Wrap(err, "can't generate clientUUID")
+			return nil, ingress.Ingress{}, errors.Wrap(err, "can't generate clientUUID")
 		}
 		namedTunnel.Client = tunnelpogs.ClientInfo{
 			ClientID: clientUUID[:],
@@ -233,103 +208,82 @@ func prepareTunnelConfig(
 		}
 		ingressRules, err = ingress.ParseIngress(config.GetConfiguration())
 		if err != nil && err != ingress.ErrNoIngressRules {
-			return nil, err
+			return nil, ingress.Ingress{}, err
 		}
 		if !ingressRules.IsEmpty() && c.IsSet("url") {
-			return nil, ingress.ErrURLIncompatibleWithIngress
-		}
-	}
-
-	var originURL string
-	if ingressRules.IsEmpty() {
-		originURL, err = config.ValidateUrl(c, compatibilityMode)
-		if err != nil {
-			logger.Errorf("Error validating origin URL: %s", err)
-			return nil, errors.Wrap(err, "Error validating origin URL")
-		}
-	}
-
-	if c.IsSet("unix-socket") {
-		unixSocket, err := config.ValidateUnixSocket(c)
-		if err != nil {
-			logger.Errorf("Error validating --unix-socket: %s", err)
-			return nil, errors.Wrap(err, "Error validating --unix-socket")
-		}
-
-		logger.Infof("Proxying tunnel requests to unix:%s", unixSocket)
-		httpTransport.DialContext = func(ctx context.Context, _, _ string) (net.Conn, error) {
-			// if --unix-socket specified, enforce network type "unix"
-			return dialContext(ctx, "unix", unixSocket)
+			return nil, ingress.Ingress{}, ingress.ErrURLIncompatibleWithIngress
 		}
 	} else {
-		logger.Infof("Proxying tunnel requests to %s", originURL)
-		httpTransport.DialContext = dialContext
-	}
-
-	if !c.IsSet("hello-world") && c.IsSet("origin-server-name") {
-		httpTransport.TLSClientConfig.ServerName = c.String("origin-server-name")
-	}
-	// If tunnel running in bastion mode, a connection to origin will not exist until initiated by the client.
-	if !c.IsSet(bastionFlag) {
-
-		// List all origin URLs that require validation
-		var originURLs []string
-		if ingressRules.IsEmpty() {
-			originURLs = append(originURLs, originURL)
-		} else {
-			for _, rule := range ingressRules.Rules {
-				originURLs = append(originURLs, rule.Service.String())
-			}
-		}
-
-		// Validate each origin URL
-		for _, u := range originURLs {
-			if err = validation.ValidateHTTPService(u, hostname, httpTransport); err != nil {
-				logger.Errorf("unable to connect to the origin: %s", err)
-			}
+		classicTunnel = &connection.ClassicTunnelConfig{
+			Hostname:   hostname,
+			OriginCert: originCert,
+			// turn off use of reconnect token and auth refresh when using named tunnels
+			UseReconnectToken: !isNamedTunnel && c.Bool("use-reconnect-token"),
 		}
 	}
 
-	toEdgeTLSConfig, err := tlsconfig.CreateTunnelConfig(c)
+	// Convert single-origin configuration into multi-origin configuration.
+	if ingressRules.IsEmpty() {
+		ingressRules, err = ingress.NewSingleOrigin(c, !isNamedTunnel)
+		if err != nil {
+			return nil, ingress.Ingress{}, err
+		}
+	}
+
+	protocolSelector, err := connection.NewProtocolSelector(c.String("protocol"), namedTunnel, edgediscovery.HTTP2Percentage, origin.ResolveTTL, log)
 	if err != nil {
-		logger.Errorf("unable to create TLS config to connect with edge: %s", err)
-		return nil, errors.Wrap(err, "unable to create TLS config to connect with edge")
+		return nil, ingress.Ingress{}, err
 	}
-	return &origin.TunnelConfig{
-		BuildInfo:          buildInfo,
-		ClientID:           clientID,
-		ClientTlsConfig:    httpTransport.TLSClientConfig,
-		CompressionQuality: c.Uint64("compression-quality"),
-		EdgeAddrs:          c.StringSlice("edge"),
-		GracePeriod:        c.Duration("grace-period"),
-		HAConnections:      c.Int("ha-connections"),
-		HTTPTransport:      httpTransport,
-		HeartbeatInterval:  c.Duration("heartbeat-interval"),
-		Hostname:           hostname,
-		HTTPHostHeader:     c.String("http-host-header"),
-		IncidentLookup:     origin.NewIncidentLookup(),
-		IsAutoupdated:      c.Bool("is-autoupdated"),
-		IsFreeTunnel:       isFreeTunnel,
-		LBPool:             c.String("lb-pool"),
-		Logger:             logger,
-		TransportLogger:    transportLogger,
-		MaxHeartbeats:      c.Uint64("heartbeat-count"),
-		Metrics:            tunnelMetrics,
+	log.Info().Msgf("Initial protocol %s", protocolSelector.Current())
+
+	edgeTLSConfigs := make(map[connection.Protocol]*tls.Config, len(connection.ProtocolList))
+	for _, p := range connection.ProtocolList {
+		edgeTLSConfig, err := tlsconfig.CreateTunnelConfig(c, p.ServerName())
+		if err != nil {
+			return nil, ingress.Ingress{}, errors.Wrap(err, "unable to create TLS config to connect with edge")
+		}
+		edgeTLSConfigs[p] = edgeTLSConfig
+	}
+
+	originClient := origin.NewClient(ingressRules, tags, log)
+	connectionConfig := &connection.Config{
+		OriginClient:    originClient,
+		GracePeriod:     c.Duration("grace-period"),
+		ReplaceExisting: c.Bool("force"),
+	}
+	muxerConfig := &connection.MuxerConfig{
+		HeartbeatInterval: c.Duration("heartbeat-interval"),
+		// Note TUN-3758 , we use Int because UInt is not supported with altsrc
+		MaxHeartbeats: uint64(c.Int("heartbeat-count")),
+		// Note TUN-3758 , we use Int because UInt is not supported with altsrc
+		CompressionSetting: h2mux.CompressionSetting(uint64(c.Int("compression-quality"))),
 		MetricsUpdateFreq:  c.Duration("metrics-update-freq"),
-		NoChunkedEncoding:  c.Bool("no-chunked-encoding"),
-		OriginCert:         originCert,
-		OriginUrl:          originURL,
-		ReportedVersion:    version,
-		Retries:            c.Uint("retries"),
-		RunFromTerminal:    isRunningFromTerminal(),
-		Tags:               tags,
-		TlsConfig:          toEdgeTLSConfig,
-		NamedTunnel:        namedTunnel,
-		ReplaceExisting:    c.Bool("force"),
-		IngressRules:       ingressRules,
-		// turn off use of reconnect token and auth refresh when using named tunnels
-		UseReconnectToken: compatibilityMode && c.Bool("use-reconnect-token"),
-	}, nil
+	}
+
+	return &origin.TunnelConfig{
+		ConnectionConfig: connectionConfig,
+		BuildInfo:        buildInfo,
+		ClientID:         clientID,
+		EdgeAddrs:        c.StringSlice("edge"),
+		HAConnections:    c.Int("ha-connections"),
+		IncidentLookup:   origin.NewIncidentLookup(),
+		IsAutoupdated:    c.Bool("is-autoupdated"),
+		IsFreeTunnel:     isFreeTunnel,
+		LBPool:           c.String("lb-pool"),
+		Tags:             tags,
+		Log:              log,
+		LogTransport:     logTransport,
+		Observer:         observer,
+		ReportedVersion:  version,
+		// Note TUN-3758 , we use Int because UInt is not supported with altsrc
+		Retries:          uint(c.Int("retries")),
+		RunFromTerminal:  isRunningFromTerminal(),
+		NamedTunnel:      namedTunnel,
+		ClassicTunnel:    classicTunnel,
+		MuxerConfig:      muxerConfig,
+		ProtocolSelector: protocolSelector,
+		EdgeTLSConfigs:   edgeTLSConfigs,
+	}, ingressRules, nil
 }
 
 func isRunningFromTerminal() bool {

cmd/cloudflared/tunnel/credential_finder.go

@@ -0,0 +1,83 @@
+package tunnel
+
+import (
+	"fmt"
+	"path/filepath"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+
+	"github.com/google/uuid"
+	"github.com/rs/zerolog"
+	"github.com/urfave/cli/v2"
+)
+
+// CredFinder can find the tunnel credentials file.
+type CredFinder interface {
+	Path() (string, error)
+}
+
+// Implements CredFinder and looks for the credentials file at the given
+// filepath.
+type staticPath struct {
+	filePath string
+	fs       fileSystem
+}
+
+func newStaticPath(filePath string, fs fileSystem) CredFinder {
+	return staticPath{
+		filePath: filePath,
+		fs:       fs,
+	}
+}
+
+func (a staticPath) Path() (string, error) {
+	if a.filePath != "" && a.fs.validFilePath(a.filePath) {
+		return a.filePath, nil
+	}
+	return "", fmt.Errorf("Tunnel credentials file '%s' doesn't exist or is not a file", a.filePath)
+}
+
+// Implements CredFinder and looks for the credentials file in several directories
+// searching for a file named <id>.json
+type searchByID struct {
+	id  uuid.UUID
+	c   *cli.Context
+	log *zerolog.Logger
+	fs  fileSystem
+}
+
+func newSearchByID(id uuid.UUID, c *cli.Context, log *zerolog.Logger, fs fileSystem) CredFinder {
+	return searchByID{
+		id:  id,
+		c:   c,
+		log: log,
+		fs:  fs,
+	}
+}
+
+func (s searchByID) Path() (string, error) {
+	originCertPath := s.c.String("origincert")
+	originCertLog := s.log.With().
+		Str(LogFieldOriginCertPath, originCertPath).
+		Logger()
+
+	// Fallback to look for tunnel credentials in the origin cert directory
+	if originCertPath, err := findOriginCert(originCertPath, &originCertLog); err == nil {
+		originCertDir := filepath.Dir(originCertPath)
+		if filePath, err := tunnelFilePath(s.id, originCertDir); err == nil {
+			if s.fs.validFilePath(filePath) {
+				return filePath, nil
+			}
+		}
+	}
+
+	// Last resort look under default config directories
+	for _, configDir := range config.DefaultConfigSearchDirectories() {
+		if filePath, err := tunnelFilePath(s.id, configDir); err == nil {
+			if s.fs.validFilePath(filePath) {
+				return filePath, nil
+			}
+		}
+	}
+	return "", fmt.Errorf("tunnel credentials file not found")
+}

cmd/cloudflared/tunnel/filesystem.go

@@ -0,0 +1,27 @@
+package tunnel
+
+import (
+	"io/ioutil"
+	"os"
+)
+
+// Abstract away details of reading files, so that SubcommandContext can read
+// from either the real filesystem, or a mock (when running unit tests).
+type fileSystem interface {
+	readFile(filePath string) ([]byte, error)
+	validFilePath(path string) bool
+}
+
+type realFileSystem struct{}
+
+func (fs realFileSystem) validFilePath(path string) bool {
+	fileStat, err := os.Stat(path)
+	if err != nil {
+		return false
+	}
+	return !fileStat.IsDir()
+}
+
+func (fs realFileSystem) readFile(filePath string) ([]byte, error) {
+	return ioutil.ReadFile(filePath)
+}

cmd/cloudflared/tunnel/ingress_subcommands.go

@@ -4,12 +4,12 @@ import (
 	"fmt"
 	"net/url"
 
-	"github.com/pkg/errors"
-	"github.com/urfave/cli/v2"
-
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
 	"github.com/cloudflare/cloudflared/ingress"
+
+	"github.com/pkg/errors"
+	"github.com/urfave/cli/v2"
 )
 
 func buildIngressSubcommand() *cli.Command {
@@ -70,6 +70,10 @@ func buildTestURLCommand() *cli.Command {
 // validateIngressCommand check the syntax of the ingress rules in the cloudflared config file
 func validateIngressCommand(c *cli.Context) error {
 	conf := config.GetConfiguration()
+	if conf.Source() == "" {
+		fmt.Println("No configuration file was found. Please create one, or use the --config flag to specify its filepath. You can use the help command to learn more about configuration files")
+		return nil
+	}
 	fmt.Println("Validating rules from", conf.Source())
 	if _, err := ingress.ParseIngress(conf); err != nil {
 		return errors.Wrap(err, "Validation failed")
@@ -103,7 +107,7 @@ func testURLCommand(c *cli.Context) error {
 		return errors.Wrap(err, "Validation failed")
 	}
 
-	i := ing.FindMatchingRule(requestURL.Hostname(), requestURL.Path)
+	_, i := ing.FindMatchingRule(requestURL.Hostname(), requestURL.Path)
 	fmt.Printf("Matched rule #%d\n", i+1)
 	fmt.Println(ing.Rules[i].MultiLineString())
 	return nil

cmd/cloudflared/tunnel/login.go

@@ -2,14 +2,15 @@ package tunnel
 
 import (
 	"fmt"
+	"io/ioutil"
 	"net/url"
 	"os"
 	"path/filepath"
 	"syscall"
 
-	homedir "github.com/mitchellh/go-homedir"
+	"github.com/mitchellh/go-homedir"
 	"github.com/pkg/errors"
-	cli "github.com/urfave/cli/v2"
+	"github.com/urfave/cli/v2"
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
@@ -39,10 +40,7 @@ func buildLoginSubcommand(hidden bool) *cli.Command {
 }
 
 func login(c *cli.Context) error {
-	logger, err := logger.New()
-	if err != nil {
-		return errors.Wrap(err, "error setting up logger")
-	}
+	log := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
 
 	path, ok, err := checkForExistingCert()
 	if ok {
@@ -58,12 +56,24 @@ func login(c *cli.Context) error {
 		return err
 	}
 
-	_, err = transfer.Run(loginURL, "cert", "callback", callbackStoreURL, path, false, false, logger)
+	resourceData, err := transfer.Run(
+		loginURL,
+		"cert",
+		"callback",
+		callbackStoreURL,
+		false,
+		false,
+		log,
+	)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "Failed to write the certificate due to the following error:\n%v\n\nYour browser will download the certificate instead. You will have to manually\ncopy it to the following path:\n\n%s\n", err, path)
 		return err
 	}
 
+	if err := ioutil.WriteFile(path, resourceData, 0600); err != nil {
+		return errors.Wrap(err, fmt.Sprintf("error writing cert to %s", path))
+	}
+
 	fmt.Fprintf(os.Stdout, "You have successfully logged in.\nIf you wish to copy your credentials to a server, they have been saved to:\n%s\n", path)
 	return nil
 }

cmd/cloudflared/tunnel/server.go

@@ -1,20 +1,19 @@
 package tunnel
 
 import (
-	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/tunneldns"
 
-	"github.com/urfave/cli/v2"
-
 	"github.com/pkg/errors"
+	"github.com/rs/zerolog"
+	"github.com/urfave/cli/v2"
 )
 
-func runDNSProxyServer(c *cli.Context, dnsReadySignal, shutdownC chan struct{}, logger logger.Service) error {
+func runDNSProxyServer(c *cli.Context, dnsReadySignal chan struct{}, shutdownC <-chan struct{}, log *zerolog.Logger) error {
 	port := c.Int("proxy-dns-port")
 	if port <= 0 || port > 65535 {
 		return errors.New("The 'proxy-dns-port' must be a valid port number in <1, 65535> range.")
 	}
-	listener, err := tunneldns.CreateListener(c.String("proxy-dns-address"), uint16(port), c.StringSlice("proxy-dns-upstream"), c.StringSlice("proxy-dns-bootstrap"), logger)
+	listener, err := tunneldns.CreateListener(c.String("proxy-dns-address"), uint16(port), c.StringSlice("proxy-dns-upstream"), c.StringSlice("proxy-dns-bootstrap"), log)
 	if err != nil {
 		close(dnsReadySignal)
 		listener.Stop()
@@ -26,6 +25,7 @@ func runDNSProxyServer(c *cli.Context, dnsReadySignal, shutdownC chan struct{},
 		return errors.Wrap(err, "Cannot start the DNS over HTTPS proxy server")
 	}
 	<-shutdownC
-	listener.Stop()
+	_ = listener.Stop()
+	log.Info().Msg("DNS server stopped")
 	return nil
 }

cmd/cloudflared/tunnel/signal.go

@@ -4,83 +4,20 @@ import (
 	"os"
 	"os/signal"
 	"syscall"
-	"time"
 
-	"github.com/cloudflare/cloudflared/logger"
+	"github.com/rs/zerolog"
 )
 
-// waitForSignal notifies all routines to shutdownC immediately by closing the
-// shutdownC when one of the routines in main exits, or when this process receives
-// SIGTERM/SIGINT
-func waitForSignal(errC chan error, shutdownC chan struct{}, logger logger.Service) error {
+// waitForSignal closes graceShutdownC to indicate that we should start graceful shutdown sequence
+func waitForSignal(graceShutdownC chan struct{}, logger *zerolog.Logger) {
 	signals := make(chan os.Signal, 10)
 	signal.Notify(signals, syscall.SIGTERM, syscall.SIGINT)
 	defer signal.Stop(signals)
 
 	select {
-	case err := <-errC:
-		logger.Infof("terminating due to error: %v", err)
-		close(shutdownC)
-		return err
 	case s := <-signals:
-		logger.Infof("terminating due to signal %s", s)
-		close(shutdownC)
-	case <-shutdownC:
-	}
-	return nil
-}
-
-// waitForSignalWithGraceShutdown notifies all routines to shutdown immediately
-// by closing the shutdownC when one of the routines in main exits.
-// When this process recieves SIGTERM/SIGINT, it closes the graceShutdownC to
-// notify certain routines to start graceful shutdown. When grace period is over,
-// or when some routine exits, it notifies the rest of the routines to shutdown
-// immediately by closing shutdownC.
-// In the case of handling commands from Windows Service Manager, closing graceShutdownC
-// initiate graceful shutdown.
-func waitForSignalWithGraceShutdown(errC chan error,
-	shutdownC, graceShutdownC chan struct{},
-	gracePeriod time.Duration,
-	logger logger.Service,
-) error {
-	signals := make(chan os.Signal, 10)
-	signal.Notify(signals, syscall.SIGTERM, syscall.SIGINT)
-	defer signal.Stop(signals)
-
-	select {
-	case err := <-errC:
-		logger.Infof("Initiating graceful shutdown due to %v ...", err)
+		logger.Info().Msgf("Initiating graceful shutdown due to signal %s ...", s)
 		close(graceShutdownC)
-		close(shutdownC)
-		return err
-	case s := <-signals:
-		logger.Infof("Initiating graceful shutdown due to signal %s ...", s)
-		close(graceShutdownC)
-		waitForGracePeriod(signals, errC, shutdownC, gracePeriod, logger)
 	case <-graceShutdownC:
-		waitForGracePeriod(signals, errC, shutdownC, gracePeriod, logger)
-	case <-shutdownC:
-		close(graceShutdownC)
 	}
-
-	return nil
-}
-
-func waitForGracePeriod(signals chan os.Signal,
-	errC chan error,
-	shutdownC chan struct{},
-	gracePeriod time.Duration,
-	logger logger.Service,
-) {
-	// Unregister signal handler early, so the client can send a second SIGTERM/SIGINT
-	// to force shutdown cloudflared
-	signal.Stop(signals)
-	graceTimerTick := time.Tick(gracePeriod)
-	// send close signal via shutdownC when grace period expires or when an
-	// error is encountered.
-	select {
-	case <-graceTimerTick:
-	case <-errC:
-	}
-	close(shutdownC)
-}
+}

cmd/cloudflared/tunnel/signal_test.go

@@ -2,11 +2,12 @@ package tunnel
 
 import (
 	"fmt"
+	"sync"
 	"syscall"
 	"testing"
 	"time"
 
-	"github.com/cloudflare/cloudflared/logger"
+	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -18,140 +19,86 @@ var (
 	graceShutdownErr = fmt.Errorf("receive grace shutdown")
 )
 
-func testChannelClosed(t *testing.T, c chan struct{}) {
+func channelClosed(c chan struct{}) bool {
 	select {
 	case <-c:
-		return
+		return true
 	default:
-		t.Fatal("Channel should be closed")
+		return false
 	}
 }
 
-func TestWaitForSignal(t *testing.T) {
-	logger := logger.NewOutputWriter(logger.NewMockWriteManager())
-
-	// Test handling server error
-	errC := make(chan error)
-	shutdownC := make(chan struct{})
-
-	go func() {
-		errC <- serverErr
-	}()
-
-	// received error, shutdownC should be closed
-	err := waitForSignal(errC, shutdownC, logger)
-	assert.Equal(t, serverErr, err)
-	testChannelClosed(t, shutdownC)
+func TestSignalShutdown(t *testing.T) {
+	log := zerolog.Nop()
 
 	// Test handling SIGTERM & SIGINT
 	for _, sig := range []syscall.Signal{syscall.SIGTERM, syscall.SIGINT} {
-		errC = make(chan error)
-		shutdownC = make(chan struct{})
-
-		go func(shutdownC chan struct{}) {
-			<-shutdownC
-			errC <- shutdownErr
-		}(shutdownC)
+		graceShutdownC := make(chan struct{})
 
 		go func(sig syscall.Signal) {
 			// sleep for a tick to prevent sending signal before calling waitForSignal
 			time.Sleep(tick)
-			syscall.Kill(syscall.Getpid(), sig)
+			_ = syscall.Kill(syscall.Getpid(), sig)
 		}(sig)
 
-		err = waitForSignal(errC, shutdownC, logger)
-		assert.Equal(t, nil, err)
-		assert.Equal(t, shutdownErr, <-errC)
-		testChannelClosed(t, shutdownC)
+		time.AfterFunc(time.Second, func() {
+			select {
+			case <-graceShutdownC:
+			default:
+				close(graceShutdownC)
+				t.Fatal("waitForSignal timed out")
+			}
+		})
+
+		waitForSignal(graceShutdownC, &log)
+		assert.True(t, channelClosed(graceShutdownC))
 	}
 }
 
-func TestWaitForSignalWithGraceShutdown(t *testing.T) {
-	// Test server returning error
-	errC := make(chan error)
-	shutdownC := make(chan struct{})
-	graceshutdownC := make(chan struct{})
+func TestWaitForShutdown(t *testing.T) {
+	log := zerolog.Nop()
 
+	errC := make(chan error)
+	graceShutdownC := make(chan struct{})
+	const gracePeriod = 5 * time.Second
+
+	contextCancelled := false
+	cancel := func() {
+		contextCancelled = true
+	}
+	var wg sync.WaitGroup
+
+	// on, error stop immediately
+	contextCancelled = false
+	startTime := time.Now()
 	go func() {
 		errC <- serverErr
 	}()
-
-	logger := logger.NewOutputWriter(logger.NewMockWriteManager())
-
-	// received error, both shutdownC and graceshutdownC should be closed
-	err := waitForSignalWithGraceShutdown(errC, shutdownC, graceshutdownC, tick, logger)
+	err := waitToShutdown(&wg, cancel, errC, graceShutdownC, gracePeriod, &log)
 	assert.Equal(t, serverErr, err)
-	testChannelClosed(t, shutdownC)
-	testChannelClosed(t, graceshutdownC)
+	assert.True(t, contextCancelled)
+	assert.False(t, channelClosed(graceShutdownC))
+	assert.True(t, time.Now().Sub(startTime) < time.Second) // check that wait ended early
 
-	// shutdownC closed, graceshutdownC should also be closed and no error
-	errC = make(chan error)
-	shutdownC = make(chan struct{})
-	graceshutdownC = make(chan struct{})
-	close(shutdownC)
-	err = waitForSignalWithGraceShutdown(errC, shutdownC, graceshutdownC, tick, logger)
-	assert.NoError(t, err)
-	testChannelClosed(t, shutdownC)
-	testChannelClosed(t, graceshutdownC)
+	// on graceful shutdown, ignore error but stop as soon as an error arrives
+	contextCancelled = false
+	startTime = time.Now()
+	go func() {
+		close(graceShutdownC)
+		time.Sleep(tick)
+		errC <- serverErr
+	}()
+	err = waitToShutdown(&wg, cancel, errC, graceShutdownC, gracePeriod, &log)
+	assert.Nil(t, err)
+	assert.True(t, contextCancelled)
+	assert.True(t, time.Now().Sub(startTime) < time.Second) // check that wait ended early
 
-	// graceshutdownC closed, shutdownC should also be closed and no error
-	errC = make(chan error)
-	shutdownC = make(chan struct{})
-	graceshutdownC = make(chan struct{})
-	close(graceshutdownC)
-	err = waitForSignalWithGraceShutdown(errC, shutdownC, graceshutdownC, tick, logger)
-	assert.NoError(t, err)
-	testChannelClosed(t, shutdownC)
-	testChannelClosed(t, graceshutdownC)
-
-	// Test handling SIGTERM & SIGINT
-	for _, sig := range []syscall.Signal{syscall.SIGTERM, syscall.SIGINT} {
-		errC := make(chan error)
-		shutdownC = make(chan struct{})
-		graceshutdownC = make(chan struct{})
-
-		go func(shutdownC, graceshutdownC chan struct{}) {
-			<-graceshutdownC
-			<-shutdownC
-			errC <- graceShutdownErr
-		}(shutdownC, graceshutdownC)
-
-		go func(sig syscall.Signal) {
-			// sleep for a tick to prevent sending signal before calling waitForSignalWithGraceShutdown
-			time.Sleep(tick)
-			syscall.Kill(syscall.Getpid(), sig)
-		}(sig)
-
-		err = waitForSignalWithGraceShutdown(errC, shutdownC, graceshutdownC, tick, logger)
-		assert.Equal(t, nil, err)
-		assert.Equal(t, graceShutdownErr, <-errC)
-		testChannelClosed(t, shutdownC)
-		testChannelClosed(t, graceshutdownC)
-	}
-
-	// Test handling SIGTERM & SIGINT, server send error before end of grace period
-	for _, sig := range []syscall.Signal{syscall.SIGTERM, syscall.SIGINT} {
-		errC := make(chan error)
-		shutdownC = make(chan struct{})
-		graceshutdownC = make(chan struct{})
-
-		go func(shutdownC, graceshutdownC chan struct{}) {
-			<-graceshutdownC
-			errC <- graceShutdownErr
-			<-shutdownC
-			errC <- shutdownErr
-		}(shutdownC, graceshutdownC)
-
-		go func(sig syscall.Signal) {
-			// sleep for a tick to prevent sending signal before calling waitForSignalWithGraceShutdown
-			time.Sleep(tick)
-			syscall.Kill(syscall.Getpid(), sig)
-		}(sig)
-
-		err = waitForSignalWithGraceShutdown(errC, shutdownC, graceshutdownC, tick, logger)
-		assert.Equal(t, nil, err)
-		assert.Equal(t, shutdownErr, <-errC)
-		testChannelClosed(t, shutdownC)
-		testChannelClosed(t, graceshutdownC)
-	}
+	// with graceShutdownC closed stop right away without grace period
+	contextCancelled = false
+	startTime = time.Now()
+	err = waitToShutdown(&wg, cancel, errC, graceShutdownC, 0, &log)
+	assert.Nil(t, err)
+	assert.True(t, contextCancelled)
+	assert.True(t, time.Now().Sub(startTime) < time.Second) // check that wait ended early
 }
+

cmd/cloudflared/tunnel/subcommand_context.go

@@ -3,52 +3,64 @@ package tunnel
 import (
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"os"
-	"path/filepath"
 	"strings"
 
 	"github.com/google/uuid"
 	"github.com/pkg/errors"
+	"github.com/rs/zerolog"
 	"github.com/urfave/cli/v2"
 
 	"github.com/cloudflare/cloudflared/certutil"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/logger"
-	"github.com/cloudflare/cloudflared/origin"
-	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	"github.com/cloudflare/cloudflared/tunnelstore"
 )
 
+type errInvalidJSONCredential struct {
+	err  error
+	path string
+}
+
+func (e errInvalidJSONCredential) Error() string {
+	return "Invalid JSON when parsing tunnel credentials file"
+}
+
 // subcommandContext carries structs shared between subcommands, to reduce number of arguments needed to
 // pass between subcommands, and make sure they are only initialized once
 type subcommandContext struct {
-	c      *cli.Context
-	logger logger.Service
+	c           *cli.Context
+	log         *zerolog.Logger
+	isUIEnabled bool
+	fs          fileSystem
 
 	// These fields should be accessed using their respective Getter
 	tunnelstoreClient tunnelstore.Client
 	userCredential    *userCredential
-
-	isUIEnabled bool
 }
 
 func newSubcommandContext(c *cli.Context) (*subcommandContext, error) {
 	isUIEnabled := c.IsSet(uiFlag) && c.String("name") != ""
 
 	// If UI is enabled, terminal log output should be disabled -- log should be written into a UI log window instead
-	logger, err := createLogger(c, false, isUIEnabled)
-	if err != nil {
-		return nil, errors.Wrap(err, "error setting up logger")
-	}
+	log := logger.CreateLoggerFromContext(c, isUIEnabled)
 
 	return &subcommandContext{
 		c:           c,
-		logger:      logger,
+		log:         log,
 		isUIEnabled: isUIEnabled,
+		fs:          realFileSystem{},
 	}, nil
 }
 
+// Returns something that can find the given tunnel's credentials file.
+func (sc *subcommandContext) credentialFinder(tunnelID uuid.UUID) CredFinder {
+	if path := sc.c.String(CredFileFlag); path != "" {
+		return newStaticPath(path, sc.fs)
+	}
+	return newSearchByID(tunnelID, sc.c, sc.log, sc.fs)
+}
+
 type userCredential struct {
 	cert     *certutil.OriginCert
 	certPath string
@@ -63,7 +75,15 @@ func (sc *subcommandContext) client() (tunnelstore.Client, error) {
 		return nil, err
 	}
 	userAgent := fmt.Sprintf("cloudflared/%s", version)
-	client, err := tunnelstore.NewRESTClient(sc.c.String("api-url"), credential.cert.AccountID, credential.cert.ZoneID, credential.cert.ServiceKey, userAgent, sc.logger)
+	client, err := tunnelstore.NewRESTClient(
+		sc.c.String("api-url"),
+		credential.cert.AccountID,
+		credential.cert.ZoneID,
+		credential.cert.ServiceKey,
+		userAgent,
+		sc.log,
+	)
+
 	if err != nil {
 		return nil, err
 	}
@@ -73,11 +93,16 @@ func (sc *subcommandContext) client() (tunnelstore.Client, error) {
 
 func (sc *subcommandContext) credential() (*userCredential, error) {
 	if sc.userCredential == nil {
-		originCertPath, err := findOriginCert(sc.c, sc.logger)
+		originCertPath := sc.c.String("origincert")
+		originCertLog := sc.log.With().
+			Str(LogFieldOriginCertPath, originCertPath).
+			Logger()
+
+		originCertPath, err := findOriginCert(originCertPath, &originCertLog)
 		if err != nil {
 			return nil, errors.Wrap(err, "Error locating origin cert")
 		}
-		blocks, err := readOriginCert(originCertPath, sc.logger)
+		blocks, err := readOriginCert(originCertPath)
 		if err != nil {
 			return nil, errors.Wrapf(err, "Can't read origin cert from %s", originCertPath)
 		}
@@ -99,53 +124,30 @@ func (sc *subcommandContext) credential() (*userCredential, error) {
 	return sc.userCredential, nil
 }
 
-func (sc *subcommandContext) readTunnelCredentials(tunnelID uuid.UUID) (*pogs.TunnelAuth, error) {
-	filePath, err := sc.tunnelCredentialsPath(tunnelID)
+func (sc *subcommandContext) readTunnelCredentials(credFinder CredFinder) (connection.Credentials, error) {
+	filePath, err := credFinder.Path()
 	if err != nil {
-		return nil, err
+		return connection.Credentials{}, err
 	}
-	body, err := ioutil.ReadFile(filePath)
+	body, err := sc.fs.readFile(filePath)
 	if err != nil {
-		return nil, errors.Wrapf(err, "couldn't read tunnel credentials from %v", filePath)
+		return connection.Credentials{}, errors.Wrapf(err, "couldn't read tunnel credentials from %v", filePath)
 	}
 
-	var auth pogs.TunnelAuth
-	if err = json.Unmarshal(body, &auth); err != nil {
-		return nil, err
+	var credentials connection.Credentials
+	if err = json.Unmarshal(body, &credentials); err != nil {
+		if strings.HasSuffix(filePath, ".pem") {
+			return connection.Credentials{}, fmt.Errorf("The tunnel credentials file should be .json but you gave a .pem. " +
+				"The tunnel credentials file was originally created by `cloudflared tunnel create`. " +
+				"You may have accidentally used the filepath to cert.pem, which is generated by `cloudflared tunnel " +
+				"login`.")
+		}
+		return connection.Credentials{}, errInvalidJSONCredential{path: filePath, err: err}
 	}
-	return &auth, nil
+	return credentials, nil
 }
 
-func (sc *subcommandContext) tunnelCredentialsPath(tunnelID uuid.UUID) (string, error) {
-	if filePath := sc.c.String("credentials-file"); filePath != "" {
-		if validFilePath(filePath) {
-			return filePath, nil
-		}
-		return "", fmt.Errorf("Tunnel credentials file %s doesn't exist or is not a file", filePath)
-	}
-
-	// Fallback to look for tunnel credentials in the origin cert directory
-	if originCertPath, err := findOriginCert(sc.c, sc.logger); err == nil {
-		originCertDir := filepath.Dir(originCertPath)
-		if filePath, err := tunnelFilePath(tunnelID, originCertDir); err == nil {
-			if validFilePath(filePath) {
-				return filePath, nil
-			}
-		}
-	}
-
-	// Last resort look under default config directories
-	for _, configDir := range config.DefaultConfigSearchDirectories() {
-		if filePath, err := tunnelFilePath(tunnelID, configDir); err == nil {
-			if validFilePath(filePath) {
-				return filePath, nil
-			}
-		}
-	}
-	return "", fmt.Errorf("Tunnel credentials file not found")
-}
-
-func (sc *subcommandContext) create(name string) (*tunnelstore.Tunnel, error) {
+func (sc *subcommandContext) create(name string, credentialsOutputPath string) (*tunnelstore.Tunnel, error) {
 	client, err := sc.client()
 	if err != nil {
 		return nil, errors.Wrap(err, "couldn't create client to talk to Argo Tunnel backend")
@@ -165,7 +167,14 @@ func (sc *subcommandContext) create(name string) (*tunnelstore.Tunnel, error) {
 	if err != nil {
 		return nil, err
 	}
-	if writeFileErr := writeTunnelCredentials(tunnel.ID, credential.cert.AccountID, credential.certPath, tunnelSecret, sc.logger); err != nil {
+	tunnelCredentials := connection.Credentials{
+		AccountTag:   credential.cert.AccountID,
+		TunnelSecret: tunnelSecret,
+		TunnelID:     tunnel.ID,
+		TunnelName:   name,
+	}
+	filePath, writeFileErr := writeTunnelCredentials(credential.certPath, credentialsOutputPath, &tunnelCredentials)
+	if writeFileErr != nil {
 		var errorLines []string
 		errorLines = append(errorLines, fmt.Sprintf("Your tunnel '%v' was created with ID %v. However, cloudflared couldn't write to the tunnel credentials file at %v.json.", tunnel.Name, tunnel.ID, tunnel.ID))
 		errorLines = append(errorLines, fmt.Sprintf("The file-writing error is: %v", writeFileErr))
@@ -178,12 +187,13 @@ func (sc *subcommandContext) create(name string) (*tunnelstore.Tunnel, error) {
 		errorMsg := strings.Join(errorLines, "\n")
 		return nil, errors.New(errorMsg)
 	}
+	sc.log.Info().Msgf("Tunnel credentials written to %v. cloudflared chose this file based on where your origin certificate was found. Keep this file secret. To revoke these credentials, delete the tunnel.", filePath)
 
 	if outputFormat := sc.c.String(outputFormatFlag.Name); outputFormat != "" {
 		return nil, renderOutput(outputFormat, &tunnel)
 	}
 
-	sc.logger.Infof("Created tunnel %s with id %s", tunnel.Name, tunnel.ID)
+	sc.log.Info().Msgf("Created tunnel %s with id %s", tunnel.Name, tunnel.ID)
 	return tunnel, nil
 }
 
@@ -228,32 +238,44 @@ func (sc *subcommandContext) delete(tunnelIDs []uuid.UUID) error {
 			return errors.Wrapf(err, "Error deleting tunnel %s", tunnel.ID)
 		}
 
-		tunnelCredentialsPath, err := sc.tunnelCredentialsPath(tunnel.ID)
-		if err != nil {
-			sc.logger.Infof("Cannot locate tunnel credentials to delete, error: %v. Please delete the file manually", err)
-			return nil
-		}
-
-		if err = os.Remove(tunnelCredentialsPath); err != nil {
-			sc.logger.Infof("Cannot delete tunnel credentials, error: %v. Please delete the file manually", err)
+		credFinder := sc.credentialFinder(id)
+		if tunnelCredentialsPath, err := credFinder.Path(); err == nil {
+			if err = os.Remove(tunnelCredentialsPath); err != nil {
+				sc.log.Info().Msgf("Tunnel %v was deleted, but we could not remove its credentials file  %s: %s. Consider deleting this file manually.", id, tunnelCredentialsPath, err)
+			}
 		}
 	}
 	return nil
 }
 
+// findCredentials will choose the right way to find the credentials file, find it,
+// and add the TunnelID into any old credentials (generated before TUN-3581 added the `TunnelID`
+// field to credentials files)
+func (sc *subcommandContext) findCredentials(tunnelID uuid.UUID) (connection.Credentials, error) {
+	credFinder := sc.credentialFinder(tunnelID)
+	credentials, err := sc.readTunnelCredentials(credFinder)
+	// This line ensures backwards compatibility with credentials files generated before
+	// TUN-3581. Those old credentials files don't have a TunnelID field, so we enrich the struct
+	// with the ID, which we have already resolved from the user input.
+	credentials.TunnelID = tunnelID
+	return credentials, err
+}
+
 func (sc *subcommandContext) run(tunnelID uuid.UUID) error {
-	credentials, err := sc.readTunnelCredentials(tunnelID)
+	credentials, err := sc.findCredentials(tunnelID)
 	if err != nil {
+		if e, ok := err.(errInvalidJSONCredential); ok {
+			sc.log.Error().Msgf("The credentials file at %s contained invalid JSON. This is probably caused by passing the wrong filepath. Reminder: the credentials file is a .json file created via `cloudflared tunnel create`.", e.path)
+			sc.log.Error().Msgf("Invalid JSON when parsing credentials file: %s", e.err.Error())
+		}
 		return err
 	}
 
 	return StartServer(
 		sc.c,
 		version,
-		shutdownC,
-		graceShutdownC,
-		&origin.NamedTunnelConfig{Auth: *credentials, ID: tunnelID},
-		sc.logger,
+		&connection.NamedTunnelConfig{Credentials: credentials},
+		sc.log,
 		sc.isUIEnabled,
 	)
 }
@@ -264,9 +286,9 @@ func (sc *subcommandContext) cleanupConnections(tunnelIDs []uuid.UUID) error {
 		return err
 	}
 	for _, tunnelID := range tunnelIDs {
-		sc.logger.Infof("Cleanup connection for tunnel %s", tunnelID)
+		sc.log.Info().Msgf("Cleanup connection for tunnel %s", tunnelID)
 		if err := client.CleanupConnections(tunnelID); err != nil {
-			sc.logger.Errorf("Error cleaning up connections for tunnel %v, error :%v", tunnelID, err)
+			sc.log.Error().Msgf("Error cleaning up connections for tunnel %v, error :%v", tunnelID, err)
 		}
 	}
 	return nil
@@ -281,6 +303,7 @@ func (sc *subcommandContext) route(tunnelID uuid.UUID, r tunnelstore.Route) (tun
 	return client.RouteTunnel(tunnelID, r)
 }
 
+// Query Tunnelstore to find the active tunnel with the given name.
 func (sc *subcommandContext) tunnelActive(name string) (*tunnelstore.Tunnel, bool, error) {
 	filter := tunnelstore.NewFilter()
 	filter.NoDeleted()
@@ -303,6 +326,15 @@ func (sc *subcommandContext) findID(input string) (uuid.UUID, error) {
 		return u, nil
 	}
 
+	// Look up name in the credentials file.
+	credFinder := newStaticPath(sc.c.String(CredFileFlag), sc.fs)
+	if credentials, err := sc.readTunnelCredentials(credFinder); err == nil {
+		if credentials.TunnelID != uuid.Nil && input == credentials.TunnelName {
+			return credentials.TunnelID, nil
+		}
+	}
+
+	// Fall back to querying Tunnelstore.
 	if tunnel, found, err := sc.tunnelActive(input); err != nil {
 		return uuid.Nil, err
 	} else if found {

cmd/cloudflared/tunnel/subcommand_context_teamnet.go

@@ -0,0 +1,42 @@
+package tunnel
+
+import (
+	"net"
+
+	"github.com/cloudflare/cloudflared/teamnet"
+	"github.com/pkg/errors"
+)
+
+const noClientMsg = "error while creating backend client"
+
+func (sc *subcommandContext) listRoutes(filter *teamnet.Filter) ([]*teamnet.DetailedRoute, error) {
+	client, err := sc.client()
+	if err != nil {
+		return nil, errors.Wrap(err, noClientMsg)
+	}
+	return client.ListRoutes(filter)
+}
+
+func (sc *subcommandContext) addRoute(newRoute teamnet.NewRoute) (teamnet.Route, error) {
+	client, err := sc.client()
+	if err != nil {
+		return teamnet.Route{}, errors.Wrap(err, noClientMsg)
+	}
+	return client.AddRoute(newRoute)
+}
+
+func (sc *subcommandContext) deleteRoute(network net.IPNet) error {
+	client, err := sc.client()
+	if err != nil {
+		return errors.Wrap(err, noClientMsg)
+	}
+	return client.DeleteRoute(network)
+}
+
+func (sc *subcommandContext) getRouteByIP(ip net.IP) (teamnet.DetailedRoute, error) {
+	client, err := sc.client()
+	if err != nil {
+		return teamnet.DetailedRoute{}, errors.Wrap(err, noClientMsg)
+	}
+	return client.GetByIP(ip)
+}

cmd/cloudflared/tunnel/subcommand_context_test.go

@@ -1,11 +1,19 @@
 package tunnel
 
 import (
+	"encoding/base64"
+	"flag"
+	"fmt"
+	"github.com/rs/zerolog"
 	"reflect"
 	"testing"
+	"time"
 
+	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/tunnelstore"
 	"github.com/google/uuid"
+	"github.com/pkg/errors"
+	"github.com/urfave/cli/v2"
 )
 
 func Test_findIDs(t *testing.T) {
@@ -80,3 +88,264 @@ func Test_findIDs(t *testing.T) {
 		})
 	}
 }
+
+type mockFileSystem struct {
+	rf  func(string) ([]byte, error)
+	vfp func(string) bool
+}
+
+func (fs mockFileSystem) validFilePath(path string) bool {
+	return fs.vfp(path)
+}
+
+func (fs mockFileSystem) readFile(filePath string) ([]byte, error) {
+	return fs.rf(filePath)
+}
+
+func Test_subcommandContext_findCredentials(t *testing.T) {
+	type fields struct {
+		c                 *cli.Context
+		log               *zerolog.Logger
+		isUIEnabled       bool
+		fs                fileSystem
+		tunnelstoreClient tunnelstore.Client
+		userCredential    *userCredential
+	}
+	type args struct {
+		tunnelID uuid.UUID
+	}
+	oldCertPath := "old_cert.json"
+	newCertPath := "new_cert.json"
+	accountTag := "0000d4d14e84bd4ae5a6a02e0000ac63"
+	secret := []byte{211, 79, 177, 245, 179, 194, 152, 127, 140, 71, 18, 46, 183, 209, 10, 24, 192, 150, 55, 249, 211, 16, 167, 30, 113, 51, 152, 168, 72, 100, 205, 144}
+	secretB64 := base64.StdEncoding.EncodeToString(secret)
+	tunnelID := uuid.MustParse("df5ed608-b8b4-4109-89f3-9f2cf199df64")
+	name := "mytunnel"
+
+	fs := mockFileSystem{
+		rf: func(filePath string) ([]byte, error) {
+			if filePath == oldCertPath {
+				// An old credentials file created before TUN-3581 added the new fields
+				return []byte(fmt.Sprintf(`{"AccountTag":"%s","TunnelSecret":"%s"}`, accountTag, secretB64)), nil
+			}
+			if filePath == newCertPath {
+				// A new credentials file created after TUN-3581 with its new fields.
+				return []byte(fmt.Sprintf(`{"AccountTag":"%s","TunnelSecret":"%s","TunnelID":"%s","TunnelName":"%s"}`, accountTag, secretB64, tunnelID, name)), nil
+			}
+			return nil, errors.New("file not found")
+		},
+		vfp: func(string) bool { return true },
+	}
+	log := zerolog.Nop()
+
+	tests := []struct {
+		name    string
+		fields  fields
+		args    args
+		want    connection.Credentials
+		wantErr bool
+	}{
+		{
+			name: "Filepath given leads to old credentials file",
+			fields: fields{
+				log: &log,
+				fs:  fs,
+				c: func() *cli.Context {
+					flagSet := flag.NewFlagSet("test0", flag.PanicOnError)
+					flagSet.String(CredFileFlag, oldCertPath, "")
+					c := cli.NewContext(cli.NewApp(), flagSet, nil)
+					_ = c.Set(CredFileFlag, oldCertPath)
+					return c
+				}(),
+			},
+			args: args{
+				tunnelID: tunnelID,
+			},
+			want: connection.Credentials{
+				AccountTag:   accountTag,
+				TunnelID:     tunnelID,
+				TunnelSecret: secret,
+			},
+		},
+		{
+			name: "Filepath given leads to new credentials file",
+			fields: fields{
+				log: &log,
+				fs:  fs,
+				c: func() *cli.Context {
+					flagSet := flag.NewFlagSet("test0", flag.PanicOnError)
+					flagSet.String(CredFileFlag, newCertPath, "")
+					c := cli.NewContext(cli.NewApp(), flagSet, nil)
+					_ = c.Set(CredFileFlag, newCertPath)
+					return c
+				}(),
+			},
+			args: args{
+				tunnelID: tunnelID,
+			},
+			want: connection.Credentials{
+				AccountTag:   accountTag,
+				TunnelID:     tunnelID,
+				TunnelSecret: secret,
+				TunnelName:   name,
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			sc := &subcommandContext{
+				c:                 tt.fields.c,
+				log:               tt.fields.log,
+				isUIEnabled:       tt.fields.isUIEnabled,
+				fs:                tt.fields.fs,
+				tunnelstoreClient: tt.fields.tunnelstoreClient,
+				userCredential:    tt.fields.userCredential,
+			}
+			got, err := sc.findCredentials(tt.args.tunnelID)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("subcommandContext.findCredentials() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("subcommandContext.findCredentials() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+type deleteMockTunnelStore struct {
+	tunnelstore.Client
+	mockTunnels      map[uuid.UUID]mockTunnelBehaviour
+	deletedTunnelIDs []uuid.UUID
+}
+
+type mockTunnelBehaviour struct {
+	tunnel     tunnelstore.Tunnel
+	deleteErr  error
+	cleanupErr error
+}
+
+func newDeleteMockTunnelStore(tunnels ...mockTunnelBehaviour) *deleteMockTunnelStore {
+	mockTunnels := make(map[uuid.UUID]mockTunnelBehaviour)
+	for _, tunnel := range tunnels {
+		mockTunnels[tunnel.tunnel.ID] = tunnel
+	}
+	return &deleteMockTunnelStore{
+		mockTunnels:      mockTunnels,
+		deletedTunnelIDs: make([]uuid.UUID, 0),
+	}
+}
+
+func (d *deleteMockTunnelStore) GetTunnel(tunnelID uuid.UUID) (*tunnelstore.Tunnel, error) {
+	tunnel, ok := d.mockTunnels[tunnelID]
+	if !ok {
+		return nil, fmt.Errorf("Couldn't find tunnel: %v", tunnelID)
+	}
+	return &tunnel.tunnel, nil
+}
+
+func (d *deleteMockTunnelStore) DeleteTunnel(tunnelID uuid.UUID) error {
+	tunnel, ok := d.mockTunnels[tunnelID]
+	if !ok {
+		return fmt.Errorf("Couldn't find tunnel: %v", tunnelID)
+	}
+
+	if tunnel.deleteErr != nil {
+		return tunnel.deleteErr
+	}
+
+	d.deletedTunnelIDs = append(d.deletedTunnelIDs, tunnelID)
+	tunnel.tunnel.DeletedAt = time.Now()
+	delete(d.mockTunnels, tunnelID)
+	return nil
+}
+
+func (d *deleteMockTunnelStore) CleanupConnections(tunnelID uuid.UUID) error {
+	tunnel, ok := d.mockTunnels[tunnelID]
+	if !ok {
+		return fmt.Errorf("Couldn't find tunnel: %v", tunnelID)
+	}
+	return tunnel.cleanupErr
+}
+
+func Test_subcommandContext_Delete(t *testing.T) {
+	type fields struct {
+		c                 *cli.Context
+		log               *zerolog.Logger
+		isUIEnabled       bool
+		fs                fileSystem
+		tunnelstoreClient *deleteMockTunnelStore
+		userCredential    *userCredential
+	}
+	type args struct {
+		tunnelIDs []uuid.UUID
+	}
+	newCertPath := "new_cert.json"
+	tunnelID1 := uuid.MustParse("df5ed608-b8b4-4109-89f3-9f2cf199df64")
+	tunnelID2 := uuid.MustParse("af5ed608-b8b4-4109-89f3-9f2cf199df64")
+	log := zerolog.Nop()
+
+	var tests = []struct {
+		name    string
+		fields  fields
+		args    args
+		want    []uuid.UUID
+		wantErr bool
+	}{
+		{
+			name: "clean up continues if credentials are not found",
+			fields: fields{
+				log: &log,
+				fs: mockFileSystem{
+					rf: func(filePath string) ([]byte, error) {
+						return nil, errors.New("file not found")
+					},
+					vfp: func(string) bool { return true },
+				},
+				c: func() *cli.Context {
+					flagSet := flag.NewFlagSet("test0", flag.PanicOnError)
+					flagSet.String(CredFileFlag, newCertPath, "")
+					c := cli.NewContext(cli.NewApp(), flagSet, nil)
+					_ = c.Set(CredFileFlag, newCertPath)
+					return c
+				}(),
+				tunnelstoreClient: newDeleteMockTunnelStore(
+					mockTunnelBehaviour{
+						tunnel: tunnelstore.Tunnel{ID: tunnelID1},
+					},
+					mockTunnelBehaviour{
+						tunnel: tunnelstore.Tunnel{ID: tunnelID2},
+					},
+				),
+			},
+
+			args: args{
+				tunnelIDs: []uuid.UUID{tunnelID1, tunnelID2},
+			},
+			want: []uuid.UUID{tunnelID1, tunnelID2},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			sc := &subcommandContext{
+				c:                 tt.fields.c,
+				log:               tt.fields.log,
+				isUIEnabled:       tt.fields.isUIEnabled,
+				fs:                tt.fields.fs,
+				tunnelstoreClient: tt.fields.tunnelstoreClient,
+				userCredential:    tt.fields.userCredential,
+			}
+			err := sc.delete(tt.args.tunnelIDs)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("subcommandContext.findCredentials() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			got := tt.fields.tunnelstoreClient.deletedTunnelIDs
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("subcommandContext.findCredentials() = %v, want %v", got, tt.want)
+				return
+			}
+		})
+	}
+}

cmd/cloudflared/tunnel/subcommands.go

@@ -23,13 +23,16 @@ import (
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
-	"github.com/cloudflare/cloudflared/logger"
-	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/tunnelstore"
 )
 
 const (
-	credFileFlagAlias = "cred-file"
+	allSortByOptions  = "name, id, createdAt, deletedAt, numConnections"
+	CredFileFlagAlias = "cred-file"
+	CredFileFlag      = "credentials-file"
+
+	LogFieldTunnelID = "tunnelID"
 )
 
 var (
@@ -65,6 +68,17 @@ var (
 		Aliases: []string{"o"},
 		Usage:   "Render output using given `FORMAT`. Valid options are 'json' or 'yaml'",
 	})
+	sortByFlag = &cli.StringFlag{
+		Name:    "sort-by",
+		Value:   "name",
+		Usage:   fmt.Sprintf("Sorts the list of tunnels by the given field. Valid options are {%s}", allSortByOptions),
+		EnvVars: []string{"TUNNEL_LIST_SORT_BY"},
+	}
+	invertSortFlag = &cli.BoolFlag{
+		Name:    "invert-sort",
+		Usage:   "Inverts the sort order of the tunnel list.",
+		EnvVars: []string{"TUNNEL_LIST_INVERT_SORT"},
+	}
 	forceFlag = altsrc.NewBoolFlag(&cli.BoolFlag{
 		Name:    "force",
 		Aliases: []string{"f"},
@@ -74,30 +88,40 @@ var (
 			"tunnels, you can do so with Cloudflare's Load Balancer product.",
 	})
 	credentialsFileFlag = altsrc.NewStringFlag(&cli.StringFlag{
-		Name:    "credentials-file",
-		Aliases: []string{credFileFlagAlias},
-		Usage:   "File path of tunnel credentials",
+		Name:    CredFileFlag,
+		Aliases: []string{CredFileFlagAlias},
+		Usage:   "Filepath at which to read/write the tunnel credentials",
+		EnvVars: []string{"TUNNEL_CRED_FILE"},
 	})
 	forceDeleteFlag = &cli.BoolFlag{
 		Name:    "force",
 		Aliases: []string{"f"},
 		Usage:   "Allows you to delete a tunnel, even if it has active connections.",
+		EnvVars: []string{"TUNNEL_RUN_FORCE_OVERWRITE"},
 	}
+	selectProtocolFlag = altsrc.NewStringFlag(&cli.StringFlag{
+		Name:    "protocol",
+		Value:   "h2mux",
+		Aliases: []string{"p"},
+		Usage:   fmt.Sprintf("Protocol implementation to connect with Cloudflare's edge network. %s", connection.AvailableProtocolFlagMessage),
+		EnvVars: []string{"TUNNEL_TRANSPORT_PROTOCOL"},
+		Hidden:  true,
+	})
 )
 
 func buildCreateCommand() *cli.Command {
 	return &cli.Command{
-		Name:               "create",
-		Action:             cliutil.ErrorHandler(createCommand),
-		Usage:              "Create a new tunnel with given name",
-		UsageText:          "cloudflared tunnel [tunnel command options] create [subcommand options] NAME",
-		Description:        `Creates a tunnel, registers it with Cloudflare edge and generates credential file used to run this tunnel.
-  Use "cloudflared tunnel route" subcommand to map a DNS name to this tunnel and "cloudflared tunnel run" to start the connection. 
+		Name:      "create",
+		Action:    cliutil.ErrorHandler(createCommand),
+		Usage:     "Create a new tunnel with given name",
+		UsageText: "cloudflared tunnel [tunnel command options] create [subcommand options] NAME",
+		Description: `Creates a tunnel, registers it with Cloudflare edge and generates credential file used to run this tunnel.
+  Use "cloudflared tunnel route" subcommand to map a DNS name to this tunnel and "cloudflared tunnel run" to start the connection.
 
   For example, to create a tunnel named 'my-tunnel' run:
 
   $ cloudflared tunnel create my-tunnel`,
-		Flags:              []cli.Flag{outputFormatFlag},
+		Flags:              []cli.Flag{outputFormatFlag, credentialsFileFlag},
 		CustomHelpTemplate: commandHelpTemplate(),
 	}
 }
@@ -120,7 +144,7 @@ func createCommand(c *cli.Context) error {
 	}
 	name := c.Args().First()
 
-	_, err = sc.create(name)
+	_, err = sc.create(name, c.String(CredFileFlag))
 	return errors.Wrap(err, "failed to create tunnel")
 }
 
@@ -130,40 +154,46 @@ func tunnelFilePath(tunnelID uuid.UUID, directory string) (string, error) {
 	return homedir.Expand(filePath)
 }
 
-func writeTunnelCredentials(tunnelID uuid.UUID, accountID, originCertPath string, tunnelSecret []byte, logger logger.Service) error {
-	originCertDir := filepath.Dir(originCertPath)
-	filePath, err := tunnelFilePath(tunnelID, originCertDir)
-	if err != nil {
-		return err
+// If an `outputFile` is given, write the credentials there.
+// Otherwise, write it to the same directory as the originCert,
+// with the filename `<tunnel id>.json`.
+func writeTunnelCredentials(
+	originCertPath, outputFile string,
+	credentials *connection.Credentials,
+) (filePath string, err error) {
+	filePath = outputFile
+	if outputFile == "" {
+		originCertDir := filepath.Dir(originCertPath)
+		filePath, err = tunnelFilePath(credentials.TunnelID, originCertDir)
 	}
-	body, err := json.Marshal(pogs.TunnelAuth{
-		AccountTag:   accountID,
-		TunnelSecret: tunnelSecret,
-	})
 	if err != nil {
-		return errors.Wrap(err, "Unable to marshal tunnel credentials to JSON")
+		return "", err
 	}
-	logger.Infof("Writing tunnel credentials to %v. cloudflared chose this file based on where your origin certificate was found.", filePath)
-	logger.Infof("Keep this file secret. To revoke these credentials, delete the tunnel.")
-	return ioutil.WriteFile(filePath, body, 400)
-}
-
-func validFilePath(path string) bool {
-	fileStat, err := os.Stat(path)
+	// Write the name and ID to the file too
+	body, err := json.Marshal(credentials)
 	if err != nil {
-		return false
+		return "", errors.Wrap(err, "Unable to marshal tunnel credentials to JSON")
 	}
-	return !fileStat.IsDir()
+	return filePath, ioutil.WriteFile(filePath, body, 400)
 }
 
 func buildListCommand() *cli.Command {
 	return &cli.Command{
-		Name:               "list",
-		Action:             cliutil.ErrorHandler(listCommand),
-		Usage:              "List existing tunnels",
-		UsageText:          "cloudflared tunnel [tunnel command options] list [subcommand options]",
-		Description:        "cloudflared tunnel list will display all active tunnels, their created time and associated connections. Use -d flag to include deleted tunnels. See the list of options to filter the list",
-		Flags:              []cli.Flag{outputFormatFlag, showDeletedFlag, listNameFlag, listExistedAtFlag, listIDFlag, showRecentlyDisconnected},
+		Name:        "list",
+		Action:      cliutil.ErrorHandler(listCommand),
+		Usage:       "List existing tunnels",
+		UsageText:   "cloudflared tunnel [tunnel command options] list [subcommand options]",
+		Description: "cloudflared tunnel list will display all active tunnels, their created time and associated connections. Use -d flag to include deleted tunnels. See the list of options to filter the list",
+		Flags: []cli.Flag{
+			outputFormatFlag,
+			showDeletedFlag,
+			listNameFlag,
+			listExistedAtFlag,
+			listIDFlag,
+			showRecentlyDisconnected,
+			sortByFlag,
+			invertSortFlag,
+		},
 		CustomHelpTemplate: commandHelpTemplate(),
 	}
 }
@@ -197,19 +227,49 @@ func listCommand(c *cli.Context) error {
 		return err
 	}
 
+	// Sort the tunnels
+	sortBy := c.String("sort-by")
+	invalidSortField := false
+	sort.Slice(tunnels, func(i, j int) bool {
+		cmp := func() bool {
+			switch sortBy {
+			case "name":
+				return tunnels[i].Name < tunnels[j].Name
+			case "id":
+				return tunnels[i].ID.String() < tunnels[j].ID.String()
+			case "createdAt":
+				return tunnels[i].CreatedAt.Unix() < tunnels[j].CreatedAt.Unix()
+			case "deletedAt":
+				return tunnels[i].DeletedAt.Unix() < tunnels[j].DeletedAt.Unix()
+			case "numConnections":
+				return len(tunnels[i].Connections) < len(tunnels[j].Connections)
+			default:
+				invalidSortField = true
+				return tunnels[i].Name < tunnels[j].Name
+			}
+		}()
+		if c.Bool("invert-sort") {
+			return !cmp
+		}
+		return cmp
+	})
+	if invalidSortField {
+		sc.log.Error().Msgf("%s is not a valid sort field. Valid sort fields are %s. Defaulting to 'name'.", sortBy, allSortByOptions)
+	}
+
 	if outputFormat := c.String(outputFormatFlag.Name); outputFormat != "" {
 		return renderOutput(outputFormat, tunnels)
 	}
 
 	if len(tunnels) > 0 {
-		fmtAndPrintTunnelList(tunnels, c.Bool("show-recently-disconnected"))
+		formatAndPrintTunnelList(tunnels, c.Bool("show-recently-disconnected"))
 	} else {
 		fmt.Println("You have no tunnels, use 'cloudflared tunnel create' to define a new tunnel")
 	}
 	return nil
 }
 
-func fmtAndPrintTunnelList(tunnels []*tunnelstore.Tunnel, showRecentlyDisconnected bool) {
+func formatAndPrintTunnelList(tunnels []*tunnelstore.Tunnel, showRecentlyDisconnected bool) {
 	const (
 		minWidth = 0
 		tabWidth = 8
@@ -222,7 +282,7 @@ func fmtAndPrintTunnelList(tunnels []*tunnelstore.Tunnel, showRecentlyDisconnect
 	defer writer.Flush()
 
 	// Print column headers with tabbed columns
-	fmt.Fprintln(writer, "ID\tNAME\tCREATED\tCONNECTIONS\t")
+	_, _ = fmt.Fprintln(writer, "ID\tNAME\tCREATED\tCONNECTIONS\t")
 
 	// Loop through tunnels, create formatted string for each, and print using tabwriter
 	for _, t := range tunnels {
@@ -233,7 +293,7 @@ func fmtAndPrintTunnelList(tunnels []*tunnelstore.Tunnel, showRecentlyDisconnect
 			t.CreatedAt.Format(time.RFC3339),
 			fmtConnections(t.Connections, showRecentlyDisconnected),
 		)
-		fmt.Fprintln(writer, formattedStr)
+		_, _ = fmt.Fprintln(writer, formattedStr)
 	}
 }
 
@@ -309,6 +369,7 @@ func buildRunCommand() *cli.Command {
 	flags := []cli.Flag{
 		forceFlag,
 		credentialsFileFlag,
+		selectProtocolFlag,
 	}
 	flags = append(flags, configureProxyFlags(false)...)
 	return &cli.Command{
@@ -317,13 +378,13 @@ func buildRunCommand() *cli.Command {
 		Before:    SetFlagsFromConfigFile,
 		Usage:     "Proxy a local web server by running the given tunnel",
 		UsageText: "cloudflared tunnel [tunnel command options] run [subcommand options] [TUNNEL]",
-		Description: `Runs the tunnel identified by name or UUUD, creating highly available connections 
+		Description: `Runs the tunnel identified by name or UUUD, creating highly available connections
   between your server and the Cloudflare edge. You can provide name or UUID of tunnel to run either as the
   last command line argument or in the configuration file using "tunnel: TUNNEL".
 
-  This command requires the tunnel credentials file created when "cloudflared tunnel create" was run, 
+  This command requires the tunnel credentials file created when "cloudflared tunnel create" was run,
   however it does not need access to cert.pem from "cloudflared login" if you identify the tunnel by UUID.
-  If you experience other problems running the tunnel, "cloudflared tunnel cleanup" may help by removing 
+  If you experience other problems running the tunnel, "cloudflared tunnel cleanup" may help by removing
   any old connection records.
 `,
 		Flags:              flags,
@@ -358,7 +419,7 @@ func runNamedTunnel(sc *subcommandContext, tunnelRef string) error {
 		return errors.Wrap(err, "error parsing tunnel ID")
 	}
 
-	sc.logger.Infof("Starting tunnel %s", tunnelID.String())
+	sc.log.Info().Str(LogFieldTunnelID, tunnelID.String()).Msg("Starting tunnel")
 
 	return sc.run(tunnelID)
 }
@@ -405,6 +466,9 @@ func buildRouteCommand() *cli.Command {
    To use this tunnel as a load balancer origin, creating pool and load balancer if necessary:
       cloudflared tunnel route lb <tunnel ID> <load balancer name> <load balancer pool>`,
 		CustomHelpTemplate: commandHelpTemplate(),
+		Subcommands: []*cli.Command{
+			buildRouteIPSubcommand(),
+		},
 	}
 }
 
@@ -419,7 +483,7 @@ func dnsRouteFromArg(c *cli.Context) (tunnelstore.Route, error) {
 	userHostname := c.Args().Get(userHostnameIndex)
 	if userHostname == "" {
 		return nil, cliutil.UsageError("The third argument should be the hostname")
-	} else if !validateHostname(userHostname) {
+	} else if !validateHostname(userHostname, true) {
 		return nil, errors.Errorf("%s is not a valid hostname", userHostname)
 	}
 	return tunnelstore.NewDNSRoute(userHostname), nil
@@ -437,14 +501,14 @@ func lbRouteFromArg(c *cli.Context) (tunnelstore.Route, error) {
 	lbName := c.Args().Get(lbNameIndex)
 	if lbName == "" {
 		return nil, cliutil.UsageError("The third argument should be the load balancer name")
-	} else if !validateHostname(lbName) {
+	} else if !validateHostname(lbName, true) {
 		return nil, errors.Errorf("%s is not a valid load balancer name", lbName)
 	}
 
 	lbPool := c.Args().Get(lbPoolIndex)
 	if lbPool == "" {
 		return nil, cliutil.UsageError("The fourth argument should be the pool name")
-	} else if !validateName(lbPool) {
+	} else if !validateName(lbPool, false) {
 		return nil, errors.Errorf("%s is not a valid pool name", lbPool)
 	}
 
@@ -452,19 +516,23 @@ func lbRouteFromArg(c *cli.Context) (tunnelstore.Route, error) {
 }
 
 var nameRegex = regexp.MustCompile("^[_a-zA-Z0-9][-_.a-zA-Z0-9]*$")
+var hostNameRegex = regexp.MustCompile("^[*_a-zA-Z0-9][-_.a-zA-Z0-9]*$")
 
-func validateName(s string) bool {
+func validateName(s string, allowWildcardSubdomain bool) bool {
+	if allowWildcardSubdomain {
+		return hostNameRegex.MatchString(s)
+	}
 	return nameRegex.MatchString(s)
 }
 
-func validateHostname(s string) bool {
+func validateHostname(s string, allowWildcardSubdomain bool) bool {
 	// Slightly stricter than PunyCodeProfile
 	idnaProfile := idna.New(
 		idna.ValidateLabels(true),
 		idna.VerifyDNSLength(true))
 
 	puny, err := idnaProfile.ToASCII(s)
-	return err == nil && validateName(puny)
+	return err == nil && validateName(puny, allowWildcardSubdomain)
 }
 
 func routeCommand(c *cli.Context) error {
@@ -509,7 +577,7 @@ func routeCommand(c *cli.Context) error {
 		return err
 	}
 
-	sc.logger.Infof(res.SuccessSummary())
+	sc.log.Info().Str(LogFieldTunnelID, tunnelID.String()).Msg(res.SuccessSummary())
 	return nil
 }
 
@@ -523,13 +591,13 @@ func commandHelpTemplate() string {
 	}
 	const template = `NAME:
 	{{.HelpName}} - {{.Usage}}
-	
+
 USAGE:
 	{{.UsageText}}
- 
+
 DESCRIPTION:
 	{{.Description}}
- 
+
 TUNNEL COMMAND OPTIONS:
 	%s
 SUBCOMMAND OPTIONS:

cmd/cloudflared/tunnel/subcommands_test.go

@@ -6,9 +6,8 @@ import (
 	"testing"
 
 	"github.com/cloudflare/cloudflared/tunnelstore"
-	"github.com/mitchellh/go-homedir"
-
 	"github.com/google/uuid"
+	"github.com/mitchellh/go-homedir"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -117,8 +116,64 @@ func TestValidateName(t *testing.T) {
 		{name: "_ab_c.-d-ef", want: true},
 	}
 	for _, tt := range tests {
-		if got := validateName(tt.name); got != tt.want {
+		if got := validateName(tt.name, false); got != tt.want {
 			t.Errorf("validateName() = %v, want %v", got, tt.want)
 		}
 	}
-}
+}
+
+func Test_validateHostname(t *testing.T) {
+	type args struct {
+		s                      string
+		allowWildcardSubdomain bool
+	}
+	tests := []struct {
+		name string
+		args args
+		want bool
+	}{
+		{
+			name: "Normal",
+			args: args{
+				s:                      "example.com",
+				allowWildcardSubdomain: true,
+			},
+			want: true,
+		},
+		{
+			name: "wildcard subdomain for TUN-358",
+			args: args{
+				s:                      "*.ehrig.io",
+				allowWildcardSubdomain: true,
+			},
+			want: true,
+		},
+		{
+			name: "Misplaced wildcard",
+			args: args{
+				s:                      "subdomain.*.ehrig.io",
+				allowWildcardSubdomain: true,
+			},
+		},
+		{
+			name: "Invalid domain",
+			args: args{
+				s:                      "..",
+				allowWildcardSubdomain: true,
+			},
+		},
+		{
+			name: "Invalid domain",
+			args: args{
+				s: "..",
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := validateHostname(tt.args.s, tt.args.allowWildcardSubdomain); got != tt.want {
+				t.Errorf("validateHostname() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

cmd/cloudflared/tunnel/teamnet_subcommands.go

@@ -0,0 +1,198 @@
+package tunnel
+
+import (
+	"fmt"
+	"net"
+	"os"
+	"text/tabwriter"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
+	"github.com/cloudflare/cloudflared/teamnet"
+	"github.com/pkg/errors"
+
+	"github.com/urfave/cli/v2"
+)
+
+func buildRouteIPSubcommand() *cli.Command {
+	return &cli.Command{
+		Name:      "ip",
+		Category:  "Tunnel",
+		Usage:     "Configure and query private routes",
+		UsageText: "cloudflared tunnel [--config FILEPATH] route COMMAND [arguments...]",
+		Hidden:    true,
+		Description: `cloudflared can provision private routes from your private IP space to origins
+		in your corporate network. Users enrolled in your Cloudflare for Teams organization can reach
+		those routes through the Cloudflare Warp client. You can also build rules to determine who
+		can reach certain routes.
+		`,
+		Subcommands: []*cli.Command{
+			{
+				Name:      "add",
+				Action:    cliutil.ErrorHandler(addRouteCommand),
+				Usage:     "Add a new Teamnet route to the table",
+				UsageText: "cloudflared tunnel [--config FILEPATH] route ip add [CIDR] [TUNNEL] [COMMENT?]",
+				Description: `Adds a private route to a CIDR in your private IP space. Requests will
+				be sent through the Cloudflare Warp client running on a user's machine, proxied
+				through the specified tunnel, and reach an IP in the given CIDR.`,
+			},
+			{
+				Name:        "show",
+				Action:      cliutil.ErrorHandler(showRoutesCommand),
+				Usage:       "Show the routing table",
+				UsageText:   "cloudflared tunnel [--config FILEPATH] route ip show [flags]",
+				Description: `Shows your organization's private route table. You can use flags to filter the results.`,
+				Flags:       teamnet.FilterFlags,
+			},
+			{
+				Name:        "delete",
+				Action:      cliutil.ErrorHandler(deleteRouteCommand),
+				Usage:       "Delete a row from your organization's private routing table",
+				UsageText:   "cloudflared tunnel [--config FILEPATH] route ip delete [CIDR]",
+				Description: `Deletes the row for a given CIDR from your routing table`,
+			},
+			{
+				Name:      "get",
+				Action:    cliutil.ErrorHandler(getRouteByIPCommand),
+				Usage:     "Check which row of the routing table matches a given IP",
+				UsageText: "cloudflared tunnel [--config FILEPATH] route ip get [IP]",
+				Description: `Checks which row of the routing table will be used to proxy a given IP.
+				This helps check and validate your config.`,
+			},
+		},
+	}
+}
+
+func showRoutesCommand(c *cli.Context) error {
+	sc, err := newSubcommandContext(c)
+	if err != nil {
+		return err
+	}
+
+	filter, err := teamnet.NewFromCLI(c)
+	if err != nil {
+		return errors.Wrap(err, "invalid config for routing filters")
+	}
+
+	routes, err := sc.listRoutes(filter)
+	if err != nil {
+		return err
+	}
+
+	if outputFormat := c.String(outputFormatFlag.Name); outputFormat != "" {
+		return renderOutput(outputFormat, routes)
+	}
+
+	if len(routes) > 0 {
+		formatAndPrintRouteList(routes)
+	} else {
+		fmt.Println("You have no routes, use 'cloudflared tunnel route ip add' to add a route")
+	}
+	return nil
+}
+
+func addRouteCommand(c *cli.Context) error {
+	sc, err := newSubcommandContext(c)
+	if err != nil {
+		return err
+	}
+	if c.NArg() < 2 {
+		return errors.New("You must supply at least 2 arguments, first the network you wish to route (in CIDR form e.g. 1.2.3.4/32) and then the tunnel ID to proxy with")
+	}
+	args := c.Args()
+	_, network, err := net.ParseCIDR(args.Get(0))
+	if err != nil {
+		return errors.Wrap(err, "Invalid network CIDR")
+	}
+	if network == nil {
+		return errors.New("Invalid network CIDR")
+	}
+	tunnelRef := args.Get(1)
+	tunnelID, err := sc.findID(tunnelRef)
+	if err != nil {
+		return errors.Wrap(err, "Invalid tunnel")
+	}
+	comment := ""
+	if c.NArg() >= 3 {
+		comment = args.Get(2)
+	}
+	_, err = sc.addRoute(teamnet.NewRoute{
+		Comment:  comment,
+		Network:  *network,
+		TunnelID: tunnelID,
+	})
+	if err != nil {
+		return errors.Wrap(err, "API error")
+	}
+	fmt.Printf("Successfully added route for %s over tunnel %s\n", network, tunnelID)
+	return nil
+}
+
+func deleteRouteCommand(c *cli.Context) error {
+	sc, err := newSubcommandContext(c)
+	if err != nil {
+		return err
+	}
+	if c.NArg() != 1 {
+		return errors.New("You must supply exactly one argument, the network whose route you want to delete (in CIDR form e.g. 1.2.3.4/32)")
+	}
+	_, network, err := net.ParseCIDR(c.Args().First())
+	if err != nil {
+		return errors.Wrap(err, "Invalid network CIDR")
+	}
+	if network == nil {
+		return errors.New("Invalid network CIDR")
+	}
+	if err := sc.deleteRoute(*network); err != nil {
+		return errors.Wrap(err, "API error")
+	}
+	fmt.Printf("Successfully deleted route for %s\n", network)
+	return nil
+}
+
+func getRouteByIPCommand(c *cli.Context) error {
+	sc, err := newSubcommandContext(c)
+	if err != nil {
+		return err
+	}
+	if c.NArg() != 1 {
+		return errors.New("You must supply exactly one argument, an IP whose route will be queried (e.g. 1.2.3.4 or 2001:0db8:::7334)")
+	}
+
+	ipInput := c.Args().First()
+	ip := net.ParseIP(ipInput)
+	if ip == nil {
+		return fmt.Errorf("Invalid IP %s", ipInput)
+	}
+	route, err := sc.getRouteByIP(ip)
+	if err != nil {
+		return errors.Wrap(err, "API error")
+	}
+	if route.IsZero() {
+		fmt.Printf("No route matches the IP %s\n", ip)
+	} else {
+		formatAndPrintRouteList([]*teamnet.DetailedRoute{&route})
+	}
+	return nil
+}
+
+func formatAndPrintRouteList(routes []*teamnet.DetailedRoute) {
+	const (
+		minWidth = 0
+		tabWidth = 8
+		padding  = 1
+		padChar  = ' '
+		flags    = 0
+	)
+
+	writer := tabwriter.NewWriter(os.Stdout, minWidth, tabWidth, padding, padChar, flags)
+	defer writer.Flush()
+
+	// Print column headers with tabbed columns
+	_, _ = fmt.Fprintln(writer, "NETWORK\tCOMMENT\tTUNNEL ID\tTUNNEL NAME\tCREATED\tDELETED\t")
+
+	// Loop through routes, create formatted string for each, and print using tabwriter
+	for _, route := range routes {
+		formattedStr := route.TableString()
+		_, _ = fmt.Fprintln(writer, formattedStr)
+	}
+}

cmd/cloudflared/ui/launch_ui.go

@@ -3,42 +3,26 @@ package ui
 import (
 	"context"
 	"fmt"
-	"time"
+	"strings"
 
-	"github.com/cloudflare/cloudflared/logger"
+	"github.com/cloudflare/cloudflared/connection"
+	"github.com/cloudflare/cloudflared/ingress"
 
 	"github.com/gdamore/tcell"
 	"github.com/rivo/tview"
+	"github.com/rs/zerolog"
 )
 
 type connState struct {
 	location string
-	state    status
-}
-
-type status int
-
-const (
-	Disconnected status = iota
-	Connected
-	Reconnecting
-	SetUrl
-	RegisteringTunnel
-)
-
-type TunnelEvent struct {
-	Index     uint8
-	EventType status
-	Location  string
-	Url       string
 }
 
 type uiModel struct {
-	version     string
-	edgeURL     string
-	metricsURL  string
-	proxyURL    string
-	connections []connState
+	version       string
+	edgeURL       string
+	metricsURL    string
+	localServices []string
+	connections   []connState
 }
 
 type palette struct {
@@ -47,29 +31,34 @@ type palette struct {
 	defaultText  string
 	disconnected string
 	reconnecting string
+	unregistered string
 }
 
-func NewUIModel(version, hostname, metricsURL, proxyURL string, haConnections int) *uiModel {
+func NewUIModel(version, hostname, metricsURL string, ing *ingress.Ingress, haConnections int) *uiModel {
+	localServices := make([]string, len(ing.Rules))
+	for i, rule := range ing.Rules {
+		localServices[i] = rule.Service.String()
+	}
 	return &uiModel{
-		version:     version,
-		edgeURL:     hostname,
-		metricsURL:  metricsURL,
-		proxyURL:    proxyURL,
-		connections: make([]connState, haConnections),
+		version:       version,
+		edgeURL:       hostname,
+		metricsURL:    metricsURL,
+		localServices: localServices,
+		connections:   make([]connState, haConnections),
 	}
 }
 
-func (data *uiModel) LaunchUI(
+func (data *uiModel) Launch(
 	ctx context.Context,
-	log logger.Service,
-	logLevels []logger.Level,
-	tunnelEventChan <-chan TunnelEvent,
-) {
+	log, transportLog *zerolog.Logger,
+) connection.EventSink {
 	// Configure the logger to stream logs into the textview
 
 	// Add TextView as a group to write output to
 	logTextView := NewDynamicColorTextView()
-	log.Add(logTextView, logger.NewUIFormatter(time.RFC3339), logLevels...)
+	// TODO: Format log for UI
+	//log.Add(logTextView, logger.NewUIFormatter(time.RFC3339), logLevels...)
+	//transportLog.Add(logTextView, logger.NewUIFormatter(time.RFC3339), logLevels...)
 
 	// Construct the UI
 	palette := palette{
@@ -78,6 +67,7 @@ func (data *uiModel) LaunchUI(
 		defaultText:  "white",
 		disconnected: "red",
 		reconnecting: "orange",
+		unregistered: "orange",
 	}
 
 	app := tview.NewApplication()
@@ -107,7 +97,8 @@ func (data *uiModel) LaunchUI(
 	tunnelHostText := tview.NewTextView().SetText(data.edgeURL)
 
 	grid.AddItem(tunnelHostText, 0, 1, 1, 1, 0, 0, false)
-	grid.AddItem(NewDynamicColorTextView().SetText(fmt.Sprintf("[%s]\u2022[%s] Proxying to [%s::b]%s", palette.connected, palette.defaultText, palette.url, data.proxyURL)), 1, 1, 1, 1, 0, 0, false)
+	status := fmt.Sprintf("[%s]\u2022[%s] Proxying to [%s::b]%s", palette.connected, palette.defaultText, palette.url, strings.Join(data.localServices, ", "))
+	grid.AddItem(NewDynamicColorTextView().SetText(status), 1, 1, 1, 1, 0, 0, false)
 
 	grid.AddItem(connTable, 2, 1, 1, 1, 0, 0, false)
 
@@ -123,35 +114,33 @@ func (data *uiModel) LaunchUI(
 	grid.AddItem(logFrame, 4, 0, 5, 2, 0, 0, false)
 
 	go func() {
-		for {
-			select {
-			case <-ctx.Done():
-				app.Stop()
-				return
-			case event := <-tunnelEventChan:
-				switch event.EventType {
-				case Connected:
-					data.setConnTableCell(event, connTable, palette)
-				case Disconnected, Reconnecting:
-					data.changeConnStatus(event, connTable, log, palette)
-				case SetUrl:
-					tunnelHostText.SetText(event.Url)
-					data.edgeURL = event.Url
-				case RegisteringTunnel:
-					if data.edgeURL == "" {
-						tunnelHostText.SetText("Registering tunnel...")
-					}
-				}
-			}
-			app.Draw()
-		}
+		<-ctx.Done()
+		app.Stop()
+		return
 	}()
 
 	go func() {
 		if err := app.SetRoot(frame, true).Run(); err != nil {
-			log.Errorf("Error launching UI: %s", err)
+			log.Error().Msgf("Error launching UI: %s", err)
 		}
 	}()
+
+	return connection.EventSinkFunc(func(event connection.Event) {
+		switch event.EventType {
+		case connection.Connected:
+			data.setConnTableCell(event, connTable, palette)
+		case connection.Disconnected, connection.Reconnecting, connection.Unregistering:
+			data.changeConnStatus(event, connTable, log, palette)
+		case connection.SetURL:
+			tunnelHostText.SetText(event.URL)
+			data.edgeURL = event.URL
+		case connection.RegisteringTunnel:
+			if data.edgeURL == "" {
+				tunnelHostText.SetText(fmt.Sprintf("Registering tunnel connection %d...", event.Index))
+			}
+		}
+		app.Draw()
+	})
 }
 
 func NewDynamicColorTextView() *tview.TextView {
@@ -167,22 +156,19 @@ func handleNewText(app *tview.Application, logTextView *tview.TextView) func() {
 	}
 }
 
-func (data *uiModel) changeConnStatus(event TunnelEvent, table *tview.Table, logger logger.Service, palette palette) {
+func (data *uiModel) changeConnStatus(event connection.Event, table *tview.Table, log *zerolog.Logger, palette palette) {
 	index := int(event.Index)
 	// Get connection location and state
 	connState := data.getConnState(index)
 	// Check if connection is already displayed in UI
 	if connState == nil {
-		logger.Info("Connection is not in the UI table")
+		log.Info().Msg("Connection is not in the UI table")
 		return
 	}
 
 	locationState := event.Location
 
-	if event.EventType == Disconnected {
-		connState.state = Disconnected
-	} else if event.EventType == Reconnecting {
-		connState.state = Reconnecting
+	if event.EventType == connection.Reconnecting {
 		locationState = "Reconnecting..."
 	}
 
@@ -203,12 +189,11 @@ func (data *uiModel) getConnState(connID int) *connState {
 	return nil
 }
 
-func (data *uiModel) setConnTableCell(event TunnelEvent, table *tview.Table, palette palette) {
+func (data *uiModel) setConnTableCell(event connection.Event, table *tview.Table, palette palette) {
 	index := int(event.Index)
 	connectionNum := index + 1
 
 	// Update slice to keep track of connection location and state in UI table
-	data.connections[index].state = Connected
 	data.connections[index].location = event.Location
 
 	// Update text in table cell to show disconnected state
@@ -217,19 +202,21 @@ func (data *uiModel) setConnTableCell(event TunnelEvent, table *tview.Table, pal
 	table.SetCell(index, 0, cell)
 }
 
-func newCellText(palette palette, connectionNum int, location string, connectedStatus status) string {
+func newCellText(palette palette, connectionNum int, location string, connectedStatus connection.Status) string {
 	// HA connection indicator formatted as: "• #<CONNECTION_INDEX>: <COLO>",
 	//  where the left middle dot's color depends on the status of the connection
 	const connFmtString = "[%s]\u2022[%s] #%d: %s"
 
 	var dotColor string
 	switch connectedStatus {
-	case Connected:
+	case connection.Connected:
 		dotColor = palette.connected
-	case Disconnected:
+	case connection.Disconnected:
 		dotColor = palette.disconnected
-	case Reconnecting:
+	case connection.Reconnecting:
 		dotColor = palette.reconnecting
+	case connection.Unregistering:
+		dotColor = palette.unregistered
 	}
 
 	return fmt.Sprintf(connFmtString, dotColor, palette.defaultText, connectionNum, location)

cmd/cloudflared/updater/update.go

@@ -3,6 +3,7 @@ package updater
 import (
 	"context"
 	"fmt"
+	"github.com/rs/zerolog"
 	"os"
 	"path/filepath"
 	"runtime"
@@ -14,18 +15,18 @@ import (
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/facebookgo/grace/gracenet"
-	"github.com/pkg/errors"
 )
 
 const (
 	DefaultCheckUpdateFreq        = time.Hour * 24
-	appID                         = "app_idCzgxYerVD"
 	noUpdateInShellMessage        = "cloudflared will not automatically update when run from the shell. To enable auto-updates, run cloudflared as a service: https://developers.cloudflare.com/argo-tunnel/reference/service/"
 	noUpdateOnWindowsMessage      = "cloudflared will not automatically update on Windows systems."
 	noUpdateManagedPackageMessage = "cloudflared will not automatically update if installed by a package manager."
 	isManagedInstallFile          = ".installedFromPackageManager"
 	UpdateURL                     = "https://update.argotunnel.com"
 	StagingUpdateURL              = "https://staging-update.argotunnel.com"
+
+	LogFieldVersion = "version"
 )
 
 var (
@@ -114,38 +115,35 @@ func checkForUpdateAndApply(options updateOptions) UpdateOutcome {
 
 // Update is the handler for the update command from the command line
 func Update(c *cli.Context) error {
-	logger, err := logger.New()
-	if err != nil {
-		return errors.Wrap(err, "error setting up logger")
-	}
+	log := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
 
 	if wasInstalledFromPackageManager() {
-		logger.Error("cloudflared was installed by a package manager. Please update using the same method.")
+		log.Error().Msg("cloudflared was installed by a package manager. Please update using the same method.")
 		return nil
 	}
 
 	isBeta := c.Bool("beta")
 	if isBeta {
-		logger.Info("cloudflared is set to update to the latest beta version")
+		log.Info().Msg("cloudflared is set to update to the latest beta version")
 	}
 
 	isStaging := c.Bool("staging")
 	if isStaging {
-		logger.Info("cloudflared is set to update from staging")
+		log.Info().Msg("cloudflared is set to update from staging")
 	}
 
 	isForced := c.Bool("force")
 	if isForced {
-		logger.Info("cloudflared is set to upgrade to the latest publish version regardless of the current version")
+		log.Info().Msg("cloudflared is set to upgrade to the latest publish version regardless of the current version")
 	}
 
-	updateOutcome := loggedUpdate(logger, updateOptions{isBeta: isBeta, isStaging: isStaging, isForced: isForced, version: c.String("version")})
+	updateOutcome := loggedUpdate(log, updateOptions{isBeta: isBeta, isStaging: isStaging, isForced: isForced, version: c.String("version")})
 	if updateOutcome.Error != nil {
 		return &statusErr{updateOutcome.Error}
 	}
 
 	if updateOutcome.noUpdate() {
-		logger.Infof("cloudflared is up to date (%s)", updateOutcome.Version)
+		log.Info().Str(LogFieldVersion, updateOutcome.Version).Msg("cloudflared is up to date")
 		return nil
 	}
 
@@ -153,13 +151,13 @@ func Update(c *cli.Context) error {
 }
 
 // Checks for an update and applies it if one is available
-func loggedUpdate(logger logger.Service, options updateOptions) UpdateOutcome {
+func loggedUpdate(log *zerolog.Logger, options updateOptions) UpdateOutcome {
 	updateOutcome := checkForUpdateAndApply(options)
 	if updateOutcome.Updated {
-		logger.Infof("cloudflared has been updated to version %s", updateOutcome.Version)
+		log.Info().Str(LogFieldVersion, updateOutcome.Version).Msg("cloudflared has been updated")
 	}
 	if updateOutcome.Error != nil {
-		logger.Errorf("update check failed: %s", updateOutcome.Error)
+		log.Err(updateOutcome.Error).Msg("update check failed: %s")
 	}
 
 	return updateOutcome
@@ -170,7 +168,7 @@ type AutoUpdater struct {
 	configurable     *configurable
 	listeners        *gracenet.Net
 	updateConfigChan chan *configurable
-	logger           logger.Service
+	log              *zerolog.Logger
 }
 
 // AutoUpdaterConfigurable is the attributes of AutoUpdater that can be reconfigured during runtime
@@ -179,7 +177,7 @@ type configurable struct {
 	freq    time.Duration
 }
 
-func NewAutoUpdater(freq time.Duration, listeners *gracenet.Net, logger logger.Service) *AutoUpdater {
+func NewAutoUpdater(freq time.Duration, listeners *gracenet.Net, log *zerolog.Logger) *AutoUpdater {
 	updaterConfigurable := &configurable{
 		enabled: true,
 		freq:    freq,
@@ -192,7 +190,7 @@ func NewAutoUpdater(freq time.Duration, listeners *gracenet.Net, logger logger.S
 		configurable:     updaterConfigurable,
 		listeners:        listeners,
 		updateConfigChan: make(chan *configurable),
-		logger:           logger,
+		log:              log,
 	}
 }
 
@@ -200,20 +198,19 @@ func (a *AutoUpdater) Run(ctx context.Context) error {
 	ticker := time.NewTicker(a.configurable.freq)
 	for {
 		if a.configurable.enabled {
-			updateOutcome := loggedUpdate(a.logger, updateOptions{})
+			updateOutcome := loggedUpdate(a.log, updateOptions{})
 			if updateOutcome.Updated {
-				os.Args = append(os.Args, "--is-autoupdated=true")
 				if IsSysV() {
 					// SysV doesn't have a mechanism to keep service alive, we have to restart the process
-					a.logger.Info("Restarting service managed by SysV...")
+					a.log.Info().Msg("Restarting service managed by SysV...")
 					pid, err := a.listeners.StartProcess()
 					if err != nil {
-						a.logger.Errorf("Unable to restart server automatically: %s", err)
+						a.log.Err(err).Msg("Unable to restart server automatically")
 						return &statusErr{err: err}
 					}
 					// stop old process after autoupdate. Otherwise we create a new process
 					// after each update
-					a.logger.Infof("PID of the new process is %d", pid)
+					a.log.Info().Msgf("PID of the new process is %d", pid)
 				}
 				return &statusSuccess{newVersion: updateOutcome.Version}
 			}
@@ -245,26 +242,26 @@ func (a *AutoUpdater) Update(newFreq time.Duration) {
 	a.updateConfigChan <- newConfigurable
 }
 
-func IsAutoupdateEnabled(c *cli.Context, l logger.Service) bool {
-	if !SupportAutoUpdate(l) {
+func IsAutoupdateEnabled(c *cli.Context, log *zerolog.Logger) bool {
+	if !SupportAutoUpdate(log) {
 		return false
 	}
 	return !c.Bool("no-autoupdate") && c.Duration("autoupdate-freq") != 0
 }
 
-func SupportAutoUpdate(logger logger.Service) bool {
+func SupportAutoUpdate(log *zerolog.Logger) bool {
 	if runtime.GOOS == "windows" {
-		logger.Info(noUpdateOnWindowsMessage)
+		log.Info().Msg(noUpdateOnWindowsMessage)
 		return false
 	}
 
 	if wasInstalledFromPackageManager() {
-		logger.Info(noUpdateManagedPackageMessage)
+		log.Info().Msg(noUpdateManagedPackageMessage)
 		return false
 	}
 
 	if isRunningFromTerminal() {
-		logger.Info(noUpdateInShellMessage)
+		log.Info().Msg(noUpdateInShellMessage)
 		return false
 	}
 	return true

cmd/cloudflared/updater/update_test.go

@@ -4,15 +4,15 @@ import (
 	"context"
 	"testing"
 
-	"github.com/cloudflare/cloudflared/logger"
 	"github.com/facebookgo/grace/gracenet"
+	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/assert"
 )
 
 func TestDisabledAutoUpdater(t *testing.T) {
 	listeners := &gracenet.Net{}
-	logger := logger.NewOutputWriter(logger.NewMockWriteManager())
-	autoupdater := NewAutoUpdater(0, listeners, logger)
+	log := zerolog.Nop()
+	autoupdater := NewAutoUpdater(0, listeners, &log)
 	ctx, cancel := context.WithCancel(context.Background())
 	errC := make(chan error)
 	go func() {

cmd/cloudflared/updater/workers_service.go

@@ -7,7 +7,6 @@ import (
 	"runtime"
 	"strconv"
 	"strings"
-	"time"
 )
 
 // Options are the update options supported by the
@@ -53,7 +52,7 @@ func NewWorkersService(currentVersion, url, targetPath string, opts Options) Ser
 // Check does a check in with the Workers API to get a new version update
 func (s *WorkersService) Check() (Version, error) {
 	client := &http.Client{
-		Timeout: time.Second * 5,
+		Timeout: clientTimeout,
 	}
 
 	req, err := http.NewRequest(http.MethodGet, s.url, nil)

cmd/cloudflared/updater/workers_update.go

@@ -18,20 +18,23 @@ import (
 	"time"
 )
 
-// stop the service
-// rename cloudflared.exe to cloudflared.exe.old
-// rename cloudflared.exe.new to cloudflared.exe
-// delete cloudflared.exe.old
-// start the service
-// delete the batch file
-const windowsUpdateCommandTemplate = `@echo off
+const (
+	clientTimeout = time.Second * 60
+	// stop the service
+	// rename cloudflared.exe to cloudflared.exe.old
+	// rename cloudflared.exe.new to cloudflared.exe
+	// delete cloudflared.exe.old
+	// start the service
+	// delete the batch file
+	windowsUpdateCommandTemplate = `@echo off
 sc stop cloudflared >nul 2>&1
 rename "{{.TargetPath}}" {{.OldName}}
 rename "{{.NewPath}}" {{.BinaryName}}
 del "{{.OldPath}}"
 sc start cloudflared >nul 2>&1
 del {{.BatchName}}`
-const batchFileName = "cfd_update.bat"
+	batchFileName = "cfd_update.bat"
+)
 
 // Prepare some data to insert into the template.
 type batchData struct {
@@ -118,7 +121,7 @@ func (v *WorkersVersion) String() string {
 // download the file from the link in the json
 func download(url, filepath string, isCompressed bool) error {
 	client := &http.Client{
-		Timeout: time.Second * 5,
+		Timeout: clientTimeout,
 	}
 	resp, err := client.Get(url)
 

cmd/cloudflared/windows_service.go

@@ -13,9 +13,8 @@ import (
 	"unsafe"
 
 	"github.com/cloudflare/cloudflared/logger"
-	"github.com/pkg/errors"
-	cli "github.com/urfave/cli/v2"
 
+	"github.com/urfave/cli/v2"
 	"golang.org/x/sys/windows"
 	"golang.org/x/sys/windows/svc"
 	"golang.org/x/sys/windows/svc/eventlog"
@@ -37,23 +36,25 @@ const (
 	// ERROR_FAILED_SERVICE_CONTROLLER_CONNECT
 	// https://docs.microsoft.com/en-us/windows/desktop/debug/system-error-codes--1000-1299-
 	serviceControllerConnectionFailure = 1063
+
+	LogFieldWindowsServiceName = "windowsServiceName"
 )
 
-func runApp(app *cli.App, shutdownC, graceShutdownC chan struct{}) {
+func runApp(app *cli.App, graceShutdownC chan struct{}) {
 	app.Commands = append(app.Commands, &cli.Command{
 		Name:  "service",
 		Usage: "Manages the Argo Tunnel Windows service",
 		Subcommands: []*cli.Command{
-			&cli.Command{
+			{
 				Name:   "install",
 				Usage:  "Install Argo Tunnel as a Windows service",
 				Action: installWindowsService,
-			},
-			&cli.Command{
+						},
+			{
 				Name:   "uninstall",
 				Usage:  "Uninstall the Argo Tunnel service",
 				Action: uninstallWindowsService,
-			},
+						},
 		},
 	})
 
@@ -67,15 +68,11 @@ func runApp(app *cli.App, shutdownC, graceShutdownC chan struct{}) {
 	//     2. get ERROR_FAILED_SERVICE_CONTROLLER_CONNECT
 	// This involves actually trying to start the service.
 
-	logger, err := logger.New()
-	if err != nil {
-		os.Exit(1)
-		return
-	}
+	log := logger.Create(nil)
 
 	isIntSess, err := svc.IsAnInteractiveSession()
 	if err != nil {
-		logger.Fatalf("failed to determine if we are running in an interactive session: %v", err)
+		log.Fatal().Err(err).Msg("failed to determine if we are running in an interactive session")
 	}
 	if isIntSess {
 		app.Run(os.Args)
@@ -85,7 +82,7 @@ func runApp(app *cli.App, shutdownC, graceShutdownC chan struct{}) {
 	// Run executes service name by calling windowsService which is a Handler
 	// interface that implements Execute method.
 	// It will set service status to stop after Execute returns
-	err = svc.Run(windowsServiceName, &windowsService{app: app, shutdownC: shutdownC, graceShutdownC: graceShutdownC})
+	err = svc.Run(windowsServiceName, &windowsService{app: app, graceShutdownC: graceShutdownC})
 	if err != nil {
 		if errno, ok := err.(syscall.Errno); ok && int(errno) == serviceControllerConnectionFailure {
 			// Hack: assume this is a false negative from the IsAnInteractiveSession() check above.
@@ -93,27 +90,22 @@ func runApp(app *cli.App, shutdownC, graceShutdownC chan struct{}) {
 			app.Run(os.Args)
 			return
 		}
-		logger.Fatalf("%s service failed: %v", windowsServiceName, err)
+		log.Fatal().Err(err).Msgf("%s service failed", windowsServiceName)
 	}
 }
 
 type windowsService struct {
 	app            *cli.App
-	shutdownC      chan struct{}
 	graceShutdownC chan struct{}
 }
 
-// called by the package code at the start of the service
+// Execute is called by the service manager when service starts, the state
+// of the service will be set to Stopped when this function returns.
 func (s *windowsService) Execute(serviceArgs []string, r <-chan svc.ChangeRequest, statusChan chan<- svc.Status) (ssec bool, errno uint32) {
-	logger, err := logger.New()
-	if err != nil {
-		os.Exit(1)
-		return
-	}
-
+	log := logger.Create(nil)
 	elog, err := eventlog.Open(windowsServiceName)
 	if err != nil {
-		logger.Errorf("Cannot open event log for %s with error: %s", windowsServiceName, err)
+		log.Err(err).Msgf("Cannot open event log for %s", windowsServiceName)
 		return
 	}
 	defer elog.Close()
@@ -136,13 +128,12 @@ func (s *windowsService) Execute(serviceArgs []string, r <-chan svc.ChangeReques
 	}
 	elog.Info(1, fmt.Sprintf("%s service arguments: %v", windowsServiceName, args))
 
-	const cmdsAccepted = svc.AcceptStop | svc.AcceptShutdown
 	statusChan <- svc.Status{State: svc.StartPending}
 	errC := make(chan error)
 	go func() {
 		errC <- s.app.Run(args)
 	}()
-	statusChan <- svc.Status{State: svc.Running, Accepts: cmdsAccepted}
+	statusChan <- svc.Status{State: svc.Running, Accepts: svc.AcceptStop | svc.AcceptShutdown}
 
 	for {
 		select {
@@ -151,17 +142,25 @@ func (s *windowsService) Execute(serviceArgs []string, r <-chan svc.ChangeReques
 			case svc.Interrogate:
 				statusChan <- c.CurrentStatus
 			case svc.Stop, svc.Shutdown:
-				close(s.graceShutdownC)
-				statusChan <- svc.Status{State: svc.Stopped, Accepts: cmdsAccepted}
+				if s.graceShutdownC != nil {
+					// start graceful shutdown
+					elog.Info(1, "cloudflared starting graceful shutdown")
+					close(s.graceShutdownC)
+					s.graceShutdownC = nil
+					statusChan <- svc.Status{State: svc.StopPending}
+					continue
+				}
+				// repeated attempts at graceful shutdown forces immediate stop
+				elog.Info(1, "cloudflared terminating immediately")
 				statusChan <- svc.Status{State: svc.StopPending}
-				return
+				return false, 0
 			default:
 				elog.Error(1, fmt.Sprintf("unexpected control request #%d", c))
 			}
 		case err := <-errC:
-			ssec = true
 			if err != nil {
 				elog.Error(1, fmt.Sprintf("cloudflared terminated with error %v", err))
+				ssec = true
 				errno = 1
 			} else {
 				elog.Info(1, "cloudflared terminated without error")
@@ -173,79 +172,76 @@ func (s *windowsService) Execute(serviceArgs []string, r <-chan svc.ChangeReques
 }
 
 func installWindowsService(c *cli.Context) error {
-	logger, err := logger.New()
-	if err != nil {
-		return errors.Wrap(err, "error setting up logger")
-	}
+	zeroLogger := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
 
-	logger.Infof("Installing Argo Tunnel Windows service")
+	zeroLogger.Info().Msg("Installing Argo Tunnel Windows service")
 	exepath, err := os.Executable()
 	if err != nil {
-		logger.Errorf("Cannot find path name that start the process")
+		zeroLogger.Err(err).Msg("Cannot find path name that start the process")
 		return err
 	}
 	m, err := mgr.Connect()
 	if err != nil {
-		logger.Errorf("Cannot establish a connection to the service control manager: %s", err)
+		zeroLogger.Err(err).Msg("Cannot establish a connection to the service control manager")
 		return err
 	}
 	defer m.Disconnect()
 	s, err := m.OpenService(windowsServiceName)
+	log := zeroLogger.With().Str(LogFieldWindowsServiceName, windowsServiceName).Logger()
 	if err == nil {
 		s.Close()
-		logger.Errorf("service %s already exists", windowsServiceName)
+		log.Err(err).Msg("service already exists")
 		return fmt.Errorf("service %s already exists", windowsServiceName)
 	}
 	config := mgr.Config{StartType: mgr.StartAutomatic, DisplayName: windowsServiceDescription}
 	s, err = m.CreateService(windowsServiceName, exepath, config)
 	if err != nil {
-		logger.Errorf("Cannot install service %s", windowsServiceName)
+		log.Error().Msg("Cannot install service")
 		return err
 	}
 	defer s.Close()
-	logger.Infof("Argo Tunnel agent service is installed")
+	log.Info().Msg("Argo Tunnel agent service is installed")
 	err = eventlog.InstallAsEventCreate(windowsServiceName, eventlog.Error|eventlog.Warning|eventlog.Info)
 	if err != nil {
 		s.Delete()
-		logger.Errorf("Cannot install event logger: %s", err)
+		log.Err(err).Msg("Cannot install event logger")
 		return fmt.Errorf("SetupEventLogSource() failed: %s", err)
 	}
 	err = configRecoveryOption(s.Handle)
 	if err != nil {
-		logger.Errorf("Cannot set service recovery actions: %s", err)
-		logger.Infof("See %s to manually configure service recovery actions", windowsServiceUrl)
+		log.Err(err).Msg("Cannot set service recovery actions")
+		log.Info().Msgf("See %s to manually configure service recovery actions", windowsServiceUrl)
 	}
 	return nil
 }
 
 func uninstallWindowsService(c *cli.Context) error {
-	logger, err := logger.New()
-	if err != nil {
-		return errors.Wrap(err, "error setting up logger")
-	}
+	log := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog).
+		With().
+		Str(LogFieldWindowsServiceName, windowsServiceName).Logger()
 
-	logger.Infof("Uninstalling Argo Tunnel Windows Service")
+	log.Info().Msg("Uninstalling Argo Tunnel Windows Service")
 	m, err := mgr.Connect()
 	if err != nil {
-		logger.Errorf("Cannot establish a connection to the service control manager")
+		log.Error().Msg("Cannot establish a connection to the service control manager")
 		return err
 	}
 	defer m.Disconnect()
 	s, err := m.OpenService(windowsServiceName)
 	if err != nil {
-		logger.Errorf("service %s is not installed", windowsServiceName)
+		log.Error().Msg("service is not installed")
 		return fmt.Errorf("service %s is not installed", windowsServiceName)
 	}
 	defer s.Close()
 	err = s.Delete()
 	if err != nil {
-		logger.Errorf("Cannot delete service %s", windowsServiceName)
+		log.Error().Msg("Cannot delete service")
 		return err
 	}
-	logger.Infof("Argo Tunnel agent service is uninstalled")
+	log.Info().Msg("Argo Tunnel agent service is uninstalled")
 	err = eventlog.Remove(windowsServiceName)
 	if err != nil {
-		logger.Errorf("Cannot remove event logger")
+		log.Error().Msg("Cannot remove event logger")
 		return fmt.Errorf("RemoveEventLogSource() failed: %s", err)
 	}
 	return nil

connection/connection.go

@@ -0,0 +1,77 @@
+package connection
+
+import (
+	"io"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/google/uuid"
+)
+
+const LogFieldConnIndex = "connIndex"
+
+type Config struct {
+	OriginClient    OriginClient
+	GracePeriod     time.Duration
+	ReplaceExisting bool
+}
+
+type NamedTunnelConfig struct {
+	Credentials Credentials
+	Client      pogs.ClientInfo
+}
+
+// Credentials are stored in the credentials file and contain all info needed to run a tunnel.
+type Credentials struct {
+	AccountTag   string
+	TunnelSecret []byte
+	TunnelID     uuid.UUID
+	TunnelName   string
+}
+
+func (c *Credentials) Auth() pogs.TunnelAuth {
+	return pogs.TunnelAuth{
+		AccountTag:   c.AccountTag,
+		TunnelSecret: c.TunnelSecret,
+	}
+}
+
+type ClassicTunnelConfig struct {
+	Hostname   string
+	OriginCert []byte
+	// feature-flag to use new edge reconnect tokens
+	UseReconnectToken bool
+}
+
+func (c *ClassicTunnelConfig) IsTrialZone() bool {
+	return c.Hostname == ""
+}
+
+type OriginClient interface {
+	Proxy(w ResponseWriter, req *http.Request, isWebsocket bool) error
+}
+
+type ResponseWriter interface {
+	WriteRespHeaders(*http.Response) error
+	WriteErrorResponse()
+	io.ReadWriter
+}
+
+type ConnectedFuse interface {
+	Connected()
+	IsConnected() bool
+}
+
+func IsServerSentEvent(headers http.Header) bool {
+	if contentType := headers.Get("content-type"); contentType != "" {
+		return strings.HasPrefix(strings.ToLower(contentType), "text/event-stream")
+	}
+	return false
+}
+
+func uint8ToString(input uint8) string {
+	return strconv.FormatUint(uint64(input), 10)
+}

connection/connection_test.go

@@ -0,0 +1,145 @@
+package connection
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"testing"
+	"time"
+
+	"github.com/gobwas/ws/wsutil"
+	"github.com/rs/zerolog"
+	"github.com/stretchr/testify/assert"
+)
+
+const (
+	largeFileSize = 2 * 1024 * 1024
+)
+
+var (
+	testConfig = &Config{
+		OriginClient: &mockOriginClient{},
+		GracePeriod:  time.Millisecond * 100,
+	}
+	log           = zerolog.Nop()
+	testOriginURL = &url.URL{
+		Scheme: "https",
+		Host:   "connectiontest.argotunnel.com",
+	}
+	testLargeResp = make([]byte, largeFileSize)
+)
+
+type testRequest struct {
+	name           string
+	endpoint       string
+	expectedStatus int
+	expectedBody   []byte
+	isProxyError   bool
+}
+
+type mockOriginClient struct {
+}
+
+func (moc *mockOriginClient) Proxy(w ResponseWriter, r *http.Request, isWebsocket bool) error {
+	if isWebsocket {
+		return wsEndpoint(w, r)
+	}
+	switch r.URL.Path {
+	case "/ok":
+		originRespEndpoint(w, http.StatusOK, []byte(http.StatusText(http.StatusOK)))
+	case "/large_file":
+		originRespEndpoint(w, http.StatusOK, testLargeResp)
+	case "/400":
+		originRespEndpoint(w, http.StatusBadRequest, []byte(http.StatusText(http.StatusBadRequest)))
+	case "/500":
+		originRespEndpoint(w, http.StatusInternalServerError, []byte(http.StatusText(http.StatusInternalServerError)))
+	case "/error":
+		return fmt.Errorf("Failed to proxy to origin")
+	default:
+		originRespEndpoint(w, http.StatusNotFound, []byte("page not found"))
+	}
+	return nil
+}
+
+type nowriter struct {
+	io.Reader
+}
+
+func (nowriter) Write(p []byte) (int, error) {
+	return 0, fmt.Errorf("Writer not implemented")
+}
+
+func wsEndpoint(w ResponseWriter, r *http.Request) error {
+	resp := &http.Response{
+		StatusCode: http.StatusSwitchingProtocols,
+	}
+	_ = w.WriteRespHeaders(resp)
+	clientReader := nowriter{r.Body}
+	go func() {
+		for {
+			data, err := wsutil.ReadClientText(clientReader)
+			if err != nil {
+				return
+			}
+			if err := wsutil.WriteServerText(w, data); err != nil {
+				return
+			}
+		}
+	}()
+	<-r.Context().Done()
+	return nil
+}
+
+func originRespEndpoint(w ResponseWriter, status int, data []byte) {
+	resp := &http.Response{
+		StatusCode: status,
+	}
+	_ = w.WriteRespHeaders(resp)
+	_, _ = w.Write(data)
+}
+
+type mockConnectedFuse struct{}
+
+func (mcf mockConnectedFuse) Connected() {}
+
+func (mcf mockConnectedFuse) IsConnected() bool {
+	return true
+}
+
+func TestIsEventStream(t *testing.T) {
+	tests := []struct {
+		headers       http.Header
+		isEventStream bool
+	}{
+		{
+			headers:       newHeader("Content-Type", "text/event-stream"),
+			isEventStream: true,
+		},
+		{
+			headers:       newHeader("content-type", "text/event-stream"),
+			isEventStream: true,
+		},
+		{
+			headers:       newHeader("Content-Type", "text/event-stream; charset=utf-8"),
+			isEventStream: true,
+		},
+		{
+			headers:       newHeader("Content-Type", "application/json"),
+			isEventStream: false,
+		},
+		{
+			headers:       http.Header{},
+			isEventStream: false,
+		},
+	}
+	for _, test := range tests {
+		assert.Equal(t, test.isEventStream, IsServerSentEvent(test.headers))
+	}
+}
+
+func newHeader(key, value string) http.Header {
+	header := http.Header{}
+	header.Add(key, value)
+	return header
+}

connection/errors.go

@@ -0,0 +1,68 @@
+package connection
+
+import (
+	"github.com/cloudflare/cloudflared/edgediscovery"
+	"github.com/cloudflare/cloudflared/h2mux"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+)
+
+const (
+	DuplicateConnectionError = "EDUPCONN"
+)
+
+type DupConnRegisterTunnelError struct{}
+
+var errDuplicationConnection = DupConnRegisterTunnelError{}
+
+func (e DupConnRegisterTunnelError) Error() string {
+	return "already connected to this server, trying another address"
+}
+
+// RegisterTunnel error from server
+type ServerRegisterTunnelError struct {
+	Cause     error
+	Permanent bool
+}
+
+func (e ServerRegisterTunnelError) Error() string {
+	return e.Cause.Error()
+}
+
+func serverRegistrationErrorFromRPC(err error) ServerRegisterTunnelError {
+	if retryable, ok := err.(*tunnelpogs.RetryableError); ok {
+		return ServerRegisterTunnelError{
+			Cause:     retryable.Unwrap(),
+			Permanent: false,
+		}
+	}
+	return ServerRegisterTunnelError{
+		Cause:     err,
+		Permanent: true,
+	}
+}
+
+type muxerShutdownError struct{}
+
+func (e muxerShutdownError) Error() string {
+	return "muxer shutdown"
+}
+
+var errMuxerStopped = muxerShutdownError{}
+
+func isHandshakeErrRecoverable(err error, connIndex uint8, observer *Observer) bool {
+	log := observer.log.With().
+		Uint8(LogFieldConnIndex, connIndex).
+		Err(err).
+		Logger()
+
+	switch err.(type) {
+	case edgediscovery.DialError:
+		log.Error().Msg("Connection unable to dial edge")
+	case h2mux.MuxerHandshakeError:
+		log.Error().Msg("Connection handshake with edge server failed")
+	default:
+		log.Error().Msg("Connection failed")
+		return false
+	}
+	return true
+}

connection/event.go

@@ -0,0 +1,27 @@
+package connection
+
+// Event is something that happened to a connection, e.g. disconnection or registration.
+type Event struct {
+	Index     uint8
+	EventType Status
+	Location  string
+	URL       string
+}
+
+// Status is the status of a connection.
+type Status int
+
+const (
+	// Disconnected means the connection to the edge was broken.
+	Disconnected Status = iota
+	// Connected means the connection to the edge was successfully established.
+	Connected
+	// Reconnecting means the connection to the edge is being re-established.
+	Reconnecting
+	// SetURL means this connection's tunnel was given a URL by the edge. Used for free tunnels.
+	SetURL
+	// RegisteringTunnel means the non-named tunnel is registering its connection.
+	RegisteringTunnel
+	// We're unregistering tunnel from the edge in preparation for a disconnect
+	Unregistering
+)

connection/h2mux.go

@@ -0,0 +1,255 @@
+package connection
+
+import (
+	"context"
+	"io"
+	"net"
+	"net/http"
+	"time"
+
+	"github.com/cloudflare/cloudflared/h2mux"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/cloudflare/cloudflared/websocket"
+
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog"
+	"golang.org/x/sync/errgroup"
+)
+
+const (
+	muxerTimeout      = 5 * time.Second
+	openStreamTimeout = 30 * time.Second
+)
+
+type h2muxConnection struct {
+	config      *Config
+	muxerConfig *MuxerConfig
+	muxer       *h2mux.Muxer
+	// connectionID is only used by metrics, and prometheus requires labels to be string
+	connIndexStr string
+	connIndex    uint8
+
+	observer          *Observer
+	gracefulShutdownC <-chan struct{}
+	stoppedGracefully bool
+
+	// newRPCClientFunc allows us to mock RPCs during testing
+	newRPCClientFunc func(context.Context, io.ReadWriteCloser, *zerolog.Logger) NamedTunnelRPCClient
+}
+
+type MuxerConfig struct {
+	HeartbeatInterval  time.Duration
+	MaxHeartbeats      uint64
+	CompressionSetting h2mux.CompressionSetting
+	MetricsUpdateFreq  time.Duration
+}
+
+func (mc *MuxerConfig) H2MuxerConfig(h h2mux.MuxedStreamHandler, log *zerolog.Logger) *h2mux.MuxerConfig {
+	return &h2mux.MuxerConfig{
+		Timeout:            muxerTimeout,
+		Handler:            h,
+		IsClient:           true,
+		HeartbeatInterval:  mc.HeartbeatInterval,
+		MaxHeartbeats:      mc.MaxHeartbeats,
+		Log:                log,
+		CompressionQuality: mc.CompressionSetting,
+	}
+}
+
+// NewTunnelHandler returns a TunnelHandler, origin LAN IP and error
+func NewH2muxConnection(
+	config *Config,
+	muxerConfig *MuxerConfig,
+	edgeConn net.Conn,
+	connIndex uint8,
+	observer *Observer,
+	gracefulShutdownC <-chan struct{},
+) (*h2muxConnection, error, bool) {
+	h := &h2muxConnection{
+		config:            config,
+		muxerConfig:       muxerConfig,
+		connIndexStr:      uint8ToString(connIndex),
+		connIndex:         connIndex,
+		observer:          observer,
+		gracefulShutdownC: gracefulShutdownC,
+		newRPCClientFunc:  newRegistrationRPCClient,
+	}
+
+	// Establish a muxed connection with the edge
+	// Client mux handshake with agent server
+	muxer, err := h2mux.Handshake(edgeConn, edgeConn, *muxerConfig.H2MuxerConfig(h, observer.logTransport), h2mux.ActiveStreams)
+	if err != nil {
+		recoverable := isHandshakeErrRecoverable(err, connIndex, observer)
+		return nil, err, recoverable
+	}
+	h.muxer = muxer
+	return h, nil, false
+}
+
+func (h *h2muxConnection) ServeNamedTunnel(ctx context.Context, namedTunnel *NamedTunnelConfig, connOptions *tunnelpogs.ConnectionOptions, connectedFuse ConnectedFuse) error {
+	errGroup, serveCtx := errgroup.WithContext(ctx)
+	errGroup.Go(func() error {
+		return h.serveMuxer(serveCtx)
+	})
+
+	errGroup.Go(func() error {
+		if err := h.registerNamedTunnel(serveCtx, namedTunnel, connOptions); err != nil {
+			return err
+		}
+		connectedFuse.Connected()
+		return nil
+	})
+
+	errGroup.Go(func() error {
+		h.controlLoop(serveCtx, connectedFuse, true)
+		return nil
+	})
+
+	err := errGroup.Wait()
+	if err == errMuxerStopped {
+		if h.stoppedGracefully {
+			return nil
+		}
+		h.observer.log.Info().Uint8(LogFieldConnIndex, h.connIndex).Msg("Unexpected muxer shutdown")
+	}
+	return err
+}
+
+func (h *h2muxConnection) ServeClassicTunnel(ctx context.Context, classicTunnel *ClassicTunnelConfig, credentialManager CredentialManager, registrationOptions *tunnelpogs.RegistrationOptions, connectedFuse ConnectedFuse) error {
+	errGroup, serveCtx := errgroup.WithContext(ctx)
+	errGroup.Go(func() error {
+		return h.serveMuxer(serveCtx)
+	})
+
+	errGroup.Go(func() (err error) {
+		defer func() {
+			if err == nil {
+				connectedFuse.Connected()
+			}
+		}()
+		if classicTunnel.UseReconnectToken && connectedFuse.IsConnected() {
+			err := h.reconnectTunnel(ctx, credentialManager, classicTunnel, registrationOptions)
+			if err == nil {
+				return nil
+			}
+			// log errors and proceed to RegisterTunnel
+			h.observer.log.Err(err).
+				Uint8(LogFieldConnIndex, h.connIndex).
+				Msg("Couldn't reconnect connection. Re-registering it instead.")
+		}
+		return h.registerTunnel(ctx, credentialManager, classicTunnel, registrationOptions)
+	})
+
+	errGroup.Go(func() error {
+		h.controlLoop(serveCtx, connectedFuse, false)
+		return nil
+	})
+
+	err := errGroup.Wait()
+	if err == errMuxerStopped {
+		if h.stoppedGracefully {
+			return nil
+		}
+		h.observer.log.Info().Uint8(LogFieldConnIndex, h.connIndex).Msg("Unexpected muxer shutdown")
+	}
+	return err
+}
+
+func (h *h2muxConnection) serveMuxer(ctx context.Context) error {
+	// All routines should stop when muxer finish serving. When muxer is shutdown
+	// gracefully, it doesn't return an error, so we need to return errMuxerShutdown
+	// here to notify other routines to stop
+	err := h.muxer.Serve(ctx)
+	if err == nil {
+		return errMuxerStopped
+	}
+	return err
+}
+
+func (h *h2muxConnection) controlLoop(ctx context.Context, connectedFuse ConnectedFuse, isNamedTunnel bool) {
+	updateMetricsTickC := time.Tick(h.muxerConfig.MetricsUpdateFreq)
+	var shutdownCompleted <-chan struct{}
+	for {
+		select {
+		case <-h.gracefulShutdownC:
+			if connectedFuse.IsConnected() {
+				h.unregister(isNamedTunnel)
+			}
+			h.stoppedGracefully = true
+			h.gracefulShutdownC = nil
+			shutdownCompleted = h.muxer.Shutdown()
+
+		case <-shutdownCompleted:
+			return
+
+		case <-ctx.Done():
+			// UnregisterTunnel blocks until the RPC call returns
+			if !h.stoppedGracefully && connectedFuse.IsConnected() {
+				h.unregister(isNamedTunnel)
+			}
+			h.muxer.Shutdown()
+			// don't wait for shutdown to finish when context is closed, this is the hard termination path
+			return
+
+		case <-updateMetricsTickC:
+			h.observer.metrics.updateMuxerMetrics(h.connIndexStr, h.muxer.Metrics())
+		}
+	}
+}
+
+func (h *h2muxConnection) newRPCStream(ctx context.Context, rpcName rpcName) (*h2mux.MuxedStream, error) {
+	openStreamCtx, openStreamCancel := context.WithTimeout(ctx, openStreamTimeout)
+	defer openStreamCancel()
+	stream, err := h.muxer.OpenRPCStream(openStreamCtx)
+	if err != nil {
+		return nil, err
+	}
+	return stream, nil
+}
+
+func (h *h2muxConnection) ServeStream(stream *h2mux.MuxedStream) error {
+	respWriter := &h2muxRespWriter{stream}
+
+	req, reqErr := h.newRequest(stream)
+	if reqErr != nil {
+		respWriter.WriteErrorResponse()
+		return reqErr
+	}
+
+	err := h.config.OriginClient.Proxy(respWriter, req, websocket.IsWebSocketUpgrade(req))
+	if err != nil {
+		respWriter.WriteErrorResponse()
+		return err
+	}
+	return nil
+}
+
+func (h *h2muxConnection) newRequest(stream *h2mux.MuxedStream) (*http.Request, error) {
+	req, err := http.NewRequest("GET", "http://localhost:8080", h2mux.MuxedStreamReader{MuxedStream: stream})
+	if err != nil {
+		return nil, errors.Wrap(err, "Unexpected error from http.NewRequest")
+	}
+	err = h2mux.H2RequestHeadersToH1Request(stream.Headers, req)
+	if err != nil {
+		return nil, errors.Wrap(err, "invalid request received")
+	}
+	return req, nil
+}
+
+type h2muxRespWriter struct {
+	*h2mux.MuxedStream
+}
+
+func (rp *h2muxRespWriter) WriteRespHeaders(resp *http.Response) error {
+	headers := h2mux.H1ResponseToH2ResponseHeaders(resp)
+	headers = append(headers, h2mux.Header{Name: ResponseMetaHeaderField, Value: responseMetaHeaderOrigin})
+	return rp.WriteHeaders(headers)
+}
+
+func (rp *h2muxRespWriter) WriteErrorResponse() {
+	_ = rp.WriteHeaders([]h2mux.Header{
+		{Name: ":status", Value: "502"},
+		{Name: ResponseMetaHeaderField, Value: responseMetaHeaderCfd},
+	})
+	_, _ = rp.Write([]byte("502 Bad Gateway"))
+}

connection/h2mux_test.go

@@ -0,0 +1,301 @@
+package connection
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"strconv"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/gobwas/ws/wsutil"
+	"github.com/rs/zerolog"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/cloudflare/cloudflared/h2mux"
+)
+
+var (
+	testMuxerConfig = &MuxerConfig{
+		HeartbeatInterval:  time.Second * 5,
+		MaxHeartbeats:      5,
+		CompressionSetting: 0,
+		MetricsUpdateFreq:  time.Second * 5,
+	}
+)
+
+func newH2MuxConnection(t require.TestingT) (*h2muxConnection, *h2mux.Muxer) {
+	edgeConn, originConn := net.Pipe()
+	edgeMuxChan := make(chan *h2mux.Muxer)
+	go func() {
+		edgeMuxConfig := h2mux.MuxerConfig{
+			Log: &log,
+			Handler: h2mux.MuxedStreamFunc(func(stream *h2mux.MuxedStream) error {
+				// we only expect RPC traffic in client->edge direction, provide minimal support for mocking
+				require.True(t, stream.IsRPCStream())
+				return stream.WriteHeaders([]h2mux.Header{
+					{Name:  ":status", Value: "200"},
+				})
+			}),
+		}
+		edgeMux, err := h2mux.Handshake(edgeConn, edgeConn, edgeMuxConfig, h2mux.ActiveStreams)
+		require.NoError(t, err)
+		edgeMuxChan <- edgeMux
+	}()
+	var connIndex = uint8(0)
+	testObserver := NewObserver(&log, &log, false)
+	h2muxConn, err, _ := NewH2muxConnection(testConfig, testMuxerConfig, originConn, connIndex, testObserver, nil)
+	require.NoError(t, err)
+	return h2muxConn, <-edgeMuxChan
+}
+
+func TestServeStreamHTTP(t *testing.T) {
+	tests := []testRequest{
+		{
+			name:           "ok",
+			endpoint:       "/ok",
+			expectedStatus: http.StatusOK,
+			expectedBody:   []byte(http.StatusText(http.StatusOK)),
+		},
+		{
+			name:           "large_file",
+			endpoint:       "/large_file",
+			expectedStatus: http.StatusOK,
+			expectedBody:   testLargeResp,
+		},
+		{
+			name:           "Bad request",
+			endpoint:       "/400",
+			expectedStatus: http.StatusBadRequest,
+			expectedBody:   []byte(http.StatusText(http.StatusBadRequest)),
+		},
+		{
+			name:           "Internal server error",
+			endpoint:       "/500",
+			expectedStatus: http.StatusInternalServerError,
+			expectedBody:   []byte(http.StatusText(http.StatusInternalServerError)),
+		},
+		{
+			name:           "Proxy error",
+			endpoint:       "/error",
+			expectedStatus: http.StatusBadGateway,
+			expectedBody:   nil,
+			isProxyError:   true,
+		},
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	h2muxConn, edgeMux := newH2MuxConnection(t)
+
+	var wg sync.WaitGroup
+	wg.Add(2)
+	go func() {
+		defer wg.Done()
+		_ = edgeMux.Serve(ctx)
+	}()
+	go func() {
+		defer wg.Done()
+		err := h2muxConn.serveMuxer(ctx)
+		require.Error(t, err)
+	}()
+
+	for _, test := range tests {
+		headers := []h2mux.Header{
+			{
+				Name:  ":path",
+				Value: test.endpoint,
+			},
+		}
+		stream, err := edgeMux.OpenStream(ctx, headers, nil)
+		require.NoError(t, err)
+		require.True(t, hasHeader(stream, ":status", strconv.Itoa(test.expectedStatus)))
+
+		if test.isProxyError {
+			assert.True(t, hasHeader(stream, ResponseMetaHeaderField, responseMetaHeaderCfd))
+		} else {
+			assert.True(t, hasHeader(stream, ResponseMetaHeaderField, responseMetaHeaderOrigin))
+			body := make([]byte, len(test.expectedBody))
+			_, err = stream.Read(body)
+			require.NoError(t, err)
+			require.Equal(t, test.expectedBody, body)
+		}
+	}
+	cancel()
+	wg.Wait()
+}
+
+func TestServeStreamWS(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	h2muxConn, edgeMux := newH2MuxConnection(t)
+
+	var wg sync.WaitGroup
+	wg.Add(2)
+	go func() {
+		defer wg.Done()
+		edgeMux.Serve(ctx)
+	}()
+	go func() {
+		defer wg.Done()
+		err := h2muxConn.serveMuxer(ctx)
+		require.Error(t, err)
+	}()
+
+	headers := []h2mux.Header{
+		{
+			Name:  ":path",
+			Value: "/ws",
+		},
+		{
+			Name:  "connection",
+			Value: "upgrade",
+		},
+		{
+			Name:  "upgrade",
+			Value: "websocket",
+		},
+	}
+
+	readPipe, writePipe := io.Pipe()
+	stream, err := edgeMux.OpenStream(ctx, headers, readPipe)
+	require.NoError(t, err)
+
+	require.True(t, hasHeader(stream, ":status", strconv.Itoa(http.StatusSwitchingProtocols)))
+	assert.True(t, hasHeader(stream, ResponseMetaHeaderField, responseMetaHeaderOrigin))
+
+	data := []byte("test websocket")
+	err = wsutil.WriteClientText(writePipe, data)
+	require.NoError(t, err)
+
+	respBody, err := wsutil.ReadServerText(stream)
+	require.NoError(t, err)
+	require.Equal(t, data, respBody, fmt.Sprintf("Expect %s, got %s", string(data), string(respBody)))
+
+	cancel()
+	wg.Wait()
+}
+
+func TestGracefulShutdownH2Mux(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	h2muxConn, edgeMux := newH2MuxConnection(t)
+
+	shutdownC := make(chan struct{})
+	unregisteredC := make(chan struct{})
+	h2muxConn.gracefulShutdownC = shutdownC
+	h2muxConn.newRPCClientFunc = func(_ context.Context, _ io.ReadWriteCloser, _ *zerolog.Logger) NamedTunnelRPCClient {
+		return &mockNamedTunnelRPCClient{
+			registered:   nil,
+			unregistered: unregisteredC,
+		}
+	}
+
+	var wg sync.WaitGroup
+	wg.Add(3)
+	go func() {
+		defer wg.Done()
+		_ = edgeMux.Serve(ctx)
+	}()
+	go func() {
+		defer wg.Done()
+		_ = h2muxConn.serveMuxer(ctx)
+	}()
+
+	go func() {
+		defer wg.Done()
+		h2muxConn.controlLoop(ctx, &mockConnectedFuse{}, true)
+	}()
+
+	time.Sleep(100 * time.Millisecond)
+	close(shutdownC)
+
+	select {
+	case <-unregisteredC:
+		break // ok
+	case <-time.Tick(time.Second):
+		assert.Fail(t, "timed out waiting for control loop to unregister")
+	}
+
+	cancel()
+	wg.Wait()
+
+	assert.True(t, h2muxConn.stoppedGracefully)
+	assert.Nil(t, h2muxConn.gracefulShutdownC)
+}
+
+func hasHeader(stream *h2mux.MuxedStream, name, val string) bool {
+	for _, header := range stream.Headers {
+		if header.Name == name && header.Value == val {
+			return true
+		}
+	}
+	return false
+}
+
+func benchmarkServeStreamHTTPSimple(b *testing.B, test testRequest) {
+	ctx, cancel := context.WithCancel(context.Background())
+	h2muxConn, edgeMux := newH2MuxConnection(b)
+
+	var wg sync.WaitGroup
+	wg.Add(2)
+	go func() {
+		defer wg.Done()
+		edgeMux.Serve(ctx)
+	}()
+	go func() {
+		defer wg.Done()
+		err := h2muxConn.serveMuxer(ctx)
+		require.Error(b, err)
+	}()
+
+	headers := []h2mux.Header{
+		{
+			Name:  ":path",
+			Value: test.endpoint,
+		},
+	}
+
+	body := make([]byte, len(test.expectedBody))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		b.StartTimer()
+		stream, openstreamErr := edgeMux.OpenStream(ctx, headers, nil)
+		_, readBodyErr := stream.Read(body)
+		b.StopTimer()
+
+		require.NoError(b, openstreamErr)
+		assert.True(b, hasHeader(stream, ResponseMetaHeaderField, responseMetaHeaderOrigin))
+		require.True(b, hasHeader(stream, ":status", strconv.Itoa(http.StatusOK)))
+		require.NoError(b, readBodyErr)
+		require.Equal(b, test.expectedBody, body)
+	}
+
+	cancel()
+	wg.Wait()
+}
+
+func BenchmarkServeStreamHTTPSimple(b *testing.B) {
+	test := testRequest{
+		name:           "ok",
+		endpoint:       "/ok",
+		expectedStatus: http.StatusOK,
+		expectedBody:   []byte(http.StatusText(http.StatusOK)),
+	}
+
+	benchmarkServeStreamHTTPSimple(b, test)
+}
+
+func BenchmarkServeStreamHTTPLargeFile(b *testing.B) {
+	test := testRequest{
+		name:           "large_file",
+		endpoint:       "/large_file",
+		expectedStatus: http.StatusOK,
+		expectedBody:   testLargeResp,
+	}
+
+	benchmarkServeStreamHTTPSimple(b, test)
+}

connection/header.go

@@ -0,0 +1,31 @@
+package connection
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/cloudflare/cloudflared/h2mux"
+)
+
+const (
+	ResponseMetaHeaderField = "cf-cloudflared-response-meta"
+)
+
+var (
+	canonicalResponseUserHeadersField = http.CanonicalHeaderKey(h2mux.ResponseUserHeadersField)
+	canonicalResponseMetaHeaderField  = http.CanonicalHeaderKey(ResponseMetaHeaderField)
+	responseMetaHeaderCfd             = mustInitRespMetaHeader("cloudflared")
+	responseMetaHeaderOrigin          = mustInitRespMetaHeader("origin")
+)
+
+type responseMetaHeader struct {
+	Source string `json:"src"`
+}
+
+func mustInitRespMetaHeader(src string) string {
+	header, err := json.Marshal(responseMetaHeader{Source: src})
+	if err != nil {
+		panic(fmt.Sprintf("Failed to serialize response meta header = %s, err: %v", src, err))
+	}
+	return string(header)
+}

connection/http2.go

@@ -0,0 +1,244 @@
+package connection
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"math"
+	"net"
+	"net/http"
+	"strings"
+	"sync"
+
+	"github.com/cloudflare/cloudflared/h2mux"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+
+	"github.com/rs/zerolog"
+	"golang.org/x/net/http2"
+)
+
+const (
+	internalUpgradeHeader = "Cf-Cloudflared-Proxy-Connection-Upgrade"
+	websocketUpgrade      = "websocket"
+	controlStreamUpgrade  = "control-stream"
+)
+
+var errEdgeConnectionClosed = fmt.Errorf("connection with edge closed")
+
+type http2Connection struct {
+	conn         net.Conn
+	server       *http2.Server
+	config       *Config
+	namedTunnel  *NamedTunnelConfig
+	connOptions  *tunnelpogs.ConnectionOptions
+	observer     *Observer
+	connIndexStr string
+	connIndex    uint8
+	// newRPCClientFunc allows us to mock RPCs during testing
+	newRPCClientFunc func(context.Context, io.ReadWriteCloser, *zerolog.Logger) NamedTunnelRPCClient
+
+	activeRequestsWG  sync.WaitGroup
+	connectedFuse     ConnectedFuse
+	gracefulShutdownC <-chan struct{}
+	stoppedGracefully bool
+	controlStreamErr  error // result of running control stream handler
+}
+
+func NewHTTP2Connection(
+	conn net.Conn,
+	config *Config,
+	namedTunnelConfig *NamedTunnelConfig,
+	connOptions *tunnelpogs.ConnectionOptions,
+	observer *Observer,
+	connIndex uint8,
+	connectedFuse ConnectedFuse,
+	gracefulShutdownC <-chan struct{},
+) *http2Connection {
+	return &http2Connection{
+		conn: conn,
+		server: &http2.Server{
+			MaxConcurrentStreams: math.MaxUint32,
+		},
+		config:            config,
+		namedTunnel:       namedTunnelConfig,
+		connOptions:       connOptions,
+		observer:          observer,
+		connIndexStr:      uint8ToString(connIndex),
+		connIndex:         connIndex,
+		newRPCClientFunc:  newRegistrationRPCClient,
+		connectedFuse:     connectedFuse,
+		gracefulShutdownC: gracefulShutdownC,
+	}
+}
+
+func (c *http2Connection) Serve(ctx context.Context) error {
+	go func() {
+		<-ctx.Done()
+		c.close()
+	}()
+	c.server.ServeConn(c.conn, &http2.ServeConnOpts{
+		Context: ctx,
+		Handler: c,
+	})
+
+	switch {
+	case c.stoppedGracefully:
+		return nil
+	case c.controlStreamErr != nil:
+		return c.controlStreamErr
+	default:
+		c.observer.log.Info().Uint8(LogFieldConnIndex, c.connIndex).Msg("Lost connection with the edge")
+		return errEdgeConnectionClosed
+	}
+}
+
+func (c *http2Connection) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	c.activeRequestsWG.Add(1)
+	defer c.activeRequestsWG.Done()
+
+	respWriter := &http2RespWriter{
+		r: r.Body,
+		w: w,
+	}
+	flusher, isFlusher := w.(http.Flusher)
+	if !isFlusher {
+		c.observer.log.Error().Msgf("%T doesn't implement http.Flusher", w)
+		respWriter.WriteErrorResponse()
+		return
+	}
+	respWriter.flusher = flusher
+	var err error
+	if isControlStreamUpgrade(r) {
+		respWriter.shouldFlush = true
+		err = c.serveControlStream(r.Context(), respWriter)
+		c.controlStreamErr = err
+	} else if isWebsocketUpgrade(r) {
+		respWriter.shouldFlush = true
+		stripWebsocketUpgradeHeader(r)
+		err = c.config.OriginClient.Proxy(respWriter, r, true)
+	} else {
+		err = c.config.OriginClient.Proxy(respWriter, r, false)
+	}
+
+	if err != nil {
+		respWriter.WriteErrorResponse()
+	}
+}
+
+func (c *http2Connection) serveControlStream(ctx context.Context, respWriter *http2RespWriter) error {
+	rpcClient := c.newRPCClientFunc(ctx, respWriter, c.observer.log)
+	defer rpcClient.Close()
+
+	if err := rpcClient.RegisterConnection(ctx, c.namedTunnel, c.connOptions, c.connIndex, c.observer); err != nil {
+		return err
+	}
+	c.connectedFuse.Connected()
+
+	// wait for connection termination or start of graceful shutdown
+	select {
+	case <-ctx.Done():
+		break
+	case <-c.gracefulShutdownC:
+		c.stoppedGracefully = true
+	}
+
+	c.observer.sendUnregisteringEvent(c.connIndex)
+	rpcClient.GracefulShutdown(ctx, c.config.GracePeriod)
+	c.observer.log.Info().Uint8(LogFieldConnIndex, c.connIndex).Msg("Unregistered tunnel connection")
+	return nil
+}
+
+func (c *http2Connection) close() {
+	// Wait for all serve HTTP handlers to return
+	c.activeRequestsWG.Wait()
+	c.conn.Close()
+}
+
+type http2RespWriter struct {
+	r           io.Reader
+	w           http.ResponseWriter
+	flusher     http.Flusher
+	shouldFlush bool
+}
+
+func (rp *http2RespWriter) WriteRespHeaders(resp *http.Response) error {
+	dest := rp.w.Header()
+	userHeaders := make(http.Header, len(resp.Header))
+	for header, values := range resp.Header {
+		// Since these are http2 headers, they're required to be lowercase
+		h2name := strings.ToLower(header)
+		for _, v := range values {
+			if h2name == "content-length" {
+				// This header has meaning in HTTP/2 and will be used by the edge,
+				// so it should be sent as an HTTP/2 response header.
+				dest.Add(h2name, v)
+				// Since these are http2 headers, they're required to be lowercase
+			} else if !h2mux.IsControlHeader(h2name) || h2mux.IsWebsocketClientHeader(h2name) {
+				// User headers, on the other hand, must all be serialized so that
+				// HTTP/2 header validation won't be applied to HTTP/1 header values
+				userHeaders.Add(h2name, v)
+			}
+		}
+	}
+
+	// Perform user header serialization and set them in the single header
+	dest.Set(canonicalResponseUserHeadersField, h2mux.SerializeHeaders(userHeaders))
+	rp.setResponseMetaHeader(responseMetaHeaderOrigin)
+	status := resp.StatusCode
+	// HTTP2 removes support for 101 Switching Protocols https://tools.ietf.org/html/rfc7540#section-8.1.1
+	if status == http.StatusSwitchingProtocols {
+		status = http.StatusOK
+	}
+	rp.w.WriteHeader(status)
+	if IsServerSentEvent(resp.Header) {
+		rp.shouldFlush = true
+	}
+	if rp.shouldFlush {
+		rp.flusher.Flush()
+	}
+	return nil
+}
+
+func (rp *http2RespWriter) WriteErrorResponse() {
+	rp.setResponseMetaHeader(responseMetaHeaderCfd)
+	rp.w.WriteHeader(http.StatusBadGateway)
+}
+
+func (rp *http2RespWriter) setResponseMetaHeader(value string) {
+	rp.w.Header().Set(canonicalResponseMetaHeaderField, value)
+}
+
+func (rp *http2RespWriter) Read(p []byte) (n int, err error) {
+	return rp.r.Read(p)
+}
+
+func (rp *http2RespWriter) Write(p []byte) (n int, err error) {
+	defer func() {
+		// Implementer of OriginClient should make sure it doesn't write to the connection after Proxy returns
+		// Register a recover routine just in case.
+		if r := recover(); r != nil {
+			println("Recover from http2 response writer panic, error", r)
+		}
+	}()
+	n, err = rp.w.Write(p)
+	if err == nil && rp.shouldFlush {
+		rp.flusher.Flush()
+	}
+	return n, err
+}
+
+func (rp *http2RespWriter) Close() error {
+	return nil
+}
+
+func isControlStreamUpgrade(r *http.Request) bool {
+	return strings.ToLower(r.Header.Get(internalUpgradeHeader)) == controlStreamUpgrade
+}
+
+func isWebsocketUpgrade(r *http.Request) bool {
+	return strings.ToLower(r.Header.Get(internalUpgradeHeader)) == websocketUpgrade
+}
+
+func stripWebsocketUpgradeHeader(r *http.Request) {
+	r.Header.Del(internalUpgradeHeader)
+}

connection/http2_test.go

@@ -0,0 +1,370 @@
+package connection
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+
+	"github.com/gobwas/ws/wsutil"
+	"github.com/rs/zerolog"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/net/http2"
+)
+
+var (
+	testTransport = http2.Transport{}
+)
+
+func newTestHTTP2Connection() (*http2Connection, net.Conn) {
+	edgeConn, originConn := net.Pipe()
+	var connIndex = uint8(0)
+	return NewHTTP2Connection(
+		originConn,
+		testConfig,
+		&NamedTunnelConfig{},
+		&pogs.ConnectionOptions{},
+		NewObserver(&log, &log, false),
+		connIndex,
+		mockConnectedFuse{},
+		nil,
+	), edgeConn
+}
+
+func TestServeHTTP(t *testing.T) {
+	tests := []testRequest{
+		{
+			name:           "ok",
+			endpoint:       "ok",
+			expectedStatus: http.StatusOK,
+			expectedBody:   []byte(http.StatusText(http.StatusOK)),
+		},
+		{
+			name:           "large_file",
+			endpoint:       "large_file",
+			expectedStatus: http.StatusOK,
+			expectedBody:   testLargeResp,
+		},
+		{
+			name:           "Bad request",
+			endpoint:       "400",
+			expectedStatus: http.StatusBadRequest,
+			expectedBody:   []byte(http.StatusText(http.StatusBadRequest)),
+		},
+		{
+			name:           "Internal server error",
+			endpoint:       "500",
+			expectedStatus: http.StatusInternalServerError,
+			expectedBody:   []byte(http.StatusText(http.StatusInternalServerError)),
+		},
+		{
+			name:           "Proxy error",
+			endpoint:       "error",
+			expectedStatus: http.StatusBadGateway,
+			expectedBody:   nil,
+			isProxyError:   true,
+		},
+	}
+
+	http2Conn, edgeConn := newTestHTTP2Connection()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		http2Conn.Serve(ctx)
+	}()
+
+	edgeHTTP2Conn, err := testTransport.NewClientConn(edgeConn)
+	require.NoError(t, err)
+
+	for _, test := range tests {
+		endpoint := fmt.Sprintf("http://localhost:8080/%s", test.endpoint)
+		req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
+		require.NoError(t, err)
+
+		resp, err := edgeHTTP2Conn.RoundTrip(req)
+		require.NoError(t, err)
+		require.Equal(t, test.expectedStatus, resp.StatusCode)
+		if test.expectedBody != nil {
+			respBody, err := ioutil.ReadAll(resp.Body)
+			require.NoError(t, err)
+			require.Equal(t, test.expectedBody, respBody)
+		}
+		if test.isProxyError {
+			require.Equal(t, responseMetaHeaderCfd, resp.Header.Get(ResponseMetaHeaderField))
+		} else {
+			require.Equal(t, responseMetaHeaderOrigin, resp.Header.Get(ResponseMetaHeaderField))
+		}
+	}
+	cancel()
+	wg.Wait()
+}
+
+type mockNamedTunnelRPCClient struct {
+	registered   chan struct{}
+	unregistered chan struct{}
+}
+
+func (mc mockNamedTunnelRPCClient) RegisterConnection(
+	c context.Context,
+	config *NamedTunnelConfig,
+	options *tunnelpogs.ConnectionOptions,
+	connIndex uint8,
+	observer *Observer,
+) error {
+	close(mc.registered)
+	return nil
+}
+
+func (mc mockNamedTunnelRPCClient) GracefulShutdown(ctx context.Context, gracePeriod time.Duration) {
+	close(mc.unregistered)
+}
+
+func (mockNamedTunnelRPCClient) Close() {}
+
+type mockRPCClientFactory struct {
+	registered   chan struct{}
+	unregistered chan struct{}
+}
+
+func (mf *mockRPCClientFactory) newMockRPCClient(context.Context, io.ReadWriteCloser, *zerolog.Logger) NamedTunnelRPCClient {
+	return mockNamedTunnelRPCClient{
+		registered:   mf.registered,
+		unregistered: mf.unregistered,
+	}
+}
+
+type wsRespWriter struct {
+	*httptest.ResponseRecorder
+	readPipe  *io.PipeReader
+	writePipe *io.PipeWriter
+}
+
+func newWSRespWriter() *wsRespWriter {
+	readPipe, writePipe := io.Pipe()
+	return &wsRespWriter{
+		httptest.NewRecorder(),
+		readPipe,
+		writePipe,
+	}
+}
+
+func (w *wsRespWriter) RespBody() io.ReadWriter {
+	return nowriter{w.readPipe}
+}
+
+func (w *wsRespWriter) Write(data []byte) (n int, err error) {
+	return w.writePipe.Write(data)
+}
+
+func TestServeWS(t *testing.T) {
+	http2Conn, _ := newTestHTTP2Connection()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		http2Conn.Serve(ctx)
+	}()
+
+	respWriter := newWSRespWriter()
+	readPipe, writePipe := io.Pipe()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://localhost:8080/ws", readPipe)
+	require.NoError(t, err)
+	req.Header.Set(internalUpgradeHeader, websocketUpgrade)
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		http2Conn.ServeHTTP(respWriter, req)
+	}()
+
+	data := []byte("test websocket")
+	err = wsutil.WriteClientText(writePipe, data)
+	require.NoError(t, err)
+
+	respBody, err := wsutil.ReadServerText(respWriter.RespBody())
+	require.NoError(t, err)
+	require.Equal(t, data, respBody, fmt.Sprintf("Expect %s, got %s", string(data), string(respBody)))
+
+	cancel()
+	resp := respWriter.Result()
+	// http2RespWriter should rewrite status 101 to 200
+	require.Equal(t, http.StatusOK, resp.StatusCode)
+	require.Equal(t, responseMetaHeaderOrigin, resp.Header.Get(ResponseMetaHeaderField))
+
+	wg.Wait()
+}
+
+func TestServeControlStream(t *testing.T) {
+	http2Conn, edgeConn := newTestHTTP2Connection()
+
+	rpcClientFactory := mockRPCClientFactory{
+		registered:   make(chan struct{}),
+		unregistered: make(chan struct{}),
+	}
+	http2Conn.newRPCClientFunc = rpcClientFactory.newMockRPCClient
+
+	ctx, cancel := context.WithCancel(context.Background())
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		http2Conn.Serve(ctx)
+	}()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://localhost:8080/", nil)
+	require.NoError(t, err)
+	req.Header.Set(internalUpgradeHeader, controlStreamUpgrade)
+
+	edgeHTTP2Conn, err := testTransport.NewClientConn(edgeConn)
+	require.NoError(t, err)
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		edgeHTTP2Conn.RoundTrip(req)
+	}()
+
+	<-rpcClientFactory.registered
+	cancel()
+	<-rpcClientFactory.unregistered
+	assert.False(t, http2Conn.stoppedGracefully)
+
+	wg.Wait()
+}
+
+func TestGracefulShutdownHTTP2(t *testing.T) {
+	http2Conn, edgeConn := newTestHTTP2Connection()
+
+	rpcClientFactory := mockRPCClientFactory{
+		registered:   make(chan struct{}),
+		unregistered: make(chan struct{}),
+	}
+	events := &eventCollectorSink{}
+	http2Conn.newRPCClientFunc = rpcClientFactory.newMockRPCClient
+	http2Conn.observer.RegisterSink(events)
+	shutdownC := make(chan struct{})
+	http2Conn.gracefulShutdownC = shutdownC
+
+	ctx, cancel := context.WithCancel(context.Background())
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		http2Conn.Serve(ctx)
+	}()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://localhost:8080/", nil)
+	require.NoError(t, err)
+	req.Header.Set(internalUpgradeHeader, controlStreamUpgrade)
+
+	edgeHTTP2Conn, err := testTransport.NewClientConn(edgeConn)
+	require.NoError(t, err)
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		_, _ = edgeHTTP2Conn.RoundTrip(req)
+	}()
+
+	select {
+	case <-rpcClientFactory.registered:
+		break //ok
+	case <-time.Tick(time.Second):
+		t.Fatal("timeout out waiting for registration")
+	}
+
+	// signal graceful shutdown
+	close(shutdownC)
+
+	select {
+	case <-rpcClientFactory.unregistered:
+		break //ok
+	case <-time.Tick(time.Second):
+		t.Fatal("timeout out waiting for unregistered signal")
+	}
+	assert.True(t, http2Conn.stoppedGracefully)
+
+	cancel()
+	wg.Wait()
+
+	events.assertSawEvent(t, Event{
+		Index:     http2Conn.connIndex,
+		EventType: Unregistering,
+	})
+}
+
+func benchmarkServeHTTP(b *testing.B, test testRequest) {
+	http2Conn, edgeConn := newTestHTTP2Connection()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		http2Conn.Serve(ctx)
+	}()
+
+	endpoint := fmt.Sprintf("http://localhost:8080/%s", test.endpoint)
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
+	require.NoError(b, err)
+
+	edgeHTTP2Conn, err := testTransport.NewClientConn(edgeConn)
+	require.NoError(b, err)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		b.StartTimer()
+		resp, err := edgeHTTP2Conn.RoundTrip(req)
+		b.StopTimer()
+		require.NoError(b, err)
+		require.Equal(b, test.expectedStatus, resp.StatusCode)
+		if test.expectedBody != nil {
+			respBody, err := ioutil.ReadAll(resp.Body)
+			require.NoError(b, err)
+			require.Equal(b, test.expectedBody, respBody)
+		}
+		resp.Body.Close()
+	}
+
+	cancel()
+	wg.Wait()
+}
+
+func BenchmarkServeHTTPSimple(b *testing.B) {
+	test := testRequest{
+		name:           "ok",
+		endpoint:       "ok",
+		expectedStatus: http.StatusOK,
+		expectedBody:   []byte(http.StatusText(http.StatusOK)),
+	}
+
+	benchmarkServeHTTP(b, test)
+}
+
+func BenchmarkServeHTTPLargeFile(b *testing.B) {
+	test := testRequest{
+		name:           "large_file",
+		endpoint:       "large_file",
+		expectedStatus: http.StatusOK,
+		expectedBody:   testLargeResp,
+	}
+
+	benchmarkServeHTTP(b, test)
+}

connection/json.go

@@ -0,0 +1,7 @@
+package connection
+
+import (
+	jsoniter "github.com/json-iterator/go"
+)
+
+var json = jsoniter.ConfigFastest

connection/metrics.go

@@ -0,0 +1,417 @@
+package connection
+
+import (
+	"sync"
+	"time"
+
+	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const (
+	MetricsNamespace = "cloudflared"
+	TunnelSubsystem  = "tunnel"
+	muxerSubsystem   = "muxer"
+)
+
+type muxerMetrics struct {
+	rtt              *prometheus.GaugeVec
+	rttMin           *prometheus.GaugeVec
+	rttMax           *prometheus.GaugeVec
+	receiveWindowAve *prometheus.GaugeVec
+	sendWindowAve    *prometheus.GaugeVec
+	receiveWindowMin *prometheus.GaugeVec
+	receiveWindowMax *prometheus.GaugeVec
+	sendWindowMin    *prometheus.GaugeVec
+	sendWindowMax    *prometheus.GaugeVec
+	inBoundRateCurr  *prometheus.GaugeVec
+	inBoundRateMin   *prometheus.GaugeVec
+	inBoundRateMax   *prometheus.GaugeVec
+	outBoundRateCurr *prometheus.GaugeVec
+	outBoundRateMin  *prometheus.GaugeVec
+	outBoundRateMax  *prometheus.GaugeVec
+	compBytesBefore  *prometheus.GaugeVec
+	compBytesAfter   *prometheus.GaugeVec
+	compRateAve      *prometheus.GaugeVec
+}
+
+type tunnelMetrics struct {
+	timerRetries    prometheus.Gauge
+	serverLocations *prometheus.GaugeVec
+	// locationLock is a mutex for oldServerLocations
+	locationLock sync.Mutex
+	// oldServerLocations stores the last server the tunnel was connected to
+	oldServerLocations map[string]string
+
+	regSuccess *prometheus.CounterVec
+	regFail    *prometheus.CounterVec
+	rpcFail    *prometheus.CounterVec
+
+	muxerMetrics        *muxerMetrics
+	tunnelsHA           tunnelsForHA
+	userHostnamesCounts *prometheus.CounterVec
+}
+
+func newMuxerMetrics() *muxerMetrics {
+	rtt := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "rtt",
+			Help:      "Round-trip time in millisecond",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(rtt)
+
+	rttMin := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "rtt_min",
+			Help:      "Shortest round-trip time in millisecond",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(rttMin)
+
+	rttMax := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "rtt_max",
+			Help:      "Longest round-trip time in millisecond",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(rttMax)
+
+	receiveWindowAve := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "receive_window_ave",
+			Help:      "Average receive window size in bytes",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(receiveWindowAve)
+
+	sendWindowAve := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "send_window_ave",
+			Help:      "Average send window size in bytes",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(sendWindowAve)
+
+	receiveWindowMin := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "receive_window_min",
+			Help:      "Smallest receive window size in bytes",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(receiveWindowMin)
+
+	receiveWindowMax := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "receive_window_max",
+			Help:      "Largest receive window size in bytes",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(receiveWindowMax)
+
+	sendWindowMin := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "send_window_min",
+			Help:      "Smallest send window size in bytes",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(sendWindowMin)
+
+	sendWindowMax := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "send_window_max",
+			Help:      "Largest send window size in bytes",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(sendWindowMax)
+
+	inBoundRateCurr := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "inbound_bytes_per_sec_curr",
+			Help:      "Current inbounding bytes per second, 0 if there is no incoming connection",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(inBoundRateCurr)
+
+	inBoundRateMin := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "inbound_bytes_per_sec_min",
+			Help:      "Minimum non-zero inbounding bytes per second",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(inBoundRateMin)
+
+	inBoundRateMax := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "inbound_bytes_per_sec_max",
+			Help:      "Maximum inbounding bytes per second",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(inBoundRateMax)
+
+	outBoundRateCurr := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "outbound_bytes_per_sec_curr",
+			Help:      "Current outbounding bytes per second, 0 if there is no outgoing traffic",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(outBoundRateCurr)
+
+	outBoundRateMin := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "outbound_bytes_per_sec_min",
+			Help:      "Minimum non-zero outbounding bytes per second",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(outBoundRateMin)
+
+	outBoundRateMax := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "outbound_bytes_per_sec_max",
+			Help:      "Maximum outbounding bytes per second",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(outBoundRateMax)
+
+	compBytesBefore := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "comp_bytes_before",
+			Help:      "Bytes sent via cross-stream compression, pre compression",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(compBytesBefore)
+
+	compBytesAfter := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "comp_bytes_after",
+			Help:      "Bytes sent via cross-stream compression, post compression",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(compBytesAfter)
+
+	compRateAve := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "comp_rate_ave",
+			Help:      "Average outbound cross-stream compression ratio",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(compRateAve)
+
+	return &muxerMetrics{
+		rtt:              rtt,
+		rttMin:           rttMin,
+		rttMax:           rttMax,
+		receiveWindowAve: receiveWindowAve,
+		sendWindowAve:    sendWindowAve,
+		receiveWindowMin: receiveWindowMin,
+		receiveWindowMax: receiveWindowMax,
+		sendWindowMin:    sendWindowMin,
+		sendWindowMax:    sendWindowMax,
+		inBoundRateCurr:  inBoundRateCurr,
+		inBoundRateMin:   inBoundRateMin,
+		inBoundRateMax:   inBoundRateMax,
+		outBoundRateCurr: outBoundRateCurr,
+		outBoundRateMin:  outBoundRateMin,
+		outBoundRateMax:  outBoundRateMax,
+		compBytesBefore:  compBytesBefore,
+		compBytesAfter:   compBytesAfter,
+		compRateAve:      compRateAve,
+	}
+}
+
+func (m *muxerMetrics) update(connectionID string, metrics *h2mux.MuxerMetrics) {
+	m.rtt.WithLabelValues(connectionID).Set(convertRTTMilliSec(metrics.RTT))
+	m.rttMin.WithLabelValues(connectionID).Set(convertRTTMilliSec(metrics.RTTMin))
+	m.rttMax.WithLabelValues(connectionID).Set(convertRTTMilliSec(metrics.RTTMax))
+	m.receiveWindowAve.WithLabelValues(connectionID).Set(metrics.ReceiveWindowAve)
+	m.sendWindowAve.WithLabelValues(connectionID).Set(metrics.SendWindowAve)
+	m.receiveWindowMin.WithLabelValues(connectionID).Set(float64(metrics.ReceiveWindowMin))
+	m.receiveWindowMax.WithLabelValues(connectionID).Set(float64(metrics.ReceiveWindowMax))
+	m.sendWindowMin.WithLabelValues(connectionID).Set(float64(metrics.SendWindowMin))
+	m.sendWindowMax.WithLabelValues(connectionID).Set(float64(metrics.SendWindowMax))
+	m.inBoundRateCurr.WithLabelValues(connectionID).Set(float64(metrics.InBoundRateCurr))
+	m.inBoundRateMin.WithLabelValues(connectionID).Set(float64(metrics.InBoundRateMin))
+	m.inBoundRateMax.WithLabelValues(connectionID).Set(float64(metrics.InBoundRateMax))
+	m.outBoundRateCurr.WithLabelValues(connectionID).Set(float64(metrics.OutBoundRateCurr))
+	m.outBoundRateMin.WithLabelValues(connectionID).Set(float64(metrics.OutBoundRateMin))
+	m.outBoundRateMax.WithLabelValues(connectionID).Set(float64(metrics.OutBoundRateMax))
+	m.compBytesBefore.WithLabelValues(connectionID).Set(float64(metrics.CompBytesBefore.Value()))
+	m.compBytesAfter.WithLabelValues(connectionID).Set(float64(metrics.CompBytesAfter.Value()))
+	m.compRateAve.WithLabelValues(connectionID).Set(float64(metrics.CompRateAve()))
+}
+
+func convertRTTMilliSec(t time.Duration) float64 {
+	return float64(t / time.Millisecond)
+}
+
+// Metrics that can be collected without asking the edge
+func initTunnelMetrics() *tunnelMetrics {
+	maxConcurrentRequestsPerTunnel := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: TunnelSubsystem,
+			Name:      "max_concurrent_requests_per_tunnel",
+			Help:      "Largest number of concurrent requests proxied through each tunnel so far",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(maxConcurrentRequestsPerTunnel)
+
+	timerRetries := prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: TunnelSubsystem,
+			Name:      "timer_retries",
+			Help:      "Unacknowledged heart beats count",
+		})
+	prometheus.MustRegister(timerRetries)
+
+	serverLocations := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: TunnelSubsystem,
+			Name:      "server_locations",
+			Help:      "Where each tunnel is connected to. 1 means current location, 0 means previous locations.",
+		},
+		[]string{"connection_id", "location"},
+	)
+	prometheus.MustRegister(serverLocations)
+
+	rpcFail := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: TunnelSubsystem,
+			Name:      "tunnel_rpc_fail",
+			Help:      "Count of RPC connection errors by type",
+		},
+		[]string{"error", "rpcName"},
+	)
+	prometheus.MustRegister(rpcFail)
+
+	registerFail := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: TunnelSubsystem,
+			Name:      "tunnel_register_fail",
+			Help:      "Count of tunnel registration errors by type",
+		},
+		[]string{"error", "rpcName"},
+	)
+	prometheus.MustRegister(registerFail)
+
+	userHostnamesCounts := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: TunnelSubsystem,
+			Name:      "user_hostnames_counts",
+			Help:      "Which user hostnames cloudflared is serving",
+		},
+		[]string{"userHostname"},
+	)
+	prometheus.MustRegister(userHostnamesCounts)
+
+	registerSuccess := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: TunnelSubsystem,
+			Name:      "tunnel_register_success",
+			Help:      "Count of successful tunnel registrations",
+		},
+		[]string{"rpcName"},
+	)
+	prometheus.MustRegister(registerSuccess)
+
+	return &tunnelMetrics{
+		timerRetries:        timerRetries,
+		serverLocations:     serverLocations,
+		oldServerLocations:  make(map[string]string),
+		muxerMetrics:        newMuxerMetrics(),
+		tunnelsHA:           newTunnelsForHA(),
+		regSuccess:          registerSuccess,
+		regFail:             registerFail,
+		rpcFail:             rpcFail,
+		userHostnamesCounts: userHostnamesCounts,
+	}
+}
+
+func (t *tunnelMetrics) updateMuxerMetrics(connectionID string, metrics *h2mux.MuxerMetrics) {
+	t.muxerMetrics.update(connectionID, metrics)
+}
+
+func (t *tunnelMetrics) registerServerLocation(connectionID, loc string) {
+	t.locationLock.Lock()
+	defer t.locationLock.Unlock()
+	if oldLoc, ok := t.oldServerLocations[connectionID]; ok && oldLoc == loc {
+		return
+	} else if ok {
+		t.serverLocations.WithLabelValues(connectionID, oldLoc).Dec()
+	}
+	t.serverLocations.WithLabelValues(connectionID, loc).Inc()
+	t.oldServerLocations[connectionID] = loc
+}
+
+var tunnelMetricsInternal struct {
+	sync.Once
+	metrics *tunnelMetrics
+}
+
+func newTunnelMetrics() *tunnelMetrics {
+	tunnelMetricsInternal.Do(func() {
+		tunnelMetricsInternal.metrics = initTunnelMetrics()
+	})
+	return tunnelMetricsInternal.metrics
+}

connection/observer.go

@@ -0,0 +1,155 @@
+package connection
+
+import (
+	"fmt"
+	"net/url"
+	"strings"
+
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+
+	"github.com/rs/zerolog"
+)
+
+const (
+	LogFieldLocation          = "location"
+	observerChannelBufferSize = 16
+)
+
+type Observer struct {
+	log             *zerolog.Logger
+	logTransport    *zerolog.Logger
+	metrics         *tunnelMetrics
+	tunnelEventChan chan Event
+	uiEnabled       bool
+	addSinkChan     chan EventSink
+}
+
+type EventSink interface {
+	OnTunnelEvent(event Event)
+}
+
+func NewObserver(log, logTransport *zerolog.Logger, uiEnabled bool) *Observer {
+	o := &Observer{
+		log:             log,
+		logTransport:    logTransport,
+		metrics:         newTunnelMetrics(),
+		uiEnabled:       uiEnabled,
+		tunnelEventChan: make(chan Event, observerChannelBufferSize),
+		addSinkChan:     make(chan EventSink, observerChannelBufferSize),
+	}
+	go o.dispatchEvents()
+	return o
+}
+
+func (o *Observer) RegisterSink(sink EventSink) {
+	o.addSinkChan <- sink
+}
+
+func (o *Observer) logServerInfo(connIndex uint8, location, msg string) {
+	o.sendEvent(Event{Index: connIndex, EventType: Connected, Location: location})
+	o.log.Info().
+		Uint8(LogFieldConnIndex, connIndex).
+		Str(LogFieldLocation, location).
+		Msg(msg)
+	o.metrics.registerServerLocation(uint8ToString(connIndex), location)
+}
+
+func (o *Observer) logTrialHostname(registration *tunnelpogs.TunnelRegistration) error {
+	// Print out the user's trial zone URL in a nice box (if they requested and got one and UI flag is not set)
+	if !o.uiEnabled {
+		if registrationURL, err := url.Parse(registration.Url); err == nil {
+			for _, line := range asciiBox(trialZoneMsg(registrationURL.String()), 2) {
+				o.log.Info().Msg(line)
+			}
+		} else {
+			o.log.Error().Msg("Failed to connect tunnel, please try again.")
+			return fmt.Errorf("empty URL in response from Cloudflare edge")
+		}
+	}
+	return nil
+}
+
+// Print out the given lines in a nice ASCII box.
+func asciiBox(lines []string, padding int) (box []string) {
+	maxLen := maxLen(lines)
+	spacer := strings.Repeat(" ", padding)
+
+	border := "+" + strings.Repeat("-", maxLen+(padding*2)) + "+"
+
+	box = append(box, border)
+	for _, line := range lines {
+		box = append(box, "|"+spacer+line+strings.Repeat(" ", maxLen-len(line))+spacer+"|")
+	}
+	box = append(box, border)
+	return
+}
+
+func maxLen(lines []string) int {
+	max := 0
+	for _, line := range lines {
+		if len(line) > max {
+			max = len(line)
+		}
+	}
+	return max
+}
+
+func trialZoneMsg(url string) []string {
+	return []string{
+		"Your free tunnel has started! Visit it:",
+		"  " + url,
+	}
+}
+
+func (o *Observer) sendRegisteringEvent(connIndex uint8) {
+	o.sendEvent(Event{Index: connIndex, EventType: RegisteringTunnel})
+}
+
+func (o *Observer) sendConnectedEvent(connIndex uint8, location string) {
+	o.sendEvent(Event{Index: connIndex, EventType: Connected, Location: location})
+}
+
+func (o *Observer) sendURL(url string) {
+	o.sendEvent(Event{EventType: SetURL, URL: url})
+}
+
+func (o *Observer) SendReconnect(connIndex uint8) {
+	o.sendEvent(Event{Index: connIndex, EventType: Reconnecting})
+}
+
+func (o *Observer) sendUnregisteringEvent(connIndex uint8) {
+	o.sendEvent(Event{Index: connIndex, EventType: Unregistering})
+}
+
+func (o *Observer) SendDisconnect(connIndex uint8) {
+	o.sendEvent(Event{Index: connIndex, EventType: Disconnected})
+}
+
+func (o *Observer) sendEvent(e Event) {
+	select {
+	case o.tunnelEventChan <- e:
+		break
+	default:
+		o.log.Warn().Msg("observer channel buffer is full")
+	}
+}
+
+func (o *Observer) dispatchEvents() {
+	var sinks []EventSink
+	for {
+		select {
+		case sink := <-o.addSinkChan:
+			sinks = append(sinks, sink)
+		case evt := <-o.tunnelEventChan:
+			for _, sink := range sinks {
+				sink.OnTunnelEvent(evt)
+			}
+		}
+	}
+}
+
+type EventSinkFunc func(event Event)
+
+func (f EventSinkFunc) OnTunnelEvent(event Event) {
+	f(event)
+}

connection/observer_test.go

@@ -0,0 +1,86 @@
+package connection
+
+import (
+	"strconv"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRegisterServerLocation(t *testing.T) {
+	m := newTunnelMetrics()
+	tunnels := 20
+	var wg sync.WaitGroup
+	wg.Add(tunnels)
+	for i := 0; i < tunnels; i++ {
+		go func(i int) {
+			id := strconv.Itoa(i)
+			m.registerServerLocation(id, "LHR")
+			wg.Done()
+		}(i)
+	}
+	wg.Wait()
+	for i := 0; i < tunnels; i++ {
+		id := strconv.Itoa(i)
+		assert.Equal(t, "LHR", m.oldServerLocations[id])
+	}
+
+	wg.Add(tunnels)
+	for i := 0; i < tunnels; i++ {
+		go func(i int) {
+			id := strconv.Itoa(i)
+			m.registerServerLocation(id, "AUS")
+			wg.Done()
+		}(i)
+	}
+	wg.Wait()
+	for i := 0; i < tunnels; i++ {
+		id := strconv.Itoa(i)
+		assert.Equal(t, "AUS", m.oldServerLocations[id])
+	}
+
+}
+
+func TestObserverEventsDontBlock(t *testing.T) {
+	observer := NewObserver(&log, &log, false)
+	var mu sync.Mutex
+	observer.RegisterSink(EventSinkFunc(func(_ Event) {
+		// callback will block if lock is already held
+		mu.Lock()
+		mu.Unlock()
+	}))
+
+	timeout := time.AfterFunc(5*time.Second, func() {
+		mu.Unlock() // release the callback on timer expiration
+		t.Fatal("observer is blocked")
+	})
+
+	mu.Lock() // block the callback
+	for i := 0; i < 2*observerChannelBufferSize; i++ {
+		observer.sendRegisteringEvent(0)
+	}
+	if pending := timeout.Stop(); pending {
+		// release the callback if timer hasn't expired yet
+		mu.Unlock()
+	}
+}
+
+
+type eventCollectorSink struct {
+	observedEvents []Event
+	mu sync.Mutex
+}
+
+func (s *eventCollectorSink) OnTunnelEvent(event Event) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.observedEvents = append(s.observedEvents, event)
+}
+
+func (s *eventCollectorSink) assertSawEvent(t *testing.T, event Event) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	assert.Contains(t, s.observedEvents, event)
+}

connection/protocol.go

@@ -0,0 +1,185 @@
+package connection
+
+import (
+	"fmt"
+	"hash/fnv"
+	"sync"
+	"time"
+
+	"github.com/rs/zerolog"
+)
+
+const (
+	AvailableProtocolFlagMessage = "Available protocols: http2 - Go's implementation, h2mux - Cloudflare's implementation of HTTP/2, and auto - automatically select between http2 and h2mux"
+	// edgeH2muxTLSServerName is the server name to establish h2mux connection with edge
+	edgeH2muxTLSServerName = "cftunnel.com"
+	// edgeH2TLSServerName is the server name to establish http2 connection with edge
+	edgeH2TLSServerName = "h2.cftunnel.com"
+	// threshold to switch back to h2mux when the user intentionally pick --protocol http2
+	explicitHTTP2FallbackThreshold = -1
+	autoSelectFlag                 = "auto"
+)
+
+var (
+	ProtocolList = []Protocol{H2mux, HTTP2}
+)
+
+type Protocol int64
+
+const (
+	H2mux Protocol = iota
+	HTTP2
+)
+
+func (p Protocol) ServerName() string {
+	switch p {
+	case H2mux:
+		return edgeH2muxTLSServerName
+	case HTTP2:
+		return edgeH2TLSServerName
+	default:
+		return ""
+	}
+}
+
+// Fallback returns the fallback protocol and whether the protocol has a fallback
+func (p Protocol) fallback() (Protocol, bool) {
+	switch p {
+	case H2mux:
+		return 0, false
+	case HTTP2:
+		return H2mux, true
+	default:
+		return 0, false
+	}
+}
+
+func (p Protocol) String() string {
+	switch p {
+	case H2mux:
+		return "h2mux"
+	case HTTP2:
+		return "http2"
+	default:
+		return fmt.Sprintf("unknown protocol")
+	}
+}
+
+type ProtocolSelector interface {
+	Current() Protocol
+	Fallback() (Protocol, bool)
+}
+
+type staticProtocolSelector struct {
+	current Protocol
+}
+
+func (s *staticProtocolSelector) Current() Protocol {
+	return s.current
+}
+
+func (s *staticProtocolSelector) Fallback() (Protocol, bool) {
+	return 0, false
+}
+
+type autoProtocolSelector struct {
+	lock           sync.RWMutex
+	current        Protocol
+	switchThrehold int32
+	fetchFunc      PercentageFetcher
+	refreshAfter   time.Time
+	ttl            time.Duration
+	log            *zerolog.Logger
+}
+
+func newAutoProtocolSelector(
+	current Protocol,
+	switchThrehold int32,
+	fetchFunc PercentageFetcher,
+	ttl time.Duration,
+	log *zerolog.Logger,
+) *autoProtocolSelector {
+	return &autoProtocolSelector{
+		current:        current,
+		switchThrehold: switchThrehold,
+		fetchFunc:      fetchFunc,
+		refreshAfter:   time.Now().Add(ttl),
+		ttl:            ttl,
+		log:            log,
+	}
+}
+
+func (s *autoProtocolSelector) Current() Protocol {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+	if time.Now().Before(s.refreshAfter) {
+		return s.current
+	}
+
+	percentage, err := s.fetchFunc()
+	if err != nil {
+		s.log.Err(err).Msg("Failed to refresh protocol")
+		return s.current
+	}
+
+	if s.switchThrehold < percentage {
+		s.current = HTTP2
+	} else {
+		s.current = H2mux
+	}
+	s.refreshAfter = time.Now().Add(s.ttl)
+	return s.current
+}
+
+func (s *autoProtocolSelector) Fallback() (Protocol, bool) {
+	s.lock.RLock()
+	defer s.lock.RUnlock()
+	return s.current.fallback()
+}
+
+type PercentageFetcher func() (int32, error)
+
+func NewProtocolSelector(
+	protocolFlag string,
+	namedTunnel *NamedTunnelConfig,
+	fetchFunc PercentageFetcher,
+	ttl time.Duration,
+	log *zerolog.Logger,
+) (ProtocolSelector, error) {
+	if namedTunnel == nil {
+		return &staticProtocolSelector{
+			current: H2mux,
+		}, nil
+	}
+	if protocolFlag == H2mux.String() {
+		return &staticProtocolSelector{
+			current: H2mux,
+		}, nil
+	}
+
+	http2Percentage, err := fetchFunc()
+	if err != nil {
+		return nil, err
+	}
+	if protocolFlag == HTTP2.String() {
+		if http2Percentage < 0 {
+			return newAutoProtocolSelector(H2mux, explicitHTTP2FallbackThreshold, fetchFunc, ttl, log), nil
+		}
+		return newAutoProtocolSelector(HTTP2, explicitHTTP2FallbackThreshold, fetchFunc, ttl, log), nil
+	}
+
+	if protocolFlag != autoSelectFlag {
+		return nil, fmt.Errorf("Unknown protocol %s, %s", protocolFlag, AvailableProtocolFlagMessage)
+	}
+	threshold := switchThreshold(namedTunnel.Credentials.AccountTag)
+	if threshold < http2Percentage {
+		return newAutoProtocolSelector(HTTP2, threshold, fetchFunc, ttl, log), nil
+	}
+	return newAutoProtocolSelector(H2mux, threshold, fetchFunc, ttl, log), nil
+}
+
+func switchThreshold(accountTag string) int32 {
+	h := fnv.New32a()
+	_, _ = h.Write([]byte(accountTag))
+	return int32(h.Sum32() % 100)
+}

connection/protocol_test.go

@@ -0,0 +1,215 @@
+package connection
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+const (
+	testNoTTL = 0
+)
+
+var (
+	testNamedTunnelConfig = &NamedTunnelConfig{
+		Credentials: Credentials{
+			AccountTag: "testAccountTag",
+		},
+	}
+)
+
+func mockFetcher(percentage int32) PercentageFetcher {
+	return func() (int32, error) {
+		return percentage, nil
+	}
+}
+
+func mockFetcherWithError() PercentageFetcher {
+	return func() (int32, error) {
+		return 0, fmt.Errorf("failed to fetch precentage")
+	}
+}
+
+type dynamicMockFetcher struct {
+	percentage int32
+	err        error
+}
+
+func (dmf *dynamicMockFetcher) fetch() PercentageFetcher {
+	return func() (int32, error) {
+		if dmf.err != nil {
+			return 0, dmf.err
+		}
+		return dmf.percentage, nil
+	}
+}
+
+func TestNewProtocolSelector(t *testing.T) {
+	tests := []struct {
+		name              string
+		protocol          string
+		expectedProtocol  Protocol
+		hasFallback       bool
+		expectedFallback  Protocol
+		namedTunnelConfig *NamedTunnelConfig
+		fetchFunc         PercentageFetcher
+		wantErr           bool
+	}{
+		{
+			name:              "classic tunnel",
+			protocol:          "h2mux",
+			expectedProtocol:  H2mux,
+			namedTunnelConfig: nil,
+		},
+		{
+			name:              "named tunnel over h2mux",
+			protocol:          "h2mux",
+			expectedProtocol:  H2mux,
+			namedTunnelConfig: testNamedTunnelConfig,
+		},
+		{
+			name:              "named tunnel over http2",
+			protocol:          "http2",
+			expectedProtocol:  HTTP2,
+			hasFallback:       true,
+			expectedFallback:  H2mux,
+			fetchFunc:         mockFetcher(0),
+			namedTunnelConfig: testNamedTunnelConfig,
+		},
+		{
+			name:              "named tunnel http2 disabled",
+			protocol:          "http2",
+			expectedProtocol:  H2mux,
+			fetchFunc:         mockFetcher(-1),
+			namedTunnelConfig: testNamedTunnelConfig,
+		},
+		{
+			name:              "named tunnel auto all http2 disabled",
+			protocol:          "auto",
+			expectedProtocol:  H2mux,
+			fetchFunc:         mockFetcher(-1),
+			namedTunnelConfig: testNamedTunnelConfig,
+		},
+		{
+			name:              "named tunnel auto to h2mux",
+			protocol:          "auto",
+			expectedProtocol:  H2mux,
+			fetchFunc:         mockFetcher(0),
+			namedTunnelConfig: testNamedTunnelConfig,
+		},
+		{
+			name:              "named tunnel auto to http2",
+			protocol:          "auto",
+			expectedProtocol:  HTTP2,
+			hasFallback:       true,
+			expectedFallback:  H2mux,
+			fetchFunc:         mockFetcher(100),
+			namedTunnelConfig: testNamedTunnelConfig,
+		},
+		{
+			// None named tunnel can only use h2mux, so specifying an unknown protocol is not an error
+			name:             "classic tunnel unknown protocol",
+			protocol:         "unknown",
+			expectedProtocol: H2mux,
+		},
+		{
+			name:              "named tunnel unknown protocol",
+			protocol:          "unknown",
+			fetchFunc:         mockFetcher(100),
+			namedTunnelConfig: testNamedTunnelConfig,
+			wantErr:           true,
+		},
+		{
+			name:              "named tunnel fetch error",
+			protocol:          "unknown",
+			fetchFunc:         mockFetcherWithError(),
+			namedTunnelConfig: testNamedTunnelConfig,
+			wantErr:           true,
+		},
+	}
+
+	for _, test := range tests {
+		selector, err := NewProtocolSelector(test.protocol, test.namedTunnelConfig, test.fetchFunc, testNoTTL, &log)
+		if test.wantErr {
+			assert.Error(t, err, fmt.Sprintf("test %s failed", test.name))
+		} else {
+			assert.NoError(t, err, fmt.Sprintf("test %s failed", test.name))
+			assert.Equal(t, test.expectedProtocol, selector.Current(), fmt.Sprintf("test %s failed", test.name))
+			fallback, ok := selector.Fallback()
+			assert.Equal(t, test.hasFallback, ok, fmt.Sprintf("test %s failed", test.name))
+			if test.hasFallback {
+				assert.Equal(t, test.expectedFallback, fallback, fmt.Sprintf("test %s failed", test.name))
+			}
+		}
+	}
+}
+
+func TestAutoProtocolSelectorRefresh(t *testing.T) {
+	fetcher := dynamicMockFetcher{}
+	selector, err := NewProtocolSelector("auto", testNamedTunnelConfig, fetcher.fetch(), testNoTTL, &log)
+	assert.NoError(t, err)
+	assert.Equal(t, H2mux, selector.Current())
+
+	fetcher.percentage = 100
+	assert.Equal(t, HTTP2, selector.Current())
+
+	fetcher.percentage = 0
+	assert.Equal(t, H2mux, selector.Current())
+
+	fetcher.percentage = 100
+	assert.Equal(t, HTTP2, selector.Current())
+
+	fetcher.err = fmt.Errorf("failed to fetch")
+	assert.Equal(t, HTTP2, selector.Current())
+
+	fetcher.percentage = -1
+	fetcher.err = nil
+	assert.Equal(t, H2mux, selector.Current())
+
+	fetcher.percentage = 0
+	assert.Equal(t, H2mux, selector.Current())
+
+	fetcher.percentage = 100
+	assert.Equal(t, HTTP2, selector.Current())
+}
+
+func TestHTTP2ProtocolSelectorRefresh(t *testing.T) {
+	fetcher := dynamicMockFetcher{}
+	selector, err := NewProtocolSelector("http2", testNamedTunnelConfig, fetcher.fetch(), testNoTTL, &log)
+	assert.NoError(t, err)
+	assert.Equal(t, HTTP2, selector.Current())
+
+	fetcher.percentage = 100
+	assert.Equal(t, HTTP2, selector.Current())
+
+	fetcher.percentage = 0
+	assert.Equal(t, HTTP2, selector.Current())
+
+	fetcher.err = fmt.Errorf("failed to fetch")
+	assert.Equal(t, HTTP2, selector.Current())
+
+	fetcher.percentage = -1
+	fetcher.err = nil
+	assert.Equal(t, H2mux, selector.Current())
+
+	fetcher.percentage = 0
+	assert.Equal(t, HTTP2, selector.Current())
+
+	fetcher.percentage = 100
+	assert.Equal(t, HTTP2, selector.Current())
+
+	fetcher.percentage = -1
+	assert.Equal(t, H2mux, selector.Current())
+}
+
+func TestProtocolSelectorRefreshTTL(t *testing.T) {
+	fetcher := dynamicMockFetcher{percentage: 100}
+	selector, err := NewProtocolSelector("auto", testNamedTunnelConfig, fetcher.fetch(), time.Hour, &log)
+	assert.NoError(t, err)
+	assert.Equal(t, HTTP2, selector.Current())
+
+	fetcher.percentage = 0
+	assert.Equal(t, HTTP2, selector.Current())
+}

connection/rpc.go

@@ -2,40 +2,318 @@ package connection
 
 import (
 	"context"
+	"fmt"
 	"io"
+	"time"
 
-	rpc "zombiezen.com/go/capnproto2/rpc"
-
-	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/tunnelrpc"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+
+	"github.com/rs/zerolog"
+	"zombiezen.com/go/capnproto2/rpc"
 )
 
-// NewTunnelRPCClient creates and returns a new RPC client, which will communicate
-// using a stream on the given muxer
-func NewTunnelRPCClient(
-	ctx context.Context,
-	stream io.ReadWriteCloser,
-	logger logger.Service,
-) (client tunnelpogs.TunnelServer_PogsClient, err error) {
-	conn := rpc.NewConn(
-		tunnelrpc.NewTransportLogger(logger, rpc.StreamTransport(stream)),
-		tunnelrpc.ConnLog(logger),
-	)
-	registrationClient := tunnelpogs.RegistrationServer_PogsClient{Client: conn.Bootstrap(ctx), Conn: conn}
-	client = tunnelpogs.TunnelServer_PogsClient{RegistrationServer_PogsClient: registrationClient, Client: conn.Bootstrap(ctx), Conn: conn}
-	return client, nil
+type tunnelServerClient struct {
+	client    tunnelpogs.TunnelServer_PogsClient
+	transport rpc.Transport
 }
 
-func NewRegistrationRPCClient(
+// NewTunnelRPCClient creates and returns a new RPC client, which will communicate using a stream on the given muxer.
+// This method is exported for supervisor to call Authenticate RPC
+func NewTunnelServerClient(
 	ctx context.Context,
 	stream io.ReadWriteCloser,
-	logger logger.Service,
-) (client tunnelpogs.RegistrationServer_PogsClient, err error) {
+	log *zerolog.Logger,
+) *tunnelServerClient {
+	transport := tunnelrpc.NewTransportLogger(log, rpc.StreamTransport(stream))
 	conn := rpc.NewConn(
-		tunnelrpc.NewTransportLogger(logger, rpc.StreamTransport(stream)),
-		tunnelrpc.ConnLog(logger),
+		transport,
+		tunnelrpc.ConnLog(log),
 	)
-	client = tunnelpogs.RegistrationServer_PogsClient{Client: conn.Bootstrap(ctx), Conn: conn}
-	return client, nil
+	registrationClient := tunnelpogs.RegistrationServer_PogsClient{Client: conn.Bootstrap(ctx), Conn: conn}
+	return &tunnelServerClient{
+		client:    tunnelpogs.TunnelServer_PogsClient{RegistrationServer_PogsClient: registrationClient, Client: conn.Bootstrap(ctx), Conn: conn},
+		transport: transport,
+	}
+}
+
+func (tsc *tunnelServerClient) Authenticate(ctx context.Context, classicTunnel *ClassicTunnelConfig, registrationOptions *tunnelpogs.RegistrationOptions) (tunnelpogs.AuthOutcome, error) {
+	authResp, err := tsc.client.Authenticate(ctx, classicTunnel.OriginCert, classicTunnel.Hostname, registrationOptions)
+	if err != nil {
+		return nil, err
+	}
+	return authResp.Outcome(), nil
+}
+
+func (tsc *tunnelServerClient) Close() {
+	// Closing the client will also close the connection
+	_ = tsc.client.Close()
+	_ = tsc.transport.Close()
+}
+
+type NamedTunnelRPCClient interface {
+	RegisterConnection(
+		c context.Context,
+		config *NamedTunnelConfig,
+		options *tunnelpogs.ConnectionOptions,
+		connIndex uint8,
+		observer *Observer,
+	) error
+	GracefulShutdown(ctx context.Context, gracePeriod time.Duration)
+	Close()
+}
+
+type registrationServerClient struct {
+	client    tunnelpogs.RegistrationServer_PogsClient
+	transport rpc.Transport
+}
+
+func newRegistrationRPCClient(
+	ctx context.Context,
+	stream io.ReadWriteCloser,
+	log *zerolog.Logger,
+) NamedTunnelRPCClient {
+	transport := tunnelrpc.NewTransportLogger(log, rpc.StreamTransport(stream))
+	conn := rpc.NewConn(
+		transport,
+		tunnelrpc.ConnLog(log),
+	)
+	return &registrationServerClient{
+		client:    tunnelpogs.RegistrationServer_PogsClient{Client: conn.Bootstrap(ctx), Conn: conn},
+		transport: transport,
+	}
+}
+
+func (rsc *registrationServerClient) RegisterConnection(
+	ctx context.Context,
+	config *NamedTunnelConfig,
+	options *tunnelpogs.ConnectionOptions,
+	connIndex uint8,
+	observer *Observer,
+) error {
+	conn, err := rsc.client.RegisterConnection(
+		ctx,
+		config.Credentials.Auth(),
+		config.Credentials.TunnelID,
+		connIndex,
+		options,
+	)
+	if err != nil {
+		if err.Error() == DuplicateConnectionError {
+			observer.metrics.regFail.WithLabelValues("dup_edge_conn", "registerConnection").Inc()
+			return errDuplicationConnection
+		}
+		observer.metrics.regFail.WithLabelValues("server_error", "registerConnection").Inc()
+		return serverRegistrationErrorFromRPC(err)
+	}
+
+	observer.metrics.regSuccess.WithLabelValues("registerConnection").Inc()
+
+	observer.logServerInfo(connIndex, conn.Location, fmt.Sprintf("Connection %s registered", conn.UUID))
+	observer.sendConnectedEvent(connIndex, conn.Location)
+
+	return nil
+}
+
+func (rsc *registrationServerClient) GracefulShutdown(ctx context.Context, gracePeriod time.Duration) {
+	ctx, cancel := context.WithTimeout(ctx, gracePeriod)
+	defer cancel()
+	_ = rsc.client.UnregisterConnection(ctx)
+}
+
+func (rsc *registrationServerClient) Close() {
+	// Closing the client will also close the connection
+	_ = rsc.client.Close()
+	// Closing the transport also closes the stream
+	_ = rsc.transport.Close()
+}
+
+type rpcName string
+
+const (
+	register     rpcName = "register"
+	reconnect    rpcName = "reconnect"
+	unregister   rpcName = "unregister"
+	authenticate rpcName = " authenticate"
+)
+
+func (h *h2muxConnection) registerTunnel(ctx context.Context, credentialSetter CredentialManager, classicTunnel *ClassicTunnelConfig, registrationOptions *tunnelpogs.RegistrationOptions) error {
+	h.observer.sendRegisteringEvent(registrationOptions.ConnectionID)
+
+	stream, err := h.newRPCStream(ctx, register)
+	if err != nil {
+		return err
+	}
+	rpcClient := NewTunnelServerClient(ctx, stream, h.observer.log)
+	defer rpcClient.Close()
+
+	_ = h.logServerInfo(ctx, rpcClient)
+	registration := rpcClient.client.RegisterTunnel(
+		ctx,
+		classicTunnel.OriginCert,
+		classicTunnel.Hostname,
+		registrationOptions,
+	)
+	if registrationErr := registration.DeserializeError(); registrationErr != nil {
+		// RegisterTunnel RPC failure
+		return h.processRegisterTunnelError(registrationErr, register)
+	}
+
+	// Send free tunnel URL to UI
+	h.observer.sendURL(registration.Url)
+	credentialSetter.SetEventDigest(h.connIndex, registration.EventDigest)
+	return h.processRegistrationSuccess(registration, register, credentialSetter, classicTunnel)
+}
+
+type CredentialManager interface {
+	ReconnectToken() ([]byte, error)
+	EventDigest(connID uint8) ([]byte, error)
+	SetEventDigest(connID uint8, digest []byte)
+	ConnDigest(connID uint8) ([]byte, error)
+	SetConnDigest(connID uint8, digest []byte)
+}
+
+func (h *h2muxConnection) processRegistrationSuccess(
+	registration *tunnelpogs.TunnelRegistration,
+	name rpcName,
+	credentialManager CredentialManager, classicTunnel *ClassicTunnelConfig,
+) error {
+	for _, logLine := range registration.LogLines {
+		h.observer.log.Info().Msg(logLine)
+	}
+
+	if registration.TunnelID != "" {
+		h.observer.metrics.tunnelsHA.AddTunnelID(h.connIndex, registration.TunnelID)
+		h.observer.log.Info().Msgf("Each HA connection's tunnel IDs: %v", h.observer.metrics.tunnelsHA.String())
+	}
+
+	// Print out the user's trial zone URL in a nice box (if they requested and got one and UI flag is not set)
+	if classicTunnel.IsTrialZone() {
+		err := h.observer.logTrialHostname(registration)
+		if err != nil {
+			return err
+		}
+	}
+
+	credentialManager.SetConnDigest(h.connIndex, registration.ConnDigest)
+	h.observer.metrics.userHostnamesCounts.WithLabelValues(registration.Url).Inc()
+
+	h.observer.log.Info().Msgf("Route propagating, it may take up to 1 minute for your new route to become functional")
+	h.observer.metrics.regSuccess.WithLabelValues(string(name)).Inc()
+	return nil
+}
+
+func (h *h2muxConnection) processRegisterTunnelError(err tunnelpogs.TunnelRegistrationError, name rpcName) error {
+	if err.Error() == DuplicateConnectionError {
+		h.observer.metrics.regFail.WithLabelValues("dup_edge_conn", string(name)).Inc()
+		return errDuplicationConnection
+	}
+	h.observer.metrics.regFail.WithLabelValues("server_error", string(name)).Inc()
+	return ServerRegisterTunnelError{
+		Cause:     err,
+		Permanent: err.IsPermanent(),
+	}
+}
+
+func (h *h2muxConnection) reconnectTunnel(ctx context.Context, credentialManager CredentialManager, classicTunnel *ClassicTunnelConfig, registrationOptions *tunnelpogs.RegistrationOptions) error {
+	token, err := credentialManager.ReconnectToken()
+	if err != nil {
+		return err
+	}
+	eventDigest, err := credentialManager.EventDigest(h.connIndex)
+	if err != nil {
+		return err
+	}
+	connDigest, err := credentialManager.ConnDigest(h.connIndex)
+	if err != nil {
+		return err
+	}
+
+	h.observer.log.Debug().Msg("initiating RPC stream to reconnect")
+	stream, err := h.newRPCStream(ctx, register)
+	if err != nil {
+		return err
+	}
+	rpcClient := NewTunnelServerClient(ctx, stream, h.observer.log)
+	defer rpcClient.Close()
+
+	_ = h.logServerInfo(ctx, rpcClient)
+	registration := rpcClient.client.ReconnectTunnel(
+		ctx,
+		token,
+		eventDigest,
+		connDigest,
+		classicTunnel.Hostname,
+		registrationOptions,
+	)
+	if registrationErr := registration.DeserializeError(); registrationErr != nil {
+		// ReconnectTunnel RPC failure
+		return h.processRegisterTunnelError(registrationErr, reconnect)
+	}
+	return h.processRegistrationSuccess(registration, reconnect, credentialManager, classicTunnel)
+}
+
+func (h *h2muxConnection) logServerInfo(ctx context.Context, rpcClient *tunnelServerClient) error {
+	// Request server info without blocking tunnel registration; must use capnp library directly.
+	serverInfoPromise := tunnelrpc.TunnelServer{Client: rpcClient.client.Client}.GetServerInfo(ctx, func(tunnelrpc.TunnelServer_getServerInfo_Params) error {
+		return nil
+	})
+	serverInfoMessage, err := serverInfoPromise.Result().Struct()
+	if err != nil {
+		h.observer.log.Err(err).Msg("Failed to retrieve server information")
+		return err
+	}
+	serverInfo, err := tunnelpogs.UnmarshalServerInfo(serverInfoMessage)
+	if err != nil {
+		h.observer.log.Err(err).Msg("Failed to retrieve server information")
+		return err
+	}
+	h.observer.logServerInfo(h.connIndex, serverInfo.LocationName, "Connection established")
+	return nil
+}
+
+func (h *h2muxConnection) registerNamedTunnel(
+	ctx context.Context,
+	namedTunnel *NamedTunnelConfig,
+	connOptions *tunnelpogs.ConnectionOptions,
+) error {
+	stream, err := h.newRPCStream(ctx, register)
+	if err != nil {
+		return err
+	}
+	rpcClient := h.newRPCClientFunc(ctx, stream, h.observer.log)
+	defer rpcClient.Close()
+
+	if err = rpcClient.RegisterConnection(ctx, namedTunnel, connOptions, h.connIndex, h.observer); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (h *h2muxConnection) unregister(isNamedTunnel bool) {
+	h.observer.sendUnregisteringEvent(h.connIndex)
+
+	unregisterCtx, cancel := context.WithTimeout(context.Background(), h.config.GracePeriod)
+	defer cancel()
+
+	stream, err := h.newRPCStream(unregisterCtx, unregister)
+	if err != nil {
+		return
+	}
+	defer stream.Close()
+
+	if isNamedTunnel {
+		rpcClient := h.newRPCClientFunc(unregisterCtx, stream, h.observer.log)
+		defer rpcClient.Close()
+
+		rpcClient.GracefulShutdown(unregisterCtx, h.config.GracePeriod)
+	} else {
+		rpcClient := NewTunnelServerClient(unregisterCtx, stream, h.observer.log)
+		defer rpcClient.Close()
+
+		// gracePeriod is encoded in int64 using capnproto
+		_ = rpcClient.client.UnregisterTunnel(unregisterCtx, h.config.GracePeriod.Nanoseconds())
+	}
+
+	h.observer.log.Info().Uint8(LogFieldConnIndex, h.connIndex).Msg("Unregistered tunnel connection")
 }

connection/tunnelsforha.go

@@ -0,0 +1,50 @@
+package connection
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// tunnelsForHA maps this cloudflared instance's HA connections to the tunnel IDs they serve.
+type tunnelsForHA struct {
+	sync.Mutex
+	metrics *prometheus.GaugeVec
+	entries map[uint8]string
+}
+
+// NewTunnelsForHA initializes the Prometheus metrics etc for a tunnelsForHA.
+func newTunnelsForHA() tunnelsForHA {
+	metrics := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "tunnel_ids",
+			Help: "The ID of all tunnels (and their corresponding HA connection ID) running in this instance of cloudflared.",
+		},
+		[]string{"tunnel_id", "ha_conn_id"},
+	)
+	prometheus.MustRegister(metrics)
+
+	return tunnelsForHA{
+		metrics: metrics,
+		entries: make(map[uint8]string),
+	}
+}
+
+// Track a new tunnel ID, removing the disconnected tunnel (if any) and update metrics.
+func (t *tunnelsForHA) AddTunnelID(haConn uint8, tunnelID string) {
+	t.Lock()
+	defer t.Unlock()
+	haStr := fmt.Sprintf("%v", haConn)
+	if oldTunnelID, ok := t.entries[haConn]; ok {
+		t.metrics.WithLabelValues(oldTunnelID, haStr).Dec()
+	}
+	t.entries[haConn] = tunnelID
+	t.metrics.WithLabelValues(tunnelID, haStr).Inc()
+}
+
+func (t *tunnelsForHA) String() string {
+	t.Lock()
+	defer t.Unlock()
+	return fmt.Sprintf("%v", t.entries)
+}

dbconnect/proxy.go

@@ -8,20 +8,25 @@ import (
 	"net"
 	"net/http"
 	"net/url"
+	"os"
 	"time"
 
 	"github.com/cloudflare/cloudflared/hello"
-	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/validation"
 	"github.com/gorilla/mux"
 	"github.com/pkg/errors"
+	"github.com/rs/zerolog"
+)
+
+const (
+	LogFieldCFRay = "cfRay"
 )
 
 // Proxy is an HTTP server that proxies requests to a Client.
 type Proxy struct {
 	client          Client
 	accessValidator *validation.Access
-	logger          logger.Service
+	log             *zerolog.Logger
 }
 
 // NewInsecureProxy creates a Proxy that talks to a Client at an origin.
@@ -43,12 +48,9 @@ func NewInsecureProxy(ctx context.Context, origin string) (*Proxy, error) {
 		return nil, errors.Wrap(err, "could not connect to the database")
 	}
 
-	logger, err := logger.New()
-	if err != nil {
-		return nil, errors.Wrap(err, "error setting up logger")
-	}
+	log := zerolog.New(os.Stderr).With().Logger() // TODO: Does not obey log configuration
 
-	return &Proxy{client, nil, logger}, nil
+	return &Proxy{client, nil, &log}, nil
 }
 
 // NewSecureProxy creates a Proxy that talks to a Client at an origin.
@@ -95,8 +97,7 @@ func (proxy *Proxy) IsAllowed(r *http.Request, verbose ...bool) bool {
 	// Warn administrators that invalid JWTs are being rejected. This is indicative
 	// of either a misconfiguration of the CLI or a massive failure of upstream systems.
 	if len(verbose) > 0 {
-		cfRay := proxy.getRayHeader(r)
-		proxy.logger.Infof("dbproxy: Failed JWT authentication: cf-ray: %s %s", cfRay, err)
+		proxy.log.Err(err).Str(LogFieldCFRay, proxy.getRayHeader(r)).Msg("dbproxy: Failed JWT authentication")
 	}
 
 	return false
@@ -151,8 +152,8 @@ func (proxy *Proxy) httpListen(ctx context.Context, listener net.Listener) error
 
 	go func() {
 		<-ctx.Done()
-		httpServer.Close()
-		listener.Close()
+		_ = httpServer.Close()
+		_ = listener.Close()
 	}()
 
 	return httpServer.Serve(listener)
@@ -241,7 +242,7 @@ func (proxy *Proxy) httpRespondErr(w http.ResponseWriter, r *http.Request, defau
 	proxy.httpRespond(w, r, status, err.Error())
 	if len(err.Error()) > 0 {
 		cfRay := proxy.getRayHeader(r)
-		proxy.logger.Infof("dbproxy: Database proxy error: cf-ray: %s %s", cfRay, err)
+		proxy.log.Err(err).Str(LogFieldCFRay, cfRay).Msg("dbproxy: Database proxy error")
 	}
 }
 

dbconnect/sql.go

@@ -46,7 +46,7 @@ func NewSQLClient(ctx context.Context, originURL *url.URL) (Client, error) {
 	// Closes the driver, will occur when the context finishes.
 	go func() {
 		<-ctx.Done()
-		driver.Close()
+		_ = driver.Close()
 	}()
 
 	return &SQLClient{driver.DriverName(), driver}, nil
@@ -260,7 +260,7 @@ func sqlRows(rows *sql.Rows) ([]map[string]interface{}, error) {
 		for i := range columns {
 			pointers[i] = &values[i]
 		}
-		rows.Scan(pointers...)
+		_ = rows.Scan(pointers...)
 
 		// Convert a row, an array of values, into an object where
 		// each key is the name of its respective column.

dev.Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.15.3 as builder
+FROM golang:1.15.7 as builder
 ENV GO111MODULE=on \
     CGO_ENABLED=0
 WORKDIR /go/src/github.com/cloudflare/cloudflared/

edgediscovery/allregions/discovery.go

@@ -7,8 +7,8 @@ import (
 	"net"
 	"time"
 
-	"github.com/cloudflare/cloudflared/logger"
 	"github.com/pkg/errors"
+	"github.com/rs/zerolog"
 )
 
 const (
@@ -21,12 +21,7 @@ const (
 	// discover HA origintunneld servers (GitHub issue #75).
 	dotServerName = "cloudflare-dns.com"
 	dotServerAddr = "1.1.1.1:853"
-	dotTimeout    = time.Duration(15 * time.Second)
-
-	// SRV record resolution TTL
-	resolveEdgeAddrTTL = 1 * time.Hour
-
-	subsystemEdgeAddrResolver = "edgeAddrResolver"
+	dotTimeout    = 15 * time.Second
 )
 
 // Redeclare network functions so they can be overridden in tests.
@@ -58,15 +53,15 @@ var friendlyDNSErrorLines = []string{
 }
 
 // EdgeDiscovery implements HA service discovery lookup.
-func edgeDiscovery(logger logger.Service) ([][]*net.TCPAddr, error) {
+func edgeDiscovery(log *zerolog.Logger) ([][]*net.TCPAddr, error) {
 	_, addrs, err := netLookupSRV(srvService, srvProto, srvName)
 	if err != nil {
 		_, fallbackAddrs, fallbackErr := fallbackLookupSRV(srvService, srvProto, srvName)
 		if fallbackErr != nil || len(fallbackAddrs) == 0 {
 			// use the original DNS error `err` in messages, not `fallbackErr`
-			logger.Errorf("Error looking up Cloudflare edge IPs: the DNS query failed: %s", err)
+			log.Err(err).Msg("Error looking up Cloudflare edge IPs: the DNS query failed")
 			for _, s := range friendlyDNSErrorLines {
-				logger.Error(s)
+				log.Error().Msg(s)
 			}
 			return nil, errors.Wrapf(err, "Could not lookup srv records on _%v._%v.%v", srvService, srvProto, srvName)
 		}
@@ -121,15 +116,15 @@ func resolveSRVToTCP(srv *net.SRV) ([]*net.TCPAddr, error) {
 }
 
 // ResolveAddrs resolves TCP address given a list of addresses. Address can be a hostname, however, it will return at most one
-// of the hostname's IP addresses
-func ResolveAddrs(addrs []string) ([]*net.TCPAddr, error) {
-	var tcpAddrs []*net.TCPAddr
+// of the hostname's IP addresses.
+func ResolveAddrs(addrs []string, log *zerolog.Logger) (resolved []*net.TCPAddr) {
 	for _, addr := range addrs {
 		tcpAddr, err := net.ResolveTCPAddr("tcp", addr)
 		if err != nil {
-			return nil, err
+			log.Err(err).Msgf("Failed to resolve %s", addr)
+		} else {
+			resolved = append(resolved, tcpAddr)
 		}
-		tcpAddrs = append(tcpAddrs, tcpAddr)
 	}
-	return tcpAddrs, nil
+	return
 }

edgediscovery/allregions/discovery_test.go

@@ -3,7 +3,7 @@ package allregions
 import (
 	"testing"
 
-	"github.com/cloudflare/cloudflared/logger"
+	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -19,8 +19,8 @@ func TestEdgeDiscovery(t *testing.T) {
 		}
 	}
 
-	l := logger.NewOutputWriter(logger.NewMockWriteManager())
-	addrLists, err := edgeDiscovery(l)
+	l := zerolog.Nop()
+	addrLists, err := edgeDiscovery(&l)
 	assert.NoError(t, err)
 	actualAddrSet := map[string]bool{}
 	for _, addrs := range addrLists {

edgediscovery/allregions/region.go

@@ -57,7 +57,6 @@ func (r Region) GetUnusedIP(excluding *net.TCPAddr) *net.TCPAddr {
 // Use the address, assigning it to a proxy connection.
 func (r Region) Use(addr *net.TCPAddr, connID int) {
 	if addr == nil {
-		//logrus.Errorf("Attempted to use nil address for connection %d", connID)
 		return
 	}
 	r.connFor[addr] = InUse(connID)

edgediscovery/allregions/regions.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"net"
 
-	"github.com/cloudflare/cloudflared/logger"
+	"github.com/rs/zerolog"
 )
 
 // Regions stores Cloudflare edge network IPs, partitioned into two regions.
@@ -19,8 +19,8 @@ type Regions struct {
 // ------------------------------------
 
 // ResolveEdge resolves the Cloudflare edge, returning all regions discovered.
-func ResolveEdge(logger logger.Service) (*Regions, error) {
-	addrLists, err := edgeDiscovery(logger)
+func ResolveEdge(log *zerolog.Logger) (*Regions, error) {
+	addrLists, err := edgeDiscovery(log)
 	if err != nil {
 		return nil, err
 	}
@@ -35,12 +35,12 @@ func ResolveEdge(logger logger.Service) (*Regions, error) {
 
 // StaticEdge creates a list of edge addresses from the list of hostnames.
 // Mainly used for testing connectivity.
-func StaticEdge(hostnames []string) (*Regions, error) {
-	addrs, err := ResolveAddrs(hostnames)
-	if err != nil {
-		return nil, err
+func StaticEdge(hostnames []string, log *zerolog.Logger) (*Regions, error) {
+	resolved := ResolveAddrs(hostnames, log)
+	if len(resolved) == 0 {
+		return nil, fmt.Errorf("failed to resolve any edge address")
 	}
-	return NewNoResolve(addrs), nil
+	return NewNoResolve(resolved), nil
 }
 
 // NewNoResolve doesn't resolve the edge. Instead it just uses the given addresses.

edgediscovery/dial.go

@@ -1,4 +1,4 @@
-package connection
+package edgediscovery
 
 import (
 	"context"
@@ -9,7 +9,7 @@ import (
 	"github.com/pkg/errors"
 )
 
-// DialEdge makes a TLS connection to a Cloudflare edge node
+// DialEdgeWithH2Mux makes a TLS connection to a Cloudflare edge node
 func DialEdge(
 	ctx context.Context,
 	timeout time.Duration,
@@ -25,6 +25,7 @@ func DialEdge(
 	if err != nil {
 		return nil, newDialError(err, "DialContext error")
 	}
+
 	tlsEdgeConn := tls.Client(edgeConn, tlsConfig)
 	tlsEdgeConn.SetDeadline(time.Now().Add(timeout))
 

edgediscovery/edgediscovery.go

@@ -6,20 +6,21 @@ import (
 	"sync"
 
 	"github.com/cloudflare/cloudflared/edgediscovery/allregions"
-	"github.com/cloudflare/cloudflared/logger"
+	"github.com/rs/zerolog"
 )
 
 const (
-	subsystem = "edgediscovery"
+	LogFieldAddress   = "address"
+	LogFieldConnIndex = "connIndex"
 )
 
-var errNoAddressesLeft = fmt.Errorf("There are no free edge addresses left")
+var errNoAddressesLeft = fmt.Errorf("there are no free edge addresses left")
 
 // Edge finds addresses on the Cloudflare edge and hands them out to connections.
 type Edge struct {
 	regions *allregions.Regions
 	sync.Mutex
-	logger logger.Service
+	log *zerolog.Logger
 }
 
 // ------------------------------------
@@ -28,34 +29,34 @@ type Edge struct {
 
 // ResolveEdge runs the initial discovery of the Cloudflare edge, finding Addrs that can be allocated
 // to connections.
-func ResolveEdge(l logger.Service) (*Edge, error) {
-	regions, err := allregions.ResolveEdge(l)
+func ResolveEdge(log *zerolog.Logger) (*Edge, error) {
+	regions, err := allregions.ResolveEdge(log)
 	if err != nil {
 		return new(Edge), err
 	}
 	return &Edge{
-		logger:  l,
+		log:     log,
 		regions: regions,
 	}, nil
 }
 
 // StaticEdge creates a list of edge addresses from the list of hostnames. Mainly used for testing connectivity.
-func StaticEdge(l logger.Service, hostnames []string) (*Edge, error) {
-	regions, err := allregions.StaticEdge(hostnames)
+func StaticEdge(log *zerolog.Logger, hostnames []string) (*Edge, error) {
+	regions, err := allregions.StaticEdge(hostnames, log)
 	if err != nil {
 		return new(Edge), err
 	}
 	return &Edge{
-		logger:  l,
+		log:     log,
 		regions: regions,
 	}, nil
 }
 
 // MockEdge creates a Cloudflare Edge from arbitrary TCP addresses. Used for testing.
-func MockEdge(l logger.Service, addrs []*net.TCPAddr) *Edge {
+func MockEdge(log *zerolog.Logger, addrs []*net.TCPAddr) *Edge {
 	regions := allregions.NewNoResolve(addrs)
 	return &Edge{
-		logger:  l,
+		log:     log,
 		regions: regions,
 	}
 }
@@ -76,42 +77,46 @@ func (ed *Edge) GetAddrForRPC() (*net.TCPAddr, error) {
 }
 
 // GetAddr gives this proxy connection an edge Addr. Prefer Addrs this connection has already used.
-func (ed *Edge) GetAddr(connID int) (*net.TCPAddr, error) {
+func (ed *Edge) GetAddr(connIndex int) (*net.TCPAddr, error) {
+	log := ed.log.With().Int(LogFieldConnIndex, connIndex).Logger()
+
 	ed.Lock()
 	defer ed.Unlock()
 
 	// If this connection has already used an edge addr, return it.
-	if addr := ed.regions.AddrUsedBy(connID); addr != nil {
-		ed.logger.Debugf("edgediscovery - GetAddr: Returning same address back to proxy connection: connID: %d", connID)
+	if addr := ed.regions.AddrUsedBy(connIndex); addr != nil {
+		log.Debug().Msg("edgediscovery - GetAddr: Returning same address back to proxy connection")
 		return addr, nil
 	}
 
 	// Otherwise, give it an unused one
-	addr := ed.regions.GetUnusedAddr(nil, connID)
+	addr := ed.regions.GetUnusedAddr(nil, connIndex)
 	if addr == nil {
-		ed.logger.Debugf("edgediscovery - GetAddr: No addresses left to give proxy connection: connID: %d", connID)
+		log.Debug().Msg("edgediscovery - GetAddr: No addresses left to give proxy connection")
 		return nil, errNoAddressesLeft
 	}
-	ed.logger.Debugf("edgediscovery - GetAddr: Giving connection its new address %s: connID: %d", addr, connID)
+	log.Debug().Str(LogFieldAddress, addr.String()).Msg("edgediscovery - GetAddr: Giving connection its new address")
 	return addr, nil
 }
 
 // GetDifferentAddr gives back the proxy connection's edge Addr and uses a new one.
-func (ed *Edge) GetDifferentAddr(connID int) (*net.TCPAddr, error) {
+func (ed *Edge) GetDifferentAddr(connIndex int) (*net.TCPAddr, error) {
+	log := ed.log.With().Int(LogFieldConnIndex, connIndex).Logger()
+
 	ed.Lock()
 	defer ed.Unlock()
 
-	oldAddr := ed.regions.AddrUsedBy(connID)
+	oldAddr := ed.regions.AddrUsedBy(connIndex)
 	if oldAddr != nil {
 		ed.regions.GiveBack(oldAddr)
 	}
-	addr := ed.regions.GetUnusedAddr(oldAddr, connID)
+	addr := ed.regions.GetUnusedAddr(oldAddr, connIndex)
 	if addr == nil {
-		ed.logger.Debugf("edgediscovery - GetDifferentAddr: No addresses left to give proxy connection: connID: %d", connID)
+		log.Debug().Msg("edgediscovery - GetDifferentAddr: No addresses left to give proxy connection")
 		// note: if oldAddr were not nil, it will become available on the next iteration
 		return nil, errNoAddressesLeft
 	}
-	ed.logger.Debugf("edgediscovery - GetDifferentAddr: Giving connection its new address %s: connID: %d", addr, connID)
+	log.Debug().Str(LogFieldAddress, addr.String()).Msg("edgediscovery - GetDifferentAddr: Giving connection its new address")
 	return addr, nil
 }
 
@@ -127,6 +132,6 @@ func (ed *Edge) AvailableAddrs() int {
 func (ed *Edge) GiveBack(addr *net.TCPAddr) bool {
 	ed.Lock()
 	defer ed.Unlock()
-	ed.logger.Debug("edgediscovery - GiveBack: Address now unused")
+	ed.log.Debug().Msg("edgediscovery - GiveBack: Address now unused")
 	return ed.regions.GiveBack(addr)
 }

edgediscovery/edgediscovery_test.go

@@ -4,7 +4,7 @@ import (
 	"net"
 	"testing"
 
-	"github.com/cloudflare/cloudflared/logger"
+	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -29,11 +29,12 @@ var (
 		Port: 8000,
 		Zone: "",
 	}
+
+	log = zerolog.Nop()
 )
 
 func TestGiveBack(t *testing.T) {
-	l := logger.NewOutputWriter(logger.NewMockWriteManager())
-	edge := MockEdge(l, []*net.TCPAddr{&addr0, &addr1, &addr2, &addr3})
+	edge := MockEdge(&log, []*net.TCPAddr{&addr0, &addr1, &addr2, &addr3})
 
 	// Give this connection an address
 	assert.Equal(t, 4, edge.AvailableAddrs())
@@ -49,10 +50,8 @@ func TestGiveBack(t *testing.T) {
 }
 
 func TestRPCAndProxyShareSingleEdgeIP(t *testing.T) {
-	l := logger.NewOutputWriter(logger.NewMockWriteManager())
-
 	// Make an edge with a single IP
-	edge := MockEdge(l, []*net.TCPAddr{&addr0})
+	edge := MockEdge(&log, []*net.TCPAddr{&addr0})
 	tunnelConnID := 0
 
 	// Use the IP for a tunnel
@@ -66,8 +65,7 @@ func TestRPCAndProxyShareSingleEdgeIP(t *testing.T) {
 }
 
 func TestGetAddrForRPC(t *testing.T) {
-	l := logger.NewOutputWriter(logger.NewMockWriteManager())
-	edge := MockEdge(l, []*net.TCPAddr{&addr0, &addr1, &addr2, &addr3})
+	edge := MockEdge(&log, []*net.TCPAddr{&addr0, &addr1, &addr2, &addr3})
 
 	// Get a connection
 	assert.Equal(t, 4, edge.AvailableAddrs())
@@ -84,10 +82,8 @@ func TestGetAddrForRPC(t *testing.T) {
 }
 
 func TestOnePerRegion(t *testing.T) {
-	l := logger.NewOutputWriter(logger.NewMockWriteManager())
-
 	// Make an edge with only one address
-	edge := MockEdge(l, []*net.TCPAddr{&addr0, &addr1})
+	edge := MockEdge(&log, []*net.TCPAddr{&addr0, &addr1})
 
 	// Use the only address
 	const connID = 0
@@ -108,10 +104,8 @@ func TestOnePerRegion(t *testing.T) {
 }
 
 func TestOnlyOneAddrLeft(t *testing.T) {
-	l := logger.NewOutputWriter(logger.NewMockWriteManager())
-
 	// Make an edge with only one address
-	edge := MockEdge(l, []*net.TCPAddr{&addr0})
+	edge := MockEdge(&log, []*net.TCPAddr{&addr0})
 
 	// Use the only address
 	const connID = 0
@@ -130,10 +124,8 @@ func TestOnlyOneAddrLeft(t *testing.T) {
 }
 
 func TestNoAddrsLeft(t *testing.T) {
-	l := logger.NewOutputWriter(logger.NewMockWriteManager())
-
 	// Make an edge with no addresses
-	edge := MockEdge(l, []*net.TCPAddr{})
+	edge := MockEdge(&log, []*net.TCPAddr{})
 
 	_, err := edge.GetAddr(2)
 	assert.Error(t, err)
@@ -142,8 +134,7 @@ func TestNoAddrsLeft(t *testing.T) {
 }
 
 func TestGetAddr(t *testing.T) {
-	l := logger.NewOutputWriter(logger.NewMockWriteManager())
-	edge := MockEdge(l, []*net.TCPAddr{&addr0, &addr1, &addr2, &addr3})
+	edge := MockEdge(&log, []*net.TCPAddr{&addr0, &addr1, &addr2, &addr3})
 
 	// Give this connection an address
 	const connID = 0
@@ -158,8 +149,7 @@ func TestGetAddr(t *testing.T) {
 }
 
 func TestGetDifferentAddr(t *testing.T) {
-	l := logger.NewOutputWriter(logger.NewMockWriteManager())
-	edge := MockEdge(l, []*net.TCPAddr{&addr0, &addr1, &addr2, &addr3})
+	edge := MockEdge(&log, []*net.TCPAddr{&addr0, &addr1, &addr2, &addr3})
 
 	// Give this connection an address
 	assert.Equal(t, 4, edge.AvailableAddrs())

edgediscovery/mocks_for_test.go

@@ -1,4 +1,4 @@
-package connection
+package edgediscovery
 
 import (
 	"fmt"

edgediscovery/protocol.go

@@ -0,0 +1,45 @@
+package edgediscovery
+
+import (
+	"fmt"
+	"net"
+	"strconv"
+	"strings"
+)
+
+const (
+	protocolRecord = "protocol.argotunnel.com"
+)
+
+var (
+	errNoProtocolRecord = fmt.Errorf("No TXT record found for %s to determine connection protocol", protocolRecord)
+)
+
+func HTTP2Percentage() (int32, error) {
+	records, err := net.LookupTXT(protocolRecord)
+	if err != nil {
+		return 0, err
+	}
+	if len(records) == 0 {
+		return 0, errNoProtocolRecord
+	}
+	return parseHTTP2Precentage(records[0])
+}
+
+// The record looks like http2=percentage
+func parseHTTP2Precentage(record string) (int32, error) {
+	const key = "http2"
+	slices := strings.Split(record, "=")
+	if len(slices) != 2 {
+		return 0, fmt.Errorf("Malformed TXT record %s, expect http2=percentage", record)
+	}
+	if slices[0] != key {
+		return 0, fmt.Errorf("Incorrect key %s, expect %s", slices[0], key)
+	}
+	percentage, err := strconv.ParseInt(slices[1], 10, 32)
+	if err != nil {
+		return 0, err
+	}
+	return int32(percentage), nil
+
+}

edgediscovery/protocol_test.go

@@ -0,0 +1,80 @@
+package edgediscovery
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestHTTP2Percentage(t *testing.T) {
+	_, err := HTTP2Percentage()
+	assert.NoError(t, err)
+}
+
+func TestParseHTTP2Precentage(t *testing.T) {
+	tests := []struct {
+		record     string
+		percentage int32
+		wantErr    bool
+	}{
+		{
+			record:     "http2=-1",
+			percentage: -1,
+			wantErr:    false,
+		},
+		{
+			record:     "http2=0",
+			percentage: 0,
+			wantErr:    false,
+		},
+		{
+			record:     "http2=50",
+			percentage: 50,
+			wantErr:    false,
+		},
+		{
+			record:     "http2=100",
+			percentage: 100,
+			wantErr:    false,
+		},
+		{
+			record:     "http2=1000",
+			percentage: 1000,
+			wantErr:    false,
+		},
+		{
+			record:  "http2=10.5",
+			wantErr: true,
+		},
+		{
+			record:  "http2=10 h2mux=90",
+			wantErr: true,
+		},
+		{
+			record:  "http2=ten",
+			wantErr: true,
+		},
+
+		{
+			record:  "h2mux=100",
+			wantErr: true,
+		},
+		{
+			record:  "http2",
+			wantErr: true,
+		},
+		{
+			record:  "http2=",
+			wantErr: true,
+		},
+	}
+
+	for _, test := range tests {
+		p, err := parseHTTP2Precentage(test.record)
+		if test.wantErr {
+			assert.Error(t, err)
+		} else {
+			assert.Equal(t, test.percentage, p)
+		}
+	}
+}

go.mod

@@ -4,10 +4,10 @@ go 1.15
 
 require (
 	github.com/DATA-DOG/go-sqlmock v1.3.3
-	github.com/acmacalister/skittles v0.0.0-20160609003031-7423546701e1
-	github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d
+	github.com/acmacalister/skittles v0.0.0-20160609003031-7423546701e1 // indirect
+	github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d // indirect
 	github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 // indirect
-	github.com/aws/aws-sdk-go v1.34.19
+	github.com/aws/aws-sdk-go v1.34.19 // indirect
 	github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894 // indirect
 	github.com/cloudflare/brotli-go v0.0.0-20191101163834-d34379f7ff93
 	github.com/cloudflare/golibs v0.0.0-20170913112048-333127dbecfc
@@ -15,7 +15,6 @@ require (
 	github.com/coreos/go-oidc v0.0.0-20171002155002-a93f71fdfe73
 	github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf
 	github.com/denisenkom/go-mssqldb v0.0.0-20191001013358-cfbb681360f0
-	github.com/equinox-io/equinox v1.2.0
 	github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 // indirect
 	github.com/facebookgo/freeport v0.0.0-20150612182905-d4adf43b75b9 // indirect
 	github.com/facebookgo/grace v0.0.0-20180706040059-75cf19382434
@@ -25,18 +24,23 @@ require (
 	github.com/fsnotify/fsnotify v1.4.9
 	github.com/gdamore/tcell v1.3.0
 	github.com/getsentry/raven-go v0.0.0-20180517221441-ed7bcb39ff10
-	github.com/gliderlabs/ssh v0.0.0-20191009160644-63518b5243e0
+	github.com/gliderlabs/ssh v0.0.0-20191009160644-63518b5243e0 // indirect
 	github.com/go-sql-driver/mysql v1.5.0
+	github.com/gobwas/httphead v0.0.0-20200921212729-da3d93bc3c58 // indirect
+	github.com/gobwas/pool v0.2.1 // indirect
+	github.com/gobwas/ws v1.0.4
 	github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3
 	github.com/google/go-cmp v0.5.2 // indirect
 	github.com/google/uuid v1.1.2
 	github.com/gorilla/mux v1.7.3
 	github.com/gorilla/websocket v1.4.2
 	github.com/jmoiron/sqlx v1.2.0
+	github.com/json-iterator/go v1.1.10
 	github.com/kr/text v0.2.0 // indirect
 	github.com/kshvakov/clickhouse v1.3.11
 	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/lib/pq v1.2.0
+	github.com/mattn/go-colorable v0.1.8
 	github.com/mattn/go-sqlite3 v1.11.0
 	github.com/miekg/dns v1.1.31
 	github.com/mitchellh/go-homedir v1.1.0
@@ -47,8 +51,8 @@ require (
 	github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 // indirect
 	github.com/prometheus/client_golang v1.7.1
 	github.com/prometheus/common v0.13.0 // indirect
-	github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5 // indirect
 	github.com/rivo/tview v0.0.0-20200712113419-c65badfc3d92
+	github.com/rs/zerolog v1.20.0
 	github.com/stretchr/testify v1.6.0
 	github.com/urfave/cli/v2 v2.2.0
 	github.com/xo/dburl v0.0.0-20191005012637-293c3298d6c0
@@ -56,11 +60,13 @@ require (
 	golang.org/x/net v0.0.0-20200904194848-62affa334b73
 	golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43 // indirect
 	golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208
-	golang.org/x/sys v0.0.0-20200909081042-eff7692f9009
+	golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4
+	golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf
 	google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d // indirect
 	google.golang.org/grpc v1.32.0 // indirect
 	gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
 	gopkg.in/coreos/go-oidc.v2 v2.1.0
+	gopkg.in/natefinch/lumberjack.v2 v2.0.0
 	gopkg.in/square/go-jose.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v2 v2.3.0
 	gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 // indirect

go.sum

@@ -100,7 +100,6 @@ github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQ
 github.com/aws/aws-sdk-go v1.23.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/aws/aws-sdk-go v1.32.1/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
-github.com/aws/aws-sdk-go v1.34.19 h1:x3MMvAJ1nfWviixEduchBSs65DgY5Y2pA2/NAcxVGOo=
 github.com/aws/aws-sdk-go v1.34.19/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
 github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
 github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f/go.mod h1:AuiFmCCPBSrqvVMvuqFuk0qogytodnVFVSN5CeJB8Gc=
@@ -109,6 +108,7 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
+github.com/bkaradzic/go-lz4 v1.0.0 h1:RXc4wYsyz985CkXXeX04y4VnZFGG8Rd43pRaHsOXAKk=
 github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4=
 github.com/caddyserver/caddy v1.0.5 h1:5B1Hs0UF2x2tggr2X9jL2qOZtDXbIWQb9YLbmlxHSuM=
 github.com/caddyserver/caddy v1.0.5/go.mod h1:AnFHB+/MrgRC+mJAvuAgQ38ePzw+wKeW0wzENpdQQKY=
@@ -144,6 +144,7 @@ github.com/coreos/go-oidc v0.0.0-20171002155002-a93f71fdfe73 h1:7CNPV0LWRCa1FNmq
 github.com/coreos/go-oidc v0.0.0-20171002155002-a93f71fdfe73/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU=
 github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk=
@@ -179,15 +180,17 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
-github.com/equinox-io/equinox v1.2.0 h1:bBS7Ou+Y7Jwgmy8TWSYxEh85WctuFn7FPlgbUzX4DBA=
-github.com/equinox-io/equinox v1.2.0/go.mod h1:6s3HJB0PYUNgs0mxmI8fHdfVl3TQ25ieA/PVfr+eyVo=
 github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/exoscale/egoscale v0.18.1/go.mod h1:Z7OOdzzTOz1Q1PjQXumlz9Wn/CddH0zSYdCF3rnBKXE=
+github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 h1:0JZ+dUmQeA8IIVUMzysrX4/AKuQwWhV2dYQuPZdvdSQ=
 github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64=
+github.com/facebookgo/freeport v0.0.0-20150612182905-d4adf43b75b9 h1:wWke/RUCl7VRjQhwPlR/v0glZXNYzBHdNUzf/Am2Nmg=
 github.com/facebookgo/freeport v0.0.0-20150612182905-d4adf43b75b9/go.mod h1:uPmAp6Sws4L7+Q/OokbWDAK1ibXYhB3PXFP1kol5hPg=
 github.com/facebookgo/grace v0.0.0-20180706040059-75cf19382434 h1:mOp33BLbcbJ8fvTAmZacbBiOASfxN+MLcLxymZCIrGE=
 github.com/facebookgo/grace v0.0.0-20180706040059-75cf19382434/go.mod h1:KigFdumBXUPSwzLDbeuzyt0elrL7+CP7TKuhrhT4bcU=
+github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojtoVVWjGfOF9635RETekkoH6Cc9SX0A=
 github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg=
+github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870 h1:E2s37DuLxFhQDg5gKsWoLBOB0n+ZW8s599zru8FJ2/Y=
 github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0=
 github.com/farsightsec/golang-framestream v0.0.0-20190425193708-fa4b164d59b8/go.mod h1:eNde4IQyEiA5br02AouhEHCu3p3UzrCdFR4LuQHklMI=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
@@ -196,6 +199,7 @@ github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjr
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
 github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4=
 github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20=
+github.com/frankban/quicktest v1.10.0 h1:Gfh+GAJZOAoKZsIZeZbdn2JF10kN1XHNvjsvQK8gVkE=
 github.com/frankban/quicktest v1.10.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
@@ -233,6 +237,12 @@ github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG
 github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/gobwas/httphead v0.0.0-20200921212729-da3d93bc3c58 h1:YyrUZvJaU8Q0QsoVo+xLFBgWDTam29PKea6GYmwvSiQ=
+github.com/gobwas/httphead v0.0.0-20200921212729-da3d93bc3c58/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo=
+github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og=
+github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
+github.com/gobwas/ws v1.0.4 h1:5eXU1CZhpQdq5kXbKb+sECH5Ia5KiO6CYzIzdlVx6Bs=
+github.com/gobwas/ws v1.0.4/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=
 github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
 github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
@@ -283,6 +293,7 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -369,6 +380,7 @@ github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV
 github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
@@ -385,13 +397,16 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxv
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kshvakov/clickhouse v1.3.11 h1:dtzTJY0fCA+MWkLyuKZaNPkmSwdX4gh8+Klic9NB1Lw=
 github.com/kshvakov/clickhouse v1.3.11/go.mod h1:/SVBAcqF3u7rxQ9sTWCZwf8jzzvxiZGeQvtmSF2BBEc=
 github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/labbsr0x/bindman-dns-webhook v1.0.2/go.mod h1:p6b+VCXIR8NYKpDr8/dg1HKfQoRHCdcsROXKvmoehKA=
 github.com/labbsr0x/goh v1.0.1/go.mod h1:8K2UhVoaWXcCU7Lxoa2omWnC8gyW8px7/lmO61c027w=
@@ -412,8 +427,12 @@ github.com/marten-seemann/chacha20 v0.2.0/go.mod h1:HSdjFau7GzYRj+ahFNwsO3ouVJr1
 github.com/marten-seemann/qpack v0.1.0/go.mod h1:LFt1NU/Ptjip0C2CPkhimBz5CGE3WGDAUWqna+CNTrI=
 github.com/marten-seemann/qtls v0.4.1/go.mod h1:pxVXcHHw1pNIt8Qo0pwSYQEoZ8yYOOPXTCZLQQunvRc=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
+github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8=
+github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
+github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/mattn/go-runewidth v0.0.8 h1:3tS41NlGYSmhhe/8fhGRzc+z3AYCw1Fe1WAyLuujKs0=
@@ -441,8 +460,10 @@ github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0Qu
 github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
@@ -459,6 +480,7 @@ github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxzi
 github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w=
 github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
 github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
 github.com/nrdcg/auroradns v1.0.0/go.mod h1:6JPXKzIRzZzMqtTDgueIhTi6rFf1QvYE/HzqidhOhjw=
 github.com/nrdcg/goinwx v0.6.1/go.mod h1:XPiut7enlbEdntAqalBIqcYcTEVhpv/dKWgDCX2SwKQ=
@@ -493,9 +515,11 @@ github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FI
 github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
 github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
+github.com/philhofer/fwd v1.0.0 h1:UbZqGr5Y38ApvM/V/jEljVxwocdweyH+vmYvRPBnbqQ=
 github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
 github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc=
 github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
+github.com/pierrec/lz4 v2.5.2+incompatible h1:WCjObylUIOlKy/+7Abdn34TLIkXiA4UWUMhxq9m9ZXI=
 github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -534,7 +558,6 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R
 github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
-github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
 github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
 github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
 github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8=
@@ -542,13 +565,15 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O
 github.com/prometheus/prometheus v2.5.0+incompatible/go.mod h1:oAIUtOny2rjMX0OWN5vPR5/q/twIROJvdqnQKDdil/s=
 github.com/rainycape/memcache v0.0.0-20150622160815-1031fa0ce2f2/go.mod h1:7tZKcyumwBO6qip7RNQ5r77yrssm9bfCowcLEBcU5IA=
 github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
-github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5/go.mod h1:GEXHk5HgEKCvEIIrSpFI3ozzG5xOKA2DVlEX/gGnewM=
 github.com/rivo/tview v0.0.0-20200712113419-c65badfc3d92 h1:rqaqSUdaW+OBbjnsrOoiaJv43mSRARuvsAuirmdxu7E=
 github.com/rivo/tview v0.0.0-20200712113419-c65badfc3d92/go.mod h1:6lkG1x+13OShEf0EaOCaTQYyB7d5nSbb181KtjlS+84=
 github.com/rivo/uniseg v0.1.0 h1:+2KBaVoUmb9XzDsrx/Ct0W/EYOSFf/nWTauy++DprtY=
 github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
+github.com/rs/zerolog v1.20.0 h1:38k9hgtUBdxFwE34yS8rTHmHBa4eN16E4DJlv177LNs=
+github.com/rs/zerolog v1.20.0/go.mod h1:IzD0RJ65iWH0w97OQQebJEvTZYvsCUm9WVLWBQrJRjo=
 github.com/russross/blackfriday v0.0.0-20170610170232-067529f716f4 h1:S9YlS71UNJIyS61OqGAmLXv3w5zclSidN+qwr80XxKs=
 github.com/russross/blackfriday v0.0.0-20170610170232-067529f716f4/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
@@ -586,7 +611,7 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5
 github.com/stretchr/testify v1.6.0 h1:jlIyCplCJFULU/01vCkhKuTyc3OorI3bJFuw6obfgho=
 github.com/stretchr/testify v1.6.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/timewasted/linode v0.0.0-20160829202747-37e84520dcf7/go.mod h1:imsgLplxEC/etjIhdr3dNzV3JeT27LbVu5pYWm0JCBY=
-github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
+github.com/tinylib/msgp v1.1.2 h1:gWmO7n0Ys2RBEb7GPYB9Ujq8Mk5p2U08lRnmMcGy6BQ=
 github.com/tinylib/msgp v1.1.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/transip/gotransip v0.0.0-20190812104329-6d8d9179b66f/go.mod h1:i0f4R4o2HM0m3DZYQWsj6/MEowD57VzoH0v3d7igeFY=
@@ -774,6 +799,7 @@ golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -791,6 +817,12 @@ golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200909081042-eff7692f9009 h1:W0lCpv29Hv0UaM1LXb9QlBHLNP8UFfcKjblhVCWftOM=
 golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 h1:myAQVi0cGEoqQVR5POX+8RR2mrocKqNN1hmeMqhX27k=
+golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf h1:MZ2shdL+ZM/XzY3ZGOnh4Nlpnxz5GSOhOmtHo3iPU6M=
+golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -821,6 +853,7 @@ golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgw
 golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
@@ -856,6 +889,7 @@ golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
@@ -952,6 +986,7 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
 gopkg.in/coreos/go-oidc.v2 v2.1.0 h1:E8PjVFdj/SLDKB0hvb70KTbMbYVHjqztiQdSkIg8E+I=
@@ -964,6 +999,7 @@ gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/ini.v1 v1.44.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/mcuadros/go-syslog.v2 v2.2.1/go.mod h1:l5LPIyOOyIdQquNg+oU6Z3524YwrcqEm0aKH+5zpt2U=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
 gopkg.in/ns1/ns1-go.v2 v2.0.0-20190730140822-b51389932cbc/go.mod h1:VV+3haRsgDiVLxyifmMBrBIuCWFBPYKbRssXB9z67Hw=
 gopkg.in/resty.v1 v1.9.1/go.mod h1:vo52Hzryw9PnPHcJfPsBiFW62XhNx5OczbV9y+IMpgc=

h2mux/activestreammap.go

@@ -7,6 +7,19 @@ import (
 	"golang.org/x/net/http2"
 )
 
+var (
+	ActiveStreams = prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace: "cloudflared",
+		Subsystem: "tunnel",
+		Name:      "active_streams",
+		Help:      "Number of active streams created by all muxers.",
+	})
+)
+
+func init() {
+	prometheus.MustRegister(ActiveStreams)
+}
+
 // activeStreamMap is used to moderate access to active streams between the read and write
 // threads, and deny access to new peer streams while shutting down.
 type activeStreamMap struct {

h2mux/activestreammap_test.go

@@ -9,7 +9,7 @@ import (
 
 func TestShutdown(t *testing.T) {
 	const numStreams = 1000
-	m := newActiveStreamMap(true, NewActiveStreamsMetrics("test", t.Name()))
+	m := newActiveStreamMap(true, ActiveStreams)
 
 	// Add all the streams
 	{
@@ -62,7 +62,7 @@ func TestShutdown(t *testing.T) {
 
 func TestEmptyBeforeShutdown(t *testing.T) {
 	const numStreams = 1000
-	m := newActiveStreamMap(true, NewActiveStreamsMetrics("test", t.Name()))
+	m := newActiveStreamMap(true, ActiveStreams)
 
 	// Add all the streams
 	{
@@ -138,7 +138,7 @@ func (_ *noopReadyList) Signal(streamID uint32) {}
 
 func TestAbort(t *testing.T) {
 	const numStreams = 1000
-	m := newActiveStreamMap(true, NewActiveStreamsMetrics("test", t.Name()))
+	m := newActiveStreamMap(true, ActiveStreams)
 
 	var openedStreams sync.Map
 

h2mux/h2mux.go

@@ -8,11 +8,10 @@ import (
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/rs/zerolog"
 	"golang.org/x/net/http2"
 	"golang.org/x/net/http2/hpack"
 	"golang.org/x/sync/errgroup"
-
-	"github.com/cloudflare/cloudflared/logger"
 )
 
 const (
@@ -50,7 +49,7 @@ type MuxerConfig struct {
 	// The minimum number of heartbeats to send before terminating the connection.
 	MaxHeartbeats uint64
 	// Logger to use
-	Logger             logger.Service
+	Log                *zerolog.Logger
 	CompressionQuality CompressionSetting
 	// Initial size for HTTP2 flow control windows
 	DefaultWindowSize uint32
@@ -138,10 +137,10 @@ func Handshake(
 	handshakeSetting := http2.Setting{ID: SettingMuxerMagic, Val: MuxerMagicEdge}
 	compressionSetting := http2.Setting{ID: SettingCompression, Val: config.CompressionQuality.toH2Setting()}
 	if CompressionIsSupported() {
-		config.Logger.Debug("muxer: Compression is supported")
+		config.Log.Debug().Msg("muxer: Compression is supported")
 		m.compressionQuality = config.CompressionQuality.getPreset()
 	} else {
-		config.Logger.Debug("muxer: Compression is not supported")
+		config.Log.Debug().Msg("muxer: Compression is not supported")
 		compressionSetting = http2.Setting{ID: SettingCompression, Val: 0}
 	}
 
@@ -178,12 +177,12 @@ func Handshake(
 	// Sanity check to enusre idelDuration is sane
 	if idleDuration == 0 || idleDuration < defaultTimeout {
 		idleDuration = defaultTimeout
-		config.Logger.Infof("muxer: Minimum idle time has been adjusted to %d", defaultTimeout)
+		config.Log.Info().Msgf("muxer: Minimum idle time has been adjusted to %d", defaultTimeout)
 	}
 	maxRetries := config.MaxHeartbeats
 	if maxRetries == 0 {
 		maxRetries = defaultRetries
-		config.Logger.Infof("muxer: Minimum number of unacked heartbeats to send before closing the connection has been adjusted to %d", maxRetries)
+		config.Log.Info().Msgf("muxer: Minimum number of unacked heartbeats to send before closing the connection has been adjusted to %d", maxRetries)
 	}
 
 	compBytesBefore, compBytesAfter := NewAtomicCounter(0), NewAtomicCounter(0)
@@ -325,7 +324,7 @@ func (m *Muxer) Serve(ctx context.Context) error {
 	errGroup.Go(func() error {
 		ch := make(chan error)
 		go func() {
-			err := m.muxReader.run(m.config.Logger)
+			err := m.muxReader.run(m.config.Log)
 			m.explicitShutdown.Fuse(false)
 			m.r.Close()
 			m.abort()
@@ -346,7 +345,7 @@ func (m *Muxer) Serve(ctx context.Context) error {
 	errGroup.Go(func() error {
 		ch := make(chan error)
 		go func() {
-			err := m.muxWriter.run(m.config.Logger)
+			err := m.muxWriter.run(m.config.Log)
 			m.explicitShutdown.Fuse(false)
 			m.w.Close()
 			m.abort()
@@ -367,7 +366,7 @@ func (m *Muxer) Serve(ctx context.Context) error {
 	errGroup.Go(func() error {
 		ch := make(chan error)
 		go func() {
-			err := m.muxMetricsUpdater.run(m.config.Logger)
+			err := m.muxMetricsUpdater.run(m.config.Log)
 			// don't block if parent goroutine quit early
 			select {
 			case ch <- err:

h2mux/h2mux_test.go

@@ -16,10 +16,9 @@ import (
 	"time"
 
 	"github.com/pkg/errors"
+	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/assert"
 	"golang.org/x/sync/errgroup"
-
-	"github.com/cloudflare/cloudflared/logger"
 )
 
 const (
@@ -27,6 +26,8 @@ const (
 	testHandshakeTimeout  = time.Millisecond * 1000
 )
 
+var log = zerolog.Nop()
+
 func TestMain(m *testing.M) {
 	if os.Getenv("VERBOSE") == "1" {
 		//TODO: set log level
@@ -52,7 +53,7 @@ func NewDefaultMuxerPair(t assert.TestingT, testName string, f MuxedStreamFunc)
 			Handler:                 f,
 			IsClient:                true,
 			Name:                    "origin",
-			Logger:                  logger.NewOutputWriter(logger.NewMockWriteManager()),
+			Log:                     &log,
 			DefaultWindowSize:       (1 << 8) - 1,
 			MaxWindowSize:           (1 << 15) - 1,
 			StreamWriteBufferMaxLen: 1024,
@@ -64,7 +65,7 @@ func NewDefaultMuxerPair(t assert.TestingT, testName string, f MuxedStreamFunc)
 			Timeout:                 testHandshakeTimeout,
 			IsClient:                false,
 			Name:                    "edge",
-			Logger:                  logger.NewOutputWriter(logger.NewMockWriteManager()),
+			Log:                     &log,
 			DefaultWindowSize:       (1 << 8) - 1,
 			MaxWindowSize:           (1 << 15) - 1,
 			StreamWriteBufferMaxLen: 1024,
@@ -87,7 +88,7 @@ func NewCompressedMuxerPair(t assert.TestingT, testName string, quality Compress
 			IsClient:           true,
 			Name:               "origin",
 			CompressionQuality: quality,
-			Logger:             logger.NewOutputWriter(logger.NewMockWriteManager()),
+			Log:                &log,
 			HeartbeatInterval:  defaultTimeout,
 			MaxHeartbeats:      defaultRetries,
 		},
@@ -97,7 +98,7 @@ func NewCompressedMuxerPair(t assert.TestingT, testName string, quality Compress
 			IsClient:           false,
 			Name:               "edge",
 			CompressionQuality: quality,
-			Logger:             logger.NewOutputWriter(logger.NewMockWriteManager()),
+			Log:                &log,
 			HeartbeatInterval:  defaultTimeout,
 			MaxHeartbeats:      defaultRetries,
 		},
@@ -113,11 +114,11 @@ func (p *DefaultMuxerPair) Handshake(testName string) error {
 	defer cancel()
 	errGroup, _ := errgroup.WithContext(ctx)
 	errGroup.Go(func() (err error) {
-		p.EdgeMux, err = Handshake(p.EdgeConn, p.EdgeConn, p.EdgeMuxConfig, NewActiveStreamsMetrics(testName, "edge"))
+		p.EdgeMux, err = Handshake(p.EdgeConn, p.EdgeConn, p.EdgeMuxConfig, ActiveStreams)
 		return errors.Wrap(err, "edge handshake failure")
 	})
 	errGroup.Go(func() (err error) {
-		p.OriginMux, err = Handshake(p.OriginConn, p.OriginConn, p.OriginMuxConfig, NewActiveStreamsMetrics(testName, "origin"))
+		p.OriginMux, err = Handshake(p.OriginConn, p.OriginConn, p.OriginMuxConfig, ActiveStreams)
 		return errors.Wrap(err, "origin handshake failure")
 	})
 
@@ -186,11 +187,11 @@ func TestSingleStream(t *testing.T) {
 		if stream.Headers[0].Value != "headerValue" {
 			t.Fatalf("expected header value %s, got %s", "headerValue", stream.Headers[0].Value)
 		}
-		stream.WriteHeaders([]Header{
+		_ = stream.WriteHeaders([]Header{
 			{Name: "response-header", Value: "responseValue"},
 		})
 		buf := []byte("Hello world")
-		stream.Write(buf)
+		_, _ = stream.Write(buf)
 		n, err := io.ReadFull(stream, buf)
 		if n > 0 {
 			t.Fatalf("read %d bytes after EOF", n)
@@ -230,7 +231,7 @@ func TestSingleStream(t *testing.T) {
 	if string(responseBody) != "Hello world" {
 		t.Fatalf("expected response body %s, got %s", "Hello world", responseBody)
 	}
-	stream.Close()
+	_ = stream.Close()
 	n, err = stream.Write([]byte("aaaaa"))
 	if n > 0 {
 		t.Fatalf("wrote %d bytes after EOF", n)
@@ -252,7 +253,7 @@ func TestSingleStreamLargeResponseBody(t *testing.T) {
 		if stream.Headers[0].Value != "headerValue" {
 			t.Fatalf("expected header value %s, got %s", "headerValue", stream.Headers[0].Value)
 		}
-		stream.WriteHeaders([]Header{
+		_ = stream.WriteHeaders([]Header{
 			{Name: "response-header", Value: "responseValue"},
 		})
 		payload := make([]byte, bodySize)
@@ -302,7 +303,6 @@ func TestSingleStreamLargeResponseBody(t *testing.T) {
 }
 
 func TestMultipleStreams(t *testing.T) {
-	l := logger.NewOutputWriter(logger.NewMockWriteManager())
 	f := MuxedStreamFunc(func(stream *MuxedStream) error {
 		if len(stream.Headers) != 1 {
 			t.Fatalf("expected %d headers, got %d", 1, len(stream.Headers))
@@ -310,13 +310,13 @@ func TestMultipleStreams(t *testing.T) {
 		if stream.Headers[0].Name != "client-token" {
 			t.Fatalf("expected header name %s, got %s", "client-token", stream.Headers[0].Name)
 		}
-		l.Debugf("Got request for stream %s", stream.Headers[0].Value)
-		stream.WriteHeaders([]Header{
+		log.Debug().Msgf("Got request for stream %s", stream.Headers[0].Value)
+		_ = stream.WriteHeaders([]Header{
 			{Name: "response-token", Value: stream.Headers[0].Value},
 		})
-		l.Debugf("Wrote headers for stream %s", stream.Headers[0].Value)
-		stream.Write([]byte("OK"))
-		l.Debugf("Wrote body for stream %s", stream.Headers[0].Value)
+		log.Debug().Msgf("Wrote headers for stream %s", stream.Headers[0].Value)
+		_, _ = stream.Write([]byte("OK"))
+		log.Debug().Msgf("Wrote body for stream %s", stream.Headers[0].Value)
 		return nil
 	})
 	muxPair := NewDefaultMuxerPair(t, t.Name(), f)
@@ -334,7 +334,7 @@ func TestMultipleStreams(t *testing.T) {
 				[]Header{{Name: "client-token", Value: tokenString}},
 				nil,
 			)
-			l.Debugf("Got headers for stream %d", tokenId)
+			log.Debug().Msgf("Got headers for stream %d", tokenId)
 			if err != nil {
 				errorsC <- err
 				return
@@ -372,7 +372,7 @@ func TestMultipleStreams(t *testing.T) {
 	testFail := false
 	for err := range errorsC {
 		testFail = true
-		l.Errorf("%s", err)
+		log.Error().Msgf("%s", err)
 	}
 	if testFail {
 		t.Fatalf("TestMultipleStreams failed")
@@ -396,7 +396,7 @@ func TestMultipleStreamsFlowControl(t *testing.T) {
 		if stream.Headers[0].Value != "headerValue" {
 			t.Fatalf("expected header value %s, got %s", "headerValue", stream.Headers[0].Value)
 		}
-		stream.WriteHeaders([]Header{
+		_ = stream.WriteHeaders([]Header{
 			{Name: "response-header", Value: "responseValue"},
 		})
 		payload := make([]byte, responseSizes[(stream.streamID-2)/2])
@@ -450,27 +450,25 @@ func TestMultipleStreamsFlowControl(t *testing.T) {
 }
 
 func TestGracefulShutdown(t *testing.T) {
-	l := logger.NewOutputWriter(logger.NewMockWriteManager())
-
 	sendC := make(chan struct{})
 	responseBuf := bytes.Repeat([]byte("Hello world"), 65536)
 
 	f := MuxedStreamFunc(func(stream *MuxedStream) error {
-		stream.WriteHeaders([]Header{
+		_ = stream.WriteHeaders([]Header{
 			{Name: "response-header", Value: "responseValue"},
 		})
 		<-sendC
-		l.Debugf("Writing %d bytes", len(responseBuf))
-		stream.Write(responseBuf)
-		stream.CloseWrite()
-		l.Debugf("Wrote %d bytes", len(responseBuf))
+		log.Debug().Msgf("Writing %d bytes", len(responseBuf))
+		_, _ = stream.Write(responseBuf)
+		_ = stream.CloseWrite()
+		log.Debug().Msgf("Wrote %d bytes", len(responseBuf))
 		// Reading from the stream will block until the edge closes its end of the stream.
 		// Otherwise, we'll close the whole connection before receiving the 'stream closed'
 		// message from the edge.
 		// Graceful shutdown works if you omit this, it just gives spurious errors for now -
 		// TODO ignore errors when writing 'stream closed' and we're shutting down.
-		stream.Read([]byte{0})
-		l.Debugf("Handler ends")
+		_, _ = stream.Read([]byte{0})
+		log.Debug().Msgf("Handler ends")
 		return nil
 	})
 	muxPair := NewDefaultMuxerPair(t, t.Name(), f)
@@ -487,7 +485,7 @@ func TestGracefulShutdown(t *testing.T) {
 	muxPair.EdgeMux.Shutdown()
 	close(sendC)
 	responseBody := make([]byte, len(responseBuf))
-	l.Debugf("Waiting for %d bytes", len(responseBuf))
+	log.Debug().Msgf("Waiting for %d bytes", len(responseBuf))
 	n, err := io.ReadFull(stream, responseBody)
 	if err != nil {
 		t.Fatalf("error from (*MuxedStream).Read with %d bytes read: %s", n, err)
@@ -498,7 +496,7 @@ func TestGracefulShutdown(t *testing.T) {
 	if !bytes.Equal(responseBuf, responseBody) {
 		t.Fatalf("response body mismatch")
 	}
-	stream.Close()
+	_ = stream.Close()
 	muxPair.Wait(t)
 }
 
@@ -509,7 +507,7 @@ func TestUnexpectedShutdown(t *testing.T) {
 
 	f := MuxedStreamFunc(func(stream *MuxedStream) error {
 		defer close(handlerFinishC)
-		stream.WriteHeaders([]Header{
+		_ = stream.WriteHeaders([]Header{
 			{Name: "response-header", Value: "responseValue"},
 		})
 		<-sendC
@@ -536,7 +534,7 @@ func TestUnexpectedShutdown(t *testing.T) {
 		nil,
 	)
 	// Close the underlying connection before telling the origin to write.
-	muxPair.EdgeConn.Close()
+	_ = muxPair.EdgeConn.Close()
 	close(sendC)
 	if err != nil {
 		t.Fatalf("error in OpenStream: %s", err)
@@ -559,18 +557,18 @@ func TestUnexpectedShutdown(t *testing.T) {
 
 func EchoHandler(stream *MuxedStream) error {
 	var buf bytes.Buffer
-	fmt.Fprintf(&buf, "Hello, world!\n\n# REQUEST HEADERS:\n\n")
+	_, _ = fmt.Fprintf(&buf, "Hello, world!\n\n# REQUEST HEADERS:\n\n")
 	for _, header := range stream.Headers {
-		fmt.Fprintf(&buf, "[%s] = %s\n", header.Name, header.Value)
+		_, _ = fmt.Fprintf(&buf, "[%s] = %s\n", header.Name, header.Value)
 	}
-	stream.WriteHeaders([]Header{
+	_ = stream.WriteHeaders([]Header{
 		{Name: ":status", Value: "200"},
 		{Name: "server", Value: "Echo-server/1.0"},
 		{Name: "date", Value: time.Now().Format(time.RFC850)},
 		{Name: "content-type", Value: "text/html; charset=utf-8"},
 		{Name: "content-length", Value: strconv.Itoa(buf.Len())},
 	})
-	buf.WriteTo(stream)
+	_, _ = buf.WriteTo(stream)
 	return nil
 }
 
@@ -582,14 +580,14 @@ func TestOpenAfterDisconnect(t *testing.T) {
 		switch i {
 		case 0:
 			// Close both directions of the connection to cause EOF on both peers.
-			muxPair.OriginConn.Close()
-			muxPair.EdgeConn.Close()
+			_ = muxPair.OriginConn.Close()
+			_ = muxPair.EdgeConn.Close()
 		case 1:
 			// Close origin conn to cause EOF on origin first.
-			muxPair.OriginConn.Close()
+			_ = muxPair.OriginConn.Close()
 		case 2:
 			// Close edge conn to cause EOF on edge first.
-			muxPair.EdgeConn.Close()
+			_ = muxPair.EdgeConn.Close()
 		}
 
 		_, err := muxPair.OpenEdgeMuxStream(
@@ -617,7 +615,7 @@ func TestHPACK(t *testing.T) {
 	if err != nil {
 		t.Fatalf("error in OpenStream: %s", err)
 	}
-	stream.Close()
+	_ = stream.Close()
 
 	for i := 0; i < 3; i++ {
 		stream, err := muxPair.OpenEdgeMuxStream(
@@ -654,8 +652,8 @@ func TestHPACK(t *testing.T) {
 		if stream.Headers[0].Value != "200" {
 			t.Fatalf("expected status 200, got %s", stream.Headers[0].Value)
 		}
-		ioutil.ReadAll(stream)
-		stream.Close()
+		_, _ = ioutil.ReadAll(stream)
+		_ = stream.Close()
 	}
 }
 
@@ -680,7 +678,7 @@ func AssertIfPipeReadable(t *testing.T, pipe io.ReadCloser) {
 }
 
 func TestMultipleStreamsWithDictionaries(t *testing.T) {
-	l := logger.NewOutputWriter(logger.NewMockWriteManager())
+	l := zerolog.Nop()
 
 	for q := CompressionNone; q <= CompressionMax; q++ {
 		htmlBody := `<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"` +
@@ -730,10 +728,10 @@ func TestMultipleStreamsWithDictionaries(t *testing.T) {
 				contentType = "img/gif"
 			}
 
-			stream.WriteHeaders([]Header{
-				Header{Name: "content-type", Value: contentType},
+			_ = stream.WriteHeaders([]Header{
+				{Name: "content-type", Value: contentType},
 			})
-			stream.Write([]byte(strings.Replace(htmlBody, "paragraph", pathHeader.Value, 1) + stream.Headers[5].Value))
+			_, _ = stream.Write([]byte(strings.Replace(htmlBody, "paragraph", pathHeader.Value, 1) + stream.Headers[5].Value))
 
 			return nil
 		})
@@ -817,7 +815,7 @@ func TestMultipleStreamsWithDictionaries(t *testing.T) {
 		testFail := false
 		for err := range errorsC {
 			testFail = true
-			l.Errorf("%s", err)
+			l.Error().Msgf("%s", err)
 		}
 		if testFail {
 			t.Fatalf("TestMultipleStreams failed")
@@ -831,8 +829,6 @@ func TestMultipleStreamsWithDictionaries(t *testing.T) {
 }
 
 func sampleSiteHandler(files map[string][]byte) MuxedStreamFunc {
-	l := logger.NewOutputWriter(logger.NewMockWriteManager())
-
 	return func(stream *MuxedStream) error {
 		var contentType string
 		var pathHeader Header
@@ -857,16 +853,16 @@ func sampleSiteHandler(files map[string][]byte) MuxedStreamFunc {
 		} else {
 			contentType = "img/gif"
 		}
-		stream.WriteHeaders([]Header{
-			Header{Name: "content-type", Value: contentType},
+		_ = stream.WriteHeaders([]Header{
+			{Name: "content-type", Value: contentType},
 		})
-		l.Debugf("Wrote headers for stream %s", pathHeader.Value)
+		log.Debug().Msgf("Wrote headers for stream %s", pathHeader.Value)
 		file, ok := files[pathHeader.Value]
 		if !ok {
 			return fmt.Errorf("%s content is not preloaded", pathHeader.Value)
 		}
-		stream.Write(file)
-		l.Debugf("Wrote body for stream %s", pathHeader.Value)
+		_, _ = stream.Write(file)
+		log.Debug().Msgf("Wrote body for stream %s", pathHeader.Value)
 		return nil
 	}
 }
@@ -1008,7 +1004,7 @@ func BenchmarkOpenStream(b *testing.B) {
 			if stream.Headers[0].Value != "headerValue" {
 				b.Fatalf("expected header value %s, got %s", "headerValue", stream.Headers[0].Value)
 			}
-			stream.WriteHeaders([]Header{
+			_ = stream.WriteHeaders([]Header{
 				{Name: "response-header", Value: "responseValue"},
 			})
 			return nil
@@ -1058,7 +1054,7 @@ func BenchmarkSingleStreamLargeResponseBody(b *testing.B) {
 		if stream.Headers[0].Value != "headerValue" {
 			b.Fatalf("expected header value %s, got %s", "headerValue", stream.Headers[0].Value)
 		}
-		stream.WriteHeaders([]Header{
+		_ = stream.WriteHeaders([]Header{
 			{Name: "response-header", Value: "responseValue"},
 		})
 		for i := 0; i < writeN; i++ {
@@ -1083,7 +1079,7 @@ func BenchmarkSingleStreamLargeResponseBody(b *testing.B) {
 			Handler:                 f,
 			IsClient:                true,
 			Name:                    "origin",
-			Logger:                  logger.NewOutputWriter(logger.NewMockWriteManager()),
+			Log:                     &log,
 			DefaultWindowSize:       defaultWindowSize,
 			MaxWindowSize:           maxWindowSize,
 			StreamWriteBufferMaxLen: defaultWriteBufferMaxLen,
@@ -1095,7 +1091,7 @@ func BenchmarkSingleStreamLargeResponseBody(b *testing.B) {
 			Timeout:                 testHandshakeTimeout,
 			IsClient:                false,
 			Name:                    "edge",
-			Logger:                  logger.NewOutputWriter(logger.NewMockWriteManager()),
+			Log:                     &log,
 			DefaultWindowSize:       defaultWindowSize,
 			MaxWindowSize:           maxWindowSize,
 			StreamWriteBufferMaxLen: defaultWriteBufferMaxLen,

h2mux/header.go

@@ -2,7 +2,6 @@ package h2mux
 
 import (
 	"encoding/base64"
-	"encoding/json"
 	"fmt"
 	"net/http"
 	"net/url"
@@ -22,10 +21,6 @@ const (
 	RequestUserHeadersField  = "cf-cloudflared-request-headers"
 	ResponseUserHeadersField = "cf-cloudflared-response-headers"
 
-	ResponseMetaHeaderField   = "cf-cloudflared-response-meta"
-	ResponseSourceCloudflared = "cloudflared"
-	ResponseSourceOrigin      = "origin"
-
 	CFAccessTokenHeader        = "cf-access-token"
 	CFJumpDestinationHeader    = "CF-Access-Jump-Destination"
 	CFAccessClientIDHeader     = "CF-Access-Client-Id"
@@ -124,7 +119,7 @@ func IsControlHeader(headerName string) bool {
 }
 
 // isWebsocketClientHeader returns true if the header name is required by the client to upgrade properly
-func isWebsocketClientHeader(headerName string) bool {
+func IsWebsocketClientHeader(headerName string) bool {
 	return headerName == "sec-websocket-accept" ||
 		headerName == "connection" ||
 		headerName == "upgrade"
@@ -143,7 +138,7 @@ func H1ResponseToH2ResponseHeaders(h1 *http.Response) (h2 []Header) {
 
 			// Since these are http2 headers, they're required to be lowercase
 			h2 = append(h2, Header{Name: "content-length", Value: values[0]})
-		} else if !IsControlHeader(h2name) || isWebsocketClientHeader(h2name) {
+		} else if !IsControlHeader(h2name) || IsWebsocketClientHeader(h2name) {
 			// User headers, on the other hand, must all be serialized so that
 			// HTTP/2 header validation won't be applied to HTTP/1 header values
 			userHeaders[header] = values
@@ -152,7 +147,6 @@ func H1ResponseToH2ResponseHeaders(h1 *http.Response) (h2 []Header) {
 
 	// Perform user header serialization and set them in the single header
 	h2 = append(h2, Header{ResponseUserHeadersField, SerializeHeaders(userHeaders)})
-
 	return h2
 }
 
@@ -238,19 +232,3 @@ func DeserializeHeaders(serializedHeaders string) ([]Header, error) {
 
 	return deserialized, nil
 }
-
-type ResponseMetaHeader struct {
-	Source string `json:"src"`
-}
-
-func CreateResponseMetaHeader(headerName, source string) Header {
-	jsonResponseMetaHeader, err := json.Marshal(ResponseMetaHeader{Source: source})
-	if err != nil {
-		panic(err)
-	}
-
-	return Header{
-		Name:  headerName,
-		Value: string(jsonResponseMetaHeader),
-	}
-}

h2mux/muxmetrics.go

@@ -4,9 +4,8 @@ import (
 	"sync"
 	"time"
 
-	"github.com/cloudflare/cloudflared/logger"
 	"github.com/golang-collections/collections/queue"
-	"github.com/prometheus/client_golang/prometheus"
+	"github.com/rs/zerolog"
 )
 
 // data points used to compute average receive window and send window size
@@ -21,7 +20,7 @@ type muxMetricsUpdater interface {
 	// metrics returns the latest metrics
 	metrics() *MuxerMetrics
 	// run is a blocking call to start the event loop
-	run(logger logger.Service) error
+	run(log *zerolog.Logger) error
 	// updateRTTChan is called by muxReader to report new RTT measurements
 	updateRTT(rtt *roundTripMeasurement)
 	//updateReceiveWindowChan is called by muxReader and muxWriter when receiveWindow size is updated
@@ -138,30 +137,30 @@ func (updater *muxMetricsUpdaterImpl) metrics() *MuxerMetrics {
 	return m
 }
 
-func (updater *muxMetricsUpdaterImpl) run(logger logger.Service) error {
-	defer logger.Debug("mux - metrics: event loop finished")
+func (updater *muxMetricsUpdaterImpl) run(log *zerolog.Logger) error {
+	defer log.Debug().Msg("mux - metrics: event loop finished")
 	for {
 		select {
 		case <-updater.abortChan:
-			logger.Infof("mux - metrics: Stopping mux metrics updater")
+			log.Debug().Msgf("mux - metrics: Stopping mux metrics updater")
 			return nil
 		case roundTripMeasurement := <-updater.updateRTTChan:
 			go updater.rttData.update(roundTripMeasurement)
-			logger.Debug("mux - metrics: Update rtt")
+			log.Debug().Msg("mux - metrics: Update rtt")
 		case receiveWindow := <-updater.updateReceiveWindowChan:
 			go updater.receiveWindowData.update(receiveWindow)
-			logger.Debug("mux - metrics: Update receive window")
+			log.Debug().Msg("mux - metrics: Update receive window")
 		case sendWindow := <-updater.updateSendWindowChan:
 			go updater.sendWindowData.update(sendWindow)
-			logger.Debug("mux - metrics: Update send window")
+			log.Debug().Msg("mux - metrics: Update send window")
 		case inBoundBytes := <-updater.updateInBoundBytesChan:
 			// inBoundBytes is bytes/sec because the update interval is 1 sec
 			go updater.inBoundRate.update(inBoundBytes)
-			logger.Debugf("mux - metrics: Inbound bytes %d", inBoundBytes)
+			log.Debug().Msgf("mux - metrics: Inbound bytes %d", inBoundBytes)
 		case outBoundBytes := <-updater.updateOutBoundBytesChan:
 			// outBoundBytes is bytes/sec because the update interval is 1 sec
 			go updater.outBoundRate.update(outBoundBytes)
-			logger.Debugf("mux - metrics: Outbound bytes %d", outBoundBytes)
+			log.Debug().Msgf("mux - metrics: Outbound bytes %d", outBoundBytes)
 		}
 	}
 }
@@ -295,14 +294,3 @@ func (r *rate) get() (curr, min, max uint64) {
 	defer r.lock.RUnlock()
 	return r.curr, r.min, r.max
 }
-
-func NewActiveStreamsMetrics(namespace, subsystem string) prometheus.Gauge {
-	activeStreams := prometheus.NewGauge(prometheus.GaugeOpts{
-		Namespace: namespace,
-		Subsystem: subsystem,
-		Name:      "active_streams",
-		Help:      "Number of active streams created by all muxers.",
-	})
-	prometheus.MustRegister(activeStreams)
-	return activeStreams
-}

h2mux/muxmetrics_test.go

@@ -5,7 +5,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/cloudflare/cloudflared/logger"
+	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -91,10 +91,10 @@ func TestMuxMetricsUpdater(t *testing.T) {
 	abortChan := make(chan struct{})
 	compBefore, compAfter := NewAtomicCounter(0), NewAtomicCounter(0)
 	m := newMuxMetricsUpdater(abortChan, compBefore, compAfter)
-	logger := logger.NewOutputWriter(logger.NewMockWriteManager())
+	log := zerolog.Nop()
 
 	go func() {
-		errChan <- m.run(logger)
+		errChan <- m.run(&log)
 	}()
 
 	var wg sync.WaitGroup

h2mux/muxreader.go

@@ -8,7 +8,7 @@ import (
 	"net/url"
 	"time"
 
-	"github.com/cloudflare/cloudflared/logger"
+	"github.com/rs/zerolog"
 	"golang.org/x/net/http2"
 )
 
@@ -68,8 +68,8 @@ func (r *MuxReader) Shutdown() <-chan struct{} {
 	return done
 }
 
-func (r *MuxReader) run(logger logger.Service) error {
-	defer logger.Debug("mux - read: event loop finished")
+func (r *MuxReader) run(log *zerolog.Logger) error {
+	defer log.Debug().Msg("mux - read: event loop finished")
 
 	// routine to periodically update bytesRead
 	go func() {
@@ -93,7 +93,7 @@ func (r *MuxReader) run(logger logger.Service) error {
 			}
 			switch e := err.(type) {
 			case http2.StreamError:
-				logger.Infof("%s: stream error", errorString)
+				log.Info().Msgf("%s: stream error", errorString)
 				// Ideally we wouldn't return here, since that aborts the muxer.
 				// We should communicate the error to the relevant MuxedStream
 				// data structure, so that callers of MuxedStream.Read() and
@@ -101,28 +101,28 @@ func (r *MuxReader) run(logger logger.Service) error {
 				// and keep the muxer going.
 				return r.streamError(e.StreamID, e.Code)
 			case http2.ConnectionError:
-				logger.Infof("%s: stream error", errorString)
+				log.Info().Msgf("%s: stream error", errorString)
 				return r.connectionError(err)
 			default:
 				if isConnectionClosedError(err) {
 					if r.streams.Len() == 0 {
 						// don't log the error here -- that would just be extra noise
-						logger.Debug("mux - read: shutting down")
+						log.Debug().Msg("mux - read: shutting down")
 						return nil
 					}
-					logger.Infof("%s: connection closed unexpectedly", errorString)
+					log.Info().Msgf("%s: connection closed unexpectedly", errorString)
 					return err
 				} else {
-					logger.Infof("%s: frame read error", errorString)
+					log.Info().Msgf("%s: frame read error", errorString)
 					return r.connectionError(err)
 				}
 			}
 		}
 		r.connActive.Signal()
-		logger.Debugf("mux - read: read frame: data %v", frame)
+		log.Debug().Msgf("mux - read: read frame: data %v", frame)
 		switch f := frame.(type) {
 		case *http2.DataFrame:
-			err = r.receiveFrameData(f, logger)
+			err = r.receiveFrameData(f, log)
 		case *http2.MetaHeadersFrame:
 			err = r.receiveHeaderData(f)
 		case *http2.RSTStreamFrame:
@@ -155,7 +155,7 @@ func (r *MuxReader) run(logger logger.Service) error {
 			err = ErrUnexpectedFrameType
 		}
 		if err != nil {
-			logger.Debugf("mux - read: read error: data %v", frame)
+			log.Debug().Msgf("mux - read: read error: data %v", frame)
 			return r.connectionError(err)
 		}
 	}
@@ -276,7 +276,7 @@ func (r *MuxReader) handleStream(stream *MuxedStream) {
 }
 
 // Receives a data frame from a stream. A non-nil error is a connection error.
-func (r *MuxReader) receiveFrameData(frame *http2.DataFrame, logger logger.Service) error {
+func (r *MuxReader) receiveFrameData(frame *http2.DataFrame, log *zerolog.Logger) error {
 	stream, err := r.getStreamForFrame(frame)
 	if err != nil {
 		return r.defaultStreamErrorHandler(err, frame.Header())
@@ -292,9 +292,9 @@ func (r *MuxReader) receiveFrameData(frame *http2.DataFrame, logger logger.Servi
 	if frame.Header().Flags.Has(http2.FlagDataEndStream) {
 		if stream.receiveEOF() {
 			r.streams.Delete(stream.streamID)
-			logger.Debugf("mux - read: stream closed: streamID: %d", frame.Header().StreamID)
+			log.Debug().Msgf("mux - read: stream closed: streamID: %d", frame.Header().StreamID)
 		} else {
-			logger.Debugf("mux - read: shutdown receive side: streamID: %d", frame.Header().StreamID)
+			log.Debug().Msgf("mux - read: shutdown receive side: streamID: %d", frame.Header().StreamID)
 		}
 		return nil
 	}

h2mux/muxwriter.go

@@ -3,10 +3,10 @@ package h2mux
 import (
 	"bytes"
 	"encoding/binary"
+	"github.com/rs/zerolog"
 	"io"
 	"time"
 
-	"github.com/cloudflare/cloudflared/logger"
 	"golang.org/x/net/http2"
 	"golang.org/x/net/http2/hpack"
 )
@@ -72,8 +72,8 @@ func tsToPingData(ts int64) [8]byte {
 	return pingData
 }
 
-func (w *MuxWriter) run(logger logger.Service) error {
-	defer logger.Debug("mux - write: event loop finished")
+func (w *MuxWriter) run(log *zerolog.Logger) error {
+	defer log.Debug().Msg("mux - write: event loop finished")
 
 	// routine to periodically communicate bytesWrote
 	go func() {
@@ -91,17 +91,17 @@ func (w *MuxWriter) run(logger logger.Service) error {
 	for {
 		select {
 		case <-w.abortChan:
-			logger.Debug("mux - write: aborting writer thread")
+			log.Debug().Msg("mux - write: aborting writer thread")
 			return nil
 		case errCode := <-w.goAwayChan:
-			logger.Debugf("mux - write: sending GOAWAY code %v", errCode)
+			log.Debug().Msgf("mux - write: sending GOAWAY code %v", errCode)
 			err := w.f.WriteGoAway(w.streams.LastPeerStreamID(), errCode, []byte{})
 			if err != nil {
 				return err
 			}
 			w.idleTimer.MarkActive()
 		case <-w.pingTimestamp.GetUpdateChan():
-			logger.Debug("mux - write: sending PING ACK")
+			log.Debug().Msg("mux - write: sending PING ACK")
 			err := w.f.WritePing(true, tsToPingData(w.pingTimestamp.Get()))
 			if err != nil {
 				return err
@@ -111,7 +111,7 @@ func (w *MuxWriter) run(logger logger.Service) error {
 			if !w.idleTimer.Retry() {
 				return ErrConnectionDropped
 			}
-			logger.Debug("mux - write: sending PING")
+			log.Debug().Msg("mux - write: sending PING")
 			err := w.f.WritePing(false, tsToPingData(time.Now().UnixNano()))
 			if err != nil {
 				return err
@@ -121,7 +121,7 @@ func (w *MuxWriter) run(logger logger.Service) error {
 			w.idleTimer.MarkActive()
 		case <-w.streamErrors.GetSignalChan():
 			for streamID, errCode := range w.streamErrors.GetErrors() {
-				logger.Debugf("mux - write: resetting stream with code: %v streamID: %d", errCode, streamID)
+				log.Debug().Msgf("mux - write: resetting stream with code: %v streamID: %d", errCode, streamID)
 				err := w.f.WriteRSTStream(streamID, errCode)
 				if err != nil {
 					return err
@@ -141,7 +141,7 @@ func (w *MuxWriter) run(logger logger.Service) error {
 			if streamRequest.body != nil {
 				go streamRequest.flushBody()
 			}
-			err := w.writeStreamData(streamRequest.stream, logger)
+			err := w.writeStreamData(streamRequest.stream, log)
 			if err != nil {
 				return err
 			}
@@ -151,7 +151,7 @@ func (w *MuxWriter) run(logger logger.Service) error {
 			if !ok {
 				continue
 			}
-			err := w.writeStreamData(stream, logger)
+			err := w.writeStreamData(stream, log)
 			if err != nil {
 				return err
 			}
@@ -159,7 +159,7 @@ func (w *MuxWriter) run(logger logger.Service) error {
 		case useDict := <-w.useDictChan:
 			err := w.writeUseDictionary(useDict)
 			if err != nil {
-				logger.Errorf("mux - write: error writing use dictionary: %s", err)
+				log.Error().Msgf("mux - write: error writing use dictionary: %s", err)
 				return err
 			}
 			w.idleTimer.MarkActive()
@@ -167,18 +167,18 @@ func (w *MuxWriter) run(logger logger.Service) error {
 	}
 }
 
-func (w *MuxWriter) writeStreamData(stream *MuxedStream, logger logger.Service) error {
-	logger.Debugf("mux - write: writable: streamID: %d", stream.streamID)
+func (w *MuxWriter) writeStreamData(stream *MuxedStream, log *zerolog.Logger) error {
+	log.Debug().Msgf("mux - write: writable: streamID: %d", stream.streamID)
 	chunk := stream.getChunk()
 	w.metricsUpdater.updateReceiveWindow(stream.getReceiveWindow())
 	w.metricsUpdater.updateSendWindow(stream.getSendWindow())
 	if chunk.sendHeadersFrame() {
 		err := w.writeHeaders(chunk.streamID, chunk.headers)
 		if err != nil {
-			logger.Errorf("mux - write: error writing headers: %s: streamID: %d", err, stream.streamID)
+			log.Error().Msgf("mux - write: error writing headers: %s: streamID: %d", err, stream.streamID)
 			return err
 		}
-		logger.Debugf("mux - write: output headers: streamID: %d", stream.streamID)
+		log.Debug().Msgf("mux - write: output headers: streamID: %d", stream.streamID)
 	}
 
 	if chunk.sendWindowUpdateFrame() {
@@ -189,22 +189,22 @@ func (w *MuxWriter) writeStreamData(stream *MuxedStream, logger logger.Service)
 		// window, unless the receiver treats this as a connection error"
 		err := w.f.WriteWindowUpdate(chunk.streamID, chunk.windowUpdate)
 		if err != nil {
-			logger.Errorf("mux - write: error writing window update: %s: streamID: %d", err, stream.streamID)
+			log.Error().Msgf("mux - write: error writing window update: %s: streamID: %d", err, stream.streamID)
 			return err
 		}
-		logger.Debugf("mux - write: increment receive window by %d streamID: %d", chunk.windowUpdate, stream.streamID)
+		log.Debug().Msgf("mux - write: increment receive window by %d streamID: %d", chunk.windowUpdate, stream.streamID)
 	}
 
 	for chunk.sendDataFrame() {
 		payload, sentEOF := chunk.nextDataFrame(int(w.maxFrameSize))
 		err := w.f.WriteData(chunk.streamID, sentEOF, payload)
 		if err != nil {
-			logger.Errorf("mux - write: error writing data: %s: streamID: %d", err, stream.streamID)
+			log.Error().Msgf("mux - write: error writing data: %s: streamID: %d", err, stream.streamID)
 			return err
 		}
 		// update the amount of data wrote
 		w.bytesWrote.IncrementBy(uint64(len(payload)))
-		logger.Debugf("mux - write: output data: %d: streamID: %d", len(payload), stream.streamID)
+		log.Debug().Msgf("mux - write: output data: %d: streamID: %d", len(payload), stream.streamID)
 
 		if sentEOF {
 			if stream.readBuffer.Closed() {
@@ -212,15 +212,15 @@ func (w *MuxWriter) writeStreamData(stream *MuxedStream, logger logger.Service)
 				if !stream.gotReceiveEOF() {
 					// the peer may send data that we no longer want to receive. Force them into the
 					// closed state.
-					logger.Debugf("mux - write: resetting stream: streamID: %d", stream.streamID)
+					log.Debug().Msgf("mux - write: resetting stream: streamID: %d", stream.streamID)
 					w.f.WriteRSTStream(chunk.streamID, http2.ErrCodeNo)
 				} else {
 					// Half-open stream transitioned into closed
-					logger.Debugf("mux - write: closing stream: streamID: %d", stream.streamID)
+					log.Debug().Msgf("mux - write: closing stream: streamID: %d", stream.streamID)
 				}
 				w.streams.Delete(chunk.streamID)
 			} else {
-				logger.Debugf("mux - write: closing stream write side: streamID: %d", stream.streamID)
+				log.Debug().Msgf("mux - write: closing stream write side: streamID: %d", stream.streamID)
 			}
 		}
 	}

hello/hello.go

@@ -12,12 +12,20 @@ import (
 	"os"
 	"time"
 
-	"github.com/cloudflare/cloudflared/logger"
 	"github.com/gorilla/websocket"
+	"github.com/rs/zerolog"
 
 	"github.com/cloudflare/cloudflared/tlsconfig"
 )
 
+const (
+	UptimeRoute    = "/uptime"
+	WSRoute        = "/ws"
+	SSERoute       = "/sse"
+	HealthRoute    = "/_health"
+	defaultSSEFreq = time.Second * 10
+)
+
 type templateData struct {
 	ServerName string
 	Request    *http.Request
@@ -91,8 +99,8 @@ const indexTemplate = `
 </html>
 `
 
-func StartHelloWorldServer(logger logger.Service, listener net.Listener, shutdownC <-chan struct{}) error {
-	logger.Infof("Starting Hello World server at %s", listener.Addr())
+func StartHelloWorldServer(log *zerolog.Logger, listener net.Listener, shutdownC <-chan struct{}) error {
+	log.Info().Msgf("Starting Hello World server at %s", listener.Addr())
 	serverName := defaultServerName
 	if hostname, err := os.Hostname(); err == nil {
 		serverName = hostname
@@ -104,13 +112,15 @@ func StartHelloWorldServer(logger logger.Service, listener net.Listener, shutdow
 	}
 
 	muxer := http.NewServeMux()
-	muxer.HandleFunc("/uptime", uptimeHandler(time.Now()))
-	muxer.HandleFunc("/ws", websocketHandler(logger, upgrader))
+	muxer.HandleFunc(UptimeRoute, uptimeHandler(time.Now()))
+	muxer.HandleFunc(WSRoute, websocketHandler(log, upgrader))
+	muxer.HandleFunc(SSERoute, sseHandler(log))
+	muxer.HandleFunc(HealthRoute, healthHandler())
 	muxer.HandleFunc("/", rootHandler(serverName))
 	httpServer := &http.Server{Addr: listener.Addr().String(), Handler: muxer}
 	go func() {
 		<-shutdownC
-		httpServer.Close()
+		_ = httpServer.Close()
 	}()
 
 	err := httpServer.Serve(listener)
@@ -142,34 +152,83 @@ func uptimeHandler(startTime time.Time) http.HandlerFunc {
 			w.WriteHeader(http.StatusInternalServerError)
 		} else {
 			w.Header().Set("Content-Type", "application/json")
-			w.Write(respJson)
+			_, _ = w.Write(respJson)
 		}
 	}
 }
 
 // This handler will echo message
-func websocketHandler(logger logger.Service, upgrader websocket.Upgrader) http.HandlerFunc {
+func websocketHandler(log *zerolog.Logger, upgrader websocket.Upgrader) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
+		// This addresses the issue of r.Host includes port but origin header doesn't
+		host, _, err := net.SplitHostPort(r.Host)
+		if err == nil {
+			r.Host = host
+		}
+
 		conn, err := upgrader.Upgrade(w, r, nil)
 		if err != nil {
+			log.Err(err).Msg("failed to upgrade to websocket connection")
 			return
 		}
 		defer conn.Close()
 		for {
 			mt, message, err := conn.ReadMessage()
 			if err != nil {
-				logger.Errorf("websocket read message error: %s", err)
+				log.Err(err).Msg("websocket read message error")
 				break
 			}
 
 			if err := conn.WriteMessage(mt, message); err != nil {
-				logger.Errorf("websocket write message error: %s", err)
+				log.Err(err).Msg("websocket write message error")
 				break
 			}
 		}
 	}
 }
 
+func sseHandler(log *zerolog.Logger) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/event-stream; charset=utf-8")
+		flusher, ok := w.(http.Flusher)
+		if !ok {
+			w.WriteHeader(http.StatusInternalServerError)
+			log.Error().Msgf("Can't support SSE. ResponseWriter %T doesn't implement http.Flusher interface", w)
+			return
+		}
+
+		freq := defaultSSEFreq
+		if requestedFreq := r.URL.Query()["freq"]; len(requestedFreq) > 0 {
+			parsedFreq, err := time.ParseDuration(requestedFreq[0])
+			if err == nil {
+				freq = parsedFreq
+			}
+		}
+		log.Info().Msgf("Server Sent Events every %s", freq)
+		ticker := time.NewTicker(freq)
+		counter := 0
+		for {
+			select {
+			case <-r.Context().Done():
+				return
+			case <-ticker.C:
+			}
+			_, err := fmt.Fprintf(w, "%d\n\n", counter)
+			if err != nil {
+				return
+			}
+			flusher.Flush()
+			counter++
+		}
+	}
+}
+
+func healthHandler() http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		w.Write([]byte("ok"))
+	}
+}
+
 func rootHandler(serverName string) http.HandlerFunc {
 	responseTemplate := template.Must(template.New("index").Parse(indexTemplate))
 	return func(w http.ResponseWriter, r *http.Request) {
@@ -188,9 +247,9 @@ func rootHandler(serverName string) http.HandlerFunc {
 		})
 		if err != nil {
 			w.WriteHeader(http.StatusInternalServerError)
-			fmt.Fprintf(w, "error: %v", err)
+			_, _ = fmt.Fprintf(w, "error: %v", err)
 		} else {
-			buffer.WriteTo(w)
+			_, _ = buffer.WriteTo(w)
 		}
 	}
 }

hello/hello_test.go

@@ -27,7 +27,7 @@ func TestCreateTLSListenerOnlyHostSuccess(t *testing.T) {
 }
 
 func TestCreateTLSListenerOnlyPortSuccess(t *testing.T) {
-	listener, err := CreateTLSListener(":8888")
+	listener, err := CreateTLSListener("localhost:8888")
 	if err != nil {
 		t.Fatal(err)
 	}

ingress/ingress.go

@@ -2,70 +2,44 @@ package ingress
 
 import (
 	"fmt"
+	"net"
 	"net/url"
 	"regexp"
+	"strconv"
 	"strings"
-
-	"github.com/pkg/errors"
+	"sync"
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog"
+	"github.com/urfave/cli/v2"
 )
 
 var (
-	ErrNoIngressRules             = errors.New("No ingress rules were specified in the config file")
-	errLastRuleNotCatchAll        = errors.New("The last ingress rule must match all hostnames (i.e. it must be missing, or must be \"*\")")
+	ErrNoIngressRules             = errors.New("The config file doesn't contain any ingress rules")
+	errLastRuleNotCatchAll        = errors.New("The last ingress rule must match all URLs (i.e. it should not have a hostname or path filter)")
 	errBadWildcard                = errors.New("Hostname patterns can have at most one wildcard character (\"*\") and it can only be used for subdomains, e.g. \"*.example.com\"")
+	errHostnameContainsPort       = errors.New("Hostname cannot contain a port")
 	ErrURLIncompatibleWithIngress = errors.New("You can't set the --url flag (or $TUNNEL_URL) when using multiple-origin ingress rules")
 )
 
-// Each rule route traffic from a hostname/path on the public
-// internet to the service running on the given URL.
-type Rule struct {
-	// Requests for this hostname will be proxied to this rule's service.
-	Hostname string
-
-	// Path is an optional regex that can specify path-driven ingress rules.
-	Path *regexp.Regexp
-
-	// A (probably local) address. Requests for a hostname which matches this
-	// rule's hostname pattern will be proxied to the service running on this
-	// address.
-	Service *url.URL
-}
-
-func (r Rule) MultiLineString() string {
-	var out strings.Builder
-	if r.Hostname != "" {
-		out.WriteString("\thostname: ")
-		out.WriteString(r.Hostname)
-		out.WriteRune('\n')
-	}
-	if r.Path != nil {
-		out.WriteString("\tpath: ")
-		out.WriteString(r.Path.String())
-		out.WriteRune('\n')
-	}
-	out.WriteString("\tservice: ")
-	out.WriteString(r.Service.String())
-	return out.String()
-}
-
-func (r *Rule) Matches(hostname, path string) bool {
-	hostMatch := r.Hostname == "" || r.Hostname == "*" || matchHost(r.Hostname, hostname)
-	pathMatch := r.Path == nil || r.Path.MatchString(path)
-	return hostMatch && pathMatch
-}
-
 // FindMatchingRule returns the index of the Ingress Rule which matches the given
 // hostname and path. This function assumes the last rule matches everything,
 // which is the case if the rules were instantiated via the ingress#Validate method
-func (ing Ingress) FindMatchingRule(hostname, path string) int {
+func (ing Ingress) FindMatchingRule(hostname, path string) (*Rule, int) {
+	// The hostname might contain port. We only want to compare the host part with the rule
+	host, _, err := net.SplitHostPort(hostname)
+	if err == nil {
+		hostname = host
+	}
 	for i, rule := range ing.Rules {
 		if rule.Matches(hostname, path) {
-			return i
+			return &rule, i
 		}
 	}
-	return len(ing.Rules) - 1
+	i := len(ing.Rules) - 1
+	return &ing.Rules[i], i
 }
 
 func matchHost(ruleHost, reqHost string) bool {
@@ -83,7 +57,54 @@ func matchHost(ruleHost, reqHost string) bool {
 
 // Ingress maps eyeball requests to origins.
 type Ingress struct {
-	Rules []Rule
+	Rules    []Rule
+	defaults OriginRequestConfig
+}
+
+// NewSingleOrigin constructs an Ingress set with only one rule, constructed from
+// legacy CLI parameters like --url or --no-chunked-encoding.
+func NewSingleOrigin(c *cli.Context, allowURLFromArgs bool) (Ingress, error) {
+
+	service, err := parseSingleOriginService(c, allowURLFromArgs)
+	if err != nil {
+		return Ingress{}, err
+	}
+
+	// Construct an Ingress with the single rule.
+	defaults := originRequestFromSingeRule(c)
+	ing := Ingress{
+		Rules: []Rule{
+			{
+				Service: service,
+				Config:  setConfig(defaults, config.OriginRequestConfig{}),
+			},
+		},
+		defaults: defaults,
+	}
+	return ing, err
+}
+
+// Get a single origin service from the CLI/config.
+func parseSingleOriginService(c *cli.Context, allowURLFromArgs bool) (OriginService, error) {
+	if c.IsSet("hello-world") {
+		return new(helloWorld), nil
+	}
+	if c.IsSet("url") || c.IsSet(config.BastionFlag) {
+		originURL, err := config.ValidateUrl(c, allowURLFromArgs)
+		if err != nil {
+			return nil, errors.Wrap(err, "Error validating origin URL")
+		}
+		return &localService{URL: originURL, RootURL: originURL}, nil
+	}
+	if c.IsSet("unix-socket") {
+		path, err := config.ValidateUnixSocket(c)
+		if err != nil {
+			return nil, errors.Wrap(err, "Error validating --unix-socket")
+		}
+		return &unixSocketPath{path: path}, nil
+	}
+	u, err := url.Parse("http://localhost:8080")
+	return &localService{URL: u, RootURL: u}, err
 }
 
 // IsEmpty checks if there are any ingress rules.
@@ -91,40 +112,76 @@ func (ing Ingress) IsEmpty() bool {
 	return len(ing.Rules) == 0
 }
 
-func validate(ingress []config.UnvalidatedIngressRule) (Ingress, error) {
+// StartOrigins will start any origin services managed by cloudflared, e.g. proxy servers or Hello World.
+func (ing Ingress) StartOrigins(
+	wg *sync.WaitGroup,
+	log *zerolog.Logger,
+	shutdownC <-chan struct{},
+	errC chan error,
+) error {
+	for _, rule := range ing.Rules {
+		if err := rule.Service.start(wg, log, shutdownC, errC, rule.Config); err != nil {
+			return errors.Wrapf(err, "Error starting local service %s", rule.Service)
+		}
+	}
+	return nil
+}
+
+// CatchAll returns the catch-all rule (i.e. the last rule)
+func (ing Ingress) CatchAll() *Rule {
+	return &ing.Rules[len(ing.Rules)-1]
+}
+
+func validate(ingress []config.UnvalidatedIngressRule, defaults OriginRequestConfig) (Ingress, error) {
 	rules := make([]Rule, len(ingress))
 	for i, r := range ingress {
-		service, err := url.Parse(r.Service)
-		if err != nil {
+		cfg := setConfig(defaults, r.OriginRequest)
+		var service OriginService
+
+		if prefix := "unix:"; strings.HasPrefix(r.Service, prefix) {
+			// No validation necessary for unix socket filepath services
+			path := strings.TrimPrefix(r.Service, prefix)
+			service = &unixSocketPath{path: path}
+		} else if prefix := "http_status:"; strings.HasPrefix(r.Service, prefix) {
+			status, err := strconv.Atoi(strings.TrimPrefix(r.Service, prefix))
+			if err != nil {
+				return Ingress{}, errors.Wrap(err, "invalid HTTP status")
+			}
+			srv := newStatusCode(status)
+			service = &srv
+		} else if r.Service == "hello_world" || r.Service == "hello-world" || r.Service == "helloworld" {
+			service = new(helloWorld)
+		} else if r.Service == "bastion" || cfg.BastionMode {
+			// Bastion mode will always start a Websocket proxy server, which will
+			// overwrite the localService.URL field when `start` is called. So,
+			// leave the URL field empty for now.
+			cfg.BastionMode = true
+			service = new(localService)
+		} else {
+			// Validate URL services
+			u, err := url.Parse(r.Service)
+			if err != nil {
+				return Ingress{}, err
+			}
+
+			if u.Scheme == "" || u.Hostname() == "" {
+				return Ingress{}, fmt.Errorf("%s is an invalid address, please make sure it has a scheme and a hostname", r.Service)
+			}
+
+			if u.Path != "" {
+				return Ingress{}, fmt.Errorf("%s is an invalid address, ingress rules don't support proxying to a different path on the origin service. The path will be the same as the eyeball request's path", r.Service)
+			}
+			serviceURL := localService{URL: u}
+			service = &serviceURL
+		}
+
+		if err := validateHostname(r, i, len(ingress)); err != nil {
 			return Ingress{}, err
 		}
-		if service.Scheme == "" || service.Hostname() == "" {
-			return Ingress{}, fmt.Errorf("The service %s must have a scheme and a hostname", r.Service)
-		}
-
-		if service.Path != "" {
-			return Ingress{}, fmt.Errorf("%s is an invalid address, ingress rules don't support proxying to a different path on the origin service. The path will be the same as the eyeball request's path.", r.Service)
-		}
-
-		// Ensure that there are no wildcards anywhere except the first character
-		// of the hostname.
-		if strings.LastIndex(r.Hostname, "*") > 0 {
-			return Ingress{}, errBadWildcard
-		}
-
-		// The last rule should catch all hostnames.
-		isCatchAllRule := (r.Hostname == "" || r.Hostname == "*") && r.Path == ""
-		isLastRule := i == len(ingress)-1
-		if isLastRule && !isCatchAllRule {
-			return Ingress{}, errLastRuleNotCatchAll
-		}
-		// ONLY the last rule should catch all hostnames.
-		if !isLastRule && isCatchAllRule {
-			return Ingress{}, errRuleShouldNotBeCatchAll{i: i, hostname: r.Hostname}
-		}
 
 		var pathRegex *regexp.Regexp
 		if r.Path != "" {
+			var err error
 			pathRegex, err = regexp.Compile(r.Path)
 			if err != nil {
 				return Ingress{}, errors.Wrapf(err, "Rule #%d has an invalid regex", i+1)
@@ -135,25 +192,52 @@ func validate(ingress []config.UnvalidatedIngressRule) (Ingress, error) {
 			Hostname: r.Hostname,
 			Service:  service,
 			Path:     pathRegex,
+			Config:   cfg,
 		}
 	}
-	return Ingress{Rules: rules}, nil
+	return Ingress{Rules: rules, defaults: defaults}, nil
+}
+
+func validateHostname(r config.UnvalidatedIngressRule, ruleIndex, totalRules int) error {
+	// Ensure that the hostname doesn't contain port
+	_, _, err := net.SplitHostPort(r.Hostname)
+	if err == nil {
+		return errHostnameContainsPort
+	}
+	// Ensure that there are no wildcards anywhere except the first character
+	// of the hostname.
+	if strings.LastIndex(r.Hostname, "*") > 0 {
+		return errBadWildcard
+	}
+
+	// The last rule should catch all hostnames.
+	isCatchAllRule := (r.Hostname == "" || r.Hostname == "*") && r.Path == ""
+	isLastRule := ruleIndex == totalRules-1
+	if isLastRule && !isCatchAllRule {
+		return errLastRuleNotCatchAll
+	}
+	// ONLY the last rule should catch all hostnames.
+	if !isLastRule && isCatchAllRule {
+		return errRuleShouldNotBeCatchAll{index: ruleIndex, hostname: r.Hostname}
+	}
+	return nil
 }
 
 type errRuleShouldNotBeCatchAll struct {
-	i        int
+	index    int
 	hostname string
 }
 
 func (e errRuleShouldNotBeCatchAll) Error() string {
 	return fmt.Sprintf("Rule #%d is matching the hostname '%s', but "+
 		"this will match every hostname, meaning the rules which follow it "+
-		"will never be triggered.", e.i+1, e.hostname)
+		"will never be triggered.", e.index+1, e.hostname)
 }
 
+// ParseIngress parses ingress rules, but does not send HTTP requests to the origins.
 func ParseIngress(conf *config.Configuration) (Ingress, error) {
 	if len(conf.Ingress) == 0 {
 		return Ingress{}, ErrNoIngressRules
 	}
-	return validate(conf.Ingress)
+	return validate(conf.Ingress, originRequestFromYAML(conf.OriginRequest))
 }

ingress/ingress_test.go

@@ -1,27 +1,47 @@
 package ingress
 
 import (
+	"flag"
+	"fmt"
 	"net/url"
 	"regexp"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"github.com/urfave/cli/v2"
 	"gopkg.in/yaml.v2"
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/tlsconfig"
 )
 
+func TestParseUnixSocket(t *testing.T) {
+	rawYAML := `
+ingress:
+- service: unix:/tmp/echo.sock
+`
+	ing, err := ParseIngress(MustReadIngress(rawYAML))
+	require.NoError(t, err)
+	_, ok := ing.Rules[0].Service.(*unixSocketPath)
+	require.True(t, ok)
+}
+
 func Test_parseIngress(t *testing.T) {
 	localhost8000 := MustParseURL(t, "https://localhost:8000")
 	localhost8001 := MustParseURL(t, "https://localhost:8001")
+	fourOhFour := newStatusCode(404)
+	defaultConfig := setConfig(originRequestFromYAML(config.OriginRequestConfig{}), config.OriginRequestConfig{})
+	require.Equal(t, defaultKeepAliveConnections, defaultConfig.KeepAliveConnections)
+	tr := true
 	type args struct {
 		rawYAML string
 	}
 	tests := []struct {
 		name    string
 		args    args
-		want    Ingress
+		want    []Rule
 		wantErr bool
 	}{
 		{
@@ -38,16 +58,18 @@ ingress:
  - hostname: "*"
    service: https://localhost:8001
 `},
-			want: Ingress{Rules: []Rule{
+			want: []Rule{
 				{
 					Hostname: "tunnel1.example.com",
-					Service:  localhost8000,
+					Service:  &localService{URL: localhost8000},
+					Config:   defaultConfig,
 				},
 				{
 					Hostname: "*",
-					Service:  localhost8001,
+					Service:  &localService{URL: localhost8001},
+					Config:   defaultConfig,
 				},
-			}},
+			},
 		},
 		{
 			name: "Extra keys",
@@ -57,12 +79,13 @@ ingress:
    service: https://localhost:8000
 extraKey: extraValue
 `},
-			want: Ingress{Rules: []Rule{
+			want: []Rule{
 				{
 					Hostname: "*",
-					Service:  localhost8000,
+					Service:  &localService{URL: localhost8000},
+					Config:   defaultConfig,
 				},
-			}},
+			},
 		},
 		{
 			name: "Hostname can be omitted",
@@ -70,11 +93,12 @@ extraKey: extraValue
 ingress:
  - service: https://localhost:8000
 `},
-			want: Ingress{Rules: []Rule{
+			want: []Rule{
 				{
-					Service: localhost8000,
+					Service: &localService{URL: localhost8000},
+					Config:  defaultConfig,
 				},
-			}},
+			},
 		},
 		{
 			name: "Invalid service",
@@ -146,6 +170,94 @@ ingress:
 			args: args{rawYAML: `
 ingress:
  - service: https://localhost:8000/static/
+`},
+			wantErr: true,
+		},
+		{
+			name: "Invalid HTTP status",
+			args: args{rawYAML: `
+ingress:
+ - service: http_status:asdf
+`},
+			wantErr: true,
+		},
+		{
+			name: "Valid HTTP status",
+			args: args{rawYAML: `
+ingress:
+ - service: http_status:404
+`},
+			want: []Rule{
+				{
+					Hostname: "",
+					Service:  &fourOhFour,
+					Config:   defaultConfig,
+				},
+			},
+		},
+		{
+			name: "Valid hello world service",
+			args: args{rawYAML: `
+ingress:
+ - service: hello_world
+`},
+			want: []Rule{
+				{
+					Hostname: "",
+					Service:  new(helloWorld),
+					Config:   defaultConfig,
+				},
+			},
+		},
+		{
+			name: "URL isn't necessary if using bastion",
+			args: args{rawYAML: `
+ingress:
+- hostname: bastion.foo.com
+  originRequest:
+    bastionMode: true
+- service: http_status:404
+`},
+			want: []Rule{
+				{
+					Hostname: "bastion.foo.com",
+					Service:  &localService{},
+					Config:   setConfig(originRequestFromYAML(config.OriginRequestConfig{}), config.OriginRequestConfig{BastionMode: &tr}),
+				},
+				{
+					Service: &fourOhFour,
+					Config:  defaultConfig,
+				},
+			},
+		},
+		{
+			name: "Bastion service",
+			args: args{rawYAML: `
+ingress:
+- hostname: bastion.foo.com
+  service: bastion
+- service: http_status:404
+`},
+			want: []Rule{
+				{
+					Hostname: "bastion.foo.com",
+					Service:  &localService{},
+					Config:   setConfig(originRequestFromYAML(config.OriginRequestConfig{}), config.OriginRequestConfig{BastionMode: &tr}),
+				},
+				{
+					Service: &fourOhFour,
+					Config:  defaultConfig,
+				},
+			},
+		},
+		{
+			name: "Hostname contains port",
+			args: args{rawYAML: `
+ingress:
+ - hostname: "test.example.com:443"
+   service: https://localhost:8000
+ - hostname: "*"
+   service: https://localhost:8001
 `},
 			wantErr: true,
 		},
@@ -157,129 +269,158 @@ ingress:
 				t.Errorf("ParseIngress() error = %v, wantErr %v", err, tt.wantErr)
 				return
 			}
-			assert.Equal(t, tt.want, got)
+			require.Equal(t, tt.want, got.Rules)
 		})
 	}
 }
 
+func TestSingleOriginSetsConfig(t *testing.T) {
+	flagSet := flag.NewFlagSet(t.Name(), flag.PanicOnError)
+	flagSet.Bool("hello-world", true, "")
+	flagSet.Duration(ProxyConnectTimeoutFlag, time.Second, "")
+	flagSet.Duration(ProxyTLSTimeoutFlag, time.Second, "")
+	flagSet.Duration(ProxyTCPKeepAlive, time.Second, "")
+	flagSet.Bool(ProxyNoHappyEyeballsFlag, true, "")
+	flagSet.Int(ProxyKeepAliveConnectionsFlag, 10, "")
+	flagSet.Duration(ProxyKeepAliveTimeoutFlag, time.Second, "")
+	flagSet.String(HTTPHostHeaderFlag, "example.com:8080", "")
+	flagSet.String(OriginServerNameFlag, "example.com", "")
+	flagSet.String(tlsconfig.OriginCAPoolFlag, "/etc/certs/ca.pem", "")
+	flagSet.Bool(NoTLSVerifyFlag, true, "")
+	flagSet.Bool(NoChunkedEncodingFlag, true, "")
+	flagSet.Bool(config.BastionFlag, true, "")
+	flagSet.String(ProxyAddressFlag, "localhost:8080", "")
+	flagSet.Uint(ProxyPortFlag, 8080, "")
+	flagSet.Bool(Socks5Flag, true, "")
+
+	cliCtx := cli.NewContext(cli.NewApp(), flagSet, nil)
+	err := cliCtx.Set("hello-world", "true")
+	require.NoError(t, err)
+	err = cliCtx.Set(ProxyConnectTimeoutFlag, "1s")
+	require.NoError(t, err)
+	err = cliCtx.Set(ProxyTLSTimeoutFlag, "1s")
+	require.NoError(t, err)
+	err = cliCtx.Set(ProxyTCPKeepAlive, "1s")
+	require.NoError(t, err)
+	err = cliCtx.Set(ProxyNoHappyEyeballsFlag, "true")
+	require.NoError(t, err)
+	err = cliCtx.Set(ProxyKeepAliveConnectionsFlag, "10")
+	require.NoError(t, err)
+	err = cliCtx.Set(ProxyKeepAliveTimeoutFlag, "1s")
+	require.NoError(t, err)
+	err = cliCtx.Set(HTTPHostHeaderFlag, "example.com:8080")
+	require.NoError(t, err)
+	err = cliCtx.Set(OriginServerNameFlag, "example.com")
+	require.NoError(t, err)
+	err = cliCtx.Set(tlsconfig.OriginCAPoolFlag, "/etc/certs/ca.pem")
+	require.NoError(t, err)
+	err = cliCtx.Set(NoTLSVerifyFlag, "true")
+	require.NoError(t, err)
+	err = cliCtx.Set(NoChunkedEncodingFlag, "true")
+	require.NoError(t, err)
+	err = cliCtx.Set(config.BastionFlag, "true")
+	require.NoError(t, err)
+	err = cliCtx.Set(ProxyAddressFlag, "localhost:8080")
+	require.NoError(t, err)
+	err = cliCtx.Set(ProxyPortFlag, "8080")
+	require.NoError(t, err)
+	err = cliCtx.Set(Socks5Flag, "true")
+	require.NoError(t, err)
+
+	allowURLFromArgs := false
+	require.NoError(t, err)
+	ingress, err := NewSingleOrigin(cliCtx, allowURLFromArgs)
+	require.NoError(t, err)
+
+	assert.Equal(t, time.Second, ingress.Rules[0].Config.ConnectTimeout)
+	assert.Equal(t, time.Second, ingress.Rules[0].Config.TLSTimeout)
+	assert.Equal(t, time.Second, ingress.Rules[0].Config.TCPKeepAlive)
+	assert.True(t, ingress.Rules[0].Config.NoHappyEyeballs)
+	assert.Equal(t, 10, ingress.Rules[0].Config.KeepAliveConnections)
+	assert.Equal(t, time.Second, ingress.Rules[0].Config.KeepAliveTimeout)
+	assert.Equal(t, "example.com:8080", ingress.Rules[0].Config.HTTPHostHeader)
+	assert.Equal(t, "example.com", ingress.Rules[0].Config.OriginServerName)
+	assert.Equal(t, "/etc/certs/ca.pem", ingress.Rules[0].Config.CAPool)
+	assert.True(t, ingress.Rules[0].Config.NoTLSVerify)
+	assert.True(t, ingress.Rules[0].Config.DisableChunkedEncoding)
+	assert.True(t, ingress.Rules[0].Config.BastionMode)
+	assert.Equal(t, "localhost:8080", ingress.Rules[0].Config.ProxyAddress)
+	assert.Equal(t, uint(8080), ingress.Rules[0].Config.ProxyPort)
+	assert.Equal(t, socksProxy, ingress.Rules[0].Config.ProxyType)
+}
+
+func TestFindMatchingRule(t *testing.T) {
+	ingress := Ingress{
+		Rules: []Rule{
+			{
+				Hostname: "tunnel-a.example.com",
+				Path:     nil,
+			},
+			{
+				Hostname: "tunnel-b.example.com",
+				Path:     mustParsePath(t, "/health"),
+			},
+			{
+				Hostname: "*",
+			},
+		},
+	}
+
+	tests := []struct {
+		host          string
+		path          string
+		wantRuleIndex int
+	}{
+		{
+			host:          "tunnel-a.example.com",
+			path:          "/",
+			wantRuleIndex: 0,
+		},
+		{
+			host:          "tunnel-a.example.com",
+			path:          "/pages/about",
+			wantRuleIndex: 0,
+		},
+		{
+			host:          "tunnel-a.example.com:443",
+			path:          "/pages/about",
+			wantRuleIndex: 0,
+		},
+		{
+			host:          "tunnel-b.example.com",
+			path:          "/health",
+			wantRuleIndex: 1,
+		},
+		{
+			host:          "tunnel-b.example.com",
+			path:          "/index.html",
+			wantRuleIndex: 2,
+		},
+		{
+			host:          "tunnel-c.example.com",
+			path:          "/",
+			wantRuleIndex: 2,
+		},
+	}
+
+	for i, test := range tests {
+		_, ruleIndex := ingress.FindMatchingRule(test.host, test.path)
+		assert.Equal(t, test.wantRuleIndex, ruleIndex, fmt.Sprintf("Expect host=%s, path=%s to match rule %d, got %d", test.host, test.path, test.wantRuleIndex, i))
+	}
+}
+
+func mustParsePath(t *testing.T, path string) *regexp.Regexp {
+	regexp, err := regexp.Compile(path)
+	assert.NoError(t, err)
+	return regexp
+}
+
 func MustParseURL(t *testing.T, rawURL string) *url.URL {
 	u, err := url.Parse(rawURL)
 	require.NoError(t, err)
 	return u
 }
 
-func Test_rule_matches(t *testing.T) {
-	type fields struct {
-		Hostname string
-		Path     *regexp.Regexp
-		Service  *url.URL
-	}
-	type args struct {
-		requestURL *url.URL
-	}
-	tests := []struct {
-		name   string
-		fields fields
-		args   args
-		want   bool
-	}{
-		{
-			name: "Just hostname, pass",
-			fields: fields{
-				Hostname: "example.com",
-			},
-			args: args{
-				requestURL: MustParseURL(t, "https://example.com"),
-			},
-			want: true,
-		},
-		{
-			name: "Entire hostname is wildcard, should match everything",
-			fields: fields{
-				Hostname: "*",
-			},
-			args: args{
-				requestURL: MustParseURL(t, "https://example.com"),
-			},
-			want: true,
-		},
-		{
-			name: "Just hostname, fail",
-			fields: fields{
-				Hostname: "example.com",
-			},
-			args: args{
-				requestURL: MustParseURL(t, "https://foo.bar"),
-			},
-			want: false,
-		},
-		{
-			name: "Just wildcard hostname, pass",
-			fields: fields{
-				Hostname: "*.example.com",
-			},
-			args: args{
-				requestURL: MustParseURL(t, "https://adam.example.com"),
-			},
-			want: true,
-		},
-		{
-			name: "Just wildcard hostname, fail",
-			fields: fields{
-				Hostname: "*.example.com",
-			},
-			args: args{
-				requestURL: MustParseURL(t, "https://tunnel.com"),
-			},
-			want: false,
-		},
-		{
-			name: "Just wildcard outside of subdomain in hostname, fail",
-			fields: fields{
-				Hostname: "*example.com",
-			},
-			args: args{
-				requestURL: MustParseURL(t, "https://www.example.com"),
-			},
-			want: false,
-		},
-		{
-			name: "Wildcard over multiple subdomains",
-			fields: fields{
-				Hostname: "*.example.com",
-			},
-			args: args{
-				requestURL: MustParseURL(t, "https://adam.chalmers.example.com"),
-			},
-			want: true,
-		},
-		{
-			name: "Hostname and path",
-			fields: fields{
-				Hostname: "*.example.com",
-				Path:     regexp.MustCompile("/static/.*\\.html"),
-			},
-			args: args{
-				requestURL: MustParseURL(t, "https://www.example.com/static/index.html"),
-			},
-			want: true,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			r := Rule{
-				Hostname: tt.fields.Hostname,
-				Path:     tt.fields.Path,
-				Service:  tt.fields.Service,
-			}
-			u := tt.args.requestURL
-			if got := r.Matches(u.Hostname(), u.Path); got != tt.want {
-				t.Errorf("rule.matches() = %v, want %v", got, tt.want)
-			}
-		})
-	}
-}
-
 func BenchmarkFindMatch(b *testing.B) {
 	rulesYAML := `
 ingress:

ingress/origin_request_config.go

@@ -0,0 +1,332 @@
+package ingress
+
+import (
+	"time"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/tlsconfig"
+	"github.com/urfave/cli/v2"
+)
+
+const (
+	defaultConnectTimeout       = 30 * time.Second
+	defaultTLSTimeout           = 10 * time.Second
+	defaultTCPKeepAlive         = 30 * time.Second
+	defaultKeepAliveConnections = 100
+	defaultKeepAliveTimeout     = 90 * time.Second
+	defaultProxyAddress         = "127.0.0.1"
+
+	SSHServerFlag                 = "ssh-server"
+	Socks5Flag                    = "socks5"
+	ProxyConnectTimeoutFlag       = "proxy-connect-timeout"
+	ProxyTLSTimeoutFlag           = "proxy-tls-timeout"
+	ProxyTCPKeepAlive             = "proxy-tcp-keepalive"
+	ProxyNoHappyEyeballsFlag      = "proxy-no-happy-eyeballs"
+	ProxyKeepAliveConnectionsFlag = "proxy-keepalive-connections"
+	ProxyKeepAliveTimeoutFlag     = "proxy-keepalive-timeout"
+	HTTPHostHeaderFlag            = "http-host-header"
+	OriginServerNameFlag          = "origin-server-name"
+	NoTLSVerifyFlag               = "no-tls-verify"
+	NoChunkedEncodingFlag         = "no-chunked-encoding"
+	ProxyAddressFlag              = "proxy-address"
+	ProxyPortFlag                 = "proxy-port"
+)
+
+const (
+	socksProxy = "socks"
+)
+
+func originRequestFromSingeRule(c *cli.Context) OriginRequestConfig {
+	var connectTimeout time.Duration = defaultConnectTimeout
+	var tlsTimeout time.Duration = defaultTLSTimeout
+	var tcpKeepAlive time.Duration = defaultTCPKeepAlive
+	var noHappyEyeballs bool
+	var keepAliveConnections int = defaultKeepAliveConnections
+	var keepAliveTimeout time.Duration = defaultKeepAliveTimeout
+	var httpHostHeader string
+	var originServerName string
+	var caPool string
+	var noTLSVerify bool
+	var disableChunkedEncoding bool
+	var bastionMode bool
+	var proxyAddress = defaultProxyAddress
+	var proxyPort uint
+	var proxyType string
+	if flag := ProxyConnectTimeoutFlag; c.IsSet(flag) {
+		connectTimeout = c.Duration(flag)
+	}
+	if flag := ProxyTLSTimeoutFlag; c.IsSet(flag) {
+		tlsTimeout = c.Duration(flag)
+	}
+	if flag := ProxyTCPKeepAlive; c.IsSet(flag) {
+		tcpKeepAlive = c.Duration(flag)
+	}
+	if flag := ProxyNoHappyEyeballsFlag; c.IsSet(flag) {
+		noHappyEyeballs = c.Bool(flag)
+	}
+	if flag := ProxyKeepAliveConnectionsFlag; c.IsSet(flag) {
+		keepAliveConnections = c.Int(flag)
+	}
+	if flag := ProxyKeepAliveTimeoutFlag; c.IsSet(flag) {
+		keepAliveTimeout = c.Duration(flag)
+	}
+	if flag := HTTPHostHeaderFlag; c.IsSet(flag) {
+		httpHostHeader = c.String(flag)
+	}
+	if flag := OriginServerNameFlag; c.IsSet(flag) {
+		originServerName = c.String(flag)
+	}
+	if flag := tlsconfig.OriginCAPoolFlag; c.IsSet(flag) {
+		caPool = c.String(flag)
+	}
+	if flag := NoTLSVerifyFlag; c.IsSet(flag) {
+		noTLSVerify = c.Bool(flag)
+	}
+	if flag := NoChunkedEncodingFlag; c.IsSet(flag) {
+		disableChunkedEncoding = c.Bool(flag)
+	}
+	if flag := config.BastionFlag; c.IsSet(flag) {
+		bastionMode = c.Bool(flag)
+	}
+	if flag := ProxyAddressFlag; c.IsSet(flag) {
+		proxyAddress = c.String(flag)
+	}
+	if flag := ProxyPortFlag; c.IsSet(flag) {
+		// Note TUN-3758 , we use Int because UInt is not supported with altsrc
+		proxyPort = uint(c.Int(flag))
+	}
+	if c.IsSet(Socks5Flag) {
+		proxyType = socksProxy
+	}
+	return OriginRequestConfig{
+		ConnectTimeout:         connectTimeout,
+		TLSTimeout:             tlsTimeout,
+		TCPKeepAlive:           tcpKeepAlive,
+		NoHappyEyeballs:        noHappyEyeballs,
+		KeepAliveConnections:   keepAliveConnections,
+		KeepAliveTimeout:       keepAliveTimeout,
+		HTTPHostHeader:         httpHostHeader,
+		OriginServerName:       originServerName,
+		CAPool:                 caPool,
+		NoTLSVerify:            noTLSVerify,
+		DisableChunkedEncoding: disableChunkedEncoding,
+		BastionMode:            bastionMode,
+		ProxyAddress:           proxyAddress,
+		ProxyPort:              proxyPort,
+		ProxyType:              proxyType,
+	}
+}
+
+func originRequestFromYAML(y config.OriginRequestConfig) OriginRequestConfig {
+	out := OriginRequestConfig{
+		ConnectTimeout:       defaultConnectTimeout,
+		TLSTimeout:           defaultTLSTimeout,
+		TCPKeepAlive:         defaultTCPKeepAlive,
+		KeepAliveConnections: defaultKeepAliveConnections,
+		KeepAliveTimeout:     defaultKeepAliveTimeout,
+		ProxyAddress:         defaultProxyAddress,
+	}
+	if y.ConnectTimeout != nil {
+		out.ConnectTimeout = *y.ConnectTimeout
+	}
+	if y.TLSTimeout != nil {
+		out.TLSTimeout = *y.TLSTimeout
+	}
+	if y.TCPKeepAlive != nil {
+		out.TCPKeepAlive = *y.TCPKeepAlive
+	}
+	if y.NoHappyEyeballs != nil {
+		out.NoHappyEyeballs = *y.NoHappyEyeballs
+	}
+	if y.KeepAliveConnections != nil {
+		out.KeepAliveConnections = *y.KeepAliveConnections
+	}
+	if y.KeepAliveTimeout != nil {
+		out.KeepAliveTimeout = *y.KeepAliveTimeout
+	}
+	if y.HTTPHostHeader != nil {
+		out.HTTPHostHeader = *y.HTTPHostHeader
+	}
+	if y.OriginServerName != nil {
+		out.OriginServerName = *y.OriginServerName
+	}
+	if y.CAPool != nil {
+		out.CAPool = *y.CAPool
+	}
+	if y.NoTLSVerify != nil {
+		out.NoTLSVerify = *y.NoTLSVerify
+	}
+	if y.DisableChunkedEncoding != nil {
+		out.DisableChunkedEncoding = *y.DisableChunkedEncoding
+	}
+	if y.BastionMode != nil {
+		out.BastionMode = *y.BastionMode
+	}
+	if y.ProxyAddress != nil {
+		out.ProxyAddress = *y.ProxyAddress
+	}
+	if y.ProxyPort != nil {
+		out.ProxyPort = *y.ProxyPort
+	}
+	if y.ProxyType != nil {
+		out.ProxyType = *y.ProxyType
+	}
+	return out
+}
+
+// OriginRequestConfig configures how Cloudflared sends requests to origin
+// services.
+// Note: To specify a time.Duration in go-yaml, use e.g. "3s" or "24h".
+type OriginRequestConfig struct {
+	// HTTP proxy timeout for establishing a new connection
+	ConnectTimeout time.Duration `yaml:"connectTimeout"`
+	// HTTP proxy timeout for completing a TLS handshake
+	TLSTimeout time.Duration `yaml:"tlsTimeout"`
+	// HTTP proxy TCP keepalive duration
+	TCPKeepAlive time.Duration `yaml:"tcpKeepAlive"`
+	// HTTP proxy should disable "happy eyeballs" for IPv4/v6 fallback
+	NoHappyEyeballs bool `yaml:"noHappyEyeballs"`
+	// HTTP proxy maximum keepalive connection pool size
+	KeepAliveConnections int `yaml:"keepAliveConnections"`
+	// HTTP proxy timeout for closing an idle connection
+	KeepAliveTimeout time.Duration `yaml:"keepAliveTimeout"`
+	// Sets the HTTP Host header for the local webserver.
+	HTTPHostHeader string `yaml:"httpHostHeader"`
+	// Hostname on the origin server certificate.
+	OriginServerName string `yaml:"originServerName"`
+	// Path to the CA for the certificate of your origin.
+	// This option should be used only if your certificate is not signed by Cloudflare.
+	CAPool string `yaml:"caPool"`
+	// Disables TLS verification of the certificate presented by your origin.
+	// Will allow any certificate from the origin to be accepted.
+	// Note: The connection from your machine to Cloudflare's Edge is still encrypted.
+	NoTLSVerify bool `yaml:"noTLSVerify"`
+	// Disables chunked transfer encoding.
+	// Useful if you are running a WSGI server.
+	DisableChunkedEncoding bool `yaml:"disableChunkedEncoding"`
+	// Runs as jump host
+	BastionMode bool `yaml:"bastionMode"`
+	// Listen address for the proxy.
+	ProxyAddress string `yaml:"proxyAddress"`
+	// Listen port for the proxy.
+	ProxyPort uint `yaml:"proxyPort"`
+	// What sort of proxy should be started
+	ProxyType string `yaml:"proxyType"`
+}
+
+func (defaults *OriginRequestConfig) setConnectTimeout(overrides config.OriginRequestConfig) {
+	if val := overrides.ConnectTimeout; val != nil {
+		defaults.ConnectTimeout = *val
+	}
+}
+
+func (defaults *OriginRequestConfig) setTLSTimeout(overrides config.OriginRequestConfig) {
+	if val := overrides.TLSTimeout; val != nil {
+		defaults.TLSTimeout = *val
+	}
+}
+
+func (defaults *OriginRequestConfig) setNoHappyEyeballs(overrides config.OriginRequestConfig) {
+	if val := overrides.NoHappyEyeballs; val != nil {
+		defaults.NoHappyEyeballs = *val
+	}
+}
+
+func (defaults *OriginRequestConfig) setKeepAliveConnections(overrides config.OriginRequestConfig) {
+	if val := overrides.KeepAliveConnections; val != nil {
+		defaults.KeepAliveConnections = *val
+	}
+}
+
+func (defaults *OriginRequestConfig) setKeepAliveTimeout(overrides config.OriginRequestConfig) {
+	if val := overrides.KeepAliveTimeout; val != nil {
+		defaults.KeepAliveTimeout = *val
+	}
+}
+
+func (defaults *OriginRequestConfig) setTCPKeepAlive(overrides config.OriginRequestConfig) {
+	if val := overrides.TCPKeepAlive; val != nil {
+		defaults.TCPKeepAlive = *val
+	}
+}
+
+func (defaults *OriginRequestConfig) setHTTPHostHeader(overrides config.OriginRequestConfig) {
+	if val := overrides.HTTPHostHeader; val != nil {
+		defaults.HTTPHostHeader = *val
+	}
+}
+
+func (defaults *OriginRequestConfig) setOriginServerName(overrides config.OriginRequestConfig) {
+	if val := overrides.OriginServerName; val != nil {
+		defaults.OriginServerName = *val
+	}
+}
+
+func (defaults *OriginRequestConfig) setCAPool(overrides config.OriginRequestConfig) {
+	if val := overrides.CAPool; val != nil {
+		defaults.CAPool = *val
+	}
+}
+
+func (defaults *OriginRequestConfig) setNoTLSVerify(overrides config.OriginRequestConfig) {
+	if val := overrides.NoTLSVerify; val != nil {
+		defaults.NoTLSVerify = *val
+	}
+}
+
+func (defaults *OriginRequestConfig) setDisableChunkedEncoding(overrides config.OriginRequestConfig) {
+	if val := overrides.DisableChunkedEncoding; val != nil {
+		defaults.DisableChunkedEncoding = *val
+	}
+}
+
+func (defaults *OriginRequestConfig) setBastionMode(overrides config.OriginRequestConfig) {
+	if val := overrides.BastionMode; val != nil {
+		defaults.BastionMode = *val
+	}
+}
+
+func (defaults *OriginRequestConfig) setProxyPort(overrides config.OriginRequestConfig) {
+	if val := overrides.ProxyPort; val != nil {
+		defaults.ProxyPort = *val
+	}
+}
+
+func (defaults *OriginRequestConfig) setProxyAddress(overrides config.OriginRequestConfig) {
+	if val := overrides.ProxyAddress; val != nil {
+		defaults.ProxyAddress = *val
+	}
+}
+
+func (defaults *OriginRequestConfig) setProxyType(overrides config.OriginRequestConfig) {
+	if val := overrides.ProxyType; val != nil {
+		defaults.ProxyType = *val
+	}
+}
+
+// SetConfig gets config for the requests that cloudflared sends to origins.
+// Each field has a setter method which sets a value for the field by trying to find:
+//   1. The user config for this rule
+//   2. The user config for the overall ingress config
+//   3. Defaults chosen by the cloudflared team
+//   4. Golang zero values for that type
+// If an earlier option isn't set, it will try the next option down.
+func setConfig(defaults OriginRequestConfig, overrides config.OriginRequestConfig) OriginRequestConfig {
+	cfg := defaults
+	cfg.setConnectTimeout(overrides)
+	cfg.setTLSTimeout(overrides)
+	cfg.setNoHappyEyeballs(overrides)
+	cfg.setKeepAliveConnections(overrides)
+	cfg.setKeepAliveTimeout(overrides)
+	cfg.setTCPKeepAlive(overrides)
+	cfg.setHTTPHostHeader(overrides)
+	cfg.setOriginServerName(overrides)
+	cfg.setCAPool(overrides)
+	cfg.setNoTLSVerify(overrides)
+	cfg.setDisableChunkedEncoding(overrides)
+	cfg.setBastionMode(overrides)
+	cfg.setProxyPort(overrides)
+	cfg.setProxyAddress(overrides)
+	cfg.setProxyType(overrides)
+	return cfg
+}

ingress/origin_request_config_test.go

@@ -0,0 +1,202 @@
+package ingress
+
+import (
+	"flag"
+	"testing"
+	"time"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/stretchr/testify/require"
+	"github.com/urfave/cli/v2"
+	"gopkg.in/yaml.v2"
+)
+
+// Ensure that the nullable config from `config` package and the
+// non-nullable config from `ingress` package have the same number of
+// fields.
+// This test ensures that programmers didn't add a new field to
+// one struct and forget to add it to the other ;)
+func TestCorrespondingFields(t *testing.T) {
+	require.Equal(
+		t,
+		CountFields(t, config.OriginRequestConfig{}),
+		CountFields(t, OriginRequestConfig{}),
+	)
+}
+
+func CountFields(t *testing.T, val interface{}) int {
+	b, err := yaml.Marshal(val)
+	require.NoError(t, err)
+	m := make(map[string]interface{}, 0)
+	err = yaml.Unmarshal(b, &m)
+	require.NoError(t, err)
+	return len(m)
+}
+
+func TestOriginRequestConfigOverrides(t *testing.T) {
+	rulesYAML := `
+originRequest:
+  connectTimeout: 1m
+  tlsTimeout: 1s
+  noHappyEyeballs: true
+  tcpKeepAlive: 1s
+  keepAliveConnections: 1
+  keepAliveTimeout: 1s
+  httpHostHeader: abc
+  originServerName: a1
+  caPool: /tmp/path0
+  noTLSVerify: true
+  disableChunkedEncoding: true
+  bastionMode: True
+  proxyAddress: 127.1.2.3
+  proxyPort: 100
+  proxyType: socks5
+ingress:
+- hostname: tun.example.com
+  service: https://localhost:8000
+- hostname: "*"
+  service: https://localhost:8001
+  originRequest:
+    connectTimeout: 2m
+    tlsTimeout: 2s
+    noHappyEyeballs: false
+    tcpKeepAlive: 2s
+    keepAliveConnections: 2
+    keepAliveTimeout: 2s
+    httpHostHeader: def
+    originServerName: b2
+    caPool: /tmp/path1
+    noTLSVerify: false
+    disableChunkedEncoding: false
+    bastionMode: false
+    proxyAddress: interface
+    proxyPort: 200
+    proxyType: ""
+`
+	ing, err := ParseIngress(MustReadIngress(rulesYAML))
+	if err != nil {
+		t.Error(err)
+	}
+
+	// Rule 0 didn't override anything, so it inherits the user-specified
+	// root-level configuration.
+	actual0 := ing.Rules[0].Config
+	expected0 := OriginRequestConfig{
+		ConnectTimeout:         1 * time.Minute,
+		TLSTimeout:             1 * time.Second,
+		NoHappyEyeballs:        true,
+		TCPKeepAlive:           1 * time.Second,
+		KeepAliveConnections:   1,
+		KeepAliveTimeout:       1 * time.Second,
+		HTTPHostHeader:         "abc",
+		OriginServerName:       "a1",
+		CAPool:                 "/tmp/path0",
+		NoTLSVerify:            true,
+		DisableChunkedEncoding: true,
+		BastionMode:            true,
+		ProxyAddress:           "127.1.2.3",
+		ProxyPort:              uint(100),
+		ProxyType:              "socks5",
+	}
+	require.Equal(t, expected0, actual0)
+
+	// Rule 1 overrode all the root-level config.
+	actual1 := ing.Rules[1].Config
+	expected1 := OriginRequestConfig{
+		ConnectTimeout:         2 * time.Minute,
+		TLSTimeout:             2 * time.Second,
+		NoHappyEyeballs:        false,
+		TCPKeepAlive:           2 * time.Second,
+		KeepAliveConnections:   2,
+		KeepAliveTimeout:       2 * time.Second,
+		HTTPHostHeader:         "def",
+		OriginServerName:       "b2",
+		CAPool:                 "/tmp/path1",
+		NoTLSVerify:            false,
+		DisableChunkedEncoding: false,
+		BastionMode:            false,
+		ProxyAddress:           "interface",
+		ProxyPort:              uint(200),
+		ProxyType:              "",
+	}
+	require.Equal(t, expected1, actual1)
+}
+
+func TestOriginRequestConfigDefaults(t *testing.T) {
+	rulesYAML := `
+ingress:
+- hostname: tun.example.com
+  service: https://localhost:8000
+- hostname: "*"
+  service: https://localhost:8001
+  originRequest:
+    connectTimeout: 2m
+    tlsTimeout: 2s
+    noHappyEyeballs: false
+    tcpKeepAlive: 2s
+    keepAliveConnections: 2
+    keepAliveTimeout: 2s
+    httpHostHeader: def
+    originServerName: b2
+    caPool: /tmp/path1
+    noTLSVerify: false
+    disableChunkedEncoding: false
+    bastionMode: false
+    proxyAddress: interface
+    proxyPort: 200
+    proxyType: ""
+`
+	ing, err := ParseIngress(MustReadIngress(rulesYAML))
+	if err != nil {
+		t.Error(err)
+	}
+
+	// Rule 0 didn't override anything, so it inherits the cloudflared defaults
+	actual0 := ing.Rules[0].Config
+	expected0 := OriginRequestConfig{
+		ConnectTimeout:       defaultConnectTimeout,
+		TLSTimeout:           defaultTLSTimeout,
+		TCPKeepAlive:         defaultTCPKeepAlive,
+		KeepAliveConnections: defaultKeepAliveConnections,
+		KeepAliveTimeout:     defaultKeepAliveTimeout,
+		ProxyAddress:         defaultProxyAddress,
+	}
+	require.Equal(t, expected0, actual0)
+
+	// Rule 1 overrode all defaults.
+	actual1 := ing.Rules[1].Config
+	expected1 := OriginRequestConfig{
+		ConnectTimeout:         2 * time.Minute,
+		TLSTimeout:             2 * time.Second,
+		NoHappyEyeballs:        false,
+		TCPKeepAlive:           2 * time.Second,
+		KeepAliveConnections:   2,
+		KeepAliveTimeout:       2 * time.Second,
+		HTTPHostHeader:         "def",
+		OriginServerName:       "b2",
+		CAPool:                 "/tmp/path1",
+		NoTLSVerify:            false,
+		DisableChunkedEncoding: false,
+		BastionMode:            false,
+		ProxyAddress:           "interface",
+		ProxyPort:              uint(200),
+		ProxyType:              "",
+	}
+	require.Equal(t, expected1, actual1)
+}
+
+func TestDefaultConfigFromCLI(t *testing.T) {
+	set := flag.NewFlagSet("contrive", 0)
+	c := cli.NewContext(nil, set, nil)
+
+	expected := OriginRequestConfig{
+		ConnectTimeout:       defaultConnectTimeout,
+		TLSTimeout:           defaultTLSTimeout,
+		TCPKeepAlive:         defaultTCPKeepAlive,
+		KeepAliveConnections: defaultKeepAliveConnections,
+		KeepAliveTimeout:     defaultKeepAliveTimeout,
+		ProxyAddress:         defaultProxyAddress,
+	}
+	actual := originRequestFromSingeRule(c)
+	require.Equal(t, expected, actual)
+}