Release 2020.12.0

Tunnel delete is successful even if we don't find the credentials
file in the user's filesystem. We no longer "error" indicating this
is a problem. This fix also enables chaining of the delete command
by removing a pre-mature return if the credentials file is not found.

Dockerfile

@@ -1,7 +1,7 @@
 # use a builder image for building cloudflare
 ARG TARGET_GOOS
 ARG TARGET_GOARCH
-FROM golang:1.15.3 as builder
+FROM golang:1.15.6 as builder
 ENV GO111MODULE=on \
     CGO_ENABLED=0 \
     TARGET_GOOS=${TARGET_GOOS} \

Makefile

@@ -89,6 +89,7 @@ define publish_package
 	for HOST in $(CF_PKG_HOSTS); do \
 		ssh-keyscan -t rsa $$HOST >> ~/.ssh/known_hosts; \
 		scp -4 cloudflared*.$(1) cfsync@$$HOST:/state/cf-pkg/staging/$(2)/$(TARGET_PUBLIC_REPO)/cloudflared/; \
+		ssh cfsync@$$HOST 'chmod g+w /state/cf-pkg/staging/$(2)/$(TARGET_PUBLIC_REPO)/cloudflared/*.$(1)'; \
 	done
 endef
 
@@ -104,7 +105,7 @@ define build_package
 	mkdir -p $(PACKAGE_DIR)
 	cp cloudflared $(PACKAGE_DIR)/cloudflared
 	cat cloudflared_man_template | sed -e 's/\$${VERSION}/$(VERSION)/; s/\$${DATE}/$(DATE)/' > $(PACKAGE_DIR)/cloudflared.1
-	fakeroot fpm -C $(PACKAGE_DIR) -s dir -t $(1) --$(1)-compression bzip2 \
+	fakeroot fpm -C $(PACKAGE_DIR) -s dir -t $(1) \
 		--description 'Cloudflare Argo tunnel daemon' \
 		--vendor 'Cloudflare' \
 		--license 'Cloudflare Service Agreement' \
@@ -225,4 +226,4 @@ vet:
 
 .PHONY: msi
 msi: cloudflared
-	go-msi make --msi cloudflared.msi --version $(MSI_VERSION)
+	go-msi make --msi cloudflared.msi --version $(MSI_VERSION)

README.md

@@ -1,9 +1,32 @@
 # Argo Tunnel client
 
-Contains the command-line client and its libraries for Argo Tunnel, a tunneling daemon that proxies any local webserver through the Cloudflare network.
+Contains the command-line client for Argo Tunnel, a tunneling daemon that proxies any local webserver through the Cloudflare network. Extensive documentation can be found in the [Argo Tunnel section](https://developers.cloudflare.com/argo-tunnel/) of the Cloudflare Docs.
 
-## Getting started
+## Before you get started
 
-    go install github.com/cloudflare/cloudflared/cmd/cloudflared
+Before you use Argo Tunnel, you'll need to complete a few steps in the Cloudflare dashboard. The website you add to Cloudflare will be used to route traffic to your Tunnel.
+
+1. [Add a website to Cloudflare](https://support.cloudflare.com/hc/en-us/articles/201720164-Creating-a-Cloudflare-account-and-adding-a-website)
+2. [Change your domain nameservers to Cloudflare](https://support.cloudflare.com/hc/en-us/articles/205195708)
+
+## Installing `cloudflared`
+
+Downloads are available as standalone binaries, a Docker image, and Debian, RPM, and Homebrew packages. You can also find releases here on the `cloudflared` GitHub repository.
+
+* You can [install on macOS](https://developers.cloudflare.com/argo-tunnel/getting-started/installation#macos) via Homebrew or by downloading the [latest Darwin amd64 release](https://github.com/cloudflare/cloudflared/releases)
+* Binaries, Debian, and RPM packages for Linux [can be found here](https://developers.cloudflare.com/argo-tunnel/getting-started/installation#linux)
+* A Docker image of `cloudflared` is [available on DockerHub](https://hub.docker.com/r/cloudflare/cloudflared)
+* You can install on Windows machines with the [steps here](https://developers.cloudflare.com/argo-tunnel/getting-started/installation#windows)
 
 User documentation for Argo Tunnel can be found at https://developers.cloudflare.com/argo-tunnel/
+
+## Creating Tunnels and routing traffic
+
+Once installed, you can authenticate `cloudflared` into your Cloudflare account and begin creating Tunnels that serve traffic for hostnames in your account.
+
+* Create a Tunnel with [these instructions](https://developers.cloudflare.com/argo-tunnel/create-tunnel)
+* Route traffic to that Tunnel with [DNS records in Cloudflare](https://developers.cloudflare.com/argo-tunnel/routing-to-tunnel/dns) or with a [Cloudflare Load Balancer](https://developers.cloudflare.com/argo-tunnel/routing-to-tunnel/lb)
+
+## TryCloudflare
+
+Want to test Argo Tunnel before adding a website to Cloudflare? You can do so with TryCloudflare using the documentation [available here](https://developers.cloudflare.com/argo-tunnel/learning/trycloudflare).

RELEASE_NOTES

@@ -1,3 +1,89 @@
+2020.12.0
+- 2020-12-04 TUN-3599: improved delete if credentials isnt found.
+- 2020-12-04 TUN-3612: Upgrade to Go 1.15.6
+- 2020-11-30 TUN-3593: /ready endpoint for k8s readiness. Move tunnel events out of UI package, into connection package.
+- 2020-11-27 TUN-3594: Log response status at debug level
+
+2020.11.11
+- 2020-11-20 TUN-3578: cloudflared tunnel route dns should allow wildcard subdomains
+- 2020-11-21 EDGEPLAT-2958 remove deb-compression, defaulting to gzip
+- 2020-11-23 TUN-3581: Tunnels can be run by name using only --credentials-file, no origin cert necessary.
+- 2020-11-15 TUN-3561: Unified logger configuration
+- 2020-11-08 AUTH-3221: Saves org token to disk and uses it to refresh the app token
+
+2020.11.10
+- 2020-11-20 TUN-3562: Fix panic when using bastion mode ingress rule
+- 2020-11-20 EDGEPLAT-2958 build cloudflared for Bullseye
+
+2020.11.9
+- 2020-11-18 TUN-3557: Detect SSE if content-type starts with text/event-stream
+- 2020-11-18 TUN-3559: Share response meta header with other packages
+- 2020-11-18 DEVTOOLS-7936: Remove redundant chgrp from publish
+- 2020-11-18 TUN-3558: cloudflared allows empty config files
+- 2020-11-18 TUN-3544: Upgrade to Go 1.15.5
+
+2020.11.8
+- 2020-11-17 TUN-3555: Single origin service should default to localhost:8080
+
+2020.11.7
+- 2020-11-13 TUN-3514: Stop setting --is-autoupdated flag after autoupdate because it can break named tunnel running in k8s
+- 2020-11-15 TUN-3548, TUN-3547: Bastion mode can be specified as a service, doesn't require URL.
+- 2020-11-16 TUN-3549: Use a separate handler for each websocket proxy
+
+2020.11.6
+- 2020-11-14 TUN-3546: Fix panic in tlsconfig.LoadOriginCA
+
+2020.11.5
+- 2020-11-12 TUN-3540: Better copy in ingress rules error messages
+- 2020-11-12 DEVTOOLS-7936: Set permissions on public packages
+- 2020-11-13 TUN-3543: ProxyAddress not using default in single-origin mode
+
+2020.11.4
+- 2020-11-11 TUN-3534: Specific error message when credentials file is a .pem not .json
+- 2020-11-02 TUN-3500: Integrate replace h2mux by http2 work with multiple origin support
+- 2020-11-09 TUN-3514: Transport logger write to UI when UI is enabled
+- 2020-10-30 TUN-3490: Make sure OriginClient implementation doesn't write after Proxy return
+- 2020-10-20 TUN-3403: Unit test for origin/proxy to test serving HTTP and Websocket
+- 2020-10-23 TUN-3480: Support SSE with http2 connection, and add SSE handler to hello-world server
+- 2020-10-27 TUN-3489: Add unit tests to cover proxy logic in connection package of cloudflared
+- 2020-10-16 TUN-3467: Serialize cf-cloudflared-response-meta during package initialization using jsoniter
+- 2020-10-14 TUN-3456: New protocol option auto to automatically select between http2 and h2mux
+- 2020-10-14 TUN-3458: Upgrade to http2 when available, fallback to h2mux when we reach max retries
+- 2020-10-08 TUN-3449: Use flag to select transport protocol implementation
+- 2020-10-08 TUN-3462: Refactor cloudflared to separate origin from connection
+- 2020-09-21 TUN-3406: Proxy websocket requests over Go http2
+- 2020-09-25 TUN-3420: Establish control plane and send RPC over control plane
+- 2020-09-11 TUN-3400: Use Go HTTP2 library as transport to connect with the edge
+
+2020.11.3
+- 2020-11-11 TUN-3533: Set config for single origin ingress
+
+2020.11.2
+
+
+2020.11.1
+- 2020-11-10 TUN-3527: More specific error for invalid YAML/JSON
+- 2020-11-06 Update README.md (#256)
+
+2020.11.0
+- 2020-11-04 TUN-3484: OriginService that responds with configured HTTP status
+- 2020-11-05 TUN-3505: Response body for status code origin returns EOF on Read
+- 2020-11-04 TUN-3503: Matching ingress rule should not take port into account
+- 2020-11-05 TUN-3506: OriginService needs to set request host and scheme for websocket requests
+- 2020-11-09 TUN-3516: Better error message when parsing invalid YAML config
+- 2020-11-09 TUN-3522: ingress validate checks that the config file exists
+- 2020-11-09 TUN-3524: Don't ignore errors from app-level action handler (#248)
+- 2020-11-09 TUN-3461: Show all origin services in the UI
+- 2020-10-30 TUN-3494: Proceed to create tunnel if at least one edge address can be resolved
+- 2020-10-30 TUN-3492: Refactor OriginService, shrink its interface
+- 2020-10-22 TUN-3478: Increase download timeout to 60s
+- 2020-10-15 TUN-2640: Users can configure per-origin config. Unify single-rule CLI flow with multi-rule config file code.
+
+2020.10.2
+- 2020-10-21 Release 2020.10.1
+- 2020-10-21 AUTH-3185 fixed indention error
+- 2020-10-19 TUN-3459: Make service install on linux use named tunnels
+
 2020.10.1
 - 2020-10-20 Split out typed config from legacy command-line switches; refactor ingress commands and fix tests
 - 2020-10-20 Move raw ingress rules to config package

cfsetup.yaml

@@ -1,6 +1,6 @@
-pinned_go: &pinned_go go=1.15.3-1
+pinned_go: &pinned_go go=1.15.6-1
 build_dir: &build_dir /cfsetup_build
-default-flavor: stretch
+default-flavor: buster
 stretch: &stretch
   build:
     build_dir: *build_dir
@@ -239,10 +239,8 @@ stretch: &stretch
       - export GOARCH=amd64
       - make publish-cloudflared-junos
 
-jessie: *stretch
-
 buster: *stretch
-
+bullseye: *stretch
 centos-7:
   publish-rpm:
     build_dir: *build_dir
@@ -250,8 +248,8 @@ centos-7:
       - https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
     pre-cache:
       - yum install -y fakeroot
-      - wget https://golang.org/dl/go1.15.3.linux-amd64.tar.gz -P /tmp/
-      - tar -C /usr/local -xzf /tmp/go1.15.3.linux-amd64.tar.gz
+      - wget https://golang.org/dl/go1.15.6.linux-amd64.tar.gz -P /tmp/
+      - tar -C /usr/local -xzf /tmp/go1.15.6.linux-amd64.tar.gz
     post-cache:
       - export PATH=$PATH:/usr/local/go/bin
       - export GOOS=linux
@@ -262,8 +260,8 @@ centos-7:
     builddeps: *el7_builddeps
     pre-cache:
       - yum install -y fakeroot
-      - wget https://golang.org/dl/go1.15.3.linux-amd64.tar.gz -P /tmp/
-      - tar -C /usr/local -xzf /tmp/go1.15.3.linux-amd64.tar.gz
+      - wget https://golang.org/dl/go1.15.6.linux-amd64.tar.gz -P /tmp/
+      - tar -C /usr/local -xzf /tmp/go1.15.6.linux-amd64.tar.gz
     post-cache:
       - export PATH=$PATH:/usr/local/go/bin
       - export GOOS=linux

cmd/cloudflared/access/carrier.go

@@ -2,7 +2,6 @@ package access
 
 import (
 	"net/http"
-	"net/url"
 	"strings"
 
 	"github.com/cloudflare/cloudflared/carrier"
@@ -12,21 +11,16 @@ import (
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/validation"
 	"github.com/pkg/errors"
-	cli "github.com/urfave/cli/v2"
+	"github.com/urfave/cli/v2"
 )
 
 // StartForwarder starts a client side websocket forward
 func StartForwarder(forwarder config.Forwarder, shutdown <-chan struct{}, logger logger.Service) error {
-	validURLString, err := validation.ValidateUrl(forwarder.Listener)
+	validURL, err := validation.ValidateUrl(forwarder.Listener)
 	if err != nil {
 		return errors.Wrap(err, "error validating origin URL")
 	}
 
-	validURL, err := url.Parse(validURLString)
-	if err != nil {
-		return errors.Wrap(err, "error parsing origin URL")
-	}
-
 	// get the headers from the config file and add to the request
 	headers := make(http.Header)
 	if forwarder.TokenClientID != "" {
@@ -58,19 +52,7 @@ func StartForwarder(forwarder config.Forwarder, shutdown <-chan struct{}, logger
 // useful for proxying other protocols (like ssh) over websockets
 // (which you can put Access in front of)
 func ssh(c *cli.Context) error {
-	logDirectory, logLevel := config.FindLogSettings()
-
-	flagLogDirectory := c.String(sshLogDirectoryFlag)
-	if flagLogDirectory != "" {
-		logDirectory = flagLogDirectory
-	}
-
-	flagLogLevel := c.String(sshLogLevelFlag)
-	if flagLogLevel != "" {
-		logLevel = flagLogLevel
-	}
-
-	logger, err := logger.New(logger.DefaultFile(logDirectory), logger.LogLevelString(logLevel))
+	logger, err := logger.CreateSSHLoggerFromContext(c, logger.EnableTerminalLog)
 	if err != nil {
 		return cliutil.PrintLoggerSetupError("error setting up logger", err)
 	}
@@ -106,12 +88,7 @@ func ssh(c *cli.Context) error {
 	wsConn := carrier.NewWSConnection(logger, false)
 
 	if c.NArg() > 0 || c.IsSet(sshURLFlag) {
-		localForwarder, err := config.ValidateUrl(c, true)
-		if err != nil {
-			logger.Errorf("Error validating origin URL: %s", err)
-			return errors.Wrap(err, "error validating origin URL")
-		}
-		forwarder, err := url.Parse(localForwarder)
+		forwarder, err := config.ValidateUrl(c, true)
 		if err != nil {
 			logger.Errorf("Error validating origin URL: %s", err)
 			return errors.Wrap(err, "error validating origin URL")

cmd/cloudflared/access/cmd.go

@@ -25,16 +25,14 @@ import (
 )
 
 const (
-	sshHostnameFlag     = "hostname"
-	sshDestinationFlag  = "destination"
-	sshURLFlag          = "url"
-	sshHeaderFlag       = "header"
-	sshTokenIDFlag      = "service-token-id"
-	sshTokenSecretFlag  = "service-token-secret"
-	sshGenCertFlag      = "short-lived-cert"
-	sshLogDirectoryFlag = "log-directory"
-	sshLogLevelFlag     = "log-level"
-	sshConfigTemplate   = `
+	sshHostnameFlag    = "hostname"
+	sshDestinationFlag = "destination"
+	sshURLFlag         = "url"
+	sshHeaderFlag      = "header"
+	sshTokenIDFlag     = "service-token-id"
+	sshTokenSecretFlag = "service-token-secret"
+	sshGenCertFlag     = "short-lived-cert"
+	sshConfigTemplate  = `
 Add to your {{.Home}}/.ssh/config:
 
 Host {{.Hostname}}
@@ -157,12 +155,12 @@ func Commands() []*cli.Command {
 							Usage:   "specify an Access service token secret you wish to use.",
 						},
 						&cli.StringFlag{
-							Name:    sshLogDirectoryFlag,
+							Name:    logger.LogSSHDirectoryFlag,
 							Aliases: []string{"logfile"}, //added to match the tunnel side
 							Usage:   "Save application log to this directory for reporting issues.",
 						},
 						&cli.StringFlag{
-							Name:    sshLogLevelFlag,
+							Name:    logger.LogSSHLevelFlag,
 							Aliases: []string{"loglevel"}, //added to match the tunnel side
 							Usage:   "Application logging level {fatal, error, info, debug}. ",
 						},
@@ -207,7 +205,7 @@ func login(c *cli.Context) error {
 		return err
 	}
 
-	logger, err := logger.New()
+	logger, err := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
 	if err != nil {
 		return errors.Wrap(err, "error setting up logger")
 	}
@@ -224,7 +222,7 @@ func login(c *cli.Context) error {
 		return err
 	}
 
-	cfdToken, err := token.GetTokenIfExists(appURL)
+	cfdToken, err := token.GetAppTokenIfExists(appURL)
 	if err != nil {
 		fmt.Fprintln(os.Stderr, "Unable to find token for provided application.")
 		return err
@@ -252,7 +250,7 @@ func curl(c *cli.Context) error {
 	if err := raven.SetDSN(sentryDSN); err != nil {
 		return err
 	}
-	logger, err := logger.New()
+	logger, err := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
 	if err != nil {
 		return errors.Wrap(err, "error setting up logger")
 	}
@@ -269,7 +267,7 @@ func curl(c *cli.Context) error {
 		return err
 	}
 
-	tok, err := token.GetTokenIfExists(appURL)
+	tok, err := token.GetAppTokenIfExists(appURL)
 	if err != nil || tok == "" {
 		if allowRequest {
 			logger.Info("You don't have an Access token set. Please run access token <access application> to fetch one.")
@@ -297,7 +295,7 @@ func generateToken(c *cli.Context) error {
 		fmt.Fprintln(os.Stderr, "Please provide a url.")
 		return err
 	}
-	tok, err := token.GetTokenIfExists(appURL)
+	tok, err := token.GetAppTokenIfExists(appURL)
 	if err != nil || tok == "" {
 		fmt.Fprintln(os.Stderr, "Unable to find token for provided application. Please run token command to generate token.")
 		return err
@@ -331,7 +329,7 @@ func sshConfig(c *cli.Context) error {
 
 // sshGen generates a short lived certificate for provided hostname
 func sshGen(c *cli.Context) error {
-	logger, err := logger.New()
+	logger, err := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
 	if err != nil {
 		return errors.Wrap(err, "error setting up logger")
 	}

cmd/cloudflared/app_service.go

@@ -37,6 +37,7 @@ func (s *AppService) Run() error {
 func (s *AppService) Shutdown() error {
 	s.configManager.Shutdown()
 	s.shutdownC <- struct{}{}
+
 	return nil
 }
 

cmd/cloudflared/cliutil/errors.go

@@ -27,17 +27,17 @@ func UsageError(format string, args ...interface{}) error {
 // Ensures exit with error code if actionFunc returns an error
 func ErrorHandler(actionFunc cli.ActionFunc) cli.ActionFunc {
 	return func(ctx *cli.Context) error {
+		defer logger.SharedWriteManager.Shutdown()
+
 		err := actionFunc(ctx)
 		if err != nil {
 			if _, ok := err.(usageError); ok {
 				msg := fmt.Sprintf("%s\nSee 'cloudflared %s --help'.", err.Error(), ctx.Command.FullName())
-				return cli.Exit(msg, -1)
+				err = cli.Exit(msg, -1)
+			} else if _, ok := err.(cli.ExitCoder); !ok {
+				err = cli.Exit(err.Error(), 1)
 			}
-			// os.Exits with error code if err is cli.ExitCoder or cli.MultiError
-			cli.HandleExitCoder(err)
-			err = cli.Exit(err.Error(), 1)
 		}
-		logger.SharedWriteManager.Shutdown()
 		return err
 	}
 }

cmd/cloudflared/config/configuration.go

@@ -1,14 +1,16 @@
 package config
 
 import (
-	"errors"
 	"fmt"
+	"io"
+	"net/url"
 	"os"
 	"path/filepath"
 	"runtime"
 	"time"
 
 	homedir "github.com/mitchellh/go-homedir"
+	"github.com/pkg/errors"
 	"github.com/urfave/cli/v2"
 	"gopkg.in/yaml.v2"
 
@@ -34,7 +36,12 @@ var (
 	ErrNoConfigFile = fmt.Errorf("Cannot determine default configuration path. No file %v in %v", DefaultConfigFiles, DefaultConfigSearchDirectories())
 )
 
-const DefaultCredentialFile = "cert.pem"
+const (
+	DefaultCredentialFile = "cert.pem"
+
+	// BastionFlag is to enable bastion, or jump host, operation
+	BastionFlag = "bastion"
+)
 
 // DefaultConfigDirectory returns the default directory of the config file
 func DefaultConfigDirectory() string {
@@ -144,35 +151,6 @@ func FindOrCreateConfigPath() string {
 	return path
 }
 
-// FindLogSettings gets the log directory and level from the config file
-func FindLogSettings() (string, string) {
-	configPath := FindOrCreateConfigPath()
-	defaultDirectory := DefaultLogDirectory()
-	defaultLevel := "info"
-
-	file, err := os.Open(configPath)
-	if err != nil {
-		return defaultDirectory, defaultLevel
-	}
-	defer file.Close()
-
-	var config Root
-	if err := yaml.NewDecoder(file).Decode(&config); err != nil {
-		return defaultDirectory, defaultLevel
-	}
-
-	directory := defaultDirectory
-	if config.LogDirectory != "" {
-		directory = config.LogDirectory
-	}
-
-	level := defaultLevel
-	if config.LogLevel != "" {
-		level = config.LogLevel
-	}
-	return directory, level
-}
-
 // ValidateUnixSocket ensures --unix-socket param is used exclusively
 // i.e. it fails if a user specifies both --url and --unix-socket
 func ValidateUnixSocket(c *cli.Context) (string, error) {
@@ -184,11 +162,11 @@ func ValidateUnixSocket(c *cli.Context) (string, error) {
 
 // ValidateUrl will validate url flag correctness. It can be either from --url or argument
 // Notice ValidateUnixSocket, it will enforce --unix-socket is not used with --url or argument
-func ValidateUrl(c *cli.Context, allowFromArgs bool) (string, error) {
+func ValidateUrl(c *cli.Context, allowURLFromArgs bool) (*url.URL, error) {
 	var url = c.String("url")
-	if allowFromArgs && c.NArg() > 0 {
+	if allowURLFromArgs && c.NArg() > 0 {
 		if c.IsSet("url") {
-			return "", errors.New("Specified origin urls using both --url and argument. Decide which one you want, I can only support one.")
+			return nil, errors.New("Specified origin urls using both --url and argument. Decide which one you want, I can only support one.")
 		}
 		url = c.Args().Get(0)
 	}
@@ -197,15 +175,59 @@ func ValidateUrl(c *cli.Context, allowFromArgs bool) (string, error) {
 }
 
 type UnvalidatedIngressRule struct {
-	Hostname string
-	Path     string
-	Service  string
+	Hostname      string
+	Path          string
+	Service       string
+	OriginRequest OriginRequestConfig `yaml:"originRequest"`
+}
+
+// OriginRequestConfig is a set of optional fields that users may set to
+// customize how cloudflared sends requests to origin services. It is used to set
+// up general config that apply to all rules, and also, specific per-rule
+// config.
+// Note: To specify a time.Duration in go-yaml, use e.g. "3s" or "24h".
+type OriginRequestConfig struct {
+	// HTTP proxy timeout for establishing a new connection
+	ConnectTimeout *time.Duration `yaml:"connectTimeout"`
+	// HTTP proxy timeout for completing a TLS handshake
+	TLSTimeout *time.Duration `yaml:"tlsTimeout"`
+	// HTTP proxy TCP keepalive duration
+	TCPKeepAlive *time.Duration `yaml:"tcpKeepAlive"`
+	// HTTP proxy should disable "happy eyeballs" for IPv4/v6 fallback
+	NoHappyEyeballs *bool `yaml:"noHappyEyeballs"`
+	// HTTP proxy maximum keepalive connection pool size
+	KeepAliveConnections *int `yaml:"keepAliveConnections"`
+	// HTTP proxy timeout for closing an idle connection
+	KeepAliveTimeout *time.Duration `yaml:"keepAliveTimeout"`
+	// Sets the HTTP Host header for the local webserver.
+	HTTPHostHeader *string `yaml:"httpHostHeader"`
+	// Hostname on the origin server certificate.
+	OriginServerName *string `yaml:"originServerName"`
+	// Path to the CA for the certificate of your origin.
+	// This option should be used only if your certificate is not signed by Cloudflare.
+	CAPool *string `yaml:"caPool"`
+	// Disables TLS verification of the certificate presented by your origin.
+	// Will allow any certificate from the origin to be accepted.
+	// Note: The connection from your machine to Cloudflare's Edge is still encrypted.
+	NoTLSVerify *bool `yaml:"noTLSVerify"`
+	// Disables chunked transfer encoding.
+	// Useful if you are running a WSGI server.
+	DisableChunkedEncoding *bool `yaml:"disableChunkedEncoding"`
+	// Runs as jump host
+	BastionMode *bool `yaml:"bastionMode"`
+	// Listen address for the proxy.
+	ProxyAddress *string `yaml:"proxyAddress"`
+	// Listen port for the proxy.
+	ProxyPort *uint `yaml:"proxyPort"`
+	// Valid options are 'socks' or empty.
+	ProxyType *string `yaml:"proxyType"`
 }
 
 type Configuration struct {
-	TunnelID   string `yaml:"tunnel"`
-	Ingress    []UnvalidatedIngressRule
-	sourceFile string
+	TunnelID      string `yaml:"tunnel"`
+	Ingress       []UnvalidatedIngressRule
+	OriginRequest OriginRequestConfig `yaml:"originRequest"`
+	sourceFile    string
 }
 
 type configFileSettings struct {
@@ -342,7 +364,11 @@ func ReadConfigFile(c *cli.Context, log logger.Service) (*configFileSettings, er
 	}
 	defer file.Close()
 	if err := yaml.NewDecoder(file).Decode(&configuration); err != nil {
-		return nil, err
+		if err == io.EOF {
+			log.Errorf("Configuration file %s was empty", configFile)
+			return &configuration, nil
+		}
+		return nil, errors.Wrap(err, "error parsing YAML in config file at "+configFile)
 	}
 	configuration.sourceFile = configFile
 	return &configuration, nil

cmd/cloudflared/config/manager.go

@@ -1,11 +1,12 @@
 package config
 
 import (
-	"errors"
+	"io"
 	"os"
 
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/watcher"
+	"github.com/pkg/errors"
 	"gopkg.in/yaml.v2"
 )
 
@@ -27,7 +28,7 @@ type FileManager struct {
 	notifier   Notifier
 	configPath string
 	logger     logger.Service
-	ReadConfig func(string) (Root, error)
+	ReadConfig func(string, logger.Service) (Root, error)
 }
 
 // NewFileManager creates a config manager
@@ -59,7 +60,7 @@ func (m *FileManager) Start(notifier Notifier) error {
 
 // GetConfig reads the yaml file from the disk
 func (m *FileManager) GetConfig() (Root, error) {
-	return m.ReadConfig(m.configPath)
+	return m.ReadConfig(m.configPath, m.logger)
 }
 
 // Shutdown stops the watcher
@@ -67,7 +68,7 @@ func (m *FileManager) Shutdown() {
 	m.watcher.Shutdown()
 }
 
-func readConfigFromPath(configPath string) (Root, error) {
+func readConfigFromPath(configPath string, log logger.Service) (Root, error) {
 	if configPath == "" {
 		return Root{}, errors.New("unable to find config file")
 	}
@@ -80,7 +81,11 @@ func readConfigFromPath(configPath string) (Root, error) {
 
 	var config Root
 	if err := yaml.NewDecoder(file).Decode(&config); err != nil {
-		return Root{}, err
+		if err == io.EOF {
+			log.Errorf("Configuration file %s was empty", configPath)
+			return Root{}, nil
+		}
+		return Root{}, errors.Wrap(err, "error parsing YAML in config file at "+configPath)
 	}
 
 	return config, nil

cmd/cloudflared/config/manager_test.go

@@ -57,7 +57,7 @@ func TestConfigChanged(t *testing.T) {
 			},
 		},
 	}
-	configRead := func(configPath string) (Root, error) {
+	configRead := func(configPath string, log logger.Service) (Root, error) {
 		return *c, nil
 	}
 	wait := make(chan struct{})

cmd/cloudflared/linux_service.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/tunnel"
 	"github.com/cloudflare/cloudflared/logger"
 )
 
@@ -211,7 +212,7 @@ func copyUserConfiguration(userConfigDir, userConfigFile, userCredentialFile str
 }
 
 func installLinuxService(c *cli.Context) error {
-	logger, err := logger.New()
+	logger, err := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
 	if err != nil {
 		return errors.Wrap(err, "error setting up logger")
 	}
@@ -250,7 +251,7 @@ func installLinuxService(c *cli.Context) error {
 			val, err := src.String(s)
 			return err == nil && val != ""
 		}
-		if src.TunnelID == "" || !configPresent("credentials-file") {
+		if src.TunnelID == "" || !configPresent(tunnel.CredFileFlag) {
 			return fmt.Errorf(`Configuration file %s must contain entries for the tunnel to run and its associated credentials:
 tunnel: TUNNEL-UUID
 credentials-file: CREDENTIALS-FILE
@@ -325,7 +326,7 @@ func installSysv(templateArgs *ServiceTemplateArgs, logger logger.Service) error
 }
 
 func uninstallLinuxService(c *cli.Context) error {
-	logger, err := logger.New()
+	logger, err := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
 	if err != nil {
 		return errors.Wrap(err, "error setting up logger")
 	}

cmd/cloudflared/macos_service.go

@@ -6,11 +6,11 @@ import (
 	"fmt"
 	"os"
 
-	"github.com/urfave/cli/v2"
-
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
 	"github.com/cloudflare/cloudflared/logger"
+
 	"github.com/pkg/errors"
+	"github.com/urfave/cli/v2"
 )
 
 const (
@@ -107,7 +107,7 @@ func stderrPath() (string, error) {
 }
 
 func installLaunchd(c *cli.Context) error {
-	logger, err := logger.New()
+	logger, err := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
 	if err != nil {
 		return errors.Wrap(err, "error setting up logger")
 	}
@@ -163,7 +163,7 @@ func installLaunchd(c *cli.Context) error {
 }
 
 func uninstallLaunchd(c *cli.Context) error {
-	logger, err := logger.New()
+	logger, err := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
 	if err != nil {
 		return errors.Wrap(err, "error setting up logger")
 	}

cmd/cloudflared/main.go

@@ -15,10 +15,9 @@ import (
 	"github.com/cloudflare/cloudflared/overwatch"
 	"github.com/cloudflare/cloudflared/tunneldns"
 	"github.com/cloudflare/cloudflared/watcher"
-
-	raven "github.com/getsentry/raven-go"
-	homedir "github.com/mitchellh/go-homedir"
-	cli "github.com/urfave/cli/v2"
+	"github.com/getsentry/raven-go"
+	"github.com/mitchellh/go-homedir"
+	"github.com/urfave/cli/v2"
 
 	"github.com/pkg/errors"
 )
@@ -145,23 +144,19 @@ func isEmptyInvocation(c *cli.Context) bool {
 }
 
 func action(version string, shutdownC, graceShutdownC chan struct{}) cli.ActionFunc {
-	return func(c *cli.Context) (err error) {
+	return cliutil.ErrorHandler(func(c *cli.Context) (err error) {
 		if isEmptyInvocation(c) {
-			return handleServiceMode(shutdownC)
+			return handleServiceMode(c, shutdownC)
 		}
 		tags := make(map[string]string)
 		tags["hostname"] = c.String("hostname")
 		raven.SetTagsContext(tags)
 		raven.CapturePanic(func() { err = tunnel.TunnelCommand(c) }, nil)
-		exitCode := 0
 		if err != nil {
-			handleError(err)
-			exitCode = 1
+			captureError(err)
 		}
-		// we already handle error printing, so we pass an empty string so we
-		// don't have to print again.
-		return cli.Exit("", exitCode)
-	}
+		return err
+	})
 }
 
 func userHomeDir() (string, error) {
@@ -177,7 +172,7 @@ func userHomeDir() (string, error) {
 }
 
 // In order to keep the amount of noise sent to Sentry low, typical network errors can be filtered out here by a substring match.
-func handleError(err error) {
+func captureError(err error) {
 	errorMessage := err.Error()
 	for _, ignoredErrorMessage := range ignoredErrors {
 		if strings.Contains(errorMessage, ignoredErrorMessage) {
@@ -188,15 +183,13 @@ func handleError(err error) {
 }
 
 // cloudflared was started without any flags
-func handleServiceMode(shutdownC chan struct{}) error {
+func handleServiceMode(c *cli.Context, shutdownC chan struct{}) error {
 	defer log.SharedWriteManager.Shutdown()
-	logDirectory, logLevel := config.FindLogSettings()
 
-	logger, err := log.New(log.DefaultFile(logDirectory), log.LogLevelString(logLevel))
+	logger, err := log.CreateLoggerFromContext(c, log.DisableTerminalLog)
 	if err != nil {
 		return cliutil.PrintLoggerSetupError("error setting up logger", err)
 	}
-	logger.Infof("logging to directory: %s", logDirectory)
 
 	// start the main run loop that reads from the config file
 	f, err := watcher.NewFile()

cmd/cloudflared/path/path.go

@@ -11,8 +11,27 @@ import (
 	"github.com/mitchellh/go-homedir"
 )
 
-// GenerateFilePathFromURL will return a filepath for given access application url
-func GenerateFilePathFromURL(url *url.URL, suffix string) (string, error) {
+// GenerateAppTokenFilePathFromURL will return a filepath for given Access org token
+func GenerateAppTokenFilePathFromURL(url *url.URL, suffix string) (string, error) {
+	configPath, err := getConfigPath()
+	if err != nil {
+		return "", err
+	}
+	name := strings.Replace(fmt.Sprintf("%s%s-%s", url.Hostname(), url.EscapedPath(), suffix), "/", "-", -1)
+	return filepath.Join(configPath, name), nil
+}
+
+// GenerateOrgTokenFilePathFromURL will return a filepath for given Access application token
+func GenerateOrgTokenFilePathFromURL(authDomain string) (string, error) {
+	configPath, err := getConfigPath()
+	if err != nil {
+		return "", err
+	}
+	name := strings.Replace(fmt.Sprintf("%s-org-token", authDomain), "/", "-", -1)
+	return filepath.Join(configPath, name), nil
+}
+
+func getConfigPath() (string, error) {
 	configPath, err := homedir.Expand(config.DefaultConfigSearchDirectories()[0])
 	if err != nil {
 		return "", err
@@ -22,9 +41,5 @@ func GenerateFilePathFromURL(url *url.URL, suffix string) (string, error) {
 		// create config directory if doesn't already exist
 		err = os.Mkdir(configPath, 0700)
 	}
-	if err != nil {
-		return "", err
-	}
-	name := strings.Replace(fmt.Sprintf("%s%s-%s", url.Hostname(), url.EscapedPath(), suffix), "/", "-", -1)
-	return filepath.Join(configPath, name), nil
+	return configPath, err
 }

cmd/cloudflared/token/token.go

@@ -5,9 +5,11 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"net/http"
 	"net/url"
 	"os"
 	"os/signal"
+	"strings"
 	"syscall"
 	"time"
 
@@ -17,10 +19,12 @@ import (
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/origin"
 	"github.com/coreos/go-oidc/jose"
+	"github.com/pkg/errors"
 )
 
 const (
-	keyName = "token"
+	keyName     = "token"
+	tokenHeader = "CF_Authorization"
 )
 
 type lock struct {
@@ -34,7 +38,7 @@ type signalHandler struct {
 	signals    []os.Signal
 }
 
-type jwtPayload struct {
+type appJWTPayload struct {
 	Aud   []string `json:"aud"`
 	Email string   `json:"email"`
 	Exp   int      `json:"exp"`
@@ -45,7 +49,17 @@ type jwtPayload struct {
 	Subt  string   `json:"sub"`
 }
 
-func (p jwtPayload) isExpired() bool {
+type orgJWTPayload struct {
+	appJWTPayload
+	Aud string `json:"aud"`
+}
+
+type transferServiceResponse struct {
+	AppToken string `json:"app_token"`
+	OrgToken string `json:"org_token"`
+}
+
+func (p appJWTPayload) isExpired() bool {
 	return int(time.Now().Unix()) > p.Exp
 }
 
@@ -141,55 +155,173 @@ func FetchToken(appURL *url.URL, logger logger.Service) (string, error) {
 
 // getToken will either load a stored token or generate a new one
 func getToken(appURL *url.URL, useHostOnly bool, logger logger.Service) (string, error) {
-	if token, err := GetTokenIfExists(appURL); token != "" && err == nil {
+	if token, err := GetAppTokenIfExists(appURL); token != "" && err == nil {
 		return token, nil
 	}
 
-	path, err := path.GenerateFilePathFromURL(appURL, keyName)
+	appTokenPath, err := path.GenerateAppTokenFilePathFromURL(appURL, keyName)
 	if err != nil {
-		return "", err
+		return "", errors.Wrap(err, "failed to generate app token file path")
 	}
 
-	fileLock := newLock(path)
-
-	err = fileLock.Acquire()
-	if err != nil {
-		return "", err
+	fileLockAppToken := newLock(appTokenPath)
+	if err = fileLockAppToken.Acquire(); err != nil {
+		return "", errors.Wrap(err, "failed to acquire app token lock")
 	}
-	defer fileLock.Release()
+	defer fileLockAppToken.Release()
 
 	// check to see if another process has gotten a token while we waited for the lock
-	if token, err := GetTokenIfExists(appURL); token != "" && err == nil {
+	if token, err := GetAppTokenIfExists(appURL); token != "" && err == nil {
 		return token, nil
 	}
 
+	// If an app token couldnt be found on disk, check for an org token and attempt to exchange it for an app token.
+	var orgTokenPath string
+	// Get auth domain to format into org token file path
+	if authDomain, err := getAuthDomain(appURL); err != nil {
+		logger.Errorf("failed to get auth domain: %s", err)
+	} else {
+		orgToken, err := GetOrgTokenIfExists(authDomain)
+		if err != nil {
+			orgTokenPath, err = path.GenerateOrgTokenFilePathFromURL(authDomain)
+			if err != nil {
+				return "", errors.Wrap(err, "failed to generate org token file path")
+			}
+
+			fileLockOrgToken := newLock(orgTokenPath)
+			if err = fileLockOrgToken.Acquire(); err != nil {
+				return "", errors.Wrap(err, "failed to acquire org token lock")
+			}
+			defer fileLockOrgToken.Release()
+			// check if an org token has been created since the lock was acquired
+			orgToken, err = GetOrgTokenIfExists(authDomain)
+		}
+		if err == nil {
+			if appToken, err := exchangeOrgToken(appURL, orgToken); err != nil {
+				logger.Debugf("failed to exchange org token for app token: %s", err)
+			} else {
+				if err := ioutil.WriteFile(appTokenPath, []byte(appToken), 0600); err != nil {
+					return "", errors.Wrap(err, "failed to write app token to disk")
+				}
+				return appToken, nil
+			}
+		}
+	}
+	return getTokensFromEdge(appURL, appTokenPath, orgTokenPath, useHostOnly, logger)
+
+}
+
+// getTokensFromEdge will attempt to use the transfer service to retrieve an app and org token, save them to disk,
+// and return the app token.
+func getTokensFromEdge(appURL *url.URL, appTokenPath, orgTokenPath string, useHostOnly bool, logger logger.Service) (string, error) {
+	// If no org token exists or if it couldnt be exchanged for an app token, then run the transfer service flow.
+
 	// this weird parameter is the resource name (token) and the key/value
 	// we want to send to the transfer service. the key is token and the value
 	// is blank (basically just the id generated in the transfer service)
-	token, err := transfer.Run(appURL, keyName, keyName, "", path, true, useHostOnly, logger)
+	resourceData, err := transfer.Run(appURL, keyName, keyName, "", true, useHostOnly, logger)
 	if err != nil {
-		return "", err
+		return "", errors.Wrap(err, "failed to run transfer service")
+	}
+	var resp transferServiceResponse
+	if err = json.Unmarshal(resourceData, &resp); err != nil {
+		return "", errors.Wrap(err, "failed to marshal transfer service response")
 	}
 
-	return string(token), nil
+	// If we were able to get the auth domain and generate an org token path, lets write it to disk.
+	if orgTokenPath != "" {
+		if err := ioutil.WriteFile(orgTokenPath, []byte(resp.OrgToken), 0600); err != nil {
+			return "", errors.Wrap(err, "failed to write org token to disk")
+		}
+	}
+
+	if err := ioutil.WriteFile(appTokenPath, []byte(resp.AppToken), 0600); err != nil {
+		return "", errors.Wrap(err, "failed to write app token to disk")
+	}
+
+	return resp.AppToken, nil
+
 }
 
-// GetTokenIfExists will return the token from local storage if it exists and not expired
-func GetTokenIfExists(url *url.URL) (string, error) {
-	path, err := path.GenerateFilePathFromURL(url, keyName)
-	if err != nil {
-		return "", err
-	}
-	content, err := ioutil.ReadFile(path)
-	if err != nil {
-		return "", err
-	}
-	token, err := jose.ParseJWT(string(content))
-	if err != nil {
-		return "", err
+// getAuthDomain makes a request to the appURL and stops at the first redirect. The 302 location header will contain the
+// auth domain
+func getAuthDomain(appURL *url.URL) (string, error) {
+	client := &http.Client{
+		// do not follow redirects
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			return http.ErrUseLastResponse
+		},
+		Timeout: time.Second * 7,
 	}
 
-	var payload jwtPayload
+	authDomainReq, err := http.NewRequest("HEAD", appURL.String(), nil)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to create auth domain request")
+	}
+	resp, err := client.Do(authDomainReq)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to get auth domain")
+	}
+	resp.Body.Close()
+	location, err := resp.Location()
+	if err != nil {
+		return "", fmt.Errorf("failed to get auth domain. Received status code %d from %s", resp.StatusCode, appURL.String())
+	}
+	return location.Hostname(), nil
+
+}
+
+// exchangeOrgToken attaches an org token to a request to the appURL and returns an app token. This uses the Access SSO
+// flow to automatically generate and return an app token without the login page.
+func exchangeOrgToken(appURL *url.URL, orgToken string) (string, error) {
+	client := &http.Client{
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			// attach org token to login request
+			if strings.Contains(req.URL.Path, "cdn-cgi/access/login") {
+				req.AddCookie(&http.Cookie{Name: tokenHeader, Value: orgToken})
+			}
+			// stop after hitting authorized endpoint since it will contain the app token
+			if strings.Contains(via[len(via)-1].URL.Path, "cdn-cgi/access/authorized") {
+				return http.ErrUseLastResponse
+			}
+			return nil
+		},
+		Timeout: time.Second * 7,
+	}
+
+	appTokenRequest, err := http.NewRequest("HEAD", appURL.String(), nil)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to create app token request")
+	}
+	resp, err := client.Do(appTokenRequest)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to get app token")
+	}
+	resp.Body.Close()
+	var appToken string
+	for _, c := range resp.Cookies() {
+		if c.Name == tokenHeader {
+			appToken = c.Value
+			break
+		}
+	}
+
+	if len(appToken) > 0 {
+		return appToken, nil
+	}
+	return "", fmt.Errorf("response from %s did not contain app token", resp.Request.URL.String())
+}
+
+func GetOrgTokenIfExists(authDomain string) (string, error) {
+	path, err := path.GenerateOrgTokenFilePathFromURL(authDomain)
+	if err != nil {
+		return "", err
+	}
+	token, err := getTokenIfExists(path)
+	if err != nil {
+		return "", err
+	}
+	var payload orgJWTPayload
 	err = json.Unmarshal(token.Payload, &payload)
 	if err != nil {
 		return "", err
@@ -199,13 +331,49 @@ func GetTokenIfExists(url *url.URL) (string, error) {
 		err := os.Remove(path)
 		return "", err
 	}
-
 	return token.Encode(), nil
 }
 
+func GetAppTokenIfExists(url *url.URL) (string, error) {
+	path, err := path.GenerateAppTokenFilePathFromURL(url, keyName)
+	if err != nil {
+		return "", err
+	}
+	token, err := getTokenIfExists(path)
+	if err != nil {
+		return "", err
+	}
+	var payload appJWTPayload
+	err = json.Unmarshal(token.Payload, &payload)
+	if err != nil {
+		return "", err
+	}
+
+	if payload.isExpired() {
+		err := os.Remove(path)
+		return "", err
+	}
+	return token.Encode(), nil
+
+}
+
+// GetTokenIfExists will return the token from local storage if it exists and not expired
+func getTokenIfExists(path string) (*jose.JWT, error) {
+	content, err := ioutil.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	token, err := jose.ParseJWT(string(content))
+	if err != nil {
+		return nil, err
+	}
+
+	return &token, nil
+}
+
 // RemoveTokenIfExists removes the a token from local storage if it exists
 func RemoveTokenIfExists(url *url.URL) error {
-	path, err := path.GenerateFilePathFromURL(url, keyName)
+	path, err := path.GenerateAppTokenFilePathFromURL(url, keyName)
 	if err != nil {
 		return err
 	}

cmd/cloudflared/transfer/transfer.go

@@ -3,10 +3,8 @@ package transfer
 import (
 	"bytes"
 	"encoding/base64"
-	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"net/url"
 	"os"
@@ -15,6 +13,7 @@ import (
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/encrypter"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/shell"
 	"github.com/cloudflare/cloudflared/logger"
+	"github.com/pkg/errors"
 )
 
 const (
@@ -28,7 +27,7 @@ const (
 // The "dance" we refer to is building a HTTP request, opening that in a browser waiting for
 // the user to complete an action, while it long polls in the background waiting for an
 // action to be completed to download the resource.
-func Run(transferURL *url.URL, resourceName, key, value, path string, shouldEncrypt bool, useHostOnly bool, logger logger.Service) ([]byte, error) {
+func Run(transferURL *url.URL, resourceName, key, value string, shouldEncrypt bool, useHostOnly bool, logger logger.Service) ([]byte, error) {
 	encrypterClient, err := encrypter.New("cloudflared_priv.pem", "cloudflared_pub.pem")
 	if err != nil {
 		return nil, err
@@ -72,11 +71,8 @@ func Run(transferURL *url.URL, resourceName, key, value, path string, shouldEncr
 		resourceData = buf
 	}
 
-	if err := ioutil.WriteFile(path, resourceData, 0600); err != nil {
-		return nil, err
-	}
-
 	return resourceData, nil
+
 }
 
 // BuildRequestURL creates a request suitable for a resource transfer.
@@ -93,6 +89,7 @@ func buildRequestURL(baseURL *url.URL, key, value string, cli, useHostOnly bool)
 		return baseURL.String(), nil
 	}
 	q.Set("redirect_url", baseURL.String()) // we add the token as a query param on both the redirect_url and the main url
+	q.Set("send_org_token", "true")         // indicates that the cli endpoint should return both the org and app token
 	baseURL.RawQuery = q.Encode()           // and this actual baseURL.
 	baseURL.Path = "cdn-cgi/access/cli"
 	return baseURL.String(), nil

cmd/cloudflared/tunnel/cmd.go

@@ -5,43 +5,33 @@ import (
 	"context"
 	"fmt"
 	"io/ioutil"
-	"net"
-	"net/http"
 	"net/url"
 	"os"
 	"reflect"
-	"runtime"
 	"runtime/trace"
-	"strconv"
 	"strings"
 	"sync"
 	"time"
 
-	"github.com/cloudflare/cloudflared/awsuploader"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/ui"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/updater"
+	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/dbconnect"
-	"github.com/cloudflare/cloudflared/h2mux"
-	"github.com/cloudflare/cloudflared/hello"
+	"github.com/cloudflare/cloudflared/ingress"
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/metrics"
 	"github.com/cloudflare/cloudflared/origin"
 	"github.com/cloudflare/cloudflared/signal"
-	"github.com/cloudflare/cloudflared/socks"
-	"github.com/cloudflare/cloudflared/sshlog"
-	"github.com/cloudflare/cloudflared/sshserver"
 	"github.com/cloudflare/cloudflared/tlsconfig"
 	"github.com/cloudflare/cloudflared/tunneldns"
 	"github.com/cloudflare/cloudflared/tunnelstore"
-	"github.com/cloudflare/cloudflared/websocket"
 
 	"github.com/coreos/go-systemd/daemon"
 	"github.com/facebookgo/grace/gracenet"
 	"github.com/getsentry/raven-go"
-	"github.com/gliderlabs/ssh"
 	"github.com/google/uuid"
 	"github.com/mitchellh/go-homedir"
 	"github.com/pkg/errors"
@@ -84,20 +74,9 @@ const (
 	// hostKeyPath is the path of the dir to save SSH host keys too
 	hostKeyPath = "host-key-path"
 
-	//sshServerFlag enables cloudflared ssh proxy server
-	sshServerFlag = "ssh-server"
-
-	// socks5Flag is to enable the socks server to deframe
-	socks5Flag = "socks5"
-
-	// bastionFlag is to enable bastion, or jump host, operation
-	bastionFlag = "bastion"
-
 	// uiFlag is to enable launching cloudflared in interactive UI mode
 	uiFlag = "ui"
 
-	logDirectoryFlag = "log-directory"
-
 	debugLevelWarning = "At debug level, request URL, method, protocol, content legnth and header will be logged. " +
 		"Response status, content length and header will also be logged in debug level."
 )
@@ -228,47 +207,13 @@ func routeFromFlag(c *cli.Context) (tunnelstore.Route, bool) {
 	return nil, false
 }
 
-func createLogger(c *cli.Context, isTransport bool, disableTerminal bool) (*logger.OutputWriter, error) {
-	var loggerOpts []logger.Option
-
-	logPath := c.String("logfile")
-	if logPath == "" {
-		logPath = c.String(logDirectoryFlag)
-	}
-
-	if logPath != "" {
-		loggerOpts = append(loggerOpts, logger.DefaultFile(logPath))
-	}
-
-	logLevel := c.String("loglevel")
-	if isTransport {
-		logLevel = c.String("transport-loglevel")
-		if logLevel == "" {
-			logLevel = "fatal"
-		}
-	}
-	loggerOpts = append(loggerOpts, logger.LogLevelString(logLevel))
-
-	if disableTerminal {
-		disableOption := logger.DisableTerminal(true)
-		loggerOpts = append(loggerOpts, disableOption)
-	}
-
-	l, err := logger.New(loggerOpts...)
-	if err != nil {
-		return nil, err
-	}
-
-	return l, nil
-}
-
 func StartServer(
 	c *cli.Context,
 	version string,
 	shutdownC,
 	graceShutdownC chan struct{},
-	namedTunnel *origin.NamedTunnelConfig,
-	log logger.Service,
+	namedTunnel *connection.NamedTunnelConfig,
+	generalLogger logger.Service,
 	isUIEnabled bool,
 ) error {
 	_ = raven.SetDSN(sentryDSN)
@@ -279,45 +224,45 @@ func StartServer(
 	dnsReadySignal := make(chan struct{})
 
 	if config.GetConfiguration().Source() == "" {
-		log.Infof(config.ErrNoConfigFile.Error())
+		generalLogger.Infof(config.ErrNoConfigFile.Error())
 	}
 
 	if c.IsSet("trace-output") {
 		tmpTraceFile, err := ioutil.TempFile("", "trace")
 		if err != nil {
-			log.Errorf("Failed to create new temporary file to save trace output: %s", err)
+			generalLogger.Errorf("Failed to create new temporary file to save trace output: %s", err)
 		}
 
 		defer func() {
 			if err := tmpTraceFile.Close(); err != nil {
-				log.Errorf("Failed to close trace output file %s with error: %s", tmpTraceFile.Name(), err)
+				generalLogger.Errorf("Failed to close trace output file %s with error: %s", tmpTraceFile.Name(), err)
 			}
 			if err := os.Rename(tmpTraceFile.Name(), c.String("trace-output")); err != nil {
-				log.Errorf("Failed to rename temporary trace output file %s to %s with error: %s", tmpTraceFile.Name(), c.String("trace-output"), err)
+				generalLogger.Errorf("Failed to rename temporary trace output file %s to %s with error: %s", tmpTraceFile.Name(), c.String("trace-output"), err)
 			} else {
 				err := os.Remove(tmpTraceFile.Name())
 				if err != nil {
-					log.Errorf("Failed to remove the temporary trace file %s with error: %s", tmpTraceFile.Name(), err)
+					generalLogger.Errorf("Failed to remove the temporary trace file %s with error: %s", tmpTraceFile.Name(), err)
 				}
 			}
 		}()
 
 		if err := trace.Start(tmpTraceFile); err != nil {
-			log.Errorf("Failed to start trace: %s", err)
+			generalLogger.Errorf("Failed to start trace: %s", err)
 			return errors.Wrap(err, "Error starting tracing")
 		}
 		defer trace.Stop()
 	}
 
 	buildInfo := buildinfo.GetBuildInfo(version)
-	buildInfo.Log(log)
-	logClientOptions(c, log)
+	buildInfo.Log(generalLogger)
+	logClientOptions(c, generalLogger)
 
 	if c.IsSet("proxy-dns") {
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
-			errC <- runDNSProxyServer(c, dnsReadySignal, shutdownC, log)
+			errC <- runDNSProxyServer(c, dnsReadySignal, shutdownC, generalLogger)
 		}()
 	} else {
 		close(dnsReadySignal)
@@ -326,26 +271,14 @@ func StartServer(
 	// Wait for proxy-dns to come up (if used)
 	<-dnsReadySignal
 
-	metricsListener, err := listeners.Listen("tcp", c.String("metrics"))
-	if err != nil {
-		log.Errorf("Error opening metrics server listener: %s", err)
-		return errors.Wrap(err, "Error opening metrics server listener")
-	}
-	defer metricsListener.Close()
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
-		errC <- metrics.ServeMetrics(metricsListener, shutdownC, log)
-	}()
-
 	go notifySystemd(connectedSignal)
 	if c.IsSet("pidfile") {
-		go writePidFile(connectedSignal, c.String("pidfile"), log)
+		go writePidFile(connectedSignal, c.String("pidfile"), generalLogger)
 	}
 
 	cloudflaredID, err := uuid.NewRandom()
 	if err != nil {
-		log.Errorf("Cannot generate cloudflared ID: %s", err)
+		generalLogger.Errorf("Cannot generate cloudflared ID: %s", err)
 		return err
 	}
 
@@ -356,12 +289,12 @@ func StartServer(
 	}()
 
 	// update needs to be after DNS proxy is up to resolve equinox server address
-	if updater.IsAutoupdateEnabled(c, log) {
-		log.Infof("Autoupdate frequency is set to %v", c.Duration("autoupdate-freq"))
+	if updater.IsAutoupdateEnabled(c, generalLogger) {
+		generalLogger.Infof("Autoupdate frequency is set to %v", c.Duration("autoupdate-freq"))
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
-			autoupdater := updater.NewAutoUpdater(c.Duration("autoupdate-freq"), &listeners, log)
+			autoupdater := updater.NewAutoUpdater(c.Duration("autoupdate-freq"), &listeners, generalLogger)
 			errC <- autoupdater.Run(ctx)
 		}()
 	}
@@ -370,133 +303,52 @@ func StartServer(
 	if dnsProxyStandAlone(c) {
 		connectedSignal.Notify()
 		// no grace period, handle SIGINT/SIGTERM immediately
-		return waitToShutdown(&wg, errC, shutdownC, graceShutdownC, 0, log)
-	}
-
-	if c.IsSet("hello-world") {
-		log.Infof("hello-world set")
-		helloListener, err := hello.CreateTLSListener("127.0.0.1:")
-		if err != nil {
-			log.Errorf("Cannot start Hello World Server: %s", err)
-			return errors.Wrap(err, "Cannot start Hello World Server")
-		}
-		defer helloListener.Close()
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			_ = hello.StartHelloWorldServer(log, helloListener, shutdownC)
-		}()
-		forceSetFlag(c, "url", "https://"+helloListener.Addr().String())
-	}
-
-	if c.IsSet(sshServerFlag) {
-		if runtime.GOOS != "darwin" && runtime.GOOS != "linux" {
-			msg := fmt.Sprintf("--ssh-server is not supported on %s", runtime.GOOS)
-			log.Error(msg)
-			return errors.New(msg)
-		}
-
-		log.Infof("ssh-server set")
-
-		logManager := sshlog.NewEmptyManager()
-		if c.IsSet(bucketNameFlag) && c.IsSet(regionNameFlag) && c.IsSet(accessKeyIDFlag) && c.IsSet(secretIDFlag) {
-			uploader, err := awsuploader.NewFileUploader(c.String(bucketNameFlag), c.String(regionNameFlag),
-				c.String(accessKeyIDFlag), c.String(secretIDFlag), c.String(sessionTokenIDFlag), c.String(s3URLFlag))
-			if err != nil {
-				msg := "Cannot create uploader for SSH Server"
-				log.Errorf("%s: %s", msg, err)
-				return errors.Wrap(err, msg)
-			}
-
-			if err := os.MkdirAll(sshLogFileDirectory, 0700); err != nil {
-				msg := fmt.Sprintf("Cannot create SSH log file directory %s", sshLogFileDirectory)
-				log.Errorf("%s: %s", msg, err)
-				return errors.Wrap(err, msg)
-			}
-
-			logManager = sshlog.New(sshLogFileDirectory)
-
-			uploadManager := awsuploader.NewDirectoryUploadManager(log, uploader, sshLogFileDirectory, 30*time.Minute, shutdownC)
-			uploadManager.Start()
-		}
-
-		localServerAddress := "127.0.0.1:" + c.String(sshPortFlag)
-		server, err := sshserver.New(logManager, log, version, localServerAddress, c.String("hostname"), c.Path(hostKeyPath), shutdownC, c.Duration(sshIdleTimeoutFlag), c.Duration(sshMaxTimeoutFlag))
-		if err != nil {
-			msg := "Cannot create new SSH Server"
-			log.Errorf("%s: %s", msg, err)
-			return errors.Wrap(err, msg)
-		}
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			if err = server.Start(); err != nil && err != ssh.ErrServerClosed {
-				log.Errorf("SSH server error: %s", err)
-				// TODO: remove when declarative tunnels are implemented.
-				close(shutdownC)
-			}
-		}()
-		forceSetFlag(c, "url", "ssh://"+localServerAddress)
+		return waitToShutdown(&wg, errC, shutdownC, graceShutdownC, 0, generalLogger)
 	}
 
 	url := c.String("url")
 	hostname := c.String("hostname")
 	if url == hostname && url != "" && hostname != "" {
 		errText := "hostname and url shouldn't match. See --help for more information"
-		log.Error(errText)
+		generalLogger.Error(errText)
 		return fmt.Errorf(errText)
 	}
 
-	if staticHost := hostnameFromURI(c.String("url")); isProxyDestinationConfigured(staticHost, c) {
-		listener, err := net.Listen("tcp", net.JoinHostPort(c.String("proxy-address"), strconv.Itoa(c.Int("proxy-port"))))
-		if err != nil {
-			log.Errorf("Cannot start Websocket Proxy Server: %s", err)
-			return errors.Wrap(err, "Cannot start Websocket Proxy Server")
-		}
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			streamHandler := websocket.DefaultStreamHandler
-			if c.IsSet(socks5Flag) {
-				log.Info("SOCKS5 server started")
-				streamHandler = func(wsConn *websocket.Conn, remoteConn net.Conn, _ http.Header) {
-					dialer := socks.NewConnDialer(remoteConn)
-					requestHandler := socks.NewRequestHandler(dialer)
-					socksServer := socks.NewConnectionHandler(requestHandler)
-
-					socksServer.Serve(wsConn)
-				}
-			} else if c.IsSet(sshServerFlag) {
-				streamHandler = func(wsConn *websocket.Conn, remoteConn net.Conn, requestHeaders http.Header) {
-					if finalDestination := requestHeaders.Get(h2mux.CFJumpDestinationHeader); finalDestination != "" {
-						token := requestHeaders.Get(h2mux.CFAccessTokenHeader)
-						if err := websocket.SendSSHPreamble(remoteConn, finalDestination, token); err != nil {
-							log.Errorf("Failed to send SSH preamble: %s", err)
-							return
-						}
-					}
-					websocket.DefaultStreamHandler(wsConn, remoteConn, requestHeaders)
-				}
-			}
-			errC <- websocket.StartProxyServer(log, listener, staticHost, shutdownC, streamHandler)
-		}()
-		forceSetFlag(c, "url", "http://"+listener.Addr().String())
-	}
-
-	transportLogger, err := createLogger(c, true, false)
+	transportLogger, err := logger.CreateTransportLoggerFromContext(c, isUIEnabled)
 	if err != nil {
 		return errors.Wrap(err, "error setting up transport logger")
 	}
 
-	tunnelConfig, err := prepareTunnelConfig(c, buildInfo, version, log, transportLogger, namedTunnel)
+	readinessCh := make(chan connection.Event, 16)
+	uiCh := make(chan connection.Event, 16)
+	eventChannels := []chan connection.Event{
+		readinessCh,
+		uiCh,
+	}
+	tunnelConfig, ingressRules, err := prepareTunnelConfig(c, buildInfo, version, generalLogger, transportLogger, namedTunnel, isUIEnabled, eventChannels)
 	if err != nil {
+		generalLogger.Errorf("Couldn't start tunnel: %v", err)
 		return err
 	}
 
+	metricsListener, err := listeners.Listen("tcp", c.String("metrics"))
+	if err != nil {
+		generalLogger.Errorf("Error opening metrics server listener: %s", err)
+		return errors.Wrap(err, "Error opening metrics server listener")
+	}
+	defer metricsListener.Close()
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		errC <- metrics.ServeMetrics(metricsListener, shutdownC, readinessCh, generalLogger)
+	}()
+
+	ingressRules.StartOrigins(&wg, generalLogger, shutdownC, errC)
+
 	reconnectCh := make(chan origin.ReconnectSignal, 1)
 	if c.IsSet("stdin-control") {
-		log.Info("Enabling control through stdin")
-		go stdinControl(reconnectCh, log)
+		generalLogger.Info("Enabling control through stdin")
+		go stdinControl(reconnectCh, generalLogger)
 	}
 
 	wg.Add(1)
@@ -506,25 +358,21 @@ func StartServer(
 	}()
 
 	if isUIEnabled {
-		const tunnelEventChanBufferSize = 16
-		tunnelEventChan := make(chan ui.TunnelEvent, tunnelEventChanBufferSize)
-		tunnelConfig.TunnelEventChan = tunnelEventChan
-
 		tunnelInfo := ui.NewUIModel(
 			version,
 			hostname,
 			metricsListener.Addr().String(),
-			tunnelConfig.OriginUrl,
+			&ingressRules,
 			tunnelConfig.HAConnections,
 		)
 		logLevels, err := logger.ParseLevelString(c.String("loglevel"))
 		if err != nil {
 			return err
 		}
-		tunnelInfo.LaunchUI(ctx, log, logLevels, tunnelEventChan)
+		tunnelInfo.LaunchUI(ctx, generalLogger, transportLogger, logLevels, uiCh)
 	}
 
-	return waitToShutdown(&wg, errC, shutdownC, graceShutdownC, c.Duration("grace-period"), log)
+	return waitToShutdown(&wg, errC, shutdownC, graceShutdownC, c.Duration("grace-period"), generalLogger)
 }
 
 // forceSetFlag attempts to set the given flag value in the closest context that has it defined
@@ -538,7 +386,7 @@ func forceSetFlag(c *cli.Context, name, value string) {
 
 func SetFlagsFromConfigFile(c *cli.Context) error {
 	const exitCode = 1
-	log, err := createLogger(c, false, false)
+	log, err := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
 	if err != nil {
 		return cliutil.PrintLoggerSetupError("error setting up logger", err)
 	}
@@ -559,11 +407,6 @@ func SetFlagsFromConfigFile(c *cli.Context) error {
 	return nil
 }
 
-// isProxyDestinationConfigured returns true if there is a static host set or if bastion mode is set.
-func isProxyDestinationConfigured(staticHost string, c *cli.Context) bool {
-	return staticHost != "" || c.IsSet(bastionFlag)
-}
-
 func waitToShutdown(wg *sync.WaitGroup,
 	errC chan error,
 	shutdownC, graceShutdownC chan struct{},
@@ -843,6 +686,7 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			Value:  false,
 			Hidden: shouldHide,
 		}),
+		selectProtocolFlag,
 	}...)
 
 	return flags
@@ -910,67 +754,67 @@ func configureProxyFlags(shouldHide bool) []cli.Flag {
 			Hidden:  shouldHide,
 		}),
 		altsrc.NewBoolFlag(&cli.BoolFlag{
-			Name:    socks5Flag,
+			Name:    ingress.Socks5Flag,
 			Usage:   "specify if this tunnel is running as a SOCK5 Server",
 			EnvVars: []string{"TUNNEL_SOCKS"},
 			Value:   false,
 			Hidden:  shouldHide,
 		}),
 		altsrc.NewDurationFlag(&cli.DurationFlag{
-			Name:   "proxy-connect-timeout",
+			Name:   ingress.ProxyConnectTimeoutFlag,
 			Usage:  "HTTP proxy timeout for establishing a new connection",
 			Value:  time.Second * 30,
 			Hidden: shouldHide,
 		}),
 		altsrc.NewDurationFlag(&cli.DurationFlag{
-			Name:   "proxy-tls-timeout",
+			Name:   ingress.ProxyTLSTimeoutFlag,
 			Usage:  "HTTP proxy timeout for completing a TLS handshake",
 			Value:  time.Second * 10,
 			Hidden: shouldHide,
 		}),
 		altsrc.NewDurationFlag(&cli.DurationFlag{
-			Name:   "proxy-tcp-keepalive",
+			Name:   ingress.ProxyTCPKeepAlive,
 			Usage:  "HTTP proxy TCP keepalive duration",
 			Value:  time.Second * 30,
 			Hidden: shouldHide,
 		}),
 		altsrc.NewBoolFlag(&cli.BoolFlag{
-			Name:   "proxy-no-happy-eyeballs",
+			Name:   ingress.ProxyNoHappyEyeballsFlag,
 			Usage:  "HTTP proxy should disable \"happy eyeballs\" for IPv4/v6 fallback",
 			Hidden: shouldHide,
 		}),
 		altsrc.NewIntFlag(&cli.IntFlag{
-			Name:   "proxy-keepalive-connections",
+			Name:   ingress.ProxyKeepAliveConnectionsFlag,
 			Usage:  "HTTP proxy maximum keepalive connection pool size",
 			Value:  100,
 			Hidden: shouldHide,
 		}),
 		altsrc.NewDurationFlag(&cli.DurationFlag{
-			Name:   "proxy-keepalive-timeout",
+			Name:   ingress.ProxyKeepAliveTimeoutFlag,
 			Usage:  "HTTP proxy timeout for closing an idle connection",
 			Value:  time.Second * 90,
 			Hidden: shouldHide,
 		}),
 		altsrc.NewDurationFlag(&cli.DurationFlag{
 			Name:   "proxy-connection-timeout",
-			Usage:  "HTTP proxy timeout for closing an idle connection",
+			Usage:  "DEPRECATED. No longer has any effect.",
 			Value:  time.Second * 90,
 			Hidden: shouldHide,
 		}),
 		altsrc.NewDurationFlag(&cli.DurationFlag{
 			Name:   "proxy-expect-continue-timeout",
-			Usage:  "HTTP proxy timeout for closing an idle connection",
+			Usage:  "DEPRECATED. No longer has any effect.",
 			Value:  time.Second * 90,
 			Hidden: shouldHide,
 		}),
 		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:    "http-host-header",
+			Name:    ingress.HTTPHostHeaderFlag,
 			Usage:   "Sets the HTTP Host header for the local webserver.",
 			EnvVars: []string{"TUNNEL_HTTP_HOST_HEADER"},
 			Hidden:  shouldHide,
 		}),
 		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:    "origin-server-name",
+			Name:    ingress.OriginServerNameFlag,
 			Usage:   "Hostname on the origin server certificate.",
 			EnvVars: []string{"TUNNEL_ORIGIN_SERVER_NAME"},
 			Hidden:  shouldHide,
@@ -988,13 +832,13 @@ func configureProxyFlags(shouldHide bool) []cli.Flag {
 			Hidden:  shouldHide,
 		}),
 		altsrc.NewBoolFlag(&cli.BoolFlag{
-			Name:    "no-tls-verify",
+			Name:    ingress.NoTLSVerifyFlag,
 			Usage:   "Disables TLS verification of the certificate presented by your origin. Will allow any certificate from the origin to be accepted. Note: The connection from your machine to Cloudflare's Edge is still encrypted.",
 			EnvVars: []string{"NO_TLS_VERIFY"},
 			Hidden:  shouldHide,
 		}),
 		altsrc.NewBoolFlag(&cli.BoolFlag{
-			Name:    "no-chunked-encoding",
+			Name:    ingress.NoChunkedEncodingFlag,
 			Usage:   "Disables chunked transfer encoding; useful if you are running a WSGI server.",
 			EnvVars: []string{"TUNNEL_NO_CHUNKED_ENCODING"},
 			Hidden:  shouldHide,
@@ -1067,28 +911,28 @@ func sshFlags(shouldHide bool) []cli.Flag {
 			Hidden:  true,
 		}),
 		altsrc.NewBoolFlag(&cli.BoolFlag{
-			Name:    sshServerFlag,
+			Name:    ingress.SSHServerFlag,
 			Value:   false,
 			Usage:   "Run an SSH Server",
 			EnvVars: []string{"TUNNEL_SSH_SERVER"},
 			Hidden:  true, // TODO: remove when feature is complete
 		}),
 		altsrc.NewBoolFlag(&cli.BoolFlag{
-			Name:    bastionFlag,
+			Name:    config.BastionFlag,
 			Value:   false,
 			Usage:   "Runs as jump host",
 			EnvVars: []string{"TUNNEL_BASTION"},
 			Hidden:  shouldHide,
 		}),
 		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:    "proxy-address",
+			Name:    ingress.ProxyAddressFlag,
 			Usage:   "Listen address for the proxy.",
 			Value:   "127.0.0.1",
 			EnvVars: []string{"TUNNEL_PROXY_ADDRESS"},
 			Hidden:  shouldHide,
 		}),
 		altsrc.NewIntFlag(&cli.IntFlag{
-			Name:    "proxy-port",
+			Name:    ingress.ProxyPortFlag,
 			Usage:   "Listen port for the proxy.",
 			Value:   0,
 			EnvVars: []string{"TUNNEL_PROXY_PORT"},
@@ -1100,28 +944,28 @@ func sshFlags(shouldHide bool) []cli.Flag {
 func configureLoggingFlags(shouldHide bool) []cli.Flag {
 	return []cli.Flag{
 		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:    "loglevel",
+			Name:    logger.LogLevelFlag,
 			Value:   "info",
 			Usage:   "Application logging level {fatal, error, info, debug}. " + debugLevelWarning,
 			EnvVars: []string{"TUNNEL_LOGLEVEL"},
 			Hidden:  shouldHide,
 		}),
 		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:    "transport-loglevel",
+			Name:    logger.LogTransportLevelFlag,
 			Aliases: []string{"proto-loglevel"}, // This flag used to be called proto-loglevel
-			Value:   "fatal",
+			Value:   "info",
 			Usage:   "Transport logging level(previously called protocol logging level) {fatal, error, info, debug}",
 			EnvVars: []string{"TUNNEL_PROTO_LOGLEVEL", "TUNNEL_TRANSPORT_LOGLEVEL"},
 			Hidden:  shouldHide,
 		}),
 		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:    "logfile",
+			Name:    logger.LogFileFlag,
 			Usage:   "Save application log to this file for reporting issues.",
 			EnvVars: []string{"TUNNEL_LOGFILE"},
 			Hidden:  shouldHide,
 		}),
 		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:    logDirectoryFlag,
+			Name:    logger.LogDirectoryFlag,
 			Usage:   "Save application log to this directory for reporting issues.",
 			EnvVars: []string{"TUNNEL_LOGDIRECTORY"},
 			Hidden:  shouldHide,

cmd/cloudflared/tunnel/configuration.go

@@ -1,19 +1,18 @@
 package tunnel
 
 import (
-	"context"
 	"crypto/tls"
 	"fmt"
 	"io/ioutil"
-	"net"
-	"net/http"
 	"os"
 	"path/filepath"
 	"strings"
-	"time"
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/connection"
+	"github.com/cloudflare/cloudflared/edgediscovery"
+	"github.com/cloudflare/cloudflared/h2mux"
 	"github.com/cloudflare/cloudflared/ingress"
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/origin"
@@ -159,28 +158,30 @@ func prepareTunnelConfig(
 	version string,
 	logger logger.Service,
 	transportLogger logger.Service,
-	namedTunnel *origin.NamedTunnelConfig,
-) (*origin.TunnelConfig, error) {
-	compatibilityMode := namedTunnel == nil
+	namedTunnel *connection.NamedTunnelConfig,
+	isUIEnabled bool,
+	eventChans []chan connection.Event,
+) (*origin.TunnelConfig, ingress.Ingress, error) {
+	isNamedTunnel := namedTunnel != nil
 
 	hostname, err := validation.ValidateHostname(c.String("hostname"))
 	if err != nil {
 		logger.Errorf("Invalid hostname: %s", err)
-		return nil, errors.Wrap(err, "Invalid hostname")
+		return nil, ingress.Ingress{}, errors.Wrap(err, "Invalid hostname")
 	}
 	isFreeTunnel := hostname == ""
 	clientID := c.String("id")
 	if !c.IsSet("id") {
 		clientID, err = generateRandomClientID(logger)
 		if err != nil {
-			return nil, err
+			return nil, ingress.Ingress{}, err
 		}
 	}
 
 	tags, err := NewTagSliceFromCLI(c.StringSlice("tag"))
 	if err != nil {
 		logger.Errorf("Tag parse failure: %s", err)
-		return nil, errors.Wrap(err, "Tag parse failure")
+		return nil, ingress.Ingress{}, errors.Wrap(err, "Tag parse failure")
 	}
 
 	tags = append(tags, tunnelpogs.Tag{Name: "ID", Value: clientID})
@@ -189,41 +190,18 @@ func prepareTunnelConfig(
 	if !isFreeTunnel {
 		originCert, err = getOriginCert(c, logger)
 		if err != nil {
-			return nil, errors.Wrap(err, "Error getting origin cert")
+			return nil, ingress.Ingress{}, errors.Wrap(err, "Error getting origin cert")
 		}
 	}
 
-	originCertPool, err := tlsconfig.LoadOriginCA(c, logger)
-	if err != nil {
-		logger.Errorf("Error loading cert pool: %s", err)
-		return nil, errors.Wrap(err, "Error loading cert pool")
-	}
-
-	tunnelMetrics := origin.NewTunnelMetrics()
-	httpTransport := &http.Transport{
-		Proxy:                 http.ProxyFromEnvironment,
-		MaxIdleConns:          c.Int("proxy-keepalive-connections"),
-		MaxIdleConnsPerHost:   c.Int("proxy-keepalive-connections"),
-		IdleConnTimeout:       c.Duration("proxy-keepalive-timeout"),
-		TLSHandshakeTimeout:   c.Duration("proxy-tls-timeout"),
-		ExpectContinueTimeout: 1 * time.Second,
-		TLSClientConfig:       &tls.Config{RootCAs: originCertPool, InsecureSkipVerify: c.IsSet("no-tls-verify")},
-	}
-
-	dialer := &net.Dialer{
-		Timeout:   c.Duration("proxy-connect-timeout"),
-		KeepAlive: c.Duration("proxy-tcp-keepalive"),
-	}
-	if c.Bool("proxy-no-happy-eyeballs") {
-		dialer.FallbackDelay = -1 // As of Golang 1.12, a negative delay disables "happy eyeballs"
-	}
-	dialContext := dialer.DialContext
-
-	var ingressRules ingress.Ingress
-	if namedTunnel != nil {
+	var (
+		ingressRules  ingress.Ingress
+		classicTunnel *connection.ClassicTunnelConfig
+	)
+	if isNamedTunnel {
 		clientUUID, err := uuid.NewRandom()
 		if err != nil {
-			return nil, errors.Wrap(err, "can't generate clientUUID")
+			return nil, ingress.Ingress{}, errors.Wrap(err, "can't generate clientUUID")
 		}
 		namedTunnel.Client = tunnelpogs.ClientInfo{
 			ClientID: clientUUID[:],
@@ -233,103 +211,79 @@ func prepareTunnelConfig(
 		}
 		ingressRules, err = ingress.ParseIngress(config.GetConfiguration())
 		if err != nil && err != ingress.ErrNoIngressRules {
-			return nil, err
+			return nil, ingress.Ingress{}, err
 		}
 		if !ingressRules.IsEmpty() && c.IsSet("url") {
-			return nil, ingress.ErrURLIncompatibleWithIngress
-		}
-	}
-
-	var originURL string
-	if ingressRules.IsEmpty() {
-		originURL, err = config.ValidateUrl(c, compatibilityMode)
-		if err != nil {
-			logger.Errorf("Error validating origin URL: %s", err)
-			return nil, errors.Wrap(err, "Error validating origin URL")
-		}
-	}
-
-	if c.IsSet("unix-socket") {
-		unixSocket, err := config.ValidateUnixSocket(c)
-		if err != nil {
-			logger.Errorf("Error validating --unix-socket: %s", err)
-			return nil, errors.Wrap(err, "Error validating --unix-socket")
-		}
-
-		logger.Infof("Proxying tunnel requests to unix:%s", unixSocket)
-		httpTransport.DialContext = func(ctx context.Context, _, _ string) (net.Conn, error) {
-			// if --unix-socket specified, enforce network type "unix"
-			return dialContext(ctx, "unix", unixSocket)
+			return nil, ingress.Ingress{}, ingress.ErrURLIncompatibleWithIngress
 		}
 	} else {
-		logger.Infof("Proxying tunnel requests to %s", originURL)
-		httpTransport.DialContext = dialContext
-	}
-
-	if !c.IsSet("hello-world") && c.IsSet("origin-server-name") {
-		httpTransport.TLSClientConfig.ServerName = c.String("origin-server-name")
-	}
-	// If tunnel running in bastion mode, a connection to origin will not exist until initiated by the client.
-	if !c.IsSet(bastionFlag) {
-
-		// List all origin URLs that require validation
-		var originURLs []string
-		if ingressRules.IsEmpty() {
-			originURLs = append(originURLs, originURL)
-		} else {
-			for _, rule := range ingressRules.Rules {
-				originURLs = append(originURLs, rule.Service.String())
-			}
-		}
-
-		// Validate each origin URL
-		for _, u := range originURLs {
-			if err = validation.ValidateHTTPService(u, hostname, httpTransport); err != nil {
-				logger.Errorf("unable to connect to the origin: %s", err)
-			}
+		classicTunnel = &connection.ClassicTunnelConfig{
+			Hostname:   hostname,
+			OriginCert: originCert,
+			// turn off use of reconnect token and auth refresh when using named tunnels
+			UseReconnectToken: !isNamedTunnel && c.Bool("use-reconnect-token"),
 		}
 	}
 
-	toEdgeTLSConfig, err := tlsconfig.CreateTunnelConfig(c)
+	// Convert single-origin configuration into multi-origin configuration.
+	if ingressRules.IsEmpty() {
+		ingressRules, err = ingress.NewSingleOrigin(c, !isNamedTunnel, logger)
+		if err != nil {
+			return nil, ingress.Ingress{}, err
+		}
+	}
+
+	protocolSelector, err := connection.NewProtocolSelector(c.String("protocol"), namedTunnel, edgediscovery.HTTP2Percentage, origin.ResolveTTL, logger)
 	if err != nil {
-		logger.Errorf("unable to create TLS config to connect with edge: %s", err)
-		return nil, errors.Wrap(err, "unable to create TLS config to connect with edge")
+		return nil, ingress.Ingress{}, err
 	}
-	return &origin.TunnelConfig{
-		BuildInfo:          buildInfo,
-		ClientID:           clientID,
-		ClientTlsConfig:    httpTransport.TLSClientConfig,
-		CompressionQuality: c.Uint64("compression-quality"),
-		EdgeAddrs:          c.StringSlice("edge"),
-		GracePeriod:        c.Duration("grace-period"),
-		HAConnections:      c.Int("ha-connections"),
-		HTTPTransport:      httpTransport,
+	logger.Infof("Initial protocol %s", protocolSelector.Current())
+
+	edgeTLSConfigs := make(map[connection.Protocol]*tls.Config, len(connection.ProtocolList))
+	for _, p := range connection.ProtocolList {
+		edgeTLSConfig, err := tlsconfig.CreateTunnelConfig(c, p.ServerName())
+		if err != nil {
+			return nil, ingress.Ingress{}, errors.Wrap(err, "unable to create TLS config to connect with edge")
+		}
+		edgeTLSConfigs[p] = edgeTLSConfig
+	}
+
+	originClient := origin.NewClient(ingressRules, tags, logger)
+	connectionConfig := &connection.Config{
+		OriginClient:    originClient,
+		GracePeriod:     c.Duration("grace-period"),
+		ReplaceExisting: c.Bool("force"),
+	}
+	muxerConfig := &connection.MuxerConfig{
 		HeartbeatInterval:  c.Duration("heartbeat-interval"),
-		Hostname:           hostname,
-		HTTPHostHeader:     c.String("http-host-header"),
-		IncidentLookup:     origin.NewIncidentLookup(),
-		IsAutoupdated:      c.Bool("is-autoupdated"),
-		IsFreeTunnel:       isFreeTunnel,
-		LBPool:             c.String("lb-pool"),
-		Logger:             logger,
-		TransportLogger:    transportLogger,
 		MaxHeartbeats:      c.Uint64("heartbeat-count"),
-		Metrics:            tunnelMetrics,
+		CompressionSetting: h2mux.CompressionSetting(c.Uint64("compression-quality")),
 		MetricsUpdateFreq:  c.Duration("metrics-update-freq"),
-		NoChunkedEncoding:  c.Bool("no-chunked-encoding"),
-		OriginCert:         originCert,
-		OriginUrl:          originURL,
-		ReportedVersion:    version,
-		Retries:            c.Uint("retries"),
-		RunFromTerminal:    isRunningFromTerminal(),
-		Tags:               tags,
-		TlsConfig:          toEdgeTLSConfig,
-		NamedTunnel:        namedTunnel,
-		ReplaceExisting:    c.Bool("force"),
-		IngressRules:       ingressRules,
-		// turn off use of reconnect token and auth refresh when using named tunnels
-		UseReconnectToken: compatibilityMode && c.Bool("use-reconnect-token"),
-	}, nil
+	}
+
+	return &origin.TunnelConfig{
+		ConnectionConfig: connectionConfig,
+		BuildInfo:        buildInfo,
+		ClientID:         clientID,
+		EdgeAddrs:        c.StringSlice("edge"),
+		HAConnections:    c.Int("ha-connections"),
+		IncidentLookup:   origin.NewIncidentLookup(),
+		IsAutoupdated:    c.Bool("is-autoupdated"),
+		IsFreeTunnel:     isFreeTunnel,
+		LBPool:           c.String("lb-pool"),
+		Tags:             tags,
+		Logger:           logger,
+		Observer:         connection.NewObserver(transportLogger, eventChans, isUIEnabled),
+		ReportedVersion:  version,
+		Retries:          c.Uint("retries"),
+		RunFromTerminal:  isRunningFromTerminal(),
+		NamedTunnel:      namedTunnel,
+		ClassicTunnel:    classicTunnel,
+		MuxerConfig:      muxerConfig,
+		TunnelEventChans: eventChans,
+		ProtocolSelector: protocolSelector,
+		EdgeTLSConfigs:   edgeTLSConfigs,
+	}, ingressRules, nil
 }
 
 func isRunningFromTerminal() bool {

cmd/cloudflared/tunnel/credential_finder.go

@@ -0,0 +1,78 @@
+package tunnel
+
+import (
+	"fmt"
+	"path/filepath"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/logger"
+	"github.com/google/uuid"
+	"github.com/urfave/cli/v2"
+)
+
+// CredFinder can find the tunnel credentials file.
+type CredFinder interface {
+	Path() (string, error)
+}
+
+// Implements CredFinder and looks for the credentials file at the given
+// filepath.
+type staticPath struct {
+	filePath string
+	fs       fileSystem
+}
+
+func newStaticPath(filePath string, fs fileSystem) CredFinder {
+	return staticPath{
+		filePath: filePath,
+		fs:       fs,
+	}
+}
+
+func (a staticPath) Path() (string, error) {
+	if a.filePath != "" && a.fs.validFilePath(a.filePath) {
+		return a.filePath, nil
+	}
+	return "", fmt.Errorf("Tunnel credentials file '%s' doesn't exist or is not a file", a.filePath)
+}
+
+// Implements CredFinder and looks for the credentials file in several directories
+// searching for a file named <id>.json
+type searchByID struct {
+	id     uuid.UUID
+	c      *cli.Context
+	logger logger.Service
+	fs     fileSystem
+}
+
+func newSearchByID(id uuid.UUID, c *cli.Context, logger logger.Service, fs fileSystem) CredFinder {
+	return searchByID{
+		id:     id,
+		c:      c,
+		logger: logger,
+		fs:     fs,
+	}
+}
+
+func (s searchByID) Path() (string, error) {
+
+	// Fallback to look for tunnel credentials in the origin cert directory
+	if originCertPath, err := findOriginCert(s.c, s.logger); err == nil {
+		originCertDir := filepath.Dir(originCertPath)
+		if filePath, err := tunnelFilePath(s.id, originCertDir); err == nil {
+			if s.fs.validFilePath(filePath) {
+				return filePath, nil
+			}
+		}
+	}
+
+	// Last resort look under default config directories
+	for _, configDir := range config.DefaultConfigSearchDirectories() {
+		if filePath, err := tunnelFilePath(s.id, configDir); err == nil {
+			if s.fs.validFilePath(filePath) {
+				return filePath, nil
+			}
+		}
+	}
+	return "", fmt.Errorf("Tunnel credentials file not found")
+}

cmd/cloudflared/tunnel/filesystem.go

@@ -0,0 +1,27 @@
+package tunnel
+
+import (
+	"io/ioutil"
+	"os"
+)
+
+// Abstract away details of reading files, so that SubcommandContext can read
+// from either the real filesystem, or a mock (when running unit tests).
+type fileSystem interface {
+	readFile(filePath string) ([]byte, error)
+	validFilePath(path string) bool
+}
+
+type realFileSystem struct{}
+
+func (fs realFileSystem) validFilePath(path string) bool {
+	fileStat, err := os.Stat(path)
+	if err != nil {
+		return false
+	}
+	return !fileStat.IsDir()
+}
+
+func (fs realFileSystem) readFile(filePath string) ([]byte, error) {
+	return ioutil.ReadFile(filePath)
+}

cmd/cloudflared/tunnel/ingress_subcommands.go

@@ -70,6 +70,10 @@ func buildTestURLCommand() *cli.Command {
 // validateIngressCommand check the syntax of the ingress rules in the cloudflared config file
 func validateIngressCommand(c *cli.Context) error {
 	conf := config.GetConfiguration()
+	if conf.Source() == "" {
+		fmt.Println("No configuration file was found. Please create one, or use the --config flag to specify its filepath. You can use the help command to learn more about configuration files")
+		return nil
+	}
 	fmt.Println("Validating rules from", conf.Source())
 	if _, err := ingress.ParseIngress(conf); err != nil {
 		return errors.Wrap(err, "Validation failed")
@@ -103,7 +107,7 @@ func testURLCommand(c *cli.Context) error {
 		return errors.Wrap(err, "Validation failed")
 	}
 
-	i := ing.FindMatchingRule(requestURL.Hostname(), requestURL.Path)
+	_, i := ing.FindMatchingRule(requestURL.Hostname(), requestURL.Path)
 	fmt.Printf("Matched rule #%d\n", i+1)
 	fmt.Println(ing.Rules[i].MultiLineString())
 	return nil

cmd/cloudflared/tunnel/login.go

@@ -2,6 +2,7 @@ package tunnel
 
 import (
 	"fmt"
+	"io/ioutil"
 	"net/url"
 	"os"
 	"path/filepath"
@@ -39,7 +40,7 @@ func buildLoginSubcommand(hidden bool) *cli.Command {
 }
 
 func login(c *cli.Context) error {
-	logger, err := logger.New()
+	logger, err := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
 	if err != nil {
 		return errors.Wrap(err, "error setting up logger")
 	}
@@ -58,12 +59,16 @@ func login(c *cli.Context) error {
 		return err
 	}
 
-	_, err = transfer.Run(loginURL, "cert", "callback", callbackStoreURL, path, false, false, logger)
+	resourceData, err := transfer.Run(loginURL, "cert", "callback", callbackStoreURL, false, false, logger)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "Failed to write the certificate due to the following error:\n%v\n\nYour browser will download the certificate instead. You will have to manually\ncopy it to the following path:\n\n%s\n", err, path)
 		return err
 	}
 
+	if err := ioutil.WriteFile(path, resourceData, 0600); err != nil {
+		return errors.Wrap(err, fmt.Sprintf("error writing cert to %s", path))
+	}
+
 	fmt.Fprintf(os.Stdout, "You have successfully logged in.\nIf you wish to copy your credentials to a server, they have been saved to:\n%s\n", path)
 	return nil
 }

cmd/cloudflared/tunnel/subcommand_context.go

@@ -3,9 +3,7 @@ package tunnel
 import (
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"os"
-	"path/filepath"
 	"strings"
 
 	"github.com/google/uuid"
@@ -13,31 +11,38 @@ import (
 	"github.com/urfave/cli/v2"
 
 	"github.com/cloudflare/cloudflared/certutil"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/logger"
-	"github.com/cloudflare/cloudflared/origin"
-	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	"github.com/cloudflare/cloudflared/tunnelstore"
 )
 
+type errInvalidJSONCredential struct {
+	err  error
+	path string
+}
+
+func (e errInvalidJSONCredential) Error() string {
+	return "Invalid JSON when parsing tunnel credentials file"
+}
+
 // subcommandContext carries structs shared between subcommands, to reduce number of arguments needed to
 // pass between subcommands, and make sure they are only initialized once
 type subcommandContext struct {
-	c      *cli.Context
-	logger logger.Service
+	c           *cli.Context
+	logger      logger.Service
+	isUIEnabled bool
+	fs          fileSystem
 
 	// These fields should be accessed using their respective Getter
 	tunnelstoreClient tunnelstore.Client
 	userCredential    *userCredential
-
-	isUIEnabled bool
 }
 
 func newSubcommandContext(c *cli.Context) (*subcommandContext, error) {
 	isUIEnabled := c.IsSet(uiFlag) && c.String("name") != ""
 
 	// If UI is enabled, terminal log output should be disabled -- log should be written into a UI log window instead
-	logger, err := createLogger(c, false, isUIEnabled)
+	logger, err := logger.CreateLoggerFromContext(c, isUIEnabled)
 	if err != nil {
 		return nil, errors.Wrap(err, "error setting up logger")
 	}
@@ -46,9 +51,18 @@ func newSubcommandContext(c *cli.Context) (*subcommandContext, error) {
 		c:           c,
 		logger:      logger,
 		isUIEnabled: isUIEnabled,
+		fs:          realFileSystem{},
 	}, nil
 }
 
+// Returns something that can find the given tunnel's credentials file.
+func (sc *subcommandContext) credentialFinder(tunnelID uuid.UUID) CredFinder {
+	if path := sc.c.String(CredFileFlag); path != "" {
+		return newStaticPath(path, sc.fs)
+	}
+	return newSearchByID(tunnelID, sc.c, sc.logger, sc.fs)
+}
+
 type userCredential struct {
 	cert     *certutil.OriginCert
 	certPath string
@@ -99,50 +113,27 @@ func (sc *subcommandContext) credential() (*userCredential, error) {
 	return sc.userCredential, nil
 }
 
-func (sc *subcommandContext) readTunnelCredentials(tunnelID uuid.UUID) (*pogs.TunnelAuth, error) {
-	filePath, err := sc.tunnelCredentialsPath(tunnelID)
+func (sc *subcommandContext) readTunnelCredentials(credFinder CredFinder) (connection.Credentials, error) {
+	filePath, err := credFinder.Path()
 	if err != nil {
-		return nil, err
+		return connection.Credentials{}, err
 	}
-	body, err := ioutil.ReadFile(filePath)
+	body, err := sc.fs.readFile(filePath)
 	if err != nil {
-		return nil, errors.Wrapf(err, "couldn't read tunnel credentials from %v", filePath)
+		return connection.Credentials{}, errors.Wrapf(err, "couldn't read tunnel credentials from %v", filePath)
 	}
 
-	var auth pogs.TunnelAuth
-	if err = json.Unmarshal(body, &auth); err != nil {
-		return nil, err
-	}
-	return &auth, nil
-}
-
-func (sc *subcommandContext) tunnelCredentialsPath(tunnelID uuid.UUID) (string, error) {
-	if filePath := sc.c.String("credentials-file"); filePath != "" {
-		if validFilePath(filePath) {
-			return filePath, nil
+	var credentials connection.Credentials
+	if err = json.Unmarshal(body, &credentials); err != nil {
+		if strings.HasSuffix(filePath, ".pem") {
+			return connection.Credentials{}, fmt.Errorf("The tunnel credentials file should be .json but you gave a .pem. " +
+				"The tunnel credentials file was originally created by `cloudflared tunnel create`. " +
+				"You may have accidentally used the filepath to cert.pem, which is generated by `cloudflared tunnel " +
+				"login`.")
 		}
-		return "", fmt.Errorf("Tunnel credentials file %s doesn't exist or is not a file", filePath)
+		return connection.Credentials{}, errInvalidJSONCredential{path: filePath, err: err}
 	}
-
-	// Fallback to look for tunnel credentials in the origin cert directory
-	if originCertPath, err := findOriginCert(sc.c, sc.logger); err == nil {
-		originCertDir := filepath.Dir(originCertPath)
-		if filePath, err := tunnelFilePath(tunnelID, originCertDir); err == nil {
-			if validFilePath(filePath) {
-				return filePath, nil
-			}
-		}
-	}
-
-	// Last resort look under default config directories
-	for _, configDir := range config.DefaultConfigSearchDirectories() {
-		if filePath, err := tunnelFilePath(tunnelID, configDir); err == nil {
-			if validFilePath(filePath) {
-				return filePath, nil
-			}
-		}
-	}
-	return "", fmt.Errorf("Tunnel credentials file not found")
+	return credentials, nil
 }
 
 func (sc *subcommandContext) create(name string) (*tunnelstore.Tunnel, error) {
@@ -165,7 +156,14 @@ func (sc *subcommandContext) create(name string) (*tunnelstore.Tunnel, error) {
 	if err != nil {
 		return nil, err
 	}
-	if writeFileErr := writeTunnelCredentials(tunnel.ID, credential.cert.AccountID, credential.certPath, tunnelSecret, sc.logger); err != nil {
+	tunnelCredentials := connection.Credentials{
+		AccountTag:   credential.cert.AccountID,
+		TunnelSecret: tunnelSecret,
+		TunnelID:     tunnel.ID,
+		TunnelName:   name,
+	}
+	filePath, writeFileErr := writeTunnelCredentials(credential.certPath, &tunnelCredentials)
+	if err != nil {
 		var errorLines []string
 		errorLines = append(errorLines, fmt.Sprintf("Your tunnel '%v' was created with ID %v. However, cloudflared couldn't write to the tunnel credentials file at %v.json.", tunnel.Name, tunnel.ID, tunnel.ID))
 		errorLines = append(errorLines, fmt.Sprintf("The file-writing error is: %v", writeFileErr))
@@ -178,6 +176,7 @@ func (sc *subcommandContext) create(name string) (*tunnelstore.Tunnel, error) {
 		errorMsg := strings.Join(errorLines, "\n")
 		return nil, errors.New(errorMsg)
 	}
+	sc.logger.Infof("Tunnel credentials written to %v. cloudflared chose this file based on where your origin certificate was found. Keep this file secret. To revoke these credentials, delete the tunnel.", filePath)
 
 	if outputFormat := sc.c.String(outputFormatFlag.Name); outputFormat != "" {
 		return nil, renderOutput(outputFormat, &tunnel)
@@ -228,31 +227,44 @@ func (sc *subcommandContext) delete(tunnelIDs []uuid.UUID) error {
 			return errors.Wrapf(err, "Error deleting tunnel %s", tunnel.ID)
 		}
 
-		tunnelCredentialsPath, err := sc.tunnelCredentialsPath(tunnel.ID)
-		if err != nil {
-			sc.logger.Infof("Cannot locate tunnel credentials to delete, error: %v. Please delete the file manually", err)
-			return nil
-		}
-
-		if err = os.Remove(tunnelCredentialsPath); err != nil {
-			sc.logger.Infof("Cannot delete tunnel credentials, error: %v. Please delete the file manually", err)
+		credFinder := sc.credentialFinder(id)
+		if tunnelCredentialsPath, err := credFinder.Path(); err == nil {
+			if err = os.Remove(tunnelCredentialsPath); err != nil {
+				sc.logger.Infof("Tunnel %v was deleted, but we could not remove its credentials file  %s: %s. Consider deleting this file manually.", id, tunnelCredentialsPath, err)
+			}
 		}
 	}
 	return nil
 }
 
+// findCredentials will choose the right way to find the credentials file, find it,
+// and add the TunnelID into any old credentials (generated before TUN-3581 added the `TunnelID`
+// field to credentials files)
+func (sc *subcommandContext) findCredentials(tunnelID uuid.UUID) (connection.Credentials, error) {
+	credFinder := sc.credentialFinder(tunnelID)
+	credentials, err := sc.readTunnelCredentials(credFinder)
+	// This line ensures backwards compatibility with credentials files generated before
+	// TUN-3581. Those old credentials files don't have a TunnelID field, so we enrich the struct
+	// with the ID, which we have already resolved from the user input.
+	credentials.TunnelID = tunnelID
+	return credentials, err
+}
+
 func (sc *subcommandContext) run(tunnelID uuid.UUID) error {
-	credentials, err := sc.readTunnelCredentials(tunnelID)
+	credentials, err := sc.findCredentials(tunnelID)
 	if err != nil {
+		if e, ok := err.(errInvalidJSONCredential); ok {
+			sc.logger.Errorf("The credentials file at %s contained invalid JSON. This is probably caused by passing the wrong filepath. Reminder: the credentials file is a .json file created via `cloudflared tunnel create`.", e.path)
+			sc.logger.Errorf("Invalid JSON when parsing credentials file: %s", e.err.Error())
+		}
 		return err
 	}
-
 	return StartServer(
 		sc.c,
 		version,
 		shutdownC,
 		graceShutdownC,
-		&origin.NamedTunnelConfig{Auth: *credentials, ID: tunnelID},
+		&connection.NamedTunnelConfig{Credentials: credentials},
 		sc.logger,
 		sc.isUIEnabled,
 	)
@@ -281,6 +293,7 @@ func (sc *subcommandContext) route(tunnelID uuid.UUID, r tunnelstore.Route) (tun
 	return client.RouteTunnel(tunnelID, r)
 }
 
+// Query Tunnelstore to find the active tunnel with the given name.
 func (sc *subcommandContext) tunnelActive(name string) (*tunnelstore.Tunnel, bool, error) {
 	filter := tunnelstore.NewFilter()
 	filter.NoDeleted()
@@ -303,6 +316,15 @@ func (sc *subcommandContext) findID(input string) (uuid.UUID, error) {
 		return u, nil
 	}
 
+	// Look up name in the credentials file.
+	credFinder := newStaticPath(sc.c.String(CredFileFlag), sc.fs)
+	if credentials, err := sc.readTunnelCredentials(credFinder); err == nil {
+		if credentials.TunnelID != uuid.Nil && input == credentials.TunnelName {
+			return credentials.TunnelID, nil
+		}
+	}
+
+	// Fall back to querying Tunnelstore.
 	if tunnel, found, err := sc.tunnelActive(input); err != nil {
 		return uuid.Nil, err
 	} else if found {

cmd/cloudflared/tunnel/subcommand_context_test.go

@@ -1,11 +1,20 @@
 package tunnel
 
 import (
+	"encoding/base64"
+	"flag"
+	"fmt"
 	"reflect"
 	"testing"
+	"time"
 
+	"github.com/cloudflare/cloudflared/connection"
+	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/tunnelstore"
 	"github.com/google/uuid"
+	"github.com/pkg/errors"
+	"github.com/stretchr/testify/require"
+	"github.com/urfave/cli/v2"
 )
 
 func Test_findIDs(t *testing.T) {
@@ -80,3 +89,266 @@ func Test_findIDs(t *testing.T) {
 		})
 	}
 }
+
+type mockFileSystem struct {
+	rf  func(string) ([]byte, error)
+	vfp func(string) bool
+}
+
+func (fs mockFileSystem) validFilePath(path string) bool {
+	return fs.vfp(path)
+}
+
+func (fs mockFileSystem) readFile(filePath string) ([]byte, error) {
+	return fs.rf(filePath)
+}
+
+func Test_subcommandContext_findCredentials(t *testing.T) {
+	type fields struct {
+		c                 *cli.Context
+		logger            logger.Service
+		isUIEnabled       bool
+		fs                fileSystem
+		tunnelstoreClient tunnelstore.Client
+		userCredential    *userCredential
+	}
+	type args struct {
+		tunnelID uuid.UUID
+	}
+	oldCertPath := "old_cert.json"
+	newCertPath := "new_cert.json"
+	accountTag := "0000d4d14e84bd4ae5a6a02e0000ac63"
+	secret := []byte{211, 79, 177, 245, 179, 194, 152, 127, 140, 71, 18, 46, 183, 209, 10, 24, 192, 150, 55, 249, 211, 16, 167, 30, 113, 51, 152, 168, 72, 100, 205, 144}
+	secretB64 := base64.StdEncoding.EncodeToString(secret)
+	tunnelID := uuid.MustParse("df5ed608-b8b4-4109-89f3-9f2cf199df64")
+	name := "mytunnel"
+
+	fs := mockFileSystem{
+		rf: func(filePath string) ([]byte, error) {
+			if filePath == oldCertPath {
+				// An old credentials file created before TUN-3581 added the new fields
+				return []byte(fmt.Sprintf(`{"AccountTag":"%s","TunnelSecret":"%s"}`, accountTag, secretB64)), nil
+			}
+			if filePath == newCertPath {
+				// A new credentials file created after TUN-3581 with its new fields.
+				return []byte(fmt.Sprintf(`{"AccountTag":"%s","TunnelSecret":"%s","TunnelID":"%s","TunnelName":"%s"}`, accountTag, secretB64, tunnelID, name)), nil
+			}
+			return nil, errors.New("file not found")
+		},
+		vfp: func(string) bool { return true },
+	}
+	logger, err := logger.New()
+	require.NoError(t, err)
+
+	tests := []struct {
+		name    string
+		fields  fields
+		args    args
+		want    connection.Credentials
+		wantErr bool
+	}{
+		{
+			name: "Filepath given leads to old credentials file",
+			fields: fields{
+				logger: logger,
+				fs:     fs,
+				c: func() *cli.Context {
+					flagSet := flag.NewFlagSet("test0", flag.PanicOnError)
+					flagSet.String(CredFileFlag, oldCertPath, "")
+					c := cli.NewContext(cli.NewApp(), flagSet, nil)
+					err = c.Set(CredFileFlag, oldCertPath)
+					return c
+				}(),
+			},
+			args: args{
+				tunnelID: tunnelID,
+			},
+			want: connection.Credentials{
+				AccountTag:   accountTag,
+				TunnelID:     tunnelID,
+				TunnelSecret: secret,
+			},
+		},
+		{
+			name: "Filepath given leads to new credentials file",
+			fields: fields{
+				logger: logger,
+				fs:     fs,
+				c: func() *cli.Context {
+					flagSet := flag.NewFlagSet("test0", flag.PanicOnError)
+					flagSet.String(CredFileFlag, newCertPath, "")
+					c := cli.NewContext(cli.NewApp(), flagSet, nil)
+					err = c.Set(CredFileFlag, newCertPath)
+					return c
+				}(),
+			},
+			args: args{
+				tunnelID: tunnelID,
+			},
+			want: connection.Credentials{
+				AccountTag:   accountTag,
+				TunnelID:     tunnelID,
+				TunnelSecret: secret,
+				TunnelName:   name,
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			sc := &subcommandContext{
+				c:                 tt.fields.c,
+				logger:            tt.fields.logger,
+				isUIEnabled:       tt.fields.isUIEnabled,
+				fs:                tt.fields.fs,
+				tunnelstoreClient: tt.fields.tunnelstoreClient,
+				userCredential:    tt.fields.userCredential,
+			}
+			got, err := sc.findCredentials(tt.args.tunnelID)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("subcommandContext.findCredentials() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("subcommandContext.findCredentials() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+type deleteMockTunnelStore struct {
+	tunnelstore.Client
+	mockTunnels      map[uuid.UUID]mockTunnelBehaviour
+	deletedTunnelIDs []uuid.UUID
+}
+
+type mockTunnelBehaviour struct {
+	tunnel     tunnelstore.Tunnel
+	deleteErr  error
+	cleanupErr error
+}
+
+func newDeleteMockTunnelStore(tunnels ...mockTunnelBehaviour) *deleteMockTunnelStore {
+	mockTunnels := make(map[uuid.UUID]mockTunnelBehaviour)
+	for _, tunnel := range tunnels {
+		mockTunnels[tunnel.tunnel.ID] = tunnel
+	}
+	return &deleteMockTunnelStore{
+		mockTunnels:      mockTunnels,
+		deletedTunnelIDs: make([]uuid.UUID, 0),
+	}
+}
+
+func (d *deleteMockTunnelStore) GetTunnel(tunnelID uuid.UUID) (*tunnelstore.Tunnel, error) {
+	tunnel, ok := d.mockTunnels[tunnelID]
+	if !ok {
+		return nil, fmt.Errorf("Couldn't find tunnel: %v", tunnelID)
+	}
+	return &tunnel.tunnel, nil
+}
+
+func (d *deleteMockTunnelStore) DeleteTunnel(tunnelID uuid.UUID) error {
+	tunnel, ok := d.mockTunnels[tunnelID]
+	if !ok {
+		return fmt.Errorf("Couldn't find tunnel: %v", tunnelID)
+	}
+
+	if tunnel.deleteErr != nil {
+		return tunnel.deleteErr
+	}
+
+	d.deletedTunnelIDs = append(d.deletedTunnelIDs, tunnelID)
+	tunnel.tunnel.DeletedAt = time.Now()
+	delete(d.mockTunnels, tunnelID)
+	return nil
+}
+
+func (d *deleteMockTunnelStore) CleanupConnections(tunnelID uuid.UUID) error {
+	tunnel, ok := d.mockTunnels[tunnelID]
+	if !ok {
+		return fmt.Errorf("Couldn't find tunnel: %v", tunnelID)
+	}
+	return tunnel.cleanupErr
+}
+
+func Test_subcommandContext_Delete(t *testing.T) {
+	type fields struct {
+		c                 *cli.Context
+		logger            logger.Service
+		isUIEnabled       bool
+		fs                fileSystem
+		tunnelstoreClient *deleteMockTunnelStore
+		userCredential    *userCredential
+	}
+	type args struct {
+		tunnelIDs []uuid.UUID
+	}
+	newCertPath := "new_cert.json"
+	tunnelID1 := uuid.MustParse("df5ed608-b8b4-4109-89f3-9f2cf199df64")
+	tunnelID2 := uuid.MustParse("af5ed608-b8b4-4109-89f3-9f2cf199df64")
+	logger, err := logger.New()
+	require.NoError(t, err)
+
+	var tests = []struct {
+		name    string
+		fields  fields
+		args    args
+		want    []uuid.UUID
+		wantErr bool
+	}{
+		{
+			name: "clean up continues if credentials are not found",
+			fields: fields{
+				logger: logger,
+				fs: mockFileSystem{
+					rf: func(filePath string) ([]byte, error) {
+						return nil, errors.New("file not found")
+					},
+					vfp: func(string) bool { return true },
+				},
+				c: func() *cli.Context {
+					flagSet := flag.NewFlagSet("test0", flag.PanicOnError)
+					flagSet.String(CredFileFlag, newCertPath, "")
+					c := cli.NewContext(cli.NewApp(), flagSet, nil)
+					err = c.Set(CredFileFlag, newCertPath)
+					return c
+				}(),
+				tunnelstoreClient: newDeleteMockTunnelStore(
+					mockTunnelBehaviour{
+						tunnel: tunnelstore.Tunnel{ID: tunnelID1},
+					},
+					mockTunnelBehaviour{
+						tunnel: tunnelstore.Tunnel{ID: tunnelID2},
+					},
+				),
+			},
+
+			args: args{
+				tunnelIDs: []uuid.UUID{tunnelID1, tunnelID2},
+			},
+			want: []uuid.UUID{tunnelID1, tunnelID2},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			sc := &subcommandContext{
+				c:                 tt.fields.c,
+				logger:            tt.fields.logger,
+				isUIEnabled:       tt.fields.isUIEnabled,
+				fs:                tt.fields.fs,
+				tunnelstoreClient: tt.fields.tunnelstoreClient,
+				userCredential:    tt.fields.userCredential,
+			}
+			err := sc.delete(tt.args.tunnelIDs)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("subcommandContext.findCredentials() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			got := tt.fields.tunnelstoreClient.deletedTunnelIDs
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("subcommandContext.findCredentials() = %v, want %v", got, tt.want)
+				return
+			}
+		})
+	}
+}

cmd/cloudflared/tunnel/subcommands.go

@@ -23,13 +23,13 @@ import (
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
-	"github.com/cloudflare/cloudflared/logger"
-	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/tunnelstore"
 )
 
 const (
-	credFileFlagAlias = "cred-file"
+	CredFileFlagAlias = "cred-file"
+	CredFileFlag      = "credentials-file"
 )
 
 var (
@@ -74,25 +74,35 @@ var (
 			"tunnels, you can do so with Cloudflare's Load Balancer product.",
 	})
 	credentialsFileFlag = altsrc.NewStringFlag(&cli.StringFlag{
-		Name:    "credentials-file",
-		Aliases: []string{credFileFlagAlias},
+		Name:    CredFileFlag,
+		Aliases: []string{CredFileFlagAlias},
 		Usage:   "File path of tunnel credentials",
+		EnvVars: []string{"TUNNEL_CRED_FILE"},
 	})
 	forceDeleteFlag = &cli.BoolFlag{
 		Name:    "force",
 		Aliases: []string{"f"},
 		Usage:   "Allows you to delete a tunnel, even if it has active connections.",
+		EnvVars: []string{"TUNNEL_RUN_FORCE_OVERWRITE"},
 	}
+	selectProtocolFlag = altsrc.NewStringFlag(&cli.StringFlag{
+		Name:    "protocol",
+		Value:   "h2mux",
+		Aliases: []string{"p"},
+		Usage:   fmt.Sprintf("Protocol implementation to connect with Cloudflare's edge network. %s", connection.AvailableProtocolFlagMessage),
+		EnvVars: []string{"TUNNEL_TRANSPORT_PROTOCOL"},
+		Hidden:  true,
+	})
 )
 
 func buildCreateCommand() *cli.Command {
 	return &cli.Command{
-		Name:               "create",
-		Action:             cliutil.ErrorHandler(createCommand),
-		Usage:              "Create a new tunnel with given name",
-		UsageText:          "cloudflared tunnel [tunnel command options] create [subcommand options] NAME",
-		Description:        `Creates a tunnel, registers it with Cloudflare edge and generates credential file used to run this tunnel.
-  Use "cloudflared tunnel route" subcommand to map a DNS name to this tunnel and "cloudflared tunnel run" to start the connection. 
+		Name:      "create",
+		Action:    cliutil.ErrorHandler(createCommand),
+		Usage:     "Create a new tunnel with given name",
+		UsageText: "cloudflared tunnel [tunnel command options] create [subcommand options] NAME",
+		Description: `Creates a tunnel, registers it with Cloudflare edge and generates credential file used to run this tunnel.
+  Use "cloudflared tunnel route" subcommand to map a DNS name to this tunnel and "cloudflared tunnel run" to start the connection.
 
   For example, to create a tunnel named 'my-tunnel' run:
 
@@ -130,30 +140,21 @@ func tunnelFilePath(tunnelID uuid.UUID, directory string) (string, error) {
 	return homedir.Expand(filePath)
 }
 
-func writeTunnelCredentials(tunnelID uuid.UUID, accountID, originCertPath string, tunnelSecret []byte, logger logger.Service) error {
+func writeTunnelCredentials(
+	originCertPath string,
+	credentials *connection.Credentials,
+) (filePath string, err error) {
 	originCertDir := filepath.Dir(originCertPath)
-	filePath, err := tunnelFilePath(tunnelID, originCertDir)
+	filePath, err = tunnelFilePath(credentials.TunnelID, originCertDir)
 	if err != nil {
-		return err
+		return "", err
 	}
-	body, err := json.Marshal(pogs.TunnelAuth{
-		AccountTag:   accountID,
-		TunnelSecret: tunnelSecret,
-	})
+	// Write the name and ID to the file too
+	body, err := json.Marshal(credentials)
 	if err != nil {
-		return errors.Wrap(err, "Unable to marshal tunnel credentials to JSON")
+		return "", errors.Wrap(err, "Unable to marshal tunnel credentials to JSON")
 	}
-	logger.Infof("Writing tunnel credentials to %v. cloudflared chose this file based on where your origin certificate was found.", filePath)
-	logger.Infof("Keep this file secret. To revoke these credentials, delete the tunnel.")
-	return ioutil.WriteFile(filePath, body, 400)
-}
-
-func validFilePath(path string) bool {
-	fileStat, err := os.Stat(path)
-	if err != nil {
-		return false
-	}
-	return !fileStat.IsDir()
+	return filePath, ioutil.WriteFile(filePath, body, 400)
 }
 
 func buildListCommand() *cli.Command {
@@ -309,6 +310,7 @@ func buildRunCommand() *cli.Command {
 	flags := []cli.Flag{
 		forceFlag,
 		credentialsFileFlag,
+		selectProtocolFlag,
 	}
 	flags = append(flags, configureProxyFlags(false)...)
 	return &cli.Command{
@@ -317,13 +319,13 @@ func buildRunCommand() *cli.Command {
 		Before:    SetFlagsFromConfigFile,
 		Usage:     "Proxy a local web server by running the given tunnel",
 		UsageText: "cloudflared tunnel [tunnel command options] run [subcommand options] [TUNNEL]",
-		Description: `Runs the tunnel identified by name or UUUD, creating highly available connections 
+		Description: `Runs the tunnel identified by name or UUUD, creating highly available connections
   between your server and the Cloudflare edge. You can provide name or UUID of tunnel to run either as the
   last command line argument or in the configuration file using "tunnel: TUNNEL".
 
-  This command requires the tunnel credentials file created when "cloudflared tunnel create" was run, 
+  This command requires the tunnel credentials file created when "cloudflared tunnel create" was run,
   however it does not need access to cert.pem from "cloudflared login" if you identify the tunnel by UUID.
-  If you experience other problems running the tunnel, "cloudflared tunnel cleanup" may help by removing 
+  If you experience other problems running the tunnel, "cloudflared tunnel cleanup" may help by removing
   any old connection records.
 `,
 		Flags:              flags,
@@ -419,7 +421,7 @@ func dnsRouteFromArg(c *cli.Context) (tunnelstore.Route, error) {
 	userHostname := c.Args().Get(userHostnameIndex)
 	if userHostname == "" {
 		return nil, cliutil.UsageError("The third argument should be the hostname")
-	} else if !validateHostname(userHostname) {
+	} else if !validateHostname(userHostname, true) {
 		return nil, errors.Errorf("%s is not a valid hostname", userHostname)
 	}
 	return tunnelstore.NewDNSRoute(userHostname), nil
@@ -437,14 +439,14 @@ func lbRouteFromArg(c *cli.Context) (tunnelstore.Route, error) {
 	lbName := c.Args().Get(lbNameIndex)
 	if lbName == "" {
 		return nil, cliutil.UsageError("The third argument should be the load balancer name")
-	} else if !validateHostname(lbName) {
+	} else if !validateHostname(lbName, true) {
 		return nil, errors.Errorf("%s is not a valid load balancer name", lbName)
 	}
 
 	lbPool := c.Args().Get(lbPoolIndex)
 	if lbPool == "" {
 		return nil, cliutil.UsageError("The fourth argument should be the pool name")
-	} else if !validateName(lbPool) {
+	} else if !validateName(lbPool, false) {
 		return nil, errors.Errorf("%s is not a valid pool name", lbPool)
 	}
 
@@ -452,19 +454,23 @@ func lbRouteFromArg(c *cli.Context) (tunnelstore.Route, error) {
 }
 
 var nameRegex = regexp.MustCompile("^[_a-zA-Z0-9][-_.a-zA-Z0-9]*$")
+var hostNameRegex = regexp.MustCompile("^[*_a-zA-Z0-9][-_.a-zA-Z0-9]*$")
 
-func validateName(s string) bool {
+func validateName(s string, allowWildcardSubdomain bool) bool {
+	if allowWildcardSubdomain {
+		return hostNameRegex.MatchString(s)
+	}
 	return nameRegex.MatchString(s)
 }
 
-func validateHostname(s string) bool {
+func validateHostname(s string, allowWildcardSubdomain bool) bool {
 	// Slightly stricter than PunyCodeProfile
 	idnaProfile := idna.New(
 		idna.ValidateLabels(true),
 		idna.VerifyDNSLength(true))
 
 	puny, err := idnaProfile.ToASCII(s)
-	return err == nil && validateName(puny)
+	return err == nil && validateName(puny, allowWildcardSubdomain)
 }
 
 func routeCommand(c *cli.Context) error {
@@ -523,13 +529,13 @@ func commandHelpTemplate() string {
 	}
 	const template = `NAME:
 	{{.HelpName}} - {{.Usage}}
-	
+
 USAGE:
 	{{.UsageText}}
- 
+
 DESCRIPTION:
 	{{.Description}}
- 
+
 TUNNEL COMMAND OPTIONS:
 	%s
 SUBCOMMAND OPTIONS:

cmd/cloudflared/tunnel/subcommands_test.go

@@ -6,9 +6,8 @@ import (
 	"testing"
 
 	"github.com/cloudflare/cloudflared/tunnelstore"
-	"github.com/mitchellh/go-homedir"
-
 	"github.com/google/uuid"
+	"github.com/mitchellh/go-homedir"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -117,8 +116,64 @@ func TestValidateName(t *testing.T) {
 		{name: "_ab_c.-d-ef", want: true},
 	}
 	for _, tt := range tests {
-		if got := validateName(tt.name); got != tt.want {
+		if got := validateName(tt.name, false); got != tt.want {
 			t.Errorf("validateName() = %v, want %v", got, tt.want)
 		}
 	}
-}
+}
+
+func Test_validateHostname(t *testing.T) {
+	type args struct {
+		s                      string
+		allowWildcardSubdomain bool
+	}
+	tests := []struct {
+		name string
+		args args
+		want bool
+	}{
+		{
+			name: "Normal",
+			args: args{
+				s:                      "example.com",
+				allowWildcardSubdomain: true,
+			},
+			want: true,
+		},
+		{
+			name: "wildcard subdomain for TUN-358",
+			args: args{
+				s:                      "*.ehrig.io",
+				allowWildcardSubdomain: true,
+			},
+			want: true,
+		},
+		{
+			name: "Misplaced wildcard",
+			args: args{
+				s:                      "subdomain.*.ehrig.io",
+				allowWildcardSubdomain: true,
+			},
+		},
+		{
+			name: "Invalid domain",
+			args: args{
+				s:                      "..",
+				allowWildcardSubdomain: true,
+			},
+		},
+		{
+			name: "Invalid domain",
+			args: args{
+				s: "..",
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := validateHostname(tt.args.s, tt.args.allowWildcardSubdomain); got != tt.want {
+				t.Errorf("validateHostname() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

cmd/cloudflared/ui/launch_ui.go

@@ -3,8 +3,11 @@ package ui
 import (
 	"context"
 	"fmt"
+	"strings"
 	"time"
 
+	"github.com/cloudflare/cloudflared/connection"
+	"github.com/cloudflare/cloudflared/ingress"
 	"github.com/cloudflare/cloudflared/logger"
 
 	"github.com/gdamore/tcell"
@@ -13,32 +16,15 @@ import (
 
 type connState struct {
 	location string
-	state    status
-}
-
-type status int
-
-const (
-	Disconnected status = iota
-	Connected
-	Reconnecting
-	SetUrl
-	RegisteringTunnel
-)
-
-type TunnelEvent struct {
-	Index     uint8
-	EventType status
-	Location  string
-	Url       string
+	state    connection.Status
 }
 
 type uiModel struct {
-	version     string
-	edgeURL     string
-	metricsURL  string
-	proxyURL    string
-	connections []connState
+	version       string
+	edgeURL       string
+	metricsURL    string
+	localServices []string
+	connections   []connState
 }
 
 type palette struct {
@@ -49,27 +35,32 @@ type palette struct {
 	reconnecting string
 }
 
-func NewUIModel(version, hostname, metricsURL, proxyURL string, haConnections int) *uiModel {
+func NewUIModel(version, hostname, metricsURL string, ing *ingress.Ingress, haConnections int) *uiModel {
+	localServices := make([]string, len(ing.Rules))
+	for i, rule := range ing.Rules {
+		localServices[i] = rule.Service.String()
+	}
 	return &uiModel{
-		version:     version,
-		edgeURL:     hostname,
-		metricsURL:  metricsURL,
-		proxyURL:    proxyURL,
-		connections: make([]connState, haConnections),
+		version:       version,
+		edgeURL:       hostname,
+		metricsURL:    metricsURL,
+		localServices: localServices,
+		connections:   make([]connState, haConnections),
 	}
 }
 
 func (data *uiModel) LaunchUI(
 	ctx context.Context,
-	log logger.Service,
+	generalLogger, transportLogger logger.Service,
 	logLevels []logger.Level,
-	tunnelEventChan <-chan TunnelEvent,
+	tunnelEventChan <-chan connection.Event,
 ) {
 	// Configure the logger to stream logs into the textview
 
 	// Add TextView as a group to write output to
 	logTextView := NewDynamicColorTextView()
-	log.Add(logTextView, logger.NewUIFormatter(time.RFC3339), logLevels...)
+	generalLogger.Add(logTextView, logger.NewUIFormatter(time.RFC3339), logLevels...)
+	transportLogger.Add(logTextView, logger.NewUIFormatter(time.RFC3339), logLevels...)
 
 	// Construct the UI
 	palette := palette{
@@ -107,7 +98,8 @@ func (data *uiModel) LaunchUI(
 	tunnelHostText := tview.NewTextView().SetText(data.edgeURL)
 
 	grid.AddItem(tunnelHostText, 0, 1, 1, 1, 0, 0, false)
-	grid.AddItem(NewDynamicColorTextView().SetText(fmt.Sprintf("[%s]\u2022[%s] Proxying to [%s::b]%s", palette.connected, palette.defaultText, palette.url, data.proxyURL)), 1, 1, 1, 1, 0, 0, false)
+	status := fmt.Sprintf("[%s]\u2022[%s] Proxying to [%s::b]%s", palette.connected, palette.defaultText, palette.url, strings.Join(data.localServices, ", "))
+	grid.AddItem(NewDynamicColorTextView().SetText(status), 1, 1, 1, 1, 0, 0, false)
 
 	grid.AddItem(connTable, 2, 1, 1, 1, 0, 0, false)
 
@@ -130,14 +122,14 @@ func (data *uiModel) LaunchUI(
 				return
 			case event := <-tunnelEventChan:
 				switch event.EventType {
-				case Connected:
+				case connection.Connected:
 					data.setConnTableCell(event, connTable, palette)
-				case Disconnected, Reconnecting:
-					data.changeConnStatus(event, connTable, log, palette)
-				case SetUrl:
-					tunnelHostText.SetText(event.Url)
-					data.edgeURL = event.Url
-				case RegisteringTunnel:
+				case connection.Disconnected, connection.Reconnecting:
+					data.changeConnStatus(event, connTable, generalLogger, palette)
+				case connection.SetURL:
+					tunnelHostText.SetText(event.URL)
+					data.edgeURL = event.URL
+				case connection.RegisteringTunnel:
 					if data.edgeURL == "" {
 						tunnelHostText.SetText("Registering tunnel...")
 					}
@@ -149,7 +141,7 @@ func (data *uiModel) LaunchUI(
 
 	go func() {
 		if err := app.SetRoot(frame, true).Run(); err != nil {
-			log.Errorf("Error launching UI: %s", err)
+			generalLogger.Errorf("Error launching UI: %s", err)
 		}
 	}()
 }
@@ -167,7 +159,7 @@ func handleNewText(app *tview.Application, logTextView *tview.TextView) func() {
 	}
 }
 
-func (data *uiModel) changeConnStatus(event TunnelEvent, table *tview.Table, logger logger.Service, palette palette) {
+func (data *uiModel) changeConnStatus(event connection.Event, table *tview.Table, logger logger.Service, palette palette) {
 	index := int(event.Index)
 	// Get connection location and state
 	connState := data.getConnState(index)
@@ -179,10 +171,10 @@ func (data *uiModel) changeConnStatus(event TunnelEvent, table *tview.Table, log
 
 	locationState := event.Location
 
-	if event.EventType == Disconnected {
-		connState.state = Disconnected
-	} else if event.EventType == Reconnecting {
-		connState.state = Reconnecting
+	if event.EventType == connection.Disconnected {
+		connState.state = connection.Disconnected
+	} else if event.EventType == connection.Reconnecting {
+		connState.state = connection.Reconnecting
 		locationState = "Reconnecting..."
 	}
 
@@ -203,12 +195,12 @@ func (data *uiModel) getConnState(connID int) *connState {
 	return nil
 }
 
-func (data *uiModel) setConnTableCell(event TunnelEvent, table *tview.Table, palette palette) {
+func (data *uiModel) setConnTableCell(event connection.Event, table *tview.Table, palette palette) {
 	index := int(event.Index)
 	connectionNum := index + 1
 
 	// Update slice to keep track of connection location and state in UI table
-	data.connections[index].state = Connected
+	data.connections[index].state = connection.Connected
 	data.connections[index].location = event.Location
 
 	// Update text in table cell to show disconnected state
@@ -217,18 +209,18 @@ func (data *uiModel) setConnTableCell(event TunnelEvent, table *tview.Table, pal
 	table.SetCell(index, 0, cell)
 }
 
-func newCellText(palette palette, connectionNum int, location string, connectedStatus status) string {
+func newCellText(palette palette, connectionNum int, location string, connectedStatus connection.Status) string {
 	// HA connection indicator formatted as: "• #<CONNECTION_INDEX>: <COLO>",
 	//  where the left middle dot's color depends on the status of the connection
 	const connFmtString = "[%s]\u2022[%s] #%d: %s"
 
 	var dotColor string
 	switch connectedStatus {
-	case Connected:
+	case connection.Connected:
 		dotColor = palette.connected
-	case Disconnected:
+	case connection.Disconnected:
 		dotColor = palette.disconnected
-	case Reconnecting:
+	case connection.Reconnecting:
 		dotColor = palette.reconnecting
 	}
 

cmd/cloudflared/updater/update.go

@@ -114,7 +114,7 @@ func checkForUpdateAndApply(options updateOptions) UpdateOutcome {
 
 // Update is the handler for the update command from the command line
 func Update(c *cli.Context) error {
-	logger, err := logger.New()
+	logger, err := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
 	if err != nil {
 		return errors.Wrap(err, "error setting up logger")
 	}
@@ -202,7 +202,6 @@ func (a *AutoUpdater) Run(ctx context.Context) error {
 		if a.configurable.enabled {
 			updateOutcome := loggedUpdate(a.logger, updateOptions{})
 			if updateOutcome.Updated {
-				os.Args = append(os.Args, "--is-autoupdated=true")
 				if IsSysV() {
 					// SysV doesn't have a mechanism to keep service alive, we have to restart the process
 					a.logger.Info("Restarting service managed by SysV...")

cmd/cloudflared/updater/workers_service.go

@@ -7,7 +7,6 @@ import (
 	"runtime"
 	"strconv"
 	"strings"
-	"time"
 )
 
 // Options are the update options supported by the
@@ -53,7 +52,7 @@ func NewWorkersService(currentVersion, url, targetPath string, opts Options) Ser
 // Check does a check in with the Workers API to get a new version update
 func (s *WorkersService) Check() (Version, error) {
 	client := &http.Client{
-		Timeout: time.Second * 5,
+		Timeout: clientTimeout,
 	}
 
 	req, err := http.NewRequest(http.MethodGet, s.url, nil)

cmd/cloudflared/updater/workers_update.go

@@ -18,20 +18,23 @@ import (
 	"time"
 )
 
-// stop the service
-// rename cloudflared.exe to cloudflared.exe.old
-// rename cloudflared.exe.new to cloudflared.exe
-// delete cloudflared.exe.old
-// start the service
-// delete the batch file
-const windowsUpdateCommandTemplate = `@echo off
+const (
+	clientTimeout = time.Second * 60
+	// stop the service
+	// rename cloudflared.exe to cloudflared.exe.old
+	// rename cloudflared.exe.new to cloudflared.exe
+	// delete cloudflared.exe.old
+	// start the service
+	// delete the batch file
+	windowsUpdateCommandTemplate = `@echo off
 sc stop cloudflared >nul 2>&1
 rename "{{.TargetPath}}" {{.OldName}}
 rename "{{.NewPath}}" {{.BinaryName}}
 del "{{.OldPath}}"
 sc start cloudflared >nul 2>&1
 del {{.BatchName}}`
-const batchFileName = "cfd_update.bat"
+	batchFileName = "cfd_update.bat"
+)
 
 // Prepare some data to insert into the template.
 type batchData struct {
@@ -118,7 +121,7 @@ func (v *WorkersVersion) String() string {
 // download the file from the link in the json
 func download(url, filepath string, isCompressed bool) error {
 	client := &http.Client{
-		Timeout: time.Second * 5,
+		Timeout: clientTimeout,
 	}
 	resp, err := client.Get(url)
 

cmd/cloudflared/windows_service.go

@@ -173,7 +173,7 @@ func (s *windowsService) Execute(serviceArgs []string, r <-chan svc.ChangeReques
 }
 
 func installWindowsService(c *cli.Context) error {
-	logger, err := logger.New()
+	logger, err := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
 	if err != nil {
 		return errors.Wrap(err, "error setting up logger")
 	}
@@ -219,7 +219,7 @@ func installWindowsService(c *cli.Context) error {
 }
 
 func uninstallWindowsService(c *cli.Context) error {
-	logger, err := logger.New()
+	logger, err := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
 	if err != nil {
 		return errors.Wrap(err, "error setting up logger")
 	}

connection/connection.go

@@ -0,0 +1,79 @@
+package connection
+
+import (
+	"io"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/google/uuid"
+)
+
+const (
+	lbProbeUserAgentPrefix = "Mozilla/5.0 (compatible; Cloudflare-Traffic-Manager/1.0; +https://www.cloudflare.com/traffic-manager/;"
+)
+
+type Config struct {
+	OriginClient    OriginClient
+	GracePeriod     time.Duration
+	ReplaceExisting bool
+}
+
+type NamedTunnelConfig struct {
+	Credentials Credentials
+	Client      pogs.ClientInfo
+}
+
+// Credentials are stored in the credentials file and contain all info needed to run a tunnel.
+type Credentials struct {
+	AccountTag   string
+	TunnelSecret []byte
+	TunnelID     uuid.UUID
+	TunnelName   string
+}
+
+func (c *Credentials) Auth() pogs.TunnelAuth {
+	return pogs.TunnelAuth{
+		AccountTag:   c.AccountTag,
+		TunnelSecret: c.TunnelSecret,
+	}
+}
+
+type ClassicTunnelConfig struct {
+	Hostname   string
+	OriginCert []byte
+	// feature-flag to use new edge reconnect tokens
+	UseReconnectToken bool
+}
+
+func (c *ClassicTunnelConfig) IsTrialZone() bool {
+	return c.Hostname == ""
+}
+
+type OriginClient interface {
+	Proxy(w ResponseWriter, req *http.Request, isWebsocket bool) error
+}
+
+type ResponseWriter interface {
+	WriteRespHeaders(*http.Response) error
+	WriteErrorResponse()
+	io.ReadWriter
+}
+
+type ConnectedFuse interface {
+	Connected()
+	IsConnected() bool
+}
+
+func IsServerSentEvent(headers http.Header) bool {
+	if contentType := headers.Get("content-type"); contentType != "" {
+		return strings.HasPrefix(strings.ToLower(contentType), "text/event-stream")
+	}
+	return false
+}
+
+func uint8ToString(input uint8) string {
+	return strconv.FormatUint(uint64(input), 10)
+}

connection/connection_test.go

@@ -0,0 +1,152 @@
+package connection
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"testing"
+	"time"
+
+	"github.com/cloudflare/cloudflared/logger"
+	"github.com/gobwas/ws/wsutil"
+	"github.com/stretchr/testify/assert"
+)
+
+const (
+	largeFileSize = 2 * 1024 * 1024
+)
+
+var (
+	testConfig = &Config{
+		OriginClient: &mockOriginClient{},
+		GracePeriod:  time.Millisecond * 100,
+	}
+	testLogger, _ = logger.New()
+	testOriginURL = &url.URL{
+		Scheme: "https",
+		Host:   "connectiontest.argotunnel.com",
+	}
+	testTunnelEventChan = make(chan Event)
+	testObserver        = &Observer{
+		testLogger,
+		m,
+		[]chan Event{testTunnelEventChan},
+		false,
+	}
+	testLargeResp = make([]byte, largeFileSize)
+)
+
+type testRequest struct {
+	name           string
+	endpoint       string
+	expectedStatus int
+	expectedBody   []byte
+	isProxyError   bool
+}
+
+type mockOriginClient struct {
+}
+
+func (moc *mockOriginClient) Proxy(w ResponseWriter, r *http.Request, isWebsocket bool) error {
+	if isWebsocket {
+		return wsEndpoint(w, r)
+	}
+	switch r.URL.Path {
+	case "/ok":
+		originRespEndpoint(w, http.StatusOK, []byte(http.StatusText(http.StatusOK)))
+	case "/large_file":
+		originRespEndpoint(w, http.StatusOK, testLargeResp)
+	case "/400":
+		originRespEndpoint(w, http.StatusBadRequest, []byte(http.StatusText(http.StatusBadRequest)))
+	case "/500":
+		originRespEndpoint(w, http.StatusInternalServerError, []byte(http.StatusText(http.StatusInternalServerError)))
+	case "/error":
+		return fmt.Errorf("Failed to proxy to origin")
+	default:
+		originRespEndpoint(w, http.StatusNotFound, []byte("page not found"))
+	}
+	return nil
+}
+
+type nowriter struct {
+	io.Reader
+}
+
+func (nowriter) Write(p []byte) (int, error) {
+	return 0, fmt.Errorf("Writer not implemented")
+}
+
+func wsEndpoint(w ResponseWriter, r *http.Request) error {
+	resp := &http.Response{
+		StatusCode: http.StatusSwitchingProtocols,
+	}
+	w.WriteRespHeaders(resp)
+	clientReader := nowriter{r.Body}
+	go func() {
+		for {
+			data, err := wsutil.ReadClientText(clientReader)
+			if err != nil {
+				return
+			}
+			if err := wsutil.WriteServerText(w, data); err != nil {
+				return
+			}
+		}
+	}()
+	<-r.Context().Done()
+	return nil
+}
+
+func originRespEndpoint(w ResponseWriter, status int, data []byte) {
+	resp := &http.Response{
+		StatusCode: status,
+	}
+	w.WriteRespHeaders(resp)
+	w.Write(data)
+}
+
+type mockConnectedFuse struct{}
+
+func (mcf mockConnectedFuse) Connected() {}
+
+func (mcf mockConnectedFuse) IsConnected() bool {
+	return true
+}
+
+func TestIsEventStream(t *testing.T) {
+	tests := []struct {
+		headers       http.Header
+		isEventStream bool
+	}{
+		{
+			headers:       newHeader("Content-Type", "text/event-stream"),
+			isEventStream: true,
+		},
+		{
+			headers:       newHeader("content-type", "text/event-stream"),
+			isEventStream: true,
+		},
+		{
+			headers:       newHeader("Content-Type", "text/event-stream; charset=utf-8"),
+			isEventStream: true,
+		},
+		{
+			headers:       newHeader("Content-Type", "application/json"),
+			isEventStream: false,
+		},
+		{
+			headers:       http.Header{},
+			isEventStream: false,
+		},
+	}
+	for _, test := range tests {
+		assert.Equal(t, test.isEventStream, IsServerSentEvent(test.headers))
+	}
+}
+
+func newHeader(key, value string) http.Header {
+	header := http.Header{}
+	header.Add(key, value)
+	return header
+}

connection/errors.go

@@ -0,0 +1,76 @@
+package connection
+
+import (
+	"github.com/cloudflare/cloudflared/edgediscovery"
+	"github.com/cloudflare/cloudflared/h2mux"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const (
+	DuplicateConnectionError = "EDUPCONN"
+)
+
+// RegisterTunnel error from client
+type clientRegisterTunnelError struct {
+	cause error
+}
+
+func newRPCError(cause error, counter *prometheus.CounterVec, name rpcName) clientRegisterTunnelError {
+	counter.WithLabelValues(cause.Error(), string(name)).Inc()
+	return clientRegisterTunnelError{cause: cause}
+}
+
+func (e clientRegisterTunnelError) Error() string {
+	return e.cause.Error()
+}
+
+type DupConnRegisterTunnelError struct{}
+
+var errDuplicationConnection = &DupConnRegisterTunnelError{}
+
+func (e DupConnRegisterTunnelError) Error() string {
+	return "already connected to this server, trying another address"
+}
+
+// RegisterTunnel error from server
+type serverRegisterTunnelError struct {
+	cause     error
+	permanent bool
+}
+
+func (e serverRegisterTunnelError) Error() string {
+	return e.cause.Error()
+}
+
+func serverRegistrationErrorFromRPC(err error) *serverRegisterTunnelError {
+	if retryable, ok := err.(*tunnelpogs.RetryableError); ok {
+		return &serverRegisterTunnelError{
+			cause:     retryable.Unwrap(),
+			permanent: false,
+		}
+	}
+	return &serverRegisterTunnelError{
+		cause:     err,
+		permanent: true,
+	}
+}
+
+type muxerShutdownError struct{}
+
+func (e muxerShutdownError) Error() string {
+	return "muxer shutdown"
+}
+
+func isHandshakeErrRecoverable(err error, connIndex uint8, observer *Observer) bool {
+	switch err.(type) {
+	case edgediscovery.DialError:
+		observer.Errorf("Connection %d unable to dial edge: %s", connIndex, err)
+	case h2mux.MuxerHandshakeError:
+		observer.Errorf("Connection %d handshake with edge server failed: %s", connIndex, err)
+	default:
+		observer.Errorf("Connection %d failed: %s", connIndex, err)
+		return false
+	}
+	return true
+}

connection/event.go

@@ -0,0 +1,25 @@
+package connection
+
+// Event is something that happened to a connection, e.g. disconnection or registration.
+type Event struct {
+	Index     uint8
+	EventType Status
+	Location  string
+	URL       string
+}
+
+// Status is the status of a connection.
+type Status int
+
+const (
+	// Disconnected means the connection to the edge was broken.
+	Disconnected Status = iota
+	// Connected means the connection to the edge was successfully established.
+	Connected
+	// Reconnecting means the connection to the edge is being re-established.
+	Reconnecting
+	// SetURL means this connection's tunnel was given a URL by the edge. Used for free tunnels.
+	SetURL
+	// RegisteringTunnel means the non-named tunnel is registering its connection.
+	RegisteringTunnel
+)

connection/h2mux.go

@@ -0,0 +1,220 @@
+package connection
+
+import (
+	"context"
+	"net"
+	"net/http"
+	"time"
+
+	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/logger"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/cloudflare/cloudflared/websocket"
+	"github.com/pkg/errors"
+	"golang.org/x/sync/errgroup"
+)
+
+const (
+	muxerTimeout      = 5 * time.Second
+	openStreamTimeout = 30 * time.Second
+)
+
+type h2muxConnection struct {
+	config      *Config
+	muxerConfig *MuxerConfig
+	muxer       *h2mux.Muxer
+	// connectionID is only used by metrics, and prometheus requires labels to be string
+	connIndexStr string
+	connIndex    uint8
+
+	observer *Observer
+}
+
+type MuxerConfig struct {
+	HeartbeatInterval  time.Duration
+	MaxHeartbeats      uint64
+	CompressionSetting h2mux.CompressionSetting
+	MetricsUpdateFreq  time.Duration
+}
+
+func (mc *MuxerConfig) H2MuxerConfig(h h2mux.MuxedStreamHandler, logger logger.Service) *h2mux.MuxerConfig {
+	return &h2mux.MuxerConfig{
+		Timeout:            muxerTimeout,
+		Handler:            h,
+		IsClient:           true,
+		HeartbeatInterval:  mc.HeartbeatInterval,
+		MaxHeartbeats:      mc.MaxHeartbeats,
+		Logger:             logger,
+		CompressionQuality: mc.CompressionSetting,
+	}
+}
+
+// NewTunnelHandler returns a TunnelHandler, origin LAN IP and error
+func NewH2muxConnection(ctx context.Context,
+	config *Config,
+	muxerConfig *MuxerConfig,
+	edgeConn net.Conn,
+	connIndex uint8,
+	observer *Observer,
+) (*h2muxConnection, error, bool) {
+	h := &h2muxConnection{
+		config:       config,
+		muxerConfig:  muxerConfig,
+		connIndexStr: uint8ToString(connIndex),
+		connIndex:    connIndex,
+		observer:     observer,
+	}
+
+	// Establish a muxed connection with the edge
+	// Client mux handshake with agent server
+	muxer, err := h2mux.Handshake(edgeConn, edgeConn, *muxerConfig.H2MuxerConfig(h, observer), h2mux.ActiveStreams)
+	if err != nil {
+		recoverable := isHandshakeErrRecoverable(err, connIndex, observer)
+		return nil, err, recoverable
+	}
+	h.muxer = muxer
+	return h, nil, false
+}
+
+func (h *h2muxConnection) ServeNamedTunnel(ctx context.Context, namedTunnel *NamedTunnelConfig, credentialManager CredentialManager, connOptions *tunnelpogs.ConnectionOptions, connectedFuse ConnectedFuse) error {
+	errGroup, serveCtx := errgroup.WithContext(ctx)
+	errGroup.Go(func() error {
+		return h.serveMuxer(serveCtx)
+	})
+
+	errGroup.Go(func() error {
+		stream, err := h.newRPCStream(serveCtx, register)
+		if err != nil {
+			return err
+		}
+		rpcClient := newRegistrationRPCClient(ctx, stream, h.observer)
+		defer rpcClient.Close()
+
+		if err = rpcClient.RegisterConnection(serveCtx, namedTunnel, connOptions, h.connIndex, h.observer); err != nil {
+			return err
+		}
+		connectedFuse.Connected()
+		return nil
+	})
+
+	errGroup.Go(func() error {
+		h.controlLoop(serveCtx, connectedFuse, true)
+		return nil
+	})
+	return errGroup.Wait()
+}
+
+func (h *h2muxConnection) ServeClassicTunnel(ctx context.Context, classicTunnel *ClassicTunnelConfig, credentialManager CredentialManager, registrationOptions *tunnelpogs.RegistrationOptions, connectedFuse ConnectedFuse) error {
+	errGroup, serveCtx := errgroup.WithContext(ctx)
+	errGroup.Go(func() error {
+		return h.serveMuxer(serveCtx)
+	})
+
+	errGroup.Go(func() (err error) {
+		defer func() {
+			if err == nil {
+				connectedFuse.Connected()
+			}
+		}()
+		if classicTunnel.UseReconnectToken && connectedFuse.IsConnected() {
+			err := h.reconnectTunnel(ctx, credentialManager, classicTunnel, registrationOptions)
+			if err == nil {
+				return nil
+			}
+			// log errors and proceed to RegisterTunnel
+			h.observer.Errorf("Couldn't reconnect connection %d. Reregistering it instead. Error was: %v", h.connIndex, err)
+		}
+		return h.registerTunnel(ctx, credentialManager, classicTunnel, registrationOptions)
+	})
+
+	errGroup.Go(func() error {
+		h.controlLoop(serveCtx, connectedFuse, false)
+		return nil
+	})
+	return errGroup.Wait()
+}
+
+func (h *h2muxConnection) serveMuxer(ctx context.Context) error {
+	// All routines should stop when muxer finish serving. When muxer is shutdown
+	// gracefully, it doesn't return an error, so we need to return errMuxerShutdown
+	// here to notify other routines to stop
+	err := h.muxer.Serve(ctx)
+	if err == nil {
+		return muxerShutdownError{}
+	}
+	return err
+}
+
+func (h *h2muxConnection) controlLoop(ctx context.Context, connectedFuse ConnectedFuse, isNamedTunnel bool) {
+	updateMetricsTickC := time.Tick(h.muxerConfig.MetricsUpdateFreq)
+	for {
+		select {
+		case <-ctx.Done():
+			// UnregisterTunnel blocks until the RPC call returns
+			if connectedFuse.IsConnected() {
+				h.unregister(isNamedTunnel)
+			}
+			h.muxer.Shutdown()
+			return
+		case <-updateMetricsTickC:
+			h.observer.metrics.updateMuxerMetrics(h.connIndexStr, h.muxer.Metrics())
+		}
+	}
+}
+
+func (h *h2muxConnection) newRPCStream(ctx context.Context, rpcName rpcName) (*h2mux.MuxedStream, error) {
+	openStreamCtx, openStreamCancel := context.WithTimeout(ctx, openStreamTimeout)
+	defer openStreamCancel()
+	stream, err := h.muxer.OpenRPCStream(openStreamCtx)
+	if err != nil {
+		return nil, err
+	}
+	return stream, nil
+}
+
+func (h *h2muxConnection) ServeStream(stream *h2mux.MuxedStream) error {
+	respWriter := &h2muxRespWriter{stream}
+
+	req, reqErr := h.newRequest(stream)
+	if reqErr != nil {
+		respWriter.WriteErrorResponse()
+		return reqErr
+	}
+
+	err := h.config.OriginClient.Proxy(respWriter, req, websocket.IsWebSocketUpgrade(req))
+	if err != nil {
+		respWriter.WriteErrorResponse()
+		return err
+	}
+	return nil
+}
+
+func (h *h2muxConnection) newRequest(stream *h2mux.MuxedStream) (*http.Request, error) {
+	req, err := http.NewRequest("GET", "http://localhost:8080", h2mux.MuxedStreamReader{MuxedStream: stream})
+	if err != nil {
+		return nil, errors.Wrap(err, "Unexpected error from http.NewRequest")
+	}
+	err = h2mux.H2RequestHeadersToH1Request(stream.Headers, req)
+	if err != nil {
+		return nil, errors.Wrap(err, "invalid request received")
+	}
+	return req, nil
+}
+
+type h2muxRespWriter struct {
+	*h2mux.MuxedStream
+}
+
+func (rp *h2muxRespWriter) WriteRespHeaders(resp *http.Response) error {
+	headers := h2mux.H1ResponseToH2ResponseHeaders(resp)
+	headers = append(headers, h2mux.Header{Name: ResponseMetaHeaderField, Value: responseMetaHeaderOrigin})
+	return rp.WriteHeaders(headers)
+}
+
+func (rp *h2muxRespWriter) WriteErrorResponse() {
+	rp.WriteHeaders([]h2mux.Header{
+		{Name: ":status", Value: "502"},
+		{Name: ResponseMetaHeaderField, Value: responseMetaHeaderCfd},
+	})
+	rp.Write([]byte("502 Bad Gateway"))
+}

connection/h2mux_test.go

@@ -0,0 +1,242 @@
+package connection
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"strconv"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/gobwas/ws/wsutil"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+var (
+	testMuxerConfig = &MuxerConfig{
+		HeartbeatInterval:  time.Second * 5,
+		MaxHeartbeats:      5,
+		CompressionSetting: 0,
+		MetricsUpdateFreq:  time.Second * 5,
+	}
+)
+
+func newH2MuxConnection(ctx context.Context, t require.TestingT) (*h2muxConnection, *h2mux.Muxer) {
+	edgeConn, originConn := net.Pipe()
+	edgeMuxChan := make(chan *h2mux.Muxer)
+	go func() {
+		edgeMuxConfig := h2mux.MuxerConfig{
+			Logger: testObserver,
+		}
+		edgeMux, err := h2mux.Handshake(edgeConn, edgeConn, edgeMuxConfig, h2mux.ActiveStreams)
+		require.NoError(t, err)
+		edgeMuxChan <- edgeMux
+	}()
+	var connIndex = uint8(0)
+	h2muxConn, err, _ := NewH2muxConnection(ctx, testConfig, testMuxerConfig, originConn, connIndex, testObserver)
+	require.NoError(t, err)
+	return h2muxConn, <-edgeMuxChan
+}
+
+func TestServeStreamHTTP(t *testing.T) {
+	tests := []testRequest{
+		{
+			name:           "ok",
+			endpoint:       "/ok",
+			expectedStatus: http.StatusOK,
+			expectedBody:   []byte(http.StatusText(http.StatusOK)),
+		},
+		{
+			name:           "large_file",
+			endpoint:       "/large_file",
+			expectedStatus: http.StatusOK,
+			expectedBody:   testLargeResp,
+		},
+		{
+			name:           "Bad request",
+			endpoint:       "/400",
+			expectedStatus: http.StatusBadRequest,
+			expectedBody:   []byte(http.StatusText(http.StatusBadRequest)),
+		},
+		{
+			name:           "Internal server error",
+			endpoint:       "/500",
+			expectedStatus: http.StatusInternalServerError,
+			expectedBody:   []byte(http.StatusText(http.StatusInternalServerError)),
+		},
+		{
+			name:           "Proxy error",
+			endpoint:       "/error",
+			expectedStatus: http.StatusBadGateway,
+			expectedBody:   nil,
+			isProxyError:   true,
+		},
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	h2muxConn, edgeMux := newH2MuxConnection(ctx, t)
+
+	var wg sync.WaitGroup
+	wg.Add(2)
+	go func() {
+		defer wg.Done()
+		edgeMux.Serve(ctx)
+	}()
+	go func() {
+		defer wg.Done()
+		err := h2muxConn.serveMuxer(ctx)
+		require.Error(t, err)
+	}()
+
+	for _, test := range tests {
+		headers := []h2mux.Header{
+			{
+				Name:  ":path",
+				Value: test.endpoint,
+			},
+		}
+		stream, err := edgeMux.OpenStream(ctx, headers, nil)
+		require.NoError(t, err)
+		require.True(t, hasHeader(stream, ":status", strconv.Itoa(test.expectedStatus)))
+
+		if test.isProxyError {
+			assert.True(t, hasHeader(stream, ResponseMetaHeaderField, responseMetaHeaderCfd))
+		} else {
+			assert.True(t, hasHeader(stream, ResponseMetaHeaderField, responseMetaHeaderOrigin))
+			body := make([]byte, len(test.expectedBody))
+			_, err = stream.Read(body)
+			require.NoError(t, err)
+			require.Equal(t, test.expectedBody, body)
+		}
+	}
+	cancel()
+	wg.Wait()
+}
+
+func TestServeStreamWS(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	h2muxConn, edgeMux := newH2MuxConnection(ctx, t)
+
+	var wg sync.WaitGroup
+	wg.Add(2)
+	go func() {
+		defer wg.Done()
+		edgeMux.Serve(ctx)
+	}()
+	go func() {
+		defer wg.Done()
+		err := h2muxConn.serveMuxer(ctx)
+		require.Error(t, err)
+	}()
+
+	headers := []h2mux.Header{
+		{
+			Name:  ":path",
+			Value: "/ws",
+		},
+		{
+			Name:  "connection",
+			Value: "upgrade",
+		},
+		{
+			Name:  "upgrade",
+			Value: "websocket",
+		},
+	}
+
+	readPipe, writePipe := io.Pipe()
+	stream, err := edgeMux.OpenStream(ctx, headers, readPipe)
+	require.NoError(t, err)
+
+	require.True(t, hasHeader(stream, ":status", strconv.Itoa(http.StatusSwitchingProtocols)))
+	assert.True(t, hasHeader(stream, ResponseMetaHeaderField, responseMetaHeaderOrigin))
+
+	data := []byte("test websocket")
+	err = wsutil.WriteClientText(writePipe, data)
+	require.NoError(t, err)
+
+	respBody, err := wsutil.ReadServerText(stream)
+	require.NoError(t, err)
+	require.Equal(t, data, respBody, fmt.Sprintf("Expect %s, got %s", string(data), string(respBody)))
+
+	cancel()
+	wg.Wait()
+}
+
+func hasHeader(stream *h2mux.MuxedStream, name, val string) bool {
+	for _, header := range stream.Headers {
+		if header.Name == name && header.Value == val {
+			return true
+		}
+	}
+	return false
+}
+
+func benchmarkServeStreamHTTPSimple(b *testing.B, test testRequest) {
+	ctx, cancel := context.WithCancel(context.Background())
+	h2muxConn, edgeMux := newH2MuxConnection(ctx, b)
+
+	var wg sync.WaitGroup
+	wg.Add(2)
+	go func() {
+		defer wg.Done()
+		edgeMux.Serve(ctx)
+	}()
+	go func() {
+		defer wg.Done()
+		err := h2muxConn.serveMuxer(ctx)
+		require.Error(b, err)
+	}()
+
+	headers := []h2mux.Header{
+		{
+			Name:  ":path",
+			Value: test.endpoint,
+		},
+	}
+
+	body := make([]byte, len(test.expectedBody))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		b.StartTimer()
+		stream, openstreamErr := edgeMux.OpenStream(ctx, headers, nil)
+		_, readBodyErr := stream.Read(body)
+		b.StopTimer()
+
+		require.NoError(b, openstreamErr)
+		assert.True(b, hasHeader(stream, ResponseMetaHeaderField, responseMetaHeaderOrigin))
+		require.True(b, hasHeader(stream, ":status", strconv.Itoa(http.StatusOK)))
+		require.NoError(b, readBodyErr)
+		require.Equal(b, test.expectedBody, body)
+	}
+
+	cancel()
+	wg.Wait()
+}
+
+func BenchmarkServeStreamHTTPSimple(b *testing.B) {
+	test := testRequest{
+		name:           "ok",
+		endpoint:       "/ok",
+		expectedStatus: http.StatusOK,
+		expectedBody:   []byte(http.StatusText(http.StatusOK)),
+	}
+
+	benchmarkServeStreamHTTPSimple(b, test)
+}
+
+func BenchmarkServeStreamHTTPLargeFile(b *testing.B) {
+	test := testRequest{
+		name:           "large_file",
+		endpoint:       "/large_file",
+		expectedStatus: http.StatusOK,
+		expectedBody:   testLargeResp,
+	}
+
+	benchmarkServeStreamHTTPSimple(b, test)
+}

connection/header.go

@@ -0,0 +1,31 @@
+package connection
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/cloudflare/cloudflared/h2mux"
+)
+
+const (
+	ResponseMetaHeaderField = "cf-cloudflared-response-meta"
+)
+
+var (
+	canonicalResponseUserHeadersField = http.CanonicalHeaderKey(h2mux.ResponseUserHeadersField)
+	canonicalResponseMetaHeaderField  = http.CanonicalHeaderKey(ResponseMetaHeaderField)
+	responseMetaHeaderCfd             = mustInitRespMetaHeader("cloudflared")
+	responseMetaHeaderOrigin          = mustInitRespMetaHeader("origin")
+)
+
+type responseMetaHeader struct {
+	Source string `json:"src"`
+}
+
+func mustInitRespMetaHeader(src string) string {
+	header, err := json.Marshal(responseMetaHeader{Source: src})
+	if err != nil {
+		panic(fmt.Sprintf("Failed to serialize response meta header = %s, err: %v", src, err))
+	}
+	return string(header)
+}

connection/http2.go

@@ -0,0 +1,221 @@
+package connection
+
+import (
+	"context"
+	"errors"
+	"io"
+	"math"
+	"net"
+	"net/http"
+	"strings"
+	"sync"
+
+	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/logger"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+
+	"golang.org/x/net/http2"
+)
+
+const (
+	internalUpgradeHeader = "Cf-Cloudflared-Proxy-Connection-Upgrade"
+	websocketUpgrade      = "websocket"
+	controlStreamUpgrade  = "control-stream"
+)
+
+var (
+	errNotFlusher = errors.New("ResponseWriter doesn't implement http.Flusher")
+)
+
+type http2Connection struct {
+	conn         net.Conn
+	server       *http2.Server
+	config       *Config
+	namedTunnel  *NamedTunnelConfig
+	connOptions  *tunnelpogs.ConnectionOptions
+	observer     *Observer
+	connIndexStr string
+	connIndex    uint8
+	wg           *sync.WaitGroup
+	// newRPCClientFunc allows us to mock RPCs during testing
+	newRPCClientFunc func(context.Context, io.ReadWriteCloser, logger.Service) NamedTunnelRPCClient
+	connectedFuse    ConnectedFuse
+}
+
+func NewHTTP2Connection(
+	conn net.Conn,
+	config *Config,
+	namedTunnelConfig *NamedTunnelConfig,
+	connOptions *tunnelpogs.ConnectionOptions,
+	observer *Observer,
+	connIndex uint8,
+	connectedFuse ConnectedFuse,
+) *http2Connection {
+	return &http2Connection{
+		conn: conn,
+		server: &http2.Server{
+			MaxConcurrentStreams: math.MaxUint32,
+		},
+		config:           config,
+		namedTunnel:      namedTunnelConfig,
+		connOptions:      connOptions,
+		observer:         observer,
+		connIndexStr:     uint8ToString(connIndex),
+		connIndex:        connIndex,
+		wg:               &sync.WaitGroup{},
+		newRPCClientFunc: newRegistrationRPCClient,
+		connectedFuse:    connectedFuse,
+	}
+}
+
+func (c *http2Connection) Serve(ctx context.Context) {
+	go func() {
+		<-ctx.Done()
+		c.close()
+	}()
+	c.server.ServeConn(c.conn, &http2.ServeConnOpts{
+		Context: ctx,
+		Handler: c,
+	})
+}
+
+func (c *http2Connection) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	c.wg.Add(1)
+	defer c.wg.Done()
+
+	respWriter := &http2RespWriter{
+		r: r.Body,
+		w: w,
+	}
+	flusher, isFlusher := w.(http.Flusher)
+	if !isFlusher {
+		c.observer.Errorf("%T doesn't implement http.Flusher", w)
+		respWriter.WriteErrorResponse()
+		return
+	}
+	respWriter.flusher = flusher
+	var err error
+	if isControlStreamUpgrade(r) {
+		respWriter.shouldFlush = true
+		err = c.serveControlStream(r.Context(), respWriter)
+	} else if isWebsocketUpgrade(r) {
+		respWriter.shouldFlush = true
+		stripWebsocketUpgradeHeader(r)
+		err = c.config.OriginClient.Proxy(respWriter, r, true)
+	} else {
+		err = c.config.OriginClient.Proxy(respWriter, r, false)
+	}
+
+	if err != nil {
+		respWriter.WriteErrorResponse()
+	}
+}
+
+func (c *http2Connection) serveControlStream(ctx context.Context, respWriter *http2RespWriter) error {
+	rpcClient := c.newRPCClientFunc(ctx, respWriter, c.observer)
+	defer rpcClient.Close()
+
+	if err := rpcClient.RegisterConnection(ctx, c.namedTunnel, c.connOptions, c.connIndex, c.observer); err != nil {
+		return err
+	}
+	c.connectedFuse.Connected()
+
+	<-ctx.Done()
+	rpcClient.GracefulShutdown(ctx, c.config.GracePeriod)
+	return nil
+}
+
+func (c *http2Connection) close() {
+	// Wait for all serve HTTP handlers to return
+	c.wg.Wait()
+	c.conn.Close()
+}
+
+type http2RespWriter struct {
+	r           io.Reader
+	w           http.ResponseWriter
+	flusher     http.Flusher
+	shouldFlush bool
+}
+
+func (rp *http2RespWriter) WriteRespHeaders(resp *http.Response) error {
+	dest := rp.w.Header()
+	userHeaders := make(http.Header, len(resp.Header))
+	for header, values := range resp.Header {
+		// Since these are http2 headers, they're required to be lowercase
+		h2name := strings.ToLower(header)
+		for _, v := range values {
+			if h2name == "content-length" {
+				// This header has meaning in HTTP/2 and will be used by the edge,
+				// so it should be sent as an HTTP/2 response header.
+				dest.Add(h2name, v)
+				// Since these are http2 headers, they're required to be lowercase
+			} else if !h2mux.IsControlHeader(h2name) || h2mux.IsWebsocketClientHeader(h2name) {
+				// User headers, on the other hand, must all be serialized so that
+				// HTTP/2 header validation won't be applied to HTTP/1 header values
+				userHeaders.Add(h2name, v)
+			}
+		}
+	}
+
+	// Perform user header serialization and set them in the single header
+	dest.Set(canonicalResponseUserHeadersField, h2mux.SerializeHeaders(userHeaders))
+	rp.setResponseMetaHeader(responseMetaHeaderOrigin)
+	status := resp.StatusCode
+	// HTTP2 removes support for 101 Switching Protocols https://tools.ietf.org/html/rfc7540#section-8.1.1
+	if status == http.StatusSwitchingProtocols {
+		status = http.StatusOK
+	}
+	rp.w.WriteHeader(status)
+	if IsServerSentEvent(resp.Header) {
+		rp.shouldFlush = true
+	}
+	if rp.shouldFlush {
+		rp.flusher.Flush()
+	}
+	return nil
+}
+
+func (rp *http2RespWriter) WriteErrorResponse() {
+	rp.setResponseMetaHeader(responseMetaHeaderCfd)
+	rp.w.WriteHeader(http.StatusBadGateway)
+}
+
+func (rp *http2RespWriter) setResponseMetaHeader(value string) {
+	rp.w.Header().Set(canonicalResponseMetaHeaderField, value)
+}
+
+func (rp *http2RespWriter) Read(p []byte) (n int, err error) {
+	return rp.r.Read(p)
+}
+
+func (rp *http2RespWriter) Write(p []byte) (n int, err error) {
+	defer func() {
+		// Implementer of OriginClient should make sure it doesn't write to the connection after Proxy returns
+		// Register a recover routine just in case.
+		if r := recover(); r != nil {
+			println("Recover from http2 response writer panic, error", r)
+		}
+	}()
+	n, err = rp.w.Write(p)
+	if err == nil && rp.shouldFlush {
+		rp.flusher.Flush()
+	}
+	return n, err
+}
+
+func (rp *http2RespWriter) Close() error {
+	return nil
+}
+
+func isControlStreamUpgrade(r *http.Request) bool {
+	return strings.ToLower(r.Header.Get(internalUpgradeHeader)) == controlStreamUpgrade
+}
+
+func isWebsocketUpgrade(r *http.Request) bool {
+	return strings.ToLower(r.Header.Get(internalUpgradeHeader)) == websocketUpgrade
+}
+
+func stripWebsocketUpgradeHeader(r *http.Request) {
+	r.Header.Del(internalUpgradeHeader)
+}

connection/http2_test.go

@@ -0,0 +1,303 @@
+package connection
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/cloudflare/cloudflared/logger"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/gobwas/ws/wsutil"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/net/http2"
+)
+
+var (
+	testTransport = http2.Transport{}
+)
+
+func newTestHTTP2Connection() (*http2Connection, net.Conn) {
+	edgeConn, originConn := net.Pipe()
+	var connIndex = uint8(0)
+	return NewHTTP2Connection(
+		originConn,
+		testConfig,
+		&NamedTunnelConfig{},
+		&pogs.ConnectionOptions{},
+		testObserver,
+		connIndex,
+		mockConnectedFuse{},
+	), edgeConn
+}
+
+func TestServeHTTP(t *testing.T) {
+	tests := []testRequest{
+		{
+			name:           "ok",
+			endpoint:       "ok",
+			expectedStatus: http.StatusOK,
+			expectedBody:   []byte(http.StatusText(http.StatusOK)),
+		},
+		{
+			name:           "large_file",
+			endpoint:       "large_file",
+			expectedStatus: http.StatusOK,
+			expectedBody:   testLargeResp,
+		},
+		{
+			name:           "Bad request",
+			endpoint:       "400",
+			expectedStatus: http.StatusBadRequest,
+			expectedBody:   []byte(http.StatusText(http.StatusBadRequest)),
+		},
+		{
+			name:           "Internal server error",
+			endpoint:       "500",
+			expectedStatus: http.StatusInternalServerError,
+			expectedBody:   []byte(http.StatusText(http.StatusInternalServerError)),
+		},
+		{
+			name:           "Proxy error",
+			endpoint:       "error",
+			expectedStatus: http.StatusBadGateway,
+			expectedBody:   nil,
+			isProxyError:   true,
+		},
+	}
+
+	http2Conn, edgeConn := newTestHTTP2Connection()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		http2Conn.Serve(ctx)
+	}()
+
+	edgeHTTP2Conn, err := testTransport.NewClientConn(edgeConn)
+	require.NoError(t, err)
+
+	for _, test := range tests {
+		endpoint := fmt.Sprintf("http://localhost:8080/%s", test.endpoint)
+		req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
+		require.NoError(t, err)
+
+		resp, err := edgeHTTP2Conn.RoundTrip(req)
+		require.NoError(t, err)
+		require.Equal(t, test.expectedStatus, resp.StatusCode)
+		if test.expectedBody != nil {
+			respBody, err := ioutil.ReadAll(resp.Body)
+			require.NoError(t, err)
+			require.Equal(t, test.expectedBody, respBody)
+		}
+		if test.isProxyError {
+			require.Equal(t, responseMetaHeaderCfd, resp.Header.Get(ResponseMetaHeaderField))
+		} else {
+			require.Equal(t, responseMetaHeaderOrigin, resp.Header.Get(ResponseMetaHeaderField))
+		}
+	}
+	cancel()
+	wg.Wait()
+}
+
+type mockNamedTunnelRPCClient struct {
+	registered   chan struct{}
+	unregistered chan struct{}
+}
+
+func (mc mockNamedTunnelRPCClient) RegisterConnection(
+	c context.Context,
+	config *NamedTunnelConfig,
+	options *tunnelpogs.ConnectionOptions,
+	connIndex uint8,
+	observer *Observer,
+) error {
+	close(mc.registered)
+	return nil
+}
+
+func (mc mockNamedTunnelRPCClient) GracefulShutdown(ctx context.Context, gracePeriod time.Duration) {
+	close(mc.unregistered)
+}
+
+func (mockNamedTunnelRPCClient) Close() {}
+
+type mockRPCClientFactory struct {
+	registered   chan struct{}
+	unregistered chan struct{}
+}
+
+func (mf *mockRPCClientFactory) newMockRPCClient(context.Context, io.ReadWriteCloser, logger.Service) NamedTunnelRPCClient {
+	return mockNamedTunnelRPCClient{
+		registered:   mf.registered,
+		unregistered: mf.unregistered,
+	}
+}
+
+type wsRespWriter struct {
+	*httptest.ResponseRecorder
+	readPipe  *io.PipeReader
+	writePipe *io.PipeWriter
+}
+
+func newWSRespWriter() *wsRespWriter {
+	readPipe, writePipe := io.Pipe()
+	return &wsRespWriter{
+		httptest.NewRecorder(),
+		readPipe,
+		writePipe,
+	}
+}
+
+func (w *wsRespWriter) RespBody() io.ReadWriter {
+	return nowriter{w.readPipe}
+}
+
+func (w *wsRespWriter) Write(data []byte) (n int, err error) {
+	return w.writePipe.Write(data)
+}
+
+func TestServeWS(t *testing.T) {
+	http2Conn, _ := newTestHTTP2Connection()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		http2Conn.Serve(ctx)
+	}()
+
+	respWriter := newWSRespWriter()
+	readPipe, writePipe := io.Pipe()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://localhost:8080/ws", readPipe)
+	require.NoError(t, err)
+	req.Header.Set(internalUpgradeHeader, websocketUpgrade)
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		http2Conn.ServeHTTP(respWriter, req)
+	}()
+
+	data := []byte("test websocket")
+	err = wsutil.WriteClientText(writePipe, data)
+	require.NoError(t, err)
+
+	respBody, err := wsutil.ReadServerText(respWriter.RespBody())
+	require.NoError(t, err)
+	require.Equal(t, data, respBody, fmt.Sprintf("Expect %s, got %s", string(data), string(respBody)))
+
+	cancel()
+	resp := respWriter.Result()
+	// http2RespWriter should rewrite status 101 to 200
+	require.Equal(t, http.StatusOK, resp.StatusCode)
+	require.Equal(t, responseMetaHeaderOrigin, resp.Header.Get(ResponseMetaHeaderField))
+
+	wg.Wait()
+}
+
+func TestServeControlStream(t *testing.T) {
+	http2Conn, edgeConn := newTestHTTP2Connection()
+
+	rpcClientFactory := mockRPCClientFactory{
+		registered:   make(chan struct{}),
+		unregistered: make(chan struct{}),
+	}
+	http2Conn.newRPCClientFunc = rpcClientFactory.newMockRPCClient
+
+	ctx, cancel := context.WithCancel(context.Background())
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		http2Conn.Serve(ctx)
+	}()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://localhost:8080/", nil)
+	require.NoError(t, err)
+	req.Header.Set(internalUpgradeHeader, controlStreamUpgrade)
+
+	edgeHTTP2Conn, err := testTransport.NewClientConn(edgeConn)
+	require.NoError(t, err)
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		edgeHTTP2Conn.RoundTrip(req)
+	}()
+
+	<-rpcClientFactory.registered
+	cancel()
+	<-rpcClientFactory.unregistered
+
+	wg.Wait()
+}
+
+func benchmarkServeHTTP(b *testing.B, test testRequest) {
+	http2Conn, edgeConn := newTestHTTP2Connection()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		http2Conn.Serve(ctx)
+	}()
+
+	endpoint := fmt.Sprintf("http://localhost:8080/%s", test.endpoint)
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
+	require.NoError(b, err)
+
+	edgeHTTP2Conn, err := testTransport.NewClientConn(edgeConn)
+	require.NoError(b, err)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		b.StartTimer()
+		resp, err := edgeHTTP2Conn.RoundTrip(req)
+		b.StopTimer()
+		require.NoError(b, err)
+		require.Equal(b, test.expectedStatus, resp.StatusCode)
+		if test.expectedBody != nil {
+			respBody, err := ioutil.ReadAll(resp.Body)
+			require.NoError(b, err)
+			require.Equal(b, test.expectedBody, respBody)
+		}
+		resp.Body.Close()
+	}
+
+	cancel()
+	wg.Wait()
+}
+func BenchmarkServeHTTPSimple(b *testing.B) {
+	test := testRequest{
+		name:           "ok",
+		endpoint:       "ok",
+		expectedStatus: http.StatusOK,
+		expectedBody:   []byte(http.StatusText(http.StatusOK)),
+	}
+
+	benchmarkServeHTTP(b, test)
+}
+
+func BenchmarkServeHTTPLargeFile(b *testing.B) {
+	test := testRequest{
+		name:           "large_file",
+		endpoint:       "large_file",
+		expectedStatus: http.StatusOK,
+		expectedBody:   testLargeResp,
+	}
+
+	benchmarkServeHTTP(b, test)
+}

connection/json.go

@@ -0,0 +1,7 @@
+package connection
+
+import (
+	jsoniter "github.com/json-iterator/go"
+)
+
+var json = jsoniter.ConfigFastest

connection/metrics.go

@@ -0,0 +1,405 @@
+package connection
+
+import (
+	"sync"
+	"time"
+
+	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const (
+	MetricsNamespace = "cloudflared"
+	TunnelSubsystem  = "tunnel"
+	muxerSubsystem   = "muxer"
+)
+
+type muxerMetrics struct {
+	rtt              *prometheus.GaugeVec
+	rttMin           *prometheus.GaugeVec
+	rttMax           *prometheus.GaugeVec
+	receiveWindowAve *prometheus.GaugeVec
+	sendWindowAve    *prometheus.GaugeVec
+	receiveWindowMin *prometheus.GaugeVec
+	receiveWindowMax *prometheus.GaugeVec
+	sendWindowMin    *prometheus.GaugeVec
+	sendWindowMax    *prometheus.GaugeVec
+	inBoundRateCurr  *prometheus.GaugeVec
+	inBoundRateMin   *prometheus.GaugeVec
+	inBoundRateMax   *prometheus.GaugeVec
+	outBoundRateCurr *prometheus.GaugeVec
+	outBoundRateMin  *prometheus.GaugeVec
+	outBoundRateMax  *prometheus.GaugeVec
+	compBytesBefore  *prometheus.GaugeVec
+	compBytesAfter   *prometheus.GaugeVec
+	compRateAve      *prometheus.GaugeVec
+}
+
+type tunnelMetrics struct {
+	timerRetries    prometheus.Gauge
+	serverLocations *prometheus.GaugeVec
+	// locationLock is a mutex for oldServerLocations
+	locationLock sync.Mutex
+	// oldServerLocations stores the last server the tunnel was connected to
+	oldServerLocations map[string]string
+
+	regSuccess *prometheus.CounterVec
+	regFail    *prometheus.CounterVec
+	rpcFail    *prometheus.CounterVec
+
+	muxerMetrics        *muxerMetrics
+	tunnelsHA           tunnelsForHA
+	userHostnamesCounts *prometheus.CounterVec
+}
+
+func newMuxerMetrics() *muxerMetrics {
+	rtt := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "rtt",
+			Help:      "Round-trip time in millisecond",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(rtt)
+
+	rttMin := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "rtt_min",
+			Help:      "Shortest round-trip time in millisecond",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(rttMin)
+
+	rttMax := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "rtt_max",
+			Help:      "Longest round-trip time in millisecond",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(rttMax)
+
+	receiveWindowAve := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "receive_window_ave",
+			Help:      "Average receive window size in bytes",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(receiveWindowAve)
+
+	sendWindowAve := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "send_window_ave",
+			Help:      "Average send window size in bytes",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(sendWindowAve)
+
+	receiveWindowMin := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "receive_window_min",
+			Help:      "Smallest receive window size in bytes",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(receiveWindowMin)
+
+	receiveWindowMax := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "receive_window_max",
+			Help:      "Largest receive window size in bytes",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(receiveWindowMax)
+
+	sendWindowMin := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "send_window_min",
+			Help:      "Smallest send window size in bytes",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(sendWindowMin)
+
+	sendWindowMax := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "send_window_max",
+			Help:      "Largest send window size in bytes",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(sendWindowMax)
+
+	inBoundRateCurr := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "inbound_bytes_per_sec_curr",
+			Help:      "Current inbounding bytes per second, 0 if there is no incoming connection",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(inBoundRateCurr)
+
+	inBoundRateMin := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "inbound_bytes_per_sec_min",
+			Help:      "Minimum non-zero inbounding bytes per second",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(inBoundRateMin)
+
+	inBoundRateMax := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "inbound_bytes_per_sec_max",
+			Help:      "Maximum inbounding bytes per second",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(inBoundRateMax)
+
+	outBoundRateCurr := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "outbound_bytes_per_sec_curr",
+			Help:      "Current outbounding bytes per second, 0 if there is no outgoing traffic",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(outBoundRateCurr)
+
+	outBoundRateMin := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "outbound_bytes_per_sec_min",
+			Help:      "Minimum non-zero outbounding bytes per second",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(outBoundRateMin)
+
+	outBoundRateMax := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "outbound_bytes_per_sec_max",
+			Help:      "Maximum outbounding bytes per second",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(outBoundRateMax)
+
+	compBytesBefore := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "comp_bytes_before",
+			Help:      "Bytes sent via cross-stream compression, pre compression",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(compBytesBefore)
+
+	compBytesAfter := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "comp_bytes_after",
+			Help:      "Bytes sent via cross-stream compression, post compression",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(compBytesAfter)
+
+	compRateAve := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "comp_rate_ave",
+			Help:      "Average outbound cross-stream compression ratio",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(compRateAve)
+
+	return &muxerMetrics{
+		rtt:              rtt,
+		rttMin:           rttMin,
+		rttMax:           rttMax,
+		receiveWindowAve: receiveWindowAve,
+		sendWindowAve:    sendWindowAve,
+		receiveWindowMin: receiveWindowMin,
+		receiveWindowMax: receiveWindowMax,
+		sendWindowMin:    sendWindowMin,
+		sendWindowMax:    sendWindowMax,
+		inBoundRateCurr:  inBoundRateCurr,
+		inBoundRateMin:   inBoundRateMin,
+		inBoundRateMax:   inBoundRateMax,
+		outBoundRateCurr: outBoundRateCurr,
+		outBoundRateMin:  outBoundRateMin,
+		outBoundRateMax:  outBoundRateMax,
+		compBytesBefore:  compBytesBefore,
+		compBytesAfter:   compBytesAfter,
+		compRateAve:      compRateAve,
+	}
+}
+
+func (m *muxerMetrics) update(connectionID string, metrics *h2mux.MuxerMetrics) {
+	m.rtt.WithLabelValues(connectionID).Set(convertRTTMilliSec(metrics.RTT))
+	m.rttMin.WithLabelValues(connectionID).Set(convertRTTMilliSec(metrics.RTTMin))
+	m.rttMax.WithLabelValues(connectionID).Set(convertRTTMilliSec(metrics.RTTMax))
+	m.receiveWindowAve.WithLabelValues(connectionID).Set(metrics.ReceiveWindowAve)
+	m.sendWindowAve.WithLabelValues(connectionID).Set(metrics.SendWindowAve)
+	m.receiveWindowMin.WithLabelValues(connectionID).Set(float64(metrics.ReceiveWindowMin))
+	m.receiveWindowMax.WithLabelValues(connectionID).Set(float64(metrics.ReceiveWindowMax))
+	m.sendWindowMin.WithLabelValues(connectionID).Set(float64(metrics.SendWindowMin))
+	m.sendWindowMax.WithLabelValues(connectionID).Set(float64(metrics.SendWindowMax))
+	m.inBoundRateCurr.WithLabelValues(connectionID).Set(float64(metrics.InBoundRateCurr))
+	m.inBoundRateMin.WithLabelValues(connectionID).Set(float64(metrics.InBoundRateMin))
+	m.inBoundRateMax.WithLabelValues(connectionID).Set(float64(metrics.InBoundRateMax))
+	m.outBoundRateCurr.WithLabelValues(connectionID).Set(float64(metrics.OutBoundRateCurr))
+	m.outBoundRateMin.WithLabelValues(connectionID).Set(float64(metrics.OutBoundRateMin))
+	m.outBoundRateMax.WithLabelValues(connectionID).Set(float64(metrics.OutBoundRateMax))
+	m.compBytesBefore.WithLabelValues(connectionID).Set(float64(metrics.CompBytesBefore.Value()))
+	m.compBytesAfter.WithLabelValues(connectionID).Set(float64(metrics.CompBytesAfter.Value()))
+	m.compRateAve.WithLabelValues(connectionID).Set(float64(metrics.CompRateAve()))
+}
+
+func convertRTTMilliSec(t time.Duration) float64 {
+	return float64(t / time.Millisecond)
+}
+
+// Metrics that can be collected without asking the edge
+func newTunnelMetrics() *tunnelMetrics {
+	maxConcurrentRequestsPerTunnel := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: TunnelSubsystem,
+			Name:      "max_concurrent_requests_per_tunnel",
+			Help:      "Largest number of concurrent requests proxied through each tunnel so far",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(maxConcurrentRequestsPerTunnel)
+
+	timerRetries := prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: TunnelSubsystem,
+			Name:      "timer_retries",
+			Help:      "Unacknowledged heart beats count",
+		})
+	prometheus.MustRegister(timerRetries)
+
+	serverLocations := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: TunnelSubsystem,
+			Name:      "server_locations",
+			Help:      "Where each tunnel is connected to. 1 means current location, 0 means previous locations.",
+		},
+		[]string{"connection_id", "location"},
+	)
+	prometheus.MustRegister(serverLocations)
+
+	rpcFail := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: TunnelSubsystem,
+			Name:      "tunnel_rpc_fail",
+			Help:      "Count of RPC connection errors by type",
+		},
+		[]string{"error", "rpcName"},
+	)
+	prometheus.MustRegister(rpcFail)
+
+	registerFail := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: TunnelSubsystem,
+			Name:      "tunnel_register_fail",
+			Help:      "Count of tunnel registration errors by type",
+		},
+		[]string{"error", "rpcName"},
+	)
+	prometheus.MustRegister(registerFail)
+
+	userHostnamesCounts := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: TunnelSubsystem,
+			Name:      "user_hostnames_counts",
+			Help:      "Which user hostnames cloudflared is serving",
+		},
+		[]string{"userHostname"},
+	)
+	prometheus.MustRegister(userHostnamesCounts)
+
+	registerSuccess := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: TunnelSubsystem,
+			Name:      "tunnel_register_success",
+			Help:      "Count of successful tunnel registrations",
+		},
+		[]string{"rpcName"},
+	)
+	prometheus.MustRegister(registerSuccess)
+
+	return &tunnelMetrics{
+		timerRetries:        timerRetries,
+		serverLocations:     serverLocations,
+		oldServerLocations:  make(map[string]string),
+		muxerMetrics:        newMuxerMetrics(),
+		tunnelsHA:           NewTunnelsForHA(),
+		regSuccess:          registerSuccess,
+		regFail:             registerFail,
+		rpcFail:             rpcFail,
+		userHostnamesCounts: userHostnamesCounts,
+	}
+}
+
+func (t *tunnelMetrics) updateMuxerMetrics(connectionID string, metrics *h2mux.MuxerMetrics) {
+	t.muxerMetrics.update(connectionID, metrics)
+}
+
+func (t *tunnelMetrics) registerServerLocation(connectionID, loc string) {
+	t.locationLock.Lock()
+	defer t.locationLock.Unlock()
+	if oldLoc, ok := t.oldServerLocations[connectionID]; ok && oldLoc == loc {
+		return
+	} else if ok {
+		t.serverLocations.WithLabelValues(connectionID, oldLoc).Dec()
+	}
+	t.serverLocations.WithLabelValues(connectionID, loc).Inc()
+	t.oldServerLocations[connectionID] = loc
+}

connection/observer.go

@@ -0,0 +1,105 @@
+package connection
+
+import (
+	"fmt"
+	"net/url"
+	"strings"
+
+	"github.com/cloudflare/cloudflared/logger"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+)
+
+type Observer struct {
+	logger.Service
+	metrics          *tunnelMetrics
+	tunnelEventChans []chan Event
+	uiEnabled        bool
+}
+
+func NewObserver(logger logger.Service, tunnelEventChans []chan Event, uiEnabled bool) *Observer {
+	return &Observer{
+		logger,
+		newTunnelMetrics(),
+		tunnelEventChans,
+		uiEnabled,
+	}
+}
+
+func (o *Observer) logServerInfo(connIndex uint8, location, msg string) {
+	o.sendEvent(Event{Index: connIndex, EventType: Connected, Location: location})
+	o.Infof(msg)
+	o.metrics.registerServerLocation(uint8ToString(connIndex), location)
+}
+
+func (o *Observer) logTrialHostname(registration *tunnelpogs.TunnelRegistration) error {
+	// Print out the user's trial zone URL in a nice box (if they requested and got one and UI flag is not set)
+	if !o.uiEnabled {
+		if registrationURL, err := url.Parse(registration.Url); err == nil {
+			for _, line := range asciiBox(trialZoneMsg(registrationURL.String()), 2) {
+				o.Info(line)
+			}
+		} else {
+			o.Error("Failed to connect tunnel, please try again.")
+			return fmt.Errorf("empty URL in response from Cloudflare edge")
+		}
+	}
+	return nil
+}
+
+// Print out the given lines in a nice ASCII box.
+func asciiBox(lines []string, padding int) (box []string) {
+	maxLen := maxLen(lines)
+	spacer := strings.Repeat(" ", padding)
+
+	border := "+" + strings.Repeat("-", maxLen+(padding*2)) + "+"
+
+	box = append(box, border)
+	for _, line := range lines {
+		box = append(box, "|"+spacer+line+strings.Repeat(" ", maxLen-len(line))+spacer+"|")
+	}
+	box = append(box, border)
+	return
+}
+
+func maxLen(lines []string) int {
+	max := 0
+	for _, line := range lines {
+		if len(line) > max {
+			max = len(line)
+		}
+	}
+	return max
+}
+
+func trialZoneMsg(url string) []string {
+	return []string{
+		"Your free tunnel has started! Visit it:",
+		"  " + url,
+	}
+}
+
+func (o *Observer) sendRegisteringEvent() {
+	o.sendEvent(Event{EventType: RegisteringTunnel})
+}
+
+func (o *Observer) sendConnectedEvent(connIndex uint8, location string) {
+	o.sendEvent(Event{Index: connIndex, EventType: Connected, Location: location})
+}
+
+func (o *Observer) sendURL(url string) {
+	o.sendEvent(Event{EventType: SetURL, URL: url})
+}
+
+func (o *Observer) SendReconnect(connIndex uint8) {
+	o.sendEvent(Event{Index: connIndex, EventType: Reconnecting})
+}
+
+func (o *Observer) SendDisconnect(connIndex uint8) {
+	o.sendEvent(Event{Index: connIndex, EventType: Disconnected})
+}
+
+func (o *Observer) sendEvent(e Event) {
+	for _, ch := range o.tunnelEventChans {
+		ch <- e
+	}
+}

connection/observer_test.go

@@ -0,0 +1,45 @@
+package connection
+
+import (
+	"strconv"
+	"sync"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// can only be called once
+var m = newTunnelMetrics()
+
+func TestRegisterServerLocation(t *testing.T) {
+	tunnels := 20
+	var wg sync.WaitGroup
+	wg.Add(tunnels)
+	for i := 0; i < tunnels; i++ {
+		go func(i int) {
+			id := strconv.Itoa(i)
+			m.registerServerLocation(id, "LHR")
+			wg.Done()
+		}(i)
+	}
+	wg.Wait()
+	for i := 0; i < tunnels; i++ {
+		id := strconv.Itoa(i)
+		assert.Equal(t, "LHR", m.oldServerLocations[id])
+	}
+
+	wg.Add(tunnels)
+	for i := 0; i < tunnels; i++ {
+		go func(i int) {
+			id := strconv.Itoa(i)
+			m.registerServerLocation(id, "AUS")
+			wg.Done()
+		}(i)
+	}
+	wg.Wait()
+	for i := 0; i < tunnels; i++ {
+		id := strconv.Itoa(i)
+		assert.Equal(t, "AUS", m.oldServerLocations[id])
+	}
+
+}

connection/protocol.go

@@ -0,0 +1,179 @@
+package connection
+
+import (
+	"fmt"
+	"hash/fnv"
+	"sync"
+	"time"
+
+	"github.com/cloudflare/cloudflared/logger"
+)
+
+const (
+	AvailableProtocolFlagMessage = "Available protocols: http2 - Go's implementation, h2mux - Cloudflare's implementation of HTTP/2, and auto - automatically select between http2 and h2mux"
+	// edgeH2muxTLSServerName is the server name to establish h2mux connection with edge
+	edgeH2muxTLSServerName = "cftunnel.com"
+	// edgeH2TLSServerName is the server name to establish http2 connection with edge
+	edgeH2TLSServerName = "h2.cftunnel.com"
+	// threshold to switch back to h2mux when the user intentionally pick --protocol http2
+	explicitHTTP2FallbackThreshold = -1
+	autoSelectFlag                 = "auto"
+)
+
+var (
+	ProtocolList = []Protocol{H2mux, HTTP2}
+)
+
+type Protocol int64
+
+const (
+	H2mux Protocol = iota
+	HTTP2
+)
+
+func (p Protocol) ServerName() string {
+	switch p {
+	case H2mux:
+		return edgeH2muxTLSServerName
+	case HTTP2:
+		return edgeH2TLSServerName
+	default:
+		return ""
+	}
+}
+
+// Fallback returns the fallback protocol and whether the protocol has a fallback
+func (p Protocol) fallback() (Protocol, bool) {
+	switch p {
+	case H2mux:
+		return 0, false
+	case HTTP2:
+		return H2mux, true
+	default:
+		return 0, false
+	}
+}
+
+func (p Protocol) String() string {
+	switch p {
+	case H2mux:
+		return "h2mux"
+	case HTTP2:
+		return "http2"
+	default:
+		return fmt.Sprintf("unknown protocol")
+	}
+}
+
+type ProtocolSelector interface {
+	Current() Protocol
+	Fallback() (Protocol, bool)
+}
+
+type staticProtocolSelector struct {
+	current Protocol
+}
+
+func (s *staticProtocolSelector) Current() Protocol {
+	return s.current
+}
+
+func (s *staticProtocolSelector) Fallback() (Protocol, bool) {
+	return 0, false
+}
+
+type autoProtocolSelector struct {
+	lock           sync.RWMutex
+	current        Protocol
+	switchThrehold int32
+	fetchFunc      PercentageFetcher
+	refreshAfter   time.Time
+	ttl            time.Duration
+	logger         logger.Service
+}
+
+func newAutoProtocolSelector(
+	current Protocol,
+	switchThrehold int32,
+	fetchFunc PercentageFetcher,
+	ttl time.Duration,
+	logger logger.Service,
+) *autoProtocolSelector {
+	return &autoProtocolSelector{
+		current:        current,
+		switchThrehold: switchThrehold,
+		fetchFunc:      fetchFunc,
+		refreshAfter:   time.Now().Add(ttl),
+		ttl:            ttl,
+		logger:         logger,
+	}
+}
+
+func (s *autoProtocolSelector) Current() Protocol {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+	if time.Now().Before(s.refreshAfter) {
+		return s.current
+	}
+
+	percentage, err := s.fetchFunc()
+	if err != nil {
+		s.logger.Errorf("Failed to refresh protocol, err: %v", err)
+		return s.current
+	}
+
+	if s.switchThrehold < percentage {
+		s.current = HTTP2
+	} else {
+		s.current = H2mux
+	}
+	s.refreshAfter = time.Now().Add(s.ttl)
+	return s.current
+}
+
+func (s *autoProtocolSelector) Fallback() (Protocol, bool) {
+	s.lock.RLock()
+	defer s.lock.RUnlock()
+	return s.current.fallback()
+}
+
+type PercentageFetcher func() (int32, error)
+
+func NewProtocolSelector(protocolFlag string, namedTunnel *NamedTunnelConfig, fetchFunc PercentageFetcher, ttl time.Duration, logger logger.Service) (ProtocolSelector, error) {
+	if namedTunnel == nil {
+		return &staticProtocolSelector{
+			current: H2mux,
+		}, nil
+	}
+	if protocolFlag == H2mux.String() {
+		return &staticProtocolSelector{
+			current: H2mux,
+		}, nil
+	}
+
+	http2Percentage, err := fetchFunc()
+	if err != nil {
+		return nil, err
+	}
+	if protocolFlag == HTTP2.String() {
+		if http2Percentage < 0 {
+			return newAutoProtocolSelector(H2mux, explicitHTTP2FallbackThreshold, fetchFunc, ttl, logger), nil
+		}
+		return newAutoProtocolSelector(HTTP2, explicitHTTP2FallbackThreshold, fetchFunc, ttl, logger), nil
+	}
+
+	if protocolFlag != autoSelectFlag {
+		return nil, fmt.Errorf("Unknown protocol %s, %s", protocolFlag, AvailableProtocolFlagMessage)
+	}
+	threshold := switchThreshold(namedTunnel.Credentials.AccountTag)
+	if threshold < http2Percentage {
+		return newAutoProtocolSelector(HTTP2, threshold, fetchFunc, ttl, logger), nil
+	}
+	return newAutoProtocolSelector(H2mux, threshold, fetchFunc, ttl, logger), nil
+}
+
+func switchThreshold(accountTag string) int32 {
+	h := fnv.New32a()
+	h.Write([]byte(accountTag))
+	return int32(h.Sum32() % 100)
+}

connection/protocol_test.go

@@ -0,0 +1,219 @@
+package connection
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/cloudflare/cloudflared/logger"
+	"github.com/stretchr/testify/assert"
+)
+
+const (
+	testNoTTL = 0
+)
+
+var (
+	testNamedTunnelConfig = &NamedTunnelConfig{
+		Credentials: Credentials{
+			AccountTag: "testAccountTag",
+		},
+	}
+)
+
+func mockFetcher(percentage int32) PercentageFetcher {
+	return func() (int32, error) {
+		return percentage, nil
+	}
+}
+
+func mockFetcherWithError() PercentageFetcher {
+	return func() (int32, error) {
+		return 0, fmt.Errorf("failed to fetch precentage")
+	}
+}
+
+type dynamicMockFetcher struct {
+	percentage int32
+	err        error
+}
+
+func (dmf *dynamicMockFetcher) fetch() PercentageFetcher {
+	return func() (int32, error) {
+		if dmf.err != nil {
+			return 0, dmf.err
+		}
+		return dmf.percentage, nil
+	}
+}
+
+func TestNewProtocolSelector(t *testing.T) {
+	tests := []struct {
+		name              string
+		protocol          string
+		expectedProtocol  Protocol
+		hasFallback       bool
+		expectedFallback  Protocol
+		namedTunnelConfig *NamedTunnelConfig
+		fetchFunc         PercentageFetcher
+		wantErr           bool
+	}{
+		{
+			name:              "classic tunnel",
+			protocol:          "h2mux",
+			expectedProtocol:  H2mux,
+			namedTunnelConfig: nil,
+		},
+		{
+			name:              "named tunnel over h2mux",
+			protocol:          "h2mux",
+			expectedProtocol:  H2mux,
+			namedTunnelConfig: testNamedTunnelConfig,
+		},
+		{
+			name:              "named tunnel over http2",
+			protocol:          "http2",
+			expectedProtocol:  HTTP2,
+			hasFallback:       true,
+			expectedFallback:  H2mux,
+			fetchFunc:         mockFetcher(0),
+			namedTunnelConfig: testNamedTunnelConfig,
+		},
+		{
+			name:              "named tunnel http2 disabled",
+			protocol:          "http2",
+			expectedProtocol:  H2mux,
+			fetchFunc:         mockFetcher(-1),
+			namedTunnelConfig: testNamedTunnelConfig,
+		},
+		{
+			name:              "named tunnel auto all http2 disabled",
+			protocol:          "auto",
+			expectedProtocol:  H2mux,
+			fetchFunc:         mockFetcher(-1),
+			namedTunnelConfig: testNamedTunnelConfig,
+		},
+		{
+			name:              "named tunnel auto to h2mux",
+			protocol:          "auto",
+			expectedProtocol:  H2mux,
+			fetchFunc:         mockFetcher(0),
+			namedTunnelConfig: testNamedTunnelConfig,
+		},
+		{
+			name:              "named tunnel auto to http2",
+			protocol:          "auto",
+			expectedProtocol:  HTTP2,
+			hasFallback:       true,
+			expectedFallback:  H2mux,
+			fetchFunc:         mockFetcher(100),
+			namedTunnelConfig: testNamedTunnelConfig,
+		},
+		{
+			// None named tunnel can only use h2mux, so specifying an unknown protocol is not an error
+			name:             "classic tunnel unknown protocol",
+			protocol:         "unknown",
+			expectedProtocol: H2mux,
+		},
+		{
+			name:              "named tunnel unknown protocol",
+			protocol:          "unknown",
+			fetchFunc:         mockFetcher(100),
+			namedTunnelConfig: testNamedTunnelConfig,
+			wantErr:           true,
+		},
+		{
+			name:              "named tunnel fetch error",
+			protocol:          "unknown",
+			fetchFunc:         mockFetcherWithError(),
+			namedTunnelConfig: testNamedTunnelConfig,
+			wantErr:           true,
+		},
+	}
+	logger, _ := logger.New()
+	for _, test := range tests {
+		selector, err := NewProtocolSelector(test.protocol, test.namedTunnelConfig, test.fetchFunc, testNoTTL, logger)
+		if test.wantErr {
+			assert.Error(t, err, fmt.Sprintf("test %s failed", test.name))
+		} else {
+			assert.NoError(t, err, fmt.Sprintf("test %s failed", test.name))
+			assert.Equal(t, test.expectedProtocol, selector.Current(), fmt.Sprintf("test %s failed", test.name))
+			fallback, ok := selector.Fallback()
+			assert.Equal(t, test.hasFallback, ok, fmt.Sprintf("test %s failed", test.name))
+			if test.hasFallback {
+				assert.Equal(t, test.expectedFallback, fallback, fmt.Sprintf("test %s failed", test.name))
+			}
+		}
+	}
+}
+
+func TestAutoProtocolSelectorRefresh(t *testing.T) {
+	logger, _ := logger.New()
+	fetcher := dynamicMockFetcher{}
+	selector, err := NewProtocolSelector("auto", testNamedTunnelConfig, fetcher.fetch(), testNoTTL, logger)
+	assert.NoError(t, err)
+	assert.Equal(t, H2mux, selector.Current())
+
+	fetcher.percentage = 100
+	assert.Equal(t, HTTP2, selector.Current())
+
+	fetcher.percentage = 0
+	assert.Equal(t, H2mux, selector.Current())
+
+	fetcher.percentage = 100
+	assert.Equal(t, HTTP2, selector.Current())
+
+	fetcher.err = fmt.Errorf("failed to fetch")
+	assert.Equal(t, HTTP2, selector.Current())
+
+	fetcher.percentage = -1
+	fetcher.err = nil
+	assert.Equal(t, H2mux, selector.Current())
+
+	fetcher.percentage = 0
+	assert.Equal(t, H2mux, selector.Current())
+
+	fetcher.percentage = 100
+	assert.Equal(t, HTTP2, selector.Current())
+}
+
+func TestHTTP2ProtocolSelectorRefresh(t *testing.T) {
+	logger, _ := logger.New()
+	fetcher := dynamicMockFetcher{}
+	selector, err := NewProtocolSelector("http2", testNamedTunnelConfig, fetcher.fetch(), testNoTTL, logger)
+	assert.NoError(t, err)
+	assert.Equal(t, HTTP2, selector.Current())
+
+	fetcher.percentage = 100
+	assert.Equal(t, HTTP2, selector.Current())
+
+	fetcher.percentage = 0
+	assert.Equal(t, HTTP2, selector.Current())
+
+	fetcher.err = fmt.Errorf("failed to fetch")
+	assert.Equal(t, HTTP2, selector.Current())
+
+	fetcher.percentage = -1
+	fetcher.err = nil
+	assert.Equal(t, H2mux, selector.Current())
+
+	fetcher.percentage = 0
+	assert.Equal(t, HTTP2, selector.Current())
+
+	fetcher.percentage = 100
+	assert.Equal(t, HTTP2, selector.Current())
+
+	fetcher.percentage = -1
+	assert.Equal(t, H2mux, selector.Current())
+}
+
+func TestProtocolSelectorRefreshTTL(t *testing.T) {
+	logger, _ := logger.New()
+	fetcher := dynamicMockFetcher{percentage: 100}
+	selector, err := NewProtocolSelector("auto", testNamedTunnelConfig, fetcher.fetch(), time.Hour, logger)
+	assert.NoError(t, err)
+	assert.Equal(t, HTTP2, selector.Current())
+
+	fetcher.percentage = 0
+	assert.Equal(t, HTTP2, selector.Current())
+}

connection/rpc.go

@@ -2,40 +2,294 @@ package connection
 
 import (
 	"context"
+	"fmt"
 	"io"
-
-	rpc "zombiezen.com/go/capnproto2/rpc"
+	"time"
 
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/tunnelrpc"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"zombiezen.com/go/capnproto2/rpc"
 )
 
-// NewTunnelRPCClient creates and returns a new RPC client, which will communicate
-// using a stream on the given muxer
-func NewTunnelRPCClient(
+type tunnelServerClient struct {
+	client    tunnelpogs.TunnelServer_PogsClient
+	transport rpc.Transport
+}
+
+// NewTunnelRPCClient creates and returns a new RPC client, which will communicate using a stream on the given muxer.
+// This method is exported for supervisor to call Authenticate RPC
+func NewTunnelServerClient(
 	ctx context.Context,
 	stream io.ReadWriteCloser,
 	logger logger.Service,
-) (client tunnelpogs.TunnelServer_PogsClient, err error) {
+) *tunnelServerClient {
+	transport := tunnelrpc.NewTransportLogger(logger, rpc.StreamTransport(stream))
 	conn := rpc.NewConn(
-		tunnelrpc.NewTransportLogger(logger, rpc.StreamTransport(stream)),
+		transport,
 		tunnelrpc.ConnLog(logger),
 	)
 	registrationClient := tunnelpogs.RegistrationServer_PogsClient{Client: conn.Bootstrap(ctx), Conn: conn}
-	client = tunnelpogs.TunnelServer_PogsClient{RegistrationServer_PogsClient: registrationClient, Client: conn.Bootstrap(ctx), Conn: conn}
-	return client, nil
+	return &tunnelServerClient{
+		client:    tunnelpogs.TunnelServer_PogsClient{RegistrationServer_PogsClient: registrationClient, Client: conn.Bootstrap(ctx), Conn: conn},
+		transport: transport,
+	}
 }
 
-func NewRegistrationRPCClient(
+func (tsc *tunnelServerClient) Authenticate(ctx context.Context, classicTunnel *ClassicTunnelConfig, registrationOptions *tunnelpogs.RegistrationOptions) (tunnelpogs.AuthOutcome, error) {
+	authResp, err := tsc.client.Authenticate(ctx, classicTunnel.OriginCert, classicTunnel.Hostname, registrationOptions)
+	if err != nil {
+		return nil, err
+	}
+	return authResp.Outcome(), nil
+}
+
+func (tsc *tunnelServerClient) Close() {
+	// Closing the client will also close the connection
+	tsc.client.Close()
+	tsc.transport.Close()
+}
+
+type NamedTunnelRPCClient interface {
+	RegisterConnection(
+		c context.Context,
+		config *NamedTunnelConfig,
+		options *tunnelpogs.ConnectionOptions,
+		connIndex uint8,
+		observer *Observer,
+	) error
+	GracefulShutdown(ctx context.Context, gracePeriod time.Duration)
+	Close()
+}
+
+type registrationServerClient struct {
+	client    tunnelpogs.RegistrationServer_PogsClient
+	transport rpc.Transport
+}
+
+func newRegistrationRPCClient(
 	ctx context.Context,
 	stream io.ReadWriteCloser,
 	logger logger.Service,
-) (client tunnelpogs.RegistrationServer_PogsClient, err error) {
+) NamedTunnelRPCClient {
+	transport := tunnelrpc.NewTransportLogger(logger, rpc.StreamTransport(stream))
 	conn := rpc.NewConn(
-		tunnelrpc.NewTransportLogger(logger, rpc.StreamTransport(stream)),
+		transport,
 		tunnelrpc.ConnLog(logger),
 	)
-	client = tunnelpogs.RegistrationServer_PogsClient{Client: conn.Bootstrap(ctx), Conn: conn}
-	return client, nil
+	return &registrationServerClient{
+		client:    tunnelpogs.RegistrationServer_PogsClient{Client: conn.Bootstrap(ctx), Conn: conn},
+		transport: transport,
+	}
+}
+
+func (rsc *registrationServerClient) RegisterConnection(
+	ctx context.Context,
+	config *NamedTunnelConfig,
+	options *tunnelpogs.ConnectionOptions,
+	connIndex uint8,
+	observer *Observer,
+) error {
+	conn, err := rsc.client.RegisterConnection(
+		ctx,
+		config.Credentials.Auth(),
+		config.Credentials.TunnelID,
+		connIndex,
+		options,
+	)
+	if err != nil {
+		if err.Error() == DuplicateConnectionError {
+			observer.metrics.regFail.WithLabelValues("dup_edge_conn", "registerConnection").Inc()
+			return errDuplicationConnection
+		}
+		observer.metrics.regFail.WithLabelValues("server_error", "registerConnection").Inc()
+		return serverRegistrationErrorFromRPC(err)
+	}
+
+	observer.metrics.regSuccess.WithLabelValues("registerConnection").Inc()
+
+	observer.logServerInfo(connIndex, conn.Location, fmt.Sprintf("Connection %d registered with %s using ID %s", connIndex, conn.Location, conn.UUID))
+	observer.sendConnectedEvent(connIndex, conn.Location)
+
+	return nil
+}
+
+func (rsc *registrationServerClient) GracefulShutdown(ctx context.Context, gracePeriod time.Duration) {
+	ctx, cancel := context.WithTimeout(ctx, gracePeriod)
+	defer cancel()
+	rsc.client.UnregisterConnection(ctx)
+}
+
+func (rsc *registrationServerClient) Close() {
+	// Closing the client will also close the connection
+	rsc.client.Close()
+	// Closing the transport also closes the stream
+	rsc.transport.Close()
+}
+
+type rpcName string
+
+const (
+	register     rpcName = "register"
+	reconnect    rpcName = "reconnect"
+	unregister   rpcName = "unregister"
+	authenticate rpcName = " authenticate"
+)
+
+func (h *h2muxConnection) registerTunnel(ctx context.Context, credentialSetter CredentialManager, classicTunnel *ClassicTunnelConfig, registrationOptions *tunnelpogs.RegistrationOptions) error {
+	h.observer.sendRegisteringEvent()
+
+	stream, err := h.newRPCStream(ctx, register)
+	if err != nil {
+		return err
+	}
+	rpcClient := NewTunnelServerClient(ctx, stream, h.observer)
+	defer rpcClient.Close()
+
+	h.logServerInfo(ctx, rpcClient)
+	registration := rpcClient.client.RegisterTunnel(
+		ctx,
+		classicTunnel.OriginCert,
+		classicTunnel.Hostname,
+		registrationOptions,
+	)
+	if registrationErr := registration.DeserializeError(); registrationErr != nil {
+		// RegisterTunnel RPC failure
+		return h.processRegisterTunnelError(registrationErr, register)
+	}
+
+	// Send free tunnel URL to UI
+	h.observer.sendURL(registration.Url)
+	credentialSetter.SetEventDigest(h.connIndex, registration.EventDigest)
+	return h.processRegistrationSuccess(registration, register, credentialSetter, classicTunnel)
+}
+
+type CredentialManager interface {
+	ReconnectToken() ([]byte, error)
+	EventDigest(connID uint8) ([]byte, error)
+	SetEventDigest(connID uint8, digest []byte)
+	ConnDigest(connID uint8) ([]byte, error)
+	SetConnDigest(connID uint8, digest []byte)
+}
+
+func (h *h2muxConnection) processRegistrationSuccess(
+	registration *tunnelpogs.TunnelRegistration,
+	name rpcName,
+	credentialManager CredentialManager, classicTunnel *ClassicTunnelConfig,
+) error {
+	for _, logLine := range registration.LogLines {
+		h.observer.Info(logLine)
+	}
+
+	if registration.TunnelID != "" {
+		h.observer.metrics.tunnelsHA.AddTunnelID(h.connIndex, registration.TunnelID)
+		h.observer.Infof("Each HA connection's tunnel IDs: %v", h.observer.metrics.tunnelsHA.String())
+	}
+
+	// Print out the user's trial zone URL in a nice box (if they requested and got one and UI flag is not set)
+	if classicTunnel.IsTrialZone() {
+		err := h.observer.logTrialHostname(registration)
+		if err != nil {
+			return err
+		}
+	}
+
+	credentialManager.SetConnDigest(h.connIndex, registration.ConnDigest)
+	h.observer.metrics.userHostnamesCounts.WithLabelValues(registration.Url).Inc()
+
+	h.observer.Infof("Route propagating, it may take up to 1 minute for your new route to become functional")
+	h.observer.metrics.regSuccess.WithLabelValues(string(name)).Inc()
+	return nil
+}
+
+func (h *h2muxConnection) processRegisterTunnelError(err tunnelpogs.TunnelRegistrationError, name rpcName) error {
+	if err.Error() == DuplicateConnectionError {
+		h.observer.metrics.regFail.WithLabelValues("dup_edge_conn", string(name)).Inc()
+		return errDuplicationConnection
+	}
+	h.observer.metrics.regFail.WithLabelValues("server_error", string(name)).Inc()
+	return serverRegisterTunnelError{
+		cause:     err,
+		permanent: err.IsPermanent(),
+	}
+}
+
+func (h *h2muxConnection) reconnectTunnel(ctx context.Context, credentialManager CredentialManager, classicTunnel *ClassicTunnelConfig, registrationOptions *tunnelpogs.RegistrationOptions) error {
+	token, err := credentialManager.ReconnectToken()
+	if err != nil {
+		return err
+	}
+	eventDigest, err := credentialManager.EventDigest(h.connIndex)
+	if err != nil {
+		return err
+	}
+	connDigest, err := credentialManager.ConnDigest(h.connIndex)
+	if err != nil {
+		return err
+	}
+
+	h.observer.Debug("initiating RPC stream to reconnect")
+	stream, err := h.newRPCStream(ctx, register)
+	if err != nil {
+		return err
+	}
+	rpcClient := NewTunnelServerClient(ctx, stream, h.observer)
+	defer rpcClient.Close()
+
+	h.logServerInfo(ctx, rpcClient)
+	registration := rpcClient.client.ReconnectTunnel(
+		ctx,
+		token,
+		eventDigest,
+		connDigest,
+		classicTunnel.Hostname,
+		registrationOptions,
+	)
+	if registrationErr := registration.DeserializeError(); registrationErr != nil {
+		// ReconnectTunnel RPC failure
+		return h.processRegisterTunnelError(registrationErr, reconnect)
+	}
+	return h.processRegistrationSuccess(registration, reconnect, credentialManager, classicTunnel)
+}
+
+func (h *h2muxConnection) logServerInfo(ctx context.Context, rpcClient *tunnelServerClient) error {
+	// Request server info without blocking tunnel registration; must use capnp library directly.
+	serverInfoPromise := tunnelrpc.TunnelServer{Client: rpcClient.client.Client}.GetServerInfo(ctx, func(tunnelrpc.TunnelServer_getServerInfo_Params) error {
+		return nil
+	})
+	serverInfoMessage, err := serverInfoPromise.Result().Struct()
+	if err != nil {
+		h.observer.Errorf("Failed to retrieve server information: %s", err)
+		return err
+	}
+	serverInfo, err := tunnelpogs.UnmarshalServerInfo(serverInfoMessage)
+	if err != nil {
+		h.observer.Errorf("Failed to retrieve server information: %s", err)
+		return err
+	}
+	h.observer.logServerInfo(h.connIndex, serverInfo.LocationName, fmt.Sprintf("Connnection %d connected to %s", h.connIndex, serverInfo.LocationName))
+	return nil
+}
+
+func (h *h2muxConnection) unregister(isNamedTunnel bool) {
+	unregisterCtx, cancel := context.WithTimeout(context.Background(), h.config.GracePeriod)
+	defer cancel()
+
+	stream, err := h.newRPCStream(unregisterCtx, register)
+	if err != nil {
+		return
+	}
+
+	if isNamedTunnel {
+		rpcClient := newRegistrationRPCClient(unregisterCtx, stream, h.observer)
+		defer rpcClient.Close()
+
+		rpcClient.GracefulShutdown(unregisterCtx, h.config.GracePeriod)
+	} else {
+		rpcClient := NewTunnelServerClient(unregisterCtx, stream, h.observer)
+		defer rpcClient.Close()
+
+		// gracePeriod is encoded in int64 using capnproto
+		rpcClient.client.UnregisterTunnel(unregisterCtx, h.config.GracePeriod.Nanoseconds())
+	}
 }

connection/tunnelsforha.go

@@ -0,0 +1,50 @@
+package connection
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// tunnelsForHA maps this cloudflared instance's HA connections to the tunnel IDs they serve.
+type tunnelsForHA struct {
+	sync.Mutex
+	metrics *prometheus.GaugeVec
+	entries map[uint8]string
+}
+
+// NewTunnelsForHA initializes the Prometheus metrics etc for a tunnelsForHA.
+func NewTunnelsForHA() tunnelsForHA {
+	metrics := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "tunnel_ids",
+			Help: "The ID of all tunnels (and their corresponding HA connection ID) running in this instance of cloudflared.",
+		},
+		[]string{"tunnel_id", "ha_conn_id"},
+	)
+	prometheus.MustRegister(metrics)
+
+	return tunnelsForHA{
+		metrics: metrics,
+		entries: make(map[uint8]string),
+	}
+}
+
+// Track a new tunnel ID, removing the disconnected tunnel (if any) and update metrics.
+func (t *tunnelsForHA) AddTunnelID(haConn uint8, tunnelID string) {
+	t.Lock()
+	defer t.Unlock()
+	haStr := fmt.Sprintf("%v", haConn)
+	if oldTunnelID, ok := t.entries[haConn]; ok {
+		t.metrics.WithLabelValues(oldTunnelID, haStr).Dec()
+	}
+	t.entries[haConn] = tunnelID
+	t.metrics.WithLabelValues(tunnelID, haStr).Inc()
+}
+
+func (t *tunnelsForHA) String() string {
+	t.Lock()
+	defer t.Unlock()
+	return fmt.Sprintf("%v", t.entries)
+}

dbconnect/proxy.go

@@ -43,7 +43,7 @@ func NewInsecureProxy(ctx context.Context, origin string) (*Proxy, error) {
 		return nil, errors.Wrap(err, "could not connect to the database")
 	}
 
-	logger, err := logger.New()
+	logger, err := logger.New() // TODO: Does not obey log configuration
 	if err != nil {
 		return nil, errors.Wrap(err, "error setting up logger")
 	}

dev.Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.15.3 as builder
+FROM golang:1.15.6 as builder
 ENV GO111MODULE=on \
     CGO_ENABLED=0
 WORKDIR /go/src/github.com/cloudflare/cloudflared/

edgediscovery/allregions/discovery.go

@@ -121,15 +121,15 @@ func resolveSRVToTCP(srv *net.SRV) ([]*net.TCPAddr, error) {
 }
 
 // ResolveAddrs resolves TCP address given a list of addresses. Address can be a hostname, however, it will return at most one
-// of the hostname's IP addresses
-func ResolveAddrs(addrs []string) ([]*net.TCPAddr, error) {
-	var tcpAddrs []*net.TCPAddr
+// of the hostname's IP addresses.
+func ResolveAddrs(addrs []string, logger logger.Service) (resolved []*net.TCPAddr) {
 	for _, addr := range addrs {
 		tcpAddr, err := net.ResolveTCPAddr("tcp", addr)
 		if err != nil {
-			return nil, err
+			logger.Errorf("Failed to resolve %s, err: %v", addr, err)
+		} else {
+			resolved = append(resolved, tcpAddr)
 		}
-		tcpAddrs = append(tcpAddrs, tcpAddr)
 	}
-	return tcpAddrs, nil
+	return
 }

edgediscovery/allregions/regions.go

@@ -35,12 +35,12 @@ func ResolveEdge(logger logger.Service) (*Regions, error) {
 
 // StaticEdge creates a list of edge addresses from the list of hostnames.
 // Mainly used for testing connectivity.
-func StaticEdge(hostnames []string) (*Regions, error) {
-	addrs, err := ResolveAddrs(hostnames)
-	if err != nil {
-		return nil, err
+func StaticEdge(hostnames []string, logger logger.Service) (*Regions, error) {
+	resolved := ResolveAddrs(hostnames, logger)
+	if len(resolved) == 0 {
+		return nil, fmt.Errorf("failed to resolve any edge address")
 	}
-	return NewNoResolve(addrs), nil
+	return NewNoResolve(resolved), nil
 }
 
 // NewNoResolve doesn't resolve the edge. Instead it just uses the given addresses.

edgediscovery/dial.go

@@ -1,4 +1,4 @@
-package connection
+package edgediscovery
 
 import (
 	"context"
@@ -9,7 +9,7 @@ import (
 	"github.com/pkg/errors"
 )
 
-// DialEdge makes a TLS connection to a Cloudflare edge node
+// DialEdgeWithH2Mux makes a TLS connection to a Cloudflare edge node
 func DialEdge(
 	ctx context.Context,
 	timeout time.Duration,
@@ -25,6 +25,7 @@ func DialEdge(
 	if err != nil {
 		return nil, newDialError(err, "DialContext error")
 	}
+
 	tlsEdgeConn := tls.Client(edgeConn, tlsConfig)
 	tlsEdgeConn.SetDeadline(time.Now().Add(timeout))
 

edgediscovery/edgediscovery.go

@@ -41,7 +41,7 @@ func ResolveEdge(l logger.Service) (*Edge, error) {
 
 // StaticEdge creates a list of edge addresses from the list of hostnames. Mainly used for testing connectivity.
 func StaticEdge(l logger.Service, hostnames []string) (*Edge, error) {
-	regions, err := allregions.StaticEdge(hostnames)
+	regions, err := allregions.StaticEdge(hostnames, l)
 	if err != nil {
 		return new(Edge), err
 	}

edgediscovery/mocks_for_test.go

@@ -1,4 +1,4 @@
-package connection
+package edgediscovery
 
 import (
 	"fmt"

edgediscovery/protocol.go

@@ -0,0 +1,45 @@
+package edgediscovery
+
+import (
+	"fmt"
+	"net"
+	"strconv"
+	"strings"
+)
+
+const (
+	protocolRecord = "protocol.argotunnel.com"
+)
+
+var (
+	errNoProtocolRecord = fmt.Errorf("No TXT record found for %s to determine connection protocol", protocolRecord)
+)
+
+func HTTP2Percentage() (int32, error) {
+	records, err := net.LookupTXT(protocolRecord)
+	if err != nil {
+		return 0, err
+	}
+	if len(records) == 0 {
+		return 0, errNoProtocolRecord
+	}
+	return parseHTTP2Precentage(records[0])
+}
+
+// The record looks like http2=percentage
+func parseHTTP2Precentage(record string) (int32, error) {
+	const key = "http2"
+	slices := strings.Split(record, "=")
+	if len(slices) != 2 {
+		return 0, fmt.Errorf("Malformed TXT record %s, expect http2=percentage", record)
+	}
+	if slices[0] != key {
+		return 0, fmt.Errorf("Incorrect key %s, expect %s", slices[0], key)
+	}
+	percentage, err := strconv.ParseInt(slices[1], 10, 32)
+	if err != nil {
+		return 0, err
+	}
+	return int32(percentage), nil
+
+}

edgediscovery/protocol_test.go

@@ -0,0 +1,80 @@
+package edgediscovery
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestHTTP2Percentage(t *testing.T) {
+	_, err := HTTP2Percentage()
+	assert.NoError(t, err)
+}
+
+func TestParseHTTP2Precentage(t *testing.T) {
+	tests := []struct {
+		record     string
+		percentage int32
+		wantErr    bool
+	}{
+		{
+			record:     "http2=-1",
+			percentage: -1,
+			wantErr:    false,
+		},
+		{
+			record:     "http2=0",
+			percentage: 0,
+			wantErr:    false,
+		},
+		{
+			record:     "http2=50",
+			percentage: 50,
+			wantErr:    false,
+		},
+		{
+			record:     "http2=100",
+			percentage: 100,
+			wantErr:    false,
+		},
+		{
+			record:     "http2=1000",
+			percentage: 1000,
+			wantErr:    false,
+		},
+		{
+			record:  "http2=10.5",
+			wantErr: true,
+		},
+		{
+			record:  "http2=10 h2mux=90",
+			wantErr: true,
+		},
+		{
+			record:  "http2=ten",
+			wantErr: true,
+		},
+
+		{
+			record:  "h2mux=100",
+			wantErr: true,
+		},
+		{
+			record:  "http2",
+			wantErr: true,
+		},
+		{
+			record:  "http2=",
+			wantErr: true,
+		},
+	}
+
+	for _, test := range tests {
+		p, err := parseHTTP2Precentage(test.record)
+		if test.wantErr {
+			assert.Error(t, err)
+		} else {
+			assert.Equal(t, test.percentage, p)
+		}
+	}
+}

github_release.py

@@ -202,7 +202,7 @@ def main():
         copy_path = os.path.join(artifact_path, args.name)
         try:
             shutil.copy(args.path, copy_path)
-       except shutil.SameFileError:
+        except shutil.SameFileError:
             pass # the macOS release copy fails with being the same file (already in the artifacts directory). Catching to ignore.
 
     except Exception as e:

go.mod

@@ -15,7 +15,7 @@ require (
 	github.com/coreos/go-oidc v0.0.0-20171002155002-a93f71fdfe73
 	github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf
 	github.com/denisenkom/go-mssqldb v0.0.0-20191001013358-cfbb681360f0
-	github.com/equinox-io/equinox v1.2.0
+	github.com/equinox-io/equinox v1.2.0 // indirect
 	github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 // indirect
 	github.com/facebookgo/freeport v0.0.0-20150612182905-d4adf43b75b9 // indirect
 	github.com/facebookgo/grace v0.0.0-20180706040059-75cf19382434
@@ -27,12 +27,16 @@ require (
 	github.com/getsentry/raven-go v0.0.0-20180517221441-ed7bcb39ff10
 	github.com/gliderlabs/ssh v0.0.0-20191009160644-63518b5243e0
 	github.com/go-sql-driver/mysql v1.5.0
+	github.com/gobwas/httphead v0.0.0-20200921212729-da3d93bc3c58 // indirect
+	github.com/gobwas/pool v0.2.1 // indirect
+	github.com/gobwas/ws v1.0.4
 	github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3
 	github.com/google/go-cmp v0.5.2 // indirect
 	github.com/google/uuid v1.1.2
 	github.com/gorilla/mux v1.7.3
 	github.com/gorilla/websocket v1.4.2
 	github.com/jmoiron/sqlx v1.2.0
+	github.com/json-iterator/go v1.1.10
 	github.com/kr/text v0.2.0 // indirect
 	github.com/kshvakov/clickhouse v1.3.11
 	github.com/kylelemons/godebug v1.1.0 // indirect

go.sum

@@ -233,6 +233,12 @@ github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG
 github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/gobwas/httphead v0.0.0-20200921212729-da3d93bc3c58 h1:YyrUZvJaU8Q0QsoVo+xLFBgWDTam29PKea6GYmwvSiQ=
+github.com/gobwas/httphead v0.0.0-20200921212729-da3d93bc3c58/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo=
+github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og=
+github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
+github.com/gobwas/ws v1.0.4 h1:5eXU1CZhpQdq5kXbKb+sECH5Ia5KiO6CYzIzdlVx6Bs=
+github.com/gobwas/ws v1.0.4/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=
 github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
 github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
@@ -369,6 +375,7 @@ github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV
 github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
@@ -441,8 +448,10 @@ github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0Qu
 github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=

h2mux/activestreammap.go

@@ -7,6 +7,19 @@ import (
 	"golang.org/x/net/http2"
 )
 
+var (
+	ActiveStreams = prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace: "cloudflared",
+		Subsystem: "tunnel",
+		Name:      "active_streams",
+		Help:      "Number of active streams created by all muxers.",
+	})
+)
+
+func init() {
+	prometheus.MustRegister(ActiveStreams)
+}
+
 // activeStreamMap is used to moderate access to active streams between the read and write
 // threads, and deny access to new peer streams while shutting down.
 type activeStreamMap struct {

h2mux/activestreammap_test.go

@@ -9,7 +9,7 @@ import (
 
 func TestShutdown(t *testing.T) {
 	const numStreams = 1000
-	m := newActiveStreamMap(true, NewActiveStreamsMetrics("test", t.Name()))
+	m := newActiveStreamMap(true, ActiveStreams)
 
 	// Add all the streams
 	{
@@ -62,7 +62,7 @@ func TestShutdown(t *testing.T) {
 
 func TestEmptyBeforeShutdown(t *testing.T) {
 	const numStreams = 1000
-	m := newActiveStreamMap(true, NewActiveStreamsMetrics("test", t.Name()))
+	m := newActiveStreamMap(true, ActiveStreams)
 
 	// Add all the streams
 	{
@@ -138,7 +138,7 @@ func (_ *noopReadyList) Signal(streamID uint32) {}
 
 func TestAbort(t *testing.T) {
 	const numStreams = 1000
-	m := newActiveStreamMap(true, NewActiveStreamsMetrics("test", t.Name()))
+	m := newActiveStreamMap(true, ActiveStreams)
 
 	var openedStreams sync.Map
 

h2mux/h2mux_test.go

@@ -113,11 +113,11 @@ func (p *DefaultMuxerPair) Handshake(testName string) error {
 	defer cancel()
 	errGroup, _ := errgroup.WithContext(ctx)
 	errGroup.Go(func() (err error) {
-		p.EdgeMux, err = Handshake(p.EdgeConn, p.EdgeConn, p.EdgeMuxConfig, NewActiveStreamsMetrics(testName, "edge"))
+		p.EdgeMux, err = Handshake(p.EdgeConn, p.EdgeConn, p.EdgeMuxConfig, ActiveStreams)
 		return errors.Wrap(err, "edge handshake failure")
 	})
 	errGroup.Go(func() (err error) {
-		p.OriginMux, err = Handshake(p.OriginConn, p.OriginConn, p.OriginMuxConfig, NewActiveStreamsMetrics(testName, "origin"))
+		p.OriginMux, err = Handshake(p.OriginConn, p.OriginConn, p.OriginMuxConfig, ActiveStreams)
 		return errors.Wrap(err, "origin handshake failure")
 	})
 

h2mux/header.go

@@ -2,7 +2,6 @@ package h2mux
 
 import (
 	"encoding/base64"
-	"encoding/json"
 	"fmt"
 	"net/http"
 	"net/url"
@@ -22,10 +21,6 @@ const (
 	RequestUserHeadersField  = "cf-cloudflared-request-headers"
 	ResponseUserHeadersField = "cf-cloudflared-response-headers"
 
-	ResponseMetaHeaderField   = "cf-cloudflared-response-meta"
-	ResponseSourceCloudflared = "cloudflared"
-	ResponseSourceOrigin      = "origin"
-
 	CFAccessTokenHeader        = "cf-access-token"
 	CFJumpDestinationHeader    = "CF-Access-Jump-Destination"
 	CFAccessClientIDHeader     = "CF-Access-Client-Id"
@@ -124,7 +119,7 @@ func IsControlHeader(headerName string) bool {
 }
 
 // isWebsocketClientHeader returns true if the header name is required by the client to upgrade properly
-func isWebsocketClientHeader(headerName string) bool {
+func IsWebsocketClientHeader(headerName string) bool {
 	return headerName == "sec-websocket-accept" ||
 		headerName == "connection" ||
 		headerName == "upgrade"
@@ -143,7 +138,7 @@ func H1ResponseToH2ResponseHeaders(h1 *http.Response) (h2 []Header) {
 
 			// Since these are http2 headers, they're required to be lowercase
 			h2 = append(h2, Header{Name: "content-length", Value: values[0]})
-		} else if !IsControlHeader(h2name) || isWebsocketClientHeader(h2name) {
+		} else if !IsControlHeader(h2name) || IsWebsocketClientHeader(h2name) {
 			// User headers, on the other hand, must all be serialized so that
 			// HTTP/2 header validation won't be applied to HTTP/1 header values
 			userHeaders[header] = values
@@ -152,7 +147,6 @@ func H1ResponseToH2ResponseHeaders(h1 *http.Response) (h2 []Header) {
 
 	// Perform user header serialization and set them in the single header
 	h2 = append(h2, Header{ResponseUserHeadersField, SerializeHeaders(userHeaders)})
-
 	return h2
 }
 
@@ -238,19 +232,3 @@ func DeserializeHeaders(serializedHeaders string) ([]Header, error) {
 
 	return deserialized, nil
 }
-
-type ResponseMetaHeader struct {
-	Source string `json:"src"`
-}
-
-func CreateResponseMetaHeader(headerName, source string) Header {
-	jsonResponseMetaHeader, err := json.Marshal(ResponseMetaHeader{Source: source})
-	if err != nil {
-		panic(err)
-	}
-
-	return Header{
-		Name:  headerName,
-		Value: string(jsonResponseMetaHeader),
-	}
-}

h2mux/muxmetrics.go

@@ -6,7 +6,6 @@ import (
 
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/golang-collections/collections/queue"
-	"github.com/prometheus/client_golang/prometheus"
 )
 
 // data points used to compute average receive window and send window size
@@ -295,14 +294,3 @@ func (r *rate) get() (curr, min, max uint64) {
 	defer r.lock.RUnlock()
 	return r.curr, r.min, r.max
 }
-
-func NewActiveStreamsMetrics(namespace, subsystem string) prometheus.Gauge {
-	activeStreams := prometheus.NewGauge(prometheus.GaugeOpts{
-		Namespace: namespace,
-		Subsystem: subsystem,
-		Name:      "active_streams",
-		Help:      "Number of active streams created by all muxers.",
-	})
-	prometheus.MustRegister(activeStreams)
-	return activeStreams
-}

hello/hello.go

@@ -18,6 +18,14 @@ import (
 	"github.com/cloudflare/cloudflared/tlsconfig"
 )
 
+const (
+	UptimeRoute    = "/uptime"
+	WSRoute        = "/ws"
+	SSERoute       = "/sse"
+	HealthRoute    = "/_health"
+	defaultSSEFreq = time.Second * 10
+)
+
 type templateData struct {
 	ServerName string
 	Request    *http.Request
@@ -104,8 +112,10 @@ func StartHelloWorldServer(logger logger.Service, listener net.Listener, shutdow
 	}
 
 	muxer := http.NewServeMux()
-	muxer.HandleFunc("/uptime", uptimeHandler(time.Now()))
-	muxer.HandleFunc("/ws", websocketHandler(logger, upgrader))
+	muxer.HandleFunc(UptimeRoute, uptimeHandler(time.Now()))
+	muxer.HandleFunc(WSRoute, websocketHandler(logger, upgrader))
+	muxer.HandleFunc(SSERoute, sseHandler(logger))
+	muxer.HandleFunc(HealthRoute, healthHandler())
 	muxer.HandleFunc("/", rootHandler(serverName))
 	httpServer := &http.Server{Addr: listener.Addr().String(), Handler: muxer}
 	go func() {
@@ -150,8 +160,15 @@ func uptimeHandler(startTime time.Time) http.HandlerFunc {
 // This handler will echo message
 func websocketHandler(logger logger.Service, upgrader websocket.Upgrader) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
+		// This addresses the issue of r.Host includes port but origin header doesn't
+		host, _, err := net.SplitHostPort(r.Host)
+		if err == nil {
+			r.Host = host
+		}
+
 		conn, err := upgrader.Upgrade(w, r, nil)
 		if err != nil {
+			logger.Errorf("failed to upgrade to websocket connection, error: %s", err)
 			return
 		}
 		defer conn.Close()
@@ -170,6 +187,48 @@ func websocketHandler(logger logger.Service, upgrader websocket.Upgrader) http.H
 	}
 }
 
+func sseHandler(logger logger.Service) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/event-stream; charset=utf-8")
+		flusher, ok := w.(http.Flusher)
+		if !ok {
+			w.WriteHeader(http.StatusInternalServerError)
+			logger.Errorf("Can't support SSE. ResponseWriter %T doesn't implement http.Flusher interface", w)
+			return
+		}
+
+		freq := defaultSSEFreq
+		if requestedFreq := r.URL.Query()["freq"]; len(requestedFreq) > 0 {
+			parsedFreq, err := time.ParseDuration(requestedFreq[0])
+			if err == nil {
+				freq = parsedFreq
+			}
+		}
+		logger.Infof("Server Sent Events every %s", freq)
+		ticker := time.NewTicker(freq)
+		counter := 0
+		for {
+			select {
+			case <-r.Context().Done():
+				return
+			case <-ticker.C:
+			}
+			_, err := fmt.Fprintf(w, "%d\n\n", counter)
+			if err != nil {
+				return
+			}
+			flusher.Flush()
+			counter++
+		}
+	}
+}
+
+func healthHandler() http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		w.Write([]byte("ok"))
+	}
+}
+
 func rootHandler(serverName string) http.HandlerFunc {
 	responseTemplate := template.Must(template.New("index").Parse(indexTemplate))
 	return func(w http.ResponseWriter, r *http.Request) {

ingress/ingress.go

@@ -2,70 +2,44 @@ package ingress
 
 import (
 	"fmt"
+	"net"
 	"net/url"
 	"regexp"
+	"strconv"
 	"strings"
+	"sync"
 
 	"github.com/pkg/errors"
+	"github.com/urfave/cli/v2"
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/logger"
 )
 
 var (
-	ErrNoIngressRules             = errors.New("No ingress rules were specified in the config file")
-	errLastRuleNotCatchAll        = errors.New("The last ingress rule must match all hostnames (i.e. it must be missing, or must be \"*\")")
+	ErrNoIngressRules             = errors.New("The config file doesn't contain any ingress rules")
+	errLastRuleNotCatchAll        = errors.New("The last ingress rule must match all URLs (i.e. it should not have a hostname or path filter)")
 	errBadWildcard                = errors.New("Hostname patterns can have at most one wildcard character (\"*\") and it can only be used for subdomains, e.g. \"*.example.com\"")
+	errHostnameContainsPort       = errors.New("Hostname cannot contain a port")
 	ErrURLIncompatibleWithIngress = errors.New("You can't set the --url flag (or $TUNNEL_URL) when using multiple-origin ingress rules")
 )
 
-// Each rule route traffic from a hostname/path on the public
-// internet to the service running on the given URL.
-type Rule struct {
-	// Requests for this hostname will be proxied to this rule's service.
-	Hostname string
-
-	// Path is an optional regex that can specify path-driven ingress rules.
-	Path *regexp.Regexp
-
-	// A (probably local) address. Requests for a hostname which matches this
-	// rule's hostname pattern will be proxied to the service running on this
-	// address.
-	Service *url.URL
-}
-
-func (r Rule) MultiLineString() string {
-	var out strings.Builder
-	if r.Hostname != "" {
-		out.WriteString("\thostname: ")
-		out.WriteString(r.Hostname)
-		out.WriteRune('\n')
-	}
-	if r.Path != nil {
-		out.WriteString("\tpath: ")
-		out.WriteString(r.Path.String())
-		out.WriteRune('\n')
-	}
-	out.WriteString("\tservice: ")
-	out.WriteString(r.Service.String())
-	return out.String()
-}
-
-func (r *Rule) Matches(hostname, path string) bool {
-	hostMatch := r.Hostname == "" || r.Hostname == "*" || matchHost(r.Hostname, hostname)
-	pathMatch := r.Path == nil || r.Path.MatchString(path)
-	return hostMatch && pathMatch
-}
-
 // FindMatchingRule returns the index of the Ingress Rule which matches the given
 // hostname and path. This function assumes the last rule matches everything,
 // which is the case if the rules were instantiated via the ingress#Validate method
-func (ing Ingress) FindMatchingRule(hostname, path string) int {
+func (ing Ingress) FindMatchingRule(hostname, path string) (*Rule, int) {
+	// The hostname might contain port. We only want to compare the host part with the rule
+	host, _, err := net.SplitHostPort(hostname)
+	if err == nil {
+		hostname = host
+	}
 	for i, rule := range ing.Rules {
 		if rule.Matches(hostname, path) {
-			return i
+			return &rule, i
 		}
 	}
-	return len(ing.Rules) - 1
+	i := len(ing.Rules) - 1
+	return &ing.Rules[i], i
 }
 
 func matchHost(ruleHost, reqHost string) bool {
@@ -83,7 +57,54 @@ func matchHost(ruleHost, reqHost string) bool {
 
 // Ingress maps eyeball requests to origins.
 type Ingress struct {
-	Rules []Rule
+	Rules    []Rule
+	defaults OriginRequestConfig
+}
+
+// NewSingleOrigin constructs an Ingress set with only one rule, constructed from
+// legacy CLI parameters like --url or --no-chunked-encoding.
+func NewSingleOrigin(c *cli.Context, allowURLFromArgs bool, logger logger.Service) (Ingress, error) {
+
+	service, err := parseSingleOriginService(c, allowURLFromArgs)
+	if err != nil {
+		return Ingress{}, err
+	}
+
+	// Construct an Ingress with the single rule.
+	defaults := originRequestFromSingeRule(c)
+	ing := Ingress{
+		Rules: []Rule{
+			{
+				Service: service,
+				Config:  setConfig(defaults, config.OriginRequestConfig{}),
+			},
+		},
+		defaults: defaults,
+	}
+	return ing, err
+}
+
+// Get a single origin service from the CLI/config.
+func parseSingleOriginService(c *cli.Context, allowURLFromArgs bool) (OriginService, error) {
+	if c.IsSet("hello-world") {
+		return new(helloWorld), nil
+	}
+	if c.IsSet("url") || c.IsSet(config.BastionFlag) {
+		originURL, err := config.ValidateUrl(c, allowURLFromArgs)
+		if err != nil {
+			return nil, errors.Wrap(err, "Error validating origin URL")
+		}
+		return &localService{URL: originURL, RootURL: originURL}, nil
+	}
+	if c.IsSet("unix-socket") {
+		path, err := config.ValidateUnixSocket(c)
+		if err != nil {
+			return nil, errors.Wrap(err, "Error validating --unix-socket")
+		}
+		return &unixSocketPath{path: path}, nil
+	}
+	u, err := url.Parse("http://localhost:8080")
+	return &localService{URL: u, RootURL: u}, err
 }
 
 // IsEmpty checks if there are any ingress rules.
@@ -91,40 +112,70 @@ func (ing Ingress) IsEmpty() bool {
 	return len(ing.Rules) == 0
 }
 
-func validate(ingress []config.UnvalidatedIngressRule) (Ingress, error) {
+// StartOrigins will start any origin services managed by cloudflared, e.g. proxy servers or Hello World.
+func (ing Ingress) StartOrigins(wg *sync.WaitGroup, log logger.Service, shutdownC <-chan struct{}, errC chan error) {
+	for _, rule := range ing.Rules {
+		if err := rule.Service.start(wg, log, shutdownC, errC, rule.Config); err != nil {
+			log.Errorf("Error starting local service %s: %s", rule.Service, err)
+		}
+	}
+}
+
+// CatchAll returns the catch-all rule (i.e. the last rule)
+func (ing Ingress) CatchAll() *Rule {
+	return &ing.Rules[len(ing.Rules)-1]
+}
+
+func validate(ingress []config.UnvalidatedIngressRule, defaults OriginRequestConfig) (Ingress, error) {
 	rules := make([]Rule, len(ingress))
 	for i, r := range ingress {
-		service, err := url.Parse(r.Service)
-		if err != nil {
+		cfg := setConfig(defaults, r.OriginRequest)
+		var service OriginService
+
+		if prefix := "unix:"; strings.HasPrefix(r.Service, prefix) {
+			// No validation necessary for unix socket filepath services
+			path := strings.TrimPrefix(r.Service, prefix)
+			service = &unixSocketPath{path: path}
+		} else if prefix := "http_status:"; strings.HasPrefix(r.Service, prefix) {
+			status, err := strconv.Atoi(strings.TrimPrefix(r.Service, prefix))
+			if err != nil {
+				return Ingress{}, errors.Wrap(err, "invalid HTTP status")
+			}
+			srv := newStatusCode(status)
+			service = &srv
+		} else if r.Service == "hello_world" || r.Service == "hello-world" || r.Service == "helloworld" {
+			service = new(helloWorld)
+		} else if r.Service == "bastion" || cfg.BastionMode {
+			// Bastion mode will always start a Websocket proxy server, which will
+			// overwrite the localService.URL field when `start` is called. So,
+			// leave the URL field empty for now.
+			cfg.BastionMode = true
+			service = new(localService)
+		} else {
+			// Validate URL services
+			u, err := url.Parse(r.Service)
+			if err != nil {
+				return Ingress{}, err
+			}
+
+			if u.Scheme == "" || u.Hostname() == "" {
+				return Ingress{}, fmt.Errorf("%s is an invalid address, please make sure it has a scheme and a hostname", r.Service)
+			}
+
+			if u.Path != "" {
+				return Ingress{}, fmt.Errorf("%s is an invalid address, ingress rules don't support proxying to a different path on the origin service. The path will be the same as the eyeball request's path", r.Service)
+			}
+			serviceURL := localService{URL: u}
+			service = &serviceURL
+		}
+
+		if err := validateHostname(r, i, len(ingress)); err != nil {
 			return Ingress{}, err
 		}
-		if service.Scheme == "" || service.Hostname() == "" {
-			return Ingress{}, fmt.Errorf("The service %s must have a scheme and a hostname", r.Service)
-		}
-
-		if service.Path != "" {
-			return Ingress{}, fmt.Errorf("%s is an invalid address, ingress rules don't support proxying to a different path on the origin service. The path will be the same as the eyeball request's path.", r.Service)
-		}
-
-		// Ensure that there are no wildcards anywhere except the first character
-		// of the hostname.
-		if strings.LastIndex(r.Hostname, "*") > 0 {
-			return Ingress{}, errBadWildcard
-		}
-
-		// The last rule should catch all hostnames.
-		isCatchAllRule := (r.Hostname == "" || r.Hostname == "*") && r.Path == ""
-		isLastRule := i == len(ingress)-1
-		if isLastRule && !isCatchAllRule {
-			return Ingress{}, errLastRuleNotCatchAll
-		}
-		// ONLY the last rule should catch all hostnames.
-		if !isLastRule && isCatchAllRule {
-			return Ingress{}, errRuleShouldNotBeCatchAll{i: i, hostname: r.Hostname}
-		}
 
 		var pathRegex *regexp.Regexp
 		if r.Path != "" {
+			var err error
 			pathRegex, err = regexp.Compile(r.Path)
 			if err != nil {
 				return Ingress{}, errors.Wrapf(err, "Rule #%d has an invalid regex", i+1)
@@ -135,25 +186,52 @@ func validate(ingress []config.UnvalidatedIngressRule) (Ingress, error) {
 			Hostname: r.Hostname,
 			Service:  service,
 			Path:     pathRegex,
+			Config:   cfg,
 		}
 	}
-	return Ingress{Rules: rules}, nil
+	return Ingress{Rules: rules, defaults: defaults}, nil
+}
+
+func validateHostname(r config.UnvalidatedIngressRule, ruleIndex, totalRules int) error {
+	// Ensure that the hostname doesn't contain port
+	_, _, err := net.SplitHostPort(r.Hostname)
+	if err == nil {
+		return errHostnameContainsPort
+	}
+	// Ensure that there are no wildcards anywhere except the first character
+	// of the hostname.
+	if strings.LastIndex(r.Hostname, "*") > 0 {
+		return errBadWildcard
+	}
+
+	// The last rule should catch all hostnames.
+	isCatchAllRule := (r.Hostname == "" || r.Hostname == "*") && r.Path == ""
+	isLastRule := ruleIndex == totalRules-1
+	if isLastRule && !isCatchAllRule {
+		return errLastRuleNotCatchAll
+	}
+	// ONLY the last rule should catch all hostnames.
+	if !isLastRule && isCatchAllRule {
+		return errRuleShouldNotBeCatchAll{index: ruleIndex, hostname: r.Hostname}
+	}
+	return nil
 }
 
 type errRuleShouldNotBeCatchAll struct {
-	i        int
+	index    int
 	hostname string
 }
 
 func (e errRuleShouldNotBeCatchAll) Error() string {
 	return fmt.Sprintf("Rule #%d is matching the hostname '%s', but "+
 		"this will match every hostname, meaning the rules which follow it "+
-		"will never be triggered.", e.i+1, e.hostname)
+		"will never be triggered.", e.index+1, e.hostname)
 }
 
+// ParseIngress parses ingress rules, but does not send HTTP requests to the origins.
 func ParseIngress(conf *config.Configuration) (Ingress, error) {
 	if len(conf.Ingress) == 0 {
 		return Ingress{}, ErrNoIngressRules
 	}
-	return validate(conf.Ingress)
+	return validate(conf.Ingress, originRequestFromYAML(conf.OriginRequest))
 }

ingress/ingress_test.go

@@ -1,27 +1,48 @@
 package ingress
 
 import (
+	"flag"
+	"fmt"
 	"net/url"
 	"regexp"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"github.com/urfave/cli/v2"
 	"gopkg.in/yaml.v2"
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/logger"
+	"github.com/cloudflare/cloudflared/tlsconfig"
 )
 
+func TestParseUnixSocket(t *testing.T) {
+	rawYAML := `
+ingress:
+- service: unix:/tmp/echo.sock
+`
+	ing, err := ParseIngress(MustReadIngress(rawYAML))
+	require.NoError(t, err)
+	_, ok := ing.Rules[0].Service.(*unixSocketPath)
+	require.True(t, ok)
+}
+
 func Test_parseIngress(t *testing.T) {
 	localhost8000 := MustParseURL(t, "https://localhost:8000")
 	localhost8001 := MustParseURL(t, "https://localhost:8001")
+	fourOhFour := newStatusCode(404)
+	defaultConfig := setConfig(originRequestFromYAML(config.OriginRequestConfig{}), config.OriginRequestConfig{})
+	require.Equal(t, defaultKeepAliveConnections, defaultConfig.KeepAliveConnections)
+	tr := true
 	type args struct {
 		rawYAML string
 	}
 	tests := []struct {
 		name    string
 		args    args
-		want    Ingress
+		want    []Rule
 		wantErr bool
 	}{
 		{
@@ -38,16 +59,18 @@ ingress:
  - hostname: "*"
    service: https://localhost:8001
 `},
-			want: Ingress{Rules: []Rule{
+			want: []Rule{
 				{
 					Hostname: "tunnel1.example.com",
-					Service:  localhost8000,
+					Service:  &localService{URL: localhost8000},
+					Config:   defaultConfig,
 				},
 				{
 					Hostname: "*",
-					Service:  localhost8001,
+					Service:  &localService{URL: localhost8001},
+					Config:   defaultConfig,
 				},
-			}},
+			},
 		},
 		{
 			name: "Extra keys",
@@ -57,12 +80,13 @@ ingress:
    service: https://localhost:8000
 extraKey: extraValue
 `},
-			want: Ingress{Rules: []Rule{
+			want: []Rule{
 				{
 					Hostname: "*",
-					Service:  localhost8000,
+					Service:  &localService{URL: localhost8000},
+					Config:   defaultConfig,
 				},
-			}},
+			},
 		},
 		{
 			name: "Hostname can be omitted",
@@ -70,11 +94,12 @@ extraKey: extraValue
 ingress:
  - service: https://localhost:8000
 `},
-			want: Ingress{Rules: []Rule{
+			want: []Rule{
 				{
-					Service: localhost8000,
+					Service: &localService{URL: localhost8000},
+					Config:  defaultConfig,
 				},
-			}},
+			},
 		},
 		{
 			name: "Invalid service",
@@ -146,6 +171,94 @@ ingress:
 			args: args{rawYAML: `
 ingress:
  - service: https://localhost:8000/static/
+`},
+			wantErr: true,
+		},
+		{
+			name: "Invalid HTTP status",
+			args: args{rawYAML: `
+ingress:
+ - service: http_status:asdf
+`},
+			wantErr: true,
+		},
+		{
+			name: "Valid HTTP status",
+			args: args{rawYAML: `
+ingress:
+ - service: http_status:404
+`},
+			want: []Rule{
+				{
+					Hostname: "",
+					Service:  &fourOhFour,
+					Config:   defaultConfig,
+				},
+			},
+		},
+		{
+			name: "Valid hello world service",
+			args: args{rawYAML: `
+ingress:
+ - service: hello_world
+`},
+			want: []Rule{
+				{
+					Hostname: "",
+					Service:  new(helloWorld),
+					Config:   defaultConfig,
+				},
+			},
+		},
+		{
+			name: "URL isn't necessary if using bastion",
+			args: args{rawYAML: `
+ingress:
+- hostname: bastion.foo.com
+  originRequest:
+    bastionMode: true
+- service: http_status:404
+`},
+			want: []Rule{
+				{
+					Hostname: "bastion.foo.com",
+					Service:  &localService{},
+					Config:   setConfig(originRequestFromYAML(config.OriginRequestConfig{}), config.OriginRequestConfig{BastionMode: &tr}),
+				},
+				{
+					Service: &fourOhFour,
+					Config:  defaultConfig,
+				},
+			},
+		},
+		{
+			name: "Bastion service",
+			args: args{rawYAML: `
+ingress:
+- hostname: bastion.foo.com
+  service: bastion
+- service: http_status:404
+`},
+			want: []Rule{
+				{
+					Hostname: "bastion.foo.com",
+					Service:  &localService{},
+					Config:   setConfig(originRequestFromYAML(config.OriginRequestConfig{}), config.OriginRequestConfig{BastionMode: &tr}),
+				},
+				{
+					Service: &fourOhFour,
+					Config:  defaultConfig,
+				},
+			},
+		},
+		{
+			name: "Hostname contains port",
+			args: args{rawYAML: `
+ingress:
+ - hostname: "test.example.com:443"
+   service: https://localhost:8000
+ - hostname: "*"
+   service: https://localhost:8001
 `},
 			wantErr: true,
 		},
@@ -157,129 +270,159 @@ ingress:
 				t.Errorf("ParseIngress() error = %v, wantErr %v", err, tt.wantErr)
 				return
 			}
-			assert.Equal(t, tt.want, got)
+			require.Equal(t, tt.want, got.Rules)
 		})
 	}
 }
 
+func TestSingleOriginSetsConfig(t *testing.T) {
+	flagSet := flag.NewFlagSet(t.Name(), flag.PanicOnError)
+	flagSet.Bool("hello-world", true, "")
+	flagSet.Duration(ProxyConnectTimeoutFlag, time.Second, "")
+	flagSet.Duration(ProxyTLSTimeoutFlag, time.Second, "")
+	flagSet.Duration(ProxyTCPKeepAlive, time.Second, "")
+	flagSet.Bool(ProxyNoHappyEyeballsFlag, true, "")
+	flagSet.Int(ProxyKeepAliveConnectionsFlag, 10, "")
+	flagSet.Duration(ProxyKeepAliveTimeoutFlag, time.Second, "")
+	flagSet.String(HTTPHostHeaderFlag, "example.com:8080", "")
+	flagSet.String(OriginServerNameFlag, "example.com", "")
+	flagSet.String(tlsconfig.OriginCAPoolFlag, "/etc/certs/ca.pem", "")
+	flagSet.Bool(NoTLSVerifyFlag, true, "")
+	flagSet.Bool(NoChunkedEncodingFlag, true, "")
+	flagSet.Bool(config.BastionFlag, true, "")
+	flagSet.String(ProxyAddressFlag, "localhost:8080", "")
+	flagSet.Uint(ProxyPortFlag, 8080, "")
+	flagSet.Bool(Socks5Flag, true, "")
+
+	cliCtx := cli.NewContext(cli.NewApp(), flagSet, nil)
+	err := cliCtx.Set("hello-world", "true")
+	require.NoError(t, err)
+	err = cliCtx.Set(ProxyConnectTimeoutFlag, "1s")
+	require.NoError(t, err)
+	err = cliCtx.Set(ProxyTLSTimeoutFlag, "1s")
+	require.NoError(t, err)
+	err = cliCtx.Set(ProxyTCPKeepAlive, "1s")
+	require.NoError(t, err)
+	err = cliCtx.Set(ProxyNoHappyEyeballsFlag, "true")
+	require.NoError(t, err)
+	err = cliCtx.Set(ProxyKeepAliveConnectionsFlag, "10")
+	require.NoError(t, err)
+	err = cliCtx.Set(ProxyKeepAliveTimeoutFlag, "1s")
+	require.NoError(t, err)
+	err = cliCtx.Set(HTTPHostHeaderFlag, "example.com:8080")
+	require.NoError(t, err)
+	err = cliCtx.Set(OriginServerNameFlag, "example.com")
+	require.NoError(t, err)
+	err = cliCtx.Set(tlsconfig.OriginCAPoolFlag, "/etc/certs/ca.pem")
+	require.NoError(t, err)
+	err = cliCtx.Set(NoTLSVerifyFlag, "true")
+	require.NoError(t, err)
+	err = cliCtx.Set(NoChunkedEncodingFlag, "true")
+	require.NoError(t, err)
+	err = cliCtx.Set(config.BastionFlag, "true")
+	require.NoError(t, err)
+	err = cliCtx.Set(ProxyAddressFlag, "localhost:8080")
+	require.NoError(t, err)
+	err = cliCtx.Set(ProxyPortFlag, "8080")
+	require.NoError(t, err)
+	err = cliCtx.Set(Socks5Flag, "true")
+	require.NoError(t, err)
+
+	allowURLFromArgs := false
+	logger, err := logger.New()
+	require.NoError(t, err)
+	ingress, err := NewSingleOrigin(cliCtx, allowURLFromArgs, logger)
+	require.NoError(t, err)
+
+	assert.Equal(t, time.Second, ingress.Rules[0].Config.ConnectTimeout)
+	assert.Equal(t, time.Second, ingress.Rules[0].Config.TLSTimeout)
+	assert.Equal(t, time.Second, ingress.Rules[0].Config.TCPKeepAlive)
+	assert.True(t, ingress.Rules[0].Config.NoHappyEyeballs)
+	assert.Equal(t, 10, ingress.Rules[0].Config.KeepAliveConnections)
+	assert.Equal(t, time.Second, ingress.Rules[0].Config.KeepAliveTimeout)
+	assert.Equal(t, "example.com:8080", ingress.Rules[0].Config.HTTPHostHeader)
+	assert.Equal(t, "example.com", ingress.Rules[0].Config.OriginServerName)
+	assert.Equal(t, "/etc/certs/ca.pem", ingress.Rules[0].Config.CAPool)
+	assert.True(t, ingress.Rules[0].Config.NoTLSVerify)
+	assert.True(t, ingress.Rules[0].Config.DisableChunkedEncoding)
+	assert.True(t, ingress.Rules[0].Config.BastionMode)
+	assert.Equal(t, "localhost:8080", ingress.Rules[0].Config.ProxyAddress)
+	assert.Equal(t, uint(8080), ingress.Rules[0].Config.ProxyPort)
+	assert.Equal(t, socksProxy, ingress.Rules[0].Config.ProxyType)
+}
+
+func TestFindMatchingRule(t *testing.T) {
+	ingress := Ingress{
+		Rules: []Rule{
+			{
+				Hostname: "tunnel-a.example.com",
+				Path:     nil,
+			},
+			{
+				Hostname: "tunnel-b.example.com",
+				Path:     mustParsePath(t, "/health"),
+			},
+			{
+				Hostname: "*",
+			},
+		},
+	}
+
+	tests := []struct {
+		host          string
+		path          string
+		wantRuleIndex int
+	}{
+		{
+			host:          "tunnel-a.example.com",
+			path:          "/",
+			wantRuleIndex: 0,
+		},
+		{
+			host:          "tunnel-a.example.com",
+			path:          "/pages/about",
+			wantRuleIndex: 0,
+		},
+		{
+			host:          "tunnel-a.example.com:443",
+			path:          "/pages/about",
+			wantRuleIndex: 0,
+		},
+		{
+			host:          "tunnel-b.example.com",
+			path:          "/health",
+			wantRuleIndex: 1,
+		},
+		{
+			host:          "tunnel-b.example.com",
+			path:          "/index.html",
+			wantRuleIndex: 2,
+		},
+		{
+			host:          "tunnel-c.example.com",
+			path:          "/",
+			wantRuleIndex: 2,
+		},
+	}
+
+	for i, test := range tests {
+		_, ruleIndex := ingress.FindMatchingRule(test.host, test.path)
+		assert.Equal(t, test.wantRuleIndex, ruleIndex, fmt.Sprintf("Expect host=%s, path=%s to match rule %d, got %d", test.host, test.path, test.wantRuleIndex, i))
+	}
+}
+
+func mustParsePath(t *testing.T, path string) *regexp.Regexp {
+	regexp, err := regexp.Compile(path)
+	assert.NoError(t, err)
+	return regexp
+}
+
 func MustParseURL(t *testing.T, rawURL string) *url.URL {
 	u, err := url.Parse(rawURL)
 	require.NoError(t, err)
 	return u
 }
 
-func Test_rule_matches(t *testing.T) {
-	type fields struct {
-		Hostname string
-		Path     *regexp.Regexp
-		Service  *url.URL
-	}
-	type args struct {
-		requestURL *url.URL
-	}
-	tests := []struct {
-		name   string
-		fields fields
-		args   args
-		want   bool
-	}{
-		{
-			name: "Just hostname, pass",
-			fields: fields{
-				Hostname: "example.com",
-			},
-			args: args{
-				requestURL: MustParseURL(t, "https://example.com"),
-			},
-			want: true,
-		},
-		{
-			name: "Entire hostname is wildcard, should match everything",
-			fields: fields{
-				Hostname: "*",
-			},
-			args: args{
-				requestURL: MustParseURL(t, "https://example.com"),
-			},
-			want: true,
-		},
-		{
-			name: "Just hostname, fail",
-			fields: fields{
-				Hostname: "example.com",
-			},
-			args: args{
-				requestURL: MustParseURL(t, "https://foo.bar"),
-			},
-			want: false,
-		},
-		{
-			name: "Just wildcard hostname, pass",
-			fields: fields{
-				Hostname: "*.example.com",
-			},
-			args: args{
-				requestURL: MustParseURL(t, "https://adam.example.com"),
-			},
-			want: true,
-		},
-		{
-			name: "Just wildcard hostname, fail",
-			fields: fields{
-				Hostname: "*.example.com",
-			},
-			args: args{
-				requestURL: MustParseURL(t, "https://tunnel.com"),
-			},
-			want: false,
-		},
-		{
-			name: "Just wildcard outside of subdomain in hostname, fail",
-			fields: fields{
-				Hostname: "*example.com",
-			},
-			args: args{
-				requestURL: MustParseURL(t, "https://www.example.com"),
-			},
-			want: false,
-		},
-		{
-			name: "Wildcard over multiple subdomains",
-			fields: fields{
-				Hostname: "*.example.com",
-			},
-			args: args{
-				requestURL: MustParseURL(t, "https://adam.chalmers.example.com"),
-			},
-			want: true,
-		},
-		{
-			name: "Hostname and path",
-			fields: fields{
-				Hostname: "*.example.com",
-				Path:     regexp.MustCompile("/static/.*\\.html"),
-			},
-			args: args{
-				requestURL: MustParseURL(t, "https://www.example.com/static/index.html"),
-			},
-			want: true,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			r := Rule{
-				Hostname: tt.fields.Hostname,
-				Path:     tt.fields.Path,
-				Service:  tt.fields.Service,
-			}
-			u := tt.args.requestURL
-			if got := r.Matches(u.Hostname(), u.Path); got != tt.want {
-				t.Errorf("rule.matches() = %v, want %v", got, tt.want)
-			}
-		})
-	}
-}
-
 func BenchmarkFindMatch(b *testing.B) {
 	rulesYAML := `
 ingress:

ingress/origin_request_config.go

@@ -0,0 +1,331 @@
+package ingress
+
+import (
+	"time"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/tlsconfig"
+	"github.com/urfave/cli/v2"
+)
+
+const (
+	defaultConnectTimeout       = 30 * time.Second
+	defaultTLSTimeout           = 10 * time.Second
+	defaultTCPKeepAlive         = 30 * time.Second
+	defaultKeepAliveConnections = 100
+	defaultKeepAliveTimeout     = 90 * time.Second
+	defaultProxyAddress         = "127.0.0.1"
+
+	SSHServerFlag                 = "ssh-server"
+	Socks5Flag                    = "socks5"
+	ProxyConnectTimeoutFlag       = "proxy-connect-timeout"
+	ProxyTLSTimeoutFlag           = "proxy-tls-timeout"
+	ProxyTCPKeepAlive             = "proxy-tcp-keepalive"
+	ProxyNoHappyEyeballsFlag      = "proxy-no-happy-eyeballs"
+	ProxyKeepAliveConnectionsFlag = "proxy-keepalive-connections"
+	ProxyKeepAliveTimeoutFlag     = "proxy-keepalive-timeout"
+	HTTPHostHeaderFlag            = "http-host-header"
+	OriginServerNameFlag          = "origin-server-name"
+	NoTLSVerifyFlag               = "no-tls-verify"
+	NoChunkedEncodingFlag         = "no-chunked-encoding"
+	ProxyAddressFlag              = "proxy-address"
+	ProxyPortFlag                 = "proxy-port"
+)
+
+const (
+	socksProxy = "socks"
+)
+
+func originRequestFromSingeRule(c *cli.Context) OriginRequestConfig {
+	var connectTimeout time.Duration = defaultConnectTimeout
+	var tlsTimeout time.Duration = defaultTLSTimeout
+	var tcpKeepAlive time.Duration = defaultTCPKeepAlive
+	var noHappyEyeballs bool
+	var keepAliveConnections int = defaultKeepAliveConnections
+	var keepAliveTimeout time.Duration = defaultKeepAliveTimeout
+	var httpHostHeader string
+	var originServerName string
+	var caPool string
+	var noTLSVerify bool
+	var disableChunkedEncoding bool
+	var bastionMode bool
+	var proxyAddress = defaultProxyAddress
+	var proxyPort uint
+	var proxyType string
+	if flag := ProxyConnectTimeoutFlag; c.IsSet(flag) {
+		connectTimeout = c.Duration(flag)
+	}
+	if flag := ProxyTLSTimeoutFlag; c.IsSet(flag) {
+		tlsTimeout = c.Duration(flag)
+	}
+	if flag := ProxyTCPKeepAlive; c.IsSet(flag) {
+		tcpKeepAlive = c.Duration(flag)
+	}
+	if flag := ProxyNoHappyEyeballsFlag; c.IsSet(flag) {
+		noHappyEyeballs = c.Bool(flag)
+	}
+	if flag := ProxyKeepAliveConnectionsFlag; c.IsSet(flag) {
+		keepAliveConnections = c.Int(flag)
+	}
+	if flag := ProxyKeepAliveTimeoutFlag; c.IsSet(flag) {
+		keepAliveTimeout = c.Duration(flag)
+	}
+	if flag := HTTPHostHeaderFlag; c.IsSet(flag) {
+		httpHostHeader = c.String(flag)
+	}
+	if flag := OriginServerNameFlag; c.IsSet(flag) {
+		originServerName = c.String(flag)
+	}
+	if flag := tlsconfig.OriginCAPoolFlag; c.IsSet(flag) {
+		caPool = c.String(flag)
+	}
+	if flag := NoTLSVerifyFlag; c.IsSet(flag) {
+		noTLSVerify = c.Bool(flag)
+	}
+	if flag := NoChunkedEncodingFlag; c.IsSet(flag) {
+		disableChunkedEncoding = c.Bool(flag)
+	}
+	if flag := config.BastionFlag; c.IsSet(flag) {
+		bastionMode = c.Bool(flag)
+	}
+	if flag := ProxyAddressFlag; c.IsSet(flag) {
+		proxyAddress = c.String(flag)
+	}
+	if flag := ProxyPortFlag; c.IsSet(flag) {
+		proxyPort = c.Uint(flag)
+	}
+	if c.IsSet(Socks5Flag) {
+		proxyType = socksProxy
+	}
+	return OriginRequestConfig{
+		ConnectTimeout:         connectTimeout,
+		TLSTimeout:             tlsTimeout,
+		TCPKeepAlive:           tcpKeepAlive,
+		NoHappyEyeballs:        noHappyEyeballs,
+		KeepAliveConnections:   keepAliveConnections,
+		KeepAliveTimeout:       keepAliveTimeout,
+		HTTPHostHeader:         httpHostHeader,
+		OriginServerName:       originServerName,
+		CAPool:                 caPool,
+		NoTLSVerify:            noTLSVerify,
+		DisableChunkedEncoding: disableChunkedEncoding,
+		BastionMode:            bastionMode,
+		ProxyAddress:           proxyAddress,
+		ProxyPort:              proxyPort,
+		ProxyType:              proxyType,
+	}
+}
+
+func originRequestFromYAML(y config.OriginRequestConfig) OriginRequestConfig {
+	out := OriginRequestConfig{
+		ConnectTimeout:       defaultConnectTimeout,
+		TLSTimeout:           defaultTLSTimeout,
+		TCPKeepAlive:         defaultTCPKeepAlive,
+		KeepAliveConnections: defaultKeepAliveConnections,
+		KeepAliveTimeout:     defaultKeepAliveTimeout,
+		ProxyAddress:         defaultProxyAddress,
+	}
+	if y.ConnectTimeout != nil {
+		out.ConnectTimeout = *y.ConnectTimeout
+	}
+	if y.TLSTimeout != nil {
+		out.TLSTimeout = *y.TLSTimeout
+	}
+	if y.TCPKeepAlive != nil {
+		out.TCPKeepAlive = *y.TCPKeepAlive
+	}
+	if y.NoHappyEyeballs != nil {
+		out.NoHappyEyeballs = *y.NoHappyEyeballs
+	}
+	if y.KeepAliveConnections != nil {
+		out.KeepAliveConnections = *y.KeepAliveConnections
+	}
+	if y.KeepAliveTimeout != nil {
+		out.KeepAliveTimeout = *y.KeepAliveTimeout
+	}
+	if y.HTTPHostHeader != nil {
+		out.HTTPHostHeader = *y.HTTPHostHeader
+	}
+	if y.OriginServerName != nil {
+		out.OriginServerName = *y.OriginServerName
+	}
+	if y.CAPool != nil {
+		out.CAPool = *y.CAPool
+	}
+	if y.NoTLSVerify != nil {
+		out.NoTLSVerify = *y.NoTLSVerify
+	}
+	if y.DisableChunkedEncoding != nil {
+		out.DisableChunkedEncoding = *y.DisableChunkedEncoding
+	}
+	if y.BastionMode != nil {
+		out.BastionMode = *y.BastionMode
+	}
+	if y.ProxyAddress != nil {
+		out.ProxyAddress = *y.ProxyAddress
+	}
+	if y.ProxyPort != nil {
+		out.ProxyPort = *y.ProxyPort
+	}
+	if y.ProxyType != nil {
+		out.ProxyType = *y.ProxyType
+	}
+	return out
+}
+
+// OriginRequestConfig configures how Cloudflared sends requests to origin
+// services.
+// Note: To specify a time.Duration in go-yaml, use e.g. "3s" or "24h".
+type OriginRequestConfig struct {
+	// HTTP proxy timeout for establishing a new connection
+	ConnectTimeout time.Duration `yaml:"connectTimeout"`
+	// HTTP proxy timeout for completing a TLS handshake
+	TLSTimeout time.Duration `yaml:"tlsTimeout"`
+	// HTTP proxy TCP keepalive duration
+	TCPKeepAlive time.Duration `yaml:"tcpKeepAlive"`
+	// HTTP proxy should disable "happy eyeballs" for IPv4/v6 fallback
+	NoHappyEyeballs bool `yaml:"noHappyEyeballs"`
+	// HTTP proxy maximum keepalive connection pool size
+	KeepAliveConnections int `yaml:"keepAliveConnections"`
+	// HTTP proxy timeout for closing an idle connection
+	KeepAliveTimeout time.Duration `yaml:"keepAliveTimeout"`
+	// Sets the HTTP Host header for the local webserver.
+	HTTPHostHeader string `yaml:"httpHostHeader"`
+	// Hostname on the origin server certificate.
+	OriginServerName string `yaml:"originServerName"`
+	// Path to the CA for the certificate of your origin.
+	// This option should be used only if your certificate is not signed by Cloudflare.
+	CAPool string `yaml:"caPool"`
+	// Disables TLS verification of the certificate presented by your origin.
+	// Will allow any certificate from the origin to be accepted.
+	// Note: The connection from your machine to Cloudflare's Edge is still encrypted.
+	NoTLSVerify bool `yaml:"noTLSVerify"`
+	// Disables chunked transfer encoding.
+	// Useful if you are running a WSGI server.
+	DisableChunkedEncoding bool `yaml:"disableChunkedEncoding"`
+	// Runs as jump host
+	BastionMode bool `yaml:"bastionMode"`
+	// Listen address for the proxy.
+	ProxyAddress string `yaml:"proxyAddress"`
+	// Listen port for the proxy.
+	ProxyPort uint `yaml:"proxyPort"`
+	// What sort of proxy should be started
+	ProxyType string `yaml:"proxyType"`
+}
+
+func (defaults *OriginRequestConfig) setConnectTimeout(overrides config.OriginRequestConfig) {
+	if val := overrides.ConnectTimeout; val != nil {
+		defaults.ConnectTimeout = *val
+	}
+}
+
+func (defaults *OriginRequestConfig) setTLSTimeout(overrides config.OriginRequestConfig) {
+	if val := overrides.TLSTimeout; val != nil {
+		defaults.TLSTimeout = *val
+	}
+}
+
+func (defaults *OriginRequestConfig) setNoHappyEyeballs(overrides config.OriginRequestConfig) {
+	if val := overrides.NoHappyEyeballs; val != nil {
+		defaults.NoHappyEyeballs = *val
+	}
+}
+
+func (defaults *OriginRequestConfig) setKeepAliveConnections(overrides config.OriginRequestConfig) {
+	if val := overrides.KeepAliveConnections; val != nil {
+		defaults.KeepAliveConnections = *val
+	}
+}
+
+func (defaults *OriginRequestConfig) setKeepAliveTimeout(overrides config.OriginRequestConfig) {
+	if val := overrides.KeepAliveTimeout; val != nil {
+		defaults.KeepAliveTimeout = *val
+	}
+}
+
+func (defaults *OriginRequestConfig) setTCPKeepAlive(overrides config.OriginRequestConfig) {
+	if val := overrides.TCPKeepAlive; val != nil {
+		defaults.TCPKeepAlive = *val
+	}
+}
+
+func (defaults *OriginRequestConfig) setHTTPHostHeader(overrides config.OriginRequestConfig) {
+	if val := overrides.HTTPHostHeader; val != nil {
+		defaults.HTTPHostHeader = *val
+	}
+}
+
+func (defaults *OriginRequestConfig) setOriginServerName(overrides config.OriginRequestConfig) {
+	if val := overrides.OriginServerName; val != nil {
+		defaults.OriginServerName = *val
+	}
+}
+
+func (defaults *OriginRequestConfig) setCAPool(overrides config.OriginRequestConfig) {
+	if val := overrides.CAPool; val != nil {
+		defaults.CAPool = *val
+	}
+}
+
+func (defaults *OriginRequestConfig) setNoTLSVerify(overrides config.OriginRequestConfig) {
+	if val := overrides.NoTLSVerify; val != nil {
+		defaults.NoTLSVerify = *val
+	}
+}
+
+func (defaults *OriginRequestConfig) setDisableChunkedEncoding(overrides config.OriginRequestConfig) {
+	if val := overrides.DisableChunkedEncoding; val != nil {
+		defaults.DisableChunkedEncoding = *val
+	}
+}
+
+func (defaults *OriginRequestConfig) setBastionMode(overrides config.OriginRequestConfig) {
+	if val := overrides.BastionMode; val != nil {
+		defaults.BastionMode = *val
+	}
+}
+
+func (defaults *OriginRequestConfig) setProxyPort(overrides config.OriginRequestConfig) {
+	if val := overrides.ProxyPort; val != nil {
+		defaults.ProxyPort = *val
+	}
+}
+
+func (defaults *OriginRequestConfig) setProxyAddress(overrides config.OriginRequestConfig) {
+	if val := overrides.ProxyAddress; val != nil {
+		defaults.ProxyAddress = *val
+	}
+}
+
+func (defaults *OriginRequestConfig) setProxyType(overrides config.OriginRequestConfig) {
+	if val := overrides.ProxyType; val != nil {
+		defaults.ProxyType = *val
+	}
+}
+
+// SetConfig gets config for the requests that cloudflared sends to origins.
+// Each field has a setter method which sets a value for the field by trying to find:
+//   1. The user config for this rule
+//   2. The user config for the overall ingress config
+//   3. Defaults chosen by the cloudflared team
+//   4. Golang zero values for that type
+// If an earlier option isn't set, it will try the next option down.
+func setConfig(defaults OriginRequestConfig, overrides config.OriginRequestConfig) OriginRequestConfig {
+	cfg := defaults
+	cfg.setConnectTimeout(overrides)
+	cfg.setTLSTimeout(overrides)
+	cfg.setNoHappyEyeballs(overrides)
+	cfg.setKeepAliveConnections(overrides)
+	cfg.setKeepAliveTimeout(overrides)
+	cfg.setTCPKeepAlive(overrides)
+	cfg.setHTTPHostHeader(overrides)
+	cfg.setOriginServerName(overrides)
+	cfg.setCAPool(overrides)
+	cfg.setNoTLSVerify(overrides)
+	cfg.setDisableChunkedEncoding(overrides)
+	cfg.setBastionMode(overrides)
+	cfg.setProxyPort(overrides)
+	cfg.setProxyAddress(overrides)
+	cfg.setProxyType(overrides)
+	return cfg
+}

ingress/origin_request_config_test.go

@@ -0,0 +1,202 @@
+package ingress
+
+import (
+	"flag"
+	"testing"
+	"time"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/stretchr/testify/require"
+	"github.com/urfave/cli/v2"
+	"gopkg.in/yaml.v2"
+)
+
+// Ensure that the nullable config from `config` package and the
+// non-nullable config from `ingress` package have the same number of
+// fields.
+// This test ensures that programmers didn't add a new field to
+// one struct and forget to add it to the other ;)
+func TestCorrespondingFields(t *testing.T) {
+	require.Equal(
+		t,
+		CountFields(t, config.OriginRequestConfig{}),
+		CountFields(t, OriginRequestConfig{}),
+	)
+}
+
+func CountFields(t *testing.T, val interface{}) int {
+	b, err := yaml.Marshal(val)
+	require.NoError(t, err)
+	m := make(map[string]interface{}, 0)
+	err = yaml.Unmarshal(b, &m)
+	require.NoError(t, err)
+	return len(m)
+}
+
+func TestOriginRequestConfigOverrides(t *testing.T) {
+	rulesYAML := `
+originRequest:
+  connectTimeout: 1m
+  tlsTimeout: 1s
+  noHappyEyeballs: true
+  tcpKeepAlive: 1s
+  keepAliveConnections: 1
+  keepAliveTimeout: 1s
+  httpHostHeader: abc
+  originServerName: a1
+  caPool: /tmp/path0
+  noTLSVerify: true
+  disableChunkedEncoding: true
+  bastionMode: True
+  proxyAddress: 127.1.2.3
+  proxyPort: 100
+  proxyType: socks5
+ingress:
+- hostname: tun.example.com
+  service: https://localhost:8000
+- hostname: "*"
+  service: https://localhost:8001
+  originRequest:
+    connectTimeout: 2m
+    tlsTimeout: 2s
+    noHappyEyeballs: false
+    tcpKeepAlive: 2s
+    keepAliveConnections: 2
+    keepAliveTimeout: 2s
+    httpHostHeader: def
+    originServerName: b2
+    caPool: /tmp/path1
+    noTLSVerify: false
+    disableChunkedEncoding: false
+    bastionMode: false
+    proxyAddress: interface
+    proxyPort: 200
+    proxyType: ""
+`
+	ing, err := ParseIngress(MustReadIngress(rulesYAML))
+	if err != nil {
+		t.Error(err)
+	}
+
+	// Rule 0 didn't override anything, so it inherits the user-specified
+	// root-level configuration.
+	actual0 := ing.Rules[0].Config
+	expected0 := OriginRequestConfig{
+		ConnectTimeout:         1 * time.Minute,
+		TLSTimeout:             1 * time.Second,
+		NoHappyEyeballs:        true,
+		TCPKeepAlive:           1 * time.Second,
+		KeepAliveConnections:   1,
+		KeepAliveTimeout:       1 * time.Second,
+		HTTPHostHeader:         "abc",
+		OriginServerName:       "a1",
+		CAPool:                 "/tmp/path0",
+		NoTLSVerify:            true,
+		DisableChunkedEncoding: true,
+		BastionMode:            true,
+		ProxyAddress:           "127.1.2.3",
+		ProxyPort:              uint(100),
+		ProxyType:              "socks5",
+	}
+	require.Equal(t, expected0, actual0)
+
+	// Rule 1 overrode all the root-level config.
+	actual1 := ing.Rules[1].Config
+	expected1 := OriginRequestConfig{
+		ConnectTimeout:         2 * time.Minute,
+		TLSTimeout:             2 * time.Second,
+		NoHappyEyeballs:        false,
+		TCPKeepAlive:           2 * time.Second,
+		KeepAliveConnections:   2,
+		KeepAliveTimeout:       2 * time.Second,
+		HTTPHostHeader:         "def",
+		OriginServerName:       "b2",
+		CAPool:                 "/tmp/path1",
+		NoTLSVerify:            false,
+		DisableChunkedEncoding: false,
+		BastionMode:            false,
+		ProxyAddress:           "interface",
+		ProxyPort:              uint(200),
+		ProxyType:              "",
+	}
+	require.Equal(t, expected1, actual1)
+}
+
+func TestOriginRequestConfigDefaults(t *testing.T) {
+	rulesYAML := `
+ingress:
+- hostname: tun.example.com
+  service: https://localhost:8000
+- hostname: "*"
+  service: https://localhost:8001
+  originRequest:
+    connectTimeout: 2m
+    tlsTimeout: 2s
+    noHappyEyeballs: false
+    tcpKeepAlive: 2s
+    keepAliveConnections: 2
+    keepAliveTimeout: 2s
+    httpHostHeader: def
+    originServerName: b2
+    caPool: /tmp/path1
+    noTLSVerify: false
+    disableChunkedEncoding: false
+    bastionMode: false
+    proxyAddress: interface
+    proxyPort: 200
+    proxyType: ""
+`
+	ing, err := ParseIngress(MustReadIngress(rulesYAML))
+	if err != nil {
+		t.Error(err)
+	}
+
+	// Rule 0 didn't override anything, so it inherits the cloudflared defaults
+	actual0 := ing.Rules[0].Config
+	expected0 := OriginRequestConfig{
+		ConnectTimeout:       defaultConnectTimeout,
+		TLSTimeout:           defaultTLSTimeout,
+		TCPKeepAlive:         defaultTCPKeepAlive,
+		KeepAliveConnections: defaultKeepAliveConnections,
+		KeepAliveTimeout:     defaultKeepAliveTimeout,
+		ProxyAddress:         defaultProxyAddress,
+	}
+	require.Equal(t, expected0, actual0)
+
+	// Rule 1 overrode all defaults.
+	actual1 := ing.Rules[1].Config
+	expected1 := OriginRequestConfig{
+		ConnectTimeout:         2 * time.Minute,
+		TLSTimeout:             2 * time.Second,
+		NoHappyEyeballs:        false,
+		TCPKeepAlive:           2 * time.Second,
+		KeepAliveConnections:   2,
+		KeepAliveTimeout:       2 * time.Second,
+		HTTPHostHeader:         "def",
+		OriginServerName:       "b2",
+		CAPool:                 "/tmp/path1",
+		NoTLSVerify:            false,
+		DisableChunkedEncoding: false,
+		BastionMode:            false,
+		ProxyAddress:           "interface",
+		ProxyPort:              uint(200),
+		ProxyType:              "",
+	}
+	require.Equal(t, expected1, actual1)
+}
+
+func TestDefaultConfigFromCLI(t *testing.T) {
+	set := flag.NewFlagSet("contrive", 0)
+	c := cli.NewContext(nil, set, nil)
+
+	expected := OriginRequestConfig{
+		ConnectTimeout:       defaultConnectTimeout,
+		TLSTimeout:           defaultTLSTimeout,
+		TCPKeepAlive:         defaultTCPKeepAlive,
+		KeepAliveConnections: defaultKeepAliveConnections,
+		KeepAliveTimeout:     defaultKeepAliveTimeout,
+		ProxyAddress:         defaultProxyAddress,
+	}
+	actual := originRequestFromSingeRule(c)
+	require.Equal(t, expected, actual)
+}

ingress/origin_service.go

@@ -0,0 +1,343 @@
+package ingress
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"net/url"
+	"strconv"
+	"sync"
+	"time"
+
+	"github.com/cloudflare/cloudflared/hello"
+	"github.com/cloudflare/cloudflared/logger"
+	"github.com/cloudflare/cloudflared/socks"
+	"github.com/cloudflare/cloudflared/tlsconfig"
+	"github.com/cloudflare/cloudflared/websocket"
+	gws "github.com/gorilla/websocket"
+	"github.com/pkg/errors"
+)
+
+// OriginService is something a tunnel can proxy traffic to.
+type OriginService interface {
+	// RoundTrip is how cloudflared proxies eyeball requests to the actual origin services
+	http.RoundTripper
+	String() string
+	// Start the origin service if it's managed by cloudflared, e.g. proxy servers or Hello World.
+	// If it's not managed by cloudflared, this is a no-op because the user is responsible for
+	// starting the origin service.
+	start(wg *sync.WaitGroup, log logger.Service, shutdownC <-chan struct{}, errC chan error, cfg OriginRequestConfig) error
+}
+
+// unixSocketPath is an OriginService representing a unix socket (which accepts HTTP)
+type unixSocketPath struct {
+	path      string
+	transport *http.Transport
+}
+
+func (o *unixSocketPath) String() string {
+	return "unix socket: " + o.path
+}
+
+func (o *unixSocketPath) start(wg *sync.WaitGroup, log logger.Service, shutdownC <-chan struct{}, errC chan error, cfg OriginRequestConfig) error {
+	transport, err := newHTTPTransport(o, cfg, log)
+	if err != nil {
+		return err
+	}
+	o.transport = transport
+	return nil
+}
+
+func (o *unixSocketPath) RoundTrip(req *http.Request) (*http.Response, error) {
+	return o.transport.RoundTrip(req)
+}
+
+func (o *unixSocketPath) Dial(reqURL *url.URL, headers http.Header) (*gws.Conn, *http.Response, error) {
+	d := &gws.Dialer{
+		NetDial:         o.transport.Dial,
+		NetDialContext:  o.transport.DialContext,
+		TLSClientConfig: o.transport.TLSClientConfig,
+	}
+	reqURL.Scheme = websocket.ChangeRequestScheme(reqURL)
+	return d.Dial(reqURL.String(), headers)
+}
+
+// localService is an OriginService listening on a TCP/IP address the user's origin can route to.
+type localService struct {
+	// The URL for the user's origin service
+	RootURL *url.URL
+	// The URL that cloudflared should send requests to.
+	// If this origin requires starting a proxy, this is the proxy's address,
+	// and that proxy points to RootURL. Otherwise, this is equal to RootURL.
+	URL       *url.URL
+	transport *http.Transport
+}
+
+func (o *localService) Dial(reqURL *url.URL, headers http.Header) (*gws.Conn, *http.Response, error) {
+	d := &gws.Dialer{TLSClientConfig: o.transport.TLSClientConfig}
+	// Rewrite the request URL so that it goes to the origin service.
+	reqURL.Host = o.URL.Host
+	reqURL.Scheme = websocket.ChangeRequestScheme(o.URL)
+	return d.Dial(reqURL.String(), headers)
+}
+
+func (o *localService) start(wg *sync.WaitGroup, log logger.Service, shutdownC <-chan struct{}, errC chan error, cfg OriginRequestConfig) error {
+	transport, err := newHTTPTransport(o, cfg, log)
+	if err != nil {
+		return err
+	}
+	o.transport = transport
+
+	// Start a proxy if one is needed
+	if staticHost := o.staticHost(); originRequiresProxy(staticHost, cfg) {
+		if err := o.startProxy(staticHost, wg, log, shutdownC, errC, cfg); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (o *localService) startProxy(staticHost string, wg *sync.WaitGroup, log logger.Service, shutdownC <-chan struct{}, errC chan error, cfg OriginRequestConfig) error {
+
+	// Start a listener for the proxy
+	proxyAddress := net.JoinHostPort(cfg.ProxyAddress, strconv.Itoa(int(cfg.ProxyPort)))
+	listener, err := net.Listen("tcp", proxyAddress)
+	if err != nil {
+		log.Errorf("Cannot start Websocket Proxy Server: %s", err)
+		return errors.Wrap(err, "Cannot start Websocket Proxy Server")
+	}
+
+	// Start the proxy itself
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		streamHandler := websocket.DefaultStreamHandler
+		// This origin's config specifies what type of proxy to start.
+		switch cfg.ProxyType {
+		case socksProxy:
+			log.Info("SOCKS5 server started")
+			streamHandler = func(wsConn *websocket.Conn, remoteConn net.Conn, _ http.Header) {
+				dialer := socks.NewConnDialer(remoteConn)
+				requestHandler := socks.NewRequestHandler(dialer)
+				socksServer := socks.NewConnectionHandler(requestHandler)
+
+				socksServer.Serve(wsConn)
+			}
+		case "":
+			log.Debug("Not starting any websocket proxy")
+		default:
+			log.Errorf("%s isn't a valid proxy (valid options are {%s})", cfg.ProxyType, socksProxy)
+		}
+
+		errC <- websocket.StartProxyServer(log, listener, staticHost, shutdownC, streamHandler)
+	}()
+
+	// Modify this origin, so that it no longer points at the origin service directly.
+	// Instead, it points at the proxy to the origin service.
+	newURL, err := url.Parse("http://" + listener.Addr().String())
+	if err != nil {
+		return err
+	}
+	o.URL = newURL
+	return nil
+}
+
+func (o *localService) String() string {
+	if o.isBastion() {
+		return "Bastion"
+	}
+	return o.URL.String()
+}
+
+func (o *localService) isBastion() bool {
+	return o.URL == nil
+}
+
+func (o *localService) RoundTrip(req *http.Request) (*http.Response, error) {
+	// Rewrite the request URL so that it goes to the origin service.
+	req.URL.Host = o.URL.Host
+	req.URL.Scheme = o.URL.Scheme
+	return o.transport.RoundTrip(req)
+}
+
+func (o *localService) staticHost() string {
+
+	if o.URL == nil {
+		return ""
+	}
+
+	addPortIfMissing := func(uri *url.URL, port int) string {
+		if uri.Port() != "" {
+			return uri.Host
+		}
+		return fmt.Sprintf("%s:%d", uri.Hostname(), port)
+	}
+
+	switch o.URL.Scheme {
+	case "ssh":
+		return addPortIfMissing(o.URL, 22)
+	case "rdp":
+		return addPortIfMissing(o.URL, 3389)
+	case "smb":
+		return addPortIfMissing(o.URL, 445)
+	case "tcp":
+		return addPortIfMissing(o.URL, 7864) // just a random port since there isn't a default in this case
+	}
+	return ""
+
+}
+
+// HelloWorld is an OriginService for the built-in Hello World server.
+// Users only use this for testing and experimenting with cloudflared.
+type helloWorld struct {
+	server    net.Listener
+	transport *http.Transport
+}
+
+func (o *helloWorld) String() string {
+	return "Hello World test origin"
+}
+
+// Start starts a HelloWorld server and stores its address in the Service receiver.
+func (o *helloWorld) start(wg *sync.WaitGroup, log logger.Service, shutdownC <-chan struct{}, errC chan error, cfg OriginRequestConfig) error {
+	transport, err := newHTTPTransport(o, cfg, log)
+	if err != nil {
+		return err
+	}
+	o.transport = transport
+	helloListener, err := hello.CreateTLSListener("127.0.0.1:")
+	if err != nil {
+		return errors.Wrap(err, "Cannot start Hello World Server")
+	}
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		_ = hello.StartHelloWorldServer(log, helloListener, shutdownC)
+	}()
+	o.server = helloListener
+	return nil
+}
+
+func (o *helloWorld) RoundTrip(req *http.Request) (*http.Response, error) {
+	// Rewrite the request URL so that it goes to the Hello World server.
+	req.URL.Host = o.server.Addr().String()
+	req.URL.Scheme = "https"
+	return o.transport.RoundTrip(req)
+}
+
+func (o *helloWorld) Dial(reqURL *url.URL, headers http.Header) (*gws.Conn, *http.Response, error) {
+	d := &gws.Dialer{
+		TLSClientConfig: o.transport.TLSClientConfig,
+	}
+	reqURL.Host = o.server.Addr().String()
+	reqURL.Scheme = "wss"
+	return d.Dial(reqURL.String(), headers)
+}
+
+func originRequiresProxy(staticHost string, cfg OriginRequestConfig) bool {
+	return staticHost != "" || cfg.BastionMode
+}
+
+// statusCode is an OriginService that just responds with a given HTTP status.
+// Typical use-case is "user wants the catch-all rule to just respond 404".
+type statusCode struct {
+	resp *http.Response
+}
+
+func newStatusCode(status int) statusCode {
+	resp := &http.Response{
+		StatusCode: status,
+		Status:     fmt.Sprintf("%d %s", status, http.StatusText(status)),
+		Body:       new(NopReadCloser),
+	}
+	return statusCode{resp: resp}
+}
+
+func (o *statusCode) String() string {
+	return fmt.Sprintf("HTTP %d", o.resp.StatusCode)
+}
+
+func (o *statusCode) start(wg *sync.WaitGroup, log logger.Service, shutdownC <-chan struct{}, errC chan error, cfg OriginRequestConfig) error {
+	return nil
+}
+
+func (o *statusCode) RoundTrip(_ *http.Request) (*http.Response, error) {
+	return o.resp, nil
+}
+
+type NopReadCloser struct{}
+
+// Read always returns EOF to signal end of input
+func (nrc *NopReadCloser) Read(buf []byte) (int, error) {
+	return 0, io.EOF
+}
+
+func (nrc *NopReadCloser) Close() error {
+	return nil
+}
+
+func newHTTPTransport(service OriginService, cfg OriginRequestConfig, log logger.Service) (*http.Transport, error) {
+	originCertPool, err := tlsconfig.LoadOriginCA(cfg.CAPool, log)
+	if err != nil {
+		return nil, errors.Wrap(err, "Error loading cert pool")
+	}
+
+	httpTransport := http.Transport{
+		Proxy:                 http.ProxyFromEnvironment,
+		MaxIdleConns:          cfg.KeepAliveConnections,
+		MaxIdleConnsPerHost:   cfg.KeepAliveConnections,
+		IdleConnTimeout:       cfg.KeepAliveTimeout,
+		TLSHandshakeTimeout:   cfg.TLSTimeout,
+		ExpectContinueTimeout: 1 * time.Second,
+		TLSClientConfig:       &tls.Config{RootCAs: originCertPool, InsecureSkipVerify: cfg.NoTLSVerify},
+	}
+	if _, isHelloWorld := service.(*helloWorld); !isHelloWorld && cfg.OriginServerName != "" {
+		httpTransport.TLSClientConfig.ServerName = cfg.OriginServerName
+	}
+
+	dialer := &net.Dialer{
+		Timeout:   cfg.ConnectTimeout,
+		KeepAlive: cfg.TCPKeepAlive,
+	}
+	if cfg.NoHappyEyeballs {
+		dialer.FallbackDelay = -1 // As of Golang 1.12, a negative delay disables "happy eyeballs"
+	}
+
+	// DialContext depends on which kind of origin is being used.
+	dialContext := dialer.DialContext
+	switch service := service.(type) {
+
+	// If this origin is a unix socket, enforce network type "unix".
+	case *unixSocketPath:
+		httpTransport.DialContext = func(ctx context.Context, _, _ string) (net.Conn, error) {
+			return dialContext(ctx, "unix", service.path)
+		}
+
+	// Otherwise, use the regular network config.
+	default:
+		httpTransport.DialContext = dialContext
+	}
+
+	return &httpTransport, nil
+}
+
+// MockOriginService should only be used by other packages to mock OriginService. Set Transport to configure desired RoundTripper behavior.
+type MockOriginService struct {
+	Transport http.RoundTripper
+}
+
+func (mos MockOriginService) RoundTrip(req *http.Request) (*http.Response, error) {
+	return mos.Transport.RoundTrip(req)
+}
+
+func (mos MockOriginService) String() string {
+	return "MockOriginService"
+}
+
+func (mos MockOriginService) start(wg *sync.WaitGroup, log logger.Service, shutdownC <-chan struct{}, errC chan error, cfg OriginRequestConfig) error {
+	return nil
+}

ingress/rule.go

@@ -0,0 +1,50 @@
+package ingress
+
+import (
+	"regexp"
+	"strings"
+)
+
+// Rule routes traffic from a hostname/path on the public internet to the
+// service running on the given URL.
+type Rule struct {
+	// Requests for this hostname will be proxied to this rule's service.
+	Hostname string
+
+	// Path is an optional regex that can specify path-driven ingress rules.
+	Path *regexp.Regexp
+
+	// A (probably local) address. Requests for a hostname which matches this
+	// rule's hostname pattern will be proxied to the service running on this
+	// address.
+	Service OriginService
+
+	// Configure the request cloudflared sends to this specific origin.
+	Config OriginRequestConfig
+}
+
+// MultiLineString is for outputting rules in a human-friendly way when Cloudflared
+// is used as a CLI tool (not as a daemon).
+func (r Rule) MultiLineString() string {
+	var out strings.Builder
+	if r.Hostname != "" {
+		out.WriteString("\thostname: ")
+		out.WriteString(r.Hostname)
+		out.WriteRune('\n')
+	}
+	if r.Path != nil {
+		out.WriteString("\tpath: ")
+		out.WriteString(r.Path.String())
+		out.WriteRune('\n')
+	}
+	out.WriteString("\tservice: ")
+	out.WriteString(r.Service.String())
+	return out.String()
+}
+
+// Matches checks if the rule matches a given hostname/path combination.
+func (r *Rule) Matches(hostname, path string) bool {
+	hostMatch := r.Hostname == "" || r.Hostname == "*" || matchHost(r.Hostname, hostname)
+	pathMatch := r.Path == nil || r.Path.MatchString(path)
+	return hostMatch && pathMatch
+}

ingress/rule_test.go

@@ -0,0 +1,146 @@
+package ingress
+
+import (
+	"io"
+	"net/http/httptest"
+	"net/url"
+	"regexp"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func Test_rule_matches(t *testing.T) {
+	type fields struct {
+		Hostname string
+		Path     *regexp.Regexp
+		Service  OriginService
+	}
+	type args struct {
+		requestURL *url.URL
+	}
+	tests := []struct {
+		name   string
+		fields fields
+		args   args
+		want   bool
+	}{
+		{
+			name: "Just hostname, pass",
+			fields: fields{
+				Hostname: "example.com",
+			},
+			args: args{
+				requestURL: MustParseURL(t, "https://example.com"),
+			},
+			want: true,
+		},
+		{
+			name: "Entire hostname is wildcard, should match everything",
+			fields: fields{
+				Hostname: "*",
+			},
+			args: args{
+				requestURL: MustParseURL(t, "https://example.com"),
+			},
+			want: true,
+		},
+		{
+			name: "Just hostname, fail",
+			fields: fields{
+				Hostname: "example.com",
+			},
+			args: args{
+				requestURL: MustParseURL(t, "https://foo.bar"),
+			},
+			want: false,
+		},
+		{
+			name: "Just wildcard hostname, pass",
+			fields: fields{
+				Hostname: "*.example.com",
+			},
+			args: args{
+				requestURL: MustParseURL(t, "https://adam.example.com"),
+			},
+			want: true,
+		},
+		{
+			name: "Just wildcard hostname, fail",
+			fields: fields{
+				Hostname: "*.example.com",
+			},
+			args: args{
+				requestURL: MustParseURL(t, "https://tunnel.com"),
+			},
+			want: false,
+		},
+		{
+			name: "Just wildcard outside of subdomain in hostname, fail",
+			fields: fields{
+				Hostname: "*example.com",
+			},
+			args: args{
+				requestURL: MustParseURL(t, "https://www.example.com"),
+			},
+			want: false,
+		},
+		{
+			name: "Wildcard over multiple subdomains",
+			fields: fields{
+				Hostname: "*.example.com",
+			},
+			args: args{
+				requestURL: MustParseURL(t, "https://adam.chalmers.example.com"),
+			},
+			want: true,
+		},
+		{
+			name: "Hostname and path",
+			fields: fields{
+				Hostname: "*.example.com",
+				Path:     regexp.MustCompile("/static/.*\\.html"),
+			},
+			args: args{
+				requestURL: MustParseURL(t, "https://www.example.com/static/index.html"),
+			},
+			want: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			r := Rule{
+				Hostname: tt.fields.Hostname,
+				Path:     tt.fields.Path,
+				Service:  tt.fields.Service,
+			}
+			u := tt.args.requestURL
+			if got := r.Matches(u.Hostname(), u.Path); got != tt.want {
+				t.Errorf("rule.matches() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestStaticHTTPStatus(t *testing.T) {
+	o := newStatusCode(404)
+	buf := make([]byte, 100)
+
+	sendReq := func() {
+		resp, err := o.RoundTrip(nil)
+		require.NoError(t, err)
+		_, err = resp.Body.Read(buf)
+		require.Equal(t, io.EOF, err)
+		require.NoError(t, resp.Body.Close())
+		require.Equal(t, 404, resp.StatusCode)
+
+		resp, err = o.RoundTrip(nil)
+		require.NoError(t, err)
+		w := httptest.NewRecorder()
+		n, err := io.Copy(w, resp.Body)
+		require.NoError(t, err)
+		require.Equal(t, int64(0), n)
+	}
+	sendReq()
+	sendReq()
+}

logger/configuration.go

@@ -0,0 +1,118 @@
+package logger
+
+var defaultConfig = createDefaultConfig()
+
+// Logging configuration
+type Config struct {
+	ConsoleConfig *ConsoleConfig // If nil, the logger will not log into the console
+	FileConfig    *FileConfig    // If nil, the logger will not use an individual log file
+	RollingConfig *RollingConfig // If nil, the logger will not use a rolling log
+
+	MinLevel string // debug | info | error | fatal
+}
+
+type ConsoleConfig struct {
+	noColor bool
+}
+
+type FileConfig struct {
+	Filepath string
+}
+
+type RollingConfig struct {
+	Directory string
+	Filename  string
+
+	maxSize    int // megabytes
+	maxBackups int // files
+	maxAge     int // days
+}
+
+func createDefaultConfig() Config {
+	const minLevel = "fatal"
+
+	const RollingMaxSize = 1    // Mb
+	const RollingMaxBackups = 5 // files
+	const RollingMaxAge = 0     // Keep forever
+	const rollingLogFilename = "cloudflared.log"
+
+	return Config{
+		ConsoleConfig: &ConsoleConfig{
+			noColor: false,
+		},
+		FileConfig: &FileConfig{
+			Filepath: "",
+		},
+		RollingConfig: &RollingConfig{
+			Directory:  "",
+			Filename:   rollingLogFilename,
+			maxSize:    RollingMaxSize,
+			maxBackups: RollingMaxBackups,
+			maxAge:     RollingMaxAge,
+		},
+		MinLevel: minLevel,
+	}
+}
+
+func CreateConfig(
+	minLevel string,
+	disableTerminal bool,
+	rollingLogPath, nonRollingLogFilePath string,
+) *Config {
+	var console *ConsoleConfig
+	if !disableTerminal {
+		console = createConsoleConfig()
+	}
+
+	var file *FileConfig
+	if nonRollingLogFilePath != "" {
+		file = createFileConfig(nonRollingLogFilePath)
+	}
+
+	var rolling *RollingConfig
+	if rollingLogPath != "" {
+		rolling = createRollingConfig(rollingLogPath)
+	}
+
+	if minLevel == "" {
+		minLevel = defaultConfig.MinLevel
+	}
+
+	return &Config{
+		ConsoleConfig: console,
+		FileConfig:    file,
+		RollingConfig: rolling,
+
+		MinLevel: minLevel,
+	}
+}
+
+func createConsoleConfig() *ConsoleConfig {
+	return &ConsoleConfig{
+		noColor: false,
+	}
+}
+
+func createFileConfig(filepath string) *FileConfig {
+	if filepath == "" {
+		filepath = defaultConfig.FileConfig.Filepath
+	}
+
+	return &FileConfig{
+		Filepath: filepath,
+	}
+}
+
+func createRollingConfig(directory string) *RollingConfig {
+	if directory == "" {
+		directory = defaultConfig.RollingConfig.Directory
+	}
+
+	return &RollingConfig{
+		Directory:  directory,
+		Filename:   defaultConfig.RollingConfig.Filename,
+		maxSize:    defaultConfig.RollingConfig.maxSize,
+		maxBackups: defaultConfig.RollingConfig.maxBackups,
+		maxAge:     defaultConfig.RollingConfig.maxAge,
+	}
+}

logger/create.go

@@ -8,6 +8,20 @@ import (
 	"time"
 
 	"github.com/alecthomas/units"
+	"github.com/urfave/cli/v2"
+)
+
+const (
+	EnableTerminalLog  = false
+	DisableTerminalLog = true
+
+	LogLevelFlag          = "loglevel"
+	LogFileFlag           = "logfile"
+	LogDirectoryFlag      = "log-directory"
+	LogTransportLevelFlag = "transport-loglevel"
+
+	LogSSHDirectoryFlag = "log-directory"
+	LogSSHLevelFlag     = "log-level"
 )
 
 // Option is to encaspulate actions that will be called by Parse and run later to build an Options struct
@@ -127,6 +141,63 @@ func New(opts ...Option) (*OutputWriter, error) {
 	return l, nil
 }
 
+func NewInHouse(loggerConfig *Config) (*OutputWriter, error) {
+	var loggerOpts []Option
+
+	var logPath string
+	if loggerConfig.FileConfig != nil {
+		logPath = loggerConfig.FileConfig.Filepath
+	}
+	if logPath == "" && loggerConfig.RollingConfig != nil {
+		logPath = loggerConfig.RollingConfig.Directory
+	}
+
+	if logPath != "" {
+		loggerOpts = append(loggerOpts, DefaultFile(logPath))
+	}
+
+	loggerOpts = append(loggerOpts, LogLevelString(loggerConfig.MinLevel))
+
+	if loggerConfig.ConsoleConfig == nil {
+		disableOption := DisableTerminal(true)
+		loggerOpts = append(loggerOpts, disableOption)
+	}
+
+	l, err := New(loggerOpts...)
+	if err != nil {
+		return nil, err
+	}
+
+	return l, nil
+}
+
+func CreateTransportLoggerFromContext(c *cli.Context, disableTerminal bool) (*OutputWriter, error) {
+	return createFromContext(c, LogTransportLevelFlag, LogDirectoryFlag, disableTerminal)
+}
+
+func CreateLoggerFromContext(c *cli.Context, disableTerminal bool) (*OutputWriter, error) {
+	return createFromContext(c, LogLevelFlag, LogDirectoryFlag, disableTerminal)
+}
+
+func CreateSSHLoggerFromContext(c *cli.Context, disableTerminal bool) (*OutputWriter, error) {
+	return createFromContext(c, LogSSHLevelFlag, LogSSHDirectoryFlag, disableTerminal)
+}
+
+func createFromContext(
+	c *cli.Context,
+	logLevelFlagName,
+	logDirectoryFlagName string,
+	disableTerminal bool,
+) (*OutputWriter, error) {
+	logLevel := c.String(logLevelFlagName)
+	logFile := c.String(LogFileFlag)
+	logDirectory := c.String(logDirectoryFlagName)
+
+	loggerConfig := CreateConfig(logLevel, disableTerminal, logDirectory, logFile)
+
+	return NewInHouse(loggerConfig)
+}
+
 // ParseLevelString returns the expected log levels based on the cmd flag
 func ParseLevelString(lvl string) ([]Level, error) {
 	switch strings.ToLower(lvl) {

metrics/metrics.go

@@ -12,6 +12,7 @@ import (
 
 	"golang.org/x/net/trace"
 
+	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
@@ -22,22 +23,35 @@ const (
 	startupTime     = time.Millisecond * 500
 )
 
-func ServeMetrics(l net.Listener, shutdownC <-chan struct{}, logger logger.Service) (err error) {
+func newMetricsHandler(connectionEvents <-chan connection.Event, log logger.Service) *http.ServeMux {
+	readyServer := NewReadyServer(connectionEvents, log)
+	mux := http.NewServeMux()
+	mux.Handle("/metrics", promhttp.Handler())
+	mux.HandleFunc("/healthcheck", func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintf(w, "OK\n")
+	})
+	mux.Handle("/ready", readyServer)
+	return mux
+}
+
+func ServeMetrics(
+	l net.Listener,
+	shutdownC <-chan struct{},
+	connectionEvents <-chan connection.Event,
+	logger logger.Service,
+) (err error) {
 	var wg sync.WaitGroup
 	// Metrics port is privileged, so no need for further access control
 	trace.AuthRequest = func(*http.Request) (bool, bool) { return true, true }
 	// TODO: parameterize ReadTimeout and WriteTimeout. The maximum time we can
 	// profile CPU usage depends on WriteTimeout
+	h := newMetricsHandler(connectionEvents, logger)
 	server := &http.Server{
 		ReadTimeout:  10 * time.Second,
 		WriteTimeout: 10 * time.Second,
+		Handler:      h,
 	}
 
-	http.Handle("/metrics", promhttp.Handler())
-	http.Handle("/healthcheck", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		fmt.Fprintf(w, "OK\n")
-	}))
-
 	wg.Add(1)
 	go func() {
 		defer wg.Done()

metrics/readiness.go

@@ -0,0 +1,80 @@
+package metrics
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"sync"
+
+	conn "github.com/cloudflare/cloudflared/connection"
+	"github.com/cloudflare/cloudflared/logger"
+)
+
+// ReadyServer serves HTTP 200 if the tunnel can serve traffic. Intended for k8s readiness checks.
+type ReadyServer struct {
+	sync.RWMutex
+	isConnected map[int]bool
+	log         logger.Service
+}
+
+// NewReadyServer initializes a ReadyServer and starts listening for dis/connection events.
+func NewReadyServer(connectionEvents <-chan conn.Event, log logger.Service) *ReadyServer {
+	rs := ReadyServer{
+		isConnected: make(map[int]bool, 0),
+		log:         log,
+	}
+	go func() {
+		for c := range connectionEvents {
+			switch c.EventType {
+			case conn.Connected:
+				rs.Lock()
+				rs.isConnected[int(c.Index)] = true
+				rs.Unlock()
+			case conn.Disconnected, conn.Reconnecting, conn.RegisteringTunnel:
+				rs.Lock()
+				rs.isConnected[int(c.Index)] = false
+				rs.Unlock()
+			case conn.SetURL:
+				continue
+			default:
+				rs.log.Errorf("Unknown connection event case %v", c)
+			}
+		}
+	}()
+	return &rs
+}
+
+type body struct {
+	Status           int `json:"status"`
+	ReadyConnections int `json:"readyConnections"`
+}
+
+// ServeHTTP responds with HTTP 200 if the tunnel is connected to the edge.
+func (rs *ReadyServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	statusCode, readyConnections := rs.makeResponse()
+	w.WriteHeader(statusCode)
+	body := body{
+		Status:           statusCode,
+		ReadyConnections: readyConnections,
+	}
+	msg, err := json.Marshal(body)
+	if err != nil {
+		fmt.Fprintf(w, `{"error": "%s"}`, err)
+	}
+	w.Write(msg)
+}
+
+// This is the bulk of the logic for ServeHTTP, broken into its own pure function
+// to make unit testing easy.
+func (rs *ReadyServer) makeResponse() (statusCode, readyConnections int) {
+	statusCode = http.StatusServiceUnavailable
+	rs.RLock()
+	defer rs.RUnlock()
+	for _, connected := range rs.isConnected {
+		if connected {
+			statusCode = http.StatusOK
+			readyConnections++
+		}
+	}
+	return statusCode, readyConnections
+}

metrics/readiness_test.go

@@ -0,0 +1,58 @@
+package metrics
+
+import (
+	"net/http"
+	"testing"
+)
+
+func TestReadyServer_makeResponse(t *testing.T) {
+	type fields struct {
+		isConnected map[int]bool
+	}
+	tests := []struct {
+		name                 string
+		fields               fields
+		wantOK               bool
+		wantReadyConnections int
+	}{
+		{
+			name: "One connection online => HTTP 200",
+			fields: fields{
+				isConnected: map[int]bool{
+					0: false,
+					1: false,
+					2: true,
+					3: false,
+				},
+			},
+			wantOK:               true,
+			wantReadyConnections: 1,
+		},
+		{
+			name: "No connections online => no HTTP 200",
+			fields: fields{
+				isConnected: map[int]bool{
+					0: false,
+					1: false,
+					2: false,
+					3: false,
+				},
+			},
+			wantReadyConnections: 0,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			rs := &ReadyServer{
+				isConnected: tt.fields.isConnected,
+			}
+			gotStatusCode, gotReadyConnections := rs.makeResponse()
+			if tt.wantOK && gotStatusCode != http.StatusOK {
+				t.Errorf("ReadyServer.makeResponse() gotStatusCode = %v, want ok = %v", gotStatusCode, tt.wantOK)
+			}
+			if gotReadyConnections != tt.wantReadyConnections {
+				t.Errorf("ReadyServer.makeResponse() gotReadyConnections = %v, want %v", gotReadyConnections, tt.wantReadyConnections)
+			}
+		})
+	}
+}

origin/backoffhandler.go

@@ -97,3 +97,11 @@ func (b BackoffHandler) GetBaseTime() time.Duration {
 func (b *BackoffHandler) Retries() int {
 	return int(b.retries)
 }
+
+func (b *BackoffHandler) ReachedMaxRetries() bool {
+	return b.retries == b.MaxRetries
+}
+
+func (b *BackoffHandler) resetNow() {
+	b.resetDeadline = time.Now()
+}

origin/metrics.go

@@ -1,540 +1,72 @@
 package origin
 
 import (
-	"sync"
-	"time"
-
-	"github.com/cloudflare/cloudflared/h2mux"
-
+	"github.com/cloudflare/cloudflared/connection"
 	"github.com/prometheus/client_golang/prometheus"
 )
 
-const (
-	metricsNamespace = "cloudflared"
-	tunnelSubsystem  = "tunnel"
-	muxerSubsystem   = "muxer"
-)
+// Metrics uses connection.MetricsNamespace(aka cloudflared) as namespace and connection.TunnelSubsystem
+// (tunnel) as subsystem to keep them consistent with the previous qualifier.
 
-type muxerMetrics struct {
-	rtt              *prometheus.GaugeVec
-	rttMin           *prometheus.GaugeVec
-	rttMax           *prometheus.GaugeVec
-	receiveWindowAve *prometheus.GaugeVec
-	sendWindowAve    *prometheus.GaugeVec
-	receiveWindowMin *prometheus.GaugeVec
-	receiveWindowMax *prometheus.GaugeVec
-	sendWindowMin    *prometheus.GaugeVec
-	sendWindowMax    *prometheus.GaugeVec
-	inBoundRateCurr  *prometheus.GaugeVec
-	inBoundRateMin   *prometheus.GaugeVec
-	inBoundRateMax   *prometheus.GaugeVec
-	outBoundRateCurr *prometheus.GaugeVec
-	outBoundRateMin  *prometheus.GaugeVec
-	outBoundRateMax  *prometheus.GaugeVec
-	compBytesBefore  *prometheus.GaugeVec
-	compBytesAfter   *prometheus.GaugeVec
-	compRateAve      *prometheus.GaugeVec
-}
-
-type TunnelMetrics struct {
-	haConnections     prometheus.Gauge
-	activeStreams     prometheus.Gauge
-	totalRequests     prometheus.Counter
-	requestsPerTunnel *prometheus.CounterVec
-	// concurrentRequestsLock is a mutex for concurrentRequests and maxConcurrentRequests
-	concurrentRequestsLock      sync.Mutex
-	concurrentRequestsPerTunnel *prometheus.GaugeVec
-	// concurrentRequests records count of concurrent requests for each tunnel
-	concurrentRequests             map[string]uint64
-	maxConcurrentRequestsPerTunnel *prometheus.GaugeVec
-	// concurrentRequests records max count of concurrent requests for each tunnel
-	maxConcurrentRequests map[string]uint64
-	timerRetries          prometheus.Gauge
-	responseByCode        *prometheus.CounterVec
-	responseCodePerTunnel *prometheus.CounterVec
-	serverLocations       *prometheus.GaugeVec
-	// locationLock is a mutex for oldServerLocations
-	locationLock sync.Mutex
-	// oldServerLocations stores the last server the tunnel was connected to
-	oldServerLocations map[string]string
-
-	regSuccess *prometheus.CounterVec
-	regFail    *prometheus.CounterVec
-	rpcFail    *prometheus.CounterVec
-
-	muxerMetrics        *muxerMetrics
-	tunnelsHA           tunnelsForHA
-	userHostnamesCounts *prometheus.CounterVec
-}
-
-func newMuxerMetrics() *muxerMetrics {
-	rtt := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "rtt",
-			Help:      "Round-trip time in millisecond",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(rtt)
-
-	rttMin := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "rtt_min",
-			Help:      "Shortest round-trip time in millisecond",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(rttMin)
-
-	rttMax := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "rtt_max",
-			Help:      "Longest round-trip time in millisecond",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(rttMax)
-
-	receiveWindowAve := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "receive_window_ave",
-			Help:      "Average receive window size in bytes",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(receiveWindowAve)
-
-	sendWindowAve := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "send_window_ave",
-			Help:      "Average send window size in bytes",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(sendWindowAve)
-
-	receiveWindowMin := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "receive_window_min",
-			Help:      "Smallest receive window size in bytes",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(receiveWindowMin)
-
-	receiveWindowMax := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "receive_window_max",
-			Help:      "Largest receive window size in bytes",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(receiveWindowMax)
-
-	sendWindowMin := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "send_window_min",
-			Help:      "Smallest send window size in bytes",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(sendWindowMin)
-
-	sendWindowMax := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "send_window_max",
-			Help:      "Largest send window size in bytes",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(sendWindowMax)
-
-	inBoundRateCurr := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "inbound_bytes_per_sec_curr",
-			Help:      "Current inbounding bytes per second, 0 if there is no incoming connection",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(inBoundRateCurr)
-
-	inBoundRateMin := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "inbound_bytes_per_sec_min",
-			Help:      "Minimum non-zero inbounding bytes per second",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(inBoundRateMin)
-
-	inBoundRateMax := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "inbound_bytes_per_sec_max",
-			Help:      "Maximum inbounding bytes per second",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(inBoundRateMax)
-
-	outBoundRateCurr := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "outbound_bytes_per_sec_curr",
-			Help:      "Current outbounding bytes per second, 0 if there is no outgoing traffic",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(outBoundRateCurr)
-
-	outBoundRateMin := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "outbound_bytes_per_sec_min",
-			Help:      "Minimum non-zero outbounding bytes per second",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(outBoundRateMin)
-
-	outBoundRateMax := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "outbound_bytes_per_sec_max",
-			Help:      "Maximum outbounding bytes per second",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(outBoundRateMax)
-
-	compBytesBefore := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "comp_bytes_before",
-			Help:      "Bytes sent via cross-stream compression, pre compression",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(compBytesBefore)
-
-	compBytesAfter := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "comp_bytes_after",
-			Help:      "Bytes sent via cross-stream compression, post compression",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(compBytesAfter)
-
-	compRateAve := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "comp_rate_ave",
-			Help:      "Average outbound cross-stream compression ratio",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(compRateAve)
-
-	return &muxerMetrics{
-		rtt:              rtt,
-		rttMin:           rttMin,
-		rttMax:           rttMax,
-		receiveWindowAve: receiveWindowAve,
-		sendWindowAve:    sendWindowAve,
-		receiveWindowMin: receiveWindowMin,
-		receiveWindowMax: receiveWindowMax,
-		sendWindowMin:    sendWindowMin,
-		sendWindowMax:    sendWindowMax,
-		inBoundRateCurr:  inBoundRateCurr,
-		inBoundRateMin:   inBoundRateMin,
-		inBoundRateMax:   inBoundRateMax,
-		outBoundRateCurr: outBoundRateCurr,
-		outBoundRateMin:  outBoundRateMin,
-		outBoundRateMax:  outBoundRateMax,
-		compBytesBefore:  compBytesBefore,
-		compBytesAfter:   compBytesAfter,
-		compRateAve:      compRateAve,
-	}
-}
-
-func (m *muxerMetrics) update(connectionID string, metrics *h2mux.MuxerMetrics) {
-	m.rtt.WithLabelValues(connectionID).Set(convertRTTMilliSec(metrics.RTT))
-	m.rttMin.WithLabelValues(connectionID).Set(convertRTTMilliSec(metrics.RTTMin))
-	m.rttMax.WithLabelValues(connectionID).Set(convertRTTMilliSec(metrics.RTTMax))
-	m.receiveWindowAve.WithLabelValues(connectionID).Set(metrics.ReceiveWindowAve)
-	m.sendWindowAve.WithLabelValues(connectionID).Set(metrics.SendWindowAve)
-	m.receiveWindowMin.WithLabelValues(connectionID).Set(float64(metrics.ReceiveWindowMin))
-	m.receiveWindowMax.WithLabelValues(connectionID).Set(float64(metrics.ReceiveWindowMax))
-	m.sendWindowMin.WithLabelValues(connectionID).Set(float64(metrics.SendWindowMin))
-	m.sendWindowMax.WithLabelValues(connectionID).Set(float64(metrics.SendWindowMax))
-	m.inBoundRateCurr.WithLabelValues(connectionID).Set(float64(metrics.InBoundRateCurr))
-	m.inBoundRateMin.WithLabelValues(connectionID).Set(float64(metrics.InBoundRateMin))
-	m.inBoundRateMax.WithLabelValues(connectionID).Set(float64(metrics.InBoundRateMax))
-	m.outBoundRateCurr.WithLabelValues(connectionID).Set(float64(metrics.OutBoundRateCurr))
-	m.outBoundRateMin.WithLabelValues(connectionID).Set(float64(metrics.OutBoundRateMin))
-	m.outBoundRateMax.WithLabelValues(connectionID).Set(float64(metrics.OutBoundRateMax))
-	m.compBytesBefore.WithLabelValues(connectionID).Set(float64(metrics.CompBytesBefore.Value()))
-	m.compBytesAfter.WithLabelValues(connectionID).Set(float64(metrics.CompBytesAfter.Value()))
-	m.compRateAve.WithLabelValues(connectionID).Set(float64(metrics.CompRateAve()))
-}
-
-func convertRTTMilliSec(t time.Duration) float64 {
-	return float64(t / time.Millisecond)
-}
-
-// Metrics that can be collected without asking the edge
-func NewTunnelMetrics() *TunnelMetrics {
-	haConnections := prometheus.NewGauge(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: tunnelSubsystem,
-			Name:      "ha_connections",
-			Help:      "Number of active ha connections",
-		})
-	prometheus.MustRegister(haConnections)
-
-	activeStreams := h2mux.NewActiveStreamsMetrics(metricsNamespace, tunnelSubsystem)
-
-	totalRequests := prometheus.NewCounter(
+var (
+	totalRequests = prometheus.NewCounter(
 		prometheus.CounterOpts{
-			Namespace: metricsNamespace,
-			Subsystem: tunnelSubsystem,
+			Namespace: connection.MetricsNamespace,
+			Subsystem: connection.TunnelSubsystem,
 			Name:      "total_requests",
 			Help:      "Amount of requests proxied through all the tunnels",
-		})
-	prometheus.MustRegister(totalRequests)
-
-	requestsPerTunnel := prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Namespace: metricsNamespace,
-			Subsystem: tunnelSubsystem,
-			Name:      "requests_per_tunnel",
-			Help:      "Amount of requests proxied through each tunnel",
 		},
-		[]string{"connection_id"},
 	)
-	prometheus.MustRegister(requestsPerTunnel)
-
-	concurrentRequestsPerTunnel := prometheus.NewGaugeVec(
+	concurrentRequests = prometheus.NewGauge(
 		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: tunnelSubsystem,
+			Namespace: connection.MetricsNamespace,
+			Subsystem: connection.TunnelSubsystem,
 			Name:      "concurrent_requests_per_tunnel",
 			Help:      "Concurrent requests proxied through each tunnel",
 		},
-		[]string{"connection_id"},
 	)
-	prometheus.MustRegister(concurrentRequestsPerTunnel)
-
-	maxConcurrentRequestsPerTunnel := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: tunnelSubsystem,
-			Name:      "max_concurrent_requests_per_tunnel",
-			Help:      "Largest number of concurrent requests proxied through each tunnel so far",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(maxConcurrentRequestsPerTunnel)
-
-	timerRetries := prometheus.NewGauge(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: tunnelSubsystem,
-			Name:      "timer_retries",
-			Help:      "Unacknowledged heart beats count",
-		})
-	prometheus.MustRegister(timerRetries)
-
-	responseByCode := prometheus.NewCounterVec(
+	responseByCode = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
-			Namespace: metricsNamespace,
-			Subsystem: tunnelSubsystem,
+			Namespace: connection.MetricsNamespace,
+			Subsystem: connection.TunnelSubsystem,
 			Name:      "response_by_code",
 			Help:      "Count of responses by HTTP status code",
 		},
 		[]string{"status_code"},
 	)
-	prometheus.MustRegister(responseByCode)
-
-	responseCodePerTunnel := prometheus.NewCounterVec(
+	requestErrors = prometheus.NewCounter(
 		prometheus.CounterOpts{
-			Namespace: metricsNamespace,
-			Subsystem: tunnelSubsystem,
-			Name:      "response_code_per_tunnel",
-			Help:      "Count of responses by HTTP status code fore each tunnel",
+			Namespace: connection.MetricsNamespace,
+			Subsystem: connection.TunnelSubsystem,
+			Name:      "request_errors",
+			Help:      "Count of error proxying to origin",
 		},
-		[]string{"connection_id", "status_code"},
 	)
-	prometheus.MustRegister(responseCodePerTunnel)
-
-	serverLocations := prometheus.NewGaugeVec(
+	haConnections = prometheus.NewGauge(
 		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: tunnelSubsystem,
-			Name:      "server_locations",
-			Help:      "Where each tunnel is connected to. 1 means current location, 0 means previous locations.",
+			Namespace: connection.MetricsNamespace,
+			Subsystem: connection.TunnelSubsystem,
+			Name:      "ha_connections",
+			Help:      "Number of active ha connections",
 		},
-		[]string{"connection_id", "location"},
 	)
-	prometheus.MustRegister(serverLocations)
+)
 
-	rpcFail := prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Namespace: metricsNamespace,
-			Subsystem: tunnelSubsystem,
-			Name:      "tunnel_rpc_fail",
-			Help:      "Count of RPC connection errors by type",
-		},
-		[]string{"error", "rpcName"},
+func init() {
+	prometheus.MustRegister(
+		totalRequests,
+		concurrentRequests,
+		responseByCode,
+		requestErrors,
+		haConnections,
 	)
-	prometheus.MustRegister(rpcFail)
-
-	registerFail := prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Namespace: metricsNamespace,
-			Subsystem: tunnelSubsystem,
-			Name:      "tunnel_register_fail",
-			Help:      "Count of tunnel registration errors by type",
-		},
-		[]string{"error", "rpcName"},
-	)
-	prometheus.MustRegister(registerFail)
-
-	userHostnamesCounts := prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Namespace: metricsNamespace,
-			Subsystem: tunnelSubsystem,
-			Name:      "user_hostnames_counts",
-			Help:      "Which user hostnames cloudflared is serving",
-		},
-		[]string{"userHostname"},
-	)
-	prometheus.MustRegister(userHostnamesCounts)
-
-	registerSuccess := prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Namespace: metricsNamespace,
-			Subsystem: tunnelSubsystem,
-			Name:      "tunnel_register_success",
-			Help:      "Count of successful tunnel registrations",
-		},
-		[]string{"rpcName"},
-	)
-	prometheus.MustRegister(registerSuccess)
-
-	return &TunnelMetrics{
-		haConnections:                  haConnections,
-		activeStreams:                  activeStreams,
-		totalRequests:                  totalRequests,
-		requestsPerTunnel:              requestsPerTunnel,
-		concurrentRequestsPerTunnel:    concurrentRequestsPerTunnel,
-		concurrentRequests:             make(map[string]uint64),
-		maxConcurrentRequestsPerTunnel: maxConcurrentRequestsPerTunnel,
-		maxConcurrentRequests:          make(map[string]uint64),
-		timerRetries:                   timerRetries,
-		responseByCode:                 responseByCode,
-		responseCodePerTunnel:          responseCodePerTunnel,
-		serverLocations:                serverLocations,
-		oldServerLocations:             make(map[string]string),
-		muxerMetrics:                   newMuxerMetrics(),
-		tunnelsHA:                      NewTunnelsForHA(),
-		regSuccess:                     registerSuccess,
-		regFail:                        registerFail,
-		rpcFail:                        rpcFail,
-		userHostnamesCounts:            userHostnamesCounts,
-	}
 }
 
-func (t *TunnelMetrics) incrementHaConnections() {
-	t.haConnections.Inc()
+func incrementRequests() {
+	totalRequests.Inc()
+	concurrentRequests.Inc()
 }
 
-func (t *TunnelMetrics) decrementHaConnections() {
-	t.haConnections.Dec()
-}
-
-func (t *TunnelMetrics) updateMuxerMetrics(connectionID string, metrics *h2mux.MuxerMetrics) {
-	t.muxerMetrics.update(connectionID, metrics)
-}
-
-func (t *TunnelMetrics) incrementRequests(connectionID string) {
-	t.concurrentRequestsLock.Lock()
-	var concurrentRequests uint64
-	var ok bool
-	if concurrentRequests, ok = t.concurrentRequests[connectionID]; ok {
-		t.concurrentRequests[connectionID]++
-		concurrentRequests++
-	} else {
-		t.concurrentRequests[connectionID] = 1
-		concurrentRequests = 1
-	}
-	if maxConcurrentRequests, ok := t.maxConcurrentRequests[connectionID]; (ok && maxConcurrentRequests < concurrentRequests) || !ok {
-		t.maxConcurrentRequests[connectionID] = concurrentRequests
-		t.maxConcurrentRequestsPerTunnel.WithLabelValues(connectionID).Set(float64(concurrentRequests))
-	}
-	t.concurrentRequestsLock.Unlock()
-
-	t.totalRequests.Inc()
-	t.requestsPerTunnel.WithLabelValues(connectionID).Inc()
-	t.concurrentRequestsPerTunnel.WithLabelValues(connectionID).Inc()
-}
-
-func (t *TunnelMetrics) decrementConcurrentRequests(connectionID string) {
-	t.concurrentRequestsLock.Lock()
-	if _, ok := t.concurrentRequests[connectionID]; ok {
-		t.concurrentRequests[connectionID]--
-	}
-	t.concurrentRequestsLock.Unlock()
-
-	t.concurrentRequestsPerTunnel.WithLabelValues(connectionID).Dec()
-}
-
-func (t *TunnelMetrics) incrementResponses(connectionID, code string) {
-	t.responseByCode.WithLabelValues(code).Inc()
-	t.responseCodePerTunnel.WithLabelValues(connectionID, code).Inc()
-
-}
-
-func (t *TunnelMetrics) registerServerLocation(connectionID, loc string) {
-	t.locationLock.Lock()
-	defer t.locationLock.Unlock()
-	if oldLoc, ok := t.oldServerLocations[connectionID]; ok && oldLoc == loc {
-		return
-	} else if ok {
-		t.serverLocations.WithLabelValues(connectionID, oldLoc).Dec()
-	}
-	t.serverLocations.WithLabelValues(connectionID, loc).Inc()
-	t.oldServerLocations[connectionID] = loc
+func decrementConcurrentRequests() {
+	concurrentRequests.Dec()
 }

origin/metrics_test.go

@@ -1,121 +0,0 @@
-package origin
-
-import (
-	"strconv"
-	"sync"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-// can only be called once
-var m = NewTunnelMetrics()
-
-func TestConcurrentRequestsSingleTunnel(t *testing.T) {
-	routines := 20
-	var wg sync.WaitGroup
-	wg.Add(routines)
-	for i := 0; i < routines; i++ {
-		go func() {
-			m.incrementRequests("0")
-			wg.Done()
-		}()
-	}
-	wg.Wait()
-	assert.Len(t, m.concurrentRequests, 1)
-	assert.Equal(t, uint64(routines), m.concurrentRequests["0"])
-	assert.Len(t, m.maxConcurrentRequests, 1)
-	assert.Equal(t, uint64(routines), m.maxConcurrentRequests["0"])
-
-	wg.Add(routines / 2)
-	for i := 0; i < routines/2; i++ {
-		go func() {
-			m.decrementConcurrentRequests("0")
-			wg.Done()
-		}()
-	}
-	wg.Wait()
-	assert.Equal(t, uint64(routines-routines/2), m.concurrentRequests["0"])
-	assert.Equal(t, uint64(routines), m.maxConcurrentRequests["0"])
-}
-
-func TestConcurrentRequestsMultiTunnel(t *testing.T) {
-	m.concurrentRequests = make(map[string]uint64)
-	m.maxConcurrentRequests = make(map[string]uint64)
-	tunnels := 20
-	var wg sync.WaitGroup
-	wg.Add(tunnels)
-	for i := 0; i < tunnels; i++ {
-		go func(i int) {
-			// if we have j < i, then tunnel 0 won't have a chance to call incrementRequests
-			for j := 0; j < i+1; j++ {
-				id := strconv.Itoa(i)
-				m.incrementRequests(id)
-			}
-			wg.Done()
-		}(i)
-	}
-	wg.Wait()
-
-	assert.Len(t, m.concurrentRequests, tunnels)
-	assert.Len(t, m.maxConcurrentRequests, tunnels)
-	for i := 0; i < tunnels; i++ {
-		id := strconv.Itoa(i)
-		assert.Equal(t, uint64(i+1), m.concurrentRequests[id])
-		assert.Equal(t, uint64(i+1), m.maxConcurrentRequests[id])
-	}
-
-	wg.Add(tunnels)
-	for i := 0; i < tunnels; i++ {
-		go func(i int) {
-			for j := 0; j < i+1; j++ {
-				id := strconv.Itoa(i)
-				m.decrementConcurrentRequests(id)
-			}
-			wg.Done()
-		}(i)
-	}
-	wg.Wait()
-
-	assert.Len(t, m.concurrentRequests, tunnels)
-	assert.Len(t, m.maxConcurrentRequests, tunnels)
-	for i := 0; i < tunnels; i++ {
-		id := strconv.Itoa(i)
-		assert.Equal(t, uint64(0), m.concurrentRequests[id])
-		assert.Equal(t, uint64(i+1), m.maxConcurrentRequests[id])
-	}
-
-}
-
-func TestRegisterServerLocation(t *testing.T) {
-	tunnels := 20
-	var wg sync.WaitGroup
-	wg.Add(tunnels)
-	for i := 0; i < tunnels; i++ {
-		go func(i int) {
-			id := strconv.Itoa(i)
-			m.registerServerLocation(id, "LHR")
-			wg.Done()
-		}(i)
-	}
-	wg.Wait()
-	for i := 0; i < tunnels; i++ {
-		id := strconv.Itoa(i)
-		assert.Equal(t, "LHR", m.oldServerLocations[id])
-	}
-
-	wg.Add(tunnels)
-	for i := 0; i < tunnels; i++ {
-		go func(i int) {
-			id := strconv.Itoa(i)
-			m.registerServerLocation(id, "AUS")
-			wg.Done()
-		}(i)
-	}
-	wg.Wait()
-	for i := 0; i < tunnels; i++ {
-		id := strconv.Itoa(i)
-		assert.Equal(t, "AUS", m.oldServerLocations[id])
-	}
-
-}

origin/proxy.go

@@ -0,0 +1,224 @@
+package origin
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"strconv"
+	"strings"
+
+	"github.com/cloudflare/cloudflared/buffer"
+	"github.com/cloudflare/cloudflared/connection"
+	"github.com/cloudflare/cloudflared/ingress"
+	"github.com/cloudflare/cloudflared/logger"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/cloudflare/cloudflared/websocket"
+	"github.com/pkg/errors"
+)
+
+const (
+	TagHeaderNamePrefix = "Cf-Warp-Tag-"
+)
+
+type client struct {
+	ingressRules ingress.Ingress
+	tags         []tunnelpogs.Tag
+	logger       logger.Service
+	bufferPool   *buffer.Pool
+}
+
+func NewClient(ingressRules ingress.Ingress, tags []tunnelpogs.Tag, logger logger.Service) connection.OriginClient {
+	return &client{
+		ingressRules: ingressRules,
+		tags:         tags,
+		logger:       logger,
+		bufferPool:   buffer.NewPool(512 * 1024),
+	}
+}
+
+func (c *client) Proxy(w connection.ResponseWriter, req *http.Request, isWebsocket bool) error {
+	incrementRequests()
+	defer decrementConcurrentRequests()
+
+	cfRay := findCfRayHeader(req)
+	lbProbe := isLBProbeRequest(req)
+
+	c.appendTagHeaders(req)
+	rule, ruleNum := c.ingressRules.FindMatchingRule(req.Host, req.URL.Path)
+	c.logRequest(req, cfRay, lbProbe, ruleNum)
+
+	var (
+		resp *http.Response
+		err  error
+	)
+	if isWebsocket {
+		resp, err = c.proxyWebsocket(w, req, rule)
+	} else {
+		resp, err = c.proxyHTTP(w, req, rule)
+	}
+	if err != nil {
+		c.logRequestError(err, cfRay, ruleNum)
+		w.WriteErrorResponse()
+		return err
+	}
+	c.logOriginResponse(resp, cfRay, lbProbe, ruleNum)
+	return nil
+}
+
+func (c *client) proxyHTTP(w connection.ResponseWriter, req *http.Request, rule *ingress.Rule) (*http.Response, error) {
+	// Support for WSGI Servers by switching transfer encoding from chunked to gzip/deflate
+	if rule.Config.DisableChunkedEncoding {
+		req.TransferEncoding = []string{"gzip", "deflate"}
+		cLength, err := strconv.Atoi(req.Header.Get("Content-Length"))
+		if err == nil {
+			req.ContentLength = int64(cLength)
+		}
+	}
+
+	// Request origin to keep connection alive to improve performance
+	req.Header.Set("Connection", "keep-alive")
+
+	if hostHeader := rule.Config.HTTPHostHeader; hostHeader != "" {
+		req.Header.Set("Host", hostHeader)
+		req.Host = hostHeader
+	}
+
+	resp, err := rule.Service.RoundTrip(req)
+	if err != nil {
+		return nil, errors.Wrap(err, "Error proxying request to origin")
+	}
+	defer resp.Body.Close()
+
+	err = w.WriteRespHeaders(resp)
+	if err != nil {
+		return nil, errors.Wrap(err, "Error writing response header")
+	}
+	if connection.IsServerSentEvent(resp.Header) {
+		c.logger.Debug("Detected Server-Side Events from Origin")
+		c.writeEventStream(w, resp.Body)
+	} else {
+		// Use CopyBuffer, because Copy only allocates a 32KiB buffer, and cross-stream
+		// compression generates dictionary on first write
+		buf := c.bufferPool.Get()
+		defer c.bufferPool.Put(buf)
+		io.CopyBuffer(w, resp.Body, buf)
+	}
+	return resp, nil
+}
+
+func (c *client) proxyWebsocket(w connection.ResponseWriter, req *http.Request, rule *ingress.Rule) (*http.Response, error) {
+	if hostHeader := rule.Config.HTTPHostHeader; hostHeader != "" {
+		req.Header.Set("Host", hostHeader)
+		req.Host = hostHeader
+	}
+
+	dialler, ok := rule.Service.(websocket.Dialler)
+	if !ok {
+		return nil, fmt.Errorf("Websockets aren't supported by the origin service '%s'", rule.Service)
+	}
+	conn, resp, err := websocket.ClientConnect(req, dialler)
+	if err != nil {
+		return nil, err
+	}
+
+	serveCtx, cancel := context.WithCancel(req.Context())
+	connClosedChan := make(chan struct{})
+	go func() {
+		// serveCtx is done if req is cancelled, or streamWebsocket returns
+		<-serveCtx.Done()
+		conn.Close()
+		close(connClosedChan)
+	}()
+
+	// Copy to/from stream to the undelying connection. Use the underlying
+	// connection because cloudflared doesn't operate on the message themselves
+	err = c.streamWebsocket(w, conn.UnderlyingConn(), resp)
+	cancel()
+
+	// We need to make sure conn is closed before returning, otherwise we might write to conn after Proxy returns
+	<-connClosedChan
+	return resp, err
+}
+
+func (c *client) streamWebsocket(w connection.ResponseWriter, conn net.Conn, resp *http.Response) error {
+	err := w.WriteRespHeaders(resp)
+	if err != nil {
+		return errors.Wrap(err, "Error writing websocket response header")
+	}
+	websocket.Stream(conn, w)
+	return nil
+}
+
+func (c *client) writeEventStream(w connection.ResponseWriter, respBody io.ReadCloser) {
+	reader := bufio.NewReader(respBody)
+	for {
+		line, err := reader.ReadBytes('\n')
+		if err != nil {
+			break
+		}
+		w.Write(line)
+	}
+}
+
+func (c *client) appendTagHeaders(r *http.Request) {
+	for _, tag := range c.tags {
+		r.Header.Add(TagHeaderNamePrefix+tag.Name, tag.Value)
+	}
+}
+
+func (c *client) logRequest(r *http.Request, cfRay string, lbProbe bool, ruleNum int) {
+	if cfRay != "" {
+		c.logger.Debugf("CF-RAY: %s %s %s %s", cfRay, r.Method, r.URL, r.Proto)
+	} else if lbProbe {
+		c.logger.Debugf("CF-RAY: %s Load Balancer health check %s %s %s", cfRay, r.Method, r.URL, r.Proto)
+	} else {
+		c.logger.Debugf("All requests should have a CF-RAY header. Please open a support ticket with Cloudflare. %s %s %s ", r.Method, r.URL, r.Proto)
+	}
+	c.logger.Debugf("CF-RAY: %s Request Headers %+v", cfRay, r.Header)
+	c.logger.Debugf("CF-RAY: %s Serving with ingress rule %d", cfRay, ruleNum)
+
+	if contentLen := r.ContentLength; contentLen == -1 {
+		c.logger.Debugf("CF-RAY: %s Request Content length unknown", cfRay)
+	} else {
+		c.logger.Debugf("CF-RAY: %s Request content length %d", cfRay, contentLen)
+	}
+}
+
+func (c *client) logOriginResponse(r *http.Response, cfRay string, lbProbe bool, ruleNum int) {
+	responseByCode.WithLabelValues(strconv.Itoa(r.StatusCode)).Inc()
+	if cfRay != "" {
+		c.logger.Debugf("CF-RAY: %s Status: %s served by ingress %d", cfRay, r.Status, ruleNum)
+	} else if lbProbe {
+		c.logger.Debugf("Response to Load Balancer health check %s", r.Status)
+	} else {
+		c.logger.Debugf("Status: %s served by ingress %d", r.Status, ruleNum)
+	}
+	c.logger.Debugf("CF-RAY: %s Response Headers %+v", cfRay, r.Header)
+
+	if contentLen := r.ContentLength; contentLen == -1 {
+		c.logger.Debugf("CF-RAY: %s Response content length unknown", cfRay)
+	} else {
+		c.logger.Debugf("CF-RAY: %s Response content length %d", cfRay, contentLen)
+	}
+}
+
+func (c *client) logRequestError(err error, cfRay string, ruleNum int) {
+	requestErrors.Inc()
+	if cfRay != "" {
+		c.logger.Errorf("CF-RAY: %s Proxying to ingress %d error: %v", cfRay, ruleNum, err)
+	} else {
+		c.logger.Errorf("Proxying to ingress %d error: %v", ruleNum, err)
+	}
+
+}
+
+func findCfRayHeader(req *http.Request) string {
+	return req.Header.Get("Cf-Ray")
+}
+
+func isLBProbeRequest(req *http.Request) bool {
+	return strings.HasPrefix(req.UserAgent(), lbProbeUserAgentPrefix)
+}

origin/proxy_test.go

@@ -0,0 +1,352 @@
+package origin
+
+import (
+	"bytes"
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/connection"
+	"github.com/cloudflare/cloudflared/hello"
+	"github.com/cloudflare/cloudflared/ingress"
+	"github.com/cloudflare/cloudflared/logger"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/urfave/cli/v2"
+
+	"github.com/gobwas/ws/wsutil"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+var (
+	testTags = []tunnelpogs.Tag(nil)
+)
+
+type mockHTTPRespWriter struct {
+	*httptest.ResponseRecorder
+}
+
+func newMockHTTPRespWriter() *mockHTTPRespWriter {
+	return &mockHTTPRespWriter{
+		httptest.NewRecorder(),
+	}
+}
+
+func (w *mockHTTPRespWriter) WriteRespHeaders(resp *http.Response) error {
+	w.WriteHeader(resp.StatusCode)
+	for header, val := range resp.Header {
+		w.Header()[header] = val
+	}
+	return nil
+}
+
+func (w *mockHTTPRespWriter) WriteErrorResponse() {
+	w.WriteHeader(http.StatusBadGateway)
+	w.Write([]byte("http response error"))
+}
+
+func (w *mockHTTPRespWriter) Read(data []byte) (int, error) {
+	return 0, fmt.Errorf("mockHTTPRespWriter doesn't implement io.Reader")
+}
+
+type mockWSRespWriter struct {
+	*mockHTTPRespWriter
+	writeNotification chan []byte
+	reader            io.Reader
+}
+
+func newMockWSRespWriter(reader io.Reader) *mockWSRespWriter {
+	return &mockWSRespWriter{
+		newMockHTTPRespWriter(),
+		make(chan []byte),
+		reader,
+	}
+}
+
+func (w *mockWSRespWriter) Write(data []byte) (int, error) {
+	w.writeNotification <- data
+	return len(data), nil
+}
+
+func (w *mockWSRespWriter) respBody() io.ReadWriter {
+	data := <-w.writeNotification
+	return bytes.NewBuffer(data)
+}
+
+func (w *mockWSRespWriter) Read(data []byte) (int, error) {
+	return w.reader.Read(data)
+}
+
+type mockSSERespWriter struct {
+	*mockHTTPRespWriter
+	writeNotification chan []byte
+}
+
+func newMockSSERespWriter() *mockSSERespWriter {
+	return &mockSSERespWriter{
+		newMockHTTPRespWriter(),
+		make(chan []byte),
+	}
+}
+
+func (w *mockSSERespWriter) Write(data []byte) (int, error) {
+	w.writeNotification <- data
+	return len(data), nil
+}
+
+func (w *mockSSERespWriter) ReadBytes() []byte {
+	return <-w.writeNotification
+}
+
+func TestProxySingleOrigin(t *testing.T) {
+	logger, err := logger.New()
+	require.NoError(t, err)
+
+	ctx, cancel := context.WithCancel(context.Background())
+
+	flagSet := flag.NewFlagSet(t.Name(), flag.PanicOnError)
+	flagSet.Bool("hello-world", true, "")
+
+	cliCtx := cli.NewContext(cli.NewApp(), flagSet, nil)
+	err = cliCtx.Set("hello-world", "true")
+	require.NoError(t, err)
+
+	allowURLFromArgs := false
+	ingressRule, err := ingress.NewSingleOrigin(cliCtx, allowURLFromArgs, logger)
+	require.NoError(t, err)
+
+	var wg sync.WaitGroup
+	errC := make(chan error)
+	ingressRule.StartOrigins(&wg, logger, ctx.Done(), errC)
+
+	client := NewClient(ingressRule, testTags, logger)
+	t.Run("testProxyHTTP", testProxyHTTP(t, client))
+	t.Run("testProxyWebsocket", testProxyWebsocket(t, client))
+	t.Run("testProxySSE", testProxySSE(t, client))
+	cancel()
+	wg.Wait()
+}
+
+func testProxyHTTP(t *testing.T, client connection.OriginClient) func(t *testing.T) {
+	return func(t *testing.T) {
+		respWriter := newMockHTTPRespWriter()
+		req, err := http.NewRequest(http.MethodGet, "http://localhost:8080", nil)
+		require.NoError(t, err)
+
+		err = client.Proxy(respWriter, req, false)
+		require.NoError(t, err)
+
+		assert.Equal(t, http.StatusOK, respWriter.Code)
+	}
+}
+
+func testProxyWebsocket(t *testing.T, client connection.OriginClient) func(t *testing.T) {
+	return func(t *testing.T) {
+		// WSRoute is a websocket echo handler
+		ctx, cancel := context.WithCancel(context.Background())
+		req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("http://localhost:8080%s", hello.WSRoute), nil)
+
+		readPipe, writePipe := io.Pipe()
+		respWriter := newMockWSRespWriter(readPipe)
+
+		var wg sync.WaitGroup
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			err = client.Proxy(respWriter, req, true)
+			require.NoError(t, err)
+
+			require.Equal(t, http.StatusSwitchingProtocols, respWriter.Code)
+		}()
+
+		msg := []byte("test websocket")
+		err = wsutil.WriteClientText(writePipe, msg)
+		require.NoError(t, err)
+
+		// ReadServerText reads next data message from rw, considering that caller represents client side.
+		returnedMsg, err := wsutil.ReadServerText(respWriter.respBody())
+		require.NoError(t, err)
+		require.Equal(t, msg, returnedMsg)
+
+		err = wsutil.WriteClientBinary(writePipe, msg)
+		require.NoError(t, err)
+
+		returnedMsg, err = wsutil.ReadServerBinary(respWriter.respBody())
+		require.NoError(t, err)
+		require.Equal(t, msg, returnedMsg)
+
+		cancel()
+		wg.Wait()
+	}
+}
+
+func testProxySSE(t *testing.T, client connection.OriginClient) func(t *testing.T) {
+	return func(t *testing.T) {
+		var (
+			pushCount = 50
+			pushFreq  = time.Duration(time.Millisecond * 10)
+		)
+		respWriter := newMockSSERespWriter()
+		ctx, cancel := context.WithCancel(context.Background())
+		req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("http://localhost:8080%s?freq=%s", hello.SSERoute, pushFreq), nil)
+		require.NoError(t, err)
+
+		var wg sync.WaitGroup
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			err = client.Proxy(respWriter, req, false)
+			require.NoError(t, err)
+
+			require.Equal(t, http.StatusOK, respWriter.Code)
+		}()
+
+		for i := 0; i < pushCount; i++ {
+			line := respWriter.ReadBytes()
+			expect := fmt.Sprintf("%d\n", i)
+			require.Equal(t, []byte(expect), line, fmt.Sprintf("Expect to read %v, got %v", expect, line))
+
+			line = respWriter.ReadBytes()
+			require.Equal(t, []byte("\n"), line, fmt.Sprintf("Expect to read '\n', got %v", line))
+		}
+
+		cancel()
+		wg.Wait()
+	}
+}
+
+func TestProxyMultipleOrigins(t *testing.T) {
+	api := httptest.NewServer(mockAPI{})
+	defer api.Close()
+
+	unvalidatedIngress := []config.UnvalidatedIngressRule{
+		{
+			Hostname: "api.example.com",
+			Service:  api.URL,
+		},
+		{
+			Hostname: "hello.example.com",
+			Service:  "hello-world",
+		},
+		{
+			Hostname: "health.example.com",
+			Path:     "/health",
+			Service:  "http_status:200",
+		},
+		{
+			Hostname: "*",
+			Service:  "http_status:404",
+		},
+	}
+
+	ingress, err := ingress.ParseIngress(&config.Configuration{
+		TunnelID: t.Name(),
+		Ingress:  unvalidatedIngress,
+	})
+	require.NoError(t, err)
+
+	logger, err := logger.New()
+	require.NoError(t, err)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	errC := make(chan error)
+	var wg sync.WaitGroup
+	ingress.StartOrigins(&wg, logger, ctx.Done(), errC)
+
+	client := NewClient(ingress, testTags, logger)
+
+	tests := []struct {
+		url            string
+		expectedStatus int
+		expectedBody   []byte
+	}{
+		{
+			url:            "http://api.example.com",
+			expectedStatus: http.StatusCreated,
+			expectedBody:   []byte("Created"),
+		},
+		{
+			url:            fmt.Sprintf("http://hello.example.com%s", hello.HealthRoute),
+			expectedStatus: http.StatusOK,
+			expectedBody:   []byte("ok"),
+		},
+		{
+			url:            "http://health.example.com/health",
+			expectedStatus: http.StatusOK,
+		},
+		{
+			url:            "http://health.example.com/",
+			expectedStatus: http.StatusNotFound,
+		},
+		{
+			url:            "http://not-found.example.com",
+			expectedStatus: http.StatusNotFound,
+		},
+	}
+
+	for _, test := range tests {
+		respWriter := newMockHTTPRespWriter()
+		req, err := http.NewRequest(http.MethodGet, test.url, nil)
+		require.NoError(t, err)
+
+		err = client.Proxy(respWriter, req, false)
+		require.NoError(t, err)
+
+		assert.Equal(t, test.expectedStatus, respWriter.Code)
+		if test.expectedBody != nil {
+			assert.Equal(t, test.expectedBody, respWriter.Body.Bytes())
+		} else {
+			assert.Equal(t, 0, respWriter.Body.Len())
+		}
+	}
+	cancel()
+	wg.Wait()
+}
+
+type mockAPI struct{}
+
+func (ma mockAPI) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	w.WriteHeader(http.StatusCreated)
+	w.Write([]byte("Created"))
+}
+
+type errorOriginTransport struct{}
+
+func (errorOriginTransport) RoundTrip(*http.Request) (*http.Response, error) {
+	return nil, fmt.Errorf("Proxy error")
+}
+
+func TestProxyError(t *testing.T) {
+	ingress := ingress.Ingress{
+		Rules: []ingress.Rule{
+			{
+				Hostname: "*",
+				Path:     nil,
+				Service: ingress.MockOriginService{
+					Transport: errorOriginTransport{},
+				},
+			},
+		},
+	}
+
+	logger, err := logger.New()
+	require.NoError(t, err)
+
+	client := NewClient(ingress, testTags, logger)
+
+	respWriter := newMockHTTPRespWriter()
+	req, err := http.NewRequest(http.MethodGet, "http://127.0.0.1", nil)
+	assert.NoError(t, err)
+
+	err = client.Proxy(respWriter, req, false)
+	assert.Error(t, err)
+	assert.Equal(t, http.StatusBadGateway, respWriter.Code)
+	assert.Equal(t, "http response error", respWriter.Body.String())
+}

origin/reconnect.go

@@ -7,11 +7,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/cloudflare/cloudflared/h2mux"
-	"github.com/cloudflare/cloudflared/logger"
-	"github.com/cloudflare/cloudflared/tunnelrpc"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
-	"github.com/google/uuid"
 	"github.com/prometheus/client_golang/prometheus"
 )
 
@@ -138,52 +134,3 @@ func (cm *reconnectCredentialManager) RefreshAuth(
 		return nil, err
 	}
 }
-
-func ReconnectTunnel(
-	ctx context.Context,
-	muxer *h2mux.Muxer,
-	config *TunnelConfig,
-	logger logger.Service,
-	connectionID uint8,
-	originLocalAddr string,
-	uuid uuid.UUID,
-	credentialManager *reconnectCredentialManager,
-) error {
-	token, err := credentialManager.ReconnectToken()
-	if err != nil {
-		return err
-	}
-	eventDigest, err := credentialManager.EventDigest(connectionID)
-	if err != nil {
-		return err
-	}
-	connDigest, err := credentialManager.ConnDigest(connectionID)
-	if err != nil {
-		return err
-	}
-
-	config.TransportLogger.Debug("initiating RPC stream to reconnect")
-	rpcClient, err := newTunnelRPCClient(ctx, muxer, config, reconnect)
-	if err != nil {
-		return err
-	}
-	defer rpcClient.Close()
-	// Request server info without blocking tunnel registration; must use capnp library directly.
-	serverInfoPromise := tunnelrpc.TunnelServer{Client: rpcClient.Client}.GetServerInfo(ctx, func(tunnelrpc.TunnelServer_getServerInfo_Params) error {
-		return nil
-	})
-	LogServerInfo(serverInfoPromise.Result(), connectionID, config.Metrics, logger, config.TunnelEventChan)
-	registration := rpcClient.ReconnectTunnel(
-		ctx,
-		token,
-		eventDigest,
-		connDigest,
-		config.Hostname,
-		config.RegistrationOptions(connectionID, originLocalAddr, uuid),
-	)
-	if registrationErr := registration.DeserializeError(); registrationErr != nil {
-		// ReconnectTunnel RPC failure
-		return processRegisterTunnelError(registrationErr, config.Metrics, reconnect)
-	}
-	return processRegistrationSuccess(config, logger, connectionID, registration, reconnect, credentialManager)
-}

origin/supervisor.go

@@ -8,7 +8,6 @@ import (
 
 	"github.com/google/uuid"
 
-	"github.com/cloudflare/cloudflared/buffer"
 	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/edgediscovery"
 	"github.com/cloudflare/cloudflared/h2mux"
@@ -18,10 +17,10 @@ import (
 )
 
 const (
+	// SRV and TXT record resolution TTL
+	ResolveTTL = time.Hour
 	// Waiting time before retrying a failed tunnel connection
 	tunnelRetryDuration = time.Second * 10
-	// SRV record resolution TTL
-	resolveTTL = time.Hour
 	// Interval between registering new tunnels
 	registrationInterval = time.Second
 
@@ -44,8 +43,6 @@ type Supervisor struct {
 	cloudflaredUUID   uuid.UUID
 	config            *TunnelConfig
 	edgeIPs           *edgediscovery.Edge
-	lastResolve       time.Time
-	resolverC         chan resolveResult
 	tunnelErrors      chan tunnelError
 	tunnelsConnecting map[int]chan struct{}
 	// nextConnectedIndex and nextConnectedSignal are used to wait for all
@@ -56,12 +53,7 @@ type Supervisor struct {
 	logger logger.Service
 
 	reconnectCredentialManager *reconnectCredentialManager
-
-	bufferPool *buffer.Pool
-}
-
-type resolveResult struct {
-	err error
+	useReconnectToken          bool
 }
 
 type tunnelError struct {
@@ -84,6 +76,11 @@ func NewSupervisor(config *TunnelConfig, cloudflaredUUID uuid.UUID) (*Supervisor
 		return nil, err
 	}
 
+	useReconnectToken := false
+	if config.ClassicTunnel != nil {
+		useReconnectToken = config.ClassicTunnel.UseReconnectToken
+	}
+
 	return &Supervisor{
 		cloudflaredUUID:            cloudflaredUUID,
 		config:                     config,
@@ -91,13 +88,12 @@ func NewSupervisor(config *TunnelConfig, cloudflaredUUID uuid.UUID) (*Supervisor
 		tunnelErrors:               make(chan tunnelError),
 		tunnelsConnecting:          map[int]chan struct{}{},
 		logger:                     config.Logger,
-		reconnectCredentialManager: newReconnectCredentialManager(metricsNamespace, tunnelSubsystem, config.HAConnections),
-		bufferPool:                 buffer.NewPool(512 * 1024),
+		reconnectCredentialManager: newReconnectCredentialManager(connection.MetricsNamespace, connection.TunnelSubsystem, config.HAConnections),
+		useReconnectToken:          useReconnectToken,
 	}, nil
 }
 
 func (s *Supervisor) Run(ctx context.Context, connectedSignal *signal.Signal, reconnectCh chan ReconnectSignal) error {
-	logger := s.config.Logger
 	if err := s.initialize(ctx, connectedSignal, reconnectCh); err != nil {
 		return err
 	}
@@ -110,11 +106,11 @@ func (s *Supervisor) Run(ctx context.Context, connectedSignal *signal.Signal, re
 	refreshAuthBackoff := &BackoffHandler{MaxRetries: refreshAuthMaxBackoff, BaseTime: refreshAuthRetryDuration, RetryForever: true}
 	var refreshAuthBackoffTimer <-chan time.Time
 
-	if s.config.UseReconnectToken {
+	if s.useReconnectToken {
 		if timer, err := s.reconnectCredentialManager.RefreshAuth(ctx, refreshAuthBackoff, s.authenticate); err == nil {
 			refreshAuthBackoffTimer = timer
 		} else {
-			logger.Errorf("supervisor: initial refreshAuth failed, retrying in %v: %s", refreshAuthRetryDuration, err)
+			s.logger.Errorf("supervisor: initial refreshAuth failed, retrying in %v: %s", refreshAuthRetryDuration, err)
 			refreshAuthBackoffTimer = time.After(refreshAuthRetryDuration)
 		}
 	}
@@ -133,7 +129,7 @@ func (s *Supervisor) Run(ctx context.Context, connectedSignal *signal.Signal, re
 		case tunnelError := <-s.tunnelErrors:
 			tunnelsActive--
 			if tunnelError.err != nil {
-				logger.Infof("supervisor: Tunnel disconnected due to error: %s", tunnelError.err)
+				s.logger.Infof("supervisor: Tunnel disconnected due to error: %s", tunnelError.err)
 				tunnelsWaiting = append(tunnelsWaiting, tunnelError.index)
 				s.waitForNextTunnel(tunnelError.index)
 
@@ -156,7 +152,7 @@ func (s *Supervisor) Run(ctx context.Context, connectedSignal *signal.Signal, re
 		case <-refreshAuthBackoffTimer:
 			newTimer, err := s.reconnectCredentialManager.RefreshAuth(ctx, refreshAuthBackoff, s.authenticate)
 			if err != nil {
-				logger.Errorf("supervisor: Authentication failed: %s", err)
+				s.logger.Errorf("supervisor: Authentication failed: %s", err)
 				// Permanent failure. Leave the `select` without setting the
 				// channel to be non-null, so we'll never hit this case of the `select` again.
 				continue
@@ -168,27 +164,15 @@ func (s *Supervisor) Run(ctx context.Context, connectedSignal *signal.Signal, re
 				// No more tunnels outstanding, clear backoff timer
 				backoff.SetGracePeriod()
 			}
-		// DNS resolution returned
-		case result := <-s.resolverC:
-			s.lastResolve = time.Now()
-			s.resolverC = nil
-			if result.err == nil {
-				logger.Debug("supervisor: Service discovery refresh complete")
-			} else {
-				logger.Errorf("supervisor: Service discovery error: %s", result.err)
-			}
 		}
 	}
 }
 
 // Returns nil if initialization succeeded, else the initialization error.
 func (s *Supervisor) initialize(ctx context.Context, connectedSignal *signal.Signal, reconnectCh chan ReconnectSignal) error {
-	logger := s.logger
-
-	s.lastResolve = time.Now()
 	availableAddrs := int(s.edgeIPs.AvailableAddrs())
 	if s.config.HAConnections > availableAddrs {
-		logger.Infof("You requested %d HA connections but I can give you at most %d.", s.config.HAConnections, availableAddrs)
+		s.logger.Infof("You requested %d HA connections but I can give you at most %d.", s.config.HAConnections, availableAddrs)
 		s.config.HAConnections = availableAddrs
 	}
 
@@ -227,7 +211,7 @@ func (s *Supervisor) startFirstTunnel(ctx context.Context, connectedSignal *sign
 		return
 	}
 
-	err = ServeTunnelLoop(ctx, s.reconnectCredentialManager, s.config, addr, firstConnIndex, connectedSignal, s.cloudflaredUUID, s.bufferPool, reconnectCh)
+	err = ServeTunnelLoop(ctx, s.reconnectCredentialManager, s.config, addr, firstConnIndex, connectedSignal, s.cloudflaredUUID, reconnectCh)
 	// If the first tunnel disconnects, keep restarting it.
 	edgeErrors := 0
 	for s.unusedIPs() {
@@ -239,7 +223,7 @@ func (s *Supervisor) startFirstTunnel(ctx context.Context, connectedSignal *sign
 			return
 		// try the next address if it was a dialError(network problem) or
 		// dupConnRegisterTunnelError
-		case connection.DialError, dupConnRegisterTunnelError:
+		case edgediscovery.DialError, connection.DupConnRegisterTunnelError:
 			edgeErrors++
 		default:
 			return
@@ -250,7 +234,7 @@ func (s *Supervisor) startFirstTunnel(ctx context.Context, connectedSignal *sign
 				return
 			}
 		}
-		err = ServeTunnelLoop(ctx, s.reconnectCredentialManager, s.config, addr, firstConnIndex, connectedSignal, s.cloudflaredUUID, s.bufferPool, reconnectCh)
+		err = ServeTunnelLoop(ctx, s.reconnectCredentialManager, s.config, addr, firstConnIndex, connectedSignal, s.cloudflaredUUID, reconnectCh)
 	}
 }
 
@@ -269,7 +253,7 @@ func (s *Supervisor) startTunnel(ctx context.Context, index int, connectedSignal
 	if err != nil {
 		return
 	}
-	err = ServeTunnelLoop(ctx, s.reconnectCredentialManager, s.config, addr, uint8(index), connectedSignal, s.cloudflaredUUID, s.bufferPool, reconnectCh)
+	err = ServeTunnelLoop(ctx, s.reconnectCredentialManager, s.config, addr, uint8(index), connectedSignal, s.cloudflaredUUID, reconnectCh)
 }
 
 func (s *Supervisor) newConnectedTunnelSignal(index int) *signal.Signal {
@@ -301,7 +285,7 @@ func (s *Supervisor) authenticate(ctx context.Context, numPreviousAttempts int)
 		return nil, err
 	}
 
-	edgeConn, err := connection.DialEdge(ctx, dialTimeout, s.config.TlsConfig, arbitraryEdgeIP)
+	edgeConn, err := edgediscovery.DialEdge(ctx, dialTimeout, s.config.EdgeTLSConfigs[connection.H2mux], arbitraryEdgeIP)
 	if err != nil {
 		return nil, err
 	}
@@ -311,8 +295,8 @@ func (s *Supervisor) authenticate(ctx context.Context, numPreviousAttempts int)
 		// This callback is invoked by h2mux when the edge initiates a stream.
 		return nil // noop
 	})
-	muxerConfig := s.config.muxerConfig(handler)
-	muxer, err := h2mux.Handshake(edgeConn, edgeConn, muxerConfig, s.config.Metrics.activeStreams)
+	muxerConfig := s.config.MuxerConfig.H2MuxerConfig(handler, s.logger)
+	muxer, err := h2mux.Handshake(edgeConn, edgeConn, *muxerConfig, h2mux.ActiveStreams)
 	if err != nil {
 		return nil, err
 	}
@@ -323,23 +307,15 @@ func (s *Supervisor) authenticate(ctx context.Context, numPreviousAttempts int)
 		<-muxer.Shutdown()
 	}()
 
-	rpcClient, err := newTunnelRPCClient(ctx, muxer, s.config, authenticate)
+	stream, err := muxer.OpenRPCStream(ctx)
 	if err != nil {
 		return nil, err
 	}
+	rpcClient := connection.NewTunnelServerClient(ctx, stream, s.logger)
 	defer rpcClient.Close()
 
 	const arbitraryConnectionID = uint8(0)
 	registrationOptions := s.config.RegistrationOptions(arbitraryConnectionID, edgeConn.LocalAddr().String(), s.cloudflaredUUID)
 	registrationOptions.NumPreviousAttempts = uint8(numPreviousAttempts)
-	authResponse, err := rpcClient.Authenticate(
-		ctx,
-		s.config.OriginCert,
-		s.config.Hostname,
-		registrationOptions,
-	)
-	if err != nil {
-		return nil, err
-	}
-	return authResponse.Outcome(), nil
+	return rpcClient.Authenticate(ctx, s.config.ClassicTunnel, registrationOptions)
 }

origin/tunnel_test.go

@@ -0,0 +1,90 @@
+package origin
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/cloudflare/cloudflared/connection"
+	"github.com/cloudflare/cloudflared/logger"
+	"github.com/stretchr/testify/assert"
+)
+
+type dynamicMockFetcher struct {
+	percentage int32
+	err        error
+}
+
+func (dmf *dynamicMockFetcher) fetch() connection.PercentageFetcher {
+	return func() (int32, error) {
+		if dmf.err != nil {
+			return 0, dmf.err
+		}
+		return dmf.percentage, nil
+	}
+}
+func TestWaitForBackoffFallback(t *testing.T) {
+	maxRetries := uint(3)
+	backoff := BackoffHandler{
+		MaxRetries: maxRetries,
+		BaseTime:   time.Millisecond * 10,
+	}
+	ctx := context.Background()
+	logger, err := logger.New()
+	assert.NoError(t, err)
+	resolveTTL := time.Duration(0)
+	namedTunnel := &connection.NamedTunnelConfig{
+		Credentials: connection.Credentials{
+			AccountTag: "test-account",
+		},
+	}
+	mockFetcher := dynamicMockFetcher{
+		percentage: 0,
+	}
+	protocolSelector, err := connection.NewProtocolSelector(connection.HTTP2.String(), namedTunnel, mockFetcher.fetch(), resolveTTL, logger)
+	assert.NoError(t, err)
+	config := &TunnelConfig{
+		Logger:           logger,
+		ProtocolSelector: protocolSelector,
+		Observer:         connection.NewObserver(nil, nil, false),
+	}
+	connIndex := uint8(1)
+
+	initProtocol := protocolSelector.Current()
+	assert.Equal(t, connection.HTTP2, initProtocol)
+
+	protocallFallback := &protocallFallback{
+		backoff,
+		initProtocol,
+		false,
+	}
+
+	// Retry #0 and #1. At retry #2, we switch protocol, so the fallback loop has one more retry than this
+	for i := 0; i < int(maxRetries-1); i++ {
+		err := waitForBackoff(ctx, protocallFallback, config, connIndex, fmt.Errorf("Some error"))
+		assert.NoError(t, err)
+		assert.Equal(t, initProtocol, protocallFallback.protocol)
+	}
+
+	// Retry fallback protocol
+	for i := 0; i < int(maxRetries); i++ {
+		err := waitForBackoff(ctx, protocallFallback, config, connIndex, fmt.Errorf("Some error"))
+		assert.NoError(t, err)
+		fallback, ok := protocolSelector.Fallback()
+		assert.True(t, ok)
+		assert.Equal(t, fallback, protocallFallback.protocol)
+	}
+
+	currentGlobalProtocol := protocolSelector.Current()
+	assert.Equal(t, initProtocol, currentGlobalProtocol)
+
+	// No protocol to fallback, return error
+	err = waitForBackoff(ctx, protocallFallback, config, connIndex, fmt.Errorf("Some error"))
+	assert.Error(t, err)
+
+	protocallFallback.reset()
+	err = waitForBackoff(ctx, protocallFallback, config, connIndex, fmt.Errorf("New error"))
+	assert.NoError(t, err)
+	assert.Equal(t, initProtocol, protocallFallback.protocol)
+}

sshgen/sshgen.go

@@ -52,7 +52,7 @@ var mockRequest func(url, contentType string, body io.Reader) (*http.Response, e
 
 // GenerateShortLivedCertificate generates and stores a keypair for short lived certs
 func GenerateShortLivedCertificate(appURL *url.URL, token string) error {
-	fullName, err := cfpath.GenerateFilePathFromURL(appURL, keyName)
+	fullName, err := cfpath.GenerateAppTokenFilePathFromURL(appURL, keyName)
 	if err != nil {
 		return err
 	}

sshgen/sshgen_test.go

@@ -35,7 +35,7 @@ func TestCertGenSuccess(t *testing.T) {
 	url, _ := url.Parse("https://cf-test-access.com/testpath")
 	token := tokenGenerator()
 
-	fullName, err := cfpath.GenerateFilePathFromURL(url, keyName)
+	fullName, err := cfpath.GenerateAppTokenFilePathFromURL(url, keyName)
 	assert.NoError(t, err)
 
 	pubKeyName := fullName + ".pub"

tlsconfig/certreloader.go

@@ -17,8 +17,6 @@ import (
 const (
 	OriginCAPoolFlag = "origin-ca-pool"
 	CaCertFlag       = "cacert"
-
-	edgeTLSServerName = "cftunnel.com"
 )
 
 // CertReloader can load and reload a TLS certificate from a particular filepath.
@@ -65,10 +63,9 @@ func (cr *CertReloader) LoadCert() error {
 	return nil
 }
 
-func LoadOriginCA(c *cli.Context, logger logger.Service) (*x509.CertPool, error) {
+func LoadOriginCA(originCAPoolFilename string, logger logger.Service) (*x509.CertPool, error) {
 	var originCustomCAPool []byte
 
-	originCAPoolFilename := c.String(OriginCAPoolFlag)
 	if originCAPoolFilename != "" {
 		var err error
 		originCustomCAPool, err = ioutil.ReadFile(originCAPoolFilename)
@@ -121,13 +118,13 @@ func LoadCustomOriginCA(originCAFilename string) (*x509.CertPool, error) {
 	return certPool, nil
 }
 
-func CreateTunnelConfig(c *cli.Context) (*tls.Config, error) {
+func CreateTunnelConfig(c *cli.Context, serverName string) (*tls.Config, error) {
 	var rootCAs []string
 	if c.String(CaCertFlag) != "" {
 		rootCAs = append(rootCAs, c.String(CaCertFlag))
 	}
 
-	userConfig := &TLSParameters{RootCAs: rootCAs, ServerName: edgeTLSServerName}
+	userConfig := &TLSParameters{RootCAs: rootCAs, ServerName: serverName}
 	tlsConfig, err := GetConfig(userConfig)
 	if err != nil {
 		return nil, err

tunneldns/tunnel.go

@@ -9,7 +9,6 @@ import (
 	"syscall"
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/metrics"
 
@@ -71,8 +70,7 @@ func Command(hidden bool) *cli.Command {
 
 // Run implements a foreground runner
 func Run(c *cli.Context) error {
-	logDirectory, logLevel := config.FindLogSettings()
-	logger, err := logger.New(logger.DefaultFile(logDirectory), logger.LogLevelString(logLevel))
+	logger, err := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
 	if err != nil {
 		return cliutil.PrintLoggerSetupError("error setting up logger", err)
 	}
@@ -82,7 +80,7 @@ func Run(c *cli.Context) error {
 		logger.Fatalf("Failed to open the metrics listener: %s", err)
 	}
 
-	go metrics.ServeMetrics(metricsListener, nil, logger)
+	go metrics.ServeMetrics(metricsListener, nil, nil, logger)
 
 	listener, err := CreateListener(c.String("address"), uint16(c.Uint("port")), c.StringSlice("upstream"), c.StringSlice("bootstrap"), logger)
 	if err != nil {

tunnelrpc/pogs/connectionrpc.go

@@ -93,6 +93,11 @@ type RegistrationServer_PogsClient struct {
 	Conn   *rpc.Conn
 }
 
+func (c RegistrationServer_PogsClient) Close() error {
+	c.Client.Close()
+	return c.Conn.Close()
+}
+
 func (c RegistrationServer_PogsClient) RegisterConnection(ctx context.Context, auth TunnelAuth, tunnelID uuid.UUID, connIndex byte, options *ConnectionOptions) (*ConnectionDetails, error) {
 	client := tunnelrpc.TunnelServer{Client: c.Client}
 	promise := client.RegisterConnection(ctx, func(p tunnelrpc.RegistrationServer_registerConnection_Params) error {

validation/validation.go

@@ -66,7 +66,15 @@ func ValidateHostname(hostname string) (string, error) {
 // but when it does not, the path is preserved:
 //   ValidateUrl("localhost:8080/api/") => "http://localhost:8080/api/"
 // This is arguably a bug, but changing it might break some cloudflared users.
-func ValidateUrl(originUrl string) (string, error) {
+func ValidateUrl(originUrl string) (*url.URL, error) {
+	urlStr, err := validateUrlString(originUrl)
+	if err != nil {
+		return nil, err
+	}
+	return url.Parse(urlStr)
+}
+
+func validateUrlString(originUrl string) (string, error) {
 	if originUrl == "" {
 		return "", fmt.Errorf("URL should not be empty")
 	}
@@ -157,6 +165,7 @@ func validateIP(scheme, host, port string) (string, error) {
 	return fmt.Sprintf("%s://%s", scheme, host), nil
 }
 
+// originURL shouldn't be a pointer, because this function might change the scheme
 func ValidateHTTPService(originURL string, hostname string, transport http.RoundTripper) error {
 	parsedURL, err := url.Parse(originURL)
 	if err != nil {
@@ -176,28 +185,32 @@ func ValidateHTTPService(originURL string, hostname string, transport http.Round
 		return err
 	}
 	initialRequest.Host = hostname
-	_, initialErr := client.Do(initialRequest)
-	if initialErr != nil {
-		// Attempt the same endpoint via the other protocol (http/https); maybe we have better luck?
-		oldScheme := parsedURL.Scheme
-		parsedURL.Scheme = toggleProtocol(parsedURL.Scheme)
+	resp, initialErr := client.Do(initialRequest)
+	if initialErr == nil {
+		resp.Body.Close()
+		return nil
+	}
 
-		secondRequest, err := http.NewRequest("GET", parsedURL.String(), nil)
-		if err != nil {
-			return err
-		}
-		secondRequest.Host = hostname
-		_, secondErr := client.Do(secondRequest)
-		if secondErr == nil { // Worked this time--advise the user to switch protocols
-			return errors.Errorf(
-				"%s doesn't seem to work over %s, but does seem to work over %s. Reason: %v. Consider changing the origin URL to %s",
-				parsedURL.Host,
-				oldScheme,
-				parsedURL.Scheme,
-				initialErr,
-				parsedURL,
-			)
-		}
+	// Attempt the same endpoint via the other protocol (http/https); maybe we have better luck?
+	oldScheme := parsedURL.Scheme
+	parsedURL.Scheme = toggleProtocol(oldScheme)
+
+	secondRequest, err := http.NewRequest("GET", parsedURL.String(), nil)
+	if err != nil {
+		return err
+	}
+	secondRequest.Host = hostname
+	resp, secondErr := client.Do(secondRequest)
+	if secondErr == nil { // Worked this time--advise the user to switch protocols
+		resp.Body.Close()
+		return errors.Errorf(
+			"%s doesn't seem to work over %s, but does seem to work over %s. Reason: %v. Consider changing the origin URL to %v",
+			parsedURL.Host,
+			oldScheme,
+			parsedURL.Scheme,
+			initialErr,
+			originURL,
+		)
 	}
 
 	return initialErr
@@ -220,12 +233,12 @@ type Access struct {
 }
 
 func NewAccessValidator(ctx context.Context, domain, issuer, applicationAUD string) (*Access, error) {
-	domainURL, err := ValidateUrl(domain)
+	domainURL, err := validateUrlString(domain)
 	if err != nil {
 		return nil, err
 	}
 
-	issuerURL, err := ValidateUrl(issuer)
+	issuerURL, err := validateUrlString(issuer)
 	if err != nil {
 		return nil, err
 	}

validation/validation_test.go

@@ -101,7 +101,7 @@ func TestValidateUrl(t *testing.T) {
 	for i, testCase := range testCases {
 		validUrl, err := ValidateUrl(testCase.input)
 		assert.NoError(t, err, "test case %v", i)
-		assert.Equal(t, testCase.expectedOutput, validUrl, "test case %v", i)
+		assert.Equal(t, testCase.expectedOutput, validUrl.String(), "test case %v", i)
 	}
 
 	validUrl, err := ValidateUrl("")
@@ -151,7 +151,7 @@ func TestValidateHTTPService_HTTP2HTTP(t *testing.T) {
 
 // Happy path 2: originURL is HTTPS, and HTTPS connections work
 func TestValidateHTTPService_HTTPS2HTTPS(t *testing.T) {
-	originURL := "https://127.0.0.1/"
+	originURL := "https://127.0.0.1:1234/"
 	hostname := "example.com"
 
 	assert.Nil(t, ValidateHTTPService(originURL, hostname, testRoundTripper(func(req *http.Request) (*http.Response, error) {

vendor/github.com/equinox-io/equinox/.travis.yml

@@ -1,13 +0,0 @@
-language: go
-go:
-  - "1.4"
-  - "1.5"
-  - "1.7"
-  - "1.9"
-  - "1.10"
-  - "1.11"
-  - "tip"
-matrix:
-  allow_failures:
-    - go: tip
-script: go test -v github.com/equinox-io/equinox github.com/equinox-io/equinox/proto

vendor/github.com/equinox-io/equinox/README.md

@@ -1,99 +0,0 @@
-# equinox client SDK [![godoc reference](https://godoc.org/github.com/equinox-io/equinox?status.png)](https://godoc.org/github.com/equinox-io/equinox)
-
-Package equinox allows applications to remotely update themselves with the [equinox.io](https://equinox.io) service.
-
-## Minimal Working Example
-
-```go
-import "github.com/equinox-io/equinox"
-
-const appID = "<YOUR EQUINOX APP ID>"
-
-var publicKey = []byte(`
------BEGIN PUBLIC KEY-----
-MFYwEAYHKoZIzj0CAQYFK4EEAAoDQgAEtrVmBxQvheRArXjg2vG1xIprWGuCyESx
-MMY8pjmjepSy2kuz+nl9aFLqmr+rDNdYvEBqQaZrYMc6k29gjvoQnQ==
------END PUBLIC KEY-----
-`)
-
-func update(channel string) error {
-    opts := equinox.Options{Channel: channel}
-    if err := opts.SetPublicKeyPEM(publicKey); err != nil {
-        return err
-    }
-
-    // check for the update
-    resp, err := equinox.Check(appID, opts)
-    switch {
-    case err == equinox.NotAvailableErr:
-        fmt.Println("No update available, already at the latest version!")
-        return nil
-    case err != nil:
-        return err
-    }
-
-    // fetch the update and apply it
-    err = resp.Apply()
-    if err != nil {
-        return err
-    }
-
-    fmt.Printf("Updated to new version: %s!\n", resp.ReleaseVersion)
-    return nil
-}
-```
-
-
-## Update To Specific Version
-
-When you specify a channel in the update options, equinox will try to update the application
-to the latest release of your application published to that channel. Instead, you may wish to
-update the application to a specific (possibly older) version. You can do this by explicitly setting
-Version in the Options struct:
-
-```go
-opts := equinox.Options{Version: "0.1.2"}
-```
-
-## Prompt For Update
-
-You may wish to ask the user for approval before updating to a new version. This is as simple
-as calling the Check function and only calling Apply on the returned result if the user approves.
-Example:
-
-```go
-// check for the update
-resp, err := equinox.Check(appID, opts)
-switch {
-case err == equinox.NotAvailableErr:
-    fmt.Println("No update available, already at the latest version!")
-    return nil
-case err != nil:
-    return err
-}
-
-fmt.Println("New version available!")
-fmt.Println("Version:", resp.ReleaseVersion)
-fmt.Println("Name:", resp.ReleaseTitle)
-fmt.Println("Details:", resp.ReleaseDescription)
-
-ok := prompt("Would you like to update?")
-
-if !ok {
-    return
-}
-
-err = resp.Apply()
-// ...
-```
-
-## Generating Keys
-
-All equinox releases must be signed with a private ECDSA key, and all updates verified with the
-public key portion. To do that, you'll need to generate a key pair. The equinox release tool can
-generate an ecdsa key pair for you easily:
-
-```shell
-equinox genkey
-```
-

vendor/github.com/equinox-io/equinox/doc.go

@@ -1,92 +0,0 @@
-/*
-Package equinox allows applications to remotely update themselves with the equinox.io service.
-
-Minimal Working Example
-
-	import "github.com/equinox-io/equinox"
-
-	const appID = "<YOUR EQUINOX APP ID>"
-
-	var publicKey = []byte(`
-	-----BEGIN PUBLIC KEY-----
-	MFYwEAYHKoZIzj0CAQYFK4EEAAoDQgAEtrVmBxQvheRArXjg2vG1xIprWGuCyESx
-	MMY8pjmjepSy2kuz+nl9aFLqmr+rDNdYvEBqQaZrYMc6k29gjvoQnQ==
-	-----END PUBLIC KEY-----
-	`)
-
-	func update(channel string) error {
-		opts := equinox.Options{Channel: channel}
-		if err := opts.SetPublicKeyPEM(publicKey); err != nil {
-			return err
-		}
-
-		// check for the update
-		resp, err := equinox.Check(appID, opts)
-		switch {
-		case err == equinox.NotAvailableErr:
-			fmt.Println("No update available, already at the latest version!")
-			return nil
-		case err != nil:
-			return err
-		}
-
-		// fetch the update and apply it
-		err = resp.Apply()
-		if err != nil {
-			return err
-		}
-
-		fmt.Printf("Updated to new version: %s!\n", resp.ReleaseVersion)
-		return nil
-	}
-
-
-Update To Specific Version
-
-When you specify a channel in the update options, equinox will try to update the application
-to the latest release of your application published to that channel. Instead, you may wish to
-update the application to a specific (possibly older) version. You can do this by explicitly setting
-Version in the Options struct:
-
-		opts := equinox.Options{Version: "0.1.2"}
-
-Prompt For Update
-
-You may wish to ask the user for approval before updating to a new version. This is as simple
-as calling the Check function and only calling Apply on the returned result if the user approves.
-Example:
-
-	// check for the update
-	resp, err := equinox.Check(appID, opts)
-	switch {
-	case err == equinox.NotAvailableErr:
-		fmt.Println("No update available, already at the latest version!")
-		return nil
-	case err != nil:
-		return err
-	}
-
-	fmt.Println("New version available!")
-	fmt.Println("Version:", resp.ReleaseVersion)
-	fmt.Println("Name:", resp.ReleaseTitle)
-	fmt.Println("Details:", resp.ReleaseDescription)
-
-	ok := prompt("Would you like to update?")
-
-	if !ok {
-		return
-	}
-
-	err = resp.Apply()
-	// ...
-
-Generating Keys
-
-All equinox releases must be signed with a private ECDSA key, and all updates verified with the
-public key portion. To do that, you'll need to generate a key pair. The equinox release tool can
-generate an ecdsa key pair for you easily:
-
-	equinox genkey
-
-*/
-package equinox

vendor/github.com/equinox-io/equinox/go.mod

@@ -1 +0,0 @@
-module github.com/equinox-io/equinox