Release 2020.12.0

Tunnel delete is successful even if we don't find the credentials
file in the user's filesystem. We no longer "error" indicating this
is a problem. This fix also enables chaining of the delete command
by removing a pre-mature return if the credentials file is not found.

Dockerfile

@@ -1,7 +1,7 @@
 # use a builder image for building cloudflare
 ARG TARGET_GOOS
 ARG TARGET_GOARCH
-FROM golang:1.13.3 as builder
+FROM golang:1.15.6 as builder
 ENV GO111MODULE=on \
     CGO_ENABLED=0 \
     TARGET_GOOS=${TARGET_GOOS} \

Makefile

@@ -89,6 +89,7 @@ define publish_package
 	for HOST in $(CF_PKG_HOSTS); do \
 		ssh-keyscan -t rsa $$HOST >> ~/.ssh/known_hosts; \
 		scp -4 cloudflared*.$(1) cfsync@$$HOST:/state/cf-pkg/staging/$(2)/$(TARGET_PUBLIC_REPO)/cloudflared/; \
+		ssh cfsync@$$HOST 'chmod g+w /state/cf-pkg/staging/$(2)/$(TARGET_PUBLIC_REPO)/cloudflared/*.$(1)'; \
 	done
 endef
 
@@ -104,7 +105,7 @@ define build_package
 	mkdir -p $(PACKAGE_DIR)
 	cp cloudflared $(PACKAGE_DIR)/cloudflared
 	cat cloudflared_man_template | sed -e 's/\$${VERSION}/$(VERSION)/; s/\$${DATE}/$(DATE)/' > $(PACKAGE_DIR)/cloudflared.1
-	fakeroot fpm -C $(PACKAGE_DIR) -s dir -t $(1) --$(1)-compression bzip2 \
+	fakeroot fpm -C $(PACKAGE_DIR) -s dir -t $(1) \
 		--description 'Cloudflare Argo tunnel daemon' \
 		--vendor 'Cloudflare' \
 		--license 'Cloudflare Service Agreement' \
@@ -225,4 +226,4 @@ vet:
 
 .PHONY: msi
 msi: cloudflared
-	go-msi make --msi cloudflared.msi --version $(MSI_VERSION)
+	go-msi make --msi cloudflared.msi --version $(MSI_VERSION)

README.md

@@ -1,9 +1,32 @@
 # Argo Tunnel client
 
-Contains the command-line client and its libraries for Argo Tunnel, a tunneling daemon that proxies any local webserver through the Cloudflare network.
+Contains the command-line client for Argo Tunnel, a tunneling daemon that proxies any local webserver through the Cloudflare network. Extensive documentation can be found in the [Argo Tunnel section](https://developers.cloudflare.com/argo-tunnel/) of the Cloudflare Docs.
 
-## Getting started
+## Before you get started
 
-    go install github.com/cloudflare/cloudflared/cmd/cloudflared
+Before you use Argo Tunnel, you'll need to complete a few steps in the Cloudflare dashboard. The website you add to Cloudflare will be used to route traffic to your Tunnel.
+
+1. [Add a website to Cloudflare](https://support.cloudflare.com/hc/en-us/articles/201720164-Creating-a-Cloudflare-account-and-adding-a-website)
+2. [Change your domain nameservers to Cloudflare](https://support.cloudflare.com/hc/en-us/articles/205195708)
+
+## Installing `cloudflared`
+
+Downloads are available as standalone binaries, a Docker image, and Debian, RPM, and Homebrew packages. You can also find releases here on the `cloudflared` GitHub repository.
+
+* You can [install on macOS](https://developers.cloudflare.com/argo-tunnel/getting-started/installation#macos) via Homebrew or by downloading the [latest Darwin amd64 release](https://github.com/cloudflare/cloudflared/releases)
+* Binaries, Debian, and RPM packages for Linux [can be found here](https://developers.cloudflare.com/argo-tunnel/getting-started/installation#linux)
+* A Docker image of `cloudflared` is [available on DockerHub](https://hub.docker.com/r/cloudflare/cloudflared)
+* You can install on Windows machines with the [steps here](https://developers.cloudflare.com/argo-tunnel/getting-started/installation#windows)
 
 User documentation for Argo Tunnel can be found at https://developers.cloudflare.com/argo-tunnel/
+
+## Creating Tunnels and routing traffic
+
+Once installed, you can authenticate `cloudflared` into your Cloudflare account and begin creating Tunnels that serve traffic for hostnames in your account.
+
+* Create a Tunnel with [these instructions](https://developers.cloudflare.com/argo-tunnel/create-tunnel)
+* Route traffic to that Tunnel with [DNS records in Cloudflare](https://developers.cloudflare.com/argo-tunnel/routing-to-tunnel/dns) or with a [Cloudflare Load Balancer](https://developers.cloudflare.com/argo-tunnel/routing-to-tunnel/lb)
+
+## TryCloudflare
+
+Want to test Argo Tunnel before adding a website to Cloudflare? You can do so with TryCloudflare using the documentation [available here](https://developers.cloudflare.com/argo-tunnel/learning/trycloudflare).

RELEASE_NOTES

@@ -1,3 +1,108 @@
+2020.12.0
+- 2020-12-04 TUN-3599: improved delete if credentials isnt found.
+- 2020-12-04 TUN-3612: Upgrade to Go 1.15.6
+- 2020-11-30 TUN-3593: /ready endpoint for k8s readiness. Move tunnel events out of UI package, into connection package.
+- 2020-11-27 TUN-3594: Log response status at debug level
+
+2020.11.11
+- 2020-11-20 TUN-3578: cloudflared tunnel route dns should allow wildcard subdomains
+- 2020-11-21 EDGEPLAT-2958 remove deb-compression, defaulting to gzip
+- 2020-11-23 TUN-3581: Tunnels can be run by name using only --credentials-file, no origin cert necessary.
+- 2020-11-15 TUN-3561: Unified logger configuration
+- 2020-11-08 AUTH-3221: Saves org token to disk and uses it to refresh the app token
+
+2020.11.10
+- 2020-11-20 TUN-3562: Fix panic when using bastion mode ingress rule
+- 2020-11-20 EDGEPLAT-2958 build cloudflared for Bullseye
+
+2020.11.9
+- 2020-11-18 TUN-3557: Detect SSE if content-type starts with text/event-stream
+- 2020-11-18 TUN-3559: Share response meta header with other packages
+- 2020-11-18 DEVTOOLS-7936: Remove redundant chgrp from publish
+- 2020-11-18 TUN-3558: cloudflared allows empty config files
+- 2020-11-18 TUN-3544: Upgrade to Go 1.15.5
+
+2020.11.8
+- 2020-11-17 TUN-3555: Single origin service should default to localhost:8080
+
+2020.11.7
+- 2020-11-13 TUN-3514: Stop setting --is-autoupdated flag after autoupdate because it can break named tunnel running in k8s
+- 2020-11-15 TUN-3548, TUN-3547: Bastion mode can be specified as a service, doesn't require URL.
+- 2020-11-16 TUN-3549: Use a separate handler for each websocket proxy
+
+2020.11.6
+- 2020-11-14 TUN-3546: Fix panic in tlsconfig.LoadOriginCA
+
+2020.11.5
+- 2020-11-12 TUN-3540: Better copy in ingress rules error messages
+- 2020-11-12 DEVTOOLS-7936: Set permissions on public packages
+- 2020-11-13 TUN-3543: ProxyAddress not using default in single-origin mode
+
+2020.11.4
+- 2020-11-11 TUN-3534: Specific error message when credentials file is a .pem not .json
+- 2020-11-02 TUN-3500: Integrate replace h2mux by http2 work with multiple origin support
+- 2020-11-09 TUN-3514: Transport logger write to UI when UI is enabled
+- 2020-10-30 TUN-3490: Make sure OriginClient implementation doesn't write after Proxy return
+- 2020-10-20 TUN-3403: Unit test for origin/proxy to test serving HTTP and Websocket
+- 2020-10-23 TUN-3480: Support SSE with http2 connection, and add SSE handler to hello-world server
+- 2020-10-27 TUN-3489: Add unit tests to cover proxy logic in connection package of cloudflared
+- 2020-10-16 TUN-3467: Serialize cf-cloudflared-response-meta during package initialization using jsoniter
+- 2020-10-14 TUN-3456: New protocol option auto to automatically select between http2 and h2mux
+- 2020-10-14 TUN-3458: Upgrade to http2 when available, fallback to h2mux when we reach max retries
+- 2020-10-08 TUN-3449: Use flag to select transport protocol implementation
+- 2020-10-08 TUN-3462: Refactor cloudflared to separate origin from connection
+- 2020-09-21 TUN-3406: Proxy websocket requests over Go http2
+- 2020-09-25 TUN-3420: Establish control plane and send RPC over control plane
+- 2020-09-11 TUN-3400: Use Go HTTP2 library as transport to connect with the edge
+
+2020.11.3
+- 2020-11-11 TUN-3533: Set config for single origin ingress
+
+2020.11.2
+
+
+2020.11.1
+- 2020-11-10 TUN-3527: More specific error for invalid YAML/JSON
+- 2020-11-06 Update README.md (#256)
+
+2020.11.0
+- 2020-11-04 TUN-3484: OriginService that responds with configured HTTP status
+- 2020-11-05 TUN-3505: Response body for status code origin returns EOF on Read
+- 2020-11-04 TUN-3503: Matching ingress rule should not take port into account
+- 2020-11-05 TUN-3506: OriginService needs to set request host and scheme for websocket requests
+- 2020-11-09 TUN-3516: Better error message when parsing invalid YAML config
+- 2020-11-09 TUN-3522: ingress validate checks that the config file exists
+- 2020-11-09 TUN-3524: Don't ignore errors from app-level action handler (#248)
+- 2020-11-09 TUN-3461: Show all origin services in the UI
+- 2020-10-30 TUN-3494: Proceed to create tunnel if at least one edge address can be resolved
+- 2020-10-30 TUN-3492: Refactor OriginService, shrink its interface
+- 2020-10-22 TUN-3478: Increase download timeout to 60s
+- 2020-10-15 TUN-2640: Users can configure per-origin config. Unify single-rule CLI flow with multi-rule config file code.
+
+2020.10.2
+- 2020-10-21 Release 2020.10.1
+- 2020-10-21 AUTH-3185 fixed indention error
+- 2020-10-19 TUN-3459: Make service install on linux use named tunnels
+
+2020.10.1
+- 2020-10-20 Split out typed config from legacy command-line switches; refactor ingress commands and fix tests
+- 2020-10-20 Move raw ingress rules to config package
+- 2020-10-21 TUN-3476: Fix conversion to string and int slice
+- 2020-10-12 TUN-3441:  Multiple-origin routing via ingress rules
+- 2020-10-15 TUN-3464: Newtype to wrap []ingress.Rule
+- 2020-10-15 TUN-3465: Use Go 1.15.3
+- 2020-10-15 TUN-3463: Let users run a named tunnel via config file setting
+- 2020-10-19 TUN-3475: Unify config file handling with typed config for new fields
+- 2020-10-19 TUN-3459: Make service install on linux use named tunnels
+- 2020-10-06 AUTH-3148 fixed cloudflared copy and match all the files in the checksum upload
+- 2020-10-06 TUN-3436, TUN-3437: Parse ingress from YAML, ensure last rule catches everything
+- 2020-10-06 TUN-3446: Use go 1.15.2 and add a step to build cloudflared in the dev Dockerfile
+- 2020-10-07 TUN-3439: 'tunnel validate' command to check ingress rules
+- 2020-10-07 TUN-3440: 'tunnel rule' command to test ingress rules
+- 2020-10-08 TUN-3451: Cloudflared tunnel ingress command
+- 2020-10-09 TUN-3452: Fix loading of flags from config file for tunnel run subcommand. This change also cleans up building of tunnel subcommand list, hides deprecated subcommands and improves help.
+- 2020-10-08 TUN-3438: move ingress into own package, read into TunnelConfig
+
 2020.10.0
 - 2020-10-02 AUTH-2993 cleaned up worker service tests
 - 2020-10-02 TUN-3443: Decode as v4api response on non-200 status

cfsetup.yaml

@@ -1,6 +1,6 @@
-pinned_go: &pinned_go go=1.15.2-1
+pinned_go: &pinned_go go=1.15.6-1
 build_dir: &build_dir /cfsetup_build
-default-flavor: stretch
+default-flavor: buster
 stretch: &stretch
   build:
     build_dir: *build_dir
@@ -239,10 +239,8 @@ stretch: &stretch
       - export GOARCH=amd64
       - make publish-cloudflared-junos
 
-jessie: *stretch
-
 buster: *stretch
-
+bullseye: *stretch
 centos-7:
   publish-rpm:
     build_dir: *build_dir
@@ -250,8 +248,8 @@ centos-7:
       - https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
     pre-cache:
       - yum install -y fakeroot
-      - wget https://golang.org/dl/go1.15.2.linux-amd64.tar.gz -P /tmp/
-      - tar -C /usr/local -xzf /tmp/go1.15.2.linux-amd64.tar.gz
+      - wget https://golang.org/dl/go1.15.6.linux-amd64.tar.gz -P /tmp/
+      - tar -C /usr/local -xzf /tmp/go1.15.6.linux-amd64.tar.gz
     post-cache:
       - export PATH=$PATH:/usr/local/go/bin
       - export GOOS=linux
@@ -262,8 +260,8 @@ centos-7:
     builddeps: *el7_builddeps
     pre-cache:
       - yum install -y fakeroot
-      - wget https://golang.org/dl/go1.15.2.linux-amd64.tar.gz -P /tmp/
-      - tar -C /usr/local -xzf /tmp/go1.15.2.linux-amd64.tar.gz
+      - wget https://golang.org/dl/go1.15.6.linux-amd64.tar.gz -P /tmp/
+      - tar -C /usr/local -xzf /tmp/go1.15.6.linux-amd64.tar.gz
     post-cache:
       - export PATH=$PATH:/usr/local/go/bin
       - export GOOS=linux

cmd/cloudflared/access/carrier.go

@@ -2,7 +2,6 @@ package access
 
 import (
 	"net/http"
-	"net/url"
 	"strings"
 
 	"github.com/cloudflare/cloudflared/carrier"
@@ -12,21 +11,16 @@ import (
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/validation"
 	"github.com/pkg/errors"
-	cli "github.com/urfave/cli/v2"
+	"github.com/urfave/cli/v2"
 )
 
 // StartForwarder starts a client side websocket forward
 func StartForwarder(forwarder config.Forwarder, shutdown <-chan struct{}, logger logger.Service) error {
-	validURLString, err := validation.ValidateUrl(forwarder.Listener)
+	validURL, err := validation.ValidateUrl(forwarder.Listener)
 	if err != nil {
 		return errors.Wrap(err, "error validating origin URL")
 	}
 
-	validURL, err := url.Parse(validURLString)
-	if err != nil {
-		return errors.Wrap(err, "error parsing origin URL")
-	}
-
 	// get the headers from the config file and add to the request
 	headers := make(http.Header)
 	if forwarder.TokenClientID != "" {
@@ -58,19 +52,7 @@ func StartForwarder(forwarder config.Forwarder, shutdown <-chan struct{}, logger
 // useful for proxying other protocols (like ssh) over websockets
 // (which you can put Access in front of)
 func ssh(c *cli.Context) error {
-	logDirectory, logLevel := config.FindLogSettings()
-
-	flagLogDirectory := c.String(sshLogDirectoryFlag)
-	if flagLogDirectory != "" {
-		logDirectory = flagLogDirectory
-	}
-
-	flagLogLevel := c.String(sshLogLevelFlag)
-	if flagLogLevel != "" {
-		logLevel = flagLogLevel
-	}
-
-	logger, err := logger.New(logger.DefaultFile(logDirectory), logger.LogLevelString(logLevel))
+	logger, err := logger.CreateSSHLoggerFromContext(c, logger.EnableTerminalLog)
 	if err != nil {
 		return cliutil.PrintLoggerSetupError("error setting up logger", err)
 	}
@@ -106,12 +88,7 @@ func ssh(c *cli.Context) error {
 	wsConn := carrier.NewWSConnection(logger, false)
 
 	if c.NArg() > 0 || c.IsSet(sshURLFlag) {
-		localForwarder, err := config.ValidateUrl(c, true)
-		if err != nil {
-			logger.Errorf("Error validating origin URL: %s", err)
-			return errors.Wrap(err, "error validating origin URL")
-		}
-		forwarder, err := url.Parse(localForwarder)
+		forwarder, err := config.ValidateUrl(c, true)
 		if err != nil {
 			logger.Errorf("Error validating origin URL: %s", err)
 			return errors.Wrap(err, "error validating origin URL")

cmd/cloudflared/access/cmd.go

@@ -25,16 +25,14 @@ import (
 )
 
 const (
-	sshHostnameFlag     = "hostname"
-	sshDestinationFlag  = "destination"
-	sshURLFlag          = "url"
-	sshHeaderFlag       = "header"
-	sshTokenIDFlag      = "service-token-id"
-	sshTokenSecretFlag  = "service-token-secret"
-	sshGenCertFlag      = "short-lived-cert"
-	sshLogDirectoryFlag = "log-directory"
-	sshLogLevelFlag     = "log-level"
-	sshConfigTemplate   = `
+	sshHostnameFlag    = "hostname"
+	sshDestinationFlag = "destination"
+	sshURLFlag         = "url"
+	sshHeaderFlag      = "header"
+	sshTokenIDFlag     = "service-token-id"
+	sshTokenSecretFlag = "service-token-secret"
+	sshGenCertFlag     = "short-lived-cert"
+	sshConfigTemplate  = `
 Add to your {{.Home}}/.ssh/config:
 
 Host {{.Hostname}}
@@ -157,12 +155,12 @@ func Commands() []*cli.Command {
 							Usage:   "specify an Access service token secret you wish to use.",
 						},
 						&cli.StringFlag{
-							Name:    sshLogDirectoryFlag,
+							Name:    logger.LogSSHDirectoryFlag,
 							Aliases: []string{"logfile"}, //added to match the tunnel side
 							Usage:   "Save application log to this directory for reporting issues.",
 						},
 						&cli.StringFlag{
-							Name:    sshLogLevelFlag,
+							Name:    logger.LogSSHLevelFlag,
 							Aliases: []string{"loglevel"}, //added to match the tunnel side
 							Usage:   "Application logging level {fatal, error, info, debug}. ",
 						},
@@ -207,7 +205,7 @@ func login(c *cli.Context) error {
 		return err
 	}
 
-	logger, err := logger.New()
+	logger, err := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
 	if err != nil {
 		return errors.Wrap(err, "error setting up logger")
 	}
@@ -224,7 +222,7 @@ func login(c *cli.Context) error {
 		return err
 	}
 
-	cfdToken, err := token.GetTokenIfExists(appURL)
+	cfdToken, err := token.GetAppTokenIfExists(appURL)
 	if err != nil {
 		fmt.Fprintln(os.Stderr, "Unable to find token for provided application.")
 		return err
@@ -252,7 +250,7 @@ func curl(c *cli.Context) error {
 	if err := raven.SetDSN(sentryDSN); err != nil {
 		return err
 	}
-	logger, err := logger.New()
+	logger, err := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
 	if err != nil {
 		return errors.Wrap(err, "error setting up logger")
 	}
@@ -269,7 +267,7 @@ func curl(c *cli.Context) error {
 		return err
 	}
 
-	tok, err := token.GetTokenIfExists(appURL)
+	tok, err := token.GetAppTokenIfExists(appURL)
 	if err != nil || tok == "" {
 		if allowRequest {
 			logger.Info("You don't have an Access token set. Please run access token <access application> to fetch one.")
@@ -297,7 +295,7 @@ func generateToken(c *cli.Context) error {
 		fmt.Fprintln(os.Stderr, "Please provide a url.")
 		return err
 	}
-	tok, err := token.GetTokenIfExists(appURL)
+	tok, err := token.GetAppTokenIfExists(appURL)
 	if err != nil || tok == "" {
 		fmt.Fprintln(os.Stderr, "Unable to find token for provided application. Please run token command to generate token.")
 		return err
@@ -331,7 +329,7 @@ func sshConfig(c *cli.Context) error {
 
 // sshGen generates a short lived certificate for provided hostname
 func sshGen(c *cli.Context) error {
-	logger, err := logger.New()
+	logger, err := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
 	if err != nil {
 		return errors.Wrap(err, "error setting up logger")
 	}

cmd/cloudflared/app_service.go

@@ -37,6 +37,7 @@ func (s *AppService) Run() error {
 func (s *AppService) Shutdown() error {
 	s.configManager.Shutdown()
 	s.shutdownC <- struct{}{}
+
 	return nil
 }
 

cmd/cloudflared/cliutil/errors.go

@@ -27,17 +27,17 @@ func UsageError(format string, args ...interface{}) error {
 // Ensures exit with error code if actionFunc returns an error
 func ErrorHandler(actionFunc cli.ActionFunc) cli.ActionFunc {
 	return func(ctx *cli.Context) error {
+		defer logger.SharedWriteManager.Shutdown()
+
 		err := actionFunc(ctx)
 		if err != nil {
 			if _, ok := err.(usageError); ok {
 				msg := fmt.Sprintf("%s\nSee 'cloudflared %s --help'.", err.Error(), ctx.Command.FullName())
-				return cli.Exit(msg, -1)
+				err = cli.Exit(msg, -1)
+			} else if _, ok := err.(cli.ExitCoder); !ok {
+				err = cli.Exit(err.Error(), 1)
 			}
-			// os.Exits with error code if err is cli.ExitCoder or cli.MultiError
-			cli.HandleExitCoder(err)
-			err = cli.Exit(err.Error(), 1)
 		}
-		logger.SharedWriteManager.Shutdown()
 		return err
 	}
 }

cmd/cloudflared/config/configuration.go

@@ -1,16 +1,20 @@
 package config
 
 import (
-	"errors"
+	"fmt"
+	"io"
+	"net/url"
 	"os"
 	"path/filepath"
 	"runtime"
+	"time"
 
 	homedir "github.com/mitchellh/go-homedir"
+	"github.com/pkg/errors"
 	"github.com/urfave/cli/v2"
-	"github.com/urfave/cli/v2/altsrc"
 	"gopkg.in/yaml.v2"
 
+	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/validation"
 )
 
@@ -28,9 +32,16 @@ var (
 	// Windows default config dir was ~/cloudflare-warp in documentation; let's keep it compatible
 	defaultUserConfigDirs = []string{"~/.cloudflared", "~/.cloudflare-warp", "~/cloudflare-warp"}
 	defaultNixConfigDirs  = []string{"/etc/cloudflared", DefaultUnixConfigLocation}
+
+	ErrNoConfigFile = fmt.Errorf("Cannot determine default configuration path. No file %v in %v", DefaultConfigFiles, DefaultConfigSearchDirectories())
 )
 
-const DefaultCredentialFile = "cert.pem"
+const (
+	DefaultCredentialFile = "cert.pem"
+
+	// BastionFlag is to enable bastion, or jump host, operation
+	BastionFlag = "bastion"
+)
 
 // DefaultConfigDirectory returns the default directory of the config file
 func DefaultConfigDirectory() string {
@@ -88,14 +99,6 @@ func FileExists(path string) (bool, error) {
 	return true, nil
 }
 
-// FindInputSourceContext pulls the input source from the config flag.
-func FindInputSourceContext(context *cli.Context) (altsrc.InputSourceContext, error) {
-	if context.String("config") != "" {
-		return altsrc.NewYamlSourceFromFile(context.String("config"))
-	}
-	return nil, nil
-}
-
 // FindDefaultConfigPath returns the first path that contains a config file.
 // If none of the combination of DefaultConfigSearchDirectories() and DefaultConfigFiles
 // contains a config file, return empty string.
@@ -148,35 +151,6 @@ func FindOrCreateConfigPath() string {
 	return path
 }
 
-// FindLogSettings gets the log directory and level from the config file
-func FindLogSettings() (string, string) {
-	configPath := FindOrCreateConfigPath()
-	defaultDirectory := DefaultLogDirectory()
-	defaultLevel := "info"
-
-	file, err := os.Open(configPath)
-	if err != nil {
-		return defaultDirectory, defaultLevel
-	}
-	defer file.Close()
-
-	var config Root
-	if err := yaml.NewDecoder(file).Decode(&config); err != nil {
-		return defaultDirectory, defaultLevel
-	}
-
-	directory := defaultDirectory
-	if config.LogDirectory != "" {
-		directory = config.LogDirectory
-	}
-
-	level := defaultLevel
-	if config.LogLevel != "" {
-		level = config.LogLevel
-	}
-	return directory, level
-}
-
 // ValidateUnixSocket ensures --unix-socket param is used exclusively
 // i.e. it fails if a user specifies both --url and --unix-socket
 func ValidateUnixSocket(c *cli.Context) (string, error) {
@@ -188,14 +162,214 @@ func ValidateUnixSocket(c *cli.Context) (string, error) {
 
 // ValidateUrl will validate url flag correctness. It can be either from --url or argument
 // Notice ValidateUnixSocket, it will enforce --unix-socket is not used with --url or argument
-func ValidateUrl(c *cli.Context, allowFromArgs bool) (string, error) {
+func ValidateUrl(c *cli.Context, allowURLFromArgs bool) (*url.URL, error) {
 	var url = c.String("url")
-	if allowFromArgs && c.NArg() > 0 {
+	if allowURLFromArgs && c.NArg() > 0 {
 		if c.IsSet("url") {
-			return "", errors.New("Specified origin urls using both --url and argument. Decide which one you want, I can only support one.")
+			return nil, errors.New("Specified origin urls using both --url and argument. Decide which one you want, I can only support one.")
 		}
 		url = c.Args().Get(0)
 	}
 	validUrl, err := validation.ValidateUrl(url)
 	return validUrl, err
 }
+
+type UnvalidatedIngressRule struct {
+	Hostname      string
+	Path          string
+	Service       string
+	OriginRequest OriginRequestConfig `yaml:"originRequest"`
+}
+
+// OriginRequestConfig is a set of optional fields that users may set to
+// customize how cloudflared sends requests to origin services. It is used to set
+// up general config that apply to all rules, and also, specific per-rule
+// config.
+// Note: To specify a time.Duration in go-yaml, use e.g. "3s" or "24h".
+type OriginRequestConfig struct {
+	// HTTP proxy timeout for establishing a new connection
+	ConnectTimeout *time.Duration `yaml:"connectTimeout"`
+	// HTTP proxy timeout for completing a TLS handshake
+	TLSTimeout *time.Duration `yaml:"tlsTimeout"`
+	// HTTP proxy TCP keepalive duration
+	TCPKeepAlive *time.Duration `yaml:"tcpKeepAlive"`
+	// HTTP proxy should disable "happy eyeballs" for IPv4/v6 fallback
+	NoHappyEyeballs *bool `yaml:"noHappyEyeballs"`
+	// HTTP proxy maximum keepalive connection pool size
+	KeepAliveConnections *int `yaml:"keepAliveConnections"`
+	// HTTP proxy timeout for closing an idle connection
+	KeepAliveTimeout *time.Duration `yaml:"keepAliveTimeout"`
+	// Sets the HTTP Host header for the local webserver.
+	HTTPHostHeader *string `yaml:"httpHostHeader"`
+	// Hostname on the origin server certificate.
+	OriginServerName *string `yaml:"originServerName"`
+	// Path to the CA for the certificate of your origin.
+	// This option should be used only if your certificate is not signed by Cloudflare.
+	CAPool *string `yaml:"caPool"`
+	// Disables TLS verification of the certificate presented by your origin.
+	// Will allow any certificate from the origin to be accepted.
+	// Note: The connection from your machine to Cloudflare's Edge is still encrypted.
+	NoTLSVerify *bool `yaml:"noTLSVerify"`
+	// Disables chunked transfer encoding.
+	// Useful if you are running a WSGI server.
+	DisableChunkedEncoding *bool `yaml:"disableChunkedEncoding"`
+	// Runs as jump host
+	BastionMode *bool `yaml:"bastionMode"`
+	// Listen address for the proxy.
+	ProxyAddress *string `yaml:"proxyAddress"`
+	// Listen port for the proxy.
+	ProxyPort *uint `yaml:"proxyPort"`
+	// Valid options are 'socks' or empty.
+	ProxyType *string `yaml:"proxyType"`
+}
+
+type Configuration struct {
+	TunnelID      string `yaml:"tunnel"`
+	Ingress       []UnvalidatedIngressRule
+	OriginRequest OriginRequestConfig `yaml:"originRequest"`
+	sourceFile    string
+}
+
+type configFileSettings struct {
+	Configuration `yaml:",inline"`
+	// older settings will be aggregated into the generic map, should be read via cli.Context
+	Settings map[string]interface{} `yaml:",inline"`
+}
+
+func (c *Configuration) Source() string {
+	return c.sourceFile
+}
+
+func (c *configFileSettings) Int(name string) (int, error) {
+	if raw, ok := c.Settings[name]; ok {
+		if v, ok := raw.(int); ok {
+			return v, nil
+		}
+		return 0, fmt.Errorf("expected int found %T for %s", raw, name)
+	}
+	return 0, nil
+}
+
+func (c *configFileSettings) Duration(name string) (time.Duration, error) {
+	if raw, ok := c.Settings[name]; ok {
+		switch v := raw.(type) {
+		case time.Duration:
+			return v, nil
+		case string:
+			return time.ParseDuration(v)
+		}
+		return 0, fmt.Errorf("expected duration found %T for %s", raw, name)
+	}
+	return 0, nil
+}
+
+func (c *configFileSettings) Float64(name string) (float64, error) {
+	if raw, ok := c.Settings[name]; ok {
+		if v, ok := raw.(float64); ok {
+			return v, nil
+		}
+		return 0, fmt.Errorf("expected float found %T for %s", raw, name)
+	}
+	return 0, nil
+}
+
+func (c *configFileSettings) String(name string) (string, error) {
+	if raw, ok := c.Settings[name]; ok {
+		if v, ok := raw.(string); ok {
+			return v, nil
+		}
+		return "", fmt.Errorf("expected string found %T for %s", raw, name)
+	}
+	return "", nil
+}
+
+func (c *configFileSettings) StringSlice(name string) ([]string, error) {
+	if raw, ok := c.Settings[name]; ok {
+		if slice, ok := raw.([]interface{}); ok {
+			strSlice := make([]string, len(slice))
+			for i, v := range slice {
+				str, ok := v.(string)
+				if !ok {
+					return nil, fmt.Errorf("expected string, found %T for %v", i, v)
+				}
+				strSlice[i] = str
+			}
+			return strSlice, nil
+		}
+		return nil, fmt.Errorf("expected string slice found %T for %s", raw, name)
+	}
+	return nil, nil
+}
+
+func (c *configFileSettings) IntSlice(name string) ([]int, error) {
+	if raw, ok := c.Settings[name]; ok {
+		if slice, ok := raw.([]interface{}); ok {
+			intSlice := make([]int, len(slice))
+			for i, v := range slice {
+				str, ok := v.(int)
+				if !ok {
+					return nil, fmt.Errorf("expected int, found %T for %v ", v, v)
+				}
+				intSlice[i] = str
+			}
+			return intSlice, nil
+		}
+		if v, ok := raw.([]int); ok {
+			return v, nil
+		}
+		return nil, fmt.Errorf("expected int slice found %T for %s", raw, name)
+	}
+	return nil, nil
+}
+
+func (c *configFileSettings) Generic(name string) (cli.Generic, error) {
+	return nil, errors.New("option type Generic not supported")
+}
+
+func (c *configFileSettings) Bool(name string) (bool, error) {
+	if raw, ok := c.Settings[name]; ok {
+		if v, ok := raw.(bool); ok {
+			return v, nil
+		}
+		return false, fmt.Errorf("expected boolean found %T for %s", raw, name)
+	}
+	return false, nil
+}
+
+var configuration configFileSettings
+
+func GetConfiguration() *Configuration {
+	return &configuration.Configuration
+}
+
+// ReadConfigFile returns InputSourceContext initialized from the configuration file.
+// On repeat calls returns with the same file, returns without reading the file again; however,
+// if value of "config" flag changes, will read the new config file
+func ReadConfigFile(c *cli.Context, log logger.Service) (*configFileSettings, error) {
+	configFile := c.String("config")
+	if configuration.Source() == configFile || configFile == "" {
+		if configuration.Source() == "" {
+			return nil, ErrNoConfigFile
+		}
+		return &configuration, nil
+	}
+
+	log.Debugf("Loading configuration from %s", configFile)
+	file, err := os.Open(configFile)
+	if err != nil {
+		if os.IsNotExist(err) {
+			err = ErrNoConfigFile
+		}
+		return nil, err
+	}
+	defer file.Close()
+	if err := yaml.NewDecoder(file).Decode(&configuration); err != nil {
+		if err == io.EOF {
+			log.Errorf("Configuration file %s was empty", configFile)
+			return &configuration, nil
+		}
+		return nil, errors.Wrap(err, "error parsing YAML in config file at "+configFile)
+	}
+	configuration.sourceFile = configFile
+	return &configuration, nil
+}

cmd/cloudflared/config/configuration_test.go

@@ -0,0 +1,76 @@
+package config
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"gopkg.in/yaml.v2"
+)
+
+func TestConfigFileSettings(t *testing.T) {
+	var (
+		firstIngress = UnvalidatedIngressRule{
+			Hostname: "tunnel1.example.com",
+			Path:     "/id",
+			Service:  "https://localhost:8000",
+		}
+		secondIngress = UnvalidatedIngressRule{
+			Hostname: "*",
+			Path:     "",
+			Service:  "https://localhost:8001",
+		}
+	)
+	rawYAML := `
+tunnel: config-file-test
+ingress:
+ - hostname: tunnel1.example.com
+   path: /id
+   service: https://localhost:8000
+ - hostname: "*"
+   service: https://localhost:8001
+retries: 5
+grace-period: 30s
+percentage: 3.14
+hostname: example.com
+tag:
+ - test
+ - central-1
+counters:
+ - 123
+ - 456
+`
+	var config configFileSettings
+	err := yaml.Unmarshal([]byte(rawYAML), &config)
+	assert.NoError(t, err)
+
+	assert.Equal(t, "config-file-test", config.TunnelID)
+	assert.Equal(t, firstIngress, config.Ingress[0])
+	assert.Equal(t, secondIngress, config.Ingress[1])
+
+	retries, err := config.Int("retries")
+	assert.NoError(t, err)
+	assert.Equal(t, 5, retries)
+
+	gracePeriod, err := config.Duration("grace-period")
+	assert.NoError(t, err)
+	assert.Equal(t, time.Second*30, gracePeriod)
+
+	percentage, err := config.Float64("percentage")
+	assert.NoError(t, err)
+	assert.Equal(t, 3.14, percentage)
+
+	hostname, err := config.String("hostname")
+	assert.NoError(t, err)
+	assert.Equal(t, "example.com", hostname)
+
+	tags, err := config.StringSlice("tag")
+	assert.NoError(t, err)
+	assert.Equal(t, "test", tags[0])
+	assert.Equal(t, "central-1", tags[1])
+
+	counters, err := config.IntSlice("counters")
+	assert.NoError(t, err)
+	assert.Equal(t, 123, counters[0])
+	assert.Equal(t, 456, counters[1])
+}

cmd/cloudflared/config/manager.go

@@ -1,11 +1,12 @@
 package config
 
 import (
-	"errors"
+	"io"
 	"os"
 
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/watcher"
+	"github.com/pkg/errors"
 	"gopkg.in/yaml.v2"
 )
 
@@ -27,7 +28,7 @@ type FileManager struct {
 	notifier   Notifier
 	configPath string
 	logger     logger.Service
-	ReadConfig func(string) (Root, error)
+	ReadConfig func(string, logger.Service) (Root, error)
 }
 
 // NewFileManager creates a config manager
@@ -59,7 +60,7 @@ func (m *FileManager) Start(notifier Notifier) error {
 
 // GetConfig reads the yaml file from the disk
 func (m *FileManager) GetConfig() (Root, error) {
-	return m.ReadConfig(m.configPath)
+	return m.ReadConfig(m.configPath, m.logger)
 }
 
 // Shutdown stops the watcher
@@ -67,7 +68,7 @@ func (m *FileManager) Shutdown() {
 	m.watcher.Shutdown()
 }
 
-func readConfigFromPath(configPath string) (Root, error) {
+func readConfigFromPath(configPath string, log logger.Service) (Root, error) {
 	if configPath == "" {
 		return Root{}, errors.New("unable to find config file")
 	}
@@ -80,7 +81,11 @@ func readConfigFromPath(configPath string) (Root, error) {
 
 	var config Root
 	if err := yaml.NewDecoder(file).Decode(&config); err != nil {
-		return Root{}, err
+		if err == io.EOF {
+			log.Errorf("Configuration file %s was empty", configPath)
+			return Root{}, nil
+		}
+		return Root{}, errors.Wrap(err, "error parsing YAML in config file at "+configPath)
 	}
 
 	return config, nil

cmd/cloudflared/config/manager_test.go

@@ -4,9 +4,10 @@ import (
 	"os"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
+
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/watcher"
-	"github.com/stretchr/testify/assert"
 )
 
 type mockNotifier struct {
@@ -56,7 +57,7 @@ func TestConfigChanged(t *testing.T) {
 			},
 		},
 	}
-	configRead := func(configPath string) (Root, error) {
+	configRead := func(configPath string, log logger.Service) (Root, error) {
 		return *c, nil
 	}
 	wait := make(chan struct{})

cmd/cloudflared/linux_service.go

@@ -7,11 +7,13 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
-	"github.com/cloudflare/cloudflared/logger"
 	"github.com/pkg/errors"
 	cli "github.com/urfave/cli/v2"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/tunnel"
+	"github.com/cloudflare/cloudflared/logger"
 )
 
 func runApp(app *cli.App, shutdownC, graceShutdownC chan struct{}) {
@@ -23,6 +25,12 @@ func runApp(app *cli.App, shutdownC, graceShutdownC chan struct{}) {
 				Name:   "install",
 				Usage:  "Install Argo Tunnel as a system service",
 				Action: cliutil.ErrorHandler(installLinuxService),
+				Flags: []cli.Flag{
+					&cli.BoolFlag{
+						Name:  "legacy",
+						Usage: "Generate service file for non-named tunnels",
+					},
+				},
 			},
 			&cli.Command{
 				Name:   "uninstall",
@@ -40,6 +48,7 @@ const (
 	serviceConfigDir      = "/etc/cloudflared"
 	serviceConfigFile     = "config.yml"
 	serviceCredentialFile = "cert.pem"
+	serviceConfigPath     = serviceConfigDir + "/" + serviceConfigFile
 )
 
 var systemdTemplates = []ServiceTemplate{
@@ -52,7 +61,7 @@ After=network.target
 [Service]
 TimeoutStartSec=0
 Type=notify
-ExecStart={{ .Path }} --config /etc/cloudflared/config.yml --origincert /etc/cloudflared/cert.pem --no-autoupdate
+ExecStart={{ .Path }} --config /etc/cloudflared/config.yml --no-autoupdate{{ range .ExtraArgs }} {{ . }}{{ end }}
 Restart=on-failure
 RestartSec=5s
 
@@ -102,7 +111,7 @@ var sysvTemplate = ServiceTemplate{
 # Description:       Argo Tunnel agent
 ### END INIT INFO
 name=$(basename $(readlink -f $0))
-cmd="{{.Path}} --config /etc/cloudflared/config.yml --origincert /etc/cloudflared/cert.pem --pidfile /var/run/$name.pid --autoupdate-freq 24h0m0s"
+cmd="{{.Path}} --config /etc/cloudflared/config.yml --pidfile /var/run/$name.pid --autoupdate-freq 24h0m0s{{ range .ExtraArgs }} {{ . }}{{ end }}"
 pid_file="/var/run/$name.pid"
 stdout_log="/var/log/$name.log"
 stderr_log="/var/log/$name.err"
@@ -182,10 +191,6 @@ func isSystemd() bool {
 }
 
 func copyUserConfiguration(userConfigDir, userConfigFile, userCredentialFile string, logger logger.Service) error {
-	if err := ensureConfigDirExists(serviceConfigDir); err != nil {
-		return err
-	}
-
 	srcCredentialPath := filepath.Join(userConfigDir, userCredentialFile)
 	destCredentialPath := filepath.Join(serviceConfigDir, serviceCredentialFile)
 	if srcCredentialPath != destCredentialPath {
@@ -207,7 +212,7 @@ func copyUserConfiguration(userConfigDir, userConfigFile, userCredentialFile str
 }
 
 func installLinuxService(c *cli.Context) error {
-	logger, err := logger.New()
+	logger, err := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
 	if err != nil {
 		return errors.Wrap(err, "error setting up logger")
 	}
@@ -216,23 +221,63 @@ func installLinuxService(c *cli.Context) error {
 	if err != nil {
 		return fmt.Errorf("error determining executable path: %v", err)
 	}
-	templateArgs := ServiceTemplateArgs{Path: etPath}
+	templateArgs := ServiceTemplateArgs{
+		Path: etPath,
+	}
 
-	userConfigDir := filepath.Dir(c.String("config"))
-	userConfigFile := filepath.Base(c.String("config"))
-	userCredentialFile := config.DefaultCredentialFile
-	if err = copyUserConfiguration(userConfigDir, userConfigFile, userCredentialFile, logger); err != nil {
-		logger.Errorf("Failed to copy user configuration: %s. Before running the service, ensure that %s contains two files, %s and %s", err,
-			serviceConfigDir, serviceCredentialFile, serviceConfigFile)
+	if err := ensureConfigDirExists(serviceConfigDir); err != nil {
 		return err
 	}
+	if c.Bool("legacy") {
+		userConfigDir := filepath.Dir(c.String("config"))
+		userConfigFile := filepath.Base(c.String("config"))
+		userCredentialFile := config.DefaultCredentialFile
+		if err = copyUserConfiguration(userConfigDir, userConfigFile, userCredentialFile, logger); err != nil {
+			logger.Errorf("Failed to copy user configuration: %s. Before running the service, ensure that %s contains two files, %s and %s", err,
+				serviceConfigDir, serviceCredentialFile, serviceConfigFile)
+			return err
+		}
+		templateArgs.ExtraArgs = []string{
+			"--origincert", serviceConfigDir + "/" + serviceCredentialFile,
+		}
+	} else {
+		src, err := config.ReadConfigFile(c, logger)
+		if err != nil {
+			return err
+		}
+
+		// can't use context because this command doesn't define "credentials-file" flag
+		configPresent := func(s string) bool {
+			val, err := src.String(s)
+			return err == nil && val != ""
+		}
+		if src.TunnelID == "" || !configPresent(tunnel.CredFileFlag) {
+			return fmt.Errorf(`Configuration file %s must contain entries for the tunnel to run and its associated credentials:
+tunnel: TUNNEL-UUID
+credentials-file: CREDENTIALS-FILE
+`, src.Source())
+		}
+		if src.Source() != serviceConfigPath {
+			if exists, err := config.FileExists(serviceConfigPath); err != nil || exists {
+				return fmt.Errorf("Possible conflicting configuration in %[1]s and %[2]s. Either remove %[2]s or run `cloudflared --config %[2]s service install`", src.Source(), serviceConfigPath)
+			}
+
+			if err := copyFile(src.Source(), serviceConfigPath); err != nil {
+				return fmt.Errorf("failed to copy %s to %s: %w", src.Source(), serviceConfigPath, err)
+			}
+		}
+
+		templateArgs.ExtraArgs = []string{
+			"tunnel", "run",
+		}
+	}
 
 	switch {
 	case isSystemd():
 		logger.Infof("Using Systemd")
 		return installSystemd(&templateArgs, logger)
 	default:
-		logger.Infof("Using Sysv")
+		logger.Infof("Using SysV")
 		return installSysv(&templateArgs, logger)
 	}
 }
@@ -281,7 +326,7 @@ func installSysv(templateArgs *ServiceTemplateArgs, logger logger.Service) error
 }
 
 func uninstallLinuxService(c *cli.Context) error {
-	logger, err := logger.New()
+	logger, err := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
 	if err != nil {
 		return errors.Wrap(err, "error setting up logger")
 	}
@@ -291,7 +336,7 @@ func uninstallLinuxService(c *cli.Context) error {
 		logger.Infof("Using Systemd")
 		return uninstallSystemd(logger)
 	default:
-		logger.Infof("Using Sysv")
+		logger.Infof("Using SysV")
 		return uninstallSysv(logger)
 	}
 }

cmd/cloudflared/macos_service.go

@@ -6,11 +6,11 @@ import (
 	"fmt"
 	"os"
 
-	"github.com/urfave/cli/v2"
-
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
 	"github.com/cloudflare/cloudflared/logger"
+
 	"github.com/pkg/errors"
+	"github.com/urfave/cli/v2"
 )
 
 const (
@@ -107,7 +107,7 @@ func stderrPath() (string, error) {
 }
 
 func installLaunchd(c *cli.Context) error {
-	logger, err := logger.New()
+	logger, err := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
 	if err != nil {
 		return errors.Wrap(err, "error setting up logger")
 	}
@@ -163,7 +163,7 @@ func installLaunchd(c *cli.Context) error {
 }
 
 func uninstallLaunchd(c *cli.Context) error {
-	logger, err := logger.New()
+	logger, err := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
 	if err != nil {
 		return errors.Wrap(err, "error setting up logger")
 	}

cmd/cloudflared/main.go

@@ -13,11 +13,11 @@ import (
 	log "github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/metrics"
 	"github.com/cloudflare/cloudflared/overwatch"
+	"github.com/cloudflare/cloudflared/tunneldns"
 	"github.com/cloudflare/cloudflared/watcher"
-
-	raven "github.com/getsentry/raven-go"
-	homedir "github.com/mitchellh/go-homedir"
-	cli "github.com/urfave/cli/v2"
+	"github.com/getsentry/raven-go"
+	"github.com/mitchellh/go-homedir"
+	"github.com/urfave/cli/v2"
 
 	"github.com/pkg/errors"
 )
@@ -76,7 +76,7 @@ func main() {
 	and configure access control.`
 	app.Flags = flags()
 	app.Action = action(Version, shutdownC, graceShutdownC)
-	app.Before = tunnel.Before
+	app.Before = tunnel.SetFlagsFromConfigFile
 	app.Commands = commands(cli.ShowVersion)
 
 	tunnel.Init(Version, shutdownC, graceShutdownC) // we need this to support the tunnel sub command...
@@ -129,6 +129,7 @@ To determine if an update happened in a script, check for error code 64.`,
 		},
 	}
 	cmds = append(cmds, tunnel.Commands()...)
+	cmds = append(cmds, tunneldns.Command(false))
 	cmds = append(cmds, access.Commands()...)
 	return cmds
 }
@@ -143,23 +144,19 @@ func isEmptyInvocation(c *cli.Context) bool {
 }
 
 func action(version string, shutdownC, graceShutdownC chan struct{}) cli.ActionFunc {
-	return func(c *cli.Context) (err error) {
+	return cliutil.ErrorHandler(func(c *cli.Context) (err error) {
 		if isEmptyInvocation(c) {
-			return handleServiceMode(shutdownC)
+			return handleServiceMode(c, shutdownC)
 		}
 		tags := make(map[string]string)
 		tags["hostname"] = c.String("hostname")
 		raven.SetTagsContext(tags)
 		raven.CapturePanic(func() { err = tunnel.TunnelCommand(c) }, nil)
-		exitCode := 0
 		if err != nil {
-			handleError(err)
-			exitCode = 1
+			captureError(err)
 		}
-		// we already handle error printing, so we pass an empty string so we
-		// don't have to print again.
-		return cli.Exit("", exitCode)
-	}
+		return err
+	})
 }
 
 func userHomeDir() (string, error) {
@@ -175,7 +172,7 @@ func userHomeDir() (string, error) {
 }
 
 // In order to keep the amount of noise sent to Sentry low, typical network errors can be filtered out here by a substring match.
-func handleError(err error) {
+func captureError(err error) {
 	errorMessage := err.Error()
 	for _, ignoredErrorMessage := range ignoredErrors {
 		if strings.Contains(errorMessage, ignoredErrorMessage) {
@@ -186,15 +183,13 @@ func handleError(err error) {
 }
 
 // cloudflared was started without any flags
-func handleServiceMode(shutdownC chan struct{}) error {
+func handleServiceMode(c *cli.Context, shutdownC chan struct{}) error {
 	defer log.SharedWriteManager.Shutdown()
-	logDirectory, logLevel := config.FindLogSettings()
 
-	logger, err := log.New(log.DefaultFile(logDirectory), log.LogLevelString(logLevel))
+	logger, err := log.CreateLoggerFromContext(c, log.DisableTerminalLog)
 	if err != nil {
 		return cliutil.PrintLoggerSetupError("error setting up logger", err)
 	}
-	logger.Infof("logging to directory: %s", logDirectory)
 
 	// start the main run loop that reads from the config file
 	f, err := watcher.NewFile()

cmd/cloudflared/path/path.go

@@ -11,8 +11,27 @@ import (
 	"github.com/mitchellh/go-homedir"
 )
 
-// GenerateFilePathFromURL will return a filepath for given access application url
-func GenerateFilePathFromURL(url *url.URL, suffix string) (string, error) {
+// GenerateAppTokenFilePathFromURL will return a filepath for given Access org token
+func GenerateAppTokenFilePathFromURL(url *url.URL, suffix string) (string, error) {
+	configPath, err := getConfigPath()
+	if err != nil {
+		return "", err
+	}
+	name := strings.Replace(fmt.Sprintf("%s%s-%s", url.Hostname(), url.EscapedPath(), suffix), "/", "-", -1)
+	return filepath.Join(configPath, name), nil
+}
+
+// GenerateOrgTokenFilePathFromURL will return a filepath for given Access application token
+func GenerateOrgTokenFilePathFromURL(authDomain string) (string, error) {
+	configPath, err := getConfigPath()
+	if err != nil {
+		return "", err
+	}
+	name := strings.Replace(fmt.Sprintf("%s-org-token", authDomain), "/", "-", -1)
+	return filepath.Join(configPath, name), nil
+}
+
+func getConfigPath() (string, error) {
 	configPath, err := homedir.Expand(config.DefaultConfigSearchDirectories()[0])
 	if err != nil {
 		return "", err
@@ -22,9 +41,5 @@ func GenerateFilePathFromURL(url *url.URL, suffix string) (string, error) {
 		// create config directory if doesn't already exist
 		err = os.Mkdir(configPath, 0700)
 	}
-	if err != nil {
-		return "", err
-	}
-	name := strings.Replace(fmt.Sprintf("%s%s-%s", url.Hostname(), url.EscapedPath(), suffix), "/", "-", -1)
-	return filepath.Join(configPath, name), nil
+	return configPath, err
 }

cmd/cloudflared/service_template.go

@@ -10,8 +10,9 @@ import (
 	"os/exec"
 	"text/template"
 
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
 	"github.com/mitchellh/go-homedir"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
 )
 
 type ServiceTemplate struct {
@@ -21,7 +22,8 @@ type ServiceTemplate struct {
 }
 
 type ServiceTemplateArgs struct {
-	Path string
+	Path      string
+	ExtraArgs []string
 }
 
 func (st *ServiceTemplate) ResolvePath() (string, error) {
@@ -139,6 +141,33 @@ func copyCredential(srcCredentialPath, destCredentialPath string) error {
 	return nil
 }
 
+func copyFile(src, dest string) error {
+	srcFile, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer srcFile.Close()
+
+	destFile, err := os.Create(dest)
+	if err != nil {
+		return err
+	}
+	ok := false
+	defer func() {
+		destFile.Close()
+		if !ok {
+			_ = os.Remove(dest)
+		}
+	}()
+
+	if _, err := io.Copy(destFile, srcFile); err != nil {
+		return err
+	}
+
+	ok = true
+	return nil
+}
+
 func copyConfig(srcConfigPath, destConfigPath string) error {
 	// Copy or create config
 	destFile, exists, err := openFile(destConfigPath, true)

cmd/cloudflared/token/token.go

@@ -5,9 +5,11 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"net/http"
 	"net/url"
 	"os"
 	"os/signal"
+	"strings"
 	"syscall"
 	"time"
 
@@ -17,10 +19,12 @@ import (
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/origin"
 	"github.com/coreos/go-oidc/jose"
+	"github.com/pkg/errors"
 )
 
 const (
-	keyName = "token"
+	keyName     = "token"
+	tokenHeader = "CF_Authorization"
 )
 
 type lock struct {
@@ -34,7 +38,7 @@ type signalHandler struct {
 	signals    []os.Signal
 }
 
-type jwtPayload struct {
+type appJWTPayload struct {
 	Aud   []string `json:"aud"`
 	Email string   `json:"email"`
 	Exp   int      `json:"exp"`
@@ -45,7 +49,17 @@ type jwtPayload struct {
 	Subt  string   `json:"sub"`
 }
 
-func (p jwtPayload) isExpired() bool {
+type orgJWTPayload struct {
+	appJWTPayload
+	Aud string `json:"aud"`
+}
+
+type transferServiceResponse struct {
+	AppToken string `json:"app_token"`
+	OrgToken string `json:"org_token"`
+}
+
+func (p appJWTPayload) isExpired() bool {
 	return int(time.Now().Unix()) > p.Exp
 }
 
@@ -141,55 +155,173 @@ func FetchToken(appURL *url.URL, logger logger.Service) (string, error) {
 
 // getToken will either load a stored token or generate a new one
 func getToken(appURL *url.URL, useHostOnly bool, logger logger.Service) (string, error) {
-	if token, err := GetTokenIfExists(appURL); token != "" && err == nil {
+	if token, err := GetAppTokenIfExists(appURL); token != "" && err == nil {
 		return token, nil
 	}
 
-	path, err := path.GenerateFilePathFromURL(appURL, keyName)
+	appTokenPath, err := path.GenerateAppTokenFilePathFromURL(appURL, keyName)
 	if err != nil {
-		return "", err
+		return "", errors.Wrap(err, "failed to generate app token file path")
 	}
 
-	fileLock := newLock(path)
-
-	err = fileLock.Acquire()
-	if err != nil {
-		return "", err
+	fileLockAppToken := newLock(appTokenPath)
+	if err = fileLockAppToken.Acquire(); err != nil {
+		return "", errors.Wrap(err, "failed to acquire app token lock")
 	}
-	defer fileLock.Release()
+	defer fileLockAppToken.Release()
 
 	// check to see if another process has gotten a token while we waited for the lock
-	if token, err := GetTokenIfExists(appURL); token != "" && err == nil {
+	if token, err := GetAppTokenIfExists(appURL); token != "" && err == nil {
 		return token, nil
 	}
 
+	// If an app token couldnt be found on disk, check for an org token and attempt to exchange it for an app token.
+	var orgTokenPath string
+	// Get auth domain to format into org token file path
+	if authDomain, err := getAuthDomain(appURL); err != nil {
+		logger.Errorf("failed to get auth domain: %s", err)
+	} else {
+		orgToken, err := GetOrgTokenIfExists(authDomain)
+		if err != nil {
+			orgTokenPath, err = path.GenerateOrgTokenFilePathFromURL(authDomain)
+			if err != nil {
+				return "", errors.Wrap(err, "failed to generate org token file path")
+			}
+
+			fileLockOrgToken := newLock(orgTokenPath)
+			if err = fileLockOrgToken.Acquire(); err != nil {
+				return "", errors.Wrap(err, "failed to acquire org token lock")
+			}
+			defer fileLockOrgToken.Release()
+			// check if an org token has been created since the lock was acquired
+			orgToken, err = GetOrgTokenIfExists(authDomain)
+		}
+		if err == nil {
+			if appToken, err := exchangeOrgToken(appURL, orgToken); err != nil {
+				logger.Debugf("failed to exchange org token for app token: %s", err)
+			} else {
+				if err := ioutil.WriteFile(appTokenPath, []byte(appToken), 0600); err != nil {
+					return "", errors.Wrap(err, "failed to write app token to disk")
+				}
+				return appToken, nil
+			}
+		}
+	}
+	return getTokensFromEdge(appURL, appTokenPath, orgTokenPath, useHostOnly, logger)
+
+}
+
+// getTokensFromEdge will attempt to use the transfer service to retrieve an app and org token, save them to disk,
+// and return the app token.
+func getTokensFromEdge(appURL *url.URL, appTokenPath, orgTokenPath string, useHostOnly bool, logger logger.Service) (string, error) {
+	// If no org token exists or if it couldnt be exchanged for an app token, then run the transfer service flow.
+
 	// this weird parameter is the resource name (token) and the key/value
 	// we want to send to the transfer service. the key is token and the value
 	// is blank (basically just the id generated in the transfer service)
-	token, err := transfer.Run(appURL, keyName, keyName, "", path, true, useHostOnly, logger)
+	resourceData, err := transfer.Run(appURL, keyName, keyName, "", true, useHostOnly, logger)
 	if err != nil {
-		return "", err
+		return "", errors.Wrap(err, "failed to run transfer service")
+	}
+	var resp transferServiceResponse
+	if err = json.Unmarshal(resourceData, &resp); err != nil {
+		return "", errors.Wrap(err, "failed to marshal transfer service response")
 	}
 
-	return string(token), nil
+	// If we were able to get the auth domain and generate an org token path, lets write it to disk.
+	if orgTokenPath != "" {
+		if err := ioutil.WriteFile(orgTokenPath, []byte(resp.OrgToken), 0600); err != nil {
+			return "", errors.Wrap(err, "failed to write org token to disk")
+		}
+	}
+
+	if err := ioutil.WriteFile(appTokenPath, []byte(resp.AppToken), 0600); err != nil {
+		return "", errors.Wrap(err, "failed to write app token to disk")
+	}
+
+	return resp.AppToken, nil
+
 }
 
-// GetTokenIfExists will return the token from local storage if it exists and not expired
-func GetTokenIfExists(url *url.URL) (string, error) {
-	path, err := path.GenerateFilePathFromURL(url, keyName)
-	if err != nil {
-		return "", err
-	}
-	content, err := ioutil.ReadFile(path)
-	if err != nil {
-		return "", err
-	}
-	token, err := jose.ParseJWT(string(content))
-	if err != nil {
-		return "", err
+// getAuthDomain makes a request to the appURL and stops at the first redirect. The 302 location header will contain the
+// auth domain
+func getAuthDomain(appURL *url.URL) (string, error) {
+	client := &http.Client{
+		// do not follow redirects
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			return http.ErrUseLastResponse
+		},
+		Timeout: time.Second * 7,
 	}
 
-	var payload jwtPayload
+	authDomainReq, err := http.NewRequest("HEAD", appURL.String(), nil)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to create auth domain request")
+	}
+	resp, err := client.Do(authDomainReq)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to get auth domain")
+	}
+	resp.Body.Close()
+	location, err := resp.Location()
+	if err != nil {
+		return "", fmt.Errorf("failed to get auth domain. Received status code %d from %s", resp.StatusCode, appURL.String())
+	}
+	return location.Hostname(), nil
+
+}
+
+// exchangeOrgToken attaches an org token to a request to the appURL and returns an app token. This uses the Access SSO
+// flow to automatically generate and return an app token without the login page.
+func exchangeOrgToken(appURL *url.URL, orgToken string) (string, error) {
+	client := &http.Client{
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			// attach org token to login request
+			if strings.Contains(req.URL.Path, "cdn-cgi/access/login") {
+				req.AddCookie(&http.Cookie{Name: tokenHeader, Value: orgToken})
+			}
+			// stop after hitting authorized endpoint since it will contain the app token
+			if strings.Contains(via[len(via)-1].URL.Path, "cdn-cgi/access/authorized") {
+				return http.ErrUseLastResponse
+			}
+			return nil
+		},
+		Timeout: time.Second * 7,
+	}
+
+	appTokenRequest, err := http.NewRequest("HEAD", appURL.String(), nil)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to create app token request")
+	}
+	resp, err := client.Do(appTokenRequest)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to get app token")
+	}
+	resp.Body.Close()
+	var appToken string
+	for _, c := range resp.Cookies() {
+		if c.Name == tokenHeader {
+			appToken = c.Value
+			break
+		}
+	}
+
+	if len(appToken) > 0 {
+		return appToken, nil
+	}
+	return "", fmt.Errorf("response from %s did not contain app token", resp.Request.URL.String())
+}
+
+func GetOrgTokenIfExists(authDomain string) (string, error) {
+	path, err := path.GenerateOrgTokenFilePathFromURL(authDomain)
+	if err != nil {
+		return "", err
+	}
+	token, err := getTokenIfExists(path)
+	if err != nil {
+		return "", err
+	}
+	var payload orgJWTPayload
 	err = json.Unmarshal(token.Payload, &payload)
 	if err != nil {
 		return "", err
@@ -199,13 +331,49 @@ func GetTokenIfExists(url *url.URL) (string, error) {
 		err := os.Remove(path)
 		return "", err
 	}
-
 	return token.Encode(), nil
 }
 
+func GetAppTokenIfExists(url *url.URL) (string, error) {
+	path, err := path.GenerateAppTokenFilePathFromURL(url, keyName)
+	if err != nil {
+		return "", err
+	}
+	token, err := getTokenIfExists(path)
+	if err != nil {
+		return "", err
+	}
+	var payload appJWTPayload
+	err = json.Unmarshal(token.Payload, &payload)
+	if err != nil {
+		return "", err
+	}
+
+	if payload.isExpired() {
+		err := os.Remove(path)
+		return "", err
+	}
+	return token.Encode(), nil
+
+}
+
+// GetTokenIfExists will return the token from local storage if it exists and not expired
+func getTokenIfExists(path string) (*jose.JWT, error) {
+	content, err := ioutil.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	token, err := jose.ParseJWT(string(content))
+	if err != nil {
+		return nil, err
+	}
+
+	return &token, nil
+}
+
 // RemoveTokenIfExists removes the a token from local storage if it exists
 func RemoveTokenIfExists(url *url.URL) error {
-	path, err := path.GenerateFilePathFromURL(url, keyName)
+	path, err := path.GenerateAppTokenFilePathFromURL(url, keyName)
 	if err != nil {
 		return err
 	}

cmd/cloudflared/transfer/transfer.go

@@ -3,10 +3,8 @@ package transfer
 import (
 	"bytes"
 	"encoding/base64"
-	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"net/url"
 	"os"
@@ -15,6 +13,7 @@ import (
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/encrypter"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/shell"
 	"github.com/cloudflare/cloudflared/logger"
+	"github.com/pkg/errors"
 )
 
 const (
@@ -28,7 +27,7 @@ const (
 // The "dance" we refer to is building a HTTP request, opening that in a browser waiting for
 // the user to complete an action, while it long polls in the background waiting for an
 // action to be completed to download the resource.
-func Run(transferURL *url.URL, resourceName, key, value, path string, shouldEncrypt bool, useHostOnly bool, logger logger.Service) ([]byte, error) {
+func Run(transferURL *url.URL, resourceName, key, value string, shouldEncrypt bool, useHostOnly bool, logger logger.Service) ([]byte, error) {
 	encrypterClient, err := encrypter.New("cloudflared_priv.pem", "cloudflared_pub.pem")
 	if err != nil {
 		return nil, err
@@ -72,11 +71,8 @@ func Run(transferURL *url.URL, resourceName, key, value, path string, shouldEncr
 		resourceData = buf
 	}
 
-	if err := ioutil.WriteFile(path, resourceData, 0600); err != nil {
-		return nil, err
-	}
-
 	return resourceData, nil
+
 }
 
 // BuildRequestURL creates a request suitable for a resource transfer.
@@ -93,6 +89,7 @@ func buildRequestURL(baseURL *url.URL, key, value string, cli, useHostOnly bool)
 		return baseURL.String(), nil
 	}
 	q.Set("redirect_url", baseURL.String()) // we add the token as a query param on both the redirect_url and the main url
+	q.Set("send_org_token", "true")         // indicates that the cli endpoint should return both the org and app token
 	baseURL.RawQuery = q.Encode()           // and this actual baseURL.
 	baseURL.Path = "cdn-cgi/access/cli"
 	return baseURL.String(), nil

cmd/cloudflared/tunnel/cmd.go

@@ -5,43 +5,33 @@ import (
 	"context"
 	"fmt"
 	"io/ioutil"
-	"net"
-	"net/http"
 	"net/url"
 	"os"
 	"reflect"
-	"runtime"
 	"runtime/trace"
-	"strconv"
 	"strings"
 	"sync"
 	"time"
 
-	"github.com/cloudflare/cloudflared/awsuploader"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/ui"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/updater"
+	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/dbconnect"
-	"github.com/cloudflare/cloudflared/h2mux"
-	"github.com/cloudflare/cloudflared/hello"
+	"github.com/cloudflare/cloudflared/ingress"
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/metrics"
 	"github.com/cloudflare/cloudflared/origin"
 	"github.com/cloudflare/cloudflared/signal"
-	"github.com/cloudflare/cloudflared/socks"
-	"github.com/cloudflare/cloudflared/sshlog"
-	"github.com/cloudflare/cloudflared/sshserver"
 	"github.com/cloudflare/cloudflared/tlsconfig"
 	"github.com/cloudflare/cloudflared/tunneldns"
 	"github.com/cloudflare/cloudflared/tunnelstore"
-	"github.com/cloudflare/cloudflared/websocket"
 
 	"github.com/coreos/go-systemd/daemon"
 	"github.com/facebookgo/grace/gracenet"
 	"github.com/getsentry/raven-go"
-	"github.com/gliderlabs/ssh"
 	"github.com/google/uuid"
 	"github.com/mitchellh/go-homedir"
 	"github.com/pkg/errors"
@@ -84,20 +74,9 @@ const (
 	// hostKeyPath is the path of the dir to save SSH host keys too
 	hostKeyPath = "host-key-path"
 
-	//sshServerFlag enables cloudflared ssh proxy server
-	sshServerFlag = "ssh-server"
-
-	// socks5Flag is to enable the socks server to deframe
-	socks5Flag = "socks5"
-
-	// bastionFlag is to enable bastion, or jump host, operation
-	bastionFlag = "bastion"
-
 	// uiFlag is to enable launching cloudflared in interactive UI mode
 	uiFlag = "ui"
 
-	logDirectoryFlag = "log-directory"
-
 	debugLevelWarning = "At debug level, request URL, method, protocol, content legnth and header will be logged. " +
 		"Response status, content length and header will also be logged in debug level."
 )
@@ -113,86 +92,33 @@ func Flags() []cli.Flag {
 }
 
 func Commands() []*cli.Command {
-	cmds := []*cli.Command{
-		{
-			Name:      "login",
-			Action:    cliutil.ErrorHandler(login),
-			Usage:     "Generate a configuration file with your login details",
-			ArgsUsage: " ",
-			Flags: []cli.Flag{
-				&cli.StringFlag{
-					Name:   "url",
-					Hidden: true,
-				},
-			},
-			Hidden: true,
-		},
-		{
-			Name:   "proxy-dns",
-			Action: cliutil.ErrorHandler(tunneldns.Run),
-			Usage:  "Run a DNS over HTTPS proxy server.",
-			Flags: []cli.Flag{
-				&cli.StringFlag{
-					Name:    "metrics",
-					Value:   "localhost:",
-					Usage:   "Listen address for metrics reporting.",
-					EnvVars: []string{"TUNNEL_METRICS"},
-				},
-				&cli.StringFlag{
-					Name:    "address",
-					Usage:   "Listen address for the DNS over HTTPS proxy server.",
-					Value:   "localhost",
-					EnvVars: []string{"TUNNEL_DNS_ADDRESS"},
-				},
-				&cli.IntFlag{
-					Name:    "port",
-					Usage:   "Listen on given port for the DNS over HTTPS proxy server.",
-					Value:   53,
-					EnvVars: []string{"TUNNEL_DNS_PORT"},
-				},
-				&cli.StringSliceFlag{
-					Name:    "upstream",
-					Usage:   "Upstream endpoint URL, you can specify multiple endpoints for redundancy.",
-					Value:   cli.NewStringSlice("https://1.1.1.1/dns-query", "https://1.0.0.1/dns-query"),
-					EnvVars: []string{"TUNNEL_DNS_UPSTREAM"},
-				},
-				&cli.StringSliceFlag{
-					Name:    "bootstrap",
-					Usage:   "bootstrap endpoint URL, you can specify multiple endpoints for redundancy.",
-					Value:   cli.NewStringSlice("https://162.159.36.1/dns-query", "https://162.159.46.1/dns-query", "https://[2606:4700:4700::1111]/dns-query", "https://[2606:4700:4700::1001]/dns-query"),
-					EnvVars: []string{"TUNNEL_DNS_BOOTSTRAP"},
-				},
-			},
-			ArgsUsage: " ", // can't be the empty string or we get the default output
-			Hidden:    false,
-		},
+	subcommands := []*cli.Command{
+		buildLoginSubcommand(false),
+		buildCreateCommand(),
+		buildRouteCommand(),
+		buildRunCommand(),
+		buildListCommand(),
+		buildIngressSubcommand(),
+		buildDeleteCommand(),
+		buildCleanupCommand(),
+		// for compatibility, allow following as tunnel subcommands
+		tunneldns.Command(true),
 		dbConnectCmd(),
 	}
 
-	var subcommands []*cli.Command
-	for _, cmd := range cmds {
-		c := *cmd
-		c.Hidden = false
-		subcommands = append(subcommands, &c)
+	return []*cli.Command{
+		buildTunnelCommand(subcommands),
+		// for compatibility, allow following as top-level subcommands
+		buildLoginSubcommand(true),
+		dbConnectCmd(),
 	}
-
-	subcommands = append(subcommands, buildCreateCommand())
-	subcommands = append(subcommands, buildListCommand())
-	subcommands = append(subcommands, buildDeleteCommand())
-	subcommands = append(subcommands, buildRunCommand())
-	subcommands = append(subcommands, buildCleanupCommand())
-	subcommands = append(subcommands, buildRouteCommand())
-
-	cmds = append(cmds, buildTunnelCommand(subcommands))
-
-	return cmds
 }
 
 func buildTunnelCommand(subcommands []*cli.Command) *cli.Command {
 	return &cli.Command{
 		Name:      "tunnel",
 		Action:    cliutil.ErrorHandler(TunnelCommand),
-		Before:    Before,
+		Before:    SetFlagsFromConfigFile,
 		Category:  "Tunnel",
 		Usage:     "Make a locally-running web service accessible over the internet using Argo Tunnel.",
 		ArgsUsage: " ",
@@ -220,24 +146,27 @@ func buildTunnelCommand(subcommands []*cli.Command) *cli.Command {
 }
 
 func TunnelCommand(c *cli.Context) error {
-	if name := c.String("name"); name != "" { // Start a named tunnel
-		return adhocNamedTunnel(c, name)
-	} else { // Start a classic tunnel
-		return classicTunnel(c)
+	sc, err := newSubcommandContext(c)
+	if err != nil {
+		return err
 	}
+	if name := c.String("name"); name != "" { // Start a named tunnel
+		return runAdhocNamedTunnel(sc, name)
+	}
+	if ref := config.GetConfiguration().TunnelID; ref != "" {
+		return fmt.Errorf("Use `cloudflared tunnel run` to start tunnel %s", ref)
+	}
+
+	// Start a classic tunnel
+	return runClassicTunnel(sc)
 }
 
 func Init(v string, s, g chan struct{}) {
 	version, shutdownC, graceShutdownC = v, s, g
 }
 
-// adhocNamedTunnel create, route and run a named tunnel in one command
-func adhocNamedTunnel(c *cli.Context, name string) error {
-	sc, err := newSubcommandContext(c)
-	if err != nil {
-		return err
-	}
-
+// runAdhocNamedTunnel create, route and run a named tunnel in one command
+func runAdhocNamedTunnel(sc *subcommandContext, name string) error {
 	tunnel, ok, err := sc.tunnelActive(name)
 	if err != nil || !ok {
 		tunnel, err = sc.create(name)
@@ -248,7 +177,7 @@ func adhocNamedTunnel(c *cli.Context, name string) error {
 		sc.logger.Infof("Tunnel already created with ID %s", tunnel.ID)
 	}
 
-	if r, ok := routeFromFlag(c); ok {
+	if r, ok := routeFromFlag(sc.c); ok {
 		if res, err := sc.route(tunnel.ID, r); err != nil {
 			sc.logger.Errorf("failed to create route, please create it manually. err: %v.", err)
 		} else {
@@ -263,14 +192,9 @@ func adhocNamedTunnel(c *cli.Context, name string) error {
 	return nil
 }
 
-// classicTunnel creates a "classic" non-named tunnel
-func classicTunnel(c *cli.Context) error {
-	sc, err := newSubcommandContext(c)
-	if err != nil {
-		return err
-	}
-
-	return StartServer(c, version, shutdownC, graceShutdownC, nil, sc.logger, sc.isUIEnabled)
+// runClassicTunnel creates a "classic" non-named tunnel
+func runClassicTunnel(sc *subcommandContext) error {
+	return StartServer(sc.c, version, shutdownC, graceShutdownC, nil, sc.logger, sc.isUIEnabled)
 }
 
 func routeFromFlag(c *cli.Context) (tunnelstore.Route, bool) {
@@ -283,47 +207,13 @@ func routeFromFlag(c *cli.Context) (tunnelstore.Route, bool) {
 	return nil, false
 }
 
-func createLogger(c *cli.Context, isTransport bool, disableTerminal bool) (*logger.OutputWriter, error) {
-	var loggerOpts []logger.Option
-
-	logPath := c.String("logfile")
-	if logPath == "" {
-		logPath = c.String(logDirectoryFlag)
-	}
-
-	if logPath != "" {
-		loggerOpts = append(loggerOpts, logger.DefaultFile(logPath))
-	}
-
-	logLevel := c.String("loglevel")
-	if isTransport {
-		logLevel = c.String("transport-loglevel")
-		if logLevel == "" {
-			logLevel = "fatal"
-		}
-	}
-	loggerOpts = append(loggerOpts, logger.LogLevelString(logLevel))
-
-	if disableTerminal {
-		disableOption := logger.DisableTerminal(true)
-		loggerOpts = append(loggerOpts, disableOption)
-	}
-
-	l, err := logger.New(loggerOpts...)
-	if err != nil {
-		return nil, err
-	}
-
-	return l, nil
-}
-
 func StartServer(
 	c *cli.Context,
 	version string,
 	shutdownC,
 	graceShutdownC chan struct{},
-	namedTunnel *origin.NamedTunnelConfig,
-	log logger.Service,
+	namedTunnel *connection.NamedTunnelConfig,
+	generalLogger logger.Service,
 	isUIEnabled bool,
 ) error {
 	_ = raven.SetDSN(sentryDSN)
@@ -333,50 +223,46 @@ func StartServer(
 	connectedSignal := signal.New(make(chan struct{}))
 	dnsReadySignal := make(chan struct{})
 
-	if c.String("config") == "" {
-		log.Infof(
-			"Cannot determine default configuration path. No file %v in %v",
-			config.DefaultConfigFiles,
-			config.DefaultConfigSearchDirectories(),
-		)
+	if config.GetConfiguration().Source() == "" {
+		generalLogger.Infof(config.ErrNoConfigFile.Error())
 	}
 
 	if c.IsSet("trace-output") {
 		tmpTraceFile, err := ioutil.TempFile("", "trace")
 		if err != nil {
-			log.Errorf("Failed to create new temporary file to save trace output: %s", err)
+			generalLogger.Errorf("Failed to create new temporary file to save trace output: %s", err)
 		}
 
 		defer func() {
 			if err := tmpTraceFile.Close(); err != nil {
-				log.Errorf("Failed to close trace output file %s with error: %s", tmpTraceFile.Name(), err)
+				generalLogger.Errorf("Failed to close trace output file %s with error: %s", tmpTraceFile.Name(), err)
 			}
 			if err := os.Rename(tmpTraceFile.Name(), c.String("trace-output")); err != nil {
-				log.Errorf("Failed to rename temporary trace output file %s to %s with error: %s", tmpTraceFile.Name(), c.String("trace-output"), err)
+				generalLogger.Errorf("Failed to rename temporary trace output file %s to %s with error: %s", tmpTraceFile.Name(), c.String("trace-output"), err)
 			} else {
 				err := os.Remove(tmpTraceFile.Name())
 				if err != nil {
-					log.Errorf("Failed to remove the temporary trace file %s with error: %s", tmpTraceFile.Name(), err)
+					generalLogger.Errorf("Failed to remove the temporary trace file %s with error: %s", tmpTraceFile.Name(), err)
 				}
 			}
 		}()
 
 		if err := trace.Start(tmpTraceFile); err != nil {
-			log.Errorf("Failed to start trace: %s", err)
+			generalLogger.Errorf("Failed to start trace: %s", err)
 			return errors.Wrap(err, "Error starting tracing")
 		}
 		defer trace.Stop()
 	}
 
 	buildInfo := buildinfo.GetBuildInfo(version)
-	buildInfo.Log(log)
-	logClientOptions(c, log)
+	buildInfo.Log(generalLogger)
+	logClientOptions(c, generalLogger)
 
 	if c.IsSet("proxy-dns") {
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
-			errC <- runDNSProxyServer(c, dnsReadySignal, shutdownC, log)
+			errC <- runDNSProxyServer(c, dnsReadySignal, shutdownC, generalLogger)
 		}()
 	} else {
 		close(dnsReadySignal)
@@ -385,26 +271,14 @@ func StartServer(
 	// Wait for proxy-dns to come up (if used)
 	<-dnsReadySignal
 
-	metricsListener, err := listeners.Listen("tcp", c.String("metrics"))
-	if err != nil {
-		log.Errorf("Error opening metrics server listener: %s", err)
-		return errors.Wrap(err, "Error opening metrics server listener")
-	}
-	defer metricsListener.Close()
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
-		errC <- metrics.ServeMetrics(metricsListener, shutdownC, log)
-	}()
-
 	go notifySystemd(connectedSignal)
 	if c.IsSet("pidfile") {
-		go writePidFile(connectedSignal, c.String("pidfile"), log)
+		go writePidFile(connectedSignal, c.String("pidfile"), generalLogger)
 	}
 
 	cloudflaredID, err := uuid.NewRandom()
 	if err != nil {
-		log.Errorf("Cannot generate cloudflared ID: %s", err)
+		generalLogger.Errorf("Cannot generate cloudflared ID: %s", err)
 		return err
 	}
 
@@ -415,12 +289,12 @@ func StartServer(
 	}()
 
 	// update needs to be after DNS proxy is up to resolve equinox server address
-	if updater.IsAutoupdateEnabled(c, log) {
-		log.Infof("Autoupdate frequency is set to %v", c.Duration("autoupdate-freq"))
+	if updater.IsAutoupdateEnabled(c, generalLogger) {
+		generalLogger.Infof("Autoupdate frequency is set to %v", c.Duration("autoupdate-freq"))
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
-			autoupdater := updater.NewAutoUpdater(c.Duration("autoupdate-freq"), &listeners, log)
+			autoupdater := updater.NewAutoUpdater(c.Duration("autoupdate-freq"), &listeners, generalLogger)
 			errC <- autoupdater.Run(ctx)
 		}()
 	}
@@ -429,133 +303,52 @@ func StartServer(
 	if dnsProxyStandAlone(c) {
 		connectedSignal.Notify()
 		// no grace period, handle SIGINT/SIGTERM immediately
-		return waitToShutdown(&wg, errC, shutdownC, graceShutdownC, 0, log)
-	}
-
-	if c.IsSet("hello-world") {
-		log.Infof("hello-world set")
-		helloListener, err := hello.CreateTLSListener("127.0.0.1:")
-		if err != nil {
-			log.Errorf("Cannot start Hello World Server: %s", err)
-			return errors.Wrap(err, "Cannot start Hello World Server")
-		}
-		defer helloListener.Close()
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			_ = hello.StartHelloWorldServer(log, helloListener, shutdownC)
-		}()
-		forceSetFlag(c, "url", "https://"+helloListener.Addr().String())
-	}
-
-	if c.IsSet(sshServerFlag) {
-		if runtime.GOOS != "darwin" && runtime.GOOS != "linux" {
-			msg := fmt.Sprintf("--ssh-server is not supported on %s", runtime.GOOS)
-			log.Error(msg)
-			return errors.New(msg)
-		}
-
-		log.Infof("ssh-server set")
-
-		logManager := sshlog.NewEmptyManager()
-		if c.IsSet(bucketNameFlag) && c.IsSet(regionNameFlag) && c.IsSet(accessKeyIDFlag) && c.IsSet(secretIDFlag) {
-			uploader, err := awsuploader.NewFileUploader(c.String(bucketNameFlag), c.String(regionNameFlag),
-				c.String(accessKeyIDFlag), c.String(secretIDFlag), c.String(sessionTokenIDFlag), c.String(s3URLFlag))
-			if err != nil {
-				msg := "Cannot create uploader for SSH Server"
-				log.Errorf("%s: %s", msg, err)
-				return errors.Wrap(err, msg)
-			}
-
-			if err := os.MkdirAll(sshLogFileDirectory, 0700); err != nil {
-				msg := fmt.Sprintf("Cannot create SSH log file directory %s", sshLogFileDirectory)
-				log.Errorf("%s: %s", msg, err)
-				return errors.Wrap(err, msg)
-			}
-
-			logManager = sshlog.New(sshLogFileDirectory)
-
-			uploadManager := awsuploader.NewDirectoryUploadManager(log, uploader, sshLogFileDirectory, 30*time.Minute, shutdownC)
-			uploadManager.Start()
-		}
-
-		localServerAddress := "127.0.0.1:" + c.String(sshPortFlag)
-		server, err := sshserver.New(logManager, log, version, localServerAddress, c.String("hostname"), c.Path(hostKeyPath), shutdownC, c.Duration(sshIdleTimeoutFlag), c.Duration(sshMaxTimeoutFlag))
-		if err != nil {
-			msg := "Cannot create new SSH Server"
-			log.Errorf("%s: %s", msg, err)
-			return errors.Wrap(err, msg)
-		}
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			if err = server.Start(); err != nil && err != ssh.ErrServerClosed {
-				log.Errorf("SSH server error: %s", err)
-				// TODO: remove when declarative tunnels are implemented.
-				close(shutdownC)
-			}
-		}()
-		forceSetFlag(c, "url", "ssh://"+localServerAddress)
+		return waitToShutdown(&wg, errC, shutdownC, graceShutdownC, 0, generalLogger)
 	}
 
 	url := c.String("url")
 	hostname := c.String("hostname")
 	if url == hostname && url != "" && hostname != "" {
 		errText := "hostname and url shouldn't match. See --help for more information"
-		log.Error(errText)
+		generalLogger.Error(errText)
 		return fmt.Errorf(errText)
 	}
 
-	if staticHost := hostnameFromURI(c.String("url")); isProxyDestinationConfigured(staticHost, c) {
-		listener, err := net.Listen("tcp", net.JoinHostPort(c.String("proxy-address"), strconv.Itoa(c.Int("proxy-port"))))
-		if err != nil {
-			log.Errorf("Cannot start Websocket Proxy Server: %s", err)
-			return errors.Wrap(err, "Cannot start Websocket Proxy Server")
-		}
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			streamHandler := websocket.DefaultStreamHandler
-			if c.IsSet(socks5Flag) {
-				log.Info("SOCKS5 server started")
-				streamHandler = func(wsConn *websocket.Conn, remoteConn net.Conn, _ http.Header) {
-					dialer := socks.NewConnDialer(remoteConn)
-					requestHandler := socks.NewRequestHandler(dialer)
-					socksServer := socks.NewConnectionHandler(requestHandler)
-
-					socksServer.Serve(wsConn)
-				}
-			} else if c.IsSet(sshServerFlag) {
-				streamHandler = func(wsConn *websocket.Conn, remoteConn net.Conn, requestHeaders http.Header) {
-					if finalDestination := requestHeaders.Get(h2mux.CFJumpDestinationHeader); finalDestination != "" {
-						token := requestHeaders.Get(h2mux.CFAccessTokenHeader)
-						if err := websocket.SendSSHPreamble(remoteConn, finalDestination, token); err != nil {
-							log.Errorf("Failed to send SSH preamble: %s", err)
-							return
-						}
-					}
-					websocket.DefaultStreamHandler(wsConn, remoteConn, requestHeaders)
-				}
-			}
-			errC <- websocket.StartProxyServer(log, listener, staticHost, shutdownC, streamHandler)
-		}()
-		forceSetFlag(c, "url", "http://"+listener.Addr().String())
-	}
-
-	transportLogger, err := createLogger(c, true, false)
+	transportLogger, err := logger.CreateTransportLoggerFromContext(c, isUIEnabled)
 	if err != nil {
 		return errors.Wrap(err, "error setting up transport logger")
 	}
 
-	tunnelConfig, err := prepareTunnelConfig(c, buildInfo, version, log, transportLogger, namedTunnel)
+	readinessCh := make(chan connection.Event, 16)
+	uiCh := make(chan connection.Event, 16)
+	eventChannels := []chan connection.Event{
+		readinessCh,
+		uiCh,
+	}
+	tunnelConfig, ingressRules, err := prepareTunnelConfig(c, buildInfo, version, generalLogger, transportLogger, namedTunnel, isUIEnabled, eventChannels)
 	if err != nil {
+		generalLogger.Errorf("Couldn't start tunnel: %v", err)
 		return err
 	}
 
+	metricsListener, err := listeners.Listen("tcp", c.String("metrics"))
+	if err != nil {
+		generalLogger.Errorf("Error opening metrics server listener: %s", err)
+		return errors.Wrap(err, "Error opening metrics server listener")
+	}
+	defer metricsListener.Close()
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		errC <- metrics.ServeMetrics(metricsListener, shutdownC, readinessCh, generalLogger)
+	}()
+
+	ingressRules.StartOrigins(&wg, generalLogger, shutdownC, errC)
+
 	reconnectCh := make(chan origin.ReconnectSignal, 1)
 	if c.IsSet("stdin-control") {
-		log.Info("Enabling control through stdin")
-		go stdinControl(reconnectCh, log)
+		generalLogger.Info("Enabling control through stdin")
+		go stdinControl(reconnectCh, generalLogger)
 	}
 
 	wg.Add(1)
@@ -565,25 +358,21 @@ func StartServer(
 	}()
 
 	if isUIEnabled {
-		const tunnelEventChanBufferSize = 16
-		tunnelEventChan := make(chan ui.TunnelEvent, tunnelEventChanBufferSize)
-		tunnelConfig.TunnelEventChan = tunnelEventChan
-
 		tunnelInfo := ui.NewUIModel(
 			version,
 			hostname,
 			metricsListener.Addr().String(),
-			tunnelConfig.OriginUrl,
+			&ingressRules,
 			tunnelConfig.HAConnections,
 		)
 		logLevels, err := logger.ParseLevelString(c.String("loglevel"))
 		if err != nil {
 			return err
 		}
-		tunnelInfo.LaunchUI(ctx, log, logLevels, tunnelEventChan)
+		tunnelInfo.LaunchUI(ctx, generalLogger, transportLogger, logLevels, uiCh)
 	}
 
-	return waitToShutdown(&wg, errC, shutdownC, graceShutdownC, c.Duration("grace-period"), log)
+	return waitToShutdown(&wg, errC, shutdownC, graceShutdownC, c.Duration("grace-period"), generalLogger)
 }
 
 // forceSetFlag attempts to set the given flag value in the closest context that has it defined
@@ -595,35 +384,29 @@ func forceSetFlag(c *cli.Context, name, value string) {
 	}
 }
 
-func Before(c *cli.Context) error {
-	logger, err := createLogger(c, false, false)
+func SetFlagsFromConfigFile(c *cli.Context) error {
+	const exitCode = 1
+	log, err := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
 	if err != nil {
 		return cliutil.PrintLoggerSetupError("error setting up logger", err)
 	}
-
-	if c.String("config") == "" {
-		logger.Debugf("Cannot determine default configuration path. No file %v in %v", config.DefaultConfigFiles, config.DefaultConfigSearchDirectories())
-	}
-	inputSource, err := config.FindInputSourceContext(c)
+	inputSource, err := config.ReadConfigFile(c, log)
 	if err != nil {
-		logger.Errorf("Cannot load configuration from %s: %s", c.String("config"), err)
-		return err
-	} else if inputSource != nil {
-		err := altsrc.ApplyInputSourceValues(c, inputSource, c.App.Flags)
-		if err != nil {
-			logger.Errorf("Cannot apply configuration from %s: %s", c.String("config"), err)
-			return err
+		if err == config.ErrNoConfigFile {
+			return nil
 		}
-		logger.Debugf("Applied configuration from %s", c.String("config"))
+		return cli.Exit(err, exitCode)
+	}
+	targetFlags := c.Command.Flags
+	if c.Command.Name == "" {
+		targetFlags = c.App.Flags
+	}
+	if err := altsrc.ApplyInputSourceValues(c, inputSource, targetFlags); err != nil {
+		return cli.Exit(err, exitCode)
 	}
 	return nil
 }
 
-// isProxyDestinationConfigured returns true if there is a static host set or if bastion mode is set.
-func isProxyDestinationConfigured(staticHost string, c *cli.Context) bool {
-	return staticHost != "" || c.IsSet(bastionFlag)
-}
-
 func waitToShutdown(wg *sync.WaitGroup,
 	errC chan error,
 	shutdownC, graceShutdownC chan struct{},
@@ -706,7 +489,7 @@ func dbConnectCmd() *cli.Command {
 
 	// Override before to run tunnel validation before dbconnect validation.
 	cmd.Before = func(c *cli.Context) error {
-		err := Before(c)
+		err := SetFlagsFromConfigFile(c)
 		if err == nil {
 			err = dbconnect.CmdBefore(c)
 		}
@@ -903,6 +686,7 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			Value:  false,
 			Hidden: shouldHide,
 		}),
+		selectProtocolFlag,
 	}...)
 
 	return flags
@@ -970,67 +754,67 @@ func configureProxyFlags(shouldHide bool) []cli.Flag {
 			Hidden:  shouldHide,
 		}),
 		altsrc.NewBoolFlag(&cli.BoolFlag{
-			Name:    socks5Flag,
+			Name:    ingress.Socks5Flag,
 			Usage:   "specify if this tunnel is running as a SOCK5 Server",
 			EnvVars: []string{"TUNNEL_SOCKS"},
 			Value:   false,
 			Hidden:  shouldHide,
 		}),
 		altsrc.NewDurationFlag(&cli.DurationFlag{
-			Name:   "proxy-connect-timeout",
+			Name:   ingress.ProxyConnectTimeoutFlag,
 			Usage:  "HTTP proxy timeout for establishing a new connection",
 			Value:  time.Second * 30,
 			Hidden: shouldHide,
 		}),
 		altsrc.NewDurationFlag(&cli.DurationFlag{
-			Name:   "proxy-tls-timeout",
+			Name:   ingress.ProxyTLSTimeoutFlag,
 			Usage:  "HTTP proxy timeout for completing a TLS handshake",
 			Value:  time.Second * 10,
 			Hidden: shouldHide,
 		}),
 		altsrc.NewDurationFlag(&cli.DurationFlag{
-			Name:   "proxy-tcp-keepalive",
+			Name:   ingress.ProxyTCPKeepAlive,
 			Usage:  "HTTP proxy TCP keepalive duration",
 			Value:  time.Second * 30,
 			Hidden: shouldHide,
 		}),
 		altsrc.NewBoolFlag(&cli.BoolFlag{
-			Name:   "proxy-no-happy-eyeballs",
+			Name:   ingress.ProxyNoHappyEyeballsFlag,
 			Usage:  "HTTP proxy should disable \"happy eyeballs\" for IPv4/v6 fallback",
 			Hidden: shouldHide,
 		}),
 		altsrc.NewIntFlag(&cli.IntFlag{
-			Name:   "proxy-keepalive-connections",
+			Name:   ingress.ProxyKeepAliveConnectionsFlag,
 			Usage:  "HTTP proxy maximum keepalive connection pool size",
 			Value:  100,
 			Hidden: shouldHide,
 		}),
 		altsrc.NewDurationFlag(&cli.DurationFlag{
-			Name:   "proxy-keepalive-timeout",
+			Name:   ingress.ProxyKeepAliveTimeoutFlag,
 			Usage:  "HTTP proxy timeout for closing an idle connection",
 			Value:  time.Second * 90,
 			Hidden: shouldHide,
 		}),
 		altsrc.NewDurationFlag(&cli.DurationFlag{
 			Name:   "proxy-connection-timeout",
-			Usage:  "HTTP proxy timeout for closing an idle connection",
+			Usage:  "DEPRECATED. No longer has any effect.",
 			Value:  time.Second * 90,
 			Hidden: shouldHide,
 		}),
 		altsrc.NewDurationFlag(&cli.DurationFlag{
 			Name:   "proxy-expect-continue-timeout",
-			Usage:  "HTTP proxy timeout for closing an idle connection",
+			Usage:  "DEPRECATED. No longer has any effect.",
 			Value:  time.Second * 90,
 			Hidden: shouldHide,
 		}),
 		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:    "http-host-header",
+			Name:    ingress.HTTPHostHeaderFlag,
 			Usage:   "Sets the HTTP Host header for the local webserver.",
 			EnvVars: []string{"TUNNEL_HTTP_HOST_HEADER"},
 			Hidden:  shouldHide,
 		}),
 		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:    "origin-server-name",
+			Name:    ingress.OriginServerNameFlag,
 			Usage:   "Hostname on the origin server certificate.",
 			EnvVars: []string{"TUNNEL_ORIGIN_SERVER_NAME"},
 			Hidden:  shouldHide,
@@ -1048,13 +832,13 @@ func configureProxyFlags(shouldHide bool) []cli.Flag {
 			Hidden:  shouldHide,
 		}),
 		altsrc.NewBoolFlag(&cli.BoolFlag{
-			Name:    "no-tls-verify",
+			Name:    ingress.NoTLSVerifyFlag,
 			Usage:   "Disables TLS verification of the certificate presented by your origin. Will allow any certificate from the origin to be accepted. Note: The connection from your machine to Cloudflare's Edge is still encrypted.",
 			EnvVars: []string{"NO_TLS_VERIFY"},
 			Hidden:  shouldHide,
 		}),
 		altsrc.NewBoolFlag(&cli.BoolFlag{
-			Name:    "no-chunked-encoding",
+			Name:    ingress.NoChunkedEncodingFlag,
 			Usage:   "Disables chunked transfer encoding; useful if you are running a WSGI server.",
 			EnvVars: []string{"TUNNEL_NO_CHUNKED_ENCODING"},
 			Hidden:  shouldHide,
@@ -1127,28 +911,28 @@ func sshFlags(shouldHide bool) []cli.Flag {
 			Hidden:  true,
 		}),
 		altsrc.NewBoolFlag(&cli.BoolFlag{
-			Name:    sshServerFlag,
+			Name:    ingress.SSHServerFlag,
 			Value:   false,
 			Usage:   "Run an SSH Server",
 			EnvVars: []string{"TUNNEL_SSH_SERVER"},
 			Hidden:  true, // TODO: remove when feature is complete
 		}),
 		altsrc.NewBoolFlag(&cli.BoolFlag{
-			Name:    bastionFlag,
+			Name:    config.BastionFlag,
 			Value:   false,
 			Usage:   "Runs as jump host",
 			EnvVars: []string{"TUNNEL_BASTION"},
 			Hidden:  shouldHide,
 		}),
 		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:    "proxy-address",
+			Name:    ingress.ProxyAddressFlag,
 			Usage:   "Listen address for the proxy.",
 			Value:   "127.0.0.1",
 			EnvVars: []string{"TUNNEL_PROXY_ADDRESS"},
 			Hidden:  shouldHide,
 		}),
 		altsrc.NewIntFlag(&cli.IntFlag{
-			Name:    "proxy-port",
+			Name:    ingress.ProxyPortFlag,
 			Usage:   "Listen port for the proxy.",
 			Value:   0,
 			EnvVars: []string{"TUNNEL_PROXY_PORT"},
@@ -1160,28 +944,28 @@ func sshFlags(shouldHide bool) []cli.Flag {
 func configureLoggingFlags(shouldHide bool) []cli.Flag {
 	return []cli.Flag{
 		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:    "loglevel",
+			Name:    logger.LogLevelFlag,
 			Value:   "info",
 			Usage:   "Application logging level {fatal, error, info, debug}. " + debugLevelWarning,
 			EnvVars: []string{"TUNNEL_LOGLEVEL"},
 			Hidden:  shouldHide,
 		}),
 		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:    "transport-loglevel",
+			Name:    logger.LogTransportLevelFlag,
 			Aliases: []string{"proto-loglevel"}, // This flag used to be called proto-loglevel
-			Value:   "fatal",
+			Value:   "info",
 			Usage:   "Transport logging level(previously called protocol logging level) {fatal, error, info, debug}",
 			EnvVars: []string{"TUNNEL_PROTO_LOGLEVEL", "TUNNEL_TRANSPORT_LOGLEVEL"},
 			Hidden:  shouldHide,
 		}),
 		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:    "logfile",
+			Name:    logger.LogFileFlag,
 			Usage:   "Save application log to this file for reporting issues.",
 			EnvVars: []string{"TUNNEL_LOGFILE"},
 			Hidden:  shouldHide,
 		}),
 		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:    logDirectoryFlag,
+			Name:    logger.LogDirectoryFlag,
 			Usage:   "Save application log to this directory for reporting issues.",
 			EnvVars: []string{"TUNNEL_LOGDIRECTORY"},
 			Hidden:  shouldHide,

cmd/cloudflared/tunnel/configuration.go

@@ -1,19 +1,19 @@
 package tunnel
 
 import (
-	"context"
 	"crypto/tls"
 	"fmt"
 	"io/ioutil"
-	"net"
-	"net/http"
 	"os"
 	"path/filepath"
 	"strings"
-	"time"
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/connection"
+	"github.com/cloudflare/cloudflared/edgediscovery"
+	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/ingress"
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/origin"
 	"github.com/cloudflare/cloudflared/tlsconfig"
@@ -158,109 +158,50 @@ func prepareTunnelConfig(
 	version string,
 	logger logger.Service,
 	transportLogger logger.Service,
-	namedTunnel *origin.NamedTunnelConfig,
-) (*origin.TunnelConfig, error) {
-	compatibilityMode := namedTunnel == nil
+	namedTunnel *connection.NamedTunnelConfig,
+	isUIEnabled bool,
+	eventChans []chan connection.Event,
+) (*origin.TunnelConfig, ingress.Ingress, error) {
+	isNamedTunnel := namedTunnel != nil
 
 	hostname, err := validation.ValidateHostname(c.String("hostname"))
 	if err != nil {
 		logger.Errorf("Invalid hostname: %s", err)
-		return nil, errors.Wrap(err, "Invalid hostname")
+		return nil, ingress.Ingress{}, errors.Wrap(err, "Invalid hostname")
 	}
 	isFreeTunnel := hostname == ""
 	clientID := c.String("id")
 	if !c.IsSet("id") {
 		clientID, err = generateRandomClientID(logger)
 		if err != nil {
-			return nil, err
+			return nil, ingress.Ingress{}, err
 		}
 	}
 
 	tags, err := NewTagSliceFromCLI(c.StringSlice("tag"))
 	if err != nil {
 		logger.Errorf("Tag parse failure: %s", err)
-		return nil, errors.Wrap(err, "Tag parse failure")
+		return nil, ingress.Ingress{}, errors.Wrap(err, "Tag parse failure")
 	}
 
 	tags = append(tags, tunnelpogs.Tag{Name: "ID", Value: clientID})
 
-	originURL, err := config.ValidateUrl(c, compatibilityMode)
-	if err != nil {
-		logger.Errorf("Error validating origin URL: %s", err)
-		return nil, errors.Wrap(err, "Error validating origin URL")
-	}
-
 	var originCert []byte
 	if !isFreeTunnel {
 		originCert, err = getOriginCert(c, logger)
 		if err != nil {
-			return nil, errors.Wrap(err, "Error getting origin cert")
+			return nil, ingress.Ingress{}, errors.Wrap(err, "Error getting origin cert")
 		}
 	}
 
-	originCertPool, err := tlsconfig.LoadOriginCA(c, logger)
-	if err != nil {
-		logger.Errorf("Error loading cert pool: %s", err)
-		return nil, errors.Wrap(err, "Error loading cert pool")
-	}
-
-	tunnelMetrics := origin.NewTunnelMetrics()
-	httpTransport := &http.Transport{
-		Proxy:                 http.ProxyFromEnvironment,
-		MaxIdleConns:          c.Int("proxy-keepalive-connections"),
-		MaxIdleConnsPerHost:   c.Int("proxy-keepalive-connections"),
-		IdleConnTimeout:       c.Duration("proxy-keepalive-timeout"),
-		TLSHandshakeTimeout:   c.Duration("proxy-tls-timeout"),
-		ExpectContinueTimeout: 1 * time.Second,
-		TLSClientConfig:       &tls.Config{RootCAs: originCertPool, InsecureSkipVerify: c.IsSet("no-tls-verify")},
-	}
-
-	dialer := &net.Dialer{
-		Timeout:   c.Duration("proxy-connect-timeout"),
-		KeepAlive: c.Duration("proxy-tcp-keepalive"),
-	}
-	if c.Bool("proxy-no-happy-eyeballs") {
-		dialer.FallbackDelay = -1 // As of Golang 1.12, a negative delay disables "happy eyeballs"
-	}
-	dialContext := dialer.DialContext
-
-	if c.IsSet("unix-socket") {
-		unixSocket, err := config.ValidateUnixSocket(c)
-		if err != nil {
-			logger.Errorf("Error validating --unix-socket: %s", err)
-			return nil, errors.Wrap(err, "Error validating --unix-socket")
-		}
-
-		logger.Infof("Proxying tunnel requests to unix:%s", unixSocket)
-		httpTransport.DialContext = func(ctx context.Context, _, _ string) (net.Conn, error) {
-			// if --unix-socket specified, enforce network type "unix"
-			return dialContext(ctx, "unix", unixSocket)
-		}
-	} else {
-		logger.Infof("Proxying tunnel requests to %s", originURL)
-		httpTransport.DialContext = dialContext
-	}
-
-	if !c.IsSet("hello-world") && c.IsSet("origin-server-name") {
-		httpTransport.TLSClientConfig.ServerName = c.String("origin-server-name")
-	}
-	// If tunnel running in bastion mode, a connection to origin will not exist until initiated by the client.
-	if !c.IsSet(bastionFlag) {
-		if err = validation.ValidateHTTPService(originURL, hostname, httpTransport); err != nil {
-			logger.Errorf("unable to connect to the origin: %s", err)
-		}
-	}
-
-	toEdgeTLSConfig, err := tlsconfig.CreateTunnelConfig(c)
-	if err != nil {
-		logger.Errorf("unable to create TLS config to connect with edge: %s", err)
-		return nil, errors.Wrap(err, "unable to create TLS config to connect with edge")
-	}
-
-	if namedTunnel != nil {
+	var (
+		ingressRules  ingress.Ingress
+		classicTunnel *connection.ClassicTunnelConfig
+	)
+	if isNamedTunnel {
 		clientUUID, err := uuid.NewRandom()
 		if err != nil {
-			return nil, errors.Wrap(err, "can't generate clientUUID")
+			return nil, ingress.Ingress{}, errors.Wrap(err, "can't generate clientUUID")
 		}
 		namedTunnel.Client = tunnelpogs.ClientInfo{
 			ClientID: clientUUID[:],
@@ -268,42 +209,81 @@ func prepareTunnelConfig(
 			Version:  version,
 			Arch:     fmt.Sprintf("%s_%s", buildInfo.GoOS, buildInfo.GoArch),
 		}
+		ingressRules, err = ingress.ParseIngress(config.GetConfiguration())
+		if err != nil && err != ingress.ErrNoIngressRules {
+			return nil, ingress.Ingress{}, err
+		}
+		if !ingressRules.IsEmpty() && c.IsSet("url") {
+			return nil, ingress.Ingress{}, ingress.ErrURLIncompatibleWithIngress
+		}
+	} else {
+		classicTunnel = &connection.ClassicTunnelConfig{
+			Hostname:   hostname,
+			OriginCert: originCert,
+			// turn off use of reconnect token and auth refresh when using named tunnels
+			UseReconnectToken: !isNamedTunnel && c.Bool("use-reconnect-token"),
+		}
+	}
+
+	// Convert single-origin configuration into multi-origin configuration.
+	if ingressRules.IsEmpty() {
+		ingressRules, err = ingress.NewSingleOrigin(c, !isNamedTunnel, logger)
+		if err != nil {
+			return nil, ingress.Ingress{}, err
+		}
+	}
+
+	protocolSelector, err := connection.NewProtocolSelector(c.String("protocol"), namedTunnel, edgediscovery.HTTP2Percentage, origin.ResolveTTL, logger)
+	if err != nil {
+		return nil, ingress.Ingress{}, err
+	}
+	logger.Infof("Initial protocol %s", protocolSelector.Current())
+
+	edgeTLSConfigs := make(map[connection.Protocol]*tls.Config, len(connection.ProtocolList))
+	for _, p := range connection.ProtocolList {
+		edgeTLSConfig, err := tlsconfig.CreateTunnelConfig(c, p.ServerName())
+		if err != nil {
+			return nil, ingress.Ingress{}, errors.Wrap(err, "unable to create TLS config to connect with edge")
+		}
+		edgeTLSConfigs[p] = edgeTLSConfig
+	}
+
+	originClient := origin.NewClient(ingressRules, tags, logger)
+	connectionConfig := &connection.Config{
+		OriginClient:    originClient,
+		GracePeriod:     c.Duration("grace-period"),
+		ReplaceExisting: c.Bool("force"),
+	}
+	muxerConfig := &connection.MuxerConfig{
+		HeartbeatInterval:  c.Duration("heartbeat-interval"),
+		MaxHeartbeats:      c.Uint64("heartbeat-count"),
+		CompressionSetting: h2mux.CompressionSetting(c.Uint64("compression-quality")),
+		MetricsUpdateFreq:  c.Duration("metrics-update-freq"),
 	}
 
 	return &origin.TunnelConfig{
-		BuildInfo:          buildInfo,
-		ClientID:           clientID,
-		ClientTlsConfig:    httpTransport.TLSClientConfig,
-		CompressionQuality: c.Uint64("compression-quality"),
-		EdgeAddrs:          c.StringSlice("edge"),
-		GracePeriod:        c.Duration("grace-period"),
-		HAConnections:      c.Int("ha-connections"),
-		HTTPTransport:      httpTransport,
-		HeartbeatInterval:  c.Duration("heartbeat-interval"),
-		Hostname:           hostname,
-		HTTPHostHeader:     c.String("http-host-header"),
-		IncidentLookup:     origin.NewIncidentLookup(),
-		IsAutoupdated:      c.Bool("is-autoupdated"),
-		IsFreeTunnel:       isFreeTunnel,
-		LBPool:             c.String("lb-pool"),
-		Logger:             logger,
-		TransportLogger:    transportLogger,
-		MaxHeartbeats:      c.Uint64("heartbeat-count"),
-		Metrics:            tunnelMetrics,
-		MetricsUpdateFreq:  c.Duration("metrics-update-freq"),
-		NoChunkedEncoding:  c.Bool("no-chunked-encoding"),
-		OriginCert:         originCert,
-		OriginUrl:          originURL,
-		ReportedVersion:    version,
-		Retries:            c.Uint("retries"),
-		RunFromTerminal:    isRunningFromTerminal(),
-		Tags:               tags,
-		TlsConfig:          toEdgeTLSConfig,
-		NamedTunnel:        namedTunnel,
-		ReplaceExisting:    c.Bool("force"),
-		// turn off use of reconnect token and auth refresh when using named tunnels
-		UseReconnectToken: compatibilityMode && c.Bool("use-reconnect-token"),
-	}, nil
+		ConnectionConfig: connectionConfig,
+		BuildInfo:        buildInfo,
+		ClientID:         clientID,
+		EdgeAddrs:        c.StringSlice("edge"),
+		HAConnections:    c.Int("ha-connections"),
+		IncidentLookup:   origin.NewIncidentLookup(),
+		IsAutoupdated:    c.Bool("is-autoupdated"),
+		IsFreeTunnel:     isFreeTunnel,
+		LBPool:           c.String("lb-pool"),
+		Tags:             tags,
+		Logger:           logger,
+		Observer:         connection.NewObserver(transportLogger, eventChans, isUIEnabled),
+		ReportedVersion:  version,
+		Retries:          c.Uint("retries"),
+		RunFromTerminal:  isRunningFromTerminal(),
+		NamedTunnel:      namedTunnel,
+		ClassicTunnel:    classicTunnel,
+		MuxerConfig:      muxerConfig,
+		TunnelEventChans: eventChans,
+		ProtocolSelector: protocolSelector,
+		EdgeTLSConfigs:   edgeTLSConfigs,
+	}, ingressRules, nil
 }
 
 func isRunningFromTerminal() bool {

cmd/cloudflared/tunnel/credential_finder.go

@@ -0,0 +1,78 @@
+package tunnel
+
+import (
+	"fmt"
+	"path/filepath"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/logger"
+	"github.com/google/uuid"
+	"github.com/urfave/cli/v2"
+)
+
+// CredFinder can find the tunnel credentials file.
+type CredFinder interface {
+	Path() (string, error)
+}
+
+// Implements CredFinder and looks for the credentials file at the given
+// filepath.
+type staticPath struct {
+	filePath string
+	fs       fileSystem
+}
+
+func newStaticPath(filePath string, fs fileSystem) CredFinder {
+	return staticPath{
+		filePath: filePath,
+		fs:       fs,
+	}
+}
+
+func (a staticPath) Path() (string, error) {
+	if a.filePath != "" && a.fs.validFilePath(a.filePath) {
+		return a.filePath, nil
+	}
+	return "", fmt.Errorf("Tunnel credentials file '%s' doesn't exist or is not a file", a.filePath)
+}
+
+// Implements CredFinder and looks for the credentials file in several directories
+// searching for a file named <id>.json
+type searchByID struct {
+	id     uuid.UUID
+	c      *cli.Context
+	logger logger.Service
+	fs     fileSystem
+}
+
+func newSearchByID(id uuid.UUID, c *cli.Context, logger logger.Service, fs fileSystem) CredFinder {
+	return searchByID{
+		id:     id,
+		c:      c,
+		logger: logger,
+		fs:     fs,
+	}
+}
+
+func (s searchByID) Path() (string, error) {
+
+	// Fallback to look for tunnel credentials in the origin cert directory
+	if originCertPath, err := findOriginCert(s.c, s.logger); err == nil {
+		originCertDir := filepath.Dir(originCertPath)
+		if filePath, err := tunnelFilePath(s.id, originCertDir); err == nil {
+			if s.fs.validFilePath(filePath) {
+				return filePath, nil
+			}
+		}
+	}
+
+	// Last resort look under default config directories
+	for _, configDir := range config.DefaultConfigSearchDirectories() {
+		if filePath, err := tunnelFilePath(s.id, configDir); err == nil {
+			if s.fs.validFilePath(filePath) {
+				return filePath, nil
+			}
+		}
+	}
+	return "", fmt.Errorf("Tunnel credentials file not found")
+}

cmd/cloudflared/tunnel/filesystem.go

@@ -0,0 +1,27 @@
+package tunnel
+
+import (
+	"io/ioutil"
+	"os"
+)
+
+// Abstract away details of reading files, so that SubcommandContext can read
+// from either the real filesystem, or a mock (when running unit tests).
+type fileSystem interface {
+	readFile(filePath string) ([]byte, error)
+	validFilePath(path string) bool
+}
+
+type realFileSystem struct{}
+
+func (fs realFileSystem) validFilePath(path string) bool {
+	fileStat, err := os.Stat(path)
+	if err != nil {
+		return false
+	}
+	return !fileStat.IsDir()
+}
+
+func (fs realFileSystem) readFile(filePath string) ([]byte, error) {
+	return ioutil.ReadFile(filePath)
+}

cmd/cloudflared/tunnel/ingress_subcommands.go

@@ -0,0 +1,114 @@
+package tunnel
+
+import (
+	"fmt"
+	"net/url"
+
+	"github.com/pkg/errors"
+	"github.com/urfave/cli/v2"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/ingress"
+)
+
+func buildIngressSubcommand() *cli.Command {
+	return &cli.Command{
+		Name:      "ingress",
+		Category:  "Tunnel",
+		Usage:     "Validate and test cloudflared tunnel's ingress configuration",
+		UsageText: "cloudflared tunnel [--config FILEPATH] ingress COMMAND [arguments...]",
+		Hidden:    true,
+		Description: ` Cloudflared lets you route traffic from the internet to multiple different addresses on your
+		origin. Multiple-origin routing is configured by a set of rules. Each rule matches traffic
+		by its hostname or path, and routes it to an address. These rules are configured under the
+		'ingress' key of your config.yaml, for example:
+
+		ingress:
+		  - hostname: www.example.com
+		    service: https://localhost:8000
+		  - hostname: *.example.xyz
+		    path: /[a-zA-Z]+.html
+		    service: https://localhost:8001
+		  - hostname: *
+		    service: https://localhost:8002
+
+		To ensure cloudflared can route all incoming requests, the last rule must be a catch-all
+		rule that matches all traffic. You can validate these rules with the 'ingress validate'
+		command, and test which rule matches a particular URL with 'ingress rule <URL>'.
+
+		Multiple-origin routing is incompatible with the --url flag.`,
+		Subcommands: []*cli.Command{buildValidateIngressCommand(), buildTestURLCommand()},
+	}
+}
+
+func buildValidateIngressCommand() *cli.Command {
+	return &cli.Command{
+		Name:        "validate",
+		Action:      cliutil.ErrorHandler(validateIngressCommand),
+		Usage:       "Validate the ingress configuration ",
+		UsageText:   "cloudflared tunnel [--config FILEPATH] ingress validate",
+		Description: "Validates the configuration file, ensuring your ingress rules are OK.",
+	}
+}
+
+func buildTestURLCommand() *cli.Command {
+	return &cli.Command{
+		Name:      "rule",
+		Action:    cliutil.ErrorHandler(testURLCommand),
+		Usage:     "Check which ingress rule matches a given request URL",
+		UsageText: "cloudflared tunnel [--config FILEPATH] ingress rule URL",
+		ArgsUsage: "URL",
+		Description: "Check which ingress rule matches a given request URL. " +
+			"Ingress rules match a request's hostname and path. Hostname is " +
+			"optional and is either a full hostname like `www.example.com` or a " +
+			"hostname with a `*` for its subdomains, e.g. `*.example.com`. Path " +
+			"is optional and matches a regular expression, like `/[a-zA-Z0-9_]+.html`",
+	}
+}
+
+// validateIngressCommand check the syntax of the ingress rules in the cloudflared config file
+func validateIngressCommand(c *cli.Context) error {
+	conf := config.GetConfiguration()
+	if conf.Source() == "" {
+		fmt.Println("No configuration file was found. Please create one, or use the --config flag to specify its filepath. You can use the help command to learn more about configuration files")
+		return nil
+	}
+	fmt.Println("Validating rules from", conf.Source())
+	if _, err := ingress.ParseIngress(conf); err != nil {
+		return errors.Wrap(err, "Validation failed")
+	}
+	if c.IsSet("url") {
+		return ingress.ErrURLIncompatibleWithIngress
+	}
+	fmt.Println("OK")
+	return nil
+}
+
+// testURLCommand checks which ingress rule matches the given URL.
+func testURLCommand(c *cli.Context) error {
+	requestArg := c.Args().First()
+	if requestArg == "" {
+		return errors.New("cloudflared tunnel rule expects a single argument, the URL to test")
+	}
+
+	requestURL, err := url.Parse(requestArg)
+	if err != nil {
+		return fmt.Errorf("%s is not a valid URL", requestArg)
+	}
+	if requestURL.Hostname() == "" && requestURL.Scheme == "" {
+		return fmt.Errorf("%s doesn't have a hostname, consider adding a scheme", requestArg)
+	}
+
+	conf := config.GetConfiguration()
+	fmt.Println("Using rules from", conf.Source())
+	ing, err := ingress.ParseIngress(conf)
+	if err != nil {
+		return errors.Wrap(err, "Validation failed")
+	}
+
+	_, i := ing.FindMatchingRule(requestURL.Hostname(), requestURL.Path)
+	fmt.Printf("Matched rule #%d\n", i+1)
+	fmt.Println(ing.Rules[i].MultiLineString())
+	return nil
+}

cmd/cloudflared/tunnel/login.go

@@ -2,17 +2,20 @@ package tunnel
 
 import (
 	"fmt"
+	"io/ioutil"
 	"net/url"
 	"os"
 	"path/filepath"
 	"syscall"
 
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/transfer"
-	"github.com/cloudflare/cloudflared/logger"
 	homedir "github.com/mitchellh/go-homedir"
 	"github.com/pkg/errors"
 	cli "github.com/urfave/cli/v2"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/transfer"
+	"github.com/cloudflare/cloudflared/logger"
 )
 
 const (
@@ -20,8 +23,24 @@ const (
 	callbackStoreURL = "https://login.argotunnel.com/"
 )
 
+func buildLoginSubcommand(hidden bool) *cli.Command {
+	return &cli.Command{
+		Name:      "login",
+		Action:    cliutil.ErrorHandler(login),
+		Usage:     "Generate a configuration file with your login details",
+		ArgsUsage: " ",
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:   "url",
+				Hidden: true,
+			},
+		},
+		Hidden: hidden,
+	}
+}
+
 func login(c *cli.Context) error {
-	logger, err := logger.New()
+	logger, err := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
 	if err != nil {
 		return errors.Wrap(err, "error setting up logger")
 	}
@@ -40,12 +59,16 @@ func login(c *cli.Context) error {
 		return err
 	}
 
-	_, err = transfer.Run(loginURL, "cert", "callback", callbackStoreURL, path, false, false, logger)
+	resourceData, err := transfer.Run(loginURL, "cert", "callback", callbackStoreURL, false, false, logger)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "Failed to write the certificate due to the following error:\n%v\n\nYour browser will download the certificate instead. You will have to manually\ncopy it to the following path:\n\n%s\n", err, path)
 		return err
 	}
 
+	if err := ioutil.WriteFile(path, resourceData, 0600); err != nil {
+		return errors.Wrap(err, fmt.Sprintf("error writing cert to %s", path))
+	}
+
 	fmt.Fprintf(os.Stdout, "You have successfully logged in.\nIf you wish to copy your credentials to a server, they have been saved to:\n%s\n", path)
 	return nil
 }

cmd/cloudflared/tunnel/subcommand_context.go

@@ -3,40 +3,46 @@ package tunnel
 import (
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"os"
-	"path/filepath"
 	"strings"
 
-	"github.com/cloudflare/cloudflared/certutil"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
-	"github.com/cloudflare/cloudflared/logger"
-	"github.com/cloudflare/cloudflared/origin"
-	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
-	"github.com/cloudflare/cloudflared/tunnelstore"
 	"github.com/google/uuid"
 	"github.com/pkg/errors"
 	"github.com/urfave/cli/v2"
+
+	"github.com/cloudflare/cloudflared/certutil"
+	"github.com/cloudflare/cloudflared/connection"
+	"github.com/cloudflare/cloudflared/logger"
+	"github.com/cloudflare/cloudflared/tunnelstore"
 )
 
+type errInvalidJSONCredential struct {
+	err  error
+	path string
+}
+
+func (e errInvalidJSONCredential) Error() string {
+	return "Invalid JSON when parsing tunnel credentials file"
+}
+
 // subcommandContext carries structs shared between subcommands, to reduce number of arguments needed to
 // pass between subcommands, and make sure they are only initialized once
 type subcommandContext struct {
-	c      *cli.Context
-	logger logger.Service
+	c           *cli.Context
+	logger      logger.Service
+	isUIEnabled bool
+	fs          fileSystem
 
 	// These fields should be accessed using their respective Getter
 	tunnelstoreClient tunnelstore.Client
 	userCredential    *userCredential
-
-	isUIEnabled bool
 }
 
 func newSubcommandContext(c *cli.Context) (*subcommandContext, error) {
 	isUIEnabled := c.IsSet(uiFlag) && c.String("name") != ""
 
 	// If UI is enabled, terminal log output should be disabled -- log should be written into a UI log window instead
-	logger, err := createLogger(c, false, isUIEnabled)
+	logger, err := logger.CreateLoggerFromContext(c, isUIEnabled)
 	if err != nil {
 		return nil, errors.Wrap(err, "error setting up logger")
 	}
@@ -45,9 +51,18 @@ func newSubcommandContext(c *cli.Context) (*subcommandContext, error) {
 		c:           c,
 		logger:      logger,
 		isUIEnabled: isUIEnabled,
+		fs:          realFileSystem{},
 	}, nil
 }
 
+// Returns something that can find the given tunnel's credentials file.
+func (sc *subcommandContext) credentialFinder(tunnelID uuid.UUID) CredFinder {
+	if path := sc.c.String(CredFileFlag); path != "" {
+		return newStaticPath(path, sc.fs)
+	}
+	return newSearchByID(tunnelID, sc.c, sc.logger, sc.fs)
+}
+
 type userCredential struct {
 	cert     *certutil.OriginCert
 	certPath string
@@ -98,49 +113,27 @@ func (sc *subcommandContext) credential() (*userCredential, error) {
 	return sc.userCredential, nil
 }
 
-func (sc *subcommandContext) readTunnelCredentials(tunnelID uuid.UUID) (*pogs.TunnelAuth, error) {
-	filePath, err := sc.tunnelCredentialsPath(tunnelID)
+func (sc *subcommandContext) readTunnelCredentials(credFinder CredFinder) (connection.Credentials, error) {
+	filePath, err := credFinder.Path()
 	if err != nil {
-		return nil, err
+		return connection.Credentials{}, err
 	}
-	body, err := ioutil.ReadFile(filePath)
+	body, err := sc.fs.readFile(filePath)
 	if err != nil {
-		return nil, errors.Wrapf(err, "couldn't read tunnel credentials from %v", filePath)
+		return connection.Credentials{}, errors.Wrapf(err, "couldn't read tunnel credentials from %v", filePath)
 	}
 
-	var auth pogs.TunnelAuth
-	if err = json.Unmarshal(body, &auth); err != nil {
-		return nil, err
-	}
-	return &auth, nil
-}
-
-func (sc *subcommandContext) tunnelCredentialsPath(tunnelID uuid.UUID) (string, error) {
-	if filePath := sc.c.String("credentials-file"); filePath != "" {
-		if validFilePath(filePath) {
-			return filePath, nil
+	var credentials connection.Credentials
+	if err = json.Unmarshal(body, &credentials); err != nil {
+		if strings.HasSuffix(filePath, ".pem") {
+			return connection.Credentials{}, fmt.Errorf("The tunnel credentials file should be .json but you gave a .pem. " +
+				"The tunnel credentials file was originally created by `cloudflared tunnel create`. " +
+				"You may have accidentally used the filepath to cert.pem, which is generated by `cloudflared tunnel " +
+				"login`.")
 		}
+		return connection.Credentials{}, errInvalidJSONCredential{path: filePath, err: err}
 	}
-
-	// Fallback to look for tunnel credentials in the origin cert directory
-	if originCertPath, err := findOriginCert(sc.c, sc.logger); err == nil {
-		originCertDir := filepath.Dir(originCertPath)
-		if filePath, err := tunnelFilePath(tunnelID, originCertDir); err == nil {
-			if validFilePath(filePath) {
-				return filePath, nil
-			}
-		}
-	}
-
-	// Last resort look under default config directories
-	for _, configDir := range config.DefaultConfigSearchDirectories() {
-		if filePath, err := tunnelFilePath(tunnelID, configDir); err == nil {
-			if validFilePath(filePath) {
-				return filePath, nil
-			}
-		}
-	}
-	return "", fmt.Errorf("Tunnel credentials file not found")
+	return credentials, nil
 }
 
 func (sc *subcommandContext) create(name string) (*tunnelstore.Tunnel, error) {
@@ -163,7 +156,14 @@ func (sc *subcommandContext) create(name string) (*tunnelstore.Tunnel, error) {
 	if err != nil {
 		return nil, err
 	}
-	if writeFileErr := writeTunnelCredentials(tunnel.ID, credential.cert.AccountID, credential.certPath, tunnelSecret, sc.logger); err != nil {
+	tunnelCredentials := connection.Credentials{
+		AccountTag:   credential.cert.AccountID,
+		TunnelSecret: tunnelSecret,
+		TunnelID:     tunnel.ID,
+		TunnelName:   name,
+	}
+	filePath, writeFileErr := writeTunnelCredentials(credential.certPath, &tunnelCredentials)
+	if err != nil {
 		var errorLines []string
 		errorLines = append(errorLines, fmt.Sprintf("Your tunnel '%v' was created with ID %v. However, cloudflared couldn't write to the tunnel credentials file at %v.json.", tunnel.Name, tunnel.ID, tunnel.ID))
 		errorLines = append(errorLines, fmt.Sprintf("The file-writing error is: %v", writeFileErr))
@@ -176,6 +176,7 @@ func (sc *subcommandContext) create(name string) (*tunnelstore.Tunnel, error) {
 		errorMsg := strings.Join(errorLines, "\n")
 		return nil, errors.New(errorMsg)
 	}
+	sc.logger.Infof("Tunnel credentials written to %v. cloudflared chose this file based on where your origin certificate was found. Keep this file secret. To revoke these credentials, delete the tunnel.", filePath)
 
 	if outputFormat := sc.c.String(outputFormatFlag.Name); outputFormat != "" {
 		return nil, renderOutput(outputFormat, &tunnel)
@@ -226,31 +227,44 @@ func (sc *subcommandContext) delete(tunnelIDs []uuid.UUID) error {
 			return errors.Wrapf(err, "Error deleting tunnel %s", tunnel.ID)
 		}
 
-		tunnelCredentialsPath, err := sc.tunnelCredentialsPath(tunnel.ID)
-		if err != nil {
-			sc.logger.Infof("Cannot locate tunnel credentials to delete, error: %v. Please delete the file manually", err)
-			return nil
-		}
-
-		if err = os.Remove(tunnelCredentialsPath); err != nil {
-			sc.logger.Infof("Cannot delete tunnel credentials, error: %v. Please delete the file manually", err)
+		credFinder := sc.credentialFinder(id)
+		if tunnelCredentialsPath, err := credFinder.Path(); err == nil {
+			if err = os.Remove(tunnelCredentialsPath); err != nil {
+				sc.logger.Infof("Tunnel %v was deleted, but we could not remove its credentials file  %s: %s. Consider deleting this file manually.", id, tunnelCredentialsPath, err)
+			}
 		}
 	}
 	return nil
 }
 
+// findCredentials will choose the right way to find the credentials file, find it,
+// and add the TunnelID into any old credentials (generated before TUN-3581 added the `TunnelID`
+// field to credentials files)
+func (sc *subcommandContext) findCredentials(tunnelID uuid.UUID) (connection.Credentials, error) {
+	credFinder := sc.credentialFinder(tunnelID)
+	credentials, err := sc.readTunnelCredentials(credFinder)
+	// This line ensures backwards compatibility with credentials files generated before
+	// TUN-3581. Those old credentials files don't have a TunnelID field, so we enrich the struct
+	// with the ID, which we have already resolved from the user input.
+	credentials.TunnelID = tunnelID
+	return credentials, err
+}
+
 func (sc *subcommandContext) run(tunnelID uuid.UUID) error {
-	credentials, err := sc.readTunnelCredentials(tunnelID)
+	credentials, err := sc.findCredentials(tunnelID)
 	if err != nil {
+		if e, ok := err.(errInvalidJSONCredential); ok {
+			sc.logger.Errorf("The credentials file at %s contained invalid JSON. This is probably caused by passing the wrong filepath. Reminder: the credentials file is a .json file created via `cloudflared tunnel create`.", e.path)
+			sc.logger.Errorf("Invalid JSON when parsing credentials file: %s", e.err.Error())
+		}
 		return err
 	}
-
 	return StartServer(
 		sc.c,
 		version,
 		shutdownC,
 		graceShutdownC,
-		&origin.NamedTunnelConfig{Auth: *credentials, ID: tunnelID},
+		&connection.NamedTunnelConfig{Credentials: credentials},
 		sc.logger,
 		sc.isUIEnabled,
 	)
@@ -279,6 +293,7 @@ func (sc *subcommandContext) route(tunnelID uuid.UUID, r tunnelstore.Route) (tun
 	return client.RouteTunnel(tunnelID, r)
 }
 
+// Query Tunnelstore to find the active tunnel with the given name.
 func (sc *subcommandContext) tunnelActive(name string) (*tunnelstore.Tunnel, bool, error) {
 	filter := tunnelstore.NewFilter()
 	filter.NoDeleted()
@@ -301,6 +316,15 @@ func (sc *subcommandContext) findID(input string) (uuid.UUID, error) {
 		return u, nil
 	}
 
+	// Look up name in the credentials file.
+	credFinder := newStaticPath(sc.c.String(CredFileFlag), sc.fs)
+	if credentials, err := sc.readTunnelCredentials(credFinder); err == nil {
+		if credentials.TunnelID != uuid.Nil && input == credentials.TunnelName {
+			return credentials.TunnelID, nil
+		}
+	}
+
+	// Fall back to querying Tunnelstore.
 	if tunnel, found, err := sc.tunnelActive(input); err != nil {
 		return uuid.Nil, err
 	} else if found {

cmd/cloudflared/tunnel/subcommand_context_test.go

@@ -1,11 +1,20 @@
 package tunnel
 
 import (
+	"encoding/base64"
+	"flag"
+	"fmt"
 	"reflect"
 	"testing"
+	"time"
 
+	"github.com/cloudflare/cloudflared/connection"
+	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/tunnelstore"
 	"github.com/google/uuid"
+	"github.com/pkg/errors"
+	"github.com/stretchr/testify/require"
+	"github.com/urfave/cli/v2"
 )
 
 func Test_findIDs(t *testing.T) {
@@ -80,3 +89,266 @@ func Test_findIDs(t *testing.T) {
 		})
 	}
 }
+
+type mockFileSystem struct {
+	rf  func(string) ([]byte, error)
+	vfp func(string) bool
+}
+
+func (fs mockFileSystem) validFilePath(path string) bool {
+	return fs.vfp(path)
+}
+
+func (fs mockFileSystem) readFile(filePath string) ([]byte, error) {
+	return fs.rf(filePath)
+}
+
+func Test_subcommandContext_findCredentials(t *testing.T) {
+	type fields struct {
+		c                 *cli.Context
+		logger            logger.Service
+		isUIEnabled       bool
+		fs                fileSystem
+		tunnelstoreClient tunnelstore.Client
+		userCredential    *userCredential
+	}
+	type args struct {
+		tunnelID uuid.UUID
+	}
+	oldCertPath := "old_cert.json"
+	newCertPath := "new_cert.json"
+	accountTag := "0000d4d14e84bd4ae5a6a02e0000ac63"
+	secret := []byte{211, 79, 177, 245, 179, 194, 152, 127, 140, 71, 18, 46, 183, 209, 10, 24, 192, 150, 55, 249, 211, 16, 167, 30, 113, 51, 152, 168, 72, 100, 205, 144}
+	secretB64 := base64.StdEncoding.EncodeToString(secret)
+	tunnelID := uuid.MustParse("df5ed608-b8b4-4109-89f3-9f2cf199df64")
+	name := "mytunnel"
+
+	fs := mockFileSystem{
+		rf: func(filePath string) ([]byte, error) {
+			if filePath == oldCertPath {
+				// An old credentials file created before TUN-3581 added the new fields
+				return []byte(fmt.Sprintf(`{"AccountTag":"%s","TunnelSecret":"%s"}`, accountTag, secretB64)), nil
+			}
+			if filePath == newCertPath {
+				// A new credentials file created after TUN-3581 with its new fields.
+				return []byte(fmt.Sprintf(`{"AccountTag":"%s","TunnelSecret":"%s","TunnelID":"%s","TunnelName":"%s"}`, accountTag, secretB64, tunnelID, name)), nil
+			}
+			return nil, errors.New("file not found")
+		},
+		vfp: func(string) bool { return true },
+	}
+	logger, err := logger.New()
+	require.NoError(t, err)
+
+	tests := []struct {
+		name    string
+		fields  fields
+		args    args
+		want    connection.Credentials
+		wantErr bool
+	}{
+		{
+			name: "Filepath given leads to old credentials file",
+			fields: fields{
+				logger: logger,
+				fs:     fs,
+				c: func() *cli.Context {
+					flagSet := flag.NewFlagSet("test0", flag.PanicOnError)
+					flagSet.String(CredFileFlag, oldCertPath, "")
+					c := cli.NewContext(cli.NewApp(), flagSet, nil)
+					err = c.Set(CredFileFlag, oldCertPath)
+					return c
+				}(),
+			},
+			args: args{
+				tunnelID: tunnelID,
+			},
+			want: connection.Credentials{
+				AccountTag:   accountTag,
+				TunnelID:     tunnelID,
+				TunnelSecret: secret,
+			},
+		},
+		{
+			name: "Filepath given leads to new credentials file",
+			fields: fields{
+				logger: logger,
+				fs:     fs,
+				c: func() *cli.Context {
+					flagSet := flag.NewFlagSet("test0", flag.PanicOnError)
+					flagSet.String(CredFileFlag, newCertPath, "")
+					c := cli.NewContext(cli.NewApp(), flagSet, nil)
+					err = c.Set(CredFileFlag, newCertPath)
+					return c
+				}(),
+			},
+			args: args{
+				tunnelID: tunnelID,
+			},
+			want: connection.Credentials{
+				AccountTag:   accountTag,
+				TunnelID:     tunnelID,
+				TunnelSecret: secret,
+				TunnelName:   name,
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			sc := &subcommandContext{
+				c:                 tt.fields.c,
+				logger:            tt.fields.logger,
+				isUIEnabled:       tt.fields.isUIEnabled,
+				fs:                tt.fields.fs,
+				tunnelstoreClient: tt.fields.tunnelstoreClient,
+				userCredential:    tt.fields.userCredential,
+			}
+			got, err := sc.findCredentials(tt.args.tunnelID)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("subcommandContext.findCredentials() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("subcommandContext.findCredentials() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+type deleteMockTunnelStore struct {
+	tunnelstore.Client
+	mockTunnels      map[uuid.UUID]mockTunnelBehaviour
+	deletedTunnelIDs []uuid.UUID
+}
+
+type mockTunnelBehaviour struct {
+	tunnel     tunnelstore.Tunnel
+	deleteErr  error
+	cleanupErr error
+}
+
+func newDeleteMockTunnelStore(tunnels ...mockTunnelBehaviour) *deleteMockTunnelStore {
+	mockTunnels := make(map[uuid.UUID]mockTunnelBehaviour)
+	for _, tunnel := range tunnels {
+		mockTunnels[tunnel.tunnel.ID] = tunnel
+	}
+	return &deleteMockTunnelStore{
+		mockTunnels:      mockTunnels,
+		deletedTunnelIDs: make([]uuid.UUID, 0),
+	}
+}
+
+func (d *deleteMockTunnelStore) GetTunnel(tunnelID uuid.UUID) (*tunnelstore.Tunnel, error) {
+	tunnel, ok := d.mockTunnels[tunnelID]
+	if !ok {
+		return nil, fmt.Errorf("Couldn't find tunnel: %v", tunnelID)
+	}
+	return &tunnel.tunnel, nil
+}
+
+func (d *deleteMockTunnelStore) DeleteTunnel(tunnelID uuid.UUID) error {
+	tunnel, ok := d.mockTunnels[tunnelID]
+	if !ok {
+		return fmt.Errorf("Couldn't find tunnel: %v", tunnelID)
+	}
+
+	if tunnel.deleteErr != nil {
+		return tunnel.deleteErr
+	}
+
+	d.deletedTunnelIDs = append(d.deletedTunnelIDs, tunnelID)
+	tunnel.tunnel.DeletedAt = time.Now()
+	delete(d.mockTunnels, tunnelID)
+	return nil
+}
+
+func (d *deleteMockTunnelStore) CleanupConnections(tunnelID uuid.UUID) error {
+	tunnel, ok := d.mockTunnels[tunnelID]
+	if !ok {
+		return fmt.Errorf("Couldn't find tunnel: %v", tunnelID)
+	}
+	return tunnel.cleanupErr
+}
+
+func Test_subcommandContext_Delete(t *testing.T) {
+	type fields struct {
+		c                 *cli.Context
+		logger            logger.Service
+		isUIEnabled       bool
+		fs                fileSystem
+		tunnelstoreClient *deleteMockTunnelStore
+		userCredential    *userCredential
+	}
+	type args struct {
+		tunnelIDs []uuid.UUID
+	}
+	newCertPath := "new_cert.json"
+	tunnelID1 := uuid.MustParse("df5ed608-b8b4-4109-89f3-9f2cf199df64")
+	tunnelID2 := uuid.MustParse("af5ed608-b8b4-4109-89f3-9f2cf199df64")
+	logger, err := logger.New()
+	require.NoError(t, err)
+
+	var tests = []struct {
+		name    string
+		fields  fields
+		args    args
+		want    []uuid.UUID
+		wantErr bool
+	}{
+		{
+			name: "clean up continues if credentials are not found",
+			fields: fields{
+				logger: logger,
+				fs: mockFileSystem{
+					rf: func(filePath string) ([]byte, error) {
+						return nil, errors.New("file not found")
+					},
+					vfp: func(string) bool { return true },
+				},
+				c: func() *cli.Context {
+					flagSet := flag.NewFlagSet("test0", flag.PanicOnError)
+					flagSet.String(CredFileFlag, newCertPath, "")
+					c := cli.NewContext(cli.NewApp(), flagSet, nil)
+					err = c.Set(CredFileFlag, newCertPath)
+					return c
+				}(),
+				tunnelstoreClient: newDeleteMockTunnelStore(
+					mockTunnelBehaviour{
+						tunnel: tunnelstore.Tunnel{ID: tunnelID1},
+					},
+					mockTunnelBehaviour{
+						tunnel: tunnelstore.Tunnel{ID: tunnelID2},
+					},
+				),
+			},
+
+			args: args{
+				tunnelIDs: []uuid.UUID{tunnelID1, tunnelID2},
+			},
+			want: []uuid.UUID{tunnelID1, tunnelID2},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			sc := &subcommandContext{
+				c:                 tt.fields.c,
+				logger:            tt.fields.logger,
+				isUIEnabled:       tt.fields.isUIEnabled,
+				fs:                tt.fields.fs,
+				tunnelstoreClient: tt.fields.tunnelstoreClient,
+				userCredential:    tt.fields.userCredential,
+			}
+			err := sc.delete(tt.args.tunnelIDs)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("subcommandContext.findCredentials() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			got := tt.fields.tunnelstoreClient.deletedTunnelIDs
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("subcommandContext.findCredentials() = %v, want %v", got, tt.want)
+				return
+			}
+		})
+	}
+}

cmd/cloudflared/tunnel/subcommands.go

@@ -17,17 +17,19 @@ import (
 	"github.com/mitchellh/go-homedir"
 	"github.com/pkg/errors"
 	"github.com/urfave/cli/v2"
+	"github.com/urfave/cli/v2/altsrc"
 	"golang.org/x/net/idna"
 	"gopkg.in/yaml.v2"
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
-	"github.com/cloudflare/cloudflared/logger"
-	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/tunnelstore"
 )
 
 const (
-	credFileFlagAlias = "cred-file"
+	CredFileFlagAlias = "cred-file"
+	CredFileFlag      = "credentials-file"
 )
 
 var (
@@ -58,39 +60,53 @@ var (
 		Aliases: []string{"rd"},
 		Usage:   "Include connections that have recently disconnected in the list",
 	}
-	outputFormatFlag = &cli.StringFlag{
+	outputFormatFlag = altsrc.NewStringFlag(&cli.StringFlag{
 		Name:    "output",
 		Aliases: []string{"o"},
 		Usage:   "Render output using given `FORMAT`. Valid options are 'json' or 'yaml'",
-	}
-	forceFlag = &cli.BoolFlag{
+	})
+	forceFlag = altsrc.NewBoolFlag(&cli.BoolFlag{
 		Name:    "force",
 		Aliases: []string{"f"},
 		Usage: "By default, if a tunnel is currently being run from a cloudflared, you can't " +
 			"simultaneously rerun it again from a second cloudflared. The --force flag lets you " +
 			"overwrite the previous tunnel. If you want to use a single hostname with multiple " +
 			"tunnels, you can do so with Cloudflare's Load Balancer product.",
-	}
-	credentialsFileFlag = &cli.StringFlag{
-		Name:    "credentials-file",
-		Aliases: []string{credFileFlagAlias},
+	})
+	credentialsFileFlag = altsrc.NewStringFlag(&cli.StringFlag{
+		Name:    CredFileFlag,
+		Aliases: []string{CredFileFlagAlias},
 		Usage:   "File path of tunnel credentials",
-	}
+		EnvVars: []string{"TUNNEL_CRED_FILE"},
+	})
 	forceDeleteFlag = &cli.BoolFlag{
 		Name:    "force",
 		Aliases: []string{"f"},
 		Usage:   "Allows you to delete a tunnel, even if it has active connections.",
+		EnvVars: []string{"TUNNEL_RUN_FORCE_OVERWRITE"},
 	}
+	selectProtocolFlag = altsrc.NewStringFlag(&cli.StringFlag{
+		Name:    "protocol",
+		Value:   "h2mux",
+		Aliases: []string{"p"},
+		Usage:   fmt.Sprintf("Protocol implementation to connect with Cloudflare's edge network. %s", connection.AvailableProtocolFlagMessage),
+		EnvVars: []string{"TUNNEL_TRANSPORT_PROTOCOL"},
+		Hidden:  true,
+	})
 )
 
 func buildCreateCommand() *cli.Command {
 	return &cli.Command{
-		Name:               "create",
-		Action:             cliutil.ErrorHandler(createCommand),
-		Usage:              "Create a new tunnel with given name",
-		UsageText:          "cloudflared tunnel [tunnel command options] create [create command options]",
-		Description:        "cloudflared tunnel create example will create a tunnel named example, and generate a credential file to run the tunnel",
-		ArgsUsage:          "TUNNEL-NAME",
+		Name:      "create",
+		Action:    cliutil.ErrorHandler(createCommand),
+		Usage:     "Create a new tunnel with given name",
+		UsageText: "cloudflared tunnel [tunnel command options] create [subcommand options] NAME",
+		Description: `Creates a tunnel, registers it with Cloudflare edge and generates credential file used to run this tunnel.
+  Use "cloudflared tunnel route" subcommand to map a DNS name to this tunnel and "cloudflared tunnel run" to start the connection.
+
+  For example, to create a tunnel named 'my-tunnel' run:
+
+  $ cloudflared tunnel create my-tunnel`,
 		Flags:              []cli.Flag{outputFormatFlag},
 		CustomHelpTemplate: commandHelpTemplate(),
 	}
@@ -124,30 +140,21 @@ func tunnelFilePath(tunnelID uuid.UUID, directory string) (string, error) {
 	return homedir.Expand(filePath)
 }
 
-func writeTunnelCredentials(tunnelID uuid.UUID, accountID, originCertPath string, tunnelSecret []byte, logger logger.Service) error {
+func writeTunnelCredentials(
+	originCertPath string,
+	credentials *connection.Credentials,
+) (filePath string, err error) {
 	originCertDir := filepath.Dir(originCertPath)
-	filePath, err := tunnelFilePath(tunnelID, originCertDir)
+	filePath, err = tunnelFilePath(credentials.TunnelID, originCertDir)
 	if err != nil {
-		return err
+		return "", err
 	}
-	body, err := json.Marshal(pogs.TunnelAuth{
-		AccountTag:   accountID,
-		TunnelSecret: tunnelSecret,
-	})
+	// Write the name and ID to the file too
+	body, err := json.Marshal(credentials)
 	if err != nil {
-		return errors.Wrap(err, "Unable to marshal tunnel credentials to JSON")
+		return "", errors.Wrap(err, "Unable to marshal tunnel credentials to JSON")
 	}
-	logger.Infof("Writing tunnel credentials to %v. cloudflared chose this file based on where your origin certificate was found.", filePath)
-	logger.Infof("Keep this file secret. To revoke these credentials, delete the tunnel.")
-	return ioutil.WriteFile(filePath, body, 400)
-}
-
-func validFilePath(path string) bool {
-	fileStat, err := os.Stat(path)
-	if err != nil {
-		return false
-	}
-	return !fileStat.IsDir()
+	return filePath, ioutil.WriteFile(filePath, body, 400)
 }
 
 func buildListCommand() *cli.Command {
@@ -156,8 +163,7 @@ func buildListCommand() *cli.Command {
 		Action:             cliutil.ErrorHandler(listCommand),
 		Usage:              "List existing tunnels",
 		UsageText:          "cloudflared tunnel [tunnel command options] list [subcommand options]",
-		Description:        "cloudflared tunnel list will return all active tunnels, their created time and associated connections. Use -d flag to include deleted tunnels. See the list of options to filter the list",
-		ArgsUsage:          " ",
+		Description:        "cloudflared tunnel list will display all active tunnels, their created time and associated connections. Use -d flag to include deleted tunnels. See the list of options to filter the list",
 		Flags:              []cli.Flag{outputFormatFlag, showDeletedFlag, listNameFlag, listExistedAtFlag, listIDFlag, showRecentlyDisconnected},
 		CustomHelpTemplate: commandHelpTemplate(),
 	}
@@ -262,9 +268,8 @@ func buildDeleteCommand() *cli.Command {
 		Name:               "delete",
 		Action:             cliutil.ErrorHandler(deleteCommand),
 		Usage:              "Delete existing tunnel by UUID or name",
-		UsageText:          "cloudflared tunnel [tunnel command options] delete [subcommand options]",
+		UsageText:          "cloudflared tunnel [tunnel command options] delete [subcommand options] TUNNEL",
 		Description:        "cloudflared tunnel delete will delete tunnels with the given tunnel UUIDs or names. A tunnel cannot be deleted if it has active connections. To delete the tunnel unconditionally, use -f flag.",
-		ArgsUsage:          "TUNNEL",
 		Flags:              []cli.Flag{credentialsFileFlag, forceDeleteFlag},
 		CustomHelpTemplate: commandHelpTemplate(),
 	}
@@ -305,20 +310,23 @@ func buildRunCommand() *cli.Command {
 	flags := []cli.Flag{
 		forceFlag,
 		credentialsFileFlag,
+		selectProtocolFlag,
 	}
 	flags = append(flags, configureProxyFlags(false)...)
 	return &cli.Command{
 		Name:      "run",
 		Action:    cliutil.ErrorHandler(runCommand),
+		Before:    SetFlagsFromConfigFile,
 		Usage:     "Proxy a local web server by running the given tunnel",
-		UsageText: "cloudflared tunnel [tunnel command options] run [subcommand options]",
-		ArgsUsage: "TUNNEL",
-		Description: `Runs the tunnel identified by name or UUUD, creating highly available connections 
-   between your server and the Cloudflare edge.
+		UsageText: "cloudflared tunnel [tunnel command options] run [subcommand options] [TUNNEL]",
+		Description: `Runs the tunnel identified by name or UUUD, creating highly available connections
+  between your server and the Cloudflare edge. You can provide name or UUID of tunnel to run either as the
+  last command line argument or in the configuration file using "tunnel: TUNNEL".
 
-   This command requires the tunnel credentials file created when "cloudflared tunnel create" was run, 
-   however it does not need access to cert.pem from "cloudflared login" if you identify the tunnel by UUID.
-   If you experience other problems running gitthe tunnel, "cloudflared tunnel cleanup" may help by removing any old connection records.
+  This command requires the tunnel credentials file created when "cloudflared tunnel create" was run,
+  however it does not need access to cert.pem from "cloudflared login" if you identify the tunnel by UUID.
+  If you experience other problems running the tunnel, "cloudflared tunnel cleanup" may help by removing
+  any old connection records.
 `,
 		Flags:              flags,
 		CustomHelpTemplate: commandHelpTemplate(),
@@ -331,14 +339,29 @@ func runCommand(c *cli.Context) error {
 		return err
 	}
 
-	if c.NArg() != 1 {
-		return cliutil.UsageError(`"cloudflared tunnel run" requires exactly 1 argument, the ID or name of the tunnel to run.`)
+	if c.NArg() > 1 {
+		return cliutil.UsageError(`"cloudflared tunnel run" accepts only one argument, the ID or name of the tunnel to run.`)
 	}
-	tunnelID, err := sc.findID(c.Args().First())
+	tunnelRef := c.Args().First()
+	if tunnelRef == "" {
+		// see if tunnel id was in the config file
+		tunnelRef = config.GetConfiguration().TunnelID
+		if tunnelRef == "" {
+			return cliutil.UsageError(`"cloudflared tunnel run" requires the ID or name of the tunnel to run as the last command line argument or in the configuration file.`)
+		}
+	}
+
+	return runNamedTunnel(sc, tunnelRef)
+}
+
+func runNamedTunnel(sc *subcommandContext, tunnelRef string) error {
+	tunnelID, err := sc.findID(tunnelRef)
 	if err != nil {
 		return errors.Wrap(err, "error parsing tunnel ID")
 	}
 
+	sc.logger.Infof("Starting tunnel %s", tunnelID.String())
+
 	return sc.run(tunnelID)
 }
 
@@ -347,9 +370,8 @@ func buildCleanupCommand() *cli.Command {
 		Name:               "cleanup",
 		Action:             cliutil.ErrorHandler(cleanupCommand),
 		Usage:              "Cleanup tunnel connections",
-		UsageText:          "cloudflared tunnel [tunnel command options] cleanup [subcommand options]",
+		UsageText:          "cloudflared tunnel [tunnel command options] cleanup [subcommand options] TUNNEL",
 		Description:        "Delete connections for tunnels with the given UUIDs or names.",
-		ArgsUsage:          "TUNNEL",
 		CustomHelpTemplate: commandHelpTemplate(),
 	}
 }
@@ -377,14 +399,13 @@ func buildRouteCommand() *cli.Command {
 		Name:      "route",
 		Action:    cliutil.ErrorHandler(routeCommand),
 		Usage:     "Define what hostname or load balancer can route to this tunnel",
-		UsageText: "cloudflared tunnel [tunnel command options] route [subcommand options]",
+		UsageText: "cloudflared tunnel [tunnel command options] route [subcommand options] dns|lb TUNNEL HOSTNAME [LB-POOL]",
 		Description: `The route defines what hostname or load balancer will proxy requests to this tunnel.
 
    To route a hostname by creating a CNAME to tunnel's address:
       cloudflared tunnel route dns <tunnel ID> <hostname>
    To use this tunnel as a load balancer origin, creating pool and load balancer if necessary:
       cloudflared tunnel route lb <tunnel ID> <load balancer name> <load balancer pool>`,
-		ArgsUsage:          "dns|lb TUNNEL HOSTNAME [LB-POOL]",
 		CustomHelpTemplate: commandHelpTemplate(),
 	}
 }
@@ -400,7 +421,7 @@ func dnsRouteFromArg(c *cli.Context) (tunnelstore.Route, error) {
 	userHostname := c.Args().Get(userHostnameIndex)
 	if userHostname == "" {
 		return nil, cliutil.UsageError("The third argument should be the hostname")
-	} else if !validateHostname(userHostname) {
+	} else if !validateHostname(userHostname, true) {
 		return nil, errors.Errorf("%s is not a valid hostname", userHostname)
 	}
 	return tunnelstore.NewDNSRoute(userHostname), nil
@@ -418,14 +439,14 @@ func lbRouteFromArg(c *cli.Context) (tunnelstore.Route, error) {
 	lbName := c.Args().Get(lbNameIndex)
 	if lbName == "" {
 		return nil, cliutil.UsageError("The third argument should be the load balancer name")
-	} else if !validateHostname(lbName) {
+	} else if !validateHostname(lbName, true) {
 		return nil, errors.Errorf("%s is not a valid load balancer name", lbName)
 	}
 
 	lbPool := c.Args().Get(lbPoolIndex)
 	if lbPool == "" {
 		return nil, cliutil.UsageError("The fourth argument should be the pool name")
-	} else if !validateName(lbPool) {
+	} else if !validateName(lbPool, false) {
 		return nil, errors.Errorf("%s is not a valid pool name", lbPool)
 	}
 
@@ -433,19 +454,23 @@ func lbRouteFromArg(c *cli.Context) (tunnelstore.Route, error) {
 }
 
 var nameRegex = regexp.MustCompile("^[_a-zA-Z0-9][-_.a-zA-Z0-9]*$")
+var hostNameRegex = regexp.MustCompile("^[*_a-zA-Z0-9][-_.a-zA-Z0-9]*$")
 
-func validateName(s string) bool {
+func validateName(s string, allowWildcardSubdomain bool) bool {
+	if allowWildcardSubdomain {
+		return hostNameRegex.MatchString(s)
+	}
 	return nameRegex.MatchString(s)
 }
 
-func validateHostname(s string) bool {
+func validateHostname(s string, allowWildcardSubdomain bool) bool {
 	// Slightly stricter than PunyCodeProfile
 	idnaProfile := idna.New(
 		idna.ValidateLabels(true),
 		idna.VerifyDNSLength(true))
 
 	puny, err := idnaProfile.ToASCII(s)
-	return err == nil && validateName(puny)
+	return err == nil && validateName(puny, allowWildcardSubdomain)
 }
 
 func routeCommand(c *cli.Context) error {
@@ -504,17 +529,16 @@ func commandHelpTemplate() string {
 	}
 	const template = `NAME:
 	{{.HelpName}} - {{.Usage}}
-	
+
 USAGE:
 	{{.UsageText}}
- 
+
 DESCRIPTION:
 	{{.Description}}
- 
+
 TUNNEL COMMAND OPTIONS:
 	%s
- 
-SUBCOMMAND COMMAND OPTIONS:
+SUBCOMMAND OPTIONS:
 	{{range .VisibleFlags}}{{.}}
 	{{end}}
 `

cmd/cloudflared/tunnel/subcommands_test.go

@@ -6,9 +6,8 @@ import (
 	"testing"
 
 	"github.com/cloudflare/cloudflared/tunnelstore"
-	"github.com/mitchellh/go-homedir"
-
 	"github.com/google/uuid"
+	"github.com/mitchellh/go-homedir"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -117,8 +116,64 @@ func TestValidateName(t *testing.T) {
 		{name: "_ab_c.-d-ef", want: true},
 	}
 	for _, tt := range tests {
-		if got := validateName(tt.name); got != tt.want {
+		if got := validateName(tt.name, false); got != tt.want {
 			t.Errorf("validateName() = %v, want %v", got, tt.want)
 		}
 	}
-}
+}
+
+func Test_validateHostname(t *testing.T) {
+	type args struct {
+		s                      string
+		allowWildcardSubdomain bool
+	}
+	tests := []struct {
+		name string
+		args args
+		want bool
+	}{
+		{
+			name: "Normal",
+			args: args{
+				s:                      "example.com",
+				allowWildcardSubdomain: true,
+			},
+			want: true,
+		},
+		{
+			name: "wildcard subdomain for TUN-358",
+			args: args{
+				s:                      "*.ehrig.io",
+				allowWildcardSubdomain: true,
+			},
+			want: true,
+		},
+		{
+			name: "Misplaced wildcard",
+			args: args{
+				s:                      "subdomain.*.ehrig.io",
+				allowWildcardSubdomain: true,
+			},
+		},
+		{
+			name: "Invalid domain",
+			args: args{
+				s:                      "..",
+				allowWildcardSubdomain: true,
+			},
+		},
+		{
+			name: "Invalid domain",
+			args: args{
+				s: "..",
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := validateHostname(tt.args.s, tt.args.allowWildcardSubdomain); got != tt.want {
+				t.Errorf("validateHostname() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

cmd/cloudflared/ui/launch_ui.go

@@ -3,8 +3,11 @@ package ui
 import (
 	"context"
 	"fmt"
+	"strings"
 	"time"
 
+	"github.com/cloudflare/cloudflared/connection"
+	"github.com/cloudflare/cloudflared/ingress"
 	"github.com/cloudflare/cloudflared/logger"
 
 	"github.com/gdamore/tcell"
@@ -13,32 +16,15 @@ import (
 
 type connState struct {
 	location string
-	state    status
-}
-
-type status int
-
-const (
-	Disconnected status = iota
-	Connected
-	Reconnecting
-	SetUrl
-	RegisteringTunnel
-)
-
-type TunnelEvent struct {
-	Index     uint8
-	EventType status
-	Location  string
-	Url       string
+	state    connection.Status
 }
 
 type uiModel struct {
-	version     string
-	edgeURL     string
-	metricsURL  string
-	proxyURL    string
-	connections []connState
+	version       string
+	edgeURL       string
+	metricsURL    string
+	localServices []string
+	connections   []connState
 }
 
 type palette struct {
@@ -49,27 +35,32 @@ type palette struct {
 	reconnecting string
 }
 
-func NewUIModel(version, hostname, metricsURL, proxyURL string, haConnections int) *uiModel {
+func NewUIModel(version, hostname, metricsURL string, ing *ingress.Ingress, haConnections int) *uiModel {
+	localServices := make([]string, len(ing.Rules))
+	for i, rule := range ing.Rules {
+		localServices[i] = rule.Service.String()
+	}
 	return &uiModel{
-		version:     version,
-		edgeURL:     hostname,
-		metricsURL:  metricsURL,
-		proxyURL:    proxyURL,
-		connections: make([]connState, haConnections),
+		version:       version,
+		edgeURL:       hostname,
+		metricsURL:    metricsURL,
+		localServices: localServices,
+		connections:   make([]connState, haConnections),
 	}
 }
 
 func (data *uiModel) LaunchUI(
 	ctx context.Context,
-	log logger.Service,
+	generalLogger, transportLogger logger.Service,
 	logLevels []logger.Level,
-	tunnelEventChan <-chan TunnelEvent,
+	tunnelEventChan <-chan connection.Event,
 ) {
 	// Configure the logger to stream logs into the textview
 
 	// Add TextView as a group to write output to
 	logTextView := NewDynamicColorTextView()
-	log.Add(logTextView, logger.NewUIFormatter(time.RFC3339), logLevels...)
+	generalLogger.Add(logTextView, logger.NewUIFormatter(time.RFC3339), logLevels...)
+	transportLogger.Add(logTextView, logger.NewUIFormatter(time.RFC3339), logLevels...)
 
 	// Construct the UI
 	palette := palette{
@@ -107,7 +98,8 @@ func (data *uiModel) LaunchUI(
 	tunnelHostText := tview.NewTextView().SetText(data.edgeURL)
 
 	grid.AddItem(tunnelHostText, 0, 1, 1, 1, 0, 0, false)
-	grid.AddItem(NewDynamicColorTextView().SetText(fmt.Sprintf("[%s]\u2022[%s] Proxying to [%s::b]%s", palette.connected, palette.defaultText, palette.url, data.proxyURL)), 1, 1, 1, 1, 0, 0, false)
+	status := fmt.Sprintf("[%s]\u2022[%s] Proxying to [%s::b]%s", palette.connected, palette.defaultText, palette.url, strings.Join(data.localServices, ", "))
+	grid.AddItem(NewDynamicColorTextView().SetText(status), 1, 1, 1, 1, 0, 0, false)
 
 	grid.AddItem(connTable, 2, 1, 1, 1, 0, 0, false)
 
@@ -130,14 +122,14 @@ func (data *uiModel) LaunchUI(
 				return
 			case event := <-tunnelEventChan:
 				switch event.EventType {
-				case Connected:
+				case connection.Connected:
 					data.setConnTableCell(event, connTable, palette)
-				case Disconnected, Reconnecting:
-					data.changeConnStatus(event, connTable, log, palette)
-				case SetUrl:
-					tunnelHostText.SetText(event.Url)
-					data.edgeURL = event.Url
-				case RegisteringTunnel:
+				case connection.Disconnected, connection.Reconnecting:
+					data.changeConnStatus(event, connTable, generalLogger, palette)
+				case connection.SetURL:
+					tunnelHostText.SetText(event.URL)
+					data.edgeURL = event.URL
+				case connection.RegisteringTunnel:
 					if data.edgeURL == "" {
 						tunnelHostText.SetText("Registering tunnel...")
 					}
@@ -149,7 +141,7 @@ func (data *uiModel) LaunchUI(
 
 	go func() {
 		if err := app.SetRoot(frame, true).Run(); err != nil {
-			log.Errorf("Error launching UI: %s", err)
+			generalLogger.Errorf("Error launching UI: %s", err)
 		}
 	}()
 }
@@ -167,7 +159,7 @@ func handleNewText(app *tview.Application, logTextView *tview.TextView) func() {
 	}
 }
 
-func (data *uiModel) changeConnStatus(event TunnelEvent, table *tview.Table, logger logger.Service, palette palette) {
+func (data *uiModel) changeConnStatus(event connection.Event, table *tview.Table, logger logger.Service, palette palette) {
 	index := int(event.Index)
 	// Get connection location and state
 	connState := data.getConnState(index)
@@ -179,10 +171,10 @@ func (data *uiModel) changeConnStatus(event TunnelEvent, table *tview.Table, log
 
 	locationState := event.Location
 
-	if event.EventType == Disconnected {
-		connState.state = Disconnected
-	} else if event.EventType == Reconnecting {
-		connState.state = Reconnecting
+	if event.EventType == connection.Disconnected {
+		connState.state = connection.Disconnected
+	} else if event.EventType == connection.Reconnecting {
+		connState.state = connection.Reconnecting
 		locationState = "Reconnecting..."
 	}
 
@@ -203,12 +195,12 @@ func (data *uiModel) getConnState(connID int) *connState {
 	return nil
 }
 
-func (data *uiModel) setConnTableCell(event TunnelEvent, table *tview.Table, palette palette) {
+func (data *uiModel) setConnTableCell(event connection.Event, table *tview.Table, palette palette) {
 	index := int(event.Index)
 	connectionNum := index + 1
 
 	// Update slice to keep track of connection location and state in UI table
-	data.connections[index].state = Connected
+	data.connections[index].state = connection.Connected
 	data.connections[index].location = event.Location
 
 	// Update text in table cell to show disconnected state
@@ -217,18 +209,18 @@ func (data *uiModel) setConnTableCell(event TunnelEvent, table *tview.Table, pal
 	table.SetCell(index, 0, cell)
 }
 
-func newCellText(palette palette, connectionNum int, location string, connectedStatus status) string {
+func newCellText(palette palette, connectionNum int, location string, connectedStatus connection.Status) string {
 	// HA connection indicator formatted as: "• #<CONNECTION_INDEX>: <COLO>",
 	//  where the left middle dot's color depends on the status of the connection
 	const connFmtString = "[%s]\u2022[%s] #%d: %s"
 
 	var dotColor string
 	switch connectedStatus {
-	case Connected:
+	case connection.Connected:
 		dotColor = palette.connected
-	case Disconnected:
+	case connection.Disconnected:
 		dotColor = palette.disconnected
-	case Reconnecting:
+	case connection.Reconnecting:
 		dotColor = palette.reconnecting
 	}
 

cmd/cloudflared/updater/update.go

@@ -114,7 +114,7 @@ func checkForUpdateAndApply(options updateOptions) UpdateOutcome {
 
 // Update is the handler for the update command from the command line
 func Update(c *cli.Context) error {
-	logger, err := logger.New()
+	logger, err := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
 	if err != nil {
 		return errors.Wrap(err, "error setting up logger")
 	}
@@ -202,7 +202,6 @@ func (a *AutoUpdater) Run(ctx context.Context) error {
 		if a.configurable.enabled {
 			updateOutcome := loggedUpdate(a.logger, updateOptions{})
 			if updateOutcome.Updated {
-				os.Args = append(os.Args, "--is-autoupdated=true")
 				if IsSysV() {
 					// SysV doesn't have a mechanism to keep service alive, we have to restart the process
 					a.logger.Info("Restarting service managed by SysV...")

cmd/cloudflared/updater/workers_service.go

@@ -7,7 +7,6 @@ import (
 	"runtime"
 	"strconv"
 	"strings"
-	"time"
 )
 
 // Options are the update options supported by the
@@ -53,7 +52,7 @@ func NewWorkersService(currentVersion, url, targetPath string, opts Options) Ser
 // Check does a check in with the Workers API to get a new version update
 func (s *WorkersService) Check() (Version, error) {
 	client := &http.Client{
-		Timeout: time.Second * 5,
+		Timeout: clientTimeout,
 	}
 
 	req, err := http.NewRequest(http.MethodGet, s.url, nil)

cmd/cloudflared/updater/workers_update.go

@@ -18,20 +18,23 @@ import (
 	"time"
 )
 
-// stop the service
-// rename cloudflared.exe to cloudflared.exe.old
-// rename cloudflared.exe.new to cloudflared.exe
-// delete cloudflared.exe.old
-// start the service
-// delete the batch file
-const windowsUpdateCommandTemplate = `@echo off
+const (
+	clientTimeout = time.Second * 60
+	// stop the service
+	// rename cloudflared.exe to cloudflared.exe.old
+	// rename cloudflared.exe.new to cloudflared.exe
+	// delete cloudflared.exe.old
+	// start the service
+	// delete the batch file
+	windowsUpdateCommandTemplate = `@echo off
 sc stop cloudflared >nul 2>&1
 rename "{{.TargetPath}}" {{.OldName}}
 rename "{{.NewPath}}" {{.BinaryName}}
 del "{{.OldPath}}"
 sc start cloudflared >nul 2>&1
 del {{.BatchName}}`
-const batchFileName = "cfd_update.bat"
+	batchFileName = "cfd_update.bat"
+)
 
 // Prepare some data to insert into the template.
 type batchData struct {
@@ -118,7 +121,7 @@ func (v *WorkersVersion) String() string {
 // download the file from the link in the json
 func download(url, filepath string, isCompressed bool) error {
 	client := &http.Client{
-		Timeout: time.Second * 5,
+		Timeout: clientTimeout,
 	}
 	resp, err := client.Get(url)
 

cmd/cloudflared/windows_service.go

@@ -173,7 +173,7 @@ func (s *windowsService) Execute(serviceArgs []string, r <-chan svc.ChangeReques
 }
 
 func installWindowsService(c *cli.Context) error {
-	logger, err := logger.New()
+	logger, err := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
 	if err != nil {
 		return errors.Wrap(err, "error setting up logger")
 	}
@@ -219,7 +219,7 @@ func installWindowsService(c *cli.Context) error {
 }
 
 func uninstallWindowsService(c *cli.Context) error {
-	logger, err := logger.New()
+	logger, err := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
 	if err != nil {
 		return errors.Wrap(err, "error setting up logger")
 	}

connection/connection.go

@@ -0,0 +1,79 @@
+package connection
+
+import (
+	"io"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/google/uuid"
+)
+
+const (
+	lbProbeUserAgentPrefix = "Mozilla/5.0 (compatible; Cloudflare-Traffic-Manager/1.0; +https://www.cloudflare.com/traffic-manager/;"
+)
+
+type Config struct {
+	OriginClient    OriginClient
+	GracePeriod     time.Duration
+	ReplaceExisting bool
+}
+
+type NamedTunnelConfig struct {
+	Credentials Credentials
+	Client      pogs.ClientInfo
+}
+
+// Credentials are stored in the credentials file and contain all info needed to run a tunnel.
+type Credentials struct {
+	AccountTag   string
+	TunnelSecret []byte
+	TunnelID     uuid.UUID
+	TunnelName   string
+}
+
+func (c *Credentials) Auth() pogs.TunnelAuth {
+	return pogs.TunnelAuth{
+		AccountTag:   c.AccountTag,
+		TunnelSecret: c.TunnelSecret,
+	}
+}
+
+type ClassicTunnelConfig struct {
+	Hostname   string
+	OriginCert []byte
+	// feature-flag to use new edge reconnect tokens
+	UseReconnectToken bool
+}
+
+func (c *ClassicTunnelConfig) IsTrialZone() bool {
+	return c.Hostname == ""
+}
+
+type OriginClient interface {
+	Proxy(w ResponseWriter, req *http.Request, isWebsocket bool) error
+}
+
+type ResponseWriter interface {
+	WriteRespHeaders(*http.Response) error
+	WriteErrorResponse()
+	io.ReadWriter
+}
+
+type ConnectedFuse interface {
+	Connected()
+	IsConnected() bool
+}
+
+func IsServerSentEvent(headers http.Header) bool {
+	if contentType := headers.Get("content-type"); contentType != "" {
+		return strings.HasPrefix(strings.ToLower(contentType), "text/event-stream")
+	}
+	return false
+}
+
+func uint8ToString(input uint8) string {
+	return strconv.FormatUint(uint64(input), 10)
+}

connection/connection_test.go

@@ -0,0 +1,152 @@
+package connection
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"testing"
+	"time"
+
+	"github.com/cloudflare/cloudflared/logger"
+	"github.com/gobwas/ws/wsutil"
+	"github.com/stretchr/testify/assert"
+)
+
+const (
+	largeFileSize = 2 * 1024 * 1024
+)
+
+var (
+	testConfig = &Config{
+		OriginClient: &mockOriginClient{},
+		GracePeriod:  time.Millisecond * 100,
+	}
+	testLogger, _ = logger.New()
+	testOriginURL = &url.URL{
+		Scheme: "https",
+		Host:   "connectiontest.argotunnel.com",
+	}
+	testTunnelEventChan = make(chan Event)
+	testObserver        = &Observer{
+		testLogger,
+		m,
+		[]chan Event{testTunnelEventChan},
+		false,
+	}
+	testLargeResp = make([]byte, largeFileSize)
+)
+
+type testRequest struct {
+	name           string
+	endpoint       string
+	expectedStatus int
+	expectedBody   []byte
+	isProxyError   bool
+}
+
+type mockOriginClient struct {
+}
+
+func (moc *mockOriginClient) Proxy(w ResponseWriter, r *http.Request, isWebsocket bool) error {
+	if isWebsocket {
+		return wsEndpoint(w, r)
+	}
+	switch r.URL.Path {
+	case "/ok":
+		originRespEndpoint(w, http.StatusOK, []byte(http.StatusText(http.StatusOK)))
+	case "/large_file":
+		originRespEndpoint(w, http.StatusOK, testLargeResp)
+	case "/400":
+		originRespEndpoint(w, http.StatusBadRequest, []byte(http.StatusText(http.StatusBadRequest)))
+	case "/500":
+		originRespEndpoint(w, http.StatusInternalServerError, []byte(http.StatusText(http.StatusInternalServerError)))
+	case "/error":
+		return fmt.Errorf("Failed to proxy to origin")
+	default:
+		originRespEndpoint(w, http.StatusNotFound, []byte("page not found"))
+	}
+	return nil
+}
+
+type nowriter struct {
+	io.Reader
+}
+
+func (nowriter) Write(p []byte) (int, error) {
+	return 0, fmt.Errorf("Writer not implemented")
+}
+
+func wsEndpoint(w ResponseWriter, r *http.Request) error {
+	resp := &http.Response{
+		StatusCode: http.StatusSwitchingProtocols,
+	}
+	w.WriteRespHeaders(resp)
+	clientReader := nowriter{r.Body}
+	go func() {
+		for {
+			data, err := wsutil.ReadClientText(clientReader)
+			if err != nil {
+				return
+			}
+			if err := wsutil.WriteServerText(w, data); err != nil {
+				return
+			}
+		}
+	}()
+	<-r.Context().Done()
+	return nil
+}
+
+func originRespEndpoint(w ResponseWriter, status int, data []byte) {
+	resp := &http.Response{
+		StatusCode: status,
+	}
+	w.WriteRespHeaders(resp)
+	w.Write(data)
+}
+
+type mockConnectedFuse struct{}
+
+func (mcf mockConnectedFuse) Connected() {}
+
+func (mcf mockConnectedFuse) IsConnected() bool {
+	return true
+}
+
+func TestIsEventStream(t *testing.T) {
+	tests := []struct {
+		headers       http.Header
+		isEventStream bool
+	}{
+		{
+			headers:       newHeader("Content-Type", "text/event-stream"),
+			isEventStream: true,
+		},
+		{
+			headers:       newHeader("content-type", "text/event-stream"),
+			isEventStream: true,
+		},
+		{
+			headers:       newHeader("Content-Type", "text/event-stream; charset=utf-8"),
+			isEventStream: true,
+		},
+		{
+			headers:       newHeader("Content-Type", "application/json"),
+			isEventStream: false,
+		},
+		{
+			headers:       http.Header{},
+			isEventStream: false,
+		},
+	}
+	for _, test := range tests {
+		assert.Equal(t, test.isEventStream, IsServerSentEvent(test.headers))
+	}
+}
+
+func newHeader(key, value string) http.Header {
+	header := http.Header{}
+	header.Add(key, value)
+	return header
+}

connection/errors.go

@@ -0,0 +1,76 @@
+package connection
+
+import (
+	"github.com/cloudflare/cloudflared/edgediscovery"
+	"github.com/cloudflare/cloudflared/h2mux"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const (
+	DuplicateConnectionError = "EDUPCONN"
+)
+
+// RegisterTunnel error from client
+type clientRegisterTunnelError struct {
+	cause error
+}
+
+func newRPCError(cause error, counter *prometheus.CounterVec, name rpcName) clientRegisterTunnelError {
+	counter.WithLabelValues(cause.Error(), string(name)).Inc()
+	return clientRegisterTunnelError{cause: cause}
+}
+
+func (e clientRegisterTunnelError) Error() string {
+	return e.cause.Error()
+}
+
+type DupConnRegisterTunnelError struct{}
+
+var errDuplicationConnection = &DupConnRegisterTunnelError{}
+
+func (e DupConnRegisterTunnelError) Error() string {
+	return "already connected to this server, trying another address"
+}
+
+// RegisterTunnel error from server
+type serverRegisterTunnelError struct {
+	cause     error
+	permanent bool
+}
+
+func (e serverRegisterTunnelError) Error() string {
+	return e.cause.Error()
+}
+
+func serverRegistrationErrorFromRPC(err error) *serverRegisterTunnelError {
+	if retryable, ok := err.(*tunnelpogs.RetryableError); ok {
+		return &serverRegisterTunnelError{
+			cause:     retryable.Unwrap(),
+			permanent: false,
+		}
+	}
+	return &serverRegisterTunnelError{
+		cause:     err,
+		permanent: true,
+	}
+}
+
+type muxerShutdownError struct{}
+
+func (e muxerShutdownError) Error() string {
+	return "muxer shutdown"
+}
+
+func isHandshakeErrRecoverable(err error, connIndex uint8, observer *Observer) bool {
+	switch err.(type) {
+	case edgediscovery.DialError:
+		observer.Errorf("Connection %d unable to dial edge: %s", connIndex, err)
+	case h2mux.MuxerHandshakeError:
+		observer.Errorf("Connection %d handshake with edge server failed: %s", connIndex, err)
+	default:
+		observer.Errorf("Connection %d failed: %s", connIndex, err)
+		return false
+	}
+	return true
+}

connection/event.go

@@ -0,0 +1,25 @@
+package connection
+
+// Event is something that happened to a connection, e.g. disconnection or registration.
+type Event struct {
+	Index     uint8
+	EventType Status
+	Location  string
+	URL       string
+}
+
+// Status is the status of a connection.
+type Status int
+
+const (
+	// Disconnected means the connection to the edge was broken.
+	Disconnected Status = iota
+	// Connected means the connection to the edge was successfully established.
+	Connected
+	// Reconnecting means the connection to the edge is being re-established.
+	Reconnecting
+	// SetURL means this connection's tunnel was given a URL by the edge. Used for free tunnels.
+	SetURL
+	// RegisteringTunnel means the non-named tunnel is registering its connection.
+	RegisteringTunnel
+)

connection/h2mux.go

@@ -0,0 +1,220 @@
+package connection
+
+import (
+	"context"
+	"net"
+	"net/http"
+	"time"
+
+	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/logger"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/cloudflare/cloudflared/websocket"
+	"github.com/pkg/errors"
+	"golang.org/x/sync/errgroup"
+)
+
+const (
+	muxerTimeout      = 5 * time.Second
+	openStreamTimeout = 30 * time.Second
+)
+
+type h2muxConnection struct {
+	config      *Config
+	muxerConfig *MuxerConfig
+	muxer       *h2mux.Muxer
+	// connectionID is only used by metrics, and prometheus requires labels to be string
+	connIndexStr string
+	connIndex    uint8
+
+	observer *Observer
+}
+
+type MuxerConfig struct {
+	HeartbeatInterval  time.Duration
+	MaxHeartbeats      uint64
+	CompressionSetting h2mux.CompressionSetting
+	MetricsUpdateFreq  time.Duration
+}
+
+func (mc *MuxerConfig) H2MuxerConfig(h h2mux.MuxedStreamHandler, logger logger.Service) *h2mux.MuxerConfig {
+	return &h2mux.MuxerConfig{
+		Timeout:            muxerTimeout,
+		Handler:            h,
+		IsClient:           true,
+		HeartbeatInterval:  mc.HeartbeatInterval,
+		MaxHeartbeats:      mc.MaxHeartbeats,
+		Logger:             logger,
+		CompressionQuality: mc.CompressionSetting,
+	}
+}
+
+// NewTunnelHandler returns a TunnelHandler, origin LAN IP and error
+func NewH2muxConnection(ctx context.Context,
+	config *Config,
+	muxerConfig *MuxerConfig,
+	edgeConn net.Conn,
+	connIndex uint8,
+	observer *Observer,
+) (*h2muxConnection, error, bool) {
+	h := &h2muxConnection{
+		config:       config,
+		muxerConfig:  muxerConfig,
+		connIndexStr: uint8ToString(connIndex),
+		connIndex:    connIndex,
+		observer:     observer,
+	}
+
+	// Establish a muxed connection with the edge
+	// Client mux handshake with agent server
+	muxer, err := h2mux.Handshake(edgeConn, edgeConn, *muxerConfig.H2MuxerConfig(h, observer), h2mux.ActiveStreams)
+	if err != nil {
+		recoverable := isHandshakeErrRecoverable(err, connIndex, observer)
+		return nil, err, recoverable
+	}
+	h.muxer = muxer
+	return h, nil, false
+}
+
+func (h *h2muxConnection) ServeNamedTunnel(ctx context.Context, namedTunnel *NamedTunnelConfig, credentialManager CredentialManager, connOptions *tunnelpogs.ConnectionOptions, connectedFuse ConnectedFuse) error {
+	errGroup, serveCtx := errgroup.WithContext(ctx)
+	errGroup.Go(func() error {
+		return h.serveMuxer(serveCtx)
+	})
+
+	errGroup.Go(func() error {
+		stream, err := h.newRPCStream(serveCtx, register)
+		if err != nil {
+			return err
+		}
+		rpcClient := newRegistrationRPCClient(ctx, stream, h.observer)
+		defer rpcClient.Close()
+
+		if err = rpcClient.RegisterConnection(serveCtx, namedTunnel, connOptions, h.connIndex, h.observer); err != nil {
+			return err
+		}
+		connectedFuse.Connected()
+		return nil
+	})
+
+	errGroup.Go(func() error {
+		h.controlLoop(serveCtx, connectedFuse, true)
+		return nil
+	})
+	return errGroup.Wait()
+}
+
+func (h *h2muxConnection) ServeClassicTunnel(ctx context.Context, classicTunnel *ClassicTunnelConfig, credentialManager CredentialManager, registrationOptions *tunnelpogs.RegistrationOptions, connectedFuse ConnectedFuse) error {
+	errGroup, serveCtx := errgroup.WithContext(ctx)
+	errGroup.Go(func() error {
+		return h.serveMuxer(serveCtx)
+	})
+
+	errGroup.Go(func() (err error) {
+		defer func() {
+			if err == nil {
+				connectedFuse.Connected()
+			}
+		}()
+		if classicTunnel.UseReconnectToken && connectedFuse.IsConnected() {
+			err := h.reconnectTunnel(ctx, credentialManager, classicTunnel, registrationOptions)
+			if err == nil {
+				return nil
+			}
+			// log errors and proceed to RegisterTunnel
+			h.observer.Errorf("Couldn't reconnect connection %d. Reregistering it instead. Error was: %v", h.connIndex, err)
+		}
+		return h.registerTunnel(ctx, credentialManager, classicTunnel, registrationOptions)
+	})
+
+	errGroup.Go(func() error {
+		h.controlLoop(serveCtx, connectedFuse, false)
+		return nil
+	})
+	return errGroup.Wait()
+}
+
+func (h *h2muxConnection) serveMuxer(ctx context.Context) error {
+	// All routines should stop when muxer finish serving. When muxer is shutdown
+	// gracefully, it doesn't return an error, so we need to return errMuxerShutdown
+	// here to notify other routines to stop
+	err := h.muxer.Serve(ctx)
+	if err == nil {
+		return muxerShutdownError{}
+	}
+	return err
+}
+
+func (h *h2muxConnection) controlLoop(ctx context.Context, connectedFuse ConnectedFuse, isNamedTunnel bool) {
+	updateMetricsTickC := time.Tick(h.muxerConfig.MetricsUpdateFreq)
+	for {
+		select {
+		case <-ctx.Done():
+			// UnregisterTunnel blocks until the RPC call returns
+			if connectedFuse.IsConnected() {
+				h.unregister(isNamedTunnel)
+			}
+			h.muxer.Shutdown()
+			return
+		case <-updateMetricsTickC:
+			h.observer.metrics.updateMuxerMetrics(h.connIndexStr, h.muxer.Metrics())
+		}
+	}
+}
+
+func (h *h2muxConnection) newRPCStream(ctx context.Context, rpcName rpcName) (*h2mux.MuxedStream, error) {
+	openStreamCtx, openStreamCancel := context.WithTimeout(ctx, openStreamTimeout)
+	defer openStreamCancel()
+	stream, err := h.muxer.OpenRPCStream(openStreamCtx)
+	if err != nil {
+		return nil, err
+	}
+	return stream, nil
+}
+
+func (h *h2muxConnection) ServeStream(stream *h2mux.MuxedStream) error {
+	respWriter := &h2muxRespWriter{stream}
+
+	req, reqErr := h.newRequest(stream)
+	if reqErr != nil {
+		respWriter.WriteErrorResponse()
+		return reqErr
+	}
+
+	err := h.config.OriginClient.Proxy(respWriter, req, websocket.IsWebSocketUpgrade(req))
+	if err != nil {
+		respWriter.WriteErrorResponse()
+		return err
+	}
+	return nil
+}
+
+func (h *h2muxConnection) newRequest(stream *h2mux.MuxedStream) (*http.Request, error) {
+	req, err := http.NewRequest("GET", "http://localhost:8080", h2mux.MuxedStreamReader{MuxedStream: stream})
+	if err != nil {
+		return nil, errors.Wrap(err, "Unexpected error from http.NewRequest")
+	}
+	err = h2mux.H2RequestHeadersToH1Request(stream.Headers, req)
+	if err != nil {
+		return nil, errors.Wrap(err, "invalid request received")
+	}
+	return req, nil
+}
+
+type h2muxRespWriter struct {
+	*h2mux.MuxedStream
+}
+
+func (rp *h2muxRespWriter) WriteRespHeaders(resp *http.Response) error {
+	headers := h2mux.H1ResponseToH2ResponseHeaders(resp)
+	headers = append(headers, h2mux.Header{Name: ResponseMetaHeaderField, Value: responseMetaHeaderOrigin})
+	return rp.WriteHeaders(headers)
+}
+
+func (rp *h2muxRespWriter) WriteErrorResponse() {
+	rp.WriteHeaders([]h2mux.Header{
+		{Name: ":status", Value: "502"},
+		{Name: ResponseMetaHeaderField, Value: responseMetaHeaderCfd},
+	})
+	rp.Write([]byte("502 Bad Gateway"))
+}

connection/h2mux_test.go

@@ -0,0 +1,242 @@
+package connection
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"strconv"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/gobwas/ws/wsutil"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+var (
+	testMuxerConfig = &MuxerConfig{
+		HeartbeatInterval:  time.Second * 5,
+		MaxHeartbeats:      5,
+		CompressionSetting: 0,
+		MetricsUpdateFreq:  time.Second * 5,
+	}
+)
+
+func newH2MuxConnection(ctx context.Context, t require.TestingT) (*h2muxConnection, *h2mux.Muxer) {
+	edgeConn, originConn := net.Pipe()
+	edgeMuxChan := make(chan *h2mux.Muxer)
+	go func() {
+		edgeMuxConfig := h2mux.MuxerConfig{
+			Logger: testObserver,
+		}
+		edgeMux, err := h2mux.Handshake(edgeConn, edgeConn, edgeMuxConfig, h2mux.ActiveStreams)
+		require.NoError(t, err)
+		edgeMuxChan <- edgeMux
+	}()
+	var connIndex = uint8(0)
+	h2muxConn, err, _ := NewH2muxConnection(ctx, testConfig, testMuxerConfig, originConn, connIndex, testObserver)
+	require.NoError(t, err)
+	return h2muxConn, <-edgeMuxChan
+}
+
+func TestServeStreamHTTP(t *testing.T) {
+	tests := []testRequest{
+		{
+			name:           "ok",
+			endpoint:       "/ok",
+			expectedStatus: http.StatusOK,
+			expectedBody:   []byte(http.StatusText(http.StatusOK)),
+		},
+		{
+			name:           "large_file",
+			endpoint:       "/large_file",
+			expectedStatus: http.StatusOK,
+			expectedBody:   testLargeResp,
+		},
+		{
+			name:           "Bad request",
+			endpoint:       "/400",
+			expectedStatus: http.StatusBadRequest,
+			expectedBody:   []byte(http.StatusText(http.StatusBadRequest)),
+		},
+		{
+			name:           "Internal server error",
+			endpoint:       "/500",
+			expectedStatus: http.StatusInternalServerError,
+			expectedBody:   []byte(http.StatusText(http.StatusInternalServerError)),
+		},
+		{
+			name:           "Proxy error",
+			endpoint:       "/error",
+			expectedStatus: http.StatusBadGateway,
+			expectedBody:   nil,
+			isProxyError:   true,
+		},
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	h2muxConn, edgeMux := newH2MuxConnection(ctx, t)
+
+	var wg sync.WaitGroup
+	wg.Add(2)
+	go func() {
+		defer wg.Done()
+		edgeMux.Serve(ctx)
+	}()
+	go func() {
+		defer wg.Done()
+		err := h2muxConn.serveMuxer(ctx)
+		require.Error(t, err)
+	}()
+
+	for _, test := range tests {
+		headers := []h2mux.Header{
+			{
+				Name:  ":path",
+				Value: test.endpoint,
+			},
+		}
+		stream, err := edgeMux.OpenStream(ctx, headers, nil)
+		require.NoError(t, err)
+		require.True(t, hasHeader(stream, ":status", strconv.Itoa(test.expectedStatus)))
+
+		if test.isProxyError {
+			assert.True(t, hasHeader(stream, ResponseMetaHeaderField, responseMetaHeaderCfd))
+		} else {
+			assert.True(t, hasHeader(stream, ResponseMetaHeaderField, responseMetaHeaderOrigin))
+			body := make([]byte, len(test.expectedBody))
+			_, err = stream.Read(body)
+			require.NoError(t, err)
+			require.Equal(t, test.expectedBody, body)
+		}
+	}
+	cancel()
+	wg.Wait()
+}
+
+func TestServeStreamWS(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	h2muxConn, edgeMux := newH2MuxConnection(ctx, t)
+
+	var wg sync.WaitGroup
+	wg.Add(2)
+	go func() {
+		defer wg.Done()
+		edgeMux.Serve(ctx)
+	}()
+	go func() {
+		defer wg.Done()
+		err := h2muxConn.serveMuxer(ctx)
+		require.Error(t, err)
+	}()
+
+	headers := []h2mux.Header{
+		{
+			Name:  ":path",
+			Value: "/ws",
+		},
+		{
+			Name:  "connection",
+			Value: "upgrade",
+		},
+		{
+			Name:  "upgrade",
+			Value: "websocket",
+		},
+	}
+
+	readPipe, writePipe := io.Pipe()
+	stream, err := edgeMux.OpenStream(ctx, headers, readPipe)
+	require.NoError(t, err)
+
+	require.True(t, hasHeader(stream, ":status", strconv.Itoa(http.StatusSwitchingProtocols)))
+	assert.True(t, hasHeader(stream, ResponseMetaHeaderField, responseMetaHeaderOrigin))
+
+	data := []byte("test websocket")
+	err = wsutil.WriteClientText(writePipe, data)
+	require.NoError(t, err)
+
+	respBody, err := wsutil.ReadServerText(stream)
+	require.NoError(t, err)
+	require.Equal(t, data, respBody, fmt.Sprintf("Expect %s, got %s", string(data), string(respBody)))
+
+	cancel()
+	wg.Wait()
+}
+
+func hasHeader(stream *h2mux.MuxedStream, name, val string) bool {
+	for _, header := range stream.Headers {
+		if header.Name == name && header.Value == val {
+			return true
+		}
+	}
+	return false
+}
+
+func benchmarkServeStreamHTTPSimple(b *testing.B, test testRequest) {
+	ctx, cancel := context.WithCancel(context.Background())
+	h2muxConn, edgeMux := newH2MuxConnection(ctx, b)
+
+	var wg sync.WaitGroup
+	wg.Add(2)
+	go func() {
+		defer wg.Done()
+		edgeMux.Serve(ctx)
+	}()
+	go func() {
+		defer wg.Done()
+		err := h2muxConn.serveMuxer(ctx)
+		require.Error(b, err)
+	}()
+
+	headers := []h2mux.Header{
+		{
+			Name:  ":path",
+			Value: test.endpoint,
+		},
+	}
+
+	body := make([]byte, len(test.expectedBody))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		b.StartTimer()
+		stream, openstreamErr := edgeMux.OpenStream(ctx, headers, nil)
+		_, readBodyErr := stream.Read(body)
+		b.StopTimer()
+
+		require.NoError(b, openstreamErr)
+		assert.True(b, hasHeader(stream, ResponseMetaHeaderField, responseMetaHeaderOrigin))
+		require.True(b, hasHeader(stream, ":status", strconv.Itoa(http.StatusOK)))
+		require.NoError(b, readBodyErr)
+		require.Equal(b, test.expectedBody, body)
+	}
+
+	cancel()
+	wg.Wait()
+}
+
+func BenchmarkServeStreamHTTPSimple(b *testing.B) {
+	test := testRequest{
+		name:           "ok",
+		endpoint:       "/ok",
+		expectedStatus: http.StatusOK,
+		expectedBody:   []byte(http.StatusText(http.StatusOK)),
+	}
+
+	benchmarkServeStreamHTTPSimple(b, test)
+}
+
+func BenchmarkServeStreamHTTPLargeFile(b *testing.B) {
+	test := testRequest{
+		name:           "large_file",
+		endpoint:       "/large_file",
+		expectedStatus: http.StatusOK,
+		expectedBody:   testLargeResp,
+	}
+
+	benchmarkServeStreamHTTPSimple(b, test)
+}

connection/header.go

@@ -0,0 +1,31 @@
+package connection
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/cloudflare/cloudflared/h2mux"
+)
+
+const (
+	ResponseMetaHeaderField = "cf-cloudflared-response-meta"
+)
+
+var (
+	canonicalResponseUserHeadersField = http.CanonicalHeaderKey(h2mux.ResponseUserHeadersField)
+	canonicalResponseMetaHeaderField  = http.CanonicalHeaderKey(ResponseMetaHeaderField)
+	responseMetaHeaderCfd             = mustInitRespMetaHeader("cloudflared")
+	responseMetaHeaderOrigin          = mustInitRespMetaHeader("origin")
+)
+
+type responseMetaHeader struct {
+	Source string `json:"src"`
+}
+
+func mustInitRespMetaHeader(src string) string {
+	header, err := json.Marshal(responseMetaHeader{Source: src})
+	if err != nil {
+		panic(fmt.Sprintf("Failed to serialize response meta header = %s, err: %v", src, err))
+	}
+	return string(header)
+}

connection/http2.go

@@ -0,0 +1,221 @@
+package connection
+
+import (
+	"context"
+	"errors"
+	"io"
+	"math"
+	"net"
+	"net/http"
+	"strings"
+	"sync"
+
+	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/logger"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+
+	"golang.org/x/net/http2"
+)
+
+const (
+	internalUpgradeHeader = "Cf-Cloudflared-Proxy-Connection-Upgrade"
+	websocketUpgrade      = "websocket"
+	controlStreamUpgrade  = "control-stream"
+)
+
+var (
+	errNotFlusher = errors.New("ResponseWriter doesn't implement http.Flusher")
+)
+
+type http2Connection struct {
+	conn         net.Conn
+	server       *http2.Server
+	config       *Config
+	namedTunnel  *NamedTunnelConfig
+	connOptions  *tunnelpogs.ConnectionOptions
+	observer     *Observer
+	connIndexStr string
+	connIndex    uint8
+	wg           *sync.WaitGroup
+	// newRPCClientFunc allows us to mock RPCs during testing
+	newRPCClientFunc func(context.Context, io.ReadWriteCloser, logger.Service) NamedTunnelRPCClient
+	connectedFuse    ConnectedFuse
+}
+
+func NewHTTP2Connection(
+	conn net.Conn,
+	config *Config,
+	namedTunnelConfig *NamedTunnelConfig,
+	connOptions *tunnelpogs.ConnectionOptions,
+	observer *Observer,
+	connIndex uint8,
+	connectedFuse ConnectedFuse,
+) *http2Connection {
+	return &http2Connection{
+		conn: conn,
+		server: &http2.Server{
+			MaxConcurrentStreams: math.MaxUint32,
+		},
+		config:           config,
+		namedTunnel:      namedTunnelConfig,
+		connOptions:      connOptions,
+		observer:         observer,
+		connIndexStr:     uint8ToString(connIndex),
+		connIndex:        connIndex,
+		wg:               &sync.WaitGroup{},
+		newRPCClientFunc: newRegistrationRPCClient,
+		connectedFuse:    connectedFuse,
+	}
+}
+
+func (c *http2Connection) Serve(ctx context.Context) {
+	go func() {
+		<-ctx.Done()
+		c.close()
+	}()
+	c.server.ServeConn(c.conn, &http2.ServeConnOpts{
+		Context: ctx,
+		Handler: c,
+	})
+}
+
+func (c *http2Connection) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	c.wg.Add(1)
+	defer c.wg.Done()
+
+	respWriter := &http2RespWriter{
+		r: r.Body,
+		w: w,
+	}
+	flusher, isFlusher := w.(http.Flusher)
+	if !isFlusher {
+		c.observer.Errorf("%T doesn't implement http.Flusher", w)
+		respWriter.WriteErrorResponse()
+		return
+	}
+	respWriter.flusher = flusher
+	var err error
+	if isControlStreamUpgrade(r) {
+		respWriter.shouldFlush = true
+		err = c.serveControlStream(r.Context(), respWriter)
+	} else if isWebsocketUpgrade(r) {
+		respWriter.shouldFlush = true
+		stripWebsocketUpgradeHeader(r)
+		err = c.config.OriginClient.Proxy(respWriter, r, true)
+	} else {
+		err = c.config.OriginClient.Proxy(respWriter, r, false)
+	}
+
+	if err != nil {
+		respWriter.WriteErrorResponse()
+	}
+}
+
+func (c *http2Connection) serveControlStream(ctx context.Context, respWriter *http2RespWriter) error {
+	rpcClient := c.newRPCClientFunc(ctx, respWriter, c.observer)
+	defer rpcClient.Close()
+
+	if err := rpcClient.RegisterConnection(ctx, c.namedTunnel, c.connOptions, c.connIndex, c.observer); err != nil {
+		return err
+	}
+	c.connectedFuse.Connected()
+
+	<-ctx.Done()
+	rpcClient.GracefulShutdown(ctx, c.config.GracePeriod)
+	return nil
+}
+
+func (c *http2Connection) close() {
+	// Wait for all serve HTTP handlers to return
+	c.wg.Wait()
+	c.conn.Close()
+}
+
+type http2RespWriter struct {
+	r           io.Reader
+	w           http.ResponseWriter
+	flusher     http.Flusher
+	shouldFlush bool
+}
+
+func (rp *http2RespWriter) WriteRespHeaders(resp *http.Response) error {
+	dest := rp.w.Header()
+	userHeaders := make(http.Header, len(resp.Header))
+	for header, values := range resp.Header {
+		// Since these are http2 headers, they're required to be lowercase
+		h2name := strings.ToLower(header)
+		for _, v := range values {
+			if h2name == "content-length" {
+				// This header has meaning in HTTP/2 and will be used by the edge,
+				// so it should be sent as an HTTP/2 response header.
+				dest.Add(h2name, v)
+				// Since these are http2 headers, they're required to be lowercase
+			} else if !h2mux.IsControlHeader(h2name) || h2mux.IsWebsocketClientHeader(h2name) {
+				// User headers, on the other hand, must all be serialized so that
+				// HTTP/2 header validation won't be applied to HTTP/1 header values
+				userHeaders.Add(h2name, v)
+			}
+		}
+	}
+
+	// Perform user header serialization and set them in the single header
+	dest.Set(canonicalResponseUserHeadersField, h2mux.SerializeHeaders(userHeaders))
+	rp.setResponseMetaHeader(responseMetaHeaderOrigin)
+	status := resp.StatusCode
+	// HTTP2 removes support for 101 Switching Protocols https://tools.ietf.org/html/rfc7540#section-8.1.1
+	if status == http.StatusSwitchingProtocols {
+		status = http.StatusOK
+	}
+	rp.w.WriteHeader(status)
+	if IsServerSentEvent(resp.Header) {
+		rp.shouldFlush = true
+	}
+	if rp.shouldFlush {
+		rp.flusher.Flush()
+	}
+	return nil
+}
+
+func (rp *http2RespWriter) WriteErrorResponse() {
+	rp.setResponseMetaHeader(responseMetaHeaderCfd)
+	rp.w.WriteHeader(http.StatusBadGateway)
+}
+
+func (rp *http2RespWriter) setResponseMetaHeader(value string) {
+	rp.w.Header().Set(canonicalResponseMetaHeaderField, value)
+}
+
+func (rp *http2RespWriter) Read(p []byte) (n int, err error) {
+	return rp.r.Read(p)
+}
+
+func (rp *http2RespWriter) Write(p []byte) (n int, err error) {
+	defer func() {
+		// Implementer of OriginClient should make sure it doesn't write to the connection after Proxy returns
+		// Register a recover routine just in case.
+		if r := recover(); r != nil {
+			println("Recover from http2 response writer panic, error", r)
+		}
+	}()
+	n, err = rp.w.Write(p)
+	if err == nil && rp.shouldFlush {
+		rp.flusher.Flush()
+	}
+	return n, err
+}
+
+func (rp *http2RespWriter) Close() error {
+	return nil
+}
+
+func isControlStreamUpgrade(r *http.Request) bool {
+	return strings.ToLower(r.Header.Get(internalUpgradeHeader)) == controlStreamUpgrade
+}
+
+func isWebsocketUpgrade(r *http.Request) bool {
+	return strings.ToLower(r.Header.Get(internalUpgradeHeader)) == websocketUpgrade
+}
+
+func stripWebsocketUpgradeHeader(r *http.Request) {
+	r.Header.Del(internalUpgradeHeader)
+}

connection/http2_test.go

@@ -0,0 +1,303 @@
+package connection
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/cloudflare/cloudflared/logger"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/gobwas/ws/wsutil"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/net/http2"
+)
+
+var (
+	testTransport = http2.Transport{}
+)
+
+func newTestHTTP2Connection() (*http2Connection, net.Conn) {
+	edgeConn, originConn := net.Pipe()
+	var connIndex = uint8(0)
+	return NewHTTP2Connection(
+		originConn,
+		testConfig,
+		&NamedTunnelConfig{},
+		&pogs.ConnectionOptions{},
+		testObserver,
+		connIndex,
+		mockConnectedFuse{},
+	), edgeConn
+}
+
+func TestServeHTTP(t *testing.T) {
+	tests := []testRequest{
+		{
+			name:           "ok",
+			endpoint:       "ok",
+			expectedStatus: http.StatusOK,
+			expectedBody:   []byte(http.StatusText(http.StatusOK)),
+		},
+		{
+			name:           "large_file",
+			endpoint:       "large_file",
+			expectedStatus: http.StatusOK,
+			expectedBody:   testLargeResp,
+		},
+		{
+			name:           "Bad request",
+			endpoint:       "400",
+			expectedStatus: http.StatusBadRequest,
+			expectedBody:   []byte(http.StatusText(http.StatusBadRequest)),
+		},
+		{
+			name:           "Internal server error",
+			endpoint:       "500",
+			expectedStatus: http.StatusInternalServerError,
+			expectedBody:   []byte(http.StatusText(http.StatusInternalServerError)),
+		},
+		{
+			name:           "Proxy error",
+			endpoint:       "error",
+			expectedStatus: http.StatusBadGateway,
+			expectedBody:   nil,
+			isProxyError:   true,
+		},
+	}
+
+	http2Conn, edgeConn := newTestHTTP2Connection()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		http2Conn.Serve(ctx)
+	}()
+
+	edgeHTTP2Conn, err := testTransport.NewClientConn(edgeConn)
+	require.NoError(t, err)
+
+	for _, test := range tests {
+		endpoint := fmt.Sprintf("http://localhost:8080/%s", test.endpoint)
+		req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
+		require.NoError(t, err)
+
+		resp, err := edgeHTTP2Conn.RoundTrip(req)
+		require.NoError(t, err)
+		require.Equal(t, test.expectedStatus, resp.StatusCode)
+		if test.expectedBody != nil {
+			respBody, err := ioutil.ReadAll(resp.Body)
+			require.NoError(t, err)
+			require.Equal(t, test.expectedBody, respBody)
+		}
+		if test.isProxyError {
+			require.Equal(t, responseMetaHeaderCfd, resp.Header.Get(ResponseMetaHeaderField))
+		} else {
+			require.Equal(t, responseMetaHeaderOrigin, resp.Header.Get(ResponseMetaHeaderField))
+		}
+	}
+	cancel()
+	wg.Wait()
+}
+
+type mockNamedTunnelRPCClient struct {
+	registered   chan struct{}
+	unregistered chan struct{}
+}
+
+func (mc mockNamedTunnelRPCClient) RegisterConnection(
+	c context.Context,
+	config *NamedTunnelConfig,
+	options *tunnelpogs.ConnectionOptions,
+	connIndex uint8,
+	observer *Observer,
+) error {
+	close(mc.registered)
+	return nil
+}
+
+func (mc mockNamedTunnelRPCClient) GracefulShutdown(ctx context.Context, gracePeriod time.Duration) {
+	close(mc.unregistered)
+}
+
+func (mockNamedTunnelRPCClient) Close() {}
+
+type mockRPCClientFactory struct {
+	registered   chan struct{}
+	unregistered chan struct{}
+}
+
+func (mf *mockRPCClientFactory) newMockRPCClient(context.Context, io.ReadWriteCloser, logger.Service) NamedTunnelRPCClient {
+	return mockNamedTunnelRPCClient{
+		registered:   mf.registered,
+		unregistered: mf.unregistered,
+	}
+}
+
+type wsRespWriter struct {
+	*httptest.ResponseRecorder
+	readPipe  *io.PipeReader
+	writePipe *io.PipeWriter
+}
+
+func newWSRespWriter() *wsRespWriter {
+	readPipe, writePipe := io.Pipe()
+	return &wsRespWriter{
+		httptest.NewRecorder(),
+		readPipe,
+		writePipe,
+	}
+}
+
+func (w *wsRespWriter) RespBody() io.ReadWriter {
+	return nowriter{w.readPipe}
+}
+
+func (w *wsRespWriter) Write(data []byte) (n int, err error) {
+	return w.writePipe.Write(data)
+}
+
+func TestServeWS(t *testing.T) {
+	http2Conn, _ := newTestHTTP2Connection()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		http2Conn.Serve(ctx)
+	}()
+
+	respWriter := newWSRespWriter()
+	readPipe, writePipe := io.Pipe()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://localhost:8080/ws", readPipe)
+	require.NoError(t, err)
+	req.Header.Set(internalUpgradeHeader, websocketUpgrade)
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		http2Conn.ServeHTTP(respWriter, req)
+	}()
+
+	data := []byte("test websocket")
+	err = wsutil.WriteClientText(writePipe, data)
+	require.NoError(t, err)
+
+	respBody, err := wsutil.ReadServerText(respWriter.RespBody())
+	require.NoError(t, err)
+	require.Equal(t, data, respBody, fmt.Sprintf("Expect %s, got %s", string(data), string(respBody)))
+
+	cancel()
+	resp := respWriter.Result()
+	// http2RespWriter should rewrite status 101 to 200
+	require.Equal(t, http.StatusOK, resp.StatusCode)
+	require.Equal(t, responseMetaHeaderOrigin, resp.Header.Get(ResponseMetaHeaderField))
+
+	wg.Wait()
+}
+
+func TestServeControlStream(t *testing.T) {
+	http2Conn, edgeConn := newTestHTTP2Connection()
+
+	rpcClientFactory := mockRPCClientFactory{
+		registered:   make(chan struct{}),
+		unregistered: make(chan struct{}),
+	}
+	http2Conn.newRPCClientFunc = rpcClientFactory.newMockRPCClient
+
+	ctx, cancel := context.WithCancel(context.Background())
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		http2Conn.Serve(ctx)
+	}()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://localhost:8080/", nil)
+	require.NoError(t, err)
+	req.Header.Set(internalUpgradeHeader, controlStreamUpgrade)
+
+	edgeHTTP2Conn, err := testTransport.NewClientConn(edgeConn)
+	require.NoError(t, err)
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		edgeHTTP2Conn.RoundTrip(req)
+	}()
+
+	<-rpcClientFactory.registered
+	cancel()
+	<-rpcClientFactory.unregistered
+
+	wg.Wait()
+}
+
+func benchmarkServeHTTP(b *testing.B, test testRequest) {
+	http2Conn, edgeConn := newTestHTTP2Connection()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		http2Conn.Serve(ctx)
+	}()
+
+	endpoint := fmt.Sprintf("http://localhost:8080/%s", test.endpoint)
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
+	require.NoError(b, err)
+
+	edgeHTTP2Conn, err := testTransport.NewClientConn(edgeConn)
+	require.NoError(b, err)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		b.StartTimer()
+		resp, err := edgeHTTP2Conn.RoundTrip(req)
+		b.StopTimer()
+		require.NoError(b, err)
+		require.Equal(b, test.expectedStatus, resp.StatusCode)
+		if test.expectedBody != nil {
+			respBody, err := ioutil.ReadAll(resp.Body)
+			require.NoError(b, err)
+			require.Equal(b, test.expectedBody, respBody)
+		}
+		resp.Body.Close()
+	}
+
+	cancel()
+	wg.Wait()
+}
+func BenchmarkServeHTTPSimple(b *testing.B) {
+	test := testRequest{
+		name:           "ok",
+		endpoint:       "ok",
+		expectedStatus: http.StatusOK,
+		expectedBody:   []byte(http.StatusText(http.StatusOK)),
+	}
+
+	benchmarkServeHTTP(b, test)
+}
+
+func BenchmarkServeHTTPLargeFile(b *testing.B) {
+	test := testRequest{
+		name:           "large_file",
+		endpoint:       "large_file",
+		expectedStatus: http.StatusOK,
+		expectedBody:   testLargeResp,
+	}
+
+	benchmarkServeHTTP(b, test)
+}

connection/json.go

@@ -0,0 +1,7 @@
+package connection
+
+import (
+	jsoniter "github.com/json-iterator/go"
+)
+
+var json = jsoniter.ConfigFastest

connection/metrics.go

@@ -0,0 +1,405 @@
+package connection
+
+import (
+	"sync"
+	"time"
+
+	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const (
+	MetricsNamespace = "cloudflared"
+	TunnelSubsystem  = "tunnel"
+	muxerSubsystem   = "muxer"
+)
+
+type muxerMetrics struct {
+	rtt              *prometheus.GaugeVec
+	rttMin           *prometheus.GaugeVec
+	rttMax           *prometheus.GaugeVec
+	receiveWindowAve *prometheus.GaugeVec
+	sendWindowAve    *prometheus.GaugeVec
+	receiveWindowMin *prometheus.GaugeVec
+	receiveWindowMax *prometheus.GaugeVec
+	sendWindowMin    *prometheus.GaugeVec
+	sendWindowMax    *prometheus.GaugeVec
+	inBoundRateCurr  *prometheus.GaugeVec
+	inBoundRateMin   *prometheus.GaugeVec
+	inBoundRateMax   *prometheus.GaugeVec
+	outBoundRateCurr *prometheus.GaugeVec
+	outBoundRateMin  *prometheus.GaugeVec
+	outBoundRateMax  *prometheus.GaugeVec
+	compBytesBefore  *prometheus.GaugeVec
+	compBytesAfter   *prometheus.GaugeVec
+	compRateAve      *prometheus.GaugeVec
+}
+
+type tunnelMetrics struct {
+	timerRetries    prometheus.Gauge
+	serverLocations *prometheus.GaugeVec
+	// locationLock is a mutex for oldServerLocations
+	locationLock sync.Mutex
+	// oldServerLocations stores the last server the tunnel was connected to
+	oldServerLocations map[string]string
+
+	regSuccess *prometheus.CounterVec
+	regFail    *prometheus.CounterVec
+	rpcFail    *prometheus.CounterVec
+
+	muxerMetrics        *muxerMetrics
+	tunnelsHA           tunnelsForHA
+	userHostnamesCounts *prometheus.CounterVec
+}
+
+func newMuxerMetrics() *muxerMetrics {
+	rtt := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "rtt",
+			Help:      "Round-trip time in millisecond",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(rtt)
+
+	rttMin := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "rtt_min",
+			Help:      "Shortest round-trip time in millisecond",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(rttMin)
+
+	rttMax := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "rtt_max",
+			Help:      "Longest round-trip time in millisecond",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(rttMax)
+
+	receiveWindowAve := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "receive_window_ave",
+			Help:      "Average receive window size in bytes",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(receiveWindowAve)
+
+	sendWindowAve := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "send_window_ave",
+			Help:      "Average send window size in bytes",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(sendWindowAve)
+
+	receiveWindowMin := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "receive_window_min",
+			Help:      "Smallest receive window size in bytes",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(receiveWindowMin)
+
+	receiveWindowMax := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "receive_window_max",
+			Help:      "Largest receive window size in bytes",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(receiveWindowMax)
+
+	sendWindowMin := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "send_window_min",
+			Help:      "Smallest send window size in bytes",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(sendWindowMin)
+
+	sendWindowMax := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "send_window_max",
+			Help:      "Largest send window size in bytes",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(sendWindowMax)
+
+	inBoundRateCurr := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "inbound_bytes_per_sec_curr",
+			Help:      "Current inbounding bytes per second, 0 if there is no incoming connection",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(inBoundRateCurr)
+
+	inBoundRateMin := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "inbound_bytes_per_sec_min",
+			Help:      "Minimum non-zero inbounding bytes per second",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(inBoundRateMin)
+
+	inBoundRateMax := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "inbound_bytes_per_sec_max",
+			Help:      "Maximum inbounding bytes per second",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(inBoundRateMax)
+
+	outBoundRateCurr := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "outbound_bytes_per_sec_curr",
+			Help:      "Current outbounding bytes per second, 0 if there is no outgoing traffic",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(outBoundRateCurr)
+
+	outBoundRateMin := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "outbound_bytes_per_sec_min",
+			Help:      "Minimum non-zero outbounding bytes per second",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(outBoundRateMin)
+
+	outBoundRateMax := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "outbound_bytes_per_sec_max",
+			Help:      "Maximum outbounding bytes per second",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(outBoundRateMax)
+
+	compBytesBefore := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "comp_bytes_before",
+			Help:      "Bytes sent via cross-stream compression, pre compression",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(compBytesBefore)
+
+	compBytesAfter := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "comp_bytes_after",
+			Help:      "Bytes sent via cross-stream compression, post compression",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(compBytesAfter)
+
+	compRateAve := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "comp_rate_ave",
+			Help:      "Average outbound cross-stream compression ratio",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(compRateAve)
+
+	return &muxerMetrics{
+		rtt:              rtt,
+		rttMin:           rttMin,
+		rttMax:           rttMax,
+		receiveWindowAve: receiveWindowAve,
+		sendWindowAve:    sendWindowAve,
+		receiveWindowMin: receiveWindowMin,
+		receiveWindowMax: receiveWindowMax,
+		sendWindowMin:    sendWindowMin,
+		sendWindowMax:    sendWindowMax,
+		inBoundRateCurr:  inBoundRateCurr,
+		inBoundRateMin:   inBoundRateMin,
+		inBoundRateMax:   inBoundRateMax,
+		outBoundRateCurr: outBoundRateCurr,
+		outBoundRateMin:  outBoundRateMin,
+		outBoundRateMax:  outBoundRateMax,
+		compBytesBefore:  compBytesBefore,
+		compBytesAfter:   compBytesAfter,
+		compRateAve:      compRateAve,
+	}
+}
+
+func (m *muxerMetrics) update(connectionID string, metrics *h2mux.MuxerMetrics) {
+	m.rtt.WithLabelValues(connectionID).Set(convertRTTMilliSec(metrics.RTT))
+	m.rttMin.WithLabelValues(connectionID).Set(convertRTTMilliSec(metrics.RTTMin))
+	m.rttMax.WithLabelValues(connectionID).Set(convertRTTMilliSec(metrics.RTTMax))
+	m.receiveWindowAve.WithLabelValues(connectionID).Set(metrics.ReceiveWindowAve)
+	m.sendWindowAve.WithLabelValues(connectionID).Set(metrics.SendWindowAve)
+	m.receiveWindowMin.WithLabelValues(connectionID).Set(float64(metrics.ReceiveWindowMin))
+	m.receiveWindowMax.WithLabelValues(connectionID).Set(float64(metrics.ReceiveWindowMax))
+	m.sendWindowMin.WithLabelValues(connectionID).Set(float64(metrics.SendWindowMin))
+	m.sendWindowMax.WithLabelValues(connectionID).Set(float64(metrics.SendWindowMax))
+	m.inBoundRateCurr.WithLabelValues(connectionID).Set(float64(metrics.InBoundRateCurr))
+	m.inBoundRateMin.WithLabelValues(connectionID).Set(float64(metrics.InBoundRateMin))
+	m.inBoundRateMax.WithLabelValues(connectionID).Set(float64(metrics.InBoundRateMax))
+	m.outBoundRateCurr.WithLabelValues(connectionID).Set(float64(metrics.OutBoundRateCurr))
+	m.outBoundRateMin.WithLabelValues(connectionID).Set(float64(metrics.OutBoundRateMin))
+	m.outBoundRateMax.WithLabelValues(connectionID).Set(float64(metrics.OutBoundRateMax))
+	m.compBytesBefore.WithLabelValues(connectionID).Set(float64(metrics.CompBytesBefore.Value()))
+	m.compBytesAfter.WithLabelValues(connectionID).Set(float64(metrics.CompBytesAfter.Value()))
+	m.compRateAve.WithLabelValues(connectionID).Set(float64(metrics.CompRateAve()))
+}
+
+func convertRTTMilliSec(t time.Duration) float64 {
+	return float64(t / time.Millisecond)
+}
+
+// Metrics that can be collected without asking the edge
+func newTunnelMetrics() *tunnelMetrics {
+	maxConcurrentRequestsPerTunnel := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: TunnelSubsystem,
+			Name:      "max_concurrent_requests_per_tunnel",
+			Help:      "Largest number of concurrent requests proxied through each tunnel so far",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(maxConcurrentRequestsPerTunnel)
+
+	timerRetries := prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: TunnelSubsystem,
+			Name:      "timer_retries",
+			Help:      "Unacknowledged heart beats count",
+		})
+	prometheus.MustRegister(timerRetries)
+
+	serverLocations := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: TunnelSubsystem,
+			Name:      "server_locations",
+			Help:      "Where each tunnel is connected to. 1 means current location, 0 means previous locations.",
+		},
+		[]string{"connection_id", "location"},
+	)
+	prometheus.MustRegister(serverLocations)
+
+	rpcFail := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: TunnelSubsystem,
+			Name:      "tunnel_rpc_fail",
+			Help:      "Count of RPC connection errors by type",
+		},
+		[]string{"error", "rpcName"},
+	)
+	prometheus.MustRegister(rpcFail)
+
+	registerFail := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: TunnelSubsystem,
+			Name:      "tunnel_register_fail",
+			Help:      "Count of tunnel registration errors by type",
+		},
+		[]string{"error", "rpcName"},
+	)
+	prometheus.MustRegister(registerFail)
+
+	userHostnamesCounts := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: TunnelSubsystem,
+			Name:      "user_hostnames_counts",
+			Help:      "Which user hostnames cloudflared is serving",
+		},
+		[]string{"userHostname"},
+	)
+	prometheus.MustRegister(userHostnamesCounts)
+
+	registerSuccess := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: TunnelSubsystem,
+			Name:      "tunnel_register_success",
+			Help:      "Count of successful tunnel registrations",
+		},
+		[]string{"rpcName"},
+	)
+	prometheus.MustRegister(registerSuccess)
+
+	return &tunnelMetrics{
+		timerRetries:        timerRetries,
+		serverLocations:     serverLocations,
+		oldServerLocations:  make(map[string]string),
+		muxerMetrics:        newMuxerMetrics(),
+		tunnelsHA:           NewTunnelsForHA(),
+		regSuccess:          registerSuccess,
+		regFail:             registerFail,
+		rpcFail:             rpcFail,
+		userHostnamesCounts: userHostnamesCounts,
+	}
+}
+
+func (t *tunnelMetrics) updateMuxerMetrics(connectionID string, metrics *h2mux.MuxerMetrics) {
+	t.muxerMetrics.update(connectionID, metrics)
+}
+
+func (t *tunnelMetrics) registerServerLocation(connectionID, loc string) {
+	t.locationLock.Lock()
+	defer t.locationLock.Unlock()
+	if oldLoc, ok := t.oldServerLocations[connectionID]; ok && oldLoc == loc {
+		return
+	} else if ok {
+		t.serverLocations.WithLabelValues(connectionID, oldLoc).Dec()
+	}
+	t.serverLocations.WithLabelValues(connectionID, loc).Inc()
+	t.oldServerLocations[connectionID] = loc
+}

connection/observer.go

@@ -0,0 +1,105 @@
+package connection
+
+import (
+	"fmt"
+	"net/url"
+	"strings"
+
+	"github.com/cloudflare/cloudflared/logger"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+)
+
+type Observer struct {
+	logger.Service
+	metrics          *tunnelMetrics
+	tunnelEventChans []chan Event
+	uiEnabled        bool
+}
+
+func NewObserver(logger logger.Service, tunnelEventChans []chan Event, uiEnabled bool) *Observer {
+	return &Observer{
+		logger,
+		newTunnelMetrics(),
+		tunnelEventChans,
+		uiEnabled,
+	}
+}
+
+func (o *Observer) logServerInfo(connIndex uint8, location, msg string) {
+	o.sendEvent(Event{Index: connIndex, EventType: Connected, Location: location})
+	o.Infof(msg)
+	o.metrics.registerServerLocation(uint8ToString(connIndex), location)
+}
+
+func (o *Observer) logTrialHostname(registration *tunnelpogs.TunnelRegistration) error {
+	// Print out the user's trial zone URL in a nice box (if they requested and got one and UI flag is not set)
+	if !o.uiEnabled {
+		if registrationURL, err := url.Parse(registration.Url); err == nil {
+			for _, line := range asciiBox(trialZoneMsg(registrationURL.String()), 2) {
+				o.Info(line)
+			}
+		} else {
+			o.Error("Failed to connect tunnel, please try again.")
+			return fmt.Errorf("empty URL in response from Cloudflare edge")
+		}
+	}
+	return nil
+}
+
+// Print out the given lines in a nice ASCII box.
+func asciiBox(lines []string, padding int) (box []string) {
+	maxLen := maxLen(lines)
+	spacer := strings.Repeat(" ", padding)
+
+	border := "+" + strings.Repeat("-", maxLen+(padding*2)) + "+"
+
+	box = append(box, border)
+	for _, line := range lines {
+		box = append(box, "|"+spacer+line+strings.Repeat(" ", maxLen-len(line))+spacer+"|")
+	}
+	box = append(box, border)
+	return
+}
+
+func maxLen(lines []string) int {
+	max := 0
+	for _, line := range lines {
+		if len(line) > max {
+			max = len(line)
+		}
+	}
+	return max
+}
+
+func trialZoneMsg(url string) []string {
+	return []string{
+		"Your free tunnel has started! Visit it:",
+		"  " + url,
+	}
+}
+
+func (o *Observer) sendRegisteringEvent() {
+	o.sendEvent(Event{EventType: RegisteringTunnel})
+}
+
+func (o *Observer) sendConnectedEvent(connIndex uint8, location string) {
+	o.sendEvent(Event{Index: connIndex, EventType: Connected, Location: location})
+}
+
+func (o *Observer) sendURL(url string) {
+	o.sendEvent(Event{EventType: SetURL, URL: url})
+}
+
+func (o *Observer) SendReconnect(connIndex uint8) {
+	o.sendEvent(Event{Index: connIndex, EventType: Reconnecting})
+}
+
+func (o *Observer) SendDisconnect(connIndex uint8) {
+	o.sendEvent(Event{Index: connIndex, EventType: Disconnected})
+}
+
+func (o *Observer) sendEvent(e Event) {
+	for _, ch := range o.tunnelEventChans {
+		ch <- e
+	}
+}

connection/observer_test.go

@@ -0,0 +1,45 @@
+package connection
+
+import (
+	"strconv"
+	"sync"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// can only be called once
+var m = newTunnelMetrics()
+
+func TestRegisterServerLocation(t *testing.T) {
+	tunnels := 20
+	var wg sync.WaitGroup
+	wg.Add(tunnels)
+	for i := 0; i < tunnels; i++ {
+		go func(i int) {
+			id := strconv.Itoa(i)
+			m.registerServerLocation(id, "LHR")
+			wg.Done()
+		}(i)
+	}
+	wg.Wait()
+	for i := 0; i < tunnels; i++ {
+		id := strconv.Itoa(i)
+		assert.Equal(t, "LHR", m.oldServerLocations[id])
+	}
+
+	wg.Add(tunnels)
+	for i := 0; i < tunnels; i++ {
+		go func(i int) {
+			id := strconv.Itoa(i)
+			m.registerServerLocation(id, "AUS")
+			wg.Done()
+		}(i)
+	}
+	wg.Wait()
+	for i := 0; i < tunnels; i++ {
+		id := strconv.Itoa(i)
+		assert.Equal(t, "AUS", m.oldServerLocations[id])
+	}
+
+}

connection/protocol.go

@@ -0,0 +1,179 @@
+package connection
+
+import (
+	"fmt"
+	"hash/fnv"
+	"sync"
+	"time"
+
+	"github.com/cloudflare/cloudflared/logger"
+)
+
+const (
+	AvailableProtocolFlagMessage = "Available protocols: http2 - Go's implementation, h2mux - Cloudflare's implementation of HTTP/2, and auto - automatically select between http2 and h2mux"
+	// edgeH2muxTLSServerName is the server name to establish h2mux connection with edge
+	edgeH2muxTLSServerName = "cftunnel.com"
+	// edgeH2TLSServerName is the server name to establish http2 connection with edge
+	edgeH2TLSServerName = "h2.cftunnel.com"
+	// threshold to switch back to h2mux when the user intentionally pick --protocol http2
+	explicitHTTP2FallbackThreshold = -1
+	autoSelectFlag                 = "auto"
+)
+
+var (
+	ProtocolList = []Protocol{H2mux, HTTP2}
+)
+
+type Protocol int64
+
+const (
+	H2mux Protocol = iota
+	HTTP2
+)
+
+func (p Protocol) ServerName() string {
+	switch p {
+	case H2mux:
+		return edgeH2muxTLSServerName
+	case HTTP2:
+		return edgeH2TLSServerName
+	default:
+		return ""
+	}
+}
+
+// Fallback returns the fallback protocol and whether the protocol has a fallback
+func (p Protocol) fallback() (Protocol, bool) {
+	switch p {
+	case H2mux:
+		return 0, false
+	case HTTP2:
+		return H2mux, true
+	default:
+		return 0, false
+	}
+}
+
+func (p Protocol) String() string {
+	switch p {
+	case H2mux:
+		return "h2mux"
+	case HTTP2:
+		return "http2"
+	default:
+		return fmt.Sprintf("unknown protocol")
+	}
+}
+
+type ProtocolSelector interface {
+	Current() Protocol
+	Fallback() (Protocol, bool)
+}
+
+type staticProtocolSelector struct {
+	current Protocol
+}
+
+func (s *staticProtocolSelector) Current() Protocol {
+	return s.current
+}
+
+func (s *staticProtocolSelector) Fallback() (Protocol, bool) {
+	return 0, false
+}
+
+type autoProtocolSelector struct {
+	lock           sync.RWMutex
+	current        Protocol
+	switchThrehold int32
+	fetchFunc      PercentageFetcher
+	refreshAfter   time.Time
+	ttl            time.Duration
+	logger         logger.Service
+}
+
+func newAutoProtocolSelector(
+	current Protocol,
+	switchThrehold int32,
+	fetchFunc PercentageFetcher,
+	ttl time.Duration,
+	logger logger.Service,
+) *autoProtocolSelector {
+	return &autoProtocolSelector{
+		current:        current,
+		switchThrehold: switchThrehold,
+		fetchFunc:      fetchFunc,
+		refreshAfter:   time.Now().Add(ttl),
+		ttl:            ttl,
+		logger:         logger,
+	}
+}
+
+func (s *autoProtocolSelector) Current() Protocol {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+	if time.Now().Before(s.refreshAfter) {
+		return s.current
+	}
+
+	percentage, err := s.fetchFunc()
+	if err != nil {
+		s.logger.Errorf("Failed to refresh protocol, err: %v", err)
+		return s.current
+	}
+
+	if s.switchThrehold < percentage {
+		s.current = HTTP2
+	} else {
+		s.current = H2mux
+	}
+	s.refreshAfter = time.Now().Add(s.ttl)
+	return s.current
+}
+
+func (s *autoProtocolSelector) Fallback() (Protocol, bool) {
+	s.lock.RLock()
+	defer s.lock.RUnlock()
+	return s.current.fallback()
+}
+
+type PercentageFetcher func() (int32, error)
+
+func NewProtocolSelector(protocolFlag string, namedTunnel *NamedTunnelConfig, fetchFunc PercentageFetcher, ttl time.Duration, logger logger.Service) (ProtocolSelector, error) {
+	if namedTunnel == nil {
+		return &staticProtocolSelector{
+			current: H2mux,
+		}, nil
+	}
+	if protocolFlag == H2mux.String() {
+		return &staticProtocolSelector{
+			current: H2mux,
+		}, nil
+	}
+
+	http2Percentage, err := fetchFunc()
+	if err != nil {
+		return nil, err
+	}
+	if protocolFlag == HTTP2.String() {
+		if http2Percentage < 0 {
+			return newAutoProtocolSelector(H2mux, explicitHTTP2FallbackThreshold, fetchFunc, ttl, logger), nil
+		}
+		return newAutoProtocolSelector(HTTP2, explicitHTTP2FallbackThreshold, fetchFunc, ttl, logger), nil
+	}
+
+	if protocolFlag != autoSelectFlag {
+		return nil, fmt.Errorf("Unknown protocol %s, %s", protocolFlag, AvailableProtocolFlagMessage)
+	}
+	threshold := switchThreshold(namedTunnel.Credentials.AccountTag)
+	if threshold < http2Percentage {
+		return newAutoProtocolSelector(HTTP2, threshold, fetchFunc, ttl, logger), nil
+	}
+	return newAutoProtocolSelector(H2mux, threshold, fetchFunc, ttl, logger), nil
+}
+
+func switchThreshold(accountTag string) int32 {
+	h := fnv.New32a()
+	h.Write([]byte(accountTag))
+	return int32(h.Sum32() % 100)
+}

connection/protocol_test.go

@@ -0,0 +1,219 @@
+package connection
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/cloudflare/cloudflared/logger"
+	"github.com/stretchr/testify/assert"
+)
+
+const (
+	testNoTTL = 0
+)
+
+var (
+	testNamedTunnelConfig = &NamedTunnelConfig{
+		Credentials: Credentials{
+			AccountTag: "testAccountTag",
+		},
+	}
+)
+
+func mockFetcher(percentage int32) PercentageFetcher {
+	return func() (int32, error) {
+		return percentage, nil
+	}
+}
+
+func mockFetcherWithError() PercentageFetcher {
+	return func() (int32, error) {
+		return 0, fmt.Errorf("failed to fetch precentage")
+	}
+}
+
+type dynamicMockFetcher struct {
+	percentage int32
+	err        error
+}
+
+func (dmf *dynamicMockFetcher) fetch() PercentageFetcher {
+	return func() (int32, error) {
+		if dmf.err != nil {
+			return 0, dmf.err
+		}
+		return dmf.percentage, nil
+	}
+}
+
+func TestNewProtocolSelector(t *testing.T) {
+	tests := []struct {
+		name              string
+		protocol          string
+		expectedProtocol  Protocol
+		hasFallback       bool
+		expectedFallback  Protocol
+		namedTunnelConfig *NamedTunnelConfig
+		fetchFunc         PercentageFetcher
+		wantErr           bool
+	}{
+		{
+			name:              "classic tunnel",
+			protocol:          "h2mux",
+			expectedProtocol:  H2mux,
+			namedTunnelConfig: nil,
+		},
+		{
+			name:              "named tunnel over h2mux",
+			protocol:          "h2mux",
+			expectedProtocol:  H2mux,
+			namedTunnelConfig: testNamedTunnelConfig,
+		},
+		{
+			name:              "named tunnel over http2",
+			protocol:          "http2",
+			expectedProtocol:  HTTP2,
+			hasFallback:       true,
+			expectedFallback:  H2mux,
+			fetchFunc:         mockFetcher(0),
+			namedTunnelConfig: testNamedTunnelConfig,
+		},
+		{
+			name:              "named tunnel http2 disabled",
+			protocol:          "http2",
+			expectedProtocol:  H2mux,
+			fetchFunc:         mockFetcher(-1),
+			namedTunnelConfig: testNamedTunnelConfig,
+		},
+		{
+			name:              "named tunnel auto all http2 disabled",
+			protocol:          "auto",
+			expectedProtocol:  H2mux,
+			fetchFunc:         mockFetcher(-1),
+			namedTunnelConfig: testNamedTunnelConfig,
+		},
+		{
+			name:              "named tunnel auto to h2mux",
+			protocol:          "auto",
+			expectedProtocol:  H2mux,
+			fetchFunc:         mockFetcher(0),
+			namedTunnelConfig: testNamedTunnelConfig,
+		},
+		{
+			name:              "named tunnel auto to http2",
+			protocol:          "auto",
+			expectedProtocol:  HTTP2,
+			hasFallback:       true,
+			expectedFallback:  H2mux,
+			fetchFunc:         mockFetcher(100),
+			namedTunnelConfig: testNamedTunnelConfig,
+		},
+		{
+			// None named tunnel can only use h2mux, so specifying an unknown protocol is not an error
+			name:             "classic tunnel unknown protocol",
+			protocol:         "unknown",
+			expectedProtocol: H2mux,
+		},
+		{
+			name:              "named tunnel unknown protocol",
+			protocol:          "unknown",
+			fetchFunc:         mockFetcher(100),
+			namedTunnelConfig: testNamedTunnelConfig,
+			wantErr:           true,
+		},
+		{
+			name:              "named tunnel fetch error",
+			protocol:          "unknown",
+			fetchFunc:         mockFetcherWithError(),
+			namedTunnelConfig: testNamedTunnelConfig,
+			wantErr:           true,
+		},
+	}
+	logger, _ := logger.New()
+	for _, test := range tests {
+		selector, err := NewProtocolSelector(test.protocol, test.namedTunnelConfig, test.fetchFunc, testNoTTL, logger)
+		if test.wantErr {
+			assert.Error(t, err, fmt.Sprintf("test %s failed", test.name))
+		} else {
+			assert.NoError(t, err, fmt.Sprintf("test %s failed", test.name))
+			assert.Equal(t, test.expectedProtocol, selector.Current(), fmt.Sprintf("test %s failed", test.name))
+			fallback, ok := selector.Fallback()
+			assert.Equal(t, test.hasFallback, ok, fmt.Sprintf("test %s failed", test.name))
+			if test.hasFallback {
+				assert.Equal(t, test.expectedFallback, fallback, fmt.Sprintf("test %s failed", test.name))
+			}
+		}
+	}
+}
+
+func TestAutoProtocolSelectorRefresh(t *testing.T) {
+	logger, _ := logger.New()
+	fetcher := dynamicMockFetcher{}
+	selector, err := NewProtocolSelector("auto", testNamedTunnelConfig, fetcher.fetch(), testNoTTL, logger)
+	assert.NoError(t, err)
+	assert.Equal(t, H2mux, selector.Current())
+
+	fetcher.percentage = 100
+	assert.Equal(t, HTTP2, selector.Current())
+
+	fetcher.percentage = 0
+	assert.Equal(t, H2mux, selector.Current())
+
+	fetcher.percentage = 100
+	assert.Equal(t, HTTP2, selector.Current())
+
+	fetcher.err = fmt.Errorf("failed to fetch")
+	assert.Equal(t, HTTP2, selector.Current())
+
+	fetcher.percentage = -1
+	fetcher.err = nil
+	assert.Equal(t, H2mux, selector.Current())
+
+	fetcher.percentage = 0
+	assert.Equal(t, H2mux, selector.Current())
+
+	fetcher.percentage = 100
+	assert.Equal(t, HTTP2, selector.Current())
+}
+
+func TestHTTP2ProtocolSelectorRefresh(t *testing.T) {
+	logger, _ := logger.New()
+	fetcher := dynamicMockFetcher{}
+	selector, err := NewProtocolSelector("http2", testNamedTunnelConfig, fetcher.fetch(), testNoTTL, logger)
+	assert.NoError(t, err)
+	assert.Equal(t, HTTP2, selector.Current())
+
+	fetcher.percentage = 100
+	assert.Equal(t, HTTP2, selector.Current())
+
+	fetcher.percentage = 0
+	assert.Equal(t, HTTP2, selector.Current())
+
+	fetcher.err = fmt.Errorf("failed to fetch")
+	assert.Equal(t, HTTP2, selector.Current())
+
+	fetcher.percentage = -1
+	fetcher.err = nil
+	assert.Equal(t, H2mux, selector.Current())
+
+	fetcher.percentage = 0
+	assert.Equal(t, HTTP2, selector.Current())
+
+	fetcher.percentage = 100
+	assert.Equal(t, HTTP2, selector.Current())
+
+	fetcher.percentage = -1
+	assert.Equal(t, H2mux, selector.Current())
+}
+
+func TestProtocolSelectorRefreshTTL(t *testing.T) {
+	logger, _ := logger.New()
+	fetcher := dynamicMockFetcher{percentage: 100}
+	selector, err := NewProtocolSelector("auto", testNamedTunnelConfig, fetcher.fetch(), time.Hour, logger)
+	assert.NoError(t, err)
+	assert.Equal(t, HTTP2, selector.Current())
+
+	fetcher.percentage = 0
+	assert.Equal(t, HTTP2, selector.Current())
+}

connection/rpc.go

@@ -2,40 +2,294 @@ package connection
 
 import (
 	"context"
+	"fmt"
 	"io"
-
-	rpc "zombiezen.com/go/capnproto2/rpc"
+	"time"
 
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/tunnelrpc"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"zombiezen.com/go/capnproto2/rpc"
 )
 
-// NewTunnelRPCClient creates and returns a new RPC client, which will communicate
-// using a stream on the given muxer
-func NewTunnelRPCClient(
+type tunnelServerClient struct {
+	client    tunnelpogs.TunnelServer_PogsClient
+	transport rpc.Transport
+}
+
+// NewTunnelRPCClient creates and returns a new RPC client, which will communicate using a stream on the given muxer.
+// This method is exported for supervisor to call Authenticate RPC
+func NewTunnelServerClient(
 	ctx context.Context,
 	stream io.ReadWriteCloser,
 	logger logger.Service,
-) (client tunnelpogs.TunnelServer_PogsClient, err error) {
+) *tunnelServerClient {
+	transport := tunnelrpc.NewTransportLogger(logger, rpc.StreamTransport(stream))
 	conn := rpc.NewConn(
-		tunnelrpc.NewTransportLogger(logger, rpc.StreamTransport(stream)),
+		transport,
 		tunnelrpc.ConnLog(logger),
 	)
 	registrationClient := tunnelpogs.RegistrationServer_PogsClient{Client: conn.Bootstrap(ctx), Conn: conn}
-	client = tunnelpogs.TunnelServer_PogsClient{RegistrationServer_PogsClient: registrationClient, Client: conn.Bootstrap(ctx), Conn: conn}
-	return client, nil
+	return &tunnelServerClient{
+		client:    tunnelpogs.TunnelServer_PogsClient{RegistrationServer_PogsClient: registrationClient, Client: conn.Bootstrap(ctx), Conn: conn},
+		transport: transport,
+	}
 }
 
-func NewRegistrationRPCClient(
+func (tsc *tunnelServerClient) Authenticate(ctx context.Context, classicTunnel *ClassicTunnelConfig, registrationOptions *tunnelpogs.RegistrationOptions) (tunnelpogs.AuthOutcome, error) {
+	authResp, err := tsc.client.Authenticate(ctx, classicTunnel.OriginCert, classicTunnel.Hostname, registrationOptions)
+	if err != nil {
+		return nil, err
+	}
+	return authResp.Outcome(), nil
+}
+
+func (tsc *tunnelServerClient) Close() {
+	// Closing the client will also close the connection
+	tsc.client.Close()
+	tsc.transport.Close()
+}
+
+type NamedTunnelRPCClient interface {
+	RegisterConnection(
+		c context.Context,
+		config *NamedTunnelConfig,
+		options *tunnelpogs.ConnectionOptions,
+		connIndex uint8,
+		observer *Observer,
+	) error
+	GracefulShutdown(ctx context.Context, gracePeriod time.Duration)
+	Close()
+}
+
+type registrationServerClient struct {
+	client    tunnelpogs.RegistrationServer_PogsClient
+	transport rpc.Transport
+}
+
+func newRegistrationRPCClient(
 	ctx context.Context,
 	stream io.ReadWriteCloser,
 	logger logger.Service,
-) (client tunnelpogs.RegistrationServer_PogsClient, err error) {
+) NamedTunnelRPCClient {
+	transport := tunnelrpc.NewTransportLogger(logger, rpc.StreamTransport(stream))
 	conn := rpc.NewConn(
-		tunnelrpc.NewTransportLogger(logger, rpc.StreamTransport(stream)),
+		transport,
 		tunnelrpc.ConnLog(logger),
 	)
-	client = tunnelpogs.RegistrationServer_PogsClient{Client: conn.Bootstrap(ctx), Conn: conn}
-	return client, nil
+	return &registrationServerClient{
+		client:    tunnelpogs.RegistrationServer_PogsClient{Client: conn.Bootstrap(ctx), Conn: conn},
+		transport: transport,
+	}
+}
+
+func (rsc *registrationServerClient) RegisterConnection(
+	ctx context.Context,
+	config *NamedTunnelConfig,
+	options *tunnelpogs.ConnectionOptions,
+	connIndex uint8,
+	observer *Observer,
+) error {
+	conn, err := rsc.client.RegisterConnection(
+		ctx,
+		config.Credentials.Auth(),
+		config.Credentials.TunnelID,
+		connIndex,
+		options,
+	)
+	if err != nil {
+		if err.Error() == DuplicateConnectionError {
+			observer.metrics.regFail.WithLabelValues("dup_edge_conn", "registerConnection").Inc()
+			return errDuplicationConnection
+		}
+		observer.metrics.regFail.WithLabelValues("server_error", "registerConnection").Inc()
+		return serverRegistrationErrorFromRPC(err)
+	}
+
+	observer.metrics.regSuccess.WithLabelValues("registerConnection").Inc()
+
+	observer.logServerInfo(connIndex, conn.Location, fmt.Sprintf("Connection %d registered with %s using ID %s", connIndex, conn.Location, conn.UUID))
+	observer.sendConnectedEvent(connIndex, conn.Location)
+
+	return nil
+}
+
+func (rsc *registrationServerClient) GracefulShutdown(ctx context.Context, gracePeriod time.Duration) {
+	ctx, cancel := context.WithTimeout(ctx, gracePeriod)
+	defer cancel()
+	rsc.client.UnregisterConnection(ctx)
+}
+
+func (rsc *registrationServerClient) Close() {
+	// Closing the client will also close the connection
+	rsc.client.Close()
+	// Closing the transport also closes the stream
+	rsc.transport.Close()
+}
+
+type rpcName string
+
+const (
+	register     rpcName = "register"
+	reconnect    rpcName = "reconnect"
+	unregister   rpcName = "unregister"
+	authenticate rpcName = " authenticate"
+)
+
+func (h *h2muxConnection) registerTunnel(ctx context.Context, credentialSetter CredentialManager, classicTunnel *ClassicTunnelConfig, registrationOptions *tunnelpogs.RegistrationOptions) error {
+	h.observer.sendRegisteringEvent()
+
+	stream, err := h.newRPCStream(ctx, register)
+	if err != nil {
+		return err
+	}
+	rpcClient := NewTunnelServerClient(ctx, stream, h.observer)
+	defer rpcClient.Close()
+
+	h.logServerInfo(ctx, rpcClient)
+	registration := rpcClient.client.RegisterTunnel(
+		ctx,
+		classicTunnel.OriginCert,
+		classicTunnel.Hostname,
+		registrationOptions,
+	)
+	if registrationErr := registration.DeserializeError(); registrationErr != nil {
+		// RegisterTunnel RPC failure
+		return h.processRegisterTunnelError(registrationErr, register)
+	}
+
+	// Send free tunnel URL to UI
+	h.observer.sendURL(registration.Url)
+	credentialSetter.SetEventDigest(h.connIndex, registration.EventDigest)
+	return h.processRegistrationSuccess(registration, register, credentialSetter, classicTunnel)
+}
+
+type CredentialManager interface {
+	ReconnectToken() ([]byte, error)
+	EventDigest(connID uint8) ([]byte, error)
+	SetEventDigest(connID uint8, digest []byte)
+	ConnDigest(connID uint8) ([]byte, error)
+	SetConnDigest(connID uint8, digest []byte)
+}
+
+func (h *h2muxConnection) processRegistrationSuccess(
+	registration *tunnelpogs.TunnelRegistration,
+	name rpcName,
+	credentialManager CredentialManager, classicTunnel *ClassicTunnelConfig,
+) error {
+	for _, logLine := range registration.LogLines {
+		h.observer.Info(logLine)
+	}
+
+	if registration.TunnelID != "" {
+		h.observer.metrics.tunnelsHA.AddTunnelID(h.connIndex, registration.TunnelID)
+		h.observer.Infof("Each HA connection's tunnel IDs: %v", h.observer.metrics.tunnelsHA.String())
+	}
+
+	// Print out the user's trial zone URL in a nice box (if they requested and got one and UI flag is not set)
+	if classicTunnel.IsTrialZone() {
+		err := h.observer.logTrialHostname(registration)
+		if err != nil {
+			return err
+		}
+	}
+
+	credentialManager.SetConnDigest(h.connIndex, registration.ConnDigest)
+	h.observer.metrics.userHostnamesCounts.WithLabelValues(registration.Url).Inc()
+
+	h.observer.Infof("Route propagating, it may take up to 1 minute for your new route to become functional")
+	h.observer.metrics.regSuccess.WithLabelValues(string(name)).Inc()
+	return nil
+}
+
+func (h *h2muxConnection) processRegisterTunnelError(err tunnelpogs.TunnelRegistrationError, name rpcName) error {
+	if err.Error() == DuplicateConnectionError {
+		h.observer.metrics.regFail.WithLabelValues("dup_edge_conn", string(name)).Inc()
+		return errDuplicationConnection
+	}
+	h.observer.metrics.regFail.WithLabelValues("server_error", string(name)).Inc()
+	return serverRegisterTunnelError{
+		cause:     err,
+		permanent: err.IsPermanent(),
+	}
+}
+
+func (h *h2muxConnection) reconnectTunnel(ctx context.Context, credentialManager CredentialManager, classicTunnel *ClassicTunnelConfig, registrationOptions *tunnelpogs.RegistrationOptions) error {
+	token, err := credentialManager.ReconnectToken()
+	if err != nil {
+		return err
+	}
+	eventDigest, err := credentialManager.EventDigest(h.connIndex)
+	if err != nil {
+		return err
+	}
+	connDigest, err := credentialManager.ConnDigest(h.connIndex)
+	if err != nil {
+		return err
+	}
+
+	h.observer.Debug("initiating RPC stream to reconnect")
+	stream, err := h.newRPCStream(ctx, register)
+	if err != nil {
+		return err
+	}
+	rpcClient := NewTunnelServerClient(ctx, stream, h.observer)
+	defer rpcClient.Close()
+
+	h.logServerInfo(ctx, rpcClient)
+	registration := rpcClient.client.ReconnectTunnel(
+		ctx,
+		token,
+		eventDigest,
+		connDigest,
+		classicTunnel.Hostname,
+		registrationOptions,
+	)
+	if registrationErr := registration.DeserializeError(); registrationErr != nil {
+		// ReconnectTunnel RPC failure
+		return h.processRegisterTunnelError(registrationErr, reconnect)
+	}
+	return h.processRegistrationSuccess(registration, reconnect, credentialManager, classicTunnel)
+}
+
+func (h *h2muxConnection) logServerInfo(ctx context.Context, rpcClient *tunnelServerClient) error {
+	// Request server info without blocking tunnel registration; must use capnp library directly.
+	serverInfoPromise := tunnelrpc.TunnelServer{Client: rpcClient.client.Client}.GetServerInfo(ctx, func(tunnelrpc.TunnelServer_getServerInfo_Params) error {
+		return nil
+	})
+	serverInfoMessage, err := serverInfoPromise.Result().Struct()
+	if err != nil {
+		h.observer.Errorf("Failed to retrieve server information: %s", err)
+		return err
+	}
+	serverInfo, err := tunnelpogs.UnmarshalServerInfo(serverInfoMessage)
+	if err != nil {
+		h.observer.Errorf("Failed to retrieve server information: %s", err)
+		return err
+	}
+	h.observer.logServerInfo(h.connIndex, serverInfo.LocationName, fmt.Sprintf("Connnection %d connected to %s", h.connIndex, serverInfo.LocationName))
+	return nil
+}
+
+func (h *h2muxConnection) unregister(isNamedTunnel bool) {
+	unregisterCtx, cancel := context.WithTimeout(context.Background(), h.config.GracePeriod)
+	defer cancel()
+
+	stream, err := h.newRPCStream(unregisterCtx, register)
+	if err != nil {
+		return
+	}
+
+	if isNamedTunnel {
+		rpcClient := newRegistrationRPCClient(unregisterCtx, stream, h.observer)
+		defer rpcClient.Close()
+
+		rpcClient.GracefulShutdown(unregisterCtx, h.config.GracePeriod)
+	} else {
+		rpcClient := NewTunnelServerClient(unregisterCtx, stream, h.observer)
+		defer rpcClient.Close()
+
+		// gracePeriod is encoded in int64 using capnproto
+		rpcClient.client.UnregisterTunnel(unregisterCtx, h.config.GracePeriod.Nanoseconds())
+	}
 }

connection/tunnelsforha.go

@@ -0,0 +1,50 @@
+package connection
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// tunnelsForHA maps this cloudflared instance's HA connections to the tunnel IDs they serve.
+type tunnelsForHA struct {
+	sync.Mutex
+	metrics *prometheus.GaugeVec
+	entries map[uint8]string
+}
+
+// NewTunnelsForHA initializes the Prometheus metrics etc for a tunnelsForHA.
+func NewTunnelsForHA() tunnelsForHA {
+	metrics := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "tunnel_ids",
+			Help: "The ID of all tunnels (and their corresponding HA connection ID) running in this instance of cloudflared.",
+		},
+		[]string{"tunnel_id", "ha_conn_id"},
+	)
+	prometheus.MustRegister(metrics)
+
+	return tunnelsForHA{
+		metrics: metrics,
+		entries: make(map[uint8]string),
+	}
+}
+
+// Track a new tunnel ID, removing the disconnected tunnel (if any) and update metrics.
+func (t *tunnelsForHA) AddTunnelID(haConn uint8, tunnelID string) {
+	t.Lock()
+	defer t.Unlock()
+	haStr := fmt.Sprintf("%v", haConn)
+	if oldTunnelID, ok := t.entries[haConn]; ok {
+		t.metrics.WithLabelValues(oldTunnelID, haStr).Dec()
+	}
+	t.entries[haConn] = tunnelID
+	t.metrics.WithLabelValues(tunnelID, haStr).Inc()
+}
+
+func (t *tunnelsForHA) String() string {
+	t.Lock()
+	defer t.Unlock()
+	return fmt.Sprintf("%v", t.entries)
+}

dbconnect/proxy.go

@@ -43,7 +43,7 @@ func NewInsecureProxy(ctx context.Context, origin string) (*Proxy, error) {
 		return nil, errors.Wrap(err, "could not connect to the database")
 	}
 
-	logger, err := logger.New()
+	logger, err := logger.New() // TODO: Does not obey log configuration
 	if err != nil {
 		return nil, errors.Wrap(err, "error setting up logger")
 	}

dev.Dockerfile

@@ -1,4 +1,9 @@
-FROM golang:1.12 as builder
+FROM golang:1.15.6 as builder
+ENV GO111MODULE=on \
+    CGO_ENABLED=0
 WORKDIR /go/src/github.com/cloudflare/cloudflared/
 RUN apt-get update
 COPY . .
+# compile cloudflared
+RUN make cloudflared
+RUN cp /go/src/github.com/cloudflare/cloudflared/cloudflared /usr/local/bin/

edgediscovery/allregions/discovery.go

@@ -121,15 +121,15 @@ func resolveSRVToTCP(srv *net.SRV) ([]*net.TCPAddr, error) {
 }
 
 // ResolveAddrs resolves TCP address given a list of addresses. Address can be a hostname, however, it will return at most one
-// of the hostname's IP addresses
-func ResolveAddrs(addrs []string) ([]*net.TCPAddr, error) {
-	var tcpAddrs []*net.TCPAddr
+// of the hostname's IP addresses.
+func ResolveAddrs(addrs []string, logger logger.Service) (resolved []*net.TCPAddr) {
 	for _, addr := range addrs {
 		tcpAddr, err := net.ResolveTCPAddr("tcp", addr)
 		if err != nil {
-			return nil, err
+			logger.Errorf("Failed to resolve %s, err: %v", addr, err)
+		} else {
+			resolved = append(resolved, tcpAddr)
 		}
-		tcpAddrs = append(tcpAddrs, tcpAddr)
 	}
-	return tcpAddrs, nil
+	return
 }

edgediscovery/allregions/regions.go

@@ -35,12 +35,12 @@ func ResolveEdge(logger logger.Service) (*Regions, error) {
 
 // StaticEdge creates a list of edge addresses from the list of hostnames.
 // Mainly used for testing connectivity.
-func StaticEdge(hostnames []string) (*Regions, error) {
-	addrs, err := ResolveAddrs(hostnames)
-	if err != nil {
-		return nil, err
+func StaticEdge(hostnames []string, logger logger.Service) (*Regions, error) {
+	resolved := ResolveAddrs(hostnames, logger)
+	if len(resolved) == 0 {
+		return nil, fmt.Errorf("failed to resolve any edge address")
 	}
-	return NewNoResolve(addrs), nil
+	return NewNoResolve(resolved), nil
 }
 
 // NewNoResolve doesn't resolve the edge. Instead it just uses the given addresses.

edgediscovery/dial.go

@@ -1,4 +1,4 @@
-package connection
+package edgediscovery
 
 import (
 	"context"
@@ -9,7 +9,7 @@ import (
 	"github.com/pkg/errors"
 )
 
-// DialEdge makes a TLS connection to a Cloudflare edge node
+// DialEdgeWithH2Mux makes a TLS connection to a Cloudflare edge node
 func DialEdge(
 	ctx context.Context,
 	timeout time.Duration,
@@ -25,6 +25,7 @@ func DialEdge(
 	if err != nil {
 		return nil, newDialError(err, "DialContext error")
 	}
+
 	tlsEdgeConn := tls.Client(edgeConn, tlsConfig)
 	tlsEdgeConn.SetDeadline(time.Now().Add(timeout))
 

edgediscovery/edgediscovery.go

@@ -41,7 +41,7 @@ func ResolveEdge(l logger.Service) (*Edge, error) {
 
 // StaticEdge creates a list of edge addresses from the list of hostnames. Mainly used for testing connectivity.
 func StaticEdge(l logger.Service, hostnames []string) (*Edge, error) {
-	regions, err := allregions.StaticEdge(hostnames)
+	regions, err := allregions.StaticEdge(hostnames, l)
 	if err != nil {
 		return new(Edge), err
 	}

edgediscovery/mocks_for_test.go

@@ -1,4 +1,4 @@
-package connection
+package edgediscovery
 
 import (
 	"fmt"

edgediscovery/protocol.go

@@ -0,0 +1,45 @@
+package edgediscovery
+
+import (
+	"fmt"
+	"net"
+	"strconv"
+	"strings"
+)
+
+const (
+	protocolRecord = "protocol.argotunnel.com"
+)
+
+var (
+	errNoProtocolRecord = fmt.Errorf("No TXT record found for %s to determine connection protocol", protocolRecord)
+)
+
+func HTTP2Percentage() (int32, error) {
+	records, err := net.LookupTXT(protocolRecord)
+	if err != nil {
+		return 0, err
+	}
+	if len(records) == 0 {
+		return 0, errNoProtocolRecord
+	}
+	return parseHTTP2Precentage(records[0])
+}
+
+// The record looks like http2=percentage
+func parseHTTP2Precentage(record string) (int32, error) {
+	const key = "http2"
+	slices := strings.Split(record, "=")
+	if len(slices) != 2 {
+		return 0, fmt.Errorf("Malformed TXT record %s, expect http2=percentage", record)
+	}
+	if slices[0] != key {
+		return 0, fmt.Errorf("Incorrect key %s, expect %s", slices[0], key)
+	}
+	percentage, err := strconv.ParseInt(slices[1], 10, 32)
+	if err != nil {
+		return 0, err
+	}
+	return int32(percentage), nil
+
+}

edgediscovery/protocol_test.go

@@ -0,0 +1,80 @@
+package edgediscovery
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestHTTP2Percentage(t *testing.T) {
+	_, err := HTTP2Percentage()
+	assert.NoError(t, err)
+}
+
+func TestParseHTTP2Precentage(t *testing.T) {
+	tests := []struct {
+		record     string
+		percentage int32
+		wantErr    bool
+	}{
+		{
+			record:     "http2=-1",
+			percentage: -1,
+			wantErr:    false,
+		},
+		{
+			record:     "http2=0",
+			percentage: 0,
+			wantErr:    false,
+		},
+		{
+			record:     "http2=50",
+			percentage: 50,
+			wantErr:    false,
+		},
+		{
+			record:     "http2=100",
+			percentage: 100,
+			wantErr:    false,
+		},
+		{
+			record:     "http2=1000",
+			percentage: 1000,
+			wantErr:    false,
+		},
+		{
+			record:  "http2=10.5",
+			wantErr: true,
+		},
+		{
+			record:  "http2=10 h2mux=90",
+			wantErr: true,
+		},
+		{
+			record:  "http2=ten",
+			wantErr: true,
+		},
+
+		{
+			record:  "h2mux=100",
+			wantErr: true,
+		},
+		{
+			record:  "http2",
+			wantErr: true,
+		},
+		{
+			record:  "http2=",
+			wantErr: true,
+		},
+	}
+
+	for _, test := range tests {
+		p, err := parseHTTP2Precentage(test.record)
+		if test.wantErr {
+			assert.Error(t, err)
+		} else {
+			assert.Equal(t, test.percentage, p)
+		}
+	}
+}

github_message.py

@@ -97,10 +97,11 @@ def main():
 
         msg = release.body
 
-        for filename in glob.glob("artifacts/*.*"):
-            pkg_hash = get_sha256(filename)
+        for filepath in glob.glob("artifacts/*"):
+            pkg_hash = get_sha256(filepath)
             # add the sha256 of the new artifact to the release message body
-            msg = update_or_add_message(msg, filename, pkg_hash)
+            name = os.path.basename(filepath)
+            msg = update_or_add_message(msg, name, pkg_hash)
 
         if args.dry_run:
             logging.info("Skipping asset upload because of dry-run")

github_release.py

@@ -200,10 +200,10 @@ def main():
 
         # copy the binary to the path
         copy_path = os.path.join(artifact_path, args.name)
-        if args.path != copy_path:
+        try:
             shutil.copy(args.path, copy_path)
-
-
+        except shutil.SameFileError:
+            pass # the macOS release copy fails with being the same file (already in the artifacts directory). Catching to ignore.
 
     except Exception as e:
         logging.exception(e)

go.mod

@@ -15,7 +15,7 @@ require (
 	github.com/coreos/go-oidc v0.0.0-20171002155002-a93f71fdfe73
 	github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf
 	github.com/denisenkom/go-mssqldb v0.0.0-20191001013358-cfbb681360f0
-	github.com/equinox-io/equinox v1.2.0
+	github.com/equinox-io/equinox v1.2.0 // indirect
 	github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 // indirect
 	github.com/facebookgo/freeport v0.0.0-20150612182905-d4adf43b75b9 // indirect
 	github.com/facebookgo/grace v0.0.0-20180706040059-75cf19382434
@@ -27,12 +27,16 @@ require (
 	github.com/getsentry/raven-go v0.0.0-20180517221441-ed7bcb39ff10
 	github.com/gliderlabs/ssh v0.0.0-20191009160644-63518b5243e0
 	github.com/go-sql-driver/mysql v1.5.0
+	github.com/gobwas/httphead v0.0.0-20200921212729-da3d93bc3c58 // indirect
+	github.com/gobwas/pool v0.2.1 // indirect
+	github.com/gobwas/ws v1.0.4
 	github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3
 	github.com/google/go-cmp v0.5.2 // indirect
 	github.com/google/uuid v1.1.2
 	github.com/gorilla/mux v1.7.3
 	github.com/gorilla/websocket v1.4.2
 	github.com/jmoiron/sqlx v1.2.0
+	github.com/json-iterator/go v1.1.10
 	github.com/kr/text v0.2.0 // indirect
 	github.com/kshvakov/clickhouse v1.3.11
 	github.com/kylelemons/godebug v1.1.0 // indirect

go.sum

@@ -233,6 +233,12 @@ github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG
 github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/gobwas/httphead v0.0.0-20200921212729-da3d93bc3c58 h1:YyrUZvJaU8Q0QsoVo+xLFBgWDTam29PKea6GYmwvSiQ=
+github.com/gobwas/httphead v0.0.0-20200921212729-da3d93bc3c58/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo=
+github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og=
+github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
+github.com/gobwas/ws v1.0.4 h1:5eXU1CZhpQdq5kXbKb+sECH5Ia5KiO6CYzIzdlVx6Bs=
+github.com/gobwas/ws v1.0.4/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=
 github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
 github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
@@ -369,6 +375,7 @@ github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV
 github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
@@ -441,8 +448,10 @@ github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0Qu
 github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=

h2mux/activestreammap.go

@@ -7,6 +7,19 @@ import (
 	"golang.org/x/net/http2"
 )
 
+var (
+	ActiveStreams = prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace: "cloudflared",
+		Subsystem: "tunnel",
+		Name:      "active_streams",
+		Help:      "Number of active streams created by all muxers.",
+	})
+)
+
+func init() {
+	prometheus.MustRegister(ActiveStreams)
+}
+
 // activeStreamMap is used to moderate access to active streams between the read and write
 // threads, and deny access to new peer streams while shutting down.
 type activeStreamMap struct {

h2mux/activestreammap_test.go

@@ -9,7 +9,7 @@ import (
 
 func TestShutdown(t *testing.T) {
 	const numStreams = 1000
-	m := newActiveStreamMap(true, NewActiveStreamsMetrics("test", t.Name()))
+	m := newActiveStreamMap(true, ActiveStreams)
 
 	// Add all the streams
 	{
@@ -62,7 +62,7 @@ func TestShutdown(t *testing.T) {
 
 func TestEmptyBeforeShutdown(t *testing.T) {
 	const numStreams = 1000
-	m := newActiveStreamMap(true, NewActiveStreamsMetrics("test", t.Name()))
+	m := newActiveStreamMap(true, ActiveStreams)
 
 	// Add all the streams
 	{
@@ -138,7 +138,7 @@ func (_ *noopReadyList) Signal(streamID uint32) {}
 
 func TestAbort(t *testing.T) {
 	const numStreams = 1000
-	m := newActiveStreamMap(true, NewActiveStreamsMetrics("test", t.Name()))
+	m := newActiveStreamMap(true, ActiveStreams)
 
 	var openedStreams sync.Map
 

h2mux/h2mux_test.go

@@ -113,11 +113,11 @@ func (p *DefaultMuxerPair) Handshake(testName string) error {
 	defer cancel()
 	errGroup, _ := errgroup.WithContext(ctx)
 	errGroup.Go(func() (err error) {
-		p.EdgeMux, err = Handshake(p.EdgeConn, p.EdgeConn, p.EdgeMuxConfig, NewActiveStreamsMetrics(testName, "edge"))
+		p.EdgeMux, err = Handshake(p.EdgeConn, p.EdgeConn, p.EdgeMuxConfig, ActiveStreams)
 		return errors.Wrap(err, "edge handshake failure")
 	})
 	errGroup.Go(func() (err error) {
-		p.OriginMux, err = Handshake(p.OriginConn, p.OriginConn, p.OriginMuxConfig, NewActiveStreamsMetrics(testName, "origin"))
+		p.OriginMux, err = Handshake(p.OriginConn, p.OriginConn, p.OriginMuxConfig, ActiveStreams)
 		return errors.Wrap(err, "origin handshake failure")
 	})
 

h2mux/header.go

@@ -2,7 +2,6 @@ package h2mux
 
 import (
 	"encoding/base64"
-	"encoding/json"
 	"fmt"
 	"net/http"
 	"net/url"
@@ -22,10 +21,6 @@ const (
 	RequestUserHeadersField  = "cf-cloudflared-request-headers"
 	ResponseUserHeadersField = "cf-cloudflared-response-headers"
 
-	ResponseMetaHeaderField   = "cf-cloudflared-response-meta"
-	ResponseSourceCloudflared = "cloudflared"
-	ResponseSourceOrigin      = "origin"
-
 	CFAccessTokenHeader        = "cf-access-token"
 	CFJumpDestinationHeader    = "CF-Access-Jump-Destination"
 	CFAccessClientIDHeader     = "CF-Access-Client-Id"
@@ -124,7 +119,7 @@ func IsControlHeader(headerName string) bool {
 }
 
 // isWebsocketClientHeader returns true if the header name is required by the client to upgrade properly
-func isWebsocketClientHeader(headerName string) bool {
+func IsWebsocketClientHeader(headerName string) bool {
 	return headerName == "sec-websocket-accept" ||
 		headerName == "connection" ||
 		headerName == "upgrade"
@@ -143,7 +138,7 @@ func H1ResponseToH2ResponseHeaders(h1 *http.Response) (h2 []Header) {
 
 			// Since these are http2 headers, they're required to be lowercase
 			h2 = append(h2, Header{Name: "content-length", Value: values[0]})
-		} else if !IsControlHeader(h2name) || isWebsocketClientHeader(h2name) {
+		} else if !IsControlHeader(h2name) || IsWebsocketClientHeader(h2name) {
 			// User headers, on the other hand, must all be serialized so that
 			// HTTP/2 header validation won't be applied to HTTP/1 header values
 			userHeaders[header] = values
@@ -152,7 +147,6 @@ func H1ResponseToH2ResponseHeaders(h1 *http.Response) (h2 []Header) {
 
 	// Perform user header serialization and set them in the single header
 	h2 = append(h2, Header{ResponseUserHeadersField, SerializeHeaders(userHeaders)})
-
 	return h2
 }
 
@@ -238,19 +232,3 @@ func DeserializeHeaders(serializedHeaders string) ([]Header, error) {
 
 	return deserialized, nil
 }
-
-type ResponseMetaHeader struct {
-	Source string `json:"src"`
-}
-
-func CreateResponseMetaHeader(headerName, source string) Header {
-	jsonResponseMetaHeader, err := json.Marshal(ResponseMetaHeader{Source: source})
-	if err != nil {
-		panic(err)
-	}
-
-	return Header{
-		Name:  headerName,
-		Value: string(jsonResponseMetaHeader),
-	}
-}

h2mux/muxmetrics.go

@@ -6,7 +6,6 @@ import (
 
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/golang-collections/collections/queue"
-	"github.com/prometheus/client_golang/prometheus"
 )
 
 // data points used to compute average receive window and send window size
@@ -295,14 +294,3 @@ func (r *rate) get() (curr, min, max uint64) {
 	defer r.lock.RUnlock()
 	return r.curr, r.min, r.max
 }
-
-func NewActiveStreamsMetrics(namespace, subsystem string) prometheus.Gauge {
-	activeStreams := prometheus.NewGauge(prometheus.GaugeOpts{
-		Namespace: namespace,
-		Subsystem: subsystem,
-		Name:      "active_streams",
-		Help:      "Number of active streams created by all muxers.",
-	})
-	prometheus.MustRegister(activeStreams)
-	return activeStreams
-}

hello/hello.go

@@ -18,6 +18,14 @@ import (
 	"github.com/cloudflare/cloudflared/tlsconfig"
 )
 
+const (
+	UptimeRoute    = "/uptime"
+	WSRoute        = "/ws"
+	SSERoute       = "/sse"
+	HealthRoute    = "/_health"
+	defaultSSEFreq = time.Second * 10
+)
+
 type templateData struct {
 	ServerName string
 	Request    *http.Request
@@ -104,8 +112,10 @@ func StartHelloWorldServer(logger logger.Service, listener net.Listener, shutdow
 	}
 
 	muxer := http.NewServeMux()
-	muxer.HandleFunc("/uptime", uptimeHandler(time.Now()))
-	muxer.HandleFunc("/ws", websocketHandler(logger, upgrader))
+	muxer.HandleFunc(UptimeRoute, uptimeHandler(time.Now()))
+	muxer.HandleFunc(WSRoute, websocketHandler(logger, upgrader))
+	muxer.HandleFunc(SSERoute, sseHandler(logger))
+	muxer.HandleFunc(HealthRoute, healthHandler())
 	muxer.HandleFunc("/", rootHandler(serverName))
 	httpServer := &http.Server{Addr: listener.Addr().String(), Handler: muxer}
 	go func() {
@@ -150,8 +160,15 @@ func uptimeHandler(startTime time.Time) http.HandlerFunc {
 // This handler will echo message
 func websocketHandler(logger logger.Service, upgrader websocket.Upgrader) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
+		// This addresses the issue of r.Host includes port but origin header doesn't
+		host, _, err := net.SplitHostPort(r.Host)
+		if err == nil {
+			r.Host = host
+		}
+
 		conn, err := upgrader.Upgrade(w, r, nil)
 		if err != nil {
+			logger.Errorf("failed to upgrade to websocket connection, error: %s", err)
 			return
 		}
 		defer conn.Close()
@@ -170,6 +187,48 @@ func websocketHandler(logger logger.Service, upgrader websocket.Upgrader) http.H
 	}
 }
 
+func sseHandler(logger logger.Service) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/event-stream; charset=utf-8")
+		flusher, ok := w.(http.Flusher)
+		if !ok {
+			w.WriteHeader(http.StatusInternalServerError)
+			logger.Errorf("Can't support SSE. ResponseWriter %T doesn't implement http.Flusher interface", w)
+			return
+		}
+
+		freq := defaultSSEFreq
+		if requestedFreq := r.URL.Query()["freq"]; len(requestedFreq) > 0 {
+			parsedFreq, err := time.ParseDuration(requestedFreq[0])
+			if err == nil {
+				freq = parsedFreq
+			}
+		}
+		logger.Infof("Server Sent Events every %s", freq)
+		ticker := time.NewTicker(freq)
+		counter := 0
+		for {
+			select {
+			case <-r.Context().Done():
+				return
+			case <-ticker.C:
+			}
+			_, err := fmt.Fprintf(w, "%d\n\n", counter)
+			if err != nil {
+				return
+			}
+			flusher.Flush()
+			counter++
+		}
+	}
+}
+
+func healthHandler() http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		w.Write([]byte("ok"))
+	}
+}
+
 func rootHandler(serverName string) http.HandlerFunc {
 	responseTemplate := template.Must(template.New("index").Parse(indexTemplate))
 	return func(w http.ResponseWriter, r *http.Request) {

ingress/ingress.go

@@ -0,0 +1,237 @@
+package ingress
+
+import (
+	"fmt"
+	"net"
+	"net/url"
+	"regexp"
+	"strconv"
+	"strings"
+	"sync"
+
+	"github.com/pkg/errors"
+	"github.com/urfave/cli/v2"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/logger"
+)
+
+var (
+	ErrNoIngressRules             = errors.New("The config file doesn't contain any ingress rules")
+	errLastRuleNotCatchAll        = errors.New("The last ingress rule must match all URLs (i.e. it should not have a hostname or path filter)")
+	errBadWildcard                = errors.New("Hostname patterns can have at most one wildcard character (\"*\") and it can only be used for subdomains, e.g. \"*.example.com\"")
+	errHostnameContainsPort       = errors.New("Hostname cannot contain a port")
+	ErrURLIncompatibleWithIngress = errors.New("You can't set the --url flag (or $TUNNEL_URL) when using multiple-origin ingress rules")
+)
+
+// FindMatchingRule returns the index of the Ingress Rule which matches the given
+// hostname and path. This function assumes the last rule matches everything,
+// which is the case if the rules were instantiated via the ingress#Validate method
+func (ing Ingress) FindMatchingRule(hostname, path string) (*Rule, int) {
+	// The hostname might contain port. We only want to compare the host part with the rule
+	host, _, err := net.SplitHostPort(hostname)
+	if err == nil {
+		hostname = host
+	}
+	for i, rule := range ing.Rules {
+		if rule.Matches(hostname, path) {
+			return &rule, i
+		}
+	}
+	i := len(ing.Rules) - 1
+	return &ing.Rules[i], i
+}
+
+func matchHost(ruleHost, reqHost string) bool {
+	if ruleHost == reqHost {
+		return true
+	}
+
+	// Validate hostnames that use wildcards at the start
+	if strings.HasPrefix(ruleHost, "*.") {
+		toMatch := strings.TrimPrefix(ruleHost, "*.")
+		return strings.HasSuffix(reqHost, toMatch)
+	}
+	return false
+}
+
+// Ingress maps eyeball requests to origins.
+type Ingress struct {
+	Rules    []Rule
+	defaults OriginRequestConfig
+}
+
+// NewSingleOrigin constructs an Ingress set with only one rule, constructed from
+// legacy CLI parameters like --url or --no-chunked-encoding.
+func NewSingleOrigin(c *cli.Context, allowURLFromArgs bool, logger logger.Service) (Ingress, error) {
+
+	service, err := parseSingleOriginService(c, allowURLFromArgs)
+	if err != nil {
+		return Ingress{}, err
+	}
+
+	// Construct an Ingress with the single rule.
+	defaults := originRequestFromSingeRule(c)
+	ing := Ingress{
+		Rules: []Rule{
+			{
+				Service: service,
+				Config:  setConfig(defaults, config.OriginRequestConfig{}),
+			},
+		},
+		defaults: defaults,
+	}
+	return ing, err
+}
+
+// Get a single origin service from the CLI/config.
+func parseSingleOriginService(c *cli.Context, allowURLFromArgs bool) (OriginService, error) {
+	if c.IsSet("hello-world") {
+		return new(helloWorld), nil
+	}
+	if c.IsSet("url") || c.IsSet(config.BastionFlag) {
+		originURL, err := config.ValidateUrl(c, allowURLFromArgs)
+		if err != nil {
+			return nil, errors.Wrap(err, "Error validating origin URL")
+		}
+		return &localService{URL: originURL, RootURL: originURL}, nil
+	}
+	if c.IsSet("unix-socket") {
+		path, err := config.ValidateUnixSocket(c)
+		if err != nil {
+			return nil, errors.Wrap(err, "Error validating --unix-socket")
+		}
+		return &unixSocketPath{path: path}, nil
+	}
+	u, err := url.Parse("http://localhost:8080")
+	return &localService{URL: u, RootURL: u}, err
+}
+
+// IsEmpty checks if there are any ingress rules.
+func (ing Ingress) IsEmpty() bool {
+	return len(ing.Rules) == 0
+}
+
+// StartOrigins will start any origin services managed by cloudflared, e.g. proxy servers or Hello World.
+func (ing Ingress) StartOrigins(wg *sync.WaitGroup, log logger.Service, shutdownC <-chan struct{}, errC chan error) {
+	for _, rule := range ing.Rules {
+		if err := rule.Service.start(wg, log, shutdownC, errC, rule.Config); err != nil {
+			log.Errorf("Error starting local service %s: %s", rule.Service, err)
+		}
+	}
+}
+
+// CatchAll returns the catch-all rule (i.e. the last rule)
+func (ing Ingress) CatchAll() *Rule {
+	return &ing.Rules[len(ing.Rules)-1]
+}
+
+func validate(ingress []config.UnvalidatedIngressRule, defaults OriginRequestConfig) (Ingress, error) {
+	rules := make([]Rule, len(ingress))
+	for i, r := range ingress {
+		cfg := setConfig(defaults, r.OriginRequest)
+		var service OriginService
+
+		if prefix := "unix:"; strings.HasPrefix(r.Service, prefix) {
+			// No validation necessary for unix socket filepath services
+			path := strings.TrimPrefix(r.Service, prefix)
+			service = &unixSocketPath{path: path}
+		} else if prefix := "http_status:"; strings.HasPrefix(r.Service, prefix) {
+			status, err := strconv.Atoi(strings.TrimPrefix(r.Service, prefix))
+			if err != nil {
+				return Ingress{}, errors.Wrap(err, "invalid HTTP status")
+			}
+			srv := newStatusCode(status)
+			service = &srv
+		} else if r.Service == "hello_world" || r.Service == "hello-world" || r.Service == "helloworld" {
+			service = new(helloWorld)
+		} else if r.Service == "bastion" || cfg.BastionMode {
+			// Bastion mode will always start a Websocket proxy server, which will
+			// overwrite the localService.URL field when `start` is called. So,
+			// leave the URL field empty for now.
+			cfg.BastionMode = true
+			service = new(localService)
+		} else {
+			// Validate URL services
+			u, err := url.Parse(r.Service)
+			if err != nil {
+				return Ingress{}, err
+			}
+
+			if u.Scheme == "" || u.Hostname() == "" {
+				return Ingress{}, fmt.Errorf("%s is an invalid address, please make sure it has a scheme and a hostname", r.Service)
+			}
+
+			if u.Path != "" {
+				return Ingress{}, fmt.Errorf("%s is an invalid address, ingress rules don't support proxying to a different path on the origin service. The path will be the same as the eyeball request's path", r.Service)
+			}
+			serviceURL := localService{URL: u}
+			service = &serviceURL
+		}
+
+		if err := validateHostname(r, i, len(ingress)); err != nil {
+			return Ingress{}, err
+		}
+
+		var pathRegex *regexp.Regexp
+		if r.Path != "" {
+			var err error
+			pathRegex, err = regexp.Compile(r.Path)
+			if err != nil {
+				return Ingress{}, errors.Wrapf(err, "Rule #%d has an invalid regex", i+1)
+			}
+		}
+
+		rules[i] = Rule{
+			Hostname: r.Hostname,
+			Service:  service,
+			Path:     pathRegex,
+			Config:   cfg,
+		}
+	}
+	return Ingress{Rules: rules, defaults: defaults}, nil
+}
+
+func validateHostname(r config.UnvalidatedIngressRule, ruleIndex, totalRules int) error {
+	// Ensure that the hostname doesn't contain port
+	_, _, err := net.SplitHostPort(r.Hostname)
+	if err == nil {
+		return errHostnameContainsPort
+	}
+	// Ensure that there are no wildcards anywhere except the first character
+	// of the hostname.
+	if strings.LastIndex(r.Hostname, "*") > 0 {
+		return errBadWildcard
+	}
+
+	// The last rule should catch all hostnames.
+	isCatchAllRule := (r.Hostname == "" || r.Hostname == "*") && r.Path == ""
+	isLastRule := ruleIndex == totalRules-1
+	if isLastRule && !isCatchAllRule {
+		return errLastRuleNotCatchAll
+	}
+	// ONLY the last rule should catch all hostnames.
+	if !isLastRule && isCatchAllRule {
+		return errRuleShouldNotBeCatchAll{index: ruleIndex, hostname: r.Hostname}
+	}
+	return nil
+}
+
+type errRuleShouldNotBeCatchAll struct {
+	index    int
+	hostname string
+}
+
+func (e errRuleShouldNotBeCatchAll) Error() string {
+	return fmt.Sprintf("Rule #%d is matching the hostname '%s', but "+
+		"this will match every hostname, meaning the rules which follow it "+
+		"will never be triggered.", e.index+1, e.hostname)
+}
+
+// ParseIngress parses ingress rules, but does not send HTTP requests to the origins.
+func ParseIngress(conf *config.Configuration) (Ingress, error) {
+	if len(conf.Ingress) == 0 {
+		return Ingress{}, ErrNoIngressRules
+	}
+	return validate(conf.Ingress, originRequestFromYAML(conf.OriginRequest))
+}

ingress/ingress_test.go

@@ -0,0 +1,455 @@
+package ingress
+
+import (
+	"flag"
+	"fmt"
+	"net/url"
+	"regexp"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/urfave/cli/v2"
+	"gopkg.in/yaml.v2"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/logger"
+	"github.com/cloudflare/cloudflared/tlsconfig"
+)
+
+func TestParseUnixSocket(t *testing.T) {
+	rawYAML := `
+ingress:
+- service: unix:/tmp/echo.sock
+`
+	ing, err := ParseIngress(MustReadIngress(rawYAML))
+	require.NoError(t, err)
+	_, ok := ing.Rules[0].Service.(*unixSocketPath)
+	require.True(t, ok)
+}
+
+func Test_parseIngress(t *testing.T) {
+	localhost8000 := MustParseURL(t, "https://localhost:8000")
+	localhost8001 := MustParseURL(t, "https://localhost:8001")
+	fourOhFour := newStatusCode(404)
+	defaultConfig := setConfig(originRequestFromYAML(config.OriginRequestConfig{}), config.OriginRequestConfig{})
+	require.Equal(t, defaultKeepAliveConnections, defaultConfig.KeepAliveConnections)
+	tr := true
+	type args struct {
+		rawYAML string
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    []Rule
+		wantErr bool
+	}{
+		{
+			name:    "Empty file",
+			args:    args{rawYAML: ""},
+			wantErr: true,
+		},
+		{
+			name: "Multiple rules",
+			args: args{rawYAML: `
+ingress:
+ - hostname: tunnel1.example.com
+   service: https://localhost:8000
+ - hostname: "*"
+   service: https://localhost:8001
+`},
+			want: []Rule{
+				{
+					Hostname: "tunnel1.example.com",
+					Service:  &localService{URL: localhost8000},
+					Config:   defaultConfig,
+				},
+				{
+					Hostname: "*",
+					Service:  &localService{URL: localhost8001},
+					Config:   defaultConfig,
+				},
+			},
+		},
+		{
+			name: "Extra keys",
+			args: args{rawYAML: `
+ingress:
+ - hostname: "*"
+   service: https://localhost:8000
+extraKey: extraValue
+`},
+			want: []Rule{
+				{
+					Hostname: "*",
+					Service:  &localService{URL: localhost8000},
+					Config:   defaultConfig,
+				},
+			},
+		},
+		{
+			name: "Hostname can be omitted",
+			args: args{rawYAML: `
+ingress:
+ - service: https://localhost:8000
+`},
+			want: []Rule{
+				{
+					Service: &localService{URL: localhost8000},
+					Config:  defaultConfig,
+				},
+			},
+		},
+		{
+			name: "Invalid service",
+			args: args{rawYAML: `
+ingress:
+ - hostname: "*"
+   service: https://local host:8000
+`},
+			wantErr: true,
+		},
+		{
+			name: "Last rule isn't catchall",
+			args: args{rawYAML: `
+ingress:
+ - hostname: example.com
+   service: https://localhost:8000
+`},
+			wantErr: true,
+		},
+		{
+			name: "First rule is catchall",
+			args: args{rawYAML: `
+ingress:
+ - service: https://localhost:8000
+ - hostname: example.com
+   service: https://localhost:8000
+`},
+			wantErr: true,
+		},
+		{
+			name: "Catch-all rule can't have a path",
+			args: args{rawYAML: `
+ingress:
+ - service: https://localhost:8001
+   path: /subpath1/(.*)/subpath2
+`},
+			wantErr: true,
+		},
+		{
+			name: "Invalid regex",
+			args: args{rawYAML: `
+ingress:
+ - hostname: example.com
+   service: https://localhost:8000
+   path: "*/subpath2"
+ - service: https://localhost:8001
+`},
+			wantErr: true,
+		},
+		{
+			name: "Service must have a scheme",
+			args: args{rawYAML: `
+ingress:
+ - service: localhost:8000
+`},
+			wantErr: true,
+		},
+		{
+			name: "Wildcard not at start",
+			args: args{rawYAML: `
+ingress:
+ - hostname: "test.*.example.com"
+   service: https://localhost:8000
+`},
+			wantErr: true,
+		},
+		{
+			name: "Service can't have a path",
+			args: args{rawYAML: `
+ingress:
+ - service: https://localhost:8000/static/
+`},
+			wantErr: true,
+		},
+		{
+			name: "Invalid HTTP status",
+			args: args{rawYAML: `
+ingress:
+ - service: http_status:asdf
+`},
+			wantErr: true,
+		},
+		{
+			name: "Valid HTTP status",
+			args: args{rawYAML: `
+ingress:
+ - service: http_status:404
+`},
+			want: []Rule{
+				{
+					Hostname: "",
+					Service:  &fourOhFour,
+					Config:   defaultConfig,
+				},
+			},
+		},
+		{
+			name: "Valid hello world service",
+			args: args{rawYAML: `
+ingress:
+ - service: hello_world
+`},
+			want: []Rule{
+				{
+					Hostname: "",
+					Service:  new(helloWorld),
+					Config:   defaultConfig,
+				},
+			},
+		},
+		{
+			name: "URL isn't necessary if using bastion",
+			args: args{rawYAML: `
+ingress:
+- hostname: bastion.foo.com
+  originRequest:
+    bastionMode: true
+- service: http_status:404
+`},
+			want: []Rule{
+				{
+					Hostname: "bastion.foo.com",
+					Service:  &localService{},
+					Config:   setConfig(originRequestFromYAML(config.OriginRequestConfig{}), config.OriginRequestConfig{BastionMode: &tr}),
+				},
+				{
+					Service: &fourOhFour,
+					Config:  defaultConfig,
+				},
+			},
+		},
+		{
+			name: "Bastion service",
+			args: args{rawYAML: `
+ingress:
+- hostname: bastion.foo.com
+  service: bastion
+- service: http_status:404
+`},
+			want: []Rule{
+				{
+					Hostname: "bastion.foo.com",
+					Service:  &localService{},
+					Config:   setConfig(originRequestFromYAML(config.OriginRequestConfig{}), config.OriginRequestConfig{BastionMode: &tr}),
+				},
+				{
+					Service: &fourOhFour,
+					Config:  defaultConfig,
+				},
+			},
+		},
+		{
+			name: "Hostname contains port",
+			args: args{rawYAML: `
+ingress:
+ - hostname: "test.example.com:443"
+   service: https://localhost:8000
+ - hostname: "*"
+   service: https://localhost:8001
+`},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := ParseIngress(MustReadIngress(tt.args.rawYAML))
+			if (err != nil) != tt.wantErr {
+				t.Errorf("ParseIngress() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			require.Equal(t, tt.want, got.Rules)
+		})
+	}
+}
+
+func TestSingleOriginSetsConfig(t *testing.T) {
+	flagSet := flag.NewFlagSet(t.Name(), flag.PanicOnError)
+	flagSet.Bool("hello-world", true, "")
+	flagSet.Duration(ProxyConnectTimeoutFlag, time.Second, "")
+	flagSet.Duration(ProxyTLSTimeoutFlag, time.Second, "")
+	flagSet.Duration(ProxyTCPKeepAlive, time.Second, "")
+	flagSet.Bool(ProxyNoHappyEyeballsFlag, true, "")
+	flagSet.Int(ProxyKeepAliveConnectionsFlag, 10, "")
+	flagSet.Duration(ProxyKeepAliveTimeoutFlag, time.Second, "")
+	flagSet.String(HTTPHostHeaderFlag, "example.com:8080", "")
+	flagSet.String(OriginServerNameFlag, "example.com", "")
+	flagSet.String(tlsconfig.OriginCAPoolFlag, "/etc/certs/ca.pem", "")
+	flagSet.Bool(NoTLSVerifyFlag, true, "")
+	flagSet.Bool(NoChunkedEncodingFlag, true, "")
+	flagSet.Bool(config.BastionFlag, true, "")
+	flagSet.String(ProxyAddressFlag, "localhost:8080", "")
+	flagSet.Uint(ProxyPortFlag, 8080, "")
+	flagSet.Bool(Socks5Flag, true, "")
+
+	cliCtx := cli.NewContext(cli.NewApp(), flagSet, nil)
+	err := cliCtx.Set("hello-world", "true")
+	require.NoError(t, err)
+	err = cliCtx.Set(ProxyConnectTimeoutFlag, "1s")
+	require.NoError(t, err)
+	err = cliCtx.Set(ProxyTLSTimeoutFlag, "1s")
+	require.NoError(t, err)
+	err = cliCtx.Set(ProxyTCPKeepAlive, "1s")
+	require.NoError(t, err)
+	err = cliCtx.Set(ProxyNoHappyEyeballsFlag, "true")
+	require.NoError(t, err)
+	err = cliCtx.Set(ProxyKeepAliveConnectionsFlag, "10")
+	require.NoError(t, err)
+	err = cliCtx.Set(ProxyKeepAliveTimeoutFlag, "1s")
+	require.NoError(t, err)
+	err = cliCtx.Set(HTTPHostHeaderFlag, "example.com:8080")
+	require.NoError(t, err)
+	err = cliCtx.Set(OriginServerNameFlag, "example.com")
+	require.NoError(t, err)
+	err = cliCtx.Set(tlsconfig.OriginCAPoolFlag, "/etc/certs/ca.pem")
+	require.NoError(t, err)
+	err = cliCtx.Set(NoTLSVerifyFlag, "true")
+	require.NoError(t, err)
+	err = cliCtx.Set(NoChunkedEncodingFlag, "true")
+	require.NoError(t, err)
+	err = cliCtx.Set(config.BastionFlag, "true")
+	require.NoError(t, err)
+	err = cliCtx.Set(ProxyAddressFlag, "localhost:8080")
+	require.NoError(t, err)
+	err = cliCtx.Set(ProxyPortFlag, "8080")
+	require.NoError(t, err)
+	err = cliCtx.Set(Socks5Flag, "true")
+	require.NoError(t, err)
+
+	allowURLFromArgs := false
+	logger, err := logger.New()
+	require.NoError(t, err)
+	ingress, err := NewSingleOrigin(cliCtx, allowURLFromArgs, logger)
+	require.NoError(t, err)
+
+	assert.Equal(t, time.Second, ingress.Rules[0].Config.ConnectTimeout)
+	assert.Equal(t, time.Second, ingress.Rules[0].Config.TLSTimeout)
+	assert.Equal(t, time.Second, ingress.Rules[0].Config.TCPKeepAlive)
+	assert.True(t, ingress.Rules[0].Config.NoHappyEyeballs)
+	assert.Equal(t, 10, ingress.Rules[0].Config.KeepAliveConnections)
+	assert.Equal(t, time.Second, ingress.Rules[0].Config.KeepAliveTimeout)
+	assert.Equal(t, "example.com:8080", ingress.Rules[0].Config.HTTPHostHeader)
+	assert.Equal(t, "example.com", ingress.Rules[0].Config.OriginServerName)
+	assert.Equal(t, "/etc/certs/ca.pem", ingress.Rules[0].Config.CAPool)
+	assert.True(t, ingress.Rules[0].Config.NoTLSVerify)
+	assert.True(t, ingress.Rules[0].Config.DisableChunkedEncoding)
+	assert.True(t, ingress.Rules[0].Config.BastionMode)
+	assert.Equal(t, "localhost:8080", ingress.Rules[0].Config.ProxyAddress)
+	assert.Equal(t, uint(8080), ingress.Rules[0].Config.ProxyPort)
+	assert.Equal(t, socksProxy, ingress.Rules[0].Config.ProxyType)
+}
+
+func TestFindMatchingRule(t *testing.T) {
+	ingress := Ingress{
+		Rules: []Rule{
+			{
+				Hostname: "tunnel-a.example.com",
+				Path:     nil,
+			},
+			{
+				Hostname: "tunnel-b.example.com",
+				Path:     mustParsePath(t, "/health"),
+			},
+			{
+				Hostname: "*",
+			},
+		},
+	}
+
+	tests := []struct {
+		host          string
+		path          string
+		wantRuleIndex int
+	}{
+		{
+			host:          "tunnel-a.example.com",
+			path:          "/",
+			wantRuleIndex: 0,
+		},
+		{
+			host:          "tunnel-a.example.com",
+			path:          "/pages/about",
+			wantRuleIndex: 0,
+		},
+		{
+			host:          "tunnel-a.example.com:443",
+			path:          "/pages/about",
+			wantRuleIndex: 0,
+		},
+		{
+			host:          "tunnel-b.example.com",
+			path:          "/health",
+			wantRuleIndex: 1,
+		},
+		{
+			host:          "tunnel-b.example.com",
+			path:          "/index.html",
+			wantRuleIndex: 2,
+		},
+		{
+			host:          "tunnel-c.example.com",
+			path:          "/",
+			wantRuleIndex: 2,
+		},
+	}
+
+	for i, test := range tests {
+		_, ruleIndex := ingress.FindMatchingRule(test.host, test.path)
+		assert.Equal(t, test.wantRuleIndex, ruleIndex, fmt.Sprintf("Expect host=%s, path=%s to match rule %d, got %d", test.host, test.path, test.wantRuleIndex, i))
+	}
+}
+
+func mustParsePath(t *testing.T, path string) *regexp.Regexp {
+	regexp, err := regexp.Compile(path)
+	assert.NoError(t, err)
+	return regexp
+}
+
+func MustParseURL(t *testing.T, rawURL string) *url.URL {
+	u, err := url.Parse(rawURL)
+	require.NoError(t, err)
+	return u
+}
+
+func BenchmarkFindMatch(b *testing.B) {
+	rulesYAML := `
+ingress:
+ - hostname: tunnel1.example.com
+   service: https://localhost:8000
+ - hostname: tunnel2.example.com
+   service: https://localhost:8001
+ - hostname: "*"
+   service: https://localhost:8002
+`
+
+	ing, err := ParseIngress(MustReadIngress(rulesYAML))
+	if err != nil {
+		b.Error(err)
+	}
+	for n := 0; n < b.N; n++ {
+		ing.FindMatchingRule("tunnel1.example.com", "")
+		ing.FindMatchingRule("tunnel2.example.com", "")
+		ing.FindMatchingRule("tunnel3.example.com", "")
+	}
+}
+
+func MustReadIngress(s string) *config.Configuration {
+	var conf config.Configuration
+	err := yaml.Unmarshal([]byte(s), &conf)
+	if err != nil {
+		panic(err)
+	}
+	return &conf
+}

ingress/origin_request_config.go

@@ -0,0 +1,331 @@
+package ingress
+
+import (
+	"time"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/tlsconfig"
+	"github.com/urfave/cli/v2"
+)
+
+const (
+	defaultConnectTimeout       = 30 * time.Second
+	defaultTLSTimeout           = 10 * time.Second
+	defaultTCPKeepAlive         = 30 * time.Second
+	defaultKeepAliveConnections = 100
+	defaultKeepAliveTimeout     = 90 * time.Second
+	defaultProxyAddress         = "127.0.0.1"
+
+	SSHServerFlag                 = "ssh-server"
+	Socks5Flag                    = "socks5"
+	ProxyConnectTimeoutFlag       = "proxy-connect-timeout"
+	ProxyTLSTimeoutFlag           = "proxy-tls-timeout"
+	ProxyTCPKeepAlive             = "proxy-tcp-keepalive"
+	ProxyNoHappyEyeballsFlag      = "proxy-no-happy-eyeballs"
+	ProxyKeepAliveConnectionsFlag = "proxy-keepalive-connections"
+	ProxyKeepAliveTimeoutFlag     = "proxy-keepalive-timeout"
+	HTTPHostHeaderFlag            = "http-host-header"
+	OriginServerNameFlag          = "origin-server-name"
+	NoTLSVerifyFlag               = "no-tls-verify"
+	NoChunkedEncodingFlag         = "no-chunked-encoding"
+	ProxyAddressFlag              = "proxy-address"
+	ProxyPortFlag                 = "proxy-port"
+)
+
+const (
+	socksProxy = "socks"
+)
+
+func originRequestFromSingeRule(c *cli.Context) OriginRequestConfig {
+	var connectTimeout time.Duration = defaultConnectTimeout
+	var tlsTimeout time.Duration = defaultTLSTimeout
+	var tcpKeepAlive time.Duration = defaultTCPKeepAlive
+	var noHappyEyeballs bool
+	var keepAliveConnections int = defaultKeepAliveConnections
+	var keepAliveTimeout time.Duration = defaultKeepAliveTimeout
+	var httpHostHeader string
+	var originServerName string
+	var caPool string
+	var noTLSVerify bool
+	var disableChunkedEncoding bool
+	var bastionMode bool
+	var proxyAddress = defaultProxyAddress
+	var proxyPort uint
+	var proxyType string
+	if flag := ProxyConnectTimeoutFlag; c.IsSet(flag) {
+		connectTimeout = c.Duration(flag)
+	}
+	if flag := ProxyTLSTimeoutFlag; c.IsSet(flag) {
+		tlsTimeout = c.Duration(flag)
+	}
+	if flag := ProxyTCPKeepAlive; c.IsSet(flag) {
+		tcpKeepAlive = c.Duration(flag)
+	}
+	if flag := ProxyNoHappyEyeballsFlag; c.IsSet(flag) {
+		noHappyEyeballs = c.Bool(flag)
+	}
+	if flag := ProxyKeepAliveConnectionsFlag; c.IsSet(flag) {
+		keepAliveConnections = c.Int(flag)
+	}
+	if flag := ProxyKeepAliveTimeoutFlag; c.IsSet(flag) {
+		keepAliveTimeout = c.Duration(flag)
+	}
+	if flag := HTTPHostHeaderFlag; c.IsSet(flag) {
+		httpHostHeader = c.String(flag)
+	}
+	if flag := OriginServerNameFlag; c.IsSet(flag) {
+		originServerName = c.String(flag)
+	}
+	if flag := tlsconfig.OriginCAPoolFlag; c.IsSet(flag) {
+		caPool = c.String(flag)
+	}
+	if flag := NoTLSVerifyFlag; c.IsSet(flag) {
+		noTLSVerify = c.Bool(flag)
+	}
+	if flag := NoChunkedEncodingFlag; c.IsSet(flag) {
+		disableChunkedEncoding = c.Bool(flag)
+	}
+	if flag := config.BastionFlag; c.IsSet(flag) {
+		bastionMode = c.Bool(flag)
+	}
+	if flag := ProxyAddressFlag; c.IsSet(flag) {
+		proxyAddress = c.String(flag)
+	}
+	if flag := ProxyPortFlag; c.IsSet(flag) {
+		proxyPort = c.Uint(flag)
+	}
+	if c.IsSet(Socks5Flag) {
+		proxyType = socksProxy
+	}
+	return OriginRequestConfig{
+		ConnectTimeout:         connectTimeout,
+		TLSTimeout:             tlsTimeout,
+		TCPKeepAlive:           tcpKeepAlive,
+		NoHappyEyeballs:        noHappyEyeballs,
+		KeepAliveConnections:   keepAliveConnections,
+		KeepAliveTimeout:       keepAliveTimeout,
+		HTTPHostHeader:         httpHostHeader,
+		OriginServerName:       originServerName,
+		CAPool:                 caPool,
+		NoTLSVerify:            noTLSVerify,
+		DisableChunkedEncoding: disableChunkedEncoding,
+		BastionMode:            bastionMode,
+		ProxyAddress:           proxyAddress,
+		ProxyPort:              proxyPort,
+		ProxyType:              proxyType,
+	}
+}
+
+func originRequestFromYAML(y config.OriginRequestConfig) OriginRequestConfig {
+	out := OriginRequestConfig{
+		ConnectTimeout:       defaultConnectTimeout,
+		TLSTimeout:           defaultTLSTimeout,
+		TCPKeepAlive:         defaultTCPKeepAlive,
+		KeepAliveConnections: defaultKeepAliveConnections,
+		KeepAliveTimeout:     defaultKeepAliveTimeout,
+		ProxyAddress:         defaultProxyAddress,
+	}
+	if y.ConnectTimeout != nil {
+		out.ConnectTimeout = *y.ConnectTimeout
+	}
+	if y.TLSTimeout != nil {
+		out.TLSTimeout = *y.TLSTimeout
+	}
+	if y.TCPKeepAlive != nil {
+		out.TCPKeepAlive = *y.TCPKeepAlive
+	}
+	if y.NoHappyEyeballs != nil {
+		out.NoHappyEyeballs = *y.NoHappyEyeballs
+	}
+	if y.KeepAliveConnections != nil {
+		out.KeepAliveConnections = *y.KeepAliveConnections
+	}
+	if y.KeepAliveTimeout != nil {
+		out.KeepAliveTimeout = *y.KeepAliveTimeout
+	}
+	if y.HTTPHostHeader != nil {
+		out.HTTPHostHeader = *y.HTTPHostHeader
+	}
+	if y.OriginServerName != nil {
+		out.OriginServerName = *y.OriginServerName
+	}
+	if y.CAPool != nil {
+		out.CAPool = *y.CAPool
+	}
+	if y.NoTLSVerify != nil {
+		out.NoTLSVerify = *y.NoTLSVerify
+	}
+	if y.DisableChunkedEncoding != nil {
+		out.DisableChunkedEncoding = *y.DisableChunkedEncoding
+	}
+	if y.BastionMode != nil {
+		out.BastionMode = *y.BastionMode
+	}
+	if y.ProxyAddress != nil {
+		out.ProxyAddress = *y.ProxyAddress
+	}
+	if y.ProxyPort != nil {
+		out.ProxyPort = *y.ProxyPort
+	}
+	if y.ProxyType != nil {
+		out.ProxyType = *y.ProxyType
+	}
+	return out
+}
+
+// OriginRequestConfig configures how Cloudflared sends requests to origin
+// services.
+// Note: To specify a time.Duration in go-yaml, use e.g. "3s" or "24h".
+type OriginRequestConfig struct {
+	// HTTP proxy timeout for establishing a new connection
+	ConnectTimeout time.Duration `yaml:"connectTimeout"`
+	// HTTP proxy timeout for completing a TLS handshake
+	TLSTimeout time.Duration `yaml:"tlsTimeout"`
+	// HTTP proxy TCP keepalive duration
+	TCPKeepAlive time.Duration `yaml:"tcpKeepAlive"`
+	// HTTP proxy should disable "happy eyeballs" for IPv4/v6 fallback
+	NoHappyEyeballs bool `yaml:"noHappyEyeballs"`
+	// HTTP proxy maximum keepalive connection pool size
+	KeepAliveConnections int `yaml:"keepAliveConnections"`
+	// HTTP proxy timeout for closing an idle connection
+	KeepAliveTimeout time.Duration `yaml:"keepAliveTimeout"`
+	// Sets the HTTP Host header for the local webserver.
+	HTTPHostHeader string `yaml:"httpHostHeader"`
+	// Hostname on the origin server certificate.
+	OriginServerName string `yaml:"originServerName"`
+	// Path to the CA for the certificate of your origin.
+	// This option should be used only if your certificate is not signed by Cloudflare.
+	CAPool string `yaml:"caPool"`
+	// Disables TLS verification of the certificate presented by your origin.
+	// Will allow any certificate from the origin to be accepted.
+	// Note: The connection from your machine to Cloudflare's Edge is still encrypted.
+	NoTLSVerify bool `yaml:"noTLSVerify"`
+	// Disables chunked transfer encoding.
+	// Useful if you are running a WSGI server.
+	DisableChunkedEncoding bool `yaml:"disableChunkedEncoding"`
+	// Runs as jump host
+	BastionMode bool `yaml:"bastionMode"`
+	// Listen address for the proxy.
+	ProxyAddress string `yaml:"proxyAddress"`
+	// Listen port for the proxy.
+	ProxyPort uint `yaml:"proxyPort"`
+	// What sort of proxy should be started
+	ProxyType string `yaml:"proxyType"`
+}
+
+func (defaults *OriginRequestConfig) setConnectTimeout(overrides config.OriginRequestConfig) {
+	if val := overrides.ConnectTimeout; val != nil {
+		defaults.ConnectTimeout = *val
+	}
+}
+
+func (defaults *OriginRequestConfig) setTLSTimeout(overrides config.OriginRequestConfig) {
+	if val := overrides.TLSTimeout; val != nil {
+		defaults.TLSTimeout = *val
+	}
+}
+
+func (defaults *OriginRequestConfig) setNoHappyEyeballs(overrides config.OriginRequestConfig) {
+	if val := overrides.NoHappyEyeballs; val != nil {
+		defaults.NoHappyEyeballs = *val
+	}
+}
+
+func (defaults *OriginRequestConfig) setKeepAliveConnections(overrides config.OriginRequestConfig) {
+	if val := overrides.KeepAliveConnections; val != nil {
+		defaults.KeepAliveConnections = *val
+	}
+}
+
+func (defaults *OriginRequestConfig) setKeepAliveTimeout(overrides config.OriginRequestConfig) {
+	if val := overrides.KeepAliveTimeout; val != nil {
+		defaults.KeepAliveTimeout = *val
+	}
+}
+
+func (defaults *OriginRequestConfig) setTCPKeepAlive(overrides config.OriginRequestConfig) {
+	if val := overrides.TCPKeepAlive; val != nil {
+		defaults.TCPKeepAlive = *val
+	}
+}
+
+func (defaults *OriginRequestConfig) setHTTPHostHeader(overrides config.OriginRequestConfig) {
+	if val := overrides.HTTPHostHeader; val != nil {
+		defaults.HTTPHostHeader = *val
+	}
+}
+
+func (defaults *OriginRequestConfig) setOriginServerName(overrides config.OriginRequestConfig) {
+	if val := overrides.OriginServerName; val != nil {
+		defaults.OriginServerName = *val
+	}
+}
+
+func (defaults *OriginRequestConfig) setCAPool(overrides config.OriginRequestConfig) {
+	if val := overrides.CAPool; val != nil {
+		defaults.CAPool = *val
+	}
+}
+
+func (defaults *OriginRequestConfig) setNoTLSVerify(overrides config.OriginRequestConfig) {
+	if val := overrides.NoTLSVerify; val != nil {
+		defaults.NoTLSVerify = *val
+	}
+}
+
+func (defaults *OriginRequestConfig) setDisableChunkedEncoding(overrides config.OriginRequestConfig) {
+	if val := overrides.DisableChunkedEncoding; val != nil {
+		defaults.DisableChunkedEncoding = *val
+	}
+}
+
+func (defaults *OriginRequestConfig) setBastionMode(overrides config.OriginRequestConfig) {
+	if val := overrides.BastionMode; val != nil {
+		defaults.BastionMode = *val
+	}
+}
+
+func (defaults *OriginRequestConfig) setProxyPort(overrides config.OriginRequestConfig) {
+	if val := overrides.ProxyPort; val != nil {
+		defaults.ProxyPort = *val
+	}
+}
+
+func (defaults *OriginRequestConfig) setProxyAddress(overrides config.OriginRequestConfig) {
+	if val := overrides.ProxyAddress; val != nil {
+		defaults.ProxyAddress = *val
+	}
+}
+
+func (defaults *OriginRequestConfig) setProxyType(overrides config.OriginRequestConfig) {
+	if val := overrides.ProxyType; val != nil {
+		defaults.ProxyType = *val
+	}
+}
+
+// SetConfig gets config for the requests that cloudflared sends to origins.
+// Each field has a setter method which sets a value for the field by trying to find:
+//   1. The user config for this rule
+//   2. The user config for the overall ingress config
+//   3. Defaults chosen by the cloudflared team
+//   4. Golang zero values for that type
+// If an earlier option isn't set, it will try the next option down.
+func setConfig(defaults OriginRequestConfig, overrides config.OriginRequestConfig) OriginRequestConfig {
+	cfg := defaults
+	cfg.setConnectTimeout(overrides)
+	cfg.setTLSTimeout(overrides)
+	cfg.setNoHappyEyeballs(overrides)
+	cfg.setKeepAliveConnections(overrides)
+	cfg.setKeepAliveTimeout(overrides)
+	cfg.setTCPKeepAlive(overrides)
+	cfg.setHTTPHostHeader(overrides)
+	cfg.setOriginServerName(overrides)
+	cfg.setCAPool(overrides)
+	cfg.setNoTLSVerify(overrides)
+	cfg.setDisableChunkedEncoding(overrides)
+	cfg.setBastionMode(overrides)
+	cfg.setProxyPort(overrides)
+	cfg.setProxyAddress(overrides)
+	cfg.setProxyType(overrides)
+	return cfg
+}

ingress/origin_request_config_test.go

@@ -0,0 +1,202 @@
+package ingress
+
+import (
+	"flag"
+	"testing"
+	"time"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/stretchr/testify/require"
+	"github.com/urfave/cli/v2"
+	"gopkg.in/yaml.v2"
+)
+
+// Ensure that the nullable config from `config` package and the
+// non-nullable config from `ingress` package have the same number of
+// fields.
+// This test ensures that programmers didn't add a new field to
+// one struct and forget to add it to the other ;)
+func TestCorrespondingFields(t *testing.T) {
+	require.Equal(
+		t,
+		CountFields(t, config.OriginRequestConfig{}),
+		CountFields(t, OriginRequestConfig{}),
+	)
+}
+
+func CountFields(t *testing.T, val interface{}) int {
+	b, err := yaml.Marshal(val)
+	require.NoError(t, err)
+	m := make(map[string]interface{}, 0)
+	err = yaml.Unmarshal(b, &m)
+	require.NoError(t, err)
+	return len(m)
+}
+
+func TestOriginRequestConfigOverrides(t *testing.T) {
+	rulesYAML := `
+originRequest:
+  connectTimeout: 1m
+  tlsTimeout: 1s
+  noHappyEyeballs: true
+  tcpKeepAlive: 1s
+  keepAliveConnections: 1
+  keepAliveTimeout: 1s
+  httpHostHeader: abc
+  originServerName: a1
+  caPool: /tmp/path0
+  noTLSVerify: true
+  disableChunkedEncoding: true
+  bastionMode: True
+  proxyAddress: 127.1.2.3
+  proxyPort: 100
+  proxyType: socks5
+ingress:
+- hostname: tun.example.com
+  service: https://localhost:8000
+- hostname: "*"
+  service: https://localhost:8001
+  originRequest:
+    connectTimeout: 2m
+    tlsTimeout: 2s
+    noHappyEyeballs: false
+    tcpKeepAlive: 2s
+    keepAliveConnections: 2
+    keepAliveTimeout: 2s
+    httpHostHeader: def
+    originServerName: b2
+    caPool: /tmp/path1
+    noTLSVerify: false
+    disableChunkedEncoding: false
+    bastionMode: false
+    proxyAddress: interface
+    proxyPort: 200
+    proxyType: ""
+`
+	ing, err := ParseIngress(MustReadIngress(rulesYAML))
+	if err != nil {
+		t.Error(err)
+	}
+
+	// Rule 0 didn't override anything, so it inherits the user-specified
+	// root-level configuration.
+	actual0 := ing.Rules[0].Config
+	expected0 := OriginRequestConfig{
+		ConnectTimeout:         1 * time.Minute,
+		TLSTimeout:             1 * time.Second,
+		NoHappyEyeballs:        true,
+		TCPKeepAlive:           1 * time.Second,
+		KeepAliveConnections:   1,
+		KeepAliveTimeout:       1 * time.Second,
+		HTTPHostHeader:         "abc",
+		OriginServerName:       "a1",
+		CAPool:                 "/tmp/path0",
+		NoTLSVerify:            true,
+		DisableChunkedEncoding: true,
+		BastionMode:            true,
+		ProxyAddress:           "127.1.2.3",
+		ProxyPort:              uint(100),
+		ProxyType:              "socks5",
+	}
+	require.Equal(t, expected0, actual0)
+
+	// Rule 1 overrode all the root-level config.
+	actual1 := ing.Rules[1].Config
+	expected1 := OriginRequestConfig{
+		ConnectTimeout:         2 * time.Minute,
+		TLSTimeout:             2 * time.Second,
+		NoHappyEyeballs:        false,
+		TCPKeepAlive:           2 * time.Second,
+		KeepAliveConnections:   2,
+		KeepAliveTimeout:       2 * time.Second,
+		HTTPHostHeader:         "def",
+		OriginServerName:       "b2",
+		CAPool:                 "/tmp/path1",
+		NoTLSVerify:            false,
+		DisableChunkedEncoding: false,
+		BastionMode:            false,
+		ProxyAddress:           "interface",
+		ProxyPort:              uint(200),
+		ProxyType:              "",
+	}
+	require.Equal(t, expected1, actual1)
+}
+
+func TestOriginRequestConfigDefaults(t *testing.T) {
+	rulesYAML := `
+ingress:
+- hostname: tun.example.com
+  service: https://localhost:8000
+- hostname: "*"
+  service: https://localhost:8001
+  originRequest:
+    connectTimeout: 2m
+    tlsTimeout: 2s
+    noHappyEyeballs: false
+    tcpKeepAlive: 2s
+    keepAliveConnections: 2
+    keepAliveTimeout: 2s
+    httpHostHeader: def
+    originServerName: b2
+    caPool: /tmp/path1
+    noTLSVerify: false
+    disableChunkedEncoding: false
+    bastionMode: false
+    proxyAddress: interface
+    proxyPort: 200
+    proxyType: ""
+`
+	ing, err := ParseIngress(MustReadIngress(rulesYAML))
+	if err != nil {
+		t.Error(err)
+	}
+
+	// Rule 0 didn't override anything, so it inherits the cloudflared defaults
+	actual0 := ing.Rules[0].Config
+	expected0 := OriginRequestConfig{
+		ConnectTimeout:       defaultConnectTimeout,
+		TLSTimeout:           defaultTLSTimeout,
+		TCPKeepAlive:         defaultTCPKeepAlive,
+		KeepAliveConnections: defaultKeepAliveConnections,
+		KeepAliveTimeout:     defaultKeepAliveTimeout,
+		ProxyAddress:         defaultProxyAddress,
+	}
+	require.Equal(t, expected0, actual0)
+
+	// Rule 1 overrode all defaults.
+	actual1 := ing.Rules[1].Config
+	expected1 := OriginRequestConfig{
+		ConnectTimeout:         2 * time.Minute,
+		TLSTimeout:             2 * time.Second,
+		NoHappyEyeballs:        false,
+		TCPKeepAlive:           2 * time.Second,
+		KeepAliveConnections:   2,
+		KeepAliveTimeout:       2 * time.Second,
+		HTTPHostHeader:         "def",
+		OriginServerName:       "b2",
+		CAPool:                 "/tmp/path1",
+		NoTLSVerify:            false,
+		DisableChunkedEncoding: false,
+		BastionMode:            false,
+		ProxyAddress:           "interface",
+		ProxyPort:              uint(200),
+		ProxyType:              "",
+	}
+	require.Equal(t, expected1, actual1)
+}
+
+func TestDefaultConfigFromCLI(t *testing.T) {
+	set := flag.NewFlagSet("contrive", 0)
+	c := cli.NewContext(nil, set, nil)
+
+	expected := OriginRequestConfig{
+		ConnectTimeout:       defaultConnectTimeout,
+		TLSTimeout:           defaultTLSTimeout,
+		TCPKeepAlive:         defaultTCPKeepAlive,
+		KeepAliveConnections: defaultKeepAliveConnections,
+		KeepAliveTimeout:     defaultKeepAliveTimeout,
+		ProxyAddress:         defaultProxyAddress,
+	}
+	actual := originRequestFromSingeRule(c)
+	require.Equal(t, expected, actual)
+}

ingress/origin_service.go

@@ -0,0 +1,343 @@
+package ingress
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"net/url"
+	"strconv"
+	"sync"
+	"time"
+
+	"github.com/cloudflare/cloudflared/hello"
+	"github.com/cloudflare/cloudflared/logger"
+	"github.com/cloudflare/cloudflared/socks"
+	"github.com/cloudflare/cloudflared/tlsconfig"
+	"github.com/cloudflare/cloudflared/websocket"
+	gws "github.com/gorilla/websocket"
+	"github.com/pkg/errors"
+)
+
+// OriginService is something a tunnel can proxy traffic to.
+type OriginService interface {
+	// RoundTrip is how cloudflared proxies eyeball requests to the actual origin services
+	http.RoundTripper
+	String() string
+	// Start the origin service if it's managed by cloudflared, e.g. proxy servers or Hello World.
+	// If it's not managed by cloudflared, this is a no-op because the user is responsible for
+	// starting the origin service.
+	start(wg *sync.WaitGroup, log logger.Service, shutdownC <-chan struct{}, errC chan error, cfg OriginRequestConfig) error
+}
+
+// unixSocketPath is an OriginService representing a unix socket (which accepts HTTP)
+type unixSocketPath struct {
+	path      string
+	transport *http.Transport
+}
+
+func (o *unixSocketPath) String() string {
+	return "unix socket: " + o.path
+}
+
+func (o *unixSocketPath) start(wg *sync.WaitGroup, log logger.Service, shutdownC <-chan struct{}, errC chan error, cfg OriginRequestConfig) error {
+	transport, err := newHTTPTransport(o, cfg, log)
+	if err != nil {
+		return err
+	}
+	o.transport = transport
+	return nil
+}
+
+func (o *unixSocketPath) RoundTrip(req *http.Request) (*http.Response, error) {
+	return o.transport.RoundTrip(req)
+}
+
+func (o *unixSocketPath) Dial(reqURL *url.URL, headers http.Header) (*gws.Conn, *http.Response, error) {
+	d := &gws.Dialer{
+		NetDial:         o.transport.Dial,
+		NetDialContext:  o.transport.DialContext,
+		TLSClientConfig: o.transport.TLSClientConfig,
+	}
+	reqURL.Scheme = websocket.ChangeRequestScheme(reqURL)
+	return d.Dial(reqURL.String(), headers)
+}
+
+// localService is an OriginService listening on a TCP/IP address the user's origin can route to.
+type localService struct {
+	// The URL for the user's origin service
+	RootURL *url.URL
+	// The URL that cloudflared should send requests to.
+	// If this origin requires starting a proxy, this is the proxy's address,
+	// and that proxy points to RootURL. Otherwise, this is equal to RootURL.
+	URL       *url.URL
+	transport *http.Transport
+}
+
+func (o *localService) Dial(reqURL *url.URL, headers http.Header) (*gws.Conn, *http.Response, error) {
+	d := &gws.Dialer{TLSClientConfig: o.transport.TLSClientConfig}
+	// Rewrite the request URL so that it goes to the origin service.
+	reqURL.Host = o.URL.Host
+	reqURL.Scheme = websocket.ChangeRequestScheme(o.URL)
+	return d.Dial(reqURL.String(), headers)
+}
+
+func (o *localService) start(wg *sync.WaitGroup, log logger.Service, shutdownC <-chan struct{}, errC chan error, cfg OriginRequestConfig) error {
+	transport, err := newHTTPTransport(o, cfg, log)
+	if err != nil {
+		return err
+	}
+	o.transport = transport
+
+	// Start a proxy if one is needed
+	if staticHost := o.staticHost(); originRequiresProxy(staticHost, cfg) {
+		if err := o.startProxy(staticHost, wg, log, shutdownC, errC, cfg); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (o *localService) startProxy(staticHost string, wg *sync.WaitGroup, log logger.Service, shutdownC <-chan struct{}, errC chan error, cfg OriginRequestConfig) error {
+
+	// Start a listener for the proxy
+	proxyAddress := net.JoinHostPort(cfg.ProxyAddress, strconv.Itoa(int(cfg.ProxyPort)))
+	listener, err := net.Listen("tcp", proxyAddress)
+	if err != nil {
+		log.Errorf("Cannot start Websocket Proxy Server: %s", err)
+		return errors.Wrap(err, "Cannot start Websocket Proxy Server")
+	}
+
+	// Start the proxy itself
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		streamHandler := websocket.DefaultStreamHandler
+		// This origin's config specifies what type of proxy to start.
+		switch cfg.ProxyType {
+		case socksProxy:
+			log.Info("SOCKS5 server started")
+			streamHandler = func(wsConn *websocket.Conn, remoteConn net.Conn, _ http.Header) {
+				dialer := socks.NewConnDialer(remoteConn)
+				requestHandler := socks.NewRequestHandler(dialer)
+				socksServer := socks.NewConnectionHandler(requestHandler)
+
+				socksServer.Serve(wsConn)
+			}
+		case "":
+			log.Debug("Not starting any websocket proxy")
+		default:
+			log.Errorf("%s isn't a valid proxy (valid options are {%s})", cfg.ProxyType, socksProxy)
+		}
+
+		errC <- websocket.StartProxyServer(log, listener, staticHost, shutdownC, streamHandler)
+	}()
+
+	// Modify this origin, so that it no longer points at the origin service directly.
+	// Instead, it points at the proxy to the origin service.
+	newURL, err := url.Parse("http://" + listener.Addr().String())
+	if err != nil {
+		return err
+	}
+	o.URL = newURL
+	return nil
+}
+
+func (o *localService) String() string {
+	if o.isBastion() {
+		return "Bastion"
+	}
+	return o.URL.String()
+}
+
+func (o *localService) isBastion() bool {
+	return o.URL == nil
+}
+
+func (o *localService) RoundTrip(req *http.Request) (*http.Response, error) {
+	// Rewrite the request URL so that it goes to the origin service.
+	req.URL.Host = o.URL.Host
+	req.URL.Scheme = o.URL.Scheme
+	return o.transport.RoundTrip(req)
+}
+
+func (o *localService) staticHost() string {
+
+	if o.URL == nil {
+		return ""
+	}
+
+	addPortIfMissing := func(uri *url.URL, port int) string {
+		if uri.Port() != "" {
+			return uri.Host
+		}
+		return fmt.Sprintf("%s:%d", uri.Hostname(), port)
+	}
+
+	switch o.URL.Scheme {
+	case "ssh":
+		return addPortIfMissing(o.URL, 22)
+	case "rdp":
+		return addPortIfMissing(o.URL, 3389)
+	case "smb":
+		return addPortIfMissing(o.URL, 445)
+	case "tcp":
+		return addPortIfMissing(o.URL, 7864) // just a random port since there isn't a default in this case
+	}
+	return ""
+
+}
+
+// HelloWorld is an OriginService for the built-in Hello World server.
+// Users only use this for testing and experimenting with cloudflared.
+type helloWorld struct {
+	server    net.Listener
+	transport *http.Transport
+}
+
+func (o *helloWorld) String() string {
+	return "Hello World test origin"
+}
+
+// Start starts a HelloWorld server and stores its address in the Service receiver.
+func (o *helloWorld) start(wg *sync.WaitGroup, log logger.Service, shutdownC <-chan struct{}, errC chan error, cfg OriginRequestConfig) error {
+	transport, err := newHTTPTransport(o, cfg, log)
+	if err != nil {
+		return err
+	}
+	o.transport = transport
+	helloListener, err := hello.CreateTLSListener("127.0.0.1:")
+	if err != nil {
+		return errors.Wrap(err, "Cannot start Hello World Server")
+	}
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		_ = hello.StartHelloWorldServer(log, helloListener, shutdownC)
+	}()
+	o.server = helloListener
+	return nil
+}
+
+func (o *helloWorld) RoundTrip(req *http.Request) (*http.Response, error) {
+	// Rewrite the request URL so that it goes to the Hello World server.
+	req.URL.Host = o.server.Addr().String()
+	req.URL.Scheme = "https"
+	return o.transport.RoundTrip(req)
+}
+
+func (o *helloWorld) Dial(reqURL *url.URL, headers http.Header) (*gws.Conn, *http.Response, error) {
+	d := &gws.Dialer{
+		TLSClientConfig: o.transport.TLSClientConfig,
+	}
+	reqURL.Host = o.server.Addr().String()
+	reqURL.Scheme = "wss"
+	return d.Dial(reqURL.String(), headers)
+}
+
+func originRequiresProxy(staticHost string, cfg OriginRequestConfig) bool {
+	return staticHost != "" || cfg.BastionMode
+}
+
+// statusCode is an OriginService that just responds with a given HTTP status.
+// Typical use-case is "user wants the catch-all rule to just respond 404".
+type statusCode struct {
+	resp *http.Response
+}
+
+func newStatusCode(status int) statusCode {
+	resp := &http.Response{
+		StatusCode: status,
+		Status:     fmt.Sprintf("%d %s", status, http.StatusText(status)),
+		Body:       new(NopReadCloser),
+	}
+	return statusCode{resp: resp}
+}
+
+func (o *statusCode) String() string {
+	return fmt.Sprintf("HTTP %d", o.resp.StatusCode)
+}
+
+func (o *statusCode) start(wg *sync.WaitGroup, log logger.Service, shutdownC <-chan struct{}, errC chan error, cfg OriginRequestConfig) error {
+	return nil
+}
+
+func (o *statusCode) RoundTrip(_ *http.Request) (*http.Response, error) {
+	return o.resp, nil
+}
+
+type NopReadCloser struct{}
+
+// Read always returns EOF to signal end of input
+func (nrc *NopReadCloser) Read(buf []byte) (int, error) {
+	return 0, io.EOF
+}
+
+func (nrc *NopReadCloser) Close() error {
+	return nil
+}
+
+func newHTTPTransport(service OriginService, cfg OriginRequestConfig, log logger.Service) (*http.Transport, error) {
+	originCertPool, err := tlsconfig.LoadOriginCA(cfg.CAPool, log)
+	if err != nil {
+		return nil, errors.Wrap(err, "Error loading cert pool")
+	}
+
+	httpTransport := http.Transport{
+		Proxy:                 http.ProxyFromEnvironment,
+		MaxIdleConns:          cfg.KeepAliveConnections,
+		MaxIdleConnsPerHost:   cfg.KeepAliveConnections,
+		IdleConnTimeout:       cfg.KeepAliveTimeout,
+		TLSHandshakeTimeout:   cfg.TLSTimeout,
+		ExpectContinueTimeout: 1 * time.Second,
+		TLSClientConfig:       &tls.Config{RootCAs: originCertPool, InsecureSkipVerify: cfg.NoTLSVerify},
+	}
+	if _, isHelloWorld := service.(*helloWorld); !isHelloWorld && cfg.OriginServerName != "" {
+		httpTransport.TLSClientConfig.ServerName = cfg.OriginServerName
+	}
+
+	dialer := &net.Dialer{
+		Timeout:   cfg.ConnectTimeout,
+		KeepAlive: cfg.TCPKeepAlive,
+	}
+	if cfg.NoHappyEyeballs {
+		dialer.FallbackDelay = -1 // As of Golang 1.12, a negative delay disables "happy eyeballs"
+	}
+
+	// DialContext depends on which kind of origin is being used.
+	dialContext := dialer.DialContext
+	switch service := service.(type) {
+
+	// If this origin is a unix socket, enforce network type "unix".
+	case *unixSocketPath:
+		httpTransport.DialContext = func(ctx context.Context, _, _ string) (net.Conn, error) {
+			return dialContext(ctx, "unix", service.path)
+		}
+
+	// Otherwise, use the regular network config.
+	default:
+		httpTransport.DialContext = dialContext
+	}
+
+	return &httpTransport, nil
+}
+
+// MockOriginService should only be used by other packages to mock OriginService. Set Transport to configure desired RoundTripper behavior.
+type MockOriginService struct {
+	Transport http.RoundTripper
+}
+
+func (mos MockOriginService) RoundTrip(req *http.Request) (*http.Response, error) {
+	return mos.Transport.RoundTrip(req)
+}
+
+func (mos MockOriginService) String() string {
+	return "MockOriginService"
+}
+
+func (mos MockOriginService) start(wg *sync.WaitGroup, log logger.Service, shutdownC <-chan struct{}, errC chan error, cfg OriginRequestConfig) error {
+	return nil
+}

ingress/rule.go

@@ -0,0 +1,50 @@
+package ingress
+
+import (
+	"regexp"
+	"strings"
+)
+
+// Rule routes traffic from a hostname/path on the public internet to the
+// service running on the given URL.
+type Rule struct {
+	// Requests for this hostname will be proxied to this rule's service.
+	Hostname string
+
+	// Path is an optional regex that can specify path-driven ingress rules.
+	Path *regexp.Regexp
+
+	// A (probably local) address. Requests for a hostname which matches this
+	// rule's hostname pattern will be proxied to the service running on this
+	// address.
+	Service OriginService
+
+	// Configure the request cloudflared sends to this specific origin.
+	Config OriginRequestConfig
+}
+
+// MultiLineString is for outputting rules in a human-friendly way when Cloudflared
+// is used as a CLI tool (not as a daemon).
+func (r Rule) MultiLineString() string {
+	var out strings.Builder
+	if r.Hostname != "" {
+		out.WriteString("\thostname: ")
+		out.WriteString(r.Hostname)
+		out.WriteRune('\n')
+	}
+	if r.Path != nil {
+		out.WriteString("\tpath: ")
+		out.WriteString(r.Path.String())
+		out.WriteRune('\n')
+	}
+	out.WriteString("\tservice: ")
+	out.WriteString(r.Service.String())
+	return out.String()
+}
+
+// Matches checks if the rule matches a given hostname/path combination.
+func (r *Rule) Matches(hostname, path string) bool {
+	hostMatch := r.Hostname == "" || r.Hostname == "*" || matchHost(r.Hostname, hostname)
+	pathMatch := r.Path == nil || r.Path.MatchString(path)
+	return hostMatch && pathMatch
+}

ingress/rule_test.go

@@ -0,0 +1,146 @@
+package ingress
+
+import (
+	"io"
+	"net/http/httptest"
+	"net/url"
+	"regexp"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func Test_rule_matches(t *testing.T) {
+	type fields struct {
+		Hostname string
+		Path     *regexp.Regexp
+		Service  OriginService
+	}
+	type args struct {
+		requestURL *url.URL
+	}
+	tests := []struct {
+		name   string
+		fields fields
+		args   args
+		want   bool
+	}{
+		{
+			name: "Just hostname, pass",
+			fields: fields{
+				Hostname: "example.com",
+			},
+			args: args{
+				requestURL: MustParseURL(t, "https://example.com"),
+			},
+			want: true,
+		},
+		{
+			name: "Entire hostname is wildcard, should match everything",
+			fields: fields{
+				Hostname: "*",
+			},
+			args: args{
+				requestURL: MustParseURL(t, "https://example.com"),
+			},
+			want: true,
+		},
+		{
+			name: "Just hostname, fail",
+			fields: fields{
+				Hostname: "example.com",
+			},
+			args: args{
+				requestURL: MustParseURL(t, "https://foo.bar"),
+			},
+			want: false,
+		},
+		{
+			name: "Just wildcard hostname, pass",
+			fields: fields{
+				Hostname: "*.example.com",
+			},
+			args: args{
+				requestURL: MustParseURL(t, "https://adam.example.com"),
+			},
+			want: true,
+		},
+		{
+			name: "Just wildcard hostname, fail",
+			fields: fields{
+				Hostname: "*.example.com",
+			},
+			args: args{
+				requestURL: MustParseURL(t, "https://tunnel.com"),
+			},
+			want: false,
+		},
+		{
+			name: "Just wildcard outside of subdomain in hostname, fail",
+			fields: fields{
+				Hostname: "*example.com",
+			},
+			args: args{
+				requestURL: MustParseURL(t, "https://www.example.com"),
+			},
+			want: false,
+		},
+		{
+			name: "Wildcard over multiple subdomains",
+			fields: fields{
+				Hostname: "*.example.com",
+			},
+			args: args{
+				requestURL: MustParseURL(t, "https://adam.chalmers.example.com"),
+			},
+			want: true,
+		},
+		{
+			name: "Hostname and path",
+			fields: fields{
+				Hostname: "*.example.com",
+				Path:     regexp.MustCompile("/static/.*\\.html"),
+			},
+			args: args{
+				requestURL: MustParseURL(t, "https://www.example.com/static/index.html"),
+			},
+			want: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			r := Rule{
+				Hostname: tt.fields.Hostname,
+				Path:     tt.fields.Path,
+				Service:  tt.fields.Service,
+			}
+			u := tt.args.requestURL
+			if got := r.Matches(u.Hostname(), u.Path); got != tt.want {
+				t.Errorf("rule.matches() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestStaticHTTPStatus(t *testing.T) {
+	o := newStatusCode(404)
+	buf := make([]byte, 100)
+
+	sendReq := func() {
+		resp, err := o.RoundTrip(nil)
+		require.NoError(t, err)
+		_, err = resp.Body.Read(buf)
+		require.Equal(t, io.EOF, err)
+		require.NoError(t, resp.Body.Close())
+		require.Equal(t, 404, resp.StatusCode)
+
+		resp, err = o.RoundTrip(nil)
+		require.NoError(t, err)
+		w := httptest.NewRecorder()
+		n, err := io.Copy(w, resp.Body)
+		require.NoError(t, err)
+		require.Equal(t, int64(0), n)
+	}
+	sendReq()
+	sendReq()
+}

logger/configuration.go

@@ -0,0 +1,118 @@
+package logger
+
+var defaultConfig = createDefaultConfig()
+
+// Logging configuration
+type Config struct {
+	ConsoleConfig *ConsoleConfig // If nil, the logger will not log into the console
+	FileConfig    *FileConfig    // If nil, the logger will not use an individual log file
+	RollingConfig *RollingConfig // If nil, the logger will not use a rolling log
+
+	MinLevel string // debug | info | error | fatal
+}
+
+type ConsoleConfig struct {
+	noColor bool
+}
+
+type FileConfig struct {
+	Filepath string
+}
+
+type RollingConfig struct {
+	Directory string
+	Filename  string
+
+	maxSize    int // megabytes
+	maxBackups int // files
+	maxAge     int // days
+}
+
+func createDefaultConfig() Config {
+	const minLevel = "fatal"
+
+	const RollingMaxSize = 1    // Mb
+	const RollingMaxBackups = 5 // files
+	const RollingMaxAge = 0     // Keep forever
+	const rollingLogFilename = "cloudflared.log"
+
+	return Config{
+		ConsoleConfig: &ConsoleConfig{
+			noColor: false,
+		},
+		FileConfig: &FileConfig{
+			Filepath: "",
+		},
+		RollingConfig: &RollingConfig{
+			Directory:  "",
+			Filename:   rollingLogFilename,
+			maxSize:    RollingMaxSize,
+			maxBackups: RollingMaxBackups,
+			maxAge:     RollingMaxAge,
+		},
+		MinLevel: minLevel,
+	}
+}
+
+func CreateConfig(
+	minLevel string,
+	disableTerminal bool,
+	rollingLogPath, nonRollingLogFilePath string,
+) *Config {
+	var console *ConsoleConfig
+	if !disableTerminal {
+		console = createConsoleConfig()
+	}
+
+	var file *FileConfig
+	if nonRollingLogFilePath != "" {
+		file = createFileConfig(nonRollingLogFilePath)
+	}
+
+	var rolling *RollingConfig
+	if rollingLogPath != "" {
+		rolling = createRollingConfig(rollingLogPath)
+	}
+
+	if minLevel == "" {
+		minLevel = defaultConfig.MinLevel
+	}
+
+	return &Config{
+		ConsoleConfig: console,
+		FileConfig:    file,
+		RollingConfig: rolling,
+
+		MinLevel: minLevel,
+	}
+}
+
+func createConsoleConfig() *ConsoleConfig {
+	return &ConsoleConfig{
+		noColor: false,
+	}
+}
+
+func createFileConfig(filepath string) *FileConfig {
+	if filepath == "" {
+		filepath = defaultConfig.FileConfig.Filepath
+	}
+
+	return &FileConfig{
+		Filepath: filepath,
+	}
+}
+
+func createRollingConfig(directory string) *RollingConfig {
+	if directory == "" {
+		directory = defaultConfig.RollingConfig.Directory
+	}
+
+	return &RollingConfig{
+		Directory:  directory,
+		Filename:   defaultConfig.RollingConfig.Filename,
+		maxSize:    defaultConfig.RollingConfig.maxSize,
+		maxBackups: defaultConfig.RollingConfig.maxBackups,
+		maxAge:     defaultConfig.RollingConfig.maxAge,
+	}
+}

logger/create.go

@@ -8,6 +8,20 @@ import (
 	"time"
 
 	"github.com/alecthomas/units"
+	"github.com/urfave/cli/v2"
+)
+
+const (
+	EnableTerminalLog  = false
+	DisableTerminalLog = true
+
+	LogLevelFlag          = "loglevel"
+	LogFileFlag           = "logfile"
+	LogDirectoryFlag      = "log-directory"
+	LogTransportLevelFlag = "transport-loglevel"
+
+	LogSSHDirectoryFlag = "log-directory"
+	LogSSHLevelFlag     = "log-level"
 )
 
 // Option is to encaspulate actions that will be called by Parse and run later to build an Options struct
@@ -127,6 +141,63 @@ func New(opts ...Option) (*OutputWriter, error) {
 	return l, nil
 }
 
+func NewInHouse(loggerConfig *Config) (*OutputWriter, error) {
+	var loggerOpts []Option
+
+	var logPath string
+	if loggerConfig.FileConfig != nil {
+		logPath = loggerConfig.FileConfig.Filepath
+	}
+	if logPath == "" && loggerConfig.RollingConfig != nil {
+		logPath = loggerConfig.RollingConfig.Directory
+	}
+
+	if logPath != "" {
+		loggerOpts = append(loggerOpts, DefaultFile(logPath))
+	}
+
+	loggerOpts = append(loggerOpts, LogLevelString(loggerConfig.MinLevel))
+
+	if loggerConfig.ConsoleConfig == nil {
+		disableOption := DisableTerminal(true)
+		loggerOpts = append(loggerOpts, disableOption)
+	}
+
+	l, err := New(loggerOpts...)
+	if err != nil {
+		return nil, err
+	}
+
+	return l, nil
+}
+
+func CreateTransportLoggerFromContext(c *cli.Context, disableTerminal bool) (*OutputWriter, error) {
+	return createFromContext(c, LogTransportLevelFlag, LogDirectoryFlag, disableTerminal)
+}
+
+func CreateLoggerFromContext(c *cli.Context, disableTerminal bool) (*OutputWriter, error) {
+	return createFromContext(c, LogLevelFlag, LogDirectoryFlag, disableTerminal)
+}
+
+func CreateSSHLoggerFromContext(c *cli.Context, disableTerminal bool) (*OutputWriter, error) {
+	return createFromContext(c, LogSSHLevelFlag, LogSSHDirectoryFlag, disableTerminal)
+}
+
+func createFromContext(
+	c *cli.Context,
+	logLevelFlagName,
+	logDirectoryFlagName string,
+	disableTerminal bool,
+) (*OutputWriter, error) {
+	logLevel := c.String(logLevelFlagName)
+	logFile := c.String(LogFileFlag)
+	logDirectory := c.String(logDirectoryFlagName)
+
+	loggerConfig := CreateConfig(logLevel, disableTerminal, logDirectory, logFile)
+
+	return NewInHouse(loggerConfig)
+}
+
 // ParseLevelString returns the expected log levels based on the cmd flag
 func ParseLevelString(lvl string) ([]Level, error) {
 	switch strings.ToLower(lvl) {

metrics/metrics.go

@@ -12,6 +12,7 @@ import (
 
 	"golang.org/x/net/trace"
 
+	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
@@ -22,22 +23,35 @@ const (
 	startupTime     = time.Millisecond * 500
 )
 
-func ServeMetrics(l net.Listener, shutdownC <-chan struct{}, logger logger.Service) (err error) {
+func newMetricsHandler(connectionEvents <-chan connection.Event, log logger.Service) *http.ServeMux {
+	readyServer := NewReadyServer(connectionEvents, log)
+	mux := http.NewServeMux()
+	mux.Handle("/metrics", promhttp.Handler())
+	mux.HandleFunc("/healthcheck", func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintf(w, "OK\n")
+	})
+	mux.Handle("/ready", readyServer)
+	return mux
+}
+
+func ServeMetrics(
+	l net.Listener,
+	shutdownC <-chan struct{},
+	connectionEvents <-chan connection.Event,
+	logger logger.Service,
+) (err error) {
 	var wg sync.WaitGroup
 	// Metrics port is privileged, so no need for further access control
 	trace.AuthRequest = func(*http.Request) (bool, bool) { return true, true }
 	// TODO: parameterize ReadTimeout and WriteTimeout. The maximum time we can
 	// profile CPU usage depends on WriteTimeout
+	h := newMetricsHandler(connectionEvents, logger)
 	server := &http.Server{
 		ReadTimeout:  10 * time.Second,
 		WriteTimeout: 10 * time.Second,
+		Handler:      h,
 	}
 
-	http.Handle("/metrics", promhttp.Handler())
-	http.Handle("/healthcheck", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		fmt.Fprintf(w, "OK\n")
-	}))
-
 	wg.Add(1)
 	go func() {
 		defer wg.Done()

metrics/readiness.go

@@ -0,0 +1,80 @@
+package metrics
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"sync"
+
+	conn "github.com/cloudflare/cloudflared/connection"
+	"github.com/cloudflare/cloudflared/logger"
+)
+
+// ReadyServer serves HTTP 200 if the tunnel can serve traffic. Intended for k8s readiness checks.
+type ReadyServer struct {
+	sync.RWMutex
+	isConnected map[int]bool
+	log         logger.Service
+}
+
+// NewReadyServer initializes a ReadyServer and starts listening for dis/connection events.
+func NewReadyServer(connectionEvents <-chan conn.Event, log logger.Service) *ReadyServer {
+	rs := ReadyServer{
+		isConnected: make(map[int]bool, 0),
+		log:         log,
+	}
+	go func() {
+		for c := range connectionEvents {
+			switch c.EventType {
+			case conn.Connected:
+				rs.Lock()
+				rs.isConnected[int(c.Index)] = true
+				rs.Unlock()
+			case conn.Disconnected, conn.Reconnecting, conn.RegisteringTunnel:
+				rs.Lock()
+				rs.isConnected[int(c.Index)] = false
+				rs.Unlock()
+			case conn.SetURL:
+				continue
+			default:
+				rs.log.Errorf("Unknown connection event case %v", c)
+			}
+		}
+	}()
+	return &rs
+}
+
+type body struct {
+	Status           int `json:"status"`
+	ReadyConnections int `json:"readyConnections"`
+}
+
+// ServeHTTP responds with HTTP 200 if the tunnel is connected to the edge.
+func (rs *ReadyServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	statusCode, readyConnections := rs.makeResponse()
+	w.WriteHeader(statusCode)
+	body := body{
+		Status:           statusCode,
+		ReadyConnections: readyConnections,
+	}
+	msg, err := json.Marshal(body)
+	if err != nil {
+		fmt.Fprintf(w, `{"error": "%s"}`, err)
+	}
+	w.Write(msg)
+}
+
+// This is the bulk of the logic for ServeHTTP, broken into its own pure function
+// to make unit testing easy.
+func (rs *ReadyServer) makeResponse() (statusCode, readyConnections int) {
+	statusCode = http.StatusServiceUnavailable
+	rs.RLock()
+	defer rs.RUnlock()
+	for _, connected := range rs.isConnected {
+		if connected {
+			statusCode = http.StatusOK
+			readyConnections++
+		}
+	}
+	return statusCode, readyConnections
+}

metrics/readiness_test.go

@@ -0,0 +1,58 @@
+package metrics
+
+import (
+	"net/http"
+	"testing"
+)
+
+func TestReadyServer_makeResponse(t *testing.T) {
+	type fields struct {
+		isConnected map[int]bool
+	}
+	tests := []struct {
+		name                 string
+		fields               fields
+		wantOK               bool
+		wantReadyConnections int
+	}{
+		{
+			name: "One connection online => HTTP 200",
+			fields: fields{
+				isConnected: map[int]bool{
+					0: false,
+					1: false,
+					2: true,
+					3: false,
+				},
+			},
+			wantOK:               true,
+			wantReadyConnections: 1,
+		},
+		{
+			name: "No connections online => no HTTP 200",
+			fields: fields{
+				isConnected: map[int]bool{
+					0: false,
+					1: false,
+					2: false,
+					3: false,
+				},
+			},
+			wantReadyConnections: 0,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			rs := &ReadyServer{
+				isConnected: tt.fields.isConnected,
+			}
+			gotStatusCode, gotReadyConnections := rs.makeResponse()
+			if tt.wantOK && gotStatusCode != http.StatusOK {
+				t.Errorf("ReadyServer.makeResponse() gotStatusCode = %v, want ok = %v", gotStatusCode, tt.wantOK)
+			}
+			if gotReadyConnections != tt.wantReadyConnections {
+				t.Errorf("ReadyServer.makeResponse() gotReadyConnections = %v, want %v", gotReadyConnections, tt.wantReadyConnections)
+			}
+		})
+	}
+}

origin/backoffhandler.go

@@ -97,3 +97,11 @@ func (b BackoffHandler) GetBaseTime() time.Duration {
 func (b *BackoffHandler) Retries() int {
 	return int(b.retries)
 }
+
+func (b *BackoffHandler) ReachedMaxRetries() bool {
+	return b.retries == b.MaxRetries
+}
+
+func (b *BackoffHandler) resetNow() {
+	b.resetDeadline = time.Now()
+}

origin/metrics.go

@@ -1,540 +1,72 @@
 package origin
 
 import (
-	"sync"
-	"time"
-
-	"github.com/cloudflare/cloudflared/h2mux"
-
+	"github.com/cloudflare/cloudflared/connection"
 	"github.com/prometheus/client_golang/prometheus"
 )
 
-const (
-	metricsNamespace = "cloudflared"
-	tunnelSubsystem  = "tunnel"
-	muxerSubsystem   = "muxer"
-)
+// Metrics uses connection.MetricsNamespace(aka cloudflared) as namespace and connection.TunnelSubsystem
+// (tunnel) as subsystem to keep them consistent with the previous qualifier.
 
-type muxerMetrics struct {
-	rtt              *prometheus.GaugeVec
-	rttMin           *prometheus.GaugeVec
-	rttMax           *prometheus.GaugeVec
-	receiveWindowAve *prometheus.GaugeVec
-	sendWindowAve    *prometheus.GaugeVec
-	receiveWindowMin *prometheus.GaugeVec
-	receiveWindowMax *prometheus.GaugeVec
-	sendWindowMin    *prometheus.GaugeVec
-	sendWindowMax    *prometheus.GaugeVec
-	inBoundRateCurr  *prometheus.GaugeVec
-	inBoundRateMin   *prometheus.GaugeVec
-	inBoundRateMax   *prometheus.GaugeVec
-	outBoundRateCurr *prometheus.GaugeVec
-	outBoundRateMin  *prometheus.GaugeVec
-	outBoundRateMax  *prometheus.GaugeVec
-	compBytesBefore  *prometheus.GaugeVec
-	compBytesAfter   *prometheus.GaugeVec
-	compRateAve      *prometheus.GaugeVec
-}
-
-type TunnelMetrics struct {
-	haConnections     prometheus.Gauge
-	activeStreams     prometheus.Gauge
-	totalRequests     prometheus.Counter
-	requestsPerTunnel *prometheus.CounterVec
-	// concurrentRequestsLock is a mutex for concurrentRequests and maxConcurrentRequests
-	concurrentRequestsLock      sync.Mutex
-	concurrentRequestsPerTunnel *prometheus.GaugeVec
-	// concurrentRequests records count of concurrent requests for each tunnel
-	concurrentRequests             map[string]uint64
-	maxConcurrentRequestsPerTunnel *prometheus.GaugeVec
-	// concurrentRequests records max count of concurrent requests for each tunnel
-	maxConcurrentRequests map[string]uint64
-	timerRetries          prometheus.Gauge
-	responseByCode        *prometheus.CounterVec
-	responseCodePerTunnel *prometheus.CounterVec
-	serverLocations       *prometheus.GaugeVec
-	// locationLock is a mutex for oldServerLocations
-	locationLock sync.Mutex
-	// oldServerLocations stores the last server the tunnel was connected to
-	oldServerLocations map[string]string
-
-	regSuccess *prometheus.CounterVec
-	regFail    *prometheus.CounterVec
-	rpcFail    *prometheus.CounterVec
-
-	muxerMetrics        *muxerMetrics
-	tunnelsHA           tunnelsForHA
-	userHostnamesCounts *prometheus.CounterVec
-}
-
-func newMuxerMetrics() *muxerMetrics {
-	rtt := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "rtt",
-			Help:      "Round-trip time in millisecond",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(rtt)
-
-	rttMin := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "rtt_min",
-			Help:      "Shortest round-trip time in millisecond",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(rttMin)
-
-	rttMax := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "rtt_max",
-			Help:      "Longest round-trip time in millisecond",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(rttMax)
-
-	receiveWindowAve := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "receive_window_ave",
-			Help:      "Average receive window size in bytes",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(receiveWindowAve)
-
-	sendWindowAve := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "send_window_ave",
-			Help:      "Average send window size in bytes",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(sendWindowAve)
-
-	receiveWindowMin := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "receive_window_min",
-			Help:      "Smallest receive window size in bytes",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(receiveWindowMin)
-
-	receiveWindowMax := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "receive_window_max",
-			Help:      "Largest receive window size in bytes",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(receiveWindowMax)
-
-	sendWindowMin := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "send_window_min",
-			Help:      "Smallest send window size in bytes",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(sendWindowMin)
-
-	sendWindowMax := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "send_window_max",
-			Help:      "Largest send window size in bytes",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(sendWindowMax)
-
-	inBoundRateCurr := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "inbound_bytes_per_sec_curr",
-			Help:      "Current inbounding bytes per second, 0 if there is no incoming connection",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(inBoundRateCurr)
-
-	inBoundRateMin := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "inbound_bytes_per_sec_min",
-			Help:      "Minimum non-zero inbounding bytes per second",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(inBoundRateMin)
-
-	inBoundRateMax := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "inbound_bytes_per_sec_max",
-			Help:      "Maximum inbounding bytes per second",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(inBoundRateMax)
-
-	outBoundRateCurr := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "outbound_bytes_per_sec_curr",
-			Help:      "Current outbounding bytes per second, 0 if there is no outgoing traffic",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(outBoundRateCurr)
-
-	outBoundRateMin := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "outbound_bytes_per_sec_min",
-			Help:      "Minimum non-zero outbounding bytes per second",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(outBoundRateMin)
-
-	outBoundRateMax := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "outbound_bytes_per_sec_max",
-			Help:      "Maximum outbounding bytes per second",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(outBoundRateMax)
-
-	compBytesBefore := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "comp_bytes_before",
-			Help:      "Bytes sent via cross-stream compression, pre compression",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(compBytesBefore)
-
-	compBytesAfter := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "comp_bytes_after",
-			Help:      "Bytes sent via cross-stream compression, post compression",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(compBytesAfter)
-
-	compRateAve := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "comp_rate_ave",
-			Help:      "Average outbound cross-stream compression ratio",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(compRateAve)
-
-	return &muxerMetrics{
-		rtt:              rtt,
-		rttMin:           rttMin,
-		rttMax:           rttMax,
-		receiveWindowAve: receiveWindowAve,
-		sendWindowAve:    sendWindowAve,
-		receiveWindowMin: receiveWindowMin,
-		receiveWindowMax: receiveWindowMax,
-		sendWindowMin:    sendWindowMin,
-		sendWindowMax:    sendWindowMax,
-		inBoundRateCurr:  inBoundRateCurr,
-		inBoundRateMin:   inBoundRateMin,
-		inBoundRateMax:   inBoundRateMax,
-		outBoundRateCurr: outBoundRateCurr,
-		outBoundRateMin:  outBoundRateMin,
-		outBoundRateMax:  outBoundRateMax,
-		compBytesBefore:  compBytesBefore,
-		compBytesAfter:   compBytesAfter,
-		compRateAve:      compRateAve,
-	}
-}
-
-func (m *muxerMetrics) update(connectionID string, metrics *h2mux.MuxerMetrics) {
-	m.rtt.WithLabelValues(connectionID).Set(convertRTTMilliSec(metrics.RTT))
-	m.rttMin.WithLabelValues(connectionID).Set(convertRTTMilliSec(metrics.RTTMin))
-	m.rttMax.WithLabelValues(connectionID).Set(convertRTTMilliSec(metrics.RTTMax))
-	m.receiveWindowAve.WithLabelValues(connectionID).Set(metrics.ReceiveWindowAve)
-	m.sendWindowAve.WithLabelValues(connectionID).Set(metrics.SendWindowAve)
-	m.receiveWindowMin.WithLabelValues(connectionID).Set(float64(metrics.ReceiveWindowMin))
-	m.receiveWindowMax.WithLabelValues(connectionID).Set(float64(metrics.ReceiveWindowMax))
-	m.sendWindowMin.WithLabelValues(connectionID).Set(float64(metrics.SendWindowMin))
-	m.sendWindowMax.WithLabelValues(connectionID).Set(float64(metrics.SendWindowMax))
-	m.inBoundRateCurr.WithLabelValues(connectionID).Set(float64(metrics.InBoundRateCurr))
-	m.inBoundRateMin.WithLabelValues(connectionID).Set(float64(metrics.InBoundRateMin))
-	m.inBoundRateMax.WithLabelValues(connectionID).Set(float64(metrics.InBoundRateMax))
-	m.outBoundRateCurr.WithLabelValues(connectionID).Set(float64(metrics.OutBoundRateCurr))
-	m.outBoundRateMin.WithLabelValues(connectionID).Set(float64(metrics.OutBoundRateMin))
-	m.outBoundRateMax.WithLabelValues(connectionID).Set(float64(metrics.OutBoundRateMax))
-	m.compBytesBefore.WithLabelValues(connectionID).Set(float64(metrics.CompBytesBefore.Value()))
-	m.compBytesAfter.WithLabelValues(connectionID).Set(float64(metrics.CompBytesAfter.Value()))
-	m.compRateAve.WithLabelValues(connectionID).Set(float64(metrics.CompRateAve()))
-}
-
-func convertRTTMilliSec(t time.Duration) float64 {
-	return float64(t / time.Millisecond)
-}
-
-// Metrics that can be collected without asking the edge
-func NewTunnelMetrics() *TunnelMetrics {
-	haConnections := prometheus.NewGauge(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: tunnelSubsystem,
-			Name:      "ha_connections",
-			Help:      "Number of active ha connections",
-		})
-	prometheus.MustRegister(haConnections)
-
-	activeStreams := h2mux.NewActiveStreamsMetrics(metricsNamespace, tunnelSubsystem)
-
-	totalRequests := prometheus.NewCounter(
+var (
+	totalRequests = prometheus.NewCounter(
 		prometheus.CounterOpts{
-			Namespace: metricsNamespace,
-			Subsystem: tunnelSubsystem,
+			Namespace: connection.MetricsNamespace,
+			Subsystem: connection.TunnelSubsystem,
 			Name:      "total_requests",
 			Help:      "Amount of requests proxied through all the tunnels",
-		})
-	prometheus.MustRegister(totalRequests)
-
-	requestsPerTunnel := prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Namespace: metricsNamespace,
-			Subsystem: tunnelSubsystem,
-			Name:      "requests_per_tunnel",
-			Help:      "Amount of requests proxied through each tunnel",
 		},
-		[]string{"connection_id"},
 	)
-	prometheus.MustRegister(requestsPerTunnel)
-
-	concurrentRequestsPerTunnel := prometheus.NewGaugeVec(
+	concurrentRequests = prometheus.NewGauge(
 		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: tunnelSubsystem,
+			Namespace: connection.MetricsNamespace,
+			Subsystem: connection.TunnelSubsystem,
 			Name:      "concurrent_requests_per_tunnel",
 			Help:      "Concurrent requests proxied through each tunnel",
 		},
-		[]string{"connection_id"},
 	)
-	prometheus.MustRegister(concurrentRequestsPerTunnel)
-
-	maxConcurrentRequestsPerTunnel := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: tunnelSubsystem,
-			Name:      "max_concurrent_requests_per_tunnel",
-			Help:      "Largest number of concurrent requests proxied through each tunnel so far",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(maxConcurrentRequestsPerTunnel)
-
-	timerRetries := prometheus.NewGauge(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: tunnelSubsystem,
-			Name:      "timer_retries",
-			Help:      "Unacknowledged heart beats count",
-		})
-	prometheus.MustRegister(timerRetries)
-
-	responseByCode := prometheus.NewCounterVec(
+	responseByCode = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
-			Namespace: metricsNamespace,
-			Subsystem: tunnelSubsystem,
+			Namespace: connection.MetricsNamespace,
+			Subsystem: connection.TunnelSubsystem,
 			Name:      "response_by_code",
 			Help:      "Count of responses by HTTP status code",
 		},
 		[]string{"status_code"},
 	)
-	prometheus.MustRegister(responseByCode)
-
-	responseCodePerTunnel := prometheus.NewCounterVec(
+	requestErrors = prometheus.NewCounter(
 		prometheus.CounterOpts{
-			Namespace: metricsNamespace,
-			Subsystem: tunnelSubsystem,
-			Name:      "response_code_per_tunnel",
-			Help:      "Count of responses by HTTP status code fore each tunnel",
+			Namespace: connection.MetricsNamespace,
+			Subsystem: connection.TunnelSubsystem,
+			Name:      "request_errors",
+			Help:      "Count of error proxying to origin",
 		},
-		[]string{"connection_id", "status_code"},
 	)
-	prometheus.MustRegister(responseCodePerTunnel)
-
-	serverLocations := prometheus.NewGaugeVec(
+	haConnections = prometheus.NewGauge(
 		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: tunnelSubsystem,
-			Name:      "server_locations",
-			Help:      "Where each tunnel is connected to. 1 means current location, 0 means previous locations.",
+			Namespace: connection.MetricsNamespace,
+			Subsystem: connection.TunnelSubsystem,
+			Name:      "ha_connections",
+			Help:      "Number of active ha connections",
 		},
-		[]string{"connection_id", "location"},
 	)
-	prometheus.MustRegister(serverLocations)
+)
 
-	rpcFail := prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Namespace: metricsNamespace,
-			Subsystem: tunnelSubsystem,
-			Name:      "tunnel_rpc_fail",
-			Help:      "Count of RPC connection errors by type",
-		},
-		[]string{"error", "rpcName"},
+func init() {
+	prometheus.MustRegister(
+		totalRequests,
+		concurrentRequests,
+		responseByCode,
+		requestErrors,
+		haConnections,
 	)
-	prometheus.MustRegister(rpcFail)
-
-	registerFail := prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Namespace: metricsNamespace,
-			Subsystem: tunnelSubsystem,
-			Name:      "tunnel_register_fail",
-			Help:      "Count of tunnel registration errors by type",
-		},
-		[]string{"error", "rpcName"},
-	)
-	prometheus.MustRegister(registerFail)
-
-	userHostnamesCounts := prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Namespace: metricsNamespace,
-			Subsystem: tunnelSubsystem,
-			Name:      "user_hostnames_counts",
-			Help:      "Which user hostnames cloudflared is serving",
-		},
-		[]string{"userHostname"},
-	)
-	prometheus.MustRegister(userHostnamesCounts)
-
-	registerSuccess := prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Namespace: metricsNamespace,
-			Subsystem: tunnelSubsystem,
-			Name:      "tunnel_register_success",
-			Help:      "Count of successful tunnel registrations",
-		},
-		[]string{"rpcName"},
-	)
-	prometheus.MustRegister(registerSuccess)
-
-	return &TunnelMetrics{
-		haConnections:                  haConnections,
-		activeStreams:                  activeStreams,
-		totalRequests:                  totalRequests,
-		requestsPerTunnel:              requestsPerTunnel,
-		concurrentRequestsPerTunnel:    concurrentRequestsPerTunnel,
-		concurrentRequests:             make(map[string]uint64),
-		maxConcurrentRequestsPerTunnel: maxConcurrentRequestsPerTunnel,
-		maxConcurrentRequests:          make(map[string]uint64),
-		timerRetries:                   timerRetries,
-		responseByCode:                 responseByCode,
-		responseCodePerTunnel:          responseCodePerTunnel,
-		serverLocations:                serverLocations,
-		oldServerLocations:             make(map[string]string),
-		muxerMetrics:                   newMuxerMetrics(),
-		tunnelsHA:                      NewTunnelsForHA(),
-		regSuccess:                     registerSuccess,
-		regFail:                        registerFail,
-		rpcFail:                        rpcFail,
-		userHostnamesCounts:            userHostnamesCounts,
-	}
 }
 
-func (t *TunnelMetrics) incrementHaConnections() {
-	t.haConnections.Inc()
+func incrementRequests() {
+	totalRequests.Inc()
+	concurrentRequests.Inc()
 }
 
-func (t *TunnelMetrics) decrementHaConnections() {
-	t.haConnections.Dec()
-}
-
-func (t *TunnelMetrics) updateMuxerMetrics(connectionID string, metrics *h2mux.MuxerMetrics) {
-	t.muxerMetrics.update(connectionID, metrics)
-}
-
-func (t *TunnelMetrics) incrementRequests(connectionID string) {
-	t.concurrentRequestsLock.Lock()
-	var concurrentRequests uint64
-	var ok bool
-	if concurrentRequests, ok = t.concurrentRequests[connectionID]; ok {
-		t.concurrentRequests[connectionID]++
-		concurrentRequests++
-	} else {
-		t.concurrentRequests[connectionID] = 1
-		concurrentRequests = 1
-	}
-	if maxConcurrentRequests, ok := t.maxConcurrentRequests[connectionID]; (ok && maxConcurrentRequests < concurrentRequests) || !ok {
-		t.maxConcurrentRequests[connectionID] = concurrentRequests
-		t.maxConcurrentRequestsPerTunnel.WithLabelValues(connectionID).Set(float64(concurrentRequests))
-	}
-	t.concurrentRequestsLock.Unlock()
-
-	t.totalRequests.Inc()
-	t.requestsPerTunnel.WithLabelValues(connectionID).Inc()
-	t.concurrentRequestsPerTunnel.WithLabelValues(connectionID).Inc()
-}
-
-func (t *TunnelMetrics) decrementConcurrentRequests(connectionID string) {
-	t.concurrentRequestsLock.Lock()
-	if _, ok := t.concurrentRequests[connectionID]; ok {
-		t.concurrentRequests[connectionID]--
-	}
-	t.concurrentRequestsLock.Unlock()
-
-	t.concurrentRequestsPerTunnel.WithLabelValues(connectionID).Dec()
-}
-
-func (t *TunnelMetrics) incrementResponses(connectionID, code string) {
-	t.responseByCode.WithLabelValues(code).Inc()
-	t.responseCodePerTunnel.WithLabelValues(connectionID, code).Inc()
-
-}
-
-func (t *TunnelMetrics) registerServerLocation(connectionID, loc string) {
-	t.locationLock.Lock()
-	defer t.locationLock.Unlock()
-	if oldLoc, ok := t.oldServerLocations[connectionID]; ok && oldLoc == loc {
-		return
-	} else if ok {
-		t.serverLocations.WithLabelValues(connectionID, oldLoc).Dec()
-	}
-	t.serverLocations.WithLabelValues(connectionID, loc).Inc()
-	t.oldServerLocations[connectionID] = loc
+func decrementConcurrentRequests() {
+	concurrentRequests.Dec()
 }

origin/metrics_test.go

@@ -1,121 +0,0 @@
-package origin
-
-import (
-	"strconv"
-	"sync"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-// can only be called once
-var m = NewTunnelMetrics()
-
-func TestConcurrentRequestsSingleTunnel(t *testing.T) {
-	routines := 20
-	var wg sync.WaitGroup
-	wg.Add(routines)
-	for i := 0; i < routines; i++ {
-		go func() {
-			m.incrementRequests("0")
-			wg.Done()
-		}()
-	}
-	wg.Wait()
-	assert.Len(t, m.concurrentRequests, 1)
-	assert.Equal(t, uint64(routines), m.concurrentRequests["0"])
-	assert.Len(t, m.maxConcurrentRequests, 1)
-	assert.Equal(t, uint64(routines), m.maxConcurrentRequests["0"])
-
-	wg.Add(routines / 2)
-	for i := 0; i < routines/2; i++ {
-		go func() {
-			m.decrementConcurrentRequests("0")
-			wg.Done()
-		}()
-	}
-	wg.Wait()
-	assert.Equal(t, uint64(routines-routines/2), m.concurrentRequests["0"])
-	assert.Equal(t, uint64(routines), m.maxConcurrentRequests["0"])
-}
-
-func TestConcurrentRequestsMultiTunnel(t *testing.T) {
-	m.concurrentRequests = make(map[string]uint64)
-	m.maxConcurrentRequests = make(map[string]uint64)
-	tunnels := 20
-	var wg sync.WaitGroup
-	wg.Add(tunnels)
-	for i := 0; i < tunnels; i++ {
-		go func(i int) {
-			// if we have j < i, then tunnel 0 won't have a chance to call incrementRequests
-			for j := 0; j < i+1; j++ {
-				id := strconv.Itoa(i)
-				m.incrementRequests(id)
-			}
-			wg.Done()
-		}(i)
-	}
-	wg.Wait()
-
-	assert.Len(t, m.concurrentRequests, tunnels)
-	assert.Len(t, m.maxConcurrentRequests, tunnels)
-	for i := 0; i < tunnels; i++ {
-		id := strconv.Itoa(i)
-		assert.Equal(t, uint64(i+1), m.concurrentRequests[id])
-		assert.Equal(t, uint64(i+1), m.maxConcurrentRequests[id])
-	}
-
-	wg.Add(tunnels)
-	for i := 0; i < tunnels; i++ {
-		go func(i int) {
-			for j := 0; j < i+1; j++ {
-				id := strconv.Itoa(i)
-				m.decrementConcurrentRequests(id)
-			}
-			wg.Done()
-		}(i)
-	}
-	wg.Wait()
-
-	assert.Len(t, m.concurrentRequests, tunnels)
-	assert.Len(t, m.maxConcurrentRequests, tunnels)
-	for i := 0; i < tunnels; i++ {
-		id := strconv.Itoa(i)
-		assert.Equal(t, uint64(0), m.concurrentRequests[id])
-		assert.Equal(t, uint64(i+1), m.maxConcurrentRequests[id])
-	}
-
-}
-
-func TestRegisterServerLocation(t *testing.T) {
-	tunnels := 20
-	var wg sync.WaitGroup
-	wg.Add(tunnels)
-	for i := 0; i < tunnels; i++ {
-		go func(i int) {
-			id := strconv.Itoa(i)
-			m.registerServerLocation(id, "LHR")
-			wg.Done()
-		}(i)
-	}
-	wg.Wait()
-	for i := 0; i < tunnels; i++ {
-		id := strconv.Itoa(i)
-		assert.Equal(t, "LHR", m.oldServerLocations[id])
-	}
-
-	wg.Add(tunnels)
-	for i := 0; i < tunnels; i++ {
-		go func(i int) {
-			id := strconv.Itoa(i)
-			m.registerServerLocation(id, "AUS")
-			wg.Done()
-		}(i)
-	}
-	wg.Wait()
-	for i := 0; i < tunnels; i++ {
-		id := strconv.Itoa(i)
-		assert.Equal(t, "AUS", m.oldServerLocations[id])
-	}
-
-}

origin/proxy.go

@@ -0,0 +1,224 @@
+package origin
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"strconv"
+	"strings"
+
+	"github.com/cloudflare/cloudflared/buffer"
+	"github.com/cloudflare/cloudflared/connection"
+	"github.com/cloudflare/cloudflared/ingress"
+	"github.com/cloudflare/cloudflared/logger"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/cloudflare/cloudflared/websocket"
+	"github.com/pkg/errors"
+)
+
+const (
+	TagHeaderNamePrefix = "Cf-Warp-Tag-"
+)
+
+type client struct {
+	ingressRules ingress.Ingress
+	tags         []tunnelpogs.Tag
+	logger       logger.Service
+	bufferPool   *buffer.Pool
+}
+
+func NewClient(ingressRules ingress.Ingress, tags []tunnelpogs.Tag, logger logger.Service) connection.OriginClient {
+	return &client{
+		ingressRules: ingressRules,
+		tags:         tags,
+		logger:       logger,
+		bufferPool:   buffer.NewPool(512 * 1024),
+	}
+}
+
+func (c *client) Proxy(w connection.ResponseWriter, req *http.Request, isWebsocket bool) error {
+	incrementRequests()
+	defer decrementConcurrentRequests()
+
+	cfRay := findCfRayHeader(req)
+	lbProbe := isLBProbeRequest(req)
+
+	c.appendTagHeaders(req)
+	rule, ruleNum := c.ingressRules.FindMatchingRule(req.Host, req.URL.Path)
+	c.logRequest(req, cfRay, lbProbe, ruleNum)
+
+	var (
+		resp *http.Response
+		err  error
+	)
+	if isWebsocket {
+		resp, err = c.proxyWebsocket(w, req, rule)
+	} else {
+		resp, err = c.proxyHTTP(w, req, rule)
+	}
+	if err != nil {
+		c.logRequestError(err, cfRay, ruleNum)
+		w.WriteErrorResponse()
+		return err
+	}
+	c.logOriginResponse(resp, cfRay, lbProbe, ruleNum)
+	return nil
+}
+
+func (c *client) proxyHTTP(w connection.ResponseWriter, req *http.Request, rule *ingress.Rule) (*http.Response, error) {
+	// Support for WSGI Servers by switching transfer encoding from chunked to gzip/deflate
+	if rule.Config.DisableChunkedEncoding {
+		req.TransferEncoding = []string{"gzip", "deflate"}
+		cLength, err := strconv.Atoi(req.Header.Get("Content-Length"))
+		if err == nil {
+			req.ContentLength = int64(cLength)
+		}
+	}
+
+	// Request origin to keep connection alive to improve performance
+	req.Header.Set("Connection", "keep-alive")
+
+	if hostHeader := rule.Config.HTTPHostHeader; hostHeader != "" {
+		req.Header.Set("Host", hostHeader)
+		req.Host = hostHeader
+	}
+
+	resp, err := rule.Service.RoundTrip(req)
+	if err != nil {
+		return nil, errors.Wrap(err, "Error proxying request to origin")
+	}
+	defer resp.Body.Close()
+
+	err = w.WriteRespHeaders(resp)
+	if err != nil {
+		return nil, errors.Wrap(err, "Error writing response header")
+	}
+	if connection.IsServerSentEvent(resp.Header) {
+		c.logger.Debug("Detected Server-Side Events from Origin")
+		c.writeEventStream(w, resp.Body)
+	} else {
+		// Use CopyBuffer, because Copy only allocates a 32KiB buffer, and cross-stream
+		// compression generates dictionary on first write
+		buf := c.bufferPool.Get()
+		defer c.bufferPool.Put(buf)
+		io.CopyBuffer(w, resp.Body, buf)
+	}
+	return resp, nil
+}
+
+func (c *client) proxyWebsocket(w connection.ResponseWriter, req *http.Request, rule *ingress.Rule) (*http.Response, error) {
+	if hostHeader := rule.Config.HTTPHostHeader; hostHeader != "" {
+		req.Header.Set("Host", hostHeader)
+		req.Host = hostHeader
+	}
+
+	dialler, ok := rule.Service.(websocket.Dialler)
+	if !ok {
+		return nil, fmt.Errorf("Websockets aren't supported by the origin service '%s'", rule.Service)
+	}
+	conn, resp, err := websocket.ClientConnect(req, dialler)
+	if err != nil {
+		return nil, err
+	}
+
+	serveCtx, cancel := context.WithCancel(req.Context())
+	connClosedChan := make(chan struct{})
+	go func() {
+		// serveCtx is done if req is cancelled, or streamWebsocket returns
+		<-serveCtx.Done()
+		conn.Close()
+		close(connClosedChan)
+	}()
+
+	// Copy to/from stream to the undelying connection. Use the underlying
+	// connection because cloudflared doesn't operate on the message themselves
+	err = c.streamWebsocket(w, conn.UnderlyingConn(), resp)
+	cancel()
+
+	// We need to make sure conn is closed before returning, otherwise we might write to conn after Proxy returns
+	<-connClosedChan
+	return resp, err
+}
+
+func (c *client) streamWebsocket(w connection.ResponseWriter, conn net.Conn, resp *http.Response) error {
+	err := w.WriteRespHeaders(resp)
+	if err != nil {
+		return errors.Wrap(err, "Error writing websocket response header")
+	}
+	websocket.Stream(conn, w)
+	return nil
+}
+
+func (c *client) writeEventStream(w connection.ResponseWriter, respBody io.ReadCloser) {
+	reader := bufio.NewReader(respBody)
+	for {
+		line, err := reader.ReadBytes('\n')
+		if err != nil {
+			break
+		}
+		w.Write(line)
+	}
+}
+
+func (c *client) appendTagHeaders(r *http.Request) {
+	for _, tag := range c.tags {
+		r.Header.Add(TagHeaderNamePrefix+tag.Name, tag.Value)
+	}
+}
+
+func (c *client) logRequest(r *http.Request, cfRay string, lbProbe bool, ruleNum int) {
+	if cfRay != "" {
+		c.logger.Debugf("CF-RAY: %s %s %s %s", cfRay, r.Method, r.URL, r.Proto)
+	} else if lbProbe {
+		c.logger.Debugf("CF-RAY: %s Load Balancer health check %s %s %s", cfRay, r.Method, r.URL, r.Proto)
+	} else {
+		c.logger.Debugf("All requests should have a CF-RAY header. Please open a support ticket with Cloudflare. %s %s %s ", r.Method, r.URL, r.Proto)
+	}
+	c.logger.Debugf("CF-RAY: %s Request Headers %+v", cfRay, r.Header)
+	c.logger.Debugf("CF-RAY: %s Serving with ingress rule %d", cfRay, ruleNum)
+
+	if contentLen := r.ContentLength; contentLen == -1 {
+		c.logger.Debugf("CF-RAY: %s Request Content length unknown", cfRay)
+	} else {
+		c.logger.Debugf("CF-RAY: %s Request content length %d", cfRay, contentLen)
+	}
+}
+
+func (c *client) logOriginResponse(r *http.Response, cfRay string, lbProbe bool, ruleNum int) {
+	responseByCode.WithLabelValues(strconv.Itoa(r.StatusCode)).Inc()
+	if cfRay != "" {
+		c.logger.Debugf("CF-RAY: %s Status: %s served by ingress %d", cfRay, r.Status, ruleNum)
+	} else if lbProbe {
+		c.logger.Debugf("Response to Load Balancer health check %s", r.Status)
+	} else {
+		c.logger.Debugf("Status: %s served by ingress %d", r.Status, ruleNum)
+	}
+	c.logger.Debugf("CF-RAY: %s Response Headers %+v", cfRay, r.Header)
+
+	if contentLen := r.ContentLength; contentLen == -1 {
+		c.logger.Debugf("CF-RAY: %s Response content length unknown", cfRay)
+	} else {
+		c.logger.Debugf("CF-RAY: %s Response content length %d", cfRay, contentLen)
+	}
+}
+
+func (c *client) logRequestError(err error, cfRay string, ruleNum int) {
+	requestErrors.Inc()
+	if cfRay != "" {
+		c.logger.Errorf("CF-RAY: %s Proxying to ingress %d error: %v", cfRay, ruleNum, err)
+	} else {
+		c.logger.Errorf("Proxying to ingress %d error: %v", ruleNum, err)
+	}
+
+}
+
+func findCfRayHeader(req *http.Request) string {
+	return req.Header.Get("Cf-Ray")
+}
+
+func isLBProbeRequest(req *http.Request) bool {
+	return strings.HasPrefix(req.UserAgent(), lbProbeUserAgentPrefix)
+}

origin/proxy_test.go

@@ -0,0 +1,352 @@
+package origin
+
+import (
+	"bytes"
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/connection"
+	"github.com/cloudflare/cloudflared/hello"
+	"github.com/cloudflare/cloudflared/ingress"
+	"github.com/cloudflare/cloudflared/logger"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/urfave/cli/v2"
+
+	"github.com/gobwas/ws/wsutil"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+var (
+	testTags = []tunnelpogs.Tag(nil)
+)
+
+type mockHTTPRespWriter struct {
+	*httptest.ResponseRecorder
+}
+
+func newMockHTTPRespWriter() *mockHTTPRespWriter {
+	return &mockHTTPRespWriter{
+		httptest.NewRecorder(),
+	}
+}
+
+func (w *mockHTTPRespWriter) WriteRespHeaders(resp *http.Response) error {
+	w.WriteHeader(resp.StatusCode)
+	for header, val := range resp.Header {
+		w.Header()[header] = val
+	}
+	return nil
+}
+
+func (w *mockHTTPRespWriter) WriteErrorResponse() {
+	w.WriteHeader(http.StatusBadGateway)
+	w.Write([]byte("http response error"))
+}
+
+func (w *mockHTTPRespWriter) Read(data []byte) (int, error) {
+	return 0, fmt.Errorf("mockHTTPRespWriter doesn't implement io.Reader")
+}
+
+type mockWSRespWriter struct {
+	*mockHTTPRespWriter
+	writeNotification chan []byte
+	reader            io.Reader
+}
+
+func newMockWSRespWriter(reader io.Reader) *mockWSRespWriter {
+	return &mockWSRespWriter{
+		newMockHTTPRespWriter(),
+		make(chan []byte),
+		reader,
+	}
+}
+
+func (w *mockWSRespWriter) Write(data []byte) (int, error) {
+	w.writeNotification <- data
+	return len(data), nil
+}
+
+func (w *mockWSRespWriter) respBody() io.ReadWriter {
+	data := <-w.writeNotification
+	return bytes.NewBuffer(data)
+}
+
+func (w *mockWSRespWriter) Read(data []byte) (int, error) {
+	return w.reader.Read(data)
+}
+
+type mockSSERespWriter struct {
+	*mockHTTPRespWriter
+	writeNotification chan []byte
+}
+
+func newMockSSERespWriter() *mockSSERespWriter {
+	return &mockSSERespWriter{
+		newMockHTTPRespWriter(),
+		make(chan []byte),
+	}
+}
+
+func (w *mockSSERespWriter) Write(data []byte) (int, error) {
+	w.writeNotification <- data
+	return len(data), nil
+}
+
+func (w *mockSSERespWriter) ReadBytes() []byte {
+	return <-w.writeNotification
+}
+
+func TestProxySingleOrigin(t *testing.T) {
+	logger, err := logger.New()
+	require.NoError(t, err)
+
+	ctx, cancel := context.WithCancel(context.Background())
+
+	flagSet := flag.NewFlagSet(t.Name(), flag.PanicOnError)
+	flagSet.Bool("hello-world", true, "")
+
+	cliCtx := cli.NewContext(cli.NewApp(), flagSet, nil)
+	err = cliCtx.Set("hello-world", "true")
+	require.NoError(t, err)
+
+	allowURLFromArgs := false
+	ingressRule, err := ingress.NewSingleOrigin(cliCtx, allowURLFromArgs, logger)
+	require.NoError(t, err)
+
+	var wg sync.WaitGroup
+	errC := make(chan error)
+	ingressRule.StartOrigins(&wg, logger, ctx.Done(), errC)
+
+	client := NewClient(ingressRule, testTags, logger)
+	t.Run("testProxyHTTP", testProxyHTTP(t, client))
+	t.Run("testProxyWebsocket", testProxyWebsocket(t, client))
+	t.Run("testProxySSE", testProxySSE(t, client))
+	cancel()
+	wg.Wait()
+}
+
+func testProxyHTTP(t *testing.T, client connection.OriginClient) func(t *testing.T) {
+	return func(t *testing.T) {
+		respWriter := newMockHTTPRespWriter()
+		req, err := http.NewRequest(http.MethodGet, "http://localhost:8080", nil)
+		require.NoError(t, err)
+
+		err = client.Proxy(respWriter, req, false)
+		require.NoError(t, err)
+
+		assert.Equal(t, http.StatusOK, respWriter.Code)
+	}
+}
+
+func testProxyWebsocket(t *testing.T, client connection.OriginClient) func(t *testing.T) {
+	return func(t *testing.T) {
+		// WSRoute is a websocket echo handler
+		ctx, cancel := context.WithCancel(context.Background())
+		req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("http://localhost:8080%s", hello.WSRoute), nil)
+
+		readPipe, writePipe := io.Pipe()
+		respWriter := newMockWSRespWriter(readPipe)
+
+		var wg sync.WaitGroup
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			err = client.Proxy(respWriter, req, true)
+			require.NoError(t, err)
+
+			require.Equal(t, http.StatusSwitchingProtocols, respWriter.Code)
+		}()
+
+		msg := []byte("test websocket")
+		err = wsutil.WriteClientText(writePipe, msg)
+		require.NoError(t, err)
+
+		// ReadServerText reads next data message from rw, considering that caller represents client side.
+		returnedMsg, err := wsutil.ReadServerText(respWriter.respBody())
+		require.NoError(t, err)
+		require.Equal(t, msg, returnedMsg)
+
+		err = wsutil.WriteClientBinary(writePipe, msg)
+		require.NoError(t, err)
+
+		returnedMsg, err = wsutil.ReadServerBinary(respWriter.respBody())
+		require.NoError(t, err)
+		require.Equal(t, msg, returnedMsg)
+
+		cancel()
+		wg.Wait()
+	}
+}
+
+func testProxySSE(t *testing.T, client connection.OriginClient) func(t *testing.T) {
+	return func(t *testing.T) {
+		var (
+			pushCount = 50
+			pushFreq  = time.Duration(time.Millisecond * 10)
+		)
+		respWriter := newMockSSERespWriter()
+		ctx, cancel := context.WithCancel(context.Background())
+		req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("http://localhost:8080%s?freq=%s", hello.SSERoute, pushFreq), nil)
+		require.NoError(t, err)
+
+		var wg sync.WaitGroup
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			err = client.Proxy(respWriter, req, false)
+			require.NoError(t, err)
+
+			require.Equal(t, http.StatusOK, respWriter.Code)
+		}()
+
+		for i := 0; i < pushCount; i++ {
+			line := respWriter.ReadBytes()
+			expect := fmt.Sprintf("%d\n", i)
+			require.Equal(t, []byte(expect), line, fmt.Sprintf("Expect to read %v, got %v", expect, line))
+
+			line = respWriter.ReadBytes()
+			require.Equal(t, []byte("\n"), line, fmt.Sprintf("Expect to read '\n', got %v", line))
+		}
+
+		cancel()
+		wg.Wait()
+	}
+}
+
+func TestProxyMultipleOrigins(t *testing.T) {
+	api := httptest.NewServer(mockAPI{})
+	defer api.Close()
+
+	unvalidatedIngress := []config.UnvalidatedIngressRule{
+		{
+			Hostname: "api.example.com",
+			Service:  api.URL,
+		},
+		{
+			Hostname: "hello.example.com",
+			Service:  "hello-world",
+		},
+		{
+			Hostname: "health.example.com",
+			Path:     "/health",
+			Service:  "http_status:200",
+		},
+		{
+			Hostname: "*",
+			Service:  "http_status:404",
+		},
+	}
+
+	ingress, err := ingress.ParseIngress(&config.Configuration{
+		TunnelID: t.Name(),
+		Ingress:  unvalidatedIngress,
+	})
+	require.NoError(t, err)
+
+	logger, err := logger.New()
+	require.NoError(t, err)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	errC := make(chan error)
+	var wg sync.WaitGroup
+	ingress.StartOrigins(&wg, logger, ctx.Done(), errC)
+
+	client := NewClient(ingress, testTags, logger)
+
+	tests := []struct {
+		url            string
+		expectedStatus int
+		expectedBody   []byte
+	}{
+		{
+			url:            "http://api.example.com",
+			expectedStatus: http.StatusCreated,
+			expectedBody:   []byte("Created"),
+		},
+		{
+			url:            fmt.Sprintf("http://hello.example.com%s", hello.HealthRoute),
+			expectedStatus: http.StatusOK,
+			expectedBody:   []byte("ok"),
+		},
+		{
+			url:            "http://health.example.com/health",
+			expectedStatus: http.StatusOK,
+		},
+		{
+			url:            "http://health.example.com/",
+			expectedStatus: http.StatusNotFound,
+		},
+		{
+			url:            "http://not-found.example.com",
+			expectedStatus: http.StatusNotFound,
+		},
+	}
+
+	for _, test := range tests {
+		respWriter := newMockHTTPRespWriter()
+		req, err := http.NewRequest(http.MethodGet, test.url, nil)
+		require.NoError(t, err)
+
+		err = client.Proxy(respWriter, req, false)
+		require.NoError(t, err)
+
+		assert.Equal(t, test.expectedStatus, respWriter.Code)
+		if test.expectedBody != nil {
+			assert.Equal(t, test.expectedBody, respWriter.Body.Bytes())
+		} else {
+			assert.Equal(t, 0, respWriter.Body.Len())
+		}
+	}
+	cancel()
+	wg.Wait()
+}
+
+type mockAPI struct{}
+
+func (ma mockAPI) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	w.WriteHeader(http.StatusCreated)
+	w.Write([]byte("Created"))
+}
+
+type errorOriginTransport struct{}
+
+func (errorOriginTransport) RoundTrip(*http.Request) (*http.Response, error) {
+	return nil, fmt.Errorf("Proxy error")
+}
+
+func TestProxyError(t *testing.T) {
+	ingress := ingress.Ingress{
+		Rules: []ingress.Rule{
+			{
+				Hostname: "*",
+				Path:     nil,
+				Service: ingress.MockOriginService{
+					Transport: errorOriginTransport{},
+				},
+			},
+		},
+	}
+
+	logger, err := logger.New()
+	require.NoError(t, err)
+
+	client := NewClient(ingress, testTags, logger)
+
+	respWriter := newMockHTTPRespWriter()
+	req, err := http.NewRequest(http.MethodGet, "http://127.0.0.1", nil)
+	assert.NoError(t, err)
+
+	err = client.Proxy(respWriter, req, false)
+	assert.Error(t, err)
+	assert.Equal(t, http.StatusBadGateway, respWriter.Code)
+	assert.Equal(t, "http response error", respWriter.Body.String())
+}

origin/reconnect.go

@@ -7,11 +7,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/cloudflare/cloudflared/h2mux"
-	"github.com/cloudflare/cloudflared/logger"
-	"github.com/cloudflare/cloudflared/tunnelrpc"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
-	"github.com/google/uuid"
 	"github.com/prometheus/client_golang/prometheus"
 )
 
@@ -138,52 +134,3 @@ func (cm *reconnectCredentialManager) RefreshAuth(
 		return nil, err
 	}
 }
-
-func ReconnectTunnel(
-	ctx context.Context,
-	muxer *h2mux.Muxer,
-	config *TunnelConfig,
-	logger logger.Service,
-	connectionID uint8,
-	originLocalAddr string,
-	uuid uuid.UUID,
-	credentialManager *reconnectCredentialManager,
-) error {
-	token, err := credentialManager.ReconnectToken()
-	if err != nil {
-		return err
-	}
-	eventDigest, err := credentialManager.EventDigest(connectionID)
-	if err != nil {
-		return err
-	}
-	connDigest, err := credentialManager.ConnDigest(connectionID)
-	if err != nil {
-		return err
-	}
-
-	config.TransportLogger.Debug("initiating RPC stream to reconnect")
-	rpcClient, err := newTunnelRPCClient(ctx, muxer, config, reconnect)
-	if err != nil {
-		return err
-	}
-	defer rpcClient.Close()
-	// Request server info without blocking tunnel registration; must use capnp library directly.
-	serverInfoPromise := tunnelrpc.TunnelServer{Client: rpcClient.Client}.GetServerInfo(ctx, func(tunnelrpc.TunnelServer_getServerInfo_Params) error {
-		return nil
-	})
-	LogServerInfo(serverInfoPromise.Result(), connectionID, config.Metrics, logger, config.TunnelEventChan)
-	registration := rpcClient.ReconnectTunnel(
-		ctx,
-		token,
-		eventDigest,
-		connDigest,
-		config.Hostname,
-		config.RegistrationOptions(connectionID, originLocalAddr, uuid),
-	)
-	if registrationErr := registration.DeserializeError(); registrationErr != nil {
-		// ReconnectTunnel RPC failure
-		return processRegisterTunnelError(registrationErr, config.Metrics, reconnect)
-	}
-	return processRegistrationSuccess(config, logger, connectionID, registration, reconnect, credentialManager)
-}

origin/supervisor.go

@@ -8,7 +8,6 @@ import (
 
 	"github.com/google/uuid"
 
-	"github.com/cloudflare/cloudflared/buffer"
 	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/edgediscovery"
 	"github.com/cloudflare/cloudflared/h2mux"
@@ -18,10 +17,10 @@ import (
 )
 
 const (
+	// SRV and TXT record resolution TTL
+	ResolveTTL = time.Hour
 	// Waiting time before retrying a failed tunnel connection
 	tunnelRetryDuration = time.Second * 10
-	// SRV record resolution TTL
-	resolveTTL = time.Hour
 	// Interval between registering new tunnels
 	registrationInterval = time.Second
 
@@ -44,8 +43,6 @@ type Supervisor struct {
 	cloudflaredUUID   uuid.UUID
 	config            *TunnelConfig
 	edgeIPs           *edgediscovery.Edge
-	lastResolve       time.Time
-	resolverC         chan resolveResult
 	tunnelErrors      chan tunnelError
 	tunnelsConnecting map[int]chan struct{}
 	// nextConnectedIndex and nextConnectedSignal are used to wait for all
@@ -56,12 +53,7 @@ type Supervisor struct {
 	logger logger.Service
 
 	reconnectCredentialManager *reconnectCredentialManager
-
-	bufferPool *buffer.Pool
-}
-
-type resolveResult struct {
-	err error
+	useReconnectToken          bool
 }
 
 type tunnelError struct {
@@ -84,6 +76,11 @@ func NewSupervisor(config *TunnelConfig, cloudflaredUUID uuid.UUID) (*Supervisor
 		return nil, err
 	}
 
+	useReconnectToken := false
+	if config.ClassicTunnel != nil {
+		useReconnectToken = config.ClassicTunnel.UseReconnectToken
+	}
+
 	return &Supervisor{
 		cloudflaredUUID:            cloudflaredUUID,
 		config:                     config,
@@ -91,13 +88,12 @@ func NewSupervisor(config *TunnelConfig, cloudflaredUUID uuid.UUID) (*Supervisor
 		tunnelErrors:               make(chan tunnelError),
 		tunnelsConnecting:          map[int]chan struct{}{},
 		logger:                     config.Logger,
-		reconnectCredentialManager: newReconnectCredentialManager(metricsNamespace, tunnelSubsystem, config.HAConnections),
-		bufferPool:                 buffer.NewPool(512 * 1024),
+		reconnectCredentialManager: newReconnectCredentialManager(connection.MetricsNamespace, connection.TunnelSubsystem, config.HAConnections),
+		useReconnectToken:          useReconnectToken,
 	}, nil
 }
 
 func (s *Supervisor) Run(ctx context.Context, connectedSignal *signal.Signal, reconnectCh chan ReconnectSignal) error {
-	logger := s.config.Logger
 	if err := s.initialize(ctx, connectedSignal, reconnectCh); err != nil {
 		return err
 	}
@@ -110,11 +106,11 @@ func (s *Supervisor) Run(ctx context.Context, connectedSignal *signal.Signal, re
 	refreshAuthBackoff := &BackoffHandler{MaxRetries: refreshAuthMaxBackoff, BaseTime: refreshAuthRetryDuration, RetryForever: true}
 	var refreshAuthBackoffTimer <-chan time.Time
 
-	if s.config.UseReconnectToken {
+	if s.useReconnectToken {
 		if timer, err := s.reconnectCredentialManager.RefreshAuth(ctx, refreshAuthBackoff, s.authenticate); err == nil {
 			refreshAuthBackoffTimer = timer
 		} else {
-			logger.Errorf("supervisor: initial refreshAuth failed, retrying in %v: %s", refreshAuthRetryDuration, err)
+			s.logger.Errorf("supervisor: initial refreshAuth failed, retrying in %v: %s", refreshAuthRetryDuration, err)
 			refreshAuthBackoffTimer = time.After(refreshAuthRetryDuration)
 		}
 	}
@@ -133,7 +129,7 @@ func (s *Supervisor) Run(ctx context.Context, connectedSignal *signal.Signal, re
 		case tunnelError := <-s.tunnelErrors:
 			tunnelsActive--
 			if tunnelError.err != nil {
-				logger.Infof("supervisor: Tunnel disconnected due to error: %s", tunnelError.err)
+				s.logger.Infof("supervisor: Tunnel disconnected due to error: %s", tunnelError.err)
 				tunnelsWaiting = append(tunnelsWaiting, tunnelError.index)
 				s.waitForNextTunnel(tunnelError.index)
 
@@ -156,7 +152,7 @@ func (s *Supervisor) Run(ctx context.Context, connectedSignal *signal.Signal, re
 		case <-refreshAuthBackoffTimer:
 			newTimer, err := s.reconnectCredentialManager.RefreshAuth(ctx, refreshAuthBackoff, s.authenticate)
 			if err != nil {
-				logger.Errorf("supervisor: Authentication failed: %s", err)
+				s.logger.Errorf("supervisor: Authentication failed: %s", err)
 				// Permanent failure. Leave the `select` without setting the
 				// channel to be non-null, so we'll never hit this case of the `select` again.
 				continue
@@ -168,27 +164,15 @@ func (s *Supervisor) Run(ctx context.Context, connectedSignal *signal.Signal, re
 				// No more tunnels outstanding, clear backoff timer
 				backoff.SetGracePeriod()
 			}
-		// DNS resolution returned
-		case result := <-s.resolverC:
-			s.lastResolve = time.Now()
-			s.resolverC = nil
-			if result.err == nil {
-				logger.Debug("supervisor: Service discovery refresh complete")
-			} else {
-				logger.Errorf("supervisor: Service discovery error: %s", result.err)
-			}
 		}
 	}
 }
 
 // Returns nil if initialization succeeded, else the initialization error.
 func (s *Supervisor) initialize(ctx context.Context, connectedSignal *signal.Signal, reconnectCh chan ReconnectSignal) error {
-	logger := s.logger
-
-	s.lastResolve = time.Now()
 	availableAddrs := int(s.edgeIPs.AvailableAddrs())
 	if s.config.HAConnections > availableAddrs {
-		logger.Infof("You requested %d HA connections but I can give you at most %d.", s.config.HAConnections, availableAddrs)
+		s.logger.Infof("You requested %d HA connections but I can give you at most %d.", s.config.HAConnections, availableAddrs)
 		s.config.HAConnections = availableAddrs
 	}
 
@@ -227,7 +211,7 @@ func (s *Supervisor) startFirstTunnel(ctx context.Context, connectedSignal *sign
 		return
 	}
 
-	err = ServeTunnelLoop(ctx, s.reconnectCredentialManager, s.config, addr, firstConnIndex, connectedSignal, s.cloudflaredUUID, s.bufferPool, reconnectCh)
+	err = ServeTunnelLoop(ctx, s.reconnectCredentialManager, s.config, addr, firstConnIndex, connectedSignal, s.cloudflaredUUID, reconnectCh)
 	// If the first tunnel disconnects, keep restarting it.
 	edgeErrors := 0
 	for s.unusedIPs() {
@@ -239,7 +223,7 @@ func (s *Supervisor) startFirstTunnel(ctx context.Context, connectedSignal *sign
 			return
 		// try the next address if it was a dialError(network problem) or
 		// dupConnRegisterTunnelError
-		case connection.DialError, dupConnRegisterTunnelError:
+		case edgediscovery.DialError, connection.DupConnRegisterTunnelError:
 			edgeErrors++
 		default:
 			return
@@ -250,7 +234,7 @@ func (s *Supervisor) startFirstTunnel(ctx context.Context, connectedSignal *sign
 				return
 			}
 		}
-		err = ServeTunnelLoop(ctx, s.reconnectCredentialManager, s.config, addr, firstConnIndex, connectedSignal, s.cloudflaredUUID, s.bufferPool, reconnectCh)
+		err = ServeTunnelLoop(ctx, s.reconnectCredentialManager, s.config, addr, firstConnIndex, connectedSignal, s.cloudflaredUUID, reconnectCh)
 	}
 }
 
@@ -269,7 +253,7 @@ func (s *Supervisor) startTunnel(ctx context.Context, index int, connectedSignal
 	if err != nil {
 		return
 	}
-	err = ServeTunnelLoop(ctx, s.reconnectCredentialManager, s.config, addr, uint8(index), connectedSignal, s.cloudflaredUUID, s.bufferPool, reconnectCh)
+	err = ServeTunnelLoop(ctx, s.reconnectCredentialManager, s.config, addr, uint8(index), connectedSignal, s.cloudflaredUUID, reconnectCh)
 }
 
 func (s *Supervisor) newConnectedTunnelSignal(index int) *signal.Signal {
@@ -301,7 +285,7 @@ func (s *Supervisor) authenticate(ctx context.Context, numPreviousAttempts int)
 		return nil, err
 	}
 
-	edgeConn, err := connection.DialEdge(ctx, dialTimeout, s.config.TlsConfig, arbitraryEdgeIP)
+	edgeConn, err := edgediscovery.DialEdge(ctx, dialTimeout, s.config.EdgeTLSConfigs[connection.H2mux], arbitraryEdgeIP)
 	if err != nil {
 		return nil, err
 	}
@@ -311,8 +295,8 @@ func (s *Supervisor) authenticate(ctx context.Context, numPreviousAttempts int)
 		// This callback is invoked by h2mux when the edge initiates a stream.
 		return nil // noop
 	})
-	muxerConfig := s.config.muxerConfig(handler)
-	muxer, err := h2mux.Handshake(edgeConn, edgeConn, muxerConfig, s.config.Metrics.activeStreams)
+	muxerConfig := s.config.MuxerConfig.H2MuxerConfig(handler, s.logger)
+	muxer, err := h2mux.Handshake(edgeConn, edgeConn, *muxerConfig, h2mux.ActiveStreams)
 	if err != nil {
 		return nil, err
 	}
@@ -323,23 +307,15 @@ func (s *Supervisor) authenticate(ctx context.Context, numPreviousAttempts int)
 		<-muxer.Shutdown()
 	}()
 
-	rpcClient, err := newTunnelRPCClient(ctx, muxer, s.config, authenticate)
+	stream, err := muxer.OpenRPCStream(ctx)
 	if err != nil {
 		return nil, err
 	}
+	rpcClient := connection.NewTunnelServerClient(ctx, stream, s.logger)
 	defer rpcClient.Close()
 
 	const arbitraryConnectionID = uint8(0)
 	registrationOptions := s.config.RegistrationOptions(arbitraryConnectionID, edgeConn.LocalAddr().String(), s.cloudflaredUUID)
 	registrationOptions.NumPreviousAttempts = uint8(numPreviousAttempts)
-	authResponse, err := rpcClient.Authenticate(
-		ctx,
-		s.config.OriginCert,
-		s.config.Hostname,
-		registrationOptions,
-	)
-	if err != nil {
-		return nil, err
-	}
-	return authResponse.Outcome(), nil
+	return rpcClient.Authenticate(ctx, s.config.ClassicTunnel, registrationOptions)
 }

origin/tunnel_test.go

@@ -0,0 +1,90 @@
+package origin
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/cloudflare/cloudflared/connection"
+	"github.com/cloudflare/cloudflared/logger"
+	"github.com/stretchr/testify/assert"
+)
+
+type dynamicMockFetcher struct {
+	percentage int32
+	err        error
+}
+
+func (dmf *dynamicMockFetcher) fetch() connection.PercentageFetcher {
+	return func() (int32, error) {
+		if dmf.err != nil {
+			return 0, dmf.err
+		}
+		return dmf.percentage, nil
+	}
+}
+func TestWaitForBackoffFallback(t *testing.T) {
+	maxRetries := uint(3)
+	backoff := BackoffHandler{
+		MaxRetries: maxRetries,
+		BaseTime:   time.Millisecond * 10,
+	}
+	ctx := context.Background()
+	logger, err := logger.New()
+	assert.NoError(t, err)
+	resolveTTL := time.Duration(0)
+	namedTunnel := &connection.NamedTunnelConfig{
+		Credentials: connection.Credentials{
+			AccountTag: "test-account",
+		},
+	}
+	mockFetcher := dynamicMockFetcher{
+		percentage: 0,
+	}
+	protocolSelector, err := connection.NewProtocolSelector(connection.HTTP2.String(), namedTunnel, mockFetcher.fetch(), resolveTTL, logger)
+	assert.NoError(t, err)
+	config := &TunnelConfig{
+		Logger:           logger,
+		ProtocolSelector: protocolSelector,
+		Observer:         connection.NewObserver(nil, nil, false),
+	}
+	connIndex := uint8(1)
+
+	initProtocol := protocolSelector.Current()
+	assert.Equal(t, connection.HTTP2, initProtocol)
+
+	protocallFallback := &protocallFallback{
+		backoff,
+		initProtocol,
+		false,
+	}
+
+	// Retry #0 and #1. At retry #2, we switch protocol, so the fallback loop has one more retry than this
+	for i := 0; i < int(maxRetries-1); i++ {
+		err := waitForBackoff(ctx, protocallFallback, config, connIndex, fmt.Errorf("Some error"))
+		assert.NoError(t, err)
+		assert.Equal(t, initProtocol, protocallFallback.protocol)
+	}
+
+	// Retry fallback protocol
+	for i := 0; i < int(maxRetries); i++ {
+		err := waitForBackoff(ctx, protocallFallback, config, connIndex, fmt.Errorf("Some error"))
+		assert.NoError(t, err)
+		fallback, ok := protocolSelector.Fallback()
+		assert.True(t, ok)
+		assert.Equal(t, fallback, protocallFallback.protocol)
+	}
+
+	currentGlobalProtocol := protocolSelector.Current()
+	assert.Equal(t, initProtocol, currentGlobalProtocol)
+
+	// No protocol to fallback, return error
+	err = waitForBackoff(ctx, protocallFallback, config, connIndex, fmt.Errorf("Some error"))
+	assert.Error(t, err)
+
+	protocallFallback.reset()
+	err = waitForBackoff(ctx, protocallFallback, config, connIndex, fmt.Errorf("New error"))
+	assert.NoError(t, err)
+	assert.Equal(t, initProtocol, protocallFallback.protocol)
+}

sshgen/sshgen.go

@@ -52,7 +52,7 @@ var mockRequest func(url, contentType string, body io.Reader) (*http.Response, e
 
 // GenerateShortLivedCertificate generates and stores a keypair for short lived certs
 func GenerateShortLivedCertificate(appURL *url.URL, token string) error {
-	fullName, err := cfpath.GenerateFilePathFromURL(appURL, keyName)
+	fullName, err := cfpath.GenerateAppTokenFilePathFromURL(appURL, keyName)
 	if err != nil {
 		return err
 	}

sshgen/sshgen_test.go

@@ -35,7 +35,7 @@ func TestCertGenSuccess(t *testing.T) {
 	url, _ := url.Parse("https://cf-test-access.com/testpath")
 	token := tokenGenerator()
 
-	fullName, err := cfpath.GenerateFilePathFromURL(url, keyName)
+	fullName, err := cfpath.GenerateAppTokenFilePathFromURL(url, keyName)
 	assert.NoError(t, err)
 
 	pubKeyName := fullName + ".pub"

tlsconfig/certreloader.go

@@ -17,8 +17,6 @@ import (
 const (
 	OriginCAPoolFlag = "origin-ca-pool"
 	CaCertFlag       = "cacert"
-
-	edgeTLSServerName = "cftunnel.com"
 )
 
 // CertReloader can load and reload a TLS certificate from a particular filepath.
@@ -65,10 +63,9 @@ func (cr *CertReloader) LoadCert() error {
 	return nil
 }
 
-func LoadOriginCA(c *cli.Context, logger logger.Service) (*x509.CertPool, error) {
+func LoadOriginCA(originCAPoolFilename string, logger logger.Service) (*x509.CertPool, error) {
 	var originCustomCAPool []byte
 
-	originCAPoolFilename := c.String(OriginCAPoolFlag)
 	if originCAPoolFilename != "" {
 		var err error
 		originCustomCAPool, err = ioutil.ReadFile(originCAPoolFilename)
@@ -121,13 +118,13 @@ func LoadCustomOriginCA(originCAFilename string) (*x509.CertPool, error) {
 	return certPool, nil
 }
 
-func CreateTunnelConfig(c *cli.Context) (*tls.Config, error) {
+func CreateTunnelConfig(c *cli.Context, serverName string) (*tls.Config, error) {
 	var rootCAs []string
 	if c.String(CaCertFlag) != "" {
 		rootCAs = append(rootCAs, c.String(CaCertFlag))
 	}
 
-	userConfig := &TLSParameters{RootCAs: rootCAs, ServerName: edgeTLSServerName}
+	userConfig := &TLSParameters{RootCAs: rootCAs, ServerName: serverName}
 	tlsConfig, err := GetConfig(userConfig)
 	if err != nil {
 		return nil, err

tunneldns/tunnel.go

@@ -9,7 +9,6 @@ import (
 	"syscall"
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/metrics"
 
@@ -27,10 +26,51 @@ type Listener struct {
 	logger logger.Service
 }
 
+func Command(hidden bool) *cli.Command {
+	return &cli.Command{
+		Name:   "proxy-dns",
+		Action: cliutil.ErrorHandler(Run),
+		Usage:  "Run a DNS over HTTPS proxy server.",
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:    "metrics",
+				Value:   "localhost:",
+				Usage:   "Listen address for metrics reporting.",
+				EnvVars: []string{"TUNNEL_METRICS"},
+			},
+			&cli.StringFlag{
+				Name:    "address",
+				Usage:   "Listen address for the DNS over HTTPS proxy server.",
+				Value:   "localhost",
+				EnvVars: []string{"TUNNEL_DNS_ADDRESS"},
+			},
+			&cli.IntFlag{
+				Name:    "port",
+				Usage:   "Listen on given port for the DNS over HTTPS proxy server.",
+				Value:   53,
+				EnvVars: []string{"TUNNEL_DNS_PORT"},
+			},
+			&cli.StringSliceFlag{
+				Name:    "upstream",
+				Usage:   "Upstream endpoint URL, you can specify multiple endpoints for redundancy.",
+				Value:   cli.NewStringSlice("https://1.1.1.1/dns-query", "https://1.0.0.1/dns-query"),
+				EnvVars: []string{"TUNNEL_DNS_UPSTREAM"},
+			},
+			&cli.StringSliceFlag{
+				Name:    "bootstrap",
+				Usage:   "bootstrap endpoint URL, you can specify multiple endpoints for redundancy.",
+				Value:   cli.NewStringSlice("https://162.159.36.1/dns-query", "https://162.159.46.1/dns-query", "https://[2606:4700:4700::1111]/dns-query", "https://[2606:4700:4700::1001]/dns-query"),
+				EnvVars: []string{"TUNNEL_DNS_BOOTSTRAP"},
+			},
+		},
+		ArgsUsage: " ", // can't be the empty string or we get the default output
+		Hidden:    hidden,
+	}
+}
+
 // Run implements a foreground runner
 func Run(c *cli.Context) error {
-	logDirectory, logLevel := config.FindLogSettings()
-	logger, err := logger.New(logger.DefaultFile(logDirectory), logger.LogLevelString(logLevel))
+	logger, err := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
 	if err != nil {
 		return cliutil.PrintLoggerSetupError("error setting up logger", err)
 	}
@@ -40,7 +80,7 @@ func Run(c *cli.Context) error {
 		logger.Fatalf("Failed to open the metrics listener: %s", err)
 	}
 
-	go metrics.ServeMetrics(metricsListener, nil, logger)
+	go metrics.ServeMetrics(metricsListener, nil, nil, logger)
 
 	listener, err := CreateListener(c.String("address"), uint16(c.Uint("port")), c.StringSlice("upstream"), c.StringSlice("bootstrap"), logger)
 	if err != nil {

tunnelrpc/pogs/connectionrpc.go

@@ -93,6 +93,11 @@ type RegistrationServer_PogsClient struct {
 	Conn   *rpc.Conn
 }
 
+func (c RegistrationServer_PogsClient) Close() error {
+	c.Client.Close()
+	return c.Conn.Close()
+}
+
 func (c RegistrationServer_PogsClient) RegisterConnection(ctx context.Context, auth TunnelAuth, tunnelID uuid.UUID, connIndex byte, options *ConnectionOptions) (*ConnectionDetails, error) {
 	client := tunnelrpc.TunnelServer{Client: c.Client}
 	promise := client.RegisterConnection(ctx, func(p tunnelrpc.RegistrationServer_registerConnection_Params) error {

validation/validation.go

@@ -66,7 +66,15 @@ func ValidateHostname(hostname string) (string, error) {
 // but when it does not, the path is preserved:
 //   ValidateUrl("localhost:8080/api/") => "http://localhost:8080/api/"
 // This is arguably a bug, but changing it might break some cloudflared users.
-func ValidateUrl(originUrl string) (string, error) {
+func ValidateUrl(originUrl string) (*url.URL, error) {
+	urlStr, err := validateUrlString(originUrl)
+	if err != nil {
+		return nil, err
+	}
+	return url.Parse(urlStr)
+}
+
+func validateUrlString(originUrl string) (string, error) {
 	if originUrl == "" {
 		return "", fmt.Errorf("URL should not be empty")
 	}
@@ -157,6 +165,7 @@ func validateIP(scheme, host, port string) (string, error) {
 	return fmt.Sprintf("%s://%s", scheme, host), nil
 }
 
+// originURL shouldn't be a pointer, because this function might change the scheme
 func ValidateHTTPService(originURL string, hostname string, transport http.RoundTripper) error {
 	parsedURL, err := url.Parse(originURL)
 	if err != nil {
@@ -176,28 +185,32 @@ func ValidateHTTPService(originURL string, hostname string, transport http.Round
 		return err
 	}
 	initialRequest.Host = hostname
-	_, initialErr := client.Do(initialRequest)
-	if initialErr != nil {
-		// Attempt the same endpoint via the other protocol (http/https); maybe we have better luck?
-		oldScheme := parsedURL.Scheme
-		parsedURL.Scheme = toggleProtocol(parsedURL.Scheme)
+	resp, initialErr := client.Do(initialRequest)
+	if initialErr == nil {
+		resp.Body.Close()
+		return nil
+	}
 
-		secondRequest, err := http.NewRequest("GET", parsedURL.String(), nil)
-		if err != nil {
-			return err
-		}
-		secondRequest.Host = hostname
-		_, secondErr := client.Do(secondRequest)
-		if secondErr == nil { // Worked this time--advise the user to switch protocols
-			return errors.Errorf(
-				"%s doesn't seem to work over %s, but does seem to work over %s. Reason: %v. Consider changing the origin URL to %s",
-				parsedURL.Host,
-				oldScheme,
-				parsedURL.Scheme,
-				initialErr,
-				parsedURL,
-			)
-		}
+	// Attempt the same endpoint via the other protocol (http/https); maybe we have better luck?
+	oldScheme := parsedURL.Scheme
+	parsedURL.Scheme = toggleProtocol(oldScheme)
+
+	secondRequest, err := http.NewRequest("GET", parsedURL.String(), nil)
+	if err != nil {
+		return err
+	}
+	secondRequest.Host = hostname
+	resp, secondErr := client.Do(secondRequest)
+	if secondErr == nil { // Worked this time--advise the user to switch protocols
+		resp.Body.Close()
+		return errors.Errorf(
+			"%s doesn't seem to work over %s, but does seem to work over %s. Reason: %v. Consider changing the origin URL to %v",
+			parsedURL.Host,
+			oldScheme,
+			parsedURL.Scheme,
+			initialErr,
+			originURL,
+		)
 	}
 
 	return initialErr
@@ -220,12 +233,12 @@ type Access struct {
 }
 
 func NewAccessValidator(ctx context.Context, domain, issuer, applicationAUD string) (*Access, error) {
-	domainURL, err := ValidateUrl(domain)
+	domainURL, err := validateUrlString(domain)
 	if err != nil {
 		return nil, err
 	}
 
-	issuerURL, err := ValidateUrl(issuer)
+	issuerURL, err := validateUrlString(issuer)
 	if err != nil {
 		return nil, err
 	}

validation/validation_test.go

@@ -101,7 +101,7 @@ func TestValidateUrl(t *testing.T) {
 	for i, testCase := range testCases {
 		validUrl, err := ValidateUrl(testCase.input)
 		assert.NoError(t, err, "test case %v", i)
-		assert.Equal(t, testCase.expectedOutput, validUrl, "test case %v", i)
+		assert.Equal(t, testCase.expectedOutput, validUrl.String(), "test case %v", i)
 	}
 
 	validUrl, err := ValidateUrl("")
@@ -151,7 +151,7 @@ func TestValidateHTTPService_HTTP2HTTP(t *testing.T) {
 
 // Happy path 2: originURL is HTTPS, and HTTPS connections work
 func TestValidateHTTPService_HTTPS2HTTPS(t *testing.T) {
-	originURL := "https://127.0.0.1/"
+	originURL := "https://127.0.0.1:1234/"
 	hostname := "example.com"
 
 	assert.Nil(t, ValidateHTTPService(originURL, hostname, testRoundTripper(func(req *http.Request) (*http.Response, error) {

vendor/github.com/equinox-io/equinox/.travis.yml

@@ -1,13 +0,0 @@
-language: go
-go:
-  - "1.4"
-  - "1.5"
-  - "1.7"
-  - "1.9"
-  - "1.10"
-  - "1.11"
-  - "tip"
-matrix:
-  allow_failures:
-    - go: tip
-script: go test -v github.com/equinox-io/equinox github.com/equinox-io/equinox/proto