Release 2020.11.8

require URL.

Makefile

@@ -89,6 +89,7 @@ define publish_package
 	for HOST in $(CF_PKG_HOSTS); do \
 		ssh-keyscan -t rsa $$HOST >> ~/.ssh/known_hosts; \
 		scp -4 cloudflared*.$(1) cfsync@$$HOST:/state/cf-pkg/staging/$(2)/$(TARGET_PUBLIC_REPO)/cloudflared/; \
+		ssh cfsync@$$HOST 'chgrp cf /state/cf-pkg/staging/$(2)/$(TARGET_PUBLIC_REPO)/cloudflared/*.$(1) && chmod g+w /state/cf-pkg/staging/$(2)/$(TARGET_PUBLIC_REPO)/cloudflared/*.$(1)'; \
 	done
 endef
 
@@ -225,4 +226,4 @@ vet:
 
 .PHONY: msi
 msi: cloudflared
-	go-msi make --msi cloudflared.msi --version $(MSI_VERSION)
+	go-msi make --msi cloudflared.msi --version $(MSI_VERSION)

README.md

@@ -1,9 +1,32 @@
 # Argo Tunnel client
 
-Contains the command-line client and its libraries for Argo Tunnel, a tunneling daemon that proxies any local webserver through the Cloudflare network.
+Contains the command-line client for Argo Tunnel, a tunneling daemon that proxies any local webserver through the Cloudflare network. Extensive documentation can be found in the [Argo Tunnel section](https://developers.cloudflare.com/argo-tunnel/) of the Cloudflare Docs.
 
-## Getting started
+## Before you get started
 
-    go install github.com/cloudflare/cloudflared/cmd/cloudflared
+Before you use Argo Tunnel, you'll need to complete a few steps in the Cloudflare dashboard. The website you add to Cloudflare will be used to route traffic to your Tunnel.
+
+1. [Add a website to Cloudflare](https://support.cloudflare.com/hc/en-us/articles/201720164-Creating-a-Cloudflare-account-and-adding-a-website)
+2. [Change your domain nameservers to Cloudflare](https://support.cloudflare.com/hc/en-us/articles/205195708)
+
+## Installing `cloudflared`
+
+Downloads are available as standalone binaries, a Docker image, and Debian, RPM, and Homebrew packages. You can also find releases here on the `cloudflared` GitHub repository.
+
+* You can [install on macOS](https://developers.cloudflare.com/argo-tunnel/getting-started/installation#macos) via Homebrew or by downloading the [latest Darwin amd64 release](https://github.com/cloudflare/cloudflared/releases)
+* Binaries, Debian, and RPM packages for Linux [can be found here](https://developers.cloudflare.com/argo-tunnel/getting-started/installation#linux)
+* A Docker image of `cloudflared` is [available on DockerHub](https://hub.docker.com/r/cloudflare/cloudflared)
+* You can install on Windows machines with the [steps here](https://developers.cloudflare.com/argo-tunnel/getting-started/installation#windows)
 
 User documentation for Argo Tunnel can be found at https://developers.cloudflare.com/argo-tunnel/
+
+## Creating Tunnels and routing traffic
+
+Once installed, you can authenticate `cloudflared` into your Cloudflare account and begin creating Tunnels that serve traffic for hostnames in your account.
+
+* Create a Tunnel with [these instructions](https://developers.cloudflare.com/argo-tunnel/create-tunnel)
+* Route traffic to that Tunnel with [DNS records in Cloudflare](https://developers.cloudflare.com/argo-tunnel/routing-to-tunnel/dns) or with a [Cloudflare Load Balancer](https://developers.cloudflare.com/argo-tunnel/routing-to-tunnel/lb)
+
+## TryCloudflare
+
+Want to test Argo Tunnel before adding a website to Cloudflare? You can do so with TryCloudflare using the documentation [available here](https://developers.cloudflare.com/argo-tunnel/learning/trycloudflare).

RELEASE_NOTES

@@ -1,3 +1,60 @@
+2020.11.8
+- 2020-11-17 TUN-3555: Single origin service should default to localhost:8080
+
+2020.11.7
+- 2020-11-13 TUN-3514: Stop setting --is-autoupdated flag after autoupdate because it can break named tunnel running in k8s
+- 2020-11-15 TUN-3548, TUN-3547: Bastion mode can be specified as a service, doesn't require URL.
+- 2020-11-16 TUN-3549: Use a separate handler for each websocket proxy
+
+2020.11.6
+- 2020-11-14 TUN-3546: Fix panic in tlsconfig.LoadOriginCA
+
+2020.11.5
+- 2020-11-12 TUN-3540: Better copy in ingress rules error messages
+- 2020-11-12 DEVTOOLS-7936: Set permissions on public packages
+- 2020-11-13 TUN-3543: ProxyAddress not using default in single-origin mode
+
+2020.11.4
+- 2020-11-11 TUN-3534: Specific error message when credentials file is a .pem not .json
+- 2020-11-02 TUN-3500: Integrate replace h2mux by http2 work with multiple origin support
+- 2020-11-09 TUN-3514: Transport logger write to UI when UI is enabled
+- 2020-10-30 TUN-3490: Make sure OriginClient implementation doesn't write after Proxy return
+- 2020-10-20 TUN-3403: Unit test for origin/proxy to test serving HTTP and Websocket
+- 2020-10-23 TUN-3480: Support SSE with http2 connection, and add SSE handler to hello-world server
+- 2020-10-27 TUN-3489: Add unit tests to cover proxy logic in connection package of cloudflared
+- 2020-10-16 TUN-3467: Serialize cf-cloudflared-response-meta during package initialization using jsoniter
+- 2020-10-14 TUN-3456: New protocol option auto to automatically select between http2 and h2mux
+- 2020-10-14 TUN-3458: Upgrade to http2 when available, fallback to h2mux when we reach max retries
+- 2020-10-08 TUN-3449: Use flag to select transport protocol implementation
+- 2020-10-08 TUN-3462: Refactor cloudflared to separate origin from connection
+- 2020-09-21 TUN-3406: Proxy websocket requests over Go http2
+- 2020-09-25 TUN-3420: Establish control plane and send RPC over control plane
+- 2020-09-11 TUN-3400: Use Go HTTP2 library as transport to connect with the edge
+
+2020.11.3
+- 2020-11-11 TUN-3533: Set config for single origin ingress
+
+2020.11.2
+
+
+2020.11.1
+- 2020-11-10 TUN-3527: More specific error for invalid YAML/JSON
+- 2020-11-06 Update README.md (#256)
+
+2020.11.0
+- 2020-11-04 TUN-3484: OriginService that responds with configured HTTP status
+- 2020-11-05 TUN-3505: Response body for status code origin returns EOF on Read
+- 2020-11-04 TUN-3503: Matching ingress rule should not take port into account
+- 2020-11-05 TUN-3506: OriginService needs to set request host and scheme for websocket requests
+- 2020-11-09 TUN-3516: Better error message when parsing invalid YAML config
+- 2020-11-09 TUN-3522: ingress validate checks that the config file exists
+- 2020-11-09 TUN-3524: Don't ignore errors from app-level action handler (#248)
+- 2020-11-09 TUN-3461: Show all origin services in the UI
+- 2020-10-30 TUN-3494: Proceed to create tunnel if at least one edge address can be resolved
+- 2020-10-30 TUN-3492: Refactor OriginService, shrink its interface
+- 2020-10-22 TUN-3478: Increase download timeout to 60s
+- 2020-10-15 TUN-2640: Users can configure per-origin config. Unify single-rule CLI flow with multi-rule config file code.
+
 2020.10.2
 - 2020-10-21 Release 2020.10.1
 - 2020-10-21 AUTH-3185 fixed indention error

cmd/cloudflared/access/carrier.go

@@ -2,7 +2,6 @@ package access
 
 import (
 	"net/http"
-	"net/url"
 	"strings"
 
 	"github.com/cloudflare/cloudflared/carrier"
@@ -17,16 +16,11 @@ import (
 
 // StartForwarder starts a client side websocket forward
 func StartForwarder(forwarder config.Forwarder, shutdown <-chan struct{}, logger logger.Service) error {
-	validURLString, err := validation.ValidateUrl(forwarder.Listener)
+	validURL, err := validation.ValidateUrl(forwarder.Listener)
 	if err != nil {
 		return errors.Wrap(err, "error validating origin URL")
 	}
 
-	validURL, err := url.Parse(validURLString)
-	if err != nil {
-		return errors.Wrap(err, "error parsing origin URL")
-	}
-
 	// get the headers from the config file and add to the request
 	headers := make(http.Header)
 	if forwarder.TokenClientID != "" {
@@ -106,12 +100,7 @@ func ssh(c *cli.Context) error {
 	wsConn := carrier.NewWSConnection(logger, false)
 
 	if c.NArg() > 0 || c.IsSet(sshURLFlag) {
-		localForwarder, err := config.ValidateUrl(c, true)
-		if err != nil {
-			logger.Errorf("Error validating origin URL: %s", err)
-			return errors.Wrap(err, "error validating origin URL")
-		}
-		forwarder, err := url.Parse(localForwarder)
+		forwarder, err := config.ValidateUrl(c, true)
 		if err != nil {
 			logger.Errorf("Error validating origin URL: %s", err)
 			return errors.Wrap(err, "error validating origin URL")

cmd/cloudflared/cliutil/errors.go

@@ -27,17 +27,17 @@ func UsageError(format string, args ...interface{}) error {
 // Ensures exit with error code if actionFunc returns an error
 func ErrorHandler(actionFunc cli.ActionFunc) cli.ActionFunc {
 	return func(ctx *cli.Context) error {
+		defer logger.SharedWriteManager.Shutdown()
+
 		err := actionFunc(ctx)
 		if err != nil {
 			if _, ok := err.(usageError); ok {
 				msg := fmt.Sprintf("%s\nSee 'cloudflared %s --help'.", err.Error(), ctx.Command.FullName())
-				return cli.Exit(msg, -1)
+				err = cli.Exit(msg, -1)
+			} else if _, ok := err.(cli.ExitCoder); !ok {
+				err = cli.Exit(err.Error(), 1)
 			}
-			// os.Exits with error code if err is cli.ExitCoder or cli.MultiError
-			cli.HandleExitCoder(err)
-			err = cli.Exit(err.Error(), 1)
 		}
-		logger.SharedWriteManager.Shutdown()
 		return err
 	}
 }

cmd/cloudflared/config/configuration.go

@@ -1,14 +1,15 @@
 package config
 
 import (
-	"errors"
 	"fmt"
+	"net/url"
 	"os"
 	"path/filepath"
 	"runtime"
 	"time"
 
 	homedir "github.com/mitchellh/go-homedir"
+	"github.com/pkg/errors"
 	"github.com/urfave/cli/v2"
 	"gopkg.in/yaml.v2"
 
@@ -34,7 +35,12 @@ var (
 	ErrNoConfigFile = fmt.Errorf("Cannot determine default configuration path. No file %v in %v", DefaultConfigFiles, DefaultConfigSearchDirectories())
 )
 
-const DefaultCredentialFile = "cert.pem"
+const (
+	DefaultCredentialFile = "cert.pem"
+
+	// BastionFlag is to enable bastion, or jump host, operation
+	BastionFlag = "bastion"
+)
 
 // DefaultConfigDirectory returns the default directory of the config file
 func DefaultConfigDirectory() string {
@@ -184,11 +190,11 @@ func ValidateUnixSocket(c *cli.Context) (string, error) {
 
 // ValidateUrl will validate url flag correctness. It can be either from --url or argument
 // Notice ValidateUnixSocket, it will enforce --unix-socket is not used with --url or argument
-func ValidateUrl(c *cli.Context, allowFromArgs bool) (string, error) {
+func ValidateUrl(c *cli.Context, allowURLFromArgs bool) (*url.URL, error) {
 	var url = c.String("url")
-	if allowFromArgs && c.NArg() > 0 {
+	if allowURLFromArgs && c.NArg() > 0 {
 		if c.IsSet("url") {
-			return "", errors.New("Specified origin urls using both --url and argument. Decide which one you want, I can only support one.")
+			return nil, errors.New("Specified origin urls using both --url and argument. Decide which one you want, I can only support one.")
 		}
 		url = c.Args().Get(0)
 	}
@@ -197,15 +203,59 @@ func ValidateUrl(c *cli.Context, allowFromArgs bool) (string, error) {
 }
 
 type UnvalidatedIngressRule struct {
-	Hostname string
-	Path     string
-	Service  string
+	Hostname      string
+	Path          string
+	Service       string
+	OriginRequest OriginRequestConfig `yaml:"originRequest"`
+}
+
+// OriginRequestConfig is a set of optional fields that users may set to
+// customize how cloudflared sends requests to origin services. It is used to set
+// up general config that apply to all rules, and also, specific per-rule
+// config.
+// Note: To specify a time.Duration in go-yaml, use e.g. "3s" or "24h".
+type OriginRequestConfig struct {
+	// HTTP proxy timeout for establishing a new connection
+	ConnectTimeout *time.Duration `yaml:"connectTimeout"`
+	// HTTP proxy timeout for completing a TLS handshake
+	TLSTimeout *time.Duration `yaml:"tlsTimeout"`
+	// HTTP proxy TCP keepalive duration
+	TCPKeepAlive *time.Duration `yaml:"tcpKeepAlive"`
+	// HTTP proxy should disable "happy eyeballs" for IPv4/v6 fallback
+	NoHappyEyeballs *bool `yaml:"noHappyEyeballs"`
+	// HTTP proxy maximum keepalive connection pool size
+	KeepAliveConnections *int `yaml:"keepAliveConnections"`
+	// HTTP proxy timeout for closing an idle connection
+	KeepAliveTimeout *time.Duration `yaml:"keepAliveTimeout"`
+	// Sets the HTTP Host header for the local webserver.
+	HTTPHostHeader *string `yaml:"httpHostHeader"`
+	// Hostname on the origin server certificate.
+	OriginServerName *string `yaml:"originServerName"`
+	// Path to the CA for the certificate of your origin.
+	// This option should be used only if your certificate is not signed by Cloudflare.
+	CAPool *string `yaml:"caPool"`
+	// Disables TLS verification of the certificate presented by your origin.
+	// Will allow any certificate from the origin to be accepted.
+	// Note: The connection from your machine to Cloudflare's Edge is still encrypted.
+	NoTLSVerify *bool `yaml:"noTLSVerify"`
+	// Disables chunked transfer encoding.
+	// Useful if you are running a WSGI server.
+	DisableChunkedEncoding *bool `yaml:"disableChunkedEncoding"`
+	// Runs as jump host
+	BastionMode *bool `yaml:"bastionMode"`
+	// Listen address for the proxy.
+	ProxyAddress *string `yaml:"proxyAddress"`
+	// Listen port for the proxy.
+	ProxyPort *uint `yaml:"proxyPort"`
+	// Valid options are 'socks' or empty.
+	ProxyType *string `yaml:"proxyType"`
 }
 
 type Configuration struct {
-	TunnelID   string `yaml:"tunnel"`
-	Ingress    []UnvalidatedIngressRule
-	sourceFile string
+	TunnelID      string `yaml:"tunnel"`
+	Ingress       []UnvalidatedIngressRule
+	OriginRequest OriginRequestConfig `yaml:"originRequest"`
+	sourceFile    string
 }
 
 type configFileSettings struct {
@@ -342,7 +392,7 @@ func ReadConfigFile(c *cli.Context, log logger.Service) (*configFileSettings, er
 	}
 	defer file.Close()
 	if err := yaml.NewDecoder(file).Decode(&configuration); err != nil {
-		return nil, err
+		return nil, errors.Wrap(err, "error parsing YAML in config file at "+configFile)
 	}
 	configuration.sourceFile = configFile
 	return &configuration, nil

cmd/cloudflared/config/manager.go

@@ -1,11 +1,11 @@
 package config
 
 import (
-	"errors"
 	"os"
 
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/watcher"
+	"github.com/pkg/errors"
 	"gopkg.in/yaml.v2"
 )
 
@@ -80,7 +80,7 @@ func readConfigFromPath(configPath string) (Root, error) {
 
 	var config Root
 	if err := yaml.NewDecoder(file).Decode(&config); err != nil {
-		return Root{}, err
+		return Root{}, errors.Wrap(err, "error parsing YAML in config file at "+configPath)
 	}
 
 	return config, nil

cmd/cloudflared/main.go

@@ -145,7 +145,7 @@ func isEmptyInvocation(c *cli.Context) bool {
 }
 
 func action(version string, shutdownC, graceShutdownC chan struct{}) cli.ActionFunc {
-	return func(c *cli.Context) (err error) {
+	return cliutil.ErrorHandler(func(c *cli.Context) (err error) {
 		if isEmptyInvocation(c) {
 			return handleServiceMode(shutdownC)
 		}
@@ -153,15 +153,11 @@ func action(version string, shutdownC, graceShutdownC chan struct{}) cli.ActionF
 		tags["hostname"] = c.String("hostname")
 		raven.SetTagsContext(tags)
 		raven.CapturePanic(func() { err = tunnel.TunnelCommand(c) }, nil)
-		exitCode := 0
 		if err != nil {
-			handleError(err)
-			exitCode = 1
+			captureError(err)
 		}
-		// we already handle error printing, so we pass an empty string so we
-		// don't have to print again.
-		return cli.Exit("", exitCode)
-	}
+		return err
+	})
 }
 
 func userHomeDir() (string, error) {
@@ -177,7 +173,7 @@ func userHomeDir() (string, error) {
 }
 
 // In order to keep the amount of noise sent to Sentry low, typical network errors can be filtered out here by a substring match.
-func handleError(err error) {
+func captureError(err error) {
 	errorMessage := err.Error()
 	for _, ignoredErrorMessage := range ignoredErrors {
 		if strings.Contains(errorMessage, ignoredErrorMessage) {

cmd/cloudflared/tunnel/cmd.go

@@ -5,43 +5,33 @@ import (
 	"context"
 	"fmt"
 	"io/ioutil"
-	"net"
-	"net/http"
 	"net/url"
 	"os"
 	"reflect"
-	"runtime"
 	"runtime/trace"
-	"strconv"
 	"strings"
 	"sync"
 	"time"
 
-	"github.com/cloudflare/cloudflared/awsuploader"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/ui"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/updater"
+	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/dbconnect"
-	"github.com/cloudflare/cloudflared/h2mux"
-	"github.com/cloudflare/cloudflared/hello"
+	"github.com/cloudflare/cloudflared/ingress"
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/metrics"
 	"github.com/cloudflare/cloudflared/origin"
 	"github.com/cloudflare/cloudflared/signal"
-	"github.com/cloudflare/cloudflared/socks"
-	"github.com/cloudflare/cloudflared/sshlog"
-	"github.com/cloudflare/cloudflared/sshserver"
 	"github.com/cloudflare/cloudflared/tlsconfig"
 	"github.com/cloudflare/cloudflared/tunneldns"
 	"github.com/cloudflare/cloudflared/tunnelstore"
-	"github.com/cloudflare/cloudflared/websocket"
 
 	"github.com/coreos/go-systemd/daemon"
 	"github.com/facebookgo/grace/gracenet"
 	"github.com/getsentry/raven-go"
-	"github.com/gliderlabs/ssh"
 	"github.com/google/uuid"
 	"github.com/mitchellh/go-homedir"
 	"github.com/pkg/errors"
@@ -84,15 +74,6 @@ const (
 	// hostKeyPath is the path of the dir to save SSH host keys too
 	hostKeyPath = "host-key-path"
 
-	//sshServerFlag enables cloudflared ssh proxy server
-	sshServerFlag = "ssh-server"
-
-	// socks5Flag is to enable the socks server to deframe
-	socks5Flag = "socks5"
-
-	// bastionFlag is to enable bastion, or jump host, operation
-	bastionFlag = "bastion"
-
 	// uiFlag is to enable launching cloudflared in interactive UI mode
 	uiFlag = "ui"
 
@@ -267,8 +248,8 @@ func StartServer(
 	version string,
 	shutdownC,
 	graceShutdownC chan struct{},
-	namedTunnel *origin.NamedTunnelConfig,
-	log logger.Service,
+	namedTunnel *connection.NamedTunnelConfig,
+	generalLogger logger.Service,
 	isUIEnabled bool,
 ) error {
 	_ = raven.SetDSN(sentryDSN)
@@ -279,45 +260,45 @@ func StartServer(
 	dnsReadySignal := make(chan struct{})
 
 	if config.GetConfiguration().Source() == "" {
-		log.Infof(config.ErrNoConfigFile.Error())
+		generalLogger.Infof(config.ErrNoConfigFile.Error())
 	}
 
 	if c.IsSet("trace-output") {
 		tmpTraceFile, err := ioutil.TempFile("", "trace")
 		if err != nil {
-			log.Errorf("Failed to create new temporary file to save trace output: %s", err)
+			generalLogger.Errorf("Failed to create new temporary file to save trace output: %s", err)
 		}
 
 		defer func() {
 			if err := tmpTraceFile.Close(); err != nil {
-				log.Errorf("Failed to close trace output file %s with error: %s", tmpTraceFile.Name(), err)
+				generalLogger.Errorf("Failed to close trace output file %s with error: %s", tmpTraceFile.Name(), err)
 			}
 			if err := os.Rename(tmpTraceFile.Name(), c.String("trace-output")); err != nil {
-				log.Errorf("Failed to rename temporary trace output file %s to %s with error: %s", tmpTraceFile.Name(), c.String("trace-output"), err)
+				generalLogger.Errorf("Failed to rename temporary trace output file %s to %s with error: %s", tmpTraceFile.Name(), c.String("trace-output"), err)
 			} else {
 				err := os.Remove(tmpTraceFile.Name())
 				if err != nil {
-					log.Errorf("Failed to remove the temporary trace file %s with error: %s", tmpTraceFile.Name(), err)
+					generalLogger.Errorf("Failed to remove the temporary trace file %s with error: %s", tmpTraceFile.Name(), err)
 				}
 			}
 		}()
 
 		if err := trace.Start(tmpTraceFile); err != nil {
-			log.Errorf("Failed to start trace: %s", err)
+			generalLogger.Errorf("Failed to start trace: %s", err)
 			return errors.Wrap(err, "Error starting tracing")
 		}
 		defer trace.Stop()
 	}
 
 	buildInfo := buildinfo.GetBuildInfo(version)
-	buildInfo.Log(log)
-	logClientOptions(c, log)
+	buildInfo.Log(generalLogger)
+	logClientOptions(c, generalLogger)
 
 	if c.IsSet("proxy-dns") {
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
-			errC <- runDNSProxyServer(c, dnsReadySignal, shutdownC, log)
+			errC <- runDNSProxyServer(c, dnsReadySignal, shutdownC, generalLogger)
 		}()
 	} else {
 		close(dnsReadySignal)
@@ -328,24 +309,24 @@ func StartServer(
 
 	metricsListener, err := listeners.Listen("tcp", c.String("metrics"))
 	if err != nil {
-		log.Errorf("Error opening metrics server listener: %s", err)
+		generalLogger.Errorf("Error opening metrics server listener: %s", err)
 		return errors.Wrap(err, "Error opening metrics server listener")
 	}
 	defer metricsListener.Close()
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
-		errC <- metrics.ServeMetrics(metricsListener, shutdownC, log)
+		errC <- metrics.ServeMetrics(metricsListener, shutdownC, generalLogger)
 	}()
 
 	go notifySystemd(connectedSignal)
 	if c.IsSet("pidfile") {
-		go writePidFile(connectedSignal, c.String("pidfile"), log)
+		go writePidFile(connectedSignal, c.String("pidfile"), generalLogger)
 	}
 
 	cloudflaredID, err := uuid.NewRandom()
 	if err != nil {
-		log.Errorf("Cannot generate cloudflared ID: %s", err)
+		generalLogger.Errorf("Cannot generate cloudflared ID: %s", err)
 		return err
 	}
 
@@ -356,12 +337,12 @@ func StartServer(
 	}()
 
 	// update needs to be after DNS proxy is up to resolve equinox server address
-	if updater.IsAutoupdateEnabled(c, log) {
-		log.Infof("Autoupdate frequency is set to %v", c.Duration("autoupdate-freq"))
+	if updater.IsAutoupdateEnabled(c, generalLogger) {
+		generalLogger.Infof("Autoupdate frequency is set to %v", c.Duration("autoupdate-freq"))
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
-			autoupdater := updater.NewAutoUpdater(c.Duration("autoupdate-freq"), &listeners, log)
+			autoupdater := updater.NewAutoUpdater(c.Duration("autoupdate-freq"), &listeners, generalLogger)
 			errC <- autoupdater.Run(ctx)
 		}()
 	}
@@ -370,133 +351,34 @@ func StartServer(
 	if dnsProxyStandAlone(c) {
 		connectedSignal.Notify()
 		// no grace period, handle SIGINT/SIGTERM immediately
-		return waitToShutdown(&wg, errC, shutdownC, graceShutdownC, 0, log)
-	}
-
-	if c.IsSet("hello-world") {
-		log.Infof("hello-world set")
-		helloListener, err := hello.CreateTLSListener("127.0.0.1:")
-		if err != nil {
-			log.Errorf("Cannot start Hello World Server: %s", err)
-			return errors.Wrap(err, "Cannot start Hello World Server")
-		}
-		defer helloListener.Close()
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			_ = hello.StartHelloWorldServer(log, helloListener, shutdownC)
-		}()
-		forceSetFlag(c, "url", "https://"+helloListener.Addr().String())
-	}
-
-	if c.IsSet(sshServerFlag) {
-		if runtime.GOOS != "darwin" && runtime.GOOS != "linux" {
-			msg := fmt.Sprintf("--ssh-server is not supported on %s", runtime.GOOS)
-			log.Error(msg)
-			return errors.New(msg)
-		}
-
-		log.Infof("ssh-server set")
-
-		logManager := sshlog.NewEmptyManager()
-		if c.IsSet(bucketNameFlag) && c.IsSet(regionNameFlag) && c.IsSet(accessKeyIDFlag) && c.IsSet(secretIDFlag) {
-			uploader, err := awsuploader.NewFileUploader(c.String(bucketNameFlag), c.String(regionNameFlag),
-				c.String(accessKeyIDFlag), c.String(secretIDFlag), c.String(sessionTokenIDFlag), c.String(s3URLFlag))
-			if err != nil {
-				msg := "Cannot create uploader for SSH Server"
-				log.Errorf("%s: %s", msg, err)
-				return errors.Wrap(err, msg)
-			}
-
-			if err := os.MkdirAll(sshLogFileDirectory, 0700); err != nil {
-				msg := fmt.Sprintf("Cannot create SSH log file directory %s", sshLogFileDirectory)
-				log.Errorf("%s: %s", msg, err)
-				return errors.Wrap(err, msg)
-			}
-
-			logManager = sshlog.New(sshLogFileDirectory)
-
-			uploadManager := awsuploader.NewDirectoryUploadManager(log, uploader, sshLogFileDirectory, 30*time.Minute, shutdownC)
-			uploadManager.Start()
-		}
-
-		localServerAddress := "127.0.0.1:" + c.String(sshPortFlag)
-		server, err := sshserver.New(logManager, log, version, localServerAddress, c.String("hostname"), c.Path(hostKeyPath), shutdownC, c.Duration(sshIdleTimeoutFlag), c.Duration(sshMaxTimeoutFlag))
-		if err != nil {
-			msg := "Cannot create new SSH Server"
-			log.Errorf("%s: %s", msg, err)
-			return errors.Wrap(err, msg)
-		}
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			if err = server.Start(); err != nil && err != ssh.ErrServerClosed {
-				log.Errorf("SSH server error: %s", err)
-				// TODO: remove when declarative tunnels are implemented.
-				close(shutdownC)
-			}
-		}()
-		forceSetFlag(c, "url", "ssh://"+localServerAddress)
+		return waitToShutdown(&wg, errC, shutdownC, graceShutdownC, 0, generalLogger)
 	}
 
 	url := c.String("url")
 	hostname := c.String("hostname")
 	if url == hostname && url != "" && hostname != "" {
 		errText := "hostname and url shouldn't match. See --help for more information"
-		log.Error(errText)
+		generalLogger.Error(errText)
 		return fmt.Errorf(errText)
 	}
 
-	if staticHost := hostnameFromURI(c.String("url")); isProxyDestinationConfigured(staticHost, c) {
-		listener, err := net.Listen("tcp", net.JoinHostPort(c.String("proxy-address"), strconv.Itoa(c.Int("proxy-port"))))
-		if err != nil {
-			log.Errorf("Cannot start Websocket Proxy Server: %s", err)
-			return errors.Wrap(err, "Cannot start Websocket Proxy Server")
-		}
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			streamHandler := websocket.DefaultStreamHandler
-			if c.IsSet(socks5Flag) {
-				log.Info("SOCKS5 server started")
-				streamHandler = func(wsConn *websocket.Conn, remoteConn net.Conn, _ http.Header) {
-					dialer := socks.NewConnDialer(remoteConn)
-					requestHandler := socks.NewRequestHandler(dialer)
-					socksServer := socks.NewConnectionHandler(requestHandler)
-
-					socksServer.Serve(wsConn)
-				}
-			} else if c.IsSet(sshServerFlag) {
-				streamHandler = func(wsConn *websocket.Conn, remoteConn net.Conn, requestHeaders http.Header) {
-					if finalDestination := requestHeaders.Get(h2mux.CFJumpDestinationHeader); finalDestination != "" {
-						token := requestHeaders.Get(h2mux.CFAccessTokenHeader)
-						if err := websocket.SendSSHPreamble(remoteConn, finalDestination, token); err != nil {
-							log.Errorf("Failed to send SSH preamble: %s", err)
-							return
-						}
-					}
-					websocket.DefaultStreamHandler(wsConn, remoteConn, requestHeaders)
-				}
-			}
-			errC <- websocket.StartProxyServer(log, listener, staticHost, shutdownC, streamHandler)
-		}()
-		forceSetFlag(c, "url", "http://"+listener.Addr().String())
-	}
-
-	transportLogger, err := createLogger(c, true, false)
+	transportLogger, err := createLogger(c, true, isUIEnabled)
 	if err != nil {
 		return errors.Wrap(err, "error setting up transport logger")
 	}
 
-	tunnelConfig, err := prepareTunnelConfig(c, buildInfo, version, log, transportLogger, namedTunnel)
+	tunnelConfig, ingressRules, err := prepareTunnelConfig(c, buildInfo, version, generalLogger, transportLogger, namedTunnel, isUIEnabled)
 	if err != nil {
+		generalLogger.Errorf("Couldn't start tunnel: %v", err)
 		return err
 	}
 
+	ingressRules.StartOrigins(&wg, generalLogger, shutdownC, errC)
+
 	reconnectCh := make(chan origin.ReconnectSignal, 1)
 	if c.IsSet("stdin-control") {
-		log.Info("Enabling control through stdin")
-		go stdinControl(reconnectCh, log)
+		generalLogger.Info("Enabling control through stdin")
+		go stdinControl(reconnectCh, generalLogger)
 	}
 
 	wg.Add(1)
@@ -506,25 +388,21 @@ func StartServer(
 	}()
 
 	if isUIEnabled {
-		const tunnelEventChanBufferSize = 16
-		tunnelEventChan := make(chan ui.TunnelEvent, tunnelEventChanBufferSize)
-		tunnelConfig.TunnelEventChan = tunnelEventChan
-
 		tunnelInfo := ui.NewUIModel(
 			version,
 			hostname,
 			metricsListener.Addr().String(),
-			tunnelConfig.OriginUrl,
+			&ingressRules,
 			tunnelConfig.HAConnections,
 		)
 		logLevels, err := logger.ParseLevelString(c.String("loglevel"))
 		if err != nil {
 			return err
 		}
-		tunnelInfo.LaunchUI(ctx, log, logLevels, tunnelEventChan)
+		tunnelInfo.LaunchUI(ctx, generalLogger, transportLogger, logLevels, tunnelConfig.TunnelEventChan)
 	}
 
-	return waitToShutdown(&wg, errC, shutdownC, graceShutdownC, c.Duration("grace-period"), log)
+	return waitToShutdown(&wg, errC, shutdownC, graceShutdownC, c.Duration("grace-period"), generalLogger)
 }
 
 // forceSetFlag attempts to set the given flag value in the closest context that has it defined
@@ -559,11 +437,6 @@ func SetFlagsFromConfigFile(c *cli.Context) error {
 	return nil
 }
 
-// isProxyDestinationConfigured returns true if there is a static host set or if bastion mode is set.
-func isProxyDestinationConfigured(staticHost string, c *cli.Context) bool {
-	return staticHost != "" || c.IsSet(bastionFlag)
-}
-
 func waitToShutdown(wg *sync.WaitGroup,
 	errC chan error,
 	shutdownC, graceShutdownC chan struct{},
@@ -843,6 +716,7 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			Value:  false,
 			Hidden: shouldHide,
 		}),
+		selectProtocolFlag,
 	}...)
 
 	return flags
@@ -910,67 +784,67 @@ func configureProxyFlags(shouldHide bool) []cli.Flag {
 			Hidden:  shouldHide,
 		}),
 		altsrc.NewBoolFlag(&cli.BoolFlag{
-			Name:    socks5Flag,
+			Name:    ingress.Socks5Flag,
 			Usage:   "specify if this tunnel is running as a SOCK5 Server",
 			EnvVars: []string{"TUNNEL_SOCKS"},
 			Value:   false,
 			Hidden:  shouldHide,
 		}),
 		altsrc.NewDurationFlag(&cli.DurationFlag{
-			Name:   "proxy-connect-timeout",
+			Name:   ingress.ProxyConnectTimeoutFlag,
 			Usage:  "HTTP proxy timeout for establishing a new connection",
 			Value:  time.Second * 30,
 			Hidden: shouldHide,
 		}),
 		altsrc.NewDurationFlag(&cli.DurationFlag{
-			Name:   "proxy-tls-timeout",
+			Name:   ingress.ProxyTLSTimeoutFlag,
 			Usage:  "HTTP proxy timeout for completing a TLS handshake",
 			Value:  time.Second * 10,
 			Hidden: shouldHide,
 		}),
 		altsrc.NewDurationFlag(&cli.DurationFlag{
-			Name:   "proxy-tcp-keepalive",
+			Name:   ingress.ProxyTCPKeepAlive,
 			Usage:  "HTTP proxy TCP keepalive duration",
 			Value:  time.Second * 30,
 			Hidden: shouldHide,
 		}),
 		altsrc.NewBoolFlag(&cli.BoolFlag{
-			Name:   "proxy-no-happy-eyeballs",
+			Name:   ingress.ProxyNoHappyEyeballsFlag,
 			Usage:  "HTTP proxy should disable \"happy eyeballs\" for IPv4/v6 fallback",
 			Hidden: shouldHide,
 		}),
 		altsrc.NewIntFlag(&cli.IntFlag{
-			Name:   "proxy-keepalive-connections",
+			Name:   ingress.ProxyKeepAliveConnectionsFlag,
 			Usage:  "HTTP proxy maximum keepalive connection pool size",
 			Value:  100,
 			Hidden: shouldHide,
 		}),
 		altsrc.NewDurationFlag(&cli.DurationFlag{
-			Name:   "proxy-keepalive-timeout",
+			Name:   ingress.ProxyKeepAliveTimeoutFlag,
 			Usage:  "HTTP proxy timeout for closing an idle connection",
 			Value:  time.Second * 90,
 			Hidden: shouldHide,
 		}),
 		altsrc.NewDurationFlag(&cli.DurationFlag{
 			Name:   "proxy-connection-timeout",
-			Usage:  "HTTP proxy timeout for closing an idle connection",
+			Usage:  "DEPRECATED. No longer has any effect.",
 			Value:  time.Second * 90,
 			Hidden: shouldHide,
 		}),
 		altsrc.NewDurationFlag(&cli.DurationFlag{
 			Name:   "proxy-expect-continue-timeout",
-			Usage:  "HTTP proxy timeout for closing an idle connection",
+			Usage:  "DEPRECATED. No longer has any effect.",
 			Value:  time.Second * 90,
 			Hidden: shouldHide,
 		}),
 		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:    "http-host-header",
+			Name:    ingress.HTTPHostHeaderFlag,
 			Usage:   "Sets the HTTP Host header for the local webserver.",
 			EnvVars: []string{"TUNNEL_HTTP_HOST_HEADER"},
 			Hidden:  shouldHide,
 		}),
 		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:    "origin-server-name",
+			Name:    ingress.OriginServerNameFlag,
 			Usage:   "Hostname on the origin server certificate.",
 			EnvVars: []string{"TUNNEL_ORIGIN_SERVER_NAME"},
 			Hidden:  shouldHide,
@@ -988,13 +862,13 @@ func configureProxyFlags(shouldHide bool) []cli.Flag {
 			Hidden:  shouldHide,
 		}),
 		altsrc.NewBoolFlag(&cli.BoolFlag{
-			Name:    "no-tls-verify",
+			Name:    ingress.NoTLSVerifyFlag,
 			Usage:   "Disables TLS verification of the certificate presented by your origin. Will allow any certificate from the origin to be accepted. Note: The connection from your machine to Cloudflare's Edge is still encrypted.",
 			EnvVars: []string{"NO_TLS_VERIFY"},
 			Hidden:  shouldHide,
 		}),
 		altsrc.NewBoolFlag(&cli.BoolFlag{
-			Name:    "no-chunked-encoding",
+			Name:    ingress.NoChunkedEncodingFlag,
 			Usage:   "Disables chunked transfer encoding; useful if you are running a WSGI server.",
 			EnvVars: []string{"TUNNEL_NO_CHUNKED_ENCODING"},
 			Hidden:  shouldHide,
@@ -1067,28 +941,28 @@ func sshFlags(shouldHide bool) []cli.Flag {
 			Hidden:  true,
 		}),
 		altsrc.NewBoolFlag(&cli.BoolFlag{
-			Name:    sshServerFlag,
+			Name:    ingress.SSHServerFlag,
 			Value:   false,
 			Usage:   "Run an SSH Server",
 			EnvVars: []string{"TUNNEL_SSH_SERVER"},
 			Hidden:  true, // TODO: remove when feature is complete
 		}),
 		altsrc.NewBoolFlag(&cli.BoolFlag{
-			Name:    bastionFlag,
+			Name:    config.BastionFlag,
 			Value:   false,
 			Usage:   "Runs as jump host",
 			EnvVars: []string{"TUNNEL_BASTION"},
 			Hidden:  shouldHide,
 		}),
 		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:    "proxy-address",
+			Name:    ingress.ProxyAddressFlag,
 			Usage:   "Listen address for the proxy.",
 			Value:   "127.0.0.1",
 			EnvVars: []string{"TUNNEL_PROXY_ADDRESS"},
 			Hidden:  shouldHide,
 		}),
 		altsrc.NewIntFlag(&cli.IntFlag{
-			Name:    "proxy-port",
+			Name:    ingress.ProxyPortFlag,
 			Usage:   "Listen port for the proxy.",
 			Value:   0,
 			EnvVars: []string{"TUNNEL_PROXY_PORT"},
@@ -1109,7 +983,7 @@ func configureLoggingFlags(shouldHide bool) []cli.Flag {
 		altsrc.NewStringFlag(&cli.StringFlag{
 			Name:    "transport-loglevel",
 			Aliases: []string{"proto-loglevel"}, // This flag used to be called proto-loglevel
-			Value:   "fatal",
+			Value:   "info",
 			Usage:   "Transport logging level(previously called protocol logging level) {fatal, error, info, debug}",
 			EnvVars: []string{"TUNNEL_PROTO_LOGLEVEL", "TUNNEL_TRANSPORT_LOGLEVEL"},
 			Hidden:  shouldHide,

cmd/cloudflared/tunnel/configuration.go

@@ -1,19 +1,19 @@
 package tunnel
 
 import (
-	"context"
 	"crypto/tls"
 	"fmt"
 	"io/ioutil"
-	"net"
-	"net/http"
 	"os"
 	"path/filepath"
 	"strings"
-	"time"
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/ui"
+	"github.com/cloudflare/cloudflared/connection"
+	"github.com/cloudflare/cloudflared/edgediscovery"
+	"github.com/cloudflare/cloudflared/h2mux"
 	"github.com/cloudflare/cloudflared/ingress"
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/origin"
@@ -159,28 +159,29 @@ func prepareTunnelConfig(
 	version string,
 	logger logger.Service,
 	transportLogger logger.Service,
-	namedTunnel *origin.NamedTunnelConfig,
-) (*origin.TunnelConfig, error) {
-	compatibilityMode := namedTunnel == nil
+	namedTunnel *connection.NamedTunnelConfig,
+	uiIsEnabled bool,
+) (*origin.TunnelConfig, ingress.Ingress, error) {
+	isNamedTunnel := namedTunnel != nil
 
 	hostname, err := validation.ValidateHostname(c.String("hostname"))
 	if err != nil {
 		logger.Errorf("Invalid hostname: %s", err)
-		return nil, errors.Wrap(err, "Invalid hostname")
+		return nil, ingress.Ingress{}, errors.Wrap(err, "Invalid hostname")
 	}
 	isFreeTunnel := hostname == ""
 	clientID := c.String("id")
 	if !c.IsSet("id") {
 		clientID, err = generateRandomClientID(logger)
 		if err != nil {
-			return nil, err
+			return nil, ingress.Ingress{}, err
 		}
 	}
 
 	tags, err := NewTagSliceFromCLI(c.StringSlice("tag"))
 	if err != nil {
 		logger.Errorf("Tag parse failure: %s", err)
-		return nil, errors.Wrap(err, "Tag parse failure")
+		return nil, ingress.Ingress{}, errors.Wrap(err, "Tag parse failure")
 	}
 
 	tags = append(tags, tunnelpogs.Tag{Name: "ID", Value: clientID})
@@ -189,41 +190,18 @@ func prepareTunnelConfig(
 	if !isFreeTunnel {
 		originCert, err = getOriginCert(c, logger)
 		if err != nil {
-			return nil, errors.Wrap(err, "Error getting origin cert")
+			return nil, ingress.Ingress{}, errors.Wrap(err, "Error getting origin cert")
 		}
 	}
 
-	originCertPool, err := tlsconfig.LoadOriginCA(c, logger)
-	if err != nil {
-		logger.Errorf("Error loading cert pool: %s", err)
-		return nil, errors.Wrap(err, "Error loading cert pool")
-	}
-
-	tunnelMetrics := origin.NewTunnelMetrics()
-	httpTransport := &http.Transport{
-		Proxy:                 http.ProxyFromEnvironment,
-		MaxIdleConns:          c.Int("proxy-keepalive-connections"),
-		MaxIdleConnsPerHost:   c.Int("proxy-keepalive-connections"),
-		IdleConnTimeout:       c.Duration("proxy-keepalive-timeout"),
-		TLSHandshakeTimeout:   c.Duration("proxy-tls-timeout"),
-		ExpectContinueTimeout: 1 * time.Second,
-		TLSClientConfig:       &tls.Config{RootCAs: originCertPool, InsecureSkipVerify: c.IsSet("no-tls-verify")},
-	}
-
-	dialer := &net.Dialer{
-		Timeout:   c.Duration("proxy-connect-timeout"),
-		KeepAlive: c.Duration("proxy-tcp-keepalive"),
-	}
-	if c.Bool("proxy-no-happy-eyeballs") {
-		dialer.FallbackDelay = -1 // As of Golang 1.12, a negative delay disables "happy eyeballs"
-	}
-	dialContext := dialer.DialContext
-
-	var ingressRules ingress.Ingress
-	if namedTunnel != nil {
+	var (
+		ingressRules  ingress.Ingress
+		classicTunnel *connection.ClassicTunnelConfig
+	)
+	if isNamedTunnel {
 		clientUUID, err := uuid.NewRandom()
 		if err != nil {
-			return nil, errors.Wrap(err, "can't generate clientUUID")
+			return nil, ingress.Ingress{}, errors.Wrap(err, "can't generate clientUUID")
 		}
 		namedTunnel.Client = tunnelpogs.ClientInfo{
 			ClientID: clientUUID[:],
@@ -233,103 +211,84 @@ func prepareTunnelConfig(
 		}
 		ingressRules, err = ingress.ParseIngress(config.GetConfiguration())
 		if err != nil && err != ingress.ErrNoIngressRules {
-			return nil, err
+			return nil, ingress.Ingress{}, err
 		}
 		if !ingressRules.IsEmpty() && c.IsSet("url") {
-			return nil, ingress.ErrURLIncompatibleWithIngress
-		}
-	}
-
-	var originURL string
-	if ingressRules.IsEmpty() {
-		originURL, err = config.ValidateUrl(c, compatibilityMode)
-		if err != nil {
-			logger.Errorf("Error validating origin URL: %s", err)
-			return nil, errors.Wrap(err, "Error validating origin URL")
-		}
-	}
-
-	if c.IsSet("unix-socket") {
-		unixSocket, err := config.ValidateUnixSocket(c)
-		if err != nil {
-			logger.Errorf("Error validating --unix-socket: %s", err)
-			return nil, errors.Wrap(err, "Error validating --unix-socket")
-		}
-
-		logger.Infof("Proxying tunnel requests to unix:%s", unixSocket)
-		httpTransport.DialContext = func(ctx context.Context, _, _ string) (net.Conn, error) {
-			// if --unix-socket specified, enforce network type "unix"
-			return dialContext(ctx, "unix", unixSocket)
+			return nil, ingress.Ingress{}, ingress.ErrURLIncompatibleWithIngress
 		}
 	} else {
-		logger.Infof("Proxying tunnel requests to %s", originURL)
-		httpTransport.DialContext = dialContext
-	}
-
-	if !c.IsSet("hello-world") && c.IsSet("origin-server-name") {
-		httpTransport.TLSClientConfig.ServerName = c.String("origin-server-name")
-	}
-	// If tunnel running in bastion mode, a connection to origin will not exist until initiated by the client.
-	if !c.IsSet(bastionFlag) {
-
-		// List all origin URLs that require validation
-		var originURLs []string
-		if ingressRules.IsEmpty() {
-			originURLs = append(originURLs, originURL)
-		} else {
-			for _, rule := range ingressRules.Rules {
-				originURLs = append(originURLs, rule.Service.String())
-			}
-		}
-
-		// Validate each origin URL
-		for _, u := range originURLs {
-			if err = validation.ValidateHTTPService(u, hostname, httpTransport); err != nil {
-				logger.Errorf("unable to connect to the origin: %s", err)
-			}
+		classicTunnel = &connection.ClassicTunnelConfig{
+			Hostname:   hostname,
+			OriginCert: originCert,
+			// turn off use of reconnect token and auth refresh when using named tunnels
+			UseReconnectToken: !isNamedTunnel && c.Bool("use-reconnect-token"),
 		}
 	}
 
-	toEdgeTLSConfig, err := tlsconfig.CreateTunnelConfig(c)
+	// Convert single-origin configuration into multi-origin configuration.
+	if ingressRules.IsEmpty() {
+		ingressRules, err = ingress.NewSingleOrigin(c, !isNamedTunnel, logger)
+		if err != nil {
+			return nil, ingress.Ingress{}, err
+		}
+	}
+
+	protocolSelector, err := connection.NewProtocolSelector(c.String("protocol"), namedTunnel, edgediscovery.HTTP2Percentage, origin.ResolveTTL, logger)
 	if err != nil {
-		logger.Errorf("unable to create TLS config to connect with edge: %s", err)
-		return nil, errors.Wrap(err, "unable to create TLS config to connect with edge")
+		return nil, ingress.Ingress{}, err
 	}
-	return &origin.TunnelConfig{
-		BuildInfo:          buildInfo,
-		ClientID:           clientID,
-		ClientTlsConfig:    httpTransport.TLSClientConfig,
-		CompressionQuality: c.Uint64("compression-quality"),
-		EdgeAddrs:          c.StringSlice("edge"),
-		GracePeriod:        c.Duration("grace-period"),
-		HAConnections:      c.Int("ha-connections"),
-		HTTPTransport:      httpTransport,
+	logger.Infof("Initial protocol %s", protocolSelector.Current())
+
+	edgeTLSConfigs := make(map[connection.Protocol]*tls.Config, len(connection.ProtocolList))
+	for _, p := range connection.ProtocolList {
+		edgeTLSConfig, err := tlsconfig.CreateTunnelConfig(c, p.ServerName())
+		if err != nil {
+			return nil, ingress.Ingress{}, errors.Wrap(err, "unable to create TLS config to connect with edge")
+		}
+		edgeTLSConfigs[p] = edgeTLSConfig
+	}
+
+	originClient := origin.NewClient(ingressRules, tags, logger)
+	connectionConfig := &connection.Config{
+		OriginClient:    originClient,
+		GracePeriod:     c.Duration("grace-period"),
+		ReplaceExisting: c.Bool("force"),
+	}
+	muxerConfig := &connection.MuxerConfig{
 		HeartbeatInterval:  c.Duration("heartbeat-interval"),
-		Hostname:           hostname,
-		HTTPHostHeader:     c.String("http-host-header"),
-		IncidentLookup:     origin.NewIncidentLookup(),
-		IsAutoupdated:      c.Bool("is-autoupdated"),
-		IsFreeTunnel:       isFreeTunnel,
-		LBPool:             c.String("lb-pool"),
-		Logger:             logger,
-		TransportLogger:    transportLogger,
 		MaxHeartbeats:      c.Uint64("heartbeat-count"),
-		Metrics:            tunnelMetrics,
+		CompressionSetting: h2mux.CompressionSetting(c.Uint64("compression-quality")),
 		MetricsUpdateFreq:  c.Duration("metrics-update-freq"),
-		NoChunkedEncoding:  c.Bool("no-chunked-encoding"),
-		OriginCert:         originCert,
-		OriginUrl:          originURL,
-		ReportedVersion:    version,
-		Retries:            c.Uint("retries"),
-		RunFromTerminal:    isRunningFromTerminal(),
-		Tags:               tags,
-		TlsConfig:          toEdgeTLSConfig,
-		NamedTunnel:        namedTunnel,
-		ReplaceExisting:    c.Bool("force"),
-		IngressRules:       ingressRules,
-		// turn off use of reconnect token and auth refresh when using named tunnels
-		UseReconnectToken: compatibilityMode && c.Bool("use-reconnect-token"),
-	}, nil
+	}
+
+	var tunnelEventChan chan ui.TunnelEvent
+	if uiIsEnabled {
+		tunnelEventChan = make(chan ui.TunnelEvent, 16)
+	}
+
+	return &origin.TunnelConfig{
+		ConnectionConfig: connectionConfig,
+		BuildInfo:        buildInfo,
+		ClientID:         clientID,
+		EdgeAddrs:        c.StringSlice("edge"),
+		HAConnections:    c.Int("ha-connections"),
+		IncidentLookup:   origin.NewIncidentLookup(),
+		IsAutoupdated:    c.Bool("is-autoupdated"),
+		IsFreeTunnel:     isFreeTunnel,
+		LBPool:           c.String("lb-pool"),
+		Tags:             tags,
+		Logger:           logger,
+		Observer:         connection.NewObserver(transportLogger, tunnelEventChan),
+		ReportedVersion:  version,
+		Retries:          c.Uint("retries"),
+		RunFromTerminal:  isRunningFromTerminal(),
+		NamedTunnel:      namedTunnel,
+		ClassicTunnel:    classicTunnel,
+		MuxerConfig:      muxerConfig,
+		TunnelEventChan:  tunnelEventChan,
+		ProtocolSelector: protocolSelector,
+		EdgeTLSConfigs:   edgeTLSConfigs,
+	}, ingressRules, nil
 }
 
 func isRunningFromTerminal() bool {

cmd/cloudflared/tunnel/ingress_subcommands.go

@@ -70,6 +70,10 @@ func buildTestURLCommand() *cli.Command {
 // validateIngressCommand check the syntax of the ingress rules in the cloudflared config file
 func validateIngressCommand(c *cli.Context) error {
 	conf := config.GetConfiguration()
+	if conf.Source() == "" {
+		fmt.Println("No configuration file was found. Please create one, or use the --config flag to specify its filepath. You can use the help command to learn more about configuration files")
+		return nil
+	}
 	fmt.Println("Validating rules from", conf.Source())
 	if _, err := ingress.ParseIngress(conf); err != nil {
 		return errors.Wrap(err, "Validation failed")
@@ -103,7 +107,7 @@ func testURLCommand(c *cli.Context) error {
 		return errors.Wrap(err, "Validation failed")
 	}
 
-	i := ing.FindMatchingRule(requestURL.Hostname(), requestURL.Path)
+	_, i := ing.FindMatchingRule(requestURL.Hostname(), requestURL.Path)
 	fmt.Printf("Matched rule #%d\n", i+1)
 	fmt.Println(ing.Rules[i].MultiLineString())
 	return nil

cmd/cloudflared/tunnel/subcommand_context.go

@@ -14,12 +14,21 @@ import (
 
 	"github.com/cloudflare/cloudflared/certutil"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/logger"
-	"github.com/cloudflare/cloudflared/origin"
 	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	"github.com/cloudflare/cloudflared/tunnelstore"
 )
 
+type errInvalidJSONCredential struct {
+	err  error
+	path string
+}
+
+func (e errInvalidJSONCredential) Error() string {
+	return "Invalid JSON when parsing tunnel credentials file"
+}
+
 // subcommandContext carries structs shared between subcommands, to reduce number of arguments needed to
 // pass between subcommands, and make sure they are only initialized once
 type subcommandContext struct {
@@ -111,7 +120,13 @@ func (sc *subcommandContext) readTunnelCredentials(tunnelID uuid.UUID) (*pogs.Tu
 
 	var auth pogs.TunnelAuth
 	if err = json.Unmarshal(body, &auth); err != nil {
-		return nil, err
+		if strings.HasSuffix(filePath, ".pem") {
+			return nil, fmt.Errorf("The tunnel credentials file should be .json but you gave a .pem. "+
+				"The tunnel credentials file was originally created by `cloudflared tunnel create` and named %s.json."+
+				"You may have accidentally used the filepath to cert.pem, which is generated by `cloudflared tunnel "+
+				"login`.", tunnelID)
+		}
+		return nil, errInvalidJSONCredential{path: filePath, err: err}
 	}
 	return &auth, nil
 }
@@ -244,6 +259,10 @@ func (sc *subcommandContext) delete(tunnelIDs []uuid.UUID) error {
 func (sc *subcommandContext) run(tunnelID uuid.UUID) error {
 	credentials, err := sc.readTunnelCredentials(tunnelID)
 	if err != nil {
+		if e, ok := err.(errInvalidJSONCredential); ok {
+			sc.logger.Errorf("The credentials file at %s contained invalid JSON. This is probably caused by passing the wrong filepath. Reminder: the credentials file is a .json file created via `cloudflared tunnel create`.", e.path)
+			sc.logger.Errorf("Invalid JSON when parsing credentials file: %s", e.err.Error())
+		}
 		return err
 	}
 
@@ -252,7 +271,7 @@ func (sc *subcommandContext) run(tunnelID uuid.UUID) error {
 		version,
 		shutdownC,
 		graceShutdownC,
-		&origin.NamedTunnelConfig{Auth: *credentials, ID: tunnelID},
+		&connection.NamedTunnelConfig{Auth: *credentials, ID: tunnelID},
 		sc.logger,
 		sc.isUIEnabled,
 	)

cmd/cloudflared/tunnel/subcommands.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	"github.com/cloudflare/cloudflared/tunnelstore"
@@ -77,21 +78,31 @@ var (
 		Name:    "credentials-file",
 		Aliases: []string{credFileFlagAlias},
 		Usage:   "File path of tunnel credentials",
+		EnvVars: []string{"TUNNEL_CRED_FILE"},
 	})
 	forceDeleteFlag = &cli.BoolFlag{
 		Name:    "force",
 		Aliases: []string{"f"},
 		Usage:   "Allows you to delete a tunnel, even if it has active connections.",
+		EnvVars: []string{"TUNNEL_RUN_FORCE_OVERWRITE"},
 	}
+	selectProtocolFlag = altsrc.NewStringFlag(&cli.StringFlag{
+		Name:    "protocol",
+		Value:   "h2mux",
+		Aliases: []string{"p"},
+		Usage:   fmt.Sprintf("Protocol implementation to connect with Cloudflare's edge network. %s", connection.AvailableProtocolFlagMessage),
+		EnvVars: []string{"TUNNEL_TRANSPORT_PROTOCOL"},
+		Hidden:  true,
+	})
 )
 
 func buildCreateCommand() *cli.Command {
 	return &cli.Command{
-		Name:               "create",
-		Action:             cliutil.ErrorHandler(createCommand),
-		Usage:              "Create a new tunnel with given name",
-		UsageText:          "cloudflared tunnel [tunnel command options] create [subcommand options] NAME",
-		Description:        `Creates a tunnel, registers it with Cloudflare edge and generates credential file used to run this tunnel.
+		Name:      "create",
+		Action:    cliutil.ErrorHandler(createCommand),
+		Usage:     "Create a new tunnel with given name",
+		UsageText: "cloudflared tunnel [tunnel command options] create [subcommand options] NAME",
+		Description: `Creates a tunnel, registers it with Cloudflare edge and generates credential file used to run this tunnel.
   Use "cloudflared tunnel route" subcommand to map a DNS name to this tunnel and "cloudflared tunnel run" to start the connection. 
 
   For example, to create a tunnel named 'my-tunnel' run:
@@ -309,6 +320,7 @@ func buildRunCommand() *cli.Command {
 	flags := []cli.Flag{
 		forceFlag,
 		credentialsFileFlag,
+		selectProtocolFlag,
 	}
 	flags = append(flags, configureProxyFlags(false)...)
 	return &cli.Command{

cmd/cloudflared/ui/launch_ui.go

@@ -3,8 +3,10 @@ package ui
 import (
 	"context"
 	"fmt"
+	"strings"
 	"time"
 
+	"github.com/cloudflare/cloudflared/ingress"
 	"github.com/cloudflare/cloudflared/logger"
 
 	"github.com/gdamore/tcell"
@@ -34,11 +36,11 @@ type TunnelEvent struct {
 }
 
 type uiModel struct {
-	version     string
-	edgeURL     string
-	metricsURL  string
-	proxyURL    string
-	connections []connState
+	version       string
+	edgeURL       string
+	metricsURL    string
+	localServices []string
+	connections   []connState
 }
 
 type palette struct {
@@ -49,19 +51,23 @@ type palette struct {
 	reconnecting string
 }
 
-func NewUIModel(version, hostname, metricsURL, proxyURL string, haConnections int) *uiModel {
+func NewUIModel(version, hostname, metricsURL string, ing *ingress.Ingress, haConnections int) *uiModel {
+	localServices := make([]string, len(ing.Rules))
+	for i, rule := range ing.Rules {
+		localServices[i] = rule.Service.String()
+	}
 	return &uiModel{
-		version:     version,
-		edgeURL:     hostname,
-		metricsURL:  metricsURL,
-		proxyURL:    proxyURL,
-		connections: make([]connState, haConnections),
+		version:       version,
+		edgeURL:       hostname,
+		metricsURL:    metricsURL,
+		localServices: localServices,
+		connections:   make([]connState, haConnections),
 	}
 }
 
 func (data *uiModel) LaunchUI(
 	ctx context.Context,
-	log logger.Service,
+	generalLogger, transportLogger logger.Service,
 	logLevels []logger.Level,
 	tunnelEventChan <-chan TunnelEvent,
 ) {
@@ -69,7 +75,8 @@ func (data *uiModel) LaunchUI(
 
 	// Add TextView as a group to write output to
 	logTextView := NewDynamicColorTextView()
-	log.Add(logTextView, logger.NewUIFormatter(time.RFC3339), logLevels...)
+	generalLogger.Add(logTextView, logger.NewUIFormatter(time.RFC3339), logLevels...)
+	transportLogger.Add(logTextView, logger.NewUIFormatter(time.RFC3339), logLevels...)
 
 	// Construct the UI
 	palette := palette{
@@ -107,7 +114,8 @@ func (data *uiModel) LaunchUI(
 	tunnelHostText := tview.NewTextView().SetText(data.edgeURL)
 
 	grid.AddItem(tunnelHostText, 0, 1, 1, 1, 0, 0, false)
-	grid.AddItem(NewDynamicColorTextView().SetText(fmt.Sprintf("[%s]\u2022[%s] Proxying to [%s::b]%s", palette.connected, palette.defaultText, palette.url, data.proxyURL)), 1, 1, 1, 1, 0, 0, false)
+	status := fmt.Sprintf("[%s]\u2022[%s] Proxying to [%s::b]%s", palette.connected, palette.defaultText, palette.url, strings.Join(data.localServices, ", "))
+	grid.AddItem(NewDynamicColorTextView().SetText(status), 1, 1, 1, 1, 0, 0, false)
 
 	grid.AddItem(connTable, 2, 1, 1, 1, 0, 0, false)
 
@@ -133,7 +141,7 @@ func (data *uiModel) LaunchUI(
 				case Connected:
 					data.setConnTableCell(event, connTable, palette)
 				case Disconnected, Reconnecting:
-					data.changeConnStatus(event, connTable, log, palette)
+					data.changeConnStatus(event, connTable, generalLogger, palette)
 				case SetUrl:
 					tunnelHostText.SetText(event.Url)
 					data.edgeURL = event.Url
@@ -149,7 +157,7 @@ func (data *uiModel) LaunchUI(
 
 	go func() {
 		if err := app.SetRoot(frame, true).Run(); err != nil {
-			log.Errorf("Error launching UI: %s", err)
+			generalLogger.Errorf("Error launching UI: %s", err)
 		}
 	}()
 }

cmd/cloudflared/updater/update.go

@@ -202,7 +202,6 @@ func (a *AutoUpdater) Run(ctx context.Context) error {
 		if a.configurable.enabled {
 			updateOutcome := loggedUpdate(a.logger, updateOptions{})
 			if updateOutcome.Updated {
-				os.Args = append(os.Args, "--is-autoupdated=true")
 				if IsSysV() {
 					// SysV doesn't have a mechanism to keep service alive, we have to restart the process
 					a.logger.Info("Restarting service managed by SysV...")

cmd/cloudflared/updater/workers_service.go

@@ -7,7 +7,6 @@ import (
 	"runtime"
 	"strconv"
 	"strings"
-	"time"
 )
 
 // Options are the update options supported by the
@@ -53,7 +52,7 @@ func NewWorkersService(currentVersion, url, targetPath string, opts Options) Ser
 // Check does a check in with the Workers API to get a new version update
 func (s *WorkersService) Check() (Version, error) {
 	client := &http.Client{
-		Timeout: time.Second * 5,
+		Timeout: clientTimeout,
 	}
 
 	req, err := http.NewRequest(http.MethodGet, s.url, nil)

cmd/cloudflared/updater/workers_update.go

@@ -18,20 +18,23 @@ import (
 	"time"
 )
 
-// stop the service
-// rename cloudflared.exe to cloudflared.exe.old
-// rename cloudflared.exe.new to cloudflared.exe
-// delete cloudflared.exe.old
-// start the service
-// delete the batch file
-const windowsUpdateCommandTemplate = `@echo off
+const (
+	clientTimeout = time.Second * 60
+	// stop the service
+	// rename cloudflared.exe to cloudflared.exe.old
+	// rename cloudflared.exe.new to cloudflared.exe
+	// delete cloudflared.exe.old
+	// start the service
+	// delete the batch file
+	windowsUpdateCommandTemplate = `@echo off
 sc stop cloudflared >nul 2>&1
 rename "{{.TargetPath}}" {{.OldName}}
 rename "{{.NewPath}}" {{.BinaryName}}
 del "{{.OldPath}}"
 sc start cloudflared >nul 2>&1
 del {{.BatchName}}`
-const batchFileName = "cfd_update.bat"
+	batchFileName = "cfd_update.bat"
+)
 
 // Prepare some data to insert into the template.
 type batchData struct {
@@ -118,7 +121,7 @@ func (v *WorkersVersion) String() string {
 // download the file from the link in the json
 func download(url, filepath string, isCompressed bool) error {
 	client := &http.Client{
-		Timeout: time.Second * 5,
+		Timeout: clientTimeout,
 	}
 	resp, err := client.Get(url)
 

connection/connection.go

@@ -0,0 +1,62 @@
+package connection
+
+import (
+	"io"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/google/uuid"
+)
+
+const (
+	lbProbeUserAgentPrefix = "Mozilla/5.0 (compatible; Cloudflare-Traffic-Manager/1.0; +https://www.cloudflare.com/traffic-manager/;"
+)
+
+type Config struct {
+	OriginClient    OriginClient
+	GracePeriod     time.Duration
+	ReplaceExisting bool
+}
+
+type NamedTunnelConfig struct {
+	Auth   pogs.TunnelAuth
+	ID     uuid.UUID
+	Client pogs.ClientInfo
+}
+
+type ClassicTunnelConfig struct {
+	Hostname   string
+	OriginCert []byte
+	// feature-flag to use new edge reconnect tokens
+	UseReconnectToken bool
+}
+
+func (c *ClassicTunnelConfig) IsTrialZone() bool {
+	return c.Hostname == ""
+}
+
+type OriginClient interface {
+	Proxy(w ResponseWriter, req *http.Request, isWebsocket bool) error
+}
+
+type ResponseWriter interface {
+	WriteRespHeaders(*http.Response) error
+	WriteErrorResponse()
+	io.ReadWriter
+}
+
+type ConnectedFuse interface {
+	Connected()
+	IsConnected() bool
+}
+
+func uint8ToString(input uint8) string {
+	return strconv.FormatUint(uint64(input), 10)
+}
+
+func isServerSentEvent(headers http.Header) bool {
+	return strings.ToLower(headers.Get("content-type")) == "text/event-stream"
+}

connection/connection_test.go

@@ -0,0 +1,113 @@
+package connection
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"time"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/ui"
+	"github.com/cloudflare/cloudflared/logger"
+	"github.com/gobwas/ws/wsutil"
+)
+
+const (
+	largeFileSize = 2 * 1024 * 1024
+)
+
+var (
+	testConfig = &Config{
+		OriginClient: &mockOriginClient{},
+		GracePeriod:  time.Millisecond * 100,
+	}
+	testLogger, _ = logger.New()
+	testOriginURL = &url.URL{
+		Scheme: "https",
+		Host:   "connectiontest.argotunnel.com",
+	}
+	testTunnelEventChan = make(chan ui.TunnelEvent)
+	testObserver        = &Observer{
+		testLogger,
+		m,
+		testTunnelEventChan,
+	}
+	testLargeResp = make([]byte, largeFileSize)
+)
+
+type testRequest struct {
+	name           string
+	endpoint       string
+	expectedStatus int
+	expectedBody   []byte
+	isProxyError   bool
+}
+
+type mockOriginClient struct {
+}
+
+func (moc *mockOriginClient) Proxy(w ResponseWriter, r *http.Request, isWebsocket bool) error {
+	if isWebsocket {
+		return wsEndpoint(w, r)
+	}
+	switch r.URL.Path {
+	case "/ok":
+		originRespEndpoint(w, http.StatusOK, []byte(http.StatusText(http.StatusOK)))
+	case "/large_file":
+		originRespEndpoint(w, http.StatusOK, testLargeResp)
+	case "/400":
+		originRespEndpoint(w, http.StatusBadRequest, []byte(http.StatusText(http.StatusBadRequest)))
+	case "/500":
+		originRespEndpoint(w, http.StatusInternalServerError, []byte(http.StatusText(http.StatusInternalServerError)))
+	case "/error":
+		return fmt.Errorf("Failed to proxy to origin")
+	default:
+		originRespEndpoint(w, http.StatusNotFound, []byte("page not found"))
+	}
+	return nil
+}
+
+type nowriter struct {
+	io.Reader
+}
+
+func (nowriter) Write(p []byte) (int, error) {
+	return 0, fmt.Errorf("Writer not implemented")
+}
+
+func wsEndpoint(w ResponseWriter, r *http.Request) error {
+	resp := &http.Response{
+		StatusCode: http.StatusSwitchingProtocols,
+	}
+	w.WriteRespHeaders(resp)
+	clientReader := nowriter{r.Body}
+	go func() {
+		for {
+			data, err := wsutil.ReadClientText(clientReader)
+			if err != nil {
+				return
+			}
+			if err := wsutil.WriteServerText(w, data); err != nil {
+				return
+			}
+		}
+	}()
+	<-r.Context().Done()
+	return nil
+}
+
+func originRespEndpoint(w ResponseWriter, status int, data []byte) {
+	resp := &http.Response{
+		StatusCode: status,
+	}
+	w.WriteRespHeaders(resp)
+	w.Write(data)
+}
+
+type mockConnectedFuse struct{}
+
+func (mcf mockConnectedFuse) Connected() {}
+
+func (mcf mockConnectedFuse) IsConnected() bool {
+	return true
+}

connection/errors.go

@@ -0,0 +1,76 @@
+package connection
+
+import (
+	"github.com/cloudflare/cloudflared/edgediscovery"
+	"github.com/cloudflare/cloudflared/h2mux"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const (
+	DuplicateConnectionError = "EDUPCONN"
+)
+
+// RegisterTunnel error from client
+type clientRegisterTunnelError struct {
+	cause error
+}
+
+func newRPCError(cause error, counter *prometheus.CounterVec, name rpcName) clientRegisterTunnelError {
+	counter.WithLabelValues(cause.Error(), string(name)).Inc()
+	return clientRegisterTunnelError{cause: cause}
+}
+
+func (e clientRegisterTunnelError) Error() string {
+	return e.cause.Error()
+}
+
+type DupConnRegisterTunnelError struct{}
+
+var errDuplicationConnection = &DupConnRegisterTunnelError{}
+
+func (e DupConnRegisterTunnelError) Error() string {
+	return "already connected to this server, trying another address"
+}
+
+// RegisterTunnel error from server
+type serverRegisterTunnelError struct {
+	cause     error
+	permanent bool
+}
+
+func (e serverRegisterTunnelError) Error() string {
+	return e.cause.Error()
+}
+
+func serverRegistrationErrorFromRPC(err error) *serverRegisterTunnelError {
+	if retryable, ok := err.(*tunnelpogs.RetryableError); ok {
+		return &serverRegisterTunnelError{
+			cause:     retryable.Unwrap(),
+			permanent: false,
+		}
+	}
+	return &serverRegisterTunnelError{
+		cause:     err,
+		permanent: true,
+	}
+}
+
+type muxerShutdownError struct{}
+
+func (e muxerShutdownError) Error() string {
+	return "muxer shutdown"
+}
+
+func isHandshakeErrRecoverable(err error, connIndex uint8, observer *Observer) bool {
+	switch err.(type) {
+	case edgediscovery.DialError:
+		observer.Errorf("Connection %d unable to dial edge: %s", connIndex, err)
+	case h2mux.MuxerHandshakeError:
+		observer.Errorf("Connection %d handshake with edge server failed: %s", connIndex, err)
+	default:
+		observer.Errorf("Connection %d failed: %s", connIndex, err)
+		return false
+	}
+	return true
+}

connection/h2mux.go

@@ -0,0 +1,220 @@
+package connection
+
+import (
+	"context"
+	"net"
+	"net/http"
+	"time"
+
+	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/logger"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/cloudflare/cloudflared/websocket"
+	"github.com/pkg/errors"
+	"golang.org/x/sync/errgroup"
+)
+
+const (
+	muxerTimeout      = 5 * time.Second
+	openStreamTimeout = 30 * time.Second
+)
+
+type h2muxConnection struct {
+	config      *Config
+	muxerConfig *MuxerConfig
+	muxer       *h2mux.Muxer
+	// connectionID is only used by metrics, and prometheus requires labels to be string
+	connIndexStr string
+	connIndex    uint8
+
+	observer *Observer
+}
+
+type MuxerConfig struct {
+	HeartbeatInterval  time.Duration
+	MaxHeartbeats      uint64
+	CompressionSetting h2mux.CompressionSetting
+	MetricsUpdateFreq  time.Duration
+}
+
+func (mc *MuxerConfig) H2MuxerConfig(h h2mux.MuxedStreamHandler, logger logger.Service) *h2mux.MuxerConfig {
+	return &h2mux.MuxerConfig{
+		Timeout:            muxerTimeout,
+		Handler:            h,
+		IsClient:           true,
+		HeartbeatInterval:  mc.HeartbeatInterval,
+		MaxHeartbeats:      mc.MaxHeartbeats,
+		Logger:             logger,
+		CompressionQuality: mc.CompressionSetting,
+	}
+}
+
+// NewTunnelHandler returns a TunnelHandler, origin LAN IP and error
+func NewH2muxConnection(ctx context.Context,
+	config *Config,
+	muxerConfig *MuxerConfig,
+	edgeConn net.Conn,
+	connIndex uint8,
+	observer *Observer,
+) (*h2muxConnection, error, bool) {
+	h := &h2muxConnection{
+		config:       config,
+		muxerConfig:  muxerConfig,
+		connIndexStr: uint8ToString(connIndex),
+		connIndex:    connIndex,
+		observer:     observer,
+	}
+
+	// Establish a muxed connection with the edge
+	// Client mux handshake with agent server
+	muxer, err := h2mux.Handshake(edgeConn, edgeConn, *muxerConfig.H2MuxerConfig(h, observer), h2mux.ActiveStreams)
+	if err != nil {
+		recoverable := isHandshakeErrRecoverable(err, connIndex, observer)
+		return nil, err, recoverable
+	}
+	h.muxer = muxer
+	return h, nil, false
+}
+
+func (h *h2muxConnection) ServeNamedTunnel(ctx context.Context, namedTunnel *NamedTunnelConfig, credentialManager CredentialManager, connOptions *tunnelpogs.ConnectionOptions, connectedFuse ConnectedFuse) error {
+	errGroup, serveCtx := errgroup.WithContext(ctx)
+	errGroup.Go(func() error {
+		return h.serveMuxer(serveCtx)
+	})
+
+	errGroup.Go(func() error {
+		stream, err := h.newRPCStream(serveCtx, register)
+		if err != nil {
+			return err
+		}
+		rpcClient := newRegistrationRPCClient(ctx, stream, h.observer)
+		defer rpcClient.Close()
+
+		if err = rpcClient.RegisterConnection(serveCtx, namedTunnel, connOptions, h.connIndex, h.observer); err != nil {
+			return err
+		}
+		connectedFuse.Connected()
+		return nil
+	})
+
+	errGroup.Go(func() error {
+		h.controlLoop(serveCtx, connectedFuse, true)
+		return nil
+	})
+	return errGroup.Wait()
+}
+
+func (h *h2muxConnection) ServeClassicTunnel(ctx context.Context, classicTunnel *ClassicTunnelConfig, credentialManager CredentialManager, registrationOptions *tunnelpogs.RegistrationOptions, connectedFuse ConnectedFuse) error {
+	errGroup, serveCtx := errgroup.WithContext(ctx)
+	errGroup.Go(func() error {
+		return h.serveMuxer(serveCtx)
+	})
+
+	errGroup.Go(func() (err error) {
+		defer func() {
+			if err == nil {
+				connectedFuse.Connected()
+			}
+		}()
+		if classicTunnel.UseReconnectToken && connectedFuse.IsConnected() {
+			err := h.reconnectTunnel(ctx, credentialManager, classicTunnel, registrationOptions)
+			if err == nil {
+				return nil
+			}
+			// log errors and proceed to RegisterTunnel
+			h.observer.Errorf("Couldn't reconnect connection %d. Reregistering it instead. Error was: %v", h.connIndex, err)
+		}
+		return h.registerTunnel(ctx, credentialManager, classicTunnel, registrationOptions)
+	})
+
+	errGroup.Go(func() error {
+		h.controlLoop(serveCtx, connectedFuse, false)
+		return nil
+	})
+	return errGroup.Wait()
+}
+
+func (h *h2muxConnection) serveMuxer(ctx context.Context) error {
+	// All routines should stop when muxer finish serving. When muxer is shutdown
+	// gracefully, it doesn't return an error, so we need to return errMuxerShutdown
+	// here to notify other routines to stop
+	err := h.muxer.Serve(ctx)
+	if err == nil {
+		return muxerShutdownError{}
+	}
+	return err
+}
+
+func (h *h2muxConnection) controlLoop(ctx context.Context, connectedFuse ConnectedFuse, isNamedTunnel bool) {
+	updateMetricsTickC := time.Tick(h.muxerConfig.MetricsUpdateFreq)
+	for {
+		select {
+		case <-ctx.Done():
+			// UnregisterTunnel blocks until the RPC call returns
+			if connectedFuse.IsConnected() {
+				h.unregister(isNamedTunnel)
+			}
+			h.muxer.Shutdown()
+			return
+		case <-updateMetricsTickC:
+			h.observer.metrics.updateMuxerMetrics(h.connIndexStr, h.muxer.Metrics())
+		}
+	}
+}
+
+func (h *h2muxConnection) newRPCStream(ctx context.Context, rpcName rpcName) (*h2mux.MuxedStream, error) {
+	openStreamCtx, openStreamCancel := context.WithTimeout(ctx, openStreamTimeout)
+	defer openStreamCancel()
+	stream, err := h.muxer.OpenRPCStream(openStreamCtx)
+	if err != nil {
+		return nil, err
+	}
+	return stream, nil
+}
+
+func (h *h2muxConnection) ServeStream(stream *h2mux.MuxedStream) error {
+	respWriter := &h2muxRespWriter{stream}
+
+	req, reqErr := h.newRequest(stream)
+	if reqErr != nil {
+		respWriter.WriteErrorResponse()
+		return reqErr
+	}
+
+	err := h.config.OriginClient.Proxy(respWriter, req, websocket.IsWebSocketUpgrade(req))
+	if err != nil {
+		respWriter.WriteErrorResponse()
+		return err
+	}
+	return nil
+}
+
+func (h *h2muxConnection) newRequest(stream *h2mux.MuxedStream) (*http.Request, error) {
+	req, err := http.NewRequest("GET", "http://localhost:8080", h2mux.MuxedStreamReader{MuxedStream: stream})
+	if err != nil {
+		return nil, errors.Wrap(err, "Unexpected error from http.NewRequest")
+	}
+	err = h2mux.H2RequestHeadersToH1Request(stream.Headers, req)
+	if err != nil {
+		return nil, errors.Wrap(err, "invalid request received")
+	}
+	return req, nil
+}
+
+type h2muxRespWriter struct {
+	*h2mux.MuxedStream
+}
+
+func (rp *h2muxRespWriter) WriteRespHeaders(resp *http.Response) error {
+	headers := h2mux.H1ResponseToH2ResponseHeaders(resp)
+	headers = append(headers, h2mux.Header{Name: responseMetaHeaderField, Value: responseMetaHeaderOrigin})
+	return rp.WriteHeaders(headers)
+}
+
+func (rp *h2muxRespWriter) WriteErrorResponse() {
+	rp.WriteHeaders([]h2mux.Header{
+		{Name: ":status", Value: "502"},
+		{Name: responseMetaHeaderField, Value: responseMetaHeaderCfd},
+	})
+	rp.Write([]byte("502 Bad Gateway"))
+}

connection/h2mux_test.go

@@ -0,0 +1,242 @@
+package connection
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"strconv"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/gobwas/ws/wsutil"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+var (
+	testMuxerConfig = &MuxerConfig{
+		HeartbeatInterval:  time.Second * 5,
+		MaxHeartbeats:      5,
+		CompressionSetting: 0,
+		MetricsUpdateFreq:  time.Second * 5,
+	}
+)
+
+func newH2MuxConnection(ctx context.Context, t require.TestingT) (*h2muxConnection, *h2mux.Muxer) {
+	edgeConn, originConn := net.Pipe()
+	edgeMuxChan := make(chan *h2mux.Muxer)
+	go func() {
+		edgeMuxConfig := h2mux.MuxerConfig{
+			Logger: testObserver,
+		}
+		edgeMux, err := h2mux.Handshake(edgeConn, edgeConn, edgeMuxConfig, h2mux.ActiveStreams)
+		require.NoError(t, err)
+		edgeMuxChan <- edgeMux
+	}()
+	var connIndex = uint8(0)
+	h2muxConn, err, _ := NewH2muxConnection(ctx, testConfig, testMuxerConfig, originConn, connIndex, testObserver)
+	require.NoError(t, err)
+	return h2muxConn, <-edgeMuxChan
+}
+
+func TestServeStreamHTTP(t *testing.T) {
+	tests := []testRequest{
+		{
+			name:           "ok",
+			endpoint:       "/ok",
+			expectedStatus: http.StatusOK,
+			expectedBody:   []byte(http.StatusText(http.StatusOK)),
+		},
+		{
+			name:           "large_file",
+			endpoint:       "/large_file",
+			expectedStatus: http.StatusOK,
+			expectedBody:   testLargeResp,
+		},
+		{
+			name:           "Bad request",
+			endpoint:       "/400",
+			expectedStatus: http.StatusBadRequest,
+			expectedBody:   []byte(http.StatusText(http.StatusBadRequest)),
+		},
+		{
+			name:           "Internal server error",
+			endpoint:       "/500",
+			expectedStatus: http.StatusInternalServerError,
+			expectedBody:   []byte(http.StatusText(http.StatusInternalServerError)),
+		},
+		{
+			name:           "Proxy error",
+			endpoint:       "/error",
+			expectedStatus: http.StatusBadGateway,
+			expectedBody:   nil,
+			isProxyError:   true,
+		},
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	h2muxConn, edgeMux := newH2MuxConnection(ctx, t)
+
+	var wg sync.WaitGroup
+	wg.Add(2)
+	go func() {
+		defer wg.Done()
+		edgeMux.Serve(ctx)
+	}()
+	go func() {
+		defer wg.Done()
+		err := h2muxConn.serveMuxer(ctx)
+		require.Error(t, err)
+	}()
+
+	for _, test := range tests {
+		headers := []h2mux.Header{
+			{
+				Name:  ":path",
+				Value: test.endpoint,
+			},
+		}
+		stream, err := edgeMux.OpenStream(ctx, headers, nil)
+		require.NoError(t, err)
+		require.True(t, hasHeader(stream, ":status", strconv.Itoa(test.expectedStatus)))
+
+		if test.isProxyError {
+			assert.True(t, hasHeader(stream, responseMetaHeaderField, responseMetaHeaderCfd))
+		} else {
+			assert.True(t, hasHeader(stream, responseMetaHeaderField, responseMetaHeaderOrigin))
+			body := make([]byte, len(test.expectedBody))
+			_, err = stream.Read(body)
+			require.NoError(t, err)
+			require.Equal(t, test.expectedBody, body)
+		}
+	}
+	cancel()
+	wg.Wait()
+}
+
+func TestServeStreamWS(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	h2muxConn, edgeMux := newH2MuxConnection(ctx, t)
+
+	var wg sync.WaitGroup
+	wg.Add(2)
+	go func() {
+		defer wg.Done()
+		edgeMux.Serve(ctx)
+	}()
+	go func() {
+		defer wg.Done()
+		err := h2muxConn.serveMuxer(ctx)
+		require.Error(t, err)
+	}()
+
+	headers := []h2mux.Header{
+		{
+			Name:  ":path",
+			Value: "/ws",
+		},
+		{
+			Name:  "connection",
+			Value: "upgrade",
+		},
+		{
+			Name:  "upgrade",
+			Value: "websocket",
+		},
+	}
+
+	readPipe, writePipe := io.Pipe()
+	stream, err := edgeMux.OpenStream(ctx, headers, readPipe)
+	require.NoError(t, err)
+
+	require.True(t, hasHeader(stream, ":status", strconv.Itoa(http.StatusSwitchingProtocols)))
+	assert.True(t, hasHeader(stream, responseMetaHeaderField, responseMetaHeaderOrigin))
+
+	data := []byte("test websocket")
+	err = wsutil.WriteClientText(writePipe, data)
+	require.NoError(t, err)
+
+	respBody, err := wsutil.ReadServerText(stream)
+	require.NoError(t, err)
+	require.Equal(t, data, respBody, fmt.Sprintf("Expect %s, got %s", string(data), string(respBody)))
+
+	cancel()
+	wg.Wait()
+}
+
+func hasHeader(stream *h2mux.MuxedStream, name, val string) bool {
+	for _, header := range stream.Headers {
+		if header.Name == name && header.Value == val {
+			return true
+		}
+	}
+	return false
+}
+
+func benchmarkServeStreamHTTPSimple(b *testing.B, test testRequest) {
+	ctx, cancel := context.WithCancel(context.Background())
+	h2muxConn, edgeMux := newH2MuxConnection(ctx, b)
+
+	var wg sync.WaitGroup
+	wg.Add(2)
+	go func() {
+		defer wg.Done()
+		edgeMux.Serve(ctx)
+	}()
+	go func() {
+		defer wg.Done()
+		err := h2muxConn.serveMuxer(ctx)
+		require.Error(b, err)
+	}()
+
+	headers := []h2mux.Header{
+		{
+			Name:  ":path",
+			Value: test.endpoint,
+		},
+	}
+
+	body := make([]byte, len(test.expectedBody))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		b.StartTimer()
+		stream, openstreamErr := edgeMux.OpenStream(ctx, headers, nil)
+		_, readBodyErr := stream.Read(body)
+		b.StopTimer()
+
+		require.NoError(b, openstreamErr)
+		assert.True(b, hasHeader(stream, responseMetaHeaderField, responseMetaHeaderOrigin))
+		require.True(b, hasHeader(stream, ":status", strconv.Itoa(http.StatusOK)))
+		require.NoError(b, readBodyErr)
+		require.Equal(b, test.expectedBody, body)
+	}
+
+	cancel()
+	wg.Wait()
+}
+
+func BenchmarkServeStreamHTTPSimple(b *testing.B) {
+	test := testRequest{
+		name:           "ok",
+		endpoint:       "/ok",
+		expectedStatus: http.StatusOK,
+		expectedBody:   []byte(http.StatusText(http.StatusOK)),
+	}
+
+	benchmarkServeStreamHTTPSimple(b, test)
+}
+
+func BenchmarkServeStreamHTTPLargeFile(b *testing.B) {
+	test := testRequest{
+		name:           "large_file",
+		endpoint:       "/large_file",
+		expectedStatus: http.StatusOK,
+		expectedBody:   testLargeResp,
+	}
+
+	benchmarkServeStreamHTTPSimple(b, test)
+}

connection/header.go

@@ -0,0 +1,31 @@
+package connection
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/cloudflare/cloudflared/h2mux"
+)
+
+const (
+	responseMetaHeaderField = "cf-cloudflared-response-meta"
+)
+
+var (
+	canonicalResponseUserHeadersField = http.CanonicalHeaderKey(h2mux.ResponseUserHeadersField)
+	canonicalResponseMetaHeaderField  = http.CanonicalHeaderKey(responseMetaHeaderField)
+	responseMetaHeaderCfd             = mustInitRespMetaHeader("cloudflared")
+	responseMetaHeaderOrigin          = mustInitRespMetaHeader("origin")
+)
+
+type responseMetaHeader struct {
+	Source string `json:"src"`
+}
+
+func mustInitRespMetaHeader(src string) string {
+	header, err := json.Marshal(responseMetaHeader{Source: src})
+	if err != nil {
+		panic(fmt.Sprintf("Failed to serialize response meta header = %s, err: %v", src, err))
+	}
+	return string(header)
+}

connection/http2.go

@@ -0,0 +1,221 @@
+package connection
+
+import (
+	"context"
+	"errors"
+	"io"
+	"math"
+	"net"
+	"net/http"
+	"strings"
+	"sync"
+
+	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/logger"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+
+	"golang.org/x/net/http2"
+)
+
+const (
+	internalUpgradeHeader = "Cf-Cloudflared-Proxy-Connection-Upgrade"
+	websocketUpgrade      = "websocket"
+	controlStreamUpgrade  = "control-stream"
+)
+
+var (
+	errNotFlusher = errors.New("ResponseWriter doesn't implement http.Flusher")
+)
+
+type http2Connection struct {
+	conn         net.Conn
+	server       *http2.Server
+	config       *Config
+	namedTunnel  *NamedTunnelConfig
+	connOptions  *tunnelpogs.ConnectionOptions
+	observer     *Observer
+	connIndexStr string
+	connIndex    uint8
+	wg           *sync.WaitGroup
+	// newRPCClientFunc allows us to mock RPCs during testing
+	newRPCClientFunc func(context.Context, io.ReadWriteCloser, logger.Service) NamedTunnelRPCClient
+	connectedFuse    ConnectedFuse
+}
+
+func NewHTTP2Connection(
+	conn net.Conn,
+	config *Config,
+	namedTunnelConfig *NamedTunnelConfig,
+	connOptions *tunnelpogs.ConnectionOptions,
+	observer *Observer,
+	connIndex uint8,
+	connectedFuse ConnectedFuse,
+) *http2Connection {
+	return &http2Connection{
+		conn: conn,
+		server: &http2.Server{
+			MaxConcurrentStreams: math.MaxUint32,
+		},
+		config:           config,
+		namedTunnel:      namedTunnelConfig,
+		connOptions:      connOptions,
+		observer:         observer,
+		connIndexStr:     uint8ToString(connIndex),
+		connIndex:        connIndex,
+		wg:               &sync.WaitGroup{},
+		newRPCClientFunc: newRegistrationRPCClient,
+		connectedFuse:    connectedFuse,
+	}
+}
+
+func (c *http2Connection) Serve(ctx context.Context) {
+	go func() {
+		<-ctx.Done()
+		c.close()
+	}()
+	c.server.ServeConn(c.conn, &http2.ServeConnOpts{
+		Context: ctx,
+		Handler: c,
+	})
+}
+
+func (c *http2Connection) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	c.wg.Add(1)
+	defer c.wg.Done()
+
+	respWriter := &http2RespWriter{
+		r: r.Body,
+		w: w,
+	}
+	flusher, isFlusher := w.(http.Flusher)
+	if !isFlusher {
+		c.observer.Errorf("%T doesn't implement http.Flusher", w)
+		respWriter.WriteErrorResponse()
+		return
+	}
+	respWriter.flusher = flusher
+	var err error
+	if isControlStreamUpgrade(r) {
+		respWriter.shouldFlush = true
+		err = c.serveControlStream(r.Context(), respWriter)
+	} else if isWebsocketUpgrade(r) {
+		respWriter.shouldFlush = true
+		stripWebsocketUpgradeHeader(r)
+		err = c.config.OriginClient.Proxy(respWriter, r, true)
+	} else {
+		err = c.config.OriginClient.Proxy(respWriter, r, false)
+	}
+
+	if err != nil {
+		respWriter.WriteErrorResponse()
+	}
+}
+
+func (c *http2Connection) serveControlStream(ctx context.Context, respWriter *http2RespWriter) error {
+	rpcClient := c.newRPCClientFunc(ctx, respWriter, c.observer)
+	defer rpcClient.Close()
+
+	if err := rpcClient.RegisterConnection(ctx, c.namedTunnel, c.connOptions, c.connIndex, c.observer); err != nil {
+		return err
+	}
+	c.connectedFuse.Connected()
+
+	<-ctx.Done()
+	rpcClient.GracefulShutdown(ctx, c.config.GracePeriod)
+	return nil
+}
+
+func (c *http2Connection) close() {
+	// Wait for all serve HTTP handlers to return
+	c.wg.Wait()
+	c.conn.Close()
+}
+
+type http2RespWriter struct {
+	r           io.Reader
+	w           http.ResponseWriter
+	flusher     http.Flusher
+	shouldFlush bool
+}
+
+func (rp *http2RespWriter) WriteRespHeaders(resp *http.Response) error {
+	dest := rp.w.Header()
+	userHeaders := make(http.Header, len(resp.Header))
+	for header, values := range resp.Header {
+		// Since these are http2 headers, they're required to be lowercase
+		h2name := strings.ToLower(header)
+		for _, v := range values {
+			if h2name == "content-length" {
+				// This header has meaning in HTTP/2 and will be used by the edge,
+				// so it should be sent as an HTTP/2 response header.
+				dest.Add(h2name, v)
+				// Since these are http2 headers, they're required to be lowercase
+			} else if !h2mux.IsControlHeader(h2name) || h2mux.IsWebsocketClientHeader(h2name) {
+				// User headers, on the other hand, must all be serialized so that
+				// HTTP/2 header validation won't be applied to HTTP/1 header values
+				userHeaders.Add(h2name, v)
+			}
+		}
+	}
+
+	// Perform user header serialization and set them in the single header
+	dest.Set(canonicalResponseUserHeadersField, h2mux.SerializeHeaders(userHeaders))
+	rp.setResponseMetaHeader(responseMetaHeaderOrigin)
+	status := resp.StatusCode
+	// HTTP2 removes support for 101 Switching Protocols https://tools.ietf.org/html/rfc7540#section-8.1.1
+	if status == http.StatusSwitchingProtocols {
+		status = http.StatusOK
+	}
+	rp.w.WriteHeader(status)
+	if isServerSentEvent(resp.Header) {
+		rp.shouldFlush = true
+	}
+	if rp.shouldFlush {
+		rp.flusher.Flush()
+	}
+	return nil
+}
+
+func (rp *http2RespWriter) WriteErrorResponse() {
+	rp.setResponseMetaHeader(responseMetaHeaderCfd)
+	rp.w.WriteHeader(http.StatusBadGateway)
+}
+
+func (rp *http2RespWriter) setResponseMetaHeader(value string) {
+	rp.w.Header().Set(canonicalResponseMetaHeaderField, value)
+}
+
+func (rp *http2RespWriter) Read(p []byte) (n int, err error) {
+	return rp.r.Read(p)
+}
+
+func (rp *http2RespWriter) Write(p []byte) (n int, err error) {
+	defer func() {
+		// Implementer of OriginClient should make sure it doesn't write to the connection after Proxy returns
+		// Register a recover routine just in case.
+		if r := recover(); r != nil {
+			println("Recover from http2 response writer panic, error", r)
+		}
+	}()
+	n, err = rp.w.Write(p)
+	if err == nil && rp.shouldFlush {
+		rp.flusher.Flush()
+	}
+	return n, err
+}
+
+func (rp *http2RespWriter) Close() error {
+	return nil
+}
+
+func isControlStreamUpgrade(r *http.Request) bool {
+	return strings.ToLower(r.Header.Get(internalUpgradeHeader)) == controlStreamUpgrade
+}
+
+func isWebsocketUpgrade(r *http.Request) bool {
+	return strings.ToLower(r.Header.Get(internalUpgradeHeader)) == websocketUpgrade
+}
+
+func stripWebsocketUpgradeHeader(r *http.Request) {
+	r.Header.Del(internalUpgradeHeader)
+}

connection/http2_test.go

@@ -0,0 +1,303 @@
+package connection
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/cloudflare/cloudflared/logger"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/gobwas/ws/wsutil"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/net/http2"
+)
+
+var (
+	testTransport = http2.Transport{}
+)
+
+func newTestHTTP2Connection() (*http2Connection, net.Conn) {
+	edgeConn, originConn := net.Pipe()
+	var connIndex = uint8(0)
+	return NewHTTP2Connection(
+		originConn,
+		testConfig,
+		&NamedTunnelConfig{},
+		&pogs.ConnectionOptions{},
+		testObserver,
+		connIndex,
+		mockConnectedFuse{},
+	), edgeConn
+}
+
+func TestServeHTTP(t *testing.T) {
+	tests := []testRequest{
+		{
+			name:           "ok",
+			endpoint:       "ok",
+			expectedStatus: http.StatusOK,
+			expectedBody:   []byte(http.StatusText(http.StatusOK)),
+		},
+		{
+			name:           "large_file",
+			endpoint:       "large_file",
+			expectedStatus: http.StatusOK,
+			expectedBody:   testLargeResp,
+		},
+		{
+			name:           "Bad request",
+			endpoint:       "400",
+			expectedStatus: http.StatusBadRequest,
+			expectedBody:   []byte(http.StatusText(http.StatusBadRequest)),
+		},
+		{
+			name:           "Internal server error",
+			endpoint:       "500",
+			expectedStatus: http.StatusInternalServerError,
+			expectedBody:   []byte(http.StatusText(http.StatusInternalServerError)),
+		},
+		{
+			name:           "Proxy error",
+			endpoint:       "error",
+			expectedStatus: http.StatusBadGateway,
+			expectedBody:   nil,
+			isProxyError:   true,
+		},
+	}
+
+	http2Conn, edgeConn := newTestHTTP2Connection()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		http2Conn.Serve(ctx)
+	}()
+
+	edgeHTTP2Conn, err := testTransport.NewClientConn(edgeConn)
+	require.NoError(t, err)
+
+	for _, test := range tests {
+		endpoint := fmt.Sprintf("http://localhost:8080/%s", test.endpoint)
+		req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
+		require.NoError(t, err)
+
+		resp, err := edgeHTTP2Conn.RoundTrip(req)
+		require.NoError(t, err)
+		require.Equal(t, test.expectedStatus, resp.StatusCode)
+		if test.expectedBody != nil {
+			respBody, err := ioutil.ReadAll(resp.Body)
+			require.NoError(t, err)
+			require.Equal(t, test.expectedBody, respBody)
+		}
+		if test.isProxyError {
+			require.Equal(t, responseMetaHeaderCfd, resp.Header.Get(responseMetaHeaderField))
+		} else {
+			require.Equal(t, responseMetaHeaderOrigin, resp.Header.Get(responseMetaHeaderField))
+		}
+	}
+	cancel()
+	wg.Wait()
+}
+
+type mockNamedTunnelRPCClient struct {
+	registered   chan struct{}
+	unregistered chan struct{}
+}
+
+func (mc mockNamedTunnelRPCClient) RegisterConnection(
+	c context.Context,
+	config *NamedTunnelConfig,
+	options *tunnelpogs.ConnectionOptions,
+	connIndex uint8,
+	observer *Observer,
+) error {
+	close(mc.registered)
+	return nil
+}
+
+func (mc mockNamedTunnelRPCClient) GracefulShutdown(ctx context.Context, gracePeriod time.Duration) {
+	close(mc.unregistered)
+}
+
+func (mockNamedTunnelRPCClient) Close() {}
+
+type mockRPCClientFactory struct {
+	registered   chan struct{}
+	unregistered chan struct{}
+}
+
+func (mf *mockRPCClientFactory) newMockRPCClient(context.Context, io.ReadWriteCloser, logger.Service) NamedTunnelRPCClient {
+	return mockNamedTunnelRPCClient{
+		registered:   mf.registered,
+		unregistered: mf.unregistered,
+	}
+}
+
+type wsRespWriter struct {
+	*httptest.ResponseRecorder
+	readPipe  *io.PipeReader
+	writePipe *io.PipeWriter
+}
+
+func newWSRespWriter() *wsRespWriter {
+	readPipe, writePipe := io.Pipe()
+	return &wsRespWriter{
+		httptest.NewRecorder(),
+		readPipe,
+		writePipe,
+	}
+}
+
+func (w *wsRespWriter) RespBody() io.ReadWriter {
+	return nowriter{w.readPipe}
+}
+
+func (w *wsRespWriter) Write(data []byte) (n int, err error) {
+	return w.writePipe.Write(data)
+}
+
+func TestServeWS(t *testing.T) {
+	http2Conn, _ := newTestHTTP2Connection()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		http2Conn.Serve(ctx)
+	}()
+
+	respWriter := newWSRespWriter()
+	readPipe, writePipe := io.Pipe()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://localhost:8080/ws", readPipe)
+	require.NoError(t, err)
+	req.Header.Set(internalUpgradeHeader, websocketUpgrade)
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		http2Conn.ServeHTTP(respWriter, req)
+	}()
+
+	data := []byte("test websocket")
+	err = wsutil.WriteClientText(writePipe, data)
+	require.NoError(t, err)
+
+	respBody, err := wsutil.ReadServerText(respWriter.RespBody())
+	require.NoError(t, err)
+	require.Equal(t, data, respBody, fmt.Sprintf("Expect %s, got %s", string(data), string(respBody)))
+
+	cancel()
+	resp := respWriter.Result()
+	// http2RespWriter should rewrite status 101 to 200
+	require.Equal(t, http.StatusOK, resp.StatusCode)
+	require.Equal(t, responseMetaHeaderOrigin, resp.Header.Get(responseMetaHeaderField))
+
+	wg.Wait()
+}
+
+func TestServeControlStream(t *testing.T) {
+	http2Conn, edgeConn := newTestHTTP2Connection()
+
+	rpcClientFactory := mockRPCClientFactory{
+		registered:   make(chan struct{}),
+		unregistered: make(chan struct{}),
+	}
+	http2Conn.newRPCClientFunc = rpcClientFactory.newMockRPCClient
+
+	ctx, cancel := context.WithCancel(context.Background())
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		http2Conn.Serve(ctx)
+	}()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://localhost:8080/", nil)
+	require.NoError(t, err)
+	req.Header.Set(internalUpgradeHeader, controlStreamUpgrade)
+
+	edgeHTTP2Conn, err := testTransport.NewClientConn(edgeConn)
+	require.NoError(t, err)
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		edgeHTTP2Conn.RoundTrip(req)
+	}()
+
+	<-rpcClientFactory.registered
+	cancel()
+	<-rpcClientFactory.unregistered
+
+	wg.Wait()
+}
+
+func benchmarkServeHTTP(b *testing.B, test testRequest) {
+	http2Conn, edgeConn := newTestHTTP2Connection()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		http2Conn.Serve(ctx)
+	}()
+
+	endpoint := fmt.Sprintf("http://localhost:8080/%s", test.endpoint)
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
+	require.NoError(b, err)
+
+	edgeHTTP2Conn, err := testTransport.NewClientConn(edgeConn)
+	require.NoError(b, err)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		b.StartTimer()
+		resp, err := edgeHTTP2Conn.RoundTrip(req)
+		b.StopTimer()
+		require.NoError(b, err)
+		require.Equal(b, test.expectedStatus, resp.StatusCode)
+		if test.expectedBody != nil {
+			respBody, err := ioutil.ReadAll(resp.Body)
+			require.NoError(b, err)
+			require.Equal(b, test.expectedBody, respBody)
+		}
+		resp.Body.Close()
+	}
+
+	cancel()
+	wg.Wait()
+}
+func BenchmarkServeHTTPSimple(b *testing.B) {
+	test := testRequest{
+		name:           "ok",
+		endpoint:       "ok",
+		expectedStatus: http.StatusOK,
+		expectedBody:   []byte(http.StatusText(http.StatusOK)),
+	}
+
+	benchmarkServeHTTP(b, test)
+}
+
+func BenchmarkServeHTTPLargeFile(b *testing.B) {
+	test := testRequest{
+		name:           "large_file",
+		endpoint:       "large_file",
+		expectedStatus: http.StatusOK,
+		expectedBody:   testLargeResp,
+	}
+
+	benchmarkServeHTTP(b, test)
+}

connection/json.go

@@ -0,0 +1,7 @@
+package connection
+
+import (
+	jsoniter "github.com/json-iterator/go"
+)
+
+var json = jsoniter.ConfigFastest

connection/metrics.go

@@ -0,0 +1,405 @@
+package connection
+
+import (
+	"sync"
+	"time"
+
+	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const (
+	MetricsNamespace = "cloudflared"
+	TunnelSubsystem  = "tunnel"
+	muxerSubsystem   = "muxer"
+)
+
+type muxerMetrics struct {
+	rtt              *prometheus.GaugeVec
+	rttMin           *prometheus.GaugeVec
+	rttMax           *prometheus.GaugeVec
+	receiveWindowAve *prometheus.GaugeVec
+	sendWindowAve    *prometheus.GaugeVec
+	receiveWindowMin *prometheus.GaugeVec
+	receiveWindowMax *prometheus.GaugeVec
+	sendWindowMin    *prometheus.GaugeVec
+	sendWindowMax    *prometheus.GaugeVec
+	inBoundRateCurr  *prometheus.GaugeVec
+	inBoundRateMin   *prometheus.GaugeVec
+	inBoundRateMax   *prometheus.GaugeVec
+	outBoundRateCurr *prometheus.GaugeVec
+	outBoundRateMin  *prometheus.GaugeVec
+	outBoundRateMax  *prometheus.GaugeVec
+	compBytesBefore  *prometheus.GaugeVec
+	compBytesAfter   *prometheus.GaugeVec
+	compRateAve      *prometheus.GaugeVec
+}
+
+type tunnelMetrics struct {
+	timerRetries    prometheus.Gauge
+	serverLocations *prometheus.GaugeVec
+	// locationLock is a mutex for oldServerLocations
+	locationLock sync.Mutex
+	// oldServerLocations stores the last server the tunnel was connected to
+	oldServerLocations map[string]string
+
+	regSuccess *prometheus.CounterVec
+	regFail    *prometheus.CounterVec
+	rpcFail    *prometheus.CounterVec
+
+	muxerMetrics        *muxerMetrics
+	tunnelsHA           tunnelsForHA
+	userHostnamesCounts *prometheus.CounterVec
+}
+
+func newMuxerMetrics() *muxerMetrics {
+	rtt := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "rtt",
+			Help:      "Round-trip time in millisecond",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(rtt)
+
+	rttMin := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "rtt_min",
+			Help:      "Shortest round-trip time in millisecond",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(rttMin)
+
+	rttMax := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "rtt_max",
+			Help:      "Longest round-trip time in millisecond",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(rttMax)
+
+	receiveWindowAve := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "receive_window_ave",
+			Help:      "Average receive window size in bytes",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(receiveWindowAve)
+
+	sendWindowAve := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "send_window_ave",
+			Help:      "Average send window size in bytes",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(sendWindowAve)
+
+	receiveWindowMin := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "receive_window_min",
+			Help:      "Smallest receive window size in bytes",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(receiveWindowMin)
+
+	receiveWindowMax := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "receive_window_max",
+			Help:      "Largest receive window size in bytes",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(receiveWindowMax)
+
+	sendWindowMin := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "send_window_min",
+			Help:      "Smallest send window size in bytes",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(sendWindowMin)
+
+	sendWindowMax := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "send_window_max",
+			Help:      "Largest send window size in bytes",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(sendWindowMax)
+
+	inBoundRateCurr := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "inbound_bytes_per_sec_curr",
+			Help:      "Current inbounding bytes per second, 0 if there is no incoming connection",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(inBoundRateCurr)
+
+	inBoundRateMin := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "inbound_bytes_per_sec_min",
+			Help:      "Minimum non-zero inbounding bytes per second",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(inBoundRateMin)
+
+	inBoundRateMax := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "inbound_bytes_per_sec_max",
+			Help:      "Maximum inbounding bytes per second",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(inBoundRateMax)
+
+	outBoundRateCurr := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "outbound_bytes_per_sec_curr",
+			Help:      "Current outbounding bytes per second, 0 if there is no outgoing traffic",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(outBoundRateCurr)
+
+	outBoundRateMin := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "outbound_bytes_per_sec_min",
+			Help:      "Minimum non-zero outbounding bytes per second",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(outBoundRateMin)
+
+	outBoundRateMax := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "outbound_bytes_per_sec_max",
+			Help:      "Maximum outbounding bytes per second",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(outBoundRateMax)
+
+	compBytesBefore := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "comp_bytes_before",
+			Help:      "Bytes sent via cross-stream compression, pre compression",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(compBytesBefore)
+
+	compBytesAfter := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "comp_bytes_after",
+			Help:      "Bytes sent via cross-stream compression, post compression",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(compBytesAfter)
+
+	compRateAve := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "comp_rate_ave",
+			Help:      "Average outbound cross-stream compression ratio",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(compRateAve)
+
+	return &muxerMetrics{
+		rtt:              rtt,
+		rttMin:           rttMin,
+		rttMax:           rttMax,
+		receiveWindowAve: receiveWindowAve,
+		sendWindowAve:    sendWindowAve,
+		receiveWindowMin: receiveWindowMin,
+		receiveWindowMax: receiveWindowMax,
+		sendWindowMin:    sendWindowMin,
+		sendWindowMax:    sendWindowMax,
+		inBoundRateCurr:  inBoundRateCurr,
+		inBoundRateMin:   inBoundRateMin,
+		inBoundRateMax:   inBoundRateMax,
+		outBoundRateCurr: outBoundRateCurr,
+		outBoundRateMin:  outBoundRateMin,
+		outBoundRateMax:  outBoundRateMax,
+		compBytesBefore:  compBytesBefore,
+		compBytesAfter:   compBytesAfter,
+		compRateAve:      compRateAve,
+	}
+}
+
+func (m *muxerMetrics) update(connectionID string, metrics *h2mux.MuxerMetrics) {
+	m.rtt.WithLabelValues(connectionID).Set(convertRTTMilliSec(metrics.RTT))
+	m.rttMin.WithLabelValues(connectionID).Set(convertRTTMilliSec(metrics.RTTMin))
+	m.rttMax.WithLabelValues(connectionID).Set(convertRTTMilliSec(metrics.RTTMax))
+	m.receiveWindowAve.WithLabelValues(connectionID).Set(metrics.ReceiveWindowAve)
+	m.sendWindowAve.WithLabelValues(connectionID).Set(metrics.SendWindowAve)
+	m.receiveWindowMin.WithLabelValues(connectionID).Set(float64(metrics.ReceiveWindowMin))
+	m.receiveWindowMax.WithLabelValues(connectionID).Set(float64(metrics.ReceiveWindowMax))
+	m.sendWindowMin.WithLabelValues(connectionID).Set(float64(metrics.SendWindowMin))
+	m.sendWindowMax.WithLabelValues(connectionID).Set(float64(metrics.SendWindowMax))
+	m.inBoundRateCurr.WithLabelValues(connectionID).Set(float64(metrics.InBoundRateCurr))
+	m.inBoundRateMin.WithLabelValues(connectionID).Set(float64(metrics.InBoundRateMin))
+	m.inBoundRateMax.WithLabelValues(connectionID).Set(float64(metrics.InBoundRateMax))
+	m.outBoundRateCurr.WithLabelValues(connectionID).Set(float64(metrics.OutBoundRateCurr))
+	m.outBoundRateMin.WithLabelValues(connectionID).Set(float64(metrics.OutBoundRateMin))
+	m.outBoundRateMax.WithLabelValues(connectionID).Set(float64(metrics.OutBoundRateMax))
+	m.compBytesBefore.WithLabelValues(connectionID).Set(float64(metrics.CompBytesBefore.Value()))
+	m.compBytesAfter.WithLabelValues(connectionID).Set(float64(metrics.CompBytesAfter.Value()))
+	m.compRateAve.WithLabelValues(connectionID).Set(float64(metrics.CompRateAve()))
+}
+
+func convertRTTMilliSec(t time.Duration) float64 {
+	return float64(t / time.Millisecond)
+}
+
+// Metrics that can be collected without asking the edge
+func newTunnelMetrics() *tunnelMetrics {
+	maxConcurrentRequestsPerTunnel := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: TunnelSubsystem,
+			Name:      "max_concurrent_requests_per_tunnel",
+			Help:      "Largest number of concurrent requests proxied through each tunnel so far",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(maxConcurrentRequestsPerTunnel)
+
+	timerRetries := prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: TunnelSubsystem,
+			Name:      "timer_retries",
+			Help:      "Unacknowledged heart beats count",
+		})
+	prometheus.MustRegister(timerRetries)
+
+	serverLocations := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: TunnelSubsystem,
+			Name:      "server_locations",
+			Help:      "Where each tunnel is connected to. 1 means current location, 0 means previous locations.",
+		},
+		[]string{"connection_id", "location"},
+	)
+	prometheus.MustRegister(serverLocations)
+
+	rpcFail := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: TunnelSubsystem,
+			Name:      "tunnel_rpc_fail",
+			Help:      "Count of RPC connection errors by type",
+		},
+		[]string{"error", "rpcName"},
+	)
+	prometheus.MustRegister(rpcFail)
+
+	registerFail := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: TunnelSubsystem,
+			Name:      "tunnel_register_fail",
+			Help:      "Count of tunnel registration errors by type",
+		},
+		[]string{"error", "rpcName"},
+	)
+	prometheus.MustRegister(registerFail)
+
+	userHostnamesCounts := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: TunnelSubsystem,
+			Name:      "user_hostnames_counts",
+			Help:      "Which user hostnames cloudflared is serving",
+		},
+		[]string{"userHostname"},
+	)
+	prometheus.MustRegister(userHostnamesCounts)
+
+	registerSuccess := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: TunnelSubsystem,
+			Name:      "tunnel_register_success",
+			Help:      "Count of successful tunnel registrations",
+		},
+		[]string{"rpcName"},
+	)
+	prometheus.MustRegister(registerSuccess)
+
+	return &tunnelMetrics{
+		timerRetries:        timerRetries,
+		serverLocations:     serverLocations,
+		oldServerLocations:  make(map[string]string),
+		muxerMetrics:        newMuxerMetrics(),
+		tunnelsHA:           NewTunnelsForHA(),
+		regSuccess:          registerSuccess,
+		regFail:             registerFail,
+		rpcFail:             rpcFail,
+		userHostnamesCounts: userHostnamesCounts,
+	}
+}
+
+func (t *tunnelMetrics) updateMuxerMetrics(connectionID string, metrics *h2mux.MuxerMetrics) {
+	t.muxerMetrics.update(connectionID, metrics)
+}
+
+func (t *tunnelMetrics) registerServerLocation(connectionID, loc string) {
+	t.locationLock.Lock()
+	defer t.locationLock.Unlock()
+	if oldLoc, ok := t.oldServerLocations[connectionID]; ok && oldLoc == loc {
+		return
+	} else if ok {
+		t.serverLocations.WithLabelValues(connectionID, oldLoc).Dec()
+	}
+	t.serverLocations.WithLabelValues(connectionID, loc).Inc()
+	t.oldServerLocations[connectionID] = loc
+}

connection/observer.go

@@ -0,0 +1,99 @@
+package connection
+
+import (
+	"fmt"
+	"net/url"
+	"strings"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/ui"
+	"github.com/cloudflare/cloudflared/logger"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+)
+
+type Observer struct {
+	logger.Service
+	metrics         *tunnelMetrics
+	tunnelEventChan chan<- ui.TunnelEvent
+}
+
+func NewObserver(logger logger.Service, tunnelEventChan chan<- ui.TunnelEvent) *Observer {
+	return &Observer{
+		logger,
+		newTunnelMetrics(),
+		tunnelEventChan,
+	}
+}
+
+func (o *Observer) logServerInfo(connIndex uint8, location, msg string) {
+	// If launch-ui flag is set, send connect msg
+	if o.tunnelEventChan != nil {
+		o.tunnelEventChan <- ui.TunnelEvent{Index: connIndex, EventType: ui.Connected, Location: location}
+	}
+	o.Infof(msg)
+	o.metrics.registerServerLocation(uint8ToString(connIndex), location)
+}
+
+func (o *Observer) logTrialHostname(registration *tunnelpogs.TunnelRegistration) error {
+	// Print out the user's trial zone URL in a nice box (if they requested and got one and UI flag is not set)
+	if o.tunnelEventChan == nil {
+		if registrationURL, err := url.Parse(registration.Url); err == nil {
+			for _, line := range asciiBox(trialZoneMsg(registrationURL.String()), 2) {
+				o.Info(line)
+			}
+		} else {
+			o.Error("Failed to connect tunnel, please try again.")
+			return fmt.Errorf("empty URL in response from Cloudflare edge")
+		}
+	}
+	return nil
+}
+
+// Print out the given lines in a nice ASCII box.
+func asciiBox(lines []string, padding int) (box []string) {
+	maxLen := maxLen(lines)
+	spacer := strings.Repeat(" ", padding)
+
+	border := "+" + strings.Repeat("-", maxLen+(padding*2)) + "+"
+
+	box = append(box, border)
+	for _, line := range lines {
+		box = append(box, "|"+spacer+line+strings.Repeat(" ", maxLen-len(line))+spacer+"|")
+	}
+	box = append(box, border)
+	return
+}
+
+func maxLen(lines []string) int {
+	max := 0
+	for _, line := range lines {
+		if len(line) > max {
+			max = len(line)
+		}
+	}
+	return max
+}
+
+func trialZoneMsg(url string) []string {
+	return []string{
+		"Your free tunnel has started! Visit it:",
+		"  " + url,
+	}
+}
+
+func (o *Observer) sendRegisteringEvent() {
+	if o.tunnelEventChan != nil {
+		o.tunnelEventChan <- ui.TunnelEvent{EventType: ui.RegisteringTunnel}
+	}
+}
+
+func (o *Observer) sendConnectedEvent(connIndex uint8, location string) {
+	if o.tunnelEventChan != nil {
+		o.tunnelEventChan <- ui.TunnelEvent{Index: connIndex, EventType: ui.Connected, Location: location}
+	}
+}
+
+func (o *Observer) sendURL(url string) {
+	if o.tunnelEventChan != nil {
+		o.tunnelEventChan <- ui.TunnelEvent{EventType: ui.SetUrl, Url: url}
+	}
+}

connection/observer_test.go

@@ -0,0 +1,45 @@
+package connection
+
+import (
+	"strconv"
+	"sync"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// can only be called once
+var m = newTunnelMetrics()
+
+func TestRegisterServerLocation(t *testing.T) {
+	tunnels := 20
+	var wg sync.WaitGroup
+	wg.Add(tunnels)
+	for i := 0; i < tunnels; i++ {
+		go func(i int) {
+			id := strconv.Itoa(i)
+			m.registerServerLocation(id, "LHR")
+			wg.Done()
+		}(i)
+	}
+	wg.Wait()
+	for i := 0; i < tunnels; i++ {
+		id := strconv.Itoa(i)
+		assert.Equal(t, "LHR", m.oldServerLocations[id])
+	}
+
+	wg.Add(tunnels)
+	for i := 0; i < tunnels; i++ {
+		go func(i int) {
+			id := strconv.Itoa(i)
+			m.registerServerLocation(id, "AUS")
+			wg.Done()
+		}(i)
+	}
+	wg.Wait()
+	for i := 0; i < tunnels; i++ {
+		id := strconv.Itoa(i)
+		assert.Equal(t, "AUS", m.oldServerLocations[id])
+	}
+
+}

connection/protocol.go

@@ -0,0 +1,179 @@
+package connection
+
+import (
+	"fmt"
+	"hash/fnv"
+	"sync"
+	"time"
+
+	"github.com/cloudflare/cloudflared/logger"
+)
+
+const (
+	AvailableProtocolFlagMessage = "Available protocols: http2 - Go's implementation, h2mux - Cloudflare's implementation of HTTP/2, and auto - automatically select between http2 and h2mux"
+	// edgeH2muxTLSServerName is the server name to establish h2mux connection with edge
+	edgeH2muxTLSServerName = "cftunnel.com"
+	// edgeH2TLSServerName is the server name to establish http2 connection with edge
+	edgeH2TLSServerName = "h2.cftunnel.com"
+	// threshold to switch back to h2mux when the user intentionally pick --protocol http2
+	explicitHTTP2FallbackThreshold = -1
+	autoSelectFlag                 = "auto"
+)
+
+var (
+	ProtocolList = []Protocol{H2mux, HTTP2}
+)
+
+type Protocol int64
+
+const (
+	H2mux Protocol = iota
+	HTTP2
+)
+
+func (p Protocol) ServerName() string {
+	switch p {
+	case H2mux:
+		return edgeH2muxTLSServerName
+	case HTTP2:
+		return edgeH2TLSServerName
+	default:
+		return ""
+	}
+}
+
+// Fallback returns the fallback protocol and whether the protocol has a fallback
+func (p Protocol) fallback() (Protocol, bool) {
+	switch p {
+	case H2mux:
+		return 0, false
+	case HTTP2:
+		return H2mux, true
+	default:
+		return 0, false
+	}
+}
+
+func (p Protocol) String() string {
+	switch p {
+	case H2mux:
+		return "h2mux"
+	case HTTP2:
+		return "http2"
+	default:
+		return fmt.Sprintf("unknown protocol")
+	}
+}
+
+type ProtocolSelector interface {
+	Current() Protocol
+	Fallback() (Protocol, bool)
+}
+
+type staticProtocolSelector struct {
+	current Protocol
+}
+
+func (s *staticProtocolSelector) Current() Protocol {
+	return s.current
+}
+
+func (s *staticProtocolSelector) Fallback() (Protocol, bool) {
+	return 0, false
+}
+
+type autoProtocolSelector struct {
+	lock           sync.RWMutex
+	current        Protocol
+	switchThrehold int32
+	fetchFunc      PercentageFetcher
+	refreshAfter   time.Time
+	ttl            time.Duration
+	logger         logger.Service
+}
+
+func newAutoProtocolSelector(
+	current Protocol,
+	switchThrehold int32,
+	fetchFunc PercentageFetcher,
+	ttl time.Duration,
+	logger logger.Service,
+) *autoProtocolSelector {
+	return &autoProtocolSelector{
+		current:        current,
+		switchThrehold: switchThrehold,
+		fetchFunc:      fetchFunc,
+		refreshAfter:   time.Now().Add(ttl),
+		ttl:            ttl,
+		logger:         logger,
+	}
+}
+
+func (s *autoProtocolSelector) Current() Protocol {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+	if time.Now().Before(s.refreshAfter) {
+		return s.current
+	}
+
+	percentage, err := s.fetchFunc()
+	if err != nil {
+		s.logger.Errorf("Failed to refresh protocol, err: %v", err)
+		return s.current
+	}
+
+	if s.switchThrehold < percentage {
+		s.current = HTTP2
+	} else {
+		s.current = H2mux
+	}
+	s.refreshAfter = time.Now().Add(s.ttl)
+	return s.current
+}
+
+func (s *autoProtocolSelector) Fallback() (Protocol, bool) {
+	s.lock.RLock()
+	defer s.lock.RUnlock()
+	return s.current.fallback()
+}
+
+type PercentageFetcher func() (int32, error)
+
+func NewProtocolSelector(protocolFlag string, namedTunnel *NamedTunnelConfig, fetchFunc PercentageFetcher, ttl time.Duration, logger logger.Service) (ProtocolSelector, error) {
+	if namedTunnel == nil {
+		return &staticProtocolSelector{
+			current: H2mux,
+		}, nil
+	}
+	if protocolFlag == H2mux.String() {
+		return &staticProtocolSelector{
+			current: H2mux,
+		}, nil
+	}
+
+	http2Percentage, err := fetchFunc()
+	if err != nil {
+		return nil, err
+	}
+	if protocolFlag == HTTP2.String() {
+		if http2Percentage < 0 {
+			return newAutoProtocolSelector(H2mux, explicitHTTP2FallbackThreshold, fetchFunc, ttl, logger), nil
+		}
+		return newAutoProtocolSelector(HTTP2, explicitHTTP2FallbackThreshold, fetchFunc, ttl, logger), nil
+	}
+
+	if protocolFlag != autoSelectFlag {
+		return nil, fmt.Errorf("Unknown protocol %s, %s", protocolFlag, AvailableProtocolFlagMessage)
+	}
+	threshold := switchThreshold(namedTunnel.Auth.AccountTag)
+	if threshold < http2Percentage {
+		return newAutoProtocolSelector(HTTP2, threshold, fetchFunc, ttl, logger), nil
+	}
+	return newAutoProtocolSelector(H2mux, threshold, fetchFunc, ttl, logger), nil
+}
+
+func switchThreshold(accountTag string) int32 {
+	h := fnv.New32a()
+	h.Write([]byte(accountTag))
+	return int32(h.Sum32() % 100)
+}

connection/protocol_test.go

@@ -0,0 +1,220 @@
+package connection
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/cloudflare/cloudflared/logger"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/stretchr/testify/assert"
+)
+
+const (
+	testNoTTL = 0
+)
+
+var (
+	testNamedTunnelConfig = &NamedTunnelConfig{
+		Auth: pogs.TunnelAuth{
+			AccountTag: "testAccountTag",
+		},
+	}
+)
+
+func mockFetcher(percentage int32) PercentageFetcher {
+	return func() (int32, error) {
+		return percentage, nil
+	}
+}
+
+func mockFetcherWithError() PercentageFetcher {
+	return func() (int32, error) {
+		return 0, fmt.Errorf("failed to fetch precentage")
+	}
+}
+
+type dynamicMockFetcher struct {
+	percentage int32
+	err        error
+}
+
+func (dmf *dynamicMockFetcher) fetch() PercentageFetcher {
+	return func() (int32, error) {
+		if dmf.err != nil {
+			return 0, dmf.err
+		}
+		return dmf.percentage, nil
+	}
+}
+
+func TestNewProtocolSelector(t *testing.T) {
+	tests := []struct {
+		name              string
+		protocol          string
+		expectedProtocol  Protocol
+		hasFallback       bool
+		expectedFallback  Protocol
+		namedTunnelConfig *NamedTunnelConfig
+		fetchFunc         PercentageFetcher
+		wantErr           bool
+	}{
+		{
+			name:              "classic tunnel",
+			protocol:          "h2mux",
+			expectedProtocol:  H2mux,
+			namedTunnelConfig: nil,
+		},
+		{
+			name:              "named tunnel over h2mux",
+			protocol:          "h2mux",
+			expectedProtocol:  H2mux,
+			namedTunnelConfig: testNamedTunnelConfig,
+		},
+		{
+			name:              "named tunnel over http2",
+			protocol:          "http2",
+			expectedProtocol:  HTTP2,
+			hasFallback:       true,
+			expectedFallback:  H2mux,
+			fetchFunc:         mockFetcher(0),
+			namedTunnelConfig: testNamedTunnelConfig,
+		},
+		{
+			name:              "named tunnel http2 disabled",
+			protocol:          "http2",
+			expectedProtocol:  H2mux,
+			fetchFunc:         mockFetcher(-1),
+			namedTunnelConfig: testNamedTunnelConfig,
+		},
+		{
+			name:              "named tunnel auto all http2 disabled",
+			protocol:          "auto",
+			expectedProtocol:  H2mux,
+			fetchFunc:         mockFetcher(-1),
+			namedTunnelConfig: testNamedTunnelConfig,
+		},
+		{
+			name:              "named tunnel auto to h2mux",
+			protocol:          "auto",
+			expectedProtocol:  H2mux,
+			fetchFunc:         mockFetcher(0),
+			namedTunnelConfig: testNamedTunnelConfig,
+		},
+		{
+			name:              "named tunnel auto to http2",
+			protocol:          "auto",
+			expectedProtocol:  HTTP2,
+			hasFallback:       true,
+			expectedFallback:  H2mux,
+			fetchFunc:         mockFetcher(100),
+			namedTunnelConfig: testNamedTunnelConfig,
+		},
+		{
+			// None named tunnel can only use h2mux, so specifying an unknown protocol is not an error
+			name:             "classic tunnel unknown protocol",
+			protocol:         "unknown",
+			expectedProtocol: H2mux,
+		},
+		{
+			name:              "named tunnel unknown protocol",
+			protocol:          "unknown",
+			fetchFunc:         mockFetcher(100),
+			namedTunnelConfig: testNamedTunnelConfig,
+			wantErr:           true,
+		},
+		{
+			name:              "named tunnel fetch error",
+			protocol:          "unknown",
+			fetchFunc:         mockFetcherWithError(),
+			namedTunnelConfig: testNamedTunnelConfig,
+			wantErr:           true,
+		},
+	}
+	logger, _ := logger.New()
+	for _, test := range tests {
+		selector, err := NewProtocolSelector(test.protocol, test.namedTunnelConfig, test.fetchFunc, testNoTTL, logger)
+		if test.wantErr {
+			assert.Error(t, err, fmt.Sprintf("test %s failed", test.name))
+		} else {
+			assert.NoError(t, err, fmt.Sprintf("test %s failed", test.name))
+			assert.Equal(t, test.expectedProtocol, selector.Current(), fmt.Sprintf("test %s failed", test.name))
+			fallback, ok := selector.Fallback()
+			assert.Equal(t, test.hasFallback, ok, fmt.Sprintf("test %s failed", test.name))
+			if test.hasFallback {
+				assert.Equal(t, test.expectedFallback, fallback, fmt.Sprintf("test %s failed", test.name))
+			}
+		}
+	}
+}
+
+func TestAutoProtocolSelectorRefresh(t *testing.T) {
+	logger, _ := logger.New()
+	fetcher := dynamicMockFetcher{}
+	selector, err := NewProtocolSelector("auto", testNamedTunnelConfig, fetcher.fetch(), testNoTTL, logger)
+	assert.NoError(t, err)
+	assert.Equal(t, H2mux, selector.Current())
+
+	fetcher.percentage = 100
+	assert.Equal(t, HTTP2, selector.Current())
+
+	fetcher.percentage = 0
+	assert.Equal(t, H2mux, selector.Current())
+
+	fetcher.percentage = 100
+	assert.Equal(t, HTTP2, selector.Current())
+
+	fetcher.err = fmt.Errorf("failed to fetch")
+	assert.Equal(t, HTTP2, selector.Current())
+
+	fetcher.percentage = -1
+	fetcher.err = nil
+	assert.Equal(t, H2mux, selector.Current())
+
+	fetcher.percentage = 0
+	assert.Equal(t, H2mux, selector.Current())
+
+	fetcher.percentage = 100
+	assert.Equal(t, HTTP2, selector.Current())
+}
+
+func TestHTTP2ProtocolSelectorRefresh(t *testing.T) {
+	logger, _ := logger.New()
+	fetcher := dynamicMockFetcher{}
+	selector, err := NewProtocolSelector("http2", testNamedTunnelConfig, fetcher.fetch(), testNoTTL, logger)
+	assert.NoError(t, err)
+	assert.Equal(t, HTTP2, selector.Current())
+
+	fetcher.percentage = 100
+	assert.Equal(t, HTTP2, selector.Current())
+
+	fetcher.percentage = 0
+	assert.Equal(t, HTTP2, selector.Current())
+
+	fetcher.err = fmt.Errorf("failed to fetch")
+	assert.Equal(t, HTTP2, selector.Current())
+
+	fetcher.percentage = -1
+	fetcher.err = nil
+	assert.Equal(t, H2mux, selector.Current())
+
+	fetcher.percentage = 0
+	assert.Equal(t, HTTP2, selector.Current())
+
+	fetcher.percentage = 100
+	assert.Equal(t, HTTP2, selector.Current())
+
+	fetcher.percentage = -1
+	assert.Equal(t, H2mux, selector.Current())
+}
+
+func TestProtocolSelectorRefreshTTL(t *testing.T) {
+	logger, _ := logger.New()
+	fetcher := dynamicMockFetcher{percentage: 100}
+	selector, err := NewProtocolSelector("auto", testNamedTunnelConfig, fetcher.fetch(), time.Hour, logger)
+	assert.NoError(t, err)
+	assert.Equal(t, HTTP2, selector.Current())
+
+	fetcher.percentage = 0
+	assert.Equal(t, HTTP2, selector.Current())
+}

connection/rpc.go

@@ -2,40 +2,294 @@ package connection
 
 import (
 	"context"
+	"fmt"
 	"io"
-
-	rpc "zombiezen.com/go/capnproto2/rpc"
+	"time"
 
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/tunnelrpc"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"zombiezen.com/go/capnproto2/rpc"
 )
 
-// NewTunnelRPCClient creates and returns a new RPC client, which will communicate
-// using a stream on the given muxer
-func NewTunnelRPCClient(
+type tunnelServerClient struct {
+	client    tunnelpogs.TunnelServer_PogsClient
+	transport rpc.Transport
+}
+
+// NewTunnelRPCClient creates and returns a new RPC client, which will communicate using a stream on the given muxer.
+// This method is exported for supervisor to call Authenticate RPC
+func NewTunnelServerClient(
 	ctx context.Context,
 	stream io.ReadWriteCloser,
 	logger logger.Service,
-) (client tunnelpogs.TunnelServer_PogsClient, err error) {
+) *tunnelServerClient {
+	transport := tunnelrpc.NewTransportLogger(logger, rpc.StreamTransport(stream))
 	conn := rpc.NewConn(
-		tunnelrpc.NewTransportLogger(logger, rpc.StreamTransport(stream)),
+		transport,
 		tunnelrpc.ConnLog(logger),
 	)
 	registrationClient := tunnelpogs.RegistrationServer_PogsClient{Client: conn.Bootstrap(ctx), Conn: conn}
-	client = tunnelpogs.TunnelServer_PogsClient{RegistrationServer_PogsClient: registrationClient, Client: conn.Bootstrap(ctx), Conn: conn}
-	return client, nil
+	return &tunnelServerClient{
+		client:    tunnelpogs.TunnelServer_PogsClient{RegistrationServer_PogsClient: registrationClient, Client: conn.Bootstrap(ctx), Conn: conn},
+		transport: transport,
+	}
 }
 
-func NewRegistrationRPCClient(
+func (tsc *tunnelServerClient) Authenticate(ctx context.Context, classicTunnel *ClassicTunnelConfig, registrationOptions *tunnelpogs.RegistrationOptions) (tunnelpogs.AuthOutcome, error) {
+	authResp, err := tsc.client.Authenticate(ctx, classicTunnel.OriginCert, classicTunnel.Hostname, registrationOptions)
+	if err != nil {
+		return nil, err
+	}
+	return authResp.Outcome(), nil
+}
+
+func (tsc *tunnelServerClient) Close() {
+	// Closing the client will also close the connection
+	tsc.client.Close()
+	tsc.transport.Close()
+}
+
+type NamedTunnelRPCClient interface {
+	RegisterConnection(
+		c context.Context,
+		config *NamedTunnelConfig,
+		options *tunnelpogs.ConnectionOptions,
+		connIndex uint8,
+		observer *Observer,
+	) error
+	GracefulShutdown(ctx context.Context, gracePeriod time.Duration)
+	Close()
+}
+
+type registrationServerClient struct {
+	client    tunnelpogs.RegistrationServer_PogsClient
+	transport rpc.Transport
+}
+
+func newRegistrationRPCClient(
 	ctx context.Context,
 	stream io.ReadWriteCloser,
 	logger logger.Service,
-) (client tunnelpogs.RegistrationServer_PogsClient, err error) {
+) NamedTunnelRPCClient {
+	transport := tunnelrpc.NewTransportLogger(logger, rpc.StreamTransport(stream))
 	conn := rpc.NewConn(
-		tunnelrpc.NewTransportLogger(logger, rpc.StreamTransport(stream)),
+		transport,
 		tunnelrpc.ConnLog(logger),
 	)
-	client = tunnelpogs.RegistrationServer_PogsClient{Client: conn.Bootstrap(ctx), Conn: conn}
-	return client, nil
+	return &registrationServerClient{
+		client:    tunnelpogs.RegistrationServer_PogsClient{Client: conn.Bootstrap(ctx), Conn: conn},
+		transport: transport,
+	}
+}
+
+func (rsc *registrationServerClient) RegisterConnection(
+	ctx context.Context,
+	config *NamedTunnelConfig,
+	options *tunnelpogs.ConnectionOptions,
+	connIndex uint8,
+	observer *Observer,
+) error {
+	conn, err := rsc.client.RegisterConnection(
+		ctx,
+		config.Auth,
+		config.ID,
+		connIndex,
+		options,
+	)
+	if err != nil {
+		if err.Error() == DuplicateConnectionError {
+			observer.metrics.regFail.WithLabelValues("dup_edge_conn", "registerConnection").Inc()
+			return errDuplicationConnection
+		}
+		observer.metrics.regFail.WithLabelValues("server_error", "registerConnection").Inc()
+		return serverRegistrationErrorFromRPC(err)
+	}
+
+	observer.metrics.regSuccess.WithLabelValues("registerConnection").Inc()
+
+	observer.logServerInfo(connIndex, conn.Location, fmt.Sprintf("Connection %d registered with %s using ID %s", connIndex, conn.Location, conn.UUID))
+	observer.sendConnectedEvent(connIndex, conn.Location)
+
+	return nil
+}
+
+func (rsc *registrationServerClient) GracefulShutdown(ctx context.Context, gracePeriod time.Duration) {
+	ctx, cancel := context.WithTimeout(ctx, gracePeriod)
+	defer cancel()
+	rsc.client.UnregisterConnection(ctx)
+}
+
+func (rsc *registrationServerClient) Close() {
+	// Closing the client will also close the connection
+	rsc.client.Close()
+	// Closing the transport also closes the stream
+	rsc.transport.Close()
+}
+
+type rpcName string
+
+const (
+	register     rpcName = "register"
+	reconnect    rpcName = "reconnect"
+	unregister   rpcName = "unregister"
+	authenticate rpcName = " authenticate"
+)
+
+func (h *h2muxConnection) registerTunnel(ctx context.Context, credentialSetter CredentialManager, classicTunnel *ClassicTunnelConfig, registrationOptions *tunnelpogs.RegistrationOptions) error {
+	h.observer.sendRegisteringEvent()
+
+	stream, err := h.newRPCStream(ctx, register)
+	if err != nil {
+		return err
+	}
+	rpcClient := NewTunnelServerClient(ctx, stream, h.observer)
+	defer rpcClient.Close()
+
+	h.logServerInfo(ctx, rpcClient)
+	registration := rpcClient.client.RegisterTunnel(
+		ctx,
+		classicTunnel.OriginCert,
+		classicTunnel.Hostname,
+		registrationOptions,
+	)
+	if registrationErr := registration.DeserializeError(); registrationErr != nil {
+		// RegisterTunnel RPC failure
+		return h.processRegisterTunnelError(registrationErr, register)
+	}
+
+	// Send free tunnel URL to UI
+	h.observer.sendURL(registration.Url)
+	credentialSetter.SetEventDigest(h.connIndex, registration.EventDigest)
+	return h.processRegistrationSuccess(registration, register, credentialSetter, classicTunnel)
+}
+
+type CredentialManager interface {
+	ReconnectToken() ([]byte, error)
+	EventDigest(connID uint8) ([]byte, error)
+	SetEventDigest(connID uint8, digest []byte)
+	ConnDigest(connID uint8) ([]byte, error)
+	SetConnDigest(connID uint8, digest []byte)
+}
+
+func (h *h2muxConnection) processRegistrationSuccess(
+	registration *tunnelpogs.TunnelRegistration,
+	name rpcName,
+	credentialManager CredentialManager, classicTunnel *ClassicTunnelConfig,
+) error {
+	for _, logLine := range registration.LogLines {
+		h.observer.Info(logLine)
+	}
+
+	if registration.TunnelID != "" {
+		h.observer.metrics.tunnelsHA.AddTunnelID(h.connIndex, registration.TunnelID)
+		h.observer.Infof("Each HA connection's tunnel IDs: %v", h.observer.metrics.tunnelsHA.String())
+	}
+
+	// Print out the user's trial zone URL in a nice box (if they requested and got one and UI flag is not set)
+	if classicTunnel.IsTrialZone() {
+		err := h.observer.logTrialHostname(registration)
+		if err != nil {
+			return err
+		}
+	}
+
+	credentialManager.SetConnDigest(h.connIndex, registration.ConnDigest)
+	h.observer.metrics.userHostnamesCounts.WithLabelValues(registration.Url).Inc()
+
+	h.observer.Infof("Route propagating, it may take up to 1 minute for your new route to become functional")
+	h.observer.metrics.regSuccess.WithLabelValues(string(name)).Inc()
+	return nil
+}
+
+func (h *h2muxConnection) processRegisterTunnelError(err tunnelpogs.TunnelRegistrationError, name rpcName) error {
+	if err.Error() == DuplicateConnectionError {
+		h.observer.metrics.regFail.WithLabelValues("dup_edge_conn", string(name)).Inc()
+		return errDuplicationConnection
+	}
+	h.observer.metrics.regFail.WithLabelValues("server_error", string(name)).Inc()
+	return serverRegisterTunnelError{
+		cause:     err,
+		permanent: err.IsPermanent(),
+	}
+}
+
+func (h *h2muxConnection) reconnectTunnel(ctx context.Context, credentialManager CredentialManager, classicTunnel *ClassicTunnelConfig, registrationOptions *tunnelpogs.RegistrationOptions) error {
+	token, err := credentialManager.ReconnectToken()
+	if err != nil {
+		return err
+	}
+	eventDigest, err := credentialManager.EventDigest(h.connIndex)
+	if err != nil {
+		return err
+	}
+	connDigest, err := credentialManager.ConnDigest(h.connIndex)
+	if err != nil {
+		return err
+	}
+
+	h.observer.Debug("initiating RPC stream to reconnect")
+	stream, err := h.newRPCStream(ctx, register)
+	if err != nil {
+		return err
+	}
+	rpcClient := NewTunnelServerClient(ctx, stream, h.observer)
+	defer rpcClient.Close()
+
+	h.logServerInfo(ctx, rpcClient)
+	registration := rpcClient.client.ReconnectTunnel(
+		ctx,
+		token,
+		eventDigest,
+		connDigest,
+		classicTunnel.Hostname,
+		registrationOptions,
+	)
+	if registrationErr := registration.DeserializeError(); registrationErr != nil {
+		// ReconnectTunnel RPC failure
+		return h.processRegisterTunnelError(registrationErr, reconnect)
+	}
+	return h.processRegistrationSuccess(registration, reconnect, credentialManager, classicTunnel)
+}
+
+func (h *h2muxConnection) logServerInfo(ctx context.Context, rpcClient *tunnelServerClient) error {
+	// Request server info without blocking tunnel registration; must use capnp library directly.
+	serverInfoPromise := tunnelrpc.TunnelServer{Client: rpcClient.client.Client}.GetServerInfo(ctx, func(tunnelrpc.TunnelServer_getServerInfo_Params) error {
+		return nil
+	})
+	serverInfoMessage, err := serverInfoPromise.Result().Struct()
+	if err != nil {
+		h.observer.Errorf("Failed to retrieve server information: %s", err)
+		return err
+	}
+	serverInfo, err := tunnelpogs.UnmarshalServerInfo(serverInfoMessage)
+	if err != nil {
+		h.observer.Errorf("Failed to retrieve server information: %s", err)
+		return err
+	}
+	h.observer.logServerInfo(h.connIndex, serverInfo.LocationName, fmt.Sprintf("Connnection %d connected to %s", h.connIndex, serverInfo.LocationName))
+	return nil
+}
+
+func (h *h2muxConnection) unregister(isNamedTunnel bool) {
+	unregisterCtx, cancel := context.WithTimeout(context.Background(), h.config.GracePeriod)
+	defer cancel()
+
+	stream, err := h.newRPCStream(unregisterCtx, register)
+	if err != nil {
+		return
+	}
+
+	if isNamedTunnel {
+		rpcClient := newRegistrationRPCClient(unregisterCtx, stream, h.observer)
+		defer rpcClient.Close()
+
+		rpcClient.GracefulShutdown(unregisterCtx, h.config.GracePeriod)
+	} else {
+		rpcClient := NewTunnelServerClient(unregisterCtx, stream, h.observer)
+		defer rpcClient.Close()
+
+		// gracePeriod is encoded in int64 using capnproto
+		rpcClient.client.UnregisterTunnel(unregisterCtx, h.config.GracePeriod.Nanoseconds())
+	}
 }

connection/tunnelsforha.go

@@ -0,0 +1,50 @@
+package connection
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// tunnelsForHA maps this cloudflared instance's HA connections to the tunnel IDs they serve.
+type tunnelsForHA struct {
+	sync.Mutex
+	metrics *prometheus.GaugeVec
+	entries map[uint8]string
+}
+
+// NewTunnelsForHA initializes the Prometheus metrics etc for a tunnelsForHA.
+func NewTunnelsForHA() tunnelsForHA {
+	metrics := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "tunnel_ids",
+			Help: "The ID of all tunnels (and their corresponding HA connection ID) running in this instance of cloudflared.",
+		},
+		[]string{"tunnel_id", "ha_conn_id"},
+	)
+	prometheus.MustRegister(metrics)
+
+	return tunnelsForHA{
+		metrics: metrics,
+		entries: make(map[uint8]string),
+	}
+}
+
+// Track a new tunnel ID, removing the disconnected tunnel (if any) and update metrics.
+func (t *tunnelsForHA) AddTunnelID(haConn uint8, tunnelID string) {
+	t.Lock()
+	defer t.Unlock()
+	haStr := fmt.Sprintf("%v", haConn)
+	if oldTunnelID, ok := t.entries[haConn]; ok {
+		t.metrics.WithLabelValues(oldTunnelID, haStr).Dec()
+	}
+	t.entries[haConn] = tunnelID
+	t.metrics.WithLabelValues(tunnelID, haStr).Inc()
+}
+
+func (t *tunnelsForHA) String() string {
+	t.Lock()
+	defer t.Unlock()
+	return fmt.Sprintf("%v", t.entries)
+}

edgediscovery/allregions/discovery.go

@@ -121,15 +121,15 @@ func resolveSRVToTCP(srv *net.SRV) ([]*net.TCPAddr, error) {
 }
 
 // ResolveAddrs resolves TCP address given a list of addresses. Address can be a hostname, however, it will return at most one
-// of the hostname's IP addresses
-func ResolveAddrs(addrs []string) ([]*net.TCPAddr, error) {
-	var tcpAddrs []*net.TCPAddr
+// of the hostname's IP addresses.
+func ResolveAddrs(addrs []string, logger logger.Service) (resolved []*net.TCPAddr) {
 	for _, addr := range addrs {
 		tcpAddr, err := net.ResolveTCPAddr("tcp", addr)
 		if err != nil {
-			return nil, err
+			logger.Errorf("Failed to resolve %s, err: %v", addr, err)
+		} else {
+			resolved = append(resolved, tcpAddr)
 		}
-		tcpAddrs = append(tcpAddrs, tcpAddr)
 	}
-	return tcpAddrs, nil
+	return
 }

edgediscovery/allregions/regions.go

@@ -35,12 +35,12 @@ func ResolveEdge(logger logger.Service) (*Regions, error) {
 
 // StaticEdge creates a list of edge addresses from the list of hostnames.
 // Mainly used for testing connectivity.
-func StaticEdge(hostnames []string) (*Regions, error) {
-	addrs, err := ResolveAddrs(hostnames)
-	if err != nil {
-		return nil, err
+func StaticEdge(hostnames []string, logger logger.Service) (*Regions, error) {
+	resolved := ResolveAddrs(hostnames, logger)
+	if len(resolved) == 0 {
+		return nil, fmt.Errorf("failed to resolve any edge address")
 	}
-	return NewNoResolve(addrs), nil
+	return NewNoResolve(resolved), nil
 }
 
 // NewNoResolve doesn't resolve the edge. Instead it just uses the given addresses.

edgediscovery/dial.go

@@ -1,4 +1,4 @@
-package connection
+package edgediscovery
 
 import (
 	"context"
@@ -9,7 +9,7 @@ import (
 	"github.com/pkg/errors"
 )
 
-// DialEdge makes a TLS connection to a Cloudflare edge node
+// DialEdgeWithH2Mux makes a TLS connection to a Cloudflare edge node
 func DialEdge(
 	ctx context.Context,
 	timeout time.Duration,
@@ -25,6 +25,7 @@ func DialEdge(
 	if err != nil {
 		return nil, newDialError(err, "DialContext error")
 	}
+
 	tlsEdgeConn := tls.Client(edgeConn, tlsConfig)
 	tlsEdgeConn.SetDeadline(time.Now().Add(timeout))
 

edgediscovery/edgediscovery.go

@@ -41,7 +41,7 @@ func ResolveEdge(l logger.Service) (*Edge, error) {
 
 // StaticEdge creates a list of edge addresses from the list of hostnames. Mainly used for testing connectivity.
 func StaticEdge(l logger.Service, hostnames []string) (*Edge, error) {
-	regions, err := allregions.StaticEdge(hostnames)
+	regions, err := allregions.StaticEdge(hostnames, l)
 	if err != nil {
 		return new(Edge), err
 	}

edgediscovery/mocks_for_test.go

@@ -1,4 +1,4 @@
-package connection
+package edgediscovery
 
 import (
 	"fmt"

edgediscovery/protocol.go

@@ -0,0 +1,45 @@
+package edgediscovery
+
+import (
+	"fmt"
+	"net"
+	"strconv"
+	"strings"
+)
+
+const (
+	protocolRecord = "protocol.argotunnel.com"
+)
+
+var (
+	errNoProtocolRecord = fmt.Errorf("No TXT record found for %s to determine connection protocol", protocolRecord)
+)
+
+func HTTP2Percentage() (int32, error) {
+	records, err := net.LookupTXT(protocolRecord)
+	if err != nil {
+		return 0, err
+	}
+	if len(records) == 0 {
+		return 0, errNoProtocolRecord
+	}
+	return parseHTTP2Precentage(records[0])
+}
+
+// The record looks like http2=percentage
+func parseHTTP2Precentage(record string) (int32, error) {
+	const key = "http2"
+	slices := strings.Split(record, "=")
+	if len(slices) != 2 {
+		return 0, fmt.Errorf("Malformed TXT record %s, expect http2=percentage", record)
+	}
+	if slices[0] != key {
+		return 0, fmt.Errorf("Incorrect key %s, expect %s", slices[0], key)
+	}
+	percentage, err := strconv.ParseInt(slices[1], 10, 32)
+	if err != nil {
+		return 0, err
+	}
+	return int32(percentage), nil
+
+}

edgediscovery/protocol_test.go

@@ -0,0 +1,80 @@
+package edgediscovery
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestHTTP2Percentage(t *testing.T) {
+	_, err := HTTP2Percentage()
+	assert.NoError(t, err)
+}
+
+func TestParseHTTP2Precentage(t *testing.T) {
+	tests := []struct {
+		record     string
+		percentage int32
+		wantErr    bool
+	}{
+		{
+			record:     "http2=-1",
+			percentage: -1,
+			wantErr:    false,
+		},
+		{
+			record:     "http2=0",
+			percentage: 0,
+			wantErr:    false,
+		},
+		{
+			record:     "http2=50",
+			percentage: 50,
+			wantErr:    false,
+		},
+		{
+			record:     "http2=100",
+			percentage: 100,
+			wantErr:    false,
+		},
+		{
+			record:     "http2=1000",
+			percentage: 1000,
+			wantErr:    false,
+		},
+		{
+			record:  "http2=10.5",
+			wantErr: true,
+		},
+		{
+			record:  "http2=10 h2mux=90",
+			wantErr: true,
+		},
+		{
+			record:  "http2=ten",
+			wantErr: true,
+		},
+
+		{
+			record:  "h2mux=100",
+			wantErr: true,
+		},
+		{
+			record:  "http2",
+			wantErr: true,
+		},
+		{
+			record:  "http2=",
+			wantErr: true,
+		},
+	}
+
+	for _, test := range tests {
+		p, err := parseHTTP2Precentage(test.record)
+		if test.wantErr {
+			assert.Error(t, err)
+		} else {
+			assert.Equal(t, test.percentage, p)
+		}
+	}
+}

go.mod

@@ -15,7 +15,7 @@ require (
 	github.com/coreos/go-oidc v0.0.0-20171002155002-a93f71fdfe73
 	github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf
 	github.com/denisenkom/go-mssqldb v0.0.0-20191001013358-cfbb681360f0
-	github.com/equinox-io/equinox v1.2.0
+	github.com/equinox-io/equinox v1.2.0 // indirect
 	github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 // indirect
 	github.com/facebookgo/freeport v0.0.0-20150612182905-d4adf43b75b9 // indirect
 	github.com/facebookgo/grace v0.0.0-20180706040059-75cf19382434
@@ -27,12 +27,16 @@ require (
 	github.com/getsentry/raven-go v0.0.0-20180517221441-ed7bcb39ff10
 	github.com/gliderlabs/ssh v0.0.0-20191009160644-63518b5243e0
 	github.com/go-sql-driver/mysql v1.5.0
+	github.com/gobwas/httphead v0.0.0-20200921212729-da3d93bc3c58 // indirect
+	github.com/gobwas/pool v0.2.1 // indirect
+	github.com/gobwas/ws v1.0.4
 	github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3
 	github.com/google/go-cmp v0.5.2 // indirect
 	github.com/google/uuid v1.1.2
 	github.com/gorilla/mux v1.7.3
 	github.com/gorilla/websocket v1.4.2
 	github.com/jmoiron/sqlx v1.2.0
+	github.com/json-iterator/go v1.1.10
 	github.com/kr/text v0.2.0 // indirect
 	github.com/kshvakov/clickhouse v1.3.11
 	github.com/kylelemons/godebug v1.1.0 // indirect

go.sum

@@ -233,6 +233,12 @@ github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG
 github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/gobwas/httphead v0.0.0-20200921212729-da3d93bc3c58 h1:YyrUZvJaU8Q0QsoVo+xLFBgWDTam29PKea6GYmwvSiQ=
+github.com/gobwas/httphead v0.0.0-20200921212729-da3d93bc3c58/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo=
+github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og=
+github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
+github.com/gobwas/ws v1.0.4 h1:5eXU1CZhpQdq5kXbKb+sECH5Ia5KiO6CYzIzdlVx6Bs=
+github.com/gobwas/ws v1.0.4/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=
 github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
 github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
@@ -369,6 +375,7 @@ github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV
 github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
@@ -441,8 +448,10 @@ github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0Qu
 github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=

h2mux/activestreammap.go

@@ -7,6 +7,19 @@ import (
 	"golang.org/x/net/http2"
 )
 
+var (
+	ActiveStreams = prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace: "cloudflared",
+		Subsystem: "tunnel",
+		Name:      "active_streams",
+		Help:      "Number of active streams created by all muxers.",
+	})
+)
+
+func init() {
+	prometheus.MustRegister(ActiveStreams)
+}
+
 // activeStreamMap is used to moderate access to active streams between the read and write
 // threads, and deny access to new peer streams while shutting down.
 type activeStreamMap struct {

h2mux/activestreammap_test.go

@@ -9,7 +9,7 @@ import (
 
 func TestShutdown(t *testing.T) {
 	const numStreams = 1000
-	m := newActiveStreamMap(true, NewActiveStreamsMetrics("test", t.Name()))
+	m := newActiveStreamMap(true, ActiveStreams)
 
 	// Add all the streams
 	{
@@ -62,7 +62,7 @@ func TestShutdown(t *testing.T) {
 
 func TestEmptyBeforeShutdown(t *testing.T) {
 	const numStreams = 1000
-	m := newActiveStreamMap(true, NewActiveStreamsMetrics("test", t.Name()))
+	m := newActiveStreamMap(true, ActiveStreams)
 
 	// Add all the streams
 	{
@@ -138,7 +138,7 @@ func (_ *noopReadyList) Signal(streamID uint32) {}
 
 func TestAbort(t *testing.T) {
 	const numStreams = 1000
-	m := newActiveStreamMap(true, NewActiveStreamsMetrics("test", t.Name()))
+	m := newActiveStreamMap(true, ActiveStreams)
 
 	var openedStreams sync.Map
 

h2mux/h2mux_test.go

@@ -113,11 +113,11 @@ func (p *DefaultMuxerPair) Handshake(testName string) error {
 	defer cancel()
 	errGroup, _ := errgroup.WithContext(ctx)
 	errGroup.Go(func() (err error) {
-		p.EdgeMux, err = Handshake(p.EdgeConn, p.EdgeConn, p.EdgeMuxConfig, NewActiveStreamsMetrics(testName, "edge"))
+		p.EdgeMux, err = Handshake(p.EdgeConn, p.EdgeConn, p.EdgeMuxConfig, ActiveStreams)
 		return errors.Wrap(err, "edge handshake failure")
 	})
 	errGroup.Go(func() (err error) {
-		p.OriginMux, err = Handshake(p.OriginConn, p.OriginConn, p.OriginMuxConfig, NewActiveStreamsMetrics(testName, "origin"))
+		p.OriginMux, err = Handshake(p.OriginConn, p.OriginConn, p.OriginMuxConfig, ActiveStreams)
 		return errors.Wrap(err, "origin handshake failure")
 	})
 

h2mux/header.go

@@ -2,7 +2,6 @@ package h2mux
 
 import (
 	"encoding/base64"
-	"encoding/json"
 	"fmt"
 	"net/http"
 	"net/url"
@@ -22,10 +21,6 @@ const (
 	RequestUserHeadersField  = "cf-cloudflared-request-headers"
 	ResponseUserHeadersField = "cf-cloudflared-response-headers"
 
-	ResponseMetaHeaderField   = "cf-cloudflared-response-meta"
-	ResponseSourceCloudflared = "cloudflared"
-	ResponseSourceOrigin      = "origin"
-
 	CFAccessTokenHeader        = "cf-access-token"
 	CFJumpDestinationHeader    = "CF-Access-Jump-Destination"
 	CFAccessClientIDHeader     = "CF-Access-Client-Id"
@@ -124,7 +119,7 @@ func IsControlHeader(headerName string) bool {
 }
 
 // isWebsocketClientHeader returns true if the header name is required by the client to upgrade properly
-func isWebsocketClientHeader(headerName string) bool {
+func IsWebsocketClientHeader(headerName string) bool {
 	return headerName == "sec-websocket-accept" ||
 		headerName == "connection" ||
 		headerName == "upgrade"
@@ -143,7 +138,7 @@ func H1ResponseToH2ResponseHeaders(h1 *http.Response) (h2 []Header) {
 
 			// Since these are http2 headers, they're required to be lowercase
 			h2 = append(h2, Header{Name: "content-length", Value: values[0]})
-		} else if !IsControlHeader(h2name) || isWebsocketClientHeader(h2name) {
+		} else if !IsControlHeader(h2name) || IsWebsocketClientHeader(h2name) {
 			// User headers, on the other hand, must all be serialized so that
 			// HTTP/2 header validation won't be applied to HTTP/1 header values
 			userHeaders[header] = values
@@ -152,7 +147,6 @@ func H1ResponseToH2ResponseHeaders(h1 *http.Response) (h2 []Header) {
 
 	// Perform user header serialization and set them in the single header
 	h2 = append(h2, Header{ResponseUserHeadersField, SerializeHeaders(userHeaders)})
-
 	return h2
 }
 
@@ -238,19 +232,3 @@ func DeserializeHeaders(serializedHeaders string) ([]Header, error) {
 
 	return deserialized, nil
 }
-
-type ResponseMetaHeader struct {
-	Source string `json:"src"`
-}
-
-func CreateResponseMetaHeader(headerName, source string) Header {
-	jsonResponseMetaHeader, err := json.Marshal(ResponseMetaHeader{Source: source})
-	if err != nil {
-		panic(err)
-	}
-
-	return Header{
-		Name:  headerName,
-		Value: string(jsonResponseMetaHeader),
-	}
-}

h2mux/muxmetrics.go

@@ -6,7 +6,6 @@ import (
 
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/golang-collections/collections/queue"
-	"github.com/prometheus/client_golang/prometheus"
 )
 
 // data points used to compute average receive window and send window size
@@ -295,14 +294,3 @@ func (r *rate) get() (curr, min, max uint64) {
 	defer r.lock.RUnlock()
 	return r.curr, r.min, r.max
 }
-
-func NewActiveStreamsMetrics(namespace, subsystem string) prometheus.Gauge {
-	activeStreams := prometheus.NewGauge(prometheus.GaugeOpts{
-		Namespace: namespace,
-		Subsystem: subsystem,
-		Name:      "active_streams",
-		Help:      "Number of active streams created by all muxers.",
-	})
-	prometheus.MustRegister(activeStreams)
-	return activeStreams
-}

hello/hello.go

@@ -18,6 +18,14 @@ import (
 	"github.com/cloudflare/cloudflared/tlsconfig"
 )
 
+const (
+	UptimeRoute    = "/uptime"
+	WSRoute        = "/ws"
+	SSERoute       = "/sse"
+	HealthRoute    = "/_health"
+	defaultSSEFreq = time.Second * 10
+)
+
 type templateData struct {
 	ServerName string
 	Request    *http.Request
@@ -104,8 +112,10 @@ func StartHelloWorldServer(logger logger.Service, listener net.Listener, shutdow
 	}
 
 	muxer := http.NewServeMux()
-	muxer.HandleFunc("/uptime", uptimeHandler(time.Now()))
-	muxer.HandleFunc("/ws", websocketHandler(logger, upgrader))
+	muxer.HandleFunc(UptimeRoute, uptimeHandler(time.Now()))
+	muxer.HandleFunc(WSRoute, websocketHandler(logger, upgrader))
+	muxer.HandleFunc(SSERoute, sseHandler(logger))
+	muxer.HandleFunc(HealthRoute, healthHandler())
 	muxer.HandleFunc("/", rootHandler(serverName))
 	httpServer := &http.Server{Addr: listener.Addr().String(), Handler: muxer}
 	go func() {
@@ -150,8 +160,15 @@ func uptimeHandler(startTime time.Time) http.HandlerFunc {
 // This handler will echo message
 func websocketHandler(logger logger.Service, upgrader websocket.Upgrader) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
+		// This addresses the issue of r.Host includes port but origin header doesn't
+		host, _, err := net.SplitHostPort(r.Host)
+		if err == nil {
+			r.Host = host
+		}
+
 		conn, err := upgrader.Upgrade(w, r, nil)
 		if err != nil {
+			logger.Errorf("failed to upgrade to websocket connection, error: %s", err)
 			return
 		}
 		defer conn.Close()
@@ -170,6 +187,48 @@ func websocketHandler(logger logger.Service, upgrader websocket.Upgrader) http.H
 	}
 }
 
+func sseHandler(logger logger.Service) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/event-stream")
+		flusher, ok := w.(http.Flusher)
+		if !ok {
+			w.WriteHeader(http.StatusInternalServerError)
+			logger.Errorf("Can't support SSE. ResponseWriter %T doesn't implement http.Flusher interface", w)
+			return
+		}
+
+		freq := defaultSSEFreq
+		if requestedFreq := r.URL.Query()["freq"]; len(requestedFreq) > 0 {
+			parsedFreq, err := time.ParseDuration(requestedFreq[0])
+			if err == nil {
+				freq = parsedFreq
+			}
+		}
+		logger.Infof("Server Sent Events every %s", freq)
+		ticker := time.NewTicker(freq)
+		counter := 0
+		for {
+			select {
+			case <-r.Context().Done():
+				return
+			case <-ticker.C:
+			}
+			_, err := fmt.Fprintf(w, "%d\n\n", counter)
+			if err != nil {
+				return
+			}
+			flusher.Flush()
+			counter++
+		}
+	}
+}
+
+func healthHandler() http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		w.Write([]byte("ok"))
+	}
+}
+
 func rootHandler(serverName string) http.HandlerFunc {
 	responseTemplate := template.Must(template.New("index").Parse(indexTemplate))
 	return func(w http.ResponseWriter, r *http.Request) {

ingress/ingress.go

@@ -2,70 +2,44 @@ package ingress
 
 import (
 	"fmt"
+	"net"
 	"net/url"
 	"regexp"
+	"strconv"
 	"strings"
+	"sync"
 
 	"github.com/pkg/errors"
+	"github.com/urfave/cli/v2"
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/logger"
 )
 
 var (
-	ErrNoIngressRules             = errors.New("No ingress rules were specified in the config file")
-	errLastRuleNotCatchAll        = errors.New("The last ingress rule must match all hostnames (i.e. it must be missing, or must be \"*\")")
+	ErrNoIngressRules             = errors.New("The config file doesn't contain any ingress rules")
+	errLastRuleNotCatchAll        = errors.New("The last ingress rule must match all URLs (i.e. it should not have a hostname or path filter)")
 	errBadWildcard                = errors.New("Hostname patterns can have at most one wildcard character (\"*\") and it can only be used for subdomains, e.g. \"*.example.com\"")
+	errHostnameContainsPort       = errors.New("Hostname cannot contain a port")
 	ErrURLIncompatibleWithIngress = errors.New("You can't set the --url flag (or $TUNNEL_URL) when using multiple-origin ingress rules")
 )
 
-// Each rule route traffic from a hostname/path on the public
-// internet to the service running on the given URL.
-type Rule struct {
-	// Requests for this hostname will be proxied to this rule's service.
-	Hostname string
-
-	// Path is an optional regex that can specify path-driven ingress rules.
-	Path *regexp.Regexp
-
-	// A (probably local) address. Requests for a hostname which matches this
-	// rule's hostname pattern will be proxied to the service running on this
-	// address.
-	Service *url.URL
-}
-
-func (r Rule) MultiLineString() string {
-	var out strings.Builder
-	if r.Hostname != "" {
-		out.WriteString("\thostname: ")
-		out.WriteString(r.Hostname)
-		out.WriteRune('\n')
-	}
-	if r.Path != nil {
-		out.WriteString("\tpath: ")
-		out.WriteString(r.Path.String())
-		out.WriteRune('\n')
-	}
-	out.WriteString("\tservice: ")
-	out.WriteString(r.Service.String())
-	return out.String()
-}
-
-func (r *Rule) Matches(hostname, path string) bool {
-	hostMatch := r.Hostname == "" || r.Hostname == "*" || matchHost(r.Hostname, hostname)
-	pathMatch := r.Path == nil || r.Path.MatchString(path)
-	return hostMatch && pathMatch
-}
-
 // FindMatchingRule returns the index of the Ingress Rule which matches the given
 // hostname and path. This function assumes the last rule matches everything,
 // which is the case if the rules were instantiated via the ingress#Validate method
-func (ing Ingress) FindMatchingRule(hostname, path string) int {
+func (ing Ingress) FindMatchingRule(hostname, path string) (*Rule, int) {
+	// The hostname might contain port. We only want to compare the host part with the rule
+	host, _, err := net.SplitHostPort(hostname)
+	if err == nil {
+		hostname = host
+	}
 	for i, rule := range ing.Rules {
 		if rule.Matches(hostname, path) {
-			return i
+			return &rule, i
 		}
 	}
-	return len(ing.Rules) - 1
+	i := len(ing.Rules) - 1
+	return &ing.Rules[i], i
 }
 
 func matchHost(ruleHost, reqHost string) bool {
@@ -83,7 +57,54 @@ func matchHost(ruleHost, reqHost string) bool {
 
 // Ingress maps eyeball requests to origins.
 type Ingress struct {
-	Rules []Rule
+	Rules    []Rule
+	defaults OriginRequestConfig
+}
+
+// NewSingleOrigin constructs an Ingress set with only one rule, constructed from
+// legacy CLI parameters like --url or --no-chunked-encoding.
+func NewSingleOrigin(c *cli.Context, allowURLFromArgs bool, logger logger.Service) (Ingress, error) {
+
+	service, err := parseSingleOriginService(c, allowURLFromArgs)
+	if err != nil {
+		return Ingress{}, err
+	}
+
+	// Construct an Ingress with the single rule.
+	defaults := originRequestFromSingeRule(c)
+	ing := Ingress{
+		Rules: []Rule{
+			{
+				Service: service,
+				Config:  setConfig(defaults, config.OriginRequestConfig{}),
+			},
+		},
+		defaults: defaults,
+	}
+	return ing, err
+}
+
+// Get a single origin service from the CLI/config.
+func parseSingleOriginService(c *cli.Context, allowURLFromArgs bool) (OriginService, error) {
+	if c.IsSet("hello-world") {
+		return new(helloWorld), nil
+	}
+	if c.IsSet("url") || c.IsSet(config.BastionFlag) {
+		originURL, err := config.ValidateUrl(c, allowURLFromArgs)
+		if err != nil {
+			return nil, errors.Wrap(err, "Error validating origin URL")
+		}
+		return &localService{URL: originURL, RootURL: originURL}, nil
+	}
+	if c.IsSet("unix-socket") {
+		path, err := config.ValidateUnixSocket(c)
+		if err != nil {
+			return nil, errors.Wrap(err, "Error validating --unix-socket")
+		}
+		return &unixSocketPath{path: path}, nil
+	}
+	u, err := url.Parse("http://localhost:8080")
+	return &localService{URL: u, RootURL: u}, err
 }
 
 // IsEmpty checks if there are any ingress rules.
@@ -91,40 +112,70 @@ func (ing Ingress) IsEmpty() bool {
 	return len(ing.Rules) == 0
 }
 
-func validate(ingress []config.UnvalidatedIngressRule) (Ingress, error) {
+// StartOrigins will start any origin services managed by cloudflared, e.g. proxy servers or Hello World.
+func (ing Ingress) StartOrigins(wg *sync.WaitGroup, log logger.Service, shutdownC <-chan struct{}, errC chan error) {
+	for _, rule := range ing.Rules {
+		if err := rule.Service.start(wg, log, shutdownC, errC, rule.Config); err != nil {
+			log.Errorf("Error starting local service %s: %s", rule.Service, err)
+		}
+	}
+}
+
+// CatchAll returns the catch-all rule (i.e. the last rule)
+func (ing Ingress) CatchAll() *Rule {
+	return &ing.Rules[len(ing.Rules)-1]
+}
+
+func validate(ingress []config.UnvalidatedIngressRule, defaults OriginRequestConfig) (Ingress, error) {
 	rules := make([]Rule, len(ingress))
 	for i, r := range ingress {
-		service, err := url.Parse(r.Service)
-		if err != nil {
+		cfg := setConfig(defaults, r.OriginRequest)
+		var service OriginService
+
+		if prefix := "unix:"; strings.HasPrefix(r.Service, prefix) {
+			// No validation necessary for unix socket filepath services
+			path := strings.TrimPrefix(r.Service, prefix)
+			service = &unixSocketPath{path: path}
+		} else if prefix := "http_status:"; strings.HasPrefix(r.Service, prefix) {
+			status, err := strconv.Atoi(strings.TrimPrefix(r.Service, prefix))
+			if err != nil {
+				return Ingress{}, errors.Wrap(err, "invalid HTTP status")
+			}
+			srv := newStatusCode(status)
+			service = &srv
+		} else if r.Service == "hello_world" || r.Service == "hello-world" || r.Service == "helloworld" {
+			service = new(helloWorld)
+		} else if r.Service == "bastion" || cfg.BastionMode {
+			// Bastion mode will always start a Websocket proxy server, which will
+			// overwrite the localService.URL field when `start` is called. So,
+			// leave the URL field empty for now.
+			cfg.BastionMode = true
+			service = new(localService)
+		} else {
+			// Validate URL services
+			u, err := url.Parse(r.Service)
+			if err != nil {
+				return Ingress{}, err
+			}
+
+			if u.Scheme == "" || u.Hostname() == "" {
+				return Ingress{}, fmt.Errorf("%s is an invalid address, please make sure it has a scheme and a hostname", r.Service)
+			}
+
+			if u.Path != "" {
+				return Ingress{}, fmt.Errorf("%s is an invalid address, ingress rules don't support proxying to a different path on the origin service. The path will be the same as the eyeball request's path", r.Service)
+			}
+			serviceURL := localService{URL: u}
+			service = &serviceURL
+		}
+
+		if err := validateHostname(r, i, len(ingress)); err != nil {
 			return Ingress{}, err
 		}
-		if service.Scheme == "" || service.Hostname() == "" {
-			return Ingress{}, fmt.Errorf("The service %s must have a scheme and a hostname", r.Service)
-		}
-
-		if service.Path != "" {
-			return Ingress{}, fmt.Errorf("%s is an invalid address, ingress rules don't support proxying to a different path on the origin service. The path will be the same as the eyeball request's path.", r.Service)
-		}
-
-		// Ensure that there are no wildcards anywhere except the first character
-		// of the hostname.
-		if strings.LastIndex(r.Hostname, "*") > 0 {
-			return Ingress{}, errBadWildcard
-		}
-
-		// The last rule should catch all hostnames.
-		isCatchAllRule := (r.Hostname == "" || r.Hostname == "*") && r.Path == ""
-		isLastRule := i == len(ingress)-1
-		if isLastRule && !isCatchAllRule {
-			return Ingress{}, errLastRuleNotCatchAll
-		}
-		// ONLY the last rule should catch all hostnames.
-		if !isLastRule && isCatchAllRule {
-			return Ingress{}, errRuleShouldNotBeCatchAll{i: i, hostname: r.Hostname}
-		}
 
 		var pathRegex *regexp.Regexp
 		if r.Path != "" {
+			var err error
 			pathRegex, err = regexp.Compile(r.Path)
 			if err != nil {
 				return Ingress{}, errors.Wrapf(err, "Rule #%d has an invalid regex", i+1)
@@ -135,25 +186,52 @@ func validate(ingress []config.UnvalidatedIngressRule) (Ingress, error) {
 			Hostname: r.Hostname,
 			Service:  service,
 			Path:     pathRegex,
+			Config:   cfg,
 		}
 	}
-	return Ingress{Rules: rules}, nil
+	return Ingress{Rules: rules, defaults: defaults}, nil
+}
+
+func validateHostname(r config.UnvalidatedIngressRule, ruleIndex, totalRules int) error {
+	// Ensure that the hostname doesn't contain port
+	_, _, err := net.SplitHostPort(r.Hostname)
+	if err == nil {
+		return errHostnameContainsPort
+	}
+	// Ensure that there are no wildcards anywhere except the first character
+	// of the hostname.
+	if strings.LastIndex(r.Hostname, "*") > 0 {
+		return errBadWildcard
+	}
+
+	// The last rule should catch all hostnames.
+	isCatchAllRule := (r.Hostname == "" || r.Hostname == "*") && r.Path == ""
+	isLastRule := ruleIndex == totalRules-1
+	if isLastRule && !isCatchAllRule {
+		return errLastRuleNotCatchAll
+	}
+	// ONLY the last rule should catch all hostnames.
+	if !isLastRule && isCatchAllRule {
+		return errRuleShouldNotBeCatchAll{index: ruleIndex, hostname: r.Hostname}
+	}
+	return nil
 }
 
 type errRuleShouldNotBeCatchAll struct {
-	i        int
+	index    int
 	hostname string
 }
 
 func (e errRuleShouldNotBeCatchAll) Error() string {
 	return fmt.Sprintf("Rule #%d is matching the hostname '%s', but "+
 		"this will match every hostname, meaning the rules which follow it "+
-		"will never be triggered.", e.i+1, e.hostname)
+		"will never be triggered.", e.index+1, e.hostname)
 }
 
+// ParseIngress parses ingress rules, but does not send HTTP requests to the origins.
 func ParseIngress(conf *config.Configuration) (Ingress, error) {
 	if len(conf.Ingress) == 0 {
 		return Ingress{}, ErrNoIngressRules
 	}
-	return validate(conf.Ingress)
+	return validate(conf.Ingress, originRequestFromYAML(conf.OriginRequest))
 }

ingress/ingress_test.go

@@ -1,27 +1,48 @@
 package ingress
 
 import (
+	"flag"
+	"fmt"
 	"net/url"
 	"regexp"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"github.com/urfave/cli/v2"
 	"gopkg.in/yaml.v2"
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/logger"
+	"github.com/cloudflare/cloudflared/tlsconfig"
 )
 
+func TestParseUnixSocket(t *testing.T) {
+	rawYAML := `
+ingress:
+- service: unix:/tmp/echo.sock
+`
+	ing, err := ParseIngress(MustReadIngress(rawYAML))
+	require.NoError(t, err)
+	_, ok := ing.Rules[0].Service.(*unixSocketPath)
+	require.True(t, ok)
+}
+
 func Test_parseIngress(t *testing.T) {
 	localhost8000 := MustParseURL(t, "https://localhost:8000")
 	localhost8001 := MustParseURL(t, "https://localhost:8001")
+	fourOhFour := newStatusCode(404)
+	defaultConfig := setConfig(originRequestFromYAML(config.OriginRequestConfig{}), config.OriginRequestConfig{})
+	require.Equal(t, defaultKeepAliveConnections, defaultConfig.KeepAliveConnections)
+	tr := true
 	type args struct {
 		rawYAML string
 	}
 	tests := []struct {
 		name    string
 		args    args
-		want    Ingress
+		want    []Rule
 		wantErr bool
 	}{
 		{
@@ -38,16 +59,18 @@ ingress:
  - hostname: "*"
    service: https://localhost:8001
 `},
-			want: Ingress{Rules: []Rule{
+			want: []Rule{
 				{
 					Hostname: "tunnel1.example.com",
-					Service:  localhost8000,
+					Service:  &localService{URL: localhost8000},
+					Config:   defaultConfig,
 				},
 				{
 					Hostname: "*",
-					Service:  localhost8001,
+					Service:  &localService{URL: localhost8001},
+					Config:   defaultConfig,
 				},
-			}},
+			},
 		},
 		{
 			name: "Extra keys",
@@ -57,12 +80,13 @@ ingress:
    service: https://localhost:8000
 extraKey: extraValue
 `},
-			want: Ingress{Rules: []Rule{
+			want: []Rule{
 				{
 					Hostname: "*",
-					Service:  localhost8000,
+					Service:  &localService{URL: localhost8000},
+					Config:   defaultConfig,
 				},
-			}},
+			},
 		},
 		{
 			name: "Hostname can be omitted",
@@ -70,11 +94,12 @@ extraKey: extraValue
 ingress:
  - service: https://localhost:8000
 `},
-			want: Ingress{Rules: []Rule{
+			want: []Rule{
 				{
-					Service: localhost8000,
+					Service: &localService{URL: localhost8000},
+					Config:  defaultConfig,
 				},
-			}},
+			},
 		},
 		{
 			name: "Invalid service",
@@ -146,6 +171,94 @@ ingress:
 			args: args{rawYAML: `
 ingress:
  - service: https://localhost:8000/static/
+`},
+			wantErr: true,
+		},
+		{
+			name: "Invalid HTTP status",
+			args: args{rawYAML: `
+ingress:
+ - service: http_status:asdf
+`},
+			wantErr: true,
+		},
+		{
+			name: "Valid HTTP status",
+			args: args{rawYAML: `
+ingress:
+ - service: http_status:404
+`},
+			want: []Rule{
+				{
+					Hostname: "",
+					Service:  &fourOhFour,
+					Config:   defaultConfig,
+				},
+			},
+		},
+		{
+			name: "Valid hello world service",
+			args: args{rawYAML: `
+ingress:
+ - service: hello_world
+`},
+			want: []Rule{
+				{
+					Hostname: "",
+					Service:  new(helloWorld),
+					Config:   defaultConfig,
+				},
+			},
+		},
+		{
+			name: "URL isn't necessary if using bastion",
+			args: args{rawYAML: `
+ingress:
+- hostname: bastion.foo.com
+  originRequest:
+    bastionMode: true
+- service: http_status:404
+`},
+			want: []Rule{
+				{
+					Hostname: "bastion.foo.com",
+					Service:  &localService{},
+					Config:   setConfig(originRequestFromYAML(config.OriginRequestConfig{}), config.OriginRequestConfig{BastionMode: &tr}),
+				},
+				{
+					Service: &fourOhFour,
+					Config:  defaultConfig,
+				},
+			},
+		},
+		{
+			name: "Bastion service",
+			args: args{rawYAML: `
+ingress:
+- hostname: bastion.foo.com
+  service: bastion
+- service: http_status:404
+`},
+			want: []Rule{
+				{
+					Hostname: "bastion.foo.com",
+					Service:  &localService{},
+					Config:   setConfig(originRequestFromYAML(config.OriginRequestConfig{}), config.OriginRequestConfig{BastionMode: &tr}),
+				},
+				{
+					Service: &fourOhFour,
+					Config:  defaultConfig,
+				},
+			},
+		},
+		{
+			name: "Hostname contains port",
+			args: args{rawYAML: `
+ingress:
+ - hostname: "test.example.com:443"
+   service: https://localhost:8000
+ - hostname: "*"
+   service: https://localhost:8001
 `},
 			wantErr: true,
 		},
@@ -157,129 +270,159 @@ ingress:
 				t.Errorf("ParseIngress() error = %v, wantErr %v", err, tt.wantErr)
 				return
 			}
-			assert.Equal(t, tt.want, got)
+			require.Equal(t, tt.want, got.Rules)
 		})
 	}
 }
 
+func TestSingleOriginSetsConfig(t *testing.T) {
+	flagSet := flag.NewFlagSet(t.Name(), flag.PanicOnError)
+	flagSet.Bool("hello-world", true, "")
+	flagSet.Duration(ProxyConnectTimeoutFlag, time.Second, "")
+	flagSet.Duration(ProxyTLSTimeoutFlag, time.Second, "")
+	flagSet.Duration(ProxyTCPKeepAlive, time.Second, "")
+	flagSet.Bool(ProxyNoHappyEyeballsFlag, true, "")
+	flagSet.Int(ProxyKeepAliveConnectionsFlag, 10, "")
+	flagSet.Duration(ProxyKeepAliveTimeoutFlag, time.Second, "")
+	flagSet.String(HTTPHostHeaderFlag, "example.com:8080", "")
+	flagSet.String(OriginServerNameFlag, "example.com", "")
+	flagSet.String(tlsconfig.OriginCAPoolFlag, "/etc/certs/ca.pem", "")
+	flagSet.Bool(NoTLSVerifyFlag, true, "")
+	flagSet.Bool(NoChunkedEncodingFlag, true, "")
+	flagSet.Bool(config.BastionFlag, true, "")
+	flagSet.String(ProxyAddressFlag, "localhost:8080", "")
+	flagSet.Uint(ProxyPortFlag, 8080, "")
+	flagSet.Bool(Socks5Flag, true, "")
+
+	cliCtx := cli.NewContext(cli.NewApp(), flagSet, nil)
+	err := cliCtx.Set("hello-world", "true")
+	require.NoError(t, err)
+	err = cliCtx.Set(ProxyConnectTimeoutFlag, "1s")
+	require.NoError(t, err)
+	err = cliCtx.Set(ProxyTLSTimeoutFlag, "1s")
+	require.NoError(t, err)
+	err = cliCtx.Set(ProxyTCPKeepAlive, "1s")
+	require.NoError(t, err)
+	err = cliCtx.Set(ProxyNoHappyEyeballsFlag, "true")
+	require.NoError(t, err)
+	err = cliCtx.Set(ProxyKeepAliveConnectionsFlag, "10")
+	require.NoError(t, err)
+	err = cliCtx.Set(ProxyKeepAliveTimeoutFlag, "1s")
+	require.NoError(t, err)
+	err = cliCtx.Set(HTTPHostHeaderFlag, "example.com:8080")
+	require.NoError(t, err)
+	err = cliCtx.Set(OriginServerNameFlag, "example.com")
+	require.NoError(t, err)
+	err = cliCtx.Set(tlsconfig.OriginCAPoolFlag, "/etc/certs/ca.pem")
+	require.NoError(t, err)
+	err = cliCtx.Set(NoTLSVerifyFlag, "true")
+	require.NoError(t, err)
+	err = cliCtx.Set(NoChunkedEncodingFlag, "true")
+	require.NoError(t, err)
+	err = cliCtx.Set(config.BastionFlag, "true")
+	require.NoError(t, err)
+	err = cliCtx.Set(ProxyAddressFlag, "localhost:8080")
+	require.NoError(t, err)
+	err = cliCtx.Set(ProxyPortFlag, "8080")
+	require.NoError(t, err)
+	err = cliCtx.Set(Socks5Flag, "true")
+	require.NoError(t, err)
+
+	allowURLFromArgs := false
+	logger, err := logger.New()
+	require.NoError(t, err)
+	ingress, err := NewSingleOrigin(cliCtx, allowURLFromArgs, logger)
+	require.NoError(t, err)
+
+	assert.Equal(t, time.Second, ingress.Rules[0].Config.ConnectTimeout)
+	assert.Equal(t, time.Second, ingress.Rules[0].Config.TLSTimeout)
+	assert.Equal(t, time.Second, ingress.Rules[0].Config.TCPKeepAlive)
+	assert.True(t, ingress.Rules[0].Config.NoHappyEyeballs)
+	assert.Equal(t, 10, ingress.Rules[0].Config.KeepAliveConnections)
+	assert.Equal(t, time.Second, ingress.Rules[0].Config.KeepAliveTimeout)
+	assert.Equal(t, "example.com:8080", ingress.Rules[0].Config.HTTPHostHeader)
+	assert.Equal(t, "example.com", ingress.Rules[0].Config.OriginServerName)
+	assert.Equal(t, "/etc/certs/ca.pem", ingress.Rules[0].Config.CAPool)
+	assert.True(t, ingress.Rules[0].Config.NoTLSVerify)
+	assert.True(t, ingress.Rules[0].Config.DisableChunkedEncoding)
+	assert.True(t, ingress.Rules[0].Config.BastionMode)
+	assert.Equal(t, "localhost:8080", ingress.Rules[0].Config.ProxyAddress)
+	assert.Equal(t, uint(8080), ingress.Rules[0].Config.ProxyPort)
+	assert.Equal(t, socksProxy, ingress.Rules[0].Config.ProxyType)
+}
+
+func TestFindMatchingRule(t *testing.T) {
+	ingress := Ingress{
+		Rules: []Rule{
+			{
+				Hostname: "tunnel-a.example.com",
+				Path:     nil,
+			},
+			{
+				Hostname: "tunnel-b.example.com",
+				Path:     mustParsePath(t, "/health"),
+			},
+			{
+				Hostname: "*",
+			},
+		},
+	}
+
+	tests := []struct {
+		host          string
+		path          string
+		wantRuleIndex int
+	}{
+		{
+			host:          "tunnel-a.example.com",
+			path:          "/",
+			wantRuleIndex: 0,
+		},
+		{
+			host:          "tunnel-a.example.com",
+			path:          "/pages/about",
+			wantRuleIndex: 0,
+		},
+		{
+			host:          "tunnel-a.example.com:443",
+			path:          "/pages/about",
+			wantRuleIndex: 0,
+		},
+		{
+			host:          "tunnel-b.example.com",
+			path:          "/health",
+			wantRuleIndex: 1,
+		},
+		{
+			host:          "tunnel-b.example.com",
+			path:          "/index.html",
+			wantRuleIndex: 2,
+		},
+		{
+			host:          "tunnel-c.example.com",
+			path:          "/",
+			wantRuleIndex: 2,
+		},
+	}
+
+	for i, test := range tests {
+		_, ruleIndex := ingress.FindMatchingRule(test.host, test.path)
+		assert.Equal(t, test.wantRuleIndex, ruleIndex, fmt.Sprintf("Expect host=%s, path=%s to match rule %d, got %d", test.host, test.path, test.wantRuleIndex, i))
+	}
+}
+
+func mustParsePath(t *testing.T, path string) *regexp.Regexp {
+	regexp, err := regexp.Compile(path)
+	assert.NoError(t, err)
+	return regexp
+}
+
 func MustParseURL(t *testing.T, rawURL string) *url.URL {
 	u, err := url.Parse(rawURL)
 	require.NoError(t, err)
 	return u
 }
 
-func Test_rule_matches(t *testing.T) {
-	type fields struct {
-		Hostname string
-		Path     *regexp.Regexp
-		Service  *url.URL
-	}
-	type args struct {
-		requestURL *url.URL
-	}
-	tests := []struct {
-		name   string
-		fields fields
-		args   args
-		want   bool
-	}{
-		{
-			name: "Just hostname, pass",
-			fields: fields{
-				Hostname: "example.com",
-			},
-			args: args{
-				requestURL: MustParseURL(t, "https://example.com"),
-			},
-			want: true,
-		},
-		{
-			name: "Entire hostname is wildcard, should match everything",
-			fields: fields{
-				Hostname: "*",
-			},
-			args: args{
-				requestURL: MustParseURL(t, "https://example.com"),
-			},
-			want: true,
-		},
-		{
-			name: "Just hostname, fail",
-			fields: fields{
-				Hostname: "example.com",
-			},
-			args: args{
-				requestURL: MustParseURL(t, "https://foo.bar"),
-			},
-			want: false,
-		},
-		{
-			name: "Just wildcard hostname, pass",
-			fields: fields{
-				Hostname: "*.example.com",
-			},
-			args: args{
-				requestURL: MustParseURL(t, "https://adam.example.com"),
-			},
-			want: true,
-		},
-		{
-			name: "Just wildcard hostname, fail",
-			fields: fields{
-				Hostname: "*.example.com",
-			},
-			args: args{
-				requestURL: MustParseURL(t, "https://tunnel.com"),
-			},
-			want: false,
-		},
-		{
-			name: "Just wildcard outside of subdomain in hostname, fail",
-			fields: fields{
-				Hostname: "*example.com",
-			},
-			args: args{
-				requestURL: MustParseURL(t, "https://www.example.com"),
-			},
-			want: false,
-		},
-		{
-			name: "Wildcard over multiple subdomains",
-			fields: fields{
-				Hostname: "*.example.com",
-			},
-			args: args{
-				requestURL: MustParseURL(t, "https://adam.chalmers.example.com"),
-			},
-			want: true,
-		},
-		{
-			name: "Hostname and path",
-			fields: fields{
-				Hostname: "*.example.com",
-				Path:     regexp.MustCompile("/static/.*\\.html"),
-			},
-			args: args{
-				requestURL: MustParseURL(t, "https://www.example.com/static/index.html"),
-			},
-			want: true,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			r := Rule{
-				Hostname: tt.fields.Hostname,
-				Path:     tt.fields.Path,
-				Service:  tt.fields.Service,
-			}
-			u := tt.args.requestURL
-			if got := r.Matches(u.Hostname(), u.Path); got != tt.want {
-				t.Errorf("rule.matches() = %v, want %v", got, tt.want)
-			}
-		})
-	}
-}
-
 func BenchmarkFindMatch(b *testing.B) {
 	rulesYAML := `
 ingress:

ingress/origin_request_config.go

@@ -0,0 +1,331 @@
+package ingress
+
+import (
+	"time"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/tlsconfig"
+	"github.com/urfave/cli/v2"
+)
+
+const (
+	defaultConnectTimeout       = 30 * time.Second
+	defaultTLSTimeout           = 10 * time.Second
+	defaultTCPKeepAlive         = 30 * time.Second
+	defaultKeepAliveConnections = 100
+	defaultKeepAliveTimeout     = 90 * time.Second
+	defaultProxyAddress         = "127.0.0.1"
+
+	SSHServerFlag                 = "ssh-server"
+	Socks5Flag                    = "socks5"
+	ProxyConnectTimeoutFlag       = "proxy-connect-timeout"
+	ProxyTLSTimeoutFlag           = "proxy-tls-timeout"
+	ProxyTCPKeepAlive             = "proxy-tcp-keepalive"
+	ProxyNoHappyEyeballsFlag      = "proxy-no-happy-eyeballs"
+	ProxyKeepAliveConnectionsFlag = "proxy-keepalive-connections"
+	ProxyKeepAliveTimeoutFlag     = "proxy-keepalive-timeout"
+	HTTPHostHeaderFlag            = "http-host-header"
+	OriginServerNameFlag          = "origin-server-name"
+	NoTLSVerifyFlag               = "no-tls-verify"
+	NoChunkedEncodingFlag         = "no-chunked-encoding"
+	ProxyAddressFlag              = "proxy-address"
+	ProxyPortFlag                 = "proxy-port"
+)
+
+const (
+	socksProxy = "socks"
+)
+
+func originRequestFromSingeRule(c *cli.Context) OriginRequestConfig {
+	var connectTimeout time.Duration = defaultConnectTimeout
+	var tlsTimeout time.Duration = defaultTLSTimeout
+	var tcpKeepAlive time.Duration = defaultTCPKeepAlive
+	var noHappyEyeballs bool
+	var keepAliveConnections int = defaultKeepAliveConnections
+	var keepAliveTimeout time.Duration = defaultKeepAliveTimeout
+	var httpHostHeader string
+	var originServerName string
+	var caPool string
+	var noTLSVerify bool
+	var disableChunkedEncoding bool
+	var bastionMode bool
+	var proxyAddress = defaultProxyAddress
+	var proxyPort uint
+	var proxyType string
+	if flag := ProxyConnectTimeoutFlag; c.IsSet(flag) {
+		connectTimeout = c.Duration(flag)
+	}
+	if flag := ProxyTLSTimeoutFlag; c.IsSet(flag) {
+		tlsTimeout = c.Duration(flag)
+	}
+	if flag := ProxyTCPKeepAlive; c.IsSet(flag) {
+		tcpKeepAlive = c.Duration(flag)
+	}
+	if flag := ProxyNoHappyEyeballsFlag; c.IsSet(flag) {
+		noHappyEyeballs = c.Bool(flag)
+	}
+	if flag := ProxyKeepAliveConnectionsFlag; c.IsSet(flag) {
+		keepAliveConnections = c.Int(flag)
+	}
+	if flag := ProxyKeepAliveTimeoutFlag; c.IsSet(flag) {
+		keepAliveTimeout = c.Duration(flag)
+	}
+	if flag := HTTPHostHeaderFlag; c.IsSet(flag) {
+		httpHostHeader = c.String(flag)
+	}
+	if flag := OriginServerNameFlag; c.IsSet(flag) {
+		originServerName = c.String(flag)
+	}
+	if flag := tlsconfig.OriginCAPoolFlag; c.IsSet(flag) {
+		caPool = c.String(flag)
+	}
+	if flag := NoTLSVerifyFlag; c.IsSet(flag) {
+		noTLSVerify = c.Bool(flag)
+	}
+	if flag := NoChunkedEncodingFlag; c.IsSet(flag) {
+		disableChunkedEncoding = c.Bool(flag)
+	}
+	if flag := config.BastionFlag; c.IsSet(flag) {
+		bastionMode = c.Bool(flag)
+	}
+	if flag := ProxyAddressFlag; c.IsSet(flag) {
+		proxyAddress = c.String(flag)
+	}
+	if flag := ProxyPortFlag; c.IsSet(flag) {
+		proxyPort = c.Uint(flag)
+	}
+	if c.IsSet(Socks5Flag) {
+		proxyType = socksProxy
+	}
+	return OriginRequestConfig{
+		ConnectTimeout:         connectTimeout,
+		TLSTimeout:             tlsTimeout,
+		TCPKeepAlive:           tcpKeepAlive,
+		NoHappyEyeballs:        noHappyEyeballs,
+		KeepAliveConnections:   keepAliveConnections,
+		KeepAliveTimeout:       keepAliveTimeout,
+		HTTPHostHeader:         httpHostHeader,
+		OriginServerName:       originServerName,
+		CAPool:                 caPool,
+		NoTLSVerify:            noTLSVerify,
+		DisableChunkedEncoding: disableChunkedEncoding,
+		BastionMode:            bastionMode,
+		ProxyAddress:           proxyAddress,
+		ProxyPort:              proxyPort,
+		ProxyType:              proxyType,
+	}
+}
+
+func originRequestFromYAML(y config.OriginRequestConfig) OriginRequestConfig {
+	out := OriginRequestConfig{
+		ConnectTimeout:       defaultConnectTimeout,
+		TLSTimeout:           defaultTLSTimeout,
+		TCPKeepAlive:         defaultTCPKeepAlive,
+		KeepAliveConnections: defaultKeepAliveConnections,
+		KeepAliveTimeout:     defaultKeepAliveTimeout,
+		ProxyAddress:         defaultProxyAddress,
+	}
+	if y.ConnectTimeout != nil {
+		out.ConnectTimeout = *y.ConnectTimeout
+	}
+	if y.TLSTimeout != nil {
+		out.TLSTimeout = *y.TLSTimeout
+	}
+	if y.TCPKeepAlive != nil {
+		out.TCPKeepAlive = *y.TCPKeepAlive
+	}
+	if y.NoHappyEyeballs != nil {
+		out.NoHappyEyeballs = *y.NoHappyEyeballs
+	}
+	if y.KeepAliveConnections != nil {
+		out.KeepAliveConnections = *y.KeepAliveConnections
+	}
+	if y.KeepAliveTimeout != nil {
+		out.KeepAliveTimeout = *y.KeepAliveTimeout
+	}
+	if y.HTTPHostHeader != nil {
+		out.HTTPHostHeader = *y.HTTPHostHeader
+	}
+	if y.OriginServerName != nil {
+		out.OriginServerName = *y.OriginServerName
+	}
+	if y.CAPool != nil {
+		out.CAPool = *y.CAPool
+	}
+	if y.NoTLSVerify != nil {
+		out.NoTLSVerify = *y.NoTLSVerify
+	}
+	if y.DisableChunkedEncoding != nil {
+		out.DisableChunkedEncoding = *y.DisableChunkedEncoding
+	}
+	if y.BastionMode != nil {
+		out.BastionMode = *y.BastionMode
+	}
+	if y.ProxyAddress != nil {
+		out.ProxyAddress = *y.ProxyAddress
+	}
+	if y.ProxyPort != nil {
+		out.ProxyPort = *y.ProxyPort
+	}
+	if y.ProxyType != nil {
+		out.ProxyType = *y.ProxyType
+	}
+	return out
+}
+
+// OriginRequestConfig configures how Cloudflared sends requests to origin
+// services.
+// Note: To specify a time.Duration in go-yaml, use e.g. "3s" or "24h".
+type OriginRequestConfig struct {
+	// HTTP proxy timeout for establishing a new connection
+	ConnectTimeout time.Duration `yaml:"connectTimeout"`
+	// HTTP proxy timeout for completing a TLS handshake
+	TLSTimeout time.Duration `yaml:"tlsTimeout"`
+	// HTTP proxy TCP keepalive duration
+	TCPKeepAlive time.Duration `yaml:"tcpKeepAlive"`
+	// HTTP proxy should disable "happy eyeballs" for IPv4/v6 fallback
+	NoHappyEyeballs bool `yaml:"noHappyEyeballs"`
+	// HTTP proxy maximum keepalive connection pool size
+	KeepAliveConnections int `yaml:"keepAliveConnections"`
+	// HTTP proxy timeout for closing an idle connection
+	KeepAliveTimeout time.Duration `yaml:"keepAliveTimeout"`
+	// Sets the HTTP Host header for the local webserver.
+	HTTPHostHeader string `yaml:"httpHostHeader"`
+	// Hostname on the origin server certificate.
+	OriginServerName string `yaml:"originServerName"`
+	// Path to the CA for the certificate of your origin.
+	// This option should be used only if your certificate is not signed by Cloudflare.
+	CAPool string `yaml:"caPool"`
+	// Disables TLS verification of the certificate presented by your origin.
+	// Will allow any certificate from the origin to be accepted.
+	// Note: The connection from your machine to Cloudflare's Edge is still encrypted.
+	NoTLSVerify bool `yaml:"noTLSVerify"`
+	// Disables chunked transfer encoding.
+	// Useful if you are running a WSGI server.
+	DisableChunkedEncoding bool `yaml:"disableChunkedEncoding"`
+	// Runs as jump host
+	BastionMode bool `yaml:"bastionMode"`
+	// Listen address for the proxy.
+	ProxyAddress string `yaml:"proxyAddress"`
+	// Listen port for the proxy.
+	ProxyPort uint `yaml:"proxyPort"`
+	// What sort of proxy should be started
+	ProxyType string `yaml:"proxyType"`
+}
+
+func (defaults *OriginRequestConfig) setConnectTimeout(overrides config.OriginRequestConfig) {
+	if val := overrides.ConnectTimeout; val != nil {
+		defaults.ConnectTimeout = *val
+	}
+}
+
+func (defaults *OriginRequestConfig) setTLSTimeout(overrides config.OriginRequestConfig) {
+	if val := overrides.TLSTimeout; val != nil {
+		defaults.TLSTimeout = *val
+	}
+}
+
+func (defaults *OriginRequestConfig) setNoHappyEyeballs(overrides config.OriginRequestConfig) {
+	if val := overrides.NoHappyEyeballs; val != nil {
+		defaults.NoHappyEyeballs = *val
+	}
+}
+
+func (defaults *OriginRequestConfig) setKeepAliveConnections(overrides config.OriginRequestConfig) {
+	if val := overrides.KeepAliveConnections; val != nil {
+		defaults.KeepAliveConnections = *val
+	}
+}
+
+func (defaults *OriginRequestConfig) setKeepAliveTimeout(overrides config.OriginRequestConfig) {
+	if val := overrides.KeepAliveTimeout; val != nil {
+		defaults.KeepAliveTimeout = *val
+	}
+}
+
+func (defaults *OriginRequestConfig) setTCPKeepAlive(overrides config.OriginRequestConfig) {
+	if val := overrides.TCPKeepAlive; val != nil {
+		defaults.TCPKeepAlive = *val
+	}
+}
+
+func (defaults *OriginRequestConfig) setHTTPHostHeader(overrides config.OriginRequestConfig) {
+	if val := overrides.HTTPHostHeader; val != nil {
+		defaults.HTTPHostHeader = *val
+	}
+}
+
+func (defaults *OriginRequestConfig) setOriginServerName(overrides config.OriginRequestConfig) {
+	if val := overrides.OriginServerName; val != nil {
+		defaults.OriginServerName = *val
+	}
+}
+
+func (defaults *OriginRequestConfig) setCAPool(overrides config.OriginRequestConfig) {
+	if val := overrides.CAPool; val != nil {
+		defaults.CAPool = *val
+	}
+}
+
+func (defaults *OriginRequestConfig) setNoTLSVerify(overrides config.OriginRequestConfig) {
+	if val := overrides.NoTLSVerify; val != nil {
+		defaults.NoTLSVerify = *val
+	}
+}
+
+func (defaults *OriginRequestConfig) setDisableChunkedEncoding(overrides config.OriginRequestConfig) {
+	if val := overrides.DisableChunkedEncoding; val != nil {
+		defaults.DisableChunkedEncoding = *val
+	}
+}
+
+func (defaults *OriginRequestConfig) setBastionMode(overrides config.OriginRequestConfig) {
+	if val := overrides.BastionMode; val != nil {
+		defaults.BastionMode = *val
+	}
+}
+
+func (defaults *OriginRequestConfig) setProxyPort(overrides config.OriginRequestConfig) {
+	if val := overrides.ProxyPort; val != nil {
+		defaults.ProxyPort = *val
+	}
+}
+
+func (defaults *OriginRequestConfig) setProxyAddress(overrides config.OriginRequestConfig) {
+	if val := overrides.ProxyAddress; val != nil {
+		defaults.ProxyAddress = *val
+	}
+}
+
+func (defaults *OriginRequestConfig) setProxyType(overrides config.OriginRequestConfig) {
+	if val := overrides.ProxyType; val != nil {
+		defaults.ProxyType = *val
+	}
+}
+
+// SetConfig gets config for the requests that cloudflared sends to origins.
+// Each field has a setter method which sets a value for the field by trying to find:
+//   1. The user config for this rule
+//   2. The user config for the overall ingress config
+//   3. Defaults chosen by the cloudflared team
+//   4. Golang zero values for that type
+// If an earlier option isn't set, it will try the next option down.
+func setConfig(defaults OriginRequestConfig, overrides config.OriginRequestConfig) OriginRequestConfig {
+	cfg := defaults
+	cfg.setConnectTimeout(overrides)
+	cfg.setTLSTimeout(overrides)
+	cfg.setNoHappyEyeballs(overrides)
+	cfg.setKeepAliveConnections(overrides)
+	cfg.setKeepAliveTimeout(overrides)
+	cfg.setTCPKeepAlive(overrides)
+	cfg.setHTTPHostHeader(overrides)
+	cfg.setOriginServerName(overrides)
+	cfg.setCAPool(overrides)
+	cfg.setNoTLSVerify(overrides)
+	cfg.setDisableChunkedEncoding(overrides)
+	cfg.setBastionMode(overrides)
+	cfg.setProxyPort(overrides)
+	cfg.setProxyAddress(overrides)
+	cfg.setProxyType(overrides)
+	return cfg
+}

ingress/origin_request_config_test.go

@@ -0,0 +1,202 @@
+package ingress
+
+import (
+	"flag"
+	"testing"
+	"time"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/stretchr/testify/require"
+	"github.com/urfave/cli/v2"
+	"gopkg.in/yaml.v2"
+)
+
+// Ensure that the nullable config from `config` package and the
+// non-nullable config from `ingress` package have the same number of
+// fields.
+// This test ensures that programmers didn't add a new field to
+// one struct and forget to add it to the other ;)
+func TestCorrespondingFields(t *testing.T) {
+	require.Equal(
+		t,
+		CountFields(t, config.OriginRequestConfig{}),
+		CountFields(t, OriginRequestConfig{}),
+	)
+}
+
+func CountFields(t *testing.T, val interface{}) int {
+	b, err := yaml.Marshal(val)
+	require.NoError(t, err)
+	m := make(map[string]interface{}, 0)
+	err = yaml.Unmarshal(b, &m)
+	require.NoError(t, err)
+	return len(m)
+}
+
+func TestOriginRequestConfigOverrides(t *testing.T) {
+	rulesYAML := `
+originRequest:
+  connectTimeout: 1m
+  tlsTimeout: 1s
+  noHappyEyeballs: true
+  tcpKeepAlive: 1s
+  keepAliveConnections: 1
+  keepAliveTimeout: 1s
+  httpHostHeader: abc
+  originServerName: a1
+  caPool: /tmp/path0
+  noTLSVerify: true
+  disableChunkedEncoding: true
+  bastionMode: True
+  proxyAddress: 127.1.2.3
+  proxyPort: 100
+  proxyType: socks5
+ingress:
+- hostname: tun.example.com
+  service: https://localhost:8000
+- hostname: "*"
+  service: https://localhost:8001
+  originRequest:
+    connectTimeout: 2m
+    tlsTimeout: 2s
+    noHappyEyeballs: false
+    tcpKeepAlive: 2s
+    keepAliveConnections: 2
+    keepAliveTimeout: 2s
+    httpHostHeader: def
+    originServerName: b2
+    caPool: /tmp/path1
+    noTLSVerify: false
+    disableChunkedEncoding: false
+    bastionMode: false
+    proxyAddress: interface
+    proxyPort: 200
+    proxyType: ""
+`
+	ing, err := ParseIngress(MustReadIngress(rulesYAML))
+	if err != nil {
+		t.Error(err)
+	}
+
+	// Rule 0 didn't override anything, so it inherits the user-specified
+	// root-level configuration.
+	actual0 := ing.Rules[0].Config
+	expected0 := OriginRequestConfig{
+		ConnectTimeout:         1 * time.Minute,
+		TLSTimeout:             1 * time.Second,
+		NoHappyEyeballs:        true,
+		TCPKeepAlive:           1 * time.Second,
+		KeepAliveConnections:   1,
+		KeepAliveTimeout:       1 * time.Second,
+		HTTPHostHeader:         "abc",
+		OriginServerName:       "a1",
+		CAPool:                 "/tmp/path0",
+		NoTLSVerify:            true,
+		DisableChunkedEncoding: true,
+		BastionMode:            true,
+		ProxyAddress:           "127.1.2.3",
+		ProxyPort:              uint(100),
+		ProxyType:              "socks5",
+	}
+	require.Equal(t, expected0, actual0)
+
+	// Rule 1 overrode all the root-level config.
+	actual1 := ing.Rules[1].Config
+	expected1 := OriginRequestConfig{
+		ConnectTimeout:         2 * time.Minute,
+		TLSTimeout:             2 * time.Second,
+		NoHappyEyeballs:        false,
+		TCPKeepAlive:           2 * time.Second,
+		KeepAliveConnections:   2,
+		KeepAliveTimeout:       2 * time.Second,
+		HTTPHostHeader:         "def",
+		OriginServerName:       "b2",
+		CAPool:                 "/tmp/path1",
+		NoTLSVerify:            false,
+		DisableChunkedEncoding: false,
+		BastionMode:            false,
+		ProxyAddress:           "interface",
+		ProxyPort:              uint(200),
+		ProxyType:              "",
+	}
+	require.Equal(t, expected1, actual1)
+}
+
+func TestOriginRequestConfigDefaults(t *testing.T) {
+	rulesYAML := `
+ingress:
+- hostname: tun.example.com
+  service: https://localhost:8000
+- hostname: "*"
+  service: https://localhost:8001
+  originRequest:
+    connectTimeout: 2m
+    tlsTimeout: 2s
+    noHappyEyeballs: false
+    tcpKeepAlive: 2s
+    keepAliveConnections: 2
+    keepAliveTimeout: 2s
+    httpHostHeader: def
+    originServerName: b2
+    caPool: /tmp/path1
+    noTLSVerify: false
+    disableChunkedEncoding: false
+    bastionMode: false
+    proxyAddress: interface
+    proxyPort: 200
+    proxyType: ""
+`
+	ing, err := ParseIngress(MustReadIngress(rulesYAML))
+	if err != nil {
+		t.Error(err)
+	}
+
+	// Rule 0 didn't override anything, so it inherits the cloudflared defaults
+	actual0 := ing.Rules[0].Config
+	expected0 := OriginRequestConfig{
+		ConnectTimeout:       defaultConnectTimeout,
+		TLSTimeout:           defaultTLSTimeout,
+		TCPKeepAlive:         defaultTCPKeepAlive,
+		KeepAliveConnections: defaultKeepAliveConnections,
+		KeepAliveTimeout:     defaultKeepAliveTimeout,
+		ProxyAddress:         defaultProxyAddress,
+	}
+	require.Equal(t, expected0, actual0)
+
+	// Rule 1 overrode all defaults.
+	actual1 := ing.Rules[1].Config
+	expected1 := OriginRequestConfig{
+		ConnectTimeout:         2 * time.Minute,
+		TLSTimeout:             2 * time.Second,
+		NoHappyEyeballs:        false,
+		TCPKeepAlive:           2 * time.Second,
+		KeepAliveConnections:   2,
+		KeepAliveTimeout:       2 * time.Second,
+		HTTPHostHeader:         "def",
+		OriginServerName:       "b2",
+		CAPool:                 "/tmp/path1",
+		NoTLSVerify:            false,
+		DisableChunkedEncoding: false,
+		BastionMode:            false,
+		ProxyAddress:           "interface",
+		ProxyPort:              uint(200),
+		ProxyType:              "",
+	}
+	require.Equal(t, expected1, actual1)
+}
+
+func TestDefaultConfigFromCLI(t *testing.T) {
+	set := flag.NewFlagSet("contrive", 0)
+	c := cli.NewContext(nil, set, nil)
+
+	expected := OriginRequestConfig{
+		ConnectTimeout:       defaultConnectTimeout,
+		TLSTimeout:           defaultTLSTimeout,
+		TCPKeepAlive:         defaultTCPKeepAlive,
+		KeepAliveConnections: defaultKeepAliveConnections,
+		KeepAliveTimeout:     defaultKeepAliveTimeout,
+		ProxyAddress:         defaultProxyAddress,
+	}
+	actual := originRequestFromSingeRule(c)
+	require.Equal(t, expected, actual)
+}

ingress/origin_service.go

@@ -0,0 +1,336 @@
+package ingress
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"net/url"
+	"strconv"
+	"sync"
+	"time"
+
+	"github.com/cloudflare/cloudflared/hello"
+	"github.com/cloudflare/cloudflared/logger"
+	"github.com/cloudflare/cloudflared/socks"
+	"github.com/cloudflare/cloudflared/tlsconfig"
+	"github.com/cloudflare/cloudflared/websocket"
+	gws "github.com/gorilla/websocket"
+	"github.com/pkg/errors"
+)
+
+// OriginService is something a tunnel can proxy traffic to.
+type OriginService interface {
+	// RoundTrip is how cloudflared proxies eyeball requests to the actual origin services
+	http.RoundTripper
+	String() string
+	// Start the origin service if it's managed by cloudflared, e.g. proxy servers or Hello World.
+	// If it's not managed by cloudflared, this is a no-op because the user is responsible for
+	// starting the origin service.
+	start(wg *sync.WaitGroup, log logger.Service, shutdownC <-chan struct{}, errC chan error, cfg OriginRequestConfig) error
+}
+
+// unixSocketPath is an OriginService representing a unix socket (which accepts HTTP)
+type unixSocketPath struct {
+	path      string
+	transport *http.Transport
+}
+
+func (o *unixSocketPath) String() string {
+	return "unix socket: " + o.path
+}
+
+func (o *unixSocketPath) start(wg *sync.WaitGroup, log logger.Service, shutdownC <-chan struct{}, errC chan error, cfg OriginRequestConfig) error {
+	transport, err := newHTTPTransport(o, cfg, log)
+	if err != nil {
+		return err
+	}
+	o.transport = transport
+	return nil
+}
+
+func (o *unixSocketPath) RoundTrip(req *http.Request) (*http.Response, error) {
+	return o.transport.RoundTrip(req)
+}
+
+func (o *unixSocketPath) Dial(reqURL *url.URL, headers http.Header) (*gws.Conn, *http.Response, error) {
+	d := &gws.Dialer{
+		NetDial:         o.transport.Dial,
+		NetDialContext:  o.transport.DialContext,
+		TLSClientConfig: o.transport.TLSClientConfig,
+	}
+	reqURL.Scheme = websocket.ChangeRequestScheme(reqURL)
+	return d.Dial(reqURL.String(), headers)
+}
+
+// localService is an OriginService listening on a TCP/IP address the user's origin can route to.
+type localService struct {
+	// The URL for the user's origin service
+	RootURL *url.URL
+	// The URL that cloudflared should send requests to.
+	// If this origin requires starting a proxy, this is the proxy's address,
+	// and that proxy points to RootURL. Otherwise, this is equal to RootURL.
+	URL       *url.URL
+	transport *http.Transport
+}
+
+func (o *localService) Dial(reqURL *url.URL, headers http.Header) (*gws.Conn, *http.Response, error) {
+	d := &gws.Dialer{TLSClientConfig: o.transport.TLSClientConfig}
+	// Rewrite the request URL so that it goes to the origin service.
+	reqURL.Host = o.URL.Host
+	reqURL.Scheme = websocket.ChangeRequestScheme(o.URL)
+	return d.Dial(reqURL.String(), headers)
+}
+
+func (o *localService) address() string {
+	return o.URL.String()
+}
+
+func (o *localService) start(wg *sync.WaitGroup, log logger.Service, shutdownC <-chan struct{}, errC chan error, cfg OriginRequestConfig) error {
+	transport, err := newHTTPTransport(o, cfg, log)
+	if err != nil {
+		return err
+	}
+	o.transport = transport
+
+	// Start a proxy if one is needed
+	if staticHost := o.staticHost(); originRequiresProxy(staticHost, cfg) {
+		if err := o.startProxy(staticHost, wg, log, shutdownC, errC, cfg); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (o *localService) startProxy(staticHost string, wg *sync.WaitGroup, log logger.Service, shutdownC <-chan struct{}, errC chan error, cfg OriginRequestConfig) error {
+
+	// Start a listener for the proxy
+	proxyAddress := net.JoinHostPort(cfg.ProxyAddress, strconv.Itoa(int(cfg.ProxyPort)))
+	listener, err := net.Listen("tcp", proxyAddress)
+	if err != nil {
+		log.Errorf("Cannot start Websocket Proxy Server: %s", err)
+		return errors.Wrap(err, "Cannot start Websocket Proxy Server")
+	}
+
+	// Start the proxy itself
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		streamHandler := websocket.DefaultStreamHandler
+		// This origin's config specifies what type of proxy to start.
+		switch cfg.ProxyType {
+		case socksProxy:
+			log.Info("SOCKS5 server started")
+			streamHandler = func(wsConn *websocket.Conn, remoteConn net.Conn, _ http.Header) {
+				dialer := socks.NewConnDialer(remoteConn)
+				requestHandler := socks.NewRequestHandler(dialer)
+				socksServer := socks.NewConnectionHandler(requestHandler)
+
+				socksServer.Serve(wsConn)
+			}
+		case "":
+			log.Debug("Not starting any websocket proxy")
+		default:
+			log.Errorf("%s isn't a valid proxy (valid options are {%s})", cfg.ProxyType, socksProxy)
+		}
+
+		errC <- websocket.StartProxyServer(log, listener, staticHost, shutdownC, streamHandler)
+	}()
+
+	// Modify this origin, so that it no longer points at the origin service directly.
+	// Instead, it points at the proxy to the origin service.
+	newURL, err := url.Parse("http://" + listener.Addr().String())
+	if err != nil {
+		return err
+	}
+	o.URL = newURL
+	return nil
+}
+
+func (o *localService) String() string {
+	return o.address()
+}
+
+func (o *localService) RoundTrip(req *http.Request) (*http.Response, error) {
+	// Rewrite the request URL so that it goes to the origin service.
+	req.URL.Host = o.URL.Host
+	req.URL.Scheme = o.URL.Scheme
+	return o.transport.RoundTrip(req)
+}
+
+func (o *localService) staticHost() string {
+
+	addPortIfMissing := func(uri *url.URL, port int) string {
+		if uri.Port() != "" {
+			return uri.Host
+		}
+		return fmt.Sprintf("%s:%d", uri.Hostname(), port)
+	}
+
+	switch o.URL.Scheme {
+	case "ssh":
+		return addPortIfMissing(o.URL, 22)
+	case "rdp":
+		return addPortIfMissing(o.URL, 3389)
+	case "smb":
+		return addPortIfMissing(o.URL, 445)
+	case "tcp":
+		return addPortIfMissing(o.URL, 7864) // just a random port since there isn't a default in this case
+	}
+	return ""
+
+}
+
+// HelloWorld is an OriginService for the built-in Hello World server.
+// Users only use this for testing and experimenting with cloudflared.
+type helloWorld struct {
+	server    net.Listener
+	transport *http.Transport
+}
+
+func (o *helloWorld) String() string {
+	return "Hello World test origin"
+}
+
+// Start starts a HelloWorld server and stores its address in the Service receiver.
+func (o *helloWorld) start(wg *sync.WaitGroup, log logger.Service, shutdownC <-chan struct{}, errC chan error, cfg OriginRequestConfig) error {
+	transport, err := newHTTPTransport(o, cfg, log)
+	if err != nil {
+		return err
+	}
+	o.transport = transport
+	helloListener, err := hello.CreateTLSListener("127.0.0.1:")
+	if err != nil {
+		return errors.Wrap(err, "Cannot start Hello World Server")
+	}
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		_ = hello.StartHelloWorldServer(log, helloListener, shutdownC)
+	}()
+	o.server = helloListener
+	return nil
+}
+
+func (o *helloWorld) RoundTrip(req *http.Request) (*http.Response, error) {
+	// Rewrite the request URL so that it goes to the Hello World server.
+	req.URL.Host = o.server.Addr().String()
+	req.URL.Scheme = "https"
+	return o.transport.RoundTrip(req)
+}
+
+func (o *helloWorld) Dial(reqURL *url.URL, headers http.Header) (*gws.Conn, *http.Response, error) {
+	d := &gws.Dialer{
+		TLSClientConfig: o.transport.TLSClientConfig,
+	}
+	reqURL.Host = o.server.Addr().String()
+	reqURL.Scheme = "wss"
+	return d.Dial(reqURL.String(), headers)
+}
+
+func originRequiresProxy(staticHost string, cfg OriginRequestConfig) bool {
+	return staticHost != "" || cfg.BastionMode
+}
+
+// statusCode is an OriginService that just responds with a given HTTP status.
+// Typical use-case is "user wants the catch-all rule to just respond 404".
+type statusCode struct {
+	resp *http.Response
+}
+
+func newStatusCode(status int) statusCode {
+	resp := &http.Response{
+		StatusCode: status,
+		Status:     fmt.Sprintf("%d %s", status, http.StatusText(status)),
+		Body:       new(NopReadCloser),
+	}
+	return statusCode{resp: resp}
+}
+
+func (o *statusCode) String() string {
+	return fmt.Sprintf("HTTP %d", o.resp.StatusCode)
+}
+
+func (o *statusCode) start(wg *sync.WaitGroup, log logger.Service, shutdownC <-chan struct{}, errC chan error, cfg OriginRequestConfig) error {
+	return nil
+}
+
+func (o *statusCode) RoundTrip(_ *http.Request) (*http.Response, error) {
+	return o.resp, nil
+}
+
+type NopReadCloser struct{}
+
+// Read always returns EOF to signal end of input
+func (nrc *NopReadCloser) Read(buf []byte) (int, error) {
+	return 0, io.EOF
+}
+
+func (nrc *NopReadCloser) Close() error {
+	return nil
+}
+
+func newHTTPTransport(service OriginService, cfg OriginRequestConfig, log logger.Service) (*http.Transport, error) {
+	originCertPool, err := tlsconfig.LoadOriginCA(cfg.CAPool, log)
+	if err != nil {
+		return nil, errors.Wrap(err, "Error loading cert pool")
+	}
+
+	httpTransport := http.Transport{
+		Proxy:                 http.ProxyFromEnvironment,
+		MaxIdleConns:          cfg.KeepAliveConnections,
+		MaxIdleConnsPerHost:   cfg.KeepAliveConnections,
+		IdleConnTimeout:       cfg.KeepAliveTimeout,
+		TLSHandshakeTimeout:   cfg.TLSTimeout,
+		ExpectContinueTimeout: 1 * time.Second,
+		TLSClientConfig:       &tls.Config{RootCAs: originCertPool, InsecureSkipVerify: cfg.NoTLSVerify},
+	}
+	if _, isHelloWorld := service.(*helloWorld); !isHelloWorld && cfg.OriginServerName != "" {
+		httpTransport.TLSClientConfig.ServerName = cfg.OriginServerName
+	}
+
+	dialer := &net.Dialer{
+		Timeout:   cfg.ConnectTimeout,
+		KeepAlive: cfg.TCPKeepAlive,
+	}
+	if cfg.NoHappyEyeballs {
+		dialer.FallbackDelay = -1 // As of Golang 1.12, a negative delay disables "happy eyeballs"
+	}
+
+	// DialContext depends on which kind of origin is being used.
+	dialContext := dialer.DialContext
+	switch service := service.(type) {
+
+	// If this origin is a unix socket, enforce network type "unix".
+	case *unixSocketPath:
+		httpTransport.DialContext = func(ctx context.Context, _, _ string) (net.Conn, error) {
+			return dialContext(ctx, "unix", service.path)
+		}
+
+	// Otherwise, use the regular network config.
+	default:
+		httpTransport.DialContext = dialContext
+	}
+
+	return &httpTransport, nil
+}
+
+// MockOriginService should only be used by other packages to mock OriginService. Set Transport to configure desired RoundTripper behavior.
+type MockOriginService struct {
+	Transport http.RoundTripper
+}
+
+func (mos MockOriginService) RoundTrip(req *http.Request) (*http.Response, error) {
+	return mos.Transport.RoundTrip(req)
+}
+
+func (mos MockOriginService) String() string {
+	return "MockOriginService"
+}
+
+func (mos MockOriginService) start(wg *sync.WaitGroup, log logger.Service, shutdownC <-chan struct{}, errC chan error, cfg OriginRequestConfig) error {
+	return nil
+}

ingress/rule.go

@@ -0,0 +1,50 @@
+package ingress
+
+import (
+	"regexp"
+	"strings"
+)
+
+// Rule routes traffic from a hostname/path on the public internet to the
+// service running on the given URL.
+type Rule struct {
+	// Requests for this hostname will be proxied to this rule's service.
+	Hostname string
+
+	// Path is an optional regex that can specify path-driven ingress rules.
+	Path *regexp.Regexp
+
+	// A (probably local) address. Requests for a hostname which matches this
+	// rule's hostname pattern will be proxied to the service running on this
+	// address.
+	Service OriginService
+
+	// Configure the request cloudflared sends to this specific origin.
+	Config OriginRequestConfig
+}
+
+// MultiLineString is for outputting rules in a human-friendly way when Cloudflared
+// is used as a CLI tool (not as a daemon).
+func (r Rule) MultiLineString() string {
+	var out strings.Builder
+	if r.Hostname != "" {
+		out.WriteString("\thostname: ")
+		out.WriteString(r.Hostname)
+		out.WriteRune('\n')
+	}
+	if r.Path != nil {
+		out.WriteString("\tpath: ")
+		out.WriteString(r.Path.String())
+		out.WriteRune('\n')
+	}
+	out.WriteString("\tservice: ")
+	out.WriteString(r.Service.String())
+	return out.String()
+}
+
+// Matches checks if the rule matches a given hostname/path combination.
+func (r *Rule) Matches(hostname, path string) bool {
+	hostMatch := r.Hostname == "" || r.Hostname == "*" || matchHost(r.Hostname, hostname)
+	pathMatch := r.Path == nil || r.Path.MatchString(path)
+	return hostMatch && pathMatch
+}

ingress/rule_test.go

@@ -0,0 +1,146 @@
+package ingress
+
+import (
+	"io"
+	"net/http/httptest"
+	"net/url"
+	"regexp"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func Test_rule_matches(t *testing.T) {
+	type fields struct {
+		Hostname string
+		Path     *regexp.Regexp
+		Service  OriginService
+	}
+	type args struct {
+		requestURL *url.URL
+	}
+	tests := []struct {
+		name   string
+		fields fields
+		args   args
+		want   bool
+	}{
+		{
+			name: "Just hostname, pass",
+			fields: fields{
+				Hostname: "example.com",
+			},
+			args: args{
+				requestURL: MustParseURL(t, "https://example.com"),
+			},
+			want: true,
+		},
+		{
+			name: "Entire hostname is wildcard, should match everything",
+			fields: fields{
+				Hostname: "*",
+			},
+			args: args{
+				requestURL: MustParseURL(t, "https://example.com"),
+			},
+			want: true,
+		},
+		{
+			name: "Just hostname, fail",
+			fields: fields{
+				Hostname: "example.com",
+			},
+			args: args{
+				requestURL: MustParseURL(t, "https://foo.bar"),
+			},
+			want: false,
+		},
+		{
+			name: "Just wildcard hostname, pass",
+			fields: fields{
+				Hostname: "*.example.com",
+			},
+			args: args{
+				requestURL: MustParseURL(t, "https://adam.example.com"),
+			},
+			want: true,
+		},
+		{
+			name: "Just wildcard hostname, fail",
+			fields: fields{
+				Hostname: "*.example.com",
+			},
+			args: args{
+				requestURL: MustParseURL(t, "https://tunnel.com"),
+			},
+			want: false,
+		},
+		{
+			name: "Just wildcard outside of subdomain in hostname, fail",
+			fields: fields{
+				Hostname: "*example.com",
+			},
+			args: args{
+				requestURL: MustParseURL(t, "https://www.example.com"),
+			},
+			want: false,
+		},
+		{
+			name: "Wildcard over multiple subdomains",
+			fields: fields{
+				Hostname: "*.example.com",
+			},
+			args: args{
+				requestURL: MustParseURL(t, "https://adam.chalmers.example.com"),
+			},
+			want: true,
+		},
+		{
+			name: "Hostname and path",
+			fields: fields{
+				Hostname: "*.example.com",
+				Path:     regexp.MustCompile("/static/.*\\.html"),
+			},
+			args: args{
+				requestURL: MustParseURL(t, "https://www.example.com/static/index.html"),
+			},
+			want: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			r := Rule{
+				Hostname: tt.fields.Hostname,
+				Path:     tt.fields.Path,
+				Service:  tt.fields.Service,
+			}
+			u := tt.args.requestURL
+			if got := r.Matches(u.Hostname(), u.Path); got != tt.want {
+				t.Errorf("rule.matches() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestStaticHTTPStatus(t *testing.T) {
+	o := newStatusCode(404)
+	buf := make([]byte, 100)
+
+	sendReq := func() {
+		resp, err := o.RoundTrip(nil)
+		require.NoError(t, err)
+		_, err = resp.Body.Read(buf)
+		require.Equal(t, io.EOF, err)
+		require.NoError(t, resp.Body.Close())
+		require.Equal(t, 404, resp.StatusCode)
+
+		resp, err = o.RoundTrip(nil)
+		require.NoError(t, err)
+		w := httptest.NewRecorder()
+		n, err := io.Copy(w, resp.Body)
+		require.NoError(t, err)
+		require.Equal(t, int64(0), n)
+	}
+	sendReq()
+	sendReq()
+}

origin/backoffhandler.go

@@ -97,3 +97,11 @@ func (b BackoffHandler) GetBaseTime() time.Duration {
 func (b *BackoffHandler) Retries() int {
 	return int(b.retries)
 }
+
+func (b *BackoffHandler) ReachedMaxRetries() bool {
+	return b.retries == b.MaxRetries
+}
+
+func (b *BackoffHandler) resetNow() {
+	b.resetDeadline = time.Now()
+}

origin/metrics.go

@@ -1,540 +1,72 @@
 package origin
 
 import (
-	"sync"
-	"time"
-
-	"github.com/cloudflare/cloudflared/h2mux"
-
+	"github.com/cloudflare/cloudflared/connection"
 	"github.com/prometheus/client_golang/prometheus"
 )
 
-const (
-	metricsNamespace = "cloudflared"
-	tunnelSubsystem  = "tunnel"
-	muxerSubsystem   = "muxer"
-)
+// Metrics uses connection.MetricsNamespace(aka cloudflared) as namespace and connection.TunnelSubsystem
+// (tunnel) as subsystem to keep them consistent with the previous qualifier.
 
-type muxerMetrics struct {
-	rtt              *prometheus.GaugeVec
-	rttMin           *prometheus.GaugeVec
-	rttMax           *prometheus.GaugeVec
-	receiveWindowAve *prometheus.GaugeVec
-	sendWindowAve    *prometheus.GaugeVec
-	receiveWindowMin *prometheus.GaugeVec
-	receiveWindowMax *prometheus.GaugeVec
-	sendWindowMin    *prometheus.GaugeVec
-	sendWindowMax    *prometheus.GaugeVec
-	inBoundRateCurr  *prometheus.GaugeVec
-	inBoundRateMin   *prometheus.GaugeVec
-	inBoundRateMax   *prometheus.GaugeVec
-	outBoundRateCurr *prometheus.GaugeVec
-	outBoundRateMin  *prometheus.GaugeVec
-	outBoundRateMax  *prometheus.GaugeVec
-	compBytesBefore  *prometheus.GaugeVec
-	compBytesAfter   *prometheus.GaugeVec
-	compRateAve      *prometheus.GaugeVec
-}
-
-type TunnelMetrics struct {
-	haConnections     prometheus.Gauge
-	activeStreams     prometheus.Gauge
-	totalRequests     prometheus.Counter
-	requestsPerTunnel *prometheus.CounterVec
-	// concurrentRequestsLock is a mutex for concurrentRequests and maxConcurrentRequests
-	concurrentRequestsLock      sync.Mutex
-	concurrentRequestsPerTunnel *prometheus.GaugeVec
-	// concurrentRequests records count of concurrent requests for each tunnel
-	concurrentRequests             map[string]uint64
-	maxConcurrentRequestsPerTunnel *prometheus.GaugeVec
-	// concurrentRequests records max count of concurrent requests for each tunnel
-	maxConcurrentRequests map[string]uint64
-	timerRetries          prometheus.Gauge
-	responseByCode        *prometheus.CounterVec
-	responseCodePerTunnel *prometheus.CounterVec
-	serverLocations       *prometheus.GaugeVec
-	// locationLock is a mutex for oldServerLocations
-	locationLock sync.Mutex
-	// oldServerLocations stores the last server the tunnel was connected to
-	oldServerLocations map[string]string
-
-	regSuccess *prometheus.CounterVec
-	regFail    *prometheus.CounterVec
-	rpcFail    *prometheus.CounterVec
-
-	muxerMetrics        *muxerMetrics
-	tunnelsHA           tunnelsForHA
-	userHostnamesCounts *prometheus.CounterVec
-}
-
-func newMuxerMetrics() *muxerMetrics {
-	rtt := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "rtt",
-			Help:      "Round-trip time in millisecond",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(rtt)
-
-	rttMin := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "rtt_min",
-			Help:      "Shortest round-trip time in millisecond",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(rttMin)
-
-	rttMax := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "rtt_max",
-			Help:      "Longest round-trip time in millisecond",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(rttMax)
-
-	receiveWindowAve := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "receive_window_ave",
-			Help:      "Average receive window size in bytes",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(receiveWindowAve)
-
-	sendWindowAve := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "send_window_ave",
-			Help:      "Average send window size in bytes",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(sendWindowAve)
-
-	receiveWindowMin := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "receive_window_min",
-			Help:      "Smallest receive window size in bytes",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(receiveWindowMin)
-
-	receiveWindowMax := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "receive_window_max",
-			Help:      "Largest receive window size in bytes",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(receiveWindowMax)
-
-	sendWindowMin := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "send_window_min",
-			Help:      "Smallest send window size in bytes",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(sendWindowMin)
-
-	sendWindowMax := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "send_window_max",
-			Help:      "Largest send window size in bytes",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(sendWindowMax)
-
-	inBoundRateCurr := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "inbound_bytes_per_sec_curr",
-			Help:      "Current inbounding bytes per second, 0 if there is no incoming connection",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(inBoundRateCurr)
-
-	inBoundRateMin := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "inbound_bytes_per_sec_min",
-			Help:      "Minimum non-zero inbounding bytes per second",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(inBoundRateMin)
-
-	inBoundRateMax := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "inbound_bytes_per_sec_max",
-			Help:      "Maximum inbounding bytes per second",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(inBoundRateMax)
-
-	outBoundRateCurr := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "outbound_bytes_per_sec_curr",
-			Help:      "Current outbounding bytes per second, 0 if there is no outgoing traffic",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(outBoundRateCurr)
-
-	outBoundRateMin := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "outbound_bytes_per_sec_min",
-			Help:      "Minimum non-zero outbounding bytes per second",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(outBoundRateMin)
-
-	outBoundRateMax := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "outbound_bytes_per_sec_max",
-			Help:      "Maximum outbounding bytes per second",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(outBoundRateMax)
-
-	compBytesBefore := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "comp_bytes_before",
-			Help:      "Bytes sent via cross-stream compression, pre compression",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(compBytesBefore)
-
-	compBytesAfter := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "comp_bytes_after",
-			Help:      "Bytes sent via cross-stream compression, post compression",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(compBytesAfter)
-
-	compRateAve := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "comp_rate_ave",
-			Help:      "Average outbound cross-stream compression ratio",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(compRateAve)
-
-	return &muxerMetrics{
-		rtt:              rtt,
-		rttMin:           rttMin,
-		rttMax:           rttMax,
-		receiveWindowAve: receiveWindowAve,
-		sendWindowAve:    sendWindowAve,
-		receiveWindowMin: receiveWindowMin,
-		receiveWindowMax: receiveWindowMax,
-		sendWindowMin:    sendWindowMin,
-		sendWindowMax:    sendWindowMax,
-		inBoundRateCurr:  inBoundRateCurr,
-		inBoundRateMin:   inBoundRateMin,
-		inBoundRateMax:   inBoundRateMax,
-		outBoundRateCurr: outBoundRateCurr,
-		outBoundRateMin:  outBoundRateMin,
-		outBoundRateMax:  outBoundRateMax,
-		compBytesBefore:  compBytesBefore,
-		compBytesAfter:   compBytesAfter,
-		compRateAve:      compRateAve,
-	}
-}
-
-func (m *muxerMetrics) update(connectionID string, metrics *h2mux.MuxerMetrics) {
-	m.rtt.WithLabelValues(connectionID).Set(convertRTTMilliSec(metrics.RTT))
-	m.rttMin.WithLabelValues(connectionID).Set(convertRTTMilliSec(metrics.RTTMin))
-	m.rttMax.WithLabelValues(connectionID).Set(convertRTTMilliSec(metrics.RTTMax))
-	m.receiveWindowAve.WithLabelValues(connectionID).Set(metrics.ReceiveWindowAve)
-	m.sendWindowAve.WithLabelValues(connectionID).Set(metrics.SendWindowAve)
-	m.receiveWindowMin.WithLabelValues(connectionID).Set(float64(metrics.ReceiveWindowMin))
-	m.receiveWindowMax.WithLabelValues(connectionID).Set(float64(metrics.ReceiveWindowMax))
-	m.sendWindowMin.WithLabelValues(connectionID).Set(float64(metrics.SendWindowMin))
-	m.sendWindowMax.WithLabelValues(connectionID).Set(float64(metrics.SendWindowMax))
-	m.inBoundRateCurr.WithLabelValues(connectionID).Set(float64(metrics.InBoundRateCurr))
-	m.inBoundRateMin.WithLabelValues(connectionID).Set(float64(metrics.InBoundRateMin))
-	m.inBoundRateMax.WithLabelValues(connectionID).Set(float64(metrics.InBoundRateMax))
-	m.outBoundRateCurr.WithLabelValues(connectionID).Set(float64(metrics.OutBoundRateCurr))
-	m.outBoundRateMin.WithLabelValues(connectionID).Set(float64(metrics.OutBoundRateMin))
-	m.outBoundRateMax.WithLabelValues(connectionID).Set(float64(metrics.OutBoundRateMax))
-	m.compBytesBefore.WithLabelValues(connectionID).Set(float64(metrics.CompBytesBefore.Value()))
-	m.compBytesAfter.WithLabelValues(connectionID).Set(float64(metrics.CompBytesAfter.Value()))
-	m.compRateAve.WithLabelValues(connectionID).Set(float64(metrics.CompRateAve()))
-}
-
-func convertRTTMilliSec(t time.Duration) float64 {
-	return float64(t / time.Millisecond)
-}
-
-// Metrics that can be collected without asking the edge
-func NewTunnelMetrics() *TunnelMetrics {
-	haConnections := prometheus.NewGauge(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: tunnelSubsystem,
-			Name:      "ha_connections",
-			Help:      "Number of active ha connections",
-		})
-	prometheus.MustRegister(haConnections)
-
-	activeStreams := h2mux.NewActiveStreamsMetrics(metricsNamespace, tunnelSubsystem)
-
-	totalRequests := prometheus.NewCounter(
+var (
+	totalRequests = prometheus.NewCounter(
 		prometheus.CounterOpts{
-			Namespace: metricsNamespace,
-			Subsystem: tunnelSubsystem,
+			Namespace: connection.MetricsNamespace,
+			Subsystem: connection.TunnelSubsystem,
 			Name:      "total_requests",
 			Help:      "Amount of requests proxied through all the tunnels",
-		})
-	prometheus.MustRegister(totalRequests)
-
-	requestsPerTunnel := prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Namespace: metricsNamespace,
-			Subsystem: tunnelSubsystem,
-			Name:      "requests_per_tunnel",
-			Help:      "Amount of requests proxied through each tunnel",
 		},
-		[]string{"connection_id"},
 	)
-	prometheus.MustRegister(requestsPerTunnel)
-
-	concurrentRequestsPerTunnel := prometheus.NewGaugeVec(
+	concurrentRequests = prometheus.NewGauge(
 		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: tunnelSubsystem,
+			Namespace: connection.MetricsNamespace,
+			Subsystem: connection.TunnelSubsystem,
 			Name:      "concurrent_requests_per_tunnel",
 			Help:      "Concurrent requests proxied through each tunnel",
 		},
-		[]string{"connection_id"},
 	)
-	prometheus.MustRegister(concurrentRequestsPerTunnel)
-
-	maxConcurrentRequestsPerTunnel := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: tunnelSubsystem,
-			Name:      "max_concurrent_requests_per_tunnel",
-			Help:      "Largest number of concurrent requests proxied through each tunnel so far",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(maxConcurrentRequestsPerTunnel)
-
-	timerRetries := prometheus.NewGauge(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: tunnelSubsystem,
-			Name:      "timer_retries",
-			Help:      "Unacknowledged heart beats count",
-		})
-	prometheus.MustRegister(timerRetries)
-
-	responseByCode := prometheus.NewCounterVec(
+	responseByCode = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
-			Namespace: metricsNamespace,
-			Subsystem: tunnelSubsystem,
+			Namespace: connection.MetricsNamespace,
+			Subsystem: connection.TunnelSubsystem,
 			Name:      "response_by_code",
 			Help:      "Count of responses by HTTP status code",
 		},
 		[]string{"status_code"},
 	)
-	prometheus.MustRegister(responseByCode)
-
-	responseCodePerTunnel := prometheus.NewCounterVec(
+	requestErrors = prometheus.NewCounter(
 		prometheus.CounterOpts{
-			Namespace: metricsNamespace,
-			Subsystem: tunnelSubsystem,
-			Name:      "response_code_per_tunnel",
-			Help:      "Count of responses by HTTP status code fore each tunnel",
+			Namespace: connection.MetricsNamespace,
+			Subsystem: connection.TunnelSubsystem,
+			Name:      "request_errors",
+			Help:      "Count of error proxying to origin",
 		},
-		[]string{"connection_id", "status_code"},
 	)
-	prometheus.MustRegister(responseCodePerTunnel)
-
-	serverLocations := prometheus.NewGaugeVec(
+	haConnections = prometheus.NewGauge(
 		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: tunnelSubsystem,
-			Name:      "server_locations",
-			Help:      "Where each tunnel is connected to. 1 means current location, 0 means previous locations.",
+			Namespace: connection.MetricsNamespace,
+			Subsystem: connection.TunnelSubsystem,
+			Name:      "ha_connections",
+			Help:      "Number of active ha connections",
 		},
-		[]string{"connection_id", "location"},
 	)
-	prometheus.MustRegister(serverLocations)
+)
 
-	rpcFail := prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Namespace: metricsNamespace,
-			Subsystem: tunnelSubsystem,
-			Name:      "tunnel_rpc_fail",
-			Help:      "Count of RPC connection errors by type",
-		},
-		[]string{"error", "rpcName"},
+func init() {
+	prometheus.MustRegister(
+		totalRequests,
+		concurrentRequests,
+		responseByCode,
+		requestErrors,
+		haConnections,
 	)
-	prometheus.MustRegister(rpcFail)
-
-	registerFail := prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Namespace: metricsNamespace,
-			Subsystem: tunnelSubsystem,
-			Name:      "tunnel_register_fail",
-			Help:      "Count of tunnel registration errors by type",
-		},
-		[]string{"error", "rpcName"},
-	)
-	prometheus.MustRegister(registerFail)
-
-	userHostnamesCounts := prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Namespace: metricsNamespace,
-			Subsystem: tunnelSubsystem,
-			Name:      "user_hostnames_counts",
-			Help:      "Which user hostnames cloudflared is serving",
-		},
-		[]string{"userHostname"},
-	)
-	prometheus.MustRegister(userHostnamesCounts)
-
-	registerSuccess := prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Namespace: metricsNamespace,
-			Subsystem: tunnelSubsystem,
-			Name:      "tunnel_register_success",
-			Help:      "Count of successful tunnel registrations",
-		},
-		[]string{"rpcName"},
-	)
-	prometheus.MustRegister(registerSuccess)
-
-	return &TunnelMetrics{
-		haConnections:                  haConnections,
-		activeStreams:                  activeStreams,
-		totalRequests:                  totalRequests,
-		requestsPerTunnel:              requestsPerTunnel,
-		concurrentRequestsPerTunnel:    concurrentRequestsPerTunnel,
-		concurrentRequests:             make(map[string]uint64),
-		maxConcurrentRequestsPerTunnel: maxConcurrentRequestsPerTunnel,
-		maxConcurrentRequests:          make(map[string]uint64),
-		timerRetries:                   timerRetries,
-		responseByCode:                 responseByCode,
-		responseCodePerTunnel:          responseCodePerTunnel,
-		serverLocations:                serverLocations,
-		oldServerLocations:             make(map[string]string),
-		muxerMetrics:                   newMuxerMetrics(),
-		tunnelsHA:                      NewTunnelsForHA(),
-		regSuccess:                     registerSuccess,
-		regFail:                        registerFail,
-		rpcFail:                        rpcFail,
-		userHostnamesCounts:            userHostnamesCounts,
-	}
 }
 
-func (t *TunnelMetrics) incrementHaConnections() {
-	t.haConnections.Inc()
+func incrementRequests() {
+	totalRequests.Inc()
+	concurrentRequests.Inc()
 }
 
-func (t *TunnelMetrics) decrementHaConnections() {
-	t.haConnections.Dec()
-}
-
-func (t *TunnelMetrics) updateMuxerMetrics(connectionID string, metrics *h2mux.MuxerMetrics) {
-	t.muxerMetrics.update(connectionID, metrics)
-}
-
-func (t *TunnelMetrics) incrementRequests(connectionID string) {
-	t.concurrentRequestsLock.Lock()
-	var concurrentRequests uint64
-	var ok bool
-	if concurrentRequests, ok = t.concurrentRequests[connectionID]; ok {
-		t.concurrentRequests[connectionID]++
-		concurrentRequests++
-	} else {
-		t.concurrentRequests[connectionID] = 1
-		concurrentRequests = 1
-	}
-	if maxConcurrentRequests, ok := t.maxConcurrentRequests[connectionID]; (ok && maxConcurrentRequests < concurrentRequests) || !ok {
-		t.maxConcurrentRequests[connectionID] = concurrentRequests
-		t.maxConcurrentRequestsPerTunnel.WithLabelValues(connectionID).Set(float64(concurrentRequests))
-	}
-	t.concurrentRequestsLock.Unlock()
-
-	t.totalRequests.Inc()
-	t.requestsPerTunnel.WithLabelValues(connectionID).Inc()
-	t.concurrentRequestsPerTunnel.WithLabelValues(connectionID).Inc()
-}
-
-func (t *TunnelMetrics) decrementConcurrentRequests(connectionID string) {
-	t.concurrentRequestsLock.Lock()
-	if _, ok := t.concurrentRequests[connectionID]; ok {
-		t.concurrentRequests[connectionID]--
-	}
-	t.concurrentRequestsLock.Unlock()
-
-	t.concurrentRequestsPerTunnel.WithLabelValues(connectionID).Dec()
-}
-
-func (t *TunnelMetrics) incrementResponses(connectionID, code string) {
-	t.responseByCode.WithLabelValues(code).Inc()
-	t.responseCodePerTunnel.WithLabelValues(connectionID, code).Inc()
-
-}
-
-func (t *TunnelMetrics) registerServerLocation(connectionID, loc string) {
-	t.locationLock.Lock()
-	defer t.locationLock.Unlock()
-	if oldLoc, ok := t.oldServerLocations[connectionID]; ok && oldLoc == loc {
-		return
-	} else if ok {
-		t.serverLocations.WithLabelValues(connectionID, oldLoc).Dec()
-	}
-	t.serverLocations.WithLabelValues(connectionID, loc).Inc()
-	t.oldServerLocations[connectionID] = loc
+func decrementConcurrentRequests() {
+	concurrentRequests.Dec()
 }

origin/metrics_test.go

@@ -1,121 +0,0 @@
-package origin
-
-import (
-	"strconv"
-	"sync"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-// can only be called once
-var m = NewTunnelMetrics()
-
-func TestConcurrentRequestsSingleTunnel(t *testing.T) {
-	routines := 20
-	var wg sync.WaitGroup
-	wg.Add(routines)
-	for i := 0; i < routines; i++ {
-		go func() {
-			m.incrementRequests("0")
-			wg.Done()
-		}()
-	}
-	wg.Wait()
-	assert.Len(t, m.concurrentRequests, 1)
-	assert.Equal(t, uint64(routines), m.concurrentRequests["0"])
-	assert.Len(t, m.maxConcurrentRequests, 1)
-	assert.Equal(t, uint64(routines), m.maxConcurrentRequests["0"])
-
-	wg.Add(routines / 2)
-	for i := 0; i < routines/2; i++ {
-		go func() {
-			m.decrementConcurrentRequests("0")
-			wg.Done()
-		}()
-	}
-	wg.Wait()
-	assert.Equal(t, uint64(routines-routines/2), m.concurrentRequests["0"])
-	assert.Equal(t, uint64(routines), m.maxConcurrentRequests["0"])
-}
-
-func TestConcurrentRequestsMultiTunnel(t *testing.T) {
-	m.concurrentRequests = make(map[string]uint64)
-	m.maxConcurrentRequests = make(map[string]uint64)
-	tunnels := 20
-	var wg sync.WaitGroup
-	wg.Add(tunnels)
-	for i := 0; i < tunnels; i++ {
-		go func(i int) {
-			// if we have j < i, then tunnel 0 won't have a chance to call incrementRequests
-			for j := 0; j < i+1; j++ {
-				id := strconv.Itoa(i)
-				m.incrementRequests(id)
-			}
-			wg.Done()
-		}(i)
-	}
-	wg.Wait()
-
-	assert.Len(t, m.concurrentRequests, tunnels)
-	assert.Len(t, m.maxConcurrentRequests, tunnels)
-	for i := 0; i < tunnels; i++ {
-		id := strconv.Itoa(i)
-		assert.Equal(t, uint64(i+1), m.concurrentRequests[id])
-		assert.Equal(t, uint64(i+1), m.maxConcurrentRequests[id])
-	}
-
-	wg.Add(tunnels)
-	for i := 0; i < tunnels; i++ {
-		go func(i int) {
-			for j := 0; j < i+1; j++ {
-				id := strconv.Itoa(i)
-				m.decrementConcurrentRequests(id)
-			}
-			wg.Done()
-		}(i)
-	}
-	wg.Wait()
-
-	assert.Len(t, m.concurrentRequests, tunnels)
-	assert.Len(t, m.maxConcurrentRequests, tunnels)
-	for i := 0; i < tunnels; i++ {
-		id := strconv.Itoa(i)
-		assert.Equal(t, uint64(0), m.concurrentRequests[id])
-		assert.Equal(t, uint64(i+1), m.maxConcurrentRequests[id])
-	}
-
-}
-
-func TestRegisterServerLocation(t *testing.T) {
-	tunnels := 20
-	var wg sync.WaitGroup
-	wg.Add(tunnels)
-	for i := 0; i < tunnels; i++ {
-		go func(i int) {
-			id := strconv.Itoa(i)
-			m.registerServerLocation(id, "LHR")
-			wg.Done()
-		}(i)
-	}
-	wg.Wait()
-	for i := 0; i < tunnels; i++ {
-		id := strconv.Itoa(i)
-		assert.Equal(t, "LHR", m.oldServerLocations[id])
-	}
-
-	wg.Add(tunnels)
-	for i := 0; i < tunnels; i++ {
-		go func(i int) {
-			id := strconv.Itoa(i)
-			m.registerServerLocation(id, "AUS")
-			wg.Done()
-		}(i)
-	}
-	wg.Wait()
-	for i := 0; i < tunnels; i++ {
-		id := strconv.Itoa(i)
-		assert.Equal(t, "AUS", m.oldServerLocations[id])
-	}
-
-}

origin/proxy.go

@@ -0,0 +1,235 @@
+package origin
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"strconv"
+	"strings"
+
+	"github.com/cloudflare/cloudflared/buffer"
+	"github.com/cloudflare/cloudflared/connection"
+	"github.com/cloudflare/cloudflared/ingress"
+	"github.com/cloudflare/cloudflared/logger"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/cloudflare/cloudflared/websocket"
+	"github.com/pkg/errors"
+)
+
+const (
+	TagHeaderNamePrefix = "Cf-Warp-Tag-"
+)
+
+type client struct {
+	ingressRules ingress.Ingress
+	tags         []tunnelpogs.Tag
+	logger       logger.Service
+	bufferPool   *buffer.Pool
+}
+
+func NewClient(ingressRules ingress.Ingress, tags []tunnelpogs.Tag, logger logger.Service) connection.OriginClient {
+	return &client{
+		ingressRules: ingressRules,
+		tags:         tags,
+		logger:       logger,
+		bufferPool:   buffer.NewPool(512 * 1024),
+	}
+}
+
+func (c *client) Proxy(w connection.ResponseWriter, req *http.Request, isWebsocket bool) error {
+	incrementRequests()
+	defer decrementConcurrentRequests()
+
+	cfRay := findCfRayHeader(req)
+	lbProbe := isLBProbeRequest(req)
+
+	c.appendTagHeaders(req)
+	rule, ruleNum := c.ingressRules.FindMatchingRule(req.Host, req.URL.Path)
+	c.logRequest(req, cfRay, lbProbe, ruleNum)
+
+	var (
+		resp *http.Response
+		err  error
+	)
+	if isWebsocket {
+		resp, err = c.proxyWebsocket(w, req, rule)
+	} else {
+		resp, err = c.proxyHTTP(w, req, rule)
+	}
+	if err != nil {
+		c.logRequestError(err, cfRay, ruleNum)
+		w.WriteErrorResponse()
+		return err
+	}
+	c.logOriginResponse(resp, cfRay, lbProbe, ruleNum)
+	return nil
+}
+
+func (c *client) proxyHTTP(w connection.ResponseWriter, req *http.Request, rule *ingress.Rule) (*http.Response, error) {
+	// Support for WSGI Servers by switching transfer encoding from chunked to gzip/deflate
+	if rule.Config.DisableChunkedEncoding {
+		req.TransferEncoding = []string{"gzip", "deflate"}
+		cLength, err := strconv.Atoi(req.Header.Get("Content-Length"))
+		if err == nil {
+			req.ContentLength = int64(cLength)
+		}
+	}
+
+	// Request origin to keep connection alive to improve performance
+	req.Header.Set("Connection", "keep-alive")
+
+	if hostHeader := rule.Config.HTTPHostHeader; hostHeader != "" {
+		req.Header.Set("Host", hostHeader)
+		req.Host = hostHeader
+	}
+
+	resp, err := rule.Service.RoundTrip(req)
+	if err != nil {
+		return nil, errors.Wrap(err, "Error proxying request to origin")
+	}
+	defer resp.Body.Close()
+
+	err = w.WriteRespHeaders(resp)
+	if err != nil {
+		return nil, errors.Wrap(err, "Error writing response header")
+	}
+	if isEventStream(resp) {
+		c.logger.Debug("Detected Server-Side Events from Origin")
+		c.writeEventStream(w, resp.Body)
+	} else {
+		// Use CopyBuffer, because Copy only allocates a 32KiB buffer, and cross-stream
+		// compression generates dictionary on first write
+		buf := c.bufferPool.Get()
+		defer c.bufferPool.Put(buf)
+		io.CopyBuffer(w, resp.Body, buf)
+	}
+	return resp, nil
+}
+
+func (c *client) proxyWebsocket(w connection.ResponseWriter, req *http.Request, rule *ingress.Rule) (*http.Response, error) {
+	if hostHeader := rule.Config.HTTPHostHeader; hostHeader != "" {
+		req.Header.Set("Host", hostHeader)
+		req.Host = hostHeader
+	}
+
+	dialler, ok := rule.Service.(websocket.Dialler)
+	if !ok {
+		return nil, fmt.Errorf("Websockets aren't supported by the origin service '%s'", rule.Service)
+	}
+	conn, resp, err := websocket.ClientConnect(req, dialler)
+	if err != nil {
+		return nil, err
+	}
+
+	serveCtx, cancel := context.WithCancel(req.Context())
+	connClosedChan := make(chan struct{})
+	go func() {
+		// serveCtx is done if req is cancelled, or streamWebsocket returns
+		<-serveCtx.Done()
+		conn.Close()
+		close(connClosedChan)
+	}()
+
+	// Copy to/from stream to the undelying connection. Use the underlying
+	// connection because cloudflared doesn't operate on the message themselves
+	err = c.streamWebsocket(w, conn.UnderlyingConn(), resp)
+	cancel()
+
+	// We need to make sure conn is closed before returning, otherwise we might write to conn after Proxy returns
+	<-connClosedChan
+	return resp, err
+}
+
+func (c *client) streamWebsocket(w connection.ResponseWriter, conn net.Conn, resp *http.Response) error {
+	err := w.WriteRespHeaders(resp)
+	if err != nil {
+		return errors.Wrap(err, "Error writing websocket response header")
+	}
+	websocket.Stream(conn, w)
+	return nil
+}
+
+func (c *client) writeEventStream(w connection.ResponseWriter, respBody io.ReadCloser) {
+	reader := bufio.NewReader(respBody)
+	for {
+		line, err := reader.ReadBytes('\n')
+		if err != nil {
+			break
+		}
+		w.Write(line)
+	}
+}
+
+func (c *client) appendTagHeaders(r *http.Request) {
+	for _, tag := range c.tags {
+		r.Header.Add(TagHeaderNamePrefix+tag.Name, tag.Value)
+	}
+}
+
+func (c *client) logRequest(r *http.Request, cfRay string, lbProbe bool, ruleNum int) {
+	if cfRay != "" {
+		c.logger.Debugf("CF-RAY: %s %s %s %s", cfRay, r.Method, r.URL, r.Proto)
+	} else if lbProbe {
+		c.logger.Debugf("CF-RAY: %s Load Balancer health check %s %s %s", cfRay, r.Method, r.URL, r.Proto)
+	} else {
+		c.logger.Debugf("All requests should have a CF-RAY header. Please open a support ticket with Cloudflare. %s %s %s ", r.Method, r.URL, r.Proto)
+	}
+	c.logger.Debugf("CF-RAY: %s Request Headers %+v", cfRay, r.Header)
+	c.logger.Debugf("CF-RAY: %s Serving with ingress rule %d", cfRay, ruleNum)
+
+	if contentLen := r.ContentLength; contentLen == -1 {
+		c.logger.Debugf("CF-RAY: %s Request Content length unknown", cfRay)
+	} else {
+		c.logger.Debugf("CF-RAY: %s Request content length %d", cfRay, contentLen)
+	}
+}
+
+func (c *client) logOriginResponse(r *http.Response, cfRay string, lbProbe bool, ruleNum int) {
+	responseByCode.WithLabelValues(strconv.Itoa(r.StatusCode)).Inc()
+	if cfRay != "" {
+		c.logger.Infof("CF-RAY: %s Status: %s served by ingress %d", cfRay, r.Status, ruleNum)
+	} else if lbProbe {
+		c.logger.Debugf("Response to Load Balancer health check %s", r.Status)
+	} else {
+		c.logger.Debugf("Status: %s served by ingress %d", r.Status, ruleNum)
+	}
+	c.logger.Debugf("CF-RAY: %s Response Headers %+v", cfRay, r.Header)
+
+	if contentLen := r.ContentLength; contentLen == -1 {
+		c.logger.Debugf("CF-RAY: %s Response content length unknown", cfRay)
+	} else {
+		c.logger.Debugf("CF-RAY: %s Response content length %d", cfRay, contentLen)
+	}
+}
+
+func (c *client) logRequestError(err error, cfRay string, ruleNum int) {
+	requestErrors.Inc()
+	if cfRay != "" {
+		c.logger.Errorf("CF-RAY: %s Proxying to ingress %d error: %v", cfRay, ruleNum, err)
+	} else {
+		c.logger.Errorf("Proxying to ingress %d error: %v", ruleNum, err)
+	}
+
+}
+
+func findCfRayHeader(req *http.Request) string {
+	return req.Header.Get("Cf-Ray")
+}
+
+func isLBProbeRequest(req *http.Request) bool {
+	return strings.HasPrefix(req.UserAgent(), lbProbeUserAgentPrefix)
+}
+
+func uint8ToString(input uint8) string {
+	return strconv.FormatUint(uint64(input), 10)
+}
+
+func isEventStream(response *http.Response) bool {
+	if response.Header.Get("content-type") == "text/event-stream" {
+		return true
+	}
+	return false
+}

origin/proxy_test.go

@@ -0,0 +1,352 @@
+package origin
+
+import (
+	"bytes"
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/connection"
+	"github.com/cloudflare/cloudflared/hello"
+	"github.com/cloudflare/cloudflared/ingress"
+	"github.com/cloudflare/cloudflared/logger"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/urfave/cli/v2"
+
+	"github.com/gobwas/ws/wsutil"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+var (
+	testTags = []tunnelpogs.Tag(nil)
+)
+
+type mockHTTPRespWriter struct {
+	*httptest.ResponseRecorder
+}
+
+func newMockHTTPRespWriter() *mockHTTPRespWriter {
+	return &mockHTTPRespWriter{
+		httptest.NewRecorder(),
+	}
+}
+
+func (w *mockHTTPRespWriter) WriteRespHeaders(resp *http.Response) error {
+	w.WriteHeader(resp.StatusCode)
+	for header, val := range resp.Header {
+		w.Header()[header] = val
+	}
+	return nil
+}
+
+func (w *mockHTTPRespWriter) WriteErrorResponse() {
+	w.WriteHeader(http.StatusBadGateway)
+	w.Write([]byte("http response error"))
+}
+
+func (w *mockHTTPRespWriter) Read(data []byte) (int, error) {
+	return 0, fmt.Errorf("mockHTTPRespWriter doesn't implement io.Reader")
+}
+
+type mockWSRespWriter struct {
+	*mockHTTPRespWriter
+	writeNotification chan []byte
+	reader            io.Reader
+}
+
+func newMockWSRespWriter(reader io.Reader) *mockWSRespWriter {
+	return &mockWSRespWriter{
+		newMockHTTPRespWriter(),
+		make(chan []byte),
+		reader,
+	}
+}
+
+func (w *mockWSRespWriter) Write(data []byte) (int, error) {
+	w.writeNotification <- data
+	return len(data), nil
+}
+
+func (w *mockWSRespWriter) respBody() io.ReadWriter {
+	data := <-w.writeNotification
+	return bytes.NewBuffer(data)
+}
+
+func (w *mockWSRespWriter) Read(data []byte) (int, error) {
+	return w.reader.Read(data)
+}
+
+type mockSSERespWriter struct {
+	*mockHTTPRespWriter
+	writeNotification chan []byte
+}
+
+func newMockSSERespWriter() *mockSSERespWriter {
+	return &mockSSERespWriter{
+		newMockHTTPRespWriter(),
+		make(chan []byte),
+	}
+}
+
+func (w *mockSSERespWriter) Write(data []byte) (int, error) {
+	w.writeNotification <- data
+	return len(data), nil
+}
+
+func (w *mockSSERespWriter) ReadBytes() []byte {
+	return <-w.writeNotification
+}
+
+func TestProxySingleOrigin(t *testing.T) {
+	logger, err := logger.New()
+	require.NoError(t, err)
+
+	ctx, cancel := context.WithCancel(context.Background())
+
+	flagSet := flag.NewFlagSet(t.Name(), flag.PanicOnError)
+	flagSet.Bool("hello-world", true, "")
+
+	cliCtx := cli.NewContext(cli.NewApp(), flagSet, nil)
+	err = cliCtx.Set("hello-world", "true")
+	require.NoError(t, err)
+
+	allowURLFromArgs := false
+	ingressRule, err := ingress.NewSingleOrigin(cliCtx, allowURLFromArgs, logger)
+	require.NoError(t, err)
+
+	var wg sync.WaitGroup
+	errC := make(chan error)
+	ingressRule.StartOrigins(&wg, logger, ctx.Done(), errC)
+
+	client := NewClient(ingressRule, testTags, logger)
+	t.Run("testProxyHTTP", testProxyHTTP(t, client))
+	t.Run("testProxyWebsocket", testProxyWebsocket(t, client))
+	t.Run("testProxySSE", testProxySSE(t, client))
+	cancel()
+	wg.Wait()
+}
+
+func testProxyHTTP(t *testing.T, client connection.OriginClient) func(t *testing.T) {
+	return func(t *testing.T) {
+		respWriter := newMockHTTPRespWriter()
+		req, err := http.NewRequest(http.MethodGet, "http://localhost:8080", nil)
+		require.NoError(t, err)
+
+		err = client.Proxy(respWriter, req, false)
+		require.NoError(t, err)
+
+		assert.Equal(t, http.StatusOK, respWriter.Code)
+	}
+}
+
+func testProxyWebsocket(t *testing.T, client connection.OriginClient) func(t *testing.T) {
+	return func(t *testing.T) {
+		// WSRoute is a websocket echo handler
+		ctx, cancel := context.WithCancel(context.Background())
+		req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("http://localhost:8080%s", hello.WSRoute), nil)
+
+		readPipe, writePipe := io.Pipe()
+		respWriter := newMockWSRespWriter(readPipe)
+
+		var wg sync.WaitGroup
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			err = client.Proxy(respWriter, req, true)
+			require.NoError(t, err)
+
+			require.Equal(t, http.StatusSwitchingProtocols, respWriter.Code)
+		}()
+
+		msg := []byte("test websocket")
+		err = wsutil.WriteClientText(writePipe, msg)
+		require.NoError(t, err)
+
+		// ReadServerText reads next data message from rw, considering that caller represents client side.
+		returnedMsg, err := wsutil.ReadServerText(respWriter.respBody())
+		require.NoError(t, err)
+		require.Equal(t, msg, returnedMsg)
+
+		err = wsutil.WriteClientBinary(writePipe, msg)
+		require.NoError(t, err)
+
+		returnedMsg, err = wsutil.ReadServerBinary(respWriter.respBody())
+		require.NoError(t, err)
+		require.Equal(t, msg, returnedMsg)
+
+		cancel()
+		wg.Wait()
+	}
+}
+
+func testProxySSE(t *testing.T, client connection.OriginClient) func(t *testing.T) {
+	return func(t *testing.T) {
+		var (
+			pushCount = 50
+			pushFreq  = time.Duration(time.Millisecond * 10)
+		)
+		respWriter := newMockSSERespWriter()
+		ctx, cancel := context.WithCancel(context.Background())
+		req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("http://localhost:8080%s?freq=%s", hello.SSERoute, pushFreq), nil)
+		require.NoError(t, err)
+
+		var wg sync.WaitGroup
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			err = client.Proxy(respWriter, req, false)
+			require.NoError(t, err)
+
+			require.Equal(t, http.StatusOK, respWriter.Code)
+		}()
+
+		for i := 0; i < pushCount; i++ {
+			line := respWriter.ReadBytes()
+			expect := fmt.Sprintf("%d\n", i)
+			require.Equal(t, []byte(expect), line, fmt.Sprintf("Expect to read %v, got %v", expect, line))
+
+			line = respWriter.ReadBytes()
+			require.Equal(t, []byte("\n"), line, fmt.Sprintf("Expect to read '\n', got %v", line))
+		}
+
+		cancel()
+		wg.Wait()
+	}
+}
+
+func TestProxyMultipleOrigins(t *testing.T) {
+	api := httptest.NewServer(mockAPI{})
+	defer api.Close()
+
+	unvalidatedIngress := []config.UnvalidatedIngressRule{
+		{
+			Hostname: "api.example.com",
+			Service:  api.URL,
+		},
+		{
+			Hostname: "hello.example.com",
+			Service:  "hello-world",
+		},
+		{
+			Hostname: "health.example.com",
+			Path:     "/health",
+			Service:  "http_status:200",
+		},
+		{
+			Hostname: "*",
+			Service:  "http_status:404",
+		},
+	}
+
+	ingress, err := ingress.ParseIngress(&config.Configuration{
+		TunnelID: t.Name(),
+		Ingress:  unvalidatedIngress,
+	})
+	require.NoError(t, err)
+
+	logger, err := logger.New()
+	require.NoError(t, err)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	errC := make(chan error)
+	var wg sync.WaitGroup
+	ingress.StartOrigins(&wg, logger, ctx.Done(), errC)
+
+	client := NewClient(ingress, testTags, logger)
+
+	tests := []struct {
+		url            string
+		expectedStatus int
+		expectedBody   []byte
+	}{
+		{
+			url:            "http://api.example.com",
+			expectedStatus: http.StatusCreated,
+			expectedBody:   []byte("Created"),
+		},
+		{
+			url:            fmt.Sprintf("http://hello.example.com%s", hello.HealthRoute),
+			expectedStatus: http.StatusOK,
+			expectedBody:   []byte("ok"),
+		},
+		{
+			url:            "http://health.example.com/health",
+			expectedStatus: http.StatusOK,
+		},
+		{
+			url:            "http://health.example.com/",
+			expectedStatus: http.StatusNotFound,
+		},
+		{
+			url:            "http://not-found.example.com",
+			expectedStatus: http.StatusNotFound,
+		},
+	}
+
+	for _, test := range tests {
+		respWriter := newMockHTTPRespWriter()
+		req, err := http.NewRequest(http.MethodGet, test.url, nil)
+		require.NoError(t, err)
+
+		err = client.Proxy(respWriter, req, false)
+		require.NoError(t, err)
+
+		assert.Equal(t, test.expectedStatus, respWriter.Code)
+		if test.expectedBody != nil {
+			assert.Equal(t, test.expectedBody, respWriter.Body.Bytes())
+		} else {
+			assert.Equal(t, 0, respWriter.Body.Len())
+		}
+	}
+	cancel()
+	wg.Wait()
+}
+
+type mockAPI struct{}
+
+func (ma mockAPI) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	w.WriteHeader(http.StatusCreated)
+	w.Write([]byte("Created"))
+}
+
+type errorOriginTransport struct{}
+
+func (errorOriginTransport) RoundTrip(*http.Request) (*http.Response, error) {
+	return nil, fmt.Errorf("Proxy error")
+}
+
+func TestProxyError(t *testing.T) {
+	ingress := ingress.Ingress{
+		Rules: []ingress.Rule{
+			{
+				Hostname: "*",
+				Path:     nil,
+				Service: ingress.MockOriginService{
+					Transport: errorOriginTransport{},
+				},
+			},
+		},
+	}
+
+	logger, err := logger.New()
+	require.NoError(t, err)
+
+	client := NewClient(ingress, testTags, logger)
+
+	respWriter := newMockHTTPRespWriter()
+	req, err := http.NewRequest(http.MethodGet, "http://127.0.0.1", nil)
+	assert.NoError(t, err)
+
+	err = client.Proxy(respWriter, req, false)
+	assert.Error(t, err)
+	assert.Equal(t, http.StatusBadGateway, respWriter.Code)
+	assert.Equal(t, "http response error", respWriter.Body.String())
+}

origin/reconnect.go

@@ -7,11 +7,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/cloudflare/cloudflared/h2mux"
-	"github.com/cloudflare/cloudflared/logger"
-	"github.com/cloudflare/cloudflared/tunnelrpc"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
-	"github.com/google/uuid"
 	"github.com/prometheus/client_golang/prometheus"
 )
 
@@ -138,52 +134,3 @@ func (cm *reconnectCredentialManager) RefreshAuth(
 		return nil, err
 	}
 }
-
-func ReconnectTunnel(
-	ctx context.Context,
-	muxer *h2mux.Muxer,
-	config *TunnelConfig,
-	logger logger.Service,
-	connectionID uint8,
-	originLocalAddr string,
-	uuid uuid.UUID,
-	credentialManager *reconnectCredentialManager,
-) error {
-	token, err := credentialManager.ReconnectToken()
-	if err != nil {
-		return err
-	}
-	eventDigest, err := credentialManager.EventDigest(connectionID)
-	if err != nil {
-		return err
-	}
-	connDigest, err := credentialManager.ConnDigest(connectionID)
-	if err != nil {
-		return err
-	}
-
-	config.TransportLogger.Debug("initiating RPC stream to reconnect")
-	rpcClient, err := newTunnelRPCClient(ctx, muxer, config, reconnect)
-	if err != nil {
-		return err
-	}
-	defer rpcClient.Close()
-	// Request server info without blocking tunnel registration; must use capnp library directly.
-	serverInfoPromise := tunnelrpc.TunnelServer{Client: rpcClient.Client}.GetServerInfo(ctx, func(tunnelrpc.TunnelServer_getServerInfo_Params) error {
-		return nil
-	})
-	LogServerInfo(serverInfoPromise.Result(), connectionID, config.Metrics, logger, config.TunnelEventChan)
-	registration := rpcClient.ReconnectTunnel(
-		ctx,
-		token,
-		eventDigest,
-		connDigest,
-		config.Hostname,
-		config.RegistrationOptions(connectionID, originLocalAddr, uuid),
-	)
-	if registrationErr := registration.DeserializeError(); registrationErr != nil {
-		// ReconnectTunnel RPC failure
-		return processRegisterTunnelError(registrationErr, config.Metrics, reconnect)
-	}
-	return processRegistrationSuccess(config, logger, connectionID, registration, reconnect, credentialManager)
-}

origin/supervisor.go

@@ -8,7 +8,6 @@ import (
 
 	"github.com/google/uuid"
 
-	"github.com/cloudflare/cloudflared/buffer"
 	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/edgediscovery"
 	"github.com/cloudflare/cloudflared/h2mux"
@@ -18,10 +17,10 @@ import (
 )
 
 const (
+	// SRV and TXT record resolution TTL
+	ResolveTTL = time.Hour
 	// Waiting time before retrying a failed tunnel connection
 	tunnelRetryDuration = time.Second * 10
-	// SRV record resolution TTL
-	resolveTTL = time.Hour
 	// Interval between registering new tunnels
 	registrationInterval = time.Second
 
@@ -44,8 +43,6 @@ type Supervisor struct {
 	cloudflaredUUID   uuid.UUID
 	config            *TunnelConfig
 	edgeIPs           *edgediscovery.Edge
-	lastResolve       time.Time
-	resolverC         chan resolveResult
 	tunnelErrors      chan tunnelError
 	tunnelsConnecting map[int]chan struct{}
 	// nextConnectedIndex and nextConnectedSignal are used to wait for all
@@ -56,12 +53,7 @@ type Supervisor struct {
 	logger logger.Service
 
 	reconnectCredentialManager *reconnectCredentialManager
-
-	bufferPool *buffer.Pool
-}
-
-type resolveResult struct {
-	err error
+	useReconnectToken          bool
 }
 
 type tunnelError struct {
@@ -84,6 +76,11 @@ func NewSupervisor(config *TunnelConfig, cloudflaredUUID uuid.UUID) (*Supervisor
 		return nil, err
 	}
 
+	useReconnectToken := false
+	if config.ClassicTunnel != nil {
+		useReconnectToken = config.ClassicTunnel.UseReconnectToken
+	}
+
 	return &Supervisor{
 		cloudflaredUUID:            cloudflaredUUID,
 		config:                     config,
@@ -91,13 +88,12 @@ func NewSupervisor(config *TunnelConfig, cloudflaredUUID uuid.UUID) (*Supervisor
 		tunnelErrors:               make(chan tunnelError),
 		tunnelsConnecting:          map[int]chan struct{}{},
 		logger:                     config.Logger,
-		reconnectCredentialManager: newReconnectCredentialManager(metricsNamespace, tunnelSubsystem, config.HAConnections),
-		bufferPool:                 buffer.NewPool(512 * 1024),
+		reconnectCredentialManager: newReconnectCredentialManager(connection.MetricsNamespace, connection.TunnelSubsystem, config.HAConnections),
+		useReconnectToken:          useReconnectToken,
 	}, nil
 }
 
 func (s *Supervisor) Run(ctx context.Context, connectedSignal *signal.Signal, reconnectCh chan ReconnectSignal) error {
-	logger := s.config.Logger
 	if err := s.initialize(ctx, connectedSignal, reconnectCh); err != nil {
 		return err
 	}
@@ -110,11 +106,11 @@ func (s *Supervisor) Run(ctx context.Context, connectedSignal *signal.Signal, re
 	refreshAuthBackoff := &BackoffHandler{MaxRetries: refreshAuthMaxBackoff, BaseTime: refreshAuthRetryDuration, RetryForever: true}
 	var refreshAuthBackoffTimer <-chan time.Time
 
-	if s.config.UseReconnectToken {
+	if s.useReconnectToken {
 		if timer, err := s.reconnectCredentialManager.RefreshAuth(ctx, refreshAuthBackoff, s.authenticate); err == nil {
 			refreshAuthBackoffTimer = timer
 		} else {
-			logger.Errorf("supervisor: initial refreshAuth failed, retrying in %v: %s", refreshAuthRetryDuration, err)
+			s.logger.Errorf("supervisor: initial refreshAuth failed, retrying in %v: %s", refreshAuthRetryDuration, err)
 			refreshAuthBackoffTimer = time.After(refreshAuthRetryDuration)
 		}
 	}
@@ -133,7 +129,7 @@ func (s *Supervisor) Run(ctx context.Context, connectedSignal *signal.Signal, re
 		case tunnelError := <-s.tunnelErrors:
 			tunnelsActive--
 			if tunnelError.err != nil {
-				logger.Infof("supervisor: Tunnel disconnected due to error: %s", tunnelError.err)
+				s.logger.Infof("supervisor: Tunnel disconnected due to error: %s", tunnelError.err)
 				tunnelsWaiting = append(tunnelsWaiting, tunnelError.index)
 				s.waitForNextTunnel(tunnelError.index)
 
@@ -156,7 +152,7 @@ func (s *Supervisor) Run(ctx context.Context, connectedSignal *signal.Signal, re
 		case <-refreshAuthBackoffTimer:
 			newTimer, err := s.reconnectCredentialManager.RefreshAuth(ctx, refreshAuthBackoff, s.authenticate)
 			if err != nil {
-				logger.Errorf("supervisor: Authentication failed: %s", err)
+				s.logger.Errorf("supervisor: Authentication failed: %s", err)
 				// Permanent failure. Leave the `select` without setting the
 				// channel to be non-null, so we'll never hit this case of the `select` again.
 				continue
@@ -168,27 +164,15 @@ func (s *Supervisor) Run(ctx context.Context, connectedSignal *signal.Signal, re
 				// No more tunnels outstanding, clear backoff timer
 				backoff.SetGracePeriod()
 			}
-		// DNS resolution returned
-		case result := <-s.resolverC:
-			s.lastResolve = time.Now()
-			s.resolverC = nil
-			if result.err == nil {
-				logger.Debug("supervisor: Service discovery refresh complete")
-			} else {
-				logger.Errorf("supervisor: Service discovery error: %s", result.err)
-			}
 		}
 	}
 }
 
 // Returns nil if initialization succeeded, else the initialization error.
 func (s *Supervisor) initialize(ctx context.Context, connectedSignal *signal.Signal, reconnectCh chan ReconnectSignal) error {
-	logger := s.logger
-
-	s.lastResolve = time.Now()
 	availableAddrs := int(s.edgeIPs.AvailableAddrs())
 	if s.config.HAConnections > availableAddrs {
-		logger.Infof("You requested %d HA connections but I can give you at most %d.", s.config.HAConnections, availableAddrs)
+		s.logger.Infof("You requested %d HA connections but I can give you at most %d.", s.config.HAConnections, availableAddrs)
 		s.config.HAConnections = availableAddrs
 	}
 
@@ -227,7 +211,7 @@ func (s *Supervisor) startFirstTunnel(ctx context.Context, connectedSignal *sign
 		return
 	}
 
-	err = ServeTunnelLoop(ctx, s.reconnectCredentialManager, s.config, addr, firstConnIndex, connectedSignal, s.cloudflaredUUID, s.bufferPool, reconnectCh)
+	err = ServeTunnelLoop(ctx, s.reconnectCredentialManager, s.config, addr, firstConnIndex, connectedSignal, s.cloudflaredUUID, reconnectCh)
 	// If the first tunnel disconnects, keep restarting it.
 	edgeErrors := 0
 	for s.unusedIPs() {
@@ -239,7 +223,7 @@ func (s *Supervisor) startFirstTunnel(ctx context.Context, connectedSignal *sign
 			return
 		// try the next address if it was a dialError(network problem) or
 		// dupConnRegisterTunnelError
-		case connection.DialError, dupConnRegisterTunnelError:
+		case edgediscovery.DialError, connection.DupConnRegisterTunnelError:
 			edgeErrors++
 		default:
 			return
@@ -250,7 +234,7 @@ func (s *Supervisor) startFirstTunnel(ctx context.Context, connectedSignal *sign
 				return
 			}
 		}
-		err = ServeTunnelLoop(ctx, s.reconnectCredentialManager, s.config, addr, firstConnIndex, connectedSignal, s.cloudflaredUUID, s.bufferPool, reconnectCh)
+		err = ServeTunnelLoop(ctx, s.reconnectCredentialManager, s.config, addr, firstConnIndex, connectedSignal, s.cloudflaredUUID, reconnectCh)
 	}
 }
 
@@ -269,7 +253,7 @@ func (s *Supervisor) startTunnel(ctx context.Context, index int, connectedSignal
 	if err != nil {
 		return
 	}
-	err = ServeTunnelLoop(ctx, s.reconnectCredentialManager, s.config, addr, uint8(index), connectedSignal, s.cloudflaredUUID, s.bufferPool, reconnectCh)
+	err = ServeTunnelLoop(ctx, s.reconnectCredentialManager, s.config, addr, uint8(index), connectedSignal, s.cloudflaredUUID, reconnectCh)
 }
 
 func (s *Supervisor) newConnectedTunnelSignal(index int) *signal.Signal {
@@ -301,7 +285,7 @@ func (s *Supervisor) authenticate(ctx context.Context, numPreviousAttempts int)
 		return nil, err
 	}
 
-	edgeConn, err := connection.DialEdge(ctx, dialTimeout, s.config.TlsConfig, arbitraryEdgeIP)
+	edgeConn, err := edgediscovery.DialEdge(ctx, dialTimeout, s.config.EdgeTLSConfigs[connection.H2mux], arbitraryEdgeIP)
 	if err != nil {
 		return nil, err
 	}
@@ -311,8 +295,8 @@ func (s *Supervisor) authenticate(ctx context.Context, numPreviousAttempts int)
 		// This callback is invoked by h2mux when the edge initiates a stream.
 		return nil // noop
 	})
-	muxerConfig := s.config.muxerConfig(handler)
-	muxer, err := h2mux.Handshake(edgeConn, edgeConn, muxerConfig, s.config.Metrics.activeStreams)
+	muxerConfig := s.config.MuxerConfig.H2MuxerConfig(handler, s.logger)
+	muxer, err := h2mux.Handshake(edgeConn, edgeConn, *muxerConfig, h2mux.ActiveStreams)
 	if err != nil {
 		return nil, err
 	}
@@ -323,23 +307,15 @@ func (s *Supervisor) authenticate(ctx context.Context, numPreviousAttempts int)
 		<-muxer.Shutdown()
 	}()
 
-	rpcClient, err := newTunnelRPCClient(ctx, muxer, s.config, authenticate)
+	stream, err := muxer.OpenRPCStream(ctx)
 	if err != nil {
 		return nil, err
 	}
+	rpcClient := connection.NewTunnelServerClient(ctx, stream, s.logger)
 	defer rpcClient.Close()
 
 	const arbitraryConnectionID = uint8(0)
 	registrationOptions := s.config.RegistrationOptions(arbitraryConnectionID, edgeConn.LocalAddr().String(), s.cloudflaredUUID)
 	registrationOptions.NumPreviousAttempts = uint8(numPreviousAttempts)
-	authResponse, err := rpcClient.Authenticate(
-		ctx,
-		s.config.OriginCert,
-		s.config.Hostname,
-		registrationOptions,
-	)
-	if err != nil {
-		return nil, err
-	}
-	return authResponse.Outcome(), nil
+	return rpcClient.Authenticate(ctx, s.config.ClassicTunnel, registrationOptions)
 }

origin/tunnel_test.go

@@ -0,0 +1,90 @@
+package origin
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/cloudflare/cloudflared/connection"
+	"github.com/cloudflare/cloudflared/logger"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/stretchr/testify/assert"
+)
+
+type dynamicMockFetcher struct {
+	percentage int32
+	err        error
+}
+
+func (dmf *dynamicMockFetcher) fetch() connection.PercentageFetcher {
+	return func() (int32, error) {
+		if dmf.err != nil {
+			return 0, dmf.err
+		}
+		return dmf.percentage, nil
+	}
+}
+func TestWaitForBackoffFallback(t *testing.T) {
+	maxRetries := uint(3)
+	backoff := BackoffHandler{
+		MaxRetries: maxRetries,
+		BaseTime:   time.Millisecond * 10,
+	}
+	ctx := context.Background()
+	logger, err := logger.New()
+	assert.NoError(t, err)
+	resolveTTL := time.Duration(0)
+	namedTunnel := &connection.NamedTunnelConfig{
+		Auth: pogs.TunnelAuth{
+			AccountTag: "test-account",
+		},
+	}
+	mockFetcher := dynamicMockFetcher{
+		percentage: 0,
+	}
+	protocolSelector, err := connection.NewProtocolSelector(connection.HTTP2.String(), namedTunnel, mockFetcher.fetch(), resolveTTL, logger)
+	assert.NoError(t, err)
+	config := &TunnelConfig{
+		Logger:           logger,
+		ProtocolSelector: protocolSelector,
+	}
+	connIndex := uint8(1)
+
+	initProtocol := protocolSelector.Current()
+	assert.Equal(t, connection.HTTP2, initProtocol)
+
+	protocallFallback := &protocallFallback{
+		backoff,
+		initProtocol,
+		false,
+	}
+
+	// Retry #0 and #1. At retry #2, we switch protocol, so the fallback loop has one more retry than this
+	for i := 0; i < int(maxRetries-1); i++ {
+		err := waitForBackoff(ctx, protocallFallback, config, connIndex, fmt.Errorf("Some error"))
+		assert.NoError(t, err)
+		assert.Equal(t, initProtocol, protocallFallback.protocol)
+	}
+
+	// Retry fallback protocol
+	for i := 0; i < int(maxRetries); i++ {
+		err := waitForBackoff(ctx, protocallFallback, config, connIndex, fmt.Errorf("Some error"))
+		assert.NoError(t, err)
+		fallback, ok := protocolSelector.Fallback()
+		assert.True(t, ok)
+		assert.Equal(t, fallback, protocallFallback.protocol)
+	}
+
+	currentGlobalProtocol := protocolSelector.Current()
+	assert.Equal(t, initProtocol, currentGlobalProtocol)
+
+	// No protocol to fallback, return error
+	err = waitForBackoff(ctx, protocallFallback, config, connIndex, fmt.Errorf("Some error"))
+	assert.Error(t, err)
+
+	protocallFallback.reset()
+	err = waitForBackoff(ctx, protocallFallback, config, connIndex, fmt.Errorf("New error"))
+	assert.NoError(t, err)
+	assert.Equal(t, initProtocol, protocallFallback.protocol)
+}

tlsconfig/certreloader.go

@@ -17,8 +17,6 @@ import (
 const (
 	OriginCAPoolFlag = "origin-ca-pool"
 	CaCertFlag       = "cacert"
-
-	edgeTLSServerName = "cftunnel.com"
 )
 
 // CertReloader can load and reload a TLS certificate from a particular filepath.
@@ -65,10 +63,9 @@ func (cr *CertReloader) LoadCert() error {
 	return nil
 }
 
-func LoadOriginCA(c *cli.Context, logger logger.Service) (*x509.CertPool, error) {
+func LoadOriginCA(originCAPoolFilename string, logger logger.Service) (*x509.CertPool, error) {
 	var originCustomCAPool []byte
 
-	originCAPoolFilename := c.String(OriginCAPoolFlag)
 	if originCAPoolFilename != "" {
 		var err error
 		originCustomCAPool, err = ioutil.ReadFile(originCAPoolFilename)
@@ -121,13 +118,13 @@ func LoadCustomOriginCA(originCAFilename string) (*x509.CertPool, error) {
 	return certPool, nil
 }
 
-func CreateTunnelConfig(c *cli.Context) (*tls.Config, error) {
+func CreateTunnelConfig(c *cli.Context, serverName string) (*tls.Config, error) {
 	var rootCAs []string
 	if c.String(CaCertFlag) != "" {
 		rootCAs = append(rootCAs, c.String(CaCertFlag))
 	}
 
-	userConfig := &TLSParameters{RootCAs: rootCAs, ServerName: edgeTLSServerName}
+	userConfig := &TLSParameters{RootCAs: rootCAs, ServerName: serverName}
 	tlsConfig, err := GetConfig(userConfig)
 	if err != nil {
 		return nil, err

tunnelrpc/pogs/connectionrpc.go

@@ -93,6 +93,11 @@ type RegistrationServer_PogsClient struct {
 	Conn   *rpc.Conn
 }
 
+func (c RegistrationServer_PogsClient) Close() error {
+	c.Client.Close()
+	return c.Conn.Close()
+}
+
 func (c RegistrationServer_PogsClient) RegisterConnection(ctx context.Context, auth TunnelAuth, tunnelID uuid.UUID, connIndex byte, options *ConnectionOptions) (*ConnectionDetails, error) {
 	client := tunnelrpc.TunnelServer{Client: c.Client}
 	promise := client.RegisterConnection(ctx, func(p tunnelrpc.RegistrationServer_registerConnection_Params) error {

validation/validation.go

@@ -66,7 +66,15 @@ func ValidateHostname(hostname string) (string, error) {
 // but when it does not, the path is preserved:
 //   ValidateUrl("localhost:8080/api/") => "http://localhost:8080/api/"
 // This is arguably a bug, but changing it might break some cloudflared users.
-func ValidateUrl(originUrl string) (string, error) {
+func ValidateUrl(originUrl string) (*url.URL, error) {
+	urlStr, err := validateUrlString(originUrl)
+	if err != nil {
+		return nil, err
+	}
+	return url.Parse(urlStr)
+}
+
+func validateUrlString(originUrl string) (string, error) {
 	if originUrl == "" {
 		return "", fmt.Errorf("URL should not be empty")
 	}
@@ -157,6 +165,7 @@ func validateIP(scheme, host, port string) (string, error) {
 	return fmt.Sprintf("%s://%s", scheme, host), nil
 }
 
+// originURL shouldn't be a pointer, because this function might change the scheme
 func ValidateHTTPService(originURL string, hostname string, transport http.RoundTripper) error {
 	parsedURL, err := url.Parse(originURL)
 	if err != nil {
@@ -176,28 +185,32 @@ func ValidateHTTPService(originURL string, hostname string, transport http.Round
 		return err
 	}
 	initialRequest.Host = hostname
-	_, initialErr := client.Do(initialRequest)
-	if initialErr != nil {
-		// Attempt the same endpoint via the other protocol (http/https); maybe we have better luck?
-		oldScheme := parsedURL.Scheme
-		parsedURL.Scheme = toggleProtocol(parsedURL.Scheme)
+	resp, initialErr := client.Do(initialRequest)
+	if initialErr == nil {
+		resp.Body.Close()
+		return nil
+	}
 
-		secondRequest, err := http.NewRequest("GET", parsedURL.String(), nil)
-		if err != nil {
-			return err
-		}
-		secondRequest.Host = hostname
-		_, secondErr := client.Do(secondRequest)
-		if secondErr == nil { // Worked this time--advise the user to switch protocols
-			return errors.Errorf(
-				"%s doesn't seem to work over %s, but does seem to work over %s. Reason: %v. Consider changing the origin URL to %s",
-				parsedURL.Host,
-				oldScheme,
-				parsedURL.Scheme,
-				initialErr,
-				parsedURL,
-			)
-		}
+	// Attempt the same endpoint via the other protocol (http/https); maybe we have better luck?
+	oldScheme := parsedURL.Scheme
+	parsedURL.Scheme = toggleProtocol(oldScheme)
+
+	secondRequest, err := http.NewRequest("GET", parsedURL.String(), nil)
+	if err != nil {
+		return err
+	}
+	secondRequest.Host = hostname
+	resp, secondErr := client.Do(secondRequest)
+	if secondErr == nil { // Worked this time--advise the user to switch protocols
+		resp.Body.Close()
+		return errors.Errorf(
+			"%s doesn't seem to work over %s, but does seem to work over %s. Reason: %v. Consider changing the origin URL to %v",
+			parsedURL.Host,
+			oldScheme,
+			parsedURL.Scheme,
+			initialErr,
+			originURL,
+		)
 	}
 
 	return initialErr
@@ -220,12 +233,12 @@ type Access struct {
 }
 
 func NewAccessValidator(ctx context.Context, domain, issuer, applicationAUD string) (*Access, error) {
-	domainURL, err := ValidateUrl(domain)
+	domainURL, err := validateUrlString(domain)
 	if err != nil {
 		return nil, err
 	}
 
-	issuerURL, err := ValidateUrl(issuer)
+	issuerURL, err := validateUrlString(issuer)
 	if err != nil {
 		return nil, err
 	}

validation/validation_test.go

@@ -101,7 +101,7 @@ func TestValidateUrl(t *testing.T) {
 	for i, testCase := range testCases {
 		validUrl, err := ValidateUrl(testCase.input)
 		assert.NoError(t, err, "test case %v", i)
-		assert.Equal(t, testCase.expectedOutput, validUrl, "test case %v", i)
+		assert.Equal(t, testCase.expectedOutput, validUrl.String(), "test case %v", i)
 	}
 
 	validUrl, err := ValidateUrl("")
@@ -151,7 +151,7 @@ func TestValidateHTTPService_HTTP2HTTP(t *testing.T) {
 
 // Happy path 2: originURL is HTTPS, and HTTPS connections work
 func TestValidateHTTPService_HTTPS2HTTPS(t *testing.T) {
-	originURL := "https://127.0.0.1/"
+	originURL := "https://127.0.0.1:1234/"
 	hostname := "example.com"
 
 	assert.Nil(t, ValidateHTTPService(originURL, hostname, testRoundTripper(func(req *http.Request) (*http.Response, error) {

vendor/github.com/equinox-io/equinox/.travis.yml

@@ -1,13 +0,0 @@
-language: go
-go:
-  - "1.4"
-  - "1.5"
-  - "1.7"
-  - "1.9"
-  - "1.10"
-  - "1.11"
-  - "tip"
-matrix:
-  allow_failures:
-    - go: tip
-script: go test -v github.com/equinox-io/equinox github.com/equinox-io/equinox/proto

vendor/github.com/equinox-io/equinox/README.md

@@ -1,99 +0,0 @@
-# equinox client SDK [![godoc reference](https://godoc.org/github.com/equinox-io/equinox?status.png)](https://godoc.org/github.com/equinox-io/equinox)
-
-Package equinox allows applications to remotely update themselves with the [equinox.io](https://equinox.io) service.
-
-## Minimal Working Example
-
-```go
-import "github.com/equinox-io/equinox"
-
-const appID = "<YOUR EQUINOX APP ID>"
-
-var publicKey = []byte(`
------BEGIN PUBLIC KEY-----
-MFYwEAYHKoZIzj0CAQYFK4EEAAoDQgAEtrVmBxQvheRArXjg2vG1xIprWGuCyESx
-MMY8pjmjepSy2kuz+nl9aFLqmr+rDNdYvEBqQaZrYMc6k29gjvoQnQ==
------END PUBLIC KEY-----
-`)
-
-func update(channel string) error {
-    opts := equinox.Options{Channel: channel}
-    if err := opts.SetPublicKeyPEM(publicKey); err != nil {
-        return err
-    }
-
-    // check for the update
-    resp, err := equinox.Check(appID, opts)
-    switch {
-    case err == equinox.NotAvailableErr:
-        fmt.Println("No update available, already at the latest version!")
-        return nil
-    case err != nil:
-        return err
-    }
-
-    // fetch the update and apply it
-    err = resp.Apply()
-    if err != nil {
-        return err
-    }
-
-    fmt.Printf("Updated to new version: %s!\n", resp.ReleaseVersion)
-    return nil
-}
-```
-
-
-## Update To Specific Version
-
-When you specify a channel in the update options, equinox will try to update the application
-to the latest release of your application published to that channel. Instead, you may wish to
-update the application to a specific (possibly older) version. You can do this by explicitly setting
-Version in the Options struct:
-
-```go
-opts := equinox.Options{Version: "0.1.2"}
-```
-
-## Prompt For Update
-
-You may wish to ask the user for approval before updating to a new version. This is as simple
-as calling the Check function and only calling Apply on the returned result if the user approves.
-Example:
-
-```go
-// check for the update
-resp, err := equinox.Check(appID, opts)
-switch {
-case err == equinox.NotAvailableErr:
-    fmt.Println("No update available, already at the latest version!")
-    return nil
-case err != nil:
-    return err
-}
-
-fmt.Println("New version available!")
-fmt.Println("Version:", resp.ReleaseVersion)
-fmt.Println("Name:", resp.ReleaseTitle)
-fmt.Println("Details:", resp.ReleaseDescription)
-
-ok := prompt("Would you like to update?")
-
-if !ok {
-    return
-}
-
-err = resp.Apply()
-// ...
-```
-
-## Generating Keys
-
-All equinox releases must be signed with a private ECDSA key, and all updates verified with the
-public key portion. To do that, you'll need to generate a key pair. The equinox release tool can
-generate an ecdsa key pair for you easily:
-
-```shell
-equinox genkey
-```
-

vendor/github.com/equinox-io/equinox/doc.go

@@ -1,92 +0,0 @@
-/*
-Package equinox allows applications to remotely update themselves with the equinox.io service.
-
-Minimal Working Example
-
-	import "github.com/equinox-io/equinox"
-
-	const appID = "<YOUR EQUINOX APP ID>"
-
-	var publicKey = []byte(`
-	-----BEGIN PUBLIC KEY-----
-	MFYwEAYHKoZIzj0CAQYFK4EEAAoDQgAEtrVmBxQvheRArXjg2vG1xIprWGuCyESx
-	MMY8pjmjepSy2kuz+nl9aFLqmr+rDNdYvEBqQaZrYMc6k29gjvoQnQ==
-	-----END PUBLIC KEY-----
-	`)
-
-	func update(channel string) error {
-		opts := equinox.Options{Channel: channel}
-		if err := opts.SetPublicKeyPEM(publicKey); err != nil {
-			return err
-		}
-
-		// check for the update
-		resp, err := equinox.Check(appID, opts)
-		switch {
-		case err == equinox.NotAvailableErr:
-			fmt.Println("No update available, already at the latest version!")
-			return nil
-		case err != nil:
-			return err
-		}
-
-		// fetch the update and apply it
-		err = resp.Apply()
-		if err != nil {
-			return err
-		}
-
-		fmt.Printf("Updated to new version: %s!\n", resp.ReleaseVersion)
-		return nil
-	}
-
-
-Update To Specific Version
-
-When you specify a channel in the update options, equinox will try to update the application
-to the latest release of your application published to that channel. Instead, you may wish to
-update the application to a specific (possibly older) version. You can do this by explicitly setting
-Version in the Options struct:
-
-		opts := equinox.Options{Version: "0.1.2"}
-
-Prompt For Update
-
-You may wish to ask the user for approval before updating to a new version. This is as simple
-as calling the Check function and only calling Apply on the returned result if the user approves.
-Example:
-
-	// check for the update
-	resp, err := equinox.Check(appID, opts)
-	switch {
-	case err == equinox.NotAvailableErr:
-		fmt.Println("No update available, already at the latest version!")
-		return nil
-	case err != nil:
-		return err
-	}
-
-	fmt.Println("New version available!")
-	fmt.Println("Version:", resp.ReleaseVersion)
-	fmt.Println("Name:", resp.ReleaseTitle)
-	fmt.Println("Details:", resp.ReleaseDescription)
-
-	ok := prompt("Would you like to update?")
-
-	if !ok {
-		return
-	}
-
-	err = resp.Apply()
-	// ...
-
-Generating Keys
-
-All equinox releases must be signed with a private ECDSA key, and all updates verified with the
-public key portion. To do that, you'll need to generate a key pair. The equinox release tool can
-generate an ecdsa key pair for you easily:
-
-	equinox genkey
-
-*/
-package equinox

vendor/github.com/equinox-io/equinox/go.mod

@@ -1 +0,0 @@
-module github.com/equinox-io/equinox

vendor/github.com/equinox-io/equinox/internal/go-update/LICENSE

@@ -1,13 +0,0 @@
-Copyright 2015 Alan Shreve
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.

vendor/github.com/equinox-io/equinox/internal/go-update/README.md

@@ -1,65 +0,0 @@
-# go-update: Build self-updating Go programs [![godoc reference](https://godoc.org/github.com/inconshreveable/go-update?status.png)](https://godoc.org/github.com/inconshreveable/go-update)
-
-Package update provides functionality to implement secure, self-updating Go programs (or other single-file targets)
-A program can update itself by replacing its executable file with a new version.
-
-It provides the flexibility to implement different updating user experiences
-like auto-updating, or manual user-initiated updates. It also boasts
-advanced features like binary patching and code signing verification.
-
-Example of updating from a URL:
-
-```go
-import (
-    "fmt"
-    "net/http"
-
-    "github.com/inconshreveable/go-update"
-)
-
-func doUpdate(url string) error {
-    resp, err := http.Get(url)
-    if err != nil {
-        return err
-    }
-    defer resp.Body.Close()
-    err := update.Apply(resp.Body, update.Options{})
-    if err != nil {
-        // error handling
-    }
-    return err
-}
-```
-
-## Features
-
-- Cross platform support (Windows too!)
-- Binary patch application
-- Checksum verification
-- Code signing verification
-- Support for updating arbitrary files
-
-## [equinox.io](https://equinox.io)
-[equinox.io](https://equinox.io) is a complete ready-to-go updating solution built on top of go-update that provides:
-
-- Hosted updates
-- Update channels (stable, beta, nightly, ...)
-- Dynamically computed binary diffs
-- Automatic key generation and code
-- Release tooling with proper code signing
-- Update/download metrics
-
-## API Compatibility Promises
-The master branch of `go-update` is *not* guaranteed to have a stable API over time. For any production application, you should vendor
-your dependency on `go-update` with a tool like git submodules, [gb](http://getgb.io/) or [govendor](https://github.com/kardianos/govendor).
-
-The `go-update` package makes the following promises about API compatibility:
-1. A list of all API-breaking changes will be documented in this README.
-1. `go-update` will strive for as few API-breaking changes as possible.
-
-## API Breaking Changes
-- **Sept 3, 2015**: The `Options` struct passed to `Apply` was changed to be passed by value instead of passed by pointer. Old API at `28de026`.
-- **Aug 9, 2015**: 2.0 API. Old API at `221d034` or `gopkg.in/inconshreveable/go-update.v0`.
-
-## License
-Apache

vendor/github.com/equinox-io/equinox/internal/go-update/apply.go

@@ -1,322 +0,0 @@
-package update
-
-import (
-	"bytes"
-	"crypto"
-	"crypto/x509"
-	"encoding/pem"
-	"errors"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"os"
-	"path/filepath"
-
-	"github.com/equinox-io/equinox/internal/go-update/internal/osext"
-)
-
-var (
-	openFile = os.OpenFile
-)
-
-// Apply performs an update of the current executable (or opts.TargetFile, if set) with the contents of the given io.Reader.
-//
-// Apply performs the following actions to ensure a safe cross-platform update:
-//
-// 1. If configured, applies the contents of the update io.Reader as a binary patch.
-//
-// 2. If configured, computes the checksum of the new executable and verifies it matches.
-//
-// 3. If configured, verifies the signature with a public key.
-//
-// 4. Creates a new file, /path/to/.target.new with the TargetMode with the contents of the updated file
-//
-// 5. Renames /path/to/target to /path/to/.target.old
-//
-// 6. Renames /path/to/.target.new to /path/to/target
-//
-// 7. If the final rename is successful, deletes /path/to/.target.old, returns no error. On Windows,
-// the removal of /path/to/target.old always fails, so instead Apply hides the old file instead.
-//
-// 8. If the final rename fails, attempts to roll back by renaming /path/to/.target.old
-// back to /path/to/target.
-//
-// If the roll back operation fails, the file system is left in an inconsistent state (betweet steps 5 and 6) where
-// there is no new executable file and the old executable file could not be be moved to its original location. In this
-// case you should notify the user of the bad news and ask them to recover manually. Applications can determine whether
-// the rollback failed by calling RollbackError, see the documentation on that function for additional detail.
-func Apply(update io.Reader, opts Options) error {
-	// validate
-	verify := false
-	switch {
-	case opts.Signature != nil && opts.PublicKey != nil:
-		// okay
-		verify = true
-	case opts.Signature != nil:
-		return errors.New("no public key to verify signature with")
-	case opts.PublicKey != nil:
-		return errors.New("No signature to verify with")
-	}
-
-	// set defaults
-	if opts.Hash == 0 {
-		opts.Hash = crypto.SHA256
-	}
-	if opts.Verifier == nil {
-		opts.Verifier = NewECDSAVerifier()
-	}
-	if opts.TargetMode == 0 {
-		opts.TargetMode = 0755
-	}
-
-	// get target path
-	var err error
-	opts.TargetPath, err = opts.getPath()
-	if err != nil {
-		return err
-	}
-
-	var newBytes []byte
-	if opts.Patcher != nil {
-		if newBytes, err = opts.applyPatch(update); err != nil {
-			return err
-		}
-	} else {
-		// no patch to apply, go on through
-		if newBytes, err = ioutil.ReadAll(update); err != nil {
-			return err
-		}
-	}
-
-	// verify checksum if requested
-	if opts.Checksum != nil {
-		if err = opts.verifyChecksum(newBytes); err != nil {
-			return err
-		}
-	}
-
-	if verify {
-		if err = opts.verifySignature(newBytes); err != nil {
-			return err
-		}
-	}
-
-	// get the directory the executable exists in
-	updateDir := filepath.Dir(opts.TargetPath)
-	filename := filepath.Base(opts.TargetPath)
-
-	// Copy the contents of newbinary to a new executable file
-	newPath := filepath.Join(updateDir, fmt.Sprintf(".%s.new", filename))
-	fp, err := openFile(newPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, opts.TargetMode)
-	if err != nil {
-		return err
-	}
-	defer fp.Close()
-
-	_, err = io.Copy(fp, bytes.NewReader(newBytes))
-	if err != nil {
-		return err
-	}
-
-	// if we don't call fp.Close(), windows won't let us move the new executable
-	// because the file will still be "in use"
-	fp.Close()
-
-	// this is where we'll move the executable to so that we can swap in the updated replacement
-	oldPath := opts.OldSavePath
-	removeOld := opts.OldSavePath == ""
-	if removeOld {
-		oldPath = filepath.Join(updateDir, fmt.Sprintf(".%s.old", filename))
-	}
-
-	// delete any existing old exec file - this is necessary on Windows for two reasons:
-	// 1. after a successful update, Windows can't remove the .old file because the process is still running
-	// 2. windows rename operations fail if the destination file already exists
-	_ = os.Remove(oldPath)
-
-	// move the existing executable to a new file in the same directory
-	err = os.Rename(opts.TargetPath, oldPath)
-	if err != nil {
-		return err
-	}
-
-	// move the new exectuable in to become the new program
-	err = os.Rename(newPath, opts.TargetPath)
-
-	if err != nil {
-		// move unsuccessful
-		//
-		// The filesystem is now in a bad state. We have successfully
-		// moved the existing binary to a new location, but we couldn't move the new
-		// binary to take its place. That means there is no file where the current executable binary
-		// used to be!
-		// Try to rollback by restoring the old binary to its original path.
-		rerr := os.Rename(oldPath, opts.TargetPath)
-		if rerr != nil {
-			return &rollbackErr{err, rerr}
-		}
-
-		return err
-	}
-
-	// move successful, remove the old binary if needed
-	if removeOld {
-		errRemove := os.Remove(oldPath)
-
-		// windows has trouble with removing old binaries, so hide it instead
-		if errRemove != nil {
-			_ = hideFile(oldPath)
-		}
-	}
-
-	return nil
-}
-
-// RollbackError takes an error value returned by Apply and returns the error, if any,
-// that occurred when attempting to roll back from a failed update. Applications should
-// always call this function on any non-nil errors returned by Apply.
-//
-// If no rollback was needed or if the rollback was successful, RollbackError returns nil,
-// otherwise it returns the error encountered when trying to roll back.
-func RollbackError(err error) error {
-	if err == nil {
-		return nil
-	}
-	if rerr, ok := err.(*rollbackErr); ok {
-		return rerr.rollbackErr
-	}
-	return nil
-}
-
-type rollbackErr struct {
-	error             // original error
-	rollbackErr error // error encountered while rolling back
-}
-
-type Options struct {
-	// TargetPath defines the path to the file to update.
-	// The emptry string means 'the executable file of the running program'.
-	TargetPath string
-
-	// Create TargetPath replacement with this file mode. If zero, defaults to 0755.
-	TargetMode os.FileMode
-
-	// Checksum of the new binary to verify against. If nil, no checksum or signature verification is done.
-	Checksum []byte
-
-	// Public key to use for signature verification. If nil, no signature verification is done.
-	PublicKey crypto.PublicKey
-
-	// Signature to verify the updated file. If nil, no signature verification is done.
-	Signature []byte
-
-	// Pluggable signature verification algorithm. If nil, ECDSA is used.
-	Verifier Verifier
-
-	// Use this hash function to generate the checksum. If not set, SHA256 is used.
-	Hash crypto.Hash
-
-	// If nil, treat the update as a complete replacement for the contents of the file at TargetPath.
-	// If non-nil, treat the update contents as a patch and use this object to apply the patch.
-	Patcher Patcher
-
-	// Store the old executable file at this path after a successful update.
-	// The empty string means the old executable file will be removed after the update.
-	OldSavePath string
-}
-
-// CheckPermissions determines whether the process has the correct permissions to
-// perform the requested update. If the update can proceed, it returns nil, otherwise
-// it returns the error that would occur if an update were attempted.
-func (o *Options) CheckPermissions() error {
-	// get the directory the file exists in
-	path, err := o.getPath()
-	if err != nil {
-		return err
-	}
-
-	fileDir := filepath.Dir(path)
-	fileName := filepath.Base(path)
-
-	// attempt to open a file in the file's directory
-	newPath := filepath.Join(fileDir, fmt.Sprintf(".%s.new", fileName))
-	fp, err := openFile(newPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, o.TargetMode)
-	if err != nil {
-		return err
-	}
-	fp.Close()
-
-	_ = os.Remove(newPath)
-	return nil
-}
-
-// SetPublicKeyPEM is a convenience method to set the PublicKey property
-// used for checking a completed update's signature by parsing a
-// Public Key formatted as PEM data.
-func (o *Options) SetPublicKeyPEM(pembytes []byte) error {
-	block, _ := pem.Decode(pembytes)
-	if block == nil {
-		return errors.New("couldn't parse PEM data")
-	}
-
-	pub, err := x509.ParsePKIXPublicKey(block.Bytes)
-	if err != nil {
-		return err
-	}
-	o.PublicKey = pub
-	return nil
-}
-
-func (o *Options) getPath() (string, error) {
-	if o.TargetPath == "" {
-		return osext.Executable()
-	} else {
-		return o.TargetPath, nil
-	}
-}
-
-func (o *Options) applyPatch(patch io.Reader) ([]byte, error) {
-	// open the file to patch
-	old, err := os.Open(o.TargetPath)
-	if err != nil {
-		return nil, err
-	}
-	defer old.Close()
-
-	// apply the patch
-	var applied bytes.Buffer
-	if err = o.Patcher.Patch(old, &applied, patch); err != nil {
-		return nil, err
-	}
-
-	return applied.Bytes(), nil
-}
-
-func (o *Options) verifyChecksum(updated []byte) error {
-	checksum, err := checksumFor(o.Hash, updated)
-	if err != nil {
-		return err
-	}
-
-	if !bytes.Equal(o.Checksum, checksum) {
-		return fmt.Errorf("Updated file has wrong checksum. Expected: %x, got: %x", o.Checksum, checksum)
-	}
-	return nil
-}
-
-func (o *Options) verifySignature(updated []byte) error {
-	checksum, err := checksumFor(o.Hash, updated)
-	if err != nil {
-		return err
-	}
-	return o.Verifier.VerifySignature(checksum, o.Signature, o.Hash, o.PublicKey)
-}
-
-func checksumFor(h crypto.Hash, payload []byte) ([]byte, error) {
-	if !h.Available() {
-		return nil, errors.New("requested hash function not available")
-	}
-	hash := h.New()
-	hash.Write(payload) // guaranteed not to error
-	return hash.Sum([]byte{}), nil
-}

vendor/github.com/equinox-io/equinox/internal/go-update/doc.go

@@ -1,172 +0,0 @@
-/*
-Package update provides functionality to implement secure, self-updating Go programs (or other single-file targets).
-
-For complete updating solutions please see Equinox (https://equinox.io) and go-tuf (https://github.com/flynn/go-tuf).
-
-Basic Example
-
-This example shows how to update a program remotely from a URL.
-
-	import (
-		"fmt"
-		"net/http"
-
-		"github.com/inconshreveable/go-update"
-	)
-
-	func doUpdate(url string) error {
-		// request the new file
-		resp, err := http.Get(url)
-		if err != nil {
-			return err
-		}
-		defer resp.Body.Close()
-		err := update.Apply(resp.Body, update.Options{})
-		if err != nil {
-			if rerr := update.RollbackError(err); rerr != nil {
-				fmt.Println("Failed to rollback from bad update: %v", rerr)
-			}
-		}
-		return err
-	}
-
-
-Binary Patching
-
-Go binaries can often be large. It can be advantageous to only ship a binary patch to a client
-instead of the complete program text of a new version.
-
-This example shows how to update a program with a bsdiff binary patch. Other patch formats
-may be applied by implementing the Patcher interface.
-
-	import (
-		"encoding/hex"
-		"io"
-
-		"github.com/inconshreveable/go-update"
-	)
-
-	func updateWithPatch(patch io.Reader) error {
-		err := update.Apply(patch, update.Options{
-			Patcher: update.NewBSDiffPatcher()
-		})
-		if err != nil {
-			// error handling
-		}
-		return err
-	}
-
-Checksum Verification
-
-Updating executable code on a computer can be a dangerous operation unless you
-take the appropriate steps to guarantee the authenticity of the new code. While
-checksum verification is important, it should always be combined with signature
-verification (next section) to guarantee that the code came from a trusted party.
-
-go-update validates SHA256 checksums by default, but this is pluggable via the Hash
-property on the Options struct.
-
-This example shows how to guarantee that the newly-updated binary is verified to
-have an appropriate checksum (that was otherwise retrived via a secure channel)
-specified as a hex string.
-
-	import (
-		"crypto"
-		_ "crypto/sha256"
-		"encoding/hex"
-		"io"
-
-		"github.com/inconshreveable/go-update"
-	)
-
-	func updateWithChecksum(binary io.Reader, hexChecksum string) error {
-		checksum, err := hex.DecodeString(hexChecksum)
-		if err != nil {
-			return err
-		}
-		err = update.Apply(binary, update.Options{
-			Hash: crypto.SHA256, 	// this is the default, you don't need to specify it
-			Checksum: checksum,
-		})
-		if err != nil {
-			// error handling
-		}
-		return err
-	}
-
-Cryptographic Signature Verification
-
-Cryptographic verification of new code from an update is an extremely important way to guarantee the
-security and integrity of your updates.
-
-Verification is performed by validating the signature of a hash of the new file. This
-means nothing changes if you apply your update with a patch.
-
-This example shows how to add signature verification to your updates. To make all of this work
-an application distributor must first create a public/private key pair and embed the public key
-into their application. When they issue a new release, the issuer must sign the new executable file
-with the private key and distribute the signature along with the update.
-
-	import (
-		"crypto"
-		_ "crypto/sha256"
-		"encoding/hex"
-		"io"
-
-		"github.com/inconshreveable/go-update"
-	)
-
-	var publicKey = []byte(`
-	-----BEGIN PUBLIC KEY-----
-	MFYwEAYHKoZIzj0CAQYFK4EEAAoDQgAEtrVmBxQvheRArXjg2vG1xIprWGuCyESx
-	MMY8pjmjepSy2kuz+nl9aFLqmr+rDNdYvEBqQaZrYMc6k29gjvoQnQ==
-	-----END PUBLIC KEY-----
-	`)
-
-	func verifiedUpdate(binary io.Reader, hexChecksum, hexSignature string) {
-		checksum, err := hex.DecodeString(hexChecksum)
-		if err != nil {
-			return err
-		}
-		signature, err := hex.DecodeString(hexSignature)
-		if err != nil {
-			return err
-		}
-		opts := update.Options{
-			Checksum: checksum,
-			Signature: signature,
-			Hash: crypto.SHA256, 	                 // this is the default, you don't need to specify it
-			Verifier: update.NewECDSAVerifier(),   // this is the default, you don't need to specify it
-		}
-		err = opts.SetPublicKeyPEM(publicKey)
-		if err != nil {
-			return err
-		}
-		err = update.Apply(binary, opts)
-		if err != nil {
-			// error handling
-		}
-		return err
-	}
-
-
-Building Single-File Go Binaries
-
-In order to update a Go application with go-update, you must distributed it as a single executable.
-This is often easy, but some applications require static assets (like HTML and CSS asset files or TLS certificates).
-In order to update applications like these, you'll want to make sure to embed those asset files into
-the distributed binary with a tool like go-bindata (my favorite): https://github.com/jteeuwen/go-bindata
-
-Non-Goals
-
-Mechanisms and protocols for determining whether an update should be applied and, if so, which one are
-out of scope for this package. Please consult go-tuf (https://github.com/flynn/go-tuf) or Equinox (https://equinox.io)
-for more complete solutions.
-
-go-update only works for self-updating applications that are distributed as a single binary, i.e.
-applications that do not have additional assets or dependency files.
-Updating application that are distributed as mutliple on-disk files is out of scope, although this
-may change in future versions of this library.
-
-*/
-package update

vendor/github.com/equinox-io/equinox/internal/go-update/hide_noop.go

@@ -1,7 +0,0 @@
-// +build !windows
-
-package update
-
-func hideFile(path string) error {
-	return nil
-}

vendor/github.com/equinox-io/equinox/internal/go-update/hide_windows.go

@@ -1,19 +0,0 @@
-package update
-
-import (
-	"syscall"
-	"unsafe"
-)
-
-func hideFile(path string) error {
-	kernel32 := syscall.NewLazyDLL("kernel32.dll")
-	setFileAttributes := kernel32.NewProc("SetFileAttributesW")
-
-	r1, _, err := setFileAttributes.Call(uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(path))), 2)
-
-	if r1 == 0 {
-		return err
-	} else {
-		return nil
-	}
-}

vendor/github.com/equinox-io/equinox/internal/go-update/internal/binarydist/License

@@ -1,22 +0,0 @@
-Copyright 2012 Keith Rarick
-
-Permission is hereby granted, free of charge, to any person
-obtaining a copy of this software and associated documentation
-files (the "Software"), to deal in the Software without
-restriction, including without limitation the rights to use,
-copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the
-Software is furnished to do so, subject to the following
-conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
-OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
-HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
-WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
-OTHER DEALINGS IN THE SOFTWARE.

vendor/github.com/equinox-io/equinox/internal/go-update/internal/binarydist/Readme.md

@@ -1,7 +0,0 @@
-# binarydist
-
-Package binarydist implements binary diff and patch as described on
-<http://www.daemonology.net/bsdiff/>. It reads and writes files
-compatible with the tools there.
-
-Documentation at <http://go.pkgdoc.org/github.com/kr/binarydist>.

vendor/github.com/equinox-io/equinox/internal/go-update/internal/binarydist/bzip2.go

@@ -1,40 +0,0 @@
-package binarydist
-
-import (
-	"io"
-	"os/exec"
-)
-
-type bzip2Writer struct {
-	c *exec.Cmd
-	w io.WriteCloser
-}
-
-func (w bzip2Writer) Write(b []byte) (int, error) {
-	return w.w.Write(b)
-}
-
-func (w bzip2Writer) Close() error {
-	if err := w.w.Close(); err != nil {
-		return err
-	}
-	return w.c.Wait()
-}
-
-// Package compress/bzip2 implements only decompression,
-// so we'll fake it by running bzip2 in another process.
-func newBzip2Writer(w io.Writer) (wc io.WriteCloser, err error) {
-	var bw bzip2Writer
-	bw.c = exec.Command("bzip2", "-c")
-	bw.c.Stdout = w
-
-	if bw.w, err = bw.c.StdinPipe(); err != nil {
-		return nil, err
-	}
-
-	if err = bw.c.Start(); err != nil {
-		return nil, err
-	}
-
-	return bw, nil
-}

vendor/github.com/equinox-io/equinox/internal/go-update/internal/binarydist/diff.go

@@ -1,408 +0,0 @@
-package binarydist
-
-import (
-	"bytes"
-	"encoding/binary"
-	"io"
-	"io/ioutil"
-)
-
-func swap(a []int, i, j int) { a[i], a[j] = a[j], a[i] }
-
-func split(I, V []int, start, length, h int) {
-	var i, j, k, x, jj, kk int
-
-	if length < 16 {
-		for k = start; k < start+length; k += j {
-			j = 1
-			x = V[I[k]+h]
-			for i = 1; k+i < start+length; i++ {
-				if V[I[k+i]+h] < x {
-					x = V[I[k+i]+h]
-					j = 0
-				}
-				if V[I[k+i]+h] == x {
-					swap(I, k+i, k+j)
-					j++
-				}
-			}
-			for i = 0; i < j; i++ {
-				V[I[k+i]] = k + j - 1
-			}
-			if j == 1 {
-				I[k] = -1
-			}
-		}
-		return
-	}
-
-	x = V[I[start+length/2]+h]
-	jj = 0
-	kk = 0
-	for i = start; i < start+length; i++ {
-		if V[I[i]+h] < x {
-			jj++
-		}
-		if V[I[i]+h] == x {
-			kk++
-		}
-	}
-	jj += start
-	kk += jj
-
-	i = start
-	j = 0
-	k = 0
-	for i < jj {
-		if V[I[i]+h] < x {
-			i++
-		} else if V[I[i]+h] == x {
-			swap(I, i, jj+j)
-			j++
-		} else {
-			swap(I, i, kk+k)
-			k++
-		}
-	}
-
-	for jj+j < kk {
-		if V[I[jj+j]+h] == x {
-			j++
-		} else {
-			swap(I, jj+j, kk+k)
-			k++
-		}
-	}
-
-	if jj > start {
-		split(I, V, start, jj-start, h)
-	}
-
-	for i = 0; i < kk-jj; i++ {
-		V[I[jj+i]] = kk - 1
-	}
-	if jj == kk-1 {
-		I[jj] = -1
-	}
-
-	if start+length > kk {
-		split(I, V, kk, start+length-kk, h)
-	}
-}
-
-func qsufsort(obuf []byte) []int {
-	var buckets [256]int
-	var i, h int
-	I := make([]int, len(obuf)+1)
-	V := make([]int, len(obuf)+1)
-
-	for _, c := range obuf {
-		buckets[c]++
-	}
-	for i = 1; i < 256; i++ {
-		buckets[i] += buckets[i-1]
-	}
-	copy(buckets[1:], buckets[:])
-	buckets[0] = 0
-
-	for i, c := range obuf {
-		buckets[c]++
-		I[buckets[c]] = i
-	}
-
-	I[0] = len(obuf)
-	for i, c := range obuf {
-		V[i] = buckets[c]
-	}
-
-	V[len(obuf)] = 0
-	for i = 1; i < 256; i++ {
-		if buckets[i] == buckets[i-1]+1 {
-			I[buckets[i]] = -1
-		}
-	}
-	I[0] = -1
-
-	for h = 1; I[0] != -(len(obuf) + 1); h += h {
-		var n int
-		for i = 0; i < len(obuf)+1; {
-			if I[i] < 0 {
-				n -= I[i]
-				i -= I[i]
-			} else {
-				if n != 0 {
-					I[i-n] = -n
-				}
-				n = V[I[i]] + 1 - i
-				split(I, V, i, n, h)
-				i += n
-				n = 0
-			}
-		}
-		if n != 0 {
-			I[i-n] = -n
-		}
-	}
-
-	for i = 0; i < len(obuf)+1; i++ {
-		I[V[i]] = i
-	}
-	return I
-}
-
-func matchlen(a, b []byte) (i int) {
-	for i < len(a) && i < len(b) && a[i] == b[i] {
-		i++
-	}
-	return i
-}
-
-func search(I []int, obuf, nbuf []byte, st, en int) (pos, n int) {
-	if en-st < 2 {
-		x := matchlen(obuf[I[st]:], nbuf)
-		y := matchlen(obuf[I[en]:], nbuf)
-
-		if x > y {
-			return I[st], x
-		} else {
-			return I[en], y
-		}
-	}
-
-	x := st + (en-st)/2
-	if bytes.Compare(obuf[I[x]:], nbuf) < 0 {
-		return search(I, obuf, nbuf, x, en)
-	} else {
-		return search(I, obuf, nbuf, st, x)
-	}
-	panic("unreached")
-}
-
-// Diff computes the difference between old and new, according to the bsdiff
-// algorithm, and writes the result to patch.
-func Diff(old, new io.Reader, patch io.Writer) error {
-	obuf, err := ioutil.ReadAll(old)
-	if err != nil {
-		return err
-	}
-
-	nbuf, err := ioutil.ReadAll(new)
-	if err != nil {
-		return err
-	}
-
-	pbuf, err := diffBytes(obuf, nbuf)
-	if err != nil {
-		return err
-	}
-
-	_, err = patch.Write(pbuf)
-	return err
-}
-
-func diffBytes(obuf, nbuf []byte) ([]byte, error) {
-	var patch seekBuffer
-	err := diff(obuf, nbuf, &patch)
-	if err != nil {
-		return nil, err
-	}
-	return patch.buf, nil
-}
-
-func diff(obuf, nbuf []byte, patch io.WriteSeeker) error {
-	var lenf int
-	I := qsufsort(obuf)
-	db := make([]byte, len(nbuf))
-	eb := make([]byte, len(nbuf))
-	var dblen, eblen int
-
-	var hdr header
-	hdr.Magic = magic
-	hdr.NewSize = int64(len(nbuf))
-	err := binary.Write(patch, signMagLittleEndian{}, &hdr)
-	if err != nil {
-		return err
-	}
-
-	// Compute the differences, writing ctrl as we go
-	pfbz2, err := newBzip2Writer(patch)
-	if err != nil {
-		return err
-	}
-	var scan, pos, length int
-	var lastscan, lastpos, lastoffset int
-	for scan < len(nbuf) {
-		var oldscore int
-		scan += length
-		for scsc := scan; scan < len(nbuf); scan++ {
-			pos, length = search(I, obuf, nbuf[scan:], 0, len(obuf))
-
-			for ; scsc < scan+length; scsc++ {
-				if scsc+lastoffset < len(obuf) &&
-					obuf[scsc+lastoffset] == nbuf[scsc] {
-					oldscore++
-				}
-			}
-
-			if (length == oldscore && length != 0) || length > oldscore+8 {
-				break
-			}
-
-			if scan+lastoffset < len(obuf) && obuf[scan+lastoffset] == nbuf[scan] {
-				oldscore--
-			}
-		}
-
-		if length != oldscore || scan == len(nbuf) {
-			var s, Sf int
-			lenf = 0
-			for i := 0; lastscan+i < scan && lastpos+i < len(obuf); {
-				if obuf[lastpos+i] == nbuf[lastscan+i] {
-					s++
-				}
-				i++
-				if s*2-i > Sf*2-lenf {
-					Sf = s
-					lenf = i
-				}
-			}
-
-			lenb := 0
-			if scan < len(nbuf) {
-				var s, Sb int
-				for i := 1; (scan >= lastscan+i) && (pos >= i); i++ {
-					if obuf[pos-i] == nbuf[scan-i] {
-						s++
-					}
-					if s*2-i > Sb*2-lenb {
-						Sb = s
-						lenb = i
-					}
-				}
-			}
-
-			if lastscan+lenf > scan-lenb {
-				overlap := (lastscan + lenf) - (scan - lenb)
-				s := 0
-				Ss := 0
-				lens := 0
-				for i := 0; i < overlap; i++ {
-					if nbuf[lastscan+lenf-overlap+i] == obuf[lastpos+lenf-overlap+i] {
-						s++
-					}
-					if nbuf[scan-lenb+i] == obuf[pos-lenb+i] {
-						s--
-					}
-					if s > Ss {
-						Ss = s
-						lens = i + 1
-					}
-				}
-
-				lenf += lens - overlap
-				lenb -= lens
-			}
-
-			for i := 0; i < lenf; i++ {
-				db[dblen+i] = nbuf[lastscan+i] - obuf[lastpos+i]
-			}
-			for i := 0; i < (scan-lenb)-(lastscan+lenf); i++ {
-				eb[eblen+i] = nbuf[lastscan+lenf+i]
-			}
-
-			dblen += lenf
-			eblen += (scan - lenb) - (lastscan + lenf)
-
-			err = binary.Write(pfbz2, signMagLittleEndian{}, int64(lenf))
-			if err != nil {
-				pfbz2.Close()
-				return err
-			}
-
-			val := (scan - lenb) - (lastscan + lenf)
-			err = binary.Write(pfbz2, signMagLittleEndian{}, int64(val))
-			if err != nil {
-				pfbz2.Close()
-				return err
-			}
-
-			val = (pos - lenb) - (lastpos + lenf)
-			err = binary.Write(pfbz2, signMagLittleEndian{}, int64(val))
-			if err != nil {
-				pfbz2.Close()
-				return err
-			}
-
-			lastscan = scan - lenb
-			lastpos = pos - lenb
-			lastoffset = pos - scan
-		}
-	}
-	err = pfbz2.Close()
-	if err != nil {
-		return err
-	}
-
-	// Compute size of compressed ctrl data
-	l64, err := patch.Seek(0, 1)
-	if err != nil {
-		return err
-	}
-	hdr.CtrlLen = int64(l64 - 32)
-
-	// Write compressed diff data
-	pfbz2, err = newBzip2Writer(patch)
-	if err != nil {
-		return err
-	}
-	n, err := pfbz2.Write(db[:dblen])
-	if err != nil {
-		pfbz2.Close()
-		return err
-	}
-	if n != dblen {
-		pfbz2.Close()
-		return io.ErrShortWrite
-	}
-	err = pfbz2.Close()
-	if err != nil {
-		return err
-	}
-
-	// Compute size of compressed diff data
-	n64, err := patch.Seek(0, 1)
-	if err != nil {
-		return err
-	}
-	hdr.DiffLen = n64 - l64
-
-	// Write compressed extra data
-	pfbz2, err = newBzip2Writer(patch)
-	if err != nil {
-		return err
-	}
-	n, err = pfbz2.Write(eb[:eblen])
-	if err != nil {
-		pfbz2.Close()
-		return err
-	}
-	if n != eblen {
-		pfbz2.Close()
-		return io.ErrShortWrite
-	}
-	err = pfbz2.Close()
-	if err != nil {
-		return err
-	}
-
-	// Seek to the beginning, write the header, and close the file
-	_, err = patch.Seek(0, 0)
-	if err != nil {
-		return err
-	}
-	err = binary.Write(patch, signMagLittleEndian{}, &hdr)
-	if err != nil {
-		return err
-	}
-	return nil
-}

vendor/github.com/equinox-io/equinox/internal/go-update/internal/binarydist/doc.go

@@ -1,24 +0,0 @@
-// Package binarydist implements binary diff and patch as described on
-// http://www.daemonology.net/bsdiff/. It reads and writes files
-// compatible with the tools there.
-package binarydist
-
-var magic = [8]byte{'B', 'S', 'D', 'I', 'F', 'F', '4', '0'}
-
-// File format:
-//   0       8    "BSDIFF40"
-//   8       8    X
-//   16      8    Y
-//   24      8    sizeof(newfile)
-//   32      X    bzip2(control block)
-//   32+X    Y    bzip2(diff block)
-//   32+X+Y  ???  bzip2(extra block)
-// with control block a set of triples (x,y,z) meaning "add x bytes
-// from oldfile to x bytes from the diff block; copy y bytes from the
-// extra block; seek forwards in oldfile by z bytes".
-type header struct {
-	Magic   [8]byte
-	CtrlLen int64
-	DiffLen int64
-	NewSize int64
-}

vendor/github.com/equinox-io/equinox/internal/go-update/internal/binarydist/encoding.go

@@ -1,53 +0,0 @@
-package binarydist
-
-// SignMagLittleEndian is the numeric encoding used by the bsdiff tools.
-// It implements binary.ByteOrder using a sign-magnitude format
-// and little-endian byte order. Only methods Uint64 and String
-// have been written; the rest panic.
-type signMagLittleEndian struct{}
-
-func (signMagLittleEndian) Uint16(b []byte) uint16 { panic("unimplemented") }
-
-func (signMagLittleEndian) PutUint16(b []byte, v uint16) { panic("unimplemented") }
-
-func (signMagLittleEndian) Uint32(b []byte) uint32 { panic("unimplemented") }
-
-func (signMagLittleEndian) PutUint32(b []byte, v uint32) { panic("unimplemented") }
-
-func (signMagLittleEndian) Uint64(b []byte) uint64 {
-	y := int64(b[0]) |
-		int64(b[1])<<8 |
-		int64(b[2])<<16 |
-		int64(b[3])<<24 |
-		int64(b[4])<<32 |
-		int64(b[5])<<40 |
-		int64(b[6])<<48 |
-		int64(b[7]&0x7f)<<56
-
-	if b[7]&0x80 != 0 {
-		y = -y
-	}
-	return uint64(y)
-}
-
-func (signMagLittleEndian) PutUint64(b []byte, v uint64) {
-	x := int64(v)
-	neg := x < 0
-	if neg {
-		x = -x
-	}
-
-	b[0] = byte(x)
-	b[1] = byte(x >> 8)
-	b[2] = byte(x >> 16)
-	b[3] = byte(x >> 24)
-	b[4] = byte(x >> 32)
-	b[5] = byte(x >> 40)
-	b[6] = byte(x >> 48)
-	b[7] = byte(x >> 56)
-	if neg {
-		b[7] |= 0x80
-	}
-}
-
-func (signMagLittleEndian) String() string { return "signMagLittleEndian" }

vendor/github.com/equinox-io/equinox/internal/go-update/internal/binarydist/patch.go

@@ -1,109 +0,0 @@
-package binarydist
-
-import (
-	"bytes"
-	"compress/bzip2"
-	"encoding/binary"
-	"errors"
-	"io"
-	"io/ioutil"
-)
-
-var ErrCorrupt = errors.New("corrupt patch")
-
-// Patch applies patch to old, according to the bspatch algorithm,
-// and writes the result to new.
-func Patch(old io.Reader, new io.Writer, patch io.Reader) error {
-	var hdr header
-	err := binary.Read(patch, signMagLittleEndian{}, &hdr)
-	if err != nil {
-		return err
-	}
-	if hdr.Magic != magic {
-		return ErrCorrupt
-	}
-	if hdr.CtrlLen < 0 || hdr.DiffLen < 0 || hdr.NewSize < 0 {
-		return ErrCorrupt
-	}
-
-	ctrlbuf := make([]byte, hdr.CtrlLen)
-	_, err = io.ReadFull(patch, ctrlbuf)
-	if err != nil {
-		return err
-	}
-	cpfbz2 := bzip2.NewReader(bytes.NewReader(ctrlbuf))
-
-	diffbuf := make([]byte, hdr.DiffLen)
-	_, err = io.ReadFull(patch, diffbuf)
-	if err != nil {
-		return err
-	}
-	dpfbz2 := bzip2.NewReader(bytes.NewReader(diffbuf))
-
-	// The entire rest of the file is the extra block.
-	epfbz2 := bzip2.NewReader(patch)
-
-	obuf, err := ioutil.ReadAll(old)
-	if err != nil {
-		return err
-	}
-
-	nbuf := make([]byte, hdr.NewSize)
-
-	var oldpos, newpos int64
-	for newpos < hdr.NewSize {
-		var ctrl struct{ Add, Copy, Seek int64 }
-		err = binary.Read(cpfbz2, signMagLittleEndian{}, &ctrl)
-		if err != nil {
-			return err
-		}
-
-		// Sanity-check
-		if newpos+ctrl.Add > hdr.NewSize {
-			return ErrCorrupt
-		}
-
-		// Read diff string
-		_, err = io.ReadFull(dpfbz2, nbuf[newpos:newpos+ctrl.Add])
-		if err != nil {
-			return ErrCorrupt
-		}
-
-		// Add old data to diff string
-		for i := int64(0); i < ctrl.Add; i++ {
-			if oldpos+i >= 0 && oldpos+i < int64(len(obuf)) {
-				nbuf[newpos+i] += obuf[oldpos+i]
-			}
-		}
-
-		// Adjust pointers
-		newpos += ctrl.Add
-		oldpos += ctrl.Add
-
-		// Sanity-check
-		if newpos+ctrl.Copy > hdr.NewSize {
-			return ErrCorrupt
-		}
-
-		// Read extra string
-		_, err = io.ReadFull(epfbz2, nbuf[newpos:newpos+ctrl.Copy])
-		if err != nil {
-			return ErrCorrupt
-		}
-
-		// Adjust pointers
-		newpos += ctrl.Copy
-		oldpos += ctrl.Seek
-	}
-
-	// Write the new file
-	for len(nbuf) > 0 {
-		n, err := new.Write(nbuf)
-		if err != nil {
-			return err
-		}
-		nbuf = nbuf[n:]
-	}
-
-	return nil
-}

vendor/github.com/equinox-io/equinox/internal/go-update/internal/binarydist/seek.go

@@ -1,43 +0,0 @@
-package binarydist
-
-import (
-	"errors"
-)
-
-type seekBuffer struct {
-	buf []byte
-	pos int
-}
-
-func (b *seekBuffer) Write(p []byte) (n int, err error) {
-	n = copy(b.buf[b.pos:], p)
-	if n == len(p) {
-		b.pos += n
-		return n, nil
-	}
-	b.buf = append(b.buf, p[n:]...)
-	b.pos += len(p)
-	return len(p), nil
-}
-
-func (b *seekBuffer) Seek(offset int64, whence int) (ret int64, err error) {
-	var abs int64
-	switch whence {
-	case 0:
-		abs = offset
-	case 1:
-		abs = int64(b.pos) + offset
-	case 2:
-		abs = int64(len(b.buf)) + offset
-	default:
-		return 0, errors.New("binarydist: invalid whence")
-	}
-	if abs < 0 {
-		return 0, errors.New("binarydist: negative position")
-	}
-	if abs >= 1<<31 {
-		return 0, errors.New("binarydist: position out of range")
-	}
-	b.pos = int(abs)
-	return abs, nil
-}

vendor/github.com/equinox-io/equinox/internal/go-update/internal/osext/LICENSE

@@ -1,27 +0,0 @@
-Copyright (c) 2012 The Go Authors. All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
-   * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-   * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
-   * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

vendor/github.com/equinox-io/equinox/internal/go-update/internal/osext/README.md

@@ -1,16 +0,0 @@
-### Extensions to the "os" package.
-
-## Find the current Executable and ExecutableFolder.
-
-There is sometimes utility in finding the current executable file
-that is running. This can be used for upgrading the current executable
-or finding resources located relative to the executable file. Both
-working directory and the os.Args[0] value are arbitrary and cannot
-be relied on; os.Args[0] can be "faked".
-
-Multi-platform and supports:
- * Linux
- * OS X
- * Windows
- * Plan 9
- * BSDs.

vendor/github.com/equinox-io/equinox/internal/go-update/internal/osext/osext.go

@@ -1,27 +0,0 @@
-// Copyright 2012 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// Extensions to the standard "os" package.
-package osext
-
-import "path/filepath"
-
-// Executable returns an absolute path that can be used to
-// re-invoke the current program.
-// It may not be valid after the current program exits.
-func Executable() (string, error) {
-	p, err := executable()
-	return filepath.Clean(p), err
-}
-
-// Returns same path as Executable, returns just the folder
-// path. Excludes the executable name and any trailing slash.
-func ExecutableFolder() (string, error) {
-	p, err := Executable()
-	if err != nil {
-		return "", err
-	}
-
-	return filepath.Dir(p), nil
-}

vendor/github.com/equinox-io/equinox/internal/go-update/internal/osext/osext_plan9.go

@@ -1,20 +0,0 @@
-// Copyright 2012 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package osext
-
-import (
-	"os"
-	"strconv"
-	"syscall"
-)
-
-func executable() (string, error) {
-	f, err := os.Open("/proc/" + strconv.Itoa(os.Getpid()) + "/text")
-	if err != nil {
-		return "", err
-	}
-	defer f.Close()
-	return syscall.Fd2path(int(f.Fd()))
-}

vendor/github.com/equinox-io/equinox/internal/go-update/internal/osext/osext_procfs.go

@@ -1,36 +0,0 @@
-// Copyright 2012 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build linux netbsd openbsd solaris dragonfly
-
-package osext
-
-import (
-	"errors"
-	"fmt"
-	"os"
-	"runtime"
-	"strings"
-)
-
-func executable() (string, error) {
-	switch runtime.GOOS {
-	case "linux":
-		const deletedTag = " (deleted)"
-		execpath, err := os.Readlink("/proc/self/exe")
-		if err != nil {
-			return execpath, err
-		}
-		execpath = strings.TrimSuffix(execpath, deletedTag)
-		execpath = strings.TrimPrefix(execpath, deletedTag)
-		return execpath, nil
-	case "netbsd":
-		return os.Readlink("/proc/curproc/exe")
-	case "openbsd", "dragonfly":
-		return os.Readlink("/proc/curproc/file")
-	case "solaris":
-		return os.Readlink(fmt.Sprintf("/proc/%d/path/a.out", os.Getpid()))
-	}
-	return "", errors.New("ExecPath not implemented for " + runtime.GOOS)
-}

vendor/github.com/equinox-io/equinox/internal/go-update/internal/osext/osext_sysctl.go

@@ -1,79 +0,0 @@
-// Copyright 2012 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build darwin freebsd
-
-package osext
-
-import (
-	"os"
-	"path/filepath"
-	"runtime"
-	"syscall"
-	"unsafe"
-)
-
-var initCwd, initCwdErr = os.Getwd()
-
-func executable() (string, error) {
-	var mib [4]int32
-	switch runtime.GOOS {
-	case "freebsd":
-		mib = [4]int32{1 /* CTL_KERN */, 14 /* KERN_PROC */, 12 /* KERN_PROC_PATHNAME */, -1}
-	case "darwin":
-		mib = [4]int32{1 /* CTL_KERN */, 38 /* KERN_PROCARGS */, int32(os.Getpid()), -1}
-	}
-
-	n := uintptr(0)
-	// Get length.
-	_, _, errNum := syscall.Syscall6(syscall.SYS___SYSCTL, uintptr(unsafe.Pointer(&mib[0])), 4, 0, uintptr(unsafe.Pointer(&n)), 0, 0)
-	if errNum != 0 {
-		return "", errNum
-	}
-	if n == 0 { // This shouldn't happen.
-		return "", nil
-	}
-	buf := make([]byte, n)
-	_, _, errNum = syscall.Syscall6(syscall.SYS___SYSCTL, uintptr(unsafe.Pointer(&mib[0])), 4, uintptr(unsafe.Pointer(&buf[0])), uintptr(unsafe.Pointer(&n)), 0, 0)
-	if errNum != 0 {
-		return "", errNum
-	}
-	if n == 0 { // This shouldn't happen.
-		return "", nil
-	}
-	for i, v := range buf {
-		if v == 0 {
-			buf = buf[:i]
-			break
-		}
-	}
-	var err error
-	execPath := string(buf)
-	// execPath will not be empty due to above checks.
-	// Try to get the absolute path if the execPath is not rooted.
-	if execPath[0] != '/' {
-		execPath, err = getAbs(execPath)
-		if err != nil {
-			return execPath, err
-		}
-	}
-	// For darwin KERN_PROCARGS may return the path to a symlink rather than the
-	// actual executable.
-	if runtime.GOOS == "darwin" {
-		if execPath, err = filepath.EvalSymlinks(execPath); err != nil {
-			return execPath, err
-		}
-	}
-	return execPath, nil
-}
-
-func getAbs(execPath string) (string, error) {
-	if initCwdErr != nil {
-		return execPath, initCwdErr
-	}
-	// The execPath may begin with a "../" or a "./" so clean it first.
-	// Join the two paths, trailing and starting slashes undetermined, so use
-	// the generic Join function.
-	return filepath.Join(initCwd, filepath.Clean(execPath)), nil
-}

vendor/github.com/equinox-io/equinox/internal/go-update/internal/osext/osext_windows.go

@@ -1,34 +0,0 @@
-// Copyright 2012 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package osext
-
-import (
-	"syscall"
-	"unicode/utf16"
-	"unsafe"
-)
-
-var (
-	kernel                = syscall.MustLoadDLL("kernel32.dll")
-	getModuleFileNameProc = kernel.MustFindProc("GetModuleFileNameW")
-)
-
-// GetModuleFileName() with hModule = NULL
-func executable() (exePath string, err error) {
-	return getModuleFileName()
-}
-
-func getModuleFileName() (string, error) {
-	var n uint32
-	b := make([]uint16, syscall.MAX_PATH)
-	size := uint32(len(b))
-
-	r0, _, e1 := getModuleFileNameProc.Call(0, uintptr(unsafe.Pointer(&b[0])), uintptr(size))
-	n = uint32(r0)
-	if n == 0 {
-		return "", e1
-	}
-	return string(utf16.Decode(b[0:n])), nil
-}

vendor/github.com/equinox-io/equinox/internal/go-update/patcher.go

@@ -1,24 +0,0 @@
-package update
-
-import (
-	"io"
-
-	"github.com/equinox-io/equinox/internal/go-update/internal/binarydist"
-)
-
-// Patcher defines an interface for applying binary patches to an old item to get an updated item.
-type Patcher interface {
-	Patch(old io.Reader, new io.Writer, patch io.Reader) error
-}
-
-type patchFn func(io.Reader, io.Writer, io.Reader) error
-
-func (fn patchFn) Patch(old io.Reader, new io.Writer, patch io.Reader) error {
-	return fn(old, new, patch)
-}
-
-// NewBSDifferPatcher returns a new Patcher that applies binary patches using
-// the bsdiff algorithm. See http://www.daemonology.net/bsdiff/
-func NewBSDiffPatcher() Patcher {
-	return patchFn(binarydist.Patch)
-}

vendor/github.com/equinox-io/equinox/internal/go-update/verifier.go

@@ -1,74 +0,0 @@
-package update
-
-import (
-	"crypto"
-	"crypto/dsa"
-	"crypto/ecdsa"
-	"crypto/rsa"
-	"encoding/asn1"
-	"errors"
-	"math/big"
-)
-
-// Verifier defines an interface for verfiying an update's signature with a public key.
-type Verifier interface {
-	VerifySignature(checksum, signature []byte, h crypto.Hash, publicKey crypto.PublicKey) error
-}
-
-type verifyFn func([]byte, []byte, crypto.Hash, crypto.PublicKey) error
-
-func (fn verifyFn) VerifySignature(checksum []byte, signature []byte, hash crypto.Hash, publicKey crypto.PublicKey) error {
-	return fn(checksum, signature, hash, publicKey)
-}
-
-// NewRSAVerifier returns a Verifier that uses the RSA algorithm to verify updates.
-func NewRSAVerifier() Verifier {
-	return verifyFn(func(checksum, signature []byte, hash crypto.Hash, publicKey crypto.PublicKey) error {
-		key, ok := publicKey.(*rsa.PublicKey)
-		if !ok {
-			return errors.New("not a valid RSA public key")
-		}
-		return rsa.VerifyPKCS1v15(key, hash, checksum, signature)
-	})
-}
-
-type rsDER struct {
-	R *big.Int
-	S *big.Int
-}
-
-// NewECDSAVerifier returns a Verifier that uses the ECDSA algorithm to verify updates.
-func NewECDSAVerifier() Verifier {
-	return verifyFn(func(checksum, signature []byte, hash crypto.Hash, publicKey crypto.PublicKey) error {
-		key, ok := publicKey.(*ecdsa.PublicKey)
-		if !ok {
-			return errors.New("not a valid ECDSA public key")
-		}
-		var rs rsDER
-		if _, err := asn1.Unmarshal(signature, &rs); err != nil {
-			return err
-		}
-		if !ecdsa.Verify(key, checksum, rs.R, rs.S) {
-			return errors.New("failed to verify ecsda signature")
-		}
-		return nil
-	})
-}
-
-// NewDSAVerifier returns a Verifier that uses the DSA algorithm to verify updates.
-func NewDSAVerifier() Verifier {
-	return verifyFn(func(checksum, signature []byte, hash crypto.Hash, publicKey crypto.PublicKey) error {
-		key, ok := publicKey.(*dsa.PublicKey)
-		if !ok {
-			return errors.New("not a valid DSA public key")
-		}
-		var rs rsDER
-		if _, err := asn1.Unmarshal(signature, &rs); err != nil {
-			return err
-		}
-		if !dsa.Verify(key, checksum, rs.R, rs.S) {
-			return errors.New("failed to verify ecsda signature")
-		}
-		return nil
-	})
-}

vendor/github.com/equinox-io/equinox/internal/osext/LICENSE

@@ -1,27 +0,0 @@
-Copyright (c) 2012 The Go Authors. All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
-   * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-   * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
-   * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

vendor/github.com/equinox-io/equinox/internal/osext/README.md

@@ -1,16 +0,0 @@
-### Extensions to the "os" package.
-
-## Find the current Executable and ExecutableFolder.
-
-There is sometimes utility in finding the current executable file
-that is running. This can be used for upgrading the current executable
-or finding resources located relative to the executable file. Both
-working directory and the os.Args[0] value are arbitrary and cannot
-be relied on; os.Args[0] can be "faked".
-
-Multi-platform and supports:
- * Linux
- * OS X
- * Windows
- * Plan 9
- * BSDs.

vendor/github.com/equinox-io/equinox/internal/osext/osext.go

@@ -1,27 +0,0 @@
-// Copyright 2012 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// Extensions to the standard "os" package.
-package osext
-
-import "path/filepath"
-
-// Executable returns an absolute path that can be used to
-// re-invoke the current program.
-// It may not be valid after the current program exits.
-func Executable() (string, error) {
-	p, err := executable()
-	return filepath.Clean(p), err
-}
-
-// Returns same path as Executable, returns just the folder
-// path. Excludes the executable name and any trailing slash.
-func ExecutableFolder() (string, error) {
-	p, err := Executable()
-	if err != nil {
-		return "", err
-	}
-
-	return filepath.Dir(p), nil
-}

vendor/github.com/equinox-io/equinox/internal/osext/osext_plan9.go

@@ -1,20 +0,0 @@
-// Copyright 2012 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package osext
-
-import (
-	"os"
-	"strconv"
-	"syscall"
-)
-
-func executable() (string, error) {
-	f, err := os.Open("/proc/" + strconv.Itoa(os.Getpid()) + "/text")
-	if err != nil {
-		return "", err
-	}
-	defer f.Close()
-	return syscall.Fd2path(int(f.Fd()))
-}

vendor/github.com/equinox-io/equinox/internal/osext/osext_procfs.go

@@ -1,36 +0,0 @@
-// Copyright 2012 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build linux netbsd openbsd solaris dragonfly
-
-package osext
-
-import (
-	"errors"
-	"fmt"
-	"os"
-	"runtime"
-	"strings"
-)
-
-func executable() (string, error) {
-	switch runtime.GOOS {
-	case "linux":
-		const deletedTag = " (deleted)"
-		execpath, err := os.Readlink("/proc/self/exe")
-		if err != nil {
-			return execpath, err
-		}
-		execpath = strings.TrimSuffix(execpath, deletedTag)
-		execpath = strings.TrimPrefix(execpath, deletedTag)
-		return execpath, nil
-	case "netbsd":
-		return os.Readlink("/proc/curproc/exe")
-	case "openbsd", "dragonfly":
-		return os.Readlink("/proc/curproc/file")
-	case "solaris":
-		return os.Readlink(fmt.Sprintf("/proc/%d/path/a.out", os.Getpid()))
-	}
-	return "", errors.New("ExecPath not implemented for " + runtime.GOOS)
-}