Release 2021.7.2

- cfsetup now has a build command `github-release-pkgs` to release linux
   and msi packages to github.
 - github_message.py now has an option to upload all assets in a provided
   directory.

.github/workflows/check.yaml

@@ -0,0 +1,20 @@
+on: [push, pull_request]
+name: Check
+jobs:
+  check:
+    strategy:
+      matrix:
+        go-version: [1.15.x]
+        os: [ubuntu-latest, macos-latest, windows-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+    - name: Install Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ matrix.go-version }}
+    - name: Install go-sumtype
+      run: go get github.com/BurntSushi/go-sumtype
+    - name: Checkout code
+      uses: actions/checkout@v2
+    - name: Test
+      run: make test

.gitignore

@@ -15,3 +15,4 @@ cscope.*
 *-session.log
 ssh_server_tests/.env
 /.cover
+built_artifacts/

CHANGES.md

@@ -1,5 +1,61 @@
 **Experimental**: This is a new format for release notes. The format and availability is subject to change.
 
+## 2021.6.0
+### Bug Fixes
+- Fixes a http2 transport (the new default for Named Tunnels) to work with unix socket origins.
+
+
+## 2021.5.10
+### Bug Fixes
+- Fixes a memory leak in h2mux transport that connects cloudflared to Cloudflare edge.
+
+
+## 2021.5.9
+### New Features
+- Uses new Worker based login helper service to facilitate token exchange in cloudflared flows.
+
+### Bug Fixes
+- Fixes Centos-7 builds.
+
+## 2021.5.8
+### New Features
+- When creating a DNS record to point a hostname at a tunnel, you can now use --overwrite-dns to overwrite any existing
+  DNS records with that hostname. This works both when using the CLI to provision DNS, as well as when starting an adhoc
+  named tunnel, e.g.:
+  - `cloudflared tunnel route dns --overwrite-dns foo-tunnel foo.example.com`
+  - `cloudflared tunnel --overwrite-dns --name foo-tunnel --hostname foo.example.com`
+
+## 2021.5.7
+### New Features
+- Named Tunnels will automatically select the protocol to connect to Cloudflare's edge network.
+
+## 2021.5.0
+
+### New Features
+- It is now possible to run the same tunnel using more than one `cloudflared` instance. This is a server-side change and
+  is compatible with any client version that uses Named Tunnels.
+
+  To get started, visit our [developer documentation](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/run-tunnel/deploy-cloudflared-replicas).
+- `cloudflared tunnel ingress validate` will now warn about unused keys in your config file. This is helpful for
+  detecting typos in your config.
+- If `cloudflared` detects it is running inside a Linux container, it will limit itself to use only the number of CPUs
+  the pod has been granted, instead of trying to use every CPU available.
+
+## 2021.4.0
+
+### Bug Fixes
+
+- Fixed proxying of websocket requests to avoid possibility of losing initial frames that were sent in the same TCP
+  packet as response headers [#345](https://github.com/cloudflare/cloudflared/issues/345).
+- `proxy-dns` option now works in conjunction with running a named tunnel [#346](https://github.com/cloudflare/cloudflared/issues/346).
+
+## 2021.3.6
+
+### Bug Fixes
+
+- Reverted 2021.3.5 improvement to use HTTP/2 in a best-effort manner between cloudflared and origin services because
+  it was found to break in some cases.
+
 ## 2021.3.5
 
 ### Improvements
@@ -16,7 +72,7 @@
 ### Improvements
 
 - Tunnel create command, as well as, running ad-hoc tunnels using `cloudflared tunnel -name NAME`, will not overwrite
-  existing files when writing tunnel credentials. 
+  existing files when writing tunnel credentials.
 
 ### Bug Fixes
 
@@ -62,7 +118,7 @@ ingress:
           ports: [80, 443]
           allow: true
 ```
-  
+
 
 ### Improvements
 
@@ -114,4 +170,3 @@ ingress:
 
 - The maximum number of upstream connections is now limited by default which should fix reported issues of cloudflared
   exhausting CPU usage when faced with connectivity issues.
-

Dockerfile

@@ -1,7 +1,7 @@
 # use a builder image for building cloudflare
 ARG TARGET_GOOS
 ARG TARGET_GOARCH
-FROM golang:1.15.7 as builder
+FROM golang:1.16.4 as builder
 ENV GO111MODULE=on \
     CGO_ENABLED=0 \
     TARGET_GOOS=${TARGET_GOOS} \

Makefile

@@ -11,6 +11,13 @@ ifneq ($(GO_BUILD_TAGS),)
 	GO_BUILD_TAGS := -tags $(GO_BUILD_TAGS)
 endif
 
+ifeq ($(NIGHTLY), true)
+	DEB_PACKAGE_NAME := cloudflared-nightly
+	NIGHTLY_FLAGS := --conflicts cloudflared --replaces cloudflared
+else
+	DEB_PACKAGE_NAME := cloudflared
+endif
+
 DATE          := $(shell date -u '+%Y-%m-%d-%H%M UTC')
 VERSION_FLAGS := -ldflags='-X "main.Version=$(VERSION)" -X "main.BuildTime=$(DATE)"'
 
@@ -81,7 +88,7 @@ clean:
 	go clean
 
 .PHONY: cloudflared
-cloudflared: tunnel-deps
+cloudflared: 
 ifeq ($(FIPS), true)
 	$(info Building cloudflared with go-fips)
 	-test -f fips/fips.go && mv fips/fips.go fips/fips.go.linux-amd64
@@ -138,7 +145,7 @@ define build_package
 		--license 'Cloudflare Service Agreement' \
 		--url 'https://github.com/cloudflare/cloudflared' \
 		-m 'Cloudflare <support@cloudflare.com>' \
-		-a $(TARGET_ARCH) -v $(VERSION) -n cloudflared --after-install postinst.sh --after-remove postrm.sh \
+		-a $(TARGET_ARCH) -v $(VERSION) -n $(DEB_PACKAGE_NAME) $(NIGHTLY_FLAGS) --after-install postinst.sh --after-remove postrm.sh \
 		cloudflared=$(INSTALL_BINDIR) cloudflared.1=$(MAN_DIR)
 endef
 
@@ -150,6 +157,14 @@ cloudflared-deb: cloudflared
 cloudflared-rpm: cloudflared
 	$(call build_package,rpm)
 
+.PHONY: cloudflared-pkg
+cloudflared-pkg: cloudflared
+	$(call build_package,osxpkg)
+
+.PHONY: cloudflared-msi
+cloudflared-msi: cloudflared
+	wixl --define Version=$(VERSION) --define Path=$(EXECUTABLE_PATH) --output cloudflared-$(VERSION)-$(TARGET_ARCH).msi cloudflared.wxs
+
 .PHONY: cloudflared-darwin-amd64.tgz
 cloudflared-darwin-amd64.tgz: cloudflared
 	tar czf cloudflared-darwin-amd64.tgz cloudflared
@@ -224,6 +239,10 @@ release: bin/equinox
 github-release: cloudflared
 	python3 github_release.py --path $(EXECUTABLE_PATH) --release-version $(VERSION)
 
+.PHONY: github-release-built-pkgs
+github-release-built-pkgs:
+	python3 github_release.py --path $(PWD)/built_artifacts --release-version $(VERSION)
+
 .PHONY: github-message
 github-message:
 	python3 github_message.py --release-version $(VERSION)
@@ -237,24 +256,25 @@ bin/equinox:
 	mkdir -p bin
 	curl -s https://bin.equinox.io/c/75JtLRTsJ3n/release-tool-beta-$(EQUINOX_PLATFORM).tgz | tar xz -C bin/
 
-.PHONY: tunnel-deps
-tunnel-deps: tunnelrpc/tunnelrpc.capnp.go
 
-tunnelrpc/tunnelrpc.capnp.go: tunnelrpc/tunnelrpc.capnp
+.PHONY: tunnelrpc-deps
+tunnelrpc-deps:
 	which capnp  # https://capnproto.org/install.html
 	which capnpc-go  # go get zombiezen.com/go/capnproto2/capnpc-go
 	capnp compile -ogo tunnelrpc/tunnelrpc.capnp
 
+.PHONY: quic-deps
+quic-deps: 
+	which capnp 
+	which capnpc-go
+	capnp compile -ogo quic/schema/quic_metadata_protocol.capnp
+
 .PHONY: vet
 vet:
 	go vet -mod=vendor ./...
 	which go-sumtype  # go get github.com/BurntSushi/go-sumtype (don't do this in build directory or this will cause vendor issues)
 	go-sumtype $$(go list -mod=vendor ./...)
 
-.PHONY: msi
-msi: cloudflared
-	go-msi make --msi cloudflared.msi --version $(MSI_VERSION)
-
 .PHONY: goimports
 goimports:
 	for d in $$(go list -mod=readonly -f '{{.Dir}}' -a ./... | fgrep -v tunnelrpc) ; do goimports -format-only -local github.com/cloudflare/cloudflared -w $$d ; done

RELEASE_NOTES

@@ -1,3 +1,85 @@
+2021.7.2
+- 2021-07-27 Fixed a syntax error with python logging.
+
+2021.7.1
+- 2021-07-21 TUN-4755: Add a windows msi release option to Make
+- 2021-07-22 TUN-4761: Added a build-all-packages target to cfsetup
+- 2021-07-26 TUN-4771: Upload deb, rpm and msi packages to github
+- 2021-07-14 TUN-4714: Name nightly package cloudflared-nightly to avoid apt conflict
+- 2021-07-16 TUN-4701: Split Proxy into ProxyHTTP and ProxyTCP
+- 2021-07-08 TUN-4596: Add QUIC application protocol for QUIC stream handshake
+- 2021-07-09 TUN-4699: Make quick tunnels the default in cloudflared
+
+2021.7.0
+- 2021-07-01 TUN-4626: Proxy non-stream based origin websockets with http Roundtrip.
+- 2021-07-01 TUN-4655: ingress.StreamBasedProxy.EstablishConnection takes dest input
+- 2021-07-09 TUN-4698: Add cloudflared metrics endpoint to serve quick tunnel hostname
+- 2021-06-21 TUN-4521: Modify cloudflared to use zoneless-tunnels-worker for free tunnels
+- 2021-04-05 AUTH-3475: Updated GetAppInfo error message
+
+2021.6.0
+- 2021-06-21 TUN-4571: Changelog for 2021.6.0
+- 2021-06-18 TUN-4571: Fix proxying to unix sockets when using HTTP2 transport to Cloudflare Edge
+- 2021-06-07 TUN-4502: Make `cloudflared tunnel route` subcommands described consistently
+- 2021-06-08 TUN-4504: Fix component tests in windows
+- 2021-05-27 TUN-4461: Log resulting DNS hostname if one is received from Cloudflare API
+
+2021.5.10
+- 2021-05-25 TUN-4456: Replaced instances of Tick() with Ticker() in h2mux paths
+
+2021.5.9
+- 2021-05-20 TUN-4426: Fix centos builds
+- 2021-05-20 Update changelog
+- 2021-04-30 AUTH-3426: Point to new transfer service URL and eliminate PUT /ok
+
+2021.5.8
+- 2021-05-14 TUN-4419: Improve error message when cloudflared cannot reach origin
+- 2021-05-19 TUN-4425: --overwrite-dns flag for in adhoc and route dns cmds
+
+2021.5.7
+- 2021-05-17 Fix typo in Changes.md
+- 2021-05-17 TUN-4421: Named Tunnels will automatically select the protocol to connect to Cloudflare's edge network
+
+2021.5.6
+- 2021-05-14 TUN-4418: Downgrade to Go 1.16.3
+
+2021.5.5
+
+
+2021.5.4
+- Fix release pipeline
+
+2021.5.1
+- 2021-05-10 TUN-4342: Fix false positive warning about unused hostname property
+- 2021-05-10 Release 2021.5.0
+
+2021.5.0
+- 2021-05-10 TUN-4384: Silence log from automaxprocs
+- 2021-05-10 AUTH-3537: AUDs in JWTs are now always arrays
+- 2021-05-10 Update changelog for 2021.5.0
+- 2021-05-03 TUN-4343: Fix broken build by setting debug field correctly
+- 2021-05-06 TUN-4356: Set AUTOMAXPROCS to the CPU limit when running in a Linux container
+- 2021-05-06 TUN-4357: Bump Go to 1.16
+- 2021-05-06 TUN-4359: Warn about unused keys in 'tunnel ingress validate'
+- 2021-04-30 debug: log host / path
+- 2021-04-20 AUTH-3513: Checks header for app info in case response is a 403/401 from the edge
+- 2021-04-29 TUN-4000: Release notes for cloudflared replica model
+- 2021-04-09 TUN-2853: rename STDIN-CONTROL env var to STDIN_CONTROL
+- 2021-04-09 TUN-4206: Better error message when user is only using one ingress rule
+
+2021.4.0
+- 2021-04-05 TUN-4178: Fix component test for running as a service in MacOS to not assume a named tunnel
+- 2021-04-05 TUN-4177: Running with proxy-dns should not prevent running Named Tunnels
+- 2021-04-02 TUN-4168: Transparently proxy websocket connections using stdlib HTTP client instead of gorilla/websocket; move websocket client code into carrier package since it's only used by access subcommands now (#345).
+- 2021-04-07 Publish change log for 2021.4.0
+
+2021.3.6
+- 2021-03-30 TUN-4150: Only show the connector table in 'tunnel info' if there are connectors. Don't show rows with zero connections.
+- 2021-03-31 TUN-4153: Revert best-effort HTTP2 usage when talking to origins
+- 2021-03-26 TUN-4141: Better error messages for tunnel info subcommand.
+- 2021-03-29 TUN-4146: Unhide and document grace-period
+- 2021-03-25 TUN-3863: Consolidate header handling logic in the connection package; move headers definitions from h2mux to packages that manage them; cleanup header conversions
+
 2021.3.5
 - 2021-03-26 TUN-3896: http-service and tunnelstore client use http2 transport.
 - 2021-03-25 TUN-4125: Change component tests to run in CI with its own dedicated resources

build-packages.sh

@@ -0,0 +1,22 @@
+# This controls the directory the built artifacts go into
+export ARTIFACT_DIR=built_artifacts/
+mkdir -p $ARTIFACT_DIR
+windowsArchs=("amd64" "386")
+export TARGET_OS=windows
+for arch in ${windowsArchs[@]}; do 
+    export TARGET_ARCH=$arch
+    make cloudflared-msi
+done
+
+mv *.msi $ARTIFACT_DIR
+
+linuxArchs=("amd64" "386" "arm")
+export TARGET_OS=linux
+for arch in ${linuxArchs[@]}; do 
+    export TARGET_ARCH=$arch
+    make cloudflared-deb
+    make cloudflared-rpm
+done
+
+mv *.deb $ARTIFACT_DIR
+mv *.rpm $ARTIFACT_DIR

carrier/carrier.go

@@ -5,20 +5,25 @@ package carrier
 
 import (
 	"crypto/tls"
+	"fmt"
 	"io"
 	"net"
 	"net/http"
+	"net/url"
 	"os"
 	"strings"
 
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
 
-	"github.com/cloudflare/cloudflared/h2mux"
 	"github.com/cloudflare/cloudflared/token"
 )
 
-const LogFieldOriginURL = "originURL"
+const (
+	LogFieldOriginURL       = "originURL"
+	CFAccessTokenHeader     = "Cf-Access-Token"
+	cfJumpDestinationHeader = "Cf-Access-Jump-Destination"
+)
 
 type StartOptions struct {
 	AppInfo         *token.AppInfo
@@ -32,15 +37,11 @@ type StartOptions struct {
 type Connection interface {
 	// ServeStream is used to forward data from the client to the edge
 	ServeStream(*StartOptions, io.ReadWriter) error
-
-	// StartServer is used to listen for incoming connections from the edge to the origin
-	StartServer(net.Listener, string, <-chan struct{}) error
 }
 
 // StdinoutStream is empty struct for wrapping stdin/stdout
 // into a single ReadWriter
-type StdinoutStream struct {
-}
+type StdinoutStream struct{}
 
 // Read will read from Stdin
 func (c *StdinoutStream) Read(p []byte) (int, error) {
@@ -149,7 +150,7 @@ func BuildAccessRequest(options *StartOptions, log *zerolog.Logger) (*http.Reque
 	if err != nil {
 		return nil, err
 	}
-	originRequest.Header.Set(h2mux.CFAccessTokenHeader, token)
+	originRequest.Header.Set(CFAccessTokenHeader, token)
 
 	for k, v := range options.Headers {
 		if len(v) >= 1 {
@@ -159,3 +160,26 @@ func BuildAccessRequest(options *StartOptions, log *zerolog.Logger) (*http.Reque
 
 	return originRequest, nil
 }
+
+func SetBastionDest(header http.Header, destination string) {
+	if destination != "" {
+		header.Set(cfJumpDestinationHeader, destination)
+	}
+}
+
+func ResolveBastionDest(r *http.Request) (string, error) {
+	jumpDestination := r.Header.Get(cfJumpDestinationHeader)
+	if jumpDestination == "" {
+		return "", fmt.Errorf("Did not receive final destination from client. The --destination flag is likely not set on the client side")
+	}
+	// Strip scheme and path set by client. Without a scheme
+	// Parsing a hostname and path without scheme might not return an error due to parsing ambiguities
+	if jumpURL, err := url.Parse(jumpDestination); err == nil && jumpURL.Host != "" {
+		return removePath(jumpURL.Host), nil
+	}
+	return removePath(jumpDestination), nil
+}
+
+func removePath(dest string) string {
+	return strings.SplitN(dest, "/", 2)[0]
+}

carrier/carrier_test.go

@@ -156,3 +156,99 @@ func testRequest(t *testing.T, url string, stream io.ReadWriter) *http.Request {
 
 	return req
 }
+
+func TestBastionDestination(t *testing.T) {
+	tests := []struct {
+		name         string
+		header       http.Header
+		expectedDest string
+		wantErr      bool
+	}{
+		{
+			name: "hostname destination",
+			header: http.Header{
+				cfJumpDestinationHeader: []string{"localhost"},
+			},
+			expectedDest: "localhost",
+		},
+		{
+			name: "hostname destination with port",
+			header: http.Header{
+				cfJumpDestinationHeader: []string{"localhost:9000"},
+			},
+			expectedDest: "localhost:9000",
+		},
+		{
+			name: "hostname destination with scheme and port",
+			header: http.Header{
+				cfJumpDestinationHeader: []string{"ssh://localhost:9000"},
+			},
+			expectedDest: "localhost:9000",
+		},
+		{
+			name: "full hostname url",
+			header: http.Header{
+				cfJumpDestinationHeader: []string{"ssh://localhost:9000/metrics"},
+			},
+			expectedDest: "localhost:9000",
+		},
+		{
+			name: "hostname destination with port and path",
+			header: http.Header{
+				cfJumpDestinationHeader: []string{"localhost:9000/metrics"},
+			},
+			expectedDest: "localhost:9000",
+		},
+		{
+			name: "ip destination",
+			header: http.Header{
+				cfJumpDestinationHeader: []string{"127.0.0.1"},
+			},
+			expectedDest: "127.0.0.1",
+		},
+		{
+			name: "ip destination with port",
+			header: http.Header{
+				cfJumpDestinationHeader: []string{"127.0.0.1:9000"},
+			},
+			expectedDest: "127.0.0.1:9000",
+		},
+		{
+			name: "ip destination with port and path",
+			header: http.Header{
+				cfJumpDestinationHeader: []string{"127.0.0.1:9000/metrics"},
+			},
+			expectedDest: "127.0.0.1:9000",
+		},
+		{
+			name: "ip destination with schem and port",
+			header: http.Header{
+				cfJumpDestinationHeader: []string{"tcp://127.0.0.1:9000"},
+			},
+			expectedDest: "127.0.0.1:9000",
+		},
+		{
+			name: "full ip url",
+			header: http.Header{
+				cfJumpDestinationHeader: []string{"ssh://127.0.0.1:9000/metrics"},
+			},
+			expectedDest: "127.0.0.1:9000",
+		},
+		{
+			name:    "no destination",
+			wantErr: true,
+		},
+	}
+	for _, test := range tests {
+		r := &http.Request{
+			Header: test.header,
+		}
+		dest, err := ResolveBastionDest(r)
+		if test.wantErr {
+			assert.Error(t, err, "Test %s expects error", test.name)
+		} else {
+			assert.NoError(t, err, "Test %s expects no error, got error %v", test.name, err)
+			assert.Equal(t, test.expectedDest, dest, "Test %s expect dest %s, got %s", test.name, test.expectedDest, dest)
+		}
+	}
+}

carrier/websocket.go

@@ -1,17 +1,14 @@
 package carrier
 
 import (
-	"fmt"
 	"io"
-	"net"
 	"net/http"
 	"net/http/httputil"
+	"net/url"
 
 	"github.com/gorilla/websocket"
 	"github.com/rs/zerolog"
 
-	"github.com/cloudflare/cloudflared/ingress"
-	"github.com/cloudflare/cloudflared/socks"
 	"github.com/cloudflare/cloudflared/token"
 	cfwebsocket "github.com/cloudflare/cloudflared/websocket"
 )
@@ -23,20 +20,6 @@ type Websocket struct {
 	isSocks bool
 }
 
-type wsdialer struct {
-	conn *cfwebsocket.GorillaConn
-}
-
-func (d *wsdialer) Dial(address string) (io.ReadWriteCloser, *socks.AddrSpec, error) {
-	local, ok := d.conn.LocalAddr().(*net.TCPAddr)
-	if !ok {
-		return nil, nil, fmt.Errorf("not a tcp connection")
-	}
-
-	addr := socks.AddrSpec{IP: local.IP, Port: local.Port}
-	return d.conn, &addr, nil
-}
-
 // NewWSConnection returns a new connection object
 func NewWSConnection(log *zerolog.Logger) Connection {
 	return &Websocket{
@@ -54,16 +37,10 @@ func (ws *Websocket) ServeStream(options *StartOptions, conn io.ReadWriter) erro
 	}
 	defer wsConn.Close()
 
-	ingress.Stream(wsConn, conn, ws.log)
+	cfwebsocket.Stream(wsConn, conn, ws.log)
 	return nil
 }
 
-// StartServer creates a Websocket server to listen for connections.
-// This is used on the origin (tunnel) side to take data from the muxer and send it to the origin
-func (ws *Websocket) StartServer(listener net.Listener, remote string, shutdownC <-chan struct{}) error {
-	return cfwebsocket.StartProxyServer(ws.log, listener, remote, shutdownC, ingress.DefaultStreamHandler)
-}
-
 // createWebsocketStream will create a WebSocket connection to stream data over
 // It also handles redirects from Access and will present that flow if
 // the token is not present on the request
@@ -84,7 +61,7 @@ func createWebsocketStream(options *StartOptions, log *zerolog.Logger) (*cfwebso
 		TLSClientConfig: options.TLSClientConfig,
 		Proxy:           http.ProxyFromEnvironment,
 	}
-	wsConn, resp, err := cfwebsocket.ClientConnect(req, dialer)
+	wsConn, resp, err := clientConnect(req, dialer)
 	defer closeRespBody(resp)
 
 	if err != nil && IsAccessResponse(resp) {
@@ -111,6 +88,63 @@ func createWebsocketStream(options *StartOptions, log *zerolog.Logger) (*cfwebso
 	return &cfwebsocket.GorillaConn{Conn: wsConn}, nil
 }
 
+var stripWebsocketHeaders = []string{
+	"Upgrade",
+	"Connection",
+	"Sec-Websocket-Key",
+	"Sec-Websocket-Version",
+	"Sec-Websocket-Extensions",
+}
+
+// the gorilla websocket library sets its own Upgrade, Connection, Sec-WebSocket-Key,
+// Sec-WebSocket-Version and Sec-Websocket-Extensions headers.
+// https://github.com/gorilla/websocket/blob/master/client.go#L189-L194.
+func websocketHeaders(req *http.Request) http.Header {
+	wsHeaders := make(http.Header)
+	for key, val := range req.Header {
+		wsHeaders[key] = val
+	}
+	// Assume the header keys are in canonical format.
+	for _, header := range stripWebsocketHeaders {
+		wsHeaders.Del(header)
+	}
+	wsHeaders.Set("Host", req.Host) // See TUN-1097
+	return wsHeaders
+}
+
+// clientConnect creates a WebSocket client connection for provided request. Caller is responsible for closing
+// the connection. The response body may not contain the entire response and does
+// not need to be closed by the application.
+func clientConnect(req *http.Request, dialler *websocket.Dialer) (*websocket.Conn, *http.Response, error) {
+	req.URL.Scheme = changeRequestScheme(req.URL)
+	wsHeaders := websocketHeaders(req)
+	if dialler == nil {
+		dialler = &websocket.Dialer{
+			Proxy: http.ProxyFromEnvironment,
+		}
+	}
+	conn, response, err := dialler.Dial(req.URL.String(), wsHeaders)
+	if err != nil {
+		return nil, response, err
+	}
+	return conn, response, nil
+}
+
+// changeRequestScheme is needed as the gorilla websocket library requires the ws scheme.
+// (even though it changes it back to http/https, but ¯\_(ツ)_/¯.)
+func changeRequestScheme(reqURL *url.URL) string {
+	switch reqURL.Scheme {
+	case "https":
+		return "wss"
+	case "http":
+		return "ws"
+	case "":
+		return "ws"
+	default:
+		return reqURL.Scheme
+	}
+}
+
 // createAccessAuthenticatedStream will try load a token from storage and make
 // a connection with the token set on the request. If it still get redirect,
 // this probably means the token in storage is invalid (expired/revoked). If that
@@ -150,7 +184,7 @@ func createAccessWebSocketStream(options *StartOptions, log *zerolog.Logger) (*w
 	dump, err := httputil.DumpRequest(req, false)
 	log.Debug().Msgf("Access Websocket request: %s", string(dump))
 
-	conn, resp, err := cfwebsocket.ClientConnect(req, nil)
+	conn, resp, err := clientConnect(req, nil)
 
 	if resp != nil {
 		r, err := httputil.DumpResponse(resp, true)

carrier/websocket_test.go

@@ -0,0 +1,123 @@
+package carrier
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"math/rand"
+	"testing"
+	"time"
+
+	gws "github.com/gorilla/websocket"
+	"github.com/rs/zerolog"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/net/websocket"
+
+	"github.com/cloudflare/cloudflared/hello"
+	"github.com/cloudflare/cloudflared/tlsconfig"
+	cfwebsocket "github.com/cloudflare/cloudflared/websocket"
+)
+
+func websocketClientTLSConfig(t *testing.T) *tls.Config {
+	certPool := x509.NewCertPool()
+	helloCert, err := tlsconfig.GetHelloCertificateX509()
+	assert.NoError(t, err)
+	certPool.AddCert(helloCert)
+	assert.NotNil(t, certPool)
+	return &tls.Config{RootCAs: certPool}
+}
+
+func TestWebsocketHeaders(t *testing.T) {
+	req := testRequest(t, "http://example.com", nil)
+	wsHeaders := websocketHeaders(req)
+	for _, header := range stripWebsocketHeaders {
+		assert.Empty(t, wsHeaders[header])
+	}
+	assert.Equal(t, "curl/7.59.0", wsHeaders.Get("User-Agent"))
+}
+
+func TestServe(t *testing.T) {
+	log := zerolog.Nop()
+	shutdownC := make(chan struct{})
+	errC := make(chan error)
+	listener, err := hello.CreateTLSListener("localhost:1111")
+	assert.NoError(t, err)
+	defer listener.Close()
+
+	go func() {
+		errC <- hello.StartHelloWorldServer(&log, listener, shutdownC)
+	}()
+
+	req := testRequest(t, "https://localhost:1111/ws", nil)
+
+	tlsConfig := websocketClientTLSConfig(t)
+	assert.NotNil(t, tlsConfig)
+	d := gws.Dialer{TLSClientConfig: tlsConfig}
+	conn, resp, err := clientConnect(req, &d)
+	assert.NoError(t, err)
+	assert.Equal(t, "websocket", resp.Header.Get("Upgrade"))
+
+	for i := 0; i < 1000; i++ {
+		messageSize := rand.Int()%2048 + 1
+		clientMessage := make([]byte, messageSize)
+		// rand.Read always returns len(clientMessage) and a nil error
+		rand.Read(clientMessage)
+		err = conn.WriteMessage(websocket.BinaryFrame, clientMessage)
+		assert.NoError(t, err)
+
+		messageType, message, err := conn.ReadMessage()
+		assert.NoError(t, err)
+		assert.Equal(t, websocket.BinaryFrame, messageType)
+		assert.Equal(t, clientMessage, message)
+	}
+
+	_ = conn.Close()
+	close(shutdownC)
+	<-errC
+}
+
+func TestWebsocketWrapper(t *testing.T) {
+	listener, err := hello.CreateTLSListener("localhost:0")
+	require.NoError(t, err)
+
+	serverErrorChan := make(chan error)
+	helloSvrCtx, cancelHelloSvr := context.WithCancel(context.Background())
+	defer func() { <-serverErrorChan }()
+	defer cancelHelloSvr()
+	go func() {
+		log := zerolog.Nop()
+		serverErrorChan <- hello.StartHelloWorldServer(&log, listener, helloSvrCtx.Done())
+	}()
+
+	tlsConfig := websocketClientTLSConfig(t)
+	d := gws.Dialer{TLSClientConfig: tlsConfig, HandshakeTimeout: time.Minute}
+	testAddr := fmt.Sprintf("https://%s/ws", listener.Addr().String())
+	req := testRequest(t, testAddr, nil)
+	conn, resp, err := clientConnect(req, &d)
+	require.NoError(t, err)
+	assert.Equal(t, "websocket", resp.Header.Get("Upgrade"))
+
+	// Websocket now connected to test server so lets check our wrapper
+	wrapper := cfwebsocket.GorillaConn{Conn: conn}
+	buf := make([]byte, 100)
+	wrapper.Write([]byte("abc"))
+	n, err := wrapper.Read(buf)
+	require.NoError(t, err)
+	require.Equal(t, n, 3)
+	require.Equal(t, "abc", string(buf[:n]))
+
+	// Test partial read, read 1 of 3 bytes in one read and the other 2 in another read
+	wrapper.Write([]byte("abc"))
+	buf = buf[:1]
+	n, err = wrapper.Read(buf)
+	require.NoError(t, err)
+	require.Equal(t, n, 1)
+	require.Equal(t, "a", string(buf[:n]))
+	buf = buf[:cap(buf)]
+	n, err = wrapper.Read(buf)
+	require.NoError(t, err)
+	require.Equal(t, n, 2)
+	require.Equal(t, "bc", string(buf[:n]))
+}

cfsetup.yaml

@@ -1,4 +1,4 @@
-pinned_go: &pinned_go go=1.15.7-1
+pinned_go: &pinned_go go=1.16.3-1
 pinned_go_fips: &pinned_go_fips go-fips=1.15.5-3
 
 build_dir: &build_dir /cfsetup_build
@@ -14,25 +14,72 @@ stretch: &stretch
       - export GOARCH=amd64
       - export FIPS=true
       - make cloudflared
-  build-deb:
+  build-all-packages: #except osxpkg
     build_dir: *build_dir
     builddeps:
       - *pinned_go_fips
       - build-essential
       - fakeroot
       - rubygem-fpm
+      - rpm
+      - wget
+      # libmsi and libgcab are libraries the wixl binary depends on.
+      - libmsi-dev
+      - libgcab-dev
+    pre-cache:
+      # TODO: https://jira.cfops.it/browse/TUN-4792 Replace this wixl with the official one once msitools supports
+      # environment.
+      - wget https://github.com/sudarshan-reddy/msitools/releases/download/v0.101b/wixl -P /usr/local/bin
+      - chmod a+x /usr/local/bin/wixl
+    post-cache:
+      - ./build-packages.sh
+  github-release-pkgs:
+    build_dir: *build_dir
+    builddeps:
+      - *pinned_go_fips
+      - build-essential
+      - fakeroot
+      - rubygem-fpm
+      - rpm
+      - wget
+      # libmsi and libgcab are libraries the wixl binary depends on.
+      - libmsi-dev
+      - libgcab-dev
+      - python3-setuptools
+      - python3-pip
+    pre-cache:
+      - wget https://github.com/sudarshan-reddy/msitools/releases/download/v0.101b/wixl -P /usr/local/bin
+      - chmod a+x /usr/local/bin/wixl
+      - pip3 install pygithub
+    post-cache:
+      # build all packages and move them to /cfsetup/built_artifacts
+      - ./build-packages.sh
+      # release the packages built and moved to /cfsetup/built_artifacts
+      - make github-release-built-pkgs
+  build-deb:
+    build_dir: *build_dir
+    builddeps: &build_deb_deps
+      - *pinned_go_fips
+      - build-essential
+      - fakeroot
+      - rubygem-fpm
     post-cache:
       - export GOOS=linux
       - export GOARCH=amd64
       - export FIPS=true
       - make cloudflared-deb
+  build-deb-nightly:
+    build_dir: *build_dir
+    builddeps: *build_deb_deps
+    post-cache:
+      - export GOOS=linux
+      - export GOARCH=amd64
+      - export FIPS=true
+      - export NIGHTLY=true
+      - make cloudflared-deb
   build-deb-arm64:
     build_dir: *build_dir
-    builddeps:
-      - *pinned_go
-      - build-essential
-      - fakeroot
-      - rubygem-fpm
+    builddeps: *build_deb_deps
     post-cache:
       - export GOOS=linux
       - export GOARCH=arm64
@@ -279,8 +326,9 @@ centos-7:
       - https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
     pre-cache:
       - yum install -y fakeroot
-      - wget https://golang.org/dl/go1.15.7.linux-amd64.tar.gz -P /tmp/
-      - tar -C /usr/local -xzf /tmp/go1.15.7.linux-amd64.tar.gz
+      - yum upgrade -y binutils-2.27-44.base.el7.x86_64
+      - wget https://golang.org/dl/go1.16.3.linux-amd64.tar.gz -P /tmp/
+      - tar -C /usr/local -xzf /tmp/go1.16.3.linux-amd64.tar.gz
     post-cache:
       - export PATH=$PATH:/usr/local/go/bin
       - export GOOS=linux
@@ -291,8 +339,9 @@ centos-7:
     builddeps: *el7_builddeps
     pre-cache:
       - yum install -y fakeroot
-      - wget https://golang.org/dl/go1.15.7.linux-amd64.tar.gz -P /tmp/
-      - tar -C /usr/local -xzf /tmp/go1.15.7.linux-amd64.tar.gz
+      - yum upgrade -y binutils-2.27-44.base.el7.x86_64
+      - wget https://golang.org/dl/go1.16.3.linux-amd64.tar.gz -P /tmp/
+      - tar -C /usr/local -xzf /tmp/go1.16.3.linux-amd64.tar.gz
     post-cache:
       - export PATH=$PATH:/usr/local/go/bin
       - export GOOS=linux

cloudflared.wxs

@@ -0,0 +1,62 @@
+<?xml version="1.0"?>
+
+<?if $(var.Platform)="x86"?>
+    <?define Program_Files="ProgramFilesFolder"?>
+<?else?>
+    <?define Program_Files="ProgramFiles64Folder"?>
+<?endif?>
+<?ifndef var.Version?>
+    <?error Undefined Version variable?>
+<?endif?>
+<?ifndef var.Path?>
+    <?error Undefined Path variable?>
+<?endif?>
+
+<Wix xmlns="http://schemas.microsoft.com/wix/2006/wi">
+    <Product Id="35e5e858-9372-4449-bf73-1cd6f7267128" 
+        UpgradeCode="23f90fdd-9328-47ea-ab52-5380855a4b12"
+        Name="cloudflared"
+        Version="$(var.Version)"
+        Manufacturer="cloudflare"
+        Language="1033">
+
+      <Package InstallerVersion="200" Compressed="yes" Comments="Windows Installer Package" InstallScope="perMachine"/>
+
+      <Media Id="1" Cabinet="product.cab" EmbedCab="yes"/>
+
+      <Upgrade Id="23f90fdd-9328-47ea-ab52-5380855a4b12">
+         <UpgradeVersion Minimum="$(var.Version)" OnlyDetect="yes" Property="NEWERVERSIONDETECTED"/>
+         <UpgradeVersion Minimum="2020.8.0" Maximum="$(var.Version)" IncludeMinimum="yes" IncludeMaximum="no"
+                         Property="OLDERVERSIONBEINGUPGRADED"/>
+      </Upgrade>
+      <Condition Message="A newer version of this software is already installed.">NOT NEWERVERSIONDETECTED</Condition>
+
+    <Directory Id="TARGETDIR" Name="SourceDir">
+        <!--This specifies where the cloudflared.exe is moved to in the windows Operation System-->
+        <Directory Id="$(var.Program_Files)">
+            <Directory Id="INSTALLDIR" Name="cloudflared">
+                <Component Id="ApplicationFiles" Guid="35e5e858-9372-4449-bf73-1cd6f7267128">
+                    <File Id="ApplicationFile0" Source="$(var.Path)"/>
+                </Component>
+            </Directory>
+        </Directory>
+        <Component Id="ENVS" Guid="6bb74449-d10d-4f4a-933e-6fc9fa006eae">
+        <!--Set the cloudflared bin location to the Path Environment Variable-->
+         <Environment Id="ENV0"
+           Name="PATH"
+           Value="[INSTALLDIR]."
+           Permanent="no"
+           Part="all"
+           Action="set"
+           System="yes" />
+        </Component>
+    </Directory>
+
+
+    <Feature Id='Complete' Level='1'>
+      <ComponentRef Id="ENVS"/>
+      <ComponentRef Id='ApplicationFiles' />
+    </Feature>
+
+    </Product>
+</Wix>

cmd/cloudflared/access/carrier.go

@@ -6,19 +6,20 @@ import (
 	"net/http"
 	"strings"
 
-	"github.com/cloudflare/cloudflared/carrier"
-	"github.com/cloudflare/cloudflared/config"
-	"github.com/cloudflare/cloudflared/h2mux"
-	"github.com/cloudflare/cloudflared/logger"
-	"github.com/cloudflare/cloudflared/validation"
-
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
 	"github.com/urfave/cli/v2"
+
+	"github.com/cloudflare/cloudflared/carrier"
+	"github.com/cloudflare/cloudflared/config"
+	"github.com/cloudflare/cloudflared/logger"
+	"github.com/cloudflare/cloudflared/validation"
 )
 
 const (
-	LogFieldHost = "host"
+	LogFieldHost               = "host"
+	cfAccessClientIDHeader     = "Cf-Access-Client-Id"
+	cfAccessClientSecretHeader = "Cf-Access-Client-Secret"
 )
 
 // StartForwarder starts a client side websocket forward
@@ -31,16 +32,14 @@ func StartForwarder(forwarder config.Forwarder, shutdown <-chan struct{}, log *z
 	// get the headers from the config file and add to the request
 	headers := make(http.Header)
 	if forwarder.TokenClientID != "" {
-		headers.Set(h2mux.CFAccessClientIDHeader, forwarder.TokenClientID)
+		headers.Set(cfAccessClientIDHeader, forwarder.TokenClientID)
 	}
 
 	if forwarder.TokenSecret != "" {
-		headers.Set(h2mux.CFAccessClientSecretHeader, forwarder.TokenSecret)
+		headers.Set(cfAccessClientSecretHeader, forwarder.TokenSecret)
 	}
 
-	if forwarder.Destination != "" {
-		headers.Add(h2mux.CFJumpDestinationHeader, forwarder.Destination)
-	}
+	carrier.SetBastionDest(headers, forwarder.Destination)
 
 	options := &carrier.StartOptions{
 		OriginURL: forwarder.URL,
@@ -72,16 +71,13 @@ func ssh(c *cli.Context) error {
 	// get the headers from the cmdline and add them
 	headers := buildRequestHeaders(c.StringSlice(sshHeaderFlag))
 	if c.IsSet(sshTokenIDFlag) {
-		headers.Set(h2mux.CFAccessClientIDHeader, c.String(sshTokenIDFlag))
+		headers.Set(cfAccessClientIDHeader, c.String(sshTokenIDFlag))
 	}
 	if c.IsSet(sshTokenSecretFlag) {
-		headers.Set(h2mux.CFAccessClientSecretHeader, c.String(sshTokenSecretFlag))
+		headers.Set(cfAccessClientSecretHeader, c.String(sshTokenSecretFlag))
 	}
 
-	destination := c.String(sshDestinationFlag)
-	if destination != "" {
-		headers.Add(h2mux.CFJumpDestinationHeader, destination)
-	}
+	carrier.SetBastionDest(headers, c.String(sshDestinationFlag))
 
 	options := &carrier.StartOptions{
 		OriginURL: originURL,

cmd/cloudflared/access/cmd.go

@@ -19,7 +19,6 @@ import (
 
 	"github.com/cloudflare/cloudflared/carrier"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
-	"github.com/cloudflare/cloudflared/h2mux"
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/sshgen"
 	"github.com/cloudflare/cloudflared/token"
@@ -286,7 +285,7 @@ func curl(c *cli.Context) error {
 	}
 
 	cmdArgs = append(cmdArgs, "-H")
-	cmdArgs = append(cmdArgs, fmt.Sprintf("%s: %s", h2mux.CFAccessTokenHeader, tok))
+	cmdArgs = append(cmdArgs, fmt.Sprintf("%s: %s", carrier.CFAccessTokenHeader, tok))
 	return run("curl", cmdArgs...)
 }
 
@@ -472,10 +471,10 @@ func isFileThere(candidate string) bool {
 func verifyTokenAtEdge(appUrl *url.URL, appInfo *token.AppInfo, c *cli.Context, log *zerolog.Logger) error {
 	headers := buildRequestHeaders(c.StringSlice(sshHeaderFlag))
 	if c.IsSet(sshTokenIDFlag) {
-		headers.Add(h2mux.CFAccessClientIDHeader, c.String(sshTokenIDFlag))
+		headers.Add(cfAccessClientIDHeader, c.String(sshTokenIDFlag))
 	}
 	if c.IsSet(sshTokenSecretFlag) {
-		headers.Add(h2mux.CFAccessClientSecretHeader, c.String(sshTokenSecretFlag))
+		headers.Add(cfAccessClientSecretHeader, c.String(sshTokenSecretFlag))
 	}
 	options := &carrier.StartOptions{AppInfo: appInfo, OriginURL: appUrl.String(), Headers: headers}
 

cmd/cloudflared/cliutil/handler.go

@@ -13,27 +13,38 @@ func Action(actionFunc cli.ActionFunc) cli.ActionFunc {
 }
 
 func ConfiguredAction(actionFunc cli.ActionFunc) cli.ActionFunc {
+	// Adapt actionFunc to the type signature required by ConfiguredActionWithWarnings
+	f := func(context *cli.Context, _ string) error {
+		return actionFunc(context)
+	}
+
+	return ConfiguredActionWithWarnings(f)
+}
+
+// Just like ConfiguredAction, but accepts a second parameter with configuration warnings.
+func ConfiguredActionWithWarnings(actionFunc func(*cli.Context, string) error) cli.ActionFunc {
 	return WithErrorHandler(func(c *cli.Context) error {
-		if err := setFlagsFromConfigFile(c); err != nil {
+		warnings, err := setFlagsFromConfigFile(c)
+		if err != nil {
 			return err
 		}
-		return actionFunc(c)
+		return actionFunc(c, warnings)
 	})
 }
 
-func setFlagsFromConfigFile(c *cli.Context) error {
+func setFlagsFromConfigFile(c *cli.Context) (configWarnings string, err error) {
 	const errorExitCode = 1
 	log := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
-	inputSource, err := config.ReadConfigFile(c, log)
+	inputSource, warnings, err := config.ReadConfigFile(c, log)
 	if err != nil {
 		if err == config.ErrNoConfigFile {
-			return nil
+			return "", nil
 		}
-		return cli.Exit(err, errorExitCode)
+		return "", cli.Exit(err, errorExitCode)
 	}
 
 	if err := altsrc.ApplyInputSource(c, inputSource); err != nil {
-		return cli.Exit(err, errorExitCode)
+		return "", cli.Exit(err, errorExitCode)
 	}
-	return nil
+	return warnings, nil
 }

cmd/cloudflared/linux_service.go

@@ -238,7 +238,7 @@ func installLinuxService(c *cli.Context) error {
 			"--origincert", serviceConfigDir + "/" + serviceCredentialFile,
 		}
 	} else {
-		src, err := config.ReadConfigFile(c, log)
+		src, _, err := config.ReadConfigFile(c, log)
 		if err != nil {
 			return err
 		}

cmd/cloudflared/main.go

@@ -10,6 +10,7 @@ import (
 	homedir "github.com/mitchellh/go-homedir"
 	"github.com/pkg/errors"
 	"github.com/urfave/cli/v2"
+	"go.uber.org/automaxprocs/maxprocs"
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/access"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
@@ -47,6 +48,7 @@ func main() {
 	rand.Seed(time.Now().UnixNano())
 	metrics.RegisterBuildInfo(BuildTime, Version)
 	raven.SetRelease(Version)
+	maxprocs.Set()
 
 	// Graceful shutdown channel used by the app. When closed, app must terminate gracefully.
 	// Windows service manager closes this channel when it receives stop command.

cmd/cloudflared/proxydns/cmd.go

@@ -73,7 +73,7 @@ func Run(c *cli.Context) error {
 		log.Fatal().Err(err).Msg("Failed to open the metrics listener")
 	}
 
-	go metrics.ServeMetrics(metricsListener, nil, nil, log)
+	go metrics.ServeMetrics(metricsListener, nil, nil, "", log)
 
 	listener, err := tunneldns.CreateListener(
 		c.String("address"),

cmd/cloudflared/tunnel/cmd.go

@@ -87,6 +87,10 @@ const (
 var (
 	graceShutdownC chan struct{}
 	version        string
+
+	routeFailMsg = fmt.Sprintf("failed to provision routing, please create it manually via Cloudflare dashboard or UI; "+
+		"most likely you already have a conflicting record there. You can also rerun this command with --%s to overwrite "+
+		"any existing DNS records for this hostname.", overwriteDNSFlag)
 )
 
 func Flags() []cli.Flag {
@@ -159,6 +163,12 @@ func TunnelCommand(c *cli.Context) error {
 		return fmt.Errorf("Use `cloudflared tunnel run` to start tunnel %s", ref)
 	}
 
+	// Unauthenticated named tunnel on <random>.<quick-tunnels-service>.com
+	// For now, default to legacy setup unless quick-service is specified
+	if c.String("hostname") == "" && c.String("quick-service") != "" {
+		return RunQuickTunnel(sc)
+	}
+
 	// Start a classic tunnel
 	return runClassicTunnel(sc)
 }
@@ -181,7 +191,7 @@ func runAdhocNamedTunnel(sc *subcommandContext, name, credentialsOutputPath stri
 
 	if r, ok := routeFromFlag(sc.c); ok {
 		if res, err := sc.route(tunnel.ID, r); err != nil {
-			sc.log.Err(err).Msg("failed to create route, please create it manually")
+			sc.log.Err(err).Str("route", r.String()).Msg(routeFailMsg)
 		} else {
 			sc.log.Info().Msg(res.SuccessSummary())
 		}
@@ -199,12 +209,12 @@ func runClassicTunnel(sc *subcommandContext) error {
 	return StartServer(sc.c, version, nil, sc.log, sc.isUIEnabled)
 }
 
-func routeFromFlag(c *cli.Context) (tunnelstore.Route, bool) {
+func routeFromFlag(c *cli.Context) (route tunnelstore.Route, ok bool) {
 	if hostname := c.String("hostname"); hostname != "" {
 		if lbPool := c.String("lb-pool"); lbPool != "" {
 			return tunnelstore.NewLBRoute(hostname, lbPool), true
 		}
-		return tunnelstore.NewDNSRoute(hostname), true
+		return tunnelstore.NewDNSRoute(hostname, c.Bool(overwriteDNSFlagName)), true
 	}
 	return nil, false
 }
@@ -296,7 +306,7 @@ func StartServer(
 	}()
 
 	// Serve DNS proxy stand-alone if no hostname or tag or app is going to run
-	if dnsProxyStandAlone(c) {
+	if dnsProxyStandAlone(c, namedTunnel) {
 		connectedSignal.Notify()
 		// no grace period, handle SIGINT/SIGTERM immediately
 		return waitToShutdown(&wg, cancel, errC, graceShutdownC, 0, log)
@@ -314,6 +324,15 @@ func StartServer(
 
 	observer := connection.NewObserver(log, logTransport, isUIEnabled)
 
+	// Send Quick Tunnel URL to UI if applicable
+	var quickTunnelURL string
+	if namedTunnel != nil {
+		quickTunnelURL = namedTunnel.QuickTunnelUrl
+	}
+	if quickTunnelURL != "" {
+		observer.SendURL(quickTunnelURL)
+	}
+
 	tunnelConfig, ingressRules, err := prepareTunnelConfig(c, buildInfo, version, log, logTransport, observer, namedTunnel)
 	if err != nil {
 		log.Err(err).Msg("Couldn't start tunnel")
@@ -331,7 +350,7 @@ func StartServer(
 		defer wg.Done()
 		readinessServer := metrics.NewReadyServer(log)
 		observer.RegisterSink(readinessServer)
-		errC <- metrics.ServeMetrics(metricsListener, ctx.Done(), readinessServer, log)
+		errC <- metrics.ServeMetrics(metricsListener, ctx.Done(), readinessServer, quickTunnelURL, log)
 	}()
 
 	if err := ingressRules.StartOrigins(&wg, log, ctx.Done(), errC); err != nil {
@@ -565,10 +584,10 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 		}),
 		altsrc.NewDurationFlag(&cli.DurationFlag{
 			Name:    "grace-period",
-			Usage:   "Duration to accept new requests after cloudflared receives first SIGINT/SIGTERM. A second SIGINT/SIGTERM will force cloudflared to shutdown immediately.",
+			Usage:   "When cloudflared receives SIGINT/SIGTERM it will stop accepting new requests, wait for in-progress requests to terminate, then shutdown. Waiting for in-progress requests will timeout after this grace period, or when a second SIGTERM/SIGINT is received.",
 			Value:   time.Second * 30,
 			EnvVars: []string{"TUNNEL_GRACE_PERIOD"},
-			Hidden:  true,
+			Hidden:  shouldHide,
 		}),
 		// Note TUN-3758 , we use Int because UInt is not supported with altsrc
 		altsrc.NewIntFlag(&cli.IntFlag{
@@ -595,7 +614,7 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 		altsrc.NewBoolFlag(&cli.BoolFlag{
 			Name:    "stdin-control",
 			Usage:   "Control the process using commands sent through stdin",
-			EnvVars: []string{"STDIN-CONTROL"},
+			EnvVars: []string{"STDIN_CONTROL"},
 			Hidden:  true,
 			Value:   false,
 		}),
@@ -612,7 +631,14 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			Value:  false,
 			Hidden: shouldHide,
 		}),
+		altsrc.NewStringFlag(&cli.StringFlag{
+			Name:   "quick-service",
+			Usage:  "URL for a service which manages unauthenticated 'quick' tunnels.",
+			Value:  "https://api.trycloudflare.com",
+			Hidden: true,
+		}),
 		selectProtocolFlag,
+		overwriteDNSFlag,
 	}...)
 
 	return flags
@@ -699,7 +725,7 @@ func configureProxyFlags(shouldHide bool) []cli.Flag {
 			Hidden: shouldHide,
 		}),
 		altsrc.NewDurationFlag(&cli.DurationFlag{
-			Name:   ingress.ProxyTCPKeepAlive,
+			Name:   ingress.ProxyTCPKeepAliveFlag,
 			Usage:  "HTTP proxy TCP keepalive duration",
 			Value:  time.Second * 30,
 			Hidden: shouldHide,

cmd/cloudflared/tunnel/configuration.go

@@ -85,8 +85,8 @@ func logClientOptions(c *cli.Context, log *zerolog.Logger) {
 	}
 }
 
-func dnsProxyStandAlone(c *cli.Context) bool {
-	return c.IsSet("proxy-dns") && (!c.IsSet("hostname") && !c.IsSet("tag") && !c.IsSet("hello-world"))
+func dnsProxyStandAlone(c *cli.Context, namedTunnel *connection.NamedTunnelConfig) bool {
+	return c.IsSet("proxy-dns") && (!c.IsSet("hostname") && !c.IsSet("tag") && !c.IsSet("hello-world") && namedTunnel == nil)
 }
 
 func findOriginCert(originCertPath string, log *zerolog.Logger) (string, error) {
@@ -160,12 +160,8 @@ func prepareTunnelConfig(
 	if err != nil {
 		log.Err(err).Str(LogFieldHostname, configHostname).Msg("Invalid hostname")
 		return nil, ingress.Ingress{}, errors.Wrap(err, "Invalid hostname")
-	} else if hostname != "" && isNamedTunnel {
-		log.Warn().Msg("The property `hostname` in your configuration is ignored because you configured a Named Tunnel " +
-			"in the property `tunnel`. Make sure to provision the routing (e.g. via `cloudflared tunnel route`) or else " +
-			"your origin will not be reachable. You should remove the `hostname` property to avoid this warning.")
 	}
-	isFreeTunnel := hostname == ""
+	isQuickTunnel := hostname == ""
 	clientID := c.String("id")
 	if !c.IsSet("id") {
 		clientID, err = generateRandomClientID(log)
@@ -183,7 +179,7 @@ func prepareTunnelConfig(
 	tags = append(tags, tunnelpogs.Tag{Name: "ID", Value: clientID})
 
 	var originCert []byte
-	if !isFreeTunnel {
+	if !isQuickTunnel {
 		originCertPath := c.String("origincert")
 		originCertLog := log.With().
 			Str(LogFieldOriginCertPath, originCertPath).
@@ -282,7 +278,6 @@ func prepareTunnelConfig(
 		HAConnections:    c.Int("ha-connections"),
 		IncidentLookup:   origin.NewIncidentLookup(),
 		IsAutoupdated:    c.Bool("is-autoupdated"),
-		IsFreeTunnel:     isFreeTunnel,
 		LBPool:           c.String("lb-pool"),
 		Tags:             tags,
 		Log:              log,

cmd/cloudflared/tunnel/ingress_subcommands.go

@@ -45,7 +45,7 @@ func buildIngressSubcommand() *cli.Command {
 func buildValidateIngressCommand() *cli.Command {
 	return &cli.Command{
 		Name:        "validate",
-		Action:      cliutil.ConfiguredAction(validateIngressCommand),
+		Action:      cliutil.ConfiguredActionWithWarnings(validateIngressCommand),
 		Usage:       "Validate the ingress configuration ",
 		UsageText:   "cloudflared tunnel [--config FILEPATH] ingress validate",
 		Description: "Validates the configuration file, ensuring your ingress rules are OK.",
@@ -68,7 +68,7 @@ func buildTestURLCommand() *cli.Command {
 }
 
 // validateIngressCommand check the syntax of the ingress rules in the cloudflared config file
-func validateIngressCommand(c *cli.Context) error {
+func validateIngressCommand(c *cli.Context, warnings string) error {
 	conf := config.GetConfiguration()
 	if conf.Source() == "" {
 		fmt.Println("No configuration file was found. Please create one, or use the --config flag to specify its filepath. You can use the help command to learn more about configuration files")
@@ -81,6 +81,11 @@ func validateIngressCommand(c *cli.Context) error {
 	if c.IsSet("url") {
 		return ingress.ErrURLIncompatibleWithIngress
 	}
+	if warnings != "" {
+		fmt.Println("Warning: unused keys detected in your config file. Here is a list of unused keys:")
+		fmt.Println(warnings)
+		return nil
+	}
 	fmt.Println("OK")
 	return nil
 }

cmd/cloudflared/tunnel/login.go

@@ -20,7 +20,7 @@ import (
 
 const (
 	baseLoginURL     = "https://dash.cloudflare.com/argotunnel"
-	callbackStoreURL = "https://login.argotunnel.com/"
+	callbackStoreURL = "https://login.cloudflareaccess.org/"
 )
 
 func buildLoginSubcommand(hidden bool) *cli.Command {

cmd/cloudflared/tunnel/quick_tunnel.go

@@ -0,0 +1,111 @@
+package tunnel
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/pkg/errors"
+
+	"github.com/cloudflare/cloudflared/connection"
+)
+
+const httpTimeout = 15 * time.Second
+
+// RunQuickTunnel requests a tunnel from the specified service.
+// We use this to power quick tunnels on trycloudflare.com, but the
+// service is open-source and could be used by anyone.
+func RunQuickTunnel(sc *subcommandContext) error {
+	sc.log.Info().Msg("Requesting new Quick Tunnel...")
+
+	client := http.Client{
+		Transport: &http.Transport{
+			TLSHandshakeTimeout:   httpTimeout,
+			ResponseHeaderTimeout: httpTimeout,
+		},
+		Timeout: httpTimeout,
+	}
+
+	resp, err := client.Post(fmt.Sprintf("%s/tunnel", sc.c.String("quick-service")), "application/json", nil)
+	if err != nil {
+		return errors.Wrap(err, "failed to request quick tunnel")
+	}
+	defer resp.Body.Close()
+
+	var data QuickTunnelResponse
+	if err := json.NewDecoder(resp.Body).Decode(&data); err != nil {
+		return errors.Wrap(err, "failed to unmarshal quick tunnel")
+	}
+
+	tunnelID, err := uuid.Parse(data.Result.ID)
+	if err != nil {
+		return errors.Wrap(err, "failed to parse quick tunnel ID")
+	}
+
+	credentials := connection.Credentials{
+		AccountTag:   data.Result.AccountTag,
+		TunnelSecret: data.Result.Secret,
+		TunnelID:     tunnelID,
+		TunnelName:   data.Result.Name,
+	}
+
+	for _, line := range AsciiBox([]string{
+		"Your Quick Tunnel has been created! Visit it at:",
+		data.Result.Hostname,
+	}, 2) {
+		sc.log.Info().Msg(line)
+	}
+
+	return StartServer(
+		sc.c,
+		version,
+		&connection.NamedTunnelConfig{Credentials: credentials, QuickTunnelUrl: data.Result.Hostname},
+		sc.log,
+		sc.isUIEnabled,
+	)
+}
+
+type QuickTunnelResponse struct {
+	Success bool
+	Result  QuickTunnel
+	Errors  []QuickTunnelError
+}
+
+type QuickTunnelError struct {
+	Code    int
+	Message string
+}
+
+type QuickTunnel struct {
+	ID         string `json:"id"`
+	Name       string `json:"name"`
+	Hostname   string `json:"hostname"`
+	AccountTag string `json:"account_tag"`
+	Secret     []byte `json:"secret"`
+}
+
+// Print out the given lines in a nice ASCII box.
+func AsciiBox(lines []string, padding int) (box []string) {
+	maxLen := maxLen(lines)
+	spacer := strings.Repeat(" ", padding)
+	border := "+" + strings.Repeat("-", maxLen+(padding*2)) + "+"
+	box = append(box, border)
+	for _, line := range lines {
+		box = append(box, "|"+spacer+line+strings.Repeat(" ", maxLen-len(line))+spacer+"|")
+	}
+	box = append(box, border)
+	return
+}
+
+func maxLen(lines []string) int {
+	max := 0
+	for _, line := range lines {
+		if len(line) > max {
+			max = len(line)
+		}
+	}
+	return max
+}

cmd/cloudflared/tunnel/signal_test.go

@@ -1,3 +1,5 @@
+// +build !windows
+
 package tunnel
 
 import (

cmd/cloudflared/tunnel/subcommands.go

@@ -29,10 +29,11 @@ import (
 )
 
 const (
-	allSortByOptions   = "name, id, createdAt, deletedAt, numConnections"
-	connsSortByOptions = "id, startedAt, numConnections, version"
-	CredFileFlagAlias  = "cred-file"
-	CredFileFlag       = "credentials-file"
+	allSortByOptions     = "name, id, createdAt, deletedAt, numConnections"
+	connsSortByOptions   = "id, startedAt, numConnections, version"
+	CredFileFlagAlias    = "cred-file"
+	CredFileFlag         = "credentials-file"
+	overwriteDNSFlagName = "overwrite-dns"
 
 	LogFieldTunnelID = "tunnelID"
 )
@@ -110,7 +111,7 @@ var (
 	}
 	selectProtocolFlag = altsrc.NewStringFlag(&cli.StringFlag{
 		Name:    "protocol",
-		Value:   "h2mux",
+		Value:   "auto",
 		Aliases: []string{"p"},
 		Usage:   fmt.Sprintf("Protocol implementation to connect with Cloudflare's edge network. %s", connection.AvailableProtocolFlagMessage),
 		EnvVars: []string{"TUNNEL_TRANSPORT_PROTOCOL"},
@@ -133,6 +134,12 @@ var (
 		Usage:   `Constraints the cleanup to stop the connections of a single Connector (by its ID). You can find the various Connectors (and their IDs) currently connected to your tunnel via 'cloudflared tunnel info <name>'.`,
 		EnvVars: []string{"TUNNEL_CLEANUP_CONNECTOR"},
 	}
+	overwriteDNSFlag = &cli.BoolFlag{
+		Name:    overwriteDNSFlagName,
+		Aliases: []string{"f"},
+		Usage:   `Overwrites existing DNS records with this hostname`,
+		EnvVars: []string{"TUNNEL_FORCE_PROVISIONING_DNS"},
+	}
 )
 
 func buildCreateCommand() *cli.Command {
@@ -368,8 +375,8 @@ func tunnelInfo(c *cli.Context) error {
 	warningChecker := updater.StartWarningCheck(c)
 	defer warningChecker.LogWarningIfAny(sc.log)
 
-	if c.NArg() > 1 {
-		return cliutil.UsageError(`"cloudflared tunnel info" accepts only one argument, the ID or name of the tunnel to run.`)
+	if c.NArg() != 1 {
+		return cliutil.UsageError(`"cloudflared tunnel info" accepts exactly one argument, the ID or name of the tunnel to get info about.`)
 	}
 	tunnelID, err := sc.findID(c.Args().First())
 	if err != nil {
@@ -454,14 +461,30 @@ func formatAndPrintConnectionsList(tunnelInfo Info, showRecentlyDisconnected boo
 	writer := tabWriter()
 	defer writer.Flush()
 
+	// Print the general tunnel info table
 	_, _ = fmt.Fprintf(writer, "NAME:     %s\nID:       %s\nCREATED:  %s\n\n", tunnelInfo.Name, tunnelInfo.ID, tunnelInfo.CreatedAt)
 
+	// Determine whether to print the connector table
+	shouldDisplayTable := false
+	for _, c := range tunnelInfo.Connectors {
+		conns := fmtConnections(c.Connections, showRecentlyDisconnected)
+		if len(conns) > 0 {
+			shouldDisplayTable = true
+		}
+	}
+	if !shouldDisplayTable {
+		fmt.Println("This tunnel has no active connectors.")
+		return
+	}
+
+	// Print the connector table
 	_, _ = fmt.Fprintln(writer, "CONNECTOR ID\tCREATED\tARCHITECTURE\tVERSION\tORIGIN IP\tEDGE\t")
 	for _, c := range tunnelInfo.Connectors {
-		var originIp = ""
-		if len(c.Connections) > 0 {
-			originIp = c.Connections[0].OriginIP.String()
+		conns := fmtConnections(c.Connections, showRecentlyDisconnected)
+		if len(conns) == 0 {
+			continue
 		}
+		originIp := c.Connections[0].OriginIP.String()
 		formattedStr := fmt.Sprintf(
 			"%s\t%s\t%s\t%s\t%s\t%s\t",
 			c.ID,
@@ -469,7 +492,7 @@ func formatAndPrintConnectionsList(tunnelInfo Info, showRecentlyDisconnected boo
 			c.Arch,
 			c.Version,
 			originIp,
-			fmtConnections(c.Connections, showRecentlyDisconnected),
+			conns,
 		)
 		_, _ = fmt.Fprintln(writer, formattedStr)
 	}
@@ -579,6 +602,12 @@ func runCommand(c *cli.Context) error {
 		}
 	}
 
+	if c.String("hostname") != "" {
+		sc.log.Warn().Msg("The property `hostname` in your configuration is ignored because you configured a Named Tunnel " +
+			"in the property `tunnel` to run. Make sure to provision the routing (e.g. via `cloudflared tunnel route dns/lb`) or else " +
+			"your origin will not be reachable. You should remove the `hostname` property to avoid this warning.")
+	}
+
 	return runNamedTunnel(sc, tunnelRef)
 }
 
@@ -626,7 +655,6 @@ func cleanupCommand(c *cli.Context) error {
 func buildRouteCommand() *cli.Command {
 	return &cli.Command{
 		Name:      "route",
-		Action:    cliutil.ConfiguredAction(routeCommand),
 		Usage:     "Define which traffic routed from Cloudflare edge to this tunnel: requests to a DNS hostname, to a Cloudflare Load Balancer, or traffic originating from Cloudflare WARP clients",
 		UsageText: "cloudflared tunnel [tunnel command options] route [subcommand options] [dns TUNNEL HOSTNAME]|[lb TUNNEL HOSTNAME LB-POOL]|[ip NETWORK TUNNEL]",
 		Description: `The route command defines how Cloudflare will proxy requests to this tunnel.
@@ -646,15 +674,30 @@ Further information about managing Cloudflare WARP traffic to your tunnel is ava
 `,
 		CustomHelpTemplate: commandHelpTemplate(),
 		Subcommands: []*cli.Command{
+			{
+				Name:        "dns",
+				Action:      cliutil.ConfiguredAction(routeDnsCommand),
+				Usage:       "Route a hostname by creating a DNS CNAME record to a tunnel",
+				UsageText:   "cloudflared tunnel route dns [TUNNEL] [HOSTNAME]",
+				Description: `Creates a DNS CNAME record hostname that points to the tunnel.`,
+				Flags:       []cli.Flag{overwriteDNSFlag},
+			},
+			{
+				Name:        "lb",
+				Action:      cliutil.ConfiguredAction(routeLbCommand),
+				Usage:       "Use this tunnel as a load balancer origin, creating pool and load balancer if necessary",
+				UsageText:   "cloudflared tunnel route dns [TUNNEL] [HOSTNAME] [LB-POOL]",
+				Description: `Creates Load Balancer with an origin pool that points to the tunnel.`,
+			},
 			buildRouteIPSubcommand(),
 		},
 	}
 }
 
-func dnsRouteFromArg(c *cli.Context) (tunnelstore.Route, error) {
+func dnsRouteFromArg(c *cli.Context, overwriteExisting bool) (tunnelstore.Route, error) {
 	const (
-		userHostnameIndex = 2
-		expectedNArgs     = 3
+		userHostnameIndex = 1
+		expectedNArgs     = 2
 	)
 	if c.NArg() != expectedNArgs {
 		return nil, cliutil.UsageError("Expected %d arguments, got %d", expectedNArgs, c.NArg())
@@ -665,14 +708,14 @@ func dnsRouteFromArg(c *cli.Context) (tunnelstore.Route, error) {
 	} else if !validateHostname(userHostname, true) {
 		return nil, errors.Errorf("%s is not a valid hostname", userHostname)
 	}
-	return tunnelstore.NewDNSRoute(userHostname), nil
+	return tunnelstore.NewDNSRoute(userHostname, overwriteExisting), nil
 }
 
 func lbRouteFromArg(c *cli.Context) (tunnelstore.Route, error) {
 	const (
-		lbNameIndex   = 2
-		lbPoolIndex   = 3
-		expectedNArgs = 4
+		lbNameIndex   = 1
+		lbPoolIndex   = 2
+		expectedNArgs = 3
 	)
 	if c.NArg() != expectedNArgs {
 		return nil, cliutil.UsageError("Expected %d arguments, got %d", expectedNArgs, c.NArg())
@@ -714,41 +757,39 @@ func validateHostname(s string, allowWildcardSubdomain bool) bool {
 	return err == nil && validateName(puny, allowWildcardSubdomain)
 }
 
-func routeCommand(c *cli.Context) error {
-	if c.NArg() < 2 {
-		return cliutil.UsageError(`"cloudflared tunnel route" requires the first argument to be the route type(dns or lb), followed by the ID or name of the tunnel`)
+func routeDnsCommand(c *cli.Context) error {
+	if c.NArg() != 2 {
+		return cliutil.UsageError(`This command expects the format "cloudflared tunnel route dns <tunnel name/id> <hostname>"`)
 	}
+	return routeCommand(c, "dns")
+}
+
+func routeLbCommand(c *cli.Context) error {
+	if c.NArg() != 3 {
+		return cliutil.UsageError(`This command expects the format "cloudflared tunnel route lb <tunnel name/id> <hostname> <load balancer pool>"`)
+	}
+	return routeCommand(c, "lb")
+}
+
+func routeCommand(c *cli.Context, routeType string) error {
 	sc, err := newSubcommandContext(c)
 	if err != nil {
 		return err
 	}
 
-	const tunnelIDIndex = 1
-
-	routeType := c.Args().First()
+	tunnelID, err := sc.findID(c.Args().Get(0))
+	if err != nil {
+		return err
+	}
 	var route tunnelstore.Route
-	var tunnelID uuid.UUID
 	switch routeType {
 	case "dns":
-		tunnelID, err = sc.findID(c.Args().Get(tunnelIDIndex))
-		if err != nil {
-			return err
-		}
-		route, err = dnsRouteFromArg(c)
-		if err != nil {
-			return err
-		}
+		route, err = dnsRouteFromArg(c, c.Bool(overwriteDNSFlagName))
 	case "lb":
-		tunnelID, err = sc.findID(c.Args().Get(tunnelIDIndex))
-		if err != nil {
-			return err
-		}
 		route, err = lbRouteFromArg(c)
-		if err != nil {
-			return err
-		}
-	default:
-		return cliutil.UsageError("%s is not a recognized route type. Supported route types are dns and lb", routeType)
+	}
+	if err != nil {
+		return err
 	}
 
 	res, err := sc.route(tunnelID, route)

cmd/cloudflared/tunnel/subcommands_test.go

@@ -1,7 +1,6 @@
 package tunnel
 
 import (
-	"fmt"
 	"path/filepath"
 	"testing"
 
@@ -95,7 +94,7 @@ func TestTunnelfilePath(t *testing.T) {
 	assert.NoError(t, err)
 	homeDir, err := homedir.Dir()
 	assert.NoError(t, err)
-	expected := fmt.Sprintf("%s/.cloudflared/%v.json", homeDir, tunnelID)
+	expected := filepath.Join(homeDir, ".cloudflared", tunnelID.String()+".json")
 	assert.Equal(t, expected, actual)
 }
 

cmd/cloudflared/updater/workers_service_test.go

@@ -1,3 +1,5 @@
+// +build !windows
+
 package updater
 
 import (
@@ -13,6 +15,7 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"os"
+	"path/filepath"
 	"runtime"
 	"strconv"
 	"strings"
@@ -21,6 +24,8 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+var testFilePath = filepath.Join(os.TempDir(), "test")
+
 func respondWithJSON(w http.ResponseWriter, v interface{}, status int) {
 	data, _ := json.Marshal(v)
 
@@ -208,7 +213,6 @@ func TestUpdateService(t *testing.T) {
 	ts := createServer()
 	defer ts.Close()
 
-	testFilePath := "tmpfile"
 	createTestFile(t, testFilePath)
 	defer os.Remove(testFilePath)
 	log.Println("server url: ", ts.URL)
@@ -229,7 +233,6 @@ func TestBetaUpdateService(t *testing.T) {
 	ts := createServer()
 	defer ts.Close()
 
-	testFilePath := "tmpfile"
 	createTestFile(t, testFilePath)
 	defer os.Remove(testFilePath)
 
@@ -249,7 +252,6 @@ func TestFailUpdateService(t *testing.T) {
 	ts := createServer()
 	defer ts.Close()
 
-	testFilePath := "tmpfile"
 	createTestFile(t, testFilePath)
 	defer os.Remove(testFilePath)
 
@@ -263,7 +265,6 @@ func TestNoUpdateService(t *testing.T) {
 	ts := createServer()
 	defer ts.Close()
 
-	testFilePath := "tmpfile"
 	createTestFile(t, testFilePath)
 	defer os.Remove(testFilePath)
 
@@ -278,7 +279,6 @@ func TestForcedUpdateService(t *testing.T) {
 	ts := createServer()
 	defer ts.Close()
 
-	testFilePath := "tmpfile"
 	createTestFile(t, testFilePath)
 	defer os.Remove(testFilePath)
 
@@ -298,7 +298,6 @@ func TestUpdateSpecificVersionService(t *testing.T) {
 	ts := createServer()
 	defer ts.Close()
 
-	testFilePath := "tmpfile"
 	createTestFile(t, testFilePath)
 	defer os.Remove(testFilePath)
 	reqVersion := "2020.9.1"
@@ -319,7 +318,6 @@ func TestCompressedUpdateService(t *testing.T) {
 	ts := createServer()
 	defer ts.Close()
 
-	testFilePath := "tmpfile"
 	createTestFile(t, testFilePath)
 	defer os.Remove(testFilePath)
 
@@ -339,7 +337,6 @@ func TestUpdateWhenRunningKnownBuggyVersion(t *testing.T) {
 	ts := createServer()
 	defer ts.Close()
 
-	testFilePath := "tmpfile"
 	createTestFile(t, testFilePath)
 	defer os.Remove(testFilePath)
 

component-tests/conftest.py

@@ -20,11 +20,21 @@ def component_tests_config():
         LOGGER.info(f"component tests base config {config}")
 
         def _component_tests_config(additional_config={}, named_tunnel=True):
+
+            # Regression test for TUN-4177, running with proxy-dns should not prevent tunnels from running
+            additional_config["proxy-dns"] = True
+            additional_config["proxy-dns-port"] = 9053
+
             if named_tunnel:
                 return NamedTunnelConfig(additional_config=additional_config,
-                                         cloudflared_binary=config['cloudflared_binary'], tunnel=config['tunnel'], credentials_file=config['credentials_file'],  ingress=config['ingress'])
+                                         cloudflared_binary=config['cloudflared_binary'],
+                                         tunnel=config['tunnel'],
+                                         credentials_file=config['credentials_file'],
+                                         ingress=config['ingress'])
+
             return ClassicTunnelConfig(
-                additional_config=additional_config, cloudflared_binary=config['cloudflared_binary'], hostname=config['classic_hostname'], origincert=config['origincert'])
+                additional_config=additional_config, cloudflared_binary=config['cloudflared_binary'],
+                hostname=config['classic_hostname'], origincert=config['origincert'])
 
         return _component_tests_config
 

component-tests/test_logging.py

@@ -7,21 +7,17 @@ from util import start_cloudflared, wait_tunnel_ready, send_requests
 # Rolling logger rotate log files after 1 MB
 rotate_after_size = 1000 * 1000
 default_log_file = "cloudflared.log"
-expect_message = "Starting tunnel"
+expect_message = "Starting Hello"
 
 
 def assert_log_to_terminal(cloudflared):
-    stderr = cloudflared.stderr.read(200)
-    # cloudflared logs the following when it first starts
-    # 2021-03-10T12:30:39Z INF Starting tunnel tunnelID=<tunnel ID>
+    stderr = cloudflared.stderr.read(1500)
     assert expect_message.encode() in stderr, f"{stderr} doesn't contain {expect_message}"
 
 
 def assert_log_in_file(file):
     with open(file, "r") as f:
-        log = f.read(200)
-        # cloudflared logs the following when it first starts
-        # {"level":"info","tunnelID":"<tunnel ID>","time":"2021-03-10T12:21:13Z","message":"Starting tunnel"}
+        log = f.read(1850)
         assert expect_message in log, f"{log} doesn't contain {expect_message}"
 
 

component-tests/test_reconnect.py

@@ -1,32 +1,36 @@
 #!/usr/bin/env python
 import copy
+import platform
 from time import sleep
 
+import pytest
 from flaky import flaky
 
 from util import start_cloudflared, wait_tunnel_ready, check_tunnel_not_connected
 
 
 @flaky(max_runs=3, min_passes=1)
-class TestReconnect():
+class TestReconnect:
     default_ha_conns = 4
-    default_reconnect_secs = 5
+    default_reconnect_secs = 15
     extra_config = {
         "stdin-control": True,
     }
 
+    @pytest.mark.skipif(platform.system() == "Windows", reason=f"Currently buggy on Windows TUN-4584")
     def test_named_reconnect(self, tmp_path, component_tests_config):
         config = component_tests_config(self.extra_config)
         with start_cloudflared(tmp_path, config, new_process=True, allow_input=True, capture_output=False) as cloudflared:
             # Repeat the test multiple times because some issues only occur after multiple reconnects
             self.assert_reconnect(config, cloudflared, 5)
 
+    @pytest.mark.skipif(platform.system() == "Windows", reason=f"Currently buggy on Windows TUN-4699")
     def test_classic_reconnect(self, tmp_path, component_tests_config):
         extra_config = copy.copy(self.extra_config)
         extra_config["hello-world"] = True
         config = component_tests_config(
             additional_config=extra_config, named_tunnel=False)
-        with start_cloudflared(tmp_path, config, cfd_args=[], new_process=True, allow_input=True, capture_output=False) as cloudflared:
+        with start_cloudflared(tmp_path, config, cfd_args=[], new_process=True, allow_input=True, capture_output=True) as cloudflared:
             self.assert_reconnect(config, cloudflared, 1)
 
     def send_reconnect(self, cloudflared, secs):
@@ -36,7 +40,7 @@ class TestReconnect():
         cloudflared.stdin.flush()
 
     def assert_reconnect(self, config, cloudflared, repeat):
-        wait_tunnel_ready(tunnel_url=config.get_url())
+        wait_tunnel_ready(tunnel_url=config.get_url(), require_min_connections=self.default_ha_conns)
         for _ in range(repeat):
             for i in range(self.default_ha_conns):
                 self.send_reconnect(cloudflared, self.default_reconnect_secs)
@@ -44,11 +48,9 @@ class TestReconnect():
                 if expect_connections > 0:
                     # Don't check if tunnel returns 200 here because there is a race condition between wait_tunnel_ready
                     # retrying to get 200 response and reconnecting
-                    wait_tunnel_ready(
-                        require_min_connections=expect_connections)
+                    wait_tunnel_ready(require_min_connections=expect_connections)
                 else:
                     check_tunnel_not_connected()
 
-            sleep(self.default_reconnect_secs + 10)
-            wait_tunnel_ready(tunnel_url=config.get_url(),
-                              require_min_connections=self.default_ha_conns)
+            sleep(self.default_reconnect_secs * 2)
+            wait_tunnel_ready(tunnel_url=config.get_url(), require_min_connections=self.default_ha_conns)

component-tests/util.py

@@ -34,7 +34,7 @@ def cloudflared_cmd(config, config_path, cfd_args, cfd_pre_args, root):
         cmd += ["sudo"]
     cmd += [config.cloudflared_binary]
     cmd += cfd_pre_args
-    cmd += ["--config", config_path]
+    cmd += ["--config", str(config_path)]
     cmd += cfd_args
     LOGGER.info(f"Run cmd {cmd} with config {config}")
     return cmd
@@ -50,8 +50,7 @@ def run_cloudflared_background(cmd, allow_input, capture_output):
     finally:
         cfd.terminate()
         if capture_output:
-            LOGGER.info(
-                f"cloudflared log: {cfd.stderr.read()}")
+            LOGGER.info(f"cloudflared log: {cfd.stderr.read()}")
 
 
 @retry(stop_max_attempt_number=MAX_RETRIES, wait_fixed=BACKOFF_SECS * 1000)

config/configuration.go

@@ -358,13 +358,13 @@ func GetConfiguration() *Configuration {
 // ReadConfigFile returns InputSourceContext initialized from the configuration file.
 // On repeat calls returns with the same file, returns without reading the file again; however,
 // if value of "config" flag changes, will read the new config file
-func ReadConfigFile(c *cli.Context, log *zerolog.Logger) (*configFileSettings, error) {
+func ReadConfigFile(c *cli.Context, log *zerolog.Logger) (settings *configFileSettings, warnings string, err error) {
 	configFile := c.String("config")
 	if configuration.Source() == configFile || configFile == "" {
 		if configuration.Source() == "" {
-			return nil, ErrNoConfigFile
+			return nil, "", ErrNoConfigFile
 		}
-		return &configuration, nil
+		return &configuration, "", nil
 	}
 
 	log.Debug().Msgf("Loading configuration from %s", configFile)
@@ -373,16 +373,27 @@ func ReadConfigFile(c *cli.Context, log *zerolog.Logger) (*configFileSettings, e
 		if os.IsNotExist(err) {
 			err = ErrNoConfigFile
 		}
-		return nil, err
+		return nil, "", err
 	}
 	defer file.Close()
 	if err := yaml.NewDecoder(file).Decode(&configuration); err != nil {
 		if err == io.EOF {
 			log.Error().Msgf("Configuration file %s was empty", configFile)
-			return &configuration, nil
+			return &configuration, "", nil
 		}
-		return nil, errors.Wrap(err, "error parsing YAML in config file at "+configFile)
+		return nil, "", errors.Wrap(err, "error parsing YAML in config file at "+configFile)
 	}
 	configuration.sourceFile = configFile
-	return &configuration, nil
+
+	// Parse it again, with strict mode, to find warnings.
+	if file, err := os.Open(configFile); err == nil {
+		decoder := yaml.NewDecoder(file)
+		decoder.SetStrict(true)
+		var unusedConfig configFileSettings
+		if err := decoder.Decode(&unusedConfig); err != nil {
+			warnings = err.Error()
+		}
+	}
+
+	return &configuration, warnings, nil
 }

connection/connection.go

@@ -1,6 +1,7 @@
 package connection
 
 import (
+	"context"
 	"fmt"
 	"io"
 	"net/http"
@@ -11,9 +12,15 @@ import (
 	"github.com/google/uuid"
 
 	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/cloudflare/cloudflared/websocket"
 )
 
-const LogFieldConnIndex = "connIndex"
+const (
+	lbProbeUserAgentPrefix = "Mozilla/5.0 (compatible; Cloudflare-Traffic-Manager/1.0; +https://www.cloudflare.com/traffic-manager/;"
+	LogFieldConnIndex      = "connIndex"
+)
+
+var switchingProtocolText = fmt.Sprintf("%d %s", http.StatusSwitchingProtocols, http.StatusText(http.StatusSwitchingProtocols))
 
 type Config struct {
 	OriginProxy     OriginProxy
@@ -22,8 +29,9 @@ type Config struct {
 }
 
 type NamedTunnelConfig struct {
-	Credentials Credentials
-	Client      pogs.ClientInfo
+	Credentials    Credentials
+	Client         pogs.ClientInfo
+	QuickTunnelUrl string
 }
 
 // Credentials are stored in the credentials file and contain all info needed to run a tunnel.
@@ -48,10 +56,6 @@ type ClassicTunnelConfig struct {
 	UseReconnectToken bool
 }
 
-func (c *ClassicTunnelConfig) IsTrialZone() bool {
-	return c.Hostname == ""
-}
-
 // Type indicates the connection type of the  connection.
 type Type int
 
@@ -87,9 +91,64 @@ func (t Type) String() string {
 	}
 }
 
+// OriginProxy is how data flows from cloudflared to the origin services running behind it.
 type OriginProxy interface {
-	// If Proxy returns an error, the caller is responsible for writing the error status to ResponseWriter
-	Proxy(w ResponseWriter, req *http.Request, sourceConnectionType Type) error
+	ProxyHTTP(w ResponseWriter, req *http.Request, isWebsocket bool) error
+	ProxyTCP(ctx context.Context, rwa ReadWriteAcker, req *TCPRequest) error
+}
+
+// TCPRequest defines the input format needed to perform a TCP proxy.
+type TCPRequest struct {
+	Dest    string
+	CFRay   string
+	LBProbe bool
+}
+
+// ReadWriteAcker is a readwriter with the ability to Acknowledge to the downstream (edge) that the origin has
+// accepted the connection.
+type ReadWriteAcker interface {
+	io.ReadWriter
+	AckConnection() error
+}
+
+// HTTPResponseReadWriteAcker is an HTTP implementation of ReadWriteAcker.
+type HTTPResponseReadWriteAcker struct {
+	r   io.Reader
+	w   ResponseWriter
+	req *http.Request
+}
+
+// NewHTTPResponseReadWriterAcker returns a new instance of HTTPResponseReadWriteAcker.
+func NewHTTPResponseReadWriterAcker(w ResponseWriter, req *http.Request) *HTTPResponseReadWriteAcker {
+	return &HTTPResponseReadWriteAcker{
+		r:   req.Body,
+		w:   w,
+		req: req,
+	}
+}
+
+func (h *HTTPResponseReadWriteAcker) Read(p []byte) (int, error) {
+	return h.r.Read(p)
+}
+
+func (h *HTTPResponseReadWriteAcker) Write(p []byte) (int, error) {
+	return h.w.Write(p)
+}
+
+// AckConnection acks an HTTP connection by sending a switch protocols status code that enables the caller to
+// upgrade to streams.
+func (h *HTTPResponseReadWriteAcker) AckConnection() error {
+	resp := &http.Response{
+		Status:        switchingProtocolText,
+		StatusCode:    http.StatusSwitchingProtocols,
+		ContentLength: -1,
+	}
+
+	if secWebsocketKey := h.req.Header.Get("Sec-WebSocket-Key"); secWebsocketKey != "" {
+		resp.Header = websocket.NewResponseHeader(h.req)
+	}
+
+	return h.w.WriteRespHeaders(resp.StatusCode, resp.Header)
 }
 
 type ResponseWriter interface {
@@ -112,3 +171,11 @@ func IsServerSentEvent(headers http.Header) bool {
 func uint8ToString(input uint8) string {
 	return strconv.FormatUint(uint64(input), 10)
 }
+
+func FindCfRayHeader(req *http.Request) string {
+	return req.Header.Get("Cf-Ray")
+}
+
+func IsLBProbeRequest(req *http.Request) bool {
+	return strings.HasPrefix(req.UserAgent(), lbProbeUserAgentPrefix)
+}

connection/connection_test.go

@@ -1,6 +1,7 @@
 package connection
 
 import (
+	"context"
 	"fmt"
 	"io"
 	"net/http"
@@ -11,6 +12,8 @@ import (
 	"github.com/gobwas/ws/wsutil"
 	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/assert"
+
+	"github.com/cloudflare/cloudflared/ingress"
 )
 
 const (
@@ -18,7 +21,8 @@ const (
 )
 
 var (
-	testConfig = &Config{
+	unusedWarpRoutingService = (*ingress.WarpRoutingService)(nil)
+	testConfig               = &Config{
 		OriginProxy: &mockOriginProxy{},
 		GracePeriod: time.Millisecond * 100,
 	}
@@ -38,14 +42,17 @@ type testRequest struct {
 	isProxyError   bool
 }
 
-type mockOriginProxy struct {
-}
+type mockOriginProxy struct{}
 
-func (moc *mockOriginProxy) Proxy(w ResponseWriter, r *http.Request, sourceConnectionType Type) error {
-	if sourceConnectionType == TypeWebsocket {
-		return wsEndpoint(w, r)
+func (moc *mockOriginProxy) ProxyHTTP(
+	w ResponseWriter,
+	req *http.Request,
+	isWebsocket bool,
+) error {
+	if isWebsocket {
+		return wsEndpoint(w, req)
 	}
-	switch r.URL.Path {
+	switch req.URL.Path {
 	case "/ok":
 		originRespEndpoint(w, http.StatusOK, []byte(http.StatusText(http.StatusOK)))
 	case "/large_file":
@@ -60,6 +67,15 @@ func (moc *mockOriginProxy) Proxy(w ResponseWriter, r *http.Request, sourceConne
 		originRespEndpoint(w, http.StatusNotFound, []byte("page not found"))
 	}
 	return nil
+
+}
+
+func (moc *mockOriginProxy) ProxyTCP(
+	ctx context.Context,
+	rwa ReadWriteAcker,
+	r *TCPRequest,
+) error {
+	return nil
 }
 
 type nowriter struct {

connection/event.go

@@ -18,7 +18,7 @@ const (
 	Connected
 	// Reconnecting means the connection to the edge is being re-established.
 	Reconnecting
-	// SetURL means this connection's tunnel was given a URL by the edge. Used for free tunnels.
+	// SetURL means this connection's tunnel was given a URL by the edge. Used for quick tunnels.
 	SetURL
 	// RegisteringTunnel means the non-named tunnel is registering its connection.
 	RegisteringTunnel

connection/h2mux.go

@@ -33,6 +33,8 @@ type h2muxConnection struct {
 	gracefulShutdownC <-chan struct{}
 	stoppedGracefully bool
 
+	log *zerolog.Logger
+
 	// newRPCClientFunc allows us to mock RPCs during testing
 	newRPCClientFunc func(context.Context, io.ReadWriteCloser, *zerolog.Logger) NamedTunnelRPCClient
 }
@@ -167,7 +169,8 @@ func (h *h2muxConnection) serveMuxer(ctx context.Context) error {
 }
 
 func (h *h2muxConnection) controlLoop(ctx context.Context, connectedFuse ConnectedFuse, isNamedTunnel bool) {
-	updateMetricsTickC := time.Tick(h.muxerConfig.MetricsUpdateFreq)
+	updateMetricsTicker := time.NewTicker(h.muxerConfig.MetricsUpdateFreq)
+	defer updateMetricsTicker.Stop()
 	var shutdownCompleted <-chan struct{}
 	for {
 		select {
@@ -191,7 +194,7 @@ func (h *h2muxConnection) controlLoop(ctx context.Context, connectedFuse Connect
 			// don't wait for shutdown to finish when context is closed, this is the hard termination path
 			return
 
-		case <-updateMetricsTickC:
+		case <-updateMetricsTicker.C:
 			h.observer.metrics.updateMuxerMetrics(h.connIndexStr, h.muxer.Metrics())
 		}
 	}
@@ -221,12 +224,11 @@ func (h *h2muxConnection) ServeStream(stream *h2mux.MuxedStream) error {
 		sourceConnectionType = TypeWebsocket
 	}
 
-	err := h.config.OriginProxy.Proxy(respWriter, req, sourceConnectionType)
+	err := h.config.OriginProxy.ProxyHTTP(respWriter, req, sourceConnectionType == TypeWebsocket)
 	if err != nil {
 		respWriter.WriteErrorResponse()
-		return err
 	}
-	return nil
+	return err
 }
 
 func (h *h2muxConnection) newRequest(stream *h2mux.MuxedStream) (*http.Request, error) {
@@ -234,7 +236,7 @@ func (h *h2muxConnection) newRequest(stream *h2mux.MuxedStream) (*http.Request,
 	if err != nil {
 		return nil, errors.Wrap(err, "Unexpected error from http.NewRequest")
 	}
-	err = h2mux.H2RequestHeadersToH1Request(stream.Headers, req)
+	err = H2RequestHeadersToH1Request(stream.Headers, req)
 	if err != nil {
 		return nil, errors.Wrap(err, "invalid request received")
 	}
@@ -246,15 +248,15 @@ type h2muxRespWriter struct {
 }
 
 func (rp *h2muxRespWriter) WriteRespHeaders(status int, header http.Header) error {
-	headers := h2mux.H1ResponseToH2ResponseHeaders(status, header)
-	headers = append(headers, h2mux.Header{Name: ResponseMetaHeaderField, Value: responseMetaHeaderOrigin})
+	headers := H1ResponseToH2ResponseHeaders(status, header)
+	headers = append(headers, h2mux.Header{Name: ResponseMetaHeader, Value: responseMetaHeaderOrigin})
 	return rp.WriteHeaders(headers)
 }
 
 func (rp *h2muxRespWriter) WriteErrorResponse() {
 	_ = rp.WriteHeaders([]h2mux.Header{
 		{Name: ":status", Value: "502"},
-		{Name: ResponseMetaHeaderField, Value: responseMetaHeaderCfd},
+		{Name: ResponseMetaHeader, Value: responseMetaHeaderCfd},
 	})
 	_, _ = rp.Write([]byte("502 Bad Gateway"))
 }

connection/h2mux_test.go

@@ -115,9 +115,9 @@ func TestServeStreamHTTP(t *testing.T) {
 		require.True(t, hasHeader(stream, ":status", strconv.Itoa(test.expectedStatus)))
 
 		if test.isProxyError {
-			assert.True(t, hasHeader(stream, ResponseMetaHeaderField, responseMetaHeaderCfd))
+			assert.True(t, hasHeader(stream, ResponseMetaHeader, responseMetaHeaderCfd))
 		} else {
-			assert.True(t, hasHeader(stream, ResponseMetaHeaderField, responseMetaHeaderOrigin))
+			assert.True(t, hasHeader(stream, ResponseMetaHeader, responseMetaHeaderOrigin))
 			body := make([]byte, len(test.expectedBody))
 			_, err = stream.Read(body)
 			require.NoError(t, err)
@@ -164,7 +164,7 @@ func TestServeStreamWS(t *testing.T) {
 	require.NoError(t, err)
 
 	require.True(t, hasHeader(stream, ":status", strconv.Itoa(http.StatusSwitchingProtocols)))
-	assert.True(t, hasHeader(stream, ResponseMetaHeaderField, responseMetaHeaderOrigin))
+	assert.True(t, hasHeader(stream, ResponseMetaHeader, responseMetaHeaderOrigin))
 
 	data := []byte("test websocket")
 	err = wsutil.WriteClientText(writePipe, data)
@@ -268,7 +268,7 @@ func benchmarkServeStreamHTTPSimple(b *testing.B, test testRequest) {
 		b.StopTimer()
 
 		require.NoError(b, openstreamErr)
-		assert.True(b, hasHeader(stream, ResponseMetaHeaderField, responseMetaHeaderOrigin))
+		assert.True(b, hasHeader(stream, ResponseMetaHeader, responseMetaHeaderOrigin))
 		require.True(b, hasHeader(stream, ":status", strconv.Itoa(http.StatusOK)))
 		require.NoError(b, readBodyErr)
 		require.Equal(b, test.expectedBody, body)

connection/header.go

@@ -1,21 +1,33 @@
 package connection
 
 import (
+	"encoding/base64"
 	"fmt"
 	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+
+	"github.com/pkg/errors"
 
 	"github.com/cloudflare/cloudflared/h2mux"
 )
 
-const (
-	ResponseMetaHeaderField = "cf-cloudflared-response-meta"
+var (
+	// h2mux-style special headers
+	RequestUserHeaders  = "cf-cloudflared-request-headers"
+	ResponseUserHeaders = "cf-cloudflared-response-headers"
+	ResponseMetaHeader  = "cf-cloudflared-response-meta"
+
+	// h2mux-style special headers
+	CanonicalResponseUserHeaders = http.CanonicalHeaderKey(ResponseUserHeaders)
+	CanonicalResponseMetaHeader  = http.CanonicalHeaderKey(ResponseMetaHeader)
 )
 
 var (
-	canonicalResponseUserHeadersField = http.CanonicalHeaderKey(h2mux.ResponseUserHeadersField)
-	canonicalResponseMetaHeaderField  = http.CanonicalHeaderKey(ResponseMetaHeaderField)
-	responseMetaHeaderCfd             = mustInitRespMetaHeader("cloudflared")
-	responseMetaHeaderOrigin          = mustInitRespMetaHeader("origin")
+	// pre-generate possible values for res
+	responseMetaHeaderCfd    = mustInitRespMetaHeader("cloudflared")
+	responseMetaHeaderOrigin = mustInitRespMetaHeader("origin")
 )
 
 type responseMetaHeader struct {
@@ -29,3 +41,204 @@ func mustInitRespMetaHeader(src string) string {
 	}
 	return string(header)
 }
+
+var headerEncoding = base64.RawStdEncoding
+
+// note: all h2mux headers should be lower-case (http/2 style)
+const ()
+
+// H2RequestHeadersToH1Request converts the HTTP/2 headers coming from origintunneld
+// to an HTTP/1 Request object destined for the local origin web service.
+// This operation includes conversion of the pseudo-headers into their closest
+// HTTP/1 equivalents. See https://tools.ietf.org/html/rfc7540#section-8.1.2.3
+func H2RequestHeadersToH1Request(h2 []h2mux.Header, h1 *http.Request) error {
+	for _, header := range h2 {
+		name := strings.ToLower(header.Name)
+		if !IsControlHeader(name) {
+			continue
+		}
+
+		switch name {
+		case ":method":
+			h1.Method = header.Value
+		case ":scheme":
+			// noop - use the preexisting scheme from h1.URL
+		case ":authority":
+			// Otherwise the host header will be based on the origin URL
+			h1.Host = header.Value
+		case ":path":
+			// We don't want to be an "opinionated" proxy, so ideally we would use :path as-is.
+			// However, this HTTP/1 Request object belongs to the Go standard library,
+			// whose URL package makes some opinionated decisions about the encoding of
+			// URL characters: see the docs of https://godoc.org/net/url#URL,
+			// in particular the EscapedPath method https://godoc.org/net/url#URL.EscapedPath,
+			// which is always used when computing url.URL.String(), whether we'd like it or not.
+			//
+			// Well, not *always*. We could circumvent this by using url.URL.Opaque. But
+			// that would present unusual difficulties when using an HTTP proxy: url.URL.Opaque
+			// is treated differently when HTTP_PROXY is set!
+			// See https://github.com/golang/go/issues/5684#issuecomment-66080888
+			//
+			// This means we are subject to the behavior of net/url's function `shouldEscape`
+			// (as invoked with mode=encodePath): https://github.com/golang/go/blob/go1.12.7/src/net/url/url.go#L101
+
+			if header.Value == "*" {
+				h1.URL.Path = "*"
+				continue
+			}
+			// Due to the behavior of validation.ValidateUrl, h1.URL may
+			// already have a partial value, with or without a trailing slash.
+			base := h1.URL.String()
+			base = strings.TrimRight(base, "/")
+			// But we know :path begins with '/', because we handled '*' above - see RFC7540
+			requestURL, err := url.Parse(base + header.Value)
+			if err != nil {
+				return errors.Wrap(err, fmt.Sprintf("invalid path '%v'", header.Value))
+			}
+			h1.URL = requestURL
+		case "content-length":
+			contentLength, err := strconv.ParseInt(header.Value, 10, 64)
+			if err != nil {
+				return fmt.Errorf("unparseable content length")
+			}
+			h1.ContentLength = contentLength
+		case RequestUserHeaders:
+			// Do not forward the serialized headers to the origin -- deserialize them, and ditch the serialized version
+			// Find and parse user headers serialized into a single one
+			userHeaders, err := DeserializeHeaders(header.Value)
+			if err != nil {
+				return errors.Wrap(err, "Unable to parse user headers")
+			}
+			for _, userHeader := range userHeaders {
+				h1.Header.Add(userHeader.Name, userHeader.Value)
+			}
+		default:
+			// All other control headers shall just be proxied transparently
+			h1.Header.Add(header.Name, header.Value)
+		}
+	}
+
+	return nil
+}
+
+func IsControlHeader(headerName string) bool {
+	return headerName == "content-length" ||
+		headerName == "connection" || headerName == "upgrade" || // Websocket headers
+		strings.HasPrefix(headerName, ":") ||
+		strings.HasPrefix(headerName, "cf-")
+}
+
+// isWebsocketClientHeader returns true if the header name is required by the client to upgrade properly
+func IsWebsocketClientHeader(headerName string) bool {
+	return headerName == "sec-websocket-accept" ||
+		headerName == "connection" ||
+		headerName == "upgrade"
+}
+
+func H1ResponseToH2ResponseHeaders(status int, h1 http.Header) (h2 []h2mux.Header) {
+	h2 = []h2mux.Header{
+		{Name: ":status", Value: strconv.Itoa(status)},
+	}
+	userHeaders := make(http.Header, len(h1))
+	for header, values := range h1 {
+		h2name := strings.ToLower(header)
+		if h2name == "content-length" {
+			// This header has meaning in HTTP/2 and will be used by the edge,
+			// so it should be sent as an HTTP/2 response header.
+
+			// Since these are http2 headers, they're required to be lowercase
+			h2 = append(h2, h2mux.Header{Name: "content-length", Value: values[0]})
+		} else if !IsControlHeader(h2name) || IsWebsocketClientHeader(h2name) {
+			// User headers, on the other hand, must all be serialized so that
+			// HTTP/2 header validation won't be applied to HTTP/1 header values
+			userHeaders[header] = values
+		}
+	}
+
+	// Perform user header serialization and set them in the single header
+	h2 = append(h2, h2mux.Header{Name: ResponseUserHeaders, Value: SerializeHeaders(userHeaders)})
+	return h2
+}
+
+// Serialize HTTP1.x headers by base64-encoding each header name and value,
+// and then joining them in the format of [key:value;]
+func SerializeHeaders(h1Headers http.Header) string {
+	// compute size of the fully serialized value and largest temp buffer we will need
+	serializedLen := 0
+	maxTempLen := 0
+	for headerName, headerValues := range h1Headers {
+		for _, headerValue := range headerValues {
+			nameLen := headerEncoding.EncodedLen(len(headerName))
+			valueLen := headerEncoding.EncodedLen(len(headerValue))
+			const delims = 2
+			serializedLen += delims + nameLen + valueLen
+			if nameLen > maxTempLen {
+				maxTempLen = nameLen
+			}
+			if valueLen > maxTempLen {
+				maxTempLen = valueLen
+			}
+		}
+	}
+	var buf strings.Builder
+	buf.Grow(serializedLen)
+
+	temp := make([]byte, maxTempLen)
+	writeB64 := func(s string) {
+		n := headerEncoding.EncodedLen(len(s))
+		if n > len(temp) {
+			temp = make([]byte, n)
+		}
+		headerEncoding.Encode(temp[:n], []byte(s))
+		buf.Write(temp[:n])
+	}
+
+	for headerName, headerValues := range h1Headers {
+		for _, headerValue := range headerValues {
+			if buf.Len() > 0 {
+				buf.WriteByte(';')
+			}
+			writeB64(headerName)
+			buf.WriteByte(':')
+			writeB64(headerValue)
+		}
+	}
+
+	return buf.String()
+}
+
+// Deserialize headers serialized by `SerializeHeader`
+func DeserializeHeaders(serializedHeaders string) ([]h2mux.Header, error) {
+	const unableToDeserializeErr = "Unable to deserialize headers"
+
+	var deserialized []h2mux.Header
+	for _, serializedPair := range strings.Split(serializedHeaders, ";") {
+		if len(serializedPair) == 0 {
+			continue
+		}
+
+		serializedHeaderParts := strings.Split(serializedPair, ":")
+		if len(serializedHeaderParts) != 2 {
+			return nil, errors.New(unableToDeserializeErr)
+		}
+
+		serializedName := serializedHeaderParts[0]
+		serializedValue := serializedHeaderParts[1]
+		deserializedName := make([]byte, headerEncoding.DecodedLen(len(serializedName)))
+		deserializedValue := make([]byte, headerEncoding.DecodedLen(len(serializedValue)))
+
+		if _, err := headerEncoding.Decode(deserializedName, []byte(serializedName)); err != nil {
+			return nil, errors.Wrap(err, unableToDeserializeErr)
+		}
+		if _, err := headerEncoding.Decode(deserializedValue, []byte(serializedValue)); err != nil {
+			return nil, errors.Wrap(err, unableToDeserializeErr)
+		}
+
+		deserialized = append(deserialized, h2mux.Header{
+			Name:  string(deserializedName),
+			Value: string(deserializedValue),
+		})
+	}
+
+	return deserialized, nil
+}

connection/header_test.go

@@ -1,4 +1,4 @@
-package h2mux
+package connection
 
 import (
 	"fmt"
@@ -14,9 +14,11 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+
+	"github.com/cloudflare/cloudflared/h2mux"
 )
 
-type ByName []Header
+type ByName []h2mux.Header
 
 func (a ByName) Len() int      { return len(a) }
 func (a ByName) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
@@ -37,16 +39,16 @@ func TestH2RequestHeadersToH1Request_RegularHeaders(t *testing.T) {
 		"Mock header 2": {"Mock value 2"},
 	}
 
-	headersConversionErr := H2RequestHeadersToH1Request(createSerializedHeaders(RequestUserHeadersField, mockHeaders), request)
+	headersConversionErr := H2RequestHeadersToH1Request(createSerializedHeaders(RequestUserHeaders, mockHeaders), request)
 
 	assert.True(t, reflect.DeepEqual(mockHeaders, request.Header))
 	assert.NoError(t, headersConversionErr)
 }
 
-func createSerializedHeaders(headersField string, headers http.Header) []Header {
-	return []Header{{
-		headersField,
-		SerializeHeaders(headers),
+func createSerializedHeaders(headersField string, headers http.Header) []h2mux.Header {
+	return []h2mux.Header{{
+		Name:  headersField,
+		Value: SerializeHeaders(headers),
 	}}
 }
 
@@ -54,15 +56,16 @@ func TestH2RequestHeadersToH1Request_NoHeaders(t *testing.T) {
 	request, err := http.NewRequest(http.MethodGet, "http://example.com", nil)
 	assert.NoError(t, err)
 
+	emptyHeaders := make(http.Header)
 	headersConversionErr := H2RequestHeadersToH1Request(
-		[]Header{{
-			RequestUserHeadersField,
-			SerializeHeaders(http.Header{}),
+		[]h2mux.Header{{
+			Name:  RequestUserHeaders,
+			Value: SerializeHeaders(emptyHeaders),
 		}},
 		request,
 	)
 
-	assert.True(t, reflect.DeepEqual(http.Header{}, request.Header))
+	assert.True(t, reflect.DeepEqual(emptyHeaders, request.Header))
 	assert.NoError(t, headersConversionErr)
 }
 
@@ -70,9 +73,9 @@ func TestH2RequestHeadersToH1Request_InvalidHostPath(t *testing.T) {
 	request, err := http.NewRequest(http.MethodGet, "http://example.com", nil)
 	assert.NoError(t, err)
 
-	mockRequestHeaders := []Header{
+	mockRequestHeaders := []h2mux.Header{
 		{Name: ":path", Value: "//bad_path/"},
-		{Name: RequestUserHeadersField, Value: SerializeHeaders(http.Header{"Mock header": {"Mock value"}})},
+		{Name: RequestUserHeaders, Value: SerializeHeaders(http.Header{"Mock header": {"Mock value"}})},
 	}
 
 	headersConversionErr := H2RequestHeadersToH1Request(mockRequestHeaders, request)
@@ -90,9 +93,9 @@ func TestH2RequestHeadersToH1Request_HostPathWithQuery(t *testing.T) {
 	request, err := http.NewRequest(http.MethodGet, "http://example.com/", nil)
 	assert.NoError(t, err)
 
-	mockRequestHeaders := []Header{
+	mockRequestHeaders := []h2mux.Header{
 		{Name: ":path", Value: "/?query=mock%20value"},
-		{Name: RequestUserHeadersField, Value: SerializeHeaders(http.Header{"Mock header": {"Mock value"}})},
+		{Name: RequestUserHeaders, Value: SerializeHeaders(http.Header{"Mock header": {"Mock value"}})},
 	}
 
 	headersConversionErr := H2RequestHeadersToH1Request(mockRequestHeaders, request)
@@ -110,9 +113,9 @@ func TestH2RequestHeadersToH1Request_HostPathWithURLEncoding(t *testing.T) {
 	request, err := http.NewRequest(http.MethodGet, "http://example.com/", nil)
 	assert.NoError(t, err)
 
-	mockRequestHeaders := []Header{
+	mockRequestHeaders := []h2mux.Header{
 		{Name: ":path", Value: "/mock%20path"},
-		{Name: RequestUserHeadersField, Value: SerializeHeaders(http.Header{"Mock header": {"Mock value"}})},
+		{Name: RequestUserHeaders, Value: SerializeHeaders(http.Header{"Mock header": {"Mock value"}})},
 	}
 
 	headersConversionErr := H2RequestHeadersToH1Request(mockRequestHeaders, request)
@@ -267,9 +270,9 @@ func TestH2RequestHeadersToH1Request_WeirdURLs(t *testing.T) {
 		request, err := http.NewRequest(http.MethodGet, requestURL, nil)
 		assert.NoError(t, err)
 
-		mockRequestHeaders := []Header{
+		mockRequestHeaders := []h2mux.Header{
 			{Name: ":path", Value: testCase.path},
-			{Name: RequestUserHeadersField, Value: SerializeHeaders(http.Header{"Mock header": {"Mock value"}})},
+			{Name: RequestUserHeaders, Value: SerializeHeaders(http.Header{"Mock header": {"Mock value"}})},
 		}
 
 		headersConversionErr := H2RequestHeadersToH1Request(mockRequestHeaders, request)
@@ -337,12 +340,12 @@ func TestH2RequestHeadersToH1Request_QuickCheck(t *testing.T) {
 				const expectedMethod = "POST"
 				const expectedHostname = "request.hostname.example.com"
 
-				h2 := []Header{
+				h2 := []h2mux.Header{
 					{Name: ":method", Value: expectedMethod},
 					{Name: ":scheme", Value: testScheme},
 					{Name: ":authority", Value: expectedHostname},
 					{Name: ":path", Value: testPath},
-					{Name: RequestUserHeadersField, Value: ""},
+					{Name: RequestUserHeaders, Value: ""},
 				}
 				h1, err := http.NewRequest("GET", testOrigin.url, nil)
 				require.NoError(t, err)
@@ -424,10 +427,10 @@ func randomHTTP2Path(t *testing.T, rand *rand.Rand) string {
 	return result
 }
 
-func stdlibHeaderToH2muxHeader(headers http.Header) (h2muxHeaders []Header) {
+func stdlibHeaderToH2muxHeader(headers http.Header) (h2muxHeaders []h2mux.Header) {
 	for name, values := range headers {
 		for _, value := range values {
-			h2muxHeaders = append(h2muxHeaders, Header{name, value})
+			h2muxHeaders = append(h2muxHeaders, h2mux.Header{Name: name, Value: value})
 		}
 	}
 
@@ -515,14 +518,14 @@ func TestParseHeaders(t *testing.T) {
 		"Mock-Header-Three": {"3"},
 	}
 
-	mockHeaders := []Header{
+	mockHeaders := []h2mux.Header{
 		{Name: "One", Value: "1"}, // will be dropped
 		{Name: "Cf-Two", Value: "cf-value-1"},
 		{Name: "Cf-Two", Value: "cf-value-2"},
-		{Name: RequestUserHeadersField, Value: SerializeHeaders(mockUserHeadersToSerialize)},
+		{Name: RequestUserHeaders, Value: SerializeHeaders(mockUserHeadersToSerialize)},
 	}
 
-	expectedHeaders := []Header{
+	expectedHeaders := []h2mux.Header{
 		{Name: "Cf-Two", Value: "cf-value-1"},
 		{Name: "Cf-Two", Value: "cf-value-2"},
 		{Name: "Mock-Header-One", Value: "1"},
@@ -583,7 +586,7 @@ func TestH1ResponseToH2ResponseHeaders(t *testing.T) {
 
 	serializedHeadersIndex := -1
 	for i, header := range headers {
-		if header.Name == ResponseUserHeadersField {
+		if header.Name == ResponseUserHeaders {
 			serializedHeadersIndex = i
 			break
 		}
@@ -593,7 +596,7 @@ func TestH1ResponseToH2ResponseHeaders(t *testing.T) {
 		headers[:serializedHeadersIndex],
 		headers[serializedHeadersIndex+1:]...,
 	)
-	expectedControlHeaders := []Header{
+	expectedControlHeaders := []h2mux.Header{
 		{Name: ":status", Value: "200"},
 		{Name: "content-length", Value: "123"},
 	}
@@ -601,7 +604,7 @@ func TestH1ResponseToH2ResponseHeaders(t *testing.T) {
 	assert.ElementsMatch(t, expectedControlHeaders, actualControlHeaders)
 
 	actualUserHeaders, err := DeserializeHeaders(headers[serializedHeadersIndex].Value)
-	expectedUserHeaders := []Header{
+	expectedUserHeaders := []h2mux.Header{
 		{Name: "User-header-one", Value: ""},
 		{Name: "User-header-two", Value: "1"},
 		{Name: "User-header-two", Value: "2"},
@@ -630,7 +633,7 @@ func TestHeaderSize(t *testing.T) {
 	}
 
 	for _, header := range serializedHeaders {
-		if header.Name != ResponseUserHeadersField {
+		if header.Name != ResponseUserHeaders {
 			continue
 		}
 

connection/http2.go

@@ -10,23 +10,26 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
 	"golang.org/x/net/http2"
 
-	"github.com/cloudflare/cloudflared/h2mux"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
+// note: these constants are exported so we can reuse them in the edge-side code
 const (
-	internalUpgradeHeader = "Cf-Cloudflared-Proxy-Connection-Upgrade"
-	tcpStreamHeader       = "Cf-Cloudflared-Proxy-Src"
-	websocketUpgrade      = "websocket"
-	controlStreamUpgrade  = "control-stream"
+	InternalUpgradeHeader     = "Cf-Cloudflared-Proxy-Connection-Upgrade"
+	InternalTCPProxySrcHeader = "Cf-Cloudflared-Proxy-Src"
+	WebsocketUpgrade          = "websocket"
+	ControlStreamUpgrade      = "control-stream"
 )
 
 var errEdgeConnectionClosed = fmt.Errorf("connection with edge closed")
 
-type http2Connection struct {
+// HTTP2Connection represents a net.Conn that uses HTTP2 frames to proxy traffic from the edge to cloudflared on the
+// origin.
+type HTTP2Connection struct {
 	conn         net.Conn
 	server       *http2.Server
 	config       *Config
@@ -38,6 +41,7 @@ type http2Connection struct {
 	// newRPCClientFunc allows us to mock RPCs during testing
 	newRPCClientFunc func(context.Context, io.ReadWriteCloser, *zerolog.Logger) NamedTunnelRPCClient
 
+	log               *zerolog.Logger
 	activeRequestsWG  sync.WaitGroup
 	connectedFuse     ConnectedFuse
 	gracefulShutdownC <-chan struct{}
@@ -45,6 +49,7 @@ type http2Connection struct {
 	controlStreamErr  error // result of running control stream handler
 }
 
+// NewHTTP2Connection returns a new instance of HTTP2Connection.
 func NewHTTP2Connection(
 	conn net.Conn,
 	config *Config,
@@ -53,9 +58,10 @@ func NewHTTP2Connection(
 	observer *Observer,
 	connIndex uint8,
 	connectedFuse ConnectedFuse,
+	log *zerolog.Logger,
 	gracefulShutdownC <-chan struct{},
-) *http2Connection {
-	return &http2Connection{
+) *HTTP2Connection {
+	return &HTTP2Connection{
 		conn: conn,
 		server: &http2.Server{
 			MaxConcurrentStreams: math.MaxUint32,
@@ -68,11 +74,13 @@ func NewHTTP2Connection(
 		connIndex:         connIndex,
 		newRPCClientFunc:  newRegistrationRPCClient,
 		connectedFuse:     connectedFuse,
+		log:               log,
 		gracefulShutdownC: gracefulShutdownC,
 	}
 }
 
-func (c *http2Connection) Serve(ctx context.Context) error {
+// Serve serves an HTTP2 server that the edge can talk to.
+func (c *HTTP2Connection) Serve(ctx context.Context) error {
 	go func() {
 		<-ctx.Done()
 		c.close()
@@ -93,34 +101,60 @@ func (c *http2Connection) Serve(ctx context.Context) error {
 	}
 }
 
-func (c *http2Connection) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+func (c *HTTP2Connection) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	c.activeRequestsWG.Add(1)
 	defer c.activeRequestsWG.Done()
 
 	connType := determineHTTP2Type(r)
+	handleMissingRequestParts(connType, r)
+
 	respWriter, err := newHTTP2RespWriter(r, w, connType)
 	if err != nil {
 		c.observer.log.Error().Msg(err.Error())
 		return
 	}
 
-	var proxyErr error
 	switch connType {
 	case TypeControlStream:
-		proxyErr = c.serveControlStream(r.Context(), respWriter)
-		c.controlStreamErr = proxyErr
-	case TypeWebsocket:
+		if err := c.serveControlStream(r.Context(), respWriter); err != nil {
+			c.controlStreamErr = err
+			c.log.Error().Err(err)
+			respWriter.WriteErrorResponse()
+		}
+
+	case TypeWebsocket, TypeHTTP:
 		stripWebsocketUpgradeHeader(r)
-		proxyErr = c.config.OriginProxy.Proxy(respWriter, r, TypeWebsocket)
+		if err := c.config.OriginProxy.ProxyHTTP(respWriter, r, connType == TypeWebsocket); err != nil {
+			err := fmt.Errorf("Failed to proxy HTTP: %w", err)
+			c.log.Error().Err(err)
+			respWriter.WriteErrorResponse()
+		}
+
+	case TypeTCP:
+		host, err := getRequestHost(r)
+		if err != nil {
+			err := fmt.Errorf(`cloudflared recieved a warp-routing request with an empty host value: %w`, err)
+			c.log.Error().Err(err)
+			respWriter.WriteErrorResponse()
+		}
+
+		rws := NewHTTPResponseReadWriterAcker(respWriter, r)
+		if err := c.config.OriginProxy.ProxyTCP(r.Context(), rws, &TCPRequest{
+			Dest:    host,
+			CFRay:   FindCfRayHeader(r),
+			LBProbe: IsLBProbeRequest(r),
+		}); err != nil {
+			respWriter.WriteErrorResponse()
+		}
+
 	default:
-		proxyErr = c.config.OriginProxy.Proxy(respWriter, r, connType)
-	}
-	if proxyErr != nil {
+		err := fmt.Errorf("Received unknown connection type: %s", connType)
+		c.log.Error().Err(err)
 		respWriter.WriteErrorResponse()
 	}
 }
 
-func (c *http2Connection) serveControlStream(ctx context.Context, respWriter *http2RespWriter) error {
+func (c *HTTP2Connection) serveControlStream(ctx context.Context, respWriter *http2RespWriter) error {
 	rpcClient := c.newRPCClientFunc(ctx, respWriter, c.observer.log)
 	defer rpcClient.Close()
 
@@ -143,7 +177,7 @@ func (c *http2Connection) serveControlStream(ctx context.Context, respWriter *ht
 	return nil
 }
 
-func (c *http2Connection) close() {
+func (c *HTTP2Connection) close() {
 	// Wait for all serve HTTP handlers to return
 	c.activeRequestsWG.Wait()
 	c.conn.Close()
@@ -178,25 +212,23 @@ func newHTTP2RespWriter(r *http.Request, w http.ResponseWriter, connType Type) (
 func (rp *http2RespWriter) WriteRespHeaders(status int, header http.Header) error {
 	dest := rp.w.Header()
 	userHeaders := make(http.Header, len(header))
-	for header, values := range header {
+	for name, values := range header {
 		// Since these are http2 headers, they're required to be lowercase
-		h2name := strings.ToLower(header)
-		for _, v := range values {
-			if h2name == "content-length" {
-				// This header has meaning in HTTP/2 and will be used by the edge,
-				// so it should be sent as an HTTP/2 response header.
-				dest.Add(h2name, v)
-				// Since these are http2 headers, they're required to be lowercase
-			} else if !h2mux.IsControlHeader(h2name) || h2mux.IsWebsocketClientHeader(h2name) {
-				// User headers, on the other hand, must all be serialized so that
-				// HTTP/2 header validation won't be applied to HTTP/1 header values
-				userHeaders.Add(h2name, v)
-			}
+		h2name := strings.ToLower(name)
+		if h2name == "content-length" {
+			// This header has meaning in HTTP/2 and will be used by the edge,
+			// so it should be sent as an HTTP/2 response header.
+			dest[name] = values
+			// Since these are http2 headers, they're required to be lowercase
+		} else if !IsControlHeader(h2name) || IsWebsocketClientHeader(h2name) {
+			// User headers, on the other hand, must all be serialized so that
+			// HTTP/2 header validation won't be applied to HTTP/1 header values
+			userHeaders[name] = values
 		}
 	}
 
 	// Perform user header serialization and set them in the single header
-	dest.Set(canonicalResponseUserHeadersField, h2mux.SerializeHeaders(userHeaders))
+	dest.Set(CanonicalResponseUserHeaders, SerializeHeaders(userHeaders))
 	rp.setResponseMetaHeader(responseMetaHeaderOrigin)
 	// HTTP2 removes support for 101 Switching Protocols https://tools.ietf.org/html/rfc7540#section-8.1.1
 	if status == http.StatusSwitchingProtocols {
@@ -218,7 +250,7 @@ func (rp *http2RespWriter) WriteErrorResponse() {
 }
 
 func (rp *http2RespWriter) setResponseMetaHeader(value string) {
-	rp.w.Header().Set(canonicalResponseMetaHeaderField, value)
+	rp.w.Header().Set(CanonicalResponseMetaHeader, value)
 }
 
 func (rp *http2RespWriter) Read(p []byte) (n int, err error) {
@@ -257,19 +289,44 @@ func determineHTTP2Type(r *http.Request) Type {
 	}
 }
 
+func handleMissingRequestParts(connType Type, r *http.Request) {
+	if connType == TypeHTTP {
+		// http library has no guarantees that we receive a filled URL. If not, then we fill it, as we reuse the request
+		// for proxying. We use the same values as we used to in h2mux. For proxying they should not matter since we
+		// control the dialer on every egress proxied.
+		if len(r.URL.Scheme) == 0 {
+			r.URL.Scheme = "http"
+		}
+		if len(r.URL.Host) == 0 {
+			r.URL.Host = "localhost:8080"
+		}
+	}
+}
+
 func isControlStreamUpgrade(r *http.Request) bool {
-	return r.Header.Get(internalUpgradeHeader) == controlStreamUpgrade
+	return r.Header.Get(InternalUpgradeHeader) == ControlStreamUpgrade
 }
 
 func isWebsocketUpgrade(r *http.Request) bool {
-	return r.Header.Get(internalUpgradeHeader) == websocketUpgrade
+	return r.Header.Get(InternalUpgradeHeader) == WebsocketUpgrade
 }
 
 // IsTCPStream discerns if the connection request needs a tcp stream proxy.
 func IsTCPStream(r *http.Request) bool {
-	return r.Header.Get(tcpStreamHeader) != ""
+	return r.Header.Get(InternalTCPProxySrcHeader) != ""
 }
 
 func stripWebsocketUpgradeHeader(r *http.Request) {
-	r.Header.Del(internalUpgradeHeader)
+	r.Header.Del(InternalUpgradeHeader)
+}
+
+// getRequestHost returns the host of the http.Request.
+func getRequestHost(r *http.Request) (string, error) {
+	if r.Host != "" {
+		return r.Host, nil
+	}
+	if r.URL != nil {
+		return r.URL.Host, nil
+	}
+	return "", errors.New("host not set in incoming request")
 }

connection/http2_test.go

@@ -26,9 +26,10 @@ var (
 	testTransport = http2.Transport{}
 )
 
-func newTestHTTP2Connection() (*http2Connection, net.Conn) {
+func newTestHTTP2Connection() (*HTTP2Connection, net.Conn) {
 	edgeConn, originConn := net.Pipe()
 	var connIndex = uint8(0)
+	log := zerolog.Nop()
 	return NewHTTP2Connection(
 		originConn,
 		testConfig,
@@ -37,6 +38,7 @@ func newTestHTTP2Connection() (*http2Connection, net.Conn) {
 		NewObserver(&log, &log, false),
 		connIndex,
 		mockConnectedFuse{},
+		&log,
 		nil,
 	), edgeConn
 }
@@ -103,9 +105,9 @@ func TestServeHTTP(t *testing.T) {
 			require.Equal(t, test.expectedBody, respBody)
 		}
 		if test.isProxyError {
-			require.Equal(t, responseMetaHeaderCfd, resp.Header.Get(ResponseMetaHeaderField))
+			require.Equal(t, responseMetaHeaderCfd, resp.Header.Get(ResponseMetaHeader))
 		} else {
-			require.Equal(t, responseMetaHeaderOrigin, resp.Header.Get(ResponseMetaHeaderField))
+			require.Equal(t, responseMetaHeaderOrigin, resp.Header.Get(ResponseMetaHeader))
 		}
 	}
 	cancel()
@@ -191,7 +193,7 @@ func TestServeWS(t *testing.T) {
 
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://localhost:8080/ws", readPipe)
 	require.NoError(t, err)
-	req.Header.Set(internalUpgradeHeader, websocketUpgrade)
+	req.Header.Set(InternalUpgradeHeader, WebsocketUpgrade)
 
 	wg.Add(1)
 	go func() {
@@ -211,7 +213,7 @@ func TestServeWS(t *testing.T) {
 	resp := respWriter.Result()
 	// http2RespWriter should rewrite status 101 to 200
 	require.Equal(t, http.StatusOK, resp.StatusCode)
-	require.Equal(t, responseMetaHeaderOrigin, resp.Header.Get(ResponseMetaHeaderField))
+	require.Equal(t, responseMetaHeaderOrigin, resp.Header.Get(ResponseMetaHeader))
 
 	wg.Wait()
 }
@@ -235,7 +237,7 @@ func TestServeControlStream(t *testing.T) {
 
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://localhost:8080/", nil)
 	require.NoError(t, err)
-	req.Header.Set(internalUpgradeHeader, controlStreamUpgrade)
+	req.Header.Set(InternalUpgradeHeader, ControlStreamUpgrade)
 
 	edgeHTTP2Conn, err := testTransport.NewClientConn(edgeConn)
 	require.NoError(t, err)
@@ -274,7 +276,7 @@ func TestFailRegistration(t *testing.T) {
 
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://localhost:8080/", nil)
 	require.NoError(t, err)
-	req.Header.Set(internalUpgradeHeader, controlStreamUpgrade)
+	req.Header.Set(InternalUpgradeHeader, ControlStreamUpgrade)
 
 	edgeHTTP2Conn, err := testTransport.NewClientConn(edgeConn)
 	require.NoError(t, err)
@@ -310,7 +312,7 @@ func TestGracefulShutdownHTTP2(t *testing.T) {
 
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://localhost:8080/", nil)
 	require.NoError(t, err)
-	req.Header.Set(internalUpgradeHeader, controlStreamUpgrade)
+	req.Header.Set(InternalUpgradeHeader, ControlStreamUpgrade)
 
 	edgeHTTP2Conn, err := testTransport.NewClientConn(edgeConn)
 	require.NoError(t, err)

connection/observer.go

@@ -1,13 +1,7 @@
 package connection
 
 import (
-	"fmt"
-	"net/url"
-	"strings"
-
 	"github.com/rs/zerolog"
-
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
 const (
@@ -54,53 +48,6 @@ func (o *Observer) logServerInfo(connIndex uint8, location, msg string) {
 	o.metrics.registerServerLocation(uint8ToString(connIndex), location)
 }
 
-func (o *Observer) logTrialHostname(registration *tunnelpogs.TunnelRegistration) error {
-	// Print out the user's trial zone URL in a nice box (if they requested and got one and UI flag is not set)
-	if !o.uiEnabled {
-		if registrationURL, err := url.Parse(registration.Url); err == nil {
-			for _, line := range asciiBox(trialZoneMsg(registrationURL.String()), 2) {
-				o.log.Info().Msg(line)
-			}
-		} else {
-			o.log.Error().Msg("Failed to connect tunnel, please try again.")
-			return fmt.Errorf("empty URL in response from Cloudflare edge")
-		}
-	}
-	return nil
-}
-
-// Print out the given lines in a nice ASCII box.
-func asciiBox(lines []string, padding int) (box []string) {
-	maxLen := maxLen(lines)
-	spacer := strings.Repeat(" ", padding)
-
-	border := "+" + strings.Repeat("-", maxLen+(padding*2)) + "+"
-
-	box = append(box, border)
-	for _, line := range lines {
-		box = append(box, "|"+spacer+line+strings.Repeat(" ", maxLen-len(line))+spacer+"|")
-	}
-	box = append(box, border)
-	return
-}
-
-func maxLen(lines []string) int {
-	max := 0
-	for _, line := range lines {
-		if len(line) > max {
-			max = len(line)
-		}
-	}
-	return max
-}
-
-func trialZoneMsg(url string) []string {
-	return []string{
-		"Your free tunnel has started! Visit it:",
-		"  " + url,
-	}
-}
-
 func (o *Observer) sendRegisteringEvent(connIndex uint8) {
 	o.sendEvent(Event{Index: connIndex, EventType: RegisteringTunnel})
 }
@@ -109,7 +56,7 @@ func (o *Observer) sendConnectedEvent(connIndex uint8, location string) {
 	o.sendEvent(Event{Index: connIndex, EventType: Connected, Location: location})
 }
 
-func (o *Observer) sendURL(url string) {
+func (o *Observer) SendURL(url string) {
 	o.sendEvent(Event{EventType: SetURL, URL: url})
 }
 

connection/rpc.go

@@ -159,8 +159,6 @@ func (h *h2muxConnection) registerTunnel(ctx context.Context, credentialSetter C
 		return h.processRegisterTunnelError(registrationErr, register)
 	}
 
-	// Send free tunnel URL to UI
-	h.observer.sendURL(registration.Url)
 	credentialSetter.SetEventDigest(h.connIndex, registration.EventDigest)
 	return h.processRegistrationSuccess(registration, register, credentialSetter, classicTunnel)
 }
@@ -187,14 +185,6 @@ func (h *h2muxConnection) processRegistrationSuccess(
 		h.observer.log.Info().Msgf("Each HA connection's tunnel IDs: %v", h.observer.metrics.tunnelsHA.String())
 	}
 
-	// Print out the user's trial zone URL in a nice box (if they requested and got one and UI flag is not set)
-	if classicTunnel.IsTrialZone() {
-		err := h.observer.logTrialHostname(registration)
-		if err != nil {
-			return err
-		}
-	}
-
 	credentialManager.SetConnDigest(h.connIndex, registration.ConnDigest)
 	h.observer.metrics.userHostnamesCounts.WithLabelValues(registration.Url).Inc()
 

dev.Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.15.7 as builder
+FROM golang:1.16.4 as builder
 ENV GO111MODULE=on \
     CGO_ENABLED=0
 WORKDIR /go/src/github.com/cloudflare/cloudflared/

github_release.py

@@ -10,6 +10,9 @@ import shutil
 import hashlib
 import requests
 import tarfile
+from os import listdir
+from os.path import isfile, join
+import re
 
 from github import Github, GithubException, UnknownObjectException
 
@@ -135,7 +138,7 @@ def parse_args():
         logging.error("Missing asset path")
         is_valid = False
 
-    if not args.name:
+    if not args.name and not os.path.isdir(args.path):
         logging.error("Missing asset name")
         is_valid = False
 
@@ -161,6 +164,57 @@ def parse_args():
     parser.print_usage()
     exit(1)
 
+def upload_asset(release, filepath, filename, release_version, kv_account_id, namespace_id, kv_api_token):
+    logging.info("Uploading asset: %s", filename)
+    release.upload_asset(filepath, name=filename)
+
+    # check and extract if the file is a tar and gzipped file (as is the case with the macos builds)
+    binary_path = filepath
+    if binary_path.endswith("tgz"):
+        try:
+            shutil.rmtree('cfd')
+        except OSError:
+            pass
+        zipfile = tarfile.open(binary_path, "r:gz")
+        zipfile.extractall('cfd') # specify which folder to extract to
+        zipfile.close()
+
+        binary_path = os.path.join(os.getcwd(), 'cfd', 'cloudflared')
+
+    # send the sha256 (the checksum) to workers kv
+    pkg_hash = get_sha256(binary_path)
+    send_hash(pkg_hash, filename, release_version, kv_account_id, namespace_id, kv_api_token)
+
+    # create the artifacts directory if it doesn't exist
+    artifact_path = os.path.join(os.getcwd(), 'artifacts')
+    if not os.path.isdir(artifact_path):
+        os.mkdir(artifact_path)
+
+    # copy the binary to the path
+    copy_path = os.path.join(artifact_path, filename)
+    try:
+        shutil.copy(filepath, copy_path)
+    except shutil.SameFileError:
+        pass # the macOS release copy fails with being the same file (already in the artifacts directory)
+
+def compile_regex() :
+    # The capture groups are:
+    # 1: the text cloudflared
+    # 2: compiled architecture
+    # 3: possible extensions
+    capture_string = r'(cloudflared).+(amd64|x86_64|386|arm)(\.msi|\.deb|\.rpm|\.exe)?'
+    return re.compile(capture_string)
+
+def extract_filename(regex, filename):
+    match_groups = regex.match(filename)
+    os = "linux"
+    name = match_groups.group(1)
+    arch = match_groups.group(2)
+    ext = match_groups.group(3)
+    if ext in ['.msi', '.exe']:
+        os = "windows"
+    return "{}-{}-{}{}".format(name, os, arch, ext if ext is not None else "")
+
 
 def main():
     """ Attempts to upload Asset to Github Release. Creates Release if it doesnt exist """
@@ -174,40 +228,20 @@ def main():
             logging.info("Skipping asset upload because of dry-run")
             return
 
-        release.upload_asset(args.path, name=args.name)
-
-        # check and extract if the file is a tar and gzipped file (as is the case with the macos builds)
-        binary_path = args.path
-        if binary_path.endswith("tgz"):
-            try:
-                shutil.rmtree('cfd') 
-            except OSError as e:
-                pass
-            zipfile = tarfile.open(binary_path, "r:gz")
-            zipfile.extractall('cfd') # specify which folder to extract to
-            zipfile.close()
-
-            binary_path = os.path.join(os.getcwd(), 'cfd', 'cloudflared') 
-
-        # send the sha256 (the checksum) to workers kv
-        pkg_hash = get_sha256(binary_path)
-        send_hash(pkg_hash, args.name, args.release_version, args.kv_account_id, args.namespace_id, args.kv_api_token)
-
-        # create the artifacts directory if it doesn't exist 
-        artifact_path = os.path.join(os.getcwd(), 'artifacts') 
-        if not os.path.isdir(artifact_path):
-            os.mkdir(artifact_path)
-
-        # copy the binary to the path
-        copy_path = os.path.join(artifact_path, args.name)
-        try:
-            shutil.copy(args.path, copy_path)
-        except shutil.SameFileError:
-            pass # the macOS release copy fails with being the same file (already in the artifacts directory). Catching to ignore.
+        if os.path.isdir(args.path):
+            regexp = compile_regex()
+            onlyfiles = [f for f in listdir(args.path) if isfile(join(args.path, f))]
+            for filename in onlyfiles:
+                binary_path = os.path.join(args.path, filename)
+                filename = extract_filename(regexp, filename)
+                upload_asset(release, binary_path, filename, args.release_version, args.kv_account_id, args.namespace_id,
+                args.kv_api_token)
+        else:
+            upload_asset(release, args.path, args.name, args.release_version, args.kv_account_id, args.namespace_id,
+                args.kv_api_token)
 
     except Exception as e:
         logging.exception(e)
         exit(1)
 
-
 main()

go.mod

@@ -43,6 +43,7 @@ require (
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/stretchr/testify v1.6.0
 	github.com/urfave/cli/v2 v2.2.0
+	go.uber.org/automaxprocs v1.4.0
 	golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a
 	golang.org/x/net v0.0.0-20200904194848-62affa334b73
 	golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43 // indirect

go.sum

@@ -609,6 +609,8 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
 go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
+go.uber.org/automaxprocs v1.4.0 h1:CpDZl6aOlLhReez+8S3eEotD7Jx0Os++lemPlMULQP0=
+go.uber.org/automaxprocs v1.4.0/go.mod h1:/mTEdr7LvHhs0v7mjdxDreTz1OG5zdZGqgOnhWiR/+Q=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
 go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4=
 go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU=

h2mux/header.go

@@ -1,234 +0,0 @@
-package h2mux
-
-import (
-	"encoding/base64"
-	"fmt"
-	"net/http"
-	"net/url"
-	"strconv"
-	"strings"
-
-	"github.com/pkg/errors"
-)
-
-type Header struct {
-	Name, Value string
-}
-
-var headerEncoding = base64.RawStdEncoding
-
-const (
-	RequestUserHeadersField  = "cf-cloudflared-request-headers"
-	ResponseUserHeadersField = "cf-cloudflared-response-headers"
-
-	CFAccessTokenHeader        = "cf-access-token"
-	CFJumpDestinationHeader    = "CF-Access-Jump-Destination"
-	CFAccessClientIDHeader     = "CF-Access-Client-Id"
-	CFAccessClientSecretHeader = "CF-Access-Client-Secret"
-)
-
-// H2RequestHeadersToH1Request converts the HTTP/2 headers coming from origintunneld
-// to an HTTP/1 Request object destined for the local origin web service.
-// This operation includes conversion of the pseudo-headers into their closest
-// HTTP/1 equivalents. See https://tools.ietf.org/html/rfc7540#section-8.1.2.3
-func H2RequestHeadersToH1Request(h2 []Header, h1 *http.Request) error {
-	for _, header := range h2 {
-		name := strings.ToLower(header.Name)
-		if !IsControlHeader(name) {
-			continue
-		}
-
-		switch name {
-		case ":method":
-			h1.Method = header.Value
-		case ":scheme":
-			// noop - use the preexisting scheme from h1.URL
-		case ":authority":
-			// Otherwise the host header will be based on the origin URL
-			h1.Host = header.Value
-		case ":path":
-			// We don't want to be an "opinionated" proxy, so ideally we would use :path as-is.
-			// However, this HTTP/1 Request object belongs to the Go standard library,
-			// whose URL package makes some opinionated decisions about the encoding of
-			// URL characters: see the docs of https://godoc.org/net/url#URL,
-			// in particular the EscapedPath method https://godoc.org/net/url#URL.EscapedPath,
-			// which is always used when computing url.URL.String(), whether we'd like it or not.
-			//
-			// Well, not *always*. We could circumvent this by using url.URL.Opaque. But
-			// that would present unusual difficulties when using an HTTP proxy: url.URL.Opaque
-			// is treated differently when HTTP_PROXY is set!
-			// See https://github.com/golang/go/issues/5684#issuecomment-66080888
-			//
-			// This means we are subject to the behavior of net/url's function `shouldEscape`
-			// (as invoked with mode=encodePath): https://github.com/golang/go/blob/go1.12.7/src/net/url/url.go#L101
-
-			if header.Value == "*" {
-				h1.URL.Path = "*"
-				continue
-			}
-			// Due to the behavior of validation.ValidateUrl, h1.URL may
-			// already have a partial value, with or without a trailing slash.
-			base := h1.URL.String()
-			base = strings.TrimRight(base, "/")
-			// But we know :path begins with '/', because we handled '*' above - see RFC7540
-			requestURL, err := url.Parse(base + header.Value)
-			if err != nil {
-				return errors.Wrap(err, fmt.Sprintf("invalid path '%v'", header.Value))
-			}
-			h1.URL = requestURL
-		case "content-length":
-			contentLength, err := strconv.ParseInt(header.Value, 10, 64)
-			if err != nil {
-				return fmt.Errorf("unparseable content length")
-			}
-			h1.ContentLength = contentLength
-		case RequestUserHeadersField:
-			// Do not forward the serialized headers to the origin -- deserialize them, and ditch the serialized version
-			// Find and parse user headers serialized into a single one
-			userHeaders, err := ParseUserHeaders(RequestUserHeadersField, h2)
-			if err != nil {
-				return errors.Wrap(err, "Unable to parse user headers")
-			}
-			for _, userHeader := range userHeaders {
-				h1.Header.Add(http.CanonicalHeaderKey(userHeader.Name), userHeader.Value)
-			}
-		default:
-			// All other control headers shall just be proxied transparently
-			h1.Header.Add(http.CanonicalHeaderKey(header.Name), header.Value)
-		}
-	}
-
-	return nil
-}
-
-func ParseUserHeaders(headerNameToParseFrom string, headers []Header) ([]Header, error) {
-	for _, header := range headers {
-		if header.Name == headerNameToParseFrom {
-			return DeserializeHeaders(header.Value)
-		}
-	}
-
-	return nil, fmt.Errorf("%v header not found", RequestUserHeadersField)
-}
-
-func IsControlHeader(headerName string) bool {
-	return headerName == "content-length" ||
-		headerName == "connection" || headerName == "upgrade" || // Websocket headers
-		strings.HasPrefix(headerName, ":") ||
-		strings.HasPrefix(headerName, "cf-")
-}
-
-// isWebsocketClientHeader returns true if the header name is required by the client to upgrade properly
-func IsWebsocketClientHeader(headerName string) bool {
-	return headerName == "sec-websocket-accept" ||
-		headerName == "connection" ||
-		headerName == "upgrade"
-}
-
-func H1ResponseToH2ResponseHeaders(status int, h1 http.Header) (h2 []Header) {
-	h2 = []Header{
-		{Name: ":status", Value: strconv.Itoa(status)},
-	}
-	userHeaders := make(http.Header, len(h1))
-	for header, values := range h1 {
-		h2name := strings.ToLower(header)
-		if h2name == "content-length" {
-			// This header has meaning in HTTP/2 and will be used by the edge,
-			// so it should be sent as an HTTP/2 response header.
-
-			// Since these are http2 headers, they're required to be lowercase
-			h2 = append(h2, Header{Name: "content-length", Value: values[0]})
-		} else if !IsControlHeader(h2name) || IsWebsocketClientHeader(h2name) {
-			// User headers, on the other hand, must all be serialized so that
-			// HTTP/2 header validation won't be applied to HTTP/1 header values
-			userHeaders[header] = values
-		}
-	}
-
-	// Perform user header serialization and set them in the single header
-	h2 = append(h2, Header{ResponseUserHeadersField, SerializeHeaders(userHeaders)})
-	return h2
-}
-
-// Serialize HTTP1.x headers by base64-encoding each header name and value,
-// and then joining them in the format of [key:value;]
-func SerializeHeaders(h1Headers http.Header) string {
-	// compute size of the fully serialized value and largest temp buffer we will need
-	serializedLen := 0
-	maxTempLen := 0
-	for headerName, headerValues := range h1Headers {
-		for _, headerValue := range headerValues {
-			nameLen := headerEncoding.EncodedLen(len(headerName))
-			valueLen := headerEncoding.EncodedLen(len(headerValue))
-			const delims = 2
-			serializedLen += delims + nameLen + valueLen
-			if nameLen > maxTempLen {
-				maxTempLen = nameLen
-			}
-			if valueLen > maxTempLen {
-				maxTempLen = valueLen
-			}
-		}
-	}
-	var buf strings.Builder
-	buf.Grow(serializedLen)
-
-	temp := make([]byte, maxTempLen)
-	writeB64 := func(s string) {
-		n := headerEncoding.EncodedLen(len(s))
-		if n > len(temp) {
-			temp = make([]byte, n)
-		}
-		headerEncoding.Encode(temp[:n], []byte(s))
-		buf.Write(temp[:n])
-	}
-
-	for headerName, headerValues := range h1Headers {
-		for _, headerValue := range headerValues {
-			if buf.Len() > 0 {
-				buf.WriteByte(';')
-			}
-			writeB64(headerName)
-			buf.WriteByte(':')
-			writeB64(headerValue)
-		}
-	}
-
-	return buf.String()
-}
-
-// Deserialize headers serialized by `SerializeHeader`
-func DeserializeHeaders(serializedHeaders string) ([]Header, error) {
-	const unableToDeserializeErr = "Unable to deserialize headers"
-
-	var deserialized []Header
-	for _, serializedPair := range strings.Split(serializedHeaders, ";") {
-		if len(serializedPair) == 0 {
-			continue
-		}
-
-		serializedHeaderParts := strings.Split(serializedPair, ":")
-		if len(serializedHeaderParts) != 2 {
-			return nil, errors.New(unableToDeserializeErr)
-		}
-
-		serializedName := serializedHeaderParts[0]
-		serializedValue := serializedHeaderParts[1]
-		deserializedName := make([]byte, headerEncoding.DecodedLen(len(serializedName)))
-		deserializedValue := make([]byte, headerEncoding.DecodedLen(len(serializedValue)))
-
-		if _, err := headerEncoding.Decode(deserializedName, []byte(serializedName)); err != nil {
-			return nil, errors.Wrap(err, unableToDeserializeErr)
-		}
-		if _, err := headerEncoding.Decode(deserializedValue, []byte(serializedValue)); err != nil {
-			return nil, errors.Wrap(err, unableToDeserializeErr)
-		}
-
-		deserialized = append(deserialized, Header{
-			Name:  string(deserializedName),
-			Value: string(deserializedValue),
-		})
-	}
-
-	return deserialized, nil
-}

h2mux/muxedstream.go

@@ -23,6 +23,10 @@ type MuxedStreamDataSignaller interface {
 	Signal(ID uint32)
 }
 
+type Header struct {
+	Name, Value string
+}
+
 // MuxedStream is logically an HTTP/2 stream, with an additional buffer for outgoing data.
 type MuxedStream struct {
 	streamID uint32
@@ -74,8 +78,6 @@ type MuxedStream struct {
 	sentEOF bool
 	// true if the peer sent us an EOF
 	receivedEOF bool
-	// If valid, tunnelHostname is used to identify which origin service is the intended recipient of the request
-	tunnelHostname TunnelHostname
 	// Compression-related fields
 	receivedUseDict bool
 	method          string
@@ -252,10 +254,6 @@ func (s *MuxedStream) IsRPCStream() bool {
 	return true
 }
 
-func (s *MuxedStream) TunnelHostname() TunnelHostname {
-	return s.tunnelHostname
-}
-
 // Block until a value is sent on writeBufferHasSpace.
 // Must be called while holding writeLock
 func (s *MuxedStream) awaitWriteBufferHasSpace() {

h2mux/muxreader.go

@@ -12,10 +12,6 @@ import (
 	"golang.org/x/net/http2"
 )
 
-const (
-	CloudflaredProxyTunnelHostnameHeader = "cf-cloudflared-proxy-tunnel-hostname"
-)
-
 type MuxReader struct {
 	// f is used to read HTTP2 frames.
 	f *http2.Framer
@@ -73,12 +69,13 @@ func (r *MuxReader) run(log *zerolog.Logger) error {
 
 	// routine to periodically update bytesRead
 	go func() {
-		tickC := time.Tick(updateFreq)
+		ticker := time.NewTicker(updateFreq)
+		defer ticker.Stop()
 		for {
 			select {
 			case <-r.abortChan:
 				return
-			case <-tickC:
+			case <-ticker.C:
 				r.metricsUpdater.updateInBoundBytes(r.bytesRead.Count())
 			}
 		}
@@ -252,8 +249,6 @@ func (r *MuxReader) receiveHeaderData(frame *http2.MetaHeadersFrame) error {
 			if r.dictionaries.write != nil {
 				continue
 			}
-		case CloudflaredProxyTunnelHostnameHeader:
-			stream.tunnelHostname = TunnelHostname(header.Value)
 		}
 		headers = append(headers, Header{Name: header.Name, Value: header.Value})
 	}

h2mux/muxreader_test.go

@@ -21,10 +21,6 @@ var (
 		Name:  ":path",
 		Value: "/api/tunnels",
 	}
-	tunnelHostnameHeader = Header{
-		Name:  CloudflaredProxyTunnelHostnameHeader,
-		Value: "tunnel.example.com",
-	}
 	respStatusHeader = Header{
 		Name:  ":status",
 		Value: "200",
@@ -42,15 +38,6 @@ func (mosh *mockOriginStreamHandler) ServeStream(stream *MuxedStream) error {
 	return nil
 }
 
-func getCloudflaredProxyTunnelHostnameHeader(stream *MuxedStream) string {
-	for _, header := range stream.Headers {
-		if header.Name == CloudflaredProxyTunnelHostnameHeader {
-			return header.Value
-		}
-	}
-	return ""
-}
-
 func assertOpenStreamSucceed(t *testing.T, stream *MuxedStream, err error) {
 	assert.NoError(t, err)
 	assert.Len(t, stream.Headers, 1)
@@ -72,13 +59,11 @@ func TestMissingHeaders(t *testing.T) {
 		},
 	}
 
-	// Request doesn't contain CloudflaredProxyTunnelHostnameHeader
 	stream, err := muxPair.EdgeMux.OpenStream(ctx, reqHeaders, nil)
 	assertOpenStreamSucceed(t, stream, err)
 
 	assert.Empty(t, originHandler.stream.method)
 	assert.Empty(t, originHandler.stream.path)
-	assert.False(t, originHandler.stream.TunnelHostname().IsSet())
 }
 
 func TestReceiveHeaderData(t *testing.T) {
@@ -90,18 +75,14 @@ func TestReceiveHeaderData(t *testing.T) {
 		methodHeader,
 		schemeHeader,
 		pathHeader,
-		tunnelHostnameHeader,
 	}
 
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
 	defer cancel()
 
-	reqHeaders = append(reqHeaders, tunnelHostnameHeader)
 	stream, err := muxPair.EdgeMux.OpenStream(ctx, reqHeaders, nil)
 	assertOpenStreamSucceed(t, stream, err)
 
 	assert.Equal(t, methodHeader.Value, originHandler.stream.method)
 	assert.Equal(t, pathHeader.Value, originHandler.stream.path)
-	assert.True(t, originHandler.stream.TunnelHostname().IsSet())
-	assert.Equal(t, tunnelHostnameHeader.Value, originHandler.stream.TunnelHostname().String())
 }

h2mux/muxwriter.go

@@ -78,12 +78,13 @@ func (w *MuxWriter) run(log *zerolog.Logger) error {
 
 	// routine to periodically communicate bytesWrote
 	go func() {
-		tickC := time.Tick(updateFreq)
+		ticker := time.NewTicker(updateFreq)
+		defer ticker.Stop()
 		for {
 			select {
 			case <-w.abortChan:
 				return
-			case <-tickC:
+			case <-ticker.C:
 				w.metricsUpdater.updateOutBoundBytes(w.bytesWrote.Count())
 			}
 		}

ingress/ingress.go

@@ -103,7 +103,7 @@ func NewWarpRoutingService() *WarpRoutingService {
 }
 
 // Get a single origin service from the CLI/config.
-func parseSingleOriginService(c *cli.Context, allowURLFromArgs bool) (originService, error) {
+func parseSingleOriginService(c *cli.Context, allowURLFromArgs bool) (OriginService, error) {
 	if c.IsSet("hello-world") {
 		return new(helloWorld), nil
 	}
@@ -138,6 +138,11 @@ func (ing Ingress) IsEmpty() bool {
 	return len(ing.Rules) == 0
 }
 
+// IsSingleRule checks if the user only specified a single ingress rule.
+func (ing Ingress) IsSingleRule() bool {
+	return len(ing.Rules) == 1
+}
+
 // StartOrigins will start any origin services managed by cloudflared, e.g. proxy servers or Hello World.
 func (ing Ingress) StartOrigins(
 	wg *sync.WaitGroup,
@@ -162,7 +167,7 @@ func validate(ingress []config.UnvalidatedIngressRule, defaults OriginRequestCon
 	rules := make([]Rule, len(ingress))
 	for i, r := range ingress {
 		cfg := setConfig(defaults, r.OriginRequest)
-		var service originService
+		var service OriginService
 
 		if prefix := "unix:"; strings.HasPrefix(r.Service, prefix) {
 			// No validation necessary for unix socket filepath services

ingress/ingress_test.go

@@ -402,7 +402,7 @@ func TestSingleOriginSetsConfig(t *testing.T) {
 	flagSet.Bool("hello-world", true, "")
 	flagSet.Duration(ProxyConnectTimeoutFlag, time.Second, "")
 	flagSet.Duration(ProxyTLSTimeoutFlag, time.Second, "")
-	flagSet.Duration(ProxyTCPKeepAlive, time.Second, "")
+	flagSet.Duration(ProxyTCPKeepAliveFlag, time.Second, "")
 	flagSet.Bool(ProxyNoHappyEyeballsFlag, true, "")
 	flagSet.Int(ProxyKeepAliveConnectionsFlag, 10, "")
 	flagSet.Duration(ProxyKeepAliveTimeoutFlag, time.Second, "")
@@ -423,7 +423,7 @@ func TestSingleOriginSetsConfig(t *testing.T) {
 	require.NoError(t, err)
 	err = cliCtx.Set(ProxyTLSTimeoutFlag, "1s")
 	require.NoError(t, err)
-	err = cliCtx.Set(ProxyTCPKeepAlive, "1s")
+	err = cliCtx.Set(ProxyTCPKeepAliveFlag, "1s")
 	require.NoError(t, err)
 	err = cliCtx.Set(ProxyNoHappyEyeballsFlag, "true")
 	require.NoError(t, err)

ingress/origin_connection.go

@@ -2,12 +2,9 @@ package ingress
 
 import (
 	"context"
-	"crypto/tls"
 	"io"
 	"net"
-	"net/http"
 
-	gws "github.com/gorilla/websocket"
 	"github.com/rs/zerolog"
 
 	"github.com/cloudflare/cloudflared/ipaccess"
@@ -25,34 +22,10 @@ type OriginConnection interface {
 
 type streamHandlerFunc func(originConn io.ReadWriter, remoteConn net.Conn, log *zerolog.Logger)
 
-// Stream copies copy data to & from provided io.ReadWriters.
-func Stream(conn, backendConn io.ReadWriter, log *zerolog.Logger) {
-	proxyDone := make(chan struct{}, 2)
-
-	go func() {
-		_, err := io.Copy(conn, backendConn)
-		if err != nil {
-			log.Debug().Msgf("conn to backendConn copy: %v", err)
-		}
-		proxyDone <- struct{}{}
-	}()
-
-	go func() {
-		_, err := io.Copy(backendConn, conn)
-		if err != nil {
-			log.Debug().Msgf("backendConn to conn copy: %v", err)
-		}
-		proxyDone <- struct{}{}
-	}()
-
-	// If one side is done, we are done.
-	<-proxyDone
-}
-
 // DefaultStreamHandler is an implementation of streamHandlerFunc that
 // performs a two way io.Copy between originConn and remoteConn.
 func DefaultStreamHandler(originConn io.ReadWriter, remoteConn net.Conn, log *zerolog.Logger) {
-	Stream(originConn, remoteConn, log)
+	websocket.Stream(originConn, remoteConn, log)
 }
 
 // tcpConnection is an OriginConnection that directly streams to raw TCP.
@@ -61,7 +34,7 @@ type tcpConnection struct {
 }
 
 func (tc *tcpConnection) Stream(ctx context.Context, tunnelConn io.ReadWriter, log *zerolog.Logger) {
-	Stream(tunnelConn, tc.conn, log)
+	websocket.Stream(tunnelConn, tc.conn, log)
 }
 
 func (tc *tcpConnection) Close() {
@@ -82,35 +55,6 @@ func (wc *tcpOverWSConnection) Close() {
 	wc.conn.Close()
 }
 
-// wsConnection is an OriginConnection that streams WS between eyeball and origin.
-type wsConnection struct {
-	wsConn *gws.Conn
-	resp   *http.Response
-}
-
-func (wsc *wsConnection) Stream(ctx context.Context, tunnelConn io.ReadWriter, log *zerolog.Logger) {
-	Stream(tunnelConn, wsc.wsConn.UnderlyingConn(), log)
-}
-
-func (wsc *wsConnection) Close() {
-	wsc.resp.Body.Close()
-	wsc.wsConn.Close()
-}
-
-func newWSConnection(clientTLSConfig *tls.Config, r *http.Request) (OriginConnection, *http.Response, error) {
-	d := &gws.Dialer{
-		TLSClientConfig: clientTLSConfig,
-	}
-	wsConn, resp, err := websocket.ClientConnect(r, d)
-	if err != nil {
-		return nil, nil, err
-	}
-	return &wsConnection{
-		wsConn,
-		resp,
-	}, resp, nil
-}
-
 // socksProxyOverWSConnection is an OriginConnection that streams SOCKS connections over WS.
 // The connection to the origin happens inside the SOCKS code as the client specifies the origin
 // details in the packet.
@@ -124,3 +68,16 @@ func (sp *socksProxyOverWSConnection) Stream(ctx context.Context, tunnelConn io.
 
 func (sp *socksProxyOverWSConnection) Close() {
 }
+
+// wsProxyConnection represents a bidirectional stream for a websocket connection to the origin
+type wsProxyConnection struct {
+	rwc io.ReadWriteCloser
+}
+
+func (conn *wsProxyConnection) Stream(ctx context.Context, tunnelConn io.ReadWriter, log *zerolog.Logger) {
+	websocket.Stream(tunnelConn, conn.rwc, log)
+}
+
+func (conn *wsProxyConnection) Close() {
+	conn.rwc.Close()
+}

ingress/origin_connection_test.go

@@ -3,7 +3,6 @@ package ingress
 import (
 	"bytes"
 	"context"
-	"crypto/tls"
 	"fmt"
 	"io/ioutil"
 	"net"
@@ -22,6 +21,7 @@ import (
 
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/socks"
+	"github.com/cloudflare/cloudflared/websocket"
 )
 
 const (
@@ -157,7 +157,7 @@ func TestSocksStreamWSOverTCPConnection(t *testing.T) {
 			require.NoError(t, err)
 			defer wsForwarderInConn.Close()
 
-			Stream(wsForwarderInConn, &wsEyeball{wsForwarderOutConn}, testLogger)
+			websocket.Stream(wsForwarderInConn, &wsEyeball{wsForwarderOutConn}, testLogger)
 			return nil
 		})
 
@@ -189,39 +189,6 @@ func TestSocksStreamWSOverTCPConnection(t *testing.T) {
 	}
 }
 
-func TestStreamWSConnection(t *testing.T) {
-	eyeballConn, edgeConn := net.Pipe()
-
-	origin := echoWSOrigin(t)
-	defer origin.Close()
-
-	req, err := http.NewRequest(http.MethodGet, origin.URL, nil)
-	require.NoError(t, err)
-	req.Header.Set("Sec-Websocket-Key", "dGhlIHNhbXBsZSBub25jZQ==")
-
-	clientTLSConfig := &tls.Config{
-		InsecureSkipVerify: true,
-	}
-	wsConn, resp, err := newWSConnection(clientTLSConfig, req)
-	require.NoError(t, err)
-	require.Equal(t, http.StatusSwitchingProtocols, resp.StatusCode)
-	require.Equal(t, "Upgrade", resp.Header.Get("Connection"))
-	require.Equal(t, "s3pPLMBiTxaQ9kYGzzhZRbK+xOo=", resp.Header.Get("Sec-Websocket-Accept"))
-	require.Equal(t, "websocket", resp.Header.Get("Upgrade"))
-
-	ctx, cancel := context.WithTimeout(context.Background(), testStreamTimeout)
-	defer cancel()
-
-	errGroup, ctx := errgroup.WithContext(ctx)
-	errGroup.Go(func() error {
-		echoWSEyeball(t, eyeballConn)
-		return nil
-	})
-
-	wsConn.Stream(ctx, edgeConn, testLogger)
-	require.NoError(t, errGroup.Wait())
-}
-
 type wsEyeball struct {
 	conn net.Conn
 }
@@ -240,17 +207,23 @@ func (wse *wsEyeball) Write(p []byte) (int, error) {
 }
 
 func echoWSEyeball(t *testing.T, conn net.Conn) {
-	require.NoError(t, wsutil.WriteClientBinary(conn, testMessage))
+	defer func() {
+		assert.NoError(t, conn.Close())
+	}()
+
+	if !assert.NoError(t, wsutil.WriteClientBinary(conn, testMessage)) {
+		return
+	}
 
 	readMsg, err := wsutil.ReadServerBinary(conn)
-	require.NoError(t, err)
+	if !assert.NoError(t, err) {
+		return
+	}
 
-	require.Equal(t, testResponse, readMsg)
-
-	require.NoError(t, conn.Close())
+	assert.Equal(t, testResponse, readMsg)
 }
 
-func echoWSOrigin(t *testing.T) *httptest.Server {
+func echoWSOrigin(t *testing.T, expectMessages bool) *httptest.Server {
 	var upgrader = gorillaWS.Upgrader{
 		ReadBufferSize:  10,
 		WriteBufferSize: 10,
@@ -267,12 +240,17 @@ func echoWSOrigin(t *testing.T) *httptest.Server {
 		require.NoError(t, err)
 		defer conn.Close()
 
+		sawMessage := false
 		for {
 			messageType, p, err := conn.ReadMessage()
 			if err != nil {
+				if expectMessages && !sawMessage {
+					t.Errorf("unexpected error: %v", err)
+				}
 				return
 			}
-			require.Equal(t, testMessage, p)
+			assert.Equal(t, testMessage, p)
+			sawMessage = true
 			if err := conn.WriteMessage(messageType, testResponse); err != nil {
 				return
 			}

ingress/origin_proxy.go

@@ -1,20 +1,14 @@
 package ingress
 
 import (
-	"fmt"
 	"net"
 	"net/http"
-	"net/url"
-	"strings"
 
 	"github.com/pkg/errors"
-
-	"github.com/cloudflare/cloudflared/h2mux"
-	"github.com/cloudflare/cloudflared/websocket"
 )
 
 var (
-	switchingProtocolText = fmt.Sprintf("%d %s", http.StatusSwitchingProtocols, http.StatusText(http.StatusSwitchingProtocols))
+	errUnsupportedConnectionType = errors.New("internal error: unsupported connection type")
 )
 
 // HTTPOriginProxy can be implemented by origin services that want to proxy http requests.
@@ -25,7 +19,7 @@ type HTTPOriginProxy interface {
 
 // StreamBasedOriginProxy can be implemented by origin services that want to proxy ws/TCP.
 type StreamBasedOriginProxy interface {
-	EstablishConnection(r *http.Request) (OriginConnection, *http.Response, error)
+	EstablishConnection(dest string) (OriginConnection, error)
 }
 
 func (o *unixSocketPath) RoundTrip(req *http.Request) (*http.Response, error) {
@@ -35,7 +29,15 @@ func (o *unixSocketPath) RoundTrip(req *http.Request) (*http.Response, error) {
 func (o *httpService) RoundTrip(req *http.Request) (*http.Response, error) {
 	// Rewrite the request URL so that it goes to the origin service.
 	req.URL.Host = o.url.Host
-	req.URL.Scheme = o.url.Scheme
+	switch o.url.Scheme {
+	case "ws":
+		req.URL.Scheme = "http"
+	case "wss":
+		req.URL.Scheme = "https"
+	default:
+		req.URL.Scheme = o.url.Scheme
+	}
+
 	if o.hostHeader != "" {
 		// For incoming requests, the Host header is promoted to the Request.Host field and removed from the Header map.
 		req.Host = o.hostHeader
@@ -43,117 +45,40 @@ func (o *httpService) RoundTrip(req *http.Request) (*http.Response, error) {
 	return o.transport.RoundTrip(req)
 }
 
-func (o *httpService) EstablishConnection(req *http.Request) (OriginConnection, *http.Response, error) {
-	req.URL.Host = o.url.Host
-	req.URL.Scheme = websocket.ChangeRequestScheme(o.url)
-	if o.hostHeader != "" {
-		// For incoming requests, the Host header is promoted to the Request.Host field and removed from the Header map.
-		req.Host = o.hostHeader
-	}
-	return newWSConnection(o.transport.TLSClientConfig, req)
-}
-
-func (o *helloWorld) RoundTrip(req *http.Request) (*http.Response, error) {
-	// Rewrite the request URL so that it goes to the Hello World server.
-	req.URL.Host = o.server.Addr().String()
-	req.URL.Scheme = "https"
-	return o.transport.RoundTrip(req)
-}
-
-func (o *helloWorld) EstablishConnection(req *http.Request) (OriginConnection, *http.Response, error) {
-	req.URL.Host = o.server.Addr().String()
-	req.URL.Scheme = "wss"
-	return newWSConnection(o.transport.TLSClientConfig, req)
-}
-
 func (o *statusCode) RoundTrip(_ *http.Request) (*http.Response, error) {
 	return o.resp, nil
 }
 
-func (o *rawTCPService) EstablishConnection(r *http.Request) (OriginConnection, *http.Response, error) {
-	dest, err := getRequestHost(r)
-	if err != nil {
-		return nil, nil, err
-	}
+func (o *rawTCPService) EstablishConnection(dest string) (OriginConnection, error) {
 	conn, err := net.Dial("tcp", dest)
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
 
 	originConn := &tcpConnection{
 		conn: conn,
 	}
-	resp := &http.Response{
-		Status:        switchingProtocolText,
-		StatusCode:    http.StatusSwitchingProtocols,
-		ContentLength: -1,
-	}
-	return originConn, resp, nil
+	return originConn, nil
 }
 
-// getRequestHost returns the host of the http.Request.
-func getRequestHost(r *http.Request) (string, error) {
-	if r.Host != "" {
-		return r.Host, nil
-	}
-	if r.URL != nil {
-		return r.URL.Host, nil
-	}
-	return "", errors.New("host not found")
-}
-
-func (o *tcpOverWSService) EstablishConnection(r *http.Request) (OriginConnection, *http.Response, error) {
+func (o *tcpOverWSService) EstablishConnection(dest string) (OriginConnection, error) {
 	var err error
-	dest := o.dest
-	if o.isBastion {
-		dest, err = o.bastionDest(r)
-		if err != nil {
-			return nil, nil, err
-		}
+	if !o.isBastion {
+		dest = o.dest
 	}
 
 	conn, err := net.Dial("tcp", dest)
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
 	originConn := &tcpOverWSConnection{
 		conn:          conn,
 		streamHandler: o.streamHandler,
 	}
-	resp := &http.Response{
-		Status:        switchingProtocolText,
-		StatusCode:    http.StatusSwitchingProtocols,
-		Header:        websocket.NewResponseHeader(r),
-		ContentLength: -1,
-	}
-	return originConn, resp, nil
+	return originConn, nil
 
 }
 
-func (o *tcpOverWSService) bastionDest(r *http.Request) (string, error) {
-	jumpDestination := r.Header.Get(h2mux.CFJumpDestinationHeader)
-	if jumpDestination == "" {
-		return "", fmt.Errorf("Did not receive final destination from client. The --destination flag is likely not set on the client side")
-	}
-	// Strip scheme and path set by client. Without a scheme
-	// Parsing a hostname and path without scheme might not return an error due to parsing ambiguities
-	if jumpURL, err := url.Parse(jumpDestination); err == nil && jumpURL.Host != "" {
-		return removePath(jumpURL.Host), nil
-	}
-	return removePath(jumpDestination), nil
-}
-
-func removePath(dest string) string {
-	return strings.SplitN(dest, "/", 2)[0]
-}
-
-func (o *socksProxyOverWSService) EstablishConnection(r *http.Request) (OriginConnection, *http.Response, error) {
-	originConn := o.conn
-	resp := &http.Response{
-		Status:        switchingProtocolText,
-		StatusCode:    http.StatusSwitchingProtocols,
-		Header:        websocket.NewResponseHeader(r),
-		ContentLength: -1,
-	}
-	return originConn, resp, nil
+func (o *socksProxyOverWSService) EstablishConnection(dest string) (OriginConnection, error) {
+	return o.conn, nil
 }

ingress/origin_proxy_test.go

@@ -2,7 +2,6 @@ package ingress
 
 import (
 	"context"
-	"crypto/tls"
 	"fmt"
 	"net"
 	"net/http"
@@ -14,75 +13,10 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/carrier"
 	"github.com/cloudflare/cloudflared/websocket"
 )
 
-// TestEstablishConnectionResponse ensures each implementation of StreamBasedOriginProxy returns
-// the expected response
-func assertEstablishConnectionResponse(t *testing.T,
-	originProxy StreamBasedOriginProxy,
-	req *http.Request,
-	expectHeader http.Header,
-) {
-	_, resp, err := originProxy.EstablishConnection(req)
-	assert.NoError(t, err)
-	assert.Equal(t, switchingProtocolText, resp.Status)
-	assert.Equal(t, http.StatusSwitchingProtocols, resp.StatusCode)
-	assert.Equal(t, expectHeader, resp.Header)
-}
-
-func TestHTTPServiceEstablishConnection(t *testing.T) {
-	origin := echoWSOrigin(t)
-	defer origin.Close()
-	originURL, err := url.Parse(origin.URL)
-	require.NoError(t, err)
-
-	httpService := &httpService{
-		url:        originURL,
-		hostHeader: origin.URL,
-		transport: &http.Transport{
-			TLSClientConfig: &tls.Config{
-				InsecureSkipVerify: true,
-			},
-		},
-	}
-	req, err := http.NewRequest(http.MethodGet, origin.URL, nil)
-	require.NoError(t, err)
-	req.Header.Set("Sec-Websocket-Key", "dGhlIHNhbXBsZSBub25jZQ==")
-	req.Header.Set("Test-Cloudflared-Echo", t.Name())
-
-	expectHeader := http.Header{
-		"Connection":            {"Upgrade"},
-		"Sec-Websocket-Accept":  {"s3pPLMBiTxaQ9kYGzzhZRbK+xOo="},
-		"Upgrade":               {"websocket"},
-		"Test-Cloudflared-Echo": {t.Name()},
-	}
-	assertEstablishConnectionResponse(t, httpService, req, expectHeader)
-}
-
-func TestHelloWorldEstablishConnection(t *testing.T) {
-	var wg sync.WaitGroup
-	shutdownC := make(chan struct{})
-	errC := make(chan error)
-	helloWorldSerivce := &helloWorld{}
-	helloWorldSerivce.start(&wg, testLogger, shutdownC, errC, OriginRequestConfig{})
-
-	// Scheme and Host of URL will be override by the Scheme and Host of the helloWorld service
-	req, err := http.NewRequest(http.MethodGet, "https://place-holder/ws", nil)
-	require.NoError(t, err)
-
-	expectHeader := http.Header{
-		"Connection": {"Upgrade"},
-		// Accept key when Sec-Websocket-Key is not specified
-		"Sec-Websocket-Accept": {"Kfh9QIsMVZcl6xEPYxPHzW8SZ8w="},
-		"Upgrade":              {"websocket"},
-	}
-	assertEstablishConnectionResponse(t, helloWorldSerivce, req, expectHeader)
-
-	close(shutdownC)
-}
-
 func TestRawTCPServiceEstablishConnection(t *testing.T) {
 	originListener, err := net.Listen("tcp", "127.0.0.1:0")
 	require.NoError(t, err)
@@ -95,8 +29,6 @@ func TestRawTCPServiceEstablishConnection(t *testing.T) {
 	req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://%s", originListener.Addr()), nil)
 	require.NoError(t, err)
 
-	assertEstablishConnectionResponse(t, rawTCPService, req, nil)
-
 	originListener.Close()
 	<-listenerClosed
 
@@ -104,9 +36,8 @@ func TestRawTCPServiceEstablishConnection(t *testing.T) {
 	require.NoError(t, err)
 
 	// Origin not listening for new connection, should return an error
-	_, resp, err := rawTCPService.EstablishConnection(req)
+	_, err = rawTCPService.EstablishConnection(req.URL.String())
 	require.Error(t, err)
-	require.Nil(t, resp)
 }
 
 func TestTCPOverWSServiceEstablishConnection(t *testing.T) {
@@ -126,28 +57,26 @@ func TestTCPOverWSServiceEstablishConnection(t *testing.T) {
 	baseReq.Header.Set("Sec-Websocket-Key", "dGhlIHNhbXBsZSBub25jZQ==")
 
 	bastionReq := baseReq.Clone(context.Background())
-	bastionReq.Header.Set(h2mux.CFJumpDestinationHeader, originListener.Addr().String())
-
-	expectHeader := http.Header{
-		"Connection":           {"Upgrade"},
-		"Sec-Websocket-Accept": {"s3pPLMBiTxaQ9kYGzzhZRbK+xOo="},
-		"Upgrade":              {"websocket"},
-	}
+	carrier.SetBastionDest(bastionReq.Header, originListener.Addr().String())
 
 	tests := []struct {
+		testCase  string
 		service   *tcpOverWSService
 		req       *http.Request
 		expectErr bool
 	}{
 		{
-			service: newTCPOverWSService(originURL),
-			req:     baseReq,
+			testCase: "specific TCP service",
+			service:  newTCPOverWSService(originURL),
+			req:      baseReq,
 		},
 		{
-			service: newBastionService(),
-			req:     bastionReq,
+			testCase: "bastion service",
+			service:  newBastionService(),
+			req:      bastionReq,
 		},
 		{
+			testCase:  "invalid bastion request",
 			service:   newBastionService(),
 			req:       baseReq,
 			expectErr: true,
@@ -155,13 +84,13 @@ func TestTCPOverWSServiceEstablishConnection(t *testing.T) {
 	}
 
 	for _, test := range tests {
-		if test.expectErr {
-			_, resp, err := test.service.EstablishConnection(test.req)
-			assert.Error(t, err)
-			assert.Nil(t, resp)
-		} else {
-			assertEstablishConnectionResponse(t, test.service, test.req, expectHeader)
-		}
+		t.Run(test.testCase, func(t *testing.T) {
+			if test.expectErr {
+				bastionHost, _ := carrier.ResolveBastionDest(test.req)
+				_, err := test.service.EstablishConnection(bastionHost)
+				assert.Error(t, err)
+			}
+		})
 	}
 
 	originListener.Close()
@@ -169,107 +98,9 @@ func TestTCPOverWSServiceEstablishConnection(t *testing.T) {
 
 	for _, service := range []*tcpOverWSService{newTCPOverWSService(originURL), newBastionService()} {
 		// Origin not listening for new connection, should return an error
-		_, resp, err := service.EstablishConnection(bastionReq)
+		bastionHost, _ := carrier.ResolveBastionDest(bastionReq)
+		_, err := service.EstablishConnection(bastionHost)
 		assert.Error(t, err)
-		assert.Nil(t, resp)
-	}
-}
-
-func TestBastionDestination(t *testing.T) {
-	canonicalJumpDestHeader := http.CanonicalHeaderKey(h2mux.CFJumpDestinationHeader)
-	tests := []struct {
-		name         string
-		header       http.Header
-		expectedDest string
-		wantErr      bool
-	}{
-		{
-			name: "hostname destination",
-			header: http.Header{
-				canonicalJumpDestHeader: []string{"localhost"},
-			},
-			expectedDest: "localhost",
-		},
-		{
-			name: "hostname destination with port",
-			header: http.Header{
-				canonicalJumpDestHeader: []string{"localhost:9000"},
-			},
-			expectedDest: "localhost:9000",
-		},
-		{
-			name: "hostname destination with scheme and port",
-			header: http.Header{
-				canonicalJumpDestHeader: []string{"ssh://localhost:9000"},
-			},
-			expectedDest: "localhost:9000",
-		},
-		{
-			name: "full hostname url",
-			header: http.Header{
-				canonicalJumpDestHeader: []string{"ssh://localhost:9000/metrics"},
-			},
-			expectedDest: "localhost:9000",
-		},
-		{
-			name: "hostname destination with port and path",
-			header: http.Header{
-				canonicalJumpDestHeader: []string{"localhost:9000/metrics"},
-			},
-			expectedDest: "localhost:9000",
-		},
-		{
-			name: "ip destination",
-			header: http.Header{
-				canonicalJumpDestHeader: []string{"127.0.0.1"},
-			},
-			expectedDest: "127.0.0.1",
-		},
-		{
-			name: "ip destination with port",
-			header: http.Header{
-				canonicalJumpDestHeader: []string{"127.0.0.1:9000"},
-			},
-			expectedDest: "127.0.0.1:9000",
-		},
-		{
-			name: "ip destination with port and path",
-			header: http.Header{
-				canonicalJumpDestHeader: []string{"127.0.0.1:9000/metrics"},
-			},
-			expectedDest: "127.0.0.1:9000",
-		},
-		{
-			name: "ip destination with schem and port",
-			header: http.Header{
-				canonicalJumpDestHeader: []string{"tcp://127.0.0.1:9000"},
-			},
-			expectedDest: "127.0.0.1:9000",
-		},
-		{
-			name: "full ip url",
-			header: http.Header{
-				canonicalJumpDestHeader: []string{"ssh://127.0.0.1:9000/metrics"},
-			},
-			expectedDest: "127.0.0.1:9000",
-		},
-		{
-			name:    "no destination",
-			wantErr: true,
-		},
-	}
-	s := newBastionService()
-	for _, test := range tests {
-		r := &http.Request{
-			Header: test.header,
-		}
-		dest, err := s.bastionDest(r)
-		if test.wantErr {
-			assert.Error(t, err, "Test %s expects error", test.name)
-		} else {
-			assert.NoError(t, err, "Test %s expects no error, got error %v", test.name, err)
-			assert.Equal(t, test.expectedDest, dest, "Test %s expect dest %s, got %s", test.name, test.expectedDest, dest)
-		}
 	}
 }
 
@@ -310,10 +141,6 @@ func TestHTTPServiceHostHeaderOverride(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, http.StatusOK, resp.StatusCode)
 
-	req = req.Clone(context.Background())
-	_, resp, err = httpService.EstablishConnection(req)
-	require.NoError(t, err)
-	require.Equal(t, http.StatusSwitchingProtocols, resp.StatusCode)
 }
 
 func tcpListenRoutine(listener net.Listener, closeChan chan struct{}) {

ingress/origin_request_config.go

@@ -22,7 +22,7 @@ const (
 	Socks5Flag                    = "socks5"
 	ProxyConnectTimeoutFlag       = "proxy-connect-timeout"
 	ProxyTLSTimeoutFlag           = "proxy-tls-timeout"
-	ProxyTCPKeepAlive             = "proxy-tcp-keepalive"
+	ProxyTCPKeepAliveFlag         = "proxy-tcp-keepalive"
 	ProxyNoHappyEyeballsFlag      = "proxy-no-happy-eyeballs"
 	ProxyKeepAliveConnectionsFlag = "proxy-keepalive-connections"
 	ProxyKeepAliveTimeoutFlag     = "proxy-keepalive-timeout"
@@ -60,7 +60,7 @@ func originRequestFromSingeRule(c *cli.Context) OriginRequestConfig {
 	if flag := ProxyTLSTimeoutFlag; c.IsSet(flag) {
 		tlsTimeout = c.Duration(flag)
 	}
-	if flag := ProxyTCPKeepAlive; c.IsSet(flag) {
+	if flag := ProxyTCPKeepAliveFlag; c.IsSet(flag) {
 		tcpKeepAlive = c.Duration(flag)
 	}
 	if flag := ProxyNoHappyEyeballsFlag; c.IsSet(flag) {

ingress/origin_service.go

@@ -11,21 +11,17 @@ import (
 	"sync"
 	"time"
 
-	gws "github.com/gorilla/websocket"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
 
-	"golang.org/x/net/http2"
-
 	"github.com/cloudflare/cloudflared/hello"
 	"github.com/cloudflare/cloudflared/ipaccess"
 	"github.com/cloudflare/cloudflared/socks"
 	"github.com/cloudflare/cloudflared/tlsconfig"
-	"github.com/cloudflare/cloudflared/websocket"
 )
 
-// originService is something a tunnel can proxy traffic to.
-type originService interface {
+// OriginService is something a tunnel can proxy traffic to.
+type OriginService interface {
 	String() string
 	// Start the origin service if it's managed by cloudflared, e.g. proxy servers or Hello World.
 	// If it's not managed by cloudflared, this is a no-op because the user is responsible for
@@ -52,16 +48,6 @@ func (o *unixSocketPath) start(wg *sync.WaitGroup, log *zerolog.Logger, shutdown
 	return nil
 }
 
-func (o *unixSocketPath) Dial(reqURL *url.URL, headers http.Header) (*gws.Conn, *http.Response, error) {
-	d := &gws.Dialer{
-		NetDial:         o.transport.Dial,
-		NetDialContext:  o.transport.DialContext,
-		TLSClientConfig: o.transport.TLSClientConfig,
-	}
-	reqURL.Scheme = websocket.ChangeRequestScheme(reqURL)
-	return d.Dial(reqURL.String(), headers)
-}
-
 type httpService struct {
 	url        *url.URL
 	hostHeader string
@@ -173,8 +159,8 @@ func (o *socksProxyOverWSService) String() string {
 // HelloWorld is an OriginService for the built-in Hello World server.
 // Users only use this for testing and experimenting with cloudflared.
 type helloWorld struct {
-	server    net.Listener
-	transport *http.Transport
+	httpService
+	server net.Listener
 }
 
 func (o *helloWorld) String() string {
@@ -189,11 +175,10 @@ func (o *helloWorld) start(
 	errC chan error,
 	cfg OriginRequestConfig,
 ) error {
-	transport, err := newHTTPTransport(o, cfg, log)
-	if err != nil {
+	if err := o.httpService.start(wg, log, shutdownC, errC, cfg); err != nil {
 		return err
 	}
-	o.transport = transport
+
 	helloListener, err := hello.CreateTLSListener("127.0.0.1:")
 	if err != nil {
 		return errors.Wrap(err, "Cannot start Hello World Server")
@@ -204,6 +189,12 @@ func (o *helloWorld) start(
 		_ = hello.StartHelloWorldServer(log, helloListener, shutdownC)
 	}()
 	o.server = helloListener
+
+	o.httpService.url = &url.URL{
+		Scheme: "https",
+		Host:   o.server.Addr().String(),
+	}
+
 	return nil
 }
 
@@ -247,7 +238,7 @@ func (nrc *NopReadCloser) Close() error {
 	return nil
 }
 
-func newHTTPTransport(service originService, cfg OriginRequestConfig, log *zerolog.Logger) (*http.Transport, error) {
+func newHTTPTransport(service OriginService, cfg OriginRequestConfig, log *zerolog.Logger) (*http.Transport, error) {
 	originCertPool, err := tlsconfig.LoadOriginCA(cfg.CAPool, log)
 	if err != nil {
 		return nil, errors.Wrap(err, "Error loading cert pool")
@@ -289,7 +280,6 @@ func newHTTPTransport(service originService, cfg OriginRequestConfig, log *zerol
 		httpTransport.DialContext = dialContext
 	}
 
-	http2.ConfigureTransport(&httpTransport)
 	return &httpTransport, nil
 }
 

ingress/rule.go

@@ -17,7 +17,7 @@ type Rule struct {
 	// A (probably local) address. Requests for a hostname which matches this
 	// rule's hostname pattern will be proxied to the service running on this
 	// address.
-	Service originService
+	Service OriginService
 
 	// Configure the request cloudflared sends to this specific origin.
 	Config OriginRequestConfig

ingress/rule_test.go

@@ -14,7 +14,7 @@ func Test_rule_matches(t *testing.T) {
 	type fields struct {
 		Hostname string
 		Path     *regexp.Regexp
-		Service  originService
+		Service  OriginService
 	}
 	type args struct {
 		requestURL *url.URL

metrics/metrics.go

@@ -22,7 +22,7 @@ const (
 	startupTime     = time.Millisecond * 500
 )
 
-func newMetricsHandler(readyServer *ReadyServer) *mux.Router {
+func newMetricsHandler(readyServer *ReadyServer, quickTunnelHostname string) *mux.Router {
 	router := mux.NewRouter()
 	router.PathPrefix("/debug/").Handler(http.DefaultServeMux)
 
@@ -33,6 +33,9 @@ func newMetricsHandler(readyServer *ReadyServer) *mux.Router {
 	if readyServer != nil {
 		router.Handle("/ready", readyServer)
 	}
+	router.HandleFunc("/quicktunnel", func(w http.ResponseWriter, r *http.Request) {
+		_, _ = fmt.Fprintf(w, `{"hostname":"%s"}`, quickTunnelHostname)
+	})
 
 	return router
 }
@@ -41,6 +44,7 @@ func ServeMetrics(
 	l net.Listener,
 	shutdownC <-chan struct{},
 	readyServer *ReadyServer,
+	quickTunnelHostname string,
 	log *zerolog.Logger,
 ) (err error) {
 	var wg sync.WaitGroup
@@ -48,7 +52,7 @@ func ServeMetrics(
 	trace.AuthRequest = func(*http.Request) (bool, bool) { return true, true }
 	// TODO: parameterize ReadTimeout and WriteTimeout. The maximum time we can
 	// profile CPU usage depends on WriteTimeout
-	h := newMetricsHandler(readyServer)
+	h := newMetricsHandler(readyServer, quickTunnelHostname)
 	server := &http.Server{
 		ReadTimeout:  10 * time.Second,
 		WriteTimeout: 10 * time.Second,

metrics/readiness.go

@@ -36,8 +36,6 @@ func (rs *ReadyServer) OnTunnelEvent(c conn.Event) {
 		rs.Lock()
 		rs.isConnected[int(c.Index)] = false
 		rs.Unlock()
-	case conn.SetURL:
-		break
 	default:
 		rs.log.Error().Msgf("Unknown connection event case %v", c)
 	}

metrics/timer.go

@@ -1,48 +0,0 @@
-package metrics
-
-import (
-	"time"
-
-	"github.com/prometheus/client_golang/prometheus"
-)
-
-// Timer assumes the metrics is partitioned by one label
-type Timer struct {
-	startTime   map[string]time.Time
-	metrics     *prometheus.HistogramVec
-	measureUnit time.Duration
-	labelKey    string
-}
-
-func NewTimer(metrics *prometheus.HistogramVec, unit time.Duration, labelKey string) *Timer {
-	return &Timer{
-		startTime:   make(map[string]time.Time),
-		measureUnit: unit,
-		metrics:     metrics,
-		labelKey:    labelKey,
-	}
-}
-
-func (i *Timer) Start(labelVal string) {
-	i.startTime[labelVal] = time.Now()
-}
-
-func (i *Timer) End(labelVal string) time.Duration {
-	if start, ok := i.startTime[labelVal]; ok {
-		return Latency(start, time.Now())
-	}
-	return 0
-}
-
-func (i *Timer) Observe(measurement time.Duration, labelVal string) {
-	metricsLabels := prometheus.Labels{i.labelKey: labelVal}
-	i.metrics.With(metricsLabels).Observe(float64(measurement / i.measureUnit))
-}
-
-func (i *Timer) EndAndObserve(labelVal string) {
-	i.Observe(i.End(labelVal), labelVal)
-}
-
-func Latency(startTime, endTime time.Time) time.Duration {
-	return endTime.Sub(startTime)
-}

metrics/timer_test.go

@@ -1,24 +0,0 @@
-package metrics
-
-import (
-	"testing"
-	"time"
-
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/stretchr/testify/assert"
-)
-
-func TestEnd(t *testing.T) {
-	m := prometheus.NewHistogramVec(
-		prometheus.HistogramOpts{
-			Namespace: "TestCallLatencyWithoutMeasurement",
-			Name:      "Latency",
-			Buckets:   prometheus.LinearBuckets(0, 50, 100),
-		},
-		[]string{"key"},
-	)
-	timer := NewTimer(m, time.Millisecond, "key")
-	assert.Equal(t, time.Duration(0), timer.End("dne"))
-	timer.Start("test")
-	assert.NotEqual(t, time.Duration(0), timer.End("test"))
-}

origin/proxy.go

@@ -7,21 +7,27 @@ import (
 	"io"
 	"net/http"
 	"strconv"
-	"strings"
 
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
 
+	"github.com/cloudflare/cloudflared/carrier"
 	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/ingress"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/cloudflare/cloudflared/websocket"
 )
 
 const (
-	TagHeaderNamePrefix = "Cf-Warp-Tag-"
+	// TagHeaderNamePrefix indicates a Cloudflared Warp Tag prefix that gets appended for warp traffic stream headers.
+	TagHeaderNamePrefix   = "Cf-Warp-Tag-"
+	LogFieldCFRay         = "cfRay"
+	LogFieldRule          = "ingressRule"
+	LogFieldOriginService = "originService"
 )
 
-type proxy struct {
+// Proxy represents a means to Proxy between cloudflared and the origin services.
+type Proxy struct {
 	ingressRules ingress.Ingress
 	warpRouting  *ingress.WarpRoutingService
 	tags         []tunnelpogs.Tag
@@ -29,13 +35,14 @@ type proxy struct {
 	bufferPool   *bufferPool
 }
 
+// NewOriginProxy returns a new instance of the Proxy struct.
 func NewOriginProxy(
 	ingressRules ingress.Ingress,
 	warpRouting *ingress.WarpRoutingService,
 	tags []tunnelpogs.Tag,
-	log *zerolog.Logger) connection.OriginProxy {
-
-	return &proxy{
+	log *zerolog.Logger,
+) *Proxy {
+	return &Proxy{
 		ingressRules: ingressRules,
 		warpRouting:  warpRouting,
 		tags:         tags,
@@ -44,35 +51,18 @@ func NewOriginProxy(
 	}
 }
 
-// Caller is responsible for writing any error to ResponseWriter
-func (p *proxy) Proxy(w connection.ResponseWriter, req *http.Request, sourceConnectionType connection.Type) error {
+// ProxyHTTP further depends on ingress rules to establish a connection with the origin service. This may be
+// a simple roundtrip or a tcp/websocket dial depending on ingres rule setup.
+func (p *Proxy) ProxyHTTP(
+	w connection.ResponseWriter,
+	req *http.Request,
+	isWebsocket bool,
+) error {
 	incrementRequests()
 	defer decrementConcurrentRequests()
 
-	cfRay := findCfRayHeader(req)
-	lbProbe := isLBProbeRequest(req)
-
-	serveCtx, cancel := context.WithCancel(req.Context())
-	defer cancel()
-
-	p.appendTagHeaders(req)
-	if sourceConnectionType == connection.TypeTCP {
-		if p.warpRouting == nil {
-			err := errors.New(`cloudflared received a request from WARP client, but your configuration has disabled ingress from WARP clients. To enable this, set "warp-routing:\n\t enabled: true" in your config.yaml`)
-			p.log.Error().Msg(err.Error())
-			return err
-		}
-		logFields := logFields{
-			cfRay:   cfRay,
-			lbProbe: lbProbe,
-			rule:    ingress.ServiceWarpRouting,
-		}
-		if err := p.proxyStreamRequest(serveCtx, w, req, sourceConnectionType, p.warpRouting.Proxy, logFields); err != nil {
-			p.logRequestError(err, cfRay, ingress.ServiceWarpRouting)
-			return err
-		}
-		return nil
-	}
+	cfRay := connection.FindCfRayHeader(req)
+	lbProbe := connection.IsLBProbeRequest(req)
 
 	rule, ruleNum := p.ingressRules.FindMatchingRule(req.Host, req.URL.Path)
 	logFields := logFields{
@@ -82,49 +72,113 @@ func (p *proxy) Proxy(w connection.ResponseWriter, req *http.Request, sourceConn
 	}
 	p.logRequest(req, logFields)
 
-	if sourceConnectionType == connection.TypeHTTP {
-		if err := p.proxyHTTPRequest(w, req, rule, logFields); err != nil {
-			p.logRequestError(err, cfRay, ruleNum)
+	switch originProxy := rule.Service.(type) {
+	case ingress.HTTPOriginProxy:
+		if err := p.proxyHTTPRequest(
+			w,
+			req,
+			originProxy,
+			isWebsocket,
+			rule.Config.DisableChunkedEncoding,
+			logFields,
+		); err != nil {
+			rule, srv := ruleField(p.ingressRules, ruleNum)
+			p.logRequestError(err, cfRay, rule, srv)
 			return err
 		}
 		return nil
-	}
 
-	connectionProxy, ok := rule.Service.(ingress.StreamBasedOriginProxy)
-	if !ok {
-		p.log.Error().Msgf("%s is not a connection-oriented service", rule.Service)
-		return fmt.Errorf("Not a connection-oriented service")
-	}
+	case ingress.StreamBasedOriginProxy:
+		dest, err := getDestFromRule(rule, req)
+		if err != nil {
+			return err
+		}
 
-	if err := p.proxyStreamRequest(serveCtx, w, req, sourceConnectionType, connectionProxy, logFields); err != nil {
-		p.logRequestError(err, cfRay, ruleNum)
+		rws := connection.NewHTTPResponseReadWriterAcker(w, req)
+		if err := p.proxyStream(req.Context(), rws, dest, originProxy, logFields); err != nil {
+			rule, srv := ruleField(p.ingressRules, ruleNum)
+			p.logRequestError(err, cfRay, rule, srv)
+			return err
+		}
+		return nil
+	default:
+		return fmt.Errorf("Unrecognized service: %s, %t", rule.Service, originProxy)
+	}
+}
+
+// ProxyTCP proxies to a TCP connection between the origin service and cloudflared.
+func (p *Proxy) ProxyTCP(
+	ctx context.Context,
+	rwa connection.ReadWriteAcker,
+	req *connection.TCPRequest,
+) error {
+	incrementRequests()
+	defer decrementConcurrentRequests()
+
+	if p.warpRouting == nil {
+		err := errors.New(`cloudflared received a request from WARP client, but your configuration has disabled ingress from WARP clients. To enable this, set "warp-routing:\n\t enabled: true" in your config.yaml`)
+		p.log.Error().Msg(err.Error())
 		return err
 	}
+
+	serveCtx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	logFields := logFields{
+		cfRay:   req.CFRay,
+		lbProbe: req.LBProbe,
+		rule:    ingress.ServiceWarpRouting,
+	}
+
+	if err := p.proxyStream(serveCtx, rwa, req.Dest, p.warpRouting.Proxy, logFields); err != nil {
+		p.logRequestError(err, req.CFRay, "", ingress.ServiceWarpRouting)
+		return err
+	}
+
 	return nil
 }
 
-func (p *proxy) proxyHTTPRequest(w connection.ResponseWriter, req *http.Request, rule *ingress.Rule, fields logFields) error {
-	// Support for WSGI Servers by switching transfer encoding from chunked to gzip/deflate
-	if rule.Config.DisableChunkedEncoding {
-		req.TransferEncoding = []string{"gzip", "deflate"}
-		cLength, err := strconv.Atoi(req.Header.Get("Content-Length"))
-		if err == nil {
-			req.ContentLength = int64(cLength)
+func ruleField(ing ingress.Ingress, ruleNum int) (ruleID string, srv string) {
+	srv = ing.Rules[ruleNum].Service.String()
+	if ing.IsSingleRule() {
+		return "", srv
+	}
+	return fmt.Sprintf("%d", ruleNum), srv
+}
+
+// ProxyHTTPRequest proxies requests of underlying type http and websocket to the origin service.
+func (p *Proxy) proxyHTTPRequest(
+	w connection.ResponseWriter,
+	req *http.Request,
+	httpService ingress.HTTPOriginProxy,
+	isWebsocket bool,
+	disableChunkedEncoding bool,
+	fields logFields,
+) error {
+	roundTripReq := req
+	if isWebsocket {
+		roundTripReq = req.Clone(req.Context())
+		roundTripReq.Header.Set("Connection", "Upgrade")
+		roundTripReq.Header.Set("Upgrade", "websocket")
+		roundTripReq.Header.Set("Sec-Websocket-Version", "13")
+		roundTripReq.ContentLength = 0
+		roundTripReq.Body = nil
+	} else {
+		// Support for WSGI Servers by switching transfer encoding from chunked to gzip/deflate
+		if disableChunkedEncoding {
+			roundTripReq.TransferEncoding = []string{"gzip", "deflate"}
+			cLength, err := strconv.Atoi(req.Header.Get("Content-Length"))
+			if err == nil {
+				roundTripReq.ContentLength = int64(cLength)
+			}
 		}
+		// Request origin to keep connection alive to improve performance
+		roundTripReq.Header.Set("Connection", "keep-alive")
 	}
 
-	// Request origin to keep connection alive to improve performance
-	req.Header.Set("Connection", "keep-alive")
-
-	httpService, ok := rule.Service.(ingress.HTTPOriginProxy)
-	if !ok {
-		p.log.Error().Msgf("%s is not a http service", rule.Service)
-		return fmt.Errorf("Not a http service")
-	}
-
-	resp, err := httpService.RoundTrip(req)
+	resp, err := httpService.RoundTrip(roundTripReq)
 	if err != nil {
-		return errors.Wrap(err, "Error proxying request to origin")
+		return errors.Wrap(err, "Unable to reach the origin service. The service may be down or it may not be responding to traffic from cloudflared")
 	}
 	defer resp.Body.Close()
 
@@ -132,6 +186,23 @@ func (p *proxy) proxyHTTPRequest(w connection.ResponseWriter, req *http.Request,
 	if err != nil {
 		return errors.Wrap(err, "Error writing response header")
 	}
+
+	if resp.StatusCode == http.StatusSwitchingProtocols {
+		rwc, ok := resp.Body.(io.ReadWriteCloser)
+		if !ok {
+			return errors.New("internal error: unsupported connection type")
+		}
+		defer rwc.Close()
+
+		eyeballStream := &bidirectionalStream{
+			writer: w,
+			reader: req.Body,
+		}
+
+		websocket.Stream(eyeballStream, rwc, p.log)
+		return nil
+	}
+
 	if connection.IsServerSentEvent(resp.Header) {
 		p.log.Debug().Msg("Detected Server-Side Events from Origin")
 		p.writeEventStream(w, resp.Body)
@@ -142,33 +213,30 @@ func (p *proxy) proxyHTTPRequest(w connection.ResponseWriter, req *http.Request,
 		defer p.bufferPool.Put(buf)
 		_, _ = io.CopyBuffer(w, resp.Body, buf)
 	}
+
 	p.logOriginResponse(resp, fields)
 	return nil
 }
 
-// proxyStreamRequest first establish a connection with origin, then it writes the status code and headers, and finally it streams data between
-// eyeball and origin.
-func (p *proxy) proxyStreamRequest(
-	serveCtx context.Context,
-	w connection.ResponseWriter,
-	req *http.Request,
-	sourceConnectionType connection.Type,
+// proxyStream proxies type TCP and other underlying types if the connection is defined as a stream oriented
+// ingress rule.
+func (p *Proxy) proxyStream(
+	ctx context.Context,
+	rwa connection.ReadWriteAcker,
+	dest string,
 	connectionProxy ingress.StreamBasedOriginProxy,
 	fields logFields,
 ) error {
-	originConn, resp, err := connectionProxy.EstablishConnection(req)
+	originConn, err := connectionProxy.EstablishConnection(dest)
 	if err != nil {
 		return err
 	}
-	if resp.Body != nil {
-		defer resp.Body.Close()
-	}
 
-	if err = w.WriteRespHeaders(resp.StatusCode, resp.Header); err != nil {
+	if err := rwa.AckConnection(); err != nil {
 		return err
 	}
 
-	streamCtx, cancel := context.WithCancel(serveCtx)
+	streamCtx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
 	go func() {
@@ -177,12 +245,7 @@ func (p *proxy) proxyStreamRequest(
 		originConn.Close()
 	}()
 
-	eyeballStream := &bidirectionalStream{
-		writer: w,
-		reader: req.Body,
-	}
-	originConn.Stream(serveCtx, eyeballStream, p.log)
-	p.logOriginResponse(resp, fields)
+	originConn.Stream(ctx, rwa, p.log)
 	return nil
 }
 
@@ -199,7 +262,7 @@ func (wr *bidirectionalStream) Write(p []byte) (n int, err error) {
 	return wr.writer.Write(p)
 }
 
-func (p *proxy) writeEventStream(w connection.ResponseWriter, respBody io.ReadCloser) {
+func (p *Proxy) writeEventStream(w connection.ResponseWriter, respBody io.ReadCloser) {
 	reader := bufio.NewReader(respBody)
 	for {
 		line, err := reader.ReadBytes('\n')
@@ -210,7 +273,7 @@ func (p *proxy) writeEventStream(w connection.ResponseWriter, respBody io.ReadCl
 	}
 }
 
-func (p *proxy) appendTagHeaders(r *http.Request) {
+func (p *Proxy) appendTagHeaders(r *http.Request) {
 	for _, tag := range p.tags {
 		r.Header.Add(TagHeaderNamePrefix+tag.Name, tag.Value)
 	}
@@ -222,7 +285,7 @@ type logFields struct {
 	rule    interface{}
 }
 
-func (p *proxy) logRequest(r *http.Request, fields logFields) {
+func (p *Proxy) logRequest(r *http.Request, fields logFields) {
 	if fields.cfRay != "" {
 		p.log.Debug().Msgf("CF-RAY: %s %s %s %s", fields.cfRay, r.Method, r.URL, r.Proto)
 	} else if fields.lbProbe {
@@ -230,8 +293,13 @@ func (p *proxy) logRequest(r *http.Request, fields logFields) {
 	} else {
 		p.log.Debug().Msgf("All requests should have a CF-RAY header. Please open a support ticket with Cloudflare. %s %s %s ", r.Method, r.URL, r.Proto)
 	}
-	p.log.Debug().Msgf("CF-RAY: %s Request Headers %+v", fields.cfRay, r.Header)
-	p.log.Debug().Msgf("CF-RAY: %s Serving with ingress rule %v", fields.cfRay, fields.rule)
+	p.log.Debug().
+		Str("CF-RAY", fields.cfRay).
+		Str("Header", fmt.Sprintf("%+v", r.Header)).
+		Str("host", r.Host).
+		Str("path", r.URL.Path).
+		Interface("rule", fields.rule).
+		Msg("Inbound request")
 
 	if contentLen := r.ContentLength; contentLen == -1 {
 		p.log.Debug().Msgf("CF-RAY: %s Request Content length unknown", fields.cfRay)
@@ -240,7 +308,7 @@ func (p *proxy) logRequest(r *http.Request, fields logFields) {
 	}
 }
 
-func (p *proxy) logOriginResponse(resp *http.Response, fields logFields) {
+func (p *Proxy) logOriginResponse(resp *http.Response, fields logFields) {
 	responseByCode.WithLabelValues(strconv.Itoa(resp.StatusCode)).Inc()
 	if fields.cfRay != "" {
 		p.log.Debug().Msgf("CF-RAY: %s Status: %s served by ingress %d", fields.cfRay, resp.Status, fields.rule)
@@ -258,19 +326,26 @@ func (p *proxy) logOriginResponse(resp *http.Response, fields logFields) {
 	}
 }
 
-func (p *proxy) logRequestError(err error, cfRay string, rule interface{}) {
+func (p *Proxy) logRequestError(err error, cfRay string, rule, service string) {
 	requestErrors.Inc()
+	log := p.log.Error().Err(err)
 	if cfRay != "" {
-		p.log.Error().Msgf("CF-RAY: %s Proxying to ingress %v error: %v", cfRay, rule, err)
-	} else {
-		p.log.Error().Msgf("Proxying to ingress %v error: %v", rule, err)
+		log = log.Str(LogFieldCFRay, cfRay)
+	}
+	if rule != "" {
+		log = log.Str(LogFieldRule, rule)
+	}
+	if service != "" {
+		log = log.Str(LogFieldOriginService, service)
+	}
+	log.Msg("")
+}
+
+func getDestFromRule(rule *ingress.Rule, req *http.Request) (string, error) {
+	switch rule.Service.String() {
+	case ingress.ServiceBastion:
+		return carrier.ResolveBastionDest(req)
+	default:
+		return rule.Service.String(), nil
 	}
 }
-
-func findCfRayHeader(req *http.Request) string {
-	return req.Header.Get("Cf-Ray")
-}
-
-func isLBProbeRequest(req *http.Request) bool {
-	return strings.HasPrefix(req.UserAgent(), lbProbeUserAgentPrefix)
-}

origin/proxy_test.go

@@ -6,9 +6,11 @@ import (
 	"flag"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"net"
 	"net/http"
 	"net/http/httptest"
+	"os"
 	"sync"
 	"testing"
 	"time"
@@ -19,6 +21,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"github.com/urfave/cli/v2"
+	"golang.org/x/sync/errgroup"
 
 	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/connection"
@@ -43,6 +46,10 @@ func newMockHTTPRespWriter() *mockHTTPRespWriter {
 	}
 }
 
+func (w *mockHTTPRespWriter) WriteResponse() error {
+	return nil
+}
+
 func (w *mockHTTPRespWriter) WriteRespHeaders(status int, header http.Header) error {
 	w.WriteHeader(status)
 	for header, val := range header {
@@ -79,6 +86,11 @@ func (w *mockWSRespWriter) respBody() io.ReadWriter {
 	return bytes.NewBuffer(data)
 }
 
+func (w *mockWSRespWriter) Close() error {
+	close(w.writeNotification)
+	return nil
+}
+
 func (w *mockWSRespWriter) Read(data []byte) (int, error) {
 	return w.reader.Read(data)
 }
@@ -125,43 +137,63 @@ func TestProxySingleOrigin(t *testing.T) {
 	require.NoError(t, ingressRule.StartOrigins(&wg, &log, ctx.Done(), errC))
 
 	proxy := NewOriginProxy(ingressRule, unusedWarpRoutingService, testTags, &log)
-	t.Run("testProxyHTTP", testProxyHTTP(t, proxy))
-	t.Run("testProxyWebsocket", testProxyWebsocket(t, proxy))
-	t.Run("testProxySSE", testProxySSE(t, proxy))
+	t.Run("testProxyHTTP", testProxyHTTP(proxy))
+	t.Run("testProxyWebsocket", testProxyWebsocket(proxy))
+	t.Run("testProxySSE", testProxySSE(proxy))
 	cancel()
 	wg.Wait()
 }
 
-func testProxyHTTP(t *testing.T, proxy connection.OriginProxy) func(t *testing.T) {
+func testProxyHTTP(proxy connection.OriginProxy) func(t *testing.T) {
 	return func(t *testing.T) {
 		responseWriter := newMockHTTPRespWriter()
 		req, err := http.NewRequest(http.MethodGet, "http://localhost:8080", nil)
 		require.NoError(t, err)
 
-		err = proxy.Proxy(responseWriter, req, connection.TypeHTTP)
+		err = proxy.ProxyHTTP(responseWriter, req, false)
 		require.NoError(t, err)
 
 		assert.Equal(t, http.StatusOK, responseWriter.Code)
 	}
 }
 
-func testProxyWebsocket(t *testing.T, proxy connection.OriginProxy) func(t *testing.T) {
+func testProxyWebsocket(proxy connection.OriginProxy) func(t *testing.T) {
 	return func(t *testing.T) {
 		// WSRoute is a websocket echo handler
-		ctx, cancel := context.WithCancel(context.Background())
+		const testTimeout = 5 * time.Second * 1000
+		ctx, cancel := context.WithTimeout(context.Background(), testTimeout)
+		defer cancel()
 		readPipe, writePipe := io.Pipe()
 		req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("http://localhost:8080%s", hello.WSRoute), readPipe)
-		responseWriter := newMockWSRespWriter(readPipe)
+		req.Header.Set("Sec-Websocket-Key", "dGhlIHNhbXBsZSBub25jZQ==")
+		req.Header.Set("Connection", "Upgrade")
+		req.Header.Set("Upgrade", "websocket")
+		responseWriter := newMockWSRespWriter(nil)
 
-		var wg sync.WaitGroup
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			err = proxy.Proxy(responseWriter, req, connection.TypeWebsocket)
+		finished := make(chan struct{})
+
+		errGroup, ctx := errgroup.WithContext(ctx)
+		errGroup.Go(func() error {
+			err = proxy.ProxyHTTP(responseWriter, req, true)
 			require.NoError(t, err)
 
 			require.Equal(t, http.StatusSwitchingProtocols, responseWriter.Code)
-		}()
+			return nil
+		})
+
+		errGroup.Go(func() error {
+			select {
+			case <-finished:
+			case <-ctx.Done():
+			}
+			if ctx.Err() == context.DeadlineExceeded {
+				t.Errorf("Test timed out")
+				readPipe.Close()
+				writePipe.Close()
+				responseWriter.Close()
+			}
+			return nil
+		})
 
 		msg := []byte("test websocket")
 		err = wsutil.WriteClientText(writePipe, msg)
@@ -179,12 +211,16 @@ func testProxyWebsocket(t *testing.T, proxy connection.OriginProxy) func(t *test
 		require.NoError(t, err)
 		require.Equal(t, msg, returnedMsg)
 
-		cancel()
-		wg.Wait()
+		_ = readPipe.Close()
+		_ = writePipe.Close()
+		_ = responseWriter.Close()
+
+		close(finished)
+		errGroup.Wait()
 	}
 }
 
-func testProxySSE(t *testing.T, proxy connection.OriginProxy) func(t *testing.T) {
+func testProxySSE(proxy connection.OriginProxy) func(t *testing.T) {
 	return func(t *testing.T) {
 		var (
 			pushCount = 50
@@ -199,7 +235,7 @@ func testProxySSE(t *testing.T, proxy connection.OriginProxy) func(t *testing.T)
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
-			err = proxy.Proxy(responseWriter, req, connection.TypeHTTP)
+			err = proxy.ProxyHTTP(responseWriter, req, false)
 			require.NoError(t, err)
 
 			require.Equal(t, http.StatusOK, responseWriter.Code)
@@ -243,26 +279,7 @@ func TestProxyMultipleOrigins(t *testing.T) {
 		},
 	}
 
-	ingress, err := ingress.ParseIngress(&config.Configuration{
-		TunnelID: t.Name(),
-		Ingress:  unvalidatedIngress,
-	})
-	require.NoError(t, err)
-
-	log := zerolog.Nop()
-
-	ctx, cancel := context.WithCancel(context.Background())
-	errC := make(chan error)
-	var wg sync.WaitGroup
-	require.NoError(t, ingress.StartOrigins(&wg, &log, ctx.Done(), errC))
-
-	proxy := NewOriginProxy(ingress, unusedWarpRoutingService, testTags, &log)
-
-	tests := []struct {
-		url            string
-		expectedStatus int
-		expectedBody   []byte
-	}{
+	tests := []MultipleIngressTest{
 		{
 			url:            "http://api.example.com",
 			expectedStatus: http.StatusCreated,
@@ -287,12 +304,37 @@ func TestProxyMultipleOrigins(t *testing.T) {
 		},
 	}
 
+	runIngressTestScenarios(t, unvalidatedIngress, tests)
+}
+
+type MultipleIngressTest struct {
+	url            string
+	expectedStatus int
+	expectedBody   []byte
+}
+
+func runIngressTestScenarios(t *testing.T, unvalidatedIngress []config.UnvalidatedIngressRule, tests []MultipleIngressTest) {
+	ingress, err := ingress.ParseIngress(&config.Configuration{
+		TunnelID: t.Name(),
+		Ingress:  unvalidatedIngress,
+	})
+	require.NoError(t, err)
+
+	log := zerolog.Nop()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	errC := make(chan error)
+	var wg sync.WaitGroup
+	require.NoError(t, ingress.StartOrigins(&wg, &log, ctx.Done(), errC))
+
+	proxy := NewOriginProxy(ingress, unusedWarpRoutingService, testTags, &log)
+
 	for _, test := range tests {
 		responseWriter := newMockHTTPRespWriter()
 		req, err := http.NewRequest(http.MethodGet, test.url, nil)
 		require.NoError(t, err)
 
-		err = proxy.Proxy(responseWriter, req, connection.TypeHTTP)
+		err = proxy.ProxyHTTP(responseWriter, req, false)
 		require.NoError(t, err)
 
 		assert.Equal(t, test.expectedStatus, responseWriter.Code)
@@ -320,7 +362,7 @@ func (errorOriginTransport) RoundTrip(*http.Request) (*http.Response, error) {
 }
 
 func TestProxyError(t *testing.T) {
-	ingress := ingress.Ingress{
+	ing := ingress.Ingress{
 		Rules: []ingress.Rule{
 			{
 				Hostname: "*",
@@ -334,13 +376,13 @@ func TestProxyError(t *testing.T) {
 
 	log := zerolog.Nop()
 
-	proxy := NewOriginProxy(ingress, unusedWarpRoutingService, testTags, &log)
+	proxy := NewOriginProxy(ing, unusedWarpRoutingService, testTags, &log)
 
 	responseWriter := newMockHTTPRespWriter()
 	req, err := http.NewRequest(http.MethodGet, "http://127.0.0.1", nil)
 	assert.NoError(t, err)
 
-	assert.Error(t, proxy.Proxy(responseWriter, req, connection.TypeHTTP))
+	assert.Error(t, proxy.ProxyHTTP(responseWriter, req, false))
 }
 
 type replayer struct {
@@ -533,8 +575,14 @@ func TestConnections(t *testing.T) {
 				},
 			},
 			want: want{
-				message: []byte{},
-				err:     true,
+				message: []byte("Forbidden\n"),
+				err:     false,
+				headers: map[string][]string{
+					"Content-Length":         {"10"},
+					"Content-Type":           {"text/plain; charset=utf-8"},
+					"Sec-Websocket-Version":  {"13"},
+					"X-Content-Type-Options": {"nosniff"},
+				},
 			},
 		},
 		{
@@ -573,6 +621,7 @@ func TestConnections(t *testing.T) {
 			ingressRule.StartOrigins(&wg, logger, ctx.Done(), errC)
 			proxy := NewOriginProxy(ingressRule, test.args.warpRoutingService, testTags, logger)
 
+			dest := ln.Addr().String()
 			req, err := http.NewRequest(
 				http.MethodGet,
 				test.args.ingressServiceScheme+ln.Addr().String(),
@@ -590,8 +639,12 @@ func TestConnections(t *testing.T) {
 					replayer.Write(resp)
 				}()
 			}
-
-			err = proxy.Proxy(respWriter, req, test.args.connectionType)
+			if test.args.connectionType == connection.TypeTCP {
+				rws := connection.NewHTTPResponseReadWriterAcker(respWriter, req)
+				err = proxy.ProxyTCP(ctx, rws, &connection.TCPRequest{Dest: dest})
+			} else {
+				err = proxy.ProxyHTTP(respWriter, req, test.args.connectionType == connection.TypeWebsocket)
+			}
 
 			cancel()
 			assert.Equal(t, test.want.err, err != nil)
@@ -603,6 +656,45 @@ func TestConnections(t *testing.T) {
 	}
 }
 
+func TestUnixSocketOrigin(t *testing.T) {
+	file, err := ioutil.TempFile("", "unix.sock")
+	require.NoError(t, err)
+	os.Remove(file.Name()) // remove the file since binding the socket expects to create it
+
+	l, err := net.Listen("unix", file.Name())
+	require.NoError(t, err)
+	defer l.Close()
+	defer os.Remove(file.Name())
+
+	api := &httptest.Server{
+		Listener: l,
+		Config:   &http.Server{Handler: mockAPI{}},
+	}
+	api.Start()
+	defer api.Close()
+
+	unvalidatedIngress := []config.UnvalidatedIngressRule{
+		{
+			Hostname: "unix.example.com",
+			Service:  "unix:" + file.Name(),
+		},
+		{
+			Hostname: "*",
+			Service:  "http_status:404",
+		},
+	}
+
+	tests := []MultipleIngressTest{
+		{
+			url:            "http://unix.example.com",
+			expectedStatus: http.StatusCreated,
+			expectedBody:   []byte("Created"),
+		},
+	}
+
+	runIngressTestScenarios(t, unvalidatedIngress, tests)
+}
+
 type requestBody struct {
 	pw *io.PipeWriter
 	pr *io.PipeReader
@@ -729,6 +821,8 @@ func (w *wsRespWriter) WriteRespHeaders(status int, header http.Header) error {
 
 // respHeaders is a test function to read respHeaders
 func (w *wsRespWriter) headers() http.Header {
+	// Removing indeterminstic header because it cannot be asserted.
+	w.responseHeaders.Del("Date")
 	return w.responseHeaders
 }
 
@@ -744,6 +838,10 @@ func newTCPRespWriter(w io.Writer) *mockTCPRespWriter {
 	}
 }
 
+func (m *mockTCPRespWriter) Read(p []byte) (n int, err error) {
+	return len(p), nil
+}
+
 func (m *mockTCPRespWriter) Write(p []byte) (n int, err error) {
 	return m.w.Write(p)
 }

origin/reconnect.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/prometheus/client_golang/prometheus"
 
+	"github.com/cloudflare/cloudflared/retry"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
@@ -103,7 +104,7 @@ func (cm *reconnectCredentialManager) SetConnDigest(connID uint8, digest []byte)
 
 func (cm *reconnectCredentialManager) RefreshAuth(
 	ctx context.Context,
-	backoff *BackoffHandler,
+	backoff *retry.BackoffHandler,
 	authenticate func(ctx context.Context, numPreviousAttempts int) (tunnelpogs.AuthOutcome, error),
 ) (retryTimer <-chan time.Time, err error) {
 	authOutcome, err := authenticate(ctx, backoff.Retries())
@@ -121,11 +122,11 @@ func (cm *reconnectCredentialManager) RefreshAuth(
 	case tunnelpogs.AuthSuccess:
 		cm.SetReconnectToken(outcome.JWT())
 		cm.authSuccess.Inc()
-		return timeAfter(outcome.RefreshAfter()), nil
+		return retry.Clock.After(outcome.RefreshAfter()), nil
 	case tunnelpogs.AuthUnknown:
 		duration := outcome.RefreshAfter()
 		cm.authFail.WithLabelValues(outcome.Error()).Inc()
-		return timeAfter(duration), nil
+		return retry.Clock.After(duration), nil
 	case tunnelpogs.AuthFail:
 		cm.authFail.WithLabelValues(outcome.Error()).Inc()
 		return nil, outcome

origin/reconnect_test.go

@@ -10,6 +10,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
+	"github.com/cloudflare/cloudflared/retry"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
@@ -17,11 +18,11 @@ func TestRefreshAuthBackoff(t *testing.T) {
 	rcm := newReconnectCredentialManager(t.Name(), t.Name(), 4)
 
 	var wait time.Duration
-	timeAfter = func(d time.Duration) <-chan time.Time {
+	retry.Clock.After = func(d time.Duration) <-chan time.Time {
 		wait = d
 		return time.After(d)
 	}
-	backoff := &BackoffHandler{MaxRetries: 3}
+	backoff := &retry.BackoffHandler{MaxRetries: 3}
 	auth := func(ctx context.Context, n int) (tunnelpogs.AuthOutcome, error) {
 		return nil, fmt.Errorf("authentication failure")
 	}
@@ -45,7 +46,7 @@ func TestRefreshAuthBackoff(t *testing.T) {
 
 	// The backoff timer should have been reset. To confirm this, make timeNow
 	// return a value after the backoff timer's grace period
-	timeNow = func() time.Time {
+	retry.Clock.Now = func() time.Time {
 		expectedGracePeriod := time.Duration(time.Second * 2 << backoff.MaxRetries)
 		return time.Now().Add(expectedGracePeriod * 2)
 	}
@@ -57,12 +58,12 @@ func TestRefreshAuthSuccess(t *testing.T) {
 	rcm := newReconnectCredentialManager(t.Name(), t.Name(), 4)
 
 	var wait time.Duration
-	timeAfter = func(d time.Duration) <-chan time.Time {
+	retry.Clock.After = func(d time.Duration) <-chan time.Time {
 		wait = d
 		return time.After(d)
 	}
 
-	backoff := &BackoffHandler{MaxRetries: 3}
+	backoff := &retry.BackoffHandler{MaxRetries: 3}
 	auth := func(ctx context.Context, n int) (tunnelpogs.AuthOutcome, error) {
 		return tunnelpogs.NewAuthSuccess([]byte("jwt"), 19), nil
 	}
@@ -81,12 +82,12 @@ func TestRefreshAuthUnknown(t *testing.T) {
 	rcm := newReconnectCredentialManager(t.Name(), t.Name(), 4)
 
 	var wait time.Duration
-	timeAfter = func(d time.Duration) <-chan time.Time {
+	retry.Clock.After = func(d time.Duration) <-chan time.Time {
 		wait = d
 		return time.After(d)
 	}
 
-	backoff := &BackoffHandler{MaxRetries: 3}
+	backoff := &retry.BackoffHandler{MaxRetries: 3}
 	auth := func(ctx context.Context, n int) (tunnelpogs.AuthOutcome, error) {
 		return tunnelpogs.NewAuthUnknown(errors.New("auth unknown"), 19), nil
 	}
@@ -104,7 +105,7 @@ func TestRefreshAuthUnknown(t *testing.T) {
 func TestRefreshAuthFail(t *testing.T) {
 	rcm := newReconnectCredentialManager(t.Name(), t.Name(), 4)
 
-	backoff := &BackoffHandler{MaxRetries: 3}
+	backoff := &retry.BackoffHandler{MaxRetries: 3}
 	auth := func(ctx context.Context, n int) (tunnelpogs.AuthOutcome, error) {
 		return tunnelpogs.NewAuthFail(errors.New("auth fail")), nil
 	}

origin/supervisor.go

@@ -13,6 +13,7 @@ import (
 	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/edgediscovery"
 	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/retry"
 	"github.com/cloudflare/cloudflared/signal"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
@@ -112,10 +113,10 @@ func (s *Supervisor) Run(
 	var tunnelsWaiting []int
 	tunnelsActive := s.config.HAConnections
 
-	backoff := BackoffHandler{MaxRetries: s.config.Retries, BaseTime: tunnelRetryDuration, RetryForever: true}
+	backoff := retry.BackoffHandler{MaxRetries: s.config.Retries, BaseTime: tunnelRetryDuration, RetryForever: true}
 	var backoffTimer <-chan time.Time
 
-	refreshAuthBackoff := &BackoffHandler{MaxRetries: refreshAuthMaxBackoff, BaseTime: refreshAuthRetryDuration, RetryForever: true}
+	refreshAuthBackoff := &retry.BackoffHandler{MaxRetries: refreshAuthMaxBackoff, BaseTime: refreshAuthRetryDuration, RetryForever: true}
 	var refreshAuthBackoffTimer <-chan time.Time
 
 	if s.useReconnectToken {

origin/tunnel.go

@@ -18,6 +18,7 @@ import (
 	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/edgediscovery"
 	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/retry"
 	"github.com/cloudflare/cloudflared/signal"
 	"github.com/cloudflare/cloudflared/tunnelrpc"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
@@ -25,7 +26,6 @@ import (
 
 const (
 	dialTimeout              = 15 * time.Second
-	lbProbeUserAgentPrefix   = "Mozilla/5.0 (compatible; Cloudflare-Traffic-Manager/1.0; +https://www.cloudflare.com/traffic-manager/;"
 	FeatureSerializedHeaders = "serialized_headers"
 	FeatureQuickReconnects   = "quick_reconnects"
 )
@@ -46,7 +46,6 @@ type TunnelConfig struct {
 	HAConnections    int
 	IncidentLookup   IncidentLookup
 	IsAutoupdated    bool
-	IsFreeTunnel     bool
 	LBPool           string
 	Tags             []tunnelpogs.Tag
 	Log              *zerolog.Logger
@@ -138,7 +137,7 @@ func ServeTunnelLoop(
 	connLog := config.Log.With().Uint8(connection.LogFieldConnIndex, connIndex).Logger()
 
 	protocolFallback := &protocolFallback{
-		BackoffHandler{MaxRetries: config.Retries},
+		retry.BackoffHandler{MaxRetries: config.Retries},
 		config.ProtocolSelector.Current(),
 		false,
 	}
@@ -195,18 +194,18 @@ func ServeTunnelLoop(
 // protocolFallback is a wrapper around backoffHandler that will try fallback option when backoff reaches
 // max retries
 type protocolFallback struct {
-	BackoffHandler
+	retry.BackoffHandler
 	protocol   connection.Protocol
 	inFallback bool
 }
 
 func (pf *protocolFallback) reset() {
-	pf.resetNow()
+	pf.ResetNow()
 	pf.inFallback = false
 }
 
 func (pf *protocolFallback) fallback(fallback connection.Protocol) {
-	pf.resetNow()
+	pf.ResetNow()
 	pf.protocol = fallback
 	pf.inFallback = true
 }
@@ -281,7 +280,7 @@ func ServeTunnel(
 	}
 
 	if protocol == connection.HTTP2 {
-		connOptions := config.ConnectionOptions(edgeConn.LocalAddr().String(), uint8(backoff.retries))
+		connOptions := config.ConnectionOptions(edgeConn.LocalAddr().String(), uint8(backoff.Retries()))
 		err = ServeHTTP2(
 			ctx,
 			connLog,
@@ -382,7 +381,7 @@ func ServeH2mux(
 
 	errGroup.Go(func() error {
 		if config.NamedTunnel != nil {
-			connOptions := config.ConnectionOptions(edgeConn.LocalAddr().String(), uint8(connectedFuse.backoff.retries))
+			connOptions := config.ConnectionOptions(edgeConn.LocalAddr().String(), uint8(connectedFuse.backoff.Retries()))
 			return handler.ServeNamedTunnel(serveCtx, config.NamedTunnel, connOptions, connectedFuse)
 		}
 		registrationOptions := config.RegistrationOptions(connIndex, edgeConn.LocalAddr().String(), cloudflaredUUID)
@@ -416,6 +415,7 @@ func ServeHTTP2(
 		config.Observer,
 		connIndex,
 		connectedFuse,
+		config.Log,
 		gracefulShutdownC,
 	)
 

origin/tunnel_test.go

@@ -8,6 +8,7 @@ import (
 	"github.com/stretchr/testify/assert"
 
 	"github.com/cloudflare/cloudflared/connection"
+	"github.com/cloudflare/cloudflared/retry"
 )
 
 type dynamicMockFetcher struct {
@@ -26,7 +27,7 @@ func (dmf *dynamicMockFetcher) fetch() connection.PercentageFetcher {
 
 func TestWaitForBackoffFallback(t *testing.T) {
 	maxRetries := uint(3)
-	backoff := BackoffHandler{
+	backoff := retry.BackoffHandler{
 		MaxRetries: maxRetries,
 		BaseTime:   time.Millisecond * 10,
 	}

quic/pogs.go

@@ -0,0 +1,88 @@
+package quic
+
+import (
+	capnp "zombiezen.com/go/capnproto2"
+	"zombiezen.com/go/capnproto2/pogs"
+
+	"github.com/cloudflare/cloudflared/quic/schema"
+)
+
+// ConnectionType indicates the type of underlying connection proxied within the QUIC stream.
+type ConnectionType uint16
+
+const (
+	ConnectionTypeHTTP ConnectionType = iota
+	ConnectionTypeWebsocket
+	ConnectionTypeTCP
+)
+
+// ConnectRequest is the representation of metadata sent at the start of a QUIC application handshake.
+type ConnectRequest struct {
+	Dest     string         `capnp:"dest"`
+	Type     ConnectionType `capnp:"type"`
+	Metadata []Metadata     `capnp:"metadata"`
+}
+
+// Metadata is a representation of key value based data sent via RequestMeta.
+type Metadata struct {
+	Key string `capnp:"key"`
+	Val string `capnp:"val"`
+}
+
+func (r *ConnectRequest) fromPogs(msg *capnp.Message) error {
+	metadata, err := schema.ReadRootConnectRequest(msg)
+	if err != nil {
+		return err
+	}
+	return pogs.Extract(r, schema.ConnectRequest_TypeID, metadata.Struct)
+}
+
+func (r *ConnectRequest) toPogs() (*capnp.Message, error) {
+	msg, seg, err := capnp.NewMessage(capnp.SingleSegment(nil))
+	if err != nil {
+		return nil, err
+	}
+
+	root, err := schema.NewRootConnectRequest(seg)
+	if err != nil {
+		return nil, err
+	}
+
+	if err := pogs.Insert(schema.ConnectRequest_TypeID, root.Struct, r); err != nil {
+		return nil, err
+	}
+
+	return msg, nil
+}
+
+// ConnectResponse is a representation of metadata sent as a response to a QUIC application handshake.
+type ConnectResponse struct {
+	Error    string     `capnp:"error"`
+	Metadata []Metadata `capnp:"metadata"`
+}
+
+func (r *ConnectResponse) fromPogs(msg *capnp.Message) error {
+	metadata, err := schema.ReadRootConnectResponse(msg)
+	if err != nil {
+		return err
+	}
+	return pogs.Extract(r, schema.ConnectResponse_TypeID, metadata.Struct)
+}
+
+func (r *ConnectResponse) toPogs() (*capnp.Message, error) {
+	msg, seg, err := capnp.NewMessage(capnp.SingleSegment(nil))
+	if err != nil {
+		return nil, err
+	}
+
+	root, err := schema.NewRootConnectResponse(seg)
+	if err != nil {
+		return nil, err
+	}
+
+	if err := pogs.Insert(schema.ConnectResponse_TypeID, root.Struct, r); err != nil {
+		return nil, err
+	}
+
+	return msg, nil
+}

quic/quic_protocol.go

@@ -0,0 +1,115 @@
+package quic
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+
+	capnp "zombiezen.com/go/capnproto2"
+)
+
+// protocolSignature is a custom protocol signature to ensure that whoever performs a handshake does not write data
+// before writing the metadata.
+var protocolSignature = []byte{0x0A, 0x36, 0xCD, 0x12, 0xA1, 0x3E}
+
+// ReadConnectRequestData reads the handshake data from a QUIC stream.
+func ReadConnectRequestData(stream io.Reader) (*ConnectRequest, error) {
+	if err := verifySignature(stream); err != nil {
+		return nil, err
+	}
+
+	msg, err := capnp.NewDecoder(stream).Decode()
+	if err != nil {
+		return nil, err
+	}
+
+	r := &ConnectRequest{}
+	if err := r.fromPogs(msg); err != nil {
+		return nil, err
+	}
+	return r, nil
+}
+
+// WriteConnectRequestData writes requestMeta to a stream.
+func WriteConnectRequestData(stream io.Writer, dest string, connectionType ConnectionType, metadata ...Metadata) error {
+	connectRequest := &ConnectRequest{
+		Dest:     dest,
+		Type:     connectionType,
+		Metadata: metadata,
+	}
+
+	msg, err := connectRequest.toPogs()
+	if err != nil {
+		return err
+	}
+
+	if err := writePreamble(stream); err != nil {
+		return err
+	}
+	return capnp.NewEncoder(stream).Encode(msg)
+}
+
+// ReadConnectResponseData reads the response to a RequestMeta in a stream.
+func ReadConnectResponseData(stream io.Reader) (*ConnectResponse, error) {
+	if err := verifySignature(stream); err != nil {
+		return nil, err
+	}
+
+	msg, err := capnp.NewDecoder(stream).Decode()
+	if err != nil {
+		return nil, err
+	}
+
+	r := &ConnectResponse{}
+	if err := r.fromPogs(msg); err != nil {
+		return nil, err
+	}
+	return r, nil
+}
+
+// WriteConnectResponseData writes response to a QUIC stream.
+func WriteConnectResponseData(stream io.Writer, respErr error, metadata ...Metadata) error {
+	var connectResponse *ConnectResponse
+	if respErr != nil {
+		connectResponse = &ConnectResponse{
+			Error: respErr.Error(),
+		}
+	} else {
+		connectResponse = &ConnectResponse{
+			Metadata: metadata,
+		}
+	}
+
+	msg, err := connectResponse.toPogs()
+	if err != nil {
+		return err
+	}
+
+	if err := writePreamble(stream); err != nil {
+		return err
+	}
+	return capnp.NewEncoder(stream).Encode(msg)
+}
+
+func writePreamble(stream io.Writer) error {
+	return writeSignature(stream)
+	// TODO : TUN-4613 Write protocol version here
+}
+
+func writeSignature(stream io.Writer) error {
+	_, err := stream.Write(protocolSignature)
+	return err
+}
+
+func verifySignature(stream io.Reader) error {
+	signature := make([]byte, len(protocolSignature))
+	if _, err := io.ReadFull(stream, signature); err != nil {
+		return err
+	}
+
+	if !bytes.Equal(signature[0:], protocolSignature) {
+		return fmt.Errorf("Wrong signature: %v", signature)
+	}
+
+	return nil
+}

quic/quic_protocol_test.go

@@ -0,0 +1,88 @@
+package quic
+
+import (
+	"bytes"
+	"errors"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestConnectRequestData(t *testing.T) {
+	var tests = []struct {
+		name           string
+		hostname       string
+		connectionType ConnectionType
+		metadata       []Metadata
+	}{
+		{
+			name:           "Signature verified and request metadata is unmarshaled and read correctly",
+			hostname:       "tunnel.com",
+			connectionType: ConnectionTypeHTTP,
+			metadata: []Metadata{
+				Metadata{
+					Key: "key",
+					Val: "1234",
+				},
+			},
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			b := &bytes.Buffer{}
+			err := WriteConnectRequestData(b, test.hostname, test.connectionType, test.metadata...)
+			require.NoError(t, err)
+			reqMeta, err := ReadConnectRequestData(b)
+			require.NoError(t, err)
+
+			assert.Equal(t, test.metadata, reqMeta.Metadata)
+			assert.Equal(t, test.hostname, reqMeta.Dest)
+			assert.Equal(t, test.connectionType, reqMeta.Type)
+		})
+	}
+}
+
+func TestConnectResponseMeta(t *testing.T) {
+	var tests = []struct {
+		name     string
+		err      error
+		metadata []Metadata
+	}{
+		{
+			name: "Signature verified and response metadata is unmarshaled and read correctly",
+			metadata: []Metadata{
+				Metadata{
+					Key: "key",
+					Val: "1234",
+				},
+			},
+		},
+		{
+			name: "If error is not empty, other fields should be blank",
+			err:  errors.New("something happened"),
+			metadata: []Metadata{
+				Metadata{
+					Key: "key",
+					Val: "1234",
+				},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			b := &bytes.Buffer{}
+			err := WriteConnectResponseData(b, test.err, test.metadata...)
+			require.NoError(t, err)
+			respMeta, err := ReadConnectResponseData(b)
+			require.NoError(t, err)
+
+			if respMeta.Error == "" {
+				assert.Equal(t, test.metadata, respMeta.Metadata)
+			} else {
+				assert.Equal(t, 0, len(respMeta.Metadata))
+			}
+		})
+	}
+}

quic/schema/quic_metadata_protocol.capnp

@@ -0,0 +1,28 @@
+using Go = import "/go.capnp";
+@0xb29021ef7421cc32;
+
+$Go.package("schema");
+$Go.import("schema");
+
+
+struct ConnectRequest{
+	dest @0 :Text;
+	type @1 :ConnectionType;
+	metadata @2 :List(Metadata);
+}
+
+enum ConnectionType{
+	http @0;
+	websocket @1;
+	tcp @2;
+}
+
+struct Metadata {
+    key @0 :Text;
+	val @1 :Text;
+}
+
+struct ConnectResponse{
+	error @0 :Text;
+	metadata @1 :List(Metadata);
+}

quic/schema/quic_metadata_protocol.capnp.go

@@ -0,0 +1,395 @@
+// Code generated by capnpc-go. DO NOT EDIT.
+
+package schema
+
+import (
+	capnp "zombiezen.com/go/capnproto2"
+	text "zombiezen.com/go/capnproto2/encoding/text"
+	schemas "zombiezen.com/go/capnproto2/schemas"
+)
+
+type ConnectRequest struct{ capnp.Struct }
+
+// ConnectRequest_TypeID is the unique identifier for the type ConnectRequest.
+const ConnectRequest_TypeID = 0xc47116a1045e4061
+
+func NewConnectRequest(s *capnp.Segment) (ConnectRequest, error) {
+	st, err := capnp.NewStruct(s, capnp.ObjectSize{DataSize: 8, PointerCount: 2})
+	return ConnectRequest{st}, err
+}
+
+func NewRootConnectRequest(s *capnp.Segment) (ConnectRequest, error) {
+	st, err := capnp.NewRootStruct(s, capnp.ObjectSize{DataSize: 8, PointerCount: 2})
+	return ConnectRequest{st}, err
+}
+
+func ReadRootConnectRequest(msg *capnp.Message) (ConnectRequest, error) {
+	root, err := msg.RootPtr()
+	return ConnectRequest{root.Struct()}, err
+}
+
+func (s ConnectRequest) String() string {
+	str, _ := text.Marshal(0xc47116a1045e4061, s.Struct)
+	return str
+}
+
+func (s ConnectRequest) Dest() (string, error) {
+	p, err := s.Struct.Ptr(0)
+	return p.Text(), err
+}
+
+func (s ConnectRequest) HasDest() bool {
+	p, err := s.Struct.Ptr(0)
+	return p.IsValid() || err != nil
+}
+
+func (s ConnectRequest) DestBytes() ([]byte, error) {
+	p, err := s.Struct.Ptr(0)
+	return p.TextBytes(), err
+}
+
+func (s ConnectRequest) SetDest(v string) error {
+	return s.Struct.SetText(0, v)
+}
+
+func (s ConnectRequest) Type() ConnectionType {
+	return ConnectionType(s.Struct.Uint16(0))
+}
+
+func (s ConnectRequest) SetType(v ConnectionType) {
+	s.Struct.SetUint16(0, uint16(v))
+}
+
+func (s ConnectRequest) Metadata() (Metadata_List, error) {
+	p, err := s.Struct.Ptr(1)
+	return Metadata_List{List: p.List()}, err
+}
+
+func (s ConnectRequest) HasMetadata() bool {
+	p, err := s.Struct.Ptr(1)
+	return p.IsValid() || err != nil
+}
+
+func (s ConnectRequest) SetMetadata(v Metadata_List) error {
+	return s.Struct.SetPtr(1, v.List.ToPtr())
+}
+
+// NewMetadata sets the metadata field to a newly
+// allocated Metadata_List, preferring placement in s's segment.
+func (s ConnectRequest) NewMetadata(n int32) (Metadata_List, error) {
+	l, err := NewMetadata_List(s.Struct.Segment(), n)
+	if err != nil {
+		return Metadata_List{}, err
+	}
+	err = s.Struct.SetPtr(1, l.List.ToPtr())
+	return l, err
+}
+
+// ConnectRequest_List is a list of ConnectRequest.
+type ConnectRequest_List struct{ capnp.List }
+
+// NewConnectRequest creates a new list of ConnectRequest.
+func NewConnectRequest_List(s *capnp.Segment, sz int32) (ConnectRequest_List, error) {
+	l, err := capnp.NewCompositeList(s, capnp.ObjectSize{DataSize: 8, PointerCount: 2}, sz)
+	return ConnectRequest_List{l}, err
+}
+
+func (s ConnectRequest_List) At(i int) ConnectRequest { return ConnectRequest{s.List.Struct(i)} }
+
+func (s ConnectRequest_List) Set(i int, v ConnectRequest) error { return s.List.SetStruct(i, v.Struct) }
+
+func (s ConnectRequest_List) String() string {
+	str, _ := text.MarshalList(0xc47116a1045e4061, s.List)
+	return str
+}
+
+// ConnectRequest_Promise is a wrapper for a ConnectRequest promised by a client call.
+type ConnectRequest_Promise struct{ *capnp.Pipeline }
+
+func (p ConnectRequest_Promise) Struct() (ConnectRequest, error) {
+	s, err := p.Pipeline.Struct()
+	return ConnectRequest{s}, err
+}
+
+type ConnectionType uint16
+
+// ConnectionType_TypeID is the unique identifier for the type ConnectionType.
+const ConnectionType_TypeID = 0xc52e1bac26d379c8
+
+// Values of ConnectionType.
+const (
+	ConnectionType_http      ConnectionType = 0
+	ConnectionType_websocket ConnectionType = 1
+	ConnectionType_tcp       ConnectionType = 2
+)
+
+// String returns the enum's constant name.
+func (c ConnectionType) String() string {
+	switch c {
+	case ConnectionType_http:
+		return "http"
+	case ConnectionType_websocket:
+		return "websocket"
+	case ConnectionType_tcp:
+		return "tcp"
+
+	default:
+		return ""
+	}
+}
+
+// ConnectionTypeFromString returns the enum value with a name,
+// or the zero value if there's no such value.
+func ConnectionTypeFromString(c string) ConnectionType {
+	switch c {
+	case "http":
+		return ConnectionType_http
+	case "websocket":
+		return ConnectionType_websocket
+	case "tcp":
+		return ConnectionType_tcp
+
+	default:
+		return 0
+	}
+}
+
+type ConnectionType_List struct{ capnp.List }
+
+func NewConnectionType_List(s *capnp.Segment, sz int32) (ConnectionType_List, error) {
+	l, err := capnp.NewUInt16List(s, sz)
+	return ConnectionType_List{l.List}, err
+}
+
+func (l ConnectionType_List) At(i int) ConnectionType {
+	ul := capnp.UInt16List{List: l.List}
+	return ConnectionType(ul.At(i))
+}
+
+func (l ConnectionType_List) Set(i int, v ConnectionType) {
+	ul := capnp.UInt16List{List: l.List}
+	ul.Set(i, uint16(v))
+}
+
+type Metadata struct{ capnp.Struct }
+
+// Metadata_TypeID is the unique identifier for the type Metadata.
+const Metadata_TypeID = 0xe1446b97bfd1cd37
+
+func NewMetadata(s *capnp.Segment) (Metadata, error) {
+	st, err := capnp.NewStruct(s, capnp.ObjectSize{DataSize: 0, PointerCount: 2})
+	return Metadata{st}, err
+}
+
+func NewRootMetadata(s *capnp.Segment) (Metadata, error) {
+	st, err := capnp.NewRootStruct(s, capnp.ObjectSize{DataSize: 0, PointerCount: 2})
+	return Metadata{st}, err
+}
+
+func ReadRootMetadata(msg *capnp.Message) (Metadata, error) {
+	root, err := msg.RootPtr()
+	return Metadata{root.Struct()}, err
+}
+
+func (s Metadata) String() string {
+	str, _ := text.Marshal(0xe1446b97bfd1cd37, s.Struct)
+	return str
+}
+
+func (s Metadata) Key() (string, error) {
+	p, err := s.Struct.Ptr(0)
+	return p.Text(), err
+}
+
+func (s Metadata) HasKey() bool {
+	p, err := s.Struct.Ptr(0)
+	return p.IsValid() || err != nil
+}
+
+func (s Metadata) KeyBytes() ([]byte, error) {
+	p, err := s.Struct.Ptr(0)
+	return p.TextBytes(), err
+}
+
+func (s Metadata) SetKey(v string) error {
+	return s.Struct.SetText(0, v)
+}
+
+func (s Metadata) Val() (string, error) {
+	p, err := s.Struct.Ptr(1)
+	return p.Text(), err
+}
+
+func (s Metadata) HasVal() bool {
+	p, err := s.Struct.Ptr(1)
+	return p.IsValid() || err != nil
+}
+
+func (s Metadata) ValBytes() ([]byte, error) {
+	p, err := s.Struct.Ptr(1)
+	return p.TextBytes(), err
+}
+
+func (s Metadata) SetVal(v string) error {
+	return s.Struct.SetText(1, v)
+}
+
+// Metadata_List is a list of Metadata.
+type Metadata_List struct{ capnp.List }
+
+// NewMetadata creates a new list of Metadata.
+func NewMetadata_List(s *capnp.Segment, sz int32) (Metadata_List, error) {
+	l, err := capnp.NewCompositeList(s, capnp.ObjectSize{DataSize: 0, PointerCount: 2}, sz)
+	return Metadata_List{l}, err
+}
+
+func (s Metadata_List) At(i int) Metadata { return Metadata{s.List.Struct(i)} }
+
+func (s Metadata_List) Set(i int, v Metadata) error { return s.List.SetStruct(i, v.Struct) }
+
+func (s Metadata_List) String() string {
+	str, _ := text.MarshalList(0xe1446b97bfd1cd37, s.List)
+	return str
+}
+
+// Metadata_Promise is a wrapper for a Metadata promised by a client call.
+type Metadata_Promise struct{ *capnp.Pipeline }
+
+func (p Metadata_Promise) Struct() (Metadata, error) {
+	s, err := p.Pipeline.Struct()
+	return Metadata{s}, err
+}
+
+type ConnectResponse struct{ capnp.Struct }
+
+// ConnectResponse_TypeID is the unique identifier for the type ConnectResponse.
+const ConnectResponse_TypeID = 0xb1032ec91cef8727
+
+func NewConnectResponse(s *capnp.Segment) (ConnectResponse, error) {
+	st, err := capnp.NewStruct(s, capnp.ObjectSize{DataSize: 0, PointerCount: 2})
+	return ConnectResponse{st}, err
+}
+
+func NewRootConnectResponse(s *capnp.Segment) (ConnectResponse, error) {
+	st, err := capnp.NewRootStruct(s, capnp.ObjectSize{DataSize: 0, PointerCount: 2})
+	return ConnectResponse{st}, err
+}
+
+func ReadRootConnectResponse(msg *capnp.Message) (ConnectResponse, error) {
+	root, err := msg.RootPtr()
+	return ConnectResponse{root.Struct()}, err
+}
+
+func (s ConnectResponse) String() string {
+	str, _ := text.Marshal(0xb1032ec91cef8727, s.Struct)
+	return str
+}
+
+func (s ConnectResponse) Error() (string, error) {
+	p, err := s.Struct.Ptr(0)
+	return p.Text(), err
+}
+
+func (s ConnectResponse) HasError() bool {
+	p, err := s.Struct.Ptr(0)
+	return p.IsValid() || err != nil
+}
+
+func (s ConnectResponse) ErrorBytes() ([]byte, error) {
+	p, err := s.Struct.Ptr(0)
+	return p.TextBytes(), err
+}
+
+func (s ConnectResponse) SetError(v string) error {
+	return s.Struct.SetText(0, v)
+}
+
+func (s ConnectResponse) Metadata() (Metadata_List, error) {
+	p, err := s.Struct.Ptr(1)
+	return Metadata_List{List: p.List()}, err
+}
+
+func (s ConnectResponse) HasMetadata() bool {
+	p, err := s.Struct.Ptr(1)
+	return p.IsValid() || err != nil
+}
+
+func (s ConnectResponse) SetMetadata(v Metadata_List) error {
+	return s.Struct.SetPtr(1, v.List.ToPtr())
+}
+
+// NewMetadata sets the metadata field to a newly
+// allocated Metadata_List, preferring placement in s's segment.
+func (s ConnectResponse) NewMetadata(n int32) (Metadata_List, error) {
+	l, err := NewMetadata_List(s.Struct.Segment(), n)
+	if err != nil {
+		return Metadata_List{}, err
+	}
+	err = s.Struct.SetPtr(1, l.List.ToPtr())
+	return l, err
+}
+
+// ConnectResponse_List is a list of ConnectResponse.
+type ConnectResponse_List struct{ capnp.List }
+
+// NewConnectResponse creates a new list of ConnectResponse.
+func NewConnectResponse_List(s *capnp.Segment, sz int32) (ConnectResponse_List, error) {
+	l, err := capnp.NewCompositeList(s, capnp.ObjectSize{DataSize: 0, PointerCount: 2}, sz)
+	return ConnectResponse_List{l}, err
+}
+
+func (s ConnectResponse_List) At(i int) ConnectResponse { return ConnectResponse{s.List.Struct(i)} }
+
+func (s ConnectResponse_List) Set(i int, v ConnectResponse) error {
+	return s.List.SetStruct(i, v.Struct)
+}
+
+func (s ConnectResponse_List) String() string {
+	str, _ := text.MarshalList(0xb1032ec91cef8727, s.List)
+	return str
+}
+
+// ConnectResponse_Promise is a wrapper for a ConnectResponse promised by a client call.
+type ConnectResponse_Promise struct{ *capnp.Pipeline }
+
+func (p ConnectResponse_Promise) Struct() (ConnectResponse, error) {
+	s, err := p.Pipeline.Struct()
+	return ConnectResponse{s}, err
+}
+
+const schema_b29021ef7421cc32 = "x\xda\xb4\x91\xcfk\x13A\x1c\xc5\xdf\x9bI\xba\x1e\xa2" +
+	"\x9b!\xd5\x8b\x8a\xa4\xf8+ES\xdb(\xa2\xa7\x80\x15" +
+	"TZ\xcc\x14\xcf\x96u;\x98\x92vw\x92\x9dZ\xf2" +
+	"\x17x\x15/\xe2\xd1\xbb \x15<\x0b\xa2\xa0\xa2\x07\x11" +
+	"\xff\x80\xfe\x05=y\xf0\xb42)\xdb@)\x08Bo" +
+	"\xdfy<\xe6}\xbe\xdfW\xfd\xd5\x16\xb3\xe5\xc7\x04t" +
+	"\xb5<\x91_x\xbas\xeaKSnA5\x98\xcf}" +
+	"\xab\xbb\x9d\xfa\xb3\xb7(\x8b\x00\x98}\xf9\x95\xea]\x00" +
+	"\xa8\xadM0\x8f\xda\x0fK\xafN\xf4?B7\xb8\xdf" +
+	"\xda\xaa\xf3\x03k7\x18\x00\xb5k|\x03\xe6\x9f\x87?" +
+	"\xcf\xbf>\xd9\xfc\x04\xd5\x10c3\xd8\xda\xf6\xce?#" +
+	"\xe7o\xde\x07\xf3\xeb\xdf\x7f\xbc\x7f\xd1\x9b\xdf>\x80\xa0" +
+	"uT<g\xed\x9c\x1fku\xe1!\xfa\x1b\xab\xf1L" +
+	"\x16w'\xccz4\xe3\x1f\xcb\xeb\xc6E+\x91\x8b\x96" +
+	"\xed ui\x9c\xae5\xe3\xc8&\xf6\xe6\xad4IL" +
+	"\xec\x96Lf\xd3$3@\x87\xd4Gd\x09(\x11P" +
+	"\x8d9@\x9f\x95\xd4W\x04\x159I/^\xbe\x07\xe8" +
+	"K\x92\xfa\x8e\xe0\x193\x18\xa4\x03V X\x01\xf3\"" +
+	"\x06\x00\x8f\x81\x1dIV\xc7\xe8\xa0\x17\xff\x87\xae\xbfa" +
+	"27b\xab\xec\xb1\xdd\x9e\x06t[R/\x08\x16h" +
+	"w\xbd6/\xa9;\x82Jp\x92\x02P\x8b\x9ewA" +
+	"Rw\x05\xc3\x15\x93\xb9\x027tCk\x18\x8e[\x00" +
+	"\x19\x1e\xda\x16\xabi\xf2`h\xcd\xee\x16#\xb0\xd3\xd3" +
+	"\xfe3u|\x09\xa0Pj\x0a\x08\xbb\xce\xd9|\xd3<" +
+	"\xca\xd2\xb8g@\x17\xb8\xd8\xeeE\x95\xff\x19\xb5\xb8\xab" +
+	"3\xdaW\xe3\xd4A5z\xf1\xa2\xa4\xbe*\x18\xf4\xcc" +
+	"\xb0\xb8J\xf0$Z+\xe6\xbf\x01\x00\x00\xff\xff\xf5\xed" +
+	"\xc9\xfe"
+
+func init() {
+	schemas.Register(schema_b29021ef7421cc32,
+		0xb1032ec91cef8727,
+		0xc47116a1045e4061,
+		0xc52e1bac26d379c8,
+		0xe1446b97bfd1cd37)
+}

retry/backoffhandler.go

@@ -1,4 +1,4 @@
-package origin
+package retry
 
 import (
 	"context"
@@ -7,10 +7,15 @@ import (
 )
 
 // Redeclare time functions so they can be overridden in tests.
-var (
-	timeNow   = time.Now
-	timeAfter = time.After
-)
+type clock struct {
+	Now   func() time.Time
+	After func(d time.Duration) <-chan time.Time
+}
+
+var Clock = clock{
+	Now:   time.Now,
+	After: time.After,
+}
 
 // BackoffHandler manages exponential backoff and limits the maximum number of retries.
 // The base time period is 1 second, doubling with each retry.
@@ -39,7 +44,7 @@ func (b BackoffHandler) GetMaxBackoffDuration(ctx context.Context) (time.Duratio
 		return time.Duration(0), false
 	default:
 	}
-	if !b.resetDeadline.IsZero() && timeNow().After(b.resetDeadline) {
+	if !b.resetDeadline.IsZero() && Clock.Now().After(b.resetDeadline) {
 		// b.retries would be set to 0 at this point
 		return time.Second, true
 	}
@@ -53,7 +58,7 @@ func (b BackoffHandler) GetMaxBackoffDuration(ctx context.Context) (time.Duratio
 // BackoffTimer returns a channel that sends the current time when the exponential backoff timeout expires.
 // Returns nil if the maximum number of retries have been used.
 func (b *BackoffHandler) BackoffTimer() <-chan time.Time {
-	if !b.resetDeadline.IsZero() && timeNow().After(b.resetDeadline) {
+	if !b.resetDeadline.IsZero() && Clock.Now().After(b.resetDeadline) {
 		b.retries = 0
 		b.resetDeadline = time.Time{}
 	}
@@ -66,7 +71,7 @@ func (b *BackoffHandler) BackoffTimer() <-chan time.Time {
 	}
 	maxTimeToWait := time.Duration(b.GetBaseTime() * 1 << (b.retries))
 	timeToWait := time.Duration(rand.Int63n(maxTimeToWait.Nanoseconds()))
-	return timeAfter(timeToWait)
+	return Clock.After(timeToWait)
 }
 
 // Backoff is used to wait according to exponential backoff. Returns false if the
@@ -89,7 +94,7 @@ func (b *BackoffHandler) Backoff(ctx context.Context) bool {
 func (b *BackoffHandler) SetGracePeriod() {
 	maxTimeToWait := b.GetBaseTime() * 2 << (b.retries + 1)
 	timeToWait := time.Duration(rand.Int63n(maxTimeToWait.Nanoseconds()))
-	b.resetDeadline = timeNow().Add(timeToWait)
+	b.resetDeadline = Clock.Now().Add(timeToWait)
 }
 
 func (b BackoffHandler) GetBaseTime() time.Duration {
@@ -108,6 +113,6 @@ func (b *BackoffHandler) ReachedMaxRetries() bool {
 	return b.retries == b.MaxRetries
 }
 
-func (b *BackoffHandler) resetNow() {
+func (b *BackoffHandler) ResetNow() {
 	b.resetDeadline = time.Now()
 }

retry/backoffhandler_test.go

@@ -1,4 +1,4 @@
-package origin
+package retry
 
 import (
 	"context"
@@ -14,7 +14,7 @@ func immediateTimeAfter(time.Duration) <-chan time.Time {
 
 func TestBackoffRetries(t *testing.T) {
 	// make backoff return immediately
-	timeAfter = immediateTimeAfter
+	Clock.After = immediateTimeAfter
 	ctx := context.Background()
 	backoff := BackoffHandler{MaxRetries: 3}
 	if !backoff.Backoff(ctx) {
@@ -33,7 +33,7 @@ func TestBackoffRetries(t *testing.T) {
 
 func TestBackoffCancel(t *testing.T) {
 	// prevent backoff from returning normally
-	timeAfter = func(time.Duration) <-chan time.Time { return make(chan time.Time) }
+	Clock.After = func(time.Duration) <-chan time.Time { return make(chan time.Time) }
 	ctx, cancelFunc := context.WithCancel(context.Background())
 	backoff := BackoffHandler{MaxRetries: 3}
 	cancelFunc()
@@ -47,10 +47,10 @@ func TestBackoffCancel(t *testing.T) {
 
 func TestBackoffGracePeriod(t *testing.T) {
 	currentTime := time.Now()
-	// make timeNow return whatever we like
-	timeNow = func() time.Time { return currentTime }
+	// make Clock.Now return whatever we like
+	Clock.Now = func() time.Time { return currentTime }
 	// make backoff return immediately
-	timeAfter = immediateTimeAfter
+	Clock.After = immediateTimeAfter
 	ctx := context.Background()
 	backoff := BackoffHandler{MaxRetries: 1}
 	if !backoff.Backoff(ctx) {
@@ -71,7 +71,7 @@ func TestBackoffGracePeriod(t *testing.T) {
 
 func TestGetMaxBackoffDurationRetries(t *testing.T) {
 	// make backoff return immediately
-	timeAfter = immediateTimeAfter
+	Clock.After = immediateTimeAfter
 	ctx := context.Background()
 	backoff := BackoffHandler{MaxRetries: 3}
 	if _, ok := backoff.GetMaxBackoffDuration(ctx); !ok {
@@ -96,7 +96,7 @@ func TestGetMaxBackoffDurationRetries(t *testing.T) {
 
 func TestGetMaxBackoffDuration(t *testing.T) {
 	// make backoff return immediately
-	timeAfter = immediateTimeAfter
+	Clock.After = immediateTimeAfter
 	ctx := context.Background()
 	backoff := BackoffHandler{MaxRetries: 3}
 	if duration, ok := backoff.GetMaxBackoffDuration(ctx); !ok || duration > time.Second*2 {
@@ -118,7 +118,7 @@ func TestGetMaxBackoffDuration(t *testing.T) {
 
 func TestBackoffRetryForever(t *testing.T) {
 	// make backoff return immediately
-	timeAfter = immediateTimeAfter
+	Clock.After = immediateTimeAfter
 	ctx := context.Background()
 	backoff := BackoffHandler{MaxRetries: 3, RetryForever: true}
 	if duration, ok := backoff.GetMaxBackoffDuration(ctx); !ok || duration > time.Second*2 {

sshgen/sshgen_test.go

@@ -1,3 +1,5 @@
+// +build !windows
+
 package sshgen
 
 import (

token/token.go

@@ -18,13 +18,14 @@ import (
 	"github.com/rs/zerolog"
 
 	"github.com/cloudflare/cloudflared/config"
-	"github.com/cloudflare/cloudflared/origin"
+	"github.com/cloudflare/cloudflared/retry"
 )
 
 const (
 	keyName               = "token"
 	tokenCookie           = "CF_Authorization"
 	appDomainHeader       = "CF-Access-Domain"
+	appAUDHeader          = "CF-Access-Aud"
 	AccessLoginWorkerPath = "/cdn-cgi/access/login"
 )
 
@@ -36,7 +37,7 @@ type AppInfo struct {
 
 type lock struct {
 	lockFilePath string
-	backoff      *origin.BackoffHandler
+	backoff      *retry.BackoffHandler
 	sigHandler   *signalHandler
 }
 
@@ -45,7 +46,7 @@ type signalHandler struct {
 	signals    []os.Signal
 }
 
-type appJWTPayload struct {
+type jwtPayload struct {
 	Aud   []string `json:"aud"`
 	Email string   `json:"email"`
 	Exp   int      `json:"exp"`
@@ -56,17 +57,12 @@ type appJWTPayload struct {
 	Subt  string   `json:"sub"`
 }
 
-type orgJWTPayload struct {
-	appJWTPayload
-	Aud string `json:"aud"`
-}
-
 type transferServiceResponse struct {
 	AppToken string `json:"app_token"`
 	OrgToken string `json:"org_token"`
 }
 
-func (p appJWTPayload) isExpired() bool {
+func (p jwtPayload) isExpired() bool {
 	return int(time.Now().Unix()) > p.Exp
 }
 
@@ -94,7 +90,7 @@ func newLock(path string) *lock {
 	lockPath := path + ".lock"
 	return &lock{
 		lockFilePath: lockPath,
-		backoff:      &origin.BackoffHandler{MaxRetries: 7},
+		backoff:      &retry.BackoffHandler{MaxRetries: 7},
 		sigHandler: &signalHandler{
 			signals: []os.Signal{syscall.SIGINT, syscall.SIGTERM},
 		},
@@ -270,14 +266,19 @@ func GetAppInfo(reqURL *url.URL) (*AppInfo, error) {
 		return nil, errors.Wrap(err, "failed to get app info")
 	}
 	resp.Body.Close()
-	location := resp.Request.URL
-	if !strings.Contains(location.Path, AccessLoginWorkerPath) {
-		return nil, fmt.Errorf("failed to get Access app info for %s", reqURL.String())
-	}
 
-	aud := resp.Request.URL.Query().Get("kid")
-	if aud == "" {
-		return nil, errors.New("Empty app aud")
+	var aud string
+	location := resp.Request.URL
+	if strings.Contains(location.Path, AccessLoginWorkerPath) {
+		aud = resp.Request.URL.Query().Get("kid")
+		if aud == "" {
+			return nil, errors.New("Empty app aud")
+		}
+	} else if audHeader := resp.Header.Get(appAUDHeader); audHeader != "" {
+		// 403/401 from the edge will have aud in a header
+		aud = audHeader
+	} else {
+		return nil, fmt.Errorf("failed to find Access application at %s", reqURL.String())
 	}
 
 	domain := resp.Header.Get(appDomainHeader)
@@ -286,7 +287,6 @@ func GetAppInfo(reqURL *url.URL) (*AppInfo, error) {
 	}
 
 	return &AppInfo{location.Hostname(), aud, domain}, nil
-
 }
 
 // exchangeOrgToken attaches an org token to a request to the appURL and returns an app token. This uses the Access SSO
@@ -341,7 +341,7 @@ func GetOrgTokenIfExists(authDomain string) (string, error) {
 	if err != nil {
 		return "", err
 	}
-	var payload orgJWTPayload
+	var payload jwtPayload
 	err = json.Unmarshal(token.Payload, &payload)
 	if err != nil {
 		return "", err
@@ -363,7 +363,7 @@ func GetAppTokenIfExists(appInfo *AppInfo) (string, error) {
 	if err != nil {
 		return "", err
 	}
-	var payload appJWTPayload
+	var payload jwtPayload
 	err = json.Unmarshal(token.Payload, &payload)
 	if err != nil {
 		return "", err

token/transfer.go

@@ -15,7 +15,7 @@ import (
 )
 
 const (
-	baseStoreURL  = "https://login.argotunnel.com/"
+	baseStoreURL  = "https://login.cloudflareaccess.org/"
 	clientTimeout = time.Second * 60
 )
 
@@ -83,11 +83,13 @@ func buildRequestURL(baseURL *url.URL, key, value string, cli, useHostOnly bool)
 	if useHostOnly {
 		baseURL.Path = ""
 	}
+	// TODO: pass arg for tunnel login
 	if !cli {
 		return baseURL.String(), nil
 	}
 	q.Set("redirect_url", baseURL.String()) // we add the token as a query param on both the redirect_url and the main url
 	q.Set("send_org_token", "true")         // indicates that the cli endpoint should return both the org and app token
+	q.Set("edge_token_transfer", "true")    // use new LoginHelper service built on workers
 	baseURL.RawQuery = q.Encode()           // and this actual baseURL.
 	baseURL.Path = "cdn-cgi/access/cli"
 	return baseURL.String(), nil
@@ -103,9 +105,6 @@ func transferRequest(requestURL string, log *zerolog.Logger) ([]byte, string, er
 		if err != nil {
 			return nil, "", err
 		} else if len(buf) > 0 {
-			if err := putSuccess(client, requestURL); err != nil {
-				log.Err(err).Msg("Failed to update resource success")
-			}
 			return buf, key, nil
 		}
 	}
@@ -136,22 +135,3 @@ func poll(client *http.Client, requestURL string, log *zerolog.Logger) ([]byte,
 	}
 	return buf.Bytes(), resp.Header.Get("service-public-key"), nil
 }
-
-// putSuccess tells the server we successfully downloaded the resource
-func putSuccess(client *http.Client, requestURL string) error {
-	req, err := http.NewRequest("PUT", requestURL+"/ok", nil)
-	if err != nil {
-		return err
-	}
-
-	resp, err := client.Do(req)
-	if err != nil {
-		return err
-	}
-
-	resp.Body.Close()
-	if resp.StatusCode != 200 {
-		return fmt.Errorf("HTTP Response Status Code: %d", resp.StatusCode)
-	}
-	return nil
-}

tunnelstore/client.go

@@ -72,6 +72,7 @@ type Route interface {
 	json.Marshaler
 	RecordType() string
 	UnmarshalResult(body io.Reader) (RouteResult, error)
+	String() string
 }
 
 type RouteResult interface {
@@ -80,27 +81,32 @@ type RouteResult interface {
 }
 
 type DNSRoute struct {
-	userHostname string
+	userHostname      string
+	overwriteExisting bool
 }
 
 type DNSRouteResult struct {
 	route *DNSRoute
 	CName Change `json:"cname"`
+	Name  string `json:"name"`
 }
 
-func NewDNSRoute(userHostname string) Route {
+func NewDNSRoute(userHostname string, overwriteExisting bool) Route {
 	return &DNSRoute{
-		userHostname: userHostname,
+		userHostname:      userHostname,
+		overwriteExisting: overwriteExisting,
 	}
 }
 
 func (dr *DNSRoute) MarshalJSON() ([]byte, error) {
 	s := struct {
-		Type         string `json:"type"`
-		UserHostname string `json:"user_hostname"`
+		Type              string `json:"type"`
+		UserHostname      string `json:"user_hostname"`
+		OverwriteExisting bool   `json:"overwrite_existing"`
 	}{
-		Type:         dr.RecordType(),
-		UserHostname: dr.userHostname,
+		Type:              dr.RecordType(),
+		UserHostname:      dr.userHostname,
+		OverwriteExisting: dr.overwriteExisting,
 	}
 	return json.Marshal(&s)
 }
@@ -116,6 +122,10 @@ func (dr *DNSRoute) RecordType() string {
 	return "dns"
 }
 
+func (dr *DNSRoute) String() string {
+	return fmt.Sprintf("%s %s", dr.RecordType(), dr.userHostname)
+}
+
 func (res *DNSRouteResult) SuccessSummary() string {
 	var msgFmt string
 	switch res.CName {
@@ -126,7 +136,16 @@ func (res *DNSRouteResult) SuccessSummary() string {
 	case ChangeUnchanged:
 		msgFmt = "%s is already configured to route to your tunnel"
 	}
-	return fmt.Sprintf(msgFmt, res.route.userHostname)
+	return fmt.Sprintf(msgFmt, res.hostname())
+}
+
+// hostname yields the resulting name for the DNS route; if that is not available from Cloudflare API, then the
+// requested name is returned instead (should not be the common path, it is just a fall-back).
+func (res *DNSRouteResult) hostname() string {
+	if res.Name != "" {
+		return res.Name
+	}
+	return res.route.userHostname
 }
 
 type LBRoute struct {
@@ -164,6 +183,10 @@ func (lr *LBRoute) RecordType() string {
 	return "lb"
 }
 
+func (lb *LBRoute) String() string {
+	return fmt.Sprintf("%s %s %s", lb.RecordType(), lb.lbName, lb.lbPool)
+}
+
 func (lr *LBRoute) UnmarshalResult(body io.Reader) (RouteResult, error) {
 	var result LBRouteResult
 	err := parseResponse(body, &result)

vendor/go.uber.org/automaxprocs/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2017 Uber Technologies, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

vendor/go.uber.org/automaxprocs/internal/cgroups/cgroup.go

@@ -0,0 +1,78 @@
+// Copyright (c) 2017 Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+// +build linux
+
+package cgroups
+
+import (
+	"bufio"
+	"io"
+	"os"
+	"path/filepath"
+	"strconv"
+)
+
+// CGroup represents the data structure for a Linux control group.
+type CGroup struct {
+	path string
+}
+
+// NewCGroup returns a new *CGroup from a given path.
+func NewCGroup(path string) *CGroup {
+	return &CGroup{path: path}
+}
+
+// Path returns the path of the CGroup*.
+func (cg *CGroup) Path() string {
+	return cg.path
+}
+
+// ParamPath returns the path of the given cgroup param under itself.
+func (cg *CGroup) ParamPath(param string) string {
+	return filepath.Join(cg.path, param)
+}
+
+// readFirstLine reads the first line from a cgroup param file.
+func (cg *CGroup) readFirstLine(param string) (string, error) {
+	paramFile, err := os.Open(cg.ParamPath(param))
+	if err != nil {
+		return "", err
+	}
+	defer paramFile.Close()
+
+	scanner := bufio.NewScanner(paramFile)
+	if scanner.Scan() {
+		return scanner.Text(), nil
+	}
+	if err := scanner.Err(); err != nil {
+		return "", err
+	}
+	return "", io.ErrUnexpectedEOF
+}
+
+// readInt parses the first line from a cgroup param file as int.
+func (cg *CGroup) readInt(param string) (int, error) {
+	text, err := cg.readFirstLine(param)
+	if err != nil {
+		return 0, err
+	}
+	return strconv.Atoi(text)
+}

vendor/go.uber.org/automaxprocs/internal/cgroups/cgroups.go

@@ -0,0 +1,117 @@
+// Copyright (c) 2017 Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+// +build linux
+
+package cgroups
+
+const (
+	// _cgroupFSType is the Linux CGroup file system type used in
+	// `/proc/$PID/mountinfo`.
+	_cgroupFSType = "cgroup"
+	// _cgroupSubsysCPU is the CPU CGroup subsystem.
+	_cgroupSubsysCPU = "cpu"
+	// _cgroupSubsysCPUAcct is the CPU accounting CGroup subsystem.
+	_cgroupSubsysCPUAcct = "cpuacct"
+	// _cgroupSubsysCPUSet is the CPUSet CGroup subsystem.
+	_cgroupSubsysCPUSet = "cpuset"
+	// _cgroupSubsysMemory is the Memory CGroup subsystem.
+	_cgroupSubsysMemory = "memory"
+
+	// _cgroupCPUCFSQuotaUsParam is the file name for the CGroup CFS quota
+	// parameter.
+	_cgroupCPUCFSQuotaUsParam = "cpu.cfs_quota_us"
+	// _cgroupCPUCFSPeriodUsParam is the file name for the CGroup CFS period
+	// parameter.
+	_cgroupCPUCFSPeriodUsParam = "cpu.cfs_period_us"
+)
+
+const (
+	_procPathCGroup    = "/proc/self/cgroup"
+	_procPathMountInfo = "/proc/self/mountinfo"
+)
+
+// CGroups is a map that associates each CGroup with its subsystem name.
+type CGroups map[string]*CGroup
+
+// NewCGroups returns a new *CGroups from given `mountinfo` and `cgroup` files
+// under for some process under `/proc` file system (see also proc(5) for more
+// information).
+func NewCGroups(procPathMountInfo, procPathCGroup string) (CGroups, error) {
+	cgroupSubsystems, err := parseCGroupSubsystems(procPathCGroup)
+	if err != nil {
+		return nil, err
+	}
+
+	cgroups := make(CGroups)
+	newMountPoint := func(mp *MountPoint) error {
+		if mp.FSType != _cgroupFSType {
+			return nil
+		}
+
+		for _, opt := range mp.SuperOptions {
+			subsys, exists := cgroupSubsystems[opt]
+			if !exists {
+				continue
+			}
+
+			cgroupPath, err := mp.Translate(subsys.Name)
+			if err != nil {
+				return err
+			}
+			cgroups[opt] = NewCGroup(cgroupPath)
+		}
+
+		return nil
+	}
+
+	if err := parseMountInfo(procPathMountInfo, newMountPoint); err != nil {
+		return nil, err
+	}
+	return cgroups, nil
+}
+
+// NewCGroupsForCurrentProcess returns a new *CGroups instance for the current
+// process.
+func NewCGroupsForCurrentProcess() (CGroups, error) {
+	return NewCGroups(_procPathMountInfo, _procPathCGroup)
+}
+
+// CPUQuota returns the CPU quota applied with the CPU cgroup controller.
+// It is a result of `cpu.cfs_quota_us / cpu.cfs_period_us`. If the value of
+// `cpu.cfs_quota_us` was not set (-1), the method returns `(-1, nil)`.
+func (cg CGroups) CPUQuota() (float64, bool, error) {
+	cpuCGroup, exists := cg[_cgroupSubsysCPU]
+	if !exists {
+		return -1, false, nil
+	}
+
+	cfsQuotaUs, err := cpuCGroup.readInt(_cgroupCPUCFSQuotaUsParam)
+	if defined := cfsQuotaUs > 0; err != nil || !defined {
+		return -1, defined, err
+	}
+
+	cfsPeriodUs, err := cpuCGroup.readInt(_cgroupCPUCFSPeriodUsParam)
+	if err != nil {
+		return -1, false, err
+	}
+
+	return float64(cfsQuotaUs) / float64(cfsPeriodUs), true, nil
+}

vendor/go.uber.org/automaxprocs/internal/cgroups/doc.go

@@ -0,0 +1,23 @@
+// Copyright (c) 2017 Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+// Package cgroups provides utilities to access Linux control group (CGroups)
+// parameters (CPU quota, for example) for a given process.
+package cgroups

vendor/go.uber.org/automaxprocs/internal/cgroups/errors.go

@@ -0,0 +1,51 @@
+// Copyright (c) 2017 Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+// +build linux
+
+package cgroups
+
+import "fmt"
+
+type cgroupSubsysFormatInvalidError struct {
+	line string
+}
+
+type mountPointFormatInvalidError struct {
+	line string
+}
+
+type pathNotExposedFromMountPointError struct {
+	mountPoint string
+	root       string
+	path       string
+}
+
+func (err cgroupSubsysFormatInvalidError) Error() string {
+	return fmt.Sprintf("invalid format for CGroupSubsys: %q", err.line)
+}
+
+func (err mountPointFormatInvalidError) Error() string {
+	return fmt.Sprintf("invalid format for MountPoint: %q", err.line)
+}
+
+func (err pathNotExposedFromMountPointError) Error() string {
+	return fmt.Sprintf("path %q is not a descendant of mount point root %q and cannot be exposed from %q", err.path, err.root, err.mountPoint)
+}

vendor/go.uber.org/automaxprocs/internal/cgroups/mountpoint.go

@@ -0,0 +1,166 @@
+// Copyright (c) 2017 Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+// +build linux
+
+package cgroups
+
+import (
+	"bufio"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+)
+
+const (
+	_mountInfoSep               = " "
+	_mountInfoOptsSep           = ","
+	_mountInfoOptionalFieldsSep = "-"
+)
+
+const (
+	_miFieldIDMountID = iota
+	_miFieldIDParentID
+	_miFieldIDDeviceID
+	_miFieldIDRoot
+	_miFieldIDMountPoint
+	_miFieldIDOptions
+	_miFieldIDOptionalFields
+
+	_miFieldCountFirstHalf
+)
+
+const (
+	_miFieldOffsetFSType = iota
+	_miFieldOffsetMountSource
+	_miFieldOffsetSuperOptions
+
+	_miFieldCountSecondHalf
+)
+
+const _miFieldCountMin = _miFieldCountFirstHalf + _miFieldCountSecondHalf
+
+// MountPoint is the data structure for the mount points in
+// `/proc/$PID/mountinfo`. See also proc(5) for more information.
+type MountPoint struct {
+	MountID        int
+	ParentID       int
+	DeviceID       string
+	Root           string
+	MountPoint     string
+	Options        []string
+	OptionalFields []string
+	FSType         string
+	MountSource    string
+	SuperOptions   []string
+}
+
+// NewMountPointFromLine parses a line read from `/proc/$PID/mountinfo` and
+// returns a new *MountPoint.
+func NewMountPointFromLine(line string) (*MountPoint, error) {
+	fields := strings.Split(line, _mountInfoSep)
+
+	if len(fields) < _miFieldCountMin {
+		return nil, mountPointFormatInvalidError{line}
+	}
+
+	mountID, err := strconv.Atoi(fields[_miFieldIDMountID])
+	if err != nil {
+		return nil, err
+	}
+
+	parentID, err := strconv.Atoi(fields[_miFieldIDParentID])
+	if err != nil {
+		return nil, err
+	}
+
+	for i, field := range fields[_miFieldIDOptionalFields:] {
+		if field == _mountInfoOptionalFieldsSep {
+			fsTypeStart := _miFieldIDOptionalFields + i + 1
+
+			if len(fields) != fsTypeStart+_miFieldCountSecondHalf {
+				return nil, mountPointFormatInvalidError{line}
+			}
+
+			miFieldIDFSType := _miFieldOffsetFSType + fsTypeStart
+			miFieldIDMountSource := _miFieldOffsetMountSource + fsTypeStart
+			miFieldIDSuperOptions := _miFieldOffsetSuperOptions + fsTypeStart
+
+			return &MountPoint{
+				MountID:        mountID,
+				ParentID:       parentID,
+				DeviceID:       fields[_miFieldIDDeviceID],
+				Root:           fields[_miFieldIDRoot],
+				MountPoint:     fields[_miFieldIDMountPoint],
+				Options:        strings.Split(fields[_miFieldIDOptions], _mountInfoOptsSep),
+				OptionalFields: fields[_miFieldIDOptionalFields:(fsTypeStart - 1)],
+				FSType:         fields[miFieldIDFSType],
+				MountSource:    fields[miFieldIDMountSource],
+				SuperOptions:   strings.Split(fields[miFieldIDSuperOptions], _mountInfoOptsSep),
+			}, nil
+		}
+	}
+
+	return nil, mountPointFormatInvalidError{line}
+}
+
+// Translate converts an absolute path inside the *MountPoint's file system to
+// the host file system path in the mount namespace the *MountPoint belongs to.
+func (mp *MountPoint) Translate(absPath string) (string, error) {
+	relPath, err := filepath.Rel(mp.Root, absPath)
+
+	if err != nil {
+		return "", err
+	}
+	if relPath == ".." || strings.HasPrefix(relPath, "../") {
+		return "", pathNotExposedFromMountPointError{
+			mountPoint: mp.MountPoint,
+			root:       mp.Root,
+			path:       absPath,
+		}
+	}
+
+	return filepath.Join(mp.MountPoint, relPath), nil
+}
+
+// parseMountInfo parses procPathMountInfo (usually at `/proc/$PID/mountinfo`)
+// and yields parsed *MountPoint into newMountPoint.
+func parseMountInfo(procPathMountInfo string, newMountPoint func(*MountPoint) error) error {
+	mountInfoFile, err := os.Open(procPathMountInfo)
+	if err != nil {
+		return err
+	}
+	defer mountInfoFile.Close()
+
+	scanner := bufio.NewScanner(mountInfoFile)
+
+	for scanner.Scan() {
+		mountPoint, err := NewMountPointFromLine(scanner.Text())
+		if err != nil {
+			return err
+		}
+		if err := newMountPoint(mountPoint); err != nil {
+			return err
+		}
+	}
+
+	return scanner.Err()
+}

vendor/go.uber.org/automaxprocs/internal/cgroups/subsys.go

@@ -0,0 +1,102 @@
+// Copyright (c) 2017 Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+// +build linux
+
+package cgroups
+
+import (
+	"bufio"
+	"os"
+	"strconv"
+	"strings"
+)
+
+const (
+	_cgroupSep       = ":"
+	_cgroupSubsysSep = ","
+)
+
+const (
+	_csFieldIDID = iota
+	_csFieldIDSubsystems
+	_csFieldIDName
+	_csFieldCount
+)
+
+// CGroupSubsys represents the data structure for entities in
+// `/proc/$PID/cgroup`. See also proc(5) for more information.
+type CGroupSubsys struct {
+	ID         int
+	Subsystems []string
+	Name       string
+}
+
+// NewCGroupSubsysFromLine returns a new *CGroupSubsys by parsing a string in
+// the format of `/proc/$PID/cgroup`
+func NewCGroupSubsysFromLine(line string) (*CGroupSubsys, error) {
+	fields := strings.SplitN(line, _cgroupSep, _csFieldCount)
+
+	if len(fields) != _csFieldCount {
+		return nil, cgroupSubsysFormatInvalidError{line}
+	}
+
+	id, err := strconv.Atoi(fields[_csFieldIDID])
+	if err != nil {
+		return nil, err
+	}
+
+	cgroup := &CGroupSubsys{
+		ID:         id,
+		Subsystems: strings.Split(fields[_csFieldIDSubsystems], _cgroupSubsysSep),
+		Name:       fields[_csFieldIDName],
+	}
+
+	return cgroup, nil
+}
+
+// parseCGroupSubsystems parses procPathCGroup (usually at `/proc/$PID/cgroup`)
+// and returns a new map[string]*CGroupSubsys.
+func parseCGroupSubsystems(procPathCGroup string) (map[string]*CGroupSubsys, error) {
+	cgroupFile, err := os.Open(procPathCGroup)
+	if err != nil {
+		return nil, err
+	}
+	defer cgroupFile.Close()
+
+	scanner := bufio.NewScanner(cgroupFile)
+	subsystems := make(map[string]*CGroupSubsys)
+
+	for scanner.Scan() {
+		cgroup, err := NewCGroupSubsysFromLine(scanner.Text())
+		if err != nil {
+			return nil, err
+		}
+		for _, subsys := range cgroup.Subsystems {
+			subsystems[subsys] = cgroup
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		return nil, err
+	}
+
+	return subsystems, nil
+}

vendor/go.uber.org/automaxprocs/internal/runtime/cpu_quota_linux.go

@@ -0,0 +1,49 @@
+// Copyright (c) 2017 Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+// +build linux
+
+package runtime
+
+import (
+	"math"
+
+	cg "go.uber.org/automaxprocs/internal/cgroups"
+)
+
+// CPUQuotaToGOMAXPROCS converts the CPU quota applied to the calling process
+// to a valid GOMAXPROCS value.
+func CPUQuotaToGOMAXPROCS(minValue int) (int, CPUQuotaStatus, error) {
+	cgroups, err := cg.NewCGroupsForCurrentProcess()
+	if err != nil {
+		return -1, CPUQuotaUndefined, err
+	}
+
+	quota, defined, err := cgroups.CPUQuota()
+	if !defined || err != nil {
+		return -1, CPUQuotaUndefined, err
+	}
+
+	maxProcs := int(math.Floor(quota))
+	if minValue > 0 && maxProcs < minValue {
+		return minValue, CPUQuotaMinUsed, nil
+	}
+	return maxProcs, CPUQuotaUsed, nil
+}

vendor/go.uber.org/automaxprocs/internal/runtime/cpu_quota_unsupported.go

@@ -0,0 +1,30 @@
+// Copyright (c) 2017 Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+// +build !linux
+
+package runtime
+
+// CPUQuotaToGOMAXPROCS converts the CPU quota applied to the calling process
+// to a valid GOMAXPROCS value. This is Linux-specific and not supported in the
+// current OS.
+func CPUQuotaToGOMAXPROCS(_ int) (int, CPUQuotaStatus, error) {
+	return -1, CPUQuotaUndefined, nil
+}

vendor/go.uber.org/automaxprocs/internal/runtime/runtime.go

@@ -0,0 +1,33 @@
+// Copyright (c) 2017 Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+package runtime
+
+// CPUQuotaStatus presents the status of how CPU quota is used
+type CPUQuotaStatus int
+
+const (
+	// CPUQuotaUndefined is returned when CPU quota is undefined
+	CPUQuotaUndefined CPUQuotaStatus = iota
+	// CPUQuotaUsed is returned when a valid CPU quota can be used
+	CPUQuotaUsed
+	// CPUQuotaMinUsed is return when CPU quota is smaller than the min value
+	CPUQuotaMinUsed
+)

vendor/go.uber.org/automaxprocs/maxprocs/maxprocs.go

@@ -0,0 +1,130 @@
+// Copyright (c) 2017 Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+// Package maxprocs lets Go programs easily configure runtime.GOMAXPROCS to
+// match the configured Linux CPU quota. Unlike the top-level automaxprocs
+// package, it lets the caller configure logging and handle errors.
+package maxprocs // import "go.uber.org/automaxprocs/maxprocs"
+
+import (
+	"os"
+	"runtime"
+
+	iruntime "go.uber.org/automaxprocs/internal/runtime"
+)
+
+const _maxProcsKey = "GOMAXPROCS"
+
+func currentMaxProcs() int {
+	return runtime.GOMAXPROCS(0)
+}
+
+type config struct {
+	printf        func(string, ...interface{})
+	procs         func(int) (int, iruntime.CPUQuotaStatus, error)
+	minGOMAXPROCS int
+}
+
+func (c *config) log(fmt string, args ...interface{}) {
+	if c.printf != nil {
+		c.printf(fmt, args...)
+	}
+}
+
+// An Option alters the behavior of Set.
+type Option interface {
+	apply(*config)
+}
+
+// Logger uses the supplied printf implementation for log output. By default,
+// Set doesn't log anything.
+func Logger(printf func(string, ...interface{})) Option {
+	return optionFunc(func(cfg *config) {
+		cfg.printf = printf
+	})
+}
+
+// Min sets the minimum GOMAXPROCS value that will be used.
+// Any value below 1 is ignored.
+func Min(n int) Option {
+	return optionFunc(func(cfg *config) {
+		if n >= 1 {
+			cfg.minGOMAXPROCS = n
+		}
+	})
+}
+
+type optionFunc func(*config)
+
+func (of optionFunc) apply(cfg *config) { of(cfg) }
+
+// Set GOMAXPROCS to match the Linux container CPU quota (if any), returning
+// any error encountered and an undo function.
+//
+// Set is a no-op on non-Linux systems and in Linux environments without a
+// configured CPU quota.
+func Set(opts ...Option) (func(), error) {
+	cfg := &config{
+		procs:         iruntime.CPUQuotaToGOMAXPROCS,
+		minGOMAXPROCS: 1,
+	}
+	for _, o := range opts {
+		o.apply(cfg)
+	}
+
+	undoNoop := func() {
+		cfg.log("maxprocs: No GOMAXPROCS change to reset")
+	}
+
+	// Honor the GOMAXPROCS environment variable if present. Otherwise, amend
+	// `runtime.GOMAXPROCS()` with the current process' CPU quota if the OS is
+	// Linux, and guarantee a minimum value of 1. The minimum guaranteed value
+	// can be overriden using `maxprocs.Min()`.
+	if max, exists := os.LookupEnv(_maxProcsKey); exists {
+		cfg.log("maxprocs: Honoring GOMAXPROCS=%q as set in environment", max)
+		return undoNoop, nil
+	}
+
+	maxProcs, status, err := cfg.procs(cfg.minGOMAXPROCS)
+	if err != nil {
+		return undoNoop, err
+	}
+
+	if status == iruntime.CPUQuotaUndefined {
+		cfg.log("maxprocs: Leaving GOMAXPROCS=%v: CPU quota undefined", currentMaxProcs())
+		return undoNoop, nil
+	}
+
+	prev := currentMaxProcs()
+	undo := func() {
+		cfg.log("maxprocs: Resetting GOMAXPROCS to %v", prev)
+		runtime.GOMAXPROCS(prev)
+	}
+
+	switch status {
+	case iruntime.CPUQuotaMinUsed:
+		cfg.log("maxprocs: Updating GOMAXPROCS=%v: using minimum allowed GOMAXPROCS", maxProcs)
+	case iruntime.CPUQuotaUsed:
+		cfg.log("maxprocs: Updating GOMAXPROCS=%v: determined from CPU quota", maxProcs)
+	}
+
+	runtime.GOMAXPROCS(maxProcs)
+	return undo, nil
+}

vendor/go.uber.org/automaxprocs/maxprocs/version.go

@@ -0,0 +1,24 @@
+// Copyright (c) 2017 Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+package maxprocs
+
+// Version is the current package version.
+const Version = "1.4.0"

vendor/modules.txt

@@ -250,6 +250,11 @@ github.com/stretchr/testify/require
 ## explicit
 github.com/urfave/cli/v2
 github.com/urfave/cli/v2/altsrc
+# go.uber.org/automaxprocs v1.4.0
+## explicit
+go.uber.org/automaxprocs/internal/cgroups
+go.uber.org/automaxprocs/internal/runtime
+go.uber.org/automaxprocs/maxprocs
 # golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a
 ## explicit
 golang.org/x/crypto/blake2b

watcher/file_test.go

@@ -1,3 +1,5 @@
+// +build !windows
+
 package watcher
 
 import (

websocket/websocket.go

@@ -3,174 +3,108 @@ package websocket
 import (
 	"crypto/sha1"
 	"encoding/base64"
+	"encoding/hex"
+	"errors"
+	"fmt"
 	"io"
-	"net"
 	"net/http"
-	"net/url"
 	"time"
 
 	"github.com/gorilla/websocket"
 	"github.com/rs/zerolog"
-
-	"github.com/cloudflare/cloudflared/h2mux"
 )
 
-var stripWebsocketHeaders = []string{
-	"Upgrade",
-	"Connection",
-	"Sec-Websocket-Key",
-	"Sec-Websocket-Version",
-	"Sec-Websocket-Extensions",
-}
-
 // IsWebSocketUpgrade checks to see if the request is a WebSocket connection.
 func IsWebSocketUpgrade(req *http.Request) bool {
 	return websocket.IsWebSocketUpgrade(req)
 }
 
-// ClientConnect creates a WebSocket client connection for provided request. Caller is responsible for closing
-// the connection. The response body may not contain the entire response and does
-// not need to be closed by the application.
-func ClientConnect(req *http.Request, dialler *websocket.Dialer) (*websocket.Conn, *http.Response, error) {
-	req.URL.Scheme = ChangeRequestScheme(req.URL)
-	wsHeaders := websocketHeaders(req)
-	if dialler == nil {
-		dialler = &websocket.Dialer{
-			Proxy: http.ProxyFromEnvironment,
-		}
-	}
-	conn, response, err := dialler.Dial(req.URL.String(), wsHeaders)
-	if err != nil {
-		return nil, response, err
-	}
-	response.Header.Set("Sec-WebSocket-Accept", generateAcceptKey(req))
-	return conn, response, nil
-}
-
-// StartProxyServer will start a websocket server that will decode
-// the websocket data and write the resulting data to the provided
-func StartProxyServer(
-	log *zerolog.Logger,
-	listener net.Listener,
-	staticHost string,
-	shutdownC <-chan struct{},
-	streamHandler func(originConn io.ReadWriter, remoteConn net.Conn, log *zerolog.Logger),
-) error {
-	upgrader := websocket.Upgrader{
-		ReadBufferSize:  1024,
-		WriteBufferSize: 1024,
-	}
-	h := handler{
-		upgrader:      upgrader,
-		log:           log,
-		staticHost:    staticHost,
-		streamHandler: streamHandler,
-	}
-
-	httpServer := &http.Server{Addr: listener.Addr().String(), Handler: &h}
-	go func() {
-		<-shutdownC
-		_ = httpServer.Close()
-	}()
-
-	return httpServer.Serve(listener)
-}
-
-// HTTP handler for the websocket proxy.
-type handler struct {
-	log           *zerolog.Logger
-	staticHost    string
-	upgrader      websocket.Upgrader
-	streamHandler func(originConn io.ReadWriter, remoteConn net.Conn, log *zerolog.Logger)
-}
-
-func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	// If remote is an empty string, get the destination from the client.
-	finalDestination := h.staticHost
-	if finalDestination == "" {
-		if jumpDestination := r.Header.Get(h2mux.CFJumpDestinationHeader); jumpDestination == "" {
-			h.log.Error().Msg("Did not receive final destination from client. The --destination flag is likely not set")
-			return
-		} else {
-			finalDestination = jumpDestination
-		}
-	}
-
-	stream, err := net.Dial("tcp", finalDestination)
-	if err != nil {
-		h.log.Err(err).Msg("Cannot connect to remote")
-		return
-	}
-	defer stream.Close()
-
-	if !websocket.IsWebSocketUpgrade(r) {
-		_, _ = w.Write(nonWebSocketRequestPage())
-		return
-	}
-	conn, err := h.upgrader.Upgrade(w, r, nil)
-	if err != nil {
-		h.log.Err(err).Msg("failed to upgrade")
-		return
-	}
-	_ = conn.SetReadDeadline(time.Now().Add(pongWait))
-	conn.SetPongHandler(func(string) error { _ = conn.SetReadDeadline(time.Now().Add(pongWait)); return nil })
-	gorillaConn := &GorillaConn{Conn: conn, log: h.log}
-	go gorillaConn.pinger(r.Context())
-	defer conn.Close()
-
-	h.streamHandler(gorillaConn, stream, h.log)
-}
-
 // NewResponseHeader returns headers needed to return to origin for completing handshake
 func NewResponseHeader(req *http.Request) http.Header {
 	header := http.Header{}
 	header.Add("Connection", "Upgrade")
-	header.Add("Sec-Websocket-Accept", generateAcceptKey(req))
+	header.Add("Sec-Websocket-Accept", generateAcceptKey(req.Header.Get("Sec-WebSocket-Key")))
 	header.Add("Upgrade", "websocket")
 	return header
 }
 
-// the gorilla websocket library sets its own Upgrade, Connection, Sec-WebSocket-Key,
-// Sec-WebSocket-Version and Sec-Websocket-Extensions headers.
-// https://github.com/gorilla/websocket/blob/master/client.go#L189-L194.
-func websocketHeaders(req *http.Request) http.Header {
-	wsHeaders := make(http.Header)
-	for key, val := range req.Header {
-		wsHeaders[key] = val
-	}
-	// Assume the header keys are in canonical format.
-	for _, header := range stripWebsocketHeaders {
-		wsHeaders.Del(header)
-	}
-	wsHeaders.Set("Host", req.Host) // See TUN-1097
-	return wsHeaders
+// Stream copies copy data to & from provided io.ReadWriters.
+func Stream(tunnelConn, originConn io.ReadWriter, log *zerolog.Logger) {
+	proxyDone := make(chan struct{}, 2)
+
+	go func() {
+		_, err := copyData(tunnelConn, originConn, "origin->tunnel")
+		if err != nil {
+			log.Debug().Msgf("origin to tunnel copy: %v", err)
+		}
+		proxyDone <- struct{}{}
+	}()
+
+	go func() {
+		_, err := copyData(originConn, tunnelConn, "tunnel->origin")
+		if err != nil {
+			log.Debug().Msgf("tunnel to origin copy: %v", err)
+		}
+		proxyDone <- struct{}{}
+	}()
+
+	// If one side is done, we are done.
+	<-proxyDone
 }
 
-// sha1Base64 sha1 and then base64 encodes str.
-func sha1Base64(str string) string {
-	hasher := sha1.New()
-	_, _ = io.WriteString(hasher, str)
-	hash := hasher.Sum(nil)
-	return base64.StdEncoding.EncodeToString(hash)
-}
+// when set to true, enables logging of content copied to/from origin and tunnel
+const debugCopy = false
 
-// generateAcceptKey returns the string needed for the Sec-WebSocket-Accept header.
-// https://tools.ietf.org/html/rfc6455#section-1.3 describes this process in more detail.
-func generateAcceptKey(req *http.Request) string {
-	return sha1Base64(req.Header.Get("Sec-WebSocket-Key") + "258EAFA5-E914-47DA-95CA-C5AB0DC85B11")
-}
-
-// ChangeRequestScheme is needed as the gorilla websocket library requires the ws scheme.
-// (even though it changes it back to http/https, but ¯\_(ツ)_/¯.)
-func ChangeRequestScheme(reqURL *url.URL) string {
-	switch reqURL.Scheme {
-	case "https":
-		return "wss"
-	case "http":
-		return "ws"
-	case "":
-		return "ws"
-	default:
-		return reqURL.Scheme
+func copyData(dst io.Writer, src io.Reader, dir string) (written int64, err error) {
+	if debugCopy {
+		// copyBuffer is based on stdio Copy implementation but shows copied data
+		copyBuffer := func(dst io.Writer, src io.Reader, dir string) (written int64, err error) {
+			var buf []byte
+			size := 32 * 1024
+			buf = make([]byte, size)
+			for {
+				t := time.Now()
+				nr, er := src.Read(buf)
+				if nr > 0 {
+					fmt.Println(dir, t.UnixNano(), "\n"+hex.Dump(buf[0:nr]))
+					nw, ew := dst.Write(buf[0:nr])
+					if nw < 0 || nr < nw {
+						nw = 0
+						if ew == nil {
+							ew = errors.New("invalid write")
+						}
+					}
+					written += int64(nw)
+					if ew != nil {
+						err = ew
+						break
+					}
+					if nr != nw {
+						err = io.ErrShortWrite
+						break
+					}
+				}
+				if er != nil {
+					if er != io.EOF {
+						err = er
+					}
+					break
+				}
+			}
+			return written, err
+		}
+		return copyBuffer(dst, src, dir)
+	} else {
+		return io.Copy(dst, src)
 	}
 }
+
+// from RFC-6455
+var keyGUID = []byte("258EAFA5-E914-47DA-95CA-C5AB0DC85B11")
+
+func generateAcceptKey(challengeKey string) string {
+	h := sha1.New()
+	h.Write([]byte(challengeKey))
+	h.Write(keyGUID)
+	return base64.StdEncoding.EncodeToString(h.Sum(nil))
+}