Release 2021.3.0

free tunnels

.gitignore

@@ -1,21 +1,17 @@
-.GOPATH/
-bin/
-tmp/
-guide/public
-/.GOPATH
+/tmp
 /bin
 .idea
 .build
 .vscode
 \#*\#
 cscope.*
-cloudflared
-cloudflared.pkg
-cloudflared.exe
-cloudflared.msi
-cloudflared-x86-64*
-!cmd/cloudflared/
+/cloudflared
+/cloudflared.pkg
+/cloudflared.exe
+/cloudflared.msi
+/cloudflared-x86-64*
+/packaging
 .DS_Store
 *-session.log
 ssh_server_tests/.env
-.cover
+/.cover

CHANGES.md

@@ -0,0 +1,77 @@
+**Experimental**: This is a new format for release notes. The format and availability is subject to change.
+
+## 2021.3.0
+
+### New Features
+
+- [Cloudflare One Routing](https://developers.cloudflare.com/cloudflare-one/tutorials/warp-to-tunnel) specific commands
+  now show up in the `cloudflared tunnel route --help` output.
+- There is a new ingress type that allows cloudflared to proxy SOCKS5 as a bastion. You can use it with an ingress
+  rule by adding `service: socks-proxy`. Traffic is routed to any destination specified by the SOCKS5 packet but only
+  if allowed by a rule. In the following example we allow proxying to a certain CIDR but explicitly forbid one address
+  within it:
+```
+ingress:
+  - hostname: socks.example.com
+    service: socks-proxy
+    originRequest:
+      ipRules:
+        - prefix: 192.168.1.8/32
+          allow: false
+        - prefix: 192.168.1.0/24
+          ports: [80, 443]
+          allow: true
+```
+  
+
+### Improvements
+
+- Nested commands, such as `cloudflared tunnel run`, now consider CLI arguments even if they appear earlier on the
+  command. For instance, `cloudflared --config config.yaml tunnel run` will now behave the same as
+  `cloudflared tunnel --config config.yaml run`
+- Warnings are now shown in the output logs whenever cloudflared is running without the most recent version and
+  `no-autoupdate` is `true`.
+- Access tokens are now stored per Access App instead of per request path. This decreases the number of times that the
+  user is required to authenticate with an Access policy redundantly.
+
+### Bug Fixes
+
+- GitHub [PR #317](https://github.com/cloudflare/cloudflared/issues/317) was broken in 2021.2.5 and is now fixed again.
+
+## 2021.2.5
+
+### New Features
+
+- We introduce [Cloudflare One Routing](https://developers.cloudflare.com/cloudflare-one/tutorials/warp-to-tunnel) in
+  beta mode. Cloudflare customer can now connect users and private networks with RFC 1918 IP addresses via the
+  Cloudflare edge network. Users running Cloudflare WARP client in the same organization can connect to the services
+  made available by Argo Tunnel IP routes. Please share your feedback in the GitHub issue tracker.
+
+## 2021.2.4
+
+### Bug Fixes
+
+- Reverts the Improvement released in 2021.2.3 for CLI arguments as it introduced a regression where cloudflared failed
+  to read URLs in configuration files.
+- cloudflared now logs the reason for failed connections if the error is recoverable.
+
+## 2021.2.3
+
+### Backward Incompatible Changes
+
+- Removes db-connect. The Cloudflare Workers product will continue to support db-connect implementations with versions
+  of cloudflared that predate this release and include support for db-connect.
+
+### New Features
+
+- Introduces support for proxy configurations with websockets in arbitrary TCP connections (#318).
+
+### Improvements
+
+- (reverted) Nested command line argument handling.
+
+### Bug Fixes
+
+- The maximum number of upstream connections is now limited by default which should fix reported issues of cloudflared
+  exhausting CPU usage when faced with connectivity issues.
+

Makefile

@@ -34,6 +34,8 @@ ifneq ($(GOARCH),)
     TARGET_ARCH ?= $(GOARCH)
 else ifeq ($(LOCAL_ARCH),x86_64)
     TARGET_ARCH ?= amd64
+else ifeq ($(LOCAL_ARCH),amd64)
+    TARGET_ARCH ?= amd64
 else ifeq ($(LOCAL_ARCH),i686)
     TARGET_ARCH ?= amd64
 else ifeq ($(shell echo $(LOCAL_ARCH) | head -c 5),armv8)
@@ -80,8 +82,18 @@ clean:
 
 .PHONY: cloudflared
 cloudflared: tunnel-deps
+ifeq ($(FIPS), true)
+	$(info Building cloudflared with go-fips)
+	-test -f fips/fips.go && mv fips/fips.go fips/fips.go.linux-amd64
+	mv fips/fips.go.linux-amd64 fips/fips.go
+endif
+
 	GOOS=$(TARGET_OS) GOARCH=$(TARGET_ARCH) go build -v -mod=vendor $(GO_BUILD_TAGS) $(VERSION_FLAGS) $(IMPORT_PATH)/cmd/cloudflared
 
+ifeq ($(FIPS), true)
+	mv fips/fips.go fips/fips.go.linux-amd64
+endif
+
 .PHONY: container
 container:
 	docker build --build-arg=TARGET_ARCH=$(TARGET_ARCH) --build-arg=TARGET_OS=$(TARGET_OS) -t cloudflare/cloudflared-$(TARGET_OS)-$(TARGET_ARCH):"$(VERSION)" .
@@ -101,10 +113,10 @@ test-ssh-server:
 	docker-compose -f ssh_server_tests/docker-compose.yml up
 
 define publish_package
+	chmod 664 cloudflared*.$(1); \
 	for HOST in $(CF_PKG_HOSTS); do \
 		ssh-keyscan -t rsa $$HOST >> ~/.ssh/known_hosts; \
-		scp -4 cloudflared*.$(1) cfsync@$$HOST:/state/cf-pkg/staging/$(2)/$(TARGET_PUBLIC_REPO)/cloudflared/; \
-		ssh cfsync@$$HOST 'chmod g+w /state/cf-pkg/staging/$(2)/$(TARGET_PUBLIC_REPO)/cloudflared/*.$(1)'; \
+		scp -p -4 cloudflared*.$(1) cfsync@$$HOST:/state/cf-pkg/staging/$(2)/$(TARGET_PUBLIC_REPO)/cloudflared/; \
 	done
 endef
 
@@ -236,7 +248,7 @@ tunnelrpc/tunnelrpc.capnp.go: tunnelrpc/tunnelrpc.capnp
 .PHONY: vet
 vet:
 	go vet -mod=vendor ./...
-	which go-sumtype  # go get github.com/BurntSushi/go-sumtype
+	which go-sumtype  # go get github.com/BurntSushi/go-sumtype (don't do this in build directory or this will cause vendor issues)
 	go-sumtype $$(go list -mod=vendor ./...)
 
 .PHONY: msi

README.md

@@ -1,6 +1,6 @@
 # Argo Tunnel client
 
-Contains the command-line client for Argo Tunnel, a tunneling daemon that proxies any local webserver through the Cloudflare network. Extensive documentation can be found in the [Argo Tunnel section](https://developers.cloudflare.com/argo-tunnel/) of the Cloudflare Docs.
+Contains the command-line client for Argo Tunnel, a tunneling daemon that proxies any local webserver through the Cloudflare network. Extensive documentation can be found in the [Argo Tunnel section](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps) of the Cloudflare Docs.
 
 ## Before you get started
 
@@ -13,29 +13,29 @@ Before you use Argo Tunnel, you'll need to complete a few steps in the Cloudflar
 
 Downloads are available as standalone binaries, a Docker image, and Debian, RPM, and Homebrew packages. You can also find releases here on the `cloudflared` GitHub repository.
 
-* You can [install on macOS](https://developers.cloudflare.com/argo-tunnel/getting-started/installation#macos) via Homebrew or by downloading the [latest Darwin amd64 release](https://github.com/cloudflare/cloudflared/releases)
-* Binaries, Debian, and RPM packages for Linux [can be found here](https://developers.cloudflare.com/argo-tunnel/getting-started/installation#linux)
+* You can [install on macOS](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/install-and-setup/installation#macos) via Homebrew or by downloading the [latest Darwin amd64 release](https://github.com/cloudflare/cloudflared/releases)
+* Binaries, Debian, and RPM packages for Linux [can be found here](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/install-and-setup/installation#linux)
 * A Docker image of `cloudflared` is [available on DockerHub](https://hub.docker.com/r/cloudflare/cloudflared)
-* You can install on Windows machines with the [steps here](https://developers.cloudflare.com/argo-tunnel/getting-started/installation#windows)
+* You can install on Windows machines with the [steps here](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/install-and-setup/installation#windows)
 
-User documentation for Argo Tunnel can be found at https://developers.cloudflare.com/argo-tunnel/
+User documentation for Argo Tunnel can be found at https://developers.cloudflare.com/cloudflare-one/connections/connect-apps
 
 ## Creating Tunnels and routing traffic
 
 Once installed, you can authenticate `cloudflared` into your Cloudflare account and begin creating Tunnels that serve traffic for hostnames in your account.
 
-* Create a Tunnel with [these instructions](https://developers.cloudflare.com/argo-tunnel/create-tunnel)
-* Route traffic to that Tunnel with [DNS records in Cloudflare](https://developers.cloudflare.com/argo-tunnel/routing-to-tunnel/dns) or with a [Cloudflare Load Balancer](https://developers.cloudflare.com/argo-tunnel/routing-to-tunnel/lb)
+* Create a Tunnel with [these instructions](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/create-tunnel)
+* Route traffic to that Tunnel with [DNS records in Cloudflare](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/routing-to-tunnel/dns) or with a [Cloudflare Load Balancer](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/routing-to-tunnel/lb)
 
 ## TryCloudflare
 
-Want to test Argo Tunnel before adding a website to Cloudflare? You can do so with TryCloudflare using the documentation [available here](https://developers.cloudflare.com/argo-tunnel/learning/trycloudflare).
+Want to test Argo Tunnel before adding a website to Cloudflare? You can do so with TryCloudflare using the documentation [available here](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/trycloudflare).
 
 ## Deprecated versions
 
 Cloudflare currently supports all versions of `cloudflared`. Starting on March 20, 2021, Cloudflare will no longer support versions released prior to 2020.5.1.
 
-All features available in versions released prior to 2020.5.1 are available in current versions. Breaking changes unrelated to feature availability may be introduced that will impact versions released prior to 2020.5.1. You can read more about upgrading `cloudflared` in our [developer documentation](https://developers.cloudflare.com/argo-tunnel/getting-started/installation#updating-cloudflared).
+All features available in versions released prior to 2020.5.1 are available in current versions. Breaking changes unrelated to feature availability may be introduced that will impact versions released prior to 2020.5.1. You can read more about upgrading `cloudflared` in our [developer documentation](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/install-and-setup/installation#updating-cloudflared).
 
 | Version(s) | Deprecation status |
 |---|---|

RELEASE_NOTES

@@ -1,3 +1,69 @@
+2021.3.0
+- 2021-03-10 TUN-4075: Dedup test should not compare order of list
+- 2021-03-10 Revert "AUTH-3394: Creates a token per app instead of per path"
+- 2021-03-11 TUN-4066: Remove unnecessary chmod during package publish to CF_PKG_HOSTS
+- 2021-03-11 TUN-4066: Set permissions in build agent before 'scp'-ing to machine hosting package repo
+- 2021-03-11 TUN-4050: Add component tests to assert reconnect behavior
+- 2021-03-10 AUTH-3394: Creates a token per app instead of per path - with fix for free tunnels
+- 2021-03-15 Publish change log for 2021.3.0
+- 2021-03-01 Issue #285 - Makefile does not detect TARGET_ARCH correctly on FreeBSD  (#325)
+- 2021-03-01 TUN-3988: Log why it cannot check if origin cert exists
+- 2021-03-02 TUN-3995: Optional --features flag for tunnel run.
+- 2021-03-02 TUN-3994: Log client_id when running a named tunnel
+- 2021-03-04 TUN-4026: Fix regression where HTTP2 edge transport was no longer propagating control plane errors
+- 2021-03-05 TUN-4055: Skeleton for component tests
+- 2021-03-08 TUN-4047: Add cfsetup target to run component test
+- 2021-03-08 TUN-4016: Delegate decision to update for Worker service
+- 2021-03-02 TUN-3905: Cannot run go mod vendor in cloudflared due to fips
+- 2021-03-08 TUN-4063: Cleanup dependencies between packages.
+- 2021-03-09 Allow partial reads from a GorillaConn; add SetDeadline (from net.Conn) (#330)
+- 2021-03-09 TUN-4069: Fix regression on support for websocket over proxy
+- 2021-03-02 AUTH-3394: Creates a token per app instead of per path
+- 2021-03-01 TUN-4017: Add support for using cloudflared as a full socks proxy.
+- 2021-03-08 TUN-4062: Read component tests config from yaml file
+- 2021-03-08 TUN-4049: Add component tests to assert logging behavior when running from terminal
+- 2021-02-23 TUN-3963: Repoint urfave/cli/v2 library at patched branch at github.com/ipostelnik/cli/v2@fixed which correctly handles reading flags declared at multiple levels of subcommands.
+- 2021-02-25 TUN-3970: Route ip show has alias route ip list
+- 2021-02-26 TUN-3978: Unhide teamnet commands and improve their help
+- 2021-02-26 TUN-3983: Renew CA certs in cloudflared
+- 2021-02-28 TUN-3989: Check in with Updater service in more situations and convey messages to user
+- 2021-02-11 TUN-3819: Remove client-side check that deleted tunnels have no connections
+
+2021.2.5
+- 2021-02-23 Publish change notes for 2021.2.5
+- 2021-02-11 TUN-3838: ResponseWriter no longer reads and origin error tests
+- 2021-02-10 TUN-3895: Tests for socks stream handler
+- 2021-02-19 TUN-3939: Add logging that shows that Warp-routing is enabled
+- 2021-02-02 TUN-3817: Adds tests for websocket based streaming regression
+- 2021-02-04 TUN-3799: extended the Stream interface to take a logger and added debug logs for io.Copy errors
+- 2021-02-03 TUN-3855: Add ability to override target of 'access ssh' command to a different host for testing
+- 2021-02-04 TUN-3853: Respond with ws headers from the origin service rather than generating our own
+- 2021-02-08 TUN-3889: Move host header override logic to httpService
+- 2021-02-05 TUN-3868: Refactor singleTCPService and bridgeService to tcpOverWSService and rawTCPService
+- 2021-01-21 TUN-3753: Select http2 protocol when warp routing is enabled
+- 2021-01-26 TUN-3809: Allow routes ip show to output as JSON or YAML
+- 2021-01-11 TUN-3615: added support to  proxy tcp streams
+- 2021-01-17 TUN-3725: Warp-routing is independent of ingress
+- 2021-01-15 TUN-3764: Actively flush data for TCP streams
+- 2020-12-09 TUN-3617: Separate service from client, and implement different client for http vs. tcp origins
+
+2021.2.4
+- 2021-02-22 TUN-3948: Log error when retrying connection
+- 2021-02-23 TUN-3964: Revert "TUN-3922: Repoint urfave/cli/v2 library at patched branch at github.com/ipostelnik/cli/v2@fixed which correctly handles reading flags declared at multiple levels of subcommands."
+- 2021-02-23 Publish release notes for 2021.2.4
+
+2021.2.3
+- 2021-02-23 Publish release notes for 2021.2.3
+- 2021-02-10 TUN-3902: Add jitter to backoffhandler
+- 2021-02-11 TUN-3913: Help gives wrong exit code for autoupdate
+- 2021-02-12 Add max upstream connections dns-proxy option (#290)
+- 2021-02-16 TUN-3924: Removed db-connect command. Added a placeholder handler for this command that informs users that command is no longer supported.
+- 2021-02-12 TUN-3922: Repoint urfave/cli/v2 library at patched branch at github.com/ipostelnik/cli/v2@fixed which correctly handles reading flags declared at multiple levels of subcommands.
+- 2021-02-19 Added support for proxy (#318)
+- 2021-02-19 TUN-3945: Fix runApp signature for generic service
+- 2021-02-09 Update README.md
+- 2021-02-09 Update the TryCloudflare link
+
 2021.2.2
 - 2021-02-04 TUN-3864: Users can choose where credentials file is written after creating a tunnel
 - 2021-02-04 TUN-3869: Improve reliability of graceful shutdown.

carrier/carrier.go

@@ -4,14 +4,15 @@
 package carrier
 
 import (
+	"crypto/tls"
 	"io"
 	"net"
 	"net/http"
 	"os"
 	"strings"
 
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/token"
 	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/token"
 
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
@@ -20,8 +21,11 @@ import (
 const LogFieldOriginURL = "originURL"
 
 type StartOptions struct {
-	OriginURL string
-	Headers   http.Header
+	AppInfo         *token.AppInfo
+	OriginURL       string
+	Headers         http.Header
+	Host            string
+	TLSClientConfig *tls.Config
 }
 
 // Connection wraps up all the needed functions to forward over the tunnel
@@ -120,7 +124,7 @@ func IsAccessResponse(resp *http.Response) bool {
 	if err != nil || location == nil {
 		return false
 	}
-	if strings.HasPrefix(location.Path, "/cdn-cgi/access/login") {
+	if strings.HasPrefix(location.Path, token.AccessLoginWorkerPath) {
 		return true
 	}
 
@@ -134,7 +138,7 @@ func BuildAccessRequest(options *StartOptions, log *zerolog.Logger) (*http.Reque
 		return nil, err
 	}
 
-	token, err := token.FetchTokenWithRedirect(req.URL, log)
+	token, err := token.FetchTokenWithRedirect(req.URL, options.AppInfo, log)
 	if err != nil {
 		return nil, err
 	}

carrier/carrier_test.go

@@ -44,7 +44,7 @@ func (s *testStreamer) Write(p []byte) (int, error) {
 func TestStartClient(t *testing.T) {
 	message := "Good morning Austin! Time for another sunny day in the great state of Texas."
 	log := zerolog.Nop()
-	wsConn := NewWSConnection(&log, false)
+	wsConn := NewWSConnection(&log)
 	ts := newTestWebSocketServer()
 	defer ts.Close()
 
@@ -70,7 +70,7 @@ func TestStartServer(t *testing.T) {
 	message := "Good morning Austin! Time for another sunny day in the great state of Texas."
 	log := zerolog.Nop()
 	shutdownC := make(chan struct{})
-	wsConn := NewWSConnection(&log, false)
+	wsConn := NewWSConnection(&log)
 	ts := newTestWebSocketServer()
 	defer ts.Close()
 	options := &StartOptions{

carrier/websocket.go

@@ -7,8 +7,9 @@ import (
 	"net/http"
 	"net/http/httputil"
 
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/token"
+	"github.com/cloudflare/cloudflared/ingress"
 	"github.com/cloudflare/cloudflared/socks"
+	"github.com/cloudflare/cloudflared/token"
 	cfwebsocket "github.com/cloudflare/cloudflared/websocket"
 
 	"github.com/gorilla/websocket"
@@ -23,7 +24,7 @@ type Websocket struct {
 }
 
 type wsdialer struct {
-	conn *cfwebsocket.Conn
+	conn *cfwebsocket.GorillaConn
 }
 
 func (d *wsdialer) Dial(address string) (io.ReadWriteCloser, *socks.AddrSpec, error) {
@@ -37,10 +38,9 @@ func (d *wsdialer) Dial(address string) (io.ReadWriteCloser, *socks.AddrSpec, er
 }
 
 // NewWSConnection returns a new connection object
-func NewWSConnection(log *zerolog.Logger, isSocks bool) Connection {
+func NewWSConnection(log *zerolog.Logger) Connection {
 	return &Websocket{
-		log:     log,
-		isSocks: isSocks,
+		log: log,
 	}
 }
 
@@ -54,41 +54,52 @@ func (ws *Websocket) ServeStream(options *StartOptions, conn io.ReadWriter) erro
 	}
 	defer wsConn.Close()
 
-	if ws.isSocks {
-		dialer := &wsdialer{conn: wsConn}
-		requestHandler := socks.NewRequestHandler(dialer)
-		socksServer := socks.NewConnectionHandler(requestHandler)
-
-		_ = socksServer.Serve(conn)
-	} else {
-		cfwebsocket.Stream(wsConn, conn)
-	}
+	ingress.Stream(wsConn, conn, ws.log)
 	return nil
 }
 
 // StartServer creates a Websocket server to listen for connections.
 // This is used on the origin (tunnel) side to take data from the muxer and send it to the origin
 func (ws *Websocket) StartServer(listener net.Listener, remote string, shutdownC <-chan struct{}) error {
-	return cfwebsocket.StartProxyServer(ws.log, listener, remote, shutdownC, cfwebsocket.DefaultStreamHandler)
+	return cfwebsocket.StartProxyServer(ws.log, listener, remote, shutdownC, ingress.DefaultStreamHandler)
 }
 
 // createWebsocketStream will create a WebSocket connection to stream data over
 // It also handles redirects from Access and will present that flow if
 // the token is not present on the request
-func createWebsocketStream(options *StartOptions, log *zerolog.Logger) (*cfwebsocket.Conn, error) {
+func createWebsocketStream(options *StartOptions, log *zerolog.Logger) (*cfwebsocket.GorillaConn, error) {
 	req, err := http.NewRequest(http.MethodGet, options.OriginURL, nil)
 	if err != nil {
 		return nil, err
 	}
 	req.Header = options.Headers
+	if options.Host != "" {
+		req.Host = options.Host
+	}
 
 	dump, err := httputil.DumpRequest(req, false)
 	log.Debug().Msgf("Websocket request: %s", string(dump))
 
-	wsConn, resp, err := cfwebsocket.ClientConnect(req, nil)
+	dialer := &websocket.Dialer{
+		TLSClientConfig: options.TLSClientConfig,
+		Proxy:           http.ProxyFromEnvironment,
+	}
+	wsConn, resp, err := cfwebsocket.ClientConnect(req, dialer)
 	defer closeRespBody(resp)
 
 	if err != nil && IsAccessResponse(resp) {
+		// Only get Access app info if we know the origin is protected by Access
+		originReq, err := http.NewRequest(http.MethodGet, options.OriginURL, nil)
+		if err != nil {
+			return nil, err
+		}
+
+		appInfo, err := token.GetAppInfo(originReq.URL)
+		if err != nil {
+			return nil, err
+		}
+		options.AppInfo = appInfo
+
 		wsConn, err = createAccessAuthenticatedStream(options, log)
 		if err != nil {
 			return nil, err
@@ -97,7 +108,7 @@ func createWebsocketStream(options *StartOptions, log *zerolog.Logger) (*cfwebso
 		return nil, err
 	}
 
-	return &cfwebsocket.Conn{Conn: wsConn}, nil
+	return &cfwebsocket.GorillaConn{Conn: wsConn}, nil
 }
 
 // createAccessAuthenticatedStream will try load a token from storage and make
@@ -117,11 +128,7 @@ func createAccessAuthenticatedStream(options *StartOptions, log *zerolog.Logger)
 	}
 
 	// Access Token is invalid for some reason. Go through regen flow
-	originReq, err := http.NewRequest(http.MethodGet, options.OriginURL, nil)
-	if err != nil {
-		return nil, err
-	}
-	if err := token.RemoveTokenIfExists(originReq.URL); err != nil {
+	if err := token.RemoveTokenIfExists(options.AppInfo); err != nil {
 		return nil, err
 	}
 	wsConn, resp, err = createAccessWebSocketStream(options, log)

cfsetup.yaml

@@ -194,6 +194,7 @@ stretch: &stretch
     builddeps:
       - *pinned_go_fips
       - build-essential
+      - gotest-to-teamcity
     post-cache:
       - export GOOS=linux
       - export GOARCH=amd64
@@ -201,7 +202,22 @@ stretch: &stretch
       # cd to a non-module directory: https://github.com/golang/go/issues/24250
       - (cd / && go get github.com/BurntSushi/go-sumtype)
       - export PATH="$HOME/go/bin:$PATH"
-      - make test
+      - make test | gotest-to-teamcity
+  component-test:
+    build_dir: *build_dir
+    builddeps:
+      - *pinned_go_fips
+      - python3.7
+      - python3-pip
+      - python3-setuptools
+    pre-cache-copy-paths:
+      - component-tests/requirements.txt
+    pre-cache:
+      - sudo pip3 install --upgrade -r component-tests/requirements.txt
+    post-cache:
+      # Constructs config file from env vars
+      - python3 component-tests/config.py
+      - pytest component-tests
   update-homebrew:
     builddeps:
       - openssh-client
@@ -275,14 +291,3 @@ centos-7:
       - export GOOS=linux
       - export GOARCH=amd64
       - make cloudflared-rpm
-
-# cfsetup compose
-default-stack: test_dbconnect
-test_dbconnect:
-  compose:
-    up-args:
-      - --renew-anon-volumes
-      - --abort-on-container-exit
-      - --exit-code-from=cloudflared
-    files:
-      - dbconnect_tests/dbconnect.yaml

cmd/cloudflared/access/carrier.go

@@ -1,11 +1,13 @@
 package access
 
 import (
+	"crypto/tls"
+	"fmt"
 	"net/http"
 	"strings"
 
 	"github.com/cloudflare/cloudflared/carrier"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/h2mux"
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/validation"
@@ -46,7 +48,7 @@ func StartForwarder(forwarder config.Forwarder, shutdown <-chan struct{}, log *z
 	}
 
 	// we could add a cmd line variable for this bool if we want the SOCK5 server to be on the client side
-	wsConn := carrier.NewWSConnection(log, false)
+	wsConn := carrier.NewWSConnection(log)
 
 	log.Info().Str(LogFieldHost, validURL.Host).Msg("Start Websocket listener")
 	return carrier.StartForwarder(wsConn, validURL.Host, shutdown, options)
@@ -84,10 +86,30 @@ func ssh(c *cli.Context) error {
 	options := &carrier.StartOptions{
 		OriginURL: originURL,
 		Headers:   headers,
+		Host:      hostname,
+	}
+
+	if connectTo := c.String(sshConnectTo); connectTo != "" {
+		parts := strings.Split(connectTo, ":")
+		switch len(parts) {
+		case 1:
+			options.OriginURL = fmt.Sprintf("https://%s", parts[0])
+		case 2:
+			options.OriginURL = fmt.Sprintf("https://%s:%s", parts[0], parts[1])
+		case 3:
+			options.OriginURL = fmt.Sprintf("https://%s:%s", parts[2], parts[1])
+			options.TLSClientConfig = &tls.Config{
+				InsecureSkipVerify: true,
+				ServerName:         parts[0],
+			}
+			log.Warn().Msgf("Using insecure SSL connection because SNI overridden to %s", parts[0])
+		default:
+			return fmt.Errorf("invalid connection override: %s", connectTo)
+		}
 	}
 
 	// we could add a cmd line variable for this bool if we want the SOCK5 server to be on the client side
-	wsConn := carrier.NewWSConnection(log, false)
+	wsConn := carrier.NewWSConnection(log)
 
 	if c.NArg() > 0 || c.IsSet(sshURLFlag) {
 		forwarder, err := config.ValidateUrl(c, true)
@@ -95,7 +117,6 @@ func ssh(c *cli.Context) error {
 			log.Err(err).Msg("Error validating origin URL")
 			return errors.Wrap(err, "error validating origin URL")
 		}
-
 		log.Info().Str(LogFieldHost, forwarder.Host).Msg("Start Websocket listener")
 		err = carrier.StartForwarder(wsConn, forwarder.Host, shutdownC, options)
 		if err != nil {

cmd/cloudflared/access/cmd.go

@@ -2,20 +2,21 @@ package access
 
 import (
 	"fmt"
+	"io"
 	"net/http"
 	"net/url"
 	"os"
+	"os/exec"
 	"strings"
 	"text/template"
 	"time"
 
 	"github.com/cloudflare/cloudflared/carrier"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/shell"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/token"
 	"github.com/cloudflare/cloudflared/h2mux"
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/sshgen"
+	"github.com/cloudflare/cloudflared/token"
 	"github.com/cloudflare/cloudflared/validation"
 
 	"github.com/getsentry/raven-go"
@@ -33,6 +34,7 @@ const (
 	sshTokenIDFlag     = "service-token-id"
 	sshTokenSecretFlag = "service-token-secret"
 	sshGenCertFlag     = "short-lived-cert"
+	sshConnectTo       = "connect-to"
 	sshConfigTemplate  = `
 Add to your {{.Home}}/.ssh/config:
 
@@ -54,7 +56,7 @@ Host cfpipe-{{.Hostname}}
 const sentryDSN = "https://56a9c9fa5c364ab28f34b14f35ea0f1b@sentry.io/189878"
 
 var (
-	shutdownC      chan struct{}
+	shutdownC chan struct{}
 )
 
 // Init will initialize and store vars from the main program
@@ -164,6 +166,11 @@ func Commands() []*cli.Command {
 							Aliases: []string{"loglevel"}, //added to match the tunnel side
 							Usage:   "Application logging level {fatal, error, info, debug}. ",
 						},
+						&cli.StringFlag{
+							Name:   sshConnectTo,
+							Hidden: true,
+							Usage:  "Connect to alternate location for testing, value is host, host:port, or sni:port:host",
+						},
 					},
 				},
 				{
@@ -214,12 +221,18 @@ func login(c *cli.Context) error {
 		log.Error().Msg("Please provide the url of the Access application")
 		return err
 	}
-	if err := verifyTokenAtEdge(appURL, c, log); err != nil {
+
+	appInfo, err := token.GetAppInfo(appURL)
+	if err != nil {
+		return err
+	}
+
+	if err := verifyTokenAtEdge(appURL, appInfo, c, log); err != nil {
 		log.Err(err).Msg("Could not verify token")
 		return err
 	}
 
-	cfdToken, err := token.GetAppTokenIfExists(appURL)
+	cfdToken, err := token.GetAppTokenIfExists(appInfo)
 	if err != nil {
 		fmt.Fprintln(os.Stderr, "Unable to find token for provided application.")
 		return err
@@ -261,13 +274,17 @@ func curl(c *cli.Context) error {
 		return err
 	}
 
-	tok, err := token.GetAppTokenIfExists(appURL)
+	appInfo, err := token.GetAppInfo(appURL)
+	if err != nil {
+		return err
+	}
+	tok, err := token.GetAppTokenIfExists(appInfo)
 	if err != nil || tok == "" {
 		if allowRequest {
 			log.Info().Msg("You don't have an Access token set. Please run access token <access application> to fetch one.")
-			return shell.Run("curl", cmdArgs...)
+			return run("curl", cmdArgs...)
 		}
-		tok, err = token.FetchToken(appURL, log)
+		tok, err = token.FetchToken(appURL, appInfo, log)
 		if err != nil {
 			log.Err(err).Msg("Failed to refresh token")
 			return err
@@ -276,7 +293,29 @@ func curl(c *cli.Context) error {
 
 	cmdArgs = append(cmdArgs, "-H")
 	cmdArgs = append(cmdArgs, fmt.Sprintf("%s: %s", h2mux.CFAccessTokenHeader, tok))
-	return shell.Run("curl", cmdArgs...)
+	return run("curl", cmdArgs...)
+}
+
+
+// run kicks off a shell task and pipe the results to the respective std pipes
+func run(cmd string, args ...string) error {
+	c := exec.Command(cmd, args...)
+	stderr, err := c.StderrPipe()
+	if err != nil {
+		return err
+	}
+	go func() {
+		io.Copy(os.Stderr, stderr)
+	}()
+
+	stdout, err := c.StdoutPipe()
+	if err != nil {
+		return err
+	}
+	go func() {
+		io.Copy(os.Stdout, stdout)
+	}()
+	return c.Run()
 }
 
 // token dumps provided token to stdout
@@ -289,7 +328,12 @@ func generateToken(c *cli.Context) error {
 		fmt.Fprintln(os.Stderr, "Please provide a url.")
 		return err
 	}
-	tok, err := token.GetAppTokenIfExists(appURL)
+
+	appInfo, err := token.GetAppInfo(appURL)
+	if err != nil {
+		return err
+	}
+	tok, err := token.GetAppTokenIfExists(appInfo)
 	if err != nil || tok == "" {
 		fmt.Fprintln(os.Stderr, "Unable to find token for provided application. Please run login command to generate token.")
 		return err
@@ -340,19 +384,24 @@ func sshGen(c *cli.Context) error {
 	// this fetchToken function mutates the appURL param. We should refactor that
 	fetchTokenURL := &url.URL{}
 	*fetchTokenURL = *originURL
-	cfdToken, err := token.FetchTokenWithRedirect(fetchTokenURL, log)
+
+	appInfo, err := token.GetAppInfo(fetchTokenURL)
+	if err != nil {
+		return err
+	}
+	cfdToken, err := token.FetchTokenWithRedirect(fetchTokenURL, appInfo, log)
 	if err != nil {
 		return err
 	}
 
-	if err := sshgen.GenerateShortLivedCertificate(originURL, cfdToken); err != nil {
+	if err := sshgen.GenerateShortLivedCertificate(appInfo, cfdToken); err != nil {
 		return err
 	}
 
 	return nil
 }
 
-// getAppURL will pull the appURL needed for fetching a user's Access token
+// getAppURL will pull the request URL needed for fetching a user's Access token
 func getAppURL(cmdArgs []string, log *zerolog.Logger) (*url.URL, error) {
 	if len(cmdArgs) < 1 {
 		log.Error().Msg("Please provide a valid URL as the first argument to curl.")
@@ -427,7 +476,7 @@ func isFileThere(candidate string) bool {
 // verifyTokenAtEdge checks for a token on disk, or generates a new one.
 // Then makes a request to to the origin with the token to ensure it is valid.
 // Returns nil if token is valid.
-func verifyTokenAtEdge(appUrl *url.URL, c *cli.Context, log *zerolog.Logger) error {
+func verifyTokenAtEdge(appUrl *url.URL, appInfo *token.AppInfo, c *cli.Context, log *zerolog.Logger) error {
 	headers := buildRequestHeaders(c.StringSlice(sshHeaderFlag))
 	if c.IsSet(sshTokenIDFlag) {
 		headers.Add(h2mux.CFAccessClientIDHeader, c.String(sshTokenIDFlag))
@@ -435,7 +484,7 @@ func verifyTokenAtEdge(appUrl *url.URL, c *cli.Context, log *zerolog.Logger) err
 	if c.IsSet(sshTokenSecretFlag) {
 		headers.Add(h2mux.CFAccessClientSecretHeader, c.String(sshTokenSecretFlag))
 	}
-	options := &carrier.StartOptions{OriginURL: appUrl.String(), Headers: headers}
+	options := &carrier.StartOptions{AppInfo: appInfo, OriginURL: appUrl.String(), Headers: headers}
 
 	if valid, err := isTokenValid(options, log); err != nil {
 		return err
@@ -443,7 +492,7 @@ func verifyTokenAtEdge(appUrl *url.URL, c *cli.Context, log *zerolog.Logger) err
 		return nil
 	}
 
-	if err := token.RemoveTokenIfExists(appUrl); err != nil {
+	if err := token.RemoveTokenIfExists(appInfo); err != nil {
 		return err
 	}
 

cmd/cloudflared/app_forward_service.go

@@ -2,7 +2,7 @@ package main
 
 import (
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/access"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/config"
 
 	"github.com/rs/zerolog"
 )

cmd/cloudflared/app_resolver_service.go

@@ -1,7 +1,7 @@
 package main
 
 import (
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/tunneldns"
 
 	"github.com/rs/zerolog"
@@ -11,8 +11,9 @@ const (
 	// ResolverServiceType is used to identify what kind of overwatch service this is
 	ResolverServiceType = "resolver"
 
-	LogFieldResolverAddress = "resolverAddress"
-	LogFieldResolverPort    = "resolverPort"
+	LogFieldResolverAddress          = "resolverAddress"
+	LogFieldResolverPort             = "resolverPort"
+	LogFieldResolverMaxUpstreamConns = "resolverMaxUpstreamConns"
 )
 
 // ResolverService is used to wrap the tunneldns package's DNS over HTTP
@@ -57,7 +58,7 @@ func (s *ResolverService) Shutdown() {
 func (s *ResolverService) Run() error {
 	// create a listener
 	l, err := tunneldns.CreateListener(s.resolver.AddressOrDefault(), s.resolver.PortOrDefault(),
-		s.resolver.UpstreamsOrDefault(), s.resolver.BootstrapsOrDefault(), s.log)
+		s.resolver.UpstreamsOrDefault(), s.resolver.BootstrapsOrDefault(), s.resolver.MaxUpstreamConnectionsOrDefault(), s.log)
 	if err != nil {
 		return err
 	}
@@ -74,6 +75,7 @@ func (s *ResolverService) Run() error {
 	resolverLog := s.log.With().
 		Str(LogFieldResolverAddress, s.resolver.AddressOrDefault()).
 		Uint16(LogFieldResolverPort, s.resolver.PortOrDefault()).
+		Int(LogFieldResolverMaxUpstreamConns, s.resolver.MaxUpstreamConnectionsOrDefault()).
 		Logger()
 
 	resolverLog.Info().Msg("Starting resolver")

cmd/cloudflared/app_service.go

@@ -1,7 +1,7 @@
 package main
 
 import (
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/overwatch"
 
 	"github.com/rs/zerolog"

cmd/cloudflared/buildinfo/build_info.go

@@ -2,6 +2,7 @@ package buildinfo
 
 import (
 	"github.com/rs/zerolog"
+	"fmt"
 	"runtime"
 )
 
@@ -25,3 +26,7 @@ func (bi *BuildInfo) Log(log *zerolog.Logger) {
 	log.Info().Msgf("Version %s", bi.CloudflaredVersion)
 	log.Info().Msgf("GOOS: %s, GOVersion: %s, GoArch: %s", bi.GoOS, bi.GoVersion, bi.GoArch)
 }
+
+func (bi *BuildInfo) OSArch() string {
+	return fmt.Sprintf("%s_%s", bi.GoOS, bi.GoArch)
+}

cmd/cloudflared/cliutil/deprecated.go

@@ -0,0 +1,21 @@
+package cliutil
+
+import (
+	"fmt"
+
+	"github.com/urfave/cli/v2"
+)
+
+func RemovedCommand(name string) *cli.Command {
+	return &cli.Command{
+		Name: name,
+		Action: ErrorHandler(func(context *cli.Context) error {
+			return cli.Exit(
+				fmt.Sprintf("%s command is no longer supported by cloudflared. Consult Argo Tunnel documentation for possible alternative solutions.", name),
+				-1,
+			)
+		}),
+		Description: fmt.Sprintf("%s is deprecated", name),
+		Hidden: true,
+	}
+}

cmd/cloudflared/generic_service.go

@@ -8,6 +8,6 @@ import (
 	cli "github.com/urfave/cli/v2"
 )
 
-func runApp(app *cli.App, shutdownC, graceShutdownC chan struct{}) {
+func runApp(app *cli.App, graceShutdownC chan struct{}) {
 	app.Run(os.Args)
 }

cmd/cloudflared/linux_service.go

@@ -8,8 +8,8 @@ import (
 	"path/filepath"
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/tunnel"
+	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/logger"
 
 	"github.com/rs/zerolog"

cmd/cloudflared/main.go

@@ -2,18 +2,19 @@ package main
 
 import (
 	"fmt"
+	"math/rand"
 	"strings"
 	"time"
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/access"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/config"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/proxydns"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/tunnel"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/updater"
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/metrics"
 	"github.com/cloudflare/cloudflared/overwatch"
-	"github.com/cloudflare/cloudflared/tunneldns"
 	"github.com/cloudflare/cloudflared/watcher"
 
 	"github.com/getsentry/raven-go"
@@ -43,6 +44,7 @@ var (
 )
 
 func main() {
+	rand.Seed(time.Now().UnixNano())
 	metrics.RegisterBuildInfo(BuildTime, Version)
 	raven.SetRelease(Version)
 
@@ -115,7 +117,7 @@ func commands(version func(c *cli.Context)) []*cli.Command {
 If a new version exists, updates the agent binary and quits.
 Otherwise, does nothing.
 
-To determine if an update happened in a script, check for error code 64.`,
+To determine if an update happened in a script, check for error code 11.`,
 		},
 		{
 			Name: "version",
@@ -128,7 +130,7 @@ To determine if an update happened in a script, check for error code 64.`,
 		},
 	}
 	cmds = append(cmds, tunnel.Commands()...)
-	cmds = append(cmds, tunneldns.Command(false))
+	cmds = append(cmds, proxydns.Command(false))
 	cmds = append(cmds, access.Commands()...)
 	return cmds
 }

cmd/cloudflared/proxydns/cmd.go

@@ -0,0 +1,115 @@
+package proxydns
+
+import (
+	"net"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"github.com/urfave/cli/v2"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
+	"github.com/cloudflare/cloudflared/logger"
+	"github.com/cloudflare/cloudflared/metrics"
+	"github.com/cloudflare/cloudflared/tunneldns"
+)
+
+func Command(hidden bool) *cli.Command {
+	return &cli.Command{
+		Name:   "proxy-dns",
+		Action: cliutil.ErrorHandler(Run),
+		Usage:  "Run a DNS over HTTPS proxy server.",
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:    "metrics",
+				Value:   "localhost:",
+				Usage:   "Listen address for metrics reporting.",
+				EnvVars: []string{"TUNNEL_METRICS"},
+			},
+			&cli.StringFlag{
+				Name:    "address",
+				Usage:   "Listen address for the DNS over HTTPS proxy server.",
+				Value:   "localhost",
+				EnvVars: []string{"TUNNEL_DNS_ADDRESS"},
+			},
+			// Note TUN-3758 , we use Int because UInt is not supported with altsrc
+			&cli.IntFlag{
+				Name:    "port",
+				Usage:   "Listen on given port for the DNS over HTTPS proxy server.",
+				Value:   53,
+				EnvVars: []string{"TUNNEL_DNS_PORT"},
+			},
+			&cli.StringSliceFlag{
+				Name:    "upstream",
+				Usage:   "Upstream endpoint URL, you can specify multiple endpoints for redundancy.",
+				Value:   cli.NewStringSlice("https://1.1.1.1/dns-query", "https://1.0.0.1/dns-query"),
+				EnvVars: []string{"TUNNEL_DNS_UPSTREAM"},
+			},
+			&cli.StringSliceFlag{
+				Name:    "bootstrap",
+				Usage:   "bootstrap endpoint URL, you can specify multiple endpoints for redundancy.",
+				Value:   cli.NewStringSlice("https://162.159.36.1/dns-query", "https://162.159.46.1/dns-query", "https://[2606:4700:4700::1111]/dns-query", "https://[2606:4700:4700::1001]/dns-query"),
+				EnvVars: []string{"TUNNEL_DNS_BOOTSTRAP"},
+			},
+			&cli.IntFlag{
+				Name:    "max-upstream-conns",
+				Usage:   "Maximum concurrent connections to upstream. Setting to 0 means unlimited.",
+				Value:   tunneldns.MaxUpstreamConnsDefault,
+				EnvVars: []string{"TUNNEL_DNS_MAX_UPSTREAM_CONNS"},
+			},
+		},
+		ArgsUsage: " ", // can't be the empty string or we get the default output
+		Hidden:    hidden,
+	}
+}
+
+// Run implements a foreground runner
+func Run(c *cli.Context) error {
+	log := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
+
+	metricsListener, err := net.Listen("tcp", c.String("metrics"))
+	if err != nil {
+		log.Fatal().Err(err).Msg("Failed to open the metrics listener")
+	}
+
+	go metrics.ServeMetrics(metricsListener, nil, nil, log)
+
+	listener, err := tunneldns.CreateListener(
+		c.String("address"),
+		// Note TUN-3758 , we use Int because UInt is not supported with altsrc
+		uint16(c.Int("port")),
+		c.StringSlice("upstream"),
+		c.StringSlice("bootstrap"),
+		c.Int("max-upstream-conns"),
+		log,
+	)
+
+	if err != nil {
+		log.Err(err).Msg("Failed to create the listeners")
+		return err
+	}
+
+	// Try to start the server
+	readySignal := make(chan struct{})
+	err = listener.Start(readySignal)
+	if err != nil {
+		log.Err(err).Msg("Failed to start the listeners")
+		return listener.Stop()
+	}
+	<-readySignal
+
+	// Wait for signal
+	signals := make(chan os.Signal, 10)
+	signal.Notify(signals, syscall.SIGTERM, syscall.SIGINT)
+	defer signal.Stop(signals)
+	<-signals
+
+	// Shut down server
+	err = listener.Stop()
+	if err != nil {
+		log.Err(err).Msg("failed to stop")
+	}
+	return err
+}
+
+

cmd/cloudflared/service_template.go

@@ -12,7 +12,7 @@ import (
 
 	"github.com/mitchellh/go-homedir"
 
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/config"
 )
 
 type ServiceTemplate struct {

cmd/cloudflared/shell/shell.go

@@ -1,33 +0,0 @@
-package shell
-
-import (
-	"io"
-	"os"
-	"os/exec"
-)
-
-// OpenBrowser opens the specified URL in the default browser of the user
-func OpenBrowser(url string) error {
-	return getBrowserCmd(url).Start()
-}
-
-// Run will kick off a shell task and pipe the results to the respective std pipes
-func Run(cmd string, args ...string) error {
-	c := exec.Command(cmd, args...)
-	stderr, err := c.StderrPipe()
-	if err != nil {
-		return err
-	}
-	go func() {
-		io.Copy(os.Stderr, stderr)
-	}()
-
-	stdout, err := c.StdoutPipe()
-	if err != nil {
-		return err
-	}
-	go func() {
-		io.Copy(os.Stdout, stdout)
-	}()
-	return c.Run()
-}

cmd/cloudflared/tunnel/cmd.go

@@ -15,11 +15,11 @@ import (
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/proxydns"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/ui"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/updater"
+	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/connection"
-	"github.com/cloudflare/cloudflared/dbconnect"
 	"github.com/cloudflare/cloudflared/ingress"
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/metrics"
@@ -105,15 +105,15 @@ func Commands() []*cli.Command {
 		buildDeleteCommand(),
 		buildCleanupCommand(),
 		// for compatibility, allow following as tunnel subcommands
-		tunneldns.Command(true),
-		dbConnectCmd(),
+		proxydns.Command(true),
+		cliutil.RemovedCommand("db-connect"),
 	}
 
 	return []*cli.Command{
 		buildTunnelCommand(subcommands),
 		// for compatibility, allow following as top-level subcommands
 		buildLoginSubcommand(true),
-		dbConnectCmd(),
+		cliutil.RemovedCommand("db-connect"),
 	}
 }
 
@@ -287,16 +287,14 @@ func StartServer(
 	}
 
 	// update needs to be after DNS proxy is up to resolve equinox server address
-	if updater.IsAutoupdateEnabled(c, log) {
-		autoupdateFreq := c.Duration("autoupdate-freq")
-		log.Info().Dur("autoupdateFreq", autoupdateFreq).Msg("Autoupdate frequency is set")
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			autoupdater := updater.NewAutoUpdater(c.Duration("autoupdate-freq"), &listeners, log)
-			errC <- autoupdater.Run(ctx)
-		}()
-	}
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		autoupdater := updater.NewAutoUpdater(
+			c.Bool("no-autoupdate"), c.Duration("autoupdate-freq"), &listeners, log,
+		)
+		errC <- autoupdater.Run(ctx)
+	}()
 
 	// Serve DNS proxy stand-alone if no hostname or tag or app is going to run
 	if dnsProxyStandAlone(c) {
@@ -479,33 +477,6 @@ func addPortIfMissing(uri *url.URL, port int) string {
 	return fmt.Sprintf("%s:%d", uri.Hostname(), port)
 }
 
-func dbConnectCmd() *cli.Command {
-	cmd := dbconnect.Cmd()
-
-	// Append the tunnel commands so users can customize the daemon settings.
-	cmd.Flags = appendFlags(Flags(), cmd.Flags...)
-
-	// Override before to run tunnel validation before dbconnect validation.
-	cmd.Before = func(c *cli.Context) error {
-		err := SetFlagsFromConfigFile(c)
-		if err == nil {
-			err = dbconnect.CmdBefore(c)
-		}
-		return err
-	}
-
-	// Override action to setup the Proxy, then if successful, start the tunnel daemon.
-	cmd.Action = cliutil.ErrorHandler(func(c *cli.Context) error {
-		err := dbconnect.CmdAction(c)
-		if err == nil {
-			err = TunnelCommand(c)
-		}
-		return err
-	})
-
-	return cmd
-}
-
 // appendFlags will append extra flags to a slice of flags.
 //
 // The cli package will panic if two flags exist with the same name,
@@ -1011,6 +982,13 @@ func configureProxyDNSFlags(shouldHide bool) []cli.Flag {
 			EnvVars: []string{"TUNNEL_DNS_UPSTREAM"},
 			Hidden:  shouldHide,
 		}),
+		altsrc.NewIntFlag(&cli.IntFlag{
+			Name:    "proxy-dns-max-upstream-conns",
+			Usage:   "Maximum concurrent connections to upstream. Setting to 0 means unlimited.",
+			Value:   tunneldns.MaxUpstreamConnsDefault,
+			Hidden:  shouldHide,
+			EnvVars: []string{"TUNNEL_DNS_MAX_UPSTREAM_CONNS"},
+		}),
 		altsrc.NewStringSliceFlag(&cli.StringSliceFlag{
 			Name:  "proxy-dns-bootstrap",
 			Usage: "bootstrap endpoint URL, you can specify multiple endpoints for redundancy.",

cmd/cloudflared/tunnel/config_test.go

@@ -0,0 +1,13 @@
+package tunnel
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestDedup(t *testing.T) {
+	expected := []string{"a", "b"}
+	actual := dedup([]string{"a", "b", "a"})
+	require.ElementsMatch(t, expected, actual)
+}

cmd/cloudflared/tunnel/configuration.go

@@ -9,7 +9,7 @@ import (
 	"strings"
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/edgediscovery"
 	"github.com/cloudflare/cloudflared/h2mux"
@@ -109,7 +109,7 @@ func findOriginCert(originCertPath string, log *zerolog.Logger) (string, error)
 	// Check that the user has acquired a certificate using the login command
 	ok, err := config.FileExists(originCertPath)
 	if err != nil {
-		log.Error().Msgf("Cannot check if origin cert exists at path %s", originCertPath)
+		log.Error().Err(err).Msgf("Cannot check if origin cert exists at path %s", originCertPath)
 		return "", fmt.Errorf("cannot check if origin cert exists at path %s", originCertPath)
 	}
 	if !ok {
@@ -195,18 +195,21 @@ func prepareTunnelConfig(
 		ingressRules  ingress.Ingress
 		classicTunnel *connection.ClassicTunnelConfig
 	)
+	cfg := config.GetConfiguration()
 	if isNamedTunnel {
 		clientUUID, err := uuid.NewRandom()
 		if err != nil {
 			return nil, ingress.Ingress{}, errors.Wrap(err, "can't generate clientUUID")
 		}
+		log.Info().Msgf("Generated Client ID: %s", clientUUID)
+		features := append(c.StringSlice("features"), origin.FeatureSerializedHeaders)
 		namedTunnel.Client = tunnelpogs.ClientInfo{
 			ClientID: clientUUID[:],
-			Features: []string{origin.FeatureSerializedHeaders},
+			Features: dedup(features),
 			Version:  version,
-			Arch:     fmt.Sprintf("%s_%s", buildInfo.GoOS, buildInfo.GoArch),
+			Arch:     buildInfo.OSArch(),
 		}
-		ingressRules, err = ingress.ParseIngress(config.GetConfiguration())
+		ingressRules, err = ingress.ParseIngress(cfg)
 		if err != nil && err != ingress.ErrNoIngressRules {
 			return nil, ingress.Ingress{}, err
 		}
@@ -230,7 +233,14 @@ func prepareTunnelConfig(
 		}
 	}
 
-	protocolSelector, err := connection.NewProtocolSelector(c.String("protocol"), namedTunnel, edgediscovery.HTTP2Percentage, origin.ResolveTTL, log)
+	var warpRoutingService *ingress.WarpRoutingService
+	warpRoutingEnabled := isWarpRoutingEnabled(cfg.WarpRouting, isNamedTunnel)
+	if warpRoutingEnabled {
+		warpRoutingService = ingress.NewWarpRoutingService()
+		log.Info().Msgf("Warp-routing is enabled")
+	}
+
+	protocolSelector, err := connection.NewProtocolSelector(c.String("protocol"), warpRoutingEnabled, namedTunnel, edgediscovery.HTTP2Percentage, origin.ResolveTTL, log)
 	if err != nil {
 		return nil, ingress.Ingress{}, err
 	}
@@ -245,9 +255,9 @@ func prepareTunnelConfig(
 		edgeTLSConfigs[p] = edgeTLSConfig
 	}
 
-	originClient := origin.NewClient(ingressRules, tags, log)
+	originProxy := origin.NewOriginProxy(ingressRules, warpRoutingService, tags, log)
 	connectionConfig := &connection.Config{
-		OriginClient:    originClient,
+		OriginProxy:     originProxy,
 		GracePeriod:     c.Duration("grace-period"),
 		ReplaceExisting: c.Bool("force"),
 	}
@@ -262,7 +272,7 @@ func prepareTunnelConfig(
 
 	return &origin.TunnelConfig{
 		ConnectionConfig: connectionConfig,
-		BuildInfo:        buildInfo,
+		OSArch:           buildInfo.OSArch(),
 		ClientID:         clientID,
 		EdgeAddrs:        c.StringSlice("edge"),
 		HAConnections:    c.Int("ha-connections"),
@@ -286,6 +296,29 @@ func prepareTunnelConfig(
 	}, ingressRules, nil
 }
 
+func isWarpRoutingEnabled(warpConfig config.WarpRoutingConfig, isNamedTunnel bool) bool {
+	return warpConfig.Enabled && isNamedTunnel
+}
+
 func isRunningFromTerminal() bool {
 	return terminal.IsTerminal(int(os.Stdout.Fd()))
 }
+
+// Remove any duplicates from the slice
+func dedup(slice []string) []string {
+
+	// Convert the slice into a set
+	set := make(map[string]bool, 0)
+	for _, str := range slice {
+		set[str] = true
+	}
+
+	// Convert the set back into a slice
+	keys := make([]string, len(set))
+	i := 0
+	for str := range set {
+		keys[i] = str
+		i++
+	}
+	return keys
+}

cmd/cloudflared/tunnel/credential_finder.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"path/filepath"
 
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/config"
 
 	"github.com/google/uuid"
 	"github.com/rs/zerolog"

cmd/cloudflared/tunnel/ingress_subcommands.go

@@ -5,7 +5,7 @@ import (
 	"net/url"
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/ingress"
 
 	"github.com/pkg/errors"

cmd/cloudflared/tunnel/login.go

@@ -13,9 +13,9 @@ import (
 	"github.com/urfave/cli/v2"
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/transfer"
+	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/logger"
+	"github.com/cloudflare/cloudflared/token"
 )
 
 const (
@@ -56,7 +56,7 @@ func login(c *cli.Context) error {
 		return err
 	}
 
-	resourceData, err := transfer.Run(
+	resourceData, err := token.RunTransfer(
 		loginURL,
 		"cert",
 		"callback",

cmd/cloudflared/tunnel/server.go

@@ -1,6 +1,8 @@
 package tunnel
 
 import (
+	"fmt"
+
 	"github.com/cloudflare/cloudflared/tunneldns"
 
 	"github.com/pkg/errors"
@@ -13,7 +15,11 @@ func runDNSProxyServer(c *cli.Context, dnsReadySignal chan struct{}, shutdownC <
 	if port <= 0 || port > 65535 {
 		return errors.New("The 'proxy-dns-port' must be a valid port number in <1, 65535> range.")
 	}
-	listener, err := tunneldns.CreateListener(c.String("proxy-dns-address"), uint16(port), c.StringSlice("proxy-dns-upstream"), c.StringSlice("proxy-dns-bootstrap"), log)
+	maxUpstreamConnections := c.Int("proxy-dns-max-upstream-conns")
+	if maxUpstreamConnections < 0 {
+		return fmt.Errorf("'%s' must be 0 or higher", "proxy-dns-max-upstream-conns")
+	}
+	listener, err := tunneldns.CreateListener(c.String("proxy-dns-address"), uint16(port), c.StringSlice("proxy-dns-upstream"), c.StringSlice("proxy-dns-bootstrap"), maxUpstreamConnections, log)
 	if err != nil {
 		close(dnsReadySignal)
 		listener.Stop()

cmd/cloudflared/tunnel/subcommand_context.go

@@ -223,12 +223,7 @@ func (sc *subcommandContext) delete(tunnelIDs []uuid.UUID) error {
 		if !tunnel.DeletedAt.IsZero() {
 			return fmt.Errorf("Tunnel %s has already been deleted", tunnel.ID)
 		}
-		// Check if tunnel has existing connections and if force flag is set, cleanup connections
-		if len(tunnel.Connections) > 0 {
-			if !forceFlagSet {
-				return fmt.Errorf("You can not delete tunnel %s because it has active connections. To see connections run the 'list' command. If you believe the tunnel is not active, you can use a -f / --force flag with this command.", id)
-			}
-
+		if forceFlagSet {
 			if err := client.CleanupConnections(tunnel.ID); err != nil {
 				return errors.Wrapf(err, "Error cleaning up connections for tunnel %s", tunnel.ID)
 			}

cmd/cloudflared/tunnel/subcommands.go

@@ -22,7 +22,8 @@ import (
 	"gopkg.in/yaml.v2"
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/updater"
+	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/tunnelstore"
 )
@@ -87,6 +88,11 @@ var (
 			"overwrite the previous tunnel. If you want to use a single hostname with multiple " +
 			"tunnels, you can do so with Cloudflare's Load Balancer product.",
 	})
+	featuresFlag = altsrc.NewStringSliceFlag(&cli.StringSliceFlag{
+		Name:    "features",
+		Aliases: []string{"F"},
+		Usage:   "Opt into various features that are still being developed or tested.",
+	})
 	credentialsFileFlag = altsrc.NewStringFlag(&cli.StringFlag{
 		Name:    CredFileFlag,
 		Aliases: []string{CredFileFlagAlias},
@@ -96,7 +102,8 @@ var (
 	forceDeleteFlag = &cli.BoolFlag{
 		Name:    "force",
 		Aliases: []string{"f"},
-		Usage:   "Allows you to delete a tunnel, even if it has active connections.",
+		Usage: "Cleans up any stale connections before the tunnel is deleted. cloudflared will not " +
+			"delete a tunnel with connections without this flag.",
 		EnvVars: []string{"TUNNEL_RUN_FORCE_OVERWRITE"},
 	}
 	selectProtocolFlag = altsrc.NewStringFlag(&cli.StringFlag{
@@ -144,6 +151,9 @@ func createCommand(c *cli.Context) error {
 	}
 	name := c.Args().First()
 
+	warningChecker := updater.StartWarningCheck(c)
+	defer warningChecker.LogWarningIfAny(sc.log)
+
 	_, err = sc.create(name, c.String(CredFileFlag))
 	return errors.Wrap(err, "failed to create tunnel")
 }
@@ -222,6 +232,9 @@ func listCommand(c *cli.Context) error {
 		filter.ByTunnelID(tunnelID)
 	}
 
+	warningChecker := updater.StartWarningCheck(c)
+	defer warningChecker.LogWarningIfAny(sc.log)
+
 	tunnels, err := sc.list(filter)
 	if err != nil {
 		return err
@@ -266,6 +279,7 @@ func listCommand(c *cli.Context) error {
 	} else {
 		fmt.Println("You have no tunnels, use 'cloudflared tunnel create' to define a new tunnel")
 	}
+
 	return nil
 }
 
@@ -344,6 +358,9 @@ func deleteCommand(c *cli.Context) error {
 		return cliutil.UsageError(`"cloudflared tunnel delete" requires at least 1 argument, the ID or name of the tunnel to delete.`)
 	}
 
+	warningChecker := updater.StartWarningCheck(c)
+	defer warningChecker.LogWarningIfAny(sc.log)
+
 	tunnelIDs, err := sc.findIDs(c.Args().Slice())
 	if err != nil {
 		return err
@@ -370,6 +387,7 @@ func buildRunCommand() *cli.Command {
 		forceFlag,
 		credentialsFileFlag,
 		selectProtocolFlag,
+		featuresFlag,
 	}
 	flags = append(flags, configureProxyFlags(false)...)
 	return &cli.Command{
@@ -457,14 +475,23 @@ func buildRouteCommand() *cli.Command {
 	return &cli.Command{
 		Name:      "route",
 		Action:    cliutil.ErrorHandler(routeCommand),
-		Usage:     "Define what hostname or load balancer can route to this tunnel",
-		UsageText: "cloudflared tunnel [tunnel command options] route [subcommand options] dns|lb TUNNEL HOSTNAME [LB-POOL]",
-		Description: `The route defines what hostname or load balancer will proxy requests to this tunnel.
+		Usage:     "Define which traffic routed from Cloudflare edge to this tunnel: requests to a DNS hostname, to a Cloudflare Load Balancer, or traffic originating from Cloudflare WARP clients",
+		UsageText: "cloudflared tunnel [tunnel command options] route [subcommand options] [dns TUNNEL HOSTNAME]|[lb TUNNEL HOSTNAME LB-POOL]|[ip NETWORK TUNNEL]",
+		Description: `The route command defines how Cloudflare will proxy requests to this tunnel.
 
-   To route a hostname by creating a CNAME to tunnel's address:
-      cloudflared tunnel route dns <tunnel ID> <hostname>
-   To use this tunnel as a load balancer origin, creating pool and load balancer if necessary:
-      cloudflared tunnel route lb <tunnel ID> <load balancer name> <load balancer pool>`,
+To route a hostname by creating a DNS CNAME record to a tunnel:
+   cloudflared tunnel route dns <tunnel ID or name> <hostname>
+You can read more at: https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/routing-to-tunnel/dns
+
+To use this tunnel as a load balancer origin, creating pool and load balancer if necessary:
+   cloudflared tunnel route lb <tunnel ID or name> <hostname> <load balancer pool>
+You can read more at: https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/routing-to-tunnel/lb
+
+For Cloudflare WARP traffic to be routed to your private network, reachable from this tunnel as origins, use:
+   cloudflared tunnel route ip <network CIDR> <tunnel ID or name>
+Further information about managing Cloudflare WARP traffic to your tunnel is available at:
+   cloudflared tunnel route ip --help
+`,
 		CustomHelpTemplate: commandHelpTemplate(),
 		Subcommands: []*cli.Command{
 			buildRouteIPSubcommand(),

cmd/cloudflared/tunnel/teamnet_subcommands.go

@@ -7,6 +7,7 @@ import (
 	"text/tabwriter"
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/updater"
 	"github.com/cloudflare/cloudflared/teamnet"
 	"github.com/pkg/errors"
 
@@ -16,44 +17,45 @@ import (
 func buildRouteIPSubcommand() *cli.Command {
 	return &cli.Command{
 		Name:      "ip",
-		Category:  "Tunnel",
-		Usage:     "Configure and query private routes",
+		Usage:     "Configure and query Cloudflare WARP routing to services or private networks available through this tunnel.",
 		UsageText: "cloudflared tunnel [--config FILEPATH] route COMMAND [arguments...]",
-		Hidden:    true,
-		Description: `cloudflared can provision private routes from your private IP space to origins
-		in your corporate network. Users enrolled in your Cloudflare for Teams organization can reach
-		those routes through the Cloudflare Warp client. You can also build rules to determine who
-		can reach certain routes.
-		`,
+		Description: `cloudflared can provision private routes from any IP space to origins in your corporate network.
+Users enrolled in your Cloudflare for Teams organization can reach those routes through the
+Cloudflare WARP client. You can also build rules to determine who can reach certain routes.`,
 		Subcommands: []*cli.Command{
 			{
 				Name:      "add",
 				Action:    cliutil.ErrorHandler(addRouteCommand),
-				Usage:     "Add a new Teamnet route to the table",
+				Usage:     "Add any new network to the routing table reachable via the tunnel",
 				UsageText: "cloudflared tunnel [--config FILEPATH] route ip add [CIDR] [TUNNEL] [COMMENT?]",
-				Description: `Adds a private route to a CIDR in your private IP space. Requests will
-				be sent through the Cloudflare Warp client running on a user's machine, proxied
-				through the specified tunnel, and reach an IP in the given CIDR.`,
+				Description: `Adds any network route space (represented as a CIDR) to your routing table.
+That network space becomes reachable for requests egressing from a user's machine
+as long as it is using Cloudflare WARP client and is enrolled in the same account
+that is running the tunnel chosen here. Further, those requests will be proxied to
+the specified tunnel, and reach an IP in the given CIDR, as long as that IP is
+reachable from the tunnel.`,
 			},
 			{
 				Name:        "show",
+				Aliases:     []string{"list"},
 				Action:      cliutil.ErrorHandler(showRoutesCommand),
 				Usage:       "Show the routing table",
 				UsageText:   "cloudflared tunnel [--config FILEPATH] route ip show [flags]",
-				Description: `Shows your organization's private route table. You can use flags to filter the results.`,
-				Flags:       teamnet.FilterFlags,
+				Description: `Shows your organization private routing table. You can use flags to filter the results.`,
+				Flags:       showRoutesFlags(),
 			},
 			{
-				Name:        "delete",
-				Action:      cliutil.ErrorHandler(deleteRouteCommand),
-				Usage:       "Delete a row from your organization's private routing table",
-				UsageText:   "cloudflared tunnel [--config FILEPATH] route ip delete [CIDR]",
-				Description: `Deletes the row for a given CIDR from your routing table`,
+				Name:      "delete",
+				Action:    cliutil.ErrorHandler(deleteRouteCommand),
+				Usage:     "Delete a row from your organization's private routing table",
+				UsageText: "cloudflared tunnel [--config FILEPATH] route ip delete [CIDR]",
+				Description: `Deletes the row for a given CIDR from your routing table. That portion
+of your network will no longer be reachable by the WARP clients.`,
 			},
 			{
 				Name:      "get",
 				Action:    cliutil.ErrorHandler(getRouteByIPCommand),
-				Usage:     "Check which row of the routing table matches a given IP",
+				Usage:     "Check which row of the routing table matches a given IP.",
 				UsageText: "cloudflared tunnel [--config FILEPATH] route ip get [IP]",
 				Description: `Checks which row of the routing table will be used to proxy a given IP.
 				This helps check and validate your config.`,
@@ -62,6 +64,13 @@ func buildRouteIPSubcommand() *cli.Command {
 	}
 }
 
+func showRoutesFlags() []cli.Flag {
+	flags := make([]cli.Flag, 0)
+	flags = append(flags, teamnet.FilterFlags...)
+	flags = append(flags, outputFormatFlag)
+	return flags
+}
+
 func showRoutesCommand(c *cli.Context) error {
 	sc, err := newSubcommandContext(c)
 	if err != nil {
@@ -73,6 +82,9 @@ func showRoutesCommand(c *cli.Context) error {
 		return errors.Wrap(err, "invalid config for routing filters")
 	}
 
+	warningChecker := updater.StartWarningCheck(c)
+	defer warningChecker.LogWarningIfAny(sc.log)
+
 	routes, err := sc.listRoutes(filter)
 	if err != nil {
 		return err
@@ -87,6 +99,7 @@ func showRoutesCommand(c *cli.Context) error {
 	} else {
 		fmt.Println("You have no routes, use 'cloudflared tunnel route ip add' to add a route")
 	}
+
 	return nil
 }
 

cmd/cloudflared/updater/check.go

@@ -0,0 +1,49 @@
+package updater
+
+import (
+	"github.com/rs/zerolog"
+	"github.com/urfave/cli/v2"
+)
+
+type VersionWarningChecker struct {
+	warningChan chan string
+}
+
+func StartWarningCheck(c *cli.Context) VersionWarningChecker {
+	checker := VersionWarningChecker{
+		warningChan: make(chan string),
+	}
+
+	go func() {
+		options := updateOptions{
+			updateDisabled:  true,
+			isBeta:          c.Bool("beta"),
+			isStaging:       c.Bool("staging"),
+			isForced:        false,
+			intendedVersion: "",
+		}
+		checkResult, err := CheckForUpdate(options)
+		if err == nil {
+			checker.warningChan <- checkResult.UserMessage()
+		}
+		close(checker.warningChan)
+	}()
+
+	return checker
+}
+
+func (checker VersionWarningChecker) getWarning() string {
+	select {
+	case message := <-checker.warningChan:
+		return message
+	default:
+		// No feedback on time, we don't wait for it, since this is best-effort.
+		return ""
+	}
+}
+
+func (checker VersionWarningChecker) LogWarningIfAny(log *zerolog.Logger) {
+	if warning := checker.getWarning(); warning != "" {
+		log.Warn().Msg(warning)
+	}
+}

cmd/cloudflared/updater/service.go

@@ -1,14 +1,15 @@
 package updater
 
-// Version is the functions needed to perform an update
-type Version interface {
+// CheckResult is the behaviour resulting from checking in with the Update Service
+type CheckResult interface {
 	Apply() error
-	String() string
+	Version() string
+	UserMessage() string
 }
 
 // Service is the functions to get check for new updates
 type Service interface {
-	Check() (Version, error)
+	Check() (CheckResult, error)
 }
 
 const (
@@ -23,4 +24,7 @@ const (
 
 	// VersionKeyName is the url parameter key to send to the checkin API to specific what version to upgrade or downgrade to
 	VersionKeyName = "version"
+
+	// ClientVersionName is the url parameter key to send the version that this cloudflared is currently running with
+	ClientVersionName = "clientVersion"
 )

cmd/cloudflared/updater/update.go

@@ -3,18 +3,18 @@ package updater
 import (
 	"context"
 	"fmt"
-	"github.com/rs/zerolog"
 	"os"
 	"path/filepath"
 	"runtime"
 	"time"
 
+	"github.com/facebookgo/grace/gracenet"
+	"github.com/rs/zerolog"
 	"github.com/urfave/cli/v2"
 	"golang.org/x/crypto/ssh/terminal"
 
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/logger"
-	"github.com/facebookgo/grace/gracenet"
 )
 
 const (
@@ -61,16 +61,18 @@ func (e *statusErr) ExitCode() int {
 }
 
 type updateOptions struct {
-	isBeta    bool
-	isStaging bool
-	isForced  bool
-	version   string
+	updateDisabled  bool
+	isBeta          bool
+	isStaging       bool
+	isForced        bool
+	intendedVersion string
 }
 
 type UpdateOutcome struct {
-	Updated bool
-	Version string
-	Error   error
+	Updated     bool
+	Version     string
+	UserMessage string
+	Error       error
 }
 
 func (uo *UpdateOutcome) noUpdate() bool {
@@ -81,10 +83,10 @@ func Init(v string) {
 	version = v
 }
 
-func checkForUpdateAndApply(options updateOptions) UpdateOutcome {
+func CheckForUpdate(options updateOptions) (CheckResult, error) {
 	cfdPath, err := os.Executable()
 	if err != nil {
-		return UpdateOutcome{Error: err}
+		return nil, err
 	}
 
 	url := UpdateURL
@@ -93,24 +95,22 @@ func checkForUpdateAndApply(options updateOptions) UpdateOutcome {
 	}
 
 	s := NewWorkersService(version, url, cfdPath, Options{IsBeta: options.isBeta,
-		IsForced: options.isForced, RequestedVersion: options.version})
+		IsForced: options.isForced, RequestedVersion: options.intendedVersion})
 
-	v, err := s.Check()
+	return s.Check()
+}
+
+func applyUpdate(options updateOptions, update CheckResult) UpdateOutcome {
+	if update.Version() == "" || options.updateDisabled {
+		return UpdateOutcome{UserMessage: update.UserMessage()}
+	}
+
+	err := update.Apply()
 	if err != nil {
 		return UpdateOutcome{Error: err}
 	}
 
-	//already on the latest version
-	if v == nil {
-		return UpdateOutcome{}
-	}
-
-	err = v.Apply()
-	if err != nil {
-		return UpdateOutcome{Error: err}
-	}
-
-	return UpdateOutcome{Updated: true, Version: v.String()}
+	return UpdateOutcome{Updated: true, Version: update.Version(), UserMessage: update.UserMessage()}
 }
 
 // Update is the handler for the update command from the command line
@@ -137,7 +137,13 @@ func Update(c *cli.Context) error {
 		log.Info().Msg("cloudflared is set to upgrade to the latest publish version regardless of the current version")
 	}
 
-	updateOutcome := loggedUpdate(log, updateOptions{isBeta: isBeta, isStaging: isStaging, isForced: isForced, version: c.String("version")})
+	updateOutcome := loggedUpdate(log, updateOptions{
+		updateDisabled:  false,
+		isBeta:          isBeta,
+		isStaging:       isStaging,
+		isForced:        isForced,
+		intendedVersion: c.String("version"),
+	})
 	if updateOutcome.Error != nil {
 		return &statusErr{updateOutcome.Error}
 	}
@@ -152,12 +158,18 @@ func Update(c *cli.Context) error {
 
 // Checks for an update and applies it if one is available
 func loggedUpdate(log *zerolog.Logger, options updateOptions) UpdateOutcome {
-	updateOutcome := checkForUpdateAndApply(options)
+	checkResult, err := CheckForUpdate(options)
+	if err != nil {
+		log.Err(err).Msg("update check failed")
+		return UpdateOutcome{Error: err}
+	}
+
+	updateOutcome := applyUpdate(options, checkResult)
 	if updateOutcome.Updated {
 		log.Info().Str(LogFieldVersion, updateOutcome.Version).Msg("cloudflared has been updated")
 	}
 	if updateOutcome.Error != nil {
-		log.Err(updateOutcome.Error).Msg("update check failed: %s")
+		log.Err(updateOutcome.Error).Msg("update failed to apply")
 	}
 
 	return updateOutcome
@@ -177,44 +189,53 @@ type configurable struct {
 	freq    time.Duration
 }
 
-func NewAutoUpdater(freq time.Duration, listeners *gracenet.Net, log *zerolog.Logger) *AutoUpdater {
-	updaterConfigurable := &configurable{
-		enabled: true,
-		freq:    freq,
-	}
-	if freq == 0 {
-		updaterConfigurable.enabled = false
-		updaterConfigurable.freq = DefaultCheckUpdateFreq
-	}
+func NewAutoUpdater(updateDisabled bool, freq time.Duration, listeners *gracenet.Net, log *zerolog.Logger) *AutoUpdater {
 	return &AutoUpdater{
-		configurable:     updaterConfigurable,
+		configurable:     createUpdateConfig(updateDisabled, freq, log),
 		listeners:        listeners,
 		updateConfigChan: make(chan *configurable),
 		log:              log,
 	}
 }
 
+func createUpdateConfig(updateDisabled bool, freq time.Duration, log *zerolog.Logger) *configurable {
+	if isAutoupdateEnabled(log, updateDisabled, freq) {
+		log.Info().Dur("autoupdateFreq", freq).Msg("Autoupdate frequency is set")
+		return &configurable{
+			enabled: true,
+			freq:    freq,
+		}
+	} else {
+		return &configurable{
+			enabled: false,
+			freq:    DefaultCheckUpdateFreq,
+		}
+	}
+}
+
 func (a *AutoUpdater) Run(ctx context.Context) error {
 	ticker := time.NewTicker(a.configurable.freq)
 	for {
-		if a.configurable.enabled {
-			updateOutcome := loggedUpdate(a.log, updateOptions{})
-			if updateOutcome.Updated {
-				if IsSysV() {
-					// SysV doesn't have a mechanism to keep service alive, we have to restart the process
-					a.log.Info().Msg("Restarting service managed by SysV...")
-					pid, err := a.listeners.StartProcess()
-					if err != nil {
-						a.log.Err(err).Msg("Unable to restart server automatically")
-						return &statusErr{err: err}
-					}
-					// stop old process after autoupdate. Otherwise we create a new process
-					// after each update
-					a.log.Info().Msgf("PID of the new process is %d", pid)
+		updateOutcome := loggedUpdate(a.log, updateOptions{updateDisabled: !a.configurable.enabled})
+		if updateOutcome.Updated {
+			Init(updateOutcome.Version)
+			if IsSysV() {
+				// SysV doesn't have a mechanism to keep service alive, we have to restart the process
+				a.log.Info().Msg("Restarting service managed by SysV...")
+				pid, err := a.listeners.StartProcess()
+				if err != nil {
+					a.log.Err(err).Msg("Unable to restart server automatically")
+					return &statusErr{err: err}
 				}
-				return &statusSuccess{newVersion: updateOutcome.Version}
+				// stop old process after autoupdate. Otherwise we create a new process
+				// after each update
+				a.log.Info().Msgf("PID of the new process is %d", pid)
 			}
+			return &statusSuccess{newVersion: updateOutcome.Version}
+		} else if updateOutcome.UserMessage != "" {
+			a.log.Warn().Msg(updateOutcome.UserMessage)
 		}
+
 		select {
 		case <-ctx.Done():
 			return ctx.Err()
@@ -229,27 +250,18 @@ func (a *AutoUpdater) Run(ctx context.Context) error {
 }
 
 // Update is the method to pass new AutoUpdaterConfigurable to a running AutoUpdater. It is safe to be called concurrently
-func (a *AutoUpdater) Update(newFreq time.Duration) {
-	newConfigurable := &configurable{
-		enabled: true,
-		freq:    newFreq,
-	}
-	// A ero duration means autoupdate is disabled
-	if newFreq == 0 {
-		newConfigurable.enabled = false
-		newConfigurable.freq = DefaultCheckUpdateFreq
-	}
-	a.updateConfigChan <- newConfigurable
+func (a *AutoUpdater) Update(updateDisabled bool, newFreq time.Duration) {
+	a.updateConfigChan <- createUpdateConfig(updateDisabled, newFreq, a.log)
 }
 
-func IsAutoupdateEnabled(c *cli.Context, log *zerolog.Logger) bool {
-	if !SupportAutoUpdate(log) {
+func isAutoupdateEnabled(log *zerolog.Logger, updateDisabled bool, updateFreq time.Duration) bool {
+	if !supportAutoUpdate(log) {
 		return false
 	}
-	return !c.Bool("no-autoupdate") && c.Duration("autoupdate-freq") != 0
+	return !updateDisabled && updateFreq != 0
 }
 
-func SupportAutoUpdate(log *zerolog.Logger) bool {
+func supportAutoUpdate(log *zerolog.Logger) bool {
 	if runtime.GOOS == "windows" {
 		log.Info().Msg(noUpdateOnWindowsMessage)
 		return false

cmd/cloudflared/updater/update_test.go

@@ -2,17 +2,19 @@ package updater
 
 import (
 	"context"
+	"flag"
 	"testing"
 
 	"github.com/facebookgo/grace/gracenet"
 	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/assert"
+	"github.com/urfave/cli/v2"
 )
 
 func TestDisabledAutoUpdater(t *testing.T) {
 	listeners := &gracenet.Net{}
 	log := zerolog.Nop()
-	autoupdater := NewAutoUpdater(0, listeners, &log)
+	autoupdater := NewAutoUpdater(false, 0, listeners, &log)
 	ctx, cancel := context.WithCancel(context.Background())
 	errC := make(chan error)
 	go func() {
@@ -26,3 +28,13 @@ func TestDisabledAutoUpdater(t *testing.T) {
 	// Make sure that autoupdater terminates after canceling the context
 	assert.Equal(t, context.Canceled, <-errC)
 }
+
+func TestCheckInWithUpdater(t *testing.T) {
+	flagSet := flag.NewFlagSet(t.Name(), flag.PanicOnError)
+	cliCtx := cli.NewContext(cli.NewApp(), flagSet, nil)
+
+	warningChecker := StartWarningCheck(cliCtx)
+	warning := warningChecker.getWarning()
+	// Assuming this runs either on a release or development version, then the Worker will never have anything to tell us.
+	assert.Empty(t, warning)
+}

cmd/cloudflared/updater/workers_service.go

@@ -5,8 +5,6 @@ import (
 	"errors"
 	"net/http"
 	"runtime"
-	"strconv"
-	"strings"
 )
 
 // Options are the update options supported by the
@@ -27,6 +25,8 @@ type VersionResponse struct {
 	Version      string `json:"version"`
 	Checksum     string `json:"checksum"`
 	IsCompressed bool   `json:"compressed"`
+	UserMessage  string `json:"userMessage"`
+	ShouldUpdate bool   `json:"shouldUpdate"`
 	Error        string `json:"error"`
 }
 
@@ -50,7 +50,7 @@ func NewWorkersService(currentVersion, url, targetPath string, opts Options) Ser
 }
 
 // Check does a check in with the Workers API to get a new version update
-func (s *WorkersService) Check() (Version, error) {
+func (s *WorkersService) Check() (CheckResult, error) {
 	client := &http.Client{
 		Timeout: clientTimeout,
 	}
@@ -59,6 +59,7 @@ func (s *WorkersService) Check() (Version, error) {
 	q := req.URL.Query()
 	q.Add(OSKeyName, runtime.GOOS)
 	q.Add(ArchitectureKeyName, runtime.GOARCH)
+	q.Add(ClientVersionName, s.currentVersion)
 
 	if s.opts.IsBeta {
 		q.Add(BetaKeyName, "true")
@@ -84,70 +85,10 @@ func (s *WorkersService) Check() (Version, error) {
 		return nil, errors.New(v.Error)
 	}
 
-	if !s.opts.IsForced && !IsNewerVersion(s.currentVersion, v.Version) {
-		return nil, nil
+	versionToUpdate := ""
+	if v.ShouldUpdate {
+		versionToUpdate = v.Version
 	}
 
-	return NewWorkersVersion(v.URL, v.Version, v.Checksum, s.targetPath, v.IsCompressed), nil
-}
-
-// IsNewerVersion checks semantic versioning for the latest version
-// cloudflared tagging is more of a date than a semantic version,
-// but the same comparision logic still holds for major.minor.patch
-// e.g. 2020.8.2 is newer than 2020.8.1.
-func IsNewerVersion(current string, check string) bool {
-	if strings.Contains(strings.ToLower(current), "dev") {
-		return false // dev builds shouldn't update
-	}
-
-	cMajor, cMinor, cPatch, err := SemanticParts(current)
-	if err != nil {
-		return false
-	}
-
-	nMajor, nMinor, nPatch, err := SemanticParts(check)
-	if err != nil {
-		return false
-	}
-
-	if nMajor > cMajor {
-		return true
-	}
-
-	if nMajor == cMajor && nMinor > cMinor {
-		return true
-	}
-
-	if nMajor == cMajor && nMinor == cMinor && nPatch > cPatch {
-		return true
-	}
-	return false
-}
-
-// SemanticParts gets the major, minor, and patch version of a semantic version string
-// e.g. 3.1.2 would return 3, 1, 2, nil
-func SemanticParts(version string) (major int, minor int, patch int, err error) {
-	major = 0
-	minor = 0
-	patch = 0
-	parts := strings.Split(version, ".")
-	if len(parts) != 3 {
-		err = errors.New("invalid version")
-		return
-	}
-	major, err = strconv.Atoi(parts[0])
-	if err != nil {
-		return
-	}
-
-	minor, err = strconv.Atoi(parts[1])
-	if err != nil {
-		return
-	}
-
-	patch, err = strconv.Atoi(parts[2])
-	if err != nil {
-		return
-	}
-	return
+	return NewWorkersVersion(v.URL, versionToUpdate, v.Checksum, s.targetPath, v.UserMessage, v.IsCompressed), nil
 }

cmd/cloudflared/updater/workers_service_test.go

@@ -6,6 +6,7 @@ import (
 	"compress/gzip"
 	"crypto/sha256"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"log"
@@ -13,6 +14,8 @@ import (
 	"net/http/httptest"
 	"os"
 	"runtime"
+	"strconv"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -32,15 +35,20 @@ func respondWithData(w http.ResponseWriter, b []byte, status int) {
 	w.Write(b)
 }
 
+const mostRecentVersion = "2021.2.5"
+const mostRecentBetaVersion = "2021.3.0"
+const knownBuggyVersion = "2020.12.0"
+const expectedUserMsg = "This message is expected when running a known buggy version"
+
 func updateHandler(w http.ResponseWriter, r *http.Request) {
-	version := "2020.08.05"
+	version := mostRecentVersion
 	host := fmt.Sprintf("http://%s", r.Host)
 	url := host + "/download"
 
 	query := r.URL.Query()
 
 	if query.Get(BetaKeyName) == "true" {
-		version = "2020.08.06"
+		version = mostRecentBetaVersion
 		url = host + "/beta"
 	}
 
@@ -59,7 +67,15 @@ func updateHandler(w http.ResponseWriter, r *http.Request) {
 	fmt.Fprint(h, version)
 	checksum := fmt.Sprintf("%x", h.Sum(nil))
 
-	v := VersionResponse{URL: url, Version: version, Checksum: checksum}
+	var userMessage = ""
+	if query.Get(ClientVersionName) == knownBuggyVersion {
+		userMessage = expectedUserMsg
+	}
+	shouldUpdate := requestedVersion != "" || IsNewerVersion(query.Get(ClientVersionName), version)
+
+	v := VersionResponse{
+		URL: url, Version: version, Checksum: checksum, UserMessage: userMessage, ShouldUpdate: shouldUpdate,
+	}
 	respondWithJSON(w, v, http.StatusOK)
 }
 
@@ -71,7 +87,7 @@ func gzipUpdateHandler(w http.ResponseWriter, r *http.Request) {
 	checksum := fmt.Sprintf("%x", h.Sum(nil))
 
 	url := fmt.Sprintf("http://%s/gzip-download.tgz", r.Host)
-	v := VersionResponse{URL: url, Version: version, Checksum: checksum}
+	v := VersionResponse{URL: url, Version: version, Checksum: checksum, ShouldUpdate: true}
 	respondWithJSON(w, v, http.StatusOK)
 }
 
@@ -96,7 +112,7 @@ func compressedDownloadHandler(w http.ResponseWriter, r *http.Request) {
 }
 
 func downloadHandler(w http.ResponseWriter, r *http.Request) {
-	version := "2020.08.05"
+	version := mostRecentVersion
 	requestedVersion := r.URL.Query().Get(VersionKeyName)
 	if requestedVersion != "" {
 		version = requestedVersion
@@ -105,13 +121,71 @@ func downloadHandler(w http.ResponseWriter, r *http.Request) {
 }
 
 func betaHandler(w http.ResponseWriter, r *http.Request) {
-	respondWithData(w, []byte("2020.08.06"), http.StatusOK)
+	respondWithData(w, []byte(mostRecentBetaVersion), http.StatusOK)
 }
 
 func failureHandler(w http.ResponseWriter, r *http.Request) {
 	respondWithJSON(w, VersionResponse{Error: "unsupported os and architecture"}, http.StatusBadRequest)
 }
 
+func IsNewerVersion(current string, check string) bool {
+	if current == "" || check == "" {
+		return false
+	}
+	if strings.Contains(strings.ToLower(current), "dev") {
+		return false // dev builds shouldn't update
+	}
+
+	cMajor, cMinor, cPatch, err := SemanticParts(current)
+	if err != nil {
+		return false
+	}
+
+	nMajor, nMinor, nPatch, err := SemanticParts(check)
+	if err != nil {
+		return false
+	}
+
+	if nMajor > cMajor {
+		return true
+	}
+
+	if nMajor == cMajor && nMinor > cMinor {
+		return true
+	}
+
+	if nMajor == cMajor && nMinor == cMinor && nPatch > cPatch {
+		return true
+	}
+	return false
+}
+
+func SemanticParts(version string) (major int, minor int, patch int, err error) {
+	major = 0
+	minor = 0
+	patch = 0
+	parts := strings.Split(version, ".")
+	if len(parts) != 3 {
+		err = errors.New("invalid version")
+		return
+	}
+	major, err = strconv.Atoi(parts[0])
+	if err != nil {
+		return
+	}
+
+	minor, err = strconv.Atoi(parts[1])
+	if err != nil {
+		return
+	}
+
+	patch, err = strconv.Atoi(parts[2])
+	if err != nil {
+		return
+	}
+	return
+}
+
 func createServer() *httptest.Server {
 	mux := http.NewServeMux()
 	mux.HandleFunc("/updater", updateHandler)
@@ -124,7 +198,7 @@ func createServer() *httptest.Server {
 }
 
 func createTestFile(t *testing.T, path string) {
-	f, err := os.Create("tmpfile")
+	f, err := os.Create(path)
 	require.NoError(t, err)
 	fmt.Fprint(f, "2020.08.04")
 	f.Close()
@@ -142,13 +216,13 @@ func TestUpdateService(t *testing.T) {
 	s := NewWorkersService("2020.8.2", fmt.Sprintf("%s/updater", ts.URL), testFilePath, Options{})
 	v, err := s.Check()
 	require.NoError(t, err)
-	require.Equal(t, v.String(), "2020.08.05")
+	require.Equal(t, v.Version(), mostRecentVersion)
 
 	require.NoError(t, v.Apply())
 	dat, err := ioutil.ReadFile(testFilePath)
 	require.NoError(t, err)
 
-	require.Equal(t, string(dat), "2020.08.05")
+	require.Equal(t, string(dat), mostRecentVersion)
 }
 
 func TestBetaUpdateService(t *testing.T) {
@@ -162,13 +236,13 @@ func TestBetaUpdateService(t *testing.T) {
 	s := NewWorkersService("2020.8.2", fmt.Sprintf("%s/updater", ts.URL), testFilePath, Options{IsBeta: true})
 	v, err := s.Check()
 	require.NoError(t, err)
-	require.Equal(t, v.String(), "2020.08.06")
+	require.Equal(t, v.Version(), mostRecentBetaVersion)
 
 	require.NoError(t, v.Apply())
 	dat, err := ioutil.ReadFile(testFilePath)
 	require.NoError(t, err)
 
-	require.Equal(t, string(dat), "2020.08.06")
+	require.Equal(t, string(dat), mostRecentBetaVersion)
 }
 
 func TestFailUpdateService(t *testing.T) {
@@ -193,10 +267,11 @@ func TestNoUpdateService(t *testing.T) {
 	createTestFile(t, testFilePath)
 	defer os.Remove(testFilePath)
 
-	s := NewWorkersService("2020.8.5", fmt.Sprintf("%s/updater", ts.URL), testFilePath, Options{})
+	s := NewWorkersService(mostRecentVersion, fmt.Sprintf("%s/updater", ts.URL), testFilePath, Options{})
 	v, err := s.Check()
 	require.NoError(t, err)
-	require.Nil(t, v)
+	require.NotNil(t, v)
+	require.Empty(t, v.Version())
 }
 
 func TestForcedUpdateService(t *testing.T) {
@@ -210,13 +285,13 @@ func TestForcedUpdateService(t *testing.T) {
 	s := NewWorkersService("2020.8.5", fmt.Sprintf("%s/updater", ts.URL), testFilePath, Options{IsForced: true})
 	v, err := s.Check()
 	require.NoError(t, err)
-	require.Equal(t, v.String(), "2020.08.05")
+	require.Equal(t, v.Version(), mostRecentVersion)
 
 	require.NoError(t, v.Apply())
 	dat, err := ioutil.ReadFile(testFilePath)
 	require.NoError(t, err)
 
-	require.Equal(t, string(dat), "2020.08.05")
+	require.Equal(t, string(dat), mostRecentVersion)
 }
 
 func TestUpdateSpecificVersionService(t *testing.T) {
@@ -231,7 +306,7 @@ func TestUpdateSpecificVersionService(t *testing.T) {
 	s := NewWorkersService("2020.8.2", fmt.Sprintf("%s/updater", ts.URL), testFilePath, Options{RequestedVersion: reqVersion})
 	v, err := s.Check()
 	require.NoError(t, err)
-	require.Equal(t, reqVersion, v.String())
+	require.Equal(t, reqVersion, v.Version())
 
 	require.NoError(t, v.Apply())
 	dat, err := ioutil.ReadFile(testFilePath)
@@ -251,7 +326,7 @@ func TestCompressedUpdateService(t *testing.T) {
 	s := NewWorkersService("2020.8.2", fmt.Sprintf("%s/compressed", ts.URL), testFilePath, Options{})
 	v, err := s.Check()
 	require.NoError(t, err)
-	require.Equal(t, "2020.09.02", v.String())
+	require.Equal(t, "2020.09.02", v.Version())
 
 	require.NoError(t, v.Apply())
 	dat, err := ioutil.ReadFile(testFilePath)
@@ -260,14 +335,17 @@ func TestCompressedUpdateService(t *testing.T) {
 	require.Equal(t, "2020.09.02", string(dat))
 }
 
-func TestVersionParsing(t *testing.T) {
-	require.False(t, IsNewerVersion("2020.8.2", "2020.8.2"))
-	require.True(t, IsNewerVersion("2020.8.2", "2020.8.3"))
-	require.True(t, IsNewerVersion("2020.8.2", "2021.1.2"))
-	require.True(t, IsNewerVersion("2020.8.2", "2020.9.1"))
-	require.True(t, IsNewerVersion("2020.8.2", "2020.12.45"))
-	require.False(t, IsNewerVersion("2020.8.2", "2020.6.3"))
-	require.False(t, IsNewerVersion("DEV", "2020.8.5"))
-	require.False(t, IsNewerVersion("2020.8.2", "asdlkfjasdf"))
-	require.True(t, IsNewerVersion("3.0.1", "4.2.1"))
+func TestUpdateWhenRunningKnownBuggyVersion(t *testing.T) {
+	ts := createServer()
+	defer ts.Close()
+
+	testFilePath := "tmpfile"
+	createTestFile(t, testFilePath)
+	defer os.Remove(testFilePath)
+
+	s := NewWorkersService(knownBuggyVersion, fmt.Sprintf("%s/updater", ts.URL), testFilePath, Options{})
+	v, err := s.Check()
+	require.NoError(t, err)
+	require.Equal(t, v.Version(), mostRecentVersion)
+	require.Equal(t, v.UserMessage(), expectedUserMsg)
 }

cmd/cloudflared/updater/workers_update.go

@@ -54,6 +54,7 @@ type WorkersVersion struct {
 	version      string
 	targetPath   string
 	isCompressed bool
+	userMessage  string
 }
 
 // NewWorkersVersion creates a new Version object. This is normally created by a WorkersService JSON checkin response
@@ -61,8 +62,17 @@ type WorkersVersion struct {
 // version is the version of this update
 // checksum is the expected checksum of the downloaded file
 // target path is where the file should be replace. Normally this the running cloudflared's path
-func NewWorkersVersion(url, version, checksum, targetPath string, isCompressed bool) Version {
-	return &WorkersVersion{downloadURL: url, version: version, checksum: checksum, targetPath: targetPath, isCompressed: isCompressed}
+// userMessage is a possible message to convey back to the user after having checked in with the Updater Service
+// isCompressed tells whether the asset to update cloudflared is compressed or not
+func NewWorkersVersion(url, version, checksum, targetPath, userMessage string, isCompressed bool) CheckResult {
+	return &WorkersVersion{
+		downloadURL:  url,
+		version:      version,
+		checksum:     checksum,
+		targetPath:   targetPath,
+		isCompressed: isCompressed,
+		userMessage:  userMessage,
+	}
 }
 
 // Apply does the actual verification and update logic.
@@ -114,10 +124,16 @@ func (v *WorkersVersion) Apply() error {
 }
 
 // String returns the version number of this update/release (e.g. 2020.08.05)
-func (v *WorkersVersion) String() string {
+func (v *WorkersVersion) Version() string {
 	return v.version
 }
 
+// String returns a possible message to convey back to user after having checked in with the Updater Service. E.g.
+// it can warn about the need to update the version currently running.
+func (v *WorkersVersion) UserMessage() string {
+	return v.userMessage
+}
+
 // download the file from the link in the json
 func download(url, filepath string, isCompressed bool) error {
 	client := &http.Client{

component-tests/.gitignore

@@ -0,0 +1,2 @@
+__pycache__
+.pytest_cache

component-tests/README.md

@@ -0,0 +1,49 @@
+# Requirements
+1. Python 3.7 or later with packages in the given `requirements.txt`
+   - E.g. with conda:
+   - `conda create -n component-tests python=3.7`  
+   - `conda activate component-tests`
+   - `pip3 install -r requirements.txt`
+
+2. Create a config yaml file, for example:
+```
+cloudflared_binary: "cloudflared"
+tunnel: "3d539f97-cd3a-4d8e-c33b-65e9099c7a8d"
+credentials_file: "/Users/tunnel/.cloudflared/3d539f97-cd3a-4d8e-c33b-65e9099c7a8d.json"
+classic_hostname: "classic-tunnel-component-tests.example.com"
+origincert: "/Users/tunnel/.cloudflared/cert.pem"
+ingress:
+- hostname: named-tunnel-component-tests.example.com
+  service: http_status:200
+- service: http_status:404
+```
+
+3. Route hostname to the tunnel. For the example config above, we can do that via
+```
+   cloudflared tunnel route dns 3d539f97-cd3a-4d8e-c33b-65e9099c7a8d named-tunnel-component-tests.example.com
+```
+
+4. Turn on linter
+If you are using Visual Studio, follow https://code.visualstudio.com/docs/python/linting to turn on linter.
+
+5. Turn on formatter
+If you are using Visual Studio, follow https://code.visualstudio.com/docs/python/editing#_formatting
+to turn on formatter and https://marketplace.visualstudio.com/items?itemName=cbrevik.toggle-format-on-save
+to turn on format on save.
+
+# How to run
+Specify path to config file via env var `COMPONENT_TESTS_CONFIG`. This is required.
+## All tests
+Run `pytest` inside this(component-tests) folder
+
+## Specific files
+Run `pytest <file 1 name>.py <file 2 name>.py`
+
+## Specific tests
+Run `pytest file.py -k <test 1 name> -k <test 2 name>`
+
+## Live Logging
+Running with `-o log_cli=true` outputs logging to CLI as the tests are. By default the log level is WARN.
+`--log-cli-level` control logging level.
+For example, to log at info level, run `pytest -o log_cli=true --log-cli-level=INFO`.
+See https://docs.pytest.org/en/latest/logging.html#live-logs for more documentation on logging.

component-tests/config.py

@@ -0,0 +1,132 @@
+#!/usr/bin/env python
+import base64
+import copy
+import os
+import yaml
+
+from dataclasses import dataclass, InitVar
+
+from constants import METRICS_PORT
+
+from util import LOGGER
+
+# frozen=True raises exception when assigning to fields. This emulates immutability
+
+
+@dataclass(frozen=True)
+class BaseConfig:
+    cloudflared_binary: str
+    no_autoupdate: bool = True
+    metrics: str = f'localhost:{METRICS_PORT}'
+
+    def merge_config(self, additional):
+        config = copy.copy(additional)
+        config['no-autoupdate'] = self.no_autoupdate
+        config['metrics'] = self.metrics
+        return config
+
+
+@dataclass(frozen=True)
+class NamedTunnelBaseConfig(BaseConfig):
+    # The attributes of the parent class are ordered before attributes in this class,
+    # so we have to use default values here and check if they are set in __post_init__
+    tunnel: str = None
+    credentials_file: str = None
+    ingress: list = None
+
+    def __post_init__(self):
+        if self.tunnel is None:
+            raise TypeError("Field tunnel is not set")
+        if self.credentials_file is None:
+            raise TypeError("Field credentials_file is not set")
+        if self.ingress is None:
+            raise TypeError("Field ingress is not set")
+
+    def merge_config(self, additional):
+        config = super(NamedTunnelBaseConfig, self).merge_config(additional)
+        config['tunnel'] = self.tunnel
+        config['credentials-file'] = self.credentials_file
+        # In some cases we want to override default ingress, such as in config tests
+        if 'ingress' not in config:
+            config['ingress'] = self.ingress
+        return config
+
+
+@dataclass(frozen=True)
+class NamedTunnelConfig(NamedTunnelBaseConfig):
+    full_config: dict = None
+    additional_config: InitVar[dict] = {}
+
+    def __post_init__(self, additional_config):
+        # Cannot call set self.full_config because the class is frozen, instead, we can use __setattr__
+        # https://docs.python.org/3/library/dataclasses.html#frozen-instances
+        object.__setattr__(self, 'full_config',
+                           self.merge_config(additional_config))
+
+    def get_url(self):
+        return "https://" + self.ingress[0]['hostname']
+
+
+@dataclass(frozen=True)
+class ClassicTunnelBaseConfig(BaseConfig):
+    hostname: str = None
+    origincert: str = None
+
+    def __post_init__(self):
+        if self.hostname is None:
+            raise TypeError("Field tunnel is not set")
+        if self.origincert is None:
+            raise TypeError("Field credentials_file is not set")
+
+    def merge_config(self, additional):
+        config = super(ClassicTunnelBaseConfig, self).merge_config(additional)
+        config['hostname'] = self.hostname
+        config['origincert'] = self.origincert
+        return config
+
+
+@dataclass(frozen=True)
+class ClassicTunnelConfig(ClassicTunnelBaseConfig):
+    full_config: dict = None
+    additional_config: InitVar[dict] = {}
+
+    def __post_init__(self, additional_config):
+        # Cannot call set self.full_config because the class is frozen, instead, we can use __setattr__
+        # https://docs.python.org/3/library/dataclasses.html#frozen-instances
+        object.__setattr__(self, 'full_config',
+                           self.merge_config(additional_config))
+
+    def get_url(self):
+        return "https://" + self.hostname
+
+
+def build_config_from_env():
+    config_path = get_env("COMPONENT_TESTS_CONFIG")
+    config_content = base64.b64decode(
+        get_env("COMPONENT_TESTS_CONFIG_CONTENT")).decode('utf-8')
+    config_yaml = yaml.safe_load(config_content)
+
+    credentials_file = get_env("COMPONENT_TESTS_CREDENTIALS_FILE")
+    write_file(credentials_file, config_yaml["credentials_file"])
+
+    origincert = get_env("COMPONENT_TESTS_ORIGINCERT")
+    write_file(origincert, config_yaml["origincert"])
+
+    write_file(config_content, config_path)
+
+
+def write_file(content, path):
+    with open(path, 'w') as outfile:
+        outfile.write(content)
+        outfile.close
+
+
+def get_env(env_name):
+    val = os.getenv(env_name)
+    if val is None:
+        raise Exception(f"{env_name} is not set")
+    return val
+
+
+if __name__ == '__main__':
+    build_config_from_env()

component-tests/conftest.py

@@ -0,0 +1,35 @@
+import os
+import pytest
+import yaml
+
+from time import sleep
+
+from config import NamedTunnelConfig, ClassicTunnelConfig
+from constants import BACKOFF_SECS
+from util import LOGGER
+
+
+@pytest.fixture(scope="session")
+def component_tests_config():
+    config_file = os.getenv("COMPONENT_TESTS_CONFIG")
+    if config_file is None:
+        raise Exception(
+            "Need to provide path to config file in COMPONENT_TESTS_CONFIG")
+    with open(config_file, 'r') as stream:
+        config = yaml.safe_load(stream)
+        LOGGER.info(f"component tests base config {config}")
+
+        def _component_tests_config(additional_config={}, named_tunnel=True):
+            if named_tunnel:
+                return NamedTunnelConfig(additional_config=additional_config,
+                                         cloudflared_binary=config['cloudflared_binary'], tunnel=config['tunnel'], credentials_file=config['credentials_file'],  ingress=config['ingress'])
+            return ClassicTunnelConfig(
+                additional_config=additional_config, cloudflared_binary=config['cloudflared_binary'], hostname=config['classic_hostname'], origincert=config['origincert'])
+
+        return _component_tests_config
+
+
+# This fixture is automatically called before each tests to make sure the previous cloudflared has been shutdown
+@pytest.fixture(autouse=True)
+def wait_previous_cloudflared():
+    sleep(BACKOFF_SECS)

component-tests/constants.py

@@ -0,0 +1,3 @@
+METRICS_PORT = 51000
+MAX_RETRIES = 3
+BACKOFF_SECS = 5

component-tests/requirements.txt

@@ -0,0 +1,4 @@
+pytest==6.2.2
+pyyaml==5.4.1
+requests==2.25.1
+retrying==1.3.3

component-tests/test_config.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+from util import start_cloudflared
+
+
+class TestConfig:
+    # tmp_path is a fixture provides a temporary directory unique to the test invocation
+    def test_validate_ingress_rules(self, tmp_path, component_tests_config):
+        extra_config = {
+            'ingress': [
+                {
+                    "hostname": "example.com",
+                    "service": "https://localhost:8000",
+                    "originRequest": {
+                        "originServerName": "test.example.com",
+                        "caPool": "/etc/certs/ca.pem"
+                    },
+                },
+                {
+                    "hostname": "api.example.com",
+                    "path": "login",
+                    "service": "https://localhost:9000",
+                },
+                {
+                    "hostname": "wss.example.com",
+                    "service": "wss://localhost:8000",
+                },
+                {
+                    "hostname": "ssh.example.com",
+                    "service": "ssh://localhost:8000",
+                },
+                {"service": "http_status:404"}
+            ],
+        }
+        config = component_tests_config(extra_config)
+        validate_args = ["ingress", "validate"]
+        _ = start_cloudflared(tmp_path, config, validate_args)
+
+        self.match_rule(tmp_path, config,
+                        "http://example.com/index.html", 1)
+        self.match_rule(tmp_path, config,
+                        "https://example.com/index.html", 1)
+        self.match_rule(tmp_path, config,
+                        "https://api.example.com/login", 2)
+        self.match_rule(tmp_path, config,
+                        "https://wss.example.com", 3)
+        self.match_rule(tmp_path, config,
+                        "https://ssh.example.com", 4)
+        self.match_rule(tmp_path, config,
+                        "https://api.example.com", 5)
+
+    # This is used to check that the command tunnel ingress url <url> matches rule number <rule_num>. Note that rule number uses 1-based indexing
+
+    def match_rule(self, tmp_path, config, url, rule_num):
+        args = ["ingress", "rule", url]
+        match_rule = start_cloudflared(tmp_path, config, args)
+
+        assert f"Matched rule #{rule_num}" .encode() in match_rule.stdout

component-tests/test_logging.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python
+import json
+import os
+from util import start_cloudflared, wait_tunnel_ready, send_requests, LOGGER
+
+
+class TestLogging:
+    # Rolling logger rotate log files after 1 MB
+    rotate_after_size = 1000 * 1000
+    default_log_file = "cloudflared.log"
+    expect_message = "Starting tunnel"
+
+    def test_logging_to_terminal(self, tmp_path, component_tests_config):
+        config = component_tests_config()
+        with start_cloudflared(tmp_path, config, new_process=True) as cloudflared:
+            wait_tunnel_ready()
+            self.assert_log_to_terminal(cloudflared)
+
+    def test_logging_to_file(self, tmp_path, component_tests_config):
+        log_file = tmp_path / self.default_log_file
+        extra_config = {
+            # Convert from pathlib.Path to str
+            "logfile": str(log_file),
+        }
+        config = component_tests_config(extra_config)
+        with start_cloudflared(tmp_path, config, new_process=True, capture_output=False):
+            wait_tunnel_ready()
+            self.assert_log_in_file(log_file)
+            self.assert_json_log(log_file)
+
+    def test_logging_to_dir(self, tmp_path, component_tests_config):
+        log_dir = tmp_path / "logs"
+        extra_config = {
+            "loglevel": "debug",
+            # Convert from pathlib.Path to str
+            "log-directory": str(log_dir),
+        }
+        config = component_tests_config(extra_config)
+        with start_cloudflared(tmp_path, config, new_process=True, capture_output=False):
+            wait_tunnel_ready()
+            self.assert_log_to_dir(config, log_dir)
+
+    def assert_log_to_terminal(self, cloudflared):
+        stderr = cloudflared.stderr.read(200)
+        # cloudflared logs the following when it first starts
+        # 2021-03-10T12:30:39Z INF Starting tunnel tunnelID=<tunnel ID>
+        assert self.expect_message.encode(
+        ) in stderr, f"{stderr} doesn't contain {self.expect_message}"
+
+    def assert_log_in_file(self, file, expect_message=""):
+        with open(file, "r") as f:
+            log = f.read(200)
+            # cloudflared logs the following when it first starts
+            # {"level":"info","tunnelID":"<tunnel ID>","time":"2021-03-10T12:21:13Z","message":"Starting tunnel"}
+            assert self.expect_message in log, f"{log} doesn't contain {self.expect_message}"
+
+    def assert_json_log(self, file):
+        with open(file, "r") as f:
+            line = f.readline()
+            json_log = json.loads(line)
+            self.assert_in_json(json_log, "level")
+            self.assert_in_json(json_log, "time")
+            self.assert_in_json(json_log, "message")
+
+    def assert_in_json(self, j, key):
+        assert key in j, f"{key} is not in j"
+
+    def assert_log_to_dir(self, config, log_dir):
+        max_batches = 3
+        batch_requests = 1000
+        for _ in range(max_batches):
+            send_requests(config.get_url(),
+                          batch_requests, require_ok=False)
+            files = os.listdir(log_dir)
+            if len(files) == 2:
+                current_log_file_index = files.index(self.default_log_file)
+                current_file = log_dir / files[current_log_file_index]
+                stats = os.stat(current_file)
+                assert stats.st_size > 0
+                self.assert_json_log(current_file)
+
+                # One file is the current log file, the other is the rotated log file
+                rotated_log_file_index = 0 if current_log_file_index == 1 else 1
+                rotated_file = log_dir / files[rotated_log_file_index]
+                stats = os.stat(rotated_file)
+                assert stats.st_size > self.rotate_after_size
+                self.assert_log_in_file(rotated_file)
+                self.assert_json_log(current_file)
+                return
+
+        raise Exception(
+            f"Log file isn't rotated after sending {max_batches * batch_requests} requests")

component-tests/test_reconnect.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+import copy
+
+from retrying import retry
+from time import sleep
+
+from util import start_cloudflared, wait_tunnel_ready, check_tunnel_not_ready, send_requests
+
+
+class TestReconnect():
+    default_ha_conns = 4
+    default_reconnect_secs = 5
+    extra_config = {
+        "stdin-control": True,
+    }
+
+    def test_named_reconnect(self, tmp_path, component_tests_config):
+        config = component_tests_config(self.extra_config)
+        with start_cloudflared(tmp_path, config, new_process=True, allow_input=True) as cloudflared:
+            # Repeat the test multiple times because some issues only occur after multiple reconnects
+            self.assert_reconnect(config, cloudflared, 5)
+
+    def test_classic_reconnect(self, tmp_path, component_tests_config):
+        extra_config = copy.copy(self.extra_config)
+        extra_config["hello-world"] = True
+        config = component_tests_config(
+            additional_config=extra_config, named_tunnel=False)
+        with start_cloudflared(tmp_path, config, cfd_args=[], new_process=True, allow_input=True) as cloudflared:
+            self.assert_reconnect(config, cloudflared, 1)
+
+    def send_reconnect(self, cloudflared, secs):
+        # Although it is recommended to use the Popen.communicate method, we cannot
+        # use it because it blocks on reading stdout and stderr until EOF is reached
+        cloudflared.stdin.write(f"reconnect {secs}s\n".encode())
+        cloudflared.stdin.flush()
+
+    def assert_reconnect(self, config, cloudflared, repeat):
+        wait_tunnel_ready()
+        for _ in range(repeat):
+            for i in range(self.default_ha_conns):
+                self.send_reconnect(cloudflared, self.default_reconnect_secs)
+                expect_connections = self.default_ha_conns-i-1
+                if expect_connections > 0:
+                    wait_tunnel_ready(expect_connections=expect_connections)
+                else:
+                    check_tunnel_not_ready()
+
+            sleep(self.default_reconnect_secs + 10)
+            wait_tunnel_ready()
+            send_requests(config.get_url(), 1)

component-tests/util.py

@@ -0,0 +1,84 @@
+from contextlib import contextmanager
+import logging
+import requests
+from retrying import retry
+import subprocess
+import yaml
+
+from time import sleep
+
+from constants import METRICS_PORT, MAX_RETRIES, BACKOFF_SECS
+
+LOGGER = logging.getLogger(__name__)
+
+
+def write_config(path, config):
+    config_path = path / "config.yaml"
+    with open(config_path, 'w') as outfile:
+        yaml.dump(config, outfile)
+    return config_path
+
+
+def start_cloudflared(path, config, cfd_args=["run"], cfd_pre_args=["tunnel"], new_process=False, allow_input=False, capture_output=True):
+    config_path = write_config(path, config.full_config)
+    cmd = [config.cloudflared_binary]
+    cmd += cfd_pre_args
+    cmd += ["--config", config_path]
+    cmd += cfd_args
+    LOGGER.info(f"Run cmd {cmd} with config {config}")
+    if new_process:
+        return run_cloudflared_background(cmd, allow_input, capture_output)
+    # By setting check=True, it will raise an exception if the process exits with non-zero exit code
+    return subprocess.run(cmd, check=True, capture_output=capture_output)
+
+
+@contextmanager
+def run_cloudflared_background(cmd, allow_input, capture_output):
+    output = subprocess.PIPE if capture_output else subprocess.DEVNULL
+    stdin = subprocess.PIPE if allow_input else None
+    try:
+        cfd = subprocess.Popen(cmd, stdin=stdin, stdout=output, stderr=output)
+        yield cfd
+    finally:
+        cfd.terminate()
+
+
+@retry(stop_max_attempt_number=MAX_RETRIES, wait_fixed=BACKOFF_SECS * 1000)
+def wait_tunnel_ready(expect_connections=4):
+    url = f'http://localhost:{METRICS_PORT}/ready'
+
+    with requests.Session() as s:
+        resp = send_request(s, url, True)
+        assert resp.json()[
+            "readyConnections"] == expect_connections, f"Ready endpoint returned {resp.json()} but we expect {expect_connections} ready connections"
+
+
+@retry(stop_max_attempt_number=MAX_RETRIES, wait_fixed=BACKOFF_SECS * 1000)
+def check_tunnel_not_ready():
+    url = f'http://localhost:{METRICS_PORT}/ready'
+
+    resp = requests.get(url, timeout=1)
+    assert resp.status_code == 503, f"Expect {url} returns 503, got {resp.status_code}"
+
+# In some cases we don't need to check response status, such as when sending batch requests to generate logs
+
+
+def send_requests(url, count, require_ok=True):
+    errors = 0
+    with requests.Session() as s:
+        for _ in range(count):
+            resp = send_request(s, url, require_ok)
+            if resp is None:
+                errors += 1
+            sleep(0.01)
+    if errors > 0:
+        LOGGER.warning(
+            f"{errors} out of {count} requests to {url} return non-200 status")
+
+
+@retry(stop_max_attempt_number=MAX_RETRIES, wait_fixed=BACKOFF_SECS * 1000)
+def send_request(session, url, require_ok):
+    resp = session.get(url, timeout=BACKOFF_SECS)
+    if require_ok:
+        assert resp.status_code == 200, f"{url} returned {resp}"
+    return resp if resp.status_code == 200 else None

config/configuration.go

@@ -11,11 +11,11 @@ import (
 
 	"github.com/mitchellh/go-homedir"
 	"github.com/pkg/errors"
+	"github.com/rs/zerolog"
 	"github.com/urfave/cli/v2"
 	"gopkg.in/yaml.v2"
 
 	"github.com/cloudflare/cloudflared/validation"
-	"github.com/rs/zerolog"
 )
 
 var (
@@ -221,15 +221,28 @@ type OriginRequestConfig struct {
 	ProxyPort *uint `yaml:"proxyPort"`
 	// Valid options are 'socks' or empty.
 	ProxyType *string `yaml:"proxyType"`
+	// IP rules for the proxy service
+	IPRules []IngressIPRule `yaml:"ipRules"`
+}
+
+type IngressIPRule struct {
+	Prefix *string `yaml:"prefix"`
+	Ports  []int   `yaml:"ports"`
+	Allow  bool    `yaml:"allow"`
 }
 
 type Configuration struct {
 	TunnelID      string `yaml:"tunnel"`
 	Ingress       []UnvalidatedIngressRule
+	WarpRouting   WarpRoutingConfig   `yaml:"warp-routing"`
 	OriginRequest OriginRequestConfig `yaml:"originRequest"`
 	sourceFile    string
 }
 
+type WarpRoutingConfig struct {
+	Enabled bool `yaml:"enabled"`
+}
+
 type configFileSettings struct {
 	Configuration `yaml:",inline"`
 	// older settings will be aggregated into the generic map, should be read via cli.Context

config/configuration_test.go

@@ -20,6 +20,9 @@ func TestConfigFileSettings(t *testing.T) {
 			Path:     "",
 			Service:  "https://localhost:8001",
 		}
+		warpRouting = WarpRoutingConfig{
+			Enabled: true,
+		}
 	)
 	rawYAML := `
 tunnel: config-file-test
@@ -29,6 +32,8 @@ ingress:
    service: https://localhost:8000
  - hostname: "*"
    service: https://localhost:8001
+warp-routing: 
+  enabled: true
 retries: 5
 grace-period: 30s
 percentage: 3.14
@@ -47,6 +52,7 @@ counters:
 	assert.Equal(t, "config-file-test", config.TunnelID)
 	assert.Equal(t, firstIngress, config.Ingress[0])
 	assert.Equal(t, secondIngress, config.Ingress[1])
+	assert.Equal(t, warpRouting, config.WarpRouting)
 
 	retries, err := config.Int("retries")
 	assert.NoError(t, err)
@@ -73,4 +79,5 @@ counters:
 	assert.NoError(t, err)
 	assert.Equal(t, 123, counters[0])
 	assert.Equal(t, 456, counters[1])
+
 }

config/model.go

@@ -5,6 +5,8 @@ import (
 	"fmt"
 	"io"
 	"strings"
+
+	"github.com/cloudflare/cloudflared/tunneldns"
 )
 
 // Forwarder represents a client side listener to forward traffic to the edge
@@ -25,11 +27,12 @@ type Tunnel struct {
 
 // DNSResolver represents a client side DNS resolver
 type DNSResolver struct {
-	Enabled    bool     `json:"enabled"`
-	Address    string   `json:"address,omitempty"`
-	Port       uint16   `json:"port,omitempty"`
-	Upstreams  []string `json:"upstreams,omitempty"`
-	Bootstraps []string `json:"bootstraps,omitempty"`
+	Enabled                bool     `json:"enabled"`
+	Address                string   `json:"address,omitempty"`
+	Port                   uint16   `json:"port,omitempty"`
+	Upstreams              []string `json:"upstreams,omitempty"`
+	Bootstraps             []string `json:"bootstraps,omitempty"`
+	MaxUpstreamConnections int      `json:"max_upstream_connections,omitempty"`
 }
 
 // Root is the base options to configure the service
@@ -59,6 +62,7 @@ func (r *DNSResolver) Hash() string {
 	io.WriteString(h, strings.Join(r.Bootstraps, ","))
 	io.WriteString(h, strings.Join(r.Upstreams, ","))
 	io.WriteString(h, fmt.Sprintf("%d", r.Port))
+	io.WriteString(h, fmt.Sprintf("%d", r.MaxUpstreamConnections))
 	io.WriteString(h, fmt.Sprintf("%v", r.Enabled))
 	return fmt.Sprintf("%x", h.Sum(nil))
 }
@@ -99,3 +103,11 @@ func (r *DNSResolver) BootstrapsOrDefault() []string {
 	}
 	return []string{"https://162.159.36.1/dns-query", "https://162.159.46.1/dns-query", "https://[2606:4700:4700::1111]/dns-query", "https://[2606:4700:4700::1001]/dns-query"}
 }
+
+// MaxUpstreamConnectionsOrDefault return the max upstream connections or returns the default if negative
+func (r *DNSResolver) MaxUpstreamConnectionsOrDefault() int {
+	if r.MaxUpstreamConnections >= 0 {
+		return r.MaxUpstreamConnections
+	}
+	return tunneldns.MaxUpstreamConnsDefault
+}

connection/connection.go

@@ -1,6 +1,7 @@
 package connection
 
 import (
+	"fmt"
 	"io"
 	"net/http"
 	"strconv"
@@ -14,7 +15,7 @@ import (
 const LogFieldConnIndex = "connIndex"
 
 type Config struct {
-	OriginClient    OriginClient
+	OriginProxy     OriginProxy
 	GracePeriod     time.Duration
 	ReplaceExisting bool
 }
@@ -50,14 +51,49 @@ func (c *ClassicTunnelConfig) IsTrialZone() bool {
 	return c.Hostname == ""
 }
 
-type OriginClient interface {
-	Proxy(w ResponseWriter, req *http.Request, isWebsocket bool) error
+// Type indicates the connection type of the  connection.
+type Type int
+
+const (
+	TypeWebsocket Type = iota
+	TypeTCP
+	TypeControlStream
+	TypeHTTP
+)
+
+// ShouldFlush returns whether this kind of connection should actively flush data
+func (t Type) shouldFlush() bool {
+	switch t {
+	case TypeWebsocket, TypeTCP, TypeControlStream:
+		return true
+	default:
+		return false
+	}
+}
+
+func (t Type) String() string {
+	switch t {
+	case TypeWebsocket:
+		return "websocket"
+	case TypeTCP:
+		return "tcp"
+	case TypeControlStream:
+		return "control stream"
+	case TypeHTTP:
+		return "http"
+	default:
+		return fmt.Sprintf("Unknown Type %d", t)
+	}
+}
+
+type OriginProxy interface {
+	// If Proxy returns an error, the caller is responsible for writing the error status to ResponseWriter
+	Proxy(w ResponseWriter, req *http.Request, sourceConnectionType Type) error
 }
 
 type ResponseWriter interface {
-	WriteRespHeaders(*http.Response) error
-	WriteErrorResponse()
-	io.ReadWriter
+	WriteRespHeaders(status int, header http.Header) error
+	io.Writer
 }
 
 type ConnectedFuse interface {

connection/connection_test.go

@@ -19,8 +19,8 @@ const (
 
 var (
 	testConfig = &Config{
-		OriginClient: &mockOriginClient{},
-		GracePeriod:  time.Millisecond * 100,
+		OriginProxy: &mockOriginProxy{},
+		GracePeriod: time.Millisecond * 100,
 	}
 	log           = zerolog.Nop()
 	testOriginURL = &url.URL{
@@ -38,11 +38,11 @@ type testRequest struct {
 	isProxyError   bool
 }
 
-type mockOriginClient struct {
+type mockOriginProxy struct {
 }
 
-func (moc *mockOriginClient) Proxy(w ResponseWriter, r *http.Request, isWebsocket bool) error {
-	if isWebsocket {
+func (moc *mockOriginProxy) Proxy(w ResponseWriter, r *http.Request, sourceConnectionType Type) error {
+	if sourceConnectionType == TypeWebsocket {
 		return wsEndpoint(w, r)
 	}
 	switch r.URL.Path {
@@ -74,7 +74,7 @@ func wsEndpoint(w ResponseWriter, r *http.Request) error {
 	resp := &http.Response{
 		StatusCode: http.StatusSwitchingProtocols,
 	}
-	_ = w.WriteRespHeaders(resp)
+	_ = w.WriteRespHeaders(resp.StatusCode, resp.Header)
 	clientReader := nowriter{r.Body}
 	go func() {
 		for {
@@ -95,7 +95,7 @@ func originRespEndpoint(w ResponseWriter, status int, data []byte) {
 	resp := &http.Response{
 		StatusCode: status,
 	}
-	_ = w.WriteRespHeaders(resp)
+	_ = w.WriteRespHeaders(resp.StatusCode, resp.Header)
 	_, _ = w.Write(data)
 }
 

connection/h2mux.go

@@ -216,7 +216,12 @@ func (h *h2muxConnection) ServeStream(stream *h2mux.MuxedStream) error {
 		return reqErr
 	}
 
-	err := h.config.OriginClient.Proxy(respWriter, req, websocket.IsWebSocketUpgrade(req))
+	var sourceConnectionType = TypeHTTP
+	if websocket.IsWebSocketUpgrade(req) {
+		sourceConnectionType = TypeWebsocket
+	}
+
+	err := h.config.OriginProxy.Proxy(respWriter, req, sourceConnectionType)
 	if err != nil {
 		respWriter.WriteErrorResponse()
 		return err
@@ -240,8 +245,8 @@ type h2muxRespWriter struct {
 	*h2mux.MuxedStream
 }
 
-func (rp *h2muxRespWriter) WriteRespHeaders(resp *http.Response) error {
-	headers := h2mux.H1ResponseToH2ResponseHeaders(resp)
+func (rp *h2muxRespWriter) WriteRespHeaders(status int, header http.Header) error {
+	headers := h2mux.H1ResponseToH2ResponseHeaders(status, header)
 	headers = append(headers, h2mux.Header{Name: ResponseMetaHeaderField, Value: responseMetaHeaderOrigin})
 	return rp.WriteHeaders(headers)
 }

connection/http2.go

@@ -19,6 +19,7 @@ import (
 
 const (
 	internalUpgradeHeader = "Cf-Cloudflared-Proxy-Connection-Upgrade"
+	tcpStreamHeader       = "Cf-Cloudflared-Proxy-Src"
 	websocketUpgrade      = "websocket"
 	controlStreamUpgrade  = "control-stream"
 )
@@ -96,31 +97,25 @@ func (c *http2Connection) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	c.activeRequestsWG.Add(1)
 	defer c.activeRequestsWG.Done()
 
-	respWriter := &http2RespWriter{
-		r: r.Body,
-		w: w,
-	}
-	flusher, isFlusher := w.(http.Flusher)
-	if !isFlusher {
-		c.observer.log.Error().Msgf("%T doesn't implement http.Flusher", w)
-		respWriter.WriteErrorResponse()
+	connType := determineHTTP2Type(r)
+	respWriter, err := newHTTP2RespWriter(r, w, connType)
+	if err != nil {
+		c.observer.log.Error().Msg(err.Error())
 		return
 	}
-	respWriter.flusher = flusher
-	var err error
-	if isControlStreamUpgrade(r) {
-		respWriter.shouldFlush = true
-		err = c.serveControlStream(r.Context(), respWriter)
-		c.controlStreamErr = err
-	} else if isWebsocketUpgrade(r) {
-		respWriter.shouldFlush = true
-		stripWebsocketUpgradeHeader(r)
-		err = c.config.OriginClient.Proxy(respWriter, r, true)
-	} else {
-		err = c.config.OriginClient.Proxy(respWriter, r, false)
-	}
 
-	if err != nil {
+	var proxyErr error
+	switch connType {
+	case TypeControlStream:
+		proxyErr = c.serveControlStream(r.Context(), respWriter)
+		c.controlStreamErr = proxyErr
+	case TypeWebsocket:
+		stripWebsocketUpgradeHeader(r)
+		proxyErr = c.config.OriginProxy.Proxy(respWriter, r, TypeWebsocket)
+	default:
+		proxyErr = c.config.OriginProxy.Proxy(respWriter, r, connType)
+	}
+	if proxyErr != nil {
 		respWriter.WriteErrorResponse()
 	}
 }
@@ -161,10 +156,29 @@ type http2RespWriter struct {
 	shouldFlush bool
 }
 
-func (rp *http2RespWriter) WriteRespHeaders(resp *http.Response) error {
+func newHTTP2RespWriter(r *http.Request, w http.ResponseWriter, connType Type) (*http2RespWriter, error) {
+	flusher, isFlusher := w.(http.Flusher)
+	if !isFlusher {
+		respWriter := &http2RespWriter{
+			r: r.Body,
+			w: w,
+		}
+		respWriter.WriteErrorResponse()
+		return nil, fmt.Errorf("%T doesn't implement http.Flusher", w)
+	}
+
+	return &http2RespWriter{
+		r:           r.Body,
+		w:           w,
+		flusher:     flusher,
+		shouldFlush: connType.shouldFlush(),
+	}, nil
+}
+
+func (rp *http2RespWriter) WriteRespHeaders(status int, header http.Header) error {
 	dest := rp.w.Header()
-	userHeaders := make(http.Header, len(resp.Header))
-	for header, values := range resp.Header {
+	userHeaders := make(http.Header, len(header))
+	for header, values := range header {
 		// Since these are http2 headers, they're required to be lowercase
 		h2name := strings.ToLower(header)
 		for _, v := range values {
@@ -184,13 +198,12 @@ func (rp *http2RespWriter) WriteRespHeaders(resp *http.Response) error {
 	// Perform user header serialization and set them in the single header
 	dest.Set(canonicalResponseUserHeadersField, h2mux.SerializeHeaders(userHeaders))
 	rp.setResponseMetaHeader(responseMetaHeaderOrigin)
-	status := resp.StatusCode
 	// HTTP2 removes support for 101 Switching Protocols https://tools.ietf.org/html/rfc7540#section-8.1.1
 	if status == http.StatusSwitchingProtocols {
 		status = http.StatusOK
 	}
 	rp.w.WriteHeader(status)
-	if IsServerSentEvent(resp.Header) {
+	if IsServerSentEvent(header) {
 		rp.shouldFlush = true
 	}
 	if rp.shouldFlush {
@@ -231,12 +244,30 @@ func (rp *http2RespWriter) Close() error {
 	return nil
 }
 
+func determineHTTP2Type(r *http.Request) Type {
+	switch {
+	case isWebsocketUpgrade(r):
+		return TypeWebsocket
+	case IsTCPStream(r):
+		return TypeTCP
+	case isControlStreamUpgrade(r):
+		return TypeControlStream
+	default:
+		return TypeHTTP
+	}
+}
+
 func isControlStreamUpgrade(r *http.Request) bool {
-	return strings.ToLower(r.Header.Get(internalUpgradeHeader)) == controlStreamUpgrade
+	return r.Header.Get(internalUpgradeHeader) == controlStreamUpgrade
 }
 
 func isWebsocketUpgrade(r *http.Request) bool {
-	return strings.ToLower(r.Header.Get(internalUpgradeHeader)) == websocketUpgrade
+	return r.Header.Get(internalUpgradeHeader) == websocketUpgrade
+}
+
+// IsTCPStream discerns if the connection request needs a tcp stream proxy.
+func IsTCPStream(r *http.Request) bool {
+	return r.Header.Get(tcpStreamHeader) != ""
 }
 
 func stripWebsocketUpgradeHeader(r *http.Request) {

connection/http2_test.go

@@ -114,6 +114,7 @@ func TestServeHTTP(t *testing.T) {
 }
 
 type mockNamedTunnelRPCClient struct {
+	shouldFail   error
 	registered   chan struct{}
 	unregistered chan struct{}
 }
@@ -125,6 +126,9 @@ func (mc mockNamedTunnelRPCClient) RegisterConnection(
 	connIndex uint8,
 	observer *Observer,
 ) error {
+	if mc.shouldFail != nil {
+		return mc.shouldFail
+	}
 	close(mc.registered)
 	return nil
 }
@@ -136,12 +140,14 @@ func (mc mockNamedTunnelRPCClient) GracefulShutdown(ctx context.Context, gracePe
 func (mockNamedTunnelRPCClient) Close() {}
 
 type mockRPCClientFactory struct {
+	shouldFail   error
 	registered   chan struct{}
 	unregistered chan struct{}
 }
 
 func (mf *mockRPCClientFactory) newMockRPCClient(context.Context, io.ReadWriteCloser, *zerolog.Logger) NamedTunnelRPCClient {
 	return mockNamedTunnelRPCClient{
+		shouldFail:   mf.shouldFail,
 		registered:   mf.registered,
 		unregistered: mf.unregistered,
 	}
@@ -249,6 +255,39 @@ func TestServeControlStream(t *testing.T) {
 	wg.Wait()
 }
 
+func TestFailRegistration(t *testing.T) {
+	http2Conn, edgeConn := newTestHTTP2Connection()
+
+	rpcClientFactory := mockRPCClientFactory{
+		shouldFail:   errDuplicationConnection,
+		registered:   make(chan struct{}),
+		unregistered: make(chan struct{}),
+	}
+	http2Conn.newRPCClientFunc = rpcClientFactory.newMockRPCClient
+
+	ctx, cancel := context.WithCancel(context.Background())
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		http2Conn.Serve(ctx)
+	}()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://localhost:8080/", nil)
+	require.NoError(t, err)
+	req.Header.Set(internalUpgradeHeader, controlStreamUpgrade)
+
+	edgeHTTP2Conn, err := testTransport.NewClientConn(edgeConn)
+	require.NoError(t, err)
+	resp, err := edgeHTTP2Conn.RoundTrip(req)
+	require.NoError(t, err)
+	require.Equal(t, http.StatusBadGateway, resp.StatusCode)
+
+	assert.NotNil(t, http2Conn.controlStreamErr)
+	cancel()
+	wg.Wait()
+}
+
 func TestGracefulShutdownHTTP2(t *testing.T) {
 	http2Conn, edgeConn := newTestHTTP2Connection()
 

connection/protocol.go

@@ -141,16 +141,29 @@ type PercentageFetcher func() (int32, error)
 
 func NewProtocolSelector(
 	protocolFlag string,
+	warpRoutingEnabled bool,
 	namedTunnel *NamedTunnelConfig,
 	fetchFunc PercentageFetcher,
 	ttl time.Duration,
 	log *zerolog.Logger,
 ) (ProtocolSelector, error) {
+	// Classic tunnel is only supported with h2mux
 	if namedTunnel == nil {
 		return &staticProtocolSelector{
 			current: H2mux,
 		}, nil
 	}
+
+	// warp routing can only be served over http2 connections
+	if warpRoutingEnabled {
+		if protocolFlag == H2mux.String() {
+			log.Warn().Msg("Warp routing is only supported by http2 protocol. Upgrading protocol to http2")
+		}
+		return &staticProtocolSelector{
+			current: HTTP2,
+		}, nil
+	}
+
 	if protocolFlag == H2mux.String() {
 		return &staticProtocolSelector{
 			current: H2mux,

connection/protocol_test.go

@@ -9,7 +9,8 @@ import (
 )
 
 const (
-	testNoTTL = 0
+	testNoTTL            = 0
+	noWarpRoutingEnabled = false
 )
 
 var (
@@ -48,14 +49,15 @@ func (dmf *dynamicMockFetcher) fetch() PercentageFetcher {
 
 func TestNewProtocolSelector(t *testing.T) {
 	tests := []struct {
-		name              string
-		protocol          string
-		expectedProtocol  Protocol
-		hasFallback       bool
-		expectedFallback  Protocol
-		namedTunnelConfig *NamedTunnelConfig
-		fetchFunc         PercentageFetcher
-		wantErr           bool
+		name               string
+		protocol           string
+		expectedProtocol   Protocol
+		hasFallback        bool
+		expectedFallback   Protocol
+		warpRoutingEnabled bool
+		namedTunnelConfig  *NamedTunnelConfig
+		fetchFunc          PercentageFetcher
+		wantErr            bool
 	}{
 		{
 			name:              "classic tunnel",
@@ -108,6 +110,36 @@ func TestNewProtocolSelector(t *testing.T) {
 			fetchFunc:         mockFetcher(100),
 			namedTunnelConfig: testNamedTunnelConfig,
 		},
+		{
+			name:               "warp routing requesting h2mux",
+			protocol:           "h2mux",
+			expectedProtocol:   HTTP2,
+			hasFallback:        false,
+			expectedFallback:   H2mux,
+			fetchFunc:          mockFetcher(100),
+			warpRoutingEnabled: true,
+			namedTunnelConfig:  testNamedTunnelConfig,
+		},
+		{
+			name:               "warp routing http2",
+			protocol:           "http2",
+			expectedProtocol:   HTTP2,
+			hasFallback:        false,
+			expectedFallback:   H2mux,
+			fetchFunc:          mockFetcher(100),
+			warpRoutingEnabled: true,
+			namedTunnelConfig:  testNamedTunnelConfig,
+		},
+		{
+			name:               "warp routing auto",
+			protocol:           "auto",
+			expectedProtocol:   HTTP2,
+			hasFallback:        false,
+			expectedFallback:   H2mux,
+			fetchFunc:          mockFetcher(100),
+			warpRoutingEnabled: true,
+			namedTunnelConfig:  testNamedTunnelConfig,
+		},
 		{
 			// None named tunnel can only use h2mux, so specifying an unknown protocol is not an error
 			name:             "classic tunnel unknown protocol",
@@ -131,7 +163,7 @@ func TestNewProtocolSelector(t *testing.T) {
 	}
 
 	for _, test := range tests {
-		selector, err := NewProtocolSelector(test.protocol, test.namedTunnelConfig, test.fetchFunc, testNoTTL, &log)
+		selector, err := NewProtocolSelector(test.protocol, test.warpRoutingEnabled, test.namedTunnelConfig, test.fetchFunc, testNoTTL, &log)
 		if test.wantErr {
 			assert.Error(t, err, fmt.Sprintf("test %s failed", test.name))
 		} else {
@@ -148,7 +180,7 @@ func TestNewProtocolSelector(t *testing.T) {
 
 func TestAutoProtocolSelectorRefresh(t *testing.T) {
 	fetcher := dynamicMockFetcher{}
-	selector, err := NewProtocolSelector("auto", testNamedTunnelConfig, fetcher.fetch(), testNoTTL, &log)
+	selector, err := NewProtocolSelector("auto", noWarpRoutingEnabled, testNamedTunnelConfig, fetcher.fetch(), testNoTTL, &log)
 	assert.NoError(t, err)
 	assert.Equal(t, H2mux, selector.Current())
 
@@ -177,7 +209,7 @@ func TestAutoProtocolSelectorRefresh(t *testing.T) {
 
 func TestHTTP2ProtocolSelectorRefresh(t *testing.T) {
 	fetcher := dynamicMockFetcher{}
-	selector, err := NewProtocolSelector("http2", testNamedTunnelConfig, fetcher.fetch(), testNoTTL, &log)
+	selector, err := NewProtocolSelector("http2", noWarpRoutingEnabled, testNamedTunnelConfig, fetcher.fetch(), testNoTTL, &log)
 	assert.NoError(t, err)
 	assert.Equal(t, HTTP2, selector.Current())
 
@@ -206,7 +238,7 @@ func TestHTTP2ProtocolSelectorRefresh(t *testing.T) {
 
 func TestProtocolSelectorRefreshTTL(t *testing.T) {
 	fetcher := dynamicMockFetcher{percentage: 100}
-	selector, err := NewProtocolSelector("auto", testNamedTunnelConfig, fetcher.fetch(), time.Hour, &log)
+	selector, err := NewProtocolSelector("auto", noWarpRoutingEnabled, testNamedTunnelConfig, fetcher.fetch(), time.Hour, &log)
 	assert.NoError(t, err)
 	assert.Equal(t, HTTP2, selector.Current())
 

dbconnect/client.go

@@ -1,145 +0,0 @@
-package dbconnect
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"net/url"
-	"strings"
-	"time"
-	"unicode"
-	"unicode/utf8"
-)
-
-// Client is an interface to talk to any database.
-//
-// Currently, the only implementation is SQLClient, but its structure
-// should be designed to handle a MongoClient or RedisClient in the future.
-type Client interface {
-	Ping(context.Context) error
-	Submit(context.Context, *Command) (interface{}, error)
-}
-
-// NewClient creates a database client based on its URL scheme.
-func NewClient(ctx context.Context, originURL *url.URL) (Client, error) {
-	return NewSQLClient(ctx, originURL)
-}
-
-// Command is a standard, non-vendor format for submitting database commands.
-//
-// When determining the scope of this struct, refer to the following litmus test:
-// Could this (roughly) conform to SQL, Document-based, and Key-value command formats?
-type Command struct {
-	Statement string        `json:"statement"`
-	Arguments Arguments     `json:"arguments,omitempty"`
-	Mode      string        `json:"mode,omitempty"`
-	Isolation string        `json:"isolation,omitempty"`
-	Timeout   time.Duration `json:"timeout,omitempty"`
-}
-
-// Validate enforces the contract of Command: non empty statement (both in length and logic),
-// lowercase mode and isolation, non-zero timeout, and valid Arguments.
-func (cmd *Command) Validate() error {
-	if cmd.Statement == "" {
-		return fmt.Errorf("cannot provide an empty statement")
-	}
-
-	if strings.Map(func(char rune) rune {
-		if char == ';' || unicode.IsSpace(char) {
-			return -1
-		}
-		return char
-	}, cmd.Statement) == "" {
-		return fmt.Errorf("cannot provide a statement with no logic: '%s'", cmd.Statement)
-	}
-
-	cmd.Mode = strings.ToLower(cmd.Mode)
-	cmd.Isolation = strings.ToLower(cmd.Isolation)
-
-	if cmd.Timeout.Nanoseconds() <= 0 {
-		cmd.Timeout = 24 * time.Hour
-	}
-
-	return cmd.Arguments.Validate()
-}
-
-// UnmarshalJSON converts a byte representation of JSON into a Command, which is also validated.
-func (cmd *Command) UnmarshalJSON(data []byte) error {
-	// Alias is required to avoid infinite recursion from the default UnmarshalJSON.
-	type Alias Command
-	alias := &struct {
-		*Alias
-	}{
-		Alias: (*Alias)(cmd),
-	}
-
-	err := json.Unmarshal(data, &alias)
-	if err == nil {
-		err = cmd.Validate()
-	}
-
-	return err
-}
-
-// Arguments is a wrapper for either map-based or array-based Command arguments.
-//
-// Each field is mutually-exclusive and some Client implementations may not
-// support both fields (eg. MySQL does not accept named arguments).
-type Arguments struct {
-	Named      map[string]interface{}
-	Positional []interface{}
-}
-
-// Validate enforces the contract of Arguments: non nil, mutually exclusive, and no empty or reserved keys.
-func (args *Arguments) Validate() error {
-	if args.Named == nil {
-		args.Named = map[string]interface{}{}
-	}
-	if args.Positional == nil {
-		args.Positional = []interface{}{}
-	}
-
-	if len(args.Named) > 0 && len(args.Positional) > 0 {
-		return fmt.Errorf("both named and positional arguments cannot be specified: %+v and %+v", args.Named, args.Positional)
-	}
-
-	for key := range args.Named {
-		if key == "" {
-			return fmt.Errorf("named arguments cannot contain an empty key: %+v", args.Named)
-		}
-		if !utf8.ValidString(key) {
-			return fmt.Errorf("named argument does not conform to UTF-8 encoding: %s", key)
-		}
-		if strings.HasPrefix(key, "_") {
-			return fmt.Errorf("named argument cannot start with a reserved keyword '_': %s", key)
-		}
-		if unicode.IsNumber([]rune(key)[0]) {
-			return fmt.Errorf("named argument cannot start with a number: %s", key)
-		}
-	}
-
-	return nil
-}
-
-// UnmarshalJSON converts a byte representation of JSON into Arguments, which is also validated.
-func (args *Arguments) UnmarshalJSON(data []byte) error {
-	var obj interface{}
-	err := json.Unmarshal(data, &obj)
-	if err != nil {
-		return err
-	}
-
-	named, ok := obj.(map[string]interface{})
-	if ok {
-		args.Named = named
-	} else {
-		positional, ok := obj.([]interface{})
-		if ok {
-			args.Positional = positional
-		} else {
-			return fmt.Errorf("arguments must either be an object {\"0\":\"val\"} or an array [\"val\"]: %s", string(data))
-		}
-	}
-
-	return args.Validate()
-}

dbconnect/client_test.go

@@ -1,183 +0,0 @@
-package dbconnect
-
-import (
-	"encoding/json"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestCommandValidateEmpty(t *testing.T) {
-	stmts := []string{
-		"",
-		";",
-		" \n\t",
-		";\n;\t;",
-	}
-
-	for _, stmt := range stmts {
-		cmd := Command{Statement: stmt}
-
-		assert.Error(t, cmd.Validate(), stmt)
-	}
-}
-
-func TestCommandValidateMode(t *testing.T) {
-	modes := []string{
-		"",
-		"query",
-		"ExEc",
-		"PREPARE",
-	}
-
-	for _, mode := range modes {
-		cmd := Command{Statement: "Ok", Mode: mode}
-
-		assert.NoError(t, cmd.Validate(), mode)
-		assert.Equal(t, strings.ToLower(mode), cmd.Mode)
-	}
-}
-
-func TestCommandValidateIsolation(t *testing.T) {
-	isos := []string{
-		"",
-		"default",
-		"read_committed",
-		"SNAPshot",
-	}
-
-	for _, iso := range isos {
-		cmd := Command{Statement: "Ok", Isolation: iso}
-
-		assert.NoError(t, cmd.Validate(), iso)
-		assert.Equal(t, strings.ToLower(iso), cmd.Isolation)
-	}
-}
-
-func TestCommandValidateTimeout(t *testing.T) {
-	cmd := Command{Statement: "Ok", Timeout: 0}
-
-	assert.NoError(t, cmd.Validate())
-	assert.NotZero(t, cmd.Timeout)
-
-	cmd = Command{Statement: "Ok", Timeout: 1 * time.Second}
-
-	assert.NoError(t, cmd.Validate())
-	assert.Equal(t, 1*time.Second, cmd.Timeout)
-}
-
-func TestCommandValidateArguments(t *testing.T) {
-	cmd := Command{Statement: "Ok", Arguments: Arguments{
-		Named:      map[string]interface{}{"key": "val"},
-		Positional: []interface{}{"val"},
-	}}
-
-	assert.Error(t, cmd.Validate())
-}
-
-func TestCommandUnmarshalJSON(t *testing.T) {
-	strs := []string{
-		"{\"statement\":\"Ok\"}",
-		"{\"statement\":\"Ok\",\"arguments\":[0, 3.14, \"apple\"],\"mode\":\"query\"}",
-		"{\"statement\":\"Ok\",\"isolation\":\"read_uncommitted\",\"timeout\":1000}",
-	}
-
-	for _, str := range strs {
-		var cmd Command
-		assert.NoError(t, json.Unmarshal([]byte(str), &cmd), str)
-	}
-
-	strs = []string{
-		"",
-		"\"",
-		"{}",
-		"{\"argument\":{\"key\":\"val\"}}",
-		"{\"statement\":[\"Ok\"]}",
-	}
-
-	for _, str := range strs {
-		var cmd Command
-		assert.Error(t, json.Unmarshal([]byte(str), &cmd), str)
-	}
-}
-
-func TestArgumentsValidateNotNil(t *testing.T) {
-	args := Arguments{}
-
-	assert.NoError(t, args.Validate())
-	assert.NotNil(t, args.Named)
-	assert.NotNil(t, args.Positional)
-}
-
-func TestArgumentsValidateMutuallyExclusive(t *testing.T) {
-	args := []Arguments{
-		Arguments{},
-		Arguments{Named: map[string]interface{}{"key": "val"}},
-		Arguments{Positional: []interface{}{"val"}},
-	}
-
-	for _, arg := range args {
-		assert.NoError(t, arg.Validate())
-		assert.False(t, len(arg.Named) > 0 && len(arg.Positional) > 0)
-	}
-
-	args = []Arguments{
-		Arguments{
-			Named:      map[string]interface{}{"key": "val"},
-			Positional: []interface{}{"val"},
-		},
-	}
-
-	for _, arg := range args {
-		assert.Error(t, arg.Validate())
-		assert.True(t, len(arg.Named) > 0 && len(arg.Positional) > 0)
-	}
-}
-
-func TestArgumentsValidateKeys(t *testing.T) {
-	keys := []string{
-		"",
-		"_",
-		"_key",
-		"1",
-		"1key",
-		"\xf0\x28\x8c\xbc", // non-utf8
-	}
-
-	for _, key := range keys {
-		args := Arguments{Named: map[string]interface{}{key: "val"}}
-
-		assert.Error(t, args.Validate(), key)
-	}
-}
-
-func TestArgumentsUnmarshalJSON(t *testing.T) {
-	strs := []string{
-		"{}",
-		"{\"key\":\"val\"}",
-		"{\"key\":[1, 3.14, {\"key\":\"val\"}]}",
-		"[]",
-		"[\"key\",\"val\"]",
-		"[{}]",
-	}
-
-	for _, str := range strs {
-		var args Arguments
-		assert.NoError(t, json.Unmarshal([]byte(str), &args), str)
-	}
-
-	strs = []string{
-		"",
-		"\"",
-		"1",
-		"\"key\"",
-		"{\"key\",\"val\"}",
-	}
-
-	for _, str := range strs {
-		var args Arguments
-		assert.Error(t, json.Unmarshal([]byte(str), &args), str)
-	}
-}

dbconnect/cmd.go

@@ -1,165 +0,0 @@
-package dbconnect
-
-import (
-	"context"
-	"log"
-	"net"
-	"strconv"
-
-	"github.com/urfave/cli/v2"
-	"github.com/urfave/cli/v2/altsrc"
-)
-
-// Cmd is the entrypoint command for dbconnect.
-//
-// The tunnel package is responsible for appending this to tunnel.Commands().
-func Cmd() *cli.Command {
-	return &cli.Command{
-		Category:  "Database Connect (ALPHA) - Deprecated",
-		Name:      "db-connect",
-		Usage:     "deprecated: Access your SQL database from Cloudflare Workers or the browser",
-		ArgsUsage: " ",
-		Description: `
-		This feature has been deprecated. 
-		Please see: 
-
-		cloudflared access tcp --help 
-
-		for setting up database connections to the cloudflare edge.
-		
-
-		Creates a connection between your database and the Cloudflare edge.
-		Now you can execute SQL commands anywhere you can send HTTPS requests.
-
-		Connect your database with any of the following commands, you can also try the "playground" without a database:
-
-			cloudflared db-connect --hostname sql.mysite.com --url postgres://user:pass@localhost?sslmode=disable \
-			                       --auth-domain mysite.cloudflareaccess.com --application-aud my-access-policy-tag
-			cloudflared db-connect --hostname sql-dev.mysite.com --url mysql://localhost --insecure
-			cloudflared db-connect --playground
-		
-		Requests should be authenticated using Cloudflare Access, learn more about how to enable it here:
-
-			https://developers.cloudflare.com/access/service-auth/service-token/
-		`,
-		Flags: []cli.Flag{
-			altsrc.NewStringFlag(&cli.StringFlag{
-				Name:    "url",
-				Usage:   "URL to the database (eg. postgres://user:pass@localhost?sslmode=disable)",
-				EnvVars: []string{"TUNNEL_URL"},
-			}),
-			altsrc.NewStringFlag(&cli.StringFlag{
-				Name:    "hostname",
-				Usage:   "Hostname to accept commands over HTTPS (eg. sql.mysite.com)",
-				EnvVars: []string{"TUNNEL_HOSTNAME"},
-			}),
-			altsrc.NewStringFlag(&cli.StringFlag{
-				Name:    "auth-domain",
-				Usage:   "Cloudflare Access authentication domain for your account (eg. mysite.cloudflareaccess.com)",
-				EnvVars: []string{"TUNNEL_ACCESS_AUTH_DOMAIN"},
-			}),
-			altsrc.NewStringFlag(&cli.StringFlag{
-				Name:    "application-aud",
-				Usage:   "Cloudflare Access application \"AUD\" to verify JWTs from requests",
-				EnvVars: []string{"TUNNEL_ACCESS_APPLICATION_AUD"},
-			}),
-			altsrc.NewBoolFlag(&cli.BoolFlag{
-				Name:    "insecure",
-				Usage:   "Disable authentication, the database will be open to the Internet",
-				Value:   false,
-				EnvVars: []string{"TUNNEL_ACCESS_INSECURE"},
-			}),
-			altsrc.NewBoolFlag(&cli.BoolFlag{
-				Name:    "playground",
-				Usage:   "Run a temporary, in-memory SQLite3 database for testing",
-				Value:   false,
-				EnvVars: []string{"TUNNEL_HELLO_WORLD"},
-			}),
-			altsrc.NewStringFlag(&cli.StringFlag{
-				Name:    "loglevel",
-				Value:   "debug", // Make it more verbose than the tunnel default 'info'.
-				EnvVars: []string{"TUNNEL_LOGLEVEL"},
-				Hidden:  true,
-			}),
-		},
-		Before: CmdBefore,
-		Action: CmdAction,
-		Hidden: true,
-	}
-}
-
-// CmdBefore runs some validation checks before running the command.
-func CmdBefore(c *cli.Context) error {
-	// Show the help text is no flags are specified.
-	if c.NumFlags() == 0 {
-		return cli.ShowSubcommandHelp(c)
-	}
-
-	// Hello-world and playground are synonymous with each other,
-	// unset hello-world to prevent tunnel from initializing the hello package.
-	if c.IsSet("hello-world") {
-		c.Set("playground", "true")
-		c.Set("hello-world", "false")
-	}
-
-	// Unix-socket database urls are supported, but the logic is the same as url.
-	if c.IsSet("unix-socket") {
-		c.Set("url", c.String("unix-socket"))
-		c.Set("unix-socket", "")
-	}
-
-	// When playground mode is enabled, run with an in-memory database.
-	if c.IsSet("playground") {
-		c.Set("url", "sqlite3::memory:?cache=shared")
-		c.Set("insecure", strconv.FormatBool(!c.IsSet("auth-domain") && !c.IsSet("application-aud")))
-	}
-
-	// At this point, insecure configurations are valid.
-	if c.Bool("insecure") {
-		return nil
-	}
-
-	// Ensure that secure configurations specify a hostname, domain, and tag for JWT validation.
-	if !c.IsSet("hostname") || !c.IsSet("auth-domain") || !c.IsSet("application-aud") {
-		log.Fatal("must specify --hostname, --auth-domain, and --application-aud unless you want to run in --insecure mode")
-	}
-
-	return nil
-}
-
-// CmdAction starts the Proxy and sets the url in cli.Context to point to the Proxy address.
-func CmdAction(c *cli.Context) error {
-	// STOR-612: sync with context in tunnel daemon.
-	ctx := context.Background()
-
-	var proxy *Proxy
-	var err error
-	if c.Bool("insecure") {
-		proxy, err = NewInsecureProxy(ctx, c.String("url"))
-	} else {
-		proxy, err = NewSecureProxy(ctx, c.String("url"), c.String("auth-domain"), c.String("application-aud"))
-	}
-
-	if err != nil {
-		log.Fatal(err)
-		return err
-	}
-
-	listenerC := make(chan net.Listener)
-	defer close(listenerC)
-
-	// Since the Proxy should only talk to the tunnel daemon, find the next available
-	// localhost port and start to listen to requests.
-	go func() {
-		err := proxy.Start(ctx, "127.0.0.1:", listenerC)
-		if err != nil {
-			log.Fatal(err)
-		}
-	}()
-
-	// Block until the the Proxy is online, retreive its address, and change the url to point to it.
-	// This is effectively "handing over" control to the tunnel package so it can run the tunnel daemon.
-	c.Set("url", "https://"+(<-listenerC).Addr().String())
-
-	return nil
-}

dbconnect/cmd_test.go

@@ -1,27 +0,0 @@
-package dbconnect
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-
-	"github.com/urfave/cli/v2"
-)
-
-func TestCmd(t *testing.T) {
-	tests := [][]string{
-		{"cloudflared", "db-connect", "--playground"},
-		{"cloudflared", "db-connect", "--playground", "--hostname", "sql.mysite.com"},
-		{"cloudflared", "db-connect", "--url", "sqlite3::memory:?cache=shared", "--insecure"},
-		{"cloudflared", "db-connect", "--url", "sqlite3::memory:?cache=shared", "--hostname", "sql.mysite.com", "--auth-domain", "mysite.cloudflareaccess.com", "--application-aud", "aud"},
-	}
-
-	app := &cli.App{
-		Name:     "cloudflared",
-		Commands: []*cli.Command{Cmd()},
-	}
-
-	for _, test := range tests {
-		assert.NoError(t, app.Run(test))
-	}
-}

dbconnect/proxy.go

@@ -1,279 +0,0 @@
-package dbconnect
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"io"
-	"net"
-	"net/http"
-	"net/url"
-	"os"
-	"time"
-
-	"github.com/cloudflare/cloudflared/hello"
-	"github.com/cloudflare/cloudflared/validation"
-	"github.com/gorilla/mux"
-	"github.com/pkg/errors"
-	"github.com/rs/zerolog"
-)
-
-const (
-	LogFieldCFRay = "cfRay"
-)
-
-// Proxy is an HTTP server that proxies requests to a Client.
-type Proxy struct {
-	client          Client
-	accessValidator *validation.Access
-	log             *zerolog.Logger
-}
-
-// NewInsecureProxy creates a Proxy that talks to a Client at an origin.
-//
-// In insecure mode, the Proxy will allow all Command requests.
-func NewInsecureProxy(ctx context.Context, origin string) (*Proxy, error) {
-	originURL, err := url.Parse(origin)
-	if err != nil {
-		return nil, errors.Wrap(err, "must provide a valid database url")
-	}
-
-	client, err := NewClient(ctx, originURL)
-	if err != nil {
-		return nil, err
-	}
-
-	err = client.Ping(ctx)
-	if err != nil {
-		return nil, errors.Wrap(err, "could not connect to the database")
-	}
-
-	log := zerolog.New(os.Stderr).With().Logger() // TODO: Does not obey log configuration
-
-	return &Proxy{client, nil, &log}, nil
-}
-
-// NewSecureProxy creates a Proxy that talks to a Client at an origin.
-//
-// In secure mode, the Proxy will reject any Command requests that are
-// not authenticated by Cloudflare Access with a valid JWT.
-func NewSecureProxy(ctx context.Context, origin, authDomain, applicationAUD string) (*Proxy, error) {
-	proxy, err := NewInsecureProxy(ctx, origin)
-	if err != nil {
-		return nil, err
-	}
-
-	validator, err := validation.NewAccessValidator(ctx, authDomain, authDomain, applicationAUD)
-	if err != nil {
-		return nil, err
-	}
-
-	proxy.accessValidator = validator
-
-	return proxy, err
-}
-
-// IsInsecure gets whether the Proxy will accept a Command from any source.
-func (proxy *Proxy) IsInsecure() bool {
-	return proxy.accessValidator == nil
-}
-
-// IsAllowed checks whether a http.Request is allowed to receive data.
-//
-// By default, requests must pass through Cloudflare Access for authentication.
-// If the proxy is explcitly set to insecure mode, all requests will be allowed.
-func (proxy *Proxy) IsAllowed(r *http.Request, verbose ...bool) bool {
-	if proxy.IsInsecure() {
-		return true
-	}
-
-	// Access and Tunnel should prevent bad JWTs from even reaching the origin,
-	// but validate tokens anyway as an abundance of caution.
-	err := proxy.accessValidator.ValidateRequest(r.Context(), r)
-	if err == nil {
-		return true
-	}
-
-	// Warn administrators that invalid JWTs are being rejected. This is indicative
-	// of either a misconfiguration of the CLI or a massive failure of upstream systems.
-	if len(verbose) > 0 {
-		proxy.log.Err(err).Str(LogFieldCFRay, proxy.getRayHeader(r)).Msg("dbproxy: Failed JWT authentication")
-	}
-
-	return false
-}
-
-// Start the Proxy at a given address and notify the listener channel when the server is online.
-func (proxy *Proxy) Start(ctx context.Context, addr string, listenerC chan<- net.Listener) error {
-	// STOR-611: use a seperate listener and consider web socket support.
-	httpListener, err := hello.CreateTLSListener(addr)
-	if err != nil {
-		return errors.Wrapf(err, "could not create listener at %s", addr)
-	}
-
-	errC := make(chan error)
-	defer close(errC)
-
-	// Starts the HTTP server and begins to serve requests.
-	go func() {
-		errC <- proxy.httpListen(ctx, httpListener)
-	}()
-
-	// Continually ping the server until it comes online or 10 attempts fail.
-	go func() {
-		var err error
-		for i := 0; i < 10; i++ {
-			_, err = http.Get("http://" + httpListener.Addr().String())
-
-			// Once no error was detected, notify the listener channel and return.
-			if err == nil {
-				listenerC <- httpListener
-				return
-			}
-
-			// Backoff between requests to ping the server.
-			<-time.After(1 * time.Second)
-		}
-		errC <- errors.Wrap(err, "took too long for the http server to start")
-	}()
-
-	return <-errC
-}
-
-// httpListen starts the httpServer and blocks until the context closes.
-func (proxy *Proxy) httpListen(ctx context.Context, listener net.Listener) error {
-	httpServer := &http.Server{
-		Addr:         listener.Addr().String(),
-		Handler:      proxy.httpRouter(),
-		ReadTimeout:  10 * time.Second,
-		WriteTimeout: 60 * time.Second,
-		IdleTimeout:  60 * time.Second,
-	}
-
-	go func() {
-		<-ctx.Done()
-		_ = httpServer.Close()
-		_ = listener.Close()
-	}()
-
-	return httpServer.Serve(listener)
-}
-
-// httpRouter creates a mux.Router for the Proxy.
-func (proxy *Proxy) httpRouter() *mux.Router {
-	router := mux.NewRouter()
-
-	router.HandleFunc("/ping", proxy.httpPing()).Methods("GET", "HEAD")
-	router.HandleFunc("/submit", proxy.httpSubmit()).Methods("POST")
-
-	return router
-}
-
-// httpPing tests the connection to the database.
-//
-// By default, this endpoint is unauthenticated to allow for health checks.
-// To enable authentication, Cloudflare Access must be enabled on this route.
-func (proxy *Proxy) httpPing() http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		ctx := r.Context()
-		err := proxy.client.Ping(ctx)
-
-		if err == nil {
-			proxy.httpRespond(w, r, http.StatusOK, "")
-		} else {
-			proxy.httpRespondErr(w, r, http.StatusInternalServerError, err)
-		}
-	}
-}
-
-// httpSubmit sends a command to the database and returns its response.
-//
-// By default, this endpoint will reject requests that do not pass through Cloudflare Access.
-// To disable authentication, the --insecure flag must be specified in the command line.
-func (proxy *Proxy) httpSubmit() http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		if !proxy.IsAllowed(r, true) {
-			proxy.httpRespondErr(w, r, http.StatusForbidden, fmt.Errorf(""))
-			return
-		}
-
-		var cmd Command
-		err := json.NewDecoder(r.Body).Decode(&cmd)
-		if err != nil {
-			proxy.httpRespondErr(w, r, http.StatusBadRequest, err)
-			return
-		}
-
-		ctx := r.Context()
-		data, err := proxy.client.Submit(ctx, &cmd)
-
-		if err != nil {
-			proxy.httpRespondErr(w, r, http.StatusUnprocessableEntity, err)
-			return
-		}
-
-		w.Header().Set("Content-type", "application/json")
-		err = json.NewEncoder(w).Encode(data)
-		if err != nil {
-			proxy.httpRespondErr(w, r, http.StatusInternalServerError, err)
-		}
-	}
-}
-
-// httpRespond writes a status code and string response to the response writer.
-func (proxy *Proxy) httpRespond(w http.ResponseWriter, r *http.Request, status int, message string) {
-	w.WriteHeader(status)
-
-	// Only expose the message detail of the reponse if the request is not HEAD
-	// and the user is authenticated. For example, this prevents an unauthenticated
-	// failed health check from accidentally leaking sensitive information about the Client.
-	if r.Method != http.MethodHead && proxy.IsAllowed(r) {
-		if message == "" {
-			message = http.StatusText(status)
-		}
-		fmt.Fprint(w, message)
-	}
-}
-
-// httpRespondErr is similar to httpRespond, except it formats errors to be more friendly.
-func (proxy *Proxy) httpRespondErr(w http.ResponseWriter, r *http.Request, defaultStatus int, err error) {
-	status, err := httpError(defaultStatus, err)
-
-	proxy.httpRespond(w, r, status, err.Error())
-	if len(err.Error()) > 0 {
-		cfRay := proxy.getRayHeader(r)
-		proxy.log.Err(err).Str(LogFieldCFRay, cfRay).Msg("dbproxy: Database proxy error")
-	}
-}
-
-// getRayHeader returns the request's Cf-ray header.
-func (proxy *Proxy) getRayHeader(r *http.Request) string {
-	return r.Header.Get("Cf-ray")
-}
-
-// httpError extracts common errors and returns an status code and friendly error.
-func httpError(defaultStatus int, err error) (int, error) {
-	if err == nil {
-		return http.StatusNotImplemented, fmt.Errorf("error expected but found none")
-	}
-
-	if err == io.EOF {
-		return http.StatusBadRequest, fmt.Errorf("request body cannot be empty")
-	}
-
-	if err == context.DeadlineExceeded {
-		return http.StatusRequestTimeout, err
-	}
-
-	_, ok := err.(net.Error)
-	if ok {
-		return http.StatusRequestTimeout, err
-	}
-
-	if err == context.Canceled {
-		// Does not exist in Golang, but would be: http.StatusClientClosedWithoutResponse
-		return 444, err
-	}
-
-	return defaultStatus, err
-}

dbconnect/proxy_test.go

@@ -1,238 +0,0 @@
-package dbconnect
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"net"
-	"net/http"
-	"net/http/httptest"
-	"strings"
-	"testing"
-
-	"github.com/gorilla/mux"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestNewInsecureProxy(t *testing.T) {
-	origins := []string{
-		"",
-		":/",
-		"http://localhost",
-		"tcp://localhost:9000?debug=true",
-		"mongodb://127.0.0.1",
-	}
-
-	for _, origin := range origins {
-		proxy, err := NewInsecureProxy(context.Background(), origin)
-
-		assert.Error(t, err)
-		assert.Empty(t, proxy)
-	}
-}
-
-func TestProxyIsAllowed(t *testing.T) {
-	proxy := helperNewProxy(t)
-	req := httptest.NewRequest("GET", "https://1.1.1.1/ping", nil)
-	assert.True(t, proxy.IsAllowed(req))
-
-	proxy = helperNewProxy(t, true)
-	req.Header.Set("Cf-access-jwt-assertion", "xxx")
-	assert.False(t, proxy.IsAllowed(req))
-}
-
-func TestProxyStart(t *testing.T) {
-	proxy := helperNewProxy(t)
-	ctx := context.Background()
-	listenerC := make(chan net.Listener)
-
-	err := proxy.Start(ctx, "1.1.1.1:", listenerC)
-	assert.Error(t, err)
-
-	err = proxy.Start(ctx, "127.0.0.1:-1", listenerC)
-	assert.Error(t, err)
-
-	ctx, cancel := context.WithTimeout(ctx, 0)
-	defer cancel()
-
-	err = proxy.Start(ctx, "127.0.0.1:", listenerC)
-	assert.IsType(t, http.ErrServerClosed, err)
-}
-
-func TestProxyHTTPRouter(t *testing.T) {
-	proxy := helperNewProxy(t)
-	router := proxy.httpRouter()
-
-	tests := []struct {
-		path   string
-		method string
-		valid  bool
-	}{
-		{"", "GET", false},
-		{"/", "GET", false},
-		{"/ping", "GET", true},
-		{"/ping", "HEAD", true},
-		{"/ping", "POST", false},
-		{"/submit", "POST", true},
-		{"/submit", "GET", false},
-		{"/submit/extra", "POST", false},
-	}
-
-	for _, test := range tests {
-		match := &mux.RouteMatch{}
-		ok := router.Match(httptest.NewRequest(test.method, "https://1.1.1.1"+test.path, nil), match)
-
-		assert.True(t, ok == test.valid, test.path)
-	}
-}
-
-func TestProxyHTTPPing(t *testing.T) {
-	proxy := helperNewProxy(t)
-
-	server := httptest.NewServer(proxy.httpPing())
-	defer server.Close()
-	client := server.Client()
-
-	res, err := client.Get(server.URL)
-	assert.NoError(t, err)
-	assert.Equal(t, http.StatusOK, res.StatusCode)
-	assert.Equal(t, int64(2), res.ContentLength)
-
-	res, err = client.Head(server.URL)
-	assert.NoError(t, err)
-	assert.Equal(t, http.StatusOK, res.StatusCode)
-	assert.Equal(t, int64(-1), res.ContentLength)
-}
-
-func TestProxyHTTPSubmit(t *testing.T) {
-	proxy := helperNewProxy(t)
-
-	server := httptest.NewServer(proxy.httpSubmit())
-	defer server.Close()
-	client := server.Client()
-
-	tests := []struct {
-		input  string
-		status int
-		output string
-	}{
-		{"", http.StatusBadRequest, "request body cannot be empty"},
-		{"{}", http.StatusBadRequest, "cannot provide an empty statement"},
-		{"{\"statement\":\"Ok\"}", http.StatusUnprocessableEntity, "cannot provide invalid sql mode: ''"},
-		{"{\"statement\":\"Ok\",\"mode\":\"query\"}", http.StatusUnprocessableEntity, "near \"Ok\": syntax error"},
-		{"{\"statement\":\"CREATE TABLE t (a INT);\",\"mode\":\"exec\"}", http.StatusOK, "{\"last_insert_id\":0,\"rows_affected\":0}\n"},
-	}
-
-	for _, test := range tests {
-		res, err := client.Post(server.URL, "application/json", strings.NewReader(test.input))
-
-		assert.NoError(t, err)
-		assert.Equal(t, test.status, res.StatusCode)
-		if res.StatusCode > http.StatusOK {
-			assert.Equal(t, "text/plain; charset=utf-8", res.Header.Get("Content-type"))
-		} else {
-			assert.Equal(t, "application/json", res.Header.Get("Content-type"))
-		}
-
-		data, err := ioutil.ReadAll(res.Body)
-		defer res.Body.Close()
-		str := string(data)
-
-		assert.NoError(t, err)
-		assert.Equal(t, test.output, str)
-	}
-}
-
-func TestProxyHTTPSubmitForbidden(t *testing.T) {
-	proxy := helperNewProxy(t, true)
-
-	server := httptest.NewServer(proxy.httpSubmit())
-	defer server.Close()
-	client := server.Client()
-
-	res, err := client.Get(server.URL)
-
-	assert.NoError(t, err)
-	assert.Equal(t, http.StatusForbidden, res.StatusCode)
-	assert.Zero(t, res.ContentLength)
-}
-
-func TestProxyHTTPRespond(t *testing.T) {
-	proxy := helperNewProxy(t)
-
-	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		proxy.httpRespond(w, r, http.StatusAccepted, "Hello")
-	}))
-	defer server.Close()
-	client := server.Client()
-
-	res, err := client.Get(server.URL)
-	assert.NoError(t, err)
-	assert.Equal(t, http.StatusAccepted, res.StatusCode)
-	assert.Equal(t, int64(5), res.ContentLength)
-
-	data, err := ioutil.ReadAll(res.Body)
-	defer res.Body.Close()
-	assert.Equal(t, []byte("Hello"), data)
-}
-
-func TestProxyHTTPRespondForbidden(t *testing.T) {
-	proxy := helperNewProxy(t, true)
-
-	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		proxy.httpRespond(w, r, http.StatusAccepted, "Hello")
-	}))
-	defer server.Close()
-	client := server.Client()
-
-	res, err := client.Get(server.URL)
-
-	assert.NoError(t, err)
-	assert.Equal(t, http.StatusAccepted, res.StatusCode)
-	assert.Equal(t, int64(0), res.ContentLength)
-}
-
-func TestHTTPError(t *testing.T) {
-	_, errTimeout := net.DialTimeout("tcp", "127.0.0.1", 0)
-	assert.Error(t, errTimeout)
-
-	tests := []struct {
-		input  error
-		status int
-		output error
-	}{
-		{nil, http.StatusNotImplemented, fmt.Errorf("error expected but found none")},
-		{io.EOF, http.StatusBadRequest, fmt.Errorf("request body cannot be empty")},
-		{context.DeadlineExceeded, http.StatusRequestTimeout, nil},
-		{context.Canceled, 444, nil},
-		{errTimeout, http.StatusRequestTimeout, nil},
-		{fmt.Errorf(""), http.StatusInternalServerError, nil},
-	}
-
-	for _, test := range tests {
-		status, err := httpError(http.StatusInternalServerError, test.input)
-
-		assert.Error(t, err)
-		assert.Equal(t, test.status, status)
-		if test.output == nil {
-			test.output = test.input
-		}
-		assert.Equal(t, test.output, err)
-	}
-}
-
-func helperNewProxy(t *testing.T, secure ...bool) *Proxy {
-	t.Helper()
-
-	proxy, err := NewSecureProxy(context.Background(), "file::memory:?cache=shared", "test.cloudflareaccess.com", "")
-	assert.NoError(t, err)
-	assert.NotNil(t, proxy)
-
-	if len(secure) == 0 {
-		proxy.accessValidator = nil // Mark as insecure
-	}
-
-	return proxy
-}

dbconnect/sql.go

@@ -1,318 +0,0 @@
-package dbconnect
-
-import (
-	"context"
-	"database/sql"
-	"encoding/json"
-	"fmt"
-	"net/url"
-	"reflect"
-	"strings"
-
-	"github.com/jmoiron/sqlx"
-	"github.com/pkg/errors"
-	"github.com/xo/dburl"
-
-	// SQL drivers self-register with the database/sql package.
-	// https://github.com/golang/go/wiki/SQLDrivers
-	_ "github.com/denisenkom/go-mssqldb"
-	_ "github.com/go-sql-driver/mysql"
-	_ "github.com/mattn/go-sqlite3"
-
-	"github.com/kshvakov/clickhouse"
-	"github.com/lib/pq"
-)
-
-// SQLClient is a Client that talks to a SQL database.
-type SQLClient struct {
-	Dialect string
-	driver  *sqlx.DB
-}
-
-// NewSQLClient creates a SQL client based on its URL scheme.
-func NewSQLClient(ctx context.Context, originURL *url.URL) (Client, error) {
-	res, err := dburl.Parse(originURL.String())
-	if err != nil {
-		helpText := fmt.Sprintf("supported drivers: %+q, see documentation for more details: %s", sql.Drivers(), "https://godoc.org/github.com/xo/dburl")
-		return nil, fmt.Errorf("could not parse sql database url '%s': %s\n%s", originURL, err.Error(), helpText)
-	}
-
-	// Establishes the driver, but does not test the connection.
-	driver, err := sqlx.Open(res.Driver, res.DSN)
-	if err != nil {
-		return nil, fmt.Errorf("could not open sql driver %s: %s\n%s", res.Driver, err.Error(), res.DSN)
-	}
-
-	// Closes the driver, will occur when the context finishes.
-	go func() {
-		<-ctx.Done()
-		_ = driver.Close()
-	}()
-
-	return &SQLClient{driver.DriverName(), driver}, nil
-}
-
-// Ping verifies a connection to the database is still alive.
-func (client *SQLClient) Ping(ctx context.Context) error {
-	return client.driver.PingContext(ctx)
-}
-
-// Submit queries or executes a command to the SQL database.
-func (client *SQLClient) Submit(ctx context.Context, cmd *Command) (interface{}, error) {
-	txx, err := cmd.ValidateSQL(client.Dialect)
-	if err != nil {
-		return nil, err
-	}
-
-	ctx, cancel := context.WithTimeout(ctx, cmd.Timeout)
-	defer cancel()
-
-	var res interface{}
-
-	// Get the next available sql.Conn and submit the Command.
-	err = sqlConn(ctx, client.driver, txx, func(conn *sql.Conn) error {
-		stmt := cmd.Statement
-		args := cmd.Arguments.Positional
-
-		if cmd.Mode == "query" {
-			res, err = sqlQuery(ctx, conn, stmt, args)
-		} else {
-			res, err = sqlExec(ctx, conn, stmt, args)
-		}
-
-		return err
-	})
-
-	return res, err
-}
-
-// ValidateSQL extends the contract of Command for SQL dialects:
-// mode is conformed, arguments are []sql.NamedArg, and isolation is a sql.IsolationLevel.
-//
-// When the command should not be wrapped in a transaction, *sql.TxOptions and error will both be nil.
-func (cmd *Command) ValidateSQL(dialect string) (*sql.TxOptions, error) {
-	err := cmd.Validate()
-	if err != nil {
-		return nil, err
-	}
-
-	mode, err := sqlMode(cmd.Mode)
-	if err != nil {
-		return nil, err
-	}
-
-	// Mutates Arguments to only use positional arguments with the type sql.NamedArg.
-	// This is a required by the sql.Driver before submitting arguments.
-	cmd.Arguments.sql(dialect)
-
-	iso, err := sqlIsolation(cmd.Isolation)
-	if err != nil {
-		return nil, err
-	}
-
-	// When isolation is out-of-range, this is indicative that no
-	// transaction should be executed and sql.TxOptions should be nil.
-	if iso < sql.LevelDefault {
-		return nil, nil
-	}
-
-	// In query mode, execute the transaction in read-only, unless it's Microsoft SQL
-	// which does not support that type of transaction.
-	readOnly := mode == "query" && dialect != "mssql"
-
-	return &sql.TxOptions{Isolation: iso, ReadOnly: readOnly}, nil
-}
-
-// sqlConn gets the next available sql.Conn in the connection pool and runs a function to use it.
-//
-// If the transaction options are nil, run the useIt function outside a transaction.
-// This is potentially an unsafe operation if the command does not clean up its state.
-func sqlConn(ctx context.Context, driver *sqlx.DB, txx *sql.TxOptions, useIt func(*sql.Conn) error) error {
-	conn, err := driver.Conn(ctx)
-	if err != nil {
-		return err
-	}
-	defer conn.Close()
-
-	// If transaction options are specified, begin and defer a rollback to catch errors.
-	var tx *sql.Tx
-	if txx != nil {
-		tx, err = conn.BeginTx(ctx, txx)
-		if err != nil {
-			return err
-		}
-		defer tx.Rollback()
-	}
-
-	err = useIt(conn)
-
-	// Check if useIt was successful and a transaction exists before committing.
-	if err == nil && tx != nil {
-		err = tx.Commit()
-	}
-
-	return err
-}
-
-// sqlQuery queries rows on a sql.Conn and returns an array of result objects.
-func sqlQuery(ctx context.Context, conn *sql.Conn, stmt string, args []interface{}) ([]map[string]interface{}, error) {
-	rows, err := conn.QueryContext(ctx, stmt, args...)
-	if err == nil {
-		return sqlRows(rows)
-	}
-	return nil, err
-}
-
-// sqlExec executes a command on a sql.Conn and returns the result of the operation.
-func sqlExec(ctx context.Context, conn *sql.Conn, stmt string, args []interface{}) (sqlResult, error) {
-	exec, err := conn.ExecContext(ctx, stmt, args...)
-	if err == nil {
-		return sqlResultFrom(exec), nil
-	}
-	return sqlResult{}, err
-}
-
-// sql mutates Arguments to contain a positional []sql.NamedArg.
-//
-// The actual return type is []interface{} due to the native Golang
-// function signatures for sql.Exec and sql.Query being generic.
-func (args *Arguments) sql(dialect string) {
-	result := args.Positional
-
-	for i, val := range result {
-		result[i] = sqlArg("", val, dialect)
-	}
-
-	for key, val := range args.Named {
-		result = append(result, sqlArg(key, val, dialect))
-	}
-
-	args.Positional = result
-	args.Named = map[string]interface{}{}
-}
-
-// sqlArg creates a sql.NamedArg from a key-value pair and an optional dialect.
-//
-// Certain dialects will need to wrap objects, such as arrays, to conform its driver requirements.
-func sqlArg(key, val interface{}, dialect string) sql.NamedArg {
-	switch reflect.ValueOf(val).Kind() {
-
-	// PostgreSQL and Clickhouse require arrays to be wrapped before
-	// being inserted into the driver interface.
-	case reflect.Slice, reflect.Array:
-		switch dialect {
-		case "postgres":
-			val = pq.Array(val)
-		case "clickhouse":
-			val = clickhouse.Array(val)
-		}
-	}
-
-	return sql.Named(fmt.Sprint(key), val)
-}
-
-// sqlIsolation tries to match a string to a sql.IsolationLevel.
-func sqlIsolation(str string) (sql.IsolationLevel, error) {
-	if str == "none" {
-		return sql.IsolationLevel(-1), nil
-	}
-
-	for iso := sql.LevelDefault; ; iso++ {
-		if iso > sql.LevelLinearizable {
-			return -1, fmt.Errorf("cannot provide an invalid sql isolation level: '%s'", str)
-		}
-
-		if str == "" || strings.EqualFold(iso.String(), strings.ReplaceAll(str, "_", " ")) {
-			return iso, nil
-		}
-	}
-}
-
-// sqlMode tries to match a string to a command mode: 'query' or 'exec' for now.
-func sqlMode(str string) (string, error) {
-	switch str {
-	case "query", "exec":
-		return str, nil
-	default:
-		return "", fmt.Errorf("cannot provide invalid sql mode: '%s'", str)
-	}
-}
-
-// sqlRows scans through a SQL result set and returns an array of objects.
-func sqlRows(rows *sql.Rows) ([]map[string]interface{}, error) {
-	columns, err := rows.Columns()
-	if err != nil {
-		return nil, errors.Wrap(err, "could not extract columns from result")
-	}
-	defer rows.Close()
-
-	types, err := rows.ColumnTypes()
-	if err != nil {
-		// Some drivers do not support type extraction, so fail silently and continue.
-		types = make([]*sql.ColumnType, len(columns))
-	}
-
-	values := make([]interface{}, len(columns))
-	pointers := make([]interface{}, len(columns))
-
-	var results []map[string]interface{}
-	for rows.Next() {
-		for i := range columns {
-			pointers[i] = &values[i]
-		}
-		_ = rows.Scan(pointers...)
-
-		// Convert a row, an array of values, into an object where
-		// each key is the name of its respective column.
-		entry := make(map[string]interface{})
-		for i, col := range columns {
-			entry[col] = sqlValue(values[i], types[i])
-		}
-		results = append(results, entry)
-	}
-
-	return results, nil
-}
-
-// sqlValue handles special cases where sql.Rows does not return a "human-readable" object.
-func sqlValue(val interface{}, col *sql.ColumnType) interface{} {
-	bytes, ok := val.([]byte)
-	if ok {
-		// Opportunistically check for embeded JSON and convert it to a first-class object.
-		var embeded interface{}
-		if json.Unmarshal(bytes, &embeded) == nil {
-			return embeded
-		}
-
-		// STOR-604: investigate a way to coerce PostgreSQL arrays '{a, b, ...}' into JSON.
-		// Although easy with strings, it becomes more difficult with special types like INET[].
-
-		return string(bytes)
-	}
-
-	return val
-}
-
-// sqlResult is a thin wrapper around sql.Result.
-type sqlResult struct {
-	LastInsertId int64 `json:"last_insert_id"`
-	RowsAffected int64 `json:"rows_affected"`
-}
-
-// sqlResultFrom converts sql.Result into a JSON-marshable sqlResult.
-func sqlResultFrom(res sql.Result) sqlResult {
-	insertID, errID := res.LastInsertId()
-	rowsAffected, errRows := res.RowsAffected()
-
-	// If an error occurs when extracting the result, it is because the
-	// driver does not support that specific field. Instead of passing this
-	// to the user, omit the field in the response.
-	if errID != nil {
-		insertID = -1
-	}
-	if errRows != nil {
-		rowsAffected = -1
-	}
-
-	return sqlResult{insertID, rowsAffected}
-}

dbconnect/sql_test.go

@@ -1,336 +0,0 @@
-package dbconnect
-
-import (
-	"context"
-	"database/sql"
-	"fmt"
-	"net/url"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/kshvakov/clickhouse"
-	"github.com/lib/pq"
-
-	"github.com/DATA-DOG/go-sqlmock"
-	"github.com/stretchr/testify/assert"
-)
-
-func TestNewSQLClient(t *testing.T) {
-	originURLs := []string{
-		"postgres://localhost",
-		"cockroachdb://localhost:1337",
-		"postgresql://user:pass@127.0.0.1",
-		"mysql://localhost",
-		"clickhouse://127.0.0.1:9000/?debug",
-		"sqlite3::memory:",
-		"file:test.db?cache=shared",
-	}
-
-	for _, originURL := range originURLs {
-		origin, _ := url.Parse(originURL)
-		_, err := NewSQLClient(context.Background(), origin)
-
-		assert.NoError(t, err, originURL)
-	}
-
-	originURLs = []string{
-		"",
-		"/",
-		"http://localhost",
-		"coolthing://user:pass@127.0.0.1",
-	}
-
-	for _, originURL := range originURLs {
-		origin, _ := url.Parse(originURL)
-		_, err := NewSQLClient(context.Background(), origin)
-
-		assert.Error(t, err, originURL)
-	}
-}
-
-func TestArgumentsSQL(t *testing.T) {
-	args := []Arguments{
-		Arguments{
-			Positional: []interface{}{
-				"val", 10, 3.14,
-			},
-		},
-		Arguments{
-			Named: map[string]interface{}{
-				"key": time.Unix(0, 0),
-			},
-		},
-	}
-
-	var nameType sql.NamedArg
-	for _, arg := range args {
-		arg.sql("")
-		for _, named := range arg.Positional {
-			assert.IsType(t, nameType, named)
-		}
-	}
-}
-
-func TestSQLArg(t *testing.T) {
-	tests := []struct {
-		key     interface{}
-		val     interface{}
-		dialect string
-		arg     sql.NamedArg
-	}{
-		{"key", "val", "mssql", sql.Named("key", "val")},
-		{0, 1, "sqlite3", sql.Named("0", 1)},
-		{1, []string{"a", "b", "c"}, "postgres", sql.Named("1", pq.Array([]string{"a", "b", "c"}))},
-		{"in", []uint{0, 1}, "clickhouse", sql.Named("in", clickhouse.Array([]uint{0, 1}))},
-		{"", time.Unix(0, 0), "", sql.Named("", time.Unix(0, 0))},
-	}
-
-	for _, test := range tests {
-		arg := sqlArg(test.key, test.val, test.dialect)
-		assert.Equal(t, test.arg, arg, test.key)
-	}
-}
-
-func TestSQLIsolation(t *testing.T) {
-	tests := []struct {
-		str string
-		iso sql.IsolationLevel
-	}{
-		{"", sql.LevelDefault},
-		{"DEFAULT", sql.LevelDefault},
-		{"read_UNcommitted", sql.LevelReadUncommitted},
-		{"serializable", sql.LevelSerializable},
-		{"none", sql.IsolationLevel(-1)},
-		{"SNAP shot", -2},
-		{"blah", -2},
-	}
-
-	for _, test := range tests {
-		iso, err := sqlIsolation(test.str)
-
-		if test.iso < -1 {
-			assert.Error(t, err, test.str)
-		} else {
-			assert.NoError(t, err)
-			assert.Equal(t, test.iso, iso, test.str)
-		}
-	}
-}
-
-func TestSQLMode(t *testing.T) {
-	modes := []string{
-		"query",
-		"exec",
-	}
-
-	for _, mode := range modes {
-		actual, err := sqlMode(mode)
-
-		assert.NoError(t, err)
-		assert.Equal(t, strings.ToLower(mode), actual, mode)
-	}
-
-	modes = []string{
-		"",
-		"blah",
-	}
-
-	for _, mode := range modes {
-		_, err := sqlMode(mode)
-
-		assert.Error(t, err)
-	}
-}
-
-func helperRows(mockRows *sqlmock.Rows) *sql.Rows {
-	db, mock, _ := sqlmock.New()
-	mock.ExpectQuery("SELECT").WillReturnRows(mockRows)
-	rows, _ := db.Query("SELECT")
-	return rows
-}
-
-func TestSQLRows(t *testing.T) {
-	actual, err := sqlRows(helperRows(sqlmock.NewRows(
-		[]string{"name", "age", "dept"}).
-		AddRow("alice", 19, "prod")))
-	expected := []map[string]interface{}{map[string]interface{}{
-		"name": "alice",
-		"age":  int64(19),
-		"dept": "prod"}}
-
-	assert.NoError(t, err)
-	assert.ElementsMatch(t, expected, actual)
-}
-
-func TestSQLValue(t *testing.T) {
-	tests := []struct {
-		input  interface{}
-		output interface{}
-	}{
-		{"hello", "hello"},
-		{1, 1},
-		{false, false},
-		{[]byte("random"), "random"},
-		{[]byte("{\"json\":true}"), map[string]interface{}{"json": true}},
-		{[]byte("[]"), []interface{}{}},
-	}
-
-	for _, test := range tests {
-		assert.Equal(t, test.output, sqlValue(test.input, nil), test.input)
-	}
-}
-
-func TestSQLResultFrom(t *testing.T) {
-	res := sqlResultFrom(sqlmock.NewResult(1, 2))
-	assert.Equal(t, sqlResult{1, 2}, res)
-
-	res = sqlResultFrom(sqlmock.NewErrorResult(fmt.Errorf("")))
-	assert.Equal(t, sqlResult{-1, -1}, res)
-}
-
-func helperSQLite3(t *testing.T) (context.Context, Client) {
-	t.Helper()
-
-	ctx := context.Background()
-	url, _ := url.Parse("file::memory:?cache=shared")
-
-	sqlite3, err := NewSQLClient(ctx, url)
-	assert.NoError(t, err)
-
-	return ctx, sqlite3
-}
-
-func TestPing(t *testing.T) {
-	ctx, sqlite3 := helperSQLite3(t)
-	err := sqlite3.Ping(ctx)
-
-	assert.NoError(t, err)
-}
-
-func TestSubmit(t *testing.T) {
-	ctx, sqlite3 := helperSQLite3(t)
-
-	res, err := sqlite3.Submit(ctx, &Command{
-		Statement: "CREATE TABLE t (a INTEGER, b FLOAT, c TEXT, d BLOB);",
-		Mode:      "exec",
-	})
-	assert.NoError(t, err)
-	assert.Equal(t, sqlResult{0, 0}, res)
-
-	res, err = sqlite3.Submit(ctx, &Command{
-		Statement: "SELECT * FROM t;",
-		Mode:      "query",
-	})
-	assert.NoError(t, err)
-	assert.Empty(t, res)
-
-	res, err = sqlite3.Submit(ctx, &Command{
-		Statement: "INSERT INTO t VALUES (?, ?, ?, ?);",
-		Mode:      "exec",
-		Arguments: Arguments{
-			Positional: []interface{}{
-				1,
-				3.14,
-				"text",
-				"blob",
-			},
-		},
-	})
-	assert.NoError(t, err)
-	assert.Equal(t, sqlResult{1, 1}, res)
-
-	res, err = sqlite3.Submit(ctx, &Command{
-		Statement: "UPDATE t SET c = NULL;",
-		Mode:      "exec",
-	})
-	assert.NoError(t, err)
-	assert.Equal(t, sqlResult{1, 1}, res)
-
-	res, err = sqlite3.Submit(ctx, &Command{
-		Statement: "SELECT * FROM t WHERE a = ?;",
-		Mode:      "query",
-		Arguments: Arguments{
-			Positional: []interface{}{1},
-		},
-	})
-	assert.NoError(t, err)
-	assert.Len(t, res, 1)
-
-	resf, ok := res.([]map[string]interface{})
-	assert.True(t, ok)
-	assert.EqualValues(t, map[string]interface{}{
-		"a": int64(1),
-		"b": 3.14,
-		"c": nil,
-		"d": "blob",
-	}, resf[0])
-
-	res, err = sqlite3.Submit(ctx, &Command{
-		Statement: "DROP TABLE t;",
-		Mode:      "exec",
-	})
-	assert.NoError(t, err)
-	assert.Equal(t, sqlResult{1, 1}, res)
-}
-
-func TestSubmitTransaction(t *testing.T) {
-	ctx, sqlite3 := helperSQLite3(t)
-
-	res, err := sqlite3.Submit(ctx, &Command{
-		Statement: "BEGIN;",
-		Mode:      "exec",
-	})
-	assert.Error(t, err)
-	assert.Empty(t, res)
-
-	res, err = sqlite3.Submit(ctx, &Command{
-		Statement: "BEGIN; CREATE TABLE tt (a INT); COMMIT;",
-		Mode:      "exec",
-		Isolation: "none",
-	})
-	assert.NoError(t, err)
-	assert.Equal(t, sqlResult{0, 0}, res)
-
-	rows, err := sqlite3.Submit(ctx, &Command{
-		Statement: "SELECT * FROM tt;",
-		Mode:      "query",
-		Isolation: "repeatable_read",
-	})
-	assert.NoError(t, err)
-	assert.Empty(t, rows)
-}
-
-func TestSubmitTimeout(t *testing.T) {
-	ctx, sqlite3 := helperSQLite3(t)
-
-	res, err := sqlite3.Submit(ctx, &Command{
-		Statement: "SELECT * FROM t;",
-		Mode:      "query",
-		Timeout:   1 * time.Nanosecond,
-	})
-	assert.Error(t, err)
-	assert.Empty(t, res)
-}
-
-func TestSubmitMode(t *testing.T) {
-	ctx, sqlite3 := helperSQLite3(t)
-
-	res, err := sqlite3.Submit(ctx, &Command{
-		Statement: "SELECT * FROM t;",
-		Mode:      "notanoption",
-	})
-	assert.Error(t, err)
-	assert.Empty(t, res)
-}
-
-func TestSubmitEmpty(t *testing.T) {
-	ctx, sqlite3 := helperSQLite3(t)
-
-	res, err := sqlite3.Submit(ctx, &Command{
-		Statement: "; ; ; ;",
-		Mode:      "query",
-	})
-	assert.Error(t, err)
-	assert.Empty(t, res)
-}

dbconnect_tests/dbconnect.yaml

@@ -1,78 +0,0 @@
-# docker-compose -f ./dbconnect_tests/dbconnect.yaml up --build --force-recreate --renew-anon-volumes --exit-code-from cloudflared
-
-version: "2.3"
-networks:
-  test-dbconnect-network:
-    driver: bridge
-services:
-  cloudflared:
-    build:
-      context: ../
-      dockerfile: dev.Dockerfile
-    command: go test github.com/cloudflare/cloudflared/dbconnect_tests -v
-    depends_on:
-      postgres:
-        condition: service_healthy
-      mysql:
-        condition: service_healthy
-      mssql:
-        condition: service_healthy
-      clickhouse:
-        condition: service_healthy
-    environment:
-      DBCONNECT_INTEGRATION_TEST: "true"
-      POSTGRESQL_URL: postgres://postgres:secret@postgres/db?sslmode=disable
-      MYSQL_URL: mysql://root:secret@mysql/db?tls=false
-      MSSQL_URL: mssql://sa:secret12345!@mssql
-      CLICKHOUSE_URL: clickhouse://clickhouse:9000/db
-    networks:
-      - test-dbconnect-network
-  postgres:
-    image: postgres:11.4-alpine
-    environment:
-      POSTGRES_DB: db
-      POSTGRES_PASSWORD: secret
-    healthcheck:
-      test: ["CMD", "pg_isready", "-U", "postgres"]
-      start_period: 3s
-      interval: 1s
-      timeout: 3s
-      retries: 10
-    networks:
-      - test-dbconnect-network
-  mysql:
-    image: mysql:8.0
-    environment:
-      MYSQL_DATABASE: db
-      MYSQL_ROOT_PASSWORD: secret
-    healthcheck:
-      test: ["CMD", "mysqladmin", "ping"]
-      start_period: 3s
-      interval: 1s
-      timeout: 3s
-      retries: 10
-    networks:
-      - test-dbconnect-network
-  mssql:
-    image: mcr.microsoft.com/mssql/server:2017-CU8-ubuntu
-    environment:
-      ACCEPT_EULA: "Y"
-      SA_PASSWORD: secret12345!
-    healthcheck:
-      test: ["CMD", "/opt/mssql-tools/bin/sqlcmd", "-S", "localhost", "-U", "sa", "-P", "secret12345!", "-Q", "SELECT 1"]
-      start_period: 3s
-      interval: 1s
-      timeout: 3s
-      retries: 10
-    networks:
-      - test-dbconnect-network
-  clickhouse:
-    image: yandex/clickhouse-server:19.11
-    healthcheck:
-      test: ["CMD", "clickhouse-client", "--query", "SELECT 1"]
-      start_period: 3s
-      interval: 1s
-      timeout: 3s
-      retries: 10
-    networks:
-      - test-dbconnect-network

dbconnect_tests/sql_test.go

@@ -1,265 +0,0 @@
-package dbconnect_tests
-
-import (
-	"context"
-	"log"
-	"net/url"
-	"os"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-
-	"github.com/cloudflare/cloudflared/dbconnect"
-)
-
-func TestIntegrationPostgreSQL(t *testing.T) {
-	ctx, pq := helperNewSQLClient(t, "POSTGRESQL_URL")
-
-	err := pq.Ping(ctx)
-	assert.NoError(t, err)
-
-	_, err = pq.Submit(ctx, &dbconnect.Command{
-		Statement: "CREATE TABLE t (a TEXT, b UUID, c JSON, d INET[], e SERIAL);",
-		Mode:      "exec",
-	})
-	assert.NoError(t, err)
-
-	_, err = pq.Submit(ctx, &dbconnect.Command{
-		Statement: "INSERT INTO t VALUES ($1, $2, $3, $4);",
-		Mode:      "exec",
-		Arguments: dbconnect.Arguments{
-			Positional: []interface{}{
-				"text",
-				"6b8d686d-bd8e-43bc-b09a-cfcbbe702c10",
-				"{\"bool\":true,\"array\":[\"a\", 1, 3.14],\"embed\":{\"num\":21}}",
-				[]string{"1.1.1.1", "1.0.0.1"},
-			},
-		},
-	})
-	assert.NoError(t, err)
-
-	_, err = pq.Submit(ctx, &dbconnect.Command{
-		Statement: "UPDATE t SET b = NULL;",
-		Mode:      "exec",
-	})
-	assert.NoError(t, err)
-
-	res, err := pq.Submit(ctx, &dbconnect.Command{
-		Statement: "SELECT * FROM t;",
-		Mode:      "query",
-	})
-	assert.NoError(t, err)
-	assert.IsType(t, make([]map[string]interface{}, 0), res)
-
-	actual := res.([]map[string]interface{})[0]
-	expected := map[string]interface{}{
-		"a": "text",
-		"b": nil,
-		"c": map[string]interface{}{
-			"bool":  true,
-			"array": []interface{}{"a", float64(1), 3.14},
-			"embed": map[string]interface{}{"num": float64(21)},
-		},
-		"d": "{1.1.1.1,1.0.0.1}",
-		"e": int64(1),
-	}
-	assert.EqualValues(t, expected, actual)
-
-	_, err = pq.Submit(ctx, &dbconnect.Command{
-		Statement: "DROP TABLE t;",
-		Mode:      "exec",
-	})
-	assert.NoError(t, err)
-}
-
-func TestIntegrationMySQL(t *testing.T) {
-	ctx, my := helperNewSQLClient(t, "MYSQL_URL")
-
-	err := my.Ping(ctx)
-	assert.NoError(t, err)
-
-	_, err = my.Submit(ctx, &dbconnect.Command{
-		Statement: "CREATE TABLE t (a CHAR, b TINYINT, c FLOAT, d JSON, e YEAR);",
-		Mode:      "exec",
-	})
-	assert.NoError(t, err)
-
-	_, err = my.Submit(ctx, &dbconnect.Command{
-		Statement: "INSERT INTO t VALUES (?, ?, ?, ?, ?);",
-		Mode:      "exec",
-		Arguments: dbconnect.Arguments{
-			Positional: []interface{}{
-				"a",
-				10,
-				3.14,
-				"{\"bool\":true}",
-				2000,
-			},
-		},
-	})
-	assert.NoError(t, err)
-
-	_, err = my.Submit(ctx, &dbconnect.Command{
-		Statement: "ALTER TABLE t ADD COLUMN f GEOMETRY;",
-		Mode:      "exec",
-	})
-	assert.NoError(t, err)
-
-	res, err := my.Submit(ctx, &dbconnect.Command{
-		Statement: "SELECT * FROM t;",
-		Mode:      "query",
-	})
-	assert.NoError(t, err)
-	assert.IsType(t, make([]map[string]interface{}, 0), res)
-
-	actual := res.([]map[string]interface{})[0]
-	expected := map[string]interface{}{
-		"a": "a",
-		"b": float64(10),
-		"c": 3.14,
-		"d": map[string]interface{}{"bool": true},
-		"e": float64(2000),
-		"f": nil,
-	}
-	assert.EqualValues(t, expected, actual)
-
-	_, err = my.Submit(ctx, &dbconnect.Command{
-		Statement: "DROP TABLE t;",
-		Mode:      "exec",
-	})
-	assert.NoError(t, err)
-}
-
-func TestIntegrationMSSQL(t *testing.T) {
-	ctx, ms := helperNewSQLClient(t, "MSSQL_URL")
-
-	err := ms.Ping(ctx)
-	assert.NoError(t, err)
-
-	_, err = ms.Submit(ctx, &dbconnect.Command{
-		Statement: "CREATE TABLE t (a BIT, b DECIMAL, c MONEY, d TEXT);",
-		Mode:      "exec"})
-	assert.NoError(t, err)
-
-	_, err = ms.Submit(ctx, &dbconnect.Command{
-		Statement: "INSERT INTO t VALUES (?, ?, ?, ?);",
-		Mode:      "exec",
-		Arguments: dbconnect.Arguments{
-			Positional: []interface{}{
-				0,
-				3,
-				"$0.99",
-				"text",
-			},
-		},
-	})
-	assert.NoError(t, err)
-
-	_, err = ms.Submit(ctx, &dbconnect.Command{
-		Statement: "UPDATE t SET d = NULL;",
-		Mode:      "exec",
-	})
-	assert.NoError(t, err)
-
-	res, err := ms.Submit(ctx, &dbconnect.Command{
-		Statement: "SELECT * FROM t;",
-		Mode:      "query",
-	})
-	assert.NoError(t, err)
-	assert.IsType(t, make([]map[string]interface{}, 0), res)
-
-	actual := res.([]map[string]interface{})[0]
-	expected := map[string]interface{}{
-		"a": false,
-		"b": float64(3),
-		"c": float64(0.99),
-		"d": nil,
-	}
-	assert.EqualValues(t, expected, actual)
-
-	_, err = ms.Submit(ctx, &dbconnect.Command{
-		Statement: "DROP TABLE t;",
-		Mode:      "exec",
-	})
-	assert.NoError(t, err)
-}
-
-func TestIntegrationClickhouse(t *testing.T) {
-	ctx, ch := helperNewSQLClient(t, "CLICKHOUSE_URL")
-
-	err := ch.Ping(ctx)
-	assert.NoError(t, err)
-
-	_, err = ch.Submit(ctx, &dbconnect.Command{
-		Statement: "CREATE TABLE t (a UUID, b String, c Float64, d UInt32, e Int16, f Array(Enum8('a'=1, 'b'=2, 'c'=3))) engine=Memory;",
-		Mode:      "exec",
-	})
-	assert.NoError(t, err)
-
-	_, err = ch.Submit(ctx, &dbconnect.Command{
-		Statement: "INSERT INTO t VALUES (?, ?, ?, ?, ?, ?);",
-		Mode:      "exec",
-		Arguments: dbconnect.Arguments{
-			Positional: []interface{}{
-				"ec65f626-6f50-4c86-9628-6314ef1edacd",
-				"",
-				3.14,
-				314,
-				-144,
-				[]string{"a", "b", "c"},
-			},
-		},
-	})
-	assert.NoError(t, err)
-
-	res, err := ch.Submit(ctx, &dbconnect.Command{
-		Statement: "SELECT * FROM t;",
-		Mode:      "query",
-	})
-	assert.NoError(t, err)
-	assert.IsType(t, make([]map[string]interface{}, 0), res)
-
-	actual := res.([]map[string]interface{})[0]
-	expected := map[string]interface{}{
-		"a": "ec65f626-6f50-4c86-9628-6314ef1edacd",
-		"b": "",
-		"c": float64(3.14),
-		"d": uint32(314),
-		"e": int16(-144),
-		"f": []string{"a", "b", "c"},
-	}
-	assert.EqualValues(t, expected, actual)
-
-	_, err = ch.Submit(ctx, &dbconnect.Command{
-		Statement: "DROP TABLE t;",
-		Mode:      "exec",
-	})
-	assert.NoError(t, err)
-}
-
-func helperNewSQLClient(t *testing.T, env string) (context.Context, dbconnect.Client) {
-	_, ok := os.LookupEnv("DBCONNECT_INTEGRATION_TEST")
-	if ok {
-		t.Helper()
-	} else {
-		t.SkipNow()
-	}
-
-	val, ok := os.LookupEnv(env)
-	if !ok {
-		log.Fatalf("must provide database url as environment variable: %s", env)
-	}
-
-	parsed, err := url.Parse(val)
-	if err != nil {
-		log.Fatalf("cannot provide invalid database url: %s=%s", env, val)
-	}
-
-	ctx := context.Background()
-	client, err := dbconnect.NewSQLClient(ctx, parsed)
-	if err != nil {
-		log.Fatalf("could not start test client: %s", err)
-	}
-
-	return ctx, client
-}

go.mod

@@ -3,29 +3,20 @@ module github.com/cloudflare/cloudflared
 go 1.15
 
 require (
-	github.com/DATA-DOG/go-sqlmock v1.3.3
-	github.com/acmacalister/skittles v0.0.0-20160609003031-7423546701e1 // indirect
-	github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d // indirect
-	github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 // indirect
-	github.com/aws/aws-sdk-go v1.34.19 // indirect
 	github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894 // indirect
 	github.com/cloudflare/brotli-go v0.0.0-20191101163834-d34379f7ff93
 	github.com/cloudflare/golibs v0.0.0-20170913112048-333127dbecfc
 	github.com/coredns/coredns v1.7.0
 	github.com/coreos/go-oidc v0.0.0-20171002155002-a93f71fdfe73
 	github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf
-	github.com/denisenkom/go-mssqldb v0.0.0-20191001013358-cfbb681360f0
 	github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 // indirect
 	github.com/facebookgo/freeport v0.0.0-20150612182905-d4adf43b75b9 // indirect
 	github.com/facebookgo/grace v0.0.0-20180706040059-75cf19382434
 	github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
 	github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870 // indirect
-	github.com/frankban/quicktest v1.10.0 // indirect
 	github.com/fsnotify/fsnotify v1.4.9
 	github.com/gdamore/tcell v1.3.0
 	github.com/getsentry/raven-go v0.0.0-20180517221441-ed7bcb39ff10
-	github.com/gliderlabs/ssh v0.0.0-20191009160644-63518b5243e0 // indirect
-	github.com/go-sql-driver/mysql v1.5.0
 	github.com/gobwas/httphead v0.0.0-20200921212729-da3d93bc3c58 // indirect
 	github.com/gobwas/pool v0.2.1 // indirect
 	github.com/gobwas/ws v1.0.4
@@ -34,19 +25,14 @@ require (
 	github.com/google/uuid v1.1.2
 	github.com/gorilla/mux v1.7.3
 	github.com/gorilla/websocket v1.4.2
-	github.com/jmoiron/sqlx v1.2.0
 	github.com/json-iterator/go v1.1.10
 	github.com/kr/text v0.2.0 // indirect
-	github.com/kshvakov/clickhouse v1.3.11
 	github.com/kylelemons/godebug v1.1.0 // indirect
-	github.com/lib/pq v1.2.0
 	github.com/mattn/go-colorable v0.1.8
-	github.com/mattn/go-sqlite3 v1.11.0
 	github.com/miekg/dns v1.1.31
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
-	github.com/pierrec/lz4 v2.5.2+incompatible // indirect
 	github.com/pkg/errors v0.9.1
 	github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 // indirect
 	github.com/prometheus/client_golang v1.7.1
@@ -55,7 +41,6 @@ require (
 	github.com/rs/zerolog v1.20.0
 	github.com/stretchr/testify v1.6.0
 	github.com/urfave/cli/v2 v2.2.0
-	github.com/xo/dburl v0.0.0-20191005012637-293c3298d6c0
 	golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a
 	golang.org/x/net v0.0.0-20200904194848-62affa334b73
 	golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43 // indirect
@@ -72,3 +57,5 @@ require (
 	gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 // indirect
 	zombiezen.com/go/capnproto2 v2.18.0+incompatible
 )
+
+replace github.com/urfave/cli/v2 => github.com/ipostelnik/cli/v2 v2.3.1-0.20210223162147-c8b4f97735e4

go.sum

@@ -62,7 +62,6 @@ github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbt
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
-github.com/DATA-DOG/go-sqlmock v1.3.3 h1:CWUqKXe0s8A2z6qCgkP4Kru7wC11YoAnoupUKFDnH08=
 github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
 github.com/DataDog/datadog-go v3.5.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
 github.com/DataDog/zstd v1.3.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
@@ -75,8 +74,6 @@ github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWX
 github.com/Shopify/sarama v1.21.0/go.mod h1:yuqtN/pe8cXRWG5zPaO7hCfNJp5MwmkoJEoLjkm5tCQ=
 github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
 github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g=
-github.com/acmacalister/skittles v0.0.0-20160609003031-7423546701e1 h1:RKnVV4C7qoN/sToLX2y1dqH7T6kKLMHcwRJlgwb9Ggk=
-github.com/acmacalister/skittles v0.0.0-20160609003031-7423546701e1/go.mod h1:gI5CyA/CEnS6eqNV22rqs4dG3aGfaSbXgPORIlwr2r0=
 github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c=
 github.com/akamai/AkamaiOPEN-edgegrid-golang v0.9.0/go.mod h1:zpDJeKyp9ScW4NNrbdr+Eyxvry3ilGPewKoXw3XGN1k=
 github.com/alangpierce/go-forceexport v0.0.0-20160317203124-8f1d6941cd75/go.mod h1:uAXEEpARkRhCZfEvy/y0Jcc888f9tHCc1W7/UeEtreE=
@@ -84,12 +81,9 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
-github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E=
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
 github.com/aliyun/alibaba-cloud-sdk-go v0.0.0-20190808125512-07798873deee/go.mod h1:myCDvQSzCW+wB1WAlocEru4wMGJxy+vlxHdhegi1CDQ=
 github.com/aliyun/aliyun-oss-go-sdk v0.0.0-20190307165228-86c17b95fcd5/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8=
-github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA=
-github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
 github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
 github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
@@ -100,7 +94,6 @@ github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQ
 github.com/aws/aws-sdk-go v1.23.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/aws/aws-sdk-go v1.32.1/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
-github.com/aws/aws-sdk-go v1.34.19/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
 github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
 github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f/go.mod h1:AuiFmCCPBSrqvVMvuqFuk0qogytodnVFVSN5CeJB8Gc=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
@@ -108,8 +101,6 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
-github.com/bkaradzic/go-lz4 v1.0.0 h1:RXc4wYsyz985CkXXeX04y4VnZFGG8Rd43pRaHsOXAKk=
-github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4=
 github.com/caddyserver/caddy v1.0.5 h1:5B1Hs0UF2x2tggr2X9jL2qOZtDXbIWQb9YLbmlxHSuM=
 github.com/caddyserver/caddy v1.0.5/go.mod h1:AnFHB+/MrgRC+mJAvuAgQ38ePzw+wKeW0wzENpdQQKY=
 github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ=
@@ -133,8 +124,6 @@ github.com/cloudflare/brotli-go v0.0.0-20191101163834-d34379f7ff93/go.mod h1:QiT
 github.com/cloudflare/cloudflare-go v0.10.2/go.mod h1:qhVI5MKwBGhdNU89ZRz2plgYutcJ5PCekLxXn56w6SY=
 github.com/cloudflare/golibs v0.0.0-20170913112048-333127dbecfc h1:Dvk3ySBsOm5EviLx6VCyILnafPcQinXGP5jbTdHUJgE=
 github.com/cloudflare/golibs v0.0.0-20170913112048-333127dbecfc/go.mod h1:HlgKKR8V5a1wroIDDIz3/A+T+9Janfq+7n1P5sEFdi0=
-github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 h1:F1EaeKL/ta07PY/k9Os/UFtwERei2/XzGemhpGnBKNg=
-github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
@@ -159,8 +148,6 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/decker502/dnspod-go v0.2.0/go.mod h1:qsurYu1FgxcDwfSwXJdLt4kRsBLZeosEb9uq4Sy+08g=
-github.com/denisenkom/go-mssqldb v0.0.0-20191001013358-cfbb681360f0 h1:epsH3lb7KVbXHYk7LYGN5EiE0MxcevHU85CKITJ0wUY=
-github.com/denisenkom/go-mssqldb v0.0.0-20191001013358-cfbb681360f0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8=
 github.com/dnaeon/go-vcr v0.0.0-20180814043457-aafff18a5cc2/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E=
@@ -199,8 +186,6 @@ github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjr
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
 github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4=
 github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20=
-github.com/frankban/quicktest v1.10.0 h1:Gfh+GAJZOAoKZsIZeZbdn2JF10kN1XHNvjsvQK8gVkE=
-github.com/frankban/quicktest v1.10.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
@@ -212,8 +197,6 @@ github.com/getsentry/raven-go v0.0.0-20180517221441-ed7bcb39ff10 h1:YO10pIIBftO/
 github.com/getsentry/raven-go v0.0.0-20180517221441-ed7bcb39ff10/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ=
 github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
-github.com/gliderlabs/ssh v0.0.0-20191009160644-63518b5243e0 h1:gF8ngtda767ddth2SH0YSAhswhz6qUkvyI9EZFYCWJA=
-github.com/gliderlabs/ssh v0.0.0-20191009160644-63518b5243e0/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
 github.com/go-acme/lego/v3 v3.1.0/go.mod h1:074uqt+JS6plx+c9Xaiz6+L+GBb+7itGtzfcDM2AhEE=
 github.com/go-acme/lego/v3 v3.2.0/go.mod h1:074uqt+JS6plx+c9Xaiz6+L+GBb+7itGtzfcDM2AhEE=
 github.com/go-cmd/cmd v1.0.5/go.mod h1:y8q8qlK5wQibcw63djSl/ntiHUHXHGdCkPk0j4QeW4s=
@@ -234,7 +217,6 @@ github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1
 github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc=
 github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I=
 github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
-github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/gobwas/httphead v0.0.0-20200921212729-da3d93bc3c58 h1:YyrUZvJaU8Q0QsoVo+xLFBgWDTam29PKea6GYmwvSiQ=
@@ -253,8 +235,6 @@ github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP
 github.com/goji/httpauth v0.0.0-20160601135302-2da839ab0f4d/go.mod h1:nnjvkQ9ptGaCkuDUx6wNykzzlUixGxvkme+H/lnzb+A=
 github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3 h1:zN2lZNZRflqFyxVaTIU61KNKQ9C0055u9CAfpmqUvo4=
 github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3/go.mod h1:nPpo7qLxd6XL3hWJG/O60sR8ZKfMCiIoNap5GvD12KU=
-github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY=
-github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -367,12 +347,11 @@ github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJ
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo=
 github.com/infobloxopen/go-trees v0.0.0-20190313150506-2af4e13f9062/go.mod h1:PcNJqIlcX/dj3DTG/+QQnRvSgTMG6CLpRMjWcv4+J6w=
+github.com/ipostelnik/cli/v2 v2.3.1-0.20210223162147-c8b4f97735e4 h1:U2OxGKZHC5puhKZnGo6wWWjuWusJCM6J/S1eF7Vt+Uk=
+github.com/ipostelnik/cli/v2 v2.3.1-0.20210223162147-c8b4f97735e4/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
 github.com/jimstudt/http-authentication v0.0.0-20140401203705-3eca13d6893a/go.mod h1:wK6yTYYcgjHE1Z1QtXACPDjcFJyBskHEdagmnq3vsP8=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
-github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc=
 github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik=
-github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA=
-github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
 github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
@@ -397,22 +376,16 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxv
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
-github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/kshvakov/clickhouse v1.3.11 h1:dtzTJY0fCA+MWkLyuKZaNPkmSwdX4gh8+Klic9NB1Lw=
-github.com/kshvakov/clickhouse v1.3.11/go.mod h1:/SVBAcqF3u7rxQ9sTWCZwf8jzzvxiZGeQvtmSF2BBEc=
 github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/labbsr0x/bindman-dns-webhook v1.0.2/go.mod h1:p6b+VCXIR8NYKpDr8/dg1HKfQoRHCdcsROXKvmoehKA=
 github.com/labbsr0x/goh v1.0.1/go.mod h1:8K2UhVoaWXcCU7Lxoa2omWnC8gyW8px7/lmO61c027w=
-github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
-github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0=
-github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM=
 github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4=
 github.com/linode/linodego v0.10.0/go.mod h1:cziNP7pbvE3mXIPneHj0oRY8L1WtGEIKlZ8LANE4eXA=
@@ -437,9 +410,6 @@ github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp
 github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/mattn/go-runewidth v0.0.8 h1:3tS41NlGYSmhhe/8fhGRzc+z3AYCw1Fe1WAyLuujKs0=
 github.com/mattn/go-runewidth v0.0.8/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
-github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
-github.com/mattn/go-sqlite3 v1.11.0 h1:LDdKkqtYlom37fkvqs8rMPFKAMe8+SgjbwZ6ex1/A/Q=
-github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/mattn/go-tty v0.0.0-20180219170247-931426f7535a/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
@@ -519,8 +489,6 @@ github.com/philhofer/fwd v1.0.0 h1:UbZqGr5Y38ApvM/V/jEljVxwocdweyH+vmYvRPBnbqQ=
 github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
 github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc=
 github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
-github.com/pierrec/lz4 v2.5.2+incompatible h1:WCjObylUIOlKy/+7Abdn34TLIkXiA4UWUMhxq9m9ZXI=
-github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@@ -619,15 +587,11 @@ github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
 github.com/urfave/cli v1.22.1 h1:+mkCCcOFKPnCmVYVcURKps1Xe+3zP90gSYGNfRkjoIY=
 github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
-github.com/urfave/cli/v2 v2.2.0 h1:JTTnM6wKzdA0Jqodd966MVj4vWbbquZykeX1sKbe2C4=
-github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ=
 github.com/vultr/govultr v0.1.4/go.mod h1:9H008Uxr/C4vFNGLqKx232C206GL0PBHzOP0809bGNA=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
 github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
 github.com/xeipuuv/gojsonschema v1.1.0/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
-github.com/xo/dburl v0.0.0-20191005012637-293c3298d6c0 h1:6DtWz8hNS4qbq0OCRPhdBMG9E2qKTSDKlwnP3dmZvuA=
-github.com/xo/dburl v0.0.0-20191005012637-293c3298d6c0/go.mod h1:A47W3pdWONaZmXuLZgfKLAVgUY0qvfTRM5vVDKS40S4=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -657,7 +621,6 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf
 golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190418165655-df01cb2cc480/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@@ -815,9 +778,6 @@ golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200909081042-eff7692f9009 h1:W0lCpv29Hv0UaM1LXb9QlBHLNP8UFfcKjblhVCWftOM=
-golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 h1:myAQVi0cGEoqQVR5POX+8RR2mrocKqNN1hmeMqhX27k=
 golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1012,6 +972,7 @@ gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRN
 gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

h2mux/header.go

@@ -125,12 +125,12 @@ func IsWebsocketClientHeader(headerName string) bool {
 		headerName == "upgrade"
 }
 
-func H1ResponseToH2ResponseHeaders(h1 *http.Response) (h2 []Header) {
+func H1ResponseToH2ResponseHeaders(status int, h1 http.Header) (h2 []Header) {
 	h2 = []Header{
-		{Name: ":status", Value: strconv.Itoa(h1.StatusCode)},
+		{Name: ":status", Value: strconv.Itoa(status)},
 	}
-	userHeaders := make(http.Header, len(h1.Header))
-	for header, values := range h1.Header {
+	userHeaders := make(http.Header, len(h1))
+	for header, values := range h1 {
 		h2name := strings.ToLower(header)
 		if h2name == "content-length" {
 			// This header has meaning in HTTP/2 and will be used by the edge,

h2mux/header_test.go

@@ -579,7 +579,7 @@ func TestH1ResponseToH2ResponseHeaders(t *testing.T) {
 		Header:     mockHeaders,
 	}
 
-	headers := H1ResponseToH2ResponseHeaders(&mockResponse)
+	headers := H1ResponseToH2ResponseHeaders(mockResponse.StatusCode, mockResponse.Header)
 
 	serializedHeadersIndex := -1
 	for i, header := range headers {
@@ -622,7 +622,7 @@ func TestHeaderSize(t *testing.T) {
 		Header:     largeHeaders,
 	}
 
-	serializedHeaders := H1ResponseToH2ResponseHeaders(&mockResponse)
+	serializedHeaders := H1ResponseToH2ResponseHeaders(mockResponse.StatusCode, mockResponse.Header)
 	request, err := http.NewRequest(http.MethodGet, "https://example.com/", nil)
 	assert.NoError(t, err)
 	for _, header := range serializedHeaders {
@@ -669,6 +669,6 @@ func BenchmarkH1ResponseToH2ResponseHeaders(b *testing.B) {
 	b.ReportAllocs()
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		_ = H1ResponseToH2ResponseHeaders(h1resp)
+		_ = H1ResponseToH2ResponseHeaders(h1resp.StatusCode, h1resp.Header)
 	}
 }

ingress/ingress.go

@@ -9,7 +9,8 @@ import (
 	"strings"
 	"sync"
 
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/config"
+	"github.com/cloudflare/cloudflared/ipaccess"
 
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
@@ -24,6 +25,12 @@ var (
 	ErrURLIncompatibleWithIngress = errors.New("You can't set the --url flag (or $TUNNEL_URL) when using multiple-origin ingress rules")
 )
 
+const (
+	ServiceBastion     = "bastion"
+	ServiceSocksProxy  = "socks-proxy"
+	ServiceWarpRouting = "warp-routing"
+)
+
 // FindMatchingRule returns the index of the Ingress Rule which matches the given
 // hostname and path. This function assumes the last rule matches everything,
 // which is the case if the rules were instantiated via the ingress#Validate method
@@ -38,6 +45,7 @@ func (ing Ingress) FindMatchingRule(hostname, path string) (*Rule, int) {
 			return &rule, i
 		}
 	}
+
 	i := len(ing.Rules) - 1
 	return &ing.Rules[i], i
 }
@@ -84,17 +92,35 @@ func NewSingleOrigin(c *cli.Context, allowURLFromArgs bool) (Ingress, error) {
 	return ing, err
 }
 
+// WarpRoutingService starts a tcp stream between the origin and requests from
+// warp clients.
+type WarpRoutingService struct {
+	Proxy StreamBasedOriginProxy
+}
+
+func NewWarpRoutingService() *WarpRoutingService {
+	return &WarpRoutingService{Proxy: &rawTCPService{name: ServiceWarpRouting}}
+}
+
 // Get a single origin service from the CLI/config.
-func parseSingleOriginService(c *cli.Context, allowURLFromArgs bool) (OriginService, error) {
+func parseSingleOriginService(c *cli.Context, allowURLFromArgs bool) (originService, error) {
 	if c.IsSet("hello-world") {
 		return new(helloWorld), nil
 	}
-	if c.IsSet("url") || c.IsSet(config.BastionFlag) {
+	if c.IsSet(config.BastionFlag) {
+		return newBastionService(), nil
+	}
+	if c.IsSet("url") {
 		originURL, err := config.ValidateUrl(c, allowURLFromArgs)
 		if err != nil {
 			return nil, errors.Wrap(err, "Error validating origin URL")
 		}
-		return &localService{URL: originURL, RootURL: originURL}, nil
+		if isHTTPService(originURL) {
+			return &httpService{
+				url: originURL,
+			}, nil
+		}
+		return newTCPOverWSService(originURL), nil
 	}
 	if c.IsSet("unix-socket") {
 		path, err := config.ValidateUnixSocket(c)
@@ -104,7 +130,7 @@ func parseSingleOriginService(c *cli.Context, allowURLFromArgs bool) (OriginServ
 		return &unixSocketPath{path: path}, nil
 	}
 	u, err := url.Parse("http://localhost:8080")
-	return &localService{URL: u, RootURL: u}, err
+	return &httpService{url: u}, err
 }
 
 // IsEmpty checks if there are any ingress rules.
@@ -136,7 +162,7 @@ func validate(ingress []config.UnvalidatedIngressRule, defaults OriginRequestCon
 	rules := make([]Rule, len(ingress))
 	for i, r := range ingress {
 		cfg := setConfig(defaults, r.OriginRequest)
-		var service OriginService
+		var service originService
 
 		if prefix := "unix:"; strings.HasPrefix(r.Service, prefix) {
 			// No validation necessary for unix socket filepath services
@@ -151,12 +177,29 @@ func validate(ingress []config.UnvalidatedIngressRule, defaults OriginRequestCon
 			service = &srv
 		} else if r.Service == "hello_world" || r.Service == "hello-world" || r.Service == "helloworld" {
 			service = new(helloWorld)
-		} else if r.Service == "bastion" || cfg.BastionMode {
+		} else if r.Service == ServiceSocksProxy {
+			rules := make([]ipaccess.Rule, len(r.OriginRequest.IPRules))
+
+			for i, ipRule := range r.OriginRequest.IPRules {
+				rule, err := ipaccess.NewRuleByCIDR(ipRule.Prefix, ipRule.Ports, ipRule.Allow)
+				if err != nil {
+					return Ingress{}, fmt.Errorf("unable to create ip rule for %s: %s", r.Service, err)
+				}
+				rules[i] = rule
+			}
+
+			accessPolicy, err := ipaccess.NewPolicy(false, rules)
+			if err != nil {
+				return Ingress{}, fmt.Errorf("unable to create ip access policy for %s: %s", r.Service, err)
+			}
+
+			service = newSocksProxyOverWSService(accessPolicy)
+		} else if r.Service == ServiceBastion || cfg.BastionMode {
 			// Bastion mode will always start a Websocket proxy server, which will
 			// overwrite the localService.URL field when `start` is called. So,
 			// leave the URL field empty for now.
 			cfg.BastionMode = true
-			service = new(localService)
+			service = newBastionService()
 		} else {
 			// Validate URL services
 			u, err := url.Parse(r.Service)
@@ -171,8 +214,11 @@ func validate(ingress []config.UnvalidatedIngressRule, defaults OriginRequestCon
 			if u.Path != "" {
 				return Ingress{}, fmt.Errorf("%s is an invalid address, ingress rules don't support proxying to a different path on the origin service. The path will be the same as the eyeball request's path", r.Service)
 			}
-			serviceURL := localService{URL: u}
-			service = &serviceURL
+			if isHTTPService(u) {
+				service = &httpService{url: u}
+			} else {
+				service = newTCPOverWSService(u)
+			}
 		}
 
 		if err := validateHostname(r, i, len(ingress)); err != nil {
@@ -241,3 +287,7 @@ func ParseIngress(conf *config.Configuration) (Ingress, error) {
 	}
 	return validate(conf.Ingress, originRequestFromYAML(conf.OriginRequest))
 }
+
+func isHTTPService(url *url.URL) bool {
+	return url.Scheme == "http" || url.Scheme == "https" || url.Scheme == "ws" || url.Scheme == "wss"
+}

ingress/ingress_test.go

@@ -3,6 +3,7 @@ package ingress
 import (
 	"flag"
 	"fmt"
+	"net/http"
 	"net/url"
 	"regexp"
 	"testing"
@@ -13,7 +14,8 @@ import (
 	"github.com/urfave/cli/v2"
 	"gopkg.in/yaml.v2"
 
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/config"
+	"github.com/cloudflare/cloudflared/ipaccess"
 	"github.com/cloudflare/cloudflared/tlsconfig"
 )
 
@@ -61,12 +63,12 @@ ingress:
 			want: []Rule{
 				{
 					Hostname: "tunnel1.example.com",
-					Service:  &localService{URL: localhost8000},
+					Service:  &httpService{url: localhost8000},
 					Config:   defaultConfig,
 				},
 				{
 					Hostname: "*",
-					Service:  &localService{URL: localhost8001},
+					Service:  &httpService{url: localhost8001},
 					Config:   defaultConfig,
 				},
 			},
@@ -82,7 +84,22 @@ extraKey: extraValue
 			want: []Rule{
 				{
 					Hostname: "*",
-					Service:  &localService{URL: localhost8000},
+					Service:  &httpService{url: localhost8000},
+					Config:   defaultConfig,
+				},
+			},
+		},
+		{
+			name: "ws service",
+			args: args{rawYAML: `
+ingress:
+ - hostname: "*"
+   service: wss://localhost:8000
+`},
+			want: []Rule{
+				{
+					Hostname: "*",
+					Service:  &httpService{url: MustParseURL(t, "wss://localhost:8000")},
 					Config:   defaultConfig,
 				},
 			},
@@ -95,7 +112,7 @@ ingress:
 `},
 			want: []Rule{
 				{
-					Service: &localService{URL: localhost8000},
+					Service: &httpService{url: localhost8000},
 					Config:  defaultConfig,
 				},
 			},
@@ -209,6 +226,112 @@ ingress:
 				},
 			},
 		},
+		{
+			name: "TCP services",
+			args: args{rawYAML: `
+ingress:
+- hostname: tcp.foo.com
+  service: tcp://127.0.0.1
+- hostname: tcp2.foo.com
+  service: tcp://localhost:8000
+- service: http_status:404
+`},
+			want: []Rule{
+				{
+					Hostname: "tcp.foo.com",
+					Service:  newTCPOverWSService(MustParseURL(t, "tcp://127.0.0.1:7864")),
+					Config:   defaultConfig,
+				},
+				{
+					Hostname: "tcp2.foo.com",
+					Service:  newTCPOverWSService(MustParseURL(t, "tcp://localhost:8000")),
+					Config:   defaultConfig,
+				},
+				{
+					Service: &fourOhFour,
+					Config:  defaultConfig,
+				},
+			},
+		},
+		{
+			name: "SSH services",
+			args: args{rawYAML: `
+ingress:
+- service: ssh://127.0.0.1
+`},
+			want: []Rule{
+				{
+					Service: newTCPOverWSService(MustParseURL(t, "ssh://127.0.0.1:22")),
+					Config:  defaultConfig,
+				},
+			},
+		},
+		{
+			name: "RDP services",
+			args: args{rawYAML: `
+ingress:
+- service: rdp://127.0.0.1
+`},
+			want: []Rule{
+				{
+					Service: newTCPOverWSService(MustParseURL(t, "rdp://127.0.0.1:3389")),
+					Config:  defaultConfig,
+				},
+			},
+		},
+		{
+			name: "SMB services",
+			args: args{rawYAML: `
+ingress:
+- service: smb://127.0.0.1
+`},
+			want: []Rule{
+				{
+					Service: newTCPOverWSService(MustParseURL(t, "smb://127.0.0.1:445")),
+					Config:  defaultConfig,
+				},
+			},
+		},
+		{
+			name: "Other TCP services",
+			args: args{rawYAML: `
+ingress:
+- service: ftp://127.0.0.1
+`},
+			want: []Rule{
+				{
+					Service: newTCPOverWSService(MustParseURL(t, "ftp://127.0.0.1")),
+					Config:  defaultConfig,
+				},
+			},
+		},
+		{
+			name: "SOCKS services",
+			args: args{rawYAML: `
+ingress:
+- hostname: socks.foo.com
+  service: socks-proxy
+  originRequest:
+    ipRules:
+      - prefix: 1.1.1.0/24
+        ports: [80, 443]
+        allow: true
+      - prefix: 0.0.0.0/0
+        allow: false
+- service: http_status:404
+`},
+			want: []Rule{
+				{
+					Hostname: "socks.foo.com",
+					Service:  newSocksProxyOverWSService(accessPolicy()),
+					Config:   defaultConfig,
+				},
+				{
+					Service: &fourOhFour,
+					Config:  defaultConfig,
+				},
+			},
+		},
 		{
 			name: "URL isn't necessary if using bastion",
 			args: args{rawYAML: `
@@ -221,7 +344,7 @@ ingress:
 			want: []Rule{
 				{
 					Hostname: "bastion.foo.com",
-					Service:  &localService{},
+					Service:  newBastionService(),
 					Config:   setConfig(originRequestFromYAML(config.OriginRequestConfig{}), config.OriginRequestConfig{BastionMode: &tr}),
 				},
 				{
@@ -241,7 +364,7 @@ ingress:
 			want: []Rule{
 				{
 					Hostname: "bastion.foo.com",
-					Service:  &localService{},
+					Service:  newBastionService(),
 					Config:   setConfig(originRequestFromYAML(config.OriginRequestConfig{}), config.OriginRequestConfig{BastionMode: &tr}),
 				},
 				{
@@ -369,6 +492,7 @@ func TestFindMatchingRule(t *testing.T) {
 	tests := []struct {
 		host          string
 		path          string
+		req           *http.Request
 		wantRuleIndex int
 	}{
 		{
@@ -403,9 +527,40 @@ func TestFindMatchingRule(t *testing.T) {
 		},
 	}
 
-	for i, test := range tests {
+	for _, test := range tests {
 		_, ruleIndex := ingress.FindMatchingRule(test.host, test.path)
-		assert.Equal(t, test.wantRuleIndex, ruleIndex, fmt.Sprintf("Expect host=%s, path=%s to match rule %d, got %d", test.host, test.path, test.wantRuleIndex, i))
+		assert.Equal(t, test.wantRuleIndex, ruleIndex, fmt.Sprintf("Expect host=%s, path=%s to match rule %d, got %d", test.host, test.path, test.wantRuleIndex, ruleIndex))
+	}
+}
+
+func TestIsHTTPService(t *testing.T) {
+	tests := []struct {
+		url    *url.URL
+		isHTTP bool
+	}{
+		{
+			url:    MustParseURL(t, "http://localhost"),
+			isHTTP: true,
+		},
+		{
+			url:    MustParseURL(t, "https://127.0.0.1:8000"),
+			isHTTP: true,
+		},
+		{
+			url:    MustParseURL(t, "ws://localhost"),
+			isHTTP: true,
+		},
+		{
+			url:    MustParseURL(t, "wss://localhost:8000"),
+			isHTTP: true,
+		},
+		{
+			url:    MustParseURL(t, "tcp://localhost:9000"),
+			isHTTP: false,
+		},
+	}
+	for _, test := range tests {
+		assert.Equal(t, test.isHTTP, isHTTPService(test.url))
 	}
 }
 
@@ -421,6 +576,16 @@ func MustParseURL(t *testing.T, rawURL string) *url.URL {
 	return u
 }
 
+func accessPolicy() *ipaccess.Policy {
+	cidr1 := "1.1.1.0/24"
+	cidr2 := "0.0.0.0/0"
+	rule1, _ := ipaccess.NewRuleByCIDR(&cidr1, []int{80, 443}, true)
+	rule2, _ := ipaccess.NewRuleByCIDR(&cidr2, nil, false)
+	rules := []ipaccess.Rule{rule1, rule2}
+	accessPolicy, _ := ipaccess.NewPolicy(false, rules)
+	return accessPolicy
+}
+
 func BenchmarkFindMatch(b *testing.B) {
 	rulesYAML := `
 ingress:
@@ -436,6 +601,7 @@ ingress:
 	if err != nil {
 		b.Error(err)
 	}
+
 	for n := 0; n < b.N; n++ {
 		ing.FindMatchingRule("tunnel1.example.com", "")
 		ing.FindMatchingRule("tunnel2.example.com", "")

ingress/origin_connection.go

@@ -0,0 +1,125 @@
+package ingress
+
+import (
+	"context"
+	"crypto/tls"
+	"io"
+	"net"
+	"net/http"
+
+	"github.com/cloudflare/cloudflared/ipaccess"
+	"github.com/cloudflare/cloudflared/socks"
+	"github.com/cloudflare/cloudflared/websocket"
+	gws "github.com/gorilla/websocket"
+	"github.com/rs/zerolog"
+)
+
+// OriginConnection is a way to stream to a service running on the user's origin.
+// Different concrete implementations will stream different protocols as long as they are io.ReadWriters.
+type OriginConnection interface {
+	// Stream should generally be implemented as a bidirectional io.Copy.
+	Stream(ctx context.Context, tunnelConn io.ReadWriter, log *zerolog.Logger)
+	Close()
+}
+
+type streamHandlerFunc func(originConn io.ReadWriter, remoteConn net.Conn, log *zerolog.Logger)
+
+// Stream copies copy data to & from provided io.ReadWriters.
+func Stream(conn, backendConn io.ReadWriter, log *zerolog.Logger) {
+	proxyDone := make(chan struct{}, 2)
+
+	go func() {
+		_, err := io.Copy(conn, backendConn)
+		if err != nil {
+			log.Debug().Msgf("conn to backendConn copy: %v", err)
+		}
+		proxyDone <- struct{}{}
+	}()
+
+	go func() {
+		_, err := io.Copy(backendConn, conn)
+		if err != nil {
+			log.Debug().Msgf("backendConn to conn copy: %v", err)
+		}
+		proxyDone <- struct{}{}
+	}()
+
+	// If one side is done, we are done.
+	<-proxyDone
+}
+
+// DefaultStreamHandler is an implementation of streamHandlerFunc that
+// performs a two way io.Copy between originConn and remoteConn.
+func DefaultStreamHandler(originConn io.ReadWriter, remoteConn net.Conn, log *zerolog.Logger) {
+	Stream(originConn, remoteConn, log)
+}
+
+// tcpConnection is an OriginConnection that directly streams to raw TCP.
+type tcpConnection struct {
+	conn net.Conn
+}
+
+func (tc *tcpConnection) Stream(ctx context.Context, tunnelConn io.ReadWriter, log *zerolog.Logger) {
+	Stream(tunnelConn, tc.conn, log)
+}
+
+func (tc *tcpConnection) Close() {
+	tc.conn.Close()
+}
+
+// tcpOverWSConnection is an OriginConnection that streams to TCP over WS.
+type tcpOverWSConnection struct {
+	conn          net.Conn
+	streamHandler streamHandlerFunc
+}
+
+func (wc *tcpOverWSConnection) Stream(ctx context.Context, tunnelConn io.ReadWriter, log *zerolog.Logger) {
+	wc.streamHandler(websocket.NewConn(ctx, tunnelConn, log), wc.conn, log)
+}
+
+func (wc *tcpOverWSConnection) Close() {
+	wc.conn.Close()
+}
+
+// wsConnection is an OriginConnection that streams WS between eyeball and origin.
+type wsConnection struct {
+	wsConn *gws.Conn
+	resp   *http.Response
+}
+
+func (wsc *wsConnection) Stream(ctx context.Context, tunnelConn io.ReadWriter, log *zerolog.Logger) {
+	Stream(tunnelConn, wsc.wsConn.UnderlyingConn(), log)
+}
+
+func (wsc *wsConnection) Close() {
+	wsc.resp.Body.Close()
+	wsc.wsConn.Close()
+}
+
+func newWSConnection(clientTLSConfig *tls.Config, r *http.Request) (OriginConnection, *http.Response, error) {
+	d := &gws.Dialer{
+		TLSClientConfig: clientTLSConfig,
+	}
+	wsConn, resp, err := websocket.ClientConnect(r, d)
+	if err != nil {
+		return nil, nil, err
+	}
+	return &wsConnection{
+		wsConn,
+		resp,
+	}, resp, nil
+}
+
+// socksProxyOverWSConnection is an OriginConnection that streams SOCKS connections over WS.
+// The connection to the origin happens inside the SOCKS code as the client specifies the origin
+// details in the packet.
+type socksProxyOverWSConnection struct {
+	accessPolicy *ipaccess.Policy
+}
+
+func (sp *socksProxyOverWSConnection) Stream(ctx context.Context, tunnelConn io.ReadWriter, log *zerolog.Logger) {
+	socks.StreamNetHandler(websocket.NewConn(ctx, tunnelConn, log), sp.accessPolicy, log)
+}
+
+func (sp *socksProxyOverWSConnection) Close() {
+}

ingress/origin_connection_test.go

@@ -0,0 +1,294 @@
+package ingress
+
+import (
+	"bytes"
+	"context"
+	"crypto/tls"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"testing"
+	"time"
+
+	"github.com/cloudflare/cloudflared/logger"
+	"github.com/cloudflare/cloudflared/socks"
+	"github.com/gobwas/ws/wsutil"
+	gorillaWS "github.com/gorilla/websocket"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/net/proxy"
+	"golang.org/x/sync/errgroup"
+)
+
+const (
+	testStreamTimeout = time.Second * 3
+	echoHeaderName    = "Test-Cloudflared-Echo"
+)
+
+var (
+	testLogger   = logger.Create(nil)
+	testMessage  = []byte("TestStreamOriginConnection")
+	testResponse = []byte(fmt.Sprintf("echo-%s", testMessage))
+)
+
+func TestStreamTCPConnection(t *testing.T) {
+	cfdConn, originConn := net.Pipe()
+	tcpConn := tcpConnection{
+		conn: cfdConn,
+	}
+
+	eyeballConn, edgeConn := net.Pipe()
+
+	ctx, cancel := context.WithTimeout(context.Background(), testStreamTimeout)
+	defer cancel()
+
+	errGroup, ctx := errgroup.WithContext(ctx)
+	errGroup.Go(func() error {
+		_, err := eyeballConn.Write(testMessage)
+
+		readBuffer := make([]byte, len(testResponse))
+		_, err = eyeballConn.Read(readBuffer)
+		require.NoError(t, err)
+
+		require.Equal(t, testResponse, readBuffer)
+
+		return nil
+	})
+	errGroup.Go(func() error {
+		echoTCPOrigin(t, originConn)
+		originConn.Close()
+		return nil
+	})
+
+	tcpConn.Stream(ctx, edgeConn, testLogger)
+	require.NoError(t, errGroup.Wait())
+}
+
+func TestDefaultStreamWSOverTCPConnection(t *testing.T) {
+	cfdConn, originConn := net.Pipe()
+	tcpOverWSConn := tcpOverWSConnection{
+		conn:          cfdConn,
+		streamHandler: DefaultStreamHandler,
+	}
+
+	eyeballConn, edgeConn := net.Pipe()
+
+	ctx, cancel := context.WithTimeout(context.Background(), testStreamTimeout)
+	defer cancel()
+
+	errGroup, ctx := errgroup.WithContext(ctx)
+	errGroup.Go(func() error {
+		echoWSEyeball(t, eyeballConn)
+		return nil
+	})
+	errGroup.Go(func() error {
+		echoTCPOrigin(t, originConn)
+		originConn.Close()
+		return nil
+	})
+
+	tcpOverWSConn.Stream(ctx, edgeConn, testLogger)
+	require.NoError(t, errGroup.Wait())
+}
+
+// TestSocksStreamWSOverTCPConnection simulates proxying in socks mode.
+// Eyeball side runs cloudflared accesss tcp with --url flag to start a websocket forwarder which
+// wraps SOCKS5 traffic in websocket
+// Origin side runs a tcpOverWSConnection with socks.StreamHandler
+func TestSocksStreamWSOverTCPConnection(t *testing.T) {
+	var (
+		sendMessage             = t.Name()
+		echoHeaderIncomingValue = fmt.Sprintf("header-%s", sendMessage)
+		echoMessage             = fmt.Sprintf("echo-%s", sendMessage)
+		echoHeaderReturnValue   = fmt.Sprintf("echo-%s", echoHeaderIncomingValue)
+	)
+
+	statusCodes := []int{
+		http.StatusOK,
+		http.StatusTemporaryRedirect,
+		http.StatusBadRequest,
+		http.StatusInternalServerError,
+	}
+	for _, status := range statusCodes {
+		handler := func(w http.ResponseWriter, r *http.Request) {
+			body, err := ioutil.ReadAll(r.Body)
+			require.NoError(t, err)
+			require.Equal(t, []byte(sendMessage), body)
+
+			require.Equal(t, echoHeaderIncomingValue, r.Header.Get(echoHeaderName))
+			w.Header().Set(echoHeaderName, echoHeaderReturnValue)
+
+			w.WriteHeader(status)
+			w.Write([]byte(echoMessage))
+		}
+		origin := httptest.NewServer(http.HandlerFunc(handler))
+		defer origin.Close()
+
+		originURL, err := url.Parse(origin.URL)
+		require.NoError(t, err)
+
+		originConn, err := net.Dial("tcp", originURL.Host)
+		require.NoError(t, err)
+
+		tcpOverWSConn := tcpOverWSConnection{
+			conn:          originConn,
+			streamHandler: socks.StreamHandler,
+		}
+
+		wsForwarderOutConn, edgeConn := net.Pipe()
+		ctx, cancel := context.WithTimeout(context.Background(), testStreamTimeout)
+		defer cancel()
+
+		errGroup, ctx := errgroup.WithContext(ctx)
+		errGroup.Go(func() error {
+			tcpOverWSConn.Stream(ctx, edgeConn, testLogger)
+			return nil
+		})
+
+		wsForwarderListener, err := net.Listen("tcp", "127.0.0.1:0")
+		require.NoError(t, err)
+
+		errGroup.Go(func() error {
+			wsForwarderInConn, err := wsForwarderListener.Accept()
+			require.NoError(t, err)
+			defer wsForwarderInConn.Close()
+
+			Stream(wsForwarderInConn, &wsEyeball{wsForwarderOutConn}, testLogger)
+			return nil
+		})
+
+		eyeballDialer, err := proxy.SOCKS5("tcp", wsForwarderListener.Addr().String(), nil, proxy.Direct)
+		require.NoError(t, err)
+
+		transport := &http.Transport{
+			Dial: eyeballDialer.Dial,
+		}
+
+		// Request URL doesn't matter because the transport is using eyeballDialer to connectq
+		req, err := http.NewRequestWithContext(ctx, "GET", "http://test-socks-stream.com", bytes.NewBuffer([]byte(sendMessage)))
+		assert.NoError(t, err)
+		req.Header.Set(echoHeaderName, echoHeaderIncomingValue)
+
+		resp, err := transport.RoundTrip(req)
+		assert.NoError(t, err)
+		assert.Equal(t, status, resp.StatusCode)
+		require.Equal(t, echoHeaderReturnValue, resp.Header.Get(echoHeaderName))
+		body, err := ioutil.ReadAll(resp.Body)
+		require.NoError(t, err)
+		require.Equal(t, []byte(echoMessage), body)
+
+		wsForwarderOutConn.Close()
+		edgeConn.Close()
+		tcpOverWSConn.Close()
+
+		require.NoError(t, errGroup.Wait())
+	}
+}
+
+func TestStreamWSConnection(t *testing.T) {
+	eyeballConn, edgeConn := net.Pipe()
+
+	origin := echoWSOrigin(t)
+	defer origin.Close()
+
+	req, err := http.NewRequest(http.MethodGet, origin.URL, nil)
+	require.NoError(t, err)
+	req.Header.Set("Sec-Websocket-Key", "dGhlIHNhbXBsZSBub25jZQ==")
+
+	clientTLSConfig := &tls.Config{
+		InsecureSkipVerify: true,
+	}
+	wsConn, resp, err := newWSConnection(clientTLSConfig, req)
+	require.NoError(t, err)
+	require.Equal(t, http.StatusSwitchingProtocols, resp.StatusCode)
+	require.Equal(t, "Upgrade", resp.Header.Get("Connection"))
+	require.Equal(t, "s3pPLMBiTxaQ9kYGzzhZRbK+xOo=", resp.Header.Get("Sec-Websocket-Accept"))
+	require.Equal(t, "websocket", resp.Header.Get("Upgrade"))
+
+	ctx, cancel := context.WithTimeout(context.Background(), testStreamTimeout)
+	defer cancel()
+
+	errGroup, ctx := errgroup.WithContext(ctx)
+	errGroup.Go(func() error {
+		echoWSEyeball(t, eyeballConn)
+		return nil
+	})
+
+	wsConn.Stream(ctx, edgeConn, testLogger)
+	require.NoError(t, errGroup.Wait())
+}
+
+type wsEyeball struct {
+	conn net.Conn
+}
+
+func (wse *wsEyeball) Read(p []byte) (int, error) {
+	data, err := wsutil.ReadServerBinary(wse.conn)
+	if err != nil {
+		return 0, err
+	}
+	return copy(p, data), nil
+}
+
+func (wse *wsEyeball) Write(p []byte) (int, error) {
+	err := wsutil.WriteClientBinary(wse.conn, p)
+	return len(p), err
+}
+
+func echoWSEyeball(t *testing.T, conn net.Conn) {
+	require.NoError(t, wsutil.WriteClientBinary(conn, testMessage))
+
+	readMsg, err := wsutil.ReadServerBinary(conn)
+	require.NoError(t, err)
+
+	require.Equal(t, testResponse, readMsg)
+
+	require.NoError(t, conn.Close())
+}
+
+func echoWSOrigin(t *testing.T) *httptest.Server {
+	var upgrader = gorillaWS.Upgrader{
+		ReadBufferSize:  10,
+		WriteBufferSize: 10,
+	}
+
+	ws := func(w http.ResponseWriter, r *http.Request) {
+		header := make(http.Header)
+		for k, vs := range r.Header {
+			if k == "Test-Cloudflared-Echo" {
+				header[k] = vs
+			}
+		}
+		conn, err := upgrader.Upgrade(w, r, header)
+		require.NoError(t, err)
+		defer conn.Close()
+
+		for {
+			messageType, p, err := conn.ReadMessage()
+			if err != nil {
+				return
+			}
+			require.Equal(t, testMessage, p)
+			if err := conn.WriteMessage(messageType, testResponse); err != nil {
+				return
+			}
+		}
+	}
+
+	// NewTLSServer starts the server in another thread
+	return httptest.NewTLSServer(http.HandlerFunc(ws))
+}
+
+func echoTCPOrigin(t *testing.T, conn net.Conn) {
+	readBuffer := make([]byte, len(testMessage))
+	_, err := conn.Read(readBuffer)
+	assert.NoError(t, err)
+
+	assert.Equal(t, testMessage, readBuffer)
+
+	_, err = conn.Write(testResponse)
+	assert.NoError(t, err)
+}

ingress/origin_proxy.go

@@ -0,0 +1,158 @@
+package ingress
+
+import (
+	"fmt"
+	"net"
+	"net/http"
+	"net/url"
+	"strings"
+
+	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/websocket"
+	"github.com/pkg/errors"
+)
+
+var (
+	switchingProtocolText = fmt.Sprintf("%d %s", http.StatusSwitchingProtocols, http.StatusText(http.StatusSwitchingProtocols))
+)
+
+// HTTPOriginProxy can be implemented by origin services that want to proxy http requests.
+type HTTPOriginProxy interface {
+	// RoundTrip is how cloudflared proxies eyeball requests to the actual origin services
+	http.RoundTripper
+}
+
+// StreamBasedOriginProxy can be implemented by origin services that want to proxy ws/TCP.
+type StreamBasedOriginProxy interface {
+	EstablishConnection(r *http.Request) (OriginConnection, *http.Response, error)
+}
+
+func (o *unixSocketPath) RoundTrip(req *http.Request) (*http.Response, error) {
+	return o.transport.RoundTrip(req)
+}
+
+func (o *httpService) RoundTrip(req *http.Request) (*http.Response, error) {
+	// Rewrite the request URL so that it goes to the origin service.
+	req.URL.Host = o.url.Host
+	req.URL.Scheme = o.url.Scheme
+	if o.hostHeader != "" {
+		// For incoming requests, the Host header is promoted to the Request.Host field and removed from the Header map.
+		req.Host = o.hostHeader
+	}
+	return o.transport.RoundTrip(req)
+}
+
+func (o *httpService) EstablishConnection(req *http.Request) (OriginConnection, *http.Response, error) {
+	req.URL.Host = o.url.Host
+	req.URL.Scheme = websocket.ChangeRequestScheme(o.url)
+	if o.hostHeader != "" {
+		// For incoming requests, the Host header is promoted to the Request.Host field and removed from the Header map.
+		req.Host = o.hostHeader
+	}
+	return newWSConnection(o.transport.TLSClientConfig, req)
+}
+
+func (o *helloWorld) RoundTrip(req *http.Request) (*http.Response, error) {
+	// Rewrite the request URL so that it goes to the Hello World server.
+	req.URL.Host = o.server.Addr().String()
+	req.URL.Scheme = "https"
+	return o.transport.RoundTrip(req)
+}
+
+func (o *helloWorld) EstablishConnection(req *http.Request) (OriginConnection, *http.Response, error) {
+	req.URL.Host = o.server.Addr().String()
+	req.URL.Scheme = "wss"
+	return newWSConnection(o.transport.TLSClientConfig, req)
+}
+
+func (o *statusCode) RoundTrip(_ *http.Request) (*http.Response, error) {
+	return o.resp, nil
+}
+
+func (o *rawTCPService) EstablishConnection(r *http.Request) (OriginConnection, *http.Response, error) {
+	dest, err := getRequestHost(r)
+	if err != nil {
+		return nil, nil, err
+	}
+	conn, err := net.Dial("tcp", dest)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	originConn := &tcpConnection{
+		conn: conn,
+	}
+	resp := &http.Response{
+		Status:        switchingProtocolText,
+		StatusCode:    http.StatusSwitchingProtocols,
+		ContentLength: -1,
+	}
+	return originConn, resp, nil
+}
+
+// getRequestHost returns the host of the http.Request.
+func getRequestHost(r *http.Request) (string, error) {
+	if r.Host != "" {
+		return r.Host, nil
+	}
+	if r.URL != nil {
+		return r.URL.Host, nil
+	}
+	return "", errors.New("host not found")
+}
+
+func (o *tcpOverWSService) EstablishConnection(r *http.Request) (OriginConnection, *http.Response, error) {
+	var err error
+	dest := o.dest
+	if o.isBastion {
+		dest, err = o.bastionDest(r)
+		if err != nil {
+			return nil, nil, err
+		}
+	}
+
+	conn, err := net.Dial("tcp", dest)
+	if err != nil {
+		return nil, nil, err
+	}
+	originConn := &tcpOverWSConnection{
+		conn:          conn,
+		streamHandler: o.streamHandler,
+	}
+	resp := &http.Response{
+		Status:        switchingProtocolText,
+		StatusCode:    http.StatusSwitchingProtocols,
+		Header:        websocket.NewResponseHeader(r),
+		ContentLength: -1,
+	}
+	return originConn, resp, nil
+
+}
+
+func (o *tcpOverWSService) bastionDest(r *http.Request) (string, error) {
+	jumpDestination := r.Header.Get(h2mux.CFJumpDestinationHeader)
+	if jumpDestination == "" {
+		return "", fmt.Errorf("Did not receive final destination from client. The --destination flag is likely not set on the client side")
+	}
+	// Strip scheme and path set by client. Without a scheme
+	// Parsing a hostname and path without scheme might not return an error due to parsing ambiguities
+	if jumpURL, err := url.Parse(jumpDestination); err == nil && jumpURL.Host != "" {
+		return removePath(jumpURL.Host), nil
+	}
+	return removePath(jumpDestination), nil
+}
+
+func removePath(dest string) string {
+	return strings.SplitN(dest, "/", 2)[0]
+}
+
+func (o *socksProxyOverWSService) EstablishConnection(r *http.Request) (OriginConnection, *http.Response, error) {
+	originConn := o.conn
+	resp := &http.Response{
+		Status:        switchingProtocolText,
+		StatusCode:    http.StatusSwitchingProtocols,
+		Header:        websocket.NewResponseHeader(r),
+		ContentLength: -1,
+	}
+	return originConn, resp, nil
+}

ingress/origin_proxy_test.go

@@ -0,0 +1,330 @@
+package ingress
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"sync"
+	"testing"
+
+	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/websocket"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestEstablishConnectionResponse ensures each implementation of StreamBasedOriginProxy returns
+// the expected response
+func assertEstablishConnectionResponse(t *testing.T,
+	originProxy StreamBasedOriginProxy,
+	req *http.Request,
+	expectHeader http.Header,
+) {
+	_, resp, err := originProxy.EstablishConnection(req)
+	assert.NoError(t, err)
+	assert.Equal(t, switchingProtocolText, resp.Status)
+	assert.Equal(t, http.StatusSwitchingProtocols, resp.StatusCode)
+	assert.Equal(t, expectHeader, resp.Header)
+}
+
+func TestHTTPServiceEstablishConnection(t *testing.T) {
+	origin := echoWSOrigin(t)
+	defer origin.Close()
+	originURL, err := url.Parse(origin.URL)
+	require.NoError(t, err)
+
+	httpService := &httpService{
+		url:        originURL,
+		hostHeader: origin.URL,
+		transport: &http.Transport{
+			TLSClientConfig: &tls.Config{
+				InsecureSkipVerify: true,
+			},
+		},
+	}
+	req, err := http.NewRequest(http.MethodGet, origin.URL, nil)
+	require.NoError(t, err)
+	req.Header.Set("Sec-Websocket-Key", "dGhlIHNhbXBsZSBub25jZQ==")
+	req.Header.Set("Test-Cloudflared-Echo", t.Name())
+
+	expectHeader := http.Header{
+		"Connection":            {"Upgrade"},
+		"Sec-Websocket-Accept":  {"s3pPLMBiTxaQ9kYGzzhZRbK+xOo="},
+		"Upgrade":               {"websocket"},
+		"Test-Cloudflared-Echo": {t.Name()},
+	}
+	assertEstablishConnectionResponse(t, httpService, req, expectHeader)
+}
+
+func TestHelloWorldEstablishConnection(t *testing.T) {
+	var wg sync.WaitGroup
+	shutdownC := make(chan struct{})
+	errC := make(chan error)
+	helloWorldSerivce := &helloWorld{}
+	helloWorldSerivce.start(&wg, testLogger, shutdownC, errC, OriginRequestConfig{})
+
+	// Scheme and Host of URL will be override by the Scheme and Host of the helloWorld service
+	req, err := http.NewRequest(http.MethodGet, "https://place-holder/ws", nil)
+	require.NoError(t, err)
+
+	expectHeader := http.Header{
+		"Connection": {"Upgrade"},
+		// Accept key when Sec-Websocket-Key is not specified
+		"Sec-Websocket-Accept": {"Kfh9QIsMVZcl6xEPYxPHzW8SZ8w="},
+		"Upgrade":              {"websocket"},
+	}
+	assertEstablishConnectionResponse(t, helloWorldSerivce, req, expectHeader)
+
+	close(shutdownC)
+}
+
+func TestRawTCPServiceEstablishConnection(t *testing.T) {
+	originListener, err := net.Listen("tcp", "127.0.0.1:0")
+	require.NoError(t, err)
+
+	listenerClosed := make(chan struct{})
+	tcpListenRoutine(originListener, listenerClosed)
+
+	rawTCPService := &rawTCPService{name: ServiceWarpRouting}
+
+	req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://%s", originListener.Addr()), nil)
+	require.NoError(t, err)
+
+	assertEstablishConnectionResponse(t, rawTCPService, req, nil)
+
+	originListener.Close()
+	<-listenerClosed
+
+	req, err = http.NewRequest(http.MethodGet, fmt.Sprintf("http://%s", originListener.Addr()), nil)
+	require.NoError(t, err)
+
+	// Origin not listening for new connection, should return an error
+	_, resp, err := rawTCPService.EstablishConnection(req)
+	require.Error(t, err)
+	require.Nil(t, resp)
+}
+
+func TestTCPOverWSServiceEstablishConnection(t *testing.T) {
+	originListener, err := net.Listen("tcp", "127.0.0.1:0")
+	require.NoError(t, err)
+
+	listenerClosed := make(chan struct{})
+	tcpListenRoutine(originListener, listenerClosed)
+
+	originURL := &url.URL{
+		Scheme: "tcp",
+		Host:   originListener.Addr().String(),
+	}
+
+	baseReq, err := http.NewRequest(http.MethodGet, "https://place-holder", nil)
+	require.NoError(t, err)
+	baseReq.Header.Set("Sec-Websocket-Key", "dGhlIHNhbXBsZSBub25jZQ==")
+
+	bastionReq := baseReq.Clone(context.Background())
+	bastionReq.Header.Set(h2mux.CFJumpDestinationHeader, originListener.Addr().String())
+
+	expectHeader := http.Header{
+		"Connection":           {"Upgrade"},
+		"Sec-Websocket-Accept": {"s3pPLMBiTxaQ9kYGzzhZRbK+xOo="},
+		"Upgrade":              {"websocket"},
+	}
+
+	tests := []struct {
+		service   *tcpOverWSService
+		req       *http.Request
+		expectErr bool
+	}{
+		{
+			service: newTCPOverWSService(originURL),
+			req:     baseReq,
+		},
+		{
+			service: newBastionService(),
+			req:     bastionReq,
+		},
+		{
+			service:   newBastionService(),
+			req:       baseReq,
+			expectErr: true,
+		},
+	}
+
+	for _, test := range tests {
+		if test.expectErr {
+			_, resp, err := test.service.EstablishConnection(test.req)
+			assert.Error(t, err)
+			assert.Nil(t, resp)
+		} else {
+			assertEstablishConnectionResponse(t, test.service, test.req, expectHeader)
+		}
+	}
+
+	originListener.Close()
+	<-listenerClosed
+
+	for _, service := range []*tcpOverWSService{newTCPOverWSService(originURL), newBastionService()} {
+		// Origin not listening for new connection, should return an error
+		_, resp, err := service.EstablishConnection(bastionReq)
+		assert.Error(t, err)
+		assert.Nil(t, resp)
+	}
+}
+
+func TestBastionDestination(t *testing.T) {
+	canonicalJumpDestHeader := http.CanonicalHeaderKey(h2mux.CFJumpDestinationHeader)
+	tests := []struct {
+		name         string
+		header       http.Header
+		expectedDest string
+		wantErr      bool
+	}{
+		{
+			name: "hostname destination",
+			header: http.Header{
+				canonicalJumpDestHeader: []string{"localhost"},
+			},
+			expectedDest: "localhost",
+		},
+		{
+			name: "hostname destination with port",
+			header: http.Header{
+				canonicalJumpDestHeader: []string{"localhost:9000"},
+			},
+			expectedDest: "localhost:9000",
+		},
+		{
+			name: "hostname destination with scheme and port",
+			header: http.Header{
+				canonicalJumpDestHeader: []string{"ssh://localhost:9000"},
+			},
+			expectedDest: "localhost:9000",
+		},
+		{
+			name: "full hostname url",
+			header: http.Header{
+				canonicalJumpDestHeader: []string{"ssh://localhost:9000/metrics"},
+			},
+			expectedDest: "localhost:9000",
+		},
+		{
+			name: "hostname destination with port and path",
+			header: http.Header{
+				canonicalJumpDestHeader: []string{"localhost:9000/metrics"},
+			},
+			expectedDest: "localhost:9000",
+		},
+		{
+			name: "ip destination",
+			header: http.Header{
+				canonicalJumpDestHeader: []string{"127.0.0.1"},
+			},
+			expectedDest: "127.0.0.1",
+		},
+		{
+			name: "ip destination with port",
+			header: http.Header{
+				canonicalJumpDestHeader: []string{"127.0.0.1:9000"},
+			},
+			expectedDest: "127.0.0.1:9000",
+		},
+		{
+			name: "ip destination with port and path",
+			header: http.Header{
+				canonicalJumpDestHeader: []string{"127.0.0.1:9000/metrics"},
+			},
+			expectedDest: "127.0.0.1:9000",
+		},
+		{
+			name: "ip destination with schem and port",
+			header: http.Header{
+				canonicalJumpDestHeader: []string{"tcp://127.0.0.1:9000"},
+			},
+			expectedDest: "127.0.0.1:9000",
+		},
+		{
+			name: "full ip url",
+			header: http.Header{
+				canonicalJumpDestHeader: []string{"ssh://127.0.0.1:9000/metrics"},
+			},
+			expectedDest: "127.0.0.1:9000",
+		},
+		{
+			name:    "no destination",
+			wantErr: true,
+		},
+	}
+	s := newBastionService()
+	for _, test := range tests {
+		r := &http.Request{
+			Header: test.header,
+		}
+		dest, err := s.bastionDest(r)
+		if test.wantErr {
+			assert.Error(t, err, "Test %s expects error", test.name)
+		} else {
+			assert.NoError(t, err, "Test %s expects no error, got error %v", test.name, err)
+			assert.Equal(t, test.expectedDest, dest, "Test %s expect dest %s, got %s", test.name, test.expectedDest, dest)
+		}
+	}
+}
+
+func TestHTTPServiceHostHeaderOverride(t *testing.T) {
+	cfg := OriginRequestConfig{
+		HTTPHostHeader: t.Name(),
+	}
+	handler := func(w http.ResponseWriter, r *http.Request) {
+		require.Equal(t, r.Host, t.Name())
+		if websocket.IsWebSocketUpgrade(r) {
+			respHeaders := websocket.NewResponseHeader(r)
+			for k, v := range respHeaders {
+				w.Header().Set(k, v[0])
+			}
+			w.WriteHeader(http.StatusSwitchingProtocols)
+			return
+		}
+		w.Write([]byte("ok"))
+	}
+	origin := httptest.NewServer(http.HandlerFunc(handler))
+	defer origin.Close()
+
+	originURL, err := url.Parse(origin.URL)
+	require.NoError(t, err)
+
+	httpService := &httpService{
+		url: originURL,
+	}
+	var wg sync.WaitGroup
+	shutdownC := make(chan struct{})
+	errC := make(chan error)
+	require.NoError(t, httpService.start(&wg, testLogger, shutdownC, errC, cfg))
+
+	req, err := http.NewRequest(http.MethodGet, originURL.String(), nil)
+	require.NoError(t, err)
+
+	resp, err := httpService.RoundTrip(req)
+	require.NoError(t, err)
+	require.Equal(t, http.StatusOK, resp.StatusCode)
+
+	req = req.Clone(context.Background())
+	_, resp, err = httpService.EstablishConnection(req)
+	require.NoError(t, err)
+	require.Equal(t, http.StatusSwitchingProtocols, resp.StatusCode)
+}
+
+func tcpListenRoutine(listener net.Listener, closeChan chan struct{}) {
+	go func() {
+		for {
+			conn, err := listener.Accept()
+			if err != nil {
+				close(closeChan)
+				return
+			}
+			// Close immediately, this test is not about testing read/write on connection
+			conn.Close()
+		}
+	}()
+}

ingress/origin_request_config.go

@@ -3,9 +3,11 @@ package ingress
 import (
 	"time"
 
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
-	"github.com/cloudflare/cloudflared/tlsconfig"
+	"github.com/cloudflare/cloudflared/ipaccess"
 	"github.com/urfave/cli/v2"
+
+	"github.com/cloudflare/cloudflared/config"
+	"github.com/cloudflare/cloudflared/tlsconfig"
 )
 
 const (
@@ -212,6 +214,8 @@ type OriginRequestConfig struct {
 	ProxyPort uint `yaml:"proxyPort"`
 	// What sort of proxy should be started
 	ProxyType string `yaml:"proxyType"`
+	// IP rules for the proxy service
+	IPRules []ipaccess.Rule `yaml:"ipRules"`
 }
 
 func (defaults *OriginRequestConfig) setConnectTimeout(overrides config.OriginRequestConfig) {

ingress/origin_request_config_test.go

@@ -5,7 +5,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/config"
 	"github.com/stretchr/testify/require"
 	"github.com/urfave/cli/v2"
 	"gopkg.in/yaml.v2"

ingress/origin_service.go

@@ -8,11 +8,11 @@ import (
 	"net"
 	"net/http"
 	"net/url"
-	"strconv"
 	"sync"
 	"time"
 
 	"github.com/cloudflare/cloudflared/hello"
+	"github.com/cloudflare/cloudflared/ipaccess"
 	"github.com/cloudflare/cloudflared/socks"
 	"github.com/cloudflare/cloudflared/tlsconfig"
 	"github.com/cloudflare/cloudflared/websocket"
@@ -21,10 +21,8 @@ import (
 	"github.com/rs/zerolog"
 )
 
-// OriginService is something a tunnel can proxy traffic to.
-type OriginService interface {
-	// RoundTrip is how cloudflared proxies eyeball requests to the actual origin services
-	http.RoundTripper
+// originService is something a tunnel can proxy traffic to.
+type originService interface {
 	String() string
 	// Start the origin service if it's managed by cloudflared, e.g. proxy servers or Hello World.
 	// If it's not managed by cloudflared, this is a no-op because the user is responsible for
@@ -51,10 +49,6 @@ func (o *unixSocketPath) start(wg *sync.WaitGroup, log *zerolog.Logger, shutdown
 	return nil
 }
 
-func (o *unixSocketPath) RoundTrip(req *http.Request) (*http.Response, error) {
-	return o.transport.RoundTrip(req)
-}
-
 func (o *unixSocketPath) Dial(reqURL *url.URL, headers http.Header) (*gws.Conn, *http.Response, error) {
 	d := &gws.Dialer{
 		NetDial:         o.transport.Dial,
@@ -65,130 +59,112 @@ func (o *unixSocketPath) Dial(reqURL *url.URL, headers http.Header) (*gws.Conn,
 	return d.Dial(reqURL.String(), headers)
 }
 
-// localService is an OriginService listening on a TCP/IP address the user's origin can route to.
-type localService struct {
-	// The URL for the user's origin service
-	RootURL *url.URL
-	// The URL that cloudflared should send requests to.
-	// If this origin requires starting a proxy, this is the proxy's address,
-	// and that proxy points to RootURL. Otherwise, this is equal to RootURL.
-	URL       *url.URL
-	transport *http.Transport
+type httpService struct {
+	url        *url.URL
+	hostHeader string
+	transport  *http.Transport
 }
 
-func (o *localService) Dial(reqURL *url.URL, headers http.Header) (*gws.Conn, *http.Response, error) {
-	d := &gws.Dialer{TLSClientConfig: o.transport.TLSClientConfig}
-	// Rewrite the request URL so that it goes to the origin service.
-	reqURL.Host = o.URL.Host
-	reqURL.Scheme = websocket.ChangeRequestScheme(o.URL)
-	return d.Dial(reqURL.String(), headers)
-}
-
-func (o *localService) start(wg *sync.WaitGroup, log *zerolog.Logger, shutdownC <-chan struct{}, errC chan error, cfg OriginRequestConfig) error {
+func (o *httpService) start(wg *sync.WaitGroup, log *zerolog.Logger, shutdownC <-chan struct{}, errC chan error, cfg OriginRequestConfig) error {
 	transport, err := newHTTPTransport(o, cfg, log)
 	if err != nil {
 		return err
 	}
+	o.hostHeader = cfg.HTTPHostHeader
 	o.transport = transport
-
-	// Start a proxy if one is needed
-	if staticHost := o.staticHost(); originRequiresProxy(staticHost, cfg) {
-		if err := o.startProxy(staticHost, wg, log, shutdownC, errC, cfg); err != nil {
-			return err
-		}
-	}
-
 	return nil
 }
 
-func (o *localService) startProxy(staticHost string, wg *sync.WaitGroup, log *zerolog.Logger, shutdownC <-chan struct{}, errC chan error, cfg OriginRequestConfig) error {
+func (o *httpService) String() string {
+	return o.url.String()
+}
 
-	// Start a listener for the proxy
-	proxyAddress := net.JoinHostPort(cfg.ProxyAddress, strconv.Itoa(int(cfg.ProxyPort)))
-	listener, err := net.Listen("tcp", proxyAddress)
-	if err != nil {
-		log.Error().Msgf("Cannot start Websocket Proxy Server: %s", err)
-		return errors.Wrap(err, "Cannot start Websocket Proxy Server")
-	}
+// rawTCPService dials TCP to the destination specified by the client
+// It's used by warp routing
+type rawTCPService struct {
+	name string
+}
 
-	// Start the proxy itself
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
-		streamHandler := websocket.DefaultStreamHandler
-		// This origin's config specifies what type of proxy to start.
-		switch cfg.ProxyType {
-		case socksProxy:
-			log.Info().Msg("SOCKS5 server started")
-			streamHandler = func(wsConn *websocket.Conn, remoteConn net.Conn, _ http.Header) {
-				dialer := socks.NewConnDialer(remoteConn)
-				requestHandler := socks.NewRequestHandler(dialer)
-				socksServer := socks.NewConnectionHandler(requestHandler)
+func (o *rawTCPService) String() string {
+	return o.name
+}
 
-				_ = socksServer.Serve(wsConn)
-			}
-		case "":
-			log.Debug().Msg("Not starting any websocket proxy")
-		default:
-			log.Error().Msgf("%s isn't a valid proxy (valid options are {%s})", cfg.ProxyType, socksProxy)
-		}
-
-		errC <- websocket.StartProxyServer(log, listener, staticHost, shutdownC, streamHandler)
-	}()
-
-	// Modify this origin, so that it no longer points at the origin service directly.
-	// Instead, it points at the proxy to the origin service.
-	newURL, err := url.Parse("http://" + listener.Addr().String())
-	if err != nil {
-		return err
-	}
-	o.URL = newURL
+func (o *rawTCPService) start(wg *sync.WaitGroup, log *zerolog.Logger, shutdownC <-chan struct{}, errC chan error, cfg OriginRequestConfig) error {
 	return nil
 }
 
-func (o *localService) String() string {
-	if o.isBastion() {
-		return "Bastion"
-	}
-	return o.URL.String()
+// tcpOverWSService models TCP origins serving eyeballs connecting over websocket, such as
+// cloudflared access commands.
+type tcpOverWSService struct {
+	dest          string
+	isBastion     bool
+	streamHandler streamHandlerFunc
 }
 
-func (o *localService) isBastion() bool {
-	return o.URL == nil
+type socksProxyOverWSService struct {
+	conn *socksProxyOverWSConnection
 }
 
-func (o *localService) RoundTrip(req *http.Request) (*http.Response, error) {
-	// Rewrite the request URL so that it goes to the origin service.
-	req.URL.Host = o.URL.Host
-	req.URL.Scheme = o.URL.Scheme
-	return o.transport.RoundTrip(req)
-}
-
-func (o *localService) staticHost() string {
-
-	if o.URL == nil {
-		return ""
-	}
-
-	addPortIfMissing := func(uri *url.URL, port int) string {
-		if uri.Port() != "" {
-			return uri.Host
-		}
-		return fmt.Sprintf("%s:%d", uri.Hostname(), port)
-	}
-
-	switch o.URL.Scheme {
+func newTCPOverWSService(url *url.URL) *tcpOverWSService {
+	switch url.Scheme {
 	case "ssh":
-		return addPortIfMissing(o.URL, 22)
+		addPortIfMissing(url, 22)
 	case "rdp":
-		return addPortIfMissing(o.URL, 3389)
+		addPortIfMissing(url, 3389)
 	case "smb":
-		return addPortIfMissing(o.URL, 445)
+		addPortIfMissing(url, 445)
 	case "tcp":
-		return addPortIfMissing(o.URL, 7864) // just a random port since there isn't a default in this case
+		addPortIfMissing(url, 7864) // just a random port since there isn't a default in this case
 	}
-	return ""
+	return &tcpOverWSService{
+		dest: url.Host,
+	}
+}
 
+func newBastionService() *tcpOverWSService {
+	return &tcpOverWSService{
+		isBastion: true,
+	}
+}
+
+func newSocksProxyOverWSService(accessPolicy *ipaccess.Policy) *socksProxyOverWSService {
+	proxy := socksProxyOverWSService{
+		conn: &socksProxyOverWSConnection{
+			accessPolicy: accessPolicy,
+		},
+	}
+
+	return &proxy
+}
+
+func addPortIfMissing(uri *url.URL, port int) {
+	if uri.Port() == "" {
+		uri.Host = fmt.Sprintf("%s:%d", uri.Hostname(), port)
+	}
+}
+
+func (o *tcpOverWSService) String() string {
+	if o.isBastion {
+		return ServiceBastion
+	}
+	return o.dest
+}
+
+func (o *tcpOverWSService) start(wg *sync.WaitGroup, log *zerolog.Logger, shutdownC <-chan struct{}, errC chan error, cfg OriginRequestConfig) error {
+	if cfg.ProxyType == socksProxy {
+		o.streamHandler = socks.StreamHandler
+	} else {
+		o.streamHandler = DefaultStreamHandler
+	}
+	return nil
+}
+
+func (o *socksProxyOverWSService) start(wg *sync.WaitGroup, log *zerolog.Logger, shutdownC <-chan struct{}, errC chan error, cfg OriginRequestConfig) error {
+	return nil
+}
+
+func (o *socksProxyOverWSService) String() string {
+	return ServiceSocksProxy
 }
 
 // HelloWorld is an OriginService for the built-in Hello World server.
@@ -228,26 +204,6 @@ func (o *helloWorld) start(
 	return nil
 }
 
-func (o *helloWorld) RoundTrip(req *http.Request) (*http.Response, error) {
-	// Rewrite the request URL so that it goes to the Hello World server.
-	req.URL.Host = o.server.Addr().String()
-	req.URL.Scheme = "https"
-	return o.transport.RoundTrip(req)
-}
-
-func (o *helloWorld) Dial(reqURL *url.URL, headers http.Header) (*gws.Conn, *http.Response, error) {
-	d := &gws.Dialer{
-		TLSClientConfig: o.transport.TLSClientConfig,
-	}
-	reqURL.Host = o.server.Addr().String()
-	reqURL.Scheme = "wss"
-	return d.Dial(reqURL.String(), headers)
-}
-
-func originRequiresProxy(staticHost string, cfg OriginRequestConfig) bool {
-	return staticHost != "" || cfg.BastionMode
-}
-
 // statusCode is an OriginService that just responds with a given HTTP status.
 // Typical use-case is "user wants the catch-all rule to just respond 404".
 type statusCode struct {
@@ -277,10 +233,6 @@ func (o *statusCode) start(
 	return nil
 }
 
-func (o *statusCode) RoundTrip(_ *http.Request) (*http.Response, error) {
-	return o.resp, nil
-}
-
 type NopReadCloser struct{}
 
 // Read always returns EOF to signal end of input
@@ -292,7 +244,7 @@ func (nrc *NopReadCloser) Close() error {
 	return nil
 }
 
-func newHTTPTransport(service OriginService, cfg OriginRequestConfig, log *zerolog.Logger) (*http.Transport, error) {
+func newHTTPTransport(service originService, cfg OriginRequestConfig, log *zerolog.Logger) (*http.Transport, error) {
 	originCertPool, err := tlsconfig.LoadOriginCA(cfg.CAPool, log)
 	if err != nil {
 		return nil, errors.Wrap(err, "Error loading cert pool")
@@ -337,19 +289,19 @@ func newHTTPTransport(service OriginService, cfg OriginRequestConfig, log *zerol
 	return &httpTransport, nil
 }
 
-// MockOriginService should only be used by other packages to mock OriginService. Set Transport to configure desired RoundTripper behavior.
-type MockOriginService struct {
+// MockOriginHTTPService should only be used by other packages to mock OriginService. Set Transport to configure desired RoundTripper behavior.
+type MockOriginHTTPService struct {
 	Transport http.RoundTripper
 }
 
-func (mos MockOriginService) RoundTrip(req *http.Request) (*http.Response, error) {
+func (mos MockOriginHTTPService) RoundTrip(req *http.Request) (*http.Response, error) {
 	return mos.Transport.RoundTrip(req)
 }
 
-func (mos MockOriginService) String() string {
+func (mos MockOriginHTTPService) String() string {
 	return "MockOriginService"
 }
 
-func (mos MockOriginService) start(wg *sync.WaitGroup, log *zerolog.Logger, shutdownC <-chan struct{}, errC chan error, cfg OriginRequestConfig) error {
+func (mos MockOriginHTTPService) start(wg *sync.WaitGroup, log *zerolog.Logger, shutdownC <-chan struct{}, errC chan error, cfg OriginRequestConfig) error {
 	return nil
 }

ingress/rule.go

@@ -17,7 +17,7 @@ type Rule struct {
 	// A (probably local) address. Requests for a hostname which matches this
 	// rule's hostname pattern will be proxied to the service running on this
 	// address.
-	Service OriginService
+	Service originService
 
 	// Configure the request cloudflared sends to this specific origin.
 	Config OriginRequestConfig

ingress/rule_test.go

@@ -14,7 +14,7 @@ func Test_rule_matches(t *testing.T) {
 	type fields struct {
 		Hostname string
 		Path     *regexp.Regexp
-		Service  OriginService
+		Service  originService
 	}
 	type args struct {
 		requestURL *url.URL

ipaccess/access.go

@@ -0,0 +1,101 @@
+package ipaccess
+
+import (
+	"fmt"
+	"net"
+	"sort"
+)
+
+type Policy struct {
+	defaultAllow bool
+	rules        []Rule
+}
+
+type Rule struct {
+	ipNet *net.IPNet
+	ports []int
+	allow bool
+}
+
+func NewPolicy(defaultAllow bool, rules []Rule) (*Policy, error) {
+	for _, rule := range rules {
+		if err := rule.Validate(); err != nil {
+			return nil, err
+		}
+	}
+
+	policy := Policy{
+		defaultAllow: defaultAllow,
+		rules:        rules,
+	}
+
+	return &policy, nil
+}
+
+func NewRuleByCIDR(prefix *string, ports []int, allow bool) (Rule, error) {
+	if prefix == nil || len(*prefix) == 0 {
+		return Rule{}, fmt.Errorf("no prefix provided")
+	}
+
+	_, ipnet, err := net.ParseCIDR(*prefix)
+	if err != nil {
+		return Rule{}, fmt.Errorf("unable to parse cidr: %s", *prefix)
+	}
+
+	return NewRule(ipnet, ports, allow)
+}
+
+func NewRule(ipnet *net.IPNet, ports []int, allow bool) (Rule, error) {
+	rule := Rule{
+		ipNet: ipnet,
+		ports: ports,
+		allow: allow,
+	}
+	return rule, rule.Validate()
+}
+
+func (r *Rule) Validate() error {
+	if r.ipNet == nil {
+		return fmt.Errorf("no ipnet set on the rule")
+	}
+
+	if len(r.ports) > 0 {
+		sort.Ints(r.ports)
+		for _, port := range r.ports {
+			if port < 1 || port > 65535 {
+				return fmt.Errorf("invalid port %d, needs to be between 1 and 65535", port)
+			}
+		}
+	}
+
+	return nil
+}
+
+func (h *Policy) Allowed(ip net.IP, port int) (bool, *Rule) {
+	if len(h.rules) == 0 {
+		return h.defaultAllow, nil
+	}
+
+	for _, rule := range h.rules {
+		if rule.ipNet.Contains(ip) {
+			if len(rule.ports) == 0 {
+				return rule.allow, &rule
+			} else if pos := sort.SearchInts(rule.ports, port); pos < len(rule.ports) && rule.ports[pos] == port {
+				return rule.allow, &rule
+			}
+		}
+	}
+
+	return h.defaultAllow, nil
+}
+
+func (ipr *Rule) String() string {
+	return fmt.Sprintf("prefix:%s/port:%s/allow:%t", ipr.ipNet, ipr.PortsString(), ipr.allow)
+}
+
+func (ipr *Rule) PortsString() string {
+	if len(ipr.ports) > 0 {
+		return fmt.Sprint(ipr.ports)
+	}
+	return "all"
+}

ipaccess/access_test.go

@@ -0,0 +1,107 @@
+package ipaccess
+
+import (
+	"bytes"
+	"net"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRuleCreation(t *testing.T) {
+	_, ipnet, _ := net.ParseCIDR("1.1.1.1/24")
+
+	_, err := NewRule(nil, []int{80}, false)
+	assert.Error(t, err, "expected error as no ipnet provided")
+
+	_, err = NewRule(ipnet, []int{65536, 80}, false)
+	assert.Error(t, err, "expected error as port higher than 65535")
+
+	_, err = NewRule(ipnet, []int{80, -1}, false)
+	assert.Error(t, err, "expected error as port less than 0")
+
+	rule, err := NewRule(ipnet, []int{443, 80}, false)
+	assert.NoError(t, err)
+	assert.True(t, ipnet.IP.Equal(rule.ipNet.IP) && bytes.Compare(ipnet.Mask, rule.ipNet.Mask) == 0, "ipnet expected to be %+v, got: %+v", ipnet, rule.ipNet)
+	assert.True(t, len(rule.ports) == 2 && rule.ports[0] == 80 && rule.ports[1] == 443, "expected ports to be sorted")
+}
+
+func TestRuleCreationByCIDR(t *testing.T) {
+	var cidr *string
+	_, err := NewRuleByCIDR(cidr, []int{80}, false)
+	assert.Error(t, err, "expected error as cidr is nil")
+
+	badCidr := "1.1.1.1"
+	cidr = &badCidr
+	_, err = NewRuleByCIDR(cidr, []int{80}, false)
+	assert.Error(t, err, "expected error as the cidr is bad")
+
+	goodCidr := "1.1.1.1/24"
+	_, ipnet, _ := net.ParseCIDR("1.1.1.0/24")
+	cidr = &goodCidr
+	rule, err := NewRuleByCIDR(cidr, []int{80}, false)
+	assert.NoError(t, err)
+	assert.True(t, ipnet.IP.Equal(rule.ipNet.IP) && bytes.Compare(ipnet.Mask, rule.ipNet.Mask) == 0, "ipnet expected to be %+v, got: %+v", ipnet, rule.ipNet)
+}
+
+func TestRulesNoRules(t *testing.T) {
+	ip, _, _ := net.ParseCIDR("1.2.3.4/24")
+
+	policy, _ := NewPolicy(true, []Rule{})
+
+	allowed, rule := policy.Allowed(ip, 80)
+	assert.True(t, allowed, "expected to be allowed as no rules and default allow")
+	assert.Nil(t, rule, "expected to be nil as no rules")
+
+	policy, _ = NewPolicy(false, []Rule{})
+
+	allowed, rule = policy.Allowed(ip, 80)
+	assert.False(t, allowed, "expected to be denied as no rules and default deny")
+	assert.Nil(t, rule, "expected to be nil as no rules")
+}
+
+func TestRulesMatchIPAndPort(t *testing.T) {
+	ip1, ipnet1, _ := net.ParseCIDR("1.2.3.4/24")
+	ip2, _, _ := net.ParseCIDR("2.3.4.5/24")
+
+	rule1, _ := NewRule(ipnet1, []int{80, 443}, true)
+	rules := []Rule{
+		rule1,
+	}
+
+	policy, _ := NewPolicy(false, rules)
+
+	allowed, rule := policy.Allowed(ip1, 80)
+	assert.True(t, allowed, "expected to be allowed as matching rule")
+	assert.True(t, rule.ipNet == ipnet1, "expected to match ipnet1")
+
+	allowed, rule = policy.Allowed(ip2, 80)
+	assert.False(t, allowed, "expected to be denied as no matching rule")
+	assert.Nil(t, rule, "expected to be nil")
+}
+
+func TestRulesMatchIPAndPort2(t *testing.T) {
+	ip1, ipnet1, _ := net.ParseCIDR("1.2.3.4/24")
+	ip2, ipnet2, _ := net.ParseCIDR("2.3.4.5/24")
+
+	rule1, _ := NewRule(ipnet1, []int{53, 80}, false)
+	rule2, _ := NewRule(ipnet2, []int{53, 80}, true)
+	rules := []Rule{
+		rule1,
+		rule2,
+	}
+
+	policy, _ := NewPolicy(false, rules)
+
+	allowed, rule := policy.Allowed(ip1, 80)
+	assert.False(t, allowed, "expected to be denied as matching rule")
+	assert.True(t, rule.ipNet == ipnet1, "expected to match ipnet1")
+
+	allowed, rule = policy.Allowed(ip2, 80)
+	assert.True(t, allowed, "expected to be allowed as matching rule")
+	assert.True(t, rule.ipNet == ipnet2, "expected to match ipnet1")
+
+	allowed, rule = policy.Allowed(ip2, 81)
+	assert.False(t, allowed, "expected to be denied as no matching rule")
+	assert.Nil(t, rule, "expected to be nil")
+}

origin/backoffhandler.go

@@ -2,6 +2,7 @@ package origin
 
 import (
 	"context"
+	"math/rand"
 	"time"
 )
 
@@ -30,7 +31,7 @@ type BackoffHandler struct {
 	resetDeadline time.Time
 }
 
-func (b BackoffHandler) GetBackoffDuration(ctx context.Context) (time.Duration, bool) {
+func (b BackoffHandler) GetMaxBackoffDuration(ctx context.Context) (time.Duration, bool) {
 	// Follows the same logic as Backoff, but without mutating the receiver.
 	// This select has to happen first to reflect the actual behaviour of the Backoff function.
 	select {
@@ -45,7 +46,8 @@ func (b BackoffHandler) GetBackoffDuration(ctx context.Context) (time.Duration,
 	if b.retries >= b.MaxRetries && !b.RetryForever {
 		return time.Duration(0), false
 	}
-	return time.Duration(b.GetBaseTime() * 1 << b.retries), true
+	maxTimeToWait := b.GetBaseTime() * 1 << (b.retries + 1)
+	return maxTimeToWait, true
 }
 
 // BackoffTimer returns a channel that sends the current time when the exponential backoff timeout expires.
@@ -62,7 +64,9 @@ func (b *BackoffHandler) BackoffTimer() <-chan time.Time {
 	} else {
 		b.retries++
 	}
-	return timeAfter(time.Duration(b.GetBaseTime() * 1 << (b.retries - 1)))
+	maxTimeToWait := time.Duration(b.GetBaseTime() * 1 << (b.retries))
+	timeToWait := time.Duration(rand.Int63n(maxTimeToWait.Nanoseconds()))
+	return timeAfter(timeToWait)
 }
 
 // Backoff is used to wait according to exponential backoff. Returns false if the
@@ -83,7 +87,9 @@ func (b *BackoffHandler) Backoff(ctx context.Context) bool {
 // Sets a grace period within which the the backoff timer is maintained. After the grace
 // period expires, the number of retries & backoff duration is reset.
 func (b *BackoffHandler) SetGracePeriod() {
-	b.resetDeadline = timeNow().Add(time.Duration(b.GetBaseTime() * 2 << b.retries))
+	maxTimeToWait := b.GetBaseTime() * 2 << (b.retries + 1)
+	timeToWait := time.Duration(rand.Int63n(maxTimeToWait.Nanoseconds()))
+	b.resetDeadline = timeNow().Add(timeToWait)
 }
 
 func (b BackoffHandler) GetBaseTime() time.Duration {

origin/backoffhandler_test.go

@@ -40,7 +40,7 @@ func TestBackoffCancel(t *testing.T) {
 	if backoff.Backoff(ctx) {
 		t.Fatalf("backoff allowed after cancel")
 	}
-	if _, ok := backoff.GetBackoffDuration(ctx); ok {
+	if _, ok := backoff.GetMaxBackoffDuration(ctx); ok {
 		t.Fatalf("backoff allowed after cancel")
 	}
 }
@@ -69,24 +69,24 @@ func TestBackoffGracePeriod(t *testing.T) {
 	}
 }
 
-func TestGetBackoffDurationRetries(t *testing.T) {
+func TestGetMaxBackoffDurationRetries(t *testing.T) {
 	// make backoff return immediately
 	timeAfter = immediateTimeAfter
 	ctx := context.Background()
 	backoff := BackoffHandler{MaxRetries: 3}
-	if _, ok := backoff.GetBackoffDuration(ctx); !ok {
+	if _, ok := backoff.GetMaxBackoffDuration(ctx); !ok {
 		t.Fatalf("backoff failed immediately")
 	}
 	backoff.Backoff(ctx) // noop
-	if _, ok := backoff.GetBackoffDuration(ctx); !ok {
+	if _, ok := backoff.GetMaxBackoffDuration(ctx); !ok {
 		t.Fatalf("backoff failed after 1 retry")
 	}
 	backoff.Backoff(ctx) // noop
-	if _, ok := backoff.GetBackoffDuration(ctx); !ok {
+	if _, ok := backoff.GetMaxBackoffDuration(ctx); !ok {
 		t.Fatalf("backoff failed after 2 retry")
 	}
 	backoff.Backoff(ctx) // noop
-	if _, ok := backoff.GetBackoffDuration(ctx); ok {
+	if _, ok := backoff.GetMaxBackoffDuration(ctx); ok {
 		t.Fatalf("backoff allowed after 3 (max) retries")
 	}
 	if backoff.Backoff(ctx) {
@@ -94,25 +94,25 @@ func TestGetBackoffDurationRetries(t *testing.T) {
 	}
 }
 
-func TestGetBackoffDuration(t *testing.T) {
+func TestGetMaxBackoffDuration(t *testing.T) {
 	// make backoff return immediately
 	timeAfter = immediateTimeAfter
 	ctx := context.Background()
 	backoff := BackoffHandler{MaxRetries: 3}
-	if duration, ok := backoff.GetBackoffDuration(ctx); !ok || duration != time.Second {
-		t.Fatalf("backoff didn't return 1 second on first retry")
+	if duration, ok := backoff.GetMaxBackoffDuration(ctx); !ok || duration > time.Second*2 {
+		t.Fatalf("backoff (%s) didn't return < 2 seconds on first retry", duration)
 	}
 	backoff.Backoff(ctx) // noop
-	if duration, ok := backoff.GetBackoffDuration(ctx); !ok || duration != time.Second*2 {
-		t.Fatalf("backoff didn't return 2 seconds on second retry")
+	if duration, ok := backoff.GetMaxBackoffDuration(ctx); !ok || duration > time.Second*4 {
+		t.Fatalf("backoff (%s) didn't return < 4 seconds on second retry", duration)
 	}
 	backoff.Backoff(ctx) // noop
-	if duration, ok := backoff.GetBackoffDuration(ctx); !ok || duration != time.Second*4 {
-		t.Fatalf("backoff didn't return 4 seconds on third retry")
+	if duration, ok := backoff.GetMaxBackoffDuration(ctx); !ok || duration > time.Second*8 {
+		t.Fatalf("backoff (%s) didn't return < 8 seconds on third retry", duration)
 	}
 	backoff.Backoff(ctx) // noop
-	if duration, ok := backoff.GetBackoffDuration(ctx); ok || duration != 0 {
-		t.Fatalf("backoff didn't return 0 seconds on fourth retry (exceeding limit)")
+	if duration, ok := backoff.GetMaxBackoffDuration(ctx); ok || duration != 0 {
+		t.Fatalf("backoff (%s) didn't return 0 seconds on fourth retry (exceeding limit)", duration)
 	}
 }
 
@@ -121,27 +121,27 @@ func TestBackoffRetryForever(t *testing.T) {
 	timeAfter = immediateTimeAfter
 	ctx := context.Background()
 	backoff := BackoffHandler{MaxRetries: 3, RetryForever: true}
-	if duration, ok := backoff.GetBackoffDuration(ctx); !ok || duration != time.Second {
-		t.Fatalf("backoff didn't return 1 second on first retry")
+	if duration, ok := backoff.GetMaxBackoffDuration(ctx); !ok || duration > time.Second*2 {
+		t.Fatalf("backoff (%s) didn't return < 2 seconds on first retry", duration)
 	}
 	backoff.Backoff(ctx) // noop
-	if duration, ok := backoff.GetBackoffDuration(ctx); !ok || duration != time.Second*2 {
-		t.Fatalf("backoff didn't return 2 seconds on second retry")
+	if duration, ok := backoff.GetMaxBackoffDuration(ctx); !ok || duration > time.Second*4 {
+		t.Fatalf("backoff (%s) didn't return < 4 seconds on second retry", duration)
 	}
 	backoff.Backoff(ctx) // noop
-	if duration, ok := backoff.GetBackoffDuration(ctx); !ok || duration != time.Second*4 {
-		t.Fatalf("backoff didn't return 4 seconds on third retry")
+	if duration, ok := backoff.GetMaxBackoffDuration(ctx); !ok || duration > time.Second*8 {
+		t.Fatalf("backoff (%s) didn't return < 8 seconds on third retry", duration)
 	}
 	if !backoff.Backoff(ctx) {
 		t.Fatalf("backoff refused on fourth retry despire RetryForever")
 	}
-	if duration, ok := backoff.GetBackoffDuration(ctx); !ok || duration != time.Second*8 {
+	if duration, ok := backoff.GetMaxBackoffDuration(ctx); !ok || duration > time.Second*16 {
 		t.Fatalf("backoff returned %v instead of 8 seconds on fourth retry", duration)
 	}
 	if !backoff.Backoff(ctx) {
 		t.Fatalf("backoff refused on fifth retry despire RetryForever")
 	}
-	if duration, ok := backoff.GetBackoffDuration(ctx); !ok || duration != time.Second*8 {
+	if duration, ok := backoff.GetMaxBackoffDuration(ctx); !ok || duration > time.Second*16 {
 		t.Fatalf("backoff returned %v instead of 8 seconds on fifth retry", duration)
 	}
 }

origin/pool.go

@@ -1,17 +1,17 @@
-package buffer
+package origin
 
 import (
 	"sync"
 )
 
-type Pool struct {
-	// A Pool must not be copied after first use.
+type bufferPool struct {
+	// A bufferPool must not be copied after first use.
 	// https://golang.org/pkg/sync/#Pool
 	buffers sync.Pool
 }
 
-func NewPool(bufferSize int) *Pool {
-	return &Pool{
+func newBufferPool(bufferSize int) *bufferPool {
+	return &bufferPool{
 		buffers: sync.Pool{
 			New: func() interface{} {
 				return make([]byte, bufferSize)
@@ -20,10 +20,10 @@ func NewPool(bufferSize int) *Pool {
 	}
 }
 
-func (p *Pool) Get() []byte {
+func (p *bufferPool) Get() []byte {
 	return p.buffers.Get().([]byte)
 }
 
-func (p *Pool) Put(buf []byte) {
+func (p *bufferPool) Put(buf []byte) {
 	p.buffers.Put(buf)
 }

origin/proxy.go

@@ -5,71 +5,105 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"net"
 	"net/http"
 	"strconv"
 	"strings"
 
-	"github.com/cloudflare/cloudflared/buffer"
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog"
+
 	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/ingress"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
-	"github.com/cloudflare/cloudflared/websocket"
-
-	"github.com/pkg/errors"
-	"github.com/rs/zerolog"
 )
 
 const (
 	TagHeaderNamePrefix = "Cf-Warp-Tag-"
 )
 
-type client struct {
+type proxy struct {
 	ingressRules ingress.Ingress
+	warpRouting  *ingress.WarpRoutingService
 	tags         []tunnelpogs.Tag
 	log          *zerolog.Logger
-	bufferPool   *buffer.Pool
+	bufferPool   *bufferPool
 }
 
-func NewClient(ingressRules ingress.Ingress, tags []tunnelpogs.Tag, log *zerolog.Logger) connection.OriginClient {
-	return &client{
+func NewOriginProxy(
+	ingressRules ingress.Ingress,
+	warpRouting *ingress.WarpRoutingService,
+	tags []tunnelpogs.Tag,
+	log *zerolog.Logger) connection.OriginProxy {
+
+	return &proxy{
 		ingressRules: ingressRules,
+		warpRouting:  warpRouting,
 		tags:         tags,
 		log:          log,
-		bufferPool:   buffer.NewPool(512 * 1024),
+		bufferPool:   newBufferPool(512 * 1024),
 	}
 }
 
-func (c *client) Proxy(w connection.ResponseWriter, req *http.Request, isWebsocket bool) error {
+// Caller is responsible for writing any error to ResponseWriter
+func (p *proxy) Proxy(w connection.ResponseWriter, req *http.Request, sourceConnectionType connection.Type) error {
 	incrementRequests()
 	defer decrementConcurrentRequests()
 
 	cfRay := findCfRayHeader(req)
 	lbProbe := isLBProbeRequest(req)
 
-	c.appendTagHeaders(req)
-	rule, ruleNum := c.ingressRules.FindMatchingRule(req.Host, req.URL.Path)
-	c.logRequest(req, cfRay, lbProbe, ruleNum)
+	serveCtx, cancel := context.WithCancel(req.Context())
+	defer cancel()
 
-	var (
-		resp *http.Response
-		err  error
-	)
-	if isWebsocket {
-		resp, err = c.proxyWebsocket(w, req, rule)
-	} else {
-		resp, err = c.proxyHTTP(w, req, rule)
+	p.appendTagHeaders(req)
+	if sourceConnectionType == connection.TypeTCP {
+		if p.warpRouting == nil {
+			err := errors.New(`cloudflared received a request from WARP client, but your configuration has disabled ingress from WARP clients. To enable this, set "warp-routing:\n\t enabled: true" in your config.yaml`)
+			p.log.Error().Msg(err.Error())
+			return err
+		}
+		logFields := logFields{
+			cfRay:   cfRay,
+			lbProbe: lbProbe,
+			rule:    ingress.ServiceWarpRouting,
+		}
+		if err := p.proxyStreamRequest(serveCtx, w, req, sourceConnectionType, p.warpRouting.Proxy, logFields); err != nil {
+			p.logRequestError(err, cfRay, ingress.ServiceWarpRouting)
+			return err
+		}
+		return nil
 	}
-	if err != nil {
-		c.logRequestError(err, cfRay, ruleNum)
-		w.WriteErrorResponse()
+
+	rule, ruleNum := p.ingressRules.FindMatchingRule(req.Host, req.URL.Path)
+	logFields := logFields{
+		cfRay:   cfRay,
+		lbProbe: lbProbe,
+		rule:    ruleNum,
+	}
+	p.logRequest(req, logFields)
+
+	if sourceConnectionType == connection.TypeHTTP {
+		if err := p.proxyHTTPRequest(w, req, rule, logFields); err != nil {
+			p.logRequestError(err, cfRay, ruleNum)
+			return err
+		}
+		return nil
+	}
+
+	connectionProxy, ok := rule.Service.(ingress.StreamBasedOriginProxy)
+	if !ok {
+		p.log.Error().Msgf("%s is not a connection-oriented service", rule.Service)
+		return fmt.Errorf("Not a connection-oriented service")
+	}
+
+	if err := p.proxyStreamRequest(serveCtx, w, req, sourceConnectionType, connectionProxy, logFields); err != nil {
+		p.logRequestError(err, cfRay, ruleNum)
 		return err
 	}
-	c.logOriginResponse(resp, cfRay, lbProbe, ruleNum)
 	return nil
 }
 
-func (c *client) proxyHTTP(w connection.ResponseWriter, req *http.Request, rule *ingress.Rule) (*http.Response, error) {
+func (p *proxy) proxyHTTPRequest(w connection.ResponseWriter, req *http.Request, rule *ingress.Rule, fields logFields) error {
 	// Support for WSGI Servers by switching transfer encoding from chunked to gzip/deflate
 	if rule.Config.DisableChunkedEncoding {
 		req.TransferEncoding = []string{"gzip", "deflate"}
@@ -82,78 +116,90 @@ func (c *client) proxyHTTP(w connection.ResponseWriter, req *http.Request, rule
 	// Request origin to keep connection alive to improve performance
 	req.Header.Set("Connection", "keep-alive")
 
-	if hostHeader := rule.Config.HTTPHostHeader; hostHeader != "" {
-		req.Header.Set("Host", hostHeader)
-		req.Host = hostHeader
+	httpService, ok := rule.Service.(ingress.HTTPOriginProxy)
+	if !ok {
+		p.log.Error().Msgf("%s is not a http service", rule.Service)
+		return fmt.Errorf("Not a http service")
 	}
 
-	resp, err := rule.Service.RoundTrip(req)
+	resp, err := httpService.RoundTrip(req)
 	if err != nil {
-		return nil, errors.Wrap(err, "Error proxying request to origin")
+		return errors.Wrap(err, "Error proxying request to origin")
 	}
 	defer resp.Body.Close()
 
-	err = w.WriteRespHeaders(resp)
+	err = w.WriteRespHeaders(resp.StatusCode, resp.Header)
 	if err != nil {
-		return nil, errors.Wrap(err, "Error writing response header")
+		return errors.Wrap(err, "Error writing response header")
 	}
 	if connection.IsServerSentEvent(resp.Header) {
-		c.log.Debug().Msg("Detected Server-Side Events from Origin")
-		c.writeEventStream(w, resp.Body)
+		p.log.Debug().Msg("Detected Server-Side Events from Origin")
+		p.writeEventStream(w, resp.Body)
 	} else {
 		// Use CopyBuffer, because Copy only allocates a 32KiB buffer, and cross-stream
 		// compression generates dictionary on first write
-		buf := c.bufferPool.Get()
-		defer c.bufferPool.Put(buf)
+		buf := p.bufferPool.Get()
+		defer p.bufferPool.Put(buf)
 		_, _ = io.CopyBuffer(w, resp.Body, buf)
 	}
-	return resp, nil
-}
-
-func (c *client) proxyWebsocket(w connection.ResponseWriter, req *http.Request, rule *ingress.Rule) (*http.Response, error) {
-	if hostHeader := rule.Config.HTTPHostHeader; hostHeader != "" {
-		req.Header.Set("Host", hostHeader)
-		req.Host = hostHeader
-	}
-
-	dialler, ok := rule.Service.(websocket.Dialler)
-	if !ok {
-		return nil, fmt.Errorf("Websockets aren't supported by the origin service '%s'", rule.Service)
-	}
-	conn, resp, err := websocket.ClientConnect(req, dialler)
-	if err != nil {
-		return nil, err
-	}
-
-	serveCtx, cancel := context.WithCancel(req.Context())
-	connClosedChan := make(chan struct{})
-	go func() {
-		// serveCtx is done if req is cancelled, or streamWebsocket returns
-		<-serveCtx.Done()
-		_ = conn.Close()
-		close(connClosedChan)
-	}()
-
-	// Copy to/from stream to the undelying connection. Use the underlying
-	// connection because cloudflared doesn't operate on the message themselves
-	err = c.streamWebsocket(w, conn.UnderlyingConn(), resp)
-	cancel()
-
-	// We need to make sure conn is closed before returning, otherwise we might write to conn after Proxy returns
-	<-connClosedChan
-	return resp, err
-}
-
-func (c *client) streamWebsocket(w connection.ResponseWriter, conn net.Conn, resp *http.Response) error {
-	err := w.WriteRespHeaders(resp)
-	if err != nil {
-		return errors.Wrap(err, "Error writing websocket response header")
-	}
-	websocket.Stream(conn, w)
+	p.logOriginResponse(resp, fields)
 	return nil
 }
 
-func (c *client) writeEventStream(w connection.ResponseWriter, respBody io.ReadCloser) {
+// proxyStreamRequest first establish a connection with origin, then it writes the status code and headers, and finally it streams data between
+// eyeball and origin.
+func (p *proxy) proxyStreamRequest(
+	serveCtx context.Context,
+	w connection.ResponseWriter,
+	req *http.Request,
+	sourceConnectionType connection.Type,
+	connectionProxy ingress.StreamBasedOriginProxy,
+	fields logFields,
+) error {
+	originConn, resp, err := connectionProxy.EstablishConnection(req)
+	if err != nil {
+		return err
+	}
+	if resp.Body != nil {
+		defer resp.Body.Close()
+	}
+
+	if err = w.WriteRespHeaders(resp.StatusCode, resp.Header); err != nil {
+		return err
+	}
+
+	streamCtx, cancel := context.WithCancel(serveCtx)
+	defer cancel()
+
+	go func() {
+		// streamCtx is done if req is cancelled or if Stream returns
+		<-streamCtx.Done()
+		originConn.Close()
+	}()
+
+	eyeballStream := &bidirectionalStream{
+		writer: w,
+		reader: req.Body,
+	}
+	originConn.Stream(serveCtx, eyeballStream, p.log)
+	p.logOriginResponse(resp, fields)
+	return nil
+}
+
+type bidirectionalStream struct {
+	reader io.Reader
+	writer io.Writer
+}
+
+func (wr *bidirectionalStream) Read(p []byte) (n int, err error) {
+	return wr.reader.Read(p)
+}
+
+func (wr *bidirectionalStream) Write(p []byte) (n int, err error) {
+	return wr.writer.Write(p)
+}
+
+func (p *proxy) writeEventStream(w connection.ResponseWriter, respBody io.ReadCloser) {
 	reader := bufio.NewReader(respBody)
 	for {
 		line, err := reader.ReadBytes('\n')
@@ -164,56 +210,61 @@ func (c *client) writeEventStream(w connection.ResponseWriter, respBody io.ReadC
 	}
 }
 
-func (c *client) appendTagHeaders(r *http.Request) {
-	for _, tag := range c.tags {
+func (p *proxy) appendTagHeaders(r *http.Request) {
+	for _, tag := range p.tags {
 		r.Header.Add(TagHeaderNamePrefix+tag.Name, tag.Value)
 	}
 }
 
-func (c *client) logRequest(r *http.Request, cfRay string, lbProbe bool, ruleNum int) {
-	if cfRay != "" {
-		c.log.Debug().Msgf("CF-RAY: %s %s %s %s", cfRay, r.Method, r.URL, r.Proto)
-	} else if lbProbe {
-		c.log.Debug().Msgf("CF-RAY: %s Load Balancer health check %s %s %s", cfRay, r.Method, r.URL, r.Proto)
+type logFields struct {
+	cfRay   string
+	lbProbe bool
+	rule    interface{}
+}
+
+func (p *proxy) logRequest(r *http.Request, fields logFields) {
+	if fields.cfRay != "" {
+		p.log.Debug().Msgf("CF-RAY: %s %s %s %s", fields.cfRay, r.Method, r.URL, r.Proto)
+	} else if fields.lbProbe {
+		p.log.Debug().Msgf("CF-RAY: %s Load Balancer health check %s %s %s", fields.cfRay, r.Method, r.URL, r.Proto)
 	} else {
-		c.log.Debug().Msgf("All requests should have a CF-RAY header. Please open a support ticket with Cloudflare. %s %s %s ", r.Method, r.URL, r.Proto)
+		p.log.Debug().Msgf("All requests should have a CF-RAY header. Please open a support ticket with Cloudflare. %s %s %s ", r.Method, r.URL, r.Proto)
 	}
-	c.log.Debug().Msgf("CF-RAY: %s Request Headers %+v", cfRay, r.Header)
-	c.log.Debug().Msgf("CF-RAY: %s Serving with ingress rule %d", cfRay, ruleNum)
+	p.log.Debug().Msgf("CF-RAY: %s Request Headers %+v", fields.cfRay, r.Header)
+	p.log.Debug().Msgf("CF-RAY: %s Serving with ingress rule %v", fields.cfRay, fields.rule)
 
 	if contentLen := r.ContentLength; contentLen == -1 {
-		c.log.Debug().Msgf("CF-RAY: %s Request Content length unknown", cfRay)
+		p.log.Debug().Msgf("CF-RAY: %s Request Content length unknown", fields.cfRay)
 	} else {
-		c.log.Debug().Msgf("CF-RAY: %s Request content length %d", cfRay, contentLen)
+		p.log.Debug().Msgf("CF-RAY: %s Request content length %d", fields.cfRay, contentLen)
 	}
 }
 
-func (c *client) logOriginResponse(r *http.Response, cfRay string, lbProbe bool, ruleNum int) {
-	responseByCode.WithLabelValues(strconv.Itoa(r.StatusCode)).Inc()
-	if cfRay != "" {
-		c.log.Debug().Msgf("CF-RAY: %s Status: %s served by ingress %d", cfRay, r.Status, ruleNum)
-	} else if lbProbe {
-		c.log.Debug().Msgf("Response to Load Balancer health check %s", r.Status)
+func (p *proxy) logOriginResponse(resp *http.Response, fields logFields) {
+	responseByCode.WithLabelValues(strconv.Itoa(resp.StatusCode)).Inc()
+	if fields.cfRay != "" {
+		p.log.Debug().Msgf("CF-RAY: %s Status: %s served by ingress %d", fields.cfRay, resp.Status, fields.rule)
+	} else if fields.lbProbe {
+		p.log.Debug().Msgf("Response to Load Balancer health check %s", resp.Status)
 	} else {
-		c.log.Debug().Msgf("Status: %s served by ingress %d", r.Status, ruleNum)
+		p.log.Debug().Msgf("Status: %s served by ingress %v", resp.Status, fields.rule)
 	}
-	c.log.Debug().Msgf("CF-RAY: %s Response Headers %+v", cfRay, r.Header)
+	p.log.Debug().Msgf("CF-RAY: %s Response Headers %+v", fields.cfRay, resp.Header)
 
-	if contentLen := r.ContentLength; contentLen == -1 {
-		c.log.Debug().Msgf("CF-RAY: %s Response content length unknown", cfRay)
+	if contentLen := resp.ContentLength; contentLen == -1 {
+		p.log.Debug().Msgf("CF-RAY: %s Response content length unknown", fields.cfRay)
 	} else {
-		c.log.Debug().Msgf("CF-RAY: %s Response content length %d", cfRay, contentLen)
+		p.log.Debug().Msgf("CF-RAY: %s Response content length %d", fields.cfRay, contentLen)
 	}
 }
 
-func (c *client) logRequestError(err error, cfRay string, ruleNum int) {
+func (p *proxy) logRequestError(err error, cfRay string, rule interface{}) {
 	requestErrors.Inc()
 	if cfRay != "" {
-		c.log.Error().Msgf("CF-RAY: %s Proxying to ingress %d error: %v", cfRay, ruleNum, err)
+		p.log.Error().Msgf("CF-RAY: %s Proxying to ingress %v error: %v", cfRay, rule, err)
 	} else {
-		c.log.Error().Msgf("Proxying to ingress %d error: %v", ruleNum, err)
+		p.log.Error().Msgf("Proxying to ingress %v error: %v", rule, err)
 	}
-
 }
 
 func findCfRayHeader(req *http.Request) string {

origin/proxy_test.go

@@ -6,17 +6,21 @@ import (
 	"flag"
 	"fmt"
 	"io"
+	"net"
 	"net/http"
 	"net/http/httptest"
 	"sync"
 	"testing"
 	"time"
 
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/logger"
+
+	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/hello"
 	"github.com/cloudflare/cloudflared/ingress"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	gorillaWS "github.com/gorilla/websocket"
 	"github.com/urfave/cli/v2"
 
 	"github.com/gobwas/ws/wsutil"
@@ -26,7 +30,8 @@ import (
 )
 
 var (
-	testTags = []tunnelpogs.Tag(nil)
+	testTags                 = []tunnelpogs.Tag(nil)
+	unusedWarpRoutingService = (*ingress.WarpRoutingService)(nil)
 )
 
 type mockHTTPRespWriter struct {
@@ -39,19 +44,14 @@ func newMockHTTPRespWriter() *mockHTTPRespWriter {
 	}
 }
 
-func (w *mockHTTPRespWriter) WriteRespHeaders(resp *http.Response) error {
-	w.WriteHeader(resp.StatusCode)
-	for header, val := range resp.Header {
+func (w *mockHTTPRespWriter) WriteRespHeaders(status int, header http.Header) error {
+	w.WriteHeader(status)
+	for header, val := range header {
 		w.Header()[header] = val
 	}
 	return nil
 }
 
-func (w *mockHTTPRespWriter) WriteErrorResponse() {
-	w.WriteHeader(http.StatusBadGateway)
-	_, _ = w.Write([]byte("http response error"))
-}
-
 func (w *mockHTTPRespWriter) Read(data []byte) (int, error) {
 	return 0, fmt.Errorf("mockHTTPRespWriter doesn't implement io.Reader")
 }
@@ -125,59 +125,58 @@ func TestProxySingleOrigin(t *testing.T) {
 	errC := make(chan error)
 	require.NoError(t, ingressRule.StartOrigins(&wg, &log, ctx.Done(), errC))
 
-	client := NewClient(ingressRule, testTags, &log)
-	t.Run("testProxyHTTP", testProxyHTTP(t, client))
-	t.Run("testProxyWebsocket", testProxyWebsocket(t, client))
-	t.Run("testProxySSE", testProxySSE(t, client))
+	proxy := NewOriginProxy(ingressRule, unusedWarpRoutingService, testTags, &log)
+	t.Run("testProxyHTTP", testProxyHTTP(t, proxy))
+	t.Run("testProxyWebsocket", testProxyWebsocket(t, proxy))
+	t.Run("testProxySSE", testProxySSE(t, proxy))
 	cancel()
 	wg.Wait()
 }
 
-func testProxyHTTP(t *testing.T, client connection.OriginClient) func(t *testing.T) {
+func testProxyHTTP(t *testing.T, proxy connection.OriginProxy) func(t *testing.T) {
 	return func(t *testing.T) {
-		respWriter := newMockHTTPRespWriter()
+		responseWriter := newMockHTTPRespWriter()
 		req, err := http.NewRequest(http.MethodGet, "http://localhost:8080", nil)
 		require.NoError(t, err)
 
-		err = client.Proxy(respWriter, req, false)
+		err = proxy.Proxy(responseWriter, req, connection.TypeHTTP)
 		require.NoError(t, err)
 
-		assert.Equal(t, http.StatusOK, respWriter.Code)
+		assert.Equal(t, http.StatusOK, responseWriter.Code)
 	}
 }
 
-func testProxyWebsocket(t *testing.T, client connection.OriginClient) func(t *testing.T) {
+func testProxyWebsocket(t *testing.T, proxy connection.OriginProxy) func(t *testing.T) {
 	return func(t *testing.T) {
 		// WSRoute is a websocket echo handler
 		ctx, cancel := context.WithCancel(context.Background())
-		req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("http://localhost:8080%s", hello.WSRoute), nil)
-
 		readPipe, writePipe := io.Pipe()
-		respWriter := newMockWSRespWriter(readPipe)
+		req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("http://localhost:8080%s", hello.WSRoute), readPipe)
+		responseWriter := newMockWSRespWriter(readPipe)
 
 		var wg sync.WaitGroup
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
-			err = client.Proxy(respWriter, req, true)
+			err = proxy.Proxy(responseWriter, req, connection.TypeWebsocket)
 			require.NoError(t, err)
 
-			require.Equal(t, http.StatusSwitchingProtocols, respWriter.Code)
+			require.Equal(t, http.StatusSwitchingProtocols, responseWriter.Code)
 		}()
 
 		msg := []byte("test websocket")
 		err = wsutil.WriteClientText(writePipe, msg)
 		require.NoError(t, err)
 
-		// ReadServerText reads next data message from rw, considering that caller represents client side.
-		returnedMsg, err := wsutil.ReadServerText(respWriter.respBody())
+		// ReadServerText reads next data message from rw, considering that caller represents proxy side.
+		returnedMsg, err := wsutil.ReadServerText(responseWriter.respBody())
 		require.NoError(t, err)
 		require.Equal(t, msg, returnedMsg)
 
 		err = wsutil.WriteClientBinary(writePipe, msg)
 		require.NoError(t, err)
 
-		returnedMsg, err = wsutil.ReadServerBinary(respWriter.respBody())
+		returnedMsg, err = wsutil.ReadServerBinary(responseWriter.respBody())
 		require.NoError(t, err)
 		require.Equal(t, msg, returnedMsg)
 
@@ -186,13 +185,13 @@ func testProxyWebsocket(t *testing.T, client connection.OriginClient) func(t *te
 	}
 }
 
-func testProxySSE(t *testing.T, client connection.OriginClient) func(t *testing.T) {
+func testProxySSE(t *testing.T, proxy connection.OriginProxy) func(t *testing.T) {
 	return func(t *testing.T) {
 		var (
 			pushCount = 50
 			pushFreq  = time.Millisecond * 10
 		)
-		respWriter := newMockSSERespWriter()
+		responseWriter := newMockSSERespWriter()
 		ctx, cancel := context.WithCancel(context.Background())
 		req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("http://localhost:8080%s?freq=%s", hello.SSERoute, pushFreq), nil)
 		require.NoError(t, err)
@@ -201,18 +200,18 @@ func testProxySSE(t *testing.T, client connection.OriginClient) func(t *testing.
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
-			err = client.Proxy(respWriter, req, false)
+			err = proxy.Proxy(responseWriter, req, connection.TypeHTTP)
 			require.NoError(t, err)
 
-			require.Equal(t, http.StatusOK, respWriter.Code)
+			require.Equal(t, http.StatusOK, responseWriter.Code)
 		}()
 
 		for i := 0; i < pushCount; i++ {
-			line := respWriter.ReadBytes()
+			line := responseWriter.ReadBytes()
 			expect := fmt.Sprintf("%d\n", i)
 			require.Equal(t, []byte(expect), line, fmt.Sprintf("Expect to read %v, got %v", expect, line))
 
-			line = respWriter.ReadBytes()
+			line = responseWriter.ReadBytes()
 			require.Equal(t, []byte("\n"), line, fmt.Sprintf("Expect to read '\n', got %v", line))
 		}
 
@@ -258,7 +257,7 @@ func TestProxyMultipleOrigins(t *testing.T) {
 	var wg sync.WaitGroup
 	require.NoError(t, ingress.StartOrigins(&wg, &log, ctx.Done(), errC))
 
-	client := NewClient(ingress, testTags, &log)
+	proxy := NewOriginProxy(ingress, unusedWarpRoutingService, testTags, &log)
 
 	tests := []struct {
 		url            string
@@ -290,18 +289,18 @@ func TestProxyMultipleOrigins(t *testing.T) {
 	}
 
 	for _, test := range tests {
-		respWriter := newMockHTTPRespWriter()
+		responseWriter := newMockHTTPRespWriter()
 		req, err := http.NewRequest(http.MethodGet, test.url, nil)
 		require.NoError(t, err)
 
-		err = client.Proxy(respWriter, req, false)
+		err = proxy.Proxy(responseWriter, req, connection.TypeHTTP)
 		require.NoError(t, err)
 
-		assert.Equal(t, test.expectedStatus, respWriter.Code)
+		assert.Equal(t, test.expectedStatus, responseWriter.Code)
 		if test.expectedBody != nil {
-			assert.Equal(t, test.expectedBody, respWriter.Body.Bytes())
+			assert.Equal(t, test.expectedBody, responseWriter.Body.Bytes())
 		} else {
-			assert.Equal(t, 0, respWriter.Body.Len())
+			assert.Equal(t, 0, responseWriter.Body.Len())
 		}
 	}
 	cancel()
@@ -327,7 +326,7 @@ func TestProxyError(t *testing.T) {
 			{
 				Hostname: "*",
 				Path:     nil,
-				Service: ingress.MockOriginService{
+				Service: ingress.MockOriginHTTPService{
 					Transport: errorOriginTransport{},
 				},
 			},
@@ -336,14 +335,509 @@ func TestProxyError(t *testing.T) {
 
 	log := zerolog.Nop()
 
-	client := NewClient(ingress, testTags, &log)
+	proxy := NewOriginProxy(ingress, unusedWarpRoutingService, testTags, &log)
 
-	respWriter := newMockHTTPRespWriter()
+	responseWriter := newMockHTTPRespWriter()
 	req, err := http.NewRequest(http.MethodGet, "http://127.0.0.1", nil)
 	assert.NoError(t, err)
 
-	err = client.Proxy(respWriter, req, false)
-	assert.Error(t, err)
-	assert.Equal(t, http.StatusBadGateway, respWriter.Code)
-	assert.Equal(t, "http response error", respWriter.Body.String())
+	assert.Error(t, proxy.Proxy(responseWriter, req, connection.TypeHTTP))
+}
+
+type replayer struct {
+	sync.RWMutex
+	writeDone chan struct{}
+	rw        *bytes.Buffer
+}
+
+func newReplayer(buffer *bytes.Buffer) {
+
+}
+
+func (r *replayer) Read(p []byte) (int, error) {
+	r.RLock()
+	defer r.RUnlock()
+	return r.rw.Read(p)
+}
+
+func (r *replayer) Write(p []byte) (int, error) {
+	r.Lock()
+	defer r.Unlock()
+	n, err := r.rw.Write(p)
+	return n, err
+}
+
+func (r *replayer) String() string {
+	r.Lock()
+	defer r.Unlock()
+	return r.rw.String()
+}
+
+func (r *replayer) Bytes() []byte {
+	r.Lock()
+	defer r.Unlock()
+	return r.rw.Bytes()
+}
+
+// TestConnections tests every possible permutation of connection protocols
+// proxied by cloudflared.
+//
+// WS - WS : When a websocket based ingress is configured on the origin and
+// the eyeball is also a websocket client streaming data.
+// TCP - TCP : When teamnet is enabled and an http or tcp service is running
+// on the origin.
+// TCP - WS: When teamnet is enabled and a websocket based service is running
+// on the origin.
+// WS - TCP: When a tcp based ingress is configured on the origin and the
+// eyeball sends tcp packets wrapped in websockets. (E.g: cloudflared access).
+func TestConnections(t *testing.T) {
+	logger := logger.Create(nil)
+	replayer := &replayer{rw: &bytes.Buffer{}}
+	type args struct {
+		ingressServiceScheme  string
+		originService         func(*testing.T, net.Listener)
+		eyeballResponseWriter connection.ResponseWriter
+		eyeballRequestBody    io.ReadCloser
+
+		// Can be set to nil to show warp routing is not enabled.
+		warpRoutingService *ingress.WarpRoutingService
+
+		// eyeball connection type.
+		connectionType connection.Type
+
+		//requestheaders to be sent in the call to proxy.Proxy
+		requestHeaders http.Header
+	}
+
+	type want struct {
+		message []byte
+		headers http.Header
+		err     bool
+	}
+
+	var tests = []struct {
+		name string
+		args args
+		want want
+	}{
+		{
+			name: "ws-ws proxy",
+			args: args{
+				ingressServiceScheme:  "ws://",
+				originService:         runEchoWSService,
+				eyeballResponseWriter: newWSRespWriter(replayer),
+				eyeballRequestBody:    newWSRequestBody([]byte("test1")),
+				connectionType:        connection.TypeWebsocket,
+				requestHeaders: map[string][]string{
+					// Example key from https://tools.ietf.org/html/rfc6455#section-1.2
+					"Sec-Websocket-Key":     {"dGhlIHNhbXBsZSBub25jZQ=="},
+					"Test-Cloudflared-Echo": {"Echo"},
+				},
+			},
+			want: want{
+				message: []byte("echo-test1"),
+				headers: map[string][]string{
+					"Connection":            {"Upgrade"},
+					"Sec-Websocket-Accept":  {"s3pPLMBiTxaQ9kYGzzhZRbK+xOo="},
+					"Upgrade":               {"websocket"},
+					"Test-Cloudflared-Echo": {"Echo"},
+				},
+			},
+		},
+		{
+			name: "tcp-tcp proxy",
+			args: args{
+				ingressServiceScheme:  "tcp://",
+				originService:         runEchoTCPService,
+				eyeballResponseWriter: newTCPRespWriter(replayer),
+				eyeballRequestBody:    newTCPRequestBody([]byte("test2")),
+				warpRoutingService:    ingress.NewWarpRoutingService(),
+				connectionType:        connection.TypeTCP,
+				requestHeaders: map[string][]string{
+					"Cf-Cloudflared-Proxy-Src": {"non-blank-value"},
+				},
+			},
+			want: want{
+				message: []byte("echo-test2"),
+			},
+		},
+		{
+			name: "tcp-ws proxy",
+			args: args{
+				ingressServiceScheme: "ws://",
+				originService:        runEchoWSService,
+				//eyeballResponseWriter gets set after roundtrip dial.
+				eyeballRequestBody: newPipedWSRequestBody([]byte("test3")),
+				warpRoutingService: ingress.NewWarpRoutingService(),
+				requestHeaders: map[string][]string{
+					"Cf-Cloudflared-Proxy-Src": {"non-blank-value"},
+				},
+				connectionType: connection.TypeTCP,
+			},
+			want: want{
+				message: []byte("echo-test3"),
+				// We expect no headers here because they are sent back via
+				// the stream.
+			},
+		},
+		{
+			name: "ws-tcp proxy",
+			args: args{
+				ingressServiceScheme:  "tcp://",
+				originService:         runEchoTCPService,
+				eyeballResponseWriter: newWSRespWriter(replayer),
+				eyeballRequestBody:    newWSRequestBody([]byte("test4")),
+				connectionType:        connection.TypeWebsocket,
+				requestHeaders: map[string][]string{
+					// Example key from https://tools.ietf.org/html/rfc6455#section-1.2
+					"Sec-Websocket-Key": {"dGhlIHNhbXBsZSBub25jZQ=="},
+				},
+			},
+			want: want{
+				message: []byte("echo-test4"),
+				headers: map[string][]string{
+					"Connection":           {"Upgrade"},
+					"Sec-Websocket-Accept": {"s3pPLMBiTxaQ9kYGzzhZRbK+xOo="},
+					"Upgrade":              {"websocket"},
+				},
+			},
+		},
+		{
+			name: "tcp-tcp proxy without warpRoutingService enabled",
+			args: args{
+				ingressServiceScheme:  "tcp://",
+				originService:         runEchoTCPService,
+				eyeballResponseWriter: newTCPRespWriter(replayer),
+				eyeballRequestBody:    newTCPRequestBody([]byte("test2")),
+				connectionType:        connection.TypeTCP,
+				requestHeaders: map[string][]string{
+					"Cf-Cloudflared-Proxy-Src": {"non-blank-value"},
+				},
+			},
+			want: want{
+				message: []byte{},
+				err:     true,
+			},
+		},
+		{
+			name: "ws-ws proxy when origin is different",
+			args: args{
+				ingressServiceScheme:  "ws://",
+				originService:         runEchoWSService,
+				eyeballResponseWriter: newWSRespWriter(replayer),
+				eyeballRequestBody:    newWSRequestBody([]byte("test1")),
+				connectionType:        connection.TypeWebsocket,
+				requestHeaders: map[string][]string{
+					// Example key from https://tools.ietf.org/html/rfc6455#section-1.2
+					"Sec-Websocket-Key": {"dGhlIHNhbXBsZSBub25jZQ=="},
+					"Origin":            {"Different origin"},
+				},
+			},
+			want: want{
+				message: []byte{},
+				err:     true,
+			},
+		},
+		{
+			name: "tcp-* proxy when origin service has already closed the connection/ is no longer running",
+			args: args{
+				ingressServiceScheme: "tcp://",
+				originService: func(t *testing.T, ln net.Listener) {
+					// closing the listener created by the test.
+					ln.Close()
+				},
+				eyeballResponseWriter: newTCPRespWriter(replayer),
+				eyeballRequestBody:    newTCPRequestBody([]byte("test2")),
+				connectionType:        connection.TypeTCP,
+				requestHeaders: map[string][]string{
+					"Cf-Cloudflared-Proxy-Src": {"non-blank-value"},
+				},
+			},
+			want: want{
+				message: []byte{},
+				err:     true,
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			ctx, cancel := context.WithCancel(context.Background())
+			ln, err := net.Listen("tcp", "127.0.0.1:0")
+			require.NoError(t, err)
+			// Starts origin service
+			test.args.originService(t, ln)
+
+			ingressRule := createSingleIngressConfig(t, test.args.ingressServiceScheme+ln.Addr().String())
+			var wg sync.WaitGroup
+			errC := make(chan error)
+			ingressRule.StartOrigins(&wg, logger, ctx.Done(), errC)
+			proxy := NewOriginProxy(ingressRule, test.args.warpRoutingService, testTags, logger)
+
+			req, err := http.NewRequest(
+				http.MethodGet,
+				test.args.ingressServiceScheme+ln.Addr().String(),
+				test.args.eyeballRequestBody,
+			)
+			require.NoError(t, err)
+
+			req.Header = test.args.requestHeaders
+			respWriter := test.args.eyeballResponseWriter
+
+			if pipedReqBody, ok := test.args.eyeballRequestBody.(*pipedRequestBody); ok {
+				respWriter = newTCPRespWriter(pipedReqBody.pipedConn)
+				go func() {
+					resp := pipedReqBody.roundtrip(test.args.ingressServiceScheme + ln.Addr().String())
+					replayer.Write(resp)
+				}()
+			}
+
+			err = proxy.Proxy(respWriter, req, test.args.connectionType)
+
+			cancel()
+			assert.Equal(t, test.want.err, err != nil)
+			assert.Equal(t, test.want.message, replayer.Bytes())
+			respPrinter := respWriter.(responsePrinter)
+			assert.Equal(t, test.want.headers, respPrinter.headers())
+			replayer.rw.Reset()
+		})
+	}
+}
+
+type requestBody struct {
+	pw *io.PipeWriter
+	pr *io.PipeReader
+}
+
+func newWSRequestBody(data []byte) *requestBody {
+	pr, pw := io.Pipe()
+	go wsutil.WriteClientBinary(pw, data)
+	return &requestBody{
+		pr: pr,
+		pw: pw,
+	}
+}
+func newTCPRequestBody(data []byte) *requestBody {
+	pr, pw := io.Pipe()
+	go pw.Write(data)
+	return &requestBody{
+		pr: pr,
+		pw: pw,
+	}
+}
+
+func (r *requestBody) Read(p []byte) (n int, err error) {
+	return r.pr.Read(p)
+}
+
+func (r *requestBody) Close() error {
+	r.pw.Close()
+	r.pr.Close()
+	return nil
+}
+
+type pipedRequestBody struct {
+	dialer         gorillaWS.Dialer
+	pipedConn      net.Conn
+	wsConn         net.Conn
+	messageToWrite []byte
+}
+
+func newPipedWSRequestBody(data []byte) *pipedRequestBody {
+	conn1, conn2 := net.Pipe()
+	dialer := gorillaWS.Dialer{
+		NetDial: func(network, addr string) (net.Conn, error) {
+			return conn2, nil
+		},
+	}
+	return &pipedRequestBody{
+		dialer:         dialer,
+		pipedConn:      conn1,
+		wsConn:         conn2,
+		messageToWrite: data,
+	}
+}
+
+func (p *pipedRequestBody) roundtrip(addr string) []byte {
+	header := http.Header{}
+	conn, resp, err := p.dialer.Dial(addr, header)
+	if err != nil {
+		panic(err)
+	}
+	defer conn.Close()
+
+	if resp.StatusCode != http.StatusSwitchingProtocols {
+		panic(fmt.Errorf("resp returned status code: %d", resp.StatusCode))
+	}
+
+	err = conn.WriteMessage(gorillaWS.TextMessage, p.messageToWrite)
+	if err != nil {
+		panic(err)
+	}
+
+	_, data, err := conn.ReadMessage()
+	if err != nil {
+		panic(err)
+	}
+
+	return data
+}
+
+func (p *pipedRequestBody) Read(data []byte) (n int, err error) {
+	return p.pipedConn.Read(data)
+}
+
+func (p *pipedRequestBody) Close() error {
+	return nil
+}
+
+type responsePrinter interface {
+	headers() http.Header
+}
+
+type wsRespWriter struct {
+	w               io.Writer
+	responseHeaders http.Header
+	code            int
+}
+
+// newWSRespWriter uses wsutil.WriteClientText to generate websocket frames.
+// and wsutil.ReadClientText to translate frames from server to byte data.
+// In essence, this acts as a wsClient.
+func newWSRespWriter(w io.Writer) *wsRespWriter {
+	return &wsRespWriter{
+		w: w,
+	}
+}
+
+// Write is written to by ingress.Stream and serves as the output to the client.
+func (w *wsRespWriter) Write(p []byte) (int, error) {
+	returnedMsg, err := wsutil.ReadServerBinary(bytes.NewBuffer(p))
+	if err != nil {
+		// The data was not returned by a websocket connecton.
+		if err != io.ErrUnexpectedEOF {
+			return w.w.Write(p)
+		}
+	}
+	return w.w.Write(returnedMsg)
+}
+
+func (w *wsRespWriter) WriteRespHeaders(status int, header http.Header) error {
+	w.responseHeaders = header
+	w.code = status
+	return nil
+}
+
+// respHeaders is a test function to read respHeaders
+func (w *wsRespWriter) headers() http.Header {
+	return w.responseHeaders
+}
+
+type mockTCPRespWriter struct {
+	w               io.Writer
+	responseHeaders http.Header
+	code            int
+}
+
+func newTCPRespWriter(w io.Writer) *mockTCPRespWriter {
+	return &mockTCPRespWriter{
+		w: w,
+	}
+}
+
+func (m *mockTCPRespWriter) Write(p []byte) (n int, err error) {
+	return m.w.Write(p)
+}
+
+func (m *mockTCPRespWriter) WriteRespHeaders(status int, header http.Header) error {
+	m.responseHeaders = header
+	m.code = status
+	return nil
+}
+
+// respHeaders is a test function to read respHeaders
+func (m *mockTCPRespWriter) headers() http.Header {
+	return m.responseHeaders
+}
+
+func createSingleIngressConfig(t *testing.T, service string) ingress.Ingress {
+	ingressConfig := &config.Configuration{
+		Ingress: []config.UnvalidatedIngressRule{
+			{
+				Hostname: "*",
+				Service:  service,
+			},
+		},
+	}
+	ingressRule, err := ingress.ParseIngress(ingressConfig)
+	require.NoError(t, err)
+	return ingressRule
+}
+
+func runEchoTCPService(t *testing.T, l net.Listener) {
+	go func() {
+		for {
+			conn, err := l.Accept()
+			require.NoError(t, err)
+			defer conn.Close()
+
+			for {
+				buf := make([]byte, 1024)
+				size, err := conn.Read(buf)
+				if err == io.EOF {
+					return
+				}
+				data := []byte("echo-")
+				data = append(data, buf[:size]...)
+				_, err = conn.Write(data)
+				if err != nil {
+					t.Log(err)
+				}
+				return
+			}
+		}
+	}()
+}
+
+func runEchoWSService(t *testing.T, l net.Listener) {
+	var upgrader = gorillaWS.Upgrader{
+		ReadBufferSize:  10,
+		WriteBufferSize: 10,
+	}
+
+	var ws = func(w http.ResponseWriter, r *http.Request) {
+		header := make(http.Header)
+		for k, vs := range r.Header {
+			if k == "Test-Cloudflared-Echo" {
+				header[k] = vs
+			}
+		}
+		conn, err := upgrader.Upgrade(w, r, header)
+		if err != nil {
+			t.Log(err)
+			return
+		}
+		defer conn.Close()
+
+		for {
+			messageType, p, err := conn.ReadMessage()
+			if err != nil {
+				return
+			}
+			data := []byte("echo-")
+			data = append(data, p...)
+			if err := conn.WriteMessage(messageType, data); err != nil {
+				return
+			}
+		}
+	}
+
+	server := http.Server{
+		Handler: http.HandlerFunc(ws),
+	}
+
+	go func() {
+		err := server.Serve(l)
+		require.NoError(t, err)
+	}()
 }

origin/reconnect.go

@@ -108,7 +108,7 @@ func (cm *reconnectCredentialManager) RefreshAuth(
 	authOutcome, err := authenticate(ctx, backoff.Retries())
 	if err != nil {
 		cm.authFail.WithLabelValues(err.Error()).Inc()
-		if _, ok := backoff.GetBackoffDuration(ctx); ok {
+		if _, ok := backoff.GetMaxBackoffDuration(ctx); ok {
 			return backoff.BackoffTimer(), nil
 		}
 		return nil, err

origin/reconnect_test.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
@@ -28,13 +29,14 @@ func TestRefreshAuthBackoff(t *testing.T) {
 	// authentication failures should consume the backoff
 	for i := uint(0); i < backoff.MaxRetries; i++ {
 		retryChan, err := rcm.RefreshAuth(context.Background(), backoff, auth)
-		assert.NoError(t, err)
-		assert.NotNil(t, retryChan)
-		assert.Equal(t, (1<<i)*time.Second, wait)
+		require.NoError(t, err)
+		require.NotNil(t, retryChan)
+		require.Greater(t, wait.Seconds(), 0.0)
+		require.Less(t, wait.Seconds(), float64((1<<(i+1))*time.Second))
 	}
 	retryChan, err := rcm.RefreshAuth(context.Background(), backoff, auth)
-	assert.Error(t, err)
-	assert.Nil(t, retryChan)
+	require.Error(t, err)
+	require.Nil(t, retryChan)
 
 	// now we actually make contact with the remote server
 	_, _ = rcm.RefreshAuth(context.Background(), backoff, func(ctx context.Context, n int) (tunnelpogs.AuthOutcome, error) {
@@ -47,8 +49,8 @@ func TestRefreshAuthBackoff(t *testing.T) {
 		expectedGracePeriod := time.Duration(time.Second * 2 << backoff.MaxRetries)
 		return time.Now().Add(expectedGracePeriod * 2)
 	}
-	_, ok := backoff.GetBackoffDuration(context.Background())
-	assert.True(t, ok)
+	_, ok := backoff.GetMaxBackoffDuration(context.Background())
+	require.True(t, ok)
 }
 
 func TestRefreshAuthSuccess(t *testing.T) {

origin/tunnel.go

@@ -15,7 +15,6 @@ import (
 	"github.com/rs/zerolog"
 	"golang.org/x/sync/errgroup"
 
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
 	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/edgediscovery"
 	"github.com/cloudflare/cloudflared/h2mux"
@@ -40,7 +39,7 @@ const (
 
 type TunnelConfig struct {
 	ConnectionConfig *connection.Config
-	BuildInfo        *buildinfo.BuildInfo
+	OSArch           string
 	ClientID         string
 	CloseConnOnce    *sync.Once // Used to close connectedSignal no more than once
 	EdgeAddrs        []string
@@ -72,7 +71,7 @@ func (c *TunnelConfig) RegistrationOptions(connectionID uint8, OriginLocalIP str
 	return &tunnelpogs.RegistrationOptions{
 		ClientID:             c.ClientID,
 		Version:              c.ReportedVersion,
-		OS:                   fmt.Sprintf("%s_%s", c.BuildInfo.GoOS, c.BuildInfo.GoArch),
+		OS:                   c.OSArch,
 		ExistingTunnelPolicy: policy,
 		PoolName:             c.LBPool,
 		Tags:                 c.Tags,
@@ -174,11 +173,11 @@ func ServeTunnelLoop(
 
 		config.Observer.SendReconnect(connIndex)
 
-		duration, ok := protocolFallback.GetBackoffDuration(ctx)
+		duration, ok := protocolFallback.GetMaxBackoffDuration(ctx)
 		if !ok {
 			return err
 		}
-		connLog.Info().Msgf("Retrying connection in %s seconds", duration)
+		connLog.Info().Msgf("Retrying connection in up to %s seconds", duration)
 
 		select {
 		case <-ctx.Done():
@@ -244,7 +243,7 @@ func selectNextProtocol(
 // on error returns a flag indicating if error can be retried
 func ServeTunnel(
 	ctx context.Context,
-	connLong *zerolog.Logger,
+	connLog *zerolog.Logger,
 	credentialManager *reconnectCredentialManager,
 	config *TunnelConfig,
 	addr *net.TCPAddr,
@@ -273,6 +272,7 @@ func ServeTunnel(
 
 	edgeConn, err := edgediscovery.DialEdge(ctx, dialTimeout, config.EdgeTLSConfigs[protocol], addr)
 	if err != nil {
+		connLog.Err(err).Msg("Unable to establish connection with Cloudflare edge")
 		return err, true
 	}
 	connectedFuse := &connectedFuse{
@@ -284,7 +284,7 @@ func ServeTunnel(
 		connOptions := config.ConnectionOptions(edgeConn.LocalAddr().String(), uint8(backoff.retries))
 		err = ServeHTTP2(
 			ctx,
-			connLong,
+			connLog,
 			config,
 			edgeConn,
 			connOptions,
@@ -296,7 +296,7 @@ func ServeTunnel(
 	} else {
 		err = ServeH2mux(
 			ctx,
-			connLong,
+			connLog,
 			credentialManager,
 			config,
 			edgeConn,
@@ -311,28 +311,29 @@ func ServeTunnel(
 	if err != nil {
 		switch err := err.(type) {
 		case connection.DupConnRegisterTunnelError:
+			connLog.Err(err).Msg("Unable to establish connection.")
 			// don't retry this connection anymore, let supervisor pick a new address
 			return err, false
 		case connection.ServerRegisterTunnelError:
-			connLong.Err(err).Msg("Register tunnel error from server side")
+			connLog.Err(err).Msg("Register tunnel error from server side")
 			// Don't send registration error return from server to Sentry. They are
 			// logged on server side
 			if incidents := config.IncidentLookup.ActiveIncidents(); len(incidents) > 0 {
-				connLong.Error().Msg(activeIncidentsMsg(incidents))
+				connLog.Error().Msg(activeIncidentsMsg(incidents))
 			}
 			return err.Cause, !err.Permanent
 		case ReconnectSignal:
-			connLong.Info().
+			connLog.Info().
 				Uint8(connection.LogFieldConnIndex, connIndex).
 				Msgf("Restarting connection due to reconnect signal in %s", err.Delay)
 			err.DelayBeforeReconnect()
 			return err, true
 		default:
 			if err == context.Canceled {
-				connLong.Debug().Err(err).Msgf("Serve tunnel error")
+				connLog.Debug().Err(err).Msgf("Serve tunnel error")
 				return err, false
 			}
-			connLong.Err(err).Msgf("Serve tunnel error")
+			connLog.Err(err).Msgf("Serve tunnel error")
 			_, permanent := err.(unrecoverableError)
 			return err, !permanent
 		}

origin/tunnel_test.go

@@ -40,8 +40,10 @@ func TestWaitForBackoffFallback(t *testing.T) {
 	mockFetcher := dynamicMockFetcher{
 		percentage: 0,
 	}
+	warpRoutingEnabled := false
 	protocolSelector, err := connection.NewProtocolSelector(
 		connection.HTTP2.String(),
+		warpRoutingEnabled,
 		namedTunnel,
 		mockFetcher.fetch(),
 		resolveTTL,

socks/connection_handler_test.go

@@ -40,7 +40,7 @@ func sendSocksRequest(t *testing.T) []byte {
 
 func startTestServer(t *testing.T, httpHandler func(w http.ResponseWriter, r *http.Request)) {
 	// create a socks server
-	requestHandler := NewRequestHandler(NewNetDialer())
+	requestHandler := NewRequestHandler(NewNetDialer(), nil)
 	socksServer := NewConnectionHandler(requestHandler)
 	listener, err := net.Listen("tcp", "localhost:8086")
 	assert.NoError(t, err)