Release 2021.7.2

- cfsetup now has a build command `github-release-pkgs` to release linux
   and msi packages to github.
 - github_message.py now has an option to upload all assets in a provided
   directory.

.github/workflows/check.yaml

@@ -0,0 +1,20 @@
+on: [push, pull_request]
+name: Check
+jobs:
+  check:
+    strategy:
+      matrix:
+        go-version: [1.15.x]
+        os: [ubuntu-latest, macos-latest, windows-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+    - name: Install Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ matrix.go-version }}
+    - name: Install go-sumtype
+      run: go get github.com/BurntSushi/go-sumtype
+    - name: Checkout code
+      uses: actions/checkout@v2
+    - name: Test
+      run: make test

.gitignore

@@ -1,21 +1,18 @@
-.GOPATH/
-bin/
-tmp/
-guide/public
-/.GOPATH
+/tmp
 /bin
 .idea
 .build
 .vscode
 \#*\#
 cscope.*
-cloudflared
-cloudflared.pkg
-cloudflared.exe
-cloudflared.msi
-cloudflared-x86-64*
-!cmd/cloudflared/
+/cloudflared
+/cloudflared.pkg
+/cloudflared.exe
+/cloudflared.msi
+/cloudflared-x86-64*
+/packaging
 .DS_Store
 *-session.log
 ssh_server_tests/.env
-.cover
+/.cover
+built_artifacts/

CHANGES.md

@@ -0,0 +1,172 @@
+**Experimental**: This is a new format for release notes. The format and availability is subject to change.
+
+## 2021.6.0
+### Bug Fixes
+- Fixes a http2 transport (the new default for Named Tunnels) to work with unix socket origins.
+
+
+## 2021.5.10
+### Bug Fixes
+- Fixes a memory leak in h2mux transport that connects cloudflared to Cloudflare edge.
+
+
+## 2021.5.9
+### New Features
+- Uses new Worker based login helper service to facilitate token exchange in cloudflared flows.
+
+### Bug Fixes
+- Fixes Centos-7 builds.
+
+## 2021.5.8
+### New Features
+- When creating a DNS record to point a hostname at a tunnel, you can now use --overwrite-dns to overwrite any existing
+  DNS records with that hostname. This works both when using the CLI to provision DNS, as well as when starting an adhoc
+  named tunnel, e.g.:
+  - `cloudflared tunnel route dns --overwrite-dns foo-tunnel foo.example.com`
+  - `cloudflared tunnel --overwrite-dns --name foo-tunnel --hostname foo.example.com`
+
+## 2021.5.7
+### New Features
+- Named Tunnels will automatically select the protocol to connect to Cloudflare's edge network.
+
+## 2021.5.0
+
+### New Features
+- It is now possible to run the same tunnel using more than one `cloudflared` instance. This is a server-side change and
+  is compatible with any client version that uses Named Tunnels.
+
+  To get started, visit our [developer documentation](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/run-tunnel/deploy-cloudflared-replicas).
+- `cloudflared tunnel ingress validate` will now warn about unused keys in your config file. This is helpful for
+  detecting typos in your config.
+- If `cloudflared` detects it is running inside a Linux container, it will limit itself to use only the number of CPUs
+  the pod has been granted, instead of trying to use every CPU available.
+
+## 2021.4.0
+
+### Bug Fixes
+
+- Fixed proxying of websocket requests to avoid possibility of losing initial frames that were sent in the same TCP
+  packet as response headers [#345](https://github.com/cloudflare/cloudflared/issues/345).
+- `proxy-dns` option now works in conjunction with running a named tunnel [#346](https://github.com/cloudflare/cloudflared/issues/346).
+
+## 2021.3.6
+
+### Bug Fixes
+
+- Reverted 2021.3.5 improvement to use HTTP/2 in a best-effort manner between cloudflared and origin services because
+  it was found to break in some cases.
+
+## 2021.3.5
+
+### Improvements
+
+ - HTTP/2 transport is now always chosen if origin server supports it and the service url scheme is HTTPS.
+   This was previously done in a best attempt manner.
+
+### Bug Fixes
+
+ - The MacOS binaries were not successfully released in 2021.3.3 and 2021.3.4. This release is aimed at addressing that.
+
+## 2021.3.3
+
+### Improvements
+
+- Tunnel create command, as well as, running ad-hoc tunnels using `cloudflared tunnel -name NAME`, will not overwrite
+  existing files when writing tunnel credentials.
+
+### Bug Fixes
+
+- Tunnel create and delete commands no longer use path to credentials from the configuration file.
+  If you need ot place tunnel credentials file at a specific location, you must use `--credentials-file` flag.
+- Access ssh-gen creates properly named keys for SSH short lived certs.
+
+
+## 2021.3.2
+
+### New Features
+
+- It is now possible to obtain more detailed information about the cloudflared connectors to Cloudflare Edge via
+  `cloudflared tunnel info <name/uuid>`. It is possible to sort the output as well as output in different formats,
+  such as: `cloudflared tunnel info --sort-by version --invert-sort --output json <name/uuid>`.
+  You can obtain more information via `cloudflared tunnel info --help`.
+
+### Bug Fixes
+
+- Don't look for configuration file in default paths when `--config FILE` flag is present after `tunnel` subcommand.
+- cloudflared access token command now functions correctly with the new token-per-app change from 2021.3.0.
+
+
+## 2021.3.0
+
+### New Features
+
+- [Cloudflare One Routing](https://developers.cloudflare.com/cloudflare-one/tutorials/warp-to-tunnel) specific commands
+  now show up in the `cloudflared tunnel route --help` output.
+- There is a new ingress type that allows cloudflared to proxy SOCKS5 as a bastion. You can use it with an ingress
+  rule by adding `service: socks-proxy`. Traffic is routed to any destination specified by the SOCKS5 packet but only
+  if allowed by a rule. In the following example we allow proxying to a certain CIDR but explicitly forbid one address
+  within it:
+```
+ingress:
+  - hostname: socks.example.com
+    service: socks-proxy
+    originRequest:
+      ipRules:
+        - prefix: 192.168.1.8/32
+          allow: false
+        - prefix: 192.168.1.0/24
+          ports: [80, 443]
+          allow: true
+```
+
+
+### Improvements
+
+- Nested commands, such as `cloudflared tunnel run`, now consider CLI arguments even if they appear earlier on the
+  command. For instance, `cloudflared --config config.yaml tunnel run` will now behave the same as
+  `cloudflared tunnel --config config.yaml run`
+- Warnings are now shown in the output logs whenever cloudflared is running without the most recent version and
+  `no-autoupdate` is `true`.
+- Access tokens are now stored per Access App instead of per request path. This decreases the number of times that the
+  user is required to authenticate with an Access policy redundantly.
+
+### Bug Fixes
+
+- GitHub [PR #317](https://github.com/cloudflare/cloudflared/issues/317) was broken in 2021.2.5 and is now fixed again.
+
+## 2021.2.5
+
+### New Features
+
+- We introduce [Cloudflare One Routing](https://developers.cloudflare.com/cloudflare-one/tutorials/warp-to-tunnel) in
+  beta mode. Cloudflare customer can now connect users and private networks with RFC 1918 IP addresses via the
+  Cloudflare edge network. Users running Cloudflare WARP client in the same organization can connect to the services
+  made available by Argo Tunnel IP routes. Please share your feedback in the GitHub issue tracker.
+
+## 2021.2.4
+
+### Bug Fixes
+
+- Reverts the Improvement released in 2021.2.3 for CLI arguments as it introduced a regression where cloudflared failed
+  to read URLs in configuration files.
+- cloudflared now logs the reason for failed connections if the error is recoverable.
+
+## 2021.2.3
+
+### Backward Incompatible Changes
+
+- Removes db-connect. The Cloudflare Workers product will continue to support db-connect implementations with versions
+  of cloudflared that predate this release and include support for db-connect.
+
+### New Features
+
+- Introduces support for proxy configurations with websockets in arbitrary TCP connections (#318).
+
+### Improvements
+
+- (reverted) Nested command line argument handling.
+
+### Bug Fixes
+
+- The maximum number of upstream connections is now limited by default which should fix reported issues of cloudflared
+  exhausting CPU usage when faced with connectivity issues.

Dockerfile

@@ -1,7 +1,7 @@
 # use a builder image for building cloudflare
 ARG TARGET_GOOS
 ARG TARGET_GOARCH
-FROM golang:1.15.7 as builder
+FROM golang:1.16.4 as builder
 ENV GO111MODULE=on \
     CGO_ENABLED=0 \
     TARGET_GOOS=${TARGET_GOOS} \

Makefile

@@ -11,6 +11,13 @@ ifneq ($(GO_BUILD_TAGS),)
 	GO_BUILD_TAGS := -tags $(GO_BUILD_TAGS)
 endif
 
+ifeq ($(NIGHTLY), true)
+	DEB_PACKAGE_NAME := cloudflared-nightly
+	NIGHTLY_FLAGS := --conflicts cloudflared --replaces cloudflared
+else
+	DEB_PACKAGE_NAME := cloudflared
+endif
+
 DATE          := $(shell date -u '+%Y-%m-%d-%H%M UTC')
 VERSION_FLAGS := -ldflags='-X "main.Version=$(VERSION)" -X "main.BuildTime=$(DATE)"'
 
@@ -34,6 +41,8 @@ ifneq ($(GOARCH),)
     TARGET_ARCH ?= $(GOARCH)
 else ifeq ($(LOCAL_ARCH),x86_64)
     TARGET_ARCH ?= amd64
+else ifeq ($(LOCAL_ARCH),amd64)
+    TARGET_ARCH ?= amd64
 else ifeq ($(LOCAL_ARCH),i686)
     TARGET_ARCH ?= amd64
 else ifeq ($(shell echo $(LOCAL_ARCH) | head -c 5),armv8)
@@ -79,9 +88,19 @@ clean:
 	go clean
 
 .PHONY: cloudflared
-cloudflared: tunnel-deps
+cloudflared: 
+ifeq ($(FIPS), true)
+	$(info Building cloudflared with go-fips)
+	-test -f fips/fips.go && mv fips/fips.go fips/fips.go.linux-amd64
+	mv fips/fips.go.linux-amd64 fips/fips.go
+endif
+
 	GOOS=$(TARGET_OS) GOARCH=$(TARGET_ARCH) go build -v -mod=vendor $(GO_BUILD_TAGS) $(VERSION_FLAGS) $(IMPORT_PATH)/cmd/cloudflared
 
+ifeq ($(FIPS), true)
+	mv fips/fips.go fips/fips.go.linux-amd64
+endif
+
 .PHONY: container
 container:
 	docker build --build-arg=TARGET_ARCH=$(TARGET_ARCH) --build-arg=TARGET_OS=$(TARGET_OS) -t cloudflare/cloudflared-$(TARGET_OS)-$(TARGET_ARCH):"$(VERSION)" .
@@ -101,10 +120,10 @@ test-ssh-server:
 	docker-compose -f ssh_server_tests/docker-compose.yml up
 
 define publish_package
+	chmod 664 cloudflared*.$(1); \
 	for HOST in $(CF_PKG_HOSTS); do \
 		ssh-keyscan -t rsa $$HOST >> ~/.ssh/known_hosts; \
-		scp -4 cloudflared*.$(1) cfsync@$$HOST:/state/cf-pkg/staging/$(2)/$(TARGET_PUBLIC_REPO)/cloudflared/; \
-		ssh cfsync@$$HOST 'chmod g+w /state/cf-pkg/staging/$(2)/$(TARGET_PUBLIC_REPO)/cloudflared/*.$(1)'; \
+		scp -p -4 cloudflared*.$(1) cfsync@$$HOST:/state/cf-pkg/staging/$(2)/$(TARGET_PUBLIC_REPO)/cloudflared/; \
 	done
 endef
 
@@ -126,7 +145,7 @@ define build_package
 		--license 'Cloudflare Service Agreement' \
 		--url 'https://github.com/cloudflare/cloudflared' \
 		-m 'Cloudflare <support@cloudflare.com>' \
-		-a $(TARGET_ARCH) -v $(VERSION) -n cloudflared --after-install postinst.sh --after-remove postrm.sh \
+		-a $(TARGET_ARCH) -v $(VERSION) -n $(DEB_PACKAGE_NAME) $(NIGHTLY_FLAGS) --after-install postinst.sh --after-remove postrm.sh \
 		cloudflared=$(INSTALL_BINDIR) cloudflared.1=$(MAN_DIR)
 endef
 
@@ -138,6 +157,14 @@ cloudflared-deb: cloudflared
 cloudflared-rpm: cloudflared
 	$(call build_package,rpm)
 
+.PHONY: cloudflared-pkg
+cloudflared-pkg: cloudflared
+	$(call build_package,osxpkg)
+
+.PHONY: cloudflared-msi
+cloudflared-msi: cloudflared
+	wixl --define Version=$(VERSION) --define Path=$(EXECUTABLE_PATH) --output cloudflared-$(VERSION)-$(TARGET_ARCH).msi cloudflared.wxs
+
 .PHONY: cloudflared-darwin-amd64.tgz
 cloudflared-darwin-amd64.tgz: cloudflared
 	tar czf cloudflared-darwin-amd64.tgz cloudflared
@@ -212,6 +239,10 @@ release: bin/equinox
 github-release: cloudflared
 	python3 github_release.py --path $(EXECUTABLE_PATH) --release-version $(VERSION)
 
+.PHONY: github-release-built-pkgs
+github-release-built-pkgs:
+	python3 github_release.py --path $(PWD)/built_artifacts --release-version $(VERSION)
+
 .PHONY: github-message
 github-message:
 	python3 github_message.py --release-version $(VERSION)
@@ -225,20 +256,25 @@ bin/equinox:
 	mkdir -p bin
 	curl -s https://bin.equinox.io/c/75JtLRTsJ3n/release-tool-beta-$(EQUINOX_PLATFORM).tgz | tar xz -C bin/
 
-.PHONY: tunnel-deps
-tunnel-deps: tunnelrpc/tunnelrpc.capnp.go
 
-tunnelrpc/tunnelrpc.capnp.go: tunnelrpc/tunnelrpc.capnp
+.PHONY: tunnelrpc-deps
+tunnelrpc-deps:
 	which capnp  # https://capnproto.org/install.html
 	which capnpc-go  # go get zombiezen.com/go/capnproto2/capnpc-go
 	capnp compile -ogo tunnelrpc/tunnelrpc.capnp
 
+.PHONY: quic-deps
+quic-deps: 
+	which capnp 
+	which capnpc-go
+	capnp compile -ogo quic/schema/quic_metadata_protocol.capnp
+
 .PHONY: vet
 vet:
 	go vet -mod=vendor ./...
-	which go-sumtype  # go get github.com/BurntSushi/go-sumtype
+	which go-sumtype  # go get github.com/BurntSushi/go-sumtype (don't do this in build directory or this will cause vendor issues)
 	go-sumtype $$(go list -mod=vendor ./...)
 
-.PHONY: msi
-msi: cloudflared
-	go-msi make --msi cloudflared.msi --version $(MSI_VERSION)
+.PHONY: goimports
+goimports:
+	for d in $$(go list -mod=readonly -f '{{.Dir}}' -a ./... | fgrep -v tunnelrpc) ; do goimports -format-only -local github.com/cloudflare/cloudflared -w $$d ; done

README.md

@@ -1,6 +1,6 @@
 # Argo Tunnel client
 
-Contains the command-line client for Argo Tunnel, a tunneling daemon that proxies any local webserver through the Cloudflare network. Extensive documentation can be found in the [Argo Tunnel section](https://developers.cloudflare.com/argo-tunnel/) of the Cloudflare Docs.
+Contains the command-line client for Argo Tunnel, a tunneling daemon that proxies any local webserver through the Cloudflare network. Extensive documentation can be found in the [Argo Tunnel section](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps) of the Cloudflare Docs.
 
 ## Before you get started
 
@@ -13,29 +13,29 @@ Before you use Argo Tunnel, you'll need to complete a few steps in the Cloudflar
 
 Downloads are available as standalone binaries, a Docker image, and Debian, RPM, and Homebrew packages. You can also find releases here on the `cloudflared` GitHub repository.
 
-* You can [install on macOS](https://developers.cloudflare.com/argo-tunnel/getting-started/installation#macos) via Homebrew or by downloading the [latest Darwin amd64 release](https://github.com/cloudflare/cloudflared/releases)
-* Binaries, Debian, and RPM packages for Linux [can be found here](https://developers.cloudflare.com/argo-tunnel/getting-started/installation#linux)
+* You can [install on macOS](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/install-and-setup/installation#macos) via Homebrew or by downloading the [latest Darwin amd64 release](https://github.com/cloudflare/cloudflared/releases)
+* Binaries, Debian, and RPM packages for Linux [can be found here](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/install-and-setup/installation#linux)
 * A Docker image of `cloudflared` is [available on DockerHub](https://hub.docker.com/r/cloudflare/cloudflared)
-* You can install on Windows machines with the [steps here](https://developers.cloudflare.com/argo-tunnel/getting-started/installation#windows)
+* You can install on Windows machines with the [steps here](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/install-and-setup/installation#windows)
 
-User documentation for Argo Tunnel can be found at https://developers.cloudflare.com/argo-tunnel/
+User documentation for Argo Tunnel can be found at https://developers.cloudflare.com/cloudflare-one/connections/connect-apps
 
 ## Creating Tunnels and routing traffic
 
 Once installed, you can authenticate `cloudflared` into your Cloudflare account and begin creating Tunnels that serve traffic for hostnames in your account.
 
-* Create a Tunnel with [these instructions](https://developers.cloudflare.com/argo-tunnel/create-tunnel)
-* Route traffic to that Tunnel with [DNS records in Cloudflare](https://developers.cloudflare.com/argo-tunnel/routing-to-tunnel/dns) or with a [Cloudflare Load Balancer](https://developers.cloudflare.com/argo-tunnel/routing-to-tunnel/lb)
+* Create a Tunnel with [these instructions](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/create-tunnel)
+* Route traffic to that Tunnel with [DNS records in Cloudflare](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/routing-to-tunnel/dns) or with a [Cloudflare Load Balancer](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/routing-to-tunnel/lb)
 
 ## TryCloudflare
 
-Want to test Argo Tunnel before adding a website to Cloudflare? You can do so with TryCloudflare using the documentation [available here](https://developers.cloudflare.com/argo-tunnel/learning/trycloudflare).
+Want to test Argo Tunnel before adding a website to Cloudflare? You can do so with TryCloudflare using the documentation [available here](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/trycloudflare).
 
 ## Deprecated versions
 
 Cloudflare currently supports all versions of `cloudflared`. Starting on March 20, 2021, Cloudflare will no longer support versions released prior to 2020.5.1.
 
-All features available in versions released prior to 2020.5.1 are available in current versions. Breaking changes unrelated to feature availability may be introduced that will impact versions released prior to 2020.5.1. You can read more about upgrading `cloudflared` in our [developer documentation](https://developers.cloudflare.com/argo-tunnel/getting-started/installation#updating-cloudflared).
+All features available in versions released prior to 2020.5.1 are available in current versions. Breaking changes unrelated to feature availability may be introduced that will impact versions released prior to 2020.5.1. You can read more about upgrading `cloudflared` in our [developer documentation](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/install-and-setup/installation#updating-cloudflared).
 
 | Version(s) | Deprecation status |
 |---|---|

RELEASE_NOTES

@@ -1,3 +1,189 @@
+2021.7.2
+- 2021-07-27 Fixed a syntax error with python logging.
+
+2021.7.1
+- 2021-07-21 TUN-4755: Add a windows msi release option to Make
+- 2021-07-22 TUN-4761: Added a build-all-packages target to cfsetup
+- 2021-07-26 TUN-4771: Upload deb, rpm and msi packages to github
+- 2021-07-14 TUN-4714: Name nightly package cloudflared-nightly to avoid apt conflict
+- 2021-07-16 TUN-4701: Split Proxy into ProxyHTTP and ProxyTCP
+- 2021-07-08 TUN-4596: Add QUIC application protocol for QUIC stream handshake
+- 2021-07-09 TUN-4699: Make quick tunnels the default in cloudflared
+
+2021.7.0
+- 2021-07-01 TUN-4626: Proxy non-stream based origin websockets with http Roundtrip.
+- 2021-07-01 TUN-4655: ingress.StreamBasedProxy.EstablishConnection takes dest input
+- 2021-07-09 TUN-4698: Add cloudflared metrics endpoint to serve quick tunnel hostname
+- 2021-06-21 TUN-4521: Modify cloudflared to use zoneless-tunnels-worker for free tunnels
+- 2021-04-05 AUTH-3475: Updated GetAppInfo error message
+
+2021.6.0
+- 2021-06-21 TUN-4571: Changelog for 2021.6.0
+- 2021-06-18 TUN-4571: Fix proxying to unix sockets when using HTTP2 transport to Cloudflare Edge
+- 2021-06-07 TUN-4502: Make `cloudflared tunnel route` subcommands described consistently
+- 2021-06-08 TUN-4504: Fix component tests in windows
+- 2021-05-27 TUN-4461: Log resulting DNS hostname if one is received from Cloudflare API
+
+2021.5.10
+- 2021-05-25 TUN-4456: Replaced instances of Tick() with Ticker() in h2mux paths
+
+2021.5.9
+- 2021-05-20 TUN-4426: Fix centos builds
+- 2021-05-20 Update changelog
+- 2021-04-30 AUTH-3426: Point to new transfer service URL and eliminate PUT /ok
+
+2021.5.8
+- 2021-05-14 TUN-4419: Improve error message when cloudflared cannot reach origin
+- 2021-05-19 TUN-4425: --overwrite-dns flag for in adhoc and route dns cmds
+
+2021.5.7
+- 2021-05-17 Fix typo in Changes.md
+- 2021-05-17 TUN-4421: Named Tunnels will automatically select the protocol to connect to Cloudflare's edge network
+
+2021.5.6
+- 2021-05-14 TUN-4418: Downgrade to Go 1.16.3
+
+2021.5.5
+
+
+2021.5.4
+- Fix release pipeline
+
+2021.5.1
+- 2021-05-10 TUN-4342: Fix false positive warning about unused hostname property
+- 2021-05-10 Release 2021.5.0
+
+2021.5.0
+- 2021-05-10 TUN-4384: Silence log from automaxprocs
+- 2021-05-10 AUTH-3537: AUDs in JWTs are now always arrays
+- 2021-05-10 Update changelog for 2021.5.0
+- 2021-05-03 TUN-4343: Fix broken build by setting debug field correctly
+- 2021-05-06 TUN-4356: Set AUTOMAXPROCS to the CPU limit when running in a Linux container
+- 2021-05-06 TUN-4357: Bump Go to 1.16
+- 2021-05-06 TUN-4359: Warn about unused keys in 'tunnel ingress validate'
+- 2021-04-30 debug: log host / path
+- 2021-04-20 AUTH-3513: Checks header for app info in case response is a 403/401 from the edge
+- 2021-04-29 TUN-4000: Release notes for cloudflared replica model
+- 2021-04-09 TUN-2853: rename STDIN-CONTROL env var to STDIN_CONTROL
+- 2021-04-09 TUN-4206: Better error message when user is only using one ingress rule
+
+2021.4.0
+- 2021-04-05 TUN-4178: Fix component test for running as a service in MacOS to not assume a named tunnel
+- 2021-04-05 TUN-4177: Running with proxy-dns should not prevent running Named Tunnels
+- 2021-04-02 TUN-4168: Transparently proxy websocket connections using stdlib HTTP client instead of gorilla/websocket; move websocket client code into carrier package since it's only used by access subcommands now (#345).
+- 2021-04-07 Publish change log for 2021.4.0
+
+2021.3.6
+- 2021-03-30 TUN-4150: Only show the connector table in 'tunnel info' if there are connectors. Don't show rows with zero connections.
+- 2021-03-31 TUN-4153: Revert best-effort HTTP2 usage when talking to origins
+- 2021-03-26 TUN-4141: Better error messages for tunnel info subcommand.
+- 2021-03-29 TUN-4146: Unhide and document grace-period
+- 2021-03-25 TUN-3863: Consolidate header handling logic in the connection package; move headers definitions from h2mux to packages that manage them; cleanup header conversions
+
+2021.3.5
+- 2021-03-26 TUN-3896: http-service and tunnelstore client use http2 transport.
+- 2021-03-25 TUN-4125: Change component tests to run in CI with its own dedicated resources
+- 2021-03-26 Publish change log for 2021.3.5
+
+2021.3.4
+
+
+2021.3.3
+- 2021-03-23 TUN-4111: Warn the user if both properties "tunnel" and "hostname" are used
+- 2021-03-23 TUN-4082: Test logging when running as a service
+- 2021-03-23 TUN-4112: Skip testing graceful shutdown with SIGINT on Windows
+- 2021-03-23 TUN-4116: Ingore credentials-file setting in configuration file during tunnel create and delete opeations.
+- 2021-03-23 TUN-4118: Don't overwrite existing file with tunnel credentials. For ad-hoc tunnels, this means tunnel won't start if there's a file in the way.
+- 2021-03-24 TUN-4123: Don't capture output in reconnect componet test
+- 2021-03-23 TUN-4067: Reformat code for consistent import order, grouping, and fix formatting. Added goimports target to the Makefile to make this easier in the future.
+- 2021-03-24 AUTH-3455: Generate short-lived ssh cert per hostname
+- 2021-03-25 Update changelog 2021.3.3
+
+2021.3.2
+- 2021-03-23 TUN-4042: Capture cloudflared output to debug component tests
+- 2021-03-23 Publish changelog for 2021.3.2
+- 2021-03-16 TUN-4089: Address flakiness in component tests for termination
+- 2021-03-16 TUN-4060: Fix Go Vet warnings (new with go 1.16) where t.Fatalf is called from a test goroutine
+- 2021-03-16 TUN-4091: Use flaky decorator to rerun reconnect component tests when they fail
+- 2021-03-12 TUN-4081: Update log severities to use Zerolog's levels
+- 2021-03-16 TUN-4094: Don't read configuration file for access commands
+- 2021-03-15 TUN-3993: New `cloudflared tunnel info` to obtain details about the active connectors for a tunnel
+- 2021-03-17 TUN-3715: Apply input source to the correct context
+- 2021-03-17 AUTH-3394: Ensure scheme on token command
+- 2021-03-18 TUN-4096: Reduce tunnel not connected assertion backoff to address flaky termination tests
+- 2021-03-19 TUN-3998: Allow to cleanup the connections of a tunnel limited to a single client
+- 2021-02-04 TUN-3715: Only read config file once, right before invoking the command
+
+2021.3.1
+- 2021-03-11 TUN-4051: Add component-tests to test graceful shutdown
+- 2021-03-12 TUN-4052: Add component tests to assert service mode behavior
+
+2021.3.0
+- 2021-03-10 TUN-4075: Dedup test should not compare order of list
+- 2021-03-10 Revert "AUTH-3394: Creates a token per app instead of per path"
+- 2021-03-11 TUN-4066: Remove unnecessary chmod during package publish to CF_PKG_HOSTS
+- 2021-03-11 TUN-4066: Set permissions in build agent before 'scp'-ing to machine hosting package repo
+- 2021-03-11 TUN-4050: Add component tests to assert reconnect behavior
+- 2021-03-10 AUTH-3394: Creates a token per app instead of per path - with fix for free tunnels
+- 2021-03-15 Publish change log for 2021.3.0
+- 2021-03-01 Issue #285 - Makefile does not detect TARGET_ARCH correctly on FreeBSD  (#325)
+- 2021-03-01 TUN-3988: Log why it cannot check if origin cert exists
+- 2021-03-02 TUN-3995: Optional --features flag for tunnel run.
+- 2021-03-02 TUN-3994: Log client_id when running a named tunnel
+- 2021-03-04 TUN-4026: Fix regression where HTTP2 edge transport was no longer propagating control plane errors
+- 2021-03-05 TUN-4055: Skeleton for component tests
+- 2021-03-08 TUN-4047: Add cfsetup target to run component test
+- 2021-03-08 TUN-4016: Delegate decision to update for Worker service
+- 2021-03-02 TUN-3905: Cannot run go mod vendor in cloudflared due to fips
+- 2021-03-08 TUN-4063: Cleanup dependencies between packages.
+- 2021-03-09 Allow partial reads from a GorillaConn; add SetDeadline (from net.Conn) (#330)
+- 2021-03-09 TUN-4069: Fix regression on support for websocket over proxy
+- 2021-03-02 AUTH-3394: Creates a token per app instead of per path
+- 2021-03-01 TUN-4017: Add support for using cloudflared as a full socks proxy.
+- 2021-03-08 TUN-4062: Read component tests config from yaml file
+- 2021-03-08 TUN-4049: Add component tests to assert logging behavior when running from terminal
+- 2021-02-23 TUN-3963: Repoint urfave/cli/v2 library at patched branch at github.com/ipostelnik/cli/v2@fixed which correctly handles reading flags declared at multiple levels of subcommands.
+- 2021-02-25 TUN-3970: Route ip show has alias route ip list
+- 2021-02-26 TUN-3978: Unhide teamnet commands and improve their help
+- 2021-02-26 TUN-3983: Renew CA certs in cloudflared
+- 2021-02-28 TUN-3989: Check in with Updater service in more situations and convey messages to user
+- 2021-02-11 TUN-3819: Remove client-side check that deleted tunnels have no connections
+
+2021.2.5
+- 2021-02-23 Publish change notes for 2021.2.5
+- 2021-02-11 TUN-3838: ResponseWriter no longer reads and origin error tests
+- 2021-02-10 TUN-3895: Tests for socks stream handler
+- 2021-02-19 TUN-3939: Add logging that shows that Warp-routing is enabled
+- 2021-02-02 TUN-3817: Adds tests for websocket based streaming regression
+- 2021-02-04 TUN-3799: extended the Stream interface to take a logger and added debug logs for io.Copy errors
+- 2021-02-03 TUN-3855: Add ability to override target of 'access ssh' command to a different host for testing
+- 2021-02-04 TUN-3853: Respond with ws headers from the origin service rather than generating our own
+- 2021-02-08 TUN-3889: Move host header override logic to httpService
+- 2021-02-05 TUN-3868: Refactor singleTCPService and bridgeService to tcpOverWSService and rawTCPService
+- 2021-01-21 TUN-3753: Select http2 protocol when warp routing is enabled
+- 2021-01-26 TUN-3809: Allow routes ip show to output as JSON or YAML
+- 2021-01-11 TUN-3615: added support to  proxy tcp streams
+- 2021-01-17 TUN-3725: Warp-routing is independent of ingress
+- 2021-01-15 TUN-3764: Actively flush data for TCP streams
+- 2020-12-09 TUN-3617: Separate service from client, and implement different client for http vs. tcp origins
+
+2021.2.4
+- 2021-02-22 TUN-3948: Log error when retrying connection
+- 2021-02-23 TUN-3964: Revert "TUN-3922: Repoint urfave/cli/v2 library at patched branch at github.com/ipostelnik/cli/v2@fixed which correctly handles reading flags declared at multiple levels of subcommands."
+- 2021-02-23 Publish release notes for 2021.2.4
+
+2021.2.3
+- 2021-02-23 Publish release notes for 2021.2.3
+- 2021-02-10 TUN-3902: Add jitter to backoffhandler
+- 2021-02-11 TUN-3913: Help gives wrong exit code for autoupdate
+- 2021-02-12 Add max upstream connections dns-proxy option (#290)
+- 2021-02-16 TUN-3924: Removed db-connect command. Added a placeholder handler for this command that informs users that command is no longer supported.
+- 2021-02-12 TUN-3922: Repoint urfave/cli/v2 library at patched branch at github.com/ipostelnik/cli/v2@fixed which correctly handles reading flags declared at multiple levels of subcommands.
+- 2021-02-19 Added support for proxy (#318)
+- 2021-02-19 TUN-3945: Fix runApp signature for generic service
+- 2021-02-09 Update README.md
+- 2021-02-09 Update the TryCloudflare link
+
 2021.2.2
 - 2021-02-04 TUN-3864: Users can choose where credentials file is written after creating a tunnel
 - 2021-02-04 TUN-3869: Improve reliability of graceful shutdown.

build-packages.sh

@@ -0,0 +1,22 @@
+# This controls the directory the built artifacts go into
+export ARTIFACT_DIR=built_artifacts/
+mkdir -p $ARTIFACT_DIR
+windowsArchs=("amd64" "386")
+export TARGET_OS=windows
+for arch in ${windowsArchs[@]}; do 
+    export TARGET_ARCH=$arch
+    make cloudflared-msi
+done
+
+mv *.msi $ARTIFACT_DIR
+
+linuxArchs=("amd64" "386" "arm")
+export TARGET_OS=linux
+for arch in ${linuxArchs[@]}; do 
+    export TARGET_ARCH=$arch
+    make cloudflared-deb
+    make cloudflared-rpm
+done
+
+mv *.deb $ARTIFACT_DIR
+mv *.rpm $ARTIFACT_DIR

carrier/carrier.go

@@ -4,39 +4,44 @@
 package carrier
 
 import (
+	"crypto/tls"
+	"fmt"
 	"io"
 	"net"
 	"net/http"
+	"net/url"
 	"os"
 	"strings"
 
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/token"
-	"github.com/cloudflare/cloudflared/h2mux"
-
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
+
+	"github.com/cloudflare/cloudflared/token"
 )
 
-const LogFieldOriginURL = "originURL"
+const (
+	LogFieldOriginURL       = "originURL"
+	CFAccessTokenHeader     = "Cf-Access-Token"
+	cfJumpDestinationHeader = "Cf-Access-Jump-Destination"
+)
 
 type StartOptions struct {
-	OriginURL string
-	Headers   http.Header
+	AppInfo         *token.AppInfo
+	OriginURL       string
+	Headers         http.Header
+	Host            string
+	TLSClientConfig *tls.Config
 }
 
 // Connection wraps up all the needed functions to forward over the tunnel
 type Connection interface {
 	// ServeStream is used to forward data from the client to the edge
 	ServeStream(*StartOptions, io.ReadWriter) error
-
-	// StartServer is used to listen for incoming connections from the edge to the origin
-	StartServer(net.Listener, string, <-chan struct{}) error
 }
 
 // StdinoutStream is empty struct for wrapping stdin/stdout
 // into a single ReadWriter
-type StdinoutStream struct {
-}
+type StdinoutStream struct{}
 
 // Read will read from Stdin
 func (c *StdinoutStream) Read(p []byte) (int, error) {
@@ -120,7 +125,7 @@ func IsAccessResponse(resp *http.Response) bool {
 	if err != nil || location == nil {
 		return false
 	}
-	if strings.HasPrefix(location.Path, "/cdn-cgi/access/login") {
+	if strings.HasPrefix(location.Path, token.AccessLoginWorkerPath) {
 		return true
 	}
 
@@ -134,7 +139,7 @@ func BuildAccessRequest(options *StartOptions, log *zerolog.Logger) (*http.Reque
 		return nil, err
 	}
 
-	token, err := token.FetchTokenWithRedirect(req.URL, log)
+	token, err := token.FetchTokenWithRedirect(req.URL, options.AppInfo, log)
 	if err != nil {
 		return nil, err
 	}
@@ -145,7 +150,7 @@ func BuildAccessRequest(options *StartOptions, log *zerolog.Logger) (*http.Reque
 	if err != nil {
 		return nil, err
 	}
-	originRequest.Header.Set(h2mux.CFAccessTokenHeader, token)
+	originRequest.Header.Set(CFAccessTokenHeader, token)
 
 	for k, v := range options.Headers {
 		if len(v) >= 1 {
@@ -155,3 +160,26 @@ func BuildAccessRequest(options *StartOptions, log *zerolog.Logger) (*http.Reque
 
 	return originRequest, nil
 }
+
+func SetBastionDest(header http.Header, destination string) {
+	if destination != "" {
+		header.Set(cfJumpDestinationHeader, destination)
+	}
+}
+
+func ResolveBastionDest(r *http.Request) (string, error) {
+	jumpDestination := r.Header.Get(cfJumpDestinationHeader)
+	if jumpDestination == "" {
+		return "", fmt.Errorf("Did not receive final destination from client. The --destination flag is likely not set on the client side")
+	}
+	// Strip scheme and path set by client. Without a scheme
+	// Parsing a hostname and path without scheme might not return an error due to parsing ambiguities
+	if jumpURL, err := url.Parse(jumpDestination); err == nil && jumpURL.Host != "" {
+		return removePath(jumpURL.Host), nil
+	}
+	return removePath(jumpDestination), nil
+}
+
+func removePath(dest string) string {
+	return strings.SplitN(dest, "/", 2)[0]
+}

carrier/carrier_test.go

@@ -44,7 +44,7 @@ func (s *testStreamer) Write(p []byte) (int, error) {
 func TestStartClient(t *testing.T) {
 	message := "Good morning Austin! Time for another sunny day in the great state of Texas."
 	log := zerolog.Nop()
-	wsConn := NewWSConnection(&log, false)
+	wsConn := NewWSConnection(&log)
 	ts := newTestWebSocketServer()
 	defer ts.Close()
 
@@ -70,7 +70,7 @@ func TestStartServer(t *testing.T) {
 	message := "Good morning Austin! Time for another sunny day in the great state of Texas."
 	log := zerolog.Nop()
 	shutdownC := make(chan struct{})
-	wsConn := NewWSConnection(&log, false)
+	wsConn := NewWSConnection(&log)
 	ts := newTestWebSocketServer()
 	defer ts.Close()
 	options := &StartOptions{
@@ -81,7 +81,8 @@ func TestStartServer(t *testing.T) {
 	go func() {
 		err := Serve(wsConn, listener, shutdownC, options)
 		if err != nil {
-			t.Fatalf("Error running server: %v", err)
+			t.Errorf("Error running server: %v", err)
+			return
 		}
 	}()
 
@@ -155,3 +156,99 @@ func testRequest(t *testing.T, url string, stream io.ReadWriter) *http.Request {
 
 	return req
 }
+
+func TestBastionDestination(t *testing.T) {
+	tests := []struct {
+		name         string
+		header       http.Header
+		expectedDest string
+		wantErr      bool
+	}{
+		{
+			name: "hostname destination",
+			header: http.Header{
+				cfJumpDestinationHeader: []string{"localhost"},
+			},
+			expectedDest: "localhost",
+		},
+		{
+			name: "hostname destination with port",
+			header: http.Header{
+				cfJumpDestinationHeader: []string{"localhost:9000"},
+			},
+			expectedDest: "localhost:9000",
+		},
+		{
+			name: "hostname destination with scheme and port",
+			header: http.Header{
+				cfJumpDestinationHeader: []string{"ssh://localhost:9000"},
+			},
+			expectedDest: "localhost:9000",
+		},
+		{
+			name: "full hostname url",
+			header: http.Header{
+				cfJumpDestinationHeader: []string{"ssh://localhost:9000/metrics"},
+			},
+			expectedDest: "localhost:9000",
+		},
+		{
+			name: "hostname destination with port and path",
+			header: http.Header{
+				cfJumpDestinationHeader: []string{"localhost:9000/metrics"},
+			},
+			expectedDest: "localhost:9000",
+		},
+		{
+			name: "ip destination",
+			header: http.Header{
+				cfJumpDestinationHeader: []string{"127.0.0.1"},
+			},
+			expectedDest: "127.0.0.1",
+		},
+		{
+			name: "ip destination with port",
+			header: http.Header{
+				cfJumpDestinationHeader: []string{"127.0.0.1:9000"},
+			},
+			expectedDest: "127.0.0.1:9000",
+		},
+		{
+			name: "ip destination with port and path",
+			header: http.Header{
+				cfJumpDestinationHeader: []string{"127.0.0.1:9000/metrics"},
+			},
+			expectedDest: "127.0.0.1:9000",
+		},
+		{
+			name: "ip destination with schem and port",
+			header: http.Header{
+				cfJumpDestinationHeader: []string{"tcp://127.0.0.1:9000"},
+			},
+			expectedDest: "127.0.0.1:9000",
+		},
+		{
+			name: "full ip url",
+			header: http.Header{
+				cfJumpDestinationHeader: []string{"ssh://127.0.0.1:9000/metrics"},
+			},
+			expectedDest: "127.0.0.1:9000",
+		},
+		{
+			name:    "no destination",
+			wantErr: true,
+		},
+	}
+	for _, test := range tests {
+		r := &http.Request{
+			Header: test.header,
+		}
+		dest, err := ResolveBastionDest(r)
+		if test.wantErr {
+			assert.Error(t, err, "Test %s expects error", test.name)
+		} else {
+			assert.NoError(t, err, "Test %s expects no error, got error %v", test.name, err)
+			assert.Equal(t, test.expectedDest, dest, "Test %s expect dest %s, got %s", test.name, test.expectedDest, dest)
+		}
+	}
+}

carrier/websocket.go

@@ -1,18 +1,16 @@
 package carrier
 
 import (
-	"fmt"
 	"io"
-	"net"
 	"net/http"
 	"net/http/httputil"
-
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/token"
-	"github.com/cloudflare/cloudflared/socks"
-	cfwebsocket "github.com/cloudflare/cloudflared/websocket"
+	"net/url"
 
 	"github.com/gorilla/websocket"
 	"github.com/rs/zerolog"
+
+	"github.com/cloudflare/cloudflared/token"
+	cfwebsocket "github.com/cloudflare/cloudflared/websocket"
 )
 
 // Websocket is used to carry data via WS binary frames over the tunnel from client to the origin
@@ -22,25 +20,10 @@ type Websocket struct {
 	isSocks bool
 }
 
-type wsdialer struct {
-	conn *cfwebsocket.Conn
-}
-
-func (d *wsdialer) Dial(address string) (io.ReadWriteCloser, *socks.AddrSpec, error) {
-	local, ok := d.conn.LocalAddr().(*net.TCPAddr)
-	if !ok {
-		return nil, nil, fmt.Errorf("not a tcp connection")
-	}
-
-	addr := socks.AddrSpec{IP: local.IP, Port: local.Port}
-	return d.conn, &addr, nil
-}
-
 // NewWSConnection returns a new connection object
-func NewWSConnection(log *zerolog.Logger, isSocks bool) Connection {
+func NewWSConnection(log *zerolog.Logger) Connection {
 	return &Websocket{
-		log:     log,
-		isSocks: isSocks,
+		log: log,
 	}
 }
 
@@ -54,41 +37,46 @@ func (ws *Websocket) ServeStream(options *StartOptions, conn io.ReadWriter) erro
 	}
 	defer wsConn.Close()
 
-	if ws.isSocks {
-		dialer := &wsdialer{conn: wsConn}
-		requestHandler := socks.NewRequestHandler(dialer)
-		socksServer := socks.NewConnectionHandler(requestHandler)
-
-		_ = socksServer.Serve(conn)
-	} else {
-		cfwebsocket.Stream(wsConn, conn)
-	}
+	cfwebsocket.Stream(wsConn, conn, ws.log)
 	return nil
 }
 
-// StartServer creates a Websocket server to listen for connections.
-// This is used on the origin (tunnel) side to take data from the muxer and send it to the origin
-func (ws *Websocket) StartServer(listener net.Listener, remote string, shutdownC <-chan struct{}) error {
-	return cfwebsocket.StartProxyServer(ws.log, listener, remote, shutdownC, cfwebsocket.DefaultStreamHandler)
-}
-
 // createWebsocketStream will create a WebSocket connection to stream data over
 // It also handles redirects from Access and will present that flow if
 // the token is not present on the request
-func createWebsocketStream(options *StartOptions, log *zerolog.Logger) (*cfwebsocket.Conn, error) {
+func createWebsocketStream(options *StartOptions, log *zerolog.Logger) (*cfwebsocket.GorillaConn, error) {
 	req, err := http.NewRequest(http.MethodGet, options.OriginURL, nil)
 	if err != nil {
 		return nil, err
 	}
 	req.Header = options.Headers
+	if options.Host != "" {
+		req.Host = options.Host
+	}
 
 	dump, err := httputil.DumpRequest(req, false)
 	log.Debug().Msgf("Websocket request: %s", string(dump))
 
-	wsConn, resp, err := cfwebsocket.ClientConnect(req, nil)
+	dialer := &websocket.Dialer{
+		TLSClientConfig: options.TLSClientConfig,
+		Proxy:           http.ProxyFromEnvironment,
+	}
+	wsConn, resp, err := clientConnect(req, dialer)
 	defer closeRespBody(resp)
 
 	if err != nil && IsAccessResponse(resp) {
+		// Only get Access app info if we know the origin is protected by Access
+		originReq, err := http.NewRequest(http.MethodGet, options.OriginURL, nil)
+		if err != nil {
+			return nil, err
+		}
+
+		appInfo, err := token.GetAppInfo(originReq.URL)
+		if err != nil {
+			return nil, err
+		}
+		options.AppInfo = appInfo
+
 		wsConn, err = createAccessAuthenticatedStream(options, log)
 		if err != nil {
 			return nil, err
@@ -97,7 +85,64 @@ func createWebsocketStream(options *StartOptions, log *zerolog.Logger) (*cfwebso
 		return nil, err
 	}
 
-	return &cfwebsocket.Conn{Conn: wsConn}, nil
+	return &cfwebsocket.GorillaConn{Conn: wsConn}, nil
+}
+
+var stripWebsocketHeaders = []string{
+	"Upgrade",
+	"Connection",
+	"Sec-Websocket-Key",
+	"Sec-Websocket-Version",
+	"Sec-Websocket-Extensions",
+}
+
+// the gorilla websocket library sets its own Upgrade, Connection, Sec-WebSocket-Key,
+// Sec-WebSocket-Version and Sec-Websocket-Extensions headers.
+// https://github.com/gorilla/websocket/blob/master/client.go#L189-L194.
+func websocketHeaders(req *http.Request) http.Header {
+	wsHeaders := make(http.Header)
+	for key, val := range req.Header {
+		wsHeaders[key] = val
+	}
+	// Assume the header keys are in canonical format.
+	for _, header := range stripWebsocketHeaders {
+		wsHeaders.Del(header)
+	}
+	wsHeaders.Set("Host", req.Host) // See TUN-1097
+	return wsHeaders
+}
+
+// clientConnect creates a WebSocket client connection for provided request. Caller is responsible for closing
+// the connection. The response body may not contain the entire response and does
+// not need to be closed by the application.
+func clientConnect(req *http.Request, dialler *websocket.Dialer) (*websocket.Conn, *http.Response, error) {
+	req.URL.Scheme = changeRequestScheme(req.URL)
+	wsHeaders := websocketHeaders(req)
+	if dialler == nil {
+		dialler = &websocket.Dialer{
+			Proxy: http.ProxyFromEnvironment,
+		}
+	}
+	conn, response, err := dialler.Dial(req.URL.String(), wsHeaders)
+	if err != nil {
+		return nil, response, err
+	}
+	return conn, response, nil
+}
+
+// changeRequestScheme is needed as the gorilla websocket library requires the ws scheme.
+// (even though it changes it back to http/https, but ¯\_(ツ)_/¯.)
+func changeRequestScheme(reqURL *url.URL) string {
+	switch reqURL.Scheme {
+	case "https":
+		return "wss"
+	case "http":
+		return "ws"
+	case "":
+		return "ws"
+	default:
+		return reqURL.Scheme
+	}
 }
 
 // createAccessAuthenticatedStream will try load a token from storage and make
@@ -117,11 +162,7 @@ func createAccessAuthenticatedStream(options *StartOptions, log *zerolog.Logger)
 	}
 
 	// Access Token is invalid for some reason. Go through regen flow
-	originReq, err := http.NewRequest(http.MethodGet, options.OriginURL, nil)
-	if err != nil {
-		return nil, err
-	}
-	if err := token.RemoveTokenIfExists(originReq.URL); err != nil {
+	if err := token.RemoveTokenIfExists(options.AppInfo); err != nil {
 		return nil, err
 	}
 	wsConn, resp, err = createAccessWebSocketStream(options, log)
@@ -143,7 +184,7 @@ func createAccessWebSocketStream(options *StartOptions, log *zerolog.Logger) (*w
 	dump, err := httputil.DumpRequest(req, false)
 	log.Debug().Msgf("Access Websocket request: %s", string(dump))
 
-	conn, resp, err := cfwebsocket.ClientConnect(req, nil)
+	conn, resp, err := clientConnect(req, nil)
 
 	if resp != nil {
 		r, err := httputil.DumpResponse(resp, true)

carrier/websocket_test.go

@@ -0,0 +1,123 @@
+package carrier
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"math/rand"
+	"testing"
+	"time"
+
+	gws "github.com/gorilla/websocket"
+	"github.com/rs/zerolog"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/net/websocket"
+
+	"github.com/cloudflare/cloudflared/hello"
+	"github.com/cloudflare/cloudflared/tlsconfig"
+	cfwebsocket "github.com/cloudflare/cloudflared/websocket"
+)
+
+func websocketClientTLSConfig(t *testing.T) *tls.Config {
+	certPool := x509.NewCertPool()
+	helloCert, err := tlsconfig.GetHelloCertificateX509()
+	assert.NoError(t, err)
+	certPool.AddCert(helloCert)
+	assert.NotNil(t, certPool)
+	return &tls.Config{RootCAs: certPool}
+}
+
+func TestWebsocketHeaders(t *testing.T) {
+	req := testRequest(t, "http://example.com", nil)
+	wsHeaders := websocketHeaders(req)
+	for _, header := range stripWebsocketHeaders {
+		assert.Empty(t, wsHeaders[header])
+	}
+	assert.Equal(t, "curl/7.59.0", wsHeaders.Get("User-Agent"))
+}
+
+func TestServe(t *testing.T) {
+	log := zerolog.Nop()
+	shutdownC := make(chan struct{})
+	errC := make(chan error)
+	listener, err := hello.CreateTLSListener("localhost:1111")
+	assert.NoError(t, err)
+	defer listener.Close()
+
+	go func() {
+		errC <- hello.StartHelloWorldServer(&log, listener, shutdownC)
+	}()
+
+	req := testRequest(t, "https://localhost:1111/ws", nil)
+
+	tlsConfig := websocketClientTLSConfig(t)
+	assert.NotNil(t, tlsConfig)
+	d := gws.Dialer{TLSClientConfig: tlsConfig}
+	conn, resp, err := clientConnect(req, &d)
+	assert.NoError(t, err)
+	assert.Equal(t, "websocket", resp.Header.Get("Upgrade"))
+
+	for i := 0; i < 1000; i++ {
+		messageSize := rand.Int()%2048 + 1
+		clientMessage := make([]byte, messageSize)
+		// rand.Read always returns len(clientMessage) and a nil error
+		rand.Read(clientMessage)
+		err = conn.WriteMessage(websocket.BinaryFrame, clientMessage)
+		assert.NoError(t, err)
+
+		messageType, message, err := conn.ReadMessage()
+		assert.NoError(t, err)
+		assert.Equal(t, websocket.BinaryFrame, messageType)
+		assert.Equal(t, clientMessage, message)
+	}
+
+	_ = conn.Close()
+	close(shutdownC)
+	<-errC
+}
+
+func TestWebsocketWrapper(t *testing.T) {
+	listener, err := hello.CreateTLSListener("localhost:0")
+	require.NoError(t, err)
+
+	serverErrorChan := make(chan error)
+	helloSvrCtx, cancelHelloSvr := context.WithCancel(context.Background())
+	defer func() { <-serverErrorChan }()
+	defer cancelHelloSvr()
+	go func() {
+		log := zerolog.Nop()
+		serverErrorChan <- hello.StartHelloWorldServer(&log, listener, helloSvrCtx.Done())
+	}()
+
+	tlsConfig := websocketClientTLSConfig(t)
+	d := gws.Dialer{TLSClientConfig: tlsConfig, HandshakeTimeout: time.Minute}
+	testAddr := fmt.Sprintf("https://%s/ws", listener.Addr().String())
+	req := testRequest(t, testAddr, nil)
+	conn, resp, err := clientConnect(req, &d)
+	require.NoError(t, err)
+	assert.Equal(t, "websocket", resp.Header.Get("Upgrade"))
+
+	// Websocket now connected to test server so lets check our wrapper
+	wrapper := cfwebsocket.GorillaConn{Conn: conn}
+	buf := make([]byte, 100)
+	wrapper.Write([]byte("abc"))
+	n, err := wrapper.Read(buf)
+	require.NoError(t, err)
+	require.Equal(t, n, 3)
+	require.Equal(t, "abc", string(buf[:n]))
+
+	// Test partial read, read 1 of 3 bytes in one read and the other 2 in another read
+	wrapper.Write([]byte("abc"))
+	buf = buf[:1]
+	n, err = wrapper.Read(buf)
+	require.NoError(t, err)
+	require.Equal(t, n, 1)
+	require.Equal(t, "a", string(buf[:n]))
+	buf = buf[:cap(buf)]
+	n, err = wrapper.Read(buf)
+	require.NoError(t, err)
+	require.Equal(t, n, 2)
+	require.Equal(t, "bc", string(buf[:n]))
+}

cfsetup.yaml

@@ -1,4 +1,4 @@
-pinned_go: &pinned_go go=1.15.7-1
+pinned_go: &pinned_go go=1.16.3-1
 pinned_go_fips: &pinned_go_fips go-fips=1.15.5-3
 
 build_dir: &build_dir /cfsetup_build
@@ -14,25 +14,72 @@ stretch: &stretch
       - export GOARCH=amd64
       - export FIPS=true
       - make cloudflared
-  build-deb:
+  build-all-packages: #except osxpkg
     build_dir: *build_dir
     builddeps:
       - *pinned_go_fips
       - build-essential
       - fakeroot
       - rubygem-fpm
+      - rpm
+      - wget
+      # libmsi and libgcab are libraries the wixl binary depends on.
+      - libmsi-dev
+      - libgcab-dev
+    pre-cache:
+      # TODO: https://jira.cfops.it/browse/TUN-4792 Replace this wixl with the official one once msitools supports
+      # environment.
+      - wget https://github.com/sudarshan-reddy/msitools/releases/download/v0.101b/wixl -P /usr/local/bin
+      - chmod a+x /usr/local/bin/wixl
+    post-cache:
+      - ./build-packages.sh
+  github-release-pkgs:
+    build_dir: *build_dir
+    builddeps:
+      - *pinned_go_fips
+      - build-essential
+      - fakeroot
+      - rubygem-fpm
+      - rpm
+      - wget
+      # libmsi and libgcab are libraries the wixl binary depends on.
+      - libmsi-dev
+      - libgcab-dev
+      - python3-setuptools
+      - python3-pip
+    pre-cache:
+      - wget https://github.com/sudarshan-reddy/msitools/releases/download/v0.101b/wixl -P /usr/local/bin
+      - chmod a+x /usr/local/bin/wixl
+      - pip3 install pygithub
+    post-cache:
+      # build all packages and move them to /cfsetup/built_artifacts
+      - ./build-packages.sh
+      # release the packages built and moved to /cfsetup/built_artifacts
+      - make github-release-built-pkgs
+  build-deb:
+    build_dir: *build_dir
+    builddeps: &build_deb_deps
+      - *pinned_go_fips
+      - build-essential
+      - fakeroot
+      - rubygem-fpm
     post-cache:
       - export GOOS=linux
       - export GOARCH=amd64
       - export FIPS=true
       - make cloudflared-deb
+  build-deb-nightly:
+    build_dir: *build_dir
+    builddeps: *build_deb_deps
+    post-cache:
+      - export GOOS=linux
+      - export GOARCH=amd64
+      - export FIPS=true
+      - export NIGHTLY=true
+      - make cloudflared-deb
   build-deb-arm64:
     build_dir: *build_dir
-    builddeps:
-      - *pinned_go
-      - build-essential
-      - fakeroot
-      - rubygem-fpm
+    builddeps: *build_deb_deps
     post-cache:
       - export GOOS=linux
       - export GOARCH=arm64
@@ -194,14 +241,37 @@ stretch: &stretch
     builddeps:
       - *pinned_go_fips
       - build-essential
+      - gotest-to-teamcity
+    pre-cache:
+      - go get golang.org/x/tools/cmd/goimports
+      - go get github.com/BurntSushi/go-sumtype
     post-cache:
       - export GOOS=linux
       - export GOARCH=amd64
       - export FIPS=true
-      # cd to a non-module directory: https://github.com/golang/go/issues/24250
-      - (cd / && go get github.com/BurntSushi/go-sumtype)
       - export PATH="$HOME/go/bin:$PATH"
-      - make test
+      - ./fmt-check.sh
+      - make test | gotest-to-teamcity
+  component-test:
+    build_dir: *build_dir
+    builddeps:
+      - *pinned_go_fips
+      - python3.7
+      - python3-pip
+      - python3-setuptools
+      # procps installs the ps command which is needed in test_sysv_service because the init script
+      # uses ps pid to determine if the agent is running
+      - procps
+    pre-cache-copy-paths:
+      - component-tests/requirements.txt
+    pre-cache:
+      - sudo pip3 install --upgrade -r component-tests/requirements.txt
+    post-cache:
+      # Creates and routes a Named Tunnel for this build. Also constructs config file from env vars.
+      - python3 component-tests/setup.py --type create
+      - pytest component-tests
+      # The Named Tunnel is deleted and its route unprovisioned here.
+      - python3 component-tests/setup.py --type cleanup
   update-homebrew:
     builddeps:
       - openssh-client
@@ -256,8 +326,9 @@ centos-7:
       - https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
     pre-cache:
       - yum install -y fakeroot
-      - wget https://golang.org/dl/go1.15.7.linux-amd64.tar.gz -P /tmp/
-      - tar -C /usr/local -xzf /tmp/go1.15.7.linux-amd64.tar.gz
+      - yum upgrade -y binutils-2.27-44.base.el7.x86_64
+      - wget https://golang.org/dl/go1.16.3.linux-amd64.tar.gz -P /tmp/
+      - tar -C /usr/local -xzf /tmp/go1.16.3.linux-amd64.tar.gz
     post-cache:
       - export PATH=$PATH:/usr/local/go/bin
       - export GOOS=linux
@@ -268,21 +339,11 @@ centos-7:
     builddeps: *el7_builddeps
     pre-cache:
       - yum install -y fakeroot
-      - wget https://golang.org/dl/go1.15.7.linux-amd64.tar.gz -P /tmp/
-      - tar -C /usr/local -xzf /tmp/go1.15.7.linux-amd64.tar.gz
+      - yum upgrade -y binutils-2.27-44.base.el7.x86_64
+      - wget https://golang.org/dl/go1.16.3.linux-amd64.tar.gz -P /tmp/
+      - tar -C /usr/local -xzf /tmp/go1.16.3.linux-amd64.tar.gz
     post-cache:
       - export PATH=$PATH:/usr/local/go/bin
       - export GOOS=linux
       - export GOARCH=amd64
       - make cloudflared-rpm
-
-# cfsetup compose
-default-stack: test_dbconnect
-test_dbconnect:
-  compose:
-    up-args:
-      - --renew-anon-volumes
-      - --abort-on-container-exit
-      - --exit-code-from=cloudflared
-    files:
-      - dbconnect_tests/dbconnect.yaml

cloudflared.wxs

@@ -0,0 +1,62 @@
+<?xml version="1.0"?>
+
+<?if $(var.Platform)="x86"?>
+    <?define Program_Files="ProgramFilesFolder"?>
+<?else?>
+    <?define Program_Files="ProgramFiles64Folder"?>
+<?endif?>
+<?ifndef var.Version?>
+    <?error Undefined Version variable?>
+<?endif?>
+<?ifndef var.Path?>
+    <?error Undefined Path variable?>
+<?endif?>
+
+<Wix xmlns="http://schemas.microsoft.com/wix/2006/wi">
+    <Product Id="35e5e858-9372-4449-bf73-1cd6f7267128" 
+        UpgradeCode="23f90fdd-9328-47ea-ab52-5380855a4b12"
+        Name="cloudflared"
+        Version="$(var.Version)"
+        Manufacturer="cloudflare"
+        Language="1033">
+
+      <Package InstallerVersion="200" Compressed="yes" Comments="Windows Installer Package" InstallScope="perMachine"/>
+
+      <Media Id="1" Cabinet="product.cab" EmbedCab="yes"/>
+
+      <Upgrade Id="23f90fdd-9328-47ea-ab52-5380855a4b12">
+         <UpgradeVersion Minimum="$(var.Version)" OnlyDetect="yes" Property="NEWERVERSIONDETECTED"/>
+         <UpgradeVersion Minimum="2020.8.0" Maximum="$(var.Version)" IncludeMinimum="yes" IncludeMaximum="no"
+                         Property="OLDERVERSIONBEINGUPGRADED"/>
+      </Upgrade>
+      <Condition Message="A newer version of this software is already installed.">NOT NEWERVERSIONDETECTED</Condition>
+
+    <Directory Id="TARGETDIR" Name="SourceDir">
+        <!--This specifies where the cloudflared.exe is moved to in the windows Operation System-->
+        <Directory Id="$(var.Program_Files)">
+            <Directory Id="INSTALLDIR" Name="cloudflared">
+                <Component Id="ApplicationFiles" Guid="35e5e858-9372-4449-bf73-1cd6f7267128">
+                    <File Id="ApplicationFile0" Source="$(var.Path)"/>
+                </Component>
+            </Directory>
+        </Directory>
+        <Component Id="ENVS" Guid="6bb74449-d10d-4f4a-933e-6fc9fa006eae">
+        <!--Set the cloudflared bin location to the Path Environment Variable-->
+         <Environment Id="ENV0"
+           Name="PATH"
+           Value="[INSTALLDIR]."
+           Permanent="no"
+           Part="all"
+           Action="set"
+           System="yes" />
+        </Component>
+    </Directory>
+
+
+    <Feature Id='Complete' Level='1'>
+      <ComponentRef Id="ENVS"/>
+      <ComponentRef Id='ApplicationFiles' />
+    </Feature>
+
+    </Product>
+</Wix>

cmd/cloudflared/access/carrier.go

@@ -1,22 +1,25 @@
 package access
 
 import (
+	"crypto/tls"
+	"fmt"
 	"net/http"
 	"strings"
 
-	"github.com/cloudflare/cloudflared/carrier"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
-	"github.com/cloudflare/cloudflared/h2mux"
-	"github.com/cloudflare/cloudflared/logger"
-	"github.com/cloudflare/cloudflared/validation"
-
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
 	"github.com/urfave/cli/v2"
+
+	"github.com/cloudflare/cloudflared/carrier"
+	"github.com/cloudflare/cloudflared/config"
+	"github.com/cloudflare/cloudflared/logger"
+	"github.com/cloudflare/cloudflared/validation"
 )
 
 const (
-	LogFieldHost = "host"
+	LogFieldHost               = "host"
+	cfAccessClientIDHeader     = "Cf-Access-Client-Id"
+	cfAccessClientSecretHeader = "Cf-Access-Client-Secret"
 )
 
 // StartForwarder starts a client side websocket forward
@@ -29,16 +32,14 @@ func StartForwarder(forwarder config.Forwarder, shutdown <-chan struct{}, log *z
 	// get the headers from the config file and add to the request
 	headers := make(http.Header)
 	if forwarder.TokenClientID != "" {
-		headers.Set(h2mux.CFAccessClientIDHeader, forwarder.TokenClientID)
+		headers.Set(cfAccessClientIDHeader, forwarder.TokenClientID)
 	}
 
 	if forwarder.TokenSecret != "" {
-		headers.Set(h2mux.CFAccessClientSecretHeader, forwarder.TokenSecret)
+		headers.Set(cfAccessClientSecretHeader, forwarder.TokenSecret)
 	}
 
-	if forwarder.Destination != "" {
-		headers.Add(h2mux.CFJumpDestinationHeader, forwarder.Destination)
-	}
+	carrier.SetBastionDest(headers, forwarder.Destination)
 
 	options := &carrier.StartOptions{
 		OriginURL: forwarder.URL,
@@ -46,7 +47,7 @@ func StartForwarder(forwarder config.Forwarder, shutdown <-chan struct{}, log *z
 	}
 
 	// we could add a cmd line variable for this bool if we want the SOCK5 server to be on the client side
-	wsConn := carrier.NewWSConnection(log, false)
+	wsConn := carrier.NewWSConnection(log)
 
 	log.Info().Str(LogFieldHost, validURL.Host).Msg("Start Websocket listener")
 	return carrier.StartForwarder(wsConn, validURL.Host, shutdown, options)
@@ -70,24 +71,41 @@ func ssh(c *cli.Context) error {
 	// get the headers from the cmdline and add them
 	headers := buildRequestHeaders(c.StringSlice(sshHeaderFlag))
 	if c.IsSet(sshTokenIDFlag) {
-		headers.Set(h2mux.CFAccessClientIDHeader, c.String(sshTokenIDFlag))
+		headers.Set(cfAccessClientIDHeader, c.String(sshTokenIDFlag))
 	}
 	if c.IsSet(sshTokenSecretFlag) {
-		headers.Set(h2mux.CFAccessClientSecretHeader, c.String(sshTokenSecretFlag))
+		headers.Set(cfAccessClientSecretHeader, c.String(sshTokenSecretFlag))
 	}
 
-	destination := c.String(sshDestinationFlag)
-	if destination != "" {
-		headers.Add(h2mux.CFJumpDestinationHeader, destination)
-	}
+	carrier.SetBastionDest(headers, c.String(sshDestinationFlag))
 
 	options := &carrier.StartOptions{
 		OriginURL: originURL,
 		Headers:   headers,
+		Host:      hostname,
+	}
+
+	if connectTo := c.String(sshConnectTo); connectTo != "" {
+		parts := strings.Split(connectTo, ":")
+		switch len(parts) {
+		case 1:
+			options.OriginURL = fmt.Sprintf("https://%s", parts[0])
+		case 2:
+			options.OriginURL = fmt.Sprintf("https://%s:%s", parts[0], parts[1])
+		case 3:
+			options.OriginURL = fmt.Sprintf("https://%s:%s", parts[2], parts[1])
+			options.TLSClientConfig = &tls.Config{
+				InsecureSkipVerify: true,
+				ServerName:         parts[0],
+			}
+			log.Warn().Msgf("Using insecure SSL connection because SNI overridden to %s", parts[0])
+		default:
+			return fmt.Errorf("invalid connection override: %s", connectTo)
+		}
 	}
 
 	// we could add a cmd line variable for this bool if we want the SOCK5 server to be on the client side
-	wsConn := carrier.NewWSConnection(log, false)
+	wsConn := carrier.NewWSConnection(log)
 
 	if c.NArg() > 0 || c.IsSet(sshURLFlag) {
 		forwarder, err := config.ValidateUrl(c, true)
@@ -95,7 +113,6 @@ func ssh(c *cli.Context) error {
 			log.Err(err).Msg("Error validating origin URL")
 			return errors.Wrap(err, "error validating origin URL")
 		}
-
 		log.Info().Str(LogFieldHost, forwarder.Host).Msg("Start Websocket listener")
 		err = carrier.StartForwarder(wsConn, forwarder.Host, shutdownC, options)
 		if err != nil {

cmd/cloudflared/access/cmd.go

@@ -2,27 +2,27 @@ package access
 
 import (
 	"fmt"
+	"io"
 	"net/http"
 	"net/url"
 	"os"
+	"os/exec"
 	"strings"
 	"text/template"
 	"time"
 
-	"github.com/cloudflare/cloudflared/carrier"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/shell"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/token"
-	"github.com/cloudflare/cloudflared/h2mux"
-	"github.com/cloudflare/cloudflared/logger"
-	"github.com/cloudflare/cloudflared/sshgen"
-	"github.com/cloudflare/cloudflared/validation"
-
 	"github.com/getsentry/raven-go"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
 	"github.com/urfave/cli/v2"
 	"golang.org/x/net/idna"
+
+	"github.com/cloudflare/cloudflared/carrier"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
+	"github.com/cloudflare/cloudflared/logger"
+	"github.com/cloudflare/cloudflared/sshgen"
+	"github.com/cloudflare/cloudflared/token"
+	"github.com/cloudflare/cloudflared/validation"
 )
 
 const (
@@ -33,12 +33,13 @@ const (
 	sshTokenIDFlag     = "service-token-id"
 	sshTokenSecretFlag = "service-token-secret"
 	sshGenCertFlag     = "short-lived-cert"
+	sshConnectTo       = "connect-to"
 	sshConfigTemplate  = `
 Add to your {{.Home}}/.ssh/config:
 
 Host {{.Hostname}}
 {{- if .ShortLivedCerts}}
-  ProxyCommand bash -c '{{.Cloudflared}} access ssh-gen --hostname %h; ssh -tt %r@cfpipe-{{.Hostname}} >&2 <&1' 
+  ProxyCommand bash -c '{{.Cloudflared}} access ssh-gen --hostname %h; ssh -tt %r@cfpipe-{{.Hostname}} >&2 <&1'
 
 Host cfpipe-{{.Hostname}}
   HostName {{.Hostname}}
@@ -54,7 +55,7 @@ Host cfpipe-{{.Hostname}}
 const sentryDSN = "https://56a9c9fa5c364ab28f34b14f35ea0f1b@sentry.io/189878"
 
 var (
-	shutdownC      chan struct{}
+	shutdownC chan struct{}
 )
 
 // Init will initialize and store vars from the main program
@@ -75,30 +76,24 @@ func Commands() []*cli.Command {
 			Aliases:  []string{"forward"},
 			Category: "Access",
 			Usage:    "access <subcommand>",
-			Description: `Cloudflare Access protects internal resources by securing, authenticating and monitoring access 
-			per-user and by application. With Cloudflare Access, only authenticated users with the required permissions are 
-			able to reach sensitive resources. The commands provided here allow you to interact with Access protected 
+			Description: `Cloudflare Access protects internal resources by securing, authenticating and monitoring access
+			per-user and by application. With Cloudflare Access, only authenticated users with the required permissions are
+			able to reach sensitive resources. The commands provided here allow you to interact with Access protected
 			applications from the command line.`,
 			Subcommands: []*cli.Command{
 				{
 					Name:   "login",
-					Action: cliutil.ErrorHandler(login),
+					Action: cliutil.Action(login),
 					Usage:  "login <url of access application>",
 					Description: `The login subcommand initiates an authentication flow with your identity provider.
 					The subcommand will launch a browser. For headless systems, a url is provided.
 					Once authenticated with your identity provider, the login command will generate a JSON Web Token (JWT)
-					scoped to your identity, the application you intend to reach, and valid for a session duration set by your 
+					scoped to your identity, the application you intend to reach, and valid for a session duration set by your
 					administrator. cloudflared stores the token in local storage.`,
-					Flags: []cli.Flag{
-						&cli.StringFlag{
-							Name:   "url",
-							Hidden: true,
-						},
-					},
 				},
 				{
 					Name:   "curl",
-					Action: cliutil.ErrorHandler(curl),
+					Action: cliutil.Action(curl),
 					Usage:  "curl [--allow-request, -ar] <url> [<curl args>...]",
 					Description: `The curl subcommand wraps curl and automatically injects the JWT into a cf-access-token
 					header when using curl to reach an application behind Access.`,
@@ -107,7 +102,7 @@ func Commands() []*cli.Command {
 				},
 				{
 					Name:        "token",
-					Action:      cliutil.ErrorHandler(generateToken),
+					Action:      cliutil.Action(generateToken),
 					Usage:       "token -app=<url of access application>",
 					ArgsUsage:   "url of Access application",
 					Description: `The token subcommand produces a JWT which can be used to authenticate requests.`,
@@ -119,7 +114,7 @@ func Commands() []*cli.Command {
 				},
 				{
 					Name:        "tcp",
-					Action:      cliutil.ErrorHandler(ssh),
+					Action:      cliutil.Action(ssh),
 					Aliases:     []string{"rdp", "ssh", "smb"},
 					Usage:       "",
 					ArgsUsage:   "",
@@ -162,13 +157,18 @@ func Commands() []*cli.Command {
 						&cli.StringFlag{
 							Name:    logger.LogSSHLevelFlag,
 							Aliases: []string{"loglevel"}, //added to match the tunnel side
-							Usage:   "Application logging level {fatal, error, info, debug}. ",
+							Usage:   "Application logging level {debug, info, warn, error, fatal}. ",
+						},
+						&cli.StringFlag{
+							Name:   sshConnectTo,
+							Hidden: true,
+							Usage:  "Connect to alternate location for testing, value is host, host:port, or sni:port:host",
 						},
 					},
 				},
 				{
 					Name:        "ssh-config",
-					Action:      cliutil.ErrorHandler(sshConfig),
+					Action:      cliutil.Action(sshConfig),
 					Usage:       "",
 					Description: `Prints an example configuration ~/.ssh/config`,
 					Flags: []cli.Flag{
@@ -184,7 +184,7 @@ func Commands() []*cli.Command {
 				},
 				{
 					Name:        "ssh-gen",
-					Action:      cliutil.ErrorHandler(sshGen),
+					Action:      cliutil.Action(sshGen),
 					Usage:       "",
 					Description: `Generates a short lived certificate for given hostname`,
 					Flags: []cli.Flag{
@@ -214,12 +214,18 @@ func login(c *cli.Context) error {
 		log.Error().Msg("Please provide the url of the Access application")
 		return err
 	}
-	if err := verifyTokenAtEdge(appURL, c, log); err != nil {
+
+	appInfo, err := token.GetAppInfo(appURL)
+	if err != nil {
+		return err
+	}
+
+	if err := verifyTokenAtEdge(appURL, appInfo, c, log); err != nil {
 		log.Err(err).Msg("Could not verify token")
 		return err
 	}
 
-	cfdToken, err := token.GetAppTokenIfExists(appURL)
+	cfdToken, err := token.GetAppTokenIfExists(appInfo)
 	if err != nil {
 		fmt.Fprintln(os.Stderr, "Unable to find token for provided application.")
 		return err
@@ -261,13 +267,17 @@ func curl(c *cli.Context) error {
 		return err
 	}
 
-	tok, err := token.GetAppTokenIfExists(appURL)
+	appInfo, err := token.GetAppInfo(appURL)
+	if err != nil {
+		return err
+	}
+	tok, err := token.GetAppTokenIfExists(appInfo)
 	if err != nil || tok == "" {
 		if allowRequest {
 			log.Info().Msg("You don't have an Access token set. Please run access token <access application> to fetch one.")
-			return shell.Run("curl", cmdArgs...)
+			return run("curl", cmdArgs...)
 		}
-		tok, err = token.FetchToken(appURL, log)
+		tok, err = token.FetchToken(appURL, appInfo, log)
 		if err != nil {
 			log.Err(err).Msg("Failed to refresh token")
 			return err
@@ -275,8 +285,29 @@ func curl(c *cli.Context) error {
 	}
 
 	cmdArgs = append(cmdArgs, "-H")
-	cmdArgs = append(cmdArgs, fmt.Sprintf("%s: %s", h2mux.CFAccessTokenHeader, tok))
-	return shell.Run("curl", cmdArgs...)
+	cmdArgs = append(cmdArgs, fmt.Sprintf("%s: %s", carrier.CFAccessTokenHeader, tok))
+	return run("curl", cmdArgs...)
+}
+
+// run kicks off a shell task and pipe the results to the respective std pipes
+func run(cmd string, args ...string) error {
+	c := exec.Command(cmd, args...)
+	stderr, err := c.StderrPipe()
+	if err != nil {
+		return err
+	}
+	go func() {
+		io.Copy(os.Stderr, stderr)
+	}()
+
+	stdout, err := c.StdoutPipe()
+	if err != nil {
+		return err
+	}
+	go func() {
+		io.Copy(os.Stdout, stdout)
+	}()
+	return c.Run()
 }
 
 // token dumps provided token to stdout
@@ -284,12 +315,17 @@ func generateToken(c *cli.Context) error {
 	if err := raven.SetDSN(sentryDSN); err != nil {
 		return err
 	}
-	appURL, err := url.Parse(c.String("app"))
+	appURL, err := url.Parse(ensureURLScheme(c.String("app")))
 	if err != nil || c.NumFlags() < 1 {
 		fmt.Fprintln(os.Stderr, "Please provide a url.")
 		return err
 	}
-	tok, err := token.GetAppTokenIfExists(appURL)
+
+	appInfo, err := token.GetAppInfo(appURL)
+	if err != nil {
+		return err
+	}
+	tok, err := token.GetAppTokenIfExists(appInfo)
 	if err != nil || tok == "" {
 		fmt.Fprintln(os.Stderr, "Unable to find token for provided application. Please run login command to generate token.")
 		return err
@@ -340,7 +376,12 @@ func sshGen(c *cli.Context) error {
 	// this fetchToken function mutates the appURL param. We should refactor that
 	fetchTokenURL := &url.URL{}
 	*fetchTokenURL = *originURL
-	cfdToken, err := token.FetchTokenWithRedirect(fetchTokenURL, log)
+
+	appInfo, err := token.GetAppInfo(fetchTokenURL)
+	if err != nil {
+		return err
+	}
+	cfdToken, err := token.FetchTokenWithRedirect(fetchTokenURL, appInfo, log)
 	if err != nil {
 		return err
 	}
@@ -352,7 +393,7 @@ func sshGen(c *cli.Context) error {
 	return nil
 }
 
-// getAppURL will pull the appURL needed for fetching a user's Access token
+// getAppURL will pull the request URL needed for fetching a user's Access token
 func getAppURL(cmdArgs []string, log *zerolog.Logger) (*url.URL, error) {
 	if len(cmdArgs) < 1 {
 		log.Error().Msg("Please provide a valid URL as the first argument to curl.")
@@ -427,15 +468,15 @@ func isFileThere(candidate string) bool {
 // verifyTokenAtEdge checks for a token on disk, or generates a new one.
 // Then makes a request to to the origin with the token to ensure it is valid.
 // Returns nil if token is valid.
-func verifyTokenAtEdge(appUrl *url.URL, c *cli.Context, log *zerolog.Logger) error {
+func verifyTokenAtEdge(appUrl *url.URL, appInfo *token.AppInfo, c *cli.Context, log *zerolog.Logger) error {
 	headers := buildRequestHeaders(c.StringSlice(sshHeaderFlag))
 	if c.IsSet(sshTokenIDFlag) {
-		headers.Add(h2mux.CFAccessClientIDHeader, c.String(sshTokenIDFlag))
+		headers.Add(cfAccessClientIDHeader, c.String(sshTokenIDFlag))
 	}
 	if c.IsSet(sshTokenSecretFlag) {
-		headers.Add(h2mux.CFAccessClientSecretHeader, c.String(sshTokenSecretFlag))
+		headers.Add(cfAccessClientSecretHeader, c.String(sshTokenSecretFlag))
 	}
-	options := &carrier.StartOptions{OriginURL: appUrl.String(), Headers: headers}
+	options := &carrier.StartOptions{AppInfo: appInfo, OriginURL: appUrl.String(), Headers: headers}
 
 	if valid, err := isTokenValid(options, log); err != nil {
 		return err
@@ -443,7 +484,7 @@ func verifyTokenAtEdge(appUrl *url.URL, c *cli.Context, log *zerolog.Logger) err
 		return nil
 	}
 
-	if err := token.RemoveTokenIfExists(appUrl); err != nil {
+	if err := token.RemoveTokenIfExists(appInfo); err != nil {
 		return err
 	}
 

cmd/cloudflared/app_forward_service.go

@@ -1,10 +1,10 @@
 package main
 
 import (
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/access"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
-
 	"github.com/rs/zerolog"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/access"
+	"github.com/cloudflare/cloudflared/config"
 )
 
 // ForwardServiceType is used to identify what kind of overwatch service this is

cmd/cloudflared/app_resolver_service.go

@@ -1,18 +1,19 @@
 package main
 
 import (
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
-	"github.com/cloudflare/cloudflared/tunneldns"
-
 	"github.com/rs/zerolog"
+
+	"github.com/cloudflare/cloudflared/config"
+	"github.com/cloudflare/cloudflared/tunneldns"
 )
 
 const (
 	// ResolverServiceType is used to identify what kind of overwatch service this is
 	ResolverServiceType = "resolver"
 
-	LogFieldResolverAddress = "resolverAddress"
-	LogFieldResolverPort    = "resolverPort"
+	LogFieldResolverAddress          = "resolverAddress"
+	LogFieldResolverPort             = "resolverPort"
+	LogFieldResolverMaxUpstreamConns = "resolverMaxUpstreamConns"
 )
 
 // ResolverService is used to wrap the tunneldns package's DNS over HTTP
@@ -57,7 +58,7 @@ func (s *ResolverService) Shutdown() {
 func (s *ResolverService) Run() error {
 	// create a listener
 	l, err := tunneldns.CreateListener(s.resolver.AddressOrDefault(), s.resolver.PortOrDefault(),
-		s.resolver.UpstreamsOrDefault(), s.resolver.BootstrapsOrDefault(), s.log)
+		s.resolver.UpstreamsOrDefault(), s.resolver.BootstrapsOrDefault(), s.resolver.MaxUpstreamConnectionsOrDefault(), s.log)
 	if err != nil {
 		return err
 	}
@@ -74,6 +75,7 @@ func (s *ResolverService) Run() error {
 	resolverLog := s.log.With().
 		Str(LogFieldResolverAddress, s.resolver.AddressOrDefault()).
 		Uint16(LogFieldResolverPort, s.resolver.PortOrDefault()).
+		Int(LogFieldResolverMaxUpstreamConns, s.resolver.MaxUpstreamConnectionsOrDefault()).
 		Logger()
 
 	resolverLog.Info().Msg("Starting resolver")

cmd/cloudflared/app_service.go

@@ -1,10 +1,10 @@
 package main
 
 import (
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
-	"github.com/cloudflare/cloudflared/overwatch"
-
 	"github.com/rs/zerolog"
+
+	"github.com/cloudflare/cloudflared/config"
+	"github.com/cloudflare/cloudflared/overwatch"
 )
 
 // AppService is the main service that runs when no command lines flags are passed to cloudflared

cmd/cloudflared/buildinfo/build_info.go

@@ -1,8 +1,10 @@
 package buildinfo
 
 import (
-	"github.com/rs/zerolog"
+	"fmt"
 	"runtime"
+
+	"github.com/rs/zerolog"
 )
 
 type BuildInfo struct {
@@ -25,3 +27,7 @@ func (bi *BuildInfo) Log(log *zerolog.Logger) {
 	log.Info().Msgf("Version %s", bi.CloudflaredVersion)
 	log.Info().Msgf("GOOS: %s, GOVersion: %s, GoArch: %s", bi.GoOS, bi.GoVersion, bi.GoArch)
 }
+
+func (bi *BuildInfo) OSArch() string {
+	return fmt.Sprintf("%s_%s", bi.GoOS, bi.GoArch)
+}

cmd/cloudflared/cliutil/deprecated.go

@@ -0,0 +1,21 @@
+package cliutil
+
+import (
+	"fmt"
+
+	"github.com/urfave/cli/v2"
+)
+
+func RemovedCommand(name string) *cli.Command {
+	return &cli.Command{
+		Name: name,
+		Action: func(context *cli.Context) error {
+			return cli.Exit(
+				fmt.Sprintf("%s command is no longer supported by cloudflared. Consult Argo Tunnel documentation for possible alternative solutions.", name),
+				-1,
+			)
+		},
+		Description: fmt.Sprintf("%s is deprecated", name),
+		Hidden:      true,
+	}
+}

cmd/cloudflared/cliutil/errors.go

@@ -2,6 +2,7 @@ package cliutil
 
 import (
 	"fmt"
+
 	"github.com/urfave/cli/v2"
 )
 
@@ -21,7 +22,7 @@ func UsageError(format string, args ...interface{}) error {
 }
 
 // Ensures exit with error code if actionFunc returns an error
-func ErrorHandler(actionFunc cli.ActionFunc) cli.ActionFunc {
+func WithErrorHandler(actionFunc cli.ActionFunc) cli.ActionFunc {
 	return func(ctx *cli.Context) error {
 		err := actionFunc(ctx)
 		if err != nil {

cmd/cloudflared/cliutil/handler.go

@@ -0,0 +1,50 @@
+package cliutil
+
+import (
+	"github.com/urfave/cli/v2"
+	"github.com/urfave/cli/v2/altsrc"
+
+	"github.com/cloudflare/cloudflared/config"
+	"github.com/cloudflare/cloudflared/logger"
+)
+
+func Action(actionFunc cli.ActionFunc) cli.ActionFunc {
+	return WithErrorHandler(actionFunc)
+}
+
+func ConfiguredAction(actionFunc cli.ActionFunc) cli.ActionFunc {
+	// Adapt actionFunc to the type signature required by ConfiguredActionWithWarnings
+	f := func(context *cli.Context, _ string) error {
+		return actionFunc(context)
+	}
+
+	return ConfiguredActionWithWarnings(f)
+}
+
+// Just like ConfiguredAction, but accepts a second parameter with configuration warnings.
+func ConfiguredActionWithWarnings(actionFunc func(*cli.Context, string) error) cli.ActionFunc {
+	return WithErrorHandler(func(c *cli.Context) error {
+		warnings, err := setFlagsFromConfigFile(c)
+		if err != nil {
+			return err
+		}
+		return actionFunc(c, warnings)
+	})
+}
+
+func setFlagsFromConfigFile(c *cli.Context) (configWarnings string, err error) {
+	const errorExitCode = 1
+	log := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
+	inputSource, warnings, err := config.ReadConfigFile(c, log)
+	if err != nil {
+		if err == config.ErrNoConfigFile {
+			return "", nil
+		}
+		return "", cli.Exit(err, errorExitCode)
+	}
+
+	if err := altsrc.ApplyInputSource(c, inputSource); err != nil {
+		return "", cli.Exit(err, errorExitCode)
+	}
+	return warnings, nil
+}

cmd/cloudflared/generic_service.go

@@ -8,6 +8,6 @@ import (
 	cli "github.com/urfave/cli/v2"
 )
 
-func runApp(app *cli.App, shutdownC, graceShutdownC chan struct{}) {
+func runApp(app *cli.App, graceShutdownC chan struct{}) {
 	app.Run(os.Args)
 }

cmd/cloudflared/linux_service.go

@@ -7,13 +7,13 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/tunnel"
-	"github.com/cloudflare/cloudflared/logger"
-
 	"github.com/rs/zerolog"
 	"github.com/urfave/cli/v2"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/tunnel"
+	"github.com/cloudflare/cloudflared/config"
+	"github.com/cloudflare/cloudflared/logger"
 )
 
 func runApp(app *cli.App, graceShutdownC chan struct{}) {
@@ -24,7 +24,7 @@ func runApp(app *cli.App, graceShutdownC chan struct{}) {
 			{
 				Name:   "install",
 				Usage:  "Install Argo Tunnel as a system service",
-				Action: cliutil.ErrorHandler(installLinuxService),
+				Action: cliutil.ConfiguredAction(installLinuxService),
 				Flags: []cli.Flag{
 					&cli.BoolFlag{
 						Name:  "legacy",
@@ -35,7 +35,7 @@ func runApp(app *cli.App, graceShutdownC chan struct{}) {
 			{
 				Name:   "uninstall",
 				Usage:  "Uninstall the Argo Tunnel service",
-				Action: cliutil.ErrorHandler(uninstallLinuxService),
+				Action: cliutil.ConfiguredAction(uninstallLinuxService),
 			},
 		},
 	})
@@ -238,7 +238,7 @@ func installLinuxService(c *cli.Context) error {
 			"--origincert", serviceConfigDir + "/" + serviceCredentialFile,
 		}
 	} else {
-		src, err := config.ReadConfigFile(c, log)
+		src, _, err := config.ReadConfigFile(c, log)
 		if err != nil {
 			return err
 		}

cmd/cloudflared/macos_service.go

@@ -6,11 +6,11 @@ import (
 	"fmt"
 	"os"
 
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
-	"github.com/cloudflare/cloudflared/logger"
-
 	"github.com/pkg/errors"
 	"github.com/urfave/cli/v2"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
+	"github.com/cloudflare/cloudflared/logger"
 )
 
 const (
@@ -25,12 +25,12 @@ func runApp(app *cli.App, graceShutdownC chan struct{}) {
 			{
 				Name:   "install",
 				Usage:  "Install Argo Tunnel as an user launch agent",
-				Action: cliutil.ErrorHandler(installLaunchd),
+				Action: cliutil.ConfiguredAction(installLaunchd),
 			},
 			{
 				Name:   "uninstall",
 				Usage:  "Uninstall the Argo Tunnel launch agent",
-				Action: cliutil.ErrorHandler(uninstallLaunchd),
+				Action: cliutil.ConfiguredAction(uninstallLaunchd),
 			},
 		},
 	})

cmd/cloudflared/main.go

@@ -2,24 +2,26 @@ package main
 
 import (
 	"fmt"
+	"math/rand"
 	"strings"
 	"time"
 
+	"github.com/getsentry/raven-go"
+	homedir "github.com/mitchellh/go-homedir"
+	"github.com/pkg/errors"
+	"github.com/urfave/cli/v2"
+	"go.uber.org/automaxprocs/maxprocs"
+
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/access"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/proxydns"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/tunnel"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/updater"
+	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/metrics"
 	"github.com/cloudflare/cloudflared/overwatch"
-	"github.com/cloudflare/cloudflared/tunneldns"
 	"github.com/cloudflare/cloudflared/watcher"
-
-	"github.com/getsentry/raven-go"
-	"github.com/mitchellh/go-homedir"
-	"github.com/pkg/errors"
-	"github.com/urfave/cli/v2"
 )
 
 const (
@@ -43,8 +45,10 @@ var (
 )
 
 func main() {
+	rand.Seed(time.Now().UnixNano())
 	metrics.RegisterBuildInfo(BuildTime, Version)
 	raven.SetRelease(Version)
+	maxprocs.Set()
 
 	// Graceful shutdown channel used by the app. When closed, app must terminate gracefully.
 	// Windows service manager closes this channel when it receives stop command.
@@ -75,7 +79,6 @@ func main() {
 	See https://developers.cloudflare.com/argo-tunnel/ for more in-depth documentation.`
 	app.Flags = flags()
 	app.Action = action(graceShutdownC)
-	app.Before = tunnel.SetFlagsFromConfigFile
 	app.Commands = commands(cli.ShowVersion)
 
 	tunnel.Init(Version, graceShutdownC) // we need this to support the tunnel sub command...
@@ -88,7 +91,7 @@ func commands(version func(c *cli.Context)) []*cli.Command {
 	cmds := []*cli.Command{
 		{
 			Name:   "update",
-			Action: cliutil.ErrorHandler(updater.Update),
+			Action: cliutil.ConfiguredAction(updater.Update),
 			Usage:  "Update the agent if a new version exists",
 			Flags: []cli.Flag{
 				&cli.BoolFlag{
@@ -115,7 +118,7 @@ func commands(version func(c *cli.Context)) []*cli.Command {
 If a new version exists, updates the agent binary and quits.
 Otherwise, does nothing.
 
-To determine if an update happened in a script, check for error code 64.`,
+To determine if an update happened in a script, check for error code 11.`,
 		},
 		{
 			Name: "version",
@@ -128,7 +131,7 @@ To determine if an update happened in a script, check for error code 64.`,
 		},
 	}
 	cmds = append(cmds, tunnel.Commands()...)
-	cmds = append(cmds, tunneldns.Command(false))
+	cmds = append(cmds, proxydns.Command(false))
 	cmds = append(cmds, access.Commands()...)
 	return cmds
 }
@@ -143,7 +146,7 @@ func isEmptyInvocation(c *cli.Context) bool {
 }
 
 func action(graceShutdownC chan struct{}) cli.ActionFunc {
-	return cliutil.ErrorHandler(func(c *cli.Context) (err error) {
+	return cliutil.ConfiguredAction(func(c *cli.Context) (err error) {
 		if isEmptyInvocation(c) {
 			return handleServiceMode(c, graceShutdownC)
 		}

cmd/cloudflared/proxydns/cmd.go

@@ -0,0 +1,114 @@
+package proxydns
+
+import (
+	"net"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"github.com/urfave/cli/v2"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
+	"github.com/cloudflare/cloudflared/logger"
+	"github.com/cloudflare/cloudflared/metrics"
+	"github.com/cloudflare/cloudflared/tunneldns"
+)
+
+func Command(hidden bool) *cli.Command {
+	return &cli.Command{
+		Name:   "proxy-dns",
+		Action: cliutil.ConfiguredAction(Run),
+
+		Usage: "Run a DNS over HTTPS proxy server.",
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:    "metrics",
+				Value:   "localhost:",
+				Usage:   "Listen address for metrics reporting.",
+				EnvVars: []string{"TUNNEL_METRICS"},
+			},
+			&cli.StringFlag{
+				Name:    "address",
+				Usage:   "Listen address for the DNS over HTTPS proxy server.",
+				Value:   "localhost",
+				EnvVars: []string{"TUNNEL_DNS_ADDRESS"},
+			},
+			// Note TUN-3758 , we use Int because UInt is not supported with altsrc
+			&cli.IntFlag{
+				Name:    "port",
+				Usage:   "Listen on given port for the DNS over HTTPS proxy server.",
+				Value:   53,
+				EnvVars: []string{"TUNNEL_DNS_PORT"},
+			},
+			&cli.StringSliceFlag{
+				Name:    "upstream",
+				Usage:   "Upstream endpoint URL, you can specify multiple endpoints for redundancy.",
+				Value:   cli.NewStringSlice("https://1.1.1.1/dns-query", "https://1.0.0.1/dns-query"),
+				EnvVars: []string{"TUNNEL_DNS_UPSTREAM"},
+			},
+			&cli.StringSliceFlag{
+				Name:    "bootstrap",
+				Usage:   "bootstrap endpoint URL, you can specify multiple endpoints for redundancy.",
+				Value:   cli.NewStringSlice("https://162.159.36.1/dns-query", "https://162.159.46.1/dns-query", "https://[2606:4700:4700::1111]/dns-query", "https://[2606:4700:4700::1001]/dns-query"),
+				EnvVars: []string{"TUNNEL_DNS_BOOTSTRAP"},
+			},
+			&cli.IntFlag{
+				Name:    "max-upstream-conns",
+				Usage:   "Maximum concurrent connections to upstream. Setting to 0 means unlimited.",
+				Value:   tunneldns.MaxUpstreamConnsDefault,
+				EnvVars: []string{"TUNNEL_DNS_MAX_UPSTREAM_CONNS"},
+			},
+		},
+		ArgsUsage: " ", // can't be the empty string or we get the default output
+		Hidden:    hidden,
+	}
+}
+
+// Run implements a foreground runner
+func Run(c *cli.Context) error {
+	log := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
+
+	metricsListener, err := net.Listen("tcp", c.String("metrics"))
+	if err != nil {
+		log.Fatal().Err(err).Msg("Failed to open the metrics listener")
+	}
+
+	go metrics.ServeMetrics(metricsListener, nil, nil, "", log)
+
+	listener, err := tunneldns.CreateListener(
+		c.String("address"),
+		// Note TUN-3758 , we use Int because UInt is not supported with altsrc
+		uint16(c.Int("port")),
+		c.StringSlice("upstream"),
+		c.StringSlice("bootstrap"),
+		c.Int("max-upstream-conns"),
+		log,
+	)
+
+	if err != nil {
+		log.Err(err).Msg("Failed to create the listeners")
+		return err
+	}
+
+	// Try to start the server
+	readySignal := make(chan struct{})
+	err = listener.Start(readySignal)
+	if err != nil {
+		log.Err(err).Msg("Failed to start the listeners")
+		return listener.Stop()
+	}
+	<-readySignal
+
+	// Wait for signal
+	signals := make(chan os.Signal, 10)
+	signal.Notify(signals, syscall.SIGTERM, syscall.SIGINT)
+	defer signal.Stop(signals)
+	<-signals
+
+	// Shut down server
+	err = listener.Stop()
+	if err != nil {
+		log.Err(err).Msg("failed to stop")
+	}
+	return err
+}

cmd/cloudflared/service_template.go

@@ -10,9 +10,9 @@ import (
 	"os/exec"
 	"text/template"
 
-	"github.com/mitchellh/go-homedir"
+	homedir "github.com/mitchellh/go-homedir"
 
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/config"
 )
 
 type ServiceTemplate struct {

cmd/cloudflared/shell/shell.go

@@ -1,33 +0,0 @@
-package shell
-
-import (
-	"io"
-	"os"
-	"os/exec"
-)
-
-// OpenBrowser opens the specified URL in the default browser of the user
-func OpenBrowser(url string) error {
-	return getBrowserCmd(url).Start()
-}
-
-// Run will kick off a shell task and pipe the results to the respective std pipes
-func Run(cmd string, args ...string) error {
-	c := exec.Command(cmd, args...)
-	stderr, err := c.StderrPipe()
-	if err != nil {
-		return err
-	}
-	go func() {
-		io.Copy(os.Stderr, stderr)
-	}()
-
-	stdout, err := c.StdoutPipe()
-	if err != nil {
-		return err
-	}
-	go func() {
-		io.Copy(os.Stdout, stdout)
-	}()
-	return c.Run()
-}

cmd/cloudflared/tunnel/cmd.go

@@ -7,19 +7,27 @@ import (
 	"io/ioutil"
 	"net/url"
 	"os"
-	"reflect"
 	"runtime/trace"
 	"strings"
 	"sync"
 	"time"
 
+	"github.com/coreos/go-systemd/daemon"
+	"github.com/facebookgo/grace/gracenet"
+	"github.com/getsentry/raven-go"
+	homedir "github.com/mitchellh/go-homedir"
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog"
+	"github.com/urfave/cli/v2"
+	"github.com/urfave/cli/v2/altsrc"
+
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/proxydns"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/ui"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/updater"
+	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/connection"
-	"github.com/cloudflare/cloudflared/dbconnect"
 	"github.com/cloudflare/cloudflared/ingress"
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/metrics"
@@ -28,15 +36,6 @@ import (
 	"github.com/cloudflare/cloudflared/tlsconfig"
 	"github.com/cloudflare/cloudflared/tunneldns"
 	"github.com/cloudflare/cloudflared/tunnelstore"
-
-	"github.com/coreos/go-systemd/daemon"
-	"github.com/facebookgo/grace/gracenet"
-	"github.com/getsentry/raven-go"
-	"github.com/mitchellh/go-homedir"
-	"github.com/pkg/errors"
-	"github.com/rs/zerolog"
-	"github.com/urfave/cli/v2"
-	"github.com/urfave/cli/v2/altsrc"
 )
 
 const (
@@ -75,8 +74,8 @@ const (
 	// uiFlag is to enable launching cloudflared in interactive UI mode
 	uiFlag = "ui"
 
-	debugLevelWarning = "At debug level, request URL, method, protocol, content legnth and header will be logged. " +
-		"Response status, content length and header will also be logged in debug level."
+	debugLevelWarning = "At debug level cloudflared will log request URL, method, protocol, content length, as well as, all request and response headers. " +
+		"This can expose sensitive information in your logs."
 
 	LogFieldCommand             = "command"
 	LogFieldExpandedPath        = "expandedPath"
@@ -88,6 +87,10 @@ const (
 var (
 	graceShutdownC chan struct{}
 	version        string
+
+	routeFailMsg = fmt.Sprintf("failed to provision routing, please create it manually via Cloudflare dashboard or UI; "+
+		"most likely you already have a conflicting record there. You can also rerun this command with --%s to overwrite "+
+		"any existing DNS records for this hostname.", overwriteDNSFlag)
 )
 
 func Flags() []cli.Flag {
@@ -101,27 +104,27 @@ func Commands() []*cli.Command {
 		buildRouteCommand(),
 		buildRunCommand(),
 		buildListCommand(),
+		buildInfoCommand(),
 		buildIngressSubcommand(),
 		buildDeleteCommand(),
 		buildCleanupCommand(),
 		// for compatibility, allow following as tunnel subcommands
-		tunneldns.Command(true),
-		dbConnectCmd(),
+		proxydns.Command(true),
+		cliutil.RemovedCommand("db-connect"),
 	}
 
 	return []*cli.Command{
 		buildTunnelCommand(subcommands),
 		// for compatibility, allow following as top-level subcommands
 		buildLoginSubcommand(true),
-		dbConnectCmd(),
+		cliutil.RemovedCommand("db-connect"),
 	}
 }
 
 func buildTunnelCommand(subcommands []*cli.Command) *cli.Command {
 	return &cli.Command{
 		Name:      "tunnel",
-		Action:    cliutil.ErrorHandler(TunnelCommand),
-		Before:    SetFlagsFromConfigFile,
+		Action:    cliutil.ConfiguredAction(TunnelCommand),
 		Category:  "Tunnel",
 		Usage:     "Make a locally-running web service accessible over the internet using Argo Tunnel.",
 		ArgsUsage: " ",
@@ -160,6 +163,12 @@ func TunnelCommand(c *cli.Context) error {
 		return fmt.Errorf("Use `cloudflared tunnel run` to start tunnel %s", ref)
 	}
 
+	// Unauthenticated named tunnel on <random>.<quick-tunnels-service>.com
+	// For now, default to legacy setup unless quick-service is specified
+	if c.String("hostname") == "" && c.String("quick-service") != "" {
+		return RunQuickTunnel(sc)
+	}
+
 	// Start a classic tunnel
 	return runClassicTunnel(sc)
 }
@@ -182,7 +191,7 @@ func runAdhocNamedTunnel(sc *subcommandContext, name, credentialsOutputPath stri
 
 	if r, ok := routeFromFlag(sc.c); ok {
 		if res, err := sc.route(tunnel.ID, r); err != nil {
-			sc.log.Err(err).Msg("failed to create route, please create it manually")
+			sc.log.Err(err).Str("route", r.String()).Msg(routeFailMsg)
 		} else {
 			sc.log.Info().Msg(res.SuccessSummary())
 		}
@@ -200,12 +209,12 @@ func runClassicTunnel(sc *subcommandContext) error {
 	return StartServer(sc.c, version, nil, sc.log, sc.isUIEnabled)
 }
 
-func routeFromFlag(c *cli.Context) (tunnelstore.Route, bool) {
+func routeFromFlag(c *cli.Context) (route tunnelstore.Route, ok bool) {
 	if hostname := c.String("hostname"); hostname != "" {
 		if lbPool := c.String("lb-pool"); lbPool != "" {
 			return tunnelstore.NewLBRoute(hostname, lbPool), true
 		}
-		return tunnelstore.NewDNSRoute(hostname), true
+		return tunnelstore.NewDNSRoute(hostname, c.Bool(overwriteDNSFlagName)), true
 	}
 	return nil, false
 }
@@ -287,19 +296,17 @@ func StartServer(
 	}
 
 	// update needs to be after DNS proxy is up to resolve equinox server address
-	if updater.IsAutoupdateEnabled(c, log) {
-		autoupdateFreq := c.Duration("autoupdate-freq")
-		log.Info().Dur("autoupdateFreq", autoupdateFreq).Msg("Autoupdate frequency is set")
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			autoupdater := updater.NewAutoUpdater(c.Duration("autoupdate-freq"), &listeners, log)
-			errC <- autoupdater.Run(ctx)
-		}()
-	}
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		autoupdater := updater.NewAutoUpdater(
+			c.Bool("no-autoupdate"), c.Duration("autoupdate-freq"), &listeners, log,
+		)
+		errC <- autoupdater.Run(ctx)
+	}()
 
 	// Serve DNS proxy stand-alone if no hostname or tag or app is going to run
-	if dnsProxyStandAlone(c) {
+	if dnsProxyStandAlone(c, namedTunnel) {
 		connectedSignal.Notify()
 		// no grace period, handle SIGINT/SIGTERM immediately
 		return waitToShutdown(&wg, cancel, errC, graceShutdownC, 0, log)
@@ -317,6 +324,15 @@ func StartServer(
 
 	observer := connection.NewObserver(log, logTransport, isUIEnabled)
 
+	// Send Quick Tunnel URL to UI if applicable
+	var quickTunnelURL string
+	if namedTunnel != nil {
+		quickTunnelURL = namedTunnel.QuickTunnelUrl
+	}
+	if quickTunnelURL != "" {
+		observer.SendURL(quickTunnelURL)
+	}
+
 	tunnelConfig, ingressRules, err := prepareTunnelConfig(c, buildInfo, version, log, logTransport, observer, namedTunnel)
 	if err != nil {
 		log.Err(err).Msg("Couldn't start tunnel")
@@ -334,7 +350,7 @@ func StartServer(
 		defer wg.Done()
 		readinessServer := metrics.NewReadyServer(log)
 		observer.RegisterSink(readinessServer)
-		errC <- metrics.ServeMetrics(metricsListener, ctx.Done(), readinessServer, log)
+		errC <- metrics.ServeMetrics(metricsListener, ctx.Done(), readinessServer, quickTunnelURL, log)
 	}()
 
 	if err := ingressRules.StartOrigins(&wg, log, ctx.Done(), errC); err != nil {
@@ -371,26 +387,6 @@ func StartServer(
 	return waitToShutdown(&wg, cancel, errC, graceShutdownC, c.Duration("grace-period"), log)
 }
 
-func SetFlagsFromConfigFile(c *cli.Context) error {
-	const exitCode = 1
-	log := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
-	inputSource, err := config.ReadConfigFile(c, log)
-	if err != nil {
-		if err == config.ErrNoConfigFile {
-			return nil
-		}
-		return cli.Exit(err, exitCode)
-	}
-	targetFlags := c.Command.Flags
-	if c.Command.Name == "" {
-		targetFlags = c.App.Flags
-	}
-	if err := altsrc.ApplyInputSourceValues(c, inputSource, targetFlags); err != nil {
-		return cli.Exit(err, exitCode)
-	}
-	return nil
-}
-
 func waitToShutdown(wg *sync.WaitGroup,
 	cancelServerContext func(),
 	errC <-chan error,
@@ -479,60 +475,6 @@ func addPortIfMissing(uri *url.URL, port int) string {
 	return fmt.Sprintf("%s:%d", uri.Hostname(), port)
 }
 
-func dbConnectCmd() *cli.Command {
-	cmd := dbconnect.Cmd()
-
-	// Append the tunnel commands so users can customize the daemon settings.
-	cmd.Flags = appendFlags(Flags(), cmd.Flags...)
-
-	// Override before to run tunnel validation before dbconnect validation.
-	cmd.Before = func(c *cli.Context) error {
-		err := SetFlagsFromConfigFile(c)
-		if err == nil {
-			err = dbconnect.CmdBefore(c)
-		}
-		return err
-	}
-
-	// Override action to setup the Proxy, then if successful, start the tunnel daemon.
-	cmd.Action = cliutil.ErrorHandler(func(c *cli.Context) error {
-		err := dbconnect.CmdAction(c)
-		if err == nil {
-			err = TunnelCommand(c)
-		}
-		return err
-	})
-
-	return cmd
-}
-
-// appendFlags will append extra flags to a slice of flags.
-//
-// The cli package will panic if two flags exist with the same name,
-// so if extraFlags contains a flag that was already defined, modify the
-// original flags to use the extra version.
-func appendFlags(flags []cli.Flag, extraFlags ...cli.Flag) []cli.Flag {
-	for _, extra := range extraFlags {
-		var found bool
-
-		// Check if an extra flag overrides an existing flag.
-		for i, flag := range flags {
-			if reflect.DeepEqual(extra.Names(), flag.Names()) {
-				flags[i] = extra
-				found = true
-				break
-			}
-		}
-
-		// Append the extra flag if it has nothing to override.
-		if !found {
-			flags = append(flags, extra)
-		}
-	}
-
-	return flags
-}
-
 func tunnelFlags(shouldHide bool) []cli.Flag {
 	flags := configureCloudflaredFlags(shouldHide)
 	flags = append(flags, configureProxyFlags(shouldHide)...)
@@ -542,7 +484,7 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 		credentialsFileFlag,
 		altsrc.NewBoolFlag(&cli.BoolFlag{
 			Name:   "is-autoupdated",
-			Usage:  "Signal the new process that Argo Tunnel client has been autoupdated",
+			Usage:  "Signal the new process that Argo Tunnel connector has been autoupdated",
 			Value:  false,
 			Hidden: true,
 		}),
@@ -642,10 +584,10 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 		}),
 		altsrc.NewDurationFlag(&cli.DurationFlag{
 			Name:    "grace-period",
-			Usage:   "Duration to accept new requests after cloudflared receives first SIGINT/SIGTERM. A second SIGINT/SIGTERM will force cloudflared to shutdown immediately.",
+			Usage:   "When cloudflared receives SIGINT/SIGTERM it will stop accepting new requests, wait for in-progress requests to terminate, then shutdown. Waiting for in-progress requests will timeout after this grace period, or when a second SIGTERM/SIGINT is received.",
 			Value:   time.Second * 30,
 			EnvVars: []string{"TUNNEL_GRACE_PERIOD"},
-			Hidden:  true,
+			Hidden:  shouldHide,
 		}),
 		// Note TUN-3758 , we use Int because UInt is not supported with altsrc
 		altsrc.NewIntFlag(&cli.IntFlag{
@@ -672,7 +614,7 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 		altsrc.NewBoolFlag(&cli.BoolFlag{
 			Name:    "stdin-control",
 			Usage:   "Control the process using commands sent through stdin",
-			EnvVars: []string{"STDIN-CONTROL"},
+			EnvVars: []string{"STDIN_CONTROL"},
 			Hidden:  true,
 			Value:   false,
 		}),
@@ -681,6 +623,7 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			Aliases: []string{"n"},
 			EnvVars: []string{"TUNNEL_NAME"},
 			Usage:   "Stable name to identify the tunnel. Using this flag will create, route and run a tunnel. For production usage, execute each command separately",
+			Hidden:  shouldHide,
 		}),
 		altsrc.NewBoolFlag(&cli.BoolFlag{
 			Name:   uiFlag,
@@ -688,7 +631,14 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			Value:  false,
 			Hidden: shouldHide,
 		}),
+		altsrc.NewStringFlag(&cli.StringFlag{
+			Name:   "quick-service",
+			Usage:  "URL for a service which manages unauthenticated 'quick' tunnels.",
+			Value:  "https://api.trycloudflare.com",
+			Hidden: true,
+		}),
 		selectProtocolFlag,
+		overwriteDNSFlag,
 	}...)
 
 	return flags
@@ -775,7 +725,7 @@ func configureProxyFlags(shouldHide bool) []cli.Flag {
 			Hidden: shouldHide,
 		}),
 		altsrc.NewDurationFlag(&cli.DurationFlag{
-			Name:   ingress.ProxyTCPKeepAlive,
+			Name:   ingress.ProxyTCPKeepAliveFlag,
 			Usage:  "HTTP proxy TCP keepalive duration",
 			Value:  time.Second * 30,
 			Hidden: shouldHide,
@@ -949,7 +899,7 @@ func configureLoggingFlags(shouldHide bool) []cli.Flag {
 		altsrc.NewStringFlag(&cli.StringFlag{
 			Name:    logger.LogLevelFlag,
 			Value:   "info",
-			Usage:   "Application logging level {fatal, error, info, debug}. " + debugLevelWarning,
+			Usage:   "Application logging level {debug, info, warn, error, fatal}. " + debugLevelWarning,
 			EnvVars: []string{"TUNNEL_LOGLEVEL"},
 			Hidden:  shouldHide,
 		}),
@@ -957,7 +907,7 @@ func configureLoggingFlags(shouldHide bool) []cli.Flag {
 			Name:    logger.LogTransportLevelFlag,
 			Aliases: []string{"proto-loglevel"}, // This flag used to be called proto-loglevel
 			Value:   "info",
-			Usage:   "Transport logging level(previously called protocol logging level) {fatal, error, info, debug}",
+			Usage:   "Transport logging level(previously called protocol logging level) {debug, info, warn, error, fatal}",
 			EnvVars: []string{"TUNNEL_PROTO_LOGLEVEL", "TUNNEL_TRANSPORT_LOGLEVEL"},
 			Hidden:  shouldHide,
 		}),
@@ -1011,6 +961,13 @@ func configureProxyDNSFlags(shouldHide bool) []cli.Flag {
 			EnvVars: []string{"TUNNEL_DNS_UPSTREAM"},
 			Hidden:  shouldHide,
 		}),
+		altsrc.NewIntFlag(&cli.IntFlag{
+			Name:    "proxy-dns-max-upstream-conns",
+			Usage:   "Maximum concurrent connections to upstream. Setting to 0 means unlimited.",
+			Value:   tunneldns.MaxUpstreamConnsDefault,
+			Hidden:  shouldHide,
+			EnvVars: []string{"TUNNEL_DNS_MAX_UPSTREAM_CONNS"},
+		}),
 		altsrc.NewStringSliceFlag(&cli.StringSliceFlag{
 			Name:  "proxy-dns-bootstrap",
 			Usage: "bootstrap endpoint URL, you can specify multiple endpoints for redundancy.",

cmd/cloudflared/tunnel/config_test.go

@@ -0,0 +1,13 @@
+package tunnel
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestDedup(t *testing.T) {
+	expected := []string{"a", "b"}
+	actual := dedup([]string{"a", "b", "a"})
+	require.ElementsMatch(t, expected, actual)
+}

cmd/cloudflared/tunnel/configuration.go

@@ -8,8 +8,15 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/google/uuid"
+	homedir "github.com/mitchellh/go-homedir"
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog"
+	"github.com/urfave/cli/v2"
+	"golang.org/x/crypto/ssh/terminal"
+
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/edgediscovery"
 	"github.com/cloudflare/cloudflared/h2mux"
@@ -18,13 +25,6 @@ import (
 	"github.com/cloudflare/cloudflared/tlsconfig"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	"github.com/cloudflare/cloudflared/validation"
-
-	"github.com/google/uuid"
-	"github.com/mitchellh/go-homedir"
-	"github.com/pkg/errors"
-	"github.com/rs/zerolog"
-	"github.com/urfave/cli/v2"
-	"golang.org/x/crypto/ssh/terminal"
 )
 
 const LogFieldOriginCertPath = "originCertPath"
@@ -85,8 +85,8 @@ func logClientOptions(c *cli.Context, log *zerolog.Logger) {
 	}
 }
 
-func dnsProxyStandAlone(c *cli.Context) bool {
-	return c.IsSet("proxy-dns") && (!c.IsSet("hostname") && !c.IsSet("tag") && !c.IsSet("hello-world"))
+func dnsProxyStandAlone(c *cli.Context, namedTunnel *connection.NamedTunnelConfig) bool {
+	return c.IsSet("proxy-dns") && (!c.IsSet("hostname") && !c.IsSet("tag") && !c.IsSet("hello-world") && namedTunnel == nil)
 }
 
 func findOriginCert(originCertPath string, log *zerolog.Logger) (string, error) {
@@ -109,7 +109,7 @@ func findOriginCert(originCertPath string, log *zerolog.Logger) (string, error)
 	// Check that the user has acquired a certificate using the login command
 	ok, err := config.FileExists(originCertPath)
 	if err != nil {
-		log.Error().Msgf("Cannot check if origin cert exists at path %s", originCertPath)
+		log.Error().Err(err).Msgf("Cannot check if origin cert exists at path %s", originCertPath)
 		return "", fmt.Errorf("cannot check if origin cert exists at path %s", originCertPath)
 	}
 	if !ok {
@@ -161,7 +161,7 @@ func prepareTunnelConfig(
 		log.Err(err).Str(LogFieldHostname, configHostname).Msg("Invalid hostname")
 		return nil, ingress.Ingress{}, errors.Wrap(err, "Invalid hostname")
 	}
-	isFreeTunnel := hostname == ""
+	isQuickTunnel := hostname == ""
 	clientID := c.String("id")
 	if !c.IsSet("id") {
 		clientID, err = generateRandomClientID(log)
@@ -179,7 +179,7 @@ func prepareTunnelConfig(
 	tags = append(tags, tunnelpogs.Tag{Name: "ID", Value: clientID})
 
 	var originCert []byte
-	if !isFreeTunnel {
+	if !isQuickTunnel {
 		originCertPath := c.String("origincert")
 		originCertLog := log.With().
 			Str(LogFieldOriginCertPath, originCertPath).
@@ -195,18 +195,21 @@ func prepareTunnelConfig(
 		ingressRules  ingress.Ingress
 		classicTunnel *connection.ClassicTunnelConfig
 	)
+	cfg := config.GetConfiguration()
 	if isNamedTunnel {
 		clientUUID, err := uuid.NewRandom()
 		if err != nil {
-			return nil, ingress.Ingress{}, errors.Wrap(err, "can't generate clientUUID")
+			return nil, ingress.Ingress{}, errors.Wrap(err, "can't generate connector UUID")
 		}
+		log.Info().Msgf("Generated Connector ID: %s", clientUUID)
+		features := append(c.StringSlice("features"), origin.FeatureSerializedHeaders)
 		namedTunnel.Client = tunnelpogs.ClientInfo{
 			ClientID: clientUUID[:],
-			Features: []string{origin.FeatureSerializedHeaders},
+			Features: dedup(features),
 			Version:  version,
-			Arch:     fmt.Sprintf("%s_%s", buildInfo.GoOS, buildInfo.GoArch),
+			Arch:     buildInfo.OSArch(),
 		}
-		ingressRules, err = ingress.ParseIngress(config.GetConfiguration())
+		ingressRules, err = ingress.ParseIngress(cfg)
 		if err != nil && err != ingress.ErrNoIngressRules {
 			return nil, ingress.Ingress{}, err
 		}
@@ -230,7 +233,14 @@ func prepareTunnelConfig(
 		}
 	}
 
-	protocolSelector, err := connection.NewProtocolSelector(c.String("protocol"), namedTunnel, edgediscovery.HTTP2Percentage, origin.ResolveTTL, log)
+	var warpRoutingService *ingress.WarpRoutingService
+	warpRoutingEnabled := isWarpRoutingEnabled(cfg.WarpRouting, isNamedTunnel)
+	if warpRoutingEnabled {
+		warpRoutingService = ingress.NewWarpRoutingService()
+		log.Info().Msgf("Warp-routing is enabled")
+	}
+
+	protocolSelector, err := connection.NewProtocolSelector(c.String("protocol"), warpRoutingEnabled, namedTunnel, edgediscovery.HTTP2Percentage, origin.ResolveTTL, log)
 	if err != nil {
 		return nil, ingress.Ingress{}, err
 	}
@@ -245,9 +255,9 @@ func prepareTunnelConfig(
 		edgeTLSConfigs[p] = edgeTLSConfig
 	}
 
-	originClient := origin.NewClient(ingressRules, tags, log)
+	originProxy := origin.NewOriginProxy(ingressRules, warpRoutingService, tags, log)
 	connectionConfig := &connection.Config{
-		OriginClient:    originClient,
+		OriginProxy:     originProxy,
 		GracePeriod:     c.Duration("grace-period"),
 		ReplaceExisting: c.Bool("force"),
 	}
@@ -262,13 +272,12 @@ func prepareTunnelConfig(
 
 	return &origin.TunnelConfig{
 		ConnectionConfig: connectionConfig,
-		BuildInfo:        buildInfo,
+		OSArch:           buildInfo.OSArch(),
 		ClientID:         clientID,
 		EdgeAddrs:        c.StringSlice("edge"),
 		HAConnections:    c.Int("ha-connections"),
 		IncidentLookup:   origin.NewIncidentLookup(),
 		IsAutoupdated:    c.Bool("is-autoupdated"),
-		IsFreeTunnel:     isFreeTunnel,
 		LBPool:           c.String("lb-pool"),
 		Tags:             tags,
 		Log:              log,
@@ -286,6 +295,29 @@ func prepareTunnelConfig(
 	}, ingressRules, nil
 }
 
+func isWarpRoutingEnabled(warpConfig config.WarpRoutingConfig, isNamedTunnel bool) bool {
+	return warpConfig.Enabled && isNamedTunnel
+}
+
 func isRunningFromTerminal() bool {
 	return terminal.IsTerminal(int(os.Stdout.Fd()))
 }
+
+// Remove any duplicates from the slice
+func dedup(slice []string) []string {
+
+	// Convert the slice into a set
+	set := make(map[string]bool, 0)
+	for _, str := range slice {
+		set[str] = true
+	}
+
+	// Convert the set back into a slice
+	keys := make([]string, len(set))
+	i := 0
+	for str := range set {
+		keys[i] = str
+		i++
+	}
+	return keys
+}

cmd/cloudflared/tunnel/credential_finder.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"path/filepath"
 
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/config"
 
 	"github.com/google/uuid"
 	"github.com/rs/zerolog"

cmd/cloudflared/tunnel/info.go

@@ -0,0 +1,16 @@
+package tunnel
+
+import (
+	"time"
+
+	"github.com/google/uuid"
+
+	"github.com/cloudflare/cloudflared/tunnelstore"
+)
+
+type Info struct {
+	ID         uuid.UUID                   `json:"id"`
+	Name       string                      `json:"name"`
+	CreatedAt  time.Time                   `json:"createdAt"`
+	Connectors []*tunnelstore.ActiveClient `json:"conns"`
+}

cmd/cloudflared/tunnel/ingress_subcommands.go

@@ -5,7 +5,7 @@ import (
 	"net/url"
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/ingress"
 
 	"github.com/pkg/errors"
@@ -45,7 +45,7 @@ func buildIngressSubcommand() *cli.Command {
 func buildValidateIngressCommand() *cli.Command {
 	return &cli.Command{
 		Name:        "validate",
-		Action:      cliutil.ErrorHandler(validateIngressCommand),
+		Action:      cliutil.ConfiguredActionWithWarnings(validateIngressCommand),
 		Usage:       "Validate the ingress configuration ",
 		UsageText:   "cloudflared tunnel [--config FILEPATH] ingress validate",
 		Description: "Validates the configuration file, ensuring your ingress rules are OK.",
@@ -55,7 +55,7 @@ func buildValidateIngressCommand() *cli.Command {
 func buildTestURLCommand() *cli.Command {
 	return &cli.Command{
 		Name:      "rule",
-		Action:    cliutil.ErrorHandler(testURLCommand),
+		Action:    cliutil.ConfiguredAction(testURLCommand),
 		Usage:     "Check which ingress rule matches a given request URL",
 		UsageText: "cloudflared tunnel [--config FILEPATH] ingress rule URL",
 		ArgsUsage: "URL",
@@ -68,7 +68,7 @@ func buildTestURLCommand() *cli.Command {
 }
 
 // validateIngressCommand check the syntax of the ingress rules in the cloudflared config file
-func validateIngressCommand(c *cli.Context) error {
+func validateIngressCommand(c *cli.Context, warnings string) error {
 	conf := config.GetConfiguration()
 	if conf.Source() == "" {
 		fmt.Println("No configuration file was found. Please create one, or use the --config flag to specify its filepath. You can use the help command to learn more about configuration files")
@@ -81,6 +81,11 @@ func validateIngressCommand(c *cli.Context) error {
 	if c.IsSet("url") {
 		return ingress.ErrURLIncompatibleWithIngress
 	}
+	if warnings != "" {
+		fmt.Println("Warning: unused keys detected in your config file. Here is a list of unused keys:")
+		fmt.Println(warnings)
+		return nil
+	}
 	fmt.Println("OK")
 	return nil
 }

cmd/cloudflared/tunnel/login.go

@@ -8,34 +8,28 @@ import (
 	"path/filepath"
 	"syscall"
 
-	"github.com/mitchellh/go-homedir"
+	homedir "github.com/mitchellh/go-homedir"
 	"github.com/pkg/errors"
 	"github.com/urfave/cli/v2"
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/transfer"
+	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/logger"
+	"github.com/cloudflare/cloudflared/token"
 )
 
 const (
 	baseLoginURL     = "https://dash.cloudflare.com/argotunnel"
-	callbackStoreURL = "https://login.argotunnel.com/"
+	callbackStoreURL = "https://login.cloudflareaccess.org/"
 )
 
 func buildLoginSubcommand(hidden bool) *cli.Command {
 	return &cli.Command{
 		Name:      "login",
-		Action:    cliutil.ErrorHandler(login),
+		Action:    cliutil.ConfiguredAction(login),
 		Usage:     "Generate a configuration file with your login details",
 		ArgsUsage: " ",
-		Flags: []cli.Flag{
-			&cli.StringFlag{
-				Name:   "url",
-				Hidden: true,
-			},
-		},
-		Hidden: hidden,
+		Hidden:    hidden,
 	}
 }
 
@@ -56,7 +50,7 @@ func login(c *cli.Context) error {
 		return err
 	}
 
-	resourceData, err := transfer.Run(
+	resourceData, err := token.RunTransfer(
 		loginURL,
 		"cert",
 		"callback",

cmd/cloudflared/tunnel/quick_tunnel.go

@@ -0,0 +1,111 @@
+package tunnel
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/pkg/errors"
+
+	"github.com/cloudflare/cloudflared/connection"
+)
+
+const httpTimeout = 15 * time.Second
+
+// RunQuickTunnel requests a tunnel from the specified service.
+// We use this to power quick tunnels on trycloudflare.com, but the
+// service is open-source and could be used by anyone.
+func RunQuickTunnel(sc *subcommandContext) error {
+	sc.log.Info().Msg("Requesting new Quick Tunnel...")
+
+	client := http.Client{
+		Transport: &http.Transport{
+			TLSHandshakeTimeout:   httpTimeout,
+			ResponseHeaderTimeout: httpTimeout,
+		},
+		Timeout: httpTimeout,
+	}
+
+	resp, err := client.Post(fmt.Sprintf("%s/tunnel", sc.c.String("quick-service")), "application/json", nil)
+	if err != nil {
+		return errors.Wrap(err, "failed to request quick tunnel")
+	}
+	defer resp.Body.Close()
+
+	var data QuickTunnelResponse
+	if err := json.NewDecoder(resp.Body).Decode(&data); err != nil {
+		return errors.Wrap(err, "failed to unmarshal quick tunnel")
+	}
+
+	tunnelID, err := uuid.Parse(data.Result.ID)
+	if err != nil {
+		return errors.Wrap(err, "failed to parse quick tunnel ID")
+	}
+
+	credentials := connection.Credentials{
+		AccountTag:   data.Result.AccountTag,
+		TunnelSecret: data.Result.Secret,
+		TunnelID:     tunnelID,
+		TunnelName:   data.Result.Name,
+	}
+
+	for _, line := range AsciiBox([]string{
+		"Your Quick Tunnel has been created! Visit it at:",
+		data.Result.Hostname,
+	}, 2) {
+		sc.log.Info().Msg(line)
+	}
+
+	return StartServer(
+		sc.c,
+		version,
+		&connection.NamedTunnelConfig{Credentials: credentials, QuickTunnelUrl: data.Result.Hostname},
+		sc.log,
+		sc.isUIEnabled,
+	)
+}
+
+type QuickTunnelResponse struct {
+	Success bool
+	Result  QuickTunnel
+	Errors  []QuickTunnelError
+}
+
+type QuickTunnelError struct {
+	Code    int
+	Message string
+}
+
+type QuickTunnel struct {
+	ID         string `json:"id"`
+	Name       string `json:"name"`
+	Hostname   string `json:"hostname"`
+	AccountTag string `json:"account_tag"`
+	Secret     []byte `json:"secret"`
+}
+
+// Print out the given lines in a nice ASCII box.
+func AsciiBox(lines []string, padding int) (box []string) {
+	maxLen := maxLen(lines)
+	spacer := strings.Repeat(" ", padding)
+	border := "+" + strings.Repeat("-", maxLen+(padding*2)) + "+"
+	box = append(box, border)
+	for _, line := range lines {
+		box = append(box, "|"+spacer+line+strings.Repeat(" ", maxLen-len(line))+spacer+"|")
+	}
+	box = append(box, border)
+	return
+}
+
+func maxLen(lines []string) int {
+	max := 0
+	for _, line := range lines {
+		if len(line) > max {
+			max = len(line)
+		}
+	}
+	return max
+}

cmd/cloudflared/tunnel/server.go

@@ -1,6 +1,8 @@
 package tunnel
 
 import (
+	"fmt"
+
 	"github.com/cloudflare/cloudflared/tunneldns"
 
 	"github.com/pkg/errors"
@@ -13,7 +15,11 @@ func runDNSProxyServer(c *cli.Context, dnsReadySignal chan struct{}, shutdownC <
 	if port <= 0 || port > 65535 {
 		return errors.New("The 'proxy-dns-port' must be a valid port number in <1, 65535> range.")
 	}
-	listener, err := tunneldns.CreateListener(c.String("proxy-dns-address"), uint16(port), c.StringSlice("proxy-dns-upstream"), c.StringSlice("proxy-dns-bootstrap"), log)
+	maxUpstreamConnections := c.Int("proxy-dns-max-upstream-conns")
+	if maxUpstreamConnections < 0 {
+		return fmt.Errorf("'%s' must be 0 or higher", "proxy-dns-max-upstream-conns")
+	}
+	listener, err := tunneldns.CreateListener(c.String("proxy-dns-address"), uint16(port), c.StringSlice("proxy-dns-upstream"), c.StringSlice("proxy-dns-bootstrap"), maxUpstreamConnections, log)
 	if err != nil {
 		close(dnsReadySignal)
 		listener.Stop()

cmd/cloudflared/tunnel/signal.go

@@ -20,4 +20,4 @@ func waitForSignal(graceShutdownC chan struct{}, logger *zerolog.Logger) {
 		close(graceShutdownC)
 	case <-graceShutdownC:
 	}
-}
+}

cmd/cloudflared/tunnel/signal_test.go

@@ -1,3 +1,5 @@
+// +build !windows
+
 package tunnel
 
 import (
@@ -101,4 +103,3 @@ func TestWaitForShutdown(t *testing.T) {
 	assert.True(t, contextCancelled)
 	assert.True(t, time.Now().Sub(startTime) < time.Second) // check that wait ended early
 }
-

cmd/cloudflared/tunnel/subcommand_context.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
+	"path/filepath"
 	"strings"
 
 	"github.com/google/uuid"
@@ -147,7 +148,7 @@ func (sc *subcommandContext) readTunnelCredentials(credFinder CredFinder) (conne
 	return credentials, nil
 }
 
-func (sc *subcommandContext) create(name string, credentialsOutputPath string) (*tunnelstore.Tunnel, error) {
+func (sc *subcommandContext) create(name string, credentialsFilePath string) (*tunnelstore.Tunnel, error) {
 	client, err := sc.client()
 	if err != nil {
 		return nil, errors.Wrap(err, "couldn't create client to talk to Argo Tunnel backend")
@@ -173,27 +174,40 @@ func (sc *subcommandContext) create(name string, credentialsOutputPath string) (
 		TunnelID:     tunnel.ID,
 		TunnelName:   name,
 	}
-	filePath, writeFileErr := writeTunnelCredentials(credential.certPath, credentialsOutputPath, &tunnelCredentials)
+	usedCertPath := false
+	if credentialsFilePath == "" {
+		originCertDir := filepath.Dir(credential.certPath)
+		credentialsFilePath, err = tunnelFilePath(tunnelCredentials.TunnelID, originCertDir)
+		if err != nil {
+			return nil, err
+		}
+		usedCertPath = true
+	}
+	writeFileErr := writeTunnelCredentials(credentialsFilePath, &tunnelCredentials)
 	if writeFileErr != nil {
 		var errorLines []string
-		errorLines = append(errorLines, fmt.Sprintf("Your tunnel '%v' was created with ID %v. However, cloudflared couldn't write to the tunnel credentials file at %v.json.", tunnel.Name, tunnel.ID, tunnel.ID))
+		errorLines = append(errorLines, fmt.Sprintf("Your tunnel '%v' was created with ID %v. However, cloudflared couldn't write tunnel credentials to %s.", tunnel.Name, tunnel.ID, credentialsFilePath))
 		errorLines = append(errorLines, fmt.Sprintf("The file-writing error is: %v", writeFileErr))
 		if deleteErr := client.DeleteTunnel(tunnel.ID); deleteErr != nil {
 			errorLines = append(errorLines, fmt.Sprintf("Cloudflared tried to delete the tunnel for you, but encountered an error. You should use `cloudflared tunnel delete %v` to delete the tunnel yourself, because the tunnel can't be run without the tunnelfile.", tunnel.ID))
 			errorLines = append(errorLines, fmt.Sprintf("The delete tunnel error is: %v", deleteErr))
 		} else {
-			errorLines = append(errorLines, fmt.Sprintf("The tunnel was deleted, because the tunnel can't be run without the tunnelfile"))
+			errorLines = append(errorLines, fmt.Sprintf("The tunnel was deleted, because the tunnel can't be run without the credentials file"))
 		}
 		errorMsg := strings.Join(errorLines, "\n")
 		return nil, errors.New(errorMsg)
 	}
-	sc.log.Info().Msgf("Tunnel credentials written to %v. cloudflared chose this file based on where your origin certificate was found. Keep this file secret. To revoke these credentials, delete the tunnel.", filePath)
 
 	if outputFormat := sc.c.String(outputFormatFlag.Name); outputFormat != "" {
 		return nil, renderOutput(outputFormat, &tunnel)
 	}
 
-	sc.log.Info().Msgf("Created tunnel %s with id %s", tunnel.Name, tunnel.ID)
+	fmt.Printf("Tunnel credentials written to %v.", credentialsFilePath)
+	if usedCertPath {
+		fmt.Print(" cloudflared chose this file based on where your origin certificate was found.")
+	}
+	fmt.Println(" Keep this file secret. To revoke these credentials, delete the tunnel.")
+	fmt.Printf("\nCreated tunnel %s with id %s\n", tunnel.Name, tunnel.ID)
 	return tunnel, nil
 }
 
@@ -223,13 +237,8 @@ func (sc *subcommandContext) delete(tunnelIDs []uuid.UUID) error {
 		if !tunnel.DeletedAt.IsZero() {
 			return fmt.Errorf("Tunnel %s has already been deleted", tunnel.ID)
 		}
-		// Check if tunnel has existing connections and if force flag is set, cleanup connections
-		if len(tunnel.Connections) > 0 {
-			if !forceFlagSet {
-				return fmt.Errorf("You can not delete tunnel %s because it has active connections. To see connections run the 'list' command. If you believe the tunnel is not active, you can use a -f / --force flag with this command.", id)
-			}
-
-			if err := client.CleanupConnections(tunnel.ID); err != nil {
+		if forceFlagSet {
+			if err := client.CleanupConnections(tunnel.ID, tunnelstore.NewCleanupParams()); err != nil {
 				return errors.Wrapf(err, "Error cleaning up connections for tunnel %s", tunnel.ID)
 			}
 		}
@@ -281,13 +290,24 @@ func (sc *subcommandContext) run(tunnelID uuid.UUID) error {
 }
 
 func (sc *subcommandContext) cleanupConnections(tunnelIDs []uuid.UUID) error {
+	params := tunnelstore.NewCleanupParams()
+	extraLog := ""
+	if connector := sc.c.String("connector-id"); connector != "" {
+		connectorID, err := uuid.Parse(connector)
+		if err != nil {
+			return errors.Wrapf(err, "%s is not a valid client ID (must be a UUID)", connector)
+		}
+		params.ForClient(connectorID)
+		extraLog = fmt.Sprintf(" for connector-id %s", connectorID.String())
+	}
+
 	client, err := sc.client()
 	if err != nil {
 		return err
 	}
 	for _, tunnelID := range tunnelIDs {
-		sc.log.Info().Msgf("Cleanup connection for tunnel %s", tunnelID)
-		if err := client.CleanupConnections(tunnelID); err != nil {
+		sc.log.Info().Msgf("Cleanup connection for tunnel %s%s", tunnelID, extraLog)
+		if err := client.CleanupConnections(tunnelID, params); err != nil {
 			sc.log.Error().Msgf("Error cleaning up connections for tunnel %v, error :%v", tunnelID, err)
 		}
 	}
@@ -348,6 +368,12 @@ func (sc *subcommandContext) findID(input string) (uuid.UUID, error) {
 // one Tunnelstore API call.
 func (sc *subcommandContext) findIDs(inputs []string) ([]uuid.UUID, error) {
 
+	// Shortcut without Tunnelstore call if we find that all inputs are already UUIDs.
+	uuids, err := convertNamesToUuids(inputs, make(map[string]uuid.UUID))
+	if err == nil {
+		return uuids, nil
+	}
+
 	// First, look up all tunnels the user has
 	filter := tunnelstore.NewFilter()
 	filter.NoDeleted()
@@ -367,7 +393,10 @@ func findIDs(tunnels []*tunnelstore.Tunnel, inputs []string) ([]uuid.UUID, error
 		nameToID[tunnel.Name] = tunnel.ID
 	}
 
-	// For each input, try to find the tunnel ID.
+	return convertNamesToUuids(inputs, nameToID)
+}
+
+func convertNamesToUuids(inputs []string, nameToID map[string]uuid.UUID) ([]uuid.UUID, error) {
 	tunnelIDs := make([]uuid.UUID, len(inputs))
 	var badInputs []string
 	for i, input := range inputs {

cmd/cloudflared/tunnel/subcommand_context_teamnet.go

@@ -3,8 +3,9 @@ package tunnel
 import (
 	"net"
 
-	"github.com/cloudflare/cloudflared/teamnet"
 	"github.com/pkg/errors"
+
+	"github.com/cloudflare/cloudflared/teamnet"
 )
 
 const noClientMsg = "error while creating backend client"

cmd/cloudflared/tunnel/subcommand_context_test.go

@@ -4,16 +4,17 @@ import (
 	"encoding/base64"
 	"flag"
 	"fmt"
-	"github.com/rs/zerolog"
 	"reflect"
 	"testing"
 	"time"
 
-	"github.com/cloudflare/cloudflared/connection"
-	"github.com/cloudflare/cloudflared/tunnelstore"
 	"github.com/google/uuid"
 	"github.com/pkg/errors"
+	"github.com/rs/zerolog"
 	"github.com/urfave/cli/v2"
+
+	"github.com/cloudflare/cloudflared/connection"
+	"github.com/cloudflare/cloudflared/tunnelstore"
 )
 
 func Test_findIDs(t *testing.T) {
@@ -260,7 +261,7 @@ func (d *deleteMockTunnelStore) DeleteTunnel(tunnelID uuid.UUID) error {
 	return nil
 }
 
-func (d *deleteMockTunnelStore) CleanupConnections(tunnelID uuid.UUID) error {
+func (d *deleteMockTunnelStore) CleanupConnections(tunnelID uuid.UUID, _ *tunnelstore.CleanupParams) error {
 	tunnel, ok := d.mockTunnels[tunnelID]
 	if !ok {
 		return fmt.Errorf("Couldn't find tunnel: %v", tunnelID)

cmd/cloudflared/tunnel/subcommands.go

@@ -14,23 +14,26 @@ import (
 	"time"
 
 	"github.com/google/uuid"
-	"github.com/mitchellh/go-homedir"
+	homedir "github.com/mitchellh/go-homedir"
 	"github.com/pkg/errors"
 	"github.com/urfave/cli/v2"
 	"github.com/urfave/cli/v2/altsrc"
 	"golang.org/x/net/idna"
-	"gopkg.in/yaml.v2"
+	yaml "gopkg.in/yaml.v2"
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/updater"
+	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/tunnelstore"
 )
 
 const (
-	allSortByOptions  = "name, id, createdAt, deletedAt, numConnections"
-	CredFileFlagAlias = "cred-file"
-	CredFileFlag      = "credentials-file"
+	allSortByOptions     = "name, id, createdAt, deletedAt, numConnections"
+	connsSortByOptions   = "id, startedAt, numConnections, version"
+	CredFileFlagAlias    = "cred-file"
+	CredFileFlag         = "credentials-file"
+	overwriteDNSFlagName = "overwrite-dns"
 
 	LogFieldTunnelID = "tunnelID"
 )
@@ -63,11 +66,11 @@ var (
 		Aliases: []string{"rd"},
 		Usage:   "Include connections that have recently disconnected in the list",
 	}
-	outputFormatFlag = altsrc.NewStringFlag(&cli.StringFlag{
+	outputFormatFlag = &cli.StringFlag{
 		Name:    "output",
 		Aliases: []string{"o"},
 		Usage:   "Render output using given `FORMAT`. Valid options are 'json' or 'yaml'",
-	})
+	}
 	sortByFlag = &cli.StringFlag{
 		Name:    "sort-by",
 		Value:   "name",
@@ -87,32 +90,62 @@ var (
 			"overwrite the previous tunnel. If you want to use a single hostname with multiple " +
 			"tunnels, you can do so with Cloudflare's Load Balancer product.",
 	})
-	credentialsFileFlag = altsrc.NewStringFlag(&cli.StringFlag{
+	featuresFlag = altsrc.NewStringSliceFlag(&cli.StringSliceFlag{
+		Name:    "features",
+		Aliases: []string{"F"},
+		Usage:   "Opt into various features that are still being developed or tested.",
+	})
+	credentialsFileFlagCLIOnly = &cli.StringFlag{
 		Name:    CredFileFlag,
 		Aliases: []string{CredFileFlagAlias},
 		Usage:   "Filepath at which to read/write the tunnel credentials",
 		EnvVars: []string{"TUNNEL_CRED_FILE"},
-	})
-	forceDeleteFlag = &cli.BoolFlag{
+	}
+	credentialsFileFlag = altsrc.NewStringFlag(credentialsFileFlagCLIOnly)
+	forceDeleteFlag     = &cli.BoolFlag{
 		Name:    "force",
 		Aliases: []string{"f"},
-		Usage:   "Allows you to delete a tunnel, even if it has active connections.",
+		Usage: "Cleans up any stale connections before the tunnel is deleted. cloudflared will not " +
+			"delete a tunnel with connections without this flag.",
 		EnvVars: []string{"TUNNEL_RUN_FORCE_OVERWRITE"},
 	}
 	selectProtocolFlag = altsrc.NewStringFlag(&cli.StringFlag{
 		Name:    "protocol",
-		Value:   "h2mux",
+		Value:   "auto",
 		Aliases: []string{"p"},
 		Usage:   fmt.Sprintf("Protocol implementation to connect with Cloudflare's edge network. %s", connection.AvailableProtocolFlagMessage),
 		EnvVars: []string{"TUNNEL_TRANSPORT_PROTOCOL"},
 		Hidden:  true,
 	})
+	sortInfoByFlag = &cli.StringFlag{
+		Name:    "sort-by",
+		Value:   "createdAt",
+		Usage:   fmt.Sprintf("Sorts the list of connections of a tunnel by the given field. Valid options are {%s}", connsSortByOptions),
+		EnvVars: []string{"TUNNEL_INFO_SORT_BY"},
+	}
+	invertInfoSortFlag = &cli.BoolFlag{
+		Name:    "invert-sort",
+		Usage:   "Inverts the sort order of the tunnel info.",
+		EnvVars: []string{"TUNNEL_INFO_INVERT_SORT"},
+	}
+	cleanupClientFlag = &cli.StringFlag{
+		Name:    "connector-id",
+		Aliases: []string{"c"},
+		Usage:   `Constraints the cleanup to stop the connections of a single Connector (by its ID). You can find the various Connectors (and their IDs) currently connected to your tunnel via 'cloudflared tunnel info <name>'.`,
+		EnvVars: []string{"TUNNEL_CLEANUP_CONNECTOR"},
+	}
+	overwriteDNSFlag = &cli.BoolFlag{
+		Name:    overwriteDNSFlagName,
+		Aliases: []string{"f"},
+		Usage:   `Overwrites existing DNS records with this hostname`,
+		EnvVars: []string{"TUNNEL_FORCE_PROVISIONING_DNS"},
+	}
 )
 
 func buildCreateCommand() *cli.Command {
 	return &cli.Command{
 		Name:      "create",
-		Action:    cliutil.ErrorHandler(createCommand),
+		Action:    cliutil.ConfiguredAction(createCommand),
 		Usage:     "Create a new tunnel with given name",
 		UsageText: "cloudflared tunnel [tunnel command options] create [subcommand options] NAME",
 		Description: `Creates a tunnel, registers it with Cloudflare edge and generates credential file used to run this tunnel.
@@ -121,7 +154,7 @@ func buildCreateCommand() *cli.Command {
   For example, to create a tunnel named 'my-tunnel' run:
 
   $ cloudflared tunnel create my-tunnel`,
-		Flags:              []cli.Flag{outputFormatFlag, credentialsFileFlag},
+		Flags:              []cli.Flag{outputFormatFlag, credentialsFileFlagCLIOnly},
 		CustomHelpTemplate: commandHelpTemplate(),
 	}
 }
@@ -144,6 +177,9 @@ func createCommand(c *cli.Context) error {
 	}
 	name := c.Args().First()
 
+	warningChecker := updater.StartWarningCheck(c)
+	defer warningChecker.LogWarningIfAny(sc.log)
+
 	_, err = sc.create(name, c.String(CredFileFlag))
 	return errors.Wrap(err, "failed to create tunnel")
 }
@@ -154,33 +190,26 @@ func tunnelFilePath(tunnelID uuid.UUID, directory string) (string, error) {
 	return homedir.Expand(filePath)
 }
 
-// If an `outputFile` is given, write the credentials there.
-// Otherwise, write it to the same directory as the originCert,
-// with the filename `<tunnel id>.json`.
-func writeTunnelCredentials(
-	originCertPath, outputFile string,
-	credentials *connection.Credentials,
-) (filePath string, err error) {
-	filePath = outputFile
-	if outputFile == "" {
-		originCertDir := filepath.Dir(originCertPath)
-		filePath, err = tunnelFilePath(credentials.TunnelID, originCertDir)
+// writeTunnelCredentials saves `credentials` as a JSON into `filePath`, only if
+// the file does not exist already
+func writeTunnelCredentials(filePath string, credentials *connection.Credentials) error {
+	if _, err := os.Stat(filePath); !os.IsNotExist(err) {
+		if err == nil {
+			return fmt.Errorf("%s already exists", filePath)
+		}
+		return err
 	}
-	if err != nil {
-		return "", err
-	}
-	// Write the name and ID to the file too
 	body, err := json.Marshal(credentials)
 	if err != nil {
-		return "", errors.Wrap(err, "Unable to marshal tunnel credentials to JSON")
+		return errors.Wrap(err, "Unable to marshal tunnel credentials to JSON")
 	}
-	return filePath, ioutil.WriteFile(filePath, body, 400)
+	return ioutil.WriteFile(filePath, body, 400)
 }
 
 func buildListCommand() *cli.Command {
 	return &cli.Command{
 		Name:        "list",
-		Action:      cliutil.ErrorHandler(listCommand),
+		Action:      cliutil.ConfiguredAction(listCommand),
 		Usage:       "List existing tunnels",
 		UsageText:   "cloudflared tunnel [tunnel command options] list [subcommand options]",
 		Description: "cloudflared tunnel list will display all active tunnels, their created time and associated connections. Use -d flag to include deleted tunnels. See the list of options to filter the list",
@@ -204,6 +233,9 @@ func listCommand(c *cli.Context) error {
 		return err
 	}
 
+	warningChecker := updater.StartWarningCheck(c)
+	defer warningChecker.LogWarningIfAny(sc.log)
+
 	filter := tunnelstore.NewFilter()
 	if !c.Bool("show-deleted") {
 		filter.NoDeleted()
@@ -266,21 +298,16 @@ func listCommand(c *cli.Context) error {
 	} else {
 		fmt.Println("You have no tunnels, use 'cloudflared tunnel create' to define a new tunnel")
 	}
+
 	return nil
 }
 
 func formatAndPrintTunnelList(tunnels []*tunnelstore.Tunnel, showRecentlyDisconnected bool) {
-	const (
-		minWidth = 0
-		tabWidth = 8
-		padding  = 1
-		padChar  = ' '
-		flags    = 0
-	)
-
-	writer := tabwriter.NewWriter(os.Stdout, minWidth, tabWidth, padding, padChar, flags)
+	writer := tabWriter()
 	defer writer.Flush()
 
+	_, _ = fmt.Fprintln(writer, "You can obtain more detailed information for each tunnel with `cloudflared tunnel info <name/uuid>`")
+
 	// Print column headers with tabbed columns
 	_, _ = fmt.Fprintln(writer, "ID\tNAME\tCREATED\tCONNECTIONS\t")
 
@@ -322,14 +349,176 @@ func fmtConnections(connections []tunnelstore.Connection, showRecentlyDisconnect
 	return strings.Join(output, ", ")
 }
 
+func buildInfoCommand() *cli.Command {
+	return &cli.Command{
+		Name:        "info",
+		Action:      cliutil.ConfiguredAction(tunnelInfo),
+		Usage:       "List details about the active connectors for a tunnel",
+		UsageText:   "cloudflared tunnel [tunnel command options] info [subcommand options] [TUNNEL]",
+		Description: "cloudflared tunnel info displays details about the active connectors for a given tunnel (identified by name or uuid).",
+		Flags: []cli.Flag{
+			outputFormatFlag,
+			showRecentlyDisconnected,
+			sortInfoByFlag,
+			invertInfoSortFlag,
+		},
+		CustomHelpTemplate: commandHelpTemplate(),
+	}
+}
+
+func tunnelInfo(c *cli.Context) error {
+	sc, err := newSubcommandContext(c)
+	if err != nil {
+		return err
+	}
+
+	warningChecker := updater.StartWarningCheck(c)
+	defer warningChecker.LogWarningIfAny(sc.log)
+
+	if c.NArg() != 1 {
+		return cliutil.UsageError(`"cloudflared tunnel info" accepts exactly one argument, the ID or name of the tunnel to get info about.`)
+	}
+	tunnelID, err := sc.findID(c.Args().First())
+	if err != nil {
+		return errors.Wrap(err, "error parsing tunnel ID")
+	}
+
+	client, err := sc.client()
+	if err != nil {
+		return err
+	}
+
+	clients, err := client.ListActiveClients(tunnelID)
+	if err != nil {
+		return err
+	}
+
+	sortBy := c.String("sort-by")
+	invalidSortField := false
+	sort.Slice(clients, func(i, j int) bool {
+		cmp := func() bool {
+			switch sortBy {
+			case "id":
+				return clients[i].ID.String() < clients[j].ID.String()
+			case "createdAt":
+				return clients[i].RunAt.Unix() < clients[j].RunAt.Unix()
+			case "numConnections":
+				return len(clients[i].Connections) < len(clients[j].Connections)
+			case "version":
+				return clients[i].Version < clients[j].Version
+			default:
+				invalidSortField = true
+				return clients[i].RunAt.Unix() < clients[j].RunAt.Unix()
+			}
+		}()
+		if c.Bool("invert-sort") {
+			return !cmp
+		}
+		return cmp
+	})
+	if invalidSortField {
+		sc.log.Error().Msgf("%s is not a valid sort field. Valid sort fields are %s. Defaulting to 'name'.", sortBy, connsSortByOptions)
+	}
+
+	tunnel, err := getTunnel(sc, tunnelID)
+	if err != nil {
+		return err
+	}
+	info := Info{
+		tunnel.ID,
+		tunnel.Name,
+		tunnel.CreatedAt,
+		clients,
+	}
+
+	if outputFormat := c.String(outputFormatFlag.Name); outputFormat != "" {
+		return renderOutput(outputFormat, info)
+	}
+
+	if len(clients) > 0 {
+		formatAndPrintConnectionsList(info, c.Bool("show-recently-disconnected"))
+	} else {
+		fmt.Printf("Your tunnel %s does not have any active connection.\n", tunnelID)
+	}
+
+	return nil
+}
+
+func getTunnel(sc *subcommandContext, tunnelID uuid.UUID) (*tunnelstore.Tunnel, error) {
+	filter := tunnelstore.NewFilter()
+	filter.ByTunnelID(tunnelID)
+	tunnels, err := sc.list(filter)
+	if err != nil {
+		return nil, err
+	}
+	if len(tunnels) != 1 {
+		return nil, errors.Errorf("Expected to find a single tunnel with uuid %v but found %d tunnels.", tunnelID, len(tunnels))
+	}
+	return tunnels[0], nil
+}
+
+func formatAndPrintConnectionsList(tunnelInfo Info, showRecentlyDisconnected bool) {
+	writer := tabWriter()
+	defer writer.Flush()
+
+	// Print the general tunnel info table
+	_, _ = fmt.Fprintf(writer, "NAME:     %s\nID:       %s\nCREATED:  %s\n\n", tunnelInfo.Name, tunnelInfo.ID, tunnelInfo.CreatedAt)
+
+	// Determine whether to print the connector table
+	shouldDisplayTable := false
+	for _, c := range tunnelInfo.Connectors {
+		conns := fmtConnections(c.Connections, showRecentlyDisconnected)
+		if len(conns) > 0 {
+			shouldDisplayTable = true
+		}
+	}
+	if !shouldDisplayTable {
+		fmt.Println("This tunnel has no active connectors.")
+		return
+	}
+
+	// Print the connector table
+	_, _ = fmt.Fprintln(writer, "CONNECTOR ID\tCREATED\tARCHITECTURE\tVERSION\tORIGIN IP\tEDGE\t")
+	for _, c := range tunnelInfo.Connectors {
+		conns := fmtConnections(c.Connections, showRecentlyDisconnected)
+		if len(conns) == 0 {
+			continue
+		}
+		originIp := c.Connections[0].OriginIP.String()
+		formattedStr := fmt.Sprintf(
+			"%s\t%s\t%s\t%s\t%s\t%s\t",
+			c.ID,
+			c.RunAt.Format(time.RFC3339),
+			c.Arch,
+			c.Version,
+			originIp,
+			conns,
+		)
+		_, _ = fmt.Fprintln(writer, formattedStr)
+	}
+}
+
+func tabWriter() *tabwriter.Writer {
+	const (
+		minWidth = 0
+		tabWidth = 8
+		padding  = 1
+		padChar  = ' '
+		flags    = 0
+	)
+
+	writer := tabwriter.NewWriter(os.Stdout, minWidth, tabWidth, padding, padChar, flags)
+	return writer
+}
+
 func buildDeleteCommand() *cli.Command {
 	return &cli.Command{
 		Name:               "delete",
-		Action:             cliutil.ErrorHandler(deleteCommand),
+		Action:             cliutil.ConfiguredAction(deleteCommand),
 		Usage:              "Delete existing tunnel by UUID or name",
 		UsageText:          "cloudflared tunnel [tunnel command options] delete [subcommand options] TUNNEL",
 		Description:        "cloudflared tunnel delete will delete tunnels with the given tunnel UUIDs or names. A tunnel cannot be deleted if it has active connections. To delete the tunnel unconditionally, use -f flag.",
-		Flags:              []cli.Flag{credentialsFileFlag, forceDeleteFlag},
+		Flags:              []cli.Flag{credentialsFileFlagCLIOnly, forceDeleteFlag},
 		CustomHelpTemplate: commandHelpTemplate(),
 	}
 }
@@ -344,6 +533,9 @@ func deleteCommand(c *cli.Context) error {
 		return cliutil.UsageError(`"cloudflared tunnel delete" requires at least 1 argument, the ID or name of the tunnel to delete.`)
 	}
 
+	warningChecker := updater.StartWarningCheck(c)
+	defer warningChecker.LogWarningIfAny(sc.log)
+
 	tunnelIDs, err := sc.findIDs(c.Args().Slice())
 	if err != nil {
 		return err
@@ -370,12 +562,12 @@ func buildRunCommand() *cli.Command {
 		forceFlag,
 		credentialsFileFlag,
 		selectProtocolFlag,
+		featuresFlag,
 	}
 	flags = append(flags, configureProxyFlags(false)...)
 	return &cli.Command{
 		Name:      "run",
-		Action:    cliutil.ErrorHandler(runCommand),
-		Before:    SetFlagsFromConfigFile,
+		Action:    cliutil.ConfiguredAction(runCommand),
 		Usage:     "Proxy a local web server by running the given tunnel",
 		UsageText: "cloudflared tunnel [tunnel command options] run [subcommand options] [TUNNEL]",
 		Description: `Runs the tunnel identified by name or UUUD, creating highly available connections
@@ -410,6 +602,12 @@ func runCommand(c *cli.Context) error {
 		}
 	}
 
+	if c.String("hostname") != "" {
+		sc.log.Warn().Msg("The property `hostname` in your configuration is ignored because you configured a Named Tunnel " +
+			"in the property `tunnel` to run. Make sure to provision the routing (e.g. via `cloudflared tunnel route dns/lb`) or else " +
+			"your origin will not be reachable. You should remove the `hostname` property to avoid this warning.")
+	}
+
 	return runNamedTunnel(sc, tunnelRef)
 }
 
@@ -427,10 +625,11 @@ func runNamedTunnel(sc *subcommandContext, tunnelRef string) error {
 func buildCleanupCommand() *cli.Command {
 	return &cli.Command{
 		Name:               "cleanup",
-		Action:             cliutil.ErrorHandler(cleanupCommand),
+		Action:             cliutil.ConfiguredAction(cleanupCommand),
 		Usage:              "Cleanup tunnel connections",
 		UsageText:          "cloudflared tunnel [tunnel command options] cleanup [subcommand options] TUNNEL",
 		Description:        "Delete connections for tunnels with the given UUIDs or names.",
+		Flags:              []cli.Flag{cleanupClientFlag},
 		CustomHelpTemplate: commandHelpTemplate(),
 	}
 }
@@ -456,26 +655,49 @@ func cleanupCommand(c *cli.Context) error {
 func buildRouteCommand() *cli.Command {
 	return &cli.Command{
 		Name:      "route",
-		Action:    cliutil.ErrorHandler(routeCommand),
-		Usage:     "Define what hostname or load balancer can route to this tunnel",
-		UsageText: "cloudflared tunnel [tunnel command options] route [subcommand options] dns|lb TUNNEL HOSTNAME [LB-POOL]",
-		Description: `The route defines what hostname or load balancer will proxy requests to this tunnel.
+		Usage:     "Define which traffic routed from Cloudflare edge to this tunnel: requests to a DNS hostname, to a Cloudflare Load Balancer, or traffic originating from Cloudflare WARP clients",
+		UsageText: "cloudflared tunnel [tunnel command options] route [subcommand options] [dns TUNNEL HOSTNAME]|[lb TUNNEL HOSTNAME LB-POOL]|[ip NETWORK TUNNEL]",
+		Description: `The route command defines how Cloudflare will proxy requests to this tunnel.
 
-   To route a hostname by creating a CNAME to tunnel's address:
-      cloudflared tunnel route dns <tunnel ID> <hostname>
-   To use this tunnel as a load balancer origin, creating pool and load balancer if necessary:
-      cloudflared tunnel route lb <tunnel ID> <load balancer name> <load balancer pool>`,
+To route a hostname by creating a DNS CNAME record to a tunnel:
+   cloudflared tunnel route dns <tunnel ID or name> <hostname>
+You can read more at: https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/routing-to-tunnel/dns
+
+To use this tunnel as a load balancer origin, creating pool and load balancer if necessary:
+   cloudflared tunnel route lb <tunnel ID or name> <hostname> <load balancer pool>
+You can read more at: https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/routing-to-tunnel/lb
+
+For Cloudflare WARP traffic to be routed to your private network, reachable from this tunnel as origins, use:
+   cloudflared tunnel route ip <network CIDR> <tunnel ID or name>
+Further information about managing Cloudflare WARP traffic to your tunnel is available at:
+   cloudflared tunnel route ip --help
+`,
 		CustomHelpTemplate: commandHelpTemplate(),
 		Subcommands: []*cli.Command{
+			{
+				Name:        "dns",
+				Action:      cliutil.ConfiguredAction(routeDnsCommand),
+				Usage:       "Route a hostname by creating a DNS CNAME record to a tunnel",
+				UsageText:   "cloudflared tunnel route dns [TUNNEL] [HOSTNAME]",
+				Description: `Creates a DNS CNAME record hostname that points to the tunnel.`,
+				Flags:       []cli.Flag{overwriteDNSFlag},
+			},
+			{
+				Name:        "lb",
+				Action:      cliutil.ConfiguredAction(routeLbCommand),
+				Usage:       "Use this tunnel as a load balancer origin, creating pool and load balancer if necessary",
+				UsageText:   "cloudflared tunnel route dns [TUNNEL] [HOSTNAME] [LB-POOL]",
+				Description: `Creates Load Balancer with an origin pool that points to the tunnel.`,
+			},
 			buildRouteIPSubcommand(),
 		},
 	}
 }
 
-func dnsRouteFromArg(c *cli.Context) (tunnelstore.Route, error) {
+func dnsRouteFromArg(c *cli.Context, overwriteExisting bool) (tunnelstore.Route, error) {
 	const (
-		userHostnameIndex = 2
-		expectedNArgs     = 3
+		userHostnameIndex = 1
+		expectedNArgs     = 2
 	)
 	if c.NArg() != expectedNArgs {
 		return nil, cliutil.UsageError("Expected %d arguments, got %d", expectedNArgs, c.NArg())
@@ -486,14 +708,14 @@ func dnsRouteFromArg(c *cli.Context) (tunnelstore.Route, error) {
 	} else if !validateHostname(userHostname, true) {
 		return nil, errors.Errorf("%s is not a valid hostname", userHostname)
 	}
-	return tunnelstore.NewDNSRoute(userHostname), nil
+	return tunnelstore.NewDNSRoute(userHostname, overwriteExisting), nil
 }
 
 func lbRouteFromArg(c *cli.Context) (tunnelstore.Route, error) {
 	const (
-		lbNameIndex   = 2
-		lbPoolIndex   = 3
-		expectedNArgs = 4
+		lbNameIndex   = 1
+		lbPoolIndex   = 2
+		expectedNArgs = 3
 	)
 	if c.NArg() != expectedNArgs {
 		return nil, cliutil.UsageError("Expected %d arguments, got %d", expectedNArgs, c.NArg())
@@ -535,41 +757,39 @@ func validateHostname(s string, allowWildcardSubdomain bool) bool {
 	return err == nil && validateName(puny, allowWildcardSubdomain)
 }
 
-func routeCommand(c *cli.Context) error {
-	if c.NArg() < 2 {
-		return cliutil.UsageError(`"cloudflared tunnel route" requires the first argument to be the route type(dns or lb), followed by the ID or name of the tunnel`)
+func routeDnsCommand(c *cli.Context) error {
+	if c.NArg() != 2 {
+		return cliutil.UsageError(`This command expects the format "cloudflared tunnel route dns <tunnel name/id> <hostname>"`)
 	}
+	return routeCommand(c, "dns")
+}
+
+func routeLbCommand(c *cli.Context) error {
+	if c.NArg() != 3 {
+		return cliutil.UsageError(`This command expects the format "cloudflared tunnel route lb <tunnel name/id> <hostname> <load balancer pool>"`)
+	}
+	return routeCommand(c, "lb")
+}
+
+func routeCommand(c *cli.Context, routeType string) error {
 	sc, err := newSubcommandContext(c)
 	if err != nil {
 		return err
 	}
 
-	const tunnelIDIndex = 1
-
-	routeType := c.Args().First()
+	tunnelID, err := sc.findID(c.Args().Get(0))
+	if err != nil {
+		return err
+	}
 	var route tunnelstore.Route
-	var tunnelID uuid.UUID
 	switch routeType {
 	case "dns":
-		tunnelID, err = sc.findID(c.Args().Get(tunnelIDIndex))
-		if err != nil {
-			return err
-		}
-		route, err = dnsRouteFromArg(c)
-		if err != nil {
-			return err
-		}
+		route, err = dnsRouteFromArg(c, c.Bool(overwriteDNSFlagName))
 	case "lb":
-		tunnelID, err = sc.findID(c.Args().Get(tunnelIDIndex))
-		if err != nil {
-			return err
-		}
 		route, err = lbRouteFromArg(c)
-		if err != nil {
-			return err
-		}
-	default:
-		return cliutil.UsageError("%s is not a recognized route type. Supported route types are dns and lb", routeType)
+	}
+	if err != nil {
+		return err
 	}
 
 	res, err := sc.route(tunnelID, route)

cmd/cloudflared/tunnel/subcommands_test.go

@@ -1,14 +1,14 @@
 package tunnel
 
 import (
-	"fmt"
 	"path/filepath"
 	"testing"
 
-	"github.com/cloudflare/cloudflared/tunnelstore"
 	"github.com/google/uuid"
-	"github.com/mitchellh/go-homedir"
+	homedir "github.com/mitchellh/go-homedir"
 	"github.com/stretchr/testify/assert"
+
+	"github.com/cloudflare/cloudflared/tunnelstore"
 )
 
 func Test_fmtConnections(t *testing.T) {
@@ -94,7 +94,7 @@ func TestTunnelfilePath(t *testing.T) {
 	assert.NoError(t, err)
 	homeDir, err := homedir.Dir()
 	assert.NoError(t, err)
-	expected := fmt.Sprintf("%s/.cloudflared/%v.json", homeDir, tunnelID)
+	expected := filepath.Join(homeDir, ".cloudflared", tunnelID.String()+".json")
 	assert.Equal(t, expected, actual)
 }
 

cmd/cloudflared/tunnel/teamnet_subcommands.go

@@ -6,54 +6,57 @@ import (
 	"os"
 	"text/tabwriter"
 
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
-	"github.com/cloudflare/cloudflared/teamnet"
 	"github.com/pkg/errors"
 
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/updater"
+	"github.com/cloudflare/cloudflared/teamnet"
+
 	"github.com/urfave/cli/v2"
 )
 
 func buildRouteIPSubcommand() *cli.Command {
 	return &cli.Command{
 		Name:      "ip",
-		Category:  "Tunnel",
-		Usage:     "Configure and query private routes",
+		Usage:     "Configure and query Cloudflare WARP routing to services or private networks available through this tunnel.",
 		UsageText: "cloudflared tunnel [--config FILEPATH] route COMMAND [arguments...]",
-		Hidden:    true,
-		Description: `cloudflared can provision private routes from your private IP space to origins
-		in your corporate network. Users enrolled in your Cloudflare for Teams organization can reach
-		those routes through the Cloudflare Warp client. You can also build rules to determine who
-		can reach certain routes.
-		`,
+		Description: `cloudflared can provision private routes from any IP space to origins in your corporate network.
+Users enrolled in your Cloudflare for Teams organization can reach those routes through the
+Cloudflare WARP client. You can also build rules to determine who can reach certain routes.`,
 		Subcommands: []*cli.Command{
 			{
 				Name:      "add",
-				Action:    cliutil.ErrorHandler(addRouteCommand),
-				Usage:     "Add a new Teamnet route to the table",
+				Action:    cliutil.ConfiguredAction(addRouteCommand),
+				Usage:     "Add any new network to the routing table reachable via the tunnel",
 				UsageText: "cloudflared tunnel [--config FILEPATH] route ip add [CIDR] [TUNNEL] [COMMENT?]",
-				Description: `Adds a private route to a CIDR in your private IP space. Requests will
-				be sent through the Cloudflare Warp client running on a user's machine, proxied
-				through the specified tunnel, and reach an IP in the given CIDR.`,
+				Description: `Adds any network route space (represented as a CIDR) to your routing table.
+That network space becomes reachable for requests egressing from a user's machine
+as long as it is using Cloudflare WARP client and is enrolled in the same account
+that is running the tunnel chosen here. Further, those requests will be proxied to
+the specified tunnel, and reach an IP in the given CIDR, as long as that IP is
+reachable from the tunnel.`,
 			},
 			{
 				Name:        "show",
-				Action:      cliutil.ErrorHandler(showRoutesCommand),
+				Aliases:     []string{"list"},
+				Action:      cliutil.ConfiguredAction(showRoutesCommand),
 				Usage:       "Show the routing table",
 				UsageText:   "cloudflared tunnel [--config FILEPATH] route ip show [flags]",
-				Description: `Shows your organization's private route table. You can use flags to filter the results.`,
-				Flags:       teamnet.FilterFlags,
+				Description: `Shows your organization private routing table. You can use flags to filter the results.`,
+				Flags:       showRoutesFlags(),
 			},
 			{
-				Name:        "delete",
-				Action:      cliutil.ErrorHandler(deleteRouteCommand),
-				Usage:       "Delete a row from your organization's private routing table",
-				UsageText:   "cloudflared tunnel [--config FILEPATH] route ip delete [CIDR]",
-				Description: `Deletes the row for a given CIDR from your routing table`,
+				Name:      "delete",
+				Action:    cliutil.ConfiguredAction(deleteRouteCommand),
+				Usage:     "Delete a row from your organization's private routing table",
+				UsageText: "cloudflared tunnel [--config FILEPATH] route ip delete [CIDR]",
+				Description: `Deletes the row for a given CIDR from your routing table. That portion
+of your network will no longer be reachable by the WARP clients.`,
 			},
 			{
 				Name:      "get",
-				Action:    cliutil.ErrorHandler(getRouteByIPCommand),
-				Usage:     "Check which row of the routing table matches a given IP",
+				Action:    cliutil.ConfiguredAction(getRouteByIPCommand),
+				Usage:     "Check which row of the routing table matches a given IP.",
 				UsageText: "cloudflared tunnel [--config FILEPATH] route ip get [IP]",
 				Description: `Checks which row of the routing table will be used to proxy a given IP.
 				This helps check and validate your config.`,
@@ -62,6 +65,13 @@ func buildRouteIPSubcommand() *cli.Command {
 	}
 }
 
+func showRoutesFlags() []cli.Flag {
+	flags := make([]cli.Flag, 0)
+	flags = append(flags, teamnet.FilterFlags...)
+	flags = append(flags, outputFormatFlag)
+	return flags
+}
+
 func showRoutesCommand(c *cli.Context) error {
 	sc, err := newSubcommandContext(c)
 	if err != nil {
@@ -73,6 +83,9 @@ func showRoutesCommand(c *cli.Context) error {
 		return errors.Wrap(err, "invalid config for routing filters")
 	}
 
+	warningChecker := updater.StartWarningCheck(c)
+	defer warningChecker.LogWarningIfAny(sc.log)
+
 	routes, err := sc.listRoutes(filter)
 	if err != nil {
 		return err
@@ -87,6 +100,7 @@ func showRoutesCommand(c *cli.Context) error {
 	} else {
 		fmt.Println("You have no routes, use 'cloudflared tunnel route ip add' to add a route")
 	}
+
 	return nil
 }
 

cmd/cloudflared/ui/launch_ui.go

@@ -5,12 +5,12 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/cloudflare/cloudflared/connection"
-	"github.com/cloudflare/cloudflared/ingress"
-
 	"github.com/gdamore/tcell"
 	"github.com/rivo/tview"
 	"github.com/rs/zerolog"
+
+	"github.com/cloudflare/cloudflared/connection"
+	"github.com/cloudflare/cloudflared/ingress"
 )
 
 type connState struct {

cmd/cloudflared/updater/check.go

@@ -0,0 +1,49 @@
+package updater
+
+import (
+	"github.com/rs/zerolog"
+	"github.com/urfave/cli/v2"
+)
+
+type VersionWarningChecker struct {
+	warningChan chan string
+}
+
+func StartWarningCheck(c *cli.Context) VersionWarningChecker {
+	checker := VersionWarningChecker{
+		warningChan: make(chan string),
+	}
+
+	go func() {
+		options := updateOptions{
+			updateDisabled:  true,
+			isBeta:          c.Bool("beta"),
+			isStaging:       c.Bool("staging"),
+			isForced:        false,
+			intendedVersion: "",
+		}
+		checkResult, err := CheckForUpdate(options)
+		if err == nil {
+			checker.warningChan <- checkResult.UserMessage()
+		}
+		close(checker.warningChan)
+	}()
+
+	return checker
+}
+
+func (checker VersionWarningChecker) getWarning() string {
+	select {
+	case message := <-checker.warningChan:
+		return message
+	default:
+		// No feedback on time, we don't wait for it, since this is best-effort.
+		return ""
+	}
+}
+
+func (checker VersionWarningChecker) LogWarningIfAny(log *zerolog.Logger) {
+	if warning := checker.getWarning(); warning != "" {
+		log.Warn().Msg(warning)
+	}
+}

cmd/cloudflared/updater/service.go

@@ -1,14 +1,15 @@
 package updater
 
-// Version is the functions needed to perform an update
-type Version interface {
+// CheckResult is the behaviour resulting from checking in with the Update Service
+type CheckResult interface {
 	Apply() error
-	String() string
+	Version() string
+	UserMessage() string
 }
 
 // Service is the functions to get check for new updates
 type Service interface {
-	Check() (Version, error)
+	Check() (CheckResult, error)
 }
 
 const (
@@ -23,4 +24,7 @@ const (
 
 	// VersionKeyName is the url parameter key to send to the checkin API to specific what version to upgrade or downgrade to
 	VersionKeyName = "version"
+
+	// ClientVersionName is the url parameter key to send the version that this cloudflared is currently running with
+	ClientVersionName = "clientVersion"
 )

cmd/cloudflared/updater/update.go

@@ -3,18 +3,18 @@ package updater
 import (
 	"context"
 	"fmt"
-	"github.com/rs/zerolog"
 	"os"
 	"path/filepath"
 	"runtime"
 	"time"
 
+	"github.com/facebookgo/grace/gracenet"
+	"github.com/rs/zerolog"
 	"github.com/urfave/cli/v2"
 	"golang.org/x/crypto/ssh/terminal"
 
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/logger"
-	"github.com/facebookgo/grace/gracenet"
 )
 
 const (
@@ -61,16 +61,18 @@ func (e *statusErr) ExitCode() int {
 }
 
 type updateOptions struct {
-	isBeta    bool
-	isStaging bool
-	isForced  bool
-	version   string
+	updateDisabled  bool
+	isBeta          bool
+	isStaging       bool
+	isForced        bool
+	intendedVersion string
 }
 
 type UpdateOutcome struct {
-	Updated bool
-	Version string
-	Error   error
+	Updated     bool
+	Version     string
+	UserMessage string
+	Error       error
 }
 
 func (uo *UpdateOutcome) noUpdate() bool {
@@ -81,10 +83,10 @@ func Init(v string) {
 	version = v
 }
 
-func checkForUpdateAndApply(options updateOptions) UpdateOutcome {
+func CheckForUpdate(options updateOptions) (CheckResult, error) {
 	cfdPath, err := os.Executable()
 	if err != nil {
-		return UpdateOutcome{Error: err}
+		return nil, err
 	}
 
 	url := UpdateURL
@@ -93,24 +95,22 @@ func checkForUpdateAndApply(options updateOptions) UpdateOutcome {
 	}
 
 	s := NewWorkersService(version, url, cfdPath, Options{IsBeta: options.isBeta,
-		IsForced: options.isForced, RequestedVersion: options.version})
+		IsForced: options.isForced, RequestedVersion: options.intendedVersion})
 
-	v, err := s.Check()
+	return s.Check()
+}
+
+func applyUpdate(options updateOptions, update CheckResult) UpdateOutcome {
+	if update.Version() == "" || options.updateDisabled {
+		return UpdateOutcome{UserMessage: update.UserMessage()}
+	}
+
+	err := update.Apply()
 	if err != nil {
 		return UpdateOutcome{Error: err}
 	}
 
-	//already on the latest version
-	if v == nil {
-		return UpdateOutcome{}
-	}
-
-	err = v.Apply()
-	if err != nil {
-		return UpdateOutcome{Error: err}
-	}
-
-	return UpdateOutcome{Updated: true, Version: v.String()}
+	return UpdateOutcome{Updated: true, Version: update.Version(), UserMessage: update.UserMessage()}
 }
 
 // Update is the handler for the update command from the command line
@@ -137,7 +137,13 @@ func Update(c *cli.Context) error {
 		log.Info().Msg("cloudflared is set to upgrade to the latest publish version regardless of the current version")
 	}
 
-	updateOutcome := loggedUpdate(log, updateOptions{isBeta: isBeta, isStaging: isStaging, isForced: isForced, version: c.String("version")})
+	updateOutcome := loggedUpdate(log, updateOptions{
+		updateDisabled:  false,
+		isBeta:          isBeta,
+		isStaging:       isStaging,
+		isForced:        isForced,
+		intendedVersion: c.String("version"),
+	})
 	if updateOutcome.Error != nil {
 		return &statusErr{updateOutcome.Error}
 	}
@@ -152,12 +158,18 @@ func Update(c *cli.Context) error {
 
 // Checks for an update and applies it if one is available
 func loggedUpdate(log *zerolog.Logger, options updateOptions) UpdateOutcome {
-	updateOutcome := checkForUpdateAndApply(options)
+	checkResult, err := CheckForUpdate(options)
+	if err != nil {
+		log.Err(err).Msg("update check failed")
+		return UpdateOutcome{Error: err}
+	}
+
+	updateOutcome := applyUpdate(options, checkResult)
 	if updateOutcome.Updated {
 		log.Info().Str(LogFieldVersion, updateOutcome.Version).Msg("cloudflared has been updated")
 	}
 	if updateOutcome.Error != nil {
-		log.Err(updateOutcome.Error).Msg("update check failed: %s")
+		log.Err(updateOutcome.Error).Msg("update failed to apply")
 	}
 
 	return updateOutcome
@@ -177,44 +189,53 @@ type configurable struct {
 	freq    time.Duration
 }
 
-func NewAutoUpdater(freq time.Duration, listeners *gracenet.Net, log *zerolog.Logger) *AutoUpdater {
-	updaterConfigurable := &configurable{
-		enabled: true,
-		freq:    freq,
-	}
-	if freq == 0 {
-		updaterConfigurable.enabled = false
-		updaterConfigurable.freq = DefaultCheckUpdateFreq
-	}
+func NewAutoUpdater(updateDisabled bool, freq time.Duration, listeners *gracenet.Net, log *zerolog.Logger) *AutoUpdater {
 	return &AutoUpdater{
-		configurable:     updaterConfigurable,
+		configurable:     createUpdateConfig(updateDisabled, freq, log),
 		listeners:        listeners,
 		updateConfigChan: make(chan *configurable),
 		log:              log,
 	}
 }
 
+func createUpdateConfig(updateDisabled bool, freq time.Duration, log *zerolog.Logger) *configurable {
+	if isAutoupdateEnabled(log, updateDisabled, freq) {
+		log.Info().Dur("autoupdateFreq", freq).Msg("Autoupdate frequency is set")
+		return &configurable{
+			enabled: true,
+			freq:    freq,
+		}
+	} else {
+		return &configurable{
+			enabled: false,
+			freq:    DefaultCheckUpdateFreq,
+		}
+	}
+}
+
 func (a *AutoUpdater) Run(ctx context.Context) error {
 	ticker := time.NewTicker(a.configurable.freq)
 	for {
-		if a.configurable.enabled {
-			updateOutcome := loggedUpdate(a.log, updateOptions{})
-			if updateOutcome.Updated {
-				if IsSysV() {
-					// SysV doesn't have a mechanism to keep service alive, we have to restart the process
-					a.log.Info().Msg("Restarting service managed by SysV...")
-					pid, err := a.listeners.StartProcess()
-					if err != nil {
-						a.log.Err(err).Msg("Unable to restart server automatically")
-						return &statusErr{err: err}
-					}
-					// stop old process after autoupdate. Otherwise we create a new process
-					// after each update
-					a.log.Info().Msgf("PID of the new process is %d", pid)
+		updateOutcome := loggedUpdate(a.log, updateOptions{updateDisabled: !a.configurable.enabled})
+		if updateOutcome.Updated {
+			Init(updateOutcome.Version)
+			if IsSysV() {
+				// SysV doesn't have a mechanism to keep service alive, we have to restart the process
+				a.log.Info().Msg("Restarting service managed by SysV...")
+				pid, err := a.listeners.StartProcess()
+				if err != nil {
+					a.log.Err(err).Msg("Unable to restart server automatically")
+					return &statusErr{err: err}
 				}
-				return &statusSuccess{newVersion: updateOutcome.Version}
+				// stop old process after autoupdate. Otherwise we create a new process
+				// after each update
+				a.log.Info().Msgf("PID of the new process is %d", pid)
 			}
+			return &statusSuccess{newVersion: updateOutcome.Version}
+		} else if updateOutcome.UserMessage != "" {
+			a.log.Warn().Msg(updateOutcome.UserMessage)
 		}
+
 		select {
 		case <-ctx.Done():
 			return ctx.Err()
@@ -229,27 +250,18 @@ func (a *AutoUpdater) Run(ctx context.Context) error {
 }
 
 // Update is the method to pass new AutoUpdaterConfigurable to a running AutoUpdater. It is safe to be called concurrently
-func (a *AutoUpdater) Update(newFreq time.Duration) {
-	newConfigurable := &configurable{
-		enabled: true,
-		freq:    newFreq,
-	}
-	// A ero duration means autoupdate is disabled
-	if newFreq == 0 {
-		newConfigurable.enabled = false
-		newConfigurable.freq = DefaultCheckUpdateFreq
-	}
-	a.updateConfigChan <- newConfigurable
+func (a *AutoUpdater) Update(updateDisabled bool, newFreq time.Duration) {
+	a.updateConfigChan <- createUpdateConfig(updateDisabled, newFreq, a.log)
 }
 
-func IsAutoupdateEnabled(c *cli.Context, log *zerolog.Logger) bool {
-	if !SupportAutoUpdate(log) {
+func isAutoupdateEnabled(log *zerolog.Logger, updateDisabled bool, updateFreq time.Duration) bool {
+	if !supportAutoUpdate(log) {
 		return false
 	}
-	return !c.Bool("no-autoupdate") && c.Duration("autoupdate-freq") != 0
+	return !updateDisabled && updateFreq != 0
 }
 
-func SupportAutoUpdate(log *zerolog.Logger) bool {
+func supportAutoUpdate(log *zerolog.Logger) bool {
 	if runtime.GOOS == "windows" {
 		log.Info().Msg(noUpdateOnWindowsMessage)
 		return false

cmd/cloudflared/updater/update_test.go

@@ -2,17 +2,19 @@ package updater
 
 import (
 	"context"
+	"flag"
 	"testing"
 
 	"github.com/facebookgo/grace/gracenet"
 	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/assert"
+	"github.com/urfave/cli/v2"
 )
 
 func TestDisabledAutoUpdater(t *testing.T) {
 	listeners := &gracenet.Net{}
 	log := zerolog.Nop()
-	autoupdater := NewAutoUpdater(0, listeners, &log)
+	autoupdater := NewAutoUpdater(false, 0, listeners, &log)
 	ctx, cancel := context.WithCancel(context.Background())
 	errC := make(chan error)
 	go func() {
@@ -26,3 +28,13 @@ func TestDisabledAutoUpdater(t *testing.T) {
 	// Make sure that autoupdater terminates after canceling the context
 	assert.Equal(t, context.Canceled, <-errC)
 }
+
+func TestCheckInWithUpdater(t *testing.T) {
+	flagSet := flag.NewFlagSet(t.Name(), flag.PanicOnError)
+	cliCtx := cli.NewContext(cli.NewApp(), flagSet, nil)
+
+	warningChecker := StartWarningCheck(cliCtx)
+	warning := warningChecker.getWarning()
+	// Assuming this runs either on a release or development version, then the Worker will never have anything to tell us.
+	assert.Empty(t, warning)
+}

cmd/cloudflared/updater/workers_service.go

@@ -5,8 +5,6 @@ import (
 	"errors"
 	"net/http"
 	"runtime"
-	"strconv"
-	"strings"
 )
 
 // Options are the update options supported by the
@@ -27,6 +25,8 @@ type VersionResponse struct {
 	Version      string `json:"version"`
 	Checksum     string `json:"checksum"`
 	IsCompressed bool   `json:"compressed"`
+	UserMessage  string `json:"userMessage"`
+	ShouldUpdate bool   `json:"shouldUpdate"`
 	Error        string `json:"error"`
 }
 
@@ -50,7 +50,7 @@ func NewWorkersService(currentVersion, url, targetPath string, opts Options) Ser
 }
 
 // Check does a check in with the Workers API to get a new version update
-func (s *WorkersService) Check() (Version, error) {
+func (s *WorkersService) Check() (CheckResult, error) {
 	client := &http.Client{
 		Timeout: clientTimeout,
 	}
@@ -59,6 +59,7 @@ func (s *WorkersService) Check() (Version, error) {
 	q := req.URL.Query()
 	q.Add(OSKeyName, runtime.GOOS)
 	q.Add(ArchitectureKeyName, runtime.GOARCH)
+	q.Add(ClientVersionName, s.currentVersion)
 
 	if s.opts.IsBeta {
 		q.Add(BetaKeyName, "true")
@@ -84,70 +85,10 @@ func (s *WorkersService) Check() (Version, error) {
 		return nil, errors.New(v.Error)
 	}
 
-	if !s.opts.IsForced && !IsNewerVersion(s.currentVersion, v.Version) {
-		return nil, nil
+	versionToUpdate := ""
+	if v.ShouldUpdate {
+		versionToUpdate = v.Version
 	}
 
-	return NewWorkersVersion(v.URL, v.Version, v.Checksum, s.targetPath, v.IsCompressed), nil
-}
-
-// IsNewerVersion checks semantic versioning for the latest version
-// cloudflared tagging is more of a date than a semantic version,
-// but the same comparision logic still holds for major.minor.patch
-// e.g. 2020.8.2 is newer than 2020.8.1.
-func IsNewerVersion(current string, check string) bool {
-	if strings.Contains(strings.ToLower(current), "dev") {
-		return false // dev builds shouldn't update
-	}
-
-	cMajor, cMinor, cPatch, err := SemanticParts(current)
-	if err != nil {
-		return false
-	}
-
-	nMajor, nMinor, nPatch, err := SemanticParts(check)
-	if err != nil {
-		return false
-	}
-
-	if nMajor > cMajor {
-		return true
-	}
-
-	if nMajor == cMajor && nMinor > cMinor {
-		return true
-	}
-
-	if nMajor == cMajor && nMinor == cMinor && nPatch > cPatch {
-		return true
-	}
-	return false
-}
-
-// SemanticParts gets the major, minor, and patch version of a semantic version string
-// e.g. 3.1.2 would return 3, 1, 2, nil
-func SemanticParts(version string) (major int, minor int, patch int, err error) {
-	major = 0
-	minor = 0
-	patch = 0
-	parts := strings.Split(version, ".")
-	if len(parts) != 3 {
-		err = errors.New("invalid version")
-		return
-	}
-	major, err = strconv.Atoi(parts[0])
-	if err != nil {
-		return
-	}
-
-	minor, err = strconv.Atoi(parts[1])
-	if err != nil {
-		return
-	}
-
-	patch, err = strconv.Atoi(parts[2])
-	if err != nil {
-		return
-	}
-	return
+	return NewWorkersVersion(v.URL, versionToUpdate, v.Checksum, s.targetPath, v.UserMessage, v.IsCompressed), nil
 }

cmd/cloudflared/updater/workers_service_test.go

@@ -1,3 +1,5 @@
+// +build !windows
+
 package updater
 
 import (
@@ -6,18 +8,24 @@ import (
 	"compress/gzip"
 	"crypto/sha256"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"log"
 	"net/http"
 	"net/http/httptest"
 	"os"
+	"path/filepath"
 	"runtime"
+	"strconv"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/require"
 )
 
+var testFilePath = filepath.Join(os.TempDir(), "test")
+
 func respondWithJSON(w http.ResponseWriter, v interface{}, status int) {
 	data, _ := json.Marshal(v)
 
@@ -32,15 +40,20 @@ func respondWithData(w http.ResponseWriter, b []byte, status int) {
 	w.Write(b)
 }
 
+const mostRecentVersion = "2021.2.5"
+const mostRecentBetaVersion = "2021.3.0"
+const knownBuggyVersion = "2020.12.0"
+const expectedUserMsg = "This message is expected when running a known buggy version"
+
 func updateHandler(w http.ResponseWriter, r *http.Request) {
-	version := "2020.08.05"
+	version := mostRecentVersion
 	host := fmt.Sprintf("http://%s", r.Host)
 	url := host + "/download"
 
 	query := r.URL.Query()
 
 	if query.Get(BetaKeyName) == "true" {
-		version = "2020.08.06"
+		version = mostRecentBetaVersion
 		url = host + "/beta"
 	}
 
@@ -59,7 +72,15 @@ func updateHandler(w http.ResponseWriter, r *http.Request) {
 	fmt.Fprint(h, version)
 	checksum := fmt.Sprintf("%x", h.Sum(nil))
 
-	v := VersionResponse{URL: url, Version: version, Checksum: checksum}
+	var userMessage = ""
+	if query.Get(ClientVersionName) == knownBuggyVersion {
+		userMessage = expectedUserMsg
+	}
+	shouldUpdate := requestedVersion != "" || IsNewerVersion(query.Get(ClientVersionName), version)
+
+	v := VersionResponse{
+		URL: url, Version: version, Checksum: checksum, UserMessage: userMessage, ShouldUpdate: shouldUpdate,
+	}
 	respondWithJSON(w, v, http.StatusOK)
 }
 
@@ -71,7 +92,7 @@ func gzipUpdateHandler(w http.ResponseWriter, r *http.Request) {
 	checksum := fmt.Sprintf("%x", h.Sum(nil))
 
 	url := fmt.Sprintf("http://%s/gzip-download.tgz", r.Host)
-	v := VersionResponse{URL: url, Version: version, Checksum: checksum}
+	v := VersionResponse{URL: url, Version: version, Checksum: checksum, ShouldUpdate: true}
 	respondWithJSON(w, v, http.StatusOK)
 }
 
@@ -96,7 +117,7 @@ func compressedDownloadHandler(w http.ResponseWriter, r *http.Request) {
 }
 
 func downloadHandler(w http.ResponseWriter, r *http.Request) {
-	version := "2020.08.05"
+	version := mostRecentVersion
 	requestedVersion := r.URL.Query().Get(VersionKeyName)
 	if requestedVersion != "" {
 		version = requestedVersion
@@ -105,13 +126,71 @@ func downloadHandler(w http.ResponseWriter, r *http.Request) {
 }
 
 func betaHandler(w http.ResponseWriter, r *http.Request) {
-	respondWithData(w, []byte("2020.08.06"), http.StatusOK)
+	respondWithData(w, []byte(mostRecentBetaVersion), http.StatusOK)
 }
 
 func failureHandler(w http.ResponseWriter, r *http.Request) {
 	respondWithJSON(w, VersionResponse{Error: "unsupported os and architecture"}, http.StatusBadRequest)
 }
 
+func IsNewerVersion(current string, check string) bool {
+	if current == "" || check == "" {
+		return false
+	}
+	if strings.Contains(strings.ToLower(current), "dev") {
+		return false // dev builds shouldn't update
+	}
+
+	cMajor, cMinor, cPatch, err := SemanticParts(current)
+	if err != nil {
+		return false
+	}
+
+	nMajor, nMinor, nPatch, err := SemanticParts(check)
+	if err != nil {
+		return false
+	}
+
+	if nMajor > cMajor {
+		return true
+	}
+
+	if nMajor == cMajor && nMinor > cMinor {
+		return true
+	}
+
+	if nMajor == cMajor && nMinor == cMinor && nPatch > cPatch {
+		return true
+	}
+	return false
+}
+
+func SemanticParts(version string) (major int, minor int, patch int, err error) {
+	major = 0
+	minor = 0
+	patch = 0
+	parts := strings.Split(version, ".")
+	if len(parts) != 3 {
+		err = errors.New("invalid version")
+		return
+	}
+	major, err = strconv.Atoi(parts[0])
+	if err != nil {
+		return
+	}
+
+	minor, err = strconv.Atoi(parts[1])
+	if err != nil {
+		return
+	}
+
+	patch, err = strconv.Atoi(parts[2])
+	if err != nil {
+		return
+	}
+	return
+}
+
 func createServer() *httptest.Server {
 	mux := http.NewServeMux()
 	mux.HandleFunc("/updater", updateHandler)
@@ -124,7 +203,7 @@ func createServer() *httptest.Server {
 }
 
 func createTestFile(t *testing.T, path string) {
-	f, err := os.Create("tmpfile")
+	f, err := os.Create(path)
 	require.NoError(t, err)
 	fmt.Fprint(f, "2020.08.04")
 	f.Close()
@@ -134,7 +213,6 @@ func TestUpdateService(t *testing.T) {
 	ts := createServer()
 	defer ts.Close()
 
-	testFilePath := "tmpfile"
 	createTestFile(t, testFilePath)
 	defer os.Remove(testFilePath)
 	log.Println("server url: ", ts.URL)
@@ -142,40 +220,38 @@ func TestUpdateService(t *testing.T) {
 	s := NewWorkersService("2020.8.2", fmt.Sprintf("%s/updater", ts.URL), testFilePath, Options{})
 	v, err := s.Check()
 	require.NoError(t, err)
-	require.Equal(t, v.String(), "2020.08.05")
+	require.Equal(t, v.Version(), mostRecentVersion)
 
 	require.NoError(t, v.Apply())
 	dat, err := ioutil.ReadFile(testFilePath)
 	require.NoError(t, err)
 
-	require.Equal(t, string(dat), "2020.08.05")
+	require.Equal(t, string(dat), mostRecentVersion)
 }
 
 func TestBetaUpdateService(t *testing.T) {
 	ts := createServer()
 	defer ts.Close()
 
-	testFilePath := "tmpfile"
 	createTestFile(t, testFilePath)
 	defer os.Remove(testFilePath)
 
 	s := NewWorkersService("2020.8.2", fmt.Sprintf("%s/updater", ts.URL), testFilePath, Options{IsBeta: true})
 	v, err := s.Check()
 	require.NoError(t, err)
-	require.Equal(t, v.String(), "2020.08.06")
+	require.Equal(t, v.Version(), mostRecentBetaVersion)
 
 	require.NoError(t, v.Apply())
 	dat, err := ioutil.ReadFile(testFilePath)
 	require.NoError(t, err)
 
-	require.Equal(t, string(dat), "2020.08.06")
+	require.Equal(t, string(dat), mostRecentBetaVersion)
 }
 
 func TestFailUpdateService(t *testing.T) {
 	ts := createServer()
 	defer ts.Close()
 
-	testFilePath := "tmpfile"
 	createTestFile(t, testFilePath)
 	defer os.Remove(testFilePath)
 
@@ -189,41 +265,39 @@ func TestNoUpdateService(t *testing.T) {
 	ts := createServer()
 	defer ts.Close()
 
-	testFilePath := "tmpfile"
 	createTestFile(t, testFilePath)
 	defer os.Remove(testFilePath)
 
-	s := NewWorkersService("2020.8.5", fmt.Sprintf("%s/updater", ts.URL), testFilePath, Options{})
+	s := NewWorkersService(mostRecentVersion, fmt.Sprintf("%s/updater", ts.URL), testFilePath, Options{})
 	v, err := s.Check()
 	require.NoError(t, err)
-	require.Nil(t, v)
+	require.NotNil(t, v)
+	require.Empty(t, v.Version())
 }
 
 func TestForcedUpdateService(t *testing.T) {
 	ts := createServer()
 	defer ts.Close()
 
-	testFilePath := "tmpfile"
 	createTestFile(t, testFilePath)
 	defer os.Remove(testFilePath)
 
 	s := NewWorkersService("2020.8.5", fmt.Sprintf("%s/updater", ts.URL), testFilePath, Options{IsForced: true})
 	v, err := s.Check()
 	require.NoError(t, err)
-	require.Equal(t, v.String(), "2020.08.05")
+	require.Equal(t, v.Version(), mostRecentVersion)
 
 	require.NoError(t, v.Apply())
 	dat, err := ioutil.ReadFile(testFilePath)
 	require.NoError(t, err)
 
-	require.Equal(t, string(dat), "2020.08.05")
+	require.Equal(t, string(dat), mostRecentVersion)
 }
 
 func TestUpdateSpecificVersionService(t *testing.T) {
 	ts := createServer()
 	defer ts.Close()
 
-	testFilePath := "tmpfile"
 	createTestFile(t, testFilePath)
 	defer os.Remove(testFilePath)
 	reqVersion := "2020.9.1"
@@ -231,7 +305,7 @@ func TestUpdateSpecificVersionService(t *testing.T) {
 	s := NewWorkersService("2020.8.2", fmt.Sprintf("%s/updater", ts.URL), testFilePath, Options{RequestedVersion: reqVersion})
 	v, err := s.Check()
 	require.NoError(t, err)
-	require.Equal(t, reqVersion, v.String())
+	require.Equal(t, reqVersion, v.Version())
 
 	require.NoError(t, v.Apply())
 	dat, err := ioutil.ReadFile(testFilePath)
@@ -244,14 +318,13 @@ func TestCompressedUpdateService(t *testing.T) {
 	ts := createServer()
 	defer ts.Close()
 
-	testFilePath := "tmpfile"
 	createTestFile(t, testFilePath)
 	defer os.Remove(testFilePath)
 
 	s := NewWorkersService("2020.8.2", fmt.Sprintf("%s/compressed", ts.URL), testFilePath, Options{})
 	v, err := s.Check()
 	require.NoError(t, err)
-	require.Equal(t, "2020.09.02", v.String())
+	require.Equal(t, "2020.09.02", v.Version())
 
 	require.NoError(t, v.Apply())
 	dat, err := ioutil.ReadFile(testFilePath)
@@ -260,14 +333,16 @@ func TestCompressedUpdateService(t *testing.T) {
 	require.Equal(t, "2020.09.02", string(dat))
 }
 
-func TestVersionParsing(t *testing.T) {
-	require.False(t, IsNewerVersion("2020.8.2", "2020.8.2"))
-	require.True(t, IsNewerVersion("2020.8.2", "2020.8.3"))
-	require.True(t, IsNewerVersion("2020.8.2", "2021.1.2"))
-	require.True(t, IsNewerVersion("2020.8.2", "2020.9.1"))
-	require.True(t, IsNewerVersion("2020.8.2", "2020.12.45"))
-	require.False(t, IsNewerVersion("2020.8.2", "2020.6.3"))
-	require.False(t, IsNewerVersion("DEV", "2020.8.5"))
-	require.False(t, IsNewerVersion("2020.8.2", "asdlkfjasdf"))
-	require.True(t, IsNewerVersion("3.0.1", "4.2.1"))
+func TestUpdateWhenRunningKnownBuggyVersion(t *testing.T) {
+	ts := createServer()
+	defer ts.Close()
+
+	createTestFile(t, testFilePath)
+	defer os.Remove(testFilePath)
+
+	s := NewWorkersService(knownBuggyVersion, fmt.Sprintf("%s/updater", ts.URL), testFilePath, Options{})
+	v, err := s.Check()
+	require.NoError(t, err)
+	require.Equal(t, v.Version(), mostRecentVersion)
+	require.Equal(t, v.UserMessage(), expectedUserMsg)
 }

cmd/cloudflared/updater/workers_update.go

@@ -54,6 +54,7 @@ type WorkersVersion struct {
 	version      string
 	targetPath   string
 	isCompressed bool
+	userMessage  string
 }
 
 // NewWorkersVersion creates a new Version object. This is normally created by a WorkersService JSON checkin response
@@ -61,8 +62,17 @@ type WorkersVersion struct {
 // version is the version of this update
 // checksum is the expected checksum of the downloaded file
 // target path is where the file should be replace. Normally this the running cloudflared's path
-func NewWorkersVersion(url, version, checksum, targetPath string, isCompressed bool) Version {
-	return &WorkersVersion{downloadURL: url, version: version, checksum: checksum, targetPath: targetPath, isCompressed: isCompressed}
+// userMessage is a possible message to convey back to the user after having checked in with the Updater Service
+// isCompressed tells whether the asset to update cloudflared is compressed or not
+func NewWorkersVersion(url, version, checksum, targetPath, userMessage string, isCompressed bool) CheckResult {
+	return &WorkersVersion{
+		downloadURL:  url,
+		version:      version,
+		checksum:     checksum,
+		targetPath:   targetPath,
+		isCompressed: isCompressed,
+		userMessage:  userMessage,
+	}
 }
 
 // Apply does the actual verification and update logic.
@@ -114,10 +124,16 @@ func (v *WorkersVersion) Apply() error {
 }
 
 // String returns the version number of this update/release (e.g. 2020.08.05)
-func (v *WorkersVersion) String() string {
+func (v *WorkersVersion) Version() string {
 	return v.version
 }
 
+// String returns a possible message to convey back to user after having checked in with the Updater Service. E.g.
+// it can warn about the need to update the version currently running.
+func (v *WorkersVersion) UserMessage() string {
+	return v.userMessage
+}
+
 // download the file from the link in the json
 func download(url, filepath string, isCompressed bool) error {
 	client := &http.Client{

cmd/cloudflared/windows_service.go

@@ -12,13 +12,15 @@ import (
 	"time"
 	"unsafe"
 
-	"github.com/cloudflare/cloudflared/logger"
-
+	"github.com/pkg/errors"
 	"github.com/urfave/cli/v2"
 	"golang.org/x/sys/windows"
 	"golang.org/x/sys/windows/svc"
 	"golang.org/x/sys/windows/svc/eventlog"
 	"golang.org/x/sys/windows/svc/mgr"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
+	"github.com/cloudflare/cloudflared/logger"
 )
 
 const (
@@ -48,13 +50,13 @@ func runApp(app *cli.App, graceShutdownC chan struct{}) {
 			{
 				Name:   "install",
 				Usage:  "Install Argo Tunnel as a Windows service",
-				Action: installWindowsService,
-						},
+				Action: cliutil.ConfiguredAction(installWindowsService),
+			},
 			{
 				Name:   "uninstall",
 				Usage:  "Uninstall the Argo Tunnel service",
-				Action: uninstallWindowsService,
-						},
+				Action: cliutil.ConfiguredAction(uninstallWindowsService),
+			},
 		},
 	})
 
@@ -177,35 +179,30 @@ func installWindowsService(c *cli.Context) error {
 	zeroLogger.Info().Msg("Installing Argo Tunnel Windows service")
 	exepath, err := os.Executable()
 	if err != nil {
-		zeroLogger.Err(err).Msg("Cannot find path name that start the process")
-		return err
+		return errors.Wrap(err, "Cannot find path name that start the process")
 	}
 	m, err := mgr.Connect()
 	if err != nil {
-		zeroLogger.Err(err).Msg("Cannot establish a connection to the service control manager")
-		return err
+		return errors.Wrap(err, "Cannot establish a connection to the service control manager")
 	}
 	defer m.Disconnect()
 	s, err := m.OpenService(windowsServiceName)
 	log := zeroLogger.With().Str(LogFieldWindowsServiceName, windowsServiceName).Logger()
 	if err == nil {
 		s.Close()
-		log.Err(err).Msg("service already exists")
-		return fmt.Errorf("service %s already exists", windowsServiceName)
+		return fmt.Errorf("Service %s already exists", windowsServiceName)
 	}
 	config := mgr.Config{StartType: mgr.StartAutomatic, DisplayName: windowsServiceDescription}
 	s, err = m.CreateService(windowsServiceName, exepath, config)
 	if err != nil {
-		log.Error().Msg("Cannot install service")
-		return err
+		return errors.Wrap(err, "Cannot install service")
 	}
 	defer s.Close()
 	log.Info().Msg("Argo Tunnel agent service is installed")
 	err = eventlog.InstallAsEventCreate(windowsServiceName, eventlog.Error|eventlog.Warning|eventlog.Info)
 	if err != nil {
 		s.Delete()
-		log.Err(err).Msg("Cannot install event logger")
-		return fmt.Errorf("SetupEventLogSource() failed: %s", err)
+		return errors.Wrap(err, "Cannot install event logger")
 	}
 	err = configRecoveryOption(s.Handle)
 	if err != nil {
@@ -223,26 +220,22 @@ func uninstallWindowsService(c *cli.Context) error {
 	log.Info().Msg("Uninstalling Argo Tunnel Windows Service")
 	m, err := mgr.Connect()
 	if err != nil {
-		log.Error().Msg("Cannot establish a connection to the service control manager")
-		return err
+		return errors.Wrap(err, "Cannot establish a connection to the service control manager")
 	}
 	defer m.Disconnect()
 	s, err := m.OpenService(windowsServiceName)
 	if err != nil {
-		log.Error().Msg("service is not installed")
-		return fmt.Errorf("service %s is not installed", windowsServiceName)
+		return fmt.Errorf("Service %s is not installed", windowsServiceName)
 	}
 	defer s.Close()
 	err = s.Delete()
 	if err != nil {
-		log.Error().Msg("Cannot delete service")
-		return err
+		return errors.Wrap(err, "Cannot delete service")
 	}
 	log.Info().Msg("Argo Tunnel agent service is uninstalled")
 	err = eventlog.Remove(windowsServiceName)
 	if err != nil {
-		log.Error().Msg("Cannot remove event logger")
-		return fmt.Errorf("RemoveEventLogSource() failed: %s", err)
+		return errors.Wrap(err, "Cannot remove event logger")
 	}
 	return nil
 }

component-tests/.gitignore

@@ -0,0 +1,2 @@
+__pycache__
+.pytest_cache

component-tests/README.md

@@ -0,0 +1,52 @@
+# Requirements
+1. Python 3.7 or later with packages in the given `requirements.txt`
+   - E.g. with conda:
+   - `conda create -n component-tests python=3.7`  
+   - `conda activate component-tests`
+   - `pip3 install -r requirements.txt`
+
+2. Create a config yaml file, for example:
+```
+cloudflared_binary: "cloudflared"
+tunnel: "3d539f97-cd3a-4d8e-c33b-65e9099c7a8d"
+credentials_file: "/Users/tunnel/.cloudflared/3d539f97-cd3a-4d8e-c33b-65e9099c7a8d.json"
+classic_hostname: "classic-tunnel-component-tests.example.com"
+origincert: "/Users/tunnel/.cloudflared/cert.pem"
+ingress:
+- hostname: named-tunnel-component-tests.example.com
+  service: http_status:200
+- service: http_status:404
+```
+
+3. Route hostname to the tunnel. For the example config above, we can do that via
+```
+   cloudflared tunnel route dns 3d539f97-cd3a-4d8e-c33b-65e9099c7a8d named-tunnel-component-tests.example.com
+```
+
+4. Turn on linter
+If you are using Visual Studio, follow https://code.visualstudio.com/docs/python/linting to turn on linter.
+
+5. Turn on formatter
+If you are using Visual Studio, follow https://code.visualstudio.com/docs/python/editing#_formatting
+to turn on formatter and https://marketplace.visualstudio.com/items?itemName=cbrevik.toggle-format-on-save
+to turn on format on save.
+
+6. If you have cloudflared running as a service on your machine, you can either stop the service or ignore the service tests
+via `--ignore test_service.py`
+
+# How to run
+Specify path to config file via env var `COMPONENT_TESTS_CONFIG`. This is required.
+## All tests
+Run `pytest` inside this(component-tests) folder
+
+## Specific files
+Run `pytest <file 1 name>.py <file 2 name>.py`
+
+## Specific tests
+Run `pytest file.py -k <test 1 name> -k <test 2 name>`
+
+## Live Logging
+Running with `-o log_cli=true` outputs logging to CLI as the tests are. By default the log level is WARN.
+`--log-cli-level` control logging level.
+For example, to log at info level, run `pytest -o log_cli=true --log-cli-level=INFO`.
+See https://docs.pytest.org/en/latest/logging.html#live-logs for more documentation on logging.

component-tests/config.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python
+import copy
+
+from dataclasses import dataclass, InitVar
+
+from constants import METRICS_PORT
+
+# frozen=True raises exception when assigning to fields. This emulates immutability
+
+
+@dataclass(frozen=True)
+class BaseConfig:
+    cloudflared_binary: str
+    no_autoupdate: bool = True
+    metrics: str = f'localhost:{METRICS_PORT}'
+
+    def merge_config(self, additional):
+        config = copy.copy(additional)
+        config['no-autoupdate'] = self.no_autoupdate
+        config['metrics'] = self.metrics
+        return config
+
+
+@dataclass(frozen=True)
+class NamedTunnelBaseConfig(BaseConfig):
+    # The attributes of the parent class are ordered before attributes in this class,
+    # so we have to use default values here and check if they are set in __post_init__
+    tunnel: str = None
+    credentials_file: str = None
+    ingress: list = None
+
+    def __post_init__(self):
+        if self.tunnel is None:
+            raise TypeError("Field tunnel is not set")
+        if self.credentials_file is None:
+            raise TypeError("Field credentials_file is not set")
+        if self.ingress is None:
+            raise TypeError("Field ingress is not set")
+
+    def merge_config(self, additional):
+        config = super(NamedTunnelBaseConfig, self).merge_config(additional)
+        config['tunnel'] = self.tunnel
+        config['credentials-file'] = self.credentials_file
+        # In some cases we want to override default ingress, such as in config tests
+        if 'ingress' not in config:
+            config['ingress'] = self.ingress
+        return config
+
+
+@dataclass(frozen=True)
+class NamedTunnelConfig(NamedTunnelBaseConfig):
+    full_config: dict = None
+    additional_config: InitVar[dict] = {}
+
+    def __post_init__(self, additional_config):
+        # Cannot call set self.full_config because the class is frozen, instead, we can use __setattr__
+        # https://docs.python.org/3/library/dataclasses.html#frozen-instances
+        object.__setattr__(self, 'full_config',
+                           self.merge_config(additional_config))
+
+    def get_url(self):
+        return "https://" + self.ingress[0]['hostname']
+
+
+@dataclass(frozen=True)
+class ClassicTunnelBaseConfig(BaseConfig):
+    hostname: str = None
+    origincert: str = None
+
+    def __post_init__(self):
+        if self.hostname is None:
+            raise TypeError("Field tunnel is not set")
+        if self.origincert is None:
+            raise TypeError("Field credentials_file is not set")
+
+    def merge_config(self, additional):
+        config = super(ClassicTunnelBaseConfig, self).merge_config(additional)
+        config['hostname'] = self.hostname
+        config['origincert'] = self.origincert
+        return config
+
+
+@dataclass(frozen=True)
+class ClassicTunnelConfig(ClassicTunnelBaseConfig):
+    full_config: dict = None
+    additional_config: InitVar[dict] = {}
+
+    def __post_init__(self, additional_config):
+        # Cannot call set self.full_config because the class is frozen, instead, we can use __setattr__
+        # https://docs.python.org/3/library/dataclasses.html#frozen-instances
+        object.__setattr__(self, 'full_config',
+                           self.merge_config(additional_config))
+
+    def get_url(self):
+        return "https://" + self.hostname

component-tests/conftest.py

@@ -0,0 +1,45 @@
+import os
+import pytest
+import yaml
+
+from time import sleep
+
+from config import NamedTunnelConfig, ClassicTunnelConfig
+from constants import BACKOFF_SECS
+from util import LOGGER
+
+
+@pytest.fixture(scope="session")
+def component_tests_config():
+    config_file = os.getenv("COMPONENT_TESTS_CONFIG")
+    if config_file is None:
+        raise Exception(
+            "Need to provide path to config file in COMPONENT_TESTS_CONFIG")
+    with open(config_file, 'r') as stream:
+        config = yaml.safe_load(stream)
+        LOGGER.info(f"component tests base config {config}")
+
+        def _component_tests_config(additional_config={}, named_tunnel=True):
+
+            # Regression test for TUN-4177, running with proxy-dns should not prevent tunnels from running
+            additional_config["proxy-dns"] = True
+            additional_config["proxy-dns-port"] = 9053
+
+            if named_tunnel:
+                return NamedTunnelConfig(additional_config=additional_config,
+                                         cloudflared_binary=config['cloudflared_binary'],
+                                         tunnel=config['tunnel'],
+                                         credentials_file=config['credentials_file'],
+                                         ingress=config['ingress'])
+
+            return ClassicTunnelConfig(
+                additional_config=additional_config, cloudflared_binary=config['cloudflared_binary'],
+                hostname=config['classic_hostname'], origincert=config['origincert'])
+
+        return _component_tests_config
+
+
+# This fixture is automatically called before each tests to make sure the previous cloudflared has been shutdown
+@pytest.fixture(autouse=True)
+def wait_previous_cloudflared():
+    sleep(BACKOFF_SECS)

component-tests/constants.py

@@ -0,0 +1,3 @@
+METRICS_PORT = 51000
+MAX_RETRIES = 5
+BACKOFF_SECS = 7

component-tests/requirements.txt

@@ -0,0 +1,6 @@
+cloudflare==2.8.15
+flaky==3.7.0
+pytest==6.2.2
+pyyaml==5.4.1
+requests==2.25.1
+retrying==1.3.3

component-tests/setup.py

@@ -0,0 +1,165 @@
+#!/usr/bin/env python
+import argparse
+import base64
+import json
+import os
+import subprocess
+import uuid
+
+import CloudFlare
+import yaml
+from retrying import retry
+
+from constants import MAX_RETRIES, BACKOFF_SECS
+from util import LOGGER
+
+
+def get_config_from_env():
+    config_content = base64.b64decode(get_env("COMPONENT_TESTS_CONFIG_CONTENT")).decode('utf-8')
+    return yaml.safe_load(config_content)
+
+
+def get_config_from_file():
+    config_path = get_env("COMPONENT_TESTS_CONFIG")
+    with open(config_path, 'r') as infile:
+        return yaml.safe_load(infile)
+
+
+def persist_config(config):
+    config_path = get_env("COMPONENT_TESTS_CONFIG")
+    with open(config_path, 'w') as outfile:
+        yaml.safe_dump(config, outfile)
+
+
+def persist_origin_cert(config):
+    origincert = get_env("COMPONENT_TESTS_ORIGINCERT")
+    path = config["origincert"]
+    with open(path, 'w') as outfile:
+        outfile.write(origincert)
+    return path
+
+
+@retry(stop_max_attempt_number=MAX_RETRIES, wait_fixed=BACKOFF_SECS * 1000)
+def create_tunnel(config, origincert_path, random_uuid):
+    # Delete any previous existing credentials file. If the agent keeps files around (that's the case in Windows) then
+    # cloudflared tunnel create will refuse to create the tunnel because it does not want to overwrite credentials
+    # files.
+    credentials_path = config["credentials_file"]
+    try:
+        os.remove(credentials_path)
+    except OSError:
+        pass
+
+    tunnel_name = "cfd_component_test-" + random_uuid
+    create_cmd = [config["cloudflared_binary"], "tunnel", "--origincert", origincert_path, "create",
+                  "--credentials-file", credentials_path, tunnel_name]
+    LOGGER.info(f"Creating tunnel with {create_cmd}")
+    subprocess.run(create_cmd, check=True)
+
+    list_cmd = [config["cloudflared_binary"], "tunnel", "--origincert", origincert_path, "list", "--name",
+                tunnel_name, "--output", "json"]
+    LOGGER.info(f"Listing tunnel with {list_cmd}")
+    cloudflared = subprocess.run(list_cmd, check=True, capture_output=True)
+    return json.loads(cloudflared.stdout)[0]["id"]
+
+
+@retry(stop_max_attempt_number=MAX_RETRIES, wait_fixed=BACKOFF_SECS * 1000)
+def delete_tunnel(config):
+    credentials_path = config["credentials_file"]
+    delete_cmd = [config["cloudflared_binary"], "tunnel", "--origincert", config["origincert"], "delete",
+                  "--credentials-file", credentials_path, "-f", config["tunnel"]]
+    LOGGER.info(f"Deleting tunnel with {delete_cmd}")
+    subprocess.run(delete_cmd, check=True)
+
+
+@retry(stop_max_attempt_number=MAX_RETRIES, wait_fixed=BACKOFF_SECS * 1000)
+def create_dns(config, hostname, type, content):
+    cf = CloudFlare.CloudFlare(debug=True, token=get_env("DNS_API_TOKEN"))
+    cf.zones.dns_records.post(
+        config["zone_tag"],
+        data={'name': hostname, 'type': type, 'content': content, 'proxied': True}
+    )
+
+
+def create_classic_dns(config, random_uuid):
+    classic_hostname = "classic-" + random_uuid + "." + config["zone_domain"]
+    create_dns(config, classic_hostname, "AAAA", "fd10:aec2:5dae::")
+    return classic_hostname
+
+
+def create_named_dns(config, random_uuid):
+    hostname = "named-" + random_uuid + "." + config["zone_domain"]
+    create_dns(config, hostname, "CNAME", config["tunnel"] + ".cfargotunnel.com")
+    return hostname
+
+
+@retry(stop_max_attempt_number=MAX_RETRIES, wait_fixed=BACKOFF_SECS * 1000)
+def delete_dns(config, hostname):
+    cf = CloudFlare.CloudFlare(debug=True, token=get_env("DNS_API_TOKEN"))
+    zone_tag = config["zone_tag"]
+    dns_records = cf.zones.dns_records.get(zone_tag, params={'name': hostname})
+    if len(dns_records) > 0:
+        cf.zones.dns_records.delete(zone_tag, dns_records[0]['id'])
+
+
+def write_file(content, path):
+    with open(path, 'w') as outfile:
+        outfile.write(content)
+
+
+def get_env(env_name):
+    val = os.getenv(env_name)
+    if val is None:
+        raise Exception(f"{env_name} is not set")
+    return val
+
+
+def create():
+    """
+    Creates the necessary resources for the components test to run.
+     - Creates a named tunnel with a random name.
+     - Creates a random CNAME DNS entry for that tunnel.
+     - Creates a random AAAA DNS entry for a classic tunnel.
+
+    Those created resources are added to the config (obtained from an environment variable).
+    The resulting configuration is persisted for the tests to use.
+    """
+    config = get_config_from_env()
+    origincert_path = persist_origin_cert(config)
+
+    random_uuid = str(uuid.uuid4())
+    config["tunnel"] = create_tunnel(config, origincert_path, random_uuid)
+    config["classic_hostname"] = create_classic_dns(config, random_uuid)
+    config["ingress"] = [
+        {
+            "hostname": create_named_dns(config, random_uuid),
+            "service": "hello_world"
+        },
+        {
+            "service": "http_status:404"
+        }
+    ]
+
+    persist_config(config)
+
+
+def cleanup():
+    """
+    Reads the persisted configuration that was created previously.
+    Deletes the resources that were created there.
+    """
+    config = get_config_from_file()
+    delete_tunnel(config)
+    delete_dns(config, config["classic_hostname"])
+    delete_dns(config, config["ingress"][0]["hostname"])
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='setup component tests')
+    parser.add_argument('--type', choices=['create', 'cleanup'], default='create')
+    args = parser.parse_args()
+
+    if args.type == 'create':
+        create()
+    else:
+        cleanup()

component-tests/test_config.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+from util import start_cloudflared
+
+
+class TestConfig:
+    # tmp_path is a fixture provides a temporary directory unique to the test invocation
+    def test_validate_ingress_rules(self, tmp_path, component_tests_config):
+        extra_config = {
+            'ingress': [
+                {
+                    "hostname": "example.com",
+                    "service": "https://localhost:8000",
+                    "originRequest": {
+                        "originServerName": "test.example.com",
+                        "caPool": "/etc/certs/ca.pem"
+                    },
+                },
+                {
+                    "hostname": "api.example.com",
+                    "path": "login",
+                    "service": "https://localhost:9000",
+                },
+                {
+                    "hostname": "wss.example.com",
+                    "service": "wss://localhost:8000",
+                },
+                {
+                    "hostname": "ssh.example.com",
+                    "service": "ssh://localhost:8000",
+                },
+                {"service": "http_status:404"}
+            ],
+        }
+        config = component_tests_config(extra_config)
+        validate_args = ["ingress", "validate"]
+        _ = start_cloudflared(tmp_path, config, validate_args)
+
+        self.match_rule(tmp_path, config,
+                        "http://example.com/index.html", 1)
+        self.match_rule(tmp_path, config,
+                        "https://example.com/index.html", 1)
+        self.match_rule(tmp_path, config,
+                        "https://api.example.com/login", 2)
+        self.match_rule(tmp_path, config,
+                        "https://wss.example.com", 3)
+        self.match_rule(tmp_path, config,
+                        "https://ssh.example.com", 4)
+        self.match_rule(tmp_path, config,
+                        "https://api.example.com", 5)
+
+    # This is used to check that the command tunnel ingress url <url> matches rule number <rule_num>. Note that rule number uses 1-based indexing
+
+    def match_rule(self, tmp_path, config, url, rule_num):
+        args = ["ingress", "rule", url]
+        match_rule = start_cloudflared(tmp_path, config, args)
+
+        assert f"Matched rule #{rule_num}" .encode() in match_rule.stdout

component-tests/test_logging.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python
+import json
+import os
+
+from util import start_cloudflared, wait_tunnel_ready, send_requests
+
+# Rolling logger rotate log files after 1 MB
+rotate_after_size = 1000 * 1000
+default_log_file = "cloudflared.log"
+expect_message = "Starting Hello"
+
+
+def assert_log_to_terminal(cloudflared):
+    stderr = cloudflared.stderr.read(1500)
+    assert expect_message.encode() in stderr, f"{stderr} doesn't contain {expect_message}"
+
+
+def assert_log_in_file(file):
+    with open(file, "r") as f:
+        log = f.read(1850)
+        assert expect_message in log, f"{log} doesn't contain {expect_message}"
+
+
+def assert_json_log(file):
+    def assert_in_json(j, key):
+        assert key in j, f"{key} is not in j"
+
+    with open(file, "r") as f:
+        line = f.readline()
+        json_log = json.loads(line)
+        assert_in_json(json_log, "level")
+        assert_in_json(json_log, "time")
+        assert_in_json(json_log, "message")
+
+
+def assert_log_to_dir(config, log_dir):
+    max_batches = 3
+    batch_requests = 1000
+    for _ in range(max_batches):
+        send_requests(config.get_url(),
+                      batch_requests, require_ok=False)
+        files = os.listdir(log_dir)
+        if len(files) == 2:
+            current_log_file_index = files.index(default_log_file)
+            current_file = log_dir / files[current_log_file_index]
+            stats = os.stat(current_file)
+            assert stats.st_size > 0
+            assert_json_log(current_file)
+
+            # One file is the current log file, the other is the rotated log file
+            rotated_log_file_index = 0 if current_log_file_index == 1 else 1
+            rotated_file = log_dir / files[rotated_log_file_index]
+            stats = os.stat(rotated_file)
+            assert stats.st_size > rotate_after_size
+            assert_log_in_file(rotated_file)
+            assert_json_log(current_file)
+            return
+
+    raise Exception(
+        f"Log file isn't rotated after sending {max_batches * batch_requests} requests")
+
+
+class TestLogging:
+    def test_logging_to_terminal(self, tmp_path, component_tests_config):
+        config = component_tests_config()
+        with start_cloudflared(tmp_path, config, new_process=True) as cloudflared:
+            wait_tunnel_ready(tunnel_url=config.get_url())
+            assert_log_to_terminal(cloudflared)
+
+    def test_logging_to_file(self, tmp_path, component_tests_config):
+        log_file = tmp_path / default_log_file
+        extra_config = {
+            # Convert from pathlib.Path to str
+            "logfile": str(log_file),
+        }
+        config = component_tests_config(extra_config)
+        with start_cloudflared(tmp_path, config, new_process=True, capture_output=False):
+            wait_tunnel_ready(tunnel_url=config.get_url())
+            assert_log_in_file(log_file)
+            assert_json_log(log_file)
+
+    def test_logging_to_dir(self, tmp_path, component_tests_config):
+        log_dir = tmp_path / "logs"
+        extra_config = {
+            "loglevel": "debug",
+            # Convert from pathlib.Path to str
+            "log-directory": str(log_dir),
+        }
+        config = component_tests_config(extra_config)
+        with start_cloudflared(tmp_path, config, new_process=True, capture_output=False):
+            wait_tunnel_ready(tunnel_url=config.get_url())
+            assert_log_to_dir(config, log_dir)

component-tests/test_reconnect.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+import copy
+import platform
+from time import sleep
+
+import pytest
+from flaky import flaky
+
+from util import start_cloudflared, wait_tunnel_ready, check_tunnel_not_connected
+
+
+@flaky(max_runs=3, min_passes=1)
+class TestReconnect:
+    default_ha_conns = 4
+    default_reconnect_secs = 15
+    extra_config = {
+        "stdin-control": True,
+    }
+
+    @pytest.mark.skipif(platform.system() == "Windows", reason=f"Currently buggy on Windows TUN-4584")
+    def test_named_reconnect(self, tmp_path, component_tests_config):
+        config = component_tests_config(self.extra_config)
+        with start_cloudflared(tmp_path, config, new_process=True, allow_input=True, capture_output=False) as cloudflared:
+            # Repeat the test multiple times because some issues only occur after multiple reconnects
+            self.assert_reconnect(config, cloudflared, 5)
+
+    @pytest.mark.skipif(platform.system() == "Windows", reason=f"Currently buggy on Windows TUN-4699")
+    def test_classic_reconnect(self, tmp_path, component_tests_config):
+        extra_config = copy.copy(self.extra_config)
+        extra_config["hello-world"] = True
+        config = component_tests_config(
+            additional_config=extra_config, named_tunnel=False)
+        with start_cloudflared(tmp_path, config, cfd_args=[], new_process=True, allow_input=True, capture_output=True) as cloudflared:
+            self.assert_reconnect(config, cloudflared, 1)
+
+    def send_reconnect(self, cloudflared, secs):
+        # Although it is recommended to use the Popen.communicate method, we cannot
+        # use it because it blocks on reading stdout and stderr until EOF is reached
+        cloudflared.stdin.write(f"reconnect {secs}s\n".encode())
+        cloudflared.stdin.flush()
+
+    def assert_reconnect(self, config, cloudflared, repeat):
+        wait_tunnel_ready(tunnel_url=config.get_url(), require_min_connections=self.default_ha_conns)
+        for _ in range(repeat):
+            for i in range(self.default_ha_conns):
+                self.send_reconnect(cloudflared, self.default_reconnect_secs)
+                expect_connections = self.default_ha_conns-i-1
+                if expect_connections > 0:
+                    # Don't check if tunnel returns 200 here because there is a race condition between wait_tunnel_ready
+                    # retrying to get 200 response and reconnecting
+                    wait_tunnel_ready(require_min_connections=expect_connections)
+                else:
+                    check_tunnel_not_connected()
+
+            sleep(self.default_reconnect_secs * 2)
+            wait_tunnel_ready(tunnel_url=config.get_url(), require_min_connections=self.default_ha_conns)

component-tests/test_service.py

@@ -0,0 +1,139 @@
+#!/usr/bin/env python
+import os
+import platform
+import subprocess
+from contextlib import contextmanager
+from pathlib import Path
+
+import pytest
+
+import test_logging
+from util import start_cloudflared, wait_tunnel_ready
+
+
+def select_platform(plat):
+    return pytest.mark.skipif(
+        platform.system() != plat, reason=f"Only runs on {plat}")
+
+
+def default_config_dir():
+    return os.path.join(Path.home(), ".cloudflared")
+
+
+def default_config_file():
+    return os.path.join(default_config_dir(), "config.yml")
+
+
+class TestServiceMode:
+    @select_platform("Darwin")
+    @pytest.mark.skipif(os.path.exists(default_config_file()), reason=f"There is already a config file in default path")
+    def test_launchd_service_log_to_file(self, tmp_path, component_tests_config):
+        log_file = tmp_path / test_logging.default_log_file
+        additional_config = {
+            # On Darwin cloudflared service defaults to run classic tunnel command
+            "hello-world": True,
+            "logfile": str(log_file),
+        }
+        config = component_tests_config(additional_config=additional_config, named_tunnel=False)
+
+        def assert_log_file():
+            test_logging.assert_log_in_file(log_file)
+            test_logging.assert_json_log(log_file)
+
+        self.launchd_service_scenario(config, assert_log_file)
+
+    @select_platform("Darwin")
+    @pytest.mark.skipif(os.path.exists(default_config_file()), reason=f"There is already a config file in default path")
+    def test_launchd_service_rotating_log(self, tmp_path, component_tests_config):
+        log_dir = tmp_path / "logs"
+        additional_config = {
+            # On Darwin cloudflared service defaults to run classic tunnel command
+            "hello-world": True,
+            "loglevel": "debug",
+            "log-directory": str(log_dir),
+        }
+        config = component_tests_config(additional_config=additional_config, named_tunnel=False)
+
+        def assert_rotating_log():
+            test_logging.assert_log_to_dir(config, log_dir)
+
+        self.launchd_service_scenario(config, assert_rotating_log)
+
+    def launchd_service_scenario(self, config, extra_assertions):
+        with self.run_service(Path(default_config_dir()), config):
+            self.launchctl_cmd("list")
+            self.launchctl_cmd("start")
+            wait_tunnel_ready(tunnel_url=config.get_url())
+            extra_assertions()
+            self.launchctl_cmd("stop")
+
+        os.remove(default_config_file())
+        self.launchctl_cmd("list", success=False)
+
+    @select_platform("Linux")
+    @pytest.mark.skipif(os.path.exists("/etc/cloudflared/config.yml"),
+                        reason=f"There is already a config file in default path")
+    def test_sysv_service_log_to_file(self, tmp_path, component_tests_config):
+        log_file = tmp_path / test_logging.default_log_file
+        additional_config = {
+            "logfile": str(log_file),
+        }
+        config = component_tests_config(additional_config=additional_config)
+
+        def assert_log_file():
+            test_logging.assert_log_in_file(log_file)
+            test_logging.assert_json_log(log_file)
+
+        self.sysv_service_scenario(config, tmp_path, assert_log_file)
+
+    @select_platform("Linux")
+    @pytest.mark.skipif(os.path.exists("/etc/cloudflared/config.yml"),
+                        reason=f"There is already a config file in default path")
+    def test_sysv_service_rotating_log(self, tmp_path, component_tests_config):
+        log_dir = tmp_path / "logs"
+        additional_config = {
+            "loglevel": "debug",
+            "log-directory": str(log_dir),
+        }
+        config = component_tests_config(additional_config=additional_config)
+
+        def assert_rotating_log():
+            # We need the folder to have executable permissions for the "stat" command in the assertions to work.
+            subprocess.check_call(['sudo', 'chmod', 'o+x', log_dir])
+            test_logging.assert_log_to_dir(config, log_dir)
+
+        self.sysv_service_scenario(config, tmp_path, assert_rotating_log)
+
+    def sysv_service_scenario(self, config, tmp_path, extra_assertions):
+        with self.run_service(tmp_path, config, root=True):
+            self.sysv_cmd("start")
+            self.sysv_cmd("status")
+            wait_tunnel_ready(tunnel_url=config.get_url())
+            extra_assertions()
+            self.sysv_cmd("stop")
+
+        # Service install copies config file to /etc/cloudflared/config.yml
+        subprocess.run(["sudo", "rm", "/etc/cloudflared/config.yml"])
+        self.sysv_cmd("status", success=False)
+
+    @contextmanager
+    def run_service(self, tmp_path, config, root=False):
+        try:
+            service = start_cloudflared(
+                tmp_path, config, cfd_args=["service", "install"], cfd_pre_args=[], capture_output=False, root=root)
+            yield service
+        finally:
+            start_cloudflared(
+                tmp_path, config, cfd_args=["service", "uninstall"], cfd_pre_args=[], capture_output=False, root=root)
+
+    def launchctl_cmd(self, action, success=True):
+        cmd = subprocess.run(
+            ["launchctl", action, "com.cloudflare.cloudflared"], check=success)
+        if not success:
+            assert cmd.returncode != 0, f"Expect {cmd.args} to fail, but it succeed"
+
+    def sysv_cmd(self, action, success=True):
+        cmd = subprocess.run(
+            ["sudo", "service", "cloudflared", action], check=success)
+        if not success:
+            assert cmd.returncode != 0, f"Expect {cmd.args} to fail, but it succeed"

component-tests/test_termination.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python
+from contextlib import contextmanager
+import platform
+import signal
+import threading
+import time
+
+import pytest
+import requests
+
+from util import start_cloudflared, wait_tunnel_ready, check_tunnel_not_connected
+
+
+def supported_signals():
+    if platform.system() == "Windows":
+        return [signal.SIGTERM]
+    return [signal.SIGTERM, signal.SIGINT]
+
+
+class TestTermination():
+    grace_period = 5
+    timeout = 10
+    extra_config = {
+        "grace-period": f"{grace_period}s",
+    }
+    sse_endpoint = "/sse?freq=1s"
+
+    @pytest.mark.parametrize("signal", supported_signals())
+    def test_graceful_shutdown(self, tmp_path, component_tests_config, signal):
+        config = component_tests_config(self.extra_config)
+        with start_cloudflared(
+                tmp_path, config, new_process=True, capture_output=False) as cloudflared:
+            wait_tunnel_ready(tunnel_url=config.get_url())
+
+            connected = threading.Condition()
+            in_flight_req = threading.Thread(
+                target=self.stream_request, args=(config, connected, False, ))
+            in_flight_req.start()
+
+            with connected:
+                connected.wait(self.timeout)
+            # Send signal after the SSE connection is established
+            self.terminate_by_signal(cloudflared, signal)
+            self.wait_eyeball_thread(
+                in_flight_req, self.grace_period + self.timeout)
+
+    # test cloudflared terminates before grace period expires when all eyeball
+    # connections are drained
+    @pytest.mark.parametrize("signal", supported_signals())
+    def test_shutdown_once_no_connection(self, tmp_path, component_tests_config, signal):
+        config = component_tests_config(self.extra_config)
+        with start_cloudflared(
+                tmp_path, config, new_process=True, capture_output=False) as cloudflared:
+            wait_tunnel_ready(tunnel_url=config.get_url())
+
+            connected = threading.Condition()
+            in_flight_req = threading.Thread(
+                target=self.stream_request, args=(config, connected, True, ))
+            in_flight_req.start()
+
+            with connected:
+                connected.wait(self.timeout)
+            with self.within_grace_period():
+                # Send signal after the SSE connection is established
+                self.terminate_by_signal(cloudflared, signal)
+                self.wait_eyeball_thread(in_flight_req, self.grace_period)
+
+    @pytest.mark.parametrize("signal", supported_signals())
+    def test_no_connection_shutdown(self, tmp_path, component_tests_config, signal):
+        config = component_tests_config(self.extra_config)
+        with start_cloudflared(
+                tmp_path, config, new_process=True, capture_output=False) as cloudflared:
+            wait_tunnel_ready(tunnel_url=config.get_url())
+            with self.within_grace_period():
+                self.terminate_by_signal(cloudflared, signal)
+
+    def terminate_by_signal(self, cloudflared, sig):
+        cloudflared.send_signal(sig)
+        check_tunnel_not_connected()
+        cloudflared.wait()
+
+    def wait_eyeball_thread(self, thread, timeout):
+        thread.join(timeout)
+        assert thread.is_alive() == False, "eyeball thread is still alive"
+
+    # Using this context asserts logic within the context is executed within grace period
+    @contextmanager
+    def within_grace_period(self):
+        try:
+            start = time.time()
+            yield
+        finally:
+            duration = time.time() - start
+            assert duration < self.grace_period
+
+    def stream_request(self, config, connected, early_terminate):
+        expected_terminate_message = "502 Bad Gateway"
+        url = config.get_url() + self.sse_endpoint
+
+        with requests.get(url, timeout=5, stream=True) as resp:
+            with connected:
+                connected.notifyAll()
+            lines = 0
+            for line in resp.iter_lines():
+                if expected_terminate_message.encode() == line:
+                    break
+                lines += 1
+                if early_terminate and lines == 2:
+                    return
+            # /sse returns count followed by 2 new lines
+            assert lines >= (self.grace_period * 2)

component-tests/util.py

@@ -0,0 +1,99 @@
+from contextlib import contextmanager
+import logging
+import requests
+from retrying import retry
+import subprocess
+import yaml
+
+from time import sleep
+
+from constants import METRICS_PORT, MAX_RETRIES, BACKOFF_SECS
+
+LOGGER = logging.getLogger(__name__)
+
+
+def write_config(directory, config):
+    config_path = directory / "config.yml"
+    with open(config_path, 'w') as outfile:
+        yaml.dump(config, outfile)
+    return config_path
+
+
+def start_cloudflared(directory, config, cfd_args=["run"], cfd_pre_args=["tunnel"], new_process=False, allow_input=False, capture_output=True, root=False):
+    config_path = write_config(directory, config.full_config)
+    cmd = cloudflared_cmd(config, config_path, cfd_args, cfd_pre_args, root)
+    if new_process:
+        return run_cloudflared_background(cmd, allow_input, capture_output)
+    # By setting check=True, it will raise an exception if the process exits with non-zero exit code
+    return subprocess.run(cmd, check=True, capture_output=capture_output)
+
+
+def cloudflared_cmd(config, config_path, cfd_args, cfd_pre_args, root):
+    cmd = []
+    if root:
+        cmd += ["sudo"]
+    cmd += [config.cloudflared_binary]
+    cmd += cfd_pre_args
+    cmd += ["--config", str(config_path)]
+    cmd += cfd_args
+    LOGGER.info(f"Run cmd {cmd} with config {config}")
+    return cmd
+
+
+@contextmanager
+def run_cloudflared_background(cmd, allow_input, capture_output):
+    output = subprocess.PIPE if capture_output else subprocess.DEVNULL
+    stdin = subprocess.PIPE if allow_input else None
+    try:
+        cfd = subprocess.Popen(cmd, stdin=stdin, stdout=output, stderr=output)
+        yield cfd
+    finally:
+        cfd.terminate()
+        if capture_output:
+            LOGGER.info(f"cloudflared log: {cfd.stderr.read()}")
+
+
+@retry(stop_max_attempt_number=MAX_RETRIES, wait_fixed=BACKOFF_SECS * 1000)
+def wait_tunnel_ready(tunnel_url=None, require_min_connections=1):
+    metrics_url = f'http://localhost:{METRICS_PORT}/ready'
+
+    with requests.Session() as s:
+        resp = send_request(s, metrics_url, True)
+        assert resp.json()[
+            "readyConnections"] >= require_min_connections, f"Ready endpoint returned {resp.json()} but we expect at least {require_min_connections} connections"
+        if tunnel_url is not None:
+            send_request(s, tunnel_url, True)
+
+
+@retry(stop_max_attempt_number=MAX_RETRIES * BACKOFF_SECS, wait_fixed=1000)
+def check_tunnel_not_connected():
+    url = f'http://localhost:{METRICS_PORT}/ready'
+
+    try:
+        resp = requests.get(url, timeout=1)
+        assert resp.status_code == 503, f"Expect {url} returns 503, got {resp.status_code}"
+    # cloudflared might already terminate
+    except requests.exceptions.ConnectionError as e:
+        LOGGER.warning(f"Failed to connect to {url}, error: {e}")
+
+
+# In some cases we don't need to check response status, such as when sending batch requests to generate logs
+def send_requests(url, count, require_ok=True):
+    errors = 0
+    with requests.Session() as s:
+        for _ in range(count):
+            resp = send_request(s, url, require_ok)
+            if resp is None:
+                errors += 1
+            sleep(0.01)
+    if errors > 0:
+        LOGGER.warning(
+            f"{errors} out of {count} requests to {url} return non-200 status")
+
+
+@retry(stop_max_attempt_number=MAX_RETRIES, wait_fixed=BACKOFF_SECS * 1000)
+def send_request(session, url, require_ok):
+    resp = session.get(url, timeout=BACKOFF_SECS)
+    if require_ok:
+        assert resp.status_code == 200, f"{url} returned {resp}"
+    return resp if resp.status_code == 200 else None

config/configuration.go

@@ -9,13 +9,13 @@ import (
 	"runtime"
 	"time"
 
-	"github.com/mitchellh/go-homedir"
+	homedir "github.com/mitchellh/go-homedir"
 	"github.com/pkg/errors"
+	"github.com/rs/zerolog"
 	"github.com/urfave/cli/v2"
-	"gopkg.in/yaml.v2"
+	yaml "gopkg.in/yaml.v2"
 
 	"github.com/cloudflare/cloudflared/validation"
-	"github.com/rs/zerolog"
 )
 
 var (
@@ -221,15 +221,28 @@ type OriginRequestConfig struct {
 	ProxyPort *uint `yaml:"proxyPort"`
 	// Valid options are 'socks' or empty.
 	ProxyType *string `yaml:"proxyType"`
+	// IP rules for the proxy service
+	IPRules []IngressIPRule `yaml:"ipRules"`
+}
+
+type IngressIPRule struct {
+	Prefix *string `yaml:"prefix"`
+	Ports  []int   `yaml:"ports"`
+	Allow  bool    `yaml:"allow"`
 }
 
 type Configuration struct {
 	TunnelID      string `yaml:"tunnel"`
 	Ingress       []UnvalidatedIngressRule
+	WarpRouting   WarpRoutingConfig   `yaml:"warp-routing"`
 	OriginRequest OriginRequestConfig `yaml:"originRequest"`
 	sourceFile    string
 }
 
+type WarpRoutingConfig struct {
+	Enabled bool `yaml:"enabled"`
+}
+
 type configFileSettings struct {
 	Configuration `yaml:",inline"`
 	// older settings will be aggregated into the generic map, should be read via cli.Context
@@ -345,13 +358,13 @@ func GetConfiguration() *Configuration {
 // ReadConfigFile returns InputSourceContext initialized from the configuration file.
 // On repeat calls returns with the same file, returns without reading the file again; however,
 // if value of "config" flag changes, will read the new config file
-func ReadConfigFile(c *cli.Context, log *zerolog.Logger) (*configFileSettings, error) {
+func ReadConfigFile(c *cli.Context, log *zerolog.Logger) (settings *configFileSettings, warnings string, err error) {
 	configFile := c.String("config")
 	if configuration.Source() == configFile || configFile == "" {
 		if configuration.Source() == "" {
-			return nil, ErrNoConfigFile
+			return nil, "", ErrNoConfigFile
 		}
-		return &configuration, nil
+		return &configuration, "", nil
 	}
 
 	log.Debug().Msgf("Loading configuration from %s", configFile)
@@ -360,16 +373,27 @@ func ReadConfigFile(c *cli.Context, log *zerolog.Logger) (*configFileSettings, e
 		if os.IsNotExist(err) {
 			err = ErrNoConfigFile
 		}
-		return nil, err
+		return nil, "", err
 	}
 	defer file.Close()
 	if err := yaml.NewDecoder(file).Decode(&configuration); err != nil {
 		if err == io.EOF {
 			log.Error().Msgf("Configuration file %s was empty", configFile)
-			return &configuration, nil
+			return &configuration, "", nil
 		}
-		return nil, errors.Wrap(err, "error parsing YAML in config file at "+configFile)
+		return nil, "", errors.Wrap(err, "error parsing YAML in config file at "+configFile)
 	}
 	configuration.sourceFile = configFile
-	return &configuration, nil
+
+	// Parse it again, with strict mode, to find warnings.
+	if file, err := os.Open(configFile); err == nil {
+		decoder := yaml.NewDecoder(file)
+		decoder.SetStrict(true)
+		var unusedConfig configFileSettings
+		if err := decoder.Decode(&unusedConfig); err != nil {
+			warnings = err.Error()
+		}
+	}
+
+	return &configuration, warnings, nil
 }

config/configuration_test.go

@@ -5,7 +5,7 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/assert"
-	"gopkg.in/yaml.v2"
+	yaml "gopkg.in/yaml.v2"
 )
 
 func TestConfigFileSettings(t *testing.T) {
@@ -20,6 +20,9 @@ func TestConfigFileSettings(t *testing.T) {
 			Path:     "",
 			Service:  "https://localhost:8001",
 		}
+		warpRouting = WarpRoutingConfig{
+			Enabled: true,
+		}
 	)
 	rawYAML := `
 tunnel: config-file-test
@@ -29,6 +32,8 @@ ingress:
    service: https://localhost:8000
  - hostname: "*"
    service: https://localhost:8001
+warp-routing: 
+  enabled: true
 retries: 5
 grace-period: 30s
 percentage: 3.14
@@ -47,6 +52,7 @@ counters:
 	assert.Equal(t, "config-file-test", config.TunnelID)
 	assert.Equal(t, firstIngress, config.Ingress[0])
 	assert.Equal(t, secondIngress, config.Ingress[1])
+	assert.Equal(t, warpRouting, config.WarpRouting)
 
 	retries, err := config.Int("retries")
 	assert.NoError(t, err)
@@ -73,4 +79,5 @@ counters:
 	assert.NoError(t, err)
 	assert.Equal(t, 123, counters[0])
 	assert.Equal(t, 456, counters[1])
+
 }

config/manager.go

@@ -4,11 +4,11 @@ import (
 	"io"
 	"os"
 
-	"github.com/cloudflare/cloudflared/watcher"
-
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
-	"gopkg.in/yaml.v2"
+	yaml "gopkg.in/yaml.v2"
+
+	"github.com/cloudflare/cloudflared/watcher"
 )
 
 // Notifier sends out config updates

config/manager_test.go

@@ -4,10 +4,10 @@ import (
 	"os"
 	"testing"
 
-	"github.com/cloudflare/cloudflared/watcher"
-
 	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/assert"
+
+	"github.com/cloudflare/cloudflared/watcher"
 )
 
 type mockNotifier struct {

config/model.go

@@ -5,6 +5,8 @@ import (
 	"fmt"
 	"io"
 	"strings"
+
+	"github.com/cloudflare/cloudflared/tunneldns"
 )
 
 // Forwarder represents a client side listener to forward traffic to the edge
@@ -25,11 +27,12 @@ type Tunnel struct {
 
 // DNSResolver represents a client side DNS resolver
 type DNSResolver struct {
-	Enabled    bool     `json:"enabled"`
-	Address    string   `json:"address,omitempty"`
-	Port       uint16   `json:"port,omitempty"`
-	Upstreams  []string `json:"upstreams,omitempty"`
-	Bootstraps []string `json:"bootstraps,omitempty"`
+	Enabled                bool     `json:"enabled"`
+	Address                string   `json:"address,omitempty"`
+	Port                   uint16   `json:"port,omitempty"`
+	Upstreams              []string `json:"upstreams,omitempty"`
+	Bootstraps             []string `json:"bootstraps,omitempty"`
+	MaxUpstreamConnections int      `json:"max_upstream_connections,omitempty"`
 }
 
 // Root is the base options to configure the service
@@ -59,6 +62,7 @@ func (r *DNSResolver) Hash() string {
 	io.WriteString(h, strings.Join(r.Bootstraps, ","))
 	io.WriteString(h, strings.Join(r.Upstreams, ","))
 	io.WriteString(h, fmt.Sprintf("%d", r.Port))
+	io.WriteString(h, fmt.Sprintf("%d", r.MaxUpstreamConnections))
 	io.WriteString(h, fmt.Sprintf("%v", r.Enabled))
 	return fmt.Sprintf("%x", h.Sum(nil))
 }
@@ -99,3 +103,11 @@ func (r *DNSResolver) BootstrapsOrDefault() []string {
 	}
 	return []string{"https://162.159.36.1/dns-query", "https://162.159.46.1/dns-query", "https://[2606:4700:4700::1111]/dns-query", "https://[2606:4700:4700::1001]/dns-query"}
 }
+
+// MaxUpstreamConnectionsOrDefault return the max upstream connections or returns the default if negative
+func (r *DNSResolver) MaxUpstreamConnectionsOrDefault() int {
+	if r.MaxUpstreamConnections >= 0 {
+		return r.MaxUpstreamConnections
+	}
+	return tunneldns.MaxUpstreamConnsDefault
+}

connection/connection.go

@@ -1,27 +1,37 @@
 package connection
 
 import (
+	"context"
+	"fmt"
 	"io"
 	"net/http"
 	"strconv"
 	"strings"
 	"time"
 
-	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	"github.com/google/uuid"
+
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/cloudflare/cloudflared/websocket"
 )
 
-const LogFieldConnIndex = "connIndex"
+const (
+	lbProbeUserAgentPrefix = "Mozilla/5.0 (compatible; Cloudflare-Traffic-Manager/1.0; +https://www.cloudflare.com/traffic-manager/;"
+	LogFieldConnIndex      = "connIndex"
+)
+
+var switchingProtocolText = fmt.Sprintf("%d %s", http.StatusSwitchingProtocols, http.StatusText(http.StatusSwitchingProtocols))
 
 type Config struct {
-	OriginClient    OriginClient
+	OriginProxy     OriginProxy
 	GracePeriod     time.Duration
 	ReplaceExisting bool
 }
 
 type NamedTunnelConfig struct {
-	Credentials Credentials
-	Client      pogs.ClientInfo
+	Credentials    Credentials
+	Client         pogs.ClientInfo
+	QuickTunnelUrl string
 }
 
 // Credentials are stored in the credentials file and contain all info needed to run a tunnel.
@@ -46,18 +56,104 @@ type ClassicTunnelConfig struct {
 	UseReconnectToken bool
 }
 
-func (c *ClassicTunnelConfig) IsTrialZone() bool {
-	return c.Hostname == ""
+// Type indicates the connection type of the  connection.
+type Type int
+
+const (
+	TypeWebsocket Type = iota
+	TypeTCP
+	TypeControlStream
+	TypeHTTP
+)
+
+// ShouldFlush returns whether this kind of connection should actively flush data
+func (t Type) shouldFlush() bool {
+	switch t {
+	case TypeWebsocket, TypeTCP, TypeControlStream:
+		return true
+	default:
+		return false
+	}
 }
 
-type OriginClient interface {
-	Proxy(w ResponseWriter, req *http.Request, isWebsocket bool) error
+func (t Type) String() string {
+	switch t {
+	case TypeWebsocket:
+		return "websocket"
+	case TypeTCP:
+		return "tcp"
+	case TypeControlStream:
+		return "control stream"
+	case TypeHTTP:
+		return "http"
+	default:
+		return fmt.Sprintf("Unknown Type %d", t)
+	}
+}
+
+// OriginProxy is how data flows from cloudflared to the origin services running behind it.
+type OriginProxy interface {
+	ProxyHTTP(w ResponseWriter, req *http.Request, isWebsocket bool) error
+	ProxyTCP(ctx context.Context, rwa ReadWriteAcker, req *TCPRequest) error
+}
+
+// TCPRequest defines the input format needed to perform a TCP proxy.
+type TCPRequest struct {
+	Dest    string
+	CFRay   string
+	LBProbe bool
+}
+
+// ReadWriteAcker is a readwriter with the ability to Acknowledge to the downstream (edge) that the origin has
+// accepted the connection.
+type ReadWriteAcker interface {
+	io.ReadWriter
+	AckConnection() error
+}
+
+// HTTPResponseReadWriteAcker is an HTTP implementation of ReadWriteAcker.
+type HTTPResponseReadWriteAcker struct {
+	r   io.Reader
+	w   ResponseWriter
+	req *http.Request
+}
+
+// NewHTTPResponseReadWriterAcker returns a new instance of HTTPResponseReadWriteAcker.
+func NewHTTPResponseReadWriterAcker(w ResponseWriter, req *http.Request) *HTTPResponseReadWriteAcker {
+	return &HTTPResponseReadWriteAcker{
+		r:   req.Body,
+		w:   w,
+		req: req,
+	}
+}
+
+func (h *HTTPResponseReadWriteAcker) Read(p []byte) (int, error) {
+	return h.r.Read(p)
+}
+
+func (h *HTTPResponseReadWriteAcker) Write(p []byte) (int, error) {
+	return h.w.Write(p)
+}
+
+// AckConnection acks an HTTP connection by sending a switch protocols status code that enables the caller to
+// upgrade to streams.
+func (h *HTTPResponseReadWriteAcker) AckConnection() error {
+	resp := &http.Response{
+		Status:        switchingProtocolText,
+		StatusCode:    http.StatusSwitchingProtocols,
+		ContentLength: -1,
+	}
+
+	if secWebsocketKey := h.req.Header.Get("Sec-WebSocket-Key"); secWebsocketKey != "" {
+		resp.Header = websocket.NewResponseHeader(h.req)
+	}
+
+	return h.w.WriteRespHeaders(resp.StatusCode, resp.Header)
 }
 
 type ResponseWriter interface {
-	WriteRespHeaders(*http.Response) error
-	WriteErrorResponse()
-	io.ReadWriter
+	WriteRespHeaders(status int, header http.Header) error
+	io.Writer
 }
 
 type ConnectedFuse interface {
@@ -75,3 +171,11 @@ func IsServerSentEvent(headers http.Header) bool {
 func uint8ToString(input uint8) string {
 	return strconv.FormatUint(uint64(input), 10)
 }
+
+func FindCfRayHeader(req *http.Request) string {
+	return req.Header.Get("Cf-Ray")
+}
+
+func IsLBProbeRequest(req *http.Request) bool {
+	return strings.HasPrefix(req.UserAgent(), lbProbeUserAgentPrefix)
+}

connection/connection_test.go

@@ -1,6 +1,7 @@
 package connection
 
 import (
+	"context"
 	"fmt"
 	"io"
 	"net/http"
@@ -11,6 +12,8 @@ import (
 	"github.com/gobwas/ws/wsutil"
 	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/assert"
+
+	"github.com/cloudflare/cloudflared/ingress"
 )
 
 const (
@@ -18,9 +21,10 @@ const (
 )
 
 var (
-	testConfig = &Config{
-		OriginClient: &mockOriginClient{},
-		GracePeriod:  time.Millisecond * 100,
+	unusedWarpRoutingService = (*ingress.WarpRoutingService)(nil)
+	testConfig               = &Config{
+		OriginProxy: &mockOriginProxy{},
+		GracePeriod: time.Millisecond * 100,
 	}
 	log           = zerolog.Nop()
 	testOriginURL = &url.URL{
@@ -38,14 +42,17 @@ type testRequest struct {
 	isProxyError   bool
 }
 
-type mockOriginClient struct {
-}
+type mockOriginProxy struct{}
 
-func (moc *mockOriginClient) Proxy(w ResponseWriter, r *http.Request, isWebsocket bool) error {
+func (moc *mockOriginProxy) ProxyHTTP(
+	w ResponseWriter,
+	req *http.Request,
+	isWebsocket bool,
+) error {
 	if isWebsocket {
-		return wsEndpoint(w, r)
+		return wsEndpoint(w, req)
 	}
-	switch r.URL.Path {
+	switch req.URL.Path {
 	case "/ok":
 		originRespEndpoint(w, http.StatusOK, []byte(http.StatusText(http.StatusOK)))
 	case "/large_file":
@@ -60,6 +67,15 @@ func (moc *mockOriginClient) Proxy(w ResponseWriter, r *http.Request, isWebsocke
 		originRespEndpoint(w, http.StatusNotFound, []byte("page not found"))
 	}
 	return nil
+
+}
+
+func (moc *mockOriginProxy) ProxyTCP(
+	ctx context.Context,
+	rwa ReadWriteAcker,
+	r *TCPRequest,
+) error {
+	return nil
 }
 
 type nowriter struct {
@@ -74,7 +90,7 @@ func wsEndpoint(w ResponseWriter, r *http.Request) error {
 	resp := &http.Response{
 		StatusCode: http.StatusSwitchingProtocols,
 	}
-	_ = w.WriteRespHeaders(resp)
+	_ = w.WriteRespHeaders(resp.StatusCode, resp.Header)
 	clientReader := nowriter{r.Body}
 	go func() {
 		for {
@@ -95,7 +111,7 @@ func originRespEndpoint(w ResponseWriter, status int, data []byte) {
 	resp := &http.Response{
 		StatusCode: status,
 	}
-	_ = w.WriteRespHeaders(resp)
+	_ = w.WriteRespHeaders(resp.StatusCode, resp.Header)
 	_, _ = w.Write(data)
 }
 

connection/event.go

@@ -18,7 +18,7 @@ const (
 	Connected
 	// Reconnecting means the connection to the edge is being re-established.
 	Reconnecting
-	// SetURL means this connection's tunnel was given a URL by the edge. Used for free tunnels.
+	// SetURL means this connection's tunnel was given a URL by the edge. Used for quick tunnels.
 	SetURL
 	// RegisteringTunnel means the non-named tunnel is registering its connection.
 	RegisteringTunnel

connection/h2mux.go

@@ -7,13 +7,13 @@ import (
 	"net/http"
 	"time"
 
-	"github.com/cloudflare/cloudflared/h2mux"
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
-	"github.com/cloudflare/cloudflared/websocket"
-
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
 	"golang.org/x/sync/errgroup"
+
+	"github.com/cloudflare/cloudflared/h2mux"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/cloudflare/cloudflared/websocket"
 )
 
 const (
@@ -33,6 +33,8 @@ type h2muxConnection struct {
 	gracefulShutdownC <-chan struct{}
 	stoppedGracefully bool
 
+	log *zerolog.Logger
+
 	// newRPCClientFunc allows us to mock RPCs during testing
 	newRPCClientFunc func(context.Context, io.ReadWriteCloser, *zerolog.Logger) NamedTunnelRPCClient
 }
@@ -167,7 +169,8 @@ func (h *h2muxConnection) serveMuxer(ctx context.Context) error {
 }
 
 func (h *h2muxConnection) controlLoop(ctx context.Context, connectedFuse ConnectedFuse, isNamedTunnel bool) {
-	updateMetricsTickC := time.Tick(h.muxerConfig.MetricsUpdateFreq)
+	updateMetricsTicker := time.NewTicker(h.muxerConfig.MetricsUpdateFreq)
+	defer updateMetricsTicker.Stop()
 	var shutdownCompleted <-chan struct{}
 	for {
 		select {
@@ -191,7 +194,7 @@ func (h *h2muxConnection) controlLoop(ctx context.Context, connectedFuse Connect
 			// don't wait for shutdown to finish when context is closed, this is the hard termination path
 			return
 
-		case <-updateMetricsTickC:
+		case <-updateMetricsTicker.C:
 			h.observer.metrics.updateMuxerMetrics(h.connIndexStr, h.muxer.Metrics())
 		}
 	}
@@ -216,12 +219,16 @@ func (h *h2muxConnection) ServeStream(stream *h2mux.MuxedStream) error {
 		return reqErr
 	}
 
-	err := h.config.OriginClient.Proxy(respWriter, req, websocket.IsWebSocketUpgrade(req))
+	var sourceConnectionType = TypeHTTP
+	if websocket.IsWebSocketUpgrade(req) {
+		sourceConnectionType = TypeWebsocket
+	}
+
+	err := h.config.OriginProxy.ProxyHTTP(respWriter, req, sourceConnectionType == TypeWebsocket)
 	if err != nil {
 		respWriter.WriteErrorResponse()
-		return err
 	}
-	return nil
+	return err
 }
 
 func (h *h2muxConnection) newRequest(stream *h2mux.MuxedStream) (*http.Request, error) {
@@ -229,7 +236,7 @@ func (h *h2muxConnection) newRequest(stream *h2mux.MuxedStream) (*http.Request,
 	if err != nil {
 		return nil, errors.Wrap(err, "Unexpected error from http.NewRequest")
 	}
-	err = h2mux.H2RequestHeadersToH1Request(stream.Headers, req)
+	err = H2RequestHeadersToH1Request(stream.Headers, req)
 	if err != nil {
 		return nil, errors.Wrap(err, "invalid request received")
 	}
@@ -240,16 +247,16 @@ type h2muxRespWriter struct {
 	*h2mux.MuxedStream
 }
 
-func (rp *h2muxRespWriter) WriteRespHeaders(resp *http.Response) error {
-	headers := h2mux.H1ResponseToH2ResponseHeaders(resp)
-	headers = append(headers, h2mux.Header{Name: ResponseMetaHeaderField, Value: responseMetaHeaderOrigin})
+func (rp *h2muxRespWriter) WriteRespHeaders(status int, header http.Header) error {
+	headers := H1ResponseToH2ResponseHeaders(status, header)
+	headers = append(headers, h2mux.Header{Name: ResponseMetaHeader, Value: responseMetaHeaderOrigin})
 	return rp.WriteHeaders(headers)
 }
 
 func (rp *h2muxRespWriter) WriteErrorResponse() {
 	_ = rp.WriteHeaders([]h2mux.Header{
 		{Name: ":status", Value: "502"},
-		{Name: ResponseMetaHeaderField, Value: responseMetaHeaderCfd},
+		{Name: ResponseMetaHeader, Value: responseMetaHeaderCfd},
 	})
 	_, _ = rp.Write([]byte("502 Bad Gateway"))
 }

connection/h2mux_test.go

@@ -38,7 +38,7 @@ func newH2MuxConnection(t require.TestingT) (*h2muxConnection, *h2mux.Muxer) {
 				// we only expect RPC traffic in client->edge direction, provide minimal support for mocking
 				require.True(t, stream.IsRPCStream())
 				return stream.WriteHeaders([]h2mux.Header{
-					{Name:  ":status", Value: "200"},
+					{Name: ":status", Value: "200"},
 				})
 			}),
 		}
@@ -115,9 +115,9 @@ func TestServeStreamHTTP(t *testing.T) {
 		require.True(t, hasHeader(stream, ":status", strconv.Itoa(test.expectedStatus)))
 
 		if test.isProxyError {
-			assert.True(t, hasHeader(stream, ResponseMetaHeaderField, responseMetaHeaderCfd))
+			assert.True(t, hasHeader(stream, ResponseMetaHeader, responseMetaHeaderCfd))
 		} else {
-			assert.True(t, hasHeader(stream, ResponseMetaHeaderField, responseMetaHeaderOrigin))
+			assert.True(t, hasHeader(stream, ResponseMetaHeader, responseMetaHeaderOrigin))
 			body := make([]byte, len(test.expectedBody))
 			_, err = stream.Read(body)
 			require.NoError(t, err)
@@ -164,7 +164,7 @@ func TestServeStreamWS(t *testing.T) {
 	require.NoError(t, err)
 
 	require.True(t, hasHeader(stream, ":status", strconv.Itoa(http.StatusSwitchingProtocols)))
-	assert.True(t, hasHeader(stream, ResponseMetaHeaderField, responseMetaHeaderOrigin))
+	assert.True(t, hasHeader(stream, ResponseMetaHeader, responseMetaHeaderOrigin))
 
 	data := []byte("test websocket")
 	err = wsutil.WriteClientText(writePipe, data)
@@ -268,7 +268,7 @@ func benchmarkServeStreamHTTPSimple(b *testing.B, test testRequest) {
 		b.StopTimer()
 
 		require.NoError(b, openstreamErr)
-		assert.True(b, hasHeader(stream, ResponseMetaHeaderField, responseMetaHeaderOrigin))
+		assert.True(b, hasHeader(stream, ResponseMetaHeader, responseMetaHeaderOrigin))
 		require.True(b, hasHeader(stream, ":status", strconv.Itoa(http.StatusOK)))
 		require.NoError(b, readBodyErr)
 		require.Equal(b, test.expectedBody, body)

connection/header.go

@@ -1,21 +1,33 @@
 package connection
 
 import (
+	"encoding/base64"
 	"fmt"
 	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+
+	"github.com/pkg/errors"
 
 	"github.com/cloudflare/cloudflared/h2mux"
 )
 
-const (
-	ResponseMetaHeaderField = "cf-cloudflared-response-meta"
+var (
+	// h2mux-style special headers
+	RequestUserHeaders  = "cf-cloudflared-request-headers"
+	ResponseUserHeaders = "cf-cloudflared-response-headers"
+	ResponseMetaHeader  = "cf-cloudflared-response-meta"
+
+	// h2mux-style special headers
+	CanonicalResponseUserHeaders = http.CanonicalHeaderKey(ResponseUserHeaders)
+	CanonicalResponseMetaHeader  = http.CanonicalHeaderKey(ResponseMetaHeader)
 )
 
 var (
-	canonicalResponseUserHeadersField = http.CanonicalHeaderKey(h2mux.ResponseUserHeadersField)
-	canonicalResponseMetaHeaderField  = http.CanonicalHeaderKey(ResponseMetaHeaderField)
-	responseMetaHeaderCfd             = mustInitRespMetaHeader("cloudflared")
-	responseMetaHeaderOrigin          = mustInitRespMetaHeader("origin")
+	// pre-generate possible values for res
+	responseMetaHeaderCfd    = mustInitRespMetaHeader("cloudflared")
+	responseMetaHeaderOrigin = mustInitRespMetaHeader("origin")
 )
 
 type responseMetaHeader struct {
@@ -29,3 +41,204 @@ func mustInitRespMetaHeader(src string) string {
 	}
 	return string(header)
 }
+
+var headerEncoding = base64.RawStdEncoding
+
+// note: all h2mux headers should be lower-case (http/2 style)
+const ()
+
+// H2RequestHeadersToH1Request converts the HTTP/2 headers coming from origintunneld
+// to an HTTP/1 Request object destined for the local origin web service.
+// This operation includes conversion of the pseudo-headers into their closest
+// HTTP/1 equivalents. See https://tools.ietf.org/html/rfc7540#section-8.1.2.3
+func H2RequestHeadersToH1Request(h2 []h2mux.Header, h1 *http.Request) error {
+	for _, header := range h2 {
+		name := strings.ToLower(header.Name)
+		if !IsControlHeader(name) {
+			continue
+		}
+
+		switch name {
+		case ":method":
+			h1.Method = header.Value
+		case ":scheme":
+			// noop - use the preexisting scheme from h1.URL
+		case ":authority":
+			// Otherwise the host header will be based on the origin URL
+			h1.Host = header.Value
+		case ":path":
+			// We don't want to be an "opinionated" proxy, so ideally we would use :path as-is.
+			// However, this HTTP/1 Request object belongs to the Go standard library,
+			// whose URL package makes some opinionated decisions about the encoding of
+			// URL characters: see the docs of https://godoc.org/net/url#URL,
+			// in particular the EscapedPath method https://godoc.org/net/url#URL.EscapedPath,
+			// which is always used when computing url.URL.String(), whether we'd like it or not.
+			//
+			// Well, not *always*. We could circumvent this by using url.URL.Opaque. But
+			// that would present unusual difficulties when using an HTTP proxy: url.URL.Opaque
+			// is treated differently when HTTP_PROXY is set!
+			// See https://github.com/golang/go/issues/5684#issuecomment-66080888
+			//
+			// This means we are subject to the behavior of net/url's function `shouldEscape`
+			// (as invoked with mode=encodePath): https://github.com/golang/go/blob/go1.12.7/src/net/url/url.go#L101
+
+			if header.Value == "*" {
+				h1.URL.Path = "*"
+				continue
+			}
+			// Due to the behavior of validation.ValidateUrl, h1.URL may
+			// already have a partial value, with or without a trailing slash.
+			base := h1.URL.String()
+			base = strings.TrimRight(base, "/")
+			// But we know :path begins with '/', because we handled '*' above - see RFC7540
+			requestURL, err := url.Parse(base + header.Value)
+			if err != nil {
+				return errors.Wrap(err, fmt.Sprintf("invalid path '%v'", header.Value))
+			}
+			h1.URL = requestURL
+		case "content-length":
+			contentLength, err := strconv.ParseInt(header.Value, 10, 64)
+			if err != nil {
+				return fmt.Errorf("unparseable content length")
+			}
+			h1.ContentLength = contentLength
+		case RequestUserHeaders:
+			// Do not forward the serialized headers to the origin -- deserialize them, and ditch the serialized version
+			// Find and parse user headers serialized into a single one
+			userHeaders, err := DeserializeHeaders(header.Value)
+			if err != nil {
+				return errors.Wrap(err, "Unable to parse user headers")
+			}
+			for _, userHeader := range userHeaders {
+				h1.Header.Add(userHeader.Name, userHeader.Value)
+			}
+		default:
+			// All other control headers shall just be proxied transparently
+			h1.Header.Add(header.Name, header.Value)
+		}
+	}
+
+	return nil
+}
+
+func IsControlHeader(headerName string) bool {
+	return headerName == "content-length" ||
+		headerName == "connection" || headerName == "upgrade" || // Websocket headers
+		strings.HasPrefix(headerName, ":") ||
+		strings.HasPrefix(headerName, "cf-")
+}
+
+// isWebsocketClientHeader returns true if the header name is required by the client to upgrade properly
+func IsWebsocketClientHeader(headerName string) bool {
+	return headerName == "sec-websocket-accept" ||
+		headerName == "connection" ||
+		headerName == "upgrade"
+}
+
+func H1ResponseToH2ResponseHeaders(status int, h1 http.Header) (h2 []h2mux.Header) {
+	h2 = []h2mux.Header{
+		{Name: ":status", Value: strconv.Itoa(status)},
+	}
+	userHeaders := make(http.Header, len(h1))
+	for header, values := range h1 {
+		h2name := strings.ToLower(header)
+		if h2name == "content-length" {
+			// This header has meaning in HTTP/2 and will be used by the edge,
+			// so it should be sent as an HTTP/2 response header.
+
+			// Since these are http2 headers, they're required to be lowercase
+			h2 = append(h2, h2mux.Header{Name: "content-length", Value: values[0]})
+		} else if !IsControlHeader(h2name) || IsWebsocketClientHeader(h2name) {
+			// User headers, on the other hand, must all be serialized so that
+			// HTTP/2 header validation won't be applied to HTTP/1 header values
+			userHeaders[header] = values
+		}
+	}
+
+	// Perform user header serialization and set them in the single header
+	h2 = append(h2, h2mux.Header{Name: ResponseUserHeaders, Value: SerializeHeaders(userHeaders)})
+	return h2
+}
+
+// Serialize HTTP1.x headers by base64-encoding each header name and value,
+// and then joining them in the format of [key:value;]
+func SerializeHeaders(h1Headers http.Header) string {
+	// compute size of the fully serialized value and largest temp buffer we will need
+	serializedLen := 0
+	maxTempLen := 0
+	for headerName, headerValues := range h1Headers {
+		for _, headerValue := range headerValues {
+			nameLen := headerEncoding.EncodedLen(len(headerName))
+			valueLen := headerEncoding.EncodedLen(len(headerValue))
+			const delims = 2
+			serializedLen += delims + nameLen + valueLen
+			if nameLen > maxTempLen {
+				maxTempLen = nameLen
+			}
+			if valueLen > maxTempLen {
+				maxTempLen = valueLen
+			}
+		}
+	}
+	var buf strings.Builder
+	buf.Grow(serializedLen)
+
+	temp := make([]byte, maxTempLen)
+	writeB64 := func(s string) {
+		n := headerEncoding.EncodedLen(len(s))
+		if n > len(temp) {
+			temp = make([]byte, n)
+		}
+		headerEncoding.Encode(temp[:n], []byte(s))
+		buf.Write(temp[:n])
+	}
+
+	for headerName, headerValues := range h1Headers {
+		for _, headerValue := range headerValues {
+			if buf.Len() > 0 {
+				buf.WriteByte(';')
+			}
+			writeB64(headerName)
+			buf.WriteByte(':')
+			writeB64(headerValue)
+		}
+	}
+
+	return buf.String()
+}
+
+// Deserialize headers serialized by `SerializeHeader`
+func DeserializeHeaders(serializedHeaders string) ([]h2mux.Header, error) {
+	const unableToDeserializeErr = "Unable to deserialize headers"
+
+	var deserialized []h2mux.Header
+	for _, serializedPair := range strings.Split(serializedHeaders, ";") {
+		if len(serializedPair) == 0 {
+			continue
+		}
+
+		serializedHeaderParts := strings.Split(serializedPair, ":")
+		if len(serializedHeaderParts) != 2 {
+			return nil, errors.New(unableToDeserializeErr)
+		}
+
+		serializedName := serializedHeaderParts[0]
+		serializedValue := serializedHeaderParts[1]
+		deserializedName := make([]byte, headerEncoding.DecodedLen(len(serializedName)))
+		deserializedValue := make([]byte, headerEncoding.DecodedLen(len(serializedValue)))
+
+		if _, err := headerEncoding.Decode(deserializedName, []byte(serializedName)); err != nil {
+			return nil, errors.Wrap(err, unableToDeserializeErr)
+		}
+		if _, err := headerEncoding.Decode(deserializedValue, []byte(serializedValue)); err != nil {
+			return nil, errors.Wrap(err, unableToDeserializeErr)
+		}
+
+		deserialized = append(deserialized, h2mux.Header{
+			Name:  string(deserializedName),
+			Value: string(deserializedValue),
+		})
+	}
+
+	return deserialized, nil
+}

connection/header_test.go

@@ -1,4 +1,4 @@
-package h2mux
+package connection
 
 import (
 	"fmt"
@@ -14,9 +14,11 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+
+	"github.com/cloudflare/cloudflared/h2mux"
 )
 
-type ByName []Header
+type ByName []h2mux.Header
 
 func (a ByName) Len() int      { return len(a) }
 func (a ByName) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
@@ -37,16 +39,16 @@ func TestH2RequestHeadersToH1Request_RegularHeaders(t *testing.T) {
 		"Mock header 2": {"Mock value 2"},
 	}
 
-	headersConversionErr := H2RequestHeadersToH1Request(createSerializedHeaders(RequestUserHeadersField, mockHeaders), request)
+	headersConversionErr := H2RequestHeadersToH1Request(createSerializedHeaders(RequestUserHeaders, mockHeaders), request)
 
 	assert.True(t, reflect.DeepEqual(mockHeaders, request.Header))
 	assert.NoError(t, headersConversionErr)
 }
 
-func createSerializedHeaders(headersField string, headers http.Header) []Header {
-	return []Header{{
-		headersField,
-		SerializeHeaders(headers),
+func createSerializedHeaders(headersField string, headers http.Header) []h2mux.Header {
+	return []h2mux.Header{{
+		Name:  headersField,
+		Value: SerializeHeaders(headers),
 	}}
 }
 
@@ -54,15 +56,16 @@ func TestH2RequestHeadersToH1Request_NoHeaders(t *testing.T) {
 	request, err := http.NewRequest(http.MethodGet, "http://example.com", nil)
 	assert.NoError(t, err)
 
+	emptyHeaders := make(http.Header)
 	headersConversionErr := H2RequestHeadersToH1Request(
-		[]Header{{
-			RequestUserHeadersField,
-			SerializeHeaders(http.Header{}),
+		[]h2mux.Header{{
+			Name:  RequestUserHeaders,
+			Value: SerializeHeaders(emptyHeaders),
 		}},
 		request,
 	)
 
-	assert.True(t, reflect.DeepEqual(http.Header{}, request.Header))
+	assert.True(t, reflect.DeepEqual(emptyHeaders, request.Header))
 	assert.NoError(t, headersConversionErr)
 }
 
@@ -70,9 +73,9 @@ func TestH2RequestHeadersToH1Request_InvalidHostPath(t *testing.T) {
 	request, err := http.NewRequest(http.MethodGet, "http://example.com", nil)
 	assert.NoError(t, err)
 
-	mockRequestHeaders := []Header{
+	mockRequestHeaders := []h2mux.Header{
 		{Name: ":path", Value: "//bad_path/"},
-		{Name: RequestUserHeadersField, Value: SerializeHeaders(http.Header{"Mock header": {"Mock value"}})},
+		{Name: RequestUserHeaders, Value: SerializeHeaders(http.Header{"Mock header": {"Mock value"}})},
 	}
 
 	headersConversionErr := H2RequestHeadersToH1Request(mockRequestHeaders, request)
@@ -90,9 +93,9 @@ func TestH2RequestHeadersToH1Request_HostPathWithQuery(t *testing.T) {
 	request, err := http.NewRequest(http.MethodGet, "http://example.com/", nil)
 	assert.NoError(t, err)
 
-	mockRequestHeaders := []Header{
+	mockRequestHeaders := []h2mux.Header{
 		{Name: ":path", Value: "/?query=mock%20value"},
-		{Name: RequestUserHeadersField, Value: SerializeHeaders(http.Header{"Mock header": {"Mock value"}})},
+		{Name: RequestUserHeaders, Value: SerializeHeaders(http.Header{"Mock header": {"Mock value"}})},
 	}
 
 	headersConversionErr := H2RequestHeadersToH1Request(mockRequestHeaders, request)
@@ -110,9 +113,9 @@ func TestH2RequestHeadersToH1Request_HostPathWithURLEncoding(t *testing.T) {
 	request, err := http.NewRequest(http.MethodGet, "http://example.com/", nil)
 	assert.NoError(t, err)
 
-	mockRequestHeaders := []Header{
+	mockRequestHeaders := []h2mux.Header{
 		{Name: ":path", Value: "/mock%20path"},
-		{Name: RequestUserHeadersField, Value: SerializeHeaders(http.Header{"Mock header": {"Mock value"}})},
+		{Name: RequestUserHeaders, Value: SerializeHeaders(http.Header{"Mock header": {"Mock value"}})},
 	}
 
 	headersConversionErr := H2RequestHeadersToH1Request(mockRequestHeaders, request)
@@ -267,9 +270,9 @@ func TestH2RequestHeadersToH1Request_WeirdURLs(t *testing.T) {
 		request, err := http.NewRequest(http.MethodGet, requestURL, nil)
 		assert.NoError(t, err)
 
-		mockRequestHeaders := []Header{
+		mockRequestHeaders := []h2mux.Header{
 			{Name: ":path", Value: testCase.path},
-			{Name: RequestUserHeadersField, Value: SerializeHeaders(http.Header{"Mock header": {"Mock value"}})},
+			{Name: RequestUserHeaders, Value: SerializeHeaders(http.Header{"Mock header": {"Mock value"}})},
 		}
 
 		headersConversionErr := H2RequestHeadersToH1Request(mockRequestHeaders, request)
@@ -337,12 +340,12 @@ func TestH2RequestHeadersToH1Request_QuickCheck(t *testing.T) {
 				const expectedMethod = "POST"
 				const expectedHostname = "request.hostname.example.com"
 
-				h2 := []Header{
+				h2 := []h2mux.Header{
 					{Name: ":method", Value: expectedMethod},
 					{Name: ":scheme", Value: testScheme},
 					{Name: ":authority", Value: expectedHostname},
 					{Name: ":path", Value: testPath},
-					{Name: RequestUserHeadersField, Value: ""},
+					{Name: RequestUserHeaders, Value: ""},
 				}
 				h1, err := http.NewRequest("GET", testOrigin.url, nil)
 				require.NoError(t, err)
@@ -424,10 +427,10 @@ func randomHTTP2Path(t *testing.T, rand *rand.Rand) string {
 	return result
 }
 
-func stdlibHeaderToH2muxHeader(headers http.Header) (h2muxHeaders []Header) {
+func stdlibHeaderToH2muxHeader(headers http.Header) (h2muxHeaders []h2mux.Header) {
 	for name, values := range headers {
 		for _, value := range values {
-			h2muxHeaders = append(h2muxHeaders, Header{name, value})
+			h2muxHeaders = append(h2muxHeaders, h2mux.Header{Name: name, Value: value})
 		}
 	}
 
@@ -515,14 +518,14 @@ func TestParseHeaders(t *testing.T) {
 		"Mock-Header-Three": {"3"},
 	}
 
-	mockHeaders := []Header{
+	mockHeaders := []h2mux.Header{
 		{Name: "One", Value: "1"}, // will be dropped
 		{Name: "Cf-Two", Value: "cf-value-1"},
 		{Name: "Cf-Two", Value: "cf-value-2"},
-		{Name: RequestUserHeadersField, Value: SerializeHeaders(mockUserHeadersToSerialize)},
+		{Name: RequestUserHeaders, Value: SerializeHeaders(mockUserHeadersToSerialize)},
 	}
 
-	expectedHeaders := []Header{
+	expectedHeaders := []h2mux.Header{
 		{Name: "Cf-Two", Value: "cf-value-1"},
 		{Name: "Cf-Two", Value: "cf-value-2"},
 		{Name: "Mock-Header-One", Value: "1"},
@@ -579,11 +582,11 @@ func TestH1ResponseToH2ResponseHeaders(t *testing.T) {
 		Header:     mockHeaders,
 	}
 
-	headers := H1ResponseToH2ResponseHeaders(&mockResponse)
+	headers := H1ResponseToH2ResponseHeaders(mockResponse.StatusCode, mockResponse.Header)
 
 	serializedHeadersIndex := -1
 	for i, header := range headers {
-		if header.Name == ResponseUserHeadersField {
+		if header.Name == ResponseUserHeaders {
 			serializedHeadersIndex = i
 			break
 		}
@@ -593,7 +596,7 @@ func TestH1ResponseToH2ResponseHeaders(t *testing.T) {
 		headers[:serializedHeadersIndex],
 		headers[serializedHeadersIndex+1:]...,
 	)
-	expectedControlHeaders := []Header{
+	expectedControlHeaders := []h2mux.Header{
 		{Name: ":status", Value: "200"},
 		{Name: "content-length", Value: "123"},
 	}
@@ -601,7 +604,7 @@ func TestH1ResponseToH2ResponseHeaders(t *testing.T) {
 	assert.ElementsMatch(t, expectedControlHeaders, actualControlHeaders)
 
 	actualUserHeaders, err := DeserializeHeaders(headers[serializedHeadersIndex].Value)
-	expectedUserHeaders := []Header{
+	expectedUserHeaders := []h2mux.Header{
 		{Name: "User-header-one", Value: ""},
 		{Name: "User-header-two", Value: "1"},
 		{Name: "User-header-two", Value: "2"},
@@ -622,7 +625,7 @@ func TestHeaderSize(t *testing.T) {
 		Header:     largeHeaders,
 	}
 
-	serializedHeaders := H1ResponseToH2ResponseHeaders(&mockResponse)
+	serializedHeaders := H1ResponseToH2ResponseHeaders(mockResponse.StatusCode, mockResponse.Header)
 	request, err := http.NewRequest(http.MethodGet, "https://example.com/", nil)
 	assert.NoError(t, err)
 	for _, header := range serializedHeaders {
@@ -630,7 +633,7 @@ func TestHeaderSize(t *testing.T) {
 	}
 
 	for _, header := range serializedHeaders {
-		if header.Name != ResponseUserHeadersField {
+		if header.Name != ResponseUserHeaders {
 			continue
 		}
 
@@ -669,6 +672,6 @@ func BenchmarkH1ResponseToH2ResponseHeaders(b *testing.B) {
 	b.ReportAllocs()
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		_ = H1ResponseToH2ResponseHeaders(h1resp)
+		_ = H1ResponseToH2ResponseHeaders(h1resp.StatusCode, h1resp.Header)
 	}
 }

connection/http2.go

@@ -10,22 +10,26 @@ import (
 	"strings"
 	"sync"
 
-	"github.com/cloudflare/cloudflared/h2mux"
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
-
+	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
 	"golang.org/x/net/http2"
+
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
+// note: these constants are exported so we can reuse them in the edge-side code
 const (
-	internalUpgradeHeader = "Cf-Cloudflared-Proxy-Connection-Upgrade"
-	websocketUpgrade      = "websocket"
-	controlStreamUpgrade  = "control-stream"
+	InternalUpgradeHeader     = "Cf-Cloudflared-Proxy-Connection-Upgrade"
+	InternalTCPProxySrcHeader = "Cf-Cloudflared-Proxy-Src"
+	WebsocketUpgrade          = "websocket"
+	ControlStreamUpgrade      = "control-stream"
 )
 
 var errEdgeConnectionClosed = fmt.Errorf("connection with edge closed")
 
-type http2Connection struct {
+// HTTP2Connection represents a net.Conn that uses HTTP2 frames to proxy traffic from the edge to cloudflared on the
+// origin.
+type HTTP2Connection struct {
 	conn         net.Conn
 	server       *http2.Server
 	config       *Config
@@ -37,6 +41,7 @@ type http2Connection struct {
 	// newRPCClientFunc allows us to mock RPCs during testing
 	newRPCClientFunc func(context.Context, io.ReadWriteCloser, *zerolog.Logger) NamedTunnelRPCClient
 
+	log               *zerolog.Logger
 	activeRequestsWG  sync.WaitGroup
 	connectedFuse     ConnectedFuse
 	gracefulShutdownC <-chan struct{}
@@ -44,6 +49,7 @@ type http2Connection struct {
 	controlStreamErr  error // result of running control stream handler
 }
 
+// NewHTTP2Connection returns a new instance of HTTP2Connection.
 func NewHTTP2Connection(
 	conn net.Conn,
 	config *Config,
@@ -52,9 +58,10 @@ func NewHTTP2Connection(
 	observer *Observer,
 	connIndex uint8,
 	connectedFuse ConnectedFuse,
+	log *zerolog.Logger,
 	gracefulShutdownC <-chan struct{},
-) *http2Connection {
-	return &http2Connection{
+) *HTTP2Connection {
+	return &HTTP2Connection{
 		conn: conn,
 		server: &http2.Server{
 			MaxConcurrentStreams: math.MaxUint32,
@@ -67,11 +74,13 @@ func NewHTTP2Connection(
 		connIndex:         connIndex,
 		newRPCClientFunc:  newRegistrationRPCClient,
 		connectedFuse:     connectedFuse,
+		log:               log,
 		gracefulShutdownC: gracefulShutdownC,
 	}
 }
 
-func (c *http2Connection) Serve(ctx context.Context) error {
+// Serve serves an HTTP2 server that the edge can talk to.
+func (c *HTTP2Connection) Serve(ctx context.Context) error {
 	go func() {
 		<-ctx.Done()
 		c.close()
@@ -92,40 +101,60 @@ func (c *http2Connection) Serve(ctx context.Context) error {
 	}
 }
 
-func (c *http2Connection) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+func (c *HTTP2Connection) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	c.activeRequestsWG.Add(1)
 	defer c.activeRequestsWG.Done()
 
-	respWriter := &http2RespWriter{
-		r: r.Body,
-		w: w,
-	}
-	flusher, isFlusher := w.(http.Flusher)
-	if !isFlusher {
-		c.observer.log.Error().Msgf("%T doesn't implement http.Flusher", w)
-		respWriter.WriteErrorResponse()
+	connType := determineHTTP2Type(r)
+	handleMissingRequestParts(connType, r)
+
+	respWriter, err := newHTTP2RespWriter(r, w, connType)
+	if err != nil {
+		c.observer.log.Error().Msg(err.Error())
 		return
 	}
-	respWriter.flusher = flusher
-	var err error
-	if isControlStreamUpgrade(r) {
-		respWriter.shouldFlush = true
-		err = c.serveControlStream(r.Context(), respWriter)
-		c.controlStreamErr = err
-	} else if isWebsocketUpgrade(r) {
-		respWriter.shouldFlush = true
-		stripWebsocketUpgradeHeader(r)
-		err = c.config.OriginClient.Proxy(respWriter, r, true)
-	} else {
-		err = c.config.OriginClient.Proxy(respWriter, r, false)
-	}
 
-	if err != nil {
+	switch connType {
+	case TypeControlStream:
+		if err := c.serveControlStream(r.Context(), respWriter); err != nil {
+			c.controlStreamErr = err
+			c.log.Error().Err(err)
+			respWriter.WriteErrorResponse()
+		}
+
+	case TypeWebsocket, TypeHTTP:
+		stripWebsocketUpgradeHeader(r)
+		if err := c.config.OriginProxy.ProxyHTTP(respWriter, r, connType == TypeWebsocket); err != nil {
+			err := fmt.Errorf("Failed to proxy HTTP: %w", err)
+			c.log.Error().Err(err)
+			respWriter.WriteErrorResponse()
+		}
+
+	case TypeTCP:
+		host, err := getRequestHost(r)
+		if err != nil {
+			err := fmt.Errorf(`cloudflared recieved a warp-routing request with an empty host value: %w`, err)
+			c.log.Error().Err(err)
+			respWriter.WriteErrorResponse()
+		}
+
+		rws := NewHTTPResponseReadWriterAcker(respWriter, r)
+		if err := c.config.OriginProxy.ProxyTCP(r.Context(), rws, &TCPRequest{
+			Dest:    host,
+			CFRay:   FindCfRayHeader(r),
+			LBProbe: IsLBProbeRequest(r),
+		}); err != nil {
+			respWriter.WriteErrorResponse()
+		}
+
+	default:
+		err := fmt.Errorf("Received unknown connection type: %s", connType)
+		c.log.Error().Err(err)
 		respWriter.WriteErrorResponse()
 	}
 }
 
-func (c *http2Connection) serveControlStream(ctx context.Context, respWriter *http2RespWriter) error {
+func (c *HTTP2Connection) serveControlStream(ctx context.Context, respWriter *http2RespWriter) error {
 	rpcClient := c.newRPCClientFunc(ctx, respWriter, c.observer.log)
 	defer rpcClient.Close()
 
@@ -148,7 +177,7 @@ func (c *http2Connection) serveControlStream(ctx context.Context, respWriter *ht
 	return nil
 }
 
-func (c *http2Connection) close() {
+func (c *HTTP2Connection) close() {
 	// Wait for all serve HTTP handlers to return
 	c.activeRequestsWG.Wait()
 	c.conn.Close()
@@ -161,36 +190,52 @@ type http2RespWriter struct {
 	shouldFlush bool
 }
 
-func (rp *http2RespWriter) WriteRespHeaders(resp *http.Response) error {
+func newHTTP2RespWriter(r *http.Request, w http.ResponseWriter, connType Type) (*http2RespWriter, error) {
+	flusher, isFlusher := w.(http.Flusher)
+	if !isFlusher {
+		respWriter := &http2RespWriter{
+			r: r.Body,
+			w: w,
+		}
+		respWriter.WriteErrorResponse()
+		return nil, fmt.Errorf("%T doesn't implement http.Flusher", w)
+	}
+
+	return &http2RespWriter{
+		r:           r.Body,
+		w:           w,
+		flusher:     flusher,
+		shouldFlush: connType.shouldFlush(),
+	}, nil
+}
+
+func (rp *http2RespWriter) WriteRespHeaders(status int, header http.Header) error {
 	dest := rp.w.Header()
-	userHeaders := make(http.Header, len(resp.Header))
-	for header, values := range resp.Header {
+	userHeaders := make(http.Header, len(header))
+	for name, values := range header {
 		// Since these are http2 headers, they're required to be lowercase
-		h2name := strings.ToLower(header)
-		for _, v := range values {
-			if h2name == "content-length" {
-				// This header has meaning in HTTP/2 and will be used by the edge,
-				// so it should be sent as an HTTP/2 response header.
-				dest.Add(h2name, v)
-				// Since these are http2 headers, they're required to be lowercase
-			} else if !h2mux.IsControlHeader(h2name) || h2mux.IsWebsocketClientHeader(h2name) {
-				// User headers, on the other hand, must all be serialized so that
-				// HTTP/2 header validation won't be applied to HTTP/1 header values
-				userHeaders.Add(h2name, v)
-			}
+		h2name := strings.ToLower(name)
+		if h2name == "content-length" {
+			// This header has meaning in HTTP/2 and will be used by the edge,
+			// so it should be sent as an HTTP/2 response header.
+			dest[name] = values
+			// Since these are http2 headers, they're required to be lowercase
+		} else if !IsControlHeader(h2name) || IsWebsocketClientHeader(h2name) {
+			// User headers, on the other hand, must all be serialized so that
+			// HTTP/2 header validation won't be applied to HTTP/1 header values
+			userHeaders[name] = values
 		}
 	}
 
 	// Perform user header serialization and set them in the single header
-	dest.Set(canonicalResponseUserHeadersField, h2mux.SerializeHeaders(userHeaders))
+	dest.Set(CanonicalResponseUserHeaders, SerializeHeaders(userHeaders))
 	rp.setResponseMetaHeader(responseMetaHeaderOrigin)
-	status := resp.StatusCode
 	// HTTP2 removes support for 101 Switching Protocols https://tools.ietf.org/html/rfc7540#section-8.1.1
 	if status == http.StatusSwitchingProtocols {
 		status = http.StatusOK
 	}
 	rp.w.WriteHeader(status)
-	if IsServerSentEvent(resp.Header) {
+	if IsServerSentEvent(header) {
 		rp.shouldFlush = true
 	}
 	if rp.shouldFlush {
@@ -205,7 +250,7 @@ func (rp *http2RespWriter) WriteErrorResponse() {
 }
 
 func (rp *http2RespWriter) setResponseMetaHeader(value string) {
-	rp.w.Header().Set(canonicalResponseMetaHeaderField, value)
+	rp.w.Header().Set(CanonicalResponseMetaHeader, value)
 }
 
 func (rp *http2RespWriter) Read(p []byte) (n int, err error) {
@@ -231,14 +276,57 @@ func (rp *http2RespWriter) Close() error {
 	return nil
 }
 
+func determineHTTP2Type(r *http.Request) Type {
+	switch {
+	case isWebsocketUpgrade(r):
+		return TypeWebsocket
+	case IsTCPStream(r):
+		return TypeTCP
+	case isControlStreamUpgrade(r):
+		return TypeControlStream
+	default:
+		return TypeHTTP
+	}
+}
+
+func handleMissingRequestParts(connType Type, r *http.Request) {
+	if connType == TypeHTTP {
+		// http library has no guarantees that we receive a filled URL. If not, then we fill it, as we reuse the request
+		// for proxying. We use the same values as we used to in h2mux. For proxying they should not matter since we
+		// control the dialer on every egress proxied.
+		if len(r.URL.Scheme) == 0 {
+			r.URL.Scheme = "http"
+		}
+		if len(r.URL.Host) == 0 {
+			r.URL.Host = "localhost:8080"
+		}
+	}
+}
+
 func isControlStreamUpgrade(r *http.Request) bool {
-	return strings.ToLower(r.Header.Get(internalUpgradeHeader)) == controlStreamUpgrade
+	return r.Header.Get(InternalUpgradeHeader) == ControlStreamUpgrade
 }
 
 func isWebsocketUpgrade(r *http.Request) bool {
-	return strings.ToLower(r.Header.Get(internalUpgradeHeader)) == websocketUpgrade
+	return r.Header.Get(InternalUpgradeHeader) == WebsocketUpgrade
+}
+
+// IsTCPStream discerns if the connection request needs a tcp stream proxy.
+func IsTCPStream(r *http.Request) bool {
+	return r.Header.Get(InternalTCPProxySrcHeader) != ""
 }
 
 func stripWebsocketUpgradeHeader(r *http.Request) {
-	r.Header.Del(internalUpgradeHeader)
+	r.Header.Del(InternalUpgradeHeader)
+}
+
+// getRequestHost returns the host of the http.Request.
+func getRequestHost(r *http.Request) (string, error) {
+	if r.Host != "" {
+		return r.Host, nil
+	}
+	if r.URL != nil {
+		return r.URL.Host, nil
+	}
+	return "", errors.New("host not set in incoming request")
 }

connection/http2_test.go

@@ -12,24 +12,24 @@ import (
 	"testing"
 	"time"
 
+	"github.com/gobwas/ws/wsutil"
+	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/net/http2"
 
 	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
-
-	"github.com/gobwas/ws/wsutil"
-	"github.com/rs/zerolog"
-	"github.com/stretchr/testify/require"
-	"golang.org/x/net/http2"
 )
 
 var (
 	testTransport = http2.Transport{}
 )
 
-func newTestHTTP2Connection() (*http2Connection, net.Conn) {
+func newTestHTTP2Connection() (*HTTP2Connection, net.Conn) {
 	edgeConn, originConn := net.Pipe()
 	var connIndex = uint8(0)
+	log := zerolog.Nop()
 	return NewHTTP2Connection(
 		originConn,
 		testConfig,
@@ -38,6 +38,7 @@ func newTestHTTP2Connection() (*http2Connection, net.Conn) {
 		NewObserver(&log, &log, false),
 		connIndex,
 		mockConnectedFuse{},
+		&log,
 		nil,
 	), edgeConn
 }
@@ -104,9 +105,9 @@ func TestServeHTTP(t *testing.T) {
 			require.Equal(t, test.expectedBody, respBody)
 		}
 		if test.isProxyError {
-			require.Equal(t, responseMetaHeaderCfd, resp.Header.Get(ResponseMetaHeaderField))
+			require.Equal(t, responseMetaHeaderCfd, resp.Header.Get(ResponseMetaHeader))
 		} else {
-			require.Equal(t, responseMetaHeaderOrigin, resp.Header.Get(ResponseMetaHeaderField))
+			require.Equal(t, responseMetaHeaderOrigin, resp.Header.Get(ResponseMetaHeader))
 		}
 	}
 	cancel()
@@ -114,6 +115,7 @@ func TestServeHTTP(t *testing.T) {
 }
 
 type mockNamedTunnelRPCClient struct {
+	shouldFail   error
 	registered   chan struct{}
 	unregistered chan struct{}
 }
@@ -125,6 +127,9 @@ func (mc mockNamedTunnelRPCClient) RegisterConnection(
 	connIndex uint8,
 	observer *Observer,
 ) error {
+	if mc.shouldFail != nil {
+		return mc.shouldFail
+	}
 	close(mc.registered)
 	return nil
 }
@@ -136,12 +141,14 @@ func (mc mockNamedTunnelRPCClient) GracefulShutdown(ctx context.Context, gracePe
 func (mockNamedTunnelRPCClient) Close() {}
 
 type mockRPCClientFactory struct {
+	shouldFail   error
 	registered   chan struct{}
 	unregistered chan struct{}
 }
 
 func (mf *mockRPCClientFactory) newMockRPCClient(context.Context, io.ReadWriteCloser, *zerolog.Logger) NamedTunnelRPCClient {
 	return mockNamedTunnelRPCClient{
+		shouldFail:   mf.shouldFail,
 		registered:   mf.registered,
 		unregistered: mf.unregistered,
 	}
@@ -186,7 +193,7 @@ func TestServeWS(t *testing.T) {
 
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://localhost:8080/ws", readPipe)
 	require.NoError(t, err)
-	req.Header.Set(internalUpgradeHeader, websocketUpgrade)
+	req.Header.Set(InternalUpgradeHeader, WebsocketUpgrade)
 
 	wg.Add(1)
 	go func() {
@@ -206,7 +213,7 @@ func TestServeWS(t *testing.T) {
 	resp := respWriter.Result()
 	// http2RespWriter should rewrite status 101 to 200
 	require.Equal(t, http.StatusOK, resp.StatusCode)
-	require.Equal(t, responseMetaHeaderOrigin, resp.Header.Get(ResponseMetaHeaderField))
+	require.Equal(t, responseMetaHeaderOrigin, resp.Header.Get(ResponseMetaHeader))
 
 	wg.Wait()
 }
@@ -230,7 +237,7 @@ func TestServeControlStream(t *testing.T) {
 
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://localhost:8080/", nil)
 	require.NoError(t, err)
-	req.Header.Set(internalUpgradeHeader, controlStreamUpgrade)
+	req.Header.Set(InternalUpgradeHeader, ControlStreamUpgrade)
 
 	edgeHTTP2Conn, err := testTransport.NewClientConn(edgeConn)
 	require.NoError(t, err)
@@ -249,6 +256,39 @@ func TestServeControlStream(t *testing.T) {
 	wg.Wait()
 }
 
+func TestFailRegistration(t *testing.T) {
+	http2Conn, edgeConn := newTestHTTP2Connection()
+
+	rpcClientFactory := mockRPCClientFactory{
+		shouldFail:   errDuplicationConnection,
+		registered:   make(chan struct{}),
+		unregistered: make(chan struct{}),
+	}
+	http2Conn.newRPCClientFunc = rpcClientFactory.newMockRPCClient
+
+	ctx, cancel := context.WithCancel(context.Background())
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		http2Conn.Serve(ctx)
+	}()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://localhost:8080/", nil)
+	require.NoError(t, err)
+	req.Header.Set(InternalUpgradeHeader, ControlStreamUpgrade)
+
+	edgeHTTP2Conn, err := testTransport.NewClientConn(edgeConn)
+	require.NoError(t, err)
+	resp, err := edgeHTTP2Conn.RoundTrip(req)
+	require.NoError(t, err)
+	require.Equal(t, http.StatusBadGateway, resp.StatusCode)
+
+	assert.NotNil(t, http2Conn.controlStreamErr)
+	cancel()
+	wg.Wait()
+}
+
 func TestGracefulShutdownHTTP2(t *testing.T) {
 	http2Conn, edgeConn := newTestHTTP2Connection()
 
@@ -272,7 +312,7 @@ func TestGracefulShutdownHTTP2(t *testing.T) {
 
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://localhost:8080/", nil)
 	require.NoError(t, err)
-	req.Header.Set(internalUpgradeHeader, controlStreamUpgrade)
+	req.Header.Set(InternalUpgradeHeader, ControlStreamUpgrade)
 
 	edgeHTTP2Conn, err := testTransport.NewClientConn(edgeConn)
 	require.NoError(t, err)

connection/metrics.go

@@ -4,8 +4,9 @@ import (
 	"sync"
 	"time"
 
-	"github.com/cloudflare/cloudflared/h2mux"
 	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/cloudflare/cloudflared/h2mux"
 )
 
 const (
@@ -414,4 +415,4 @@ func newTunnelMetrics() *tunnelMetrics {
 		tunnelMetricsInternal.metrics = initTunnelMetrics()
 	})
 	return tunnelMetricsInternal.metrics
-}
+}

connection/observer.go

@@ -1,12 +1,6 @@
 package connection
 
 import (
-	"fmt"
-	"net/url"
-	"strings"
-
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
-
 	"github.com/rs/zerolog"
 )
 
@@ -54,53 +48,6 @@ func (o *Observer) logServerInfo(connIndex uint8, location, msg string) {
 	o.metrics.registerServerLocation(uint8ToString(connIndex), location)
 }
 
-func (o *Observer) logTrialHostname(registration *tunnelpogs.TunnelRegistration) error {
-	// Print out the user's trial zone URL in a nice box (if they requested and got one and UI flag is not set)
-	if !o.uiEnabled {
-		if registrationURL, err := url.Parse(registration.Url); err == nil {
-			for _, line := range asciiBox(trialZoneMsg(registrationURL.String()), 2) {
-				o.log.Info().Msg(line)
-			}
-		} else {
-			o.log.Error().Msg("Failed to connect tunnel, please try again.")
-			return fmt.Errorf("empty URL in response from Cloudflare edge")
-		}
-	}
-	return nil
-}
-
-// Print out the given lines in a nice ASCII box.
-func asciiBox(lines []string, padding int) (box []string) {
-	maxLen := maxLen(lines)
-	spacer := strings.Repeat(" ", padding)
-
-	border := "+" + strings.Repeat("-", maxLen+(padding*2)) + "+"
-
-	box = append(box, border)
-	for _, line := range lines {
-		box = append(box, "|"+spacer+line+strings.Repeat(" ", maxLen-len(line))+spacer+"|")
-	}
-	box = append(box, border)
-	return
-}
-
-func maxLen(lines []string) int {
-	max := 0
-	for _, line := range lines {
-		if len(line) > max {
-			max = len(line)
-		}
-	}
-	return max
-}
-
-func trialZoneMsg(url string) []string {
-	return []string{
-		"Your free tunnel has started! Visit it:",
-		"  " + url,
-	}
-}
-
 func (o *Observer) sendRegisteringEvent(connIndex uint8) {
 	o.sendEvent(Event{Index: connIndex, EventType: RegisteringTunnel})
 }
@@ -109,7 +56,7 @@ func (o *Observer) sendConnectedEvent(connIndex uint8, location string) {
 	o.sendEvent(Event{Index: connIndex, EventType: Connected, Location: location})
 }
 
-func (o *Observer) sendURL(url string) {
+func (o *Observer) SendURL(url string) {
 	o.sendEvent(Event{EventType: SetURL, URL: url})
 }
 

connection/observer_test.go

@@ -67,10 +67,9 @@ func TestObserverEventsDontBlock(t *testing.T) {
 	}
 }
 
-
 type eventCollectorSink struct {
 	observedEvents []Event
-	mu sync.Mutex
+	mu             sync.Mutex
 }
 
 func (s *eventCollectorSink) OnTunnelEvent(event Event) {
@@ -83,4 +82,4 @@ func (s *eventCollectorSink) assertSawEvent(t *testing.T, event Event) {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 	assert.Contains(t, s.observedEvents, event)
-}
+}

connection/protocol.go

@@ -141,16 +141,29 @@ type PercentageFetcher func() (int32, error)
 
 func NewProtocolSelector(
 	protocolFlag string,
+	warpRoutingEnabled bool,
 	namedTunnel *NamedTunnelConfig,
 	fetchFunc PercentageFetcher,
 	ttl time.Duration,
 	log *zerolog.Logger,
 ) (ProtocolSelector, error) {
+	// Classic tunnel is only supported with h2mux
 	if namedTunnel == nil {
 		return &staticProtocolSelector{
 			current: H2mux,
 		}, nil
 	}
+
+	// warp routing can only be served over http2 connections
+	if warpRoutingEnabled {
+		if protocolFlag == H2mux.String() {
+			log.Warn().Msg("Warp routing is only supported by http2 protocol. Upgrading protocol to http2")
+		}
+		return &staticProtocolSelector{
+			current: HTTP2,
+		}, nil
+	}
+
 	if protocolFlag == H2mux.String() {
 		return &staticProtocolSelector{
 			current: H2mux,

connection/protocol_test.go

@@ -9,7 +9,8 @@ import (
 )
 
 const (
-	testNoTTL = 0
+	testNoTTL            = 0
+	noWarpRoutingEnabled = false
 )
 
 var (
@@ -48,14 +49,15 @@ func (dmf *dynamicMockFetcher) fetch() PercentageFetcher {
 
 func TestNewProtocolSelector(t *testing.T) {
 	tests := []struct {
-		name              string
-		protocol          string
-		expectedProtocol  Protocol
-		hasFallback       bool
-		expectedFallback  Protocol
-		namedTunnelConfig *NamedTunnelConfig
-		fetchFunc         PercentageFetcher
-		wantErr           bool
+		name               string
+		protocol           string
+		expectedProtocol   Protocol
+		hasFallback        bool
+		expectedFallback   Protocol
+		warpRoutingEnabled bool
+		namedTunnelConfig  *NamedTunnelConfig
+		fetchFunc          PercentageFetcher
+		wantErr            bool
 	}{
 		{
 			name:              "classic tunnel",
@@ -108,6 +110,36 @@ func TestNewProtocolSelector(t *testing.T) {
 			fetchFunc:         mockFetcher(100),
 			namedTunnelConfig: testNamedTunnelConfig,
 		},
+		{
+			name:               "warp routing requesting h2mux",
+			protocol:           "h2mux",
+			expectedProtocol:   HTTP2,
+			hasFallback:        false,
+			expectedFallback:   H2mux,
+			fetchFunc:          mockFetcher(100),
+			warpRoutingEnabled: true,
+			namedTunnelConfig:  testNamedTunnelConfig,
+		},
+		{
+			name:               "warp routing http2",
+			protocol:           "http2",
+			expectedProtocol:   HTTP2,
+			hasFallback:        false,
+			expectedFallback:   H2mux,
+			fetchFunc:          mockFetcher(100),
+			warpRoutingEnabled: true,
+			namedTunnelConfig:  testNamedTunnelConfig,
+		},
+		{
+			name:               "warp routing auto",
+			protocol:           "auto",
+			expectedProtocol:   HTTP2,
+			hasFallback:        false,
+			expectedFallback:   H2mux,
+			fetchFunc:          mockFetcher(100),
+			warpRoutingEnabled: true,
+			namedTunnelConfig:  testNamedTunnelConfig,
+		},
 		{
 			// None named tunnel can only use h2mux, so specifying an unknown protocol is not an error
 			name:             "classic tunnel unknown protocol",
@@ -131,7 +163,7 @@ func TestNewProtocolSelector(t *testing.T) {
 	}
 
 	for _, test := range tests {
-		selector, err := NewProtocolSelector(test.protocol, test.namedTunnelConfig, test.fetchFunc, testNoTTL, &log)
+		selector, err := NewProtocolSelector(test.protocol, test.warpRoutingEnabled, test.namedTunnelConfig, test.fetchFunc, testNoTTL, &log)
 		if test.wantErr {
 			assert.Error(t, err, fmt.Sprintf("test %s failed", test.name))
 		} else {
@@ -148,7 +180,7 @@ func TestNewProtocolSelector(t *testing.T) {
 
 func TestAutoProtocolSelectorRefresh(t *testing.T) {
 	fetcher := dynamicMockFetcher{}
-	selector, err := NewProtocolSelector("auto", testNamedTunnelConfig, fetcher.fetch(), testNoTTL, &log)
+	selector, err := NewProtocolSelector("auto", noWarpRoutingEnabled, testNamedTunnelConfig, fetcher.fetch(), testNoTTL, &log)
 	assert.NoError(t, err)
 	assert.Equal(t, H2mux, selector.Current())
 
@@ -177,7 +209,7 @@ func TestAutoProtocolSelectorRefresh(t *testing.T) {
 
 func TestHTTP2ProtocolSelectorRefresh(t *testing.T) {
 	fetcher := dynamicMockFetcher{}
-	selector, err := NewProtocolSelector("http2", testNamedTunnelConfig, fetcher.fetch(), testNoTTL, &log)
+	selector, err := NewProtocolSelector("http2", noWarpRoutingEnabled, testNamedTunnelConfig, fetcher.fetch(), testNoTTL, &log)
 	assert.NoError(t, err)
 	assert.Equal(t, HTTP2, selector.Current())
 
@@ -206,7 +238,7 @@ func TestHTTP2ProtocolSelectorRefresh(t *testing.T) {
 
 func TestProtocolSelectorRefreshTTL(t *testing.T) {
 	fetcher := dynamicMockFetcher{percentage: 100}
-	selector, err := NewProtocolSelector("auto", testNamedTunnelConfig, fetcher.fetch(), time.Hour, &log)
+	selector, err := NewProtocolSelector("auto", noWarpRoutingEnabled, testNamedTunnelConfig, fetcher.fetch(), time.Hour, &log)
 	assert.NoError(t, err)
 	assert.Equal(t, HTTP2, selector.Current())
 

connection/rpc.go

@@ -6,11 +6,11 @@ import (
 	"io"
 	"time"
 
-	"github.com/cloudflare/cloudflared/tunnelrpc"
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
-
 	"github.com/rs/zerolog"
 	"zombiezen.com/go/capnproto2/rpc"
+
+	"github.com/cloudflare/cloudflared/tunnelrpc"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
 type tunnelServerClient struct {
@@ -159,8 +159,6 @@ func (h *h2muxConnection) registerTunnel(ctx context.Context, credentialSetter C
 		return h.processRegisterTunnelError(registrationErr, register)
 	}
 
-	// Send free tunnel URL to UI
-	h.observer.sendURL(registration.Url)
 	credentialSetter.SetEventDigest(h.connIndex, registration.EventDigest)
 	return h.processRegistrationSuccess(registration, register, credentialSetter, classicTunnel)
 }
@@ -187,14 +185,6 @@ func (h *h2muxConnection) processRegistrationSuccess(
 		h.observer.log.Info().Msgf("Each HA connection's tunnel IDs: %v", h.observer.metrics.tunnelsHA.String())
 	}
 
-	// Print out the user's trial zone URL in a nice box (if they requested and got one and UI flag is not set)
-	if classicTunnel.IsTrialZone() {
-		err := h.observer.logTrialHostname(registration)
-		if err != nil {
-			return err
-		}
-	}
-
 	credentialManager.SetConnDigest(h.connIndex, registration.ConnDigest)
 	h.observer.metrics.userHostnamesCounts.WithLabelValues(registration.Url).Inc()
 

dbconnect/client.go

@@ -1,145 +0,0 @@
-package dbconnect
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"net/url"
-	"strings"
-	"time"
-	"unicode"
-	"unicode/utf8"
-)
-
-// Client is an interface to talk to any database.
-//
-// Currently, the only implementation is SQLClient, but its structure
-// should be designed to handle a MongoClient or RedisClient in the future.
-type Client interface {
-	Ping(context.Context) error
-	Submit(context.Context, *Command) (interface{}, error)
-}
-
-// NewClient creates a database client based on its URL scheme.
-func NewClient(ctx context.Context, originURL *url.URL) (Client, error) {
-	return NewSQLClient(ctx, originURL)
-}
-
-// Command is a standard, non-vendor format for submitting database commands.
-//
-// When determining the scope of this struct, refer to the following litmus test:
-// Could this (roughly) conform to SQL, Document-based, and Key-value command formats?
-type Command struct {
-	Statement string        `json:"statement"`
-	Arguments Arguments     `json:"arguments,omitempty"`
-	Mode      string        `json:"mode,omitempty"`
-	Isolation string        `json:"isolation,omitempty"`
-	Timeout   time.Duration `json:"timeout,omitempty"`
-}
-
-// Validate enforces the contract of Command: non empty statement (both in length and logic),
-// lowercase mode and isolation, non-zero timeout, and valid Arguments.
-func (cmd *Command) Validate() error {
-	if cmd.Statement == "" {
-		return fmt.Errorf("cannot provide an empty statement")
-	}
-
-	if strings.Map(func(char rune) rune {
-		if char == ';' || unicode.IsSpace(char) {
-			return -1
-		}
-		return char
-	}, cmd.Statement) == "" {
-		return fmt.Errorf("cannot provide a statement with no logic: '%s'", cmd.Statement)
-	}
-
-	cmd.Mode = strings.ToLower(cmd.Mode)
-	cmd.Isolation = strings.ToLower(cmd.Isolation)
-
-	if cmd.Timeout.Nanoseconds() <= 0 {
-		cmd.Timeout = 24 * time.Hour
-	}
-
-	return cmd.Arguments.Validate()
-}
-
-// UnmarshalJSON converts a byte representation of JSON into a Command, which is also validated.
-func (cmd *Command) UnmarshalJSON(data []byte) error {
-	// Alias is required to avoid infinite recursion from the default UnmarshalJSON.
-	type Alias Command
-	alias := &struct {
-		*Alias
-	}{
-		Alias: (*Alias)(cmd),
-	}
-
-	err := json.Unmarshal(data, &alias)
-	if err == nil {
-		err = cmd.Validate()
-	}
-
-	return err
-}
-
-// Arguments is a wrapper for either map-based or array-based Command arguments.
-//
-// Each field is mutually-exclusive and some Client implementations may not
-// support both fields (eg. MySQL does not accept named arguments).
-type Arguments struct {
-	Named      map[string]interface{}
-	Positional []interface{}
-}
-
-// Validate enforces the contract of Arguments: non nil, mutually exclusive, and no empty or reserved keys.
-func (args *Arguments) Validate() error {
-	if args.Named == nil {
-		args.Named = map[string]interface{}{}
-	}
-	if args.Positional == nil {
-		args.Positional = []interface{}{}
-	}
-
-	if len(args.Named) > 0 && len(args.Positional) > 0 {
-		return fmt.Errorf("both named and positional arguments cannot be specified: %+v and %+v", args.Named, args.Positional)
-	}
-
-	for key := range args.Named {
-		if key == "" {
-			return fmt.Errorf("named arguments cannot contain an empty key: %+v", args.Named)
-		}
-		if !utf8.ValidString(key) {
-			return fmt.Errorf("named argument does not conform to UTF-8 encoding: %s", key)
-		}
-		if strings.HasPrefix(key, "_") {
-			return fmt.Errorf("named argument cannot start with a reserved keyword '_': %s", key)
-		}
-		if unicode.IsNumber([]rune(key)[0]) {
-			return fmt.Errorf("named argument cannot start with a number: %s", key)
-		}
-	}
-
-	return nil
-}
-
-// UnmarshalJSON converts a byte representation of JSON into Arguments, which is also validated.
-func (args *Arguments) UnmarshalJSON(data []byte) error {
-	var obj interface{}
-	err := json.Unmarshal(data, &obj)
-	if err != nil {
-		return err
-	}
-
-	named, ok := obj.(map[string]interface{})
-	if ok {
-		args.Named = named
-	} else {
-		positional, ok := obj.([]interface{})
-		if ok {
-			args.Positional = positional
-		} else {
-			return fmt.Errorf("arguments must either be an object {\"0\":\"val\"} or an array [\"val\"]: %s", string(data))
-		}
-	}
-
-	return args.Validate()
-}

dbconnect/client_test.go

@@ -1,183 +0,0 @@
-package dbconnect
-
-import (
-	"encoding/json"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestCommandValidateEmpty(t *testing.T) {
-	stmts := []string{
-		"",
-		";",
-		" \n\t",
-		";\n;\t;",
-	}
-
-	for _, stmt := range stmts {
-		cmd := Command{Statement: stmt}
-
-		assert.Error(t, cmd.Validate(), stmt)
-	}
-}
-
-func TestCommandValidateMode(t *testing.T) {
-	modes := []string{
-		"",
-		"query",
-		"ExEc",
-		"PREPARE",
-	}
-
-	for _, mode := range modes {
-		cmd := Command{Statement: "Ok", Mode: mode}
-
-		assert.NoError(t, cmd.Validate(), mode)
-		assert.Equal(t, strings.ToLower(mode), cmd.Mode)
-	}
-}
-
-func TestCommandValidateIsolation(t *testing.T) {
-	isos := []string{
-		"",
-		"default",
-		"read_committed",
-		"SNAPshot",
-	}
-
-	for _, iso := range isos {
-		cmd := Command{Statement: "Ok", Isolation: iso}
-
-		assert.NoError(t, cmd.Validate(), iso)
-		assert.Equal(t, strings.ToLower(iso), cmd.Isolation)
-	}
-}
-
-func TestCommandValidateTimeout(t *testing.T) {
-	cmd := Command{Statement: "Ok", Timeout: 0}
-
-	assert.NoError(t, cmd.Validate())
-	assert.NotZero(t, cmd.Timeout)
-
-	cmd = Command{Statement: "Ok", Timeout: 1 * time.Second}
-
-	assert.NoError(t, cmd.Validate())
-	assert.Equal(t, 1*time.Second, cmd.Timeout)
-}
-
-func TestCommandValidateArguments(t *testing.T) {
-	cmd := Command{Statement: "Ok", Arguments: Arguments{
-		Named:      map[string]interface{}{"key": "val"},
-		Positional: []interface{}{"val"},
-	}}
-
-	assert.Error(t, cmd.Validate())
-}
-
-func TestCommandUnmarshalJSON(t *testing.T) {
-	strs := []string{
-		"{\"statement\":\"Ok\"}",
-		"{\"statement\":\"Ok\",\"arguments\":[0, 3.14, \"apple\"],\"mode\":\"query\"}",
-		"{\"statement\":\"Ok\",\"isolation\":\"read_uncommitted\",\"timeout\":1000}",
-	}
-
-	for _, str := range strs {
-		var cmd Command
-		assert.NoError(t, json.Unmarshal([]byte(str), &cmd), str)
-	}
-
-	strs = []string{
-		"",
-		"\"",
-		"{}",
-		"{\"argument\":{\"key\":\"val\"}}",
-		"{\"statement\":[\"Ok\"]}",
-	}
-
-	for _, str := range strs {
-		var cmd Command
-		assert.Error(t, json.Unmarshal([]byte(str), &cmd), str)
-	}
-}
-
-func TestArgumentsValidateNotNil(t *testing.T) {
-	args := Arguments{}
-
-	assert.NoError(t, args.Validate())
-	assert.NotNil(t, args.Named)
-	assert.NotNil(t, args.Positional)
-}
-
-func TestArgumentsValidateMutuallyExclusive(t *testing.T) {
-	args := []Arguments{
-		Arguments{},
-		Arguments{Named: map[string]interface{}{"key": "val"}},
-		Arguments{Positional: []interface{}{"val"}},
-	}
-
-	for _, arg := range args {
-		assert.NoError(t, arg.Validate())
-		assert.False(t, len(arg.Named) > 0 && len(arg.Positional) > 0)
-	}
-
-	args = []Arguments{
-		Arguments{
-			Named:      map[string]interface{}{"key": "val"},
-			Positional: []interface{}{"val"},
-		},
-	}
-
-	for _, arg := range args {
-		assert.Error(t, arg.Validate())
-		assert.True(t, len(arg.Named) > 0 && len(arg.Positional) > 0)
-	}
-}
-
-func TestArgumentsValidateKeys(t *testing.T) {
-	keys := []string{
-		"",
-		"_",
-		"_key",
-		"1",
-		"1key",
-		"\xf0\x28\x8c\xbc", // non-utf8
-	}
-
-	for _, key := range keys {
-		args := Arguments{Named: map[string]interface{}{key: "val"}}
-
-		assert.Error(t, args.Validate(), key)
-	}
-}
-
-func TestArgumentsUnmarshalJSON(t *testing.T) {
-	strs := []string{
-		"{}",
-		"{\"key\":\"val\"}",
-		"{\"key\":[1, 3.14, {\"key\":\"val\"}]}",
-		"[]",
-		"[\"key\",\"val\"]",
-		"[{}]",
-	}
-
-	for _, str := range strs {
-		var args Arguments
-		assert.NoError(t, json.Unmarshal([]byte(str), &args), str)
-	}
-
-	strs = []string{
-		"",
-		"\"",
-		"1",
-		"\"key\"",
-		"{\"key\",\"val\"}",
-	}
-
-	for _, str := range strs {
-		var args Arguments
-		assert.Error(t, json.Unmarshal([]byte(str), &args), str)
-	}
-}

dbconnect/cmd.go

@@ -1,165 +0,0 @@
-package dbconnect
-
-import (
-	"context"
-	"log"
-	"net"
-	"strconv"
-
-	"github.com/urfave/cli/v2"
-	"github.com/urfave/cli/v2/altsrc"
-)
-
-// Cmd is the entrypoint command for dbconnect.
-//
-// The tunnel package is responsible for appending this to tunnel.Commands().
-func Cmd() *cli.Command {
-	return &cli.Command{
-		Category:  "Database Connect (ALPHA) - Deprecated",
-		Name:      "db-connect",
-		Usage:     "deprecated: Access your SQL database from Cloudflare Workers or the browser",
-		ArgsUsage: " ",
-		Description: `
-		This feature has been deprecated. 
-		Please see: 
-
-		cloudflared access tcp --help 
-
-		for setting up database connections to the cloudflare edge.
-		
-
-		Creates a connection between your database and the Cloudflare edge.
-		Now you can execute SQL commands anywhere you can send HTTPS requests.
-
-		Connect your database with any of the following commands, you can also try the "playground" without a database:
-
-			cloudflared db-connect --hostname sql.mysite.com --url postgres://user:pass@localhost?sslmode=disable \
-			                       --auth-domain mysite.cloudflareaccess.com --application-aud my-access-policy-tag
-			cloudflared db-connect --hostname sql-dev.mysite.com --url mysql://localhost --insecure
-			cloudflared db-connect --playground
-		
-		Requests should be authenticated using Cloudflare Access, learn more about how to enable it here:
-
-			https://developers.cloudflare.com/access/service-auth/service-token/
-		`,
-		Flags: []cli.Flag{
-			altsrc.NewStringFlag(&cli.StringFlag{
-				Name:    "url",
-				Usage:   "URL to the database (eg. postgres://user:pass@localhost?sslmode=disable)",
-				EnvVars: []string{"TUNNEL_URL"},
-			}),
-			altsrc.NewStringFlag(&cli.StringFlag{
-				Name:    "hostname",
-				Usage:   "Hostname to accept commands over HTTPS (eg. sql.mysite.com)",
-				EnvVars: []string{"TUNNEL_HOSTNAME"},
-			}),
-			altsrc.NewStringFlag(&cli.StringFlag{
-				Name:    "auth-domain",
-				Usage:   "Cloudflare Access authentication domain for your account (eg. mysite.cloudflareaccess.com)",
-				EnvVars: []string{"TUNNEL_ACCESS_AUTH_DOMAIN"},
-			}),
-			altsrc.NewStringFlag(&cli.StringFlag{
-				Name:    "application-aud",
-				Usage:   "Cloudflare Access application \"AUD\" to verify JWTs from requests",
-				EnvVars: []string{"TUNNEL_ACCESS_APPLICATION_AUD"},
-			}),
-			altsrc.NewBoolFlag(&cli.BoolFlag{
-				Name:    "insecure",
-				Usage:   "Disable authentication, the database will be open to the Internet",
-				Value:   false,
-				EnvVars: []string{"TUNNEL_ACCESS_INSECURE"},
-			}),
-			altsrc.NewBoolFlag(&cli.BoolFlag{
-				Name:    "playground",
-				Usage:   "Run a temporary, in-memory SQLite3 database for testing",
-				Value:   false,
-				EnvVars: []string{"TUNNEL_HELLO_WORLD"},
-			}),
-			altsrc.NewStringFlag(&cli.StringFlag{
-				Name:    "loglevel",
-				Value:   "debug", // Make it more verbose than the tunnel default 'info'.
-				EnvVars: []string{"TUNNEL_LOGLEVEL"},
-				Hidden:  true,
-			}),
-		},
-		Before: CmdBefore,
-		Action: CmdAction,
-		Hidden: true,
-	}
-}
-
-// CmdBefore runs some validation checks before running the command.
-func CmdBefore(c *cli.Context) error {
-	// Show the help text is no flags are specified.
-	if c.NumFlags() == 0 {
-		return cli.ShowSubcommandHelp(c)
-	}
-
-	// Hello-world and playground are synonymous with each other,
-	// unset hello-world to prevent tunnel from initializing the hello package.
-	if c.IsSet("hello-world") {
-		c.Set("playground", "true")
-		c.Set("hello-world", "false")
-	}
-
-	// Unix-socket database urls are supported, but the logic is the same as url.
-	if c.IsSet("unix-socket") {
-		c.Set("url", c.String("unix-socket"))
-		c.Set("unix-socket", "")
-	}
-
-	// When playground mode is enabled, run with an in-memory database.
-	if c.IsSet("playground") {
-		c.Set("url", "sqlite3::memory:?cache=shared")
-		c.Set("insecure", strconv.FormatBool(!c.IsSet("auth-domain") && !c.IsSet("application-aud")))
-	}
-
-	// At this point, insecure configurations are valid.
-	if c.Bool("insecure") {
-		return nil
-	}
-
-	// Ensure that secure configurations specify a hostname, domain, and tag for JWT validation.
-	if !c.IsSet("hostname") || !c.IsSet("auth-domain") || !c.IsSet("application-aud") {
-		log.Fatal("must specify --hostname, --auth-domain, and --application-aud unless you want to run in --insecure mode")
-	}
-
-	return nil
-}
-
-// CmdAction starts the Proxy and sets the url in cli.Context to point to the Proxy address.
-func CmdAction(c *cli.Context) error {
-	// STOR-612: sync with context in tunnel daemon.
-	ctx := context.Background()
-
-	var proxy *Proxy
-	var err error
-	if c.Bool("insecure") {
-		proxy, err = NewInsecureProxy(ctx, c.String("url"))
-	} else {
-		proxy, err = NewSecureProxy(ctx, c.String("url"), c.String("auth-domain"), c.String("application-aud"))
-	}
-
-	if err != nil {
-		log.Fatal(err)
-		return err
-	}
-
-	listenerC := make(chan net.Listener)
-	defer close(listenerC)
-
-	// Since the Proxy should only talk to the tunnel daemon, find the next available
-	// localhost port and start to listen to requests.
-	go func() {
-		err := proxy.Start(ctx, "127.0.0.1:", listenerC)
-		if err != nil {
-			log.Fatal(err)
-		}
-	}()
-
-	// Block until the the Proxy is online, retreive its address, and change the url to point to it.
-	// This is effectively "handing over" control to the tunnel package so it can run the tunnel daemon.
-	c.Set("url", "https://"+(<-listenerC).Addr().String())
-
-	return nil
-}

dbconnect/cmd_test.go

@@ -1,27 +0,0 @@
-package dbconnect
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-
-	"github.com/urfave/cli/v2"
-)
-
-func TestCmd(t *testing.T) {
-	tests := [][]string{
-		{"cloudflared", "db-connect", "--playground"},
-		{"cloudflared", "db-connect", "--playground", "--hostname", "sql.mysite.com"},
-		{"cloudflared", "db-connect", "--url", "sqlite3::memory:?cache=shared", "--insecure"},
-		{"cloudflared", "db-connect", "--url", "sqlite3::memory:?cache=shared", "--hostname", "sql.mysite.com", "--auth-domain", "mysite.cloudflareaccess.com", "--application-aud", "aud"},
-	}
-
-	app := &cli.App{
-		Name:     "cloudflared",
-		Commands: []*cli.Command{Cmd()},
-	}
-
-	for _, test := range tests {
-		assert.NoError(t, app.Run(test))
-	}
-}

dbconnect/proxy.go

@@ -1,279 +0,0 @@
-package dbconnect
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"io"
-	"net"
-	"net/http"
-	"net/url"
-	"os"
-	"time"
-
-	"github.com/cloudflare/cloudflared/hello"
-	"github.com/cloudflare/cloudflared/validation"
-	"github.com/gorilla/mux"
-	"github.com/pkg/errors"
-	"github.com/rs/zerolog"
-)
-
-const (
-	LogFieldCFRay = "cfRay"
-)
-
-// Proxy is an HTTP server that proxies requests to a Client.
-type Proxy struct {
-	client          Client
-	accessValidator *validation.Access
-	log             *zerolog.Logger
-}
-
-// NewInsecureProxy creates a Proxy that talks to a Client at an origin.
-//
-// In insecure mode, the Proxy will allow all Command requests.
-func NewInsecureProxy(ctx context.Context, origin string) (*Proxy, error) {
-	originURL, err := url.Parse(origin)
-	if err != nil {
-		return nil, errors.Wrap(err, "must provide a valid database url")
-	}
-
-	client, err := NewClient(ctx, originURL)
-	if err != nil {
-		return nil, err
-	}
-
-	err = client.Ping(ctx)
-	if err != nil {
-		return nil, errors.Wrap(err, "could not connect to the database")
-	}
-
-	log := zerolog.New(os.Stderr).With().Logger() // TODO: Does not obey log configuration
-
-	return &Proxy{client, nil, &log}, nil
-}
-
-// NewSecureProxy creates a Proxy that talks to a Client at an origin.
-//
-// In secure mode, the Proxy will reject any Command requests that are
-// not authenticated by Cloudflare Access with a valid JWT.
-func NewSecureProxy(ctx context.Context, origin, authDomain, applicationAUD string) (*Proxy, error) {
-	proxy, err := NewInsecureProxy(ctx, origin)
-	if err != nil {
-		return nil, err
-	}
-
-	validator, err := validation.NewAccessValidator(ctx, authDomain, authDomain, applicationAUD)
-	if err != nil {
-		return nil, err
-	}
-
-	proxy.accessValidator = validator
-
-	return proxy, err
-}
-
-// IsInsecure gets whether the Proxy will accept a Command from any source.
-func (proxy *Proxy) IsInsecure() bool {
-	return proxy.accessValidator == nil
-}
-
-// IsAllowed checks whether a http.Request is allowed to receive data.
-//
-// By default, requests must pass through Cloudflare Access for authentication.
-// If the proxy is explcitly set to insecure mode, all requests will be allowed.
-func (proxy *Proxy) IsAllowed(r *http.Request, verbose ...bool) bool {
-	if proxy.IsInsecure() {
-		return true
-	}
-
-	// Access and Tunnel should prevent bad JWTs from even reaching the origin,
-	// but validate tokens anyway as an abundance of caution.
-	err := proxy.accessValidator.ValidateRequest(r.Context(), r)
-	if err == nil {
-		return true
-	}
-
-	// Warn administrators that invalid JWTs are being rejected. This is indicative
-	// of either a misconfiguration of the CLI or a massive failure of upstream systems.
-	if len(verbose) > 0 {
-		proxy.log.Err(err).Str(LogFieldCFRay, proxy.getRayHeader(r)).Msg("dbproxy: Failed JWT authentication")
-	}
-
-	return false
-}
-
-// Start the Proxy at a given address and notify the listener channel when the server is online.
-func (proxy *Proxy) Start(ctx context.Context, addr string, listenerC chan<- net.Listener) error {
-	// STOR-611: use a seperate listener and consider web socket support.
-	httpListener, err := hello.CreateTLSListener(addr)
-	if err != nil {
-		return errors.Wrapf(err, "could not create listener at %s", addr)
-	}
-
-	errC := make(chan error)
-	defer close(errC)
-
-	// Starts the HTTP server and begins to serve requests.
-	go func() {
-		errC <- proxy.httpListen(ctx, httpListener)
-	}()
-
-	// Continually ping the server until it comes online or 10 attempts fail.
-	go func() {
-		var err error
-		for i := 0; i < 10; i++ {
-			_, err = http.Get("http://" + httpListener.Addr().String())
-
-			// Once no error was detected, notify the listener channel and return.
-			if err == nil {
-				listenerC <- httpListener
-				return
-			}
-
-			// Backoff between requests to ping the server.
-			<-time.After(1 * time.Second)
-		}
-		errC <- errors.Wrap(err, "took too long for the http server to start")
-	}()
-
-	return <-errC
-}
-
-// httpListen starts the httpServer and blocks until the context closes.
-func (proxy *Proxy) httpListen(ctx context.Context, listener net.Listener) error {
-	httpServer := &http.Server{
-		Addr:         listener.Addr().String(),
-		Handler:      proxy.httpRouter(),
-		ReadTimeout:  10 * time.Second,
-		WriteTimeout: 60 * time.Second,
-		IdleTimeout:  60 * time.Second,
-	}
-
-	go func() {
-		<-ctx.Done()
-		_ = httpServer.Close()
-		_ = listener.Close()
-	}()
-
-	return httpServer.Serve(listener)
-}
-
-// httpRouter creates a mux.Router for the Proxy.
-func (proxy *Proxy) httpRouter() *mux.Router {
-	router := mux.NewRouter()
-
-	router.HandleFunc("/ping", proxy.httpPing()).Methods("GET", "HEAD")
-	router.HandleFunc("/submit", proxy.httpSubmit()).Methods("POST")
-
-	return router
-}
-
-// httpPing tests the connection to the database.
-//
-// By default, this endpoint is unauthenticated to allow for health checks.
-// To enable authentication, Cloudflare Access must be enabled on this route.
-func (proxy *Proxy) httpPing() http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		ctx := r.Context()
-		err := proxy.client.Ping(ctx)
-
-		if err == nil {
-			proxy.httpRespond(w, r, http.StatusOK, "")
-		} else {
-			proxy.httpRespondErr(w, r, http.StatusInternalServerError, err)
-		}
-	}
-}
-
-// httpSubmit sends a command to the database and returns its response.
-//
-// By default, this endpoint will reject requests that do not pass through Cloudflare Access.
-// To disable authentication, the --insecure flag must be specified in the command line.
-func (proxy *Proxy) httpSubmit() http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		if !proxy.IsAllowed(r, true) {
-			proxy.httpRespondErr(w, r, http.StatusForbidden, fmt.Errorf(""))
-			return
-		}
-
-		var cmd Command
-		err := json.NewDecoder(r.Body).Decode(&cmd)
-		if err != nil {
-			proxy.httpRespondErr(w, r, http.StatusBadRequest, err)
-			return
-		}
-
-		ctx := r.Context()
-		data, err := proxy.client.Submit(ctx, &cmd)
-
-		if err != nil {
-			proxy.httpRespondErr(w, r, http.StatusUnprocessableEntity, err)
-			return
-		}
-
-		w.Header().Set("Content-type", "application/json")
-		err = json.NewEncoder(w).Encode(data)
-		if err != nil {
-			proxy.httpRespondErr(w, r, http.StatusInternalServerError, err)
-		}
-	}
-}
-
-// httpRespond writes a status code and string response to the response writer.
-func (proxy *Proxy) httpRespond(w http.ResponseWriter, r *http.Request, status int, message string) {
-	w.WriteHeader(status)
-
-	// Only expose the message detail of the reponse if the request is not HEAD
-	// and the user is authenticated. For example, this prevents an unauthenticated
-	// failed health check from accidentally leaking sensitive information about the Client.
-	if r.Method != http.MethodHead && proxy.IsAllowed(r) {
-		if message == "" {
-			message = http.StatusText(status)
-		}
-		fmt.Fprint(w, message)
-	}
-}
-
-// httpRespondErr is similar to httpRespond, except it formats errors to be more friendly.
-func (proxy *Proxy) httpRespondErr(w http.ResponseWriter, r *http.Request, defaultStatus int, err error) {
-	status, err := httpError(defaultStatus, err)
-
-	proxy.httpRespond(w, r, status, err.Error())
-	if len(err.Error()) > 0 {
-		cfRay := proxy.getRayHeader(r)
-		proxy.log.Err(err).Str(LogFieldCFRay, cfRay).Msg("dbproxy: Database proxy error")
-	}
-}
-
-// getRayHeader returns the request's Cf-ray header.
-func (proxy *Proxy) getRayHeader(r *http.Request) string {
-	return r.Header.Get("Cf-ray")
-}
-
-// httpError extracts common errors and returns an status code and friendly error.
-func httpError(defaultStatus int, err error) (int, error) {
-	if err == nil {
-		return http.StatusNotImplemented, fmt.Errorf("error expected but found none")
-	}
-
-	if err == io.EOF {
-		return http.StatusBadRequest, fmt.Errorf("request body cannot be empty")
-	}
-
-	if err == context.DeadlineExceeded {
-		return http.StatusRequestTimeout, err
-	}
-
-	_, ok := err.(net.Error)
-	if ok {
-		return http.StatusRequestTimeout, err
-	}
-
-	if err == context.Canceled {
-		// Does not exist in Golang, but would be: http.StatusClientClosedWithoutResponse
-		return 444, err
-	}
-
-	return defaultStatus, err
-}

dbconnect/proxy_test.go

@@ -1,238 +0,0 @@
-package dbconnect
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"net"
-	"net/http"
-	"net/http/httptest"
-	"strings"
-	"testing"
-
-	"github.com/gorilla/mux"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestNewInsecureProxy(t *testing.T) {
-	origins := []string{
-		"",
-		":/",
-		"http://localhost",
-		"tcp://localhost:9000?debug=true",
-		"mongodb://127.0.0.1",
-	}
-
-	for _, origin := range origins {
-		proxy, err := NewInsecureProxy(context.Background(), origin)
-
-		assert.Error(t, err)
-		assert.Empty(t, proxy)
-	}
-}
-
-func TestProxyIsAllowed(t *testing.T) {
-	proxy := helperNewProxy(t)
-	req := httptest.NewRequest("GET", "https://1.1.1.1/ping", nil)
-	assert.True(t, proxy.IsAllowed(req))
-
-	proxy = helperNewProxy(t, true)
-	req.Header.Set("Cf-access-jwt-assertion", "xxx")
-	assert.False(t, proxy.IsAllowed(req))
-}
-
-func TestProxyStart(t *testing.T) {
-	proxy := helperNewProxy(t)
-	ctx := context.Background()
-	listenerC := make(chan net.Listener)
-
-	err := proxy.Start(ctx, "1.1.1.1:", listenerC)
-	assert.Error(t, err)
-
-	err = proxy.Start(ctx, "127.0.0.1:-1", listenerC)
-	assert.Error(t, err)
-
-	ctx, cancel := context.WithTimeout(ctx, 0)
-	defer cancel()
-
-	err = proxy.Start(ctx, "127.0.0.1:", listenerC)
-	assert.IsType(t, http.ErrServerClosed, err)
-}
-
-func TestProxyHTTPRouter(t *testing.T) {
-	proxy := helperNewProxy(t)
-	router := proxy.httpRouter()
-
-	tests := []struct {
-		path   string
-		method string
-		valid  bool
-	}{
-		{"", "GET", false},
-		{"/", "GET", false},
-		{"/ping", "GET", true},
-		{"/ping", "HEAD", true},
-		{"/ping", "POST", false},
-		{"/submit", "POST", true},
-		{"/submit", "GET", false},
-		{"/submit/extra", "POST", false},
-	}
-
-	for _, test := range tests {
-		match := &mux.RouteMatch{}
-		ok := router.Match(httptest.NewRequest(test.method, "https://1.1.1.1"+test.path, nil), match)
-
-		assert.True(t, ok == test.valid, test.path)
-	}
-}
-
-func TestProxyHTTPPing(t *testing.T) {
-	proxy := helperNewProxy(t)
-
-	server := httptest.NewServer(proxy.httpPing())
-	defer server.Close()
-	client := server.Client()
-
-	res, err := client.Get(server.URL)
-	assert.NoError(t, err)
-	assert.Equal(t, http.StatusOK, res.StatusCode)
-	assert.Equal(t, int64(2), res.ContentLength)
-
-	res, err = client.Head(server.URL)
-	assert.NoError(t, err)
-	assert.Equal(t, http.StatusOK, res.StatusCode)
-	assert.Equal(t, int64(-1), res.ContentLength)
-}
-
-func TestProxyHTTPSubmit(t *testing.T) {
-	proxy := helperNewProxy(t)
-
-	server := httptest.NewServer(proxy.httpSubmit())
-	defer server.Close()
-	client := server.Client()
-
-	tests := []struct {
-		input  string
-		status int
-		output string
-	}{
-		{"", http.StatusBadRequest, "request body cannot be empty"},
-		{"{}", http.StatusBadRequest, "cannot provide an empty statement"},
-		{"{\"statement\":\"Ok\"}", http.StatusUnprocessableEntity, "cannot provide invalid sql mode: ''"},
-		{"{\"statement\":\"Ok\",\"mode\":\"query\"}", http.StatusUnprocessableEntity, "near \"Ok\": syntax error"},
-		{"{\"statement\":\"CREATE TABLE t (a INT);\",\"mode\":\"exec\"}", http.StatusOK, "{\"last_insert_id\":0,\"rows_affected\":0}\n"},
-	}
-
-	for _, test := range tests {
-		res, err := client.Post(server.URL, "application/json", strings.NewReader(test.input))
-
-		assert.NoError(t, err)
-		assert.Equal(t, test.status, res.StatusCode)
-		if res.StatusCode > http.StatusOK {
-			assert.Equal(t, "text/plain; charset=utf-8", res.Header.Get("Content-type"))
-		} else {
-			assert.Equal(t, "application/json", res.Header.Get("Content-type"))
-		}
-
-		data, err := ioutil.ReadAll(res.Body)
-		defer res.Body.Close()
-		str := string(data)
-
-		assert.NoError(t, err)
-		assert.Equal(t, test.output, str)
-	}
-}
-
-func TestProxyHTTPSubmitForbidden(t *testing.T) {
-	proxy := helperNewProxy(t, true)
-
-	server := httptest.NewServer(proxy.httpSubmit())
-	defer server.Close()
-	client := server.Client()
-
-	res, err := client.Get(server.URL)
-
-	assert.NoError(t, err)
-	assert.Equal(t, http.StatusForbidden, res.StatusCode)
-	assert.Zero(t, res.ContentLength)
-}
-
-func TestProxyHTTPRespond(t *testing.T) {
-	proxy := helperNewProxy(t)
-
-	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		proxy.httpRespond(w, r, http.StatusAccepted, "Hello")
-	}))
-	defer server.Close()
-	client := server.Client()
-
-	res, err := client.Get(server.URL)
-	assert.NoError(t, err)
-	assert.Equal(t, http.StatusAccepted, res.StatusCode)
-	assert.Equal(t, int64(5), res.ContentLength)
-
-	data, err := ioutil.ReadAll(res.Body)
-	defer res.Body.Close()
-	assert.Equal(t, []byte("Hello"), data)
-}
-
-func TestProxyHTTPRespondForbidden(t *testing.T) {
-	proxy := helperNewProxy(t, true)
-
-	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		proxy.httpRespond(w, r, http.StatusAccepted, "Hello")
-	}))
-	defer server.Close()
-	client := server.Client()
-
-	res, err := client.Get(server.URL)
-
-	assert.NoError(t, err)
-	assert.Equal(t, http.StatusAccepted, res.StatusCode)
-	assert.Equal(t, int64(0), res.ContentLength)
-}
-
-func TestHTTPError(t *testing.T) {
-	_, errTimeout := net.DialTimeout("tcp", "127.0.0.1", 0)
-	assert.Error(t, errTimeout)
-
-	tests := []struct {
-		input  error
-		status int
-		output error
-	}{
-		{nil, http.StatusNotImplemented, fmt.Errorf("error expected but found none")},
-		{io.EOF, http.StatusBadRequest, fmt.Errorf("request body cannot be empty")},
-		{context.DeadlineExceeded, http.StatusRequestTimeout, nil},
-		{context.Canceled, 444, nil},
-		{errTimeout, http.StatusRequestTimeout, nil},
-		{fmt.Errorf(""), http.StatusInternalServerError, nil},
-	}
-
-	for _, test := range tests {
-		status, err := httpError(http.StatusInternalServerError, test.input)
-
-		assert.Error(t, err)
-		assert.Equal(t, test.status, status)
-		if test.output == nil {
-			test.output = test.input
-		}
-		assert.Equal(t, test.output, err)
-	}
-}
-
-func helperNewProxy(t *testing.T, secure ...bool) *Proxy {
-	t.Helper()
-
-	proxy, err := NewSecureProxy(context.Background(), "file::memory:?cache=shared", "test.cloudflareaccess.com", "")
-	assert.NoError(t, err)
-	assert.NotNil(t, proxy)
-
-	if len(secure) == 0 {
-		proxy.accessValidator = nil // Mark as insecure
-	}
-
-	return proxy
-}

dbconnect/sql.go

@@ -1,318 +0,0 @@
-package dbconnect
-
-import (
-	"context"
-	"database/sql"
-	"encoding/json"
-	"fmt"
-	"net/url"
-	"reflect"
-	"strings"
-
-	"github.com/jmoiron/sqlx"
-	"github.com/pkg/errors"
-	"github.com/xo/dburl"
-
-	// SQL drivers self-register with the database/sql package.
-	// https://github.com/golang/go/wiki/SQLDrivers
-	_ "github.com/denisenkom/go-mssqldb"
-	_ "github.com/go-sql-driver/mysql"
-	_ "github.com/mattn/go-sqlite3"
-
-	"github.com/kshvakov/clickhouse"
-	"github.com/lib/pq"
-)
-
-// SQLClient is a Client that talks to a SQL database.
-type SQLClient struct {
-	Dialect string
-	driver  *sqlx.DB
-}
-
-// NewSQLClient creates a SQL client based on its URL scheme.
-func NewSQLClient(ctx context.Context, originURL *url.URL) (Client, error) {
-	res, err := dburl.Parse(originURL.String())
-	if err != nil {
-		helpText := fmt.Sprintf("supported drivers: %+q, see documentation for more details: %s", sql.Drivers(), "https://godoc.org/github.com/xo/dburl")
-		return nil, fmt.Errorf("could not parse sql database url '%s': %s\n%s", originURL, err.Error(), helpText)
-	}
-
-	// Establishes the driver, but does not test the connection.
-	driver, err := sqlx.Open(res.Driver, res.DSN)
-	if err != nil {
-		return nil, fmt.Errorf("could not open sql driver %s: %s\n%s", res.Driver, err.Error(), res.DSN)
-	}
-
-	// Closes the driver, will occur when the context finishes.
-	go func() {
-		<-ctx.Done()
-		_ = driver.Close()
-	}()
-
-	return &SQLClient{driver.DriverName(), driver}, nil
-}
-
-// Ping verifies a connection to the database is still alive.
-func (client *SQLClient) Ping(ctx context.Context) error {
-	return client.driver.PingContext(ctx)
-}
-
-// Submit queries or executes a command to the SQL database.
-func (client *SQLClient) Submit(ctx context.Context, cmd *Command) (interface{}, error) {
-	txx, err := cmd.ValidateSQL(client.Dialect)
-	if err != nil {
-		return nil, err
-	}
-
-	ctx, cancel := context.WithTimeout(ctx, cmd.Timeout)
-	defer cancel()
-
-	var res interface{}
-
-	// Get the next available sql.Conn and submit the Command.
-	err = sqlConn(ctx, client.driver, txx, func(conn *sql.Conn) error {
-		stmt := cmd.Statement
-		args := cmd.Arguments.Positional
-
-		if cmd.Mode == "query" {
-			res, err = sqlQuery(ctx, conn, stmt, args)
-		} else {
-			res, err = sqlExec(ctx, conn, stmt, args)
-		}
-
-		return err
-	})
-
-	return res, err
-}
-
-// ValidateSQL extends the contract of Command for SQL dialects:
-// mode is conformed, arguments are []sql.NamedArg, and isolation is a sql.IsolationLevel.
-//
-// When the command should not be wrapped in a transaction, *sql.TxOptions and error will both be nil.
-func (cmd *Command) ValidateSQL(dialect string) (*sql.TxOptions, error) {
-	err := cmd.Validate()
-	if err != nil {
-		return nil, err
-	}
-
-	mode, err := sqlMode(cmd.Mode)
-	if err != nil {
-		return nil, err
-	}
-
-	// Mutates Arguments to only use positional arguments with the type sql.NamedArg.
-	// This is a required by the sql.Driver before submitting arguments.
-	cmd.Arguments.sql(dialect)
-
-	iso, err := sqlIsolation(cmd.Isolation)
-	if err != nil {
-		return nil, err
-	}
-
-	// When isolation is out-of-range, this is indicative that no
-	// transaction should be executed and sql.TxOptions should be nil.
-	if iso < sql.LevelDefault {
-		return nil, nil
-	}
-
-	// In query mode, execute the transaction in read-only, unless it's Microsoft SQL
-	// which does not support that type of transaction.
-	readOnly := mode == "query" && dialect != "mssql"
-
-	return &sql.TxOptions{Isolation: iso, ReadOnly: readOnly}, nil
-}
-
-// sqlConn gets the next available sql.Conn in the connection pool and runs a function to use it.
-//
-// If the transaction options are nil, run the useIt function outside a transaction.
-// This is potentially an unsafe operation if the command does not clean up its state.
-func sqlConn(ctx context.Context, driver *sqlx.DB, txx *sql.TxOptions, useIt func(*sql.Conn) error) error {
-	conn, err := driver.Conn(ctx)
-	if err != nil {
-		return err
-	}
-	defer conn.Close()
-
-	// If transaction options are specified, begin and defer a rollback to catch errors.
-	var tx *sql.Tx
-	if txx != nil {
-		tx, err = conn.BeginTx(ctx, txx)
-		if err != nil {
-			return err
-		}
-		defer tx.Rollback()
-	}
-
-	err = useIt(conn)
-
-	// Check if useIt was successful and a transaction exists before committing.
-	if err == nil && tx != nil {
-		err = tx.Commit()
-	}
-
-	return err
-}
-
-// sqlQuery queries rows on a sql.Conn and returns an array of result objects.
-func sqlQuery(ctx context.Context, conn *sql.Conn, stmt string, args []interface{}) ([]map[string]interface{}, error) {
-	rows, err := conn.QueryContext(ctx, stmt, args...)
-	if err == nil {
-		return sqlRows(rows)
-	}
-	return nil, err
-}
-
-// sqlExec executes a command on a sql.Conn and returns the result of the operation.
-func sqlExec(ctx context.Context, conn *sql.Conn, stmt string, args []interface{}) (sqlResult, error) {
-	exec, err := conn.ExecContext(ctx, stmt, args...)
-	if err == nil {
-		return sqlResultFrom(exec), nil
-	}
-	return sqlResult{}, err
-}
-
-// sql mutates Arguments to contain a positional []sql.NamedArg.
-//
-// The actual return type is []interface{} due to the native Golang
-// function signatures for sql.Exec and sql.Query being generic.
-func (args *Arguments) sql(dialect string) {
-	result := args.Positional
-
-	for i, val := range result {
-		result[i] = sqlArg("", val, dialect)
-	}
-
-	for key, val := range args.Named {
-		result = append(result, sqlArg(key, val, dialect))
-	}
-
-	args.Positional = result
-	args.Named = map[string]interface{}{}
-}
-
-// sqlArg creates a sql.NamedArg from a key-value pair and an optional dialect.
-//
-// Certain dialects will need to wrap objects, such as arrays, to conform its driver requirements.
-func sqlArg(key, val interface{}, dialect string) sql.NamedArg {
-	switch reflect.ValueOf(val).Kind() {
-
-	// PostgreSQL and Clickhouse require arrays to be wrapped before
-	// being inserted into the driver interface.
-	case reflect.Slice, reflect.Array:
-		switch dialect {
-		case "postgres":
-			val = pq.Array(val)
-		case "clickhouse":
-			val = clickhouse.Array(val)
-		}
-	}
-
-	return sql.Named(fmt.Sprint(key), val)
-}
-
-// sqlIsolation tries to match a string to a sql.IsolationLevel.
-func sqlIsolation(str string) (sql.IsolationLevel, error) {
-	if str == "none" {
-		return sql.IsolationLevel(-1), nil
-	}
-
-	for iso := sql.LevelDefault; ; iso++ {
-		if iso > sql.LevelLinearizable {
-			return -1, fmt.Errorf("cannot provide an invalid sql isolation level: '%s'", str)
-		}
-
-		if str == "" || strings.EqualFold(iso.String(), strings.ReplaceAll(str, "_", " ")) {
-			return iso, nil
-		}
-	}
-}
-
-// sqlMode tries to match a string to a command mode: 'query' or 'exec' for now.
-func sqlMode(str string) (string, error) {
-	switch str {
-	case "query", "exec":
-		return str, nil
-	default:
-		return "", fmt.Errorf("cannot provide invalid sql mode: '%s'", str)
-	}
-}
-
-// sqlRows scans through a SQL result set and returns an array of objects.
-func sqlRows(rows *sql.Rows) ([]map[string]interface{}, error) {
-	columns, err := rows.Columns()
-	if err != nil {
-		return nil, errors.Wrap(err, "could not extract columns from result")
-	}
-	defer rows.Close()
-
-	types, err := rows.ColumnTypes()
-	if err != nil {
-		// Some drivers do not support type extraction, so fail silently and continue.
-		types = make([]*sql.ColumnType, len(columns))
-	}
-
-	values := make([]interface{}, len(columns))
-	pointers := make([]interface{}, len(columns))
-
-	var results []map[string]interface{}
-	for rows.Next() {
-		for i := range columns {
-			pointers[i] = &values[i]
-		}
-		_ = rows.Scan(pointers...)
-
-		// Convert a row, an array of values, into an object where
-		// each key is the name of its respective column.
-		entry := make(map[string]interface{})
-		for i, col := range columns {
-			entry[col] = sqlValue(values[i], types[i])
-		}
-		results = append(results, entry)
-	}
-
-	return results, nil
-}
-
-// sqlValue handles special cases where sql.Rows does not return a "human-readable" object.
-func sqlValue(val interface{}, col *sql.ColumnType) interface{} {
-	bytes, ok := val.([]byte)
-	if ok {
-		// Opportunistically check for embeded JSON and convert it to a first-class object.
-		var embeded interface{}
-		if json.Unmarshal(bytes, &embeded) == nil {
-			return embeded
-		}
-
-		// STOR-604: investigate a way to coerce PostgreSQL arrays '{a, b, ...}' into JSON.
-		// Although easy with strings, it becomes more difficult with special types like INET[].
-
-		return string(bytes)
-	}
-
-	return val
-}
-
-// sqlResult is a thin wrapper around sql.Result.
-type sqlResult struct {
-	LastInsertId int64 `json:"last_insert_id"`
-	RowsAffected int64 `json:"rows_affected"`
-}
-
-// sqlResultFrom converts sql.Result into a JSON-marshable sqlResult.
-func sqlResultFrom(res sql.Result) sqlResult {
-	insertID, errID := res.LastInsertId()
-	rowsAffected, errRows := res.RowsAffected()
-
-	// If an error occurs when extracting the result, it is because the
-	// driver does not support that specific field. Instead of passing this
-	// to the user, omit the field in the response.
-	if errID != nil {
-		insertID = -1
-	}
-	if errRows != nil {
-		rowsAffected = -1
-	}
-
-	return sqlResult{insertID, rowsAffected}
-}

dbconnect/sql_test.go

@@ -1,336 +0,0 @@
-package dbconnect
-
-import (
-	"context"
-	"database/sql"
-	"fmt"
-	"net/url"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/kshvakov/clickhouse"
-	"github.com/lib/pq"
-
-	"github.com/DATA-DOG/go-sqlmock"
-	"github.com/stretchr/testify/assert"
-)
-
-func TestNewSQLClient(t *testing.T) {
-	originURLs := []string{
-		"postgres://localhost",
-		"cockroachdb://localhost:1337",
-		"postgresql://user:pass@127.0.0.1",
-		"mysql://localhost",
-		"clickhouse://127.0.0.1:9000/?debug",
-		"sqlite3::memory:",
-		"file:test.db?cache=shared",
-	}
-
-	for _, originURL := range originURLs {
-		origin, _ := url.Parse(originURL)
-		_, err := NewSQLClient(context.Background(), origin)
-
-		assert.NoError(t, err, originURL)
-	}
-
-	originURLs = []string{
-		"",
-		"/",
-		"http://localhost",
-		"coolthing://user:pass@127.0.0.1",
-	}
-
-	for _, originURL := range originURLs {
-		origin, _ := url.Parse(originURL)
-		_, err := NewSQLClient(context.Background(), origin)
-
-		assert.Error(t, err, originURL)
-	}
-}
-
-func TestArgumentsSQL(t *testing.T) {
-	args := []Arguments{
-		Arguments{
-			Positional: []interface{}{
-				"val", 10, 3.14,
-			},
-		},
-		Arguments{
-			Named: map[string]interface{}{
-				"key": time.Unix(0, 0),
-			},
-		},
-	}
-
-	var nameType sql.NamedArg
-	for _, arg := range args {
-		arg.sql("")
-		for _, named := range arg.Positional {
-			assert.IsType(t, nameType, named)
-		}
-	}
-}
-
-func TestSQLArg(t *testing.T) {
-	tests := []struct {
-		key     interface{}
-		val     interface{}
-		dialect string
-		arg     sql.NamedArg
-	}{
-		{"key", "val", "mssql", sql.Named("key", "val")},
-		{0, 1, "sqlite3", sql.Named("0", 1)},
-		{1, []string{"a", "b", "c"}, "postgres", sql.Named("1", pq.Array([]string{"a", "b", "c"}))},
-		{"in", []uint{0, 1}, "clickhouse", sql.Named("in", clickhouse.Array([]uint{0, 1}))},
-		{"", time.Unix(0, 0), "", sql.Named("", time.Unix(0, 0))},
-	}
-
-	for _, test := range tests {
-		arg := sqlArg(test.key, test.val, test.dialect)
-		assert.Equal(t, test.arg, arg, test.key)
-	}
-}
-
-func TestSQLIsolation(t *testing.T) {
-	tests := []struct {
-		str string
-		iso sql.IsolationLevel
-	}{
-		{"", sql.LevelDefault},
-		{"DEFAULT", sql.LevelDefault},
-		{"read_UNcommitted", sql.LevelReadUncommitted},
-		{"serializable", sql.LevelSerializable},
-		{"none", sql.IsolationLevel(-1)},
-		{"SNAP shot", -2},
-		{"blah", -2},
-	}
-
-	for _, test := range tests {
-		iso, err := sqlIsolation(test.str)
-
-		if test.iso < -1 {
-			assert.Error(t, err, test.str)
-		} else {
-			assert.NoError(t, err)
-			assert.Equal(t, test.iso, iso, test.str)
-		}
-	}
-}
-
-func TestSQLMode(t *testing.T) {
-	modes := []string{
-		"query",
-		"exec",
-	}
-
-	for _, mode := range modes {
-		actual, err := sqlMode(mode)
-
-		assert.NoError(t, err)
-		assert.Equal(t, strings.ToLower(mode), actual, mode)
-	}
-
-	modes = []string{
-		"",
-		"blah",
-	}
-
-	for _, mode := range modes {
-		_, err := sqlMode(mode)
-
-		assert.Error(t, err)
-	}
-}
-
-func helperRows(mockRows *sqlmock.Rows) *sql.Rows {
-	db, mock, _ := sqlmock.New()
-	mock.ExpectQuery("SELECT").WillReturnRows(mockRows)
-	rows, _ := db.Query("SELECT")
-	return rows
-}
-
-func TestSQLRows(t *testing.T) {
-	actual, err := sqlRows(helperRows(sqlmock.NewRows(
-		[]string{"name", "age", "dept"}).
-		AddRow("alice", 19, "prod")))
-	expected := []map[string]interface{}{map[string]interface{}{
-		"name": "alice",
-		"age":  int64(19),
-		"dept": "prod"}}
-
-	assert.NoError(t, err)
-	assert.ElementsMatch(t, expected, actual)
-}
-
-func TestSQLValue(t *testing.T) {
-	tests := []struct {
-		input  interface{}
-		output interface{}
-	}{
-		{"hello", "hello"},
-		{1, 1},
-		{false, false},
-		{[]byte("random"), "random"},
-		{[]byte("{\"json\":true}"), map[string]interface{}{"json": true}},
-		{[]byte("[]"), []interface{}{}},
-	}
-
-	for _, test := range tests {
-		assert.Equal(t, test.output, sqlValue(test.input, nil), test.input)
-	}
-}
-
-func TestSQLResultFrom(t *testing.T) {
-	res := sqlResultFrom(sqlmock.NewResult(1, 2))
-	assert.Equal(t, sqlResult{1, 2}, res)
-
-	res = sqlResultFrom(sqlmock.NewErrorResult(fmt.Errorf("")))
-	assert.Equal(t, sqlResult{-1, -1}, res)
-}
-
-func helperSQLite3(t *testing.T) (context.Context, Client) {
-	t.Helper()
-
-	ctx := context.Background()
-	url, _ := url.Parse("file::memory:?cache=shared")
-
-	sqlite3, err := NewSQLClient(ctx, url)
-	assert.NoError(t, err)
-
-	return ctx, sqlite3
-}
-
-func TestPing(t *testing.T) {
-	ctx, sqlite3 := helperSQLite3(t)
-	err := sqlite3.Ping(ctx)
-
-	assert.NoError(t, err)
-}
-
-func TestSubmit(t *testing.T) {
-	ctx, sqlite3 := helperSQLite3(t)
-
-	res, err := sqlite3.Submit(ctx, &Command{
-		Statement: "CREATE TABLE t (a INTEGER, b FLOAT, c TEXT, d BLOB);",
-		Mode:      "exec",
-	})
-	assert.NoError(t, err)
-	assert.Equal(t, sqlResult{0, 0}, res)
-
-	res, err = sqlite3.Submit(ctx, &Command{
-		Statement: "SELECT * FROM t;",
-		Mode:      "query",
-	})
-	assert.NoError(t, err)
-	assert.Empty(t, res)
-
-	res, err = sqlite3.Submit(ctx, &Command{
-		Statement: "INSERT INTO t VALUES (?, ?, ?, ?);",
-		Mode:      "exec",
-		Arguments: Arguments{
-			Positional: []interface{}{
-				1,
-				3.14,
-				"text",
-				"blob",
-			},
-		},
-	})
-	assert.NoError(t, err)
-	assert.Equal(t, sqlResult{1, 1}, res)
-
-	res, err = sqlite3.Submit(ctx, &Command{
-		Statement: "UPDATE t SET c = NULL;",
-		Mode:      "exec",
-	})
-	assert.NoError(t, err)
-	assert.Equal(t, sqlResult{1, 1}, res)
-
-	res, err = sqlite3.Submit(ctx, &Command{
-		Statement: "SELECT * FROM t WHERE a = ?;",
-		Mode:      "query",
-		Arguments: Arguments{
-			Positional: []interface{}{1},
-		},
-	})
-	assert.NoError(t, err)
-	assert.Len(t, res, 1)
-
-	resf, ok := res.([]map[string]interface{})
-	assert.True(t, ok)
-	assert.EqualValues(t, map[string]interface{}{
-		"a": int64(1),
-		"b": 3.14,
-		"c": nil,
-		"d": "blob",
-	}, resf[0])
-
-	res, err = sqlite3.Submit(ctx, &Command{
-		Statement: "DROP TABLE t;",
-		Mode:      "exec",
-	})
-	assert.NoError(t, err)
-	assert.Equal(t, sqlResult{1, 1}, res)
-}
-
-func TestSubmitTransaction(t *testing.T) {
-	ctx, sqlite3 := helperSQLite3(t)
-
-	res, err := sqlite3.Submit(ctx, &Command{
-		Statement: "BEGIN;",
-		Mode:      "exec",
-	})
-	assert.Error(t, err)
-	assert.Empty(t, res)
-
-	res, err = sqlite3.Submit(ctx, &Command{
-		Statement: "BEGIN; CREATE TABLE tt (a INT); COMMIT;",
-		Mode:      "exec",
-		Isolation: "none",
-	})
-	assert.NoError(t, err)
-	assert.Equal(t, sqlResult{0, 0}, res)
-
-	rows, err := sqlite3.Submit(ctx, &Command{
-		Statement: "SELECT * FROM tt;",
-		Mode:      "query",
-		Isolation: "repeatable_read",
-	})
-	assert.NoError(t, err)
-	assert.Empty(t, rows)
-}
-
-func TestSubmitTimeout(t *testing.T) {
-	ctx, sqlite3 := helperSQLite3(t)
-
-	res, err := sqlite3.Submit(ctx, &Command{
-		Statement: "SELECT * FROM t;",
-		Mode:      "query",
-		Timeout:   1 * time.Nanosecond,
-	})
-	assert.Error(t, err)
-	assert.Empty(t, res)
-}
-
-func TestSubmitMode(t *testing.T) {
-	ctx, sqlite3 := helperSQLite3(t)
-
-	res, err := sqlite3.Submit(ctx, &Command{
-		Statement: "SELECT * FROM t;",
-		Mode:      "notanoption",
-	})
-	assert.Error(t, err)
-	assert.Empty(t, res)
-}
-
-func TestSubmitEmpty(t *testing.T) {
-	ctx, sqlite3 := helperSQLite3(t)
-
-	res, err := sqlite3.Submit(ctx, &Command{
-		Statement: "; ; ; ;",
-		Mode:      "query",
-	})
-	assert.Error(t, err)
-	assert.Empty(t, res)
-}

dbconnect_tests/dbconnect.yaml

@@ -1,78 +0,0 @@
-# docker-compose -f ./dbconnect_tests/dbconnect.yaml up --build --force-recreate --renew-anon-volumes --exit-code-from cloudflared
-
-version: "2.3"
-networks:
-  test-dbconnect-network:
-    driver: bridge
-services:
-  cloudflared:
-    build:
-      context: ../
-      dockerfile: dev.Dockerfile
-    command: go test github.com/cloudflare/cloudflared/dbconnect_tests -v
-    depends_on:
-      postgres:
-        condition: service_healthy
-      mysql:
-        condition: service_healthy
-      mssql:
-        condition: service_healthy
-      clickhouse:
-        condition: service_healthy
-    environment:
-      DBCONNECT_INTEGRATION_TEST: "true"
-      POSTGRESQL_URL: postgres://postgres:secret@postgres/db?sslmode=disable
-      MYSQL_URL: mysql://root:secret@mysql/db?tls=false
-      MSSQL_URL: mssql://sa:secret12345!@mssql
-      CLICKHOUSE_URL: clickhouse://clickhouse:9000/db
-    networks:
-      - test-dbconnect-network
-  postgres:
-    image: postgres:11.4-alpine
-    environment:
-      POSTGRES_DB: db
-      POSTGRES_PASSWORD: secret
-    healthcheck:
-      test: ["CMD", "pg_isready", "-U", "postgres"]
-      start_period: 3s
-      interval: 1s
-      timeout: 3s
-      retries: 10
-    networks:
-      - test-dbconnect-network
-  mysql:
-    image: mysql:8.0
-    environment:
-      MYSQL_DATABASE: db
-      MYSQL_ROOT_PASSWORD: secret
-    healthcheck:
-      test: ["CMD", "mysqladmin", "ping"]
-      start_period: 3s
-      interval: 1s
-      timeout: 3s
-      retries: 10
-    networks:
-      - test-dbconnect-network
-  mssql:
-    image: mcr.microsoft.com/mssql/server:2017-CU8-ubuntu
-    environment:
-      ACCEPT_EULA: "Y"
-      SA_PASSWORD: secret12345!
-    healthcheck:
-      test: ["CMD", "/opt/mssql-tools/bin/sqlcmd", "-S", "localhost", "-U", "sa", "-P", "secret12345!", "-Q", "SELECT 1"]
-      start_period: 3s
-      interval: 1s
-      timeout: 3s
-      retries: 10
-    networks:
-      - test-dbconnect-network
-  clickhouse:
-    image: yandex/clickhouse-server:19.11
-    healthcheck:
-      test: ["CMD", "clickhouse-client", "--query", "SELECT 1"]
-      start_period: 3s
-      interval: 1s
-      timeout: 3s
-      retries: 10
-    networks:
-      - test-dbconnect-network

dbconnect_tests/sql_test.go

@@ -1,265 +0,0 @@
-package dbconnect_tests
-
-import (
-	"context"
-	"log"
-	"net/url"
-	"os"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-
-	"github.com/cloudflare/cloudflared/dbconnect"
-)
-
-func TestIntegrationPostgreSQL(t *testing.T) {
-	ctx, pq := helperNewSQLClient(t, "POSTGRESQL_URL")
-
-	err := pq.Ping(ctx)
-	assert.NoError(t, err)
-
-	_, err = pq.Submit(ctx, &dbconnect.Command{
-		Statement: "CREATE TABLE t (a TEXT, b UUID, c JSON, d INET[], e SERIAL);",
-		Mode:      "exec",
-	})
-	assert.NoError(t, err)
-
-	_, err = pq.Submit(ctx, &dbconnect.Command{
-		Statement: "INSERT INTO t VALUES ($1, $2, $3, $4);",
-		Mode:      "exec",
-		Arguments: dbconnect.Arguments{
-			Positional: []interface{}{
-				"text",
-				"6b8d686d-bd8e-43bc-b09a-cfcbbe702c10",
-				"{\"bool\":true,\"array\":[\"a\", 1, 3.14],\"embed\":{\"num\":21}}",
-				[]string{"1.1.1.1", "1.0.0.1"},
-			},
-		},
-	})
-	assert.NoError(t, err)
-
-	_, err = pq.Submit(ctx, &dbconnect.Command{
-		Statement: "UPDATE t SET b = NULL;",
-		Mode:      "exec",
-	})
-	assert.NoError(t, err)
-
-	res, err := pq.Submit(ctx, &dbconnect.Command{
-		Statement: "SELECT * FROM t;",
-		Mode:      "query",
-	})
-	assert.NoError(t, err)
-	assert.IsType(t, make([]map[string]interface{}, 0), res)
-
-	actual := res.([]map[string]interface{})[0]
-	expected := map[string]interface{}{
-		"a": "text",
-		"b": nil,
-		"c": map[string]interface{}{
-			"bool":  true,
-			"array": []interface{}{"a", float64(1), 3.14},
-			"embed": map[string]interface{}{"num": float64(21)},
-		},
-		"d": "{1.1.1.1,1.0.0.1}",
-		"e": int64(1),
-	}
-	assert.EqualValues(t, expected, actual)
-
-	_, err = pq.Submit(ctx, &dbconnect.Command{
-		Statement: "DROP TABLE t;",
-		Mode:      "exec",
-	})
-	assert.NoError(t, err)
-}
-
-func TestIntegrationMySQL(t *testing.T) {
-	ctx, my := helperNewSQLClient(t, "MYSQL_URL")
-
-	err := my.Ping(ctx)
-	assert.NoError(t, err)
-
-	_, err = my.Submit(ctx, &dbconnect.Command{
-		Statement: "CREATE TABLE t (a CHAR, b TINYINT, c FLOAT, d JSON, e YEAR);",
-		Mode:      "exec",
-	})
-	assert.NoError(t, err)
-
-	_, err = my.Submit(ctx, &dbconnect.Command{
-		Statement: "INSERT INTO t VALUES (?, ?, ?, ?, ?);",
-		Mode:      "exec",
-		Arguments: dbconnect.Arguments{
-			Positional: []interface{}{
-				"a",
-				10,
-				3.14,
-				"{\"bool\":true}",
-				2000,
-			},
-		},
-	})
-	assert.NoError(t, err)
-
-	_, err = my.Submit(ctx, &dbconnect.Command{
-		Statement: "ALTER TABLE t ADD COLUMN f GEOMETRY;",
-		Mode:      "exec",
-	})
-	assert.NoError(t, err)
-
-	res, err := my.Submit(ctx, &dbconnect.Command{
-		Statement: "SELECT * FROM t;",
-		Mode:      "query",
-	})
-	assert.NoError(t, err)
-	assert.IsType(t, make([]map[string]interface{}, 0), res)
-
-	actual := res.([]map[string]interface{})[0]
-	expected := map[string]interface{}{
-		"a": "a",
-		"b": float64(10),
-		"c": 3.14,
-		"d": map[string]interface{}{"bool": true},
-		"e": float64(2000),
-		"f": nil,
-	}
-	assert.EqualValues(t, expected, actual)
-
-	_, err = my.Submit(ctx, &dbconnect.Command{
-		Statement: "DROP TABLE t;",
-		Mode:      "exec",
-	})
-	assert.NoError(t, err)
-}
-
-func TestIntegrationMSSQL(t *testing.T) {
-	ctx, ms := helperNewSQLClient(t, "MSSQL_URL")
-
-	err := ms.Ping(ctx)
-	assert.NoError(t, err)
-
-	_, err = ms.Submit(ctx, &dbconnect.Command{
-		Statement: "CREATE TABLE t (a BIT, b DECIMAL, c MONEY, d TEXT);",
-		Mode:      "exec"})
-	assert.NoError(t, err)
-
-	_, err = ms.Submit(ctx, &dbconnect.Command{
-		Statement: "INSERT INTO t VALUES (?, ?, ?, ?);",
-		Mode:      "exec",
-		Arguments: dbconnect.Arguments{
-			Positional: []interface{}{
-				0,
-				3,
-				"$0.99",
-				"text",
-			},
-		},
-	})
-	assert.NoError(t, err)
-
-	_, err = ms.Submit(ctx, &dbconnect.Command{
-		Statement: "UPDATE t SET d = NULL;",
-		Mode:      "exec",
-	})
-	assert.NoError(t, err)
-
-	res, err := ms.Submit(ctx, &dbconnect.Command{
-		Statement: "SELECT * FROM t;",
-		Mode:      "query",
-	})
-	assert.NoError(t, err)
-	assert.IsType(t, make([]map[string]interface{}, 0), res)
-
-	actual := res.([]map[string]interface{})[0]
-	expected := map[string]interface{}{
-		"a": false,
-		"b": float64(3),
-		"c": float64(0.99),
-		"d": nil,
-	}
-	assert.EqualValues(t, expected, actual)
-
-	_, err = ms.Submit(ctx, &dbconnect.Command{
-		Statement: "DROP TABLE t;",
-		Mode:      "exec",
-	})
-	assert.NoError(t, err)
-}
-
-func TestIntegrationClickhouse(t *testing.T) {
-	ctx, ch := helperNewSQLClient(t, "CLICKHOUSE_URL")
-
-	err := ch.Ping(ctx)
-	assert.NoError(t, err)
-
-	_, err = ch.Submit(ctx, &dbconnect.Command{
-		Statement: "CREATE TABLE t (a UUID, b String, c Float64, d UInt32, e Int16, f Array(Enum8('a'=1, 'b'=2, 'c'=3))) engine=Memory;",
-		Mode:      "exec",
-	})
-	assert.NoError(t, err)
-
-	_, err = ch.Submit(ctx, &dbconnect.Command{
-		Statement: "INSERT INTO t VALUES (?, ?, ?, ?, ?, ?);",
-		Mode:      "exec",
-		Arguments: dbconnect.Arguments{
-			Positional: []interface{}{
-				"ec65f626-6f50-4c86-9628-6314ef1edacd",
-				"",
-				3.14,
-				314,
-				-144,
-				[]string{"a", "b", "c"},
-			},
-		},
-	})
-	assert.NoError(t, err)
-
-	res, err := ch.Submit(ctx, &dbconnect.Command{
-		Statement: "SELECT * FROM t;",
-		Mode:      "query",
-	})
-	assert.NoError(t, err)
-	assert.IsType(t, make([]map[string]interface{}, 0), res)
-
-	actual := res.([]map[string]interface{})[0]
-	expected := map[string]interface{}{
-		"a": "ec65f626-6f50-4c86-9628-6314ef1edacd",
-		"b": "",
-		"c": float64(3.14),
-		"d": uint32(314),
-		"e": int16(-144),
-		"f": []string{"a", "b", "c"},
-	}
-	assert.EqualValues(t, expected, actual)
-
-	_, err = ch.Submit(ctx, &dbconnect.Command{
-		Statement: "DROP TABLE t;",
-		Mode:      "exec",
-	})
-	assert.NoError(t, err)
-}
-
-func helperNewSQLClient(t *testing.T, env string) (context.Context, dbconnect.Client) {
-	_, ok := os.LookupEnv("DBCONNECT_INTEGRATION_TEST")
-	if ok {
-		t.Helper()
-	} else {
-		t.SkipNow()
-	}
-
-	val, ok := os.LookupEnv(env)
-	if !ok {
-		log.Fatalf("must provide database url as environment variable: %s", env)
-	}
-
-	parsed, err := url.Parse(val)
-	if err != nil {
-		log.Fatalf("cannot provide invalid database url: %s=%s", env, val)
-	}
-
-	ctx := context.Background()
-	client, err := dbconnect.NewSQLClient(ctx, parsed)
-	if err != nil {
-		log.Fatalf("could not start test client: %s", err)
-	}
-
-	return ctx, client
-}

dev.Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.15.7 as builder
+FROM golang:1.16.4 as builder
 ENV GO111MODULE=on \
     CGO_ENABLED=0
 WORKDIR /go/src/github.com/cloudflare/cloudflared/